omp-conductor 0.4.2 → 0.4.4

package/src/tracker/github.ts

@@ -113,18 +113,29 @@ interface GhPrVerification {
 
 /** Carries the captured stderr so callers can classify a failure without
  *  re-running the command or parsing the message text of a plain Error. */
-class GhError extends Error {
+export class GhError extends Error {
   readonly argv: string[];
   readonly code: number;
   readonly stderr: string;
-
-  constructor(argv: string[], code: number, stderr: string) {
+  /**
+   * Whatever the command printed before it failed.
+   *
+   * Carried because a nonzero exit is not always a failure to *read*: `gh pr
+   * checks` exits 1 when a check failed and 8 when one is pending, having
+   * already written the complete JSON payload asked for. Discarding stdout here
+   * made both CI failure classes unreachable on exactly the pull requests they
+   * exist to classify — a red PR answered "no checks" (#132 follow-up).
+   */
+  readonly stdout: string;
+
+  constructor(argv: string[], code: number, stderr: string, stdout = "") {
     const detail = stderr.trim() || "(no stderr)";
     super(`\`gh ${argv.join(" ")}\` exited ${code}: ${detail}`);
     this.name = "GhError";
     this.argv = argv;
     this.code = code;
     this.stderr = stderr;
+    this.stdout = stdout;
   }
 }
 
@@ -155,7 +166,7 @@ async function gh(argv: string[], stdin?: string): Promise<string> {
 
   const signal = proc.signalCode;
   if (code !== 0 || signal) {
-    throw new GhError(argv, code, signal ? `${stderr}\nterminated by ${signal}` : stderr);
+    throw new GhError(argv, code, signal ? `${stderr}\nterminated by ${signal}` : stderr, stdout);
   }
   return stdout;
 }
@@ -381,10 +392,104 @@ export function parentNumberFrom(raw: string): number | undefined {
   return n;
 }
 
+/**
+ * Parse `gh pr checks --json name,state,link`.
+ *
+ * Tolerant on purpose: a payload shape this build does not recognise yields an
+ * empty list, which the classifier reads as "could not tell" — the same posture
+ * every other reader in this adapter takes.
+ */
+export function checkConclusionsFrom(raw: string): { name: string; state: string; link?: string }[] {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw) as unknown;
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(parsed)) return [];
+  const checks: { name: string; state: string; link?: string }[] = [];
+  for (const entry of parsed) {
+    if (entry === null || typeof entry !== "object") continue;
+    const row = entry as { readonly [key: string]: unknown };
+    const name = row["name"];
+    const state = row["state"];
+    if (typeof name !== "string" || typeof state !== "string") continue;
+    const link = row["link"];
+    checks.push({
+      name,
+      state,
+      ...(typeof link === "string" && link.length > 0 ? { link } : {}),
+    });
+  }
+  return checks;
+}
+
+/** GitHub's mergeability spelling, fail-open to `unknown` on anything else. */
+export function mergeableFrom(raw: string): "conflicting" | "clean" | "unknown" {
+  switch (raw.trim().replaceAll('"', "")) {
+    case "CONFLICTING":
+      return "conflicting";
+    case "MERGEABLE":
+      return "clean";
+    default:
+      // `UNKNOWN` is GitHub still computing the merge, and it is common on a PR
+      // pushed seconds ago. Reading it as clean would let a conflict recovery
+      // fire on a PR nobody has assessed yet.
+      return "unknown";
+  }
+}
+
+/**
+ * Distinct workflow-run ids behind the non-success checks of a PR.
+ *
+ * Extracted from each check's `link` because that is the only run identifier
+ * `gh pr checks` reports. Deduplicated: one workflow run usually backs several
+ * checks, and re-running it once is the whole point.
+ */
+export function failedRunIds(
+  checks: readonly { name: string; state: string; link?: string }[],
+): string[] {
+  const ids: string[] = [];
+  for (const check of checks) {
+    const state = check.state.trim().toLowerCase();
+    if (state === "success" || state === "neutral") continue;
+    const match = /\/actions\/runs\/(\d+)\//.exec(check.link ?? "");
+    const id = match?.[1];
+    if (id !== undefined && !ids.includes(id)) ids.push(id);
+  }
+  return ids;
+}
+
+/** Parse `[{number, state}]` from `gh issue list` or the sub-issues API. */
+export function labeledIssuesFrom(raw: string): { number: number; state: IssueState }[] {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw) as unknown;
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(parsed)) return [];
+  const issues: { number: number; state: IssueState }[] = [];
+  for (const entry of parsed) {
+    if (entry === null || typeof entry !== "object") continue;
+    const row = entry as { readonly [key: string]: unknown };
+    const number = row["number"];
+    const rawState = row["state"];
+    if (typeof number !== "number" || !Number.isInteger(number) || typeof rawState !== "string") continue;
+    // The REST API answers lowercase, the CLI uppercase. Both are the same fact.
+    const state = issueStateFrom(rawState.toUpperCase());
+    if (state === undefined) continue;
+    issues.push({ number, state });
+  }
+  return issues;
+}
+
 export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
   const repo = p.tracker.repo;
 
-  return {
+  // Named rather than returned inline, so `rerunFailedChecks` can reuse
+  // `checkConclusions` instead of re-implementing the same `gh` call.
+  const tracker: Tracker = {
     async listReady(): Promise<ReadyIssue[]> {
       const raw = await runGh([
         "issue",
@@ -573,5 +678,83 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
         return undefined;
       }
     },
+    async checkConclusions(prUrl: string): Promise<{ name: string; state: string; link?: string }[]> {
+      if (!PR_URL.test(prUrl)) return [];
+      try {
+        return checkConclusionsFrom(await runGh(["pr", "checks", prUrl, "--json", "name,state,link"]));
+      } catch (err) {
+        // A nonzero exit is the *documented* answer for a PR with a failing or
+        // pending check — `gh pr checks` exits 1 and 8 respectively, having
+        // already printed the whole payload. Reading only the throw is what made
+        // `ci-infra` and `ci-deterministic` unreachable on live data: every red
+        // PR answered "no checks", so it classified `unknown` and escalated.
+        //
+        // So the payload wins when there is one, and only a genuinely unreadable
+        // answer falls through to "could not tell".
+        const stdout = err instanceof GhError ? err.stdout : "";
+        return checkConclusionsFrom(stdout);
+      }
+    },
+
+    async mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown"> {
+      if (!PR_URL.test(prUrl)) return "unknown";
+      try {
+        return mergeableFrom(await runGh(["pr", "view", prUrl, "--json", "mergeable", "--jq", ".mergeable"]));
+      } catch {
+        return "unknown";
+      }
+    },
+
+    async rerunFailedChecks(prUrl: string): Promise<void> {
+      const runIds = failedRunIds(await tracker.checkConclusions(prUrl));
+      for (const id of runIds) {
+        try {
+          await runGh(["run", "rerun", id, "--failed"]);
+        } catch {
+          // Per run id: a workflow run too old to re-run, or one already
+          // re-running, must not stop the others. The sweep re-verifies the PR
+          // against its head on a later tick either way.
+        }
+      }
+    },
+
+    async listLabeled(label: string, limit = 50): Promise<{ number: number; state: IssueState }[]> {
+      try {
+        return labeledIssuesFrom(
+          await runGh([
+            "issue",
+            "list",
+            "--repo",
+            repo,
+            "--label",
+            label,
+            "--state",
+            "all",
+            "--json",
+            "number,state",
+            "--limit",
+            String(limit),
+          ]),
+        );
+      } catch {
+        // A reconcile that cannot list must remove no labels: an empty answer is
+        // read as "no evidence", never as "nothing carries this label".
+        return [];
+      }
+    },
+
+    async childrenOf(issue: number): Promise<{ number: number; state: IssueState }[]> {
+      try {
+        return labeledIssuesFrom(
+          await runGh(["api", `repos/${repo}/issues/${issue}/sub_issues`, "--jq", "[.[] | {number, state}]"]),
+        );
+      } catch {
+        // Also the answer on a repo without the sub-issues API, or an issue that
+        // was never decomposed. All three mean the same thing to the caller.
+        return [];
+      }
+    },
   };
+
+  return tracker;
 }

package/src/types.ts

@@ -114,10 +114,17 @@ export interface RepoTarget {
 
 /**
  * How much the orchestrator says out loud without being asked. Declared as data
- * so the validator, the wizard and the brief all enumerate the same two values:
- * a third scope cannot be added while one of them still knows only two.
+ * so the validator, the wizard and the brief all enumerate the same three
+ * values: a fourth scope cannot be added while one of them still knows only
+ * three.
+ *
+ * `escalations` interrupts only for a tier-2 decision. `material` reports every
+ * material event as it happens. `decisions` sits between them: a decision the
+ * session needs, or a condition that stops the fleet, interrupts immediately;
+ * every other material event accumulates and ships with the next tick report —
+ * one message on a schedule instead of a ping per merge (#138).
  */
-export const REPORT_SCOPES = ["escalations", "material"] as const;
+export const REPORT_SCOPES = ["escalations", "decisions", "material"] as const;
 
 export type ReportScope = (typeof REPORT_SCOPES)[number];
 
@@ -608,16 +615,6 @@ export interface ProjectConfig {
   workspaceRoot: string;
   /** Cache of bare clones, so N runs share one fetch instead of N. */
   mirrorRoot: string;
-  /**
-   * Extra roots an orchestrator session may *read*, on top of the state
-   * directory and its own briefs. Absolute (or `~`-expandable) paths only,
-   * rejected at load rather than resolved against a cwd nobody can name.
-   *
-   * It cannot widen the gate past the roots #127 exists to close: the denied
-   * list — every worker checkout, the mirror cache, this package's own
-   * install — outranks anything named here.
-   */
-  orchestratorReadPaths?: string[];
 }
 
 /**
@@ -836,8 +833,84 @@ export interface Tracker {
    * better answer on a later tick.
    */
   prDiff(url: string): Promise<PrDiff | undefined>;
+  /**
+   * Every check on a PR with the state the tracker reports for it (#132).
+   *
+   * Deliberately separate from {@link Tracker.verifyPr}, which answers "may this
+   * merge" and collapses every non-success into one verdict. Classification asks
+   * a different question — *why* is it not green — and the answer is the
+   * difference between re-running a cancelled runner for free and charging an
+   * implementation attempt for a real test failure.
+   *
+   * An empty array means "could not tell", the same posture as the rest of this
+   * port: nothing downstream may read it as "no checks failed".
+   */
+  checkConclusions(prUrl: string): Promise<{ name: string; state: string; link?: string }[]>;
+  /** Whether the PR can merge into its base. `unknown` on any doubt, so a
+   *  mergeability nobody could read never becomes a conflict recovery. */
+  mergeable(prUrl: string): Promise<"conflicting" | "clean" | "unknown">;
+  /** Re-run the failed jobs behind a PR's non-success checks. Best effort per
+   *  workflow run: one un-rerunnable run must not stop the others. */
+  rerunFailedChecks(prUrl: string): Promise<void>;
+  /** Issues carrying `label`, open or closed, bounded. Empty on any failure —
+   *  a reconcile that cannot list must remove no labels. */
+  listLabeled(label: string, limit?: number): Promise<{ number: number; state: IssueState }[]>;
+  /** Sub-issues of `issue`, with their states. Empty when there are none *or*
+   *  when the lookup failed: both mean "no evidence this was decomposed", and
+   *  the reconcile below only ever acts on positive evidence. */
+  childrenOf(issue: number): Promise<{ number: number; state: IssueState }[]>;
 }
 
+/**
+ * Why a terminal run ended badly (#132).
+ *
+ * Declared as data here for the same reason `REPORT_SCOPES` is: the classifier,
+ * the store's budget counters, `status` and the board all enumerate these, so a
+ * new class must fail to compile in each of those places rather than silently
+ * resolve to "unclassified" in one of them. The decision table that maps signals
+ * onto them lives in `failure-class.ts`.
+ *
+ * The issue's `question-answerable` / `question-decision` split is deliberately
+ * one `question`: telling them apart needs semantic judgement about the worker's
+ * question, which is the orchestrator's job rather than a decision table's. The
+ * escalation carries the worker's own blocking report, so the orchestrator
+ * answers instead of re-investigating.
+ */
+export const FAILURE_CLASSES = [
+  "turn-cap-progress",
+  "turn-cap-spinning",
+  "admin-kill",
+  "ci-infra",
+  "ci-deterministic",
+  "merge-conflict",
+  "question",
+  "orphan-clean",
+  "orphan-dirty",
+  "settlement-stuck",
+  "unknown",
+] as const;
+
+export type FailureClass = (typeof FAILURE_CLASSES)[number];
+
+/**
+ * What the daemon does about a class.
+ *
+ * `none` completes the vocabulary rather than naming an outcome: an
+ * unclassifiable run escalates and never silently retries, so nothing in the
+ * decision table maps to it.
+ */
+export const RECOVERY_ACTIONS = [
+  "requeue",
+  "continue",
+  "rerun-checks",
+  "settle",
+  "escalate",
+  "hold",
+  "none",
+] as const;
+
+export type RecoveryAction = (typeof RECOVERY_ACTIONS)[number];
+
 /**
  * Execution state is separate from the tracker's own labels on purpose: labels
  * are coarse and human-editable, while the loop needs to distinguish "pushed
@@ -902,6 +975,17 @@ export interface RunRecord {
    *  exactly as an unflagged one does, and the flags are evidence for whoever
    *  reviews the PR. Absent means the audit found nothing, or never ran. */
   settlementFlags?: SettlementFlag[];
+  /**
+   * Why this run ended badly, and what the daemon did about it (#132). Absent
+   * means the sweep has not looked at the row yet — never "nothing was wrong":
+   * every budget counter treats an unclassified terminal row exactly as it did
+   * before classification existed.
+   */
+  failureClass?: FailureClass;
+  recoveryAction?: RecoveryAction;
+  /** When the recovery for that class ran. Absent means still unrecovered, which
+   *  is what `status` counts and what the board's card suffix reports. */
+  recoveredAt?: number;
 }
 
 export type AdmissionHoldReason =
@@ -1073,6 +1157,60 @@ export interface ReportEnqueue {
   deduped: boolean;
 }
 
+/**
+ * Where one operator decision stands (#136).
+ *
+ * `open` is the only state that owes anybody anything, which is why the tick
+ * digest reads exactly that set. The three terminal states are distinct because
+ * they describe different histories: `answered` carries what the operator
+ * decided, `withdrawn` records that the session stopped needing it (and why),
+ * and `expired` is the ledger closing a question nobody answered inside
+ * {@link DECISION_TTL_MS} rather than letting it accumulate forever.
+ */
+export const DECISION_STATES = ["open", "answered", "withdrawn", "expired"] as const;
+
+export type DecisionState = (typeof DECISION_STATES)[number];
+
+/**
+ * How long an unanswered question stays open — the seven days the floor's
+ * parked-amendment protocol already promised, now enforced instead of
+ * remembered.
+ */
+export const DECISION_TTL_MS = 7 * 24 * 60 * 60_000;
+
+/** One question put to the operator, and its answer if it has one. */
+export interface DecisionRecord {
+  id: string;
+  project: string;
+  /** The question verbatim, as it was sent. Re-asking must not reword it. */
+  question: string;
+  /** What is waiting on the answer — an issue, a release, a PR. Free text,
+   *  because the point is that a human reads it in a digest. */
+  blocks?: string;
+  askedAt: number;
+  expiresAt: number;
+  /** Raw machine-checkable precondition, e.g. `pr-merged:<url>`. Stored as
+   *  written and parsed on read, so a grammar this build does not know is a row
+   *  that still lists rather than a row that fails to load. */
+  condition?: string;
+  /** When that precondition was first observed true. Set once; the digest
+   *  promotes the row from "parked" to "act on this now". */
+  conditionMetAt?: number;
+  state: DecisionState;
+  resolvedAt?: number;
+  /** The answer, the withdrawal reason, or the expiry note. */
+  resolution?: string;
+}
+
+/** What a caller hands over. The store owns the id, the state and the expiry. */
+export interface DecisionDraft {
+  project: string;
+  question: string;
+  blocks?: string;
+  condition?: string;
+  at: number;
+}
+
 /**
  * Bookkeeping only — GitHub labels remain the source of truth. The store
  * exists to answer cap questions cheaply and to survive a restart; if it is
@@ -1160,6 +1298,38 @@ export interface Store {
   recoverSendingReports(project: string, staleAt: number, at: number): ReportRecord[];
   /** Everything an operator still has to care about: pending, sending, failed. */
   openReports(project: string): ReportRecord[];
+  /**
+   * Terminal rows the classification sweep still owes something to, newest first
+   * and bounded (#132): never classified, or classified with a retryable recovery
+   * that has not run. The second half is what makes "retrying next tick" true
+   * when a tracker write fails mid-recovery.
+   *
+   * Bounded because each row costs tracker calls to gather facts for: a fleet
+   * with a long unclassified history works through it over several ticks rather
+   * than spending one tick on all of it.
+   */
+  runsNeedingClassification(project: string, limit?: number): RunRecord[];
+  /** Unrecovered rows per class, for `status`. Empty when nothing is carrying one. */
+  failureClassCounts(project: string): { cls: FailureClass; n: number }[];
+  /** Rows whose recovery ran at or after `since`, newest first — the tick's
+   *  "auto-recovered since last tick" line, and its count. */
+  recoveredSince(project: string, since: number): RunRecord[];
+  /**
+   * Record a question put to the operator (#136). Expiry is the store's, not the
+   * caller's: a session that computed its own deadline would be back to
+   * remembering things.
+   */
+  createDecision(draft: DecisionDraft): DecisionRecord;
+  /** Everything still owed an answer, oldest first — what a tick digest reads. */
+  openDecisions(project: string): DecisionRecord[];
+  /** Answer or withdraw one. `false` when the id is unknown or already closed,
+   *  so a double-resolve cannot overwrite the first answer. */
+  resolveDecision(id: string, state: "answered" | "withdrawn", resolution: string, at: number): boolean;
+  /** First observation that a row's condition came true. Idempotent. */
+  markDecisionConditionMet(id: string, at: number): boolean;
+  /** Close every open row past its deadline and return them, so the caller can
+   *  say what it just closed rather than reporting a count. */
+  expireDueDecisions(project: string, now: number): DecisionRecord[];
   /**
    * Append one decided verb call to the run-scoped action ledger (#126).
    * Written for refusals as well as approvals, and written *before* the

package/src/upgrade.ts

@@ -3,7 +3,7 @@ import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
 import { setPaused, statusSnapshot } from "./daemon.ts";
 import { fleetLayers, telegramStateDir, type DispatchLayer, type FleetLayers } from "./fleet.ts";
 import { livingDaemon, restartDaemon } from "./lifecycle.ts";
-import { configPath, findProject, loadConfig, resolveCaps, writeConfigFile } from "./config.ts";
+import { configPath, findProject, loadConfig, resolveCaps, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";
 import {
   STAGED_SERVICE_NAME,
@@ -276,6 +276,21 @@ async function waitForRecovery(
   throw new Error(`upgrade verification failed: ${problem}`);
 }
 
+/**
+ * Brings the brief up with the freshly installed package.
+ *
+ * `brief-upgrade --migrate --apply` is the cross-version ABI: this code runs
+ * from the *old* CLI while the new one is already installed, so the verb has to
+ * keep working across the boundary. When it does not — a flag renamed, the verb
+ * gone — the tolerable outcome depends entirely on the layout:
+ *
+ *  - `overlay` fleets need nothing. The floor is recomposed from the installed
+ *    package on every tick, so a failed migrate costs nothing and stopping the
+ *    upgrade over it would be the worse trade.
+ *  - a legacy layout needs it. Skipping the migration silently would leave the
+ *    fleet on a single-file brief whose floor no longer tracks the package —
+ *    the exact drift this verb exists to end — so it fails loudly and rolls back.
+ */
 async function upgradeBrief(
   deps: UpgradeDeps,
   kind: BriefLayout["kind"],
@@ -283,9 +298,16 @@ async function upgradeBrief(
 ): Promise<void> {
   const selected = project === undefined ? [] : ["--project", project];
   if (kind === "legacy-handwritten") {
+    // Ungated on purpose: a hand-written brief cannot be migrated without it.
     await mustRun(deps, "omp-conductor", ["brief-upgrade", "--retrofit", "--apply", ...selected]);
   }
-  await mustRun(deps, "omp-conductor", ["brief-upgrade", "--migrate", "--apply", ...selected]);
+  try {
+    await mustRun(deps, "omp-conductor", ["brief-upgrade", "--migrate", "--apply", ...selected]);
+  } catch (err) {
+    if (kind !== "overlay") throw err;
+    const msg = err instanceof Error ? err.message : String(err);
+    deps.log(`brief: brief-upgrade unavailable in the target CLI (${msg}) — overlay recomposes each tick, continuing`);
+  }
 }
 
 async function rollbackUpgrade(
@@ -314,7 +336,9 @@ async function rollbackUpgrade(
       const path = configPath();
       if (readFileSync(path, "utf8") !== configBefore) {
         deps.log("rollback: conductor config.json");
-        writeConfigFile(JSON.parse(configBefore));
+        // Exact bytes. `writeConfigFile(JSON.parse(...))` returns canonically
+        // formatted output, which restores the keys but not the file.
+        writeConfigRaw(configBefore);
       }
     } catch (err) {
       failures.push(`could not restore config.json: ${err instanceof Error ? err.message : String(err)}`);