omp-conductor 0.16.2 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +38 -4
  2. package/REFERENCE.md +18 -12
  3. package/package.json +2 -1
  4. package/schema/config.schema.json +16 -0
  5. package/src/admission.ts +159 -43
  6. package/src/availability.ts +27 -1
  7. package/src/briefs/worker.md +2 -0
  8. package/src/clack-ui.ts +83 -0
  9. package/src/command-manifest.ts +16 -7
  10. package/src/commands/arm.ts +11 -3
  11. package/src/commands/decision.ts +17 -7
  12. package/src/commands/doctor.ts +18 -1
  13. package/src/commands/hold.ts +9 -7
  14. package/src/commands/ledger.ts +25 -4
  15. package/src/commands/message.ts +32 -4
  16. package/src/commands/setup.ts +61 -10
  17. package/src/commands/stats.ts +9 -5
  18. package/src/commands/status.ts +32 -5
  19. package/src/commands/tail.ts +13 -1
  20. package/src/commands/watch.ts +16 -7
  21. package/src/config-schema.ts +20 -0
  22. package/src/config.ts +37 -0
  23. package/src/daemon.ts +1240 -18
  24. package/src/doctor.ts +310 -22
  25. package/src/escalate.ts +560 -57
  26. package/src/failure-class.ts +56 -13
  27. package/src/fleet.ts +224 -47
  28. package/src/gitops.ts +103 -24
  29. package/src/lifecycle.ts +7 -2
  30. package/src/orchestrator-tick.ts +372 -157
  31. package/src/privileged.ts +3 -0
  32. package/src/release-policy.ts +177 -5
  33. package/src/setup-answers.ts +135 -0
  34. package/src/setup-host.ts +193 -4
  35. package/src/setup-install.ts +2 -0
  36. package/src/setup-probe.ts +1 -0
  37. package/src/setup-wizard.ts +1296 -101
  38. package/src/setup.ts +60 -3
  39. package/src/status-render.ts +11 -1
  40. package/src/store.ts +333 -12
  41. package/src/tracker/github.ts +562 -13
  42. package/src/types.ts +204 -2
  43. package/src/ui/progress.ts +32 -0
  44. package/src/ui/style.ts +11 -0
  45. package/src/upgrade.ts +50 -19
  46. package/src/verbs/actions.ts +66 -18
  47. package/src/verbs/protocol.ts +45 -0
  48. package/src/verbs/server.ts +212 -11
  49. package/src/wizard-ui.ts +14 -5
  50. package/src/worker.ts +26 -0
  51. package/systemd/omp-conductor-recover.sh +73 -0
  52. package/systemd/recover-unit-test.sh +61 -0
package/src/daemon.ts CHANGED
@@ -8,7 +8,7 @@
8
8
  * here and enforced before anything is claimed.
9
9
  */
10
10
  import { createHash } from "node:crypto";
11
- import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
11
+ import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
12
12
  import { dirname, join, relative } from "node:path";
13
13
  import {
14
14
  configPath,
@@ -31,7 +31,7 @@ import { digestScheduleState, type DigestScheduleState } from "./digest-schedule
31
31
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
32
32
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
33
33
  import { graphHint } from "./graph.ts";
34
- import { acquireOnceLease, healthCheck, livingDaemon } from "./lifecycle.ts";
34
+ import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
35
35
  import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
36
36
  import { runDoctor } from "./doctor.ts";
37
37
  import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
@@ -52,17 +52,23 @@ import {
52
52
  } from "./reports.ts";
53
53
  import {
54
54
  recordReleaseBlock,
55
+ sharedHostBriefNotice,
55
56
  type GateShape,
56
57
  type ReleaseBlockContext,
57
58
  } from "./release-policy.ts";
58
59
  import { branchName, effectiveLabels, route } from "./routing.ts";
59
60
  import type { Routed, UnroutableReason } from "./routing.ts";
60
- import { admitCandidates, hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
61
- import type { Admission, AdmissionHold } from "./admission.ts";
61
+ import {
62
+ admitCandidates,
63
+ effectiveLane,
64
+ hasContinuationBudget,
65
+ hasFailedAttemptBudget,
66
+ } from "./admission.ts";
67
+ import type { Admission, AdmissionHold, FileLane } from "./admission.ts";
62
68
  import { log, errText, safeEscalate } from "./log.ts";
63
69
  import { materializeOmpSettings } from "./omp-settings.ts";
64
70
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
65
- import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
71
+ import { classifyRun, infraLogSignature, infraSignatureVersion, normalise, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
66
72
  import {
67
73
  DEFAULT_MODEL_FALLBACK_THRESHOLD,
68
74
  fallbackClause,
@@ -73,7 +79,7 @@ import {
73
79
  } from "./model-fallback.ts";
74
80
  import { projectLabels } from "./label-projection.ts";
75
81
  import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
76
- import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
82
+ import { GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
77
83
  import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
78
84
  import type {
79
85
  BaseFreeze,
@@ -97,6 +103,8 @@ import type {
97
103
  ResolvedGrants,
98
104
  FailureClass,
99
105
  RecoveryAction,
106
+ ReviewRevisionOutcome,
107
+ ReviewRevisionRecord,
100
108
  RunRecord,
101
109
  RunState,
102
110
  SettlementFlag,
@@ -104,6 +112,7 @@ import type {
104
112
  Tracker,
105
113
  VerbLedgerEntry,
106
114
  TurnOverride,
115
+ WorkflowRun,
107
116
  } from "./types.ts";
108
117
  import {
109
118
  type KilledBy,
@@ -113,6 +122,7 @@ import {
113
122
  type RunWorkerDeps,
114
123
  ORPHAN_RESUME_PROMPT,
115
124
  renderBrief,
125
+ renderReviewRevisionPrompt,
116
126
  runWorker,
117
127
  } from "./worker.ts";
118
128
  import {
@@ -289,7 +299,9 @@ interface Deps {
289
299
  /**
290
300
  * Reads one active run's file lane for the admission file-lane interlock
291
301
  * (#555): the union of its uncommitted worktree changes and its branch-vs-base
292
- * diff. Wired by `runDaemon` to the mirror/worktree-backed
302
+ * diff. Reconciliation is not occupancy: while the run merges its base, the
303
+ * probe reports only files it has actually diverged on, never the files the
304
+ * merge merely staged (#684). Wired by `runDaemon` to the mirror/worktree-backed
293
305
  * {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert
294
306
  * (no lane is ever known occupied), which is the issue's "fail open": the
295
307
  * gate adds holds, it never refuses a well-formed issue for lack of this
@@ -610,6 +622,135 @@ export function setPaused(
610
622
  }
611
623
  }
612
624
 
625
+ // ----------------------------------------------------------------- admission
626
+ // acknowledgement (#651, review #3)
627
+ //
628
+ // The setup-fence's "acknowledgement" must come from the daemon itself, not a
629
+ // second synchronous worker count: a tick that passed its pause gate before
630
+ // the fence landed can sit in awaited tracker/routing/admission work and claim
631
+ // after the count and before setup mutates. Two places make that impossible
632
+ // and record it durably:
633
+ //
634
+ // - the tick's pause gate re-acknowledges every held pass (the daemon has
635
+ // reached its admission boundary and claims nothing);
636
+ // - the claim itself re-checks the pause immediately before creating the run
637
+ // row, so a tick already past its gate when the fence landed refuses at
638
+ // the claim — and writes the acknowledgement — instead of admitting.
639
+ //
640
+ // The acknowledgement file names the exact pause instance observed and the
641
+ // daemon generation that observed it, so the setup barrier can prove the
642
+ // acknowledged fence is the fence it froze and the acknowledgement belongs to
643
+ // the daemon it began with.
644
+
645
+ /** The durable admission acknowledgement a daemon writes when it observes a
646
+ * pause fence at an admission boundary. */
647
+ export interface AdmissionAckRecord {
648
+ /** The pause instance observed (source token, reason, creation instant). */
649
+ pause: { source: string; reason?: string; since: number };
650
+ /** The daemon generation that observed it — {@link daemonGeneration}. */
651
+ daemon: string;
652
+ /** When the daemon observed the fence. */
653
+ observedAt: number;
654
+ }
655
+
656
+ /** The acknowledgement file path. One per host: the fence is host-global. */
657
+ export function admissionAckPath(): string {
658
+ return join(stateDir(), "admission-ack.json");
659
+ }
660
+
661
+ /**
662
+ * The generation identity of the running daemon, or `undefined` when nothing
663
+ * provably runs. The live pidfile record wins (its pid and boot instant
664
+ * identify the exact instance, #377); a record-less ACTIVE unit is still a
665
+ * running daemon and its MainPID is the generation that lets a fence spot a
666
+ * supervisor restart (#651 review #2). Shared by the ack writer and the setup
667
+ * barrier's identity so both sides of the acknowledgement name the same
668
+ * instance by the same rule.
669
+ */
670
+ export function daemonGeneration(): string | undefined {
671
+ const daemon = livingDaemon();
672
+ if (daemon !== undefined) return `${daemon.pid}@${daemon.startedAt}`;
673
+ const ownership = probeUnit(SYSTEMD_UNIT);
674
+ return ownership.kind === "active" ? `systemd:${ownership.pid}` : undefined;
675
+ }
676
+
677
+ /** Reads the current admission acknowledgement, if one is readable. A corrupt
678
+ * or unreadable file is absence — the barrier fails closed on the absence. */
679
+ export function readAdmissionAck(): AdmissionAckRecord | undefined {
680
+ let parsed: unknown;
681
+ try {
682
+ parsed = JSON.parse(readFileSync(admissionAckPath(), "utf8"));
683
+ } catch {
684
+ return undefined;
685
+ }
686
+ if (parsed === null || typeof parsed !== "object") return undefined;
687
+ const r = parsed as Record<string, unknown>;
688
+ const pause = r["pause"];
689
+ const daemon = r["daemon"];
690
+ const observedAt = r["observedAt"];
691
+ if (pause === null || typeof pause !== "object") return undefined;
692
+ const p = pause as Record<string, unknown>;
693
+ const source = p["source"];
694
+ const since = p["since"];
695
+ const reason = p["reason"];
696
+ if (typeof source !== "string" || source.length === 0) return undefined;
697
+ if (typeof since !== "number" || !Number.isFinite(since)) return undefined;
698
+ if (reason !== undefined && typeof reason !== "string") return undefined;
699
+ if (typeof daemon !== "string" || daemon.length === 0) return undefined;
700
+ if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return undefined;
701
+ return { pause: { source, since, ...(reason === undefined ? {} : { reason }) }, daemon, observedAt };
702
+ }
703
+
704
+ /**
705
+ * Writes the admission acknowledgement for the pause the daemon just observed.
706
+ * The observed instance is the most restrictive fence in force: a host-global
707
+ * sentinel gates every project and is the pause a host-wide transaction (setup,
708
+ * an all-projects hold) froze, so it wins over a project-scoped sentinel;
709
+ * otherwise the project's own sentinel is the effective one for the daemon's
710
+ * claims, matching {@link pauseInstance}. Recording the project sentinel while
711
+ * a global fence is also in force would name a narrower, older hold instead of
712
+ * the fence that actually gates the host — and a barrier that froze the global
713
+ * sentinel could then never match its own acknowledgement (#651 review #4). A
714
+ * no-op when no pause is readable: there is nothing to acknowledge. The
715
+ * generation is the writing daemon's own (the record it owns, or its
716
+ * supervised MainPID).
717
+ */
718
+ export function writeAdmissionAck(project: string): void {
719
+ // `pauseInstance()` reads only the global sentinel; while it is in force it
720
+ // is the fence that gates every project, and the durable record must name it
721
+ // rather than a per-project hold that predates it.
722
+ const pause = pauseInstance() ?? pauseInstance(project);
723
+ if (pause === undefined) return;
724
+ const generation = daemonGeneration();
725
+ if (generation === undefined) return;
726
+ const path = admissionAckPath();
727
+ mkdirSync(dirname(path), { recursive: true });
728
+ const record: AdmissionAckRecord = { pause, daemon: generation, observedAt: Date.now() };
729
+ const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
730
+ try {
731
+ writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
732
+ renameSync(tmp, path);
733
+ } catch (err) {
734
+ rmSync(tmp, { force: true });
735
+ throw err;
736
+ }
737
+ }
738
+
739
+ /** Wake the daemon's dispatch loop to prompt a pass (best-effort; used by the
740
+ * setup barrier so a live daemon acknowledges the fence without waiting for
741
+ * its next five-minute tick). A refused/failed wake just lengthens the wait;
742
+ * the barrier's deadline is what bounds it. */
743
+ export async function wakeDaemon(port: number): Promise<void> {
744
+ try {
745
+ await fetch(`http://127.0.0.1:${port}/wake`, {
746
+ method: "POST",
747
+ signal: AbortSignal.timeout(1_500),
748
+ });
749
+ } catch {
750
+ // Best-effort by contract: the caller's bounded wait is the backstop.
751
+ }
752
+ }
753
+
613
754
  // ----------------------------------------------------------- package integrity
614
755
 
615
756
  /** Enough differing paths to tell a deploy from a tamper at a glance; the full
@@ -775,16 +916,29 @@ const DISCUSSION_CHARS_BUDGET = 8_000;
775
916
  * the #517 failure. An empty comment list renders nothing, so a commentless
776
917
  * issue's brief stays byte-identical to what this package has always shipped.
777
918
  */
778
- function renderDiscussion(comments: IssueComment[] | "unread"): string {
919
+ function renderDiscussion(comments: IssueComment[] | "unread", lane?: FileLane): string {
779
920
  if (comments === "unread") {
780
- return [
921
+ const lines = [
781
922
  "## Discussion",
782
923
  "",
783
924
  "_The issue's comments could not be read at dispatch time. The live read below is_",
784
925
  "_the only path to them; if it prints nothing, that is a failed read, not an_",
785
926
  "_absence of discussion._",
786
927
  "",
787
- ].join("\n");
928
+ ];
929
+ // The lane admission enforced is carried through dispatch, so even an
930
+ // unreadable thread cannot hide it from the worker (#608): the gate's
931
+ // effective declaration renders from the admission snapshot, not from the
932
+ // read that just failed. A body declaration needs no note — the body
933
+ // always renders.
934
+ if (lane !== undefined && lane.at !== "body") {
935
+ lines.push(
936
+ "_The effective file lane below was enforced at admission — it supersedes any earlier declaration._",
937
+ lane.source,
938
+ "",
939
+ );
940
+ }
941
+ return lines.join("\n");
788
942
  }
789
943
  if (comments.length === 0) return "";
790
944
  const total = comments.length;
@@ -812,9 +966,51 @@ function renderDiscussion(comments: IssueComment[] | "unread"): string {
812
966
  "",
813
967
  );
814
968
  }
969
+ // The file-lane gate resolves its effective lane across the whole thread,
970
+ // not just what this budget renders, so a winning declaration beyond the
971
+ // budget would otherwise control admission while the worker never sees it
972
+ // (#608). Reproduce the winning declaration verbatim here — `at` records
973
+ // the comment it came from — so the rendered brief and the gate agree on
974
+ // the same lane, and the note itself re-parses through the same
975
+ // `File lane:` grammar. The note is suppressed only when the winning
976
+ // declaration is provably visible in the rendered thread (the body always
977
+ // renders, so a body winner needs none); a carried admission lane whose
978
+ // comment the dispatch re-read shifted or dropped still renders here,
979
+ // whether or not the thread was truncated.
980
+ if (
981
+ lane !== undefined &&
982
+ lane.at !== "body" &&
983
+ !comments.slice(0, shown).some((c) => c.body.includes(lane.source))
984
+ ) {
985
+ lines.push(
986
+ `_The effective file lane was declared in comment ${lane.at + 1} — it supersedes any earlier declaration._`,
987
+ lane.source,
988
+ "",
989
+ );
990
+ }
815
991
  return lines.join("\n");
816
992
  }
817
993
 
994
+ /**
995
+ * What an orphan-resumed worker is told about the file lane on top of the
996
+ * continuation notice (#608). The original brief already in the transcript
997
+ * may show an earlier declaration, while admission enforces the current one —
998
+ * replaying the whole brief would re-do the work, but continuing under a stale
999
+ * lane is the collision the gate exists to stop. The declaration is rendered
1000
+ * verbatim, so it re-parses through the same `File lane:` grammar every
1001
+ * surfaced declaration uses.
1002
+ */
1003
+ function resumeLaneBlock(lane: FileLane): string {
1004
+ return [
1005
+ "",
1006
+ "The file lane admission enforced for this continuation is below. It supersedes",
1007
+ "any lane declaration in the brief already in your transcript:",
1008
+ "",
1009
+ lane.source,
1010
+ "",
1011
+ ].join("\n");
1012
+ }
1013
+
818
1014
  /**
819
1015
  * Record a state-label swap for projection (#201). The add enqueues before the
820
1016
  * remove — the reverse order would leave a window where the issue carries no
@@ -1066,6 +1262,15 @@ export async function buildBrief(
1066
1262
  * when the tracker refused at dispatch — rare, but it must never read as
1067
1263
  * "no comments" (the #517 failure mode). Absent means an empty list. */
1068
1264
  comments?: IssueComment[] | "unread";
1265
+ /**
1266
+ * The effective file lane admission resolved for this candidate (#608).
1267
+ * When carried, the brief renders exactly this value — the gate's own
1268
+ * snapshot — instead of recomputing from the dispatch-time comment read,
1269
+ * so a changed or failed second read can neither hide nor reword the
1270
+ * lane admission enforced. Absent (unit-level callers), the lane is
1271
+ * resolved from the rendered thread itself.
1272
+ */
1273
+ lane?: FileLane;
1069
1274
  } = {},
1070
1275
  ): Promise<string> {
1071
1276
  // Read per dispatch rather than caching: editing the brief then takes effect
@@ -1103,6 +1308,13 @@ export async function buildBrief(
1103
1308
  "",
1104
1309
  ].join("\n")
1105
1310
  : "";
1311
+ // The comments as rendered, and the effective file lane the brief must show
1312
+ // — the admission-carried value when dispatch has one, else the lane the
1313
+ // thread itself resolves to — passed to the renderer so a winning
1314
+ // declaration can be reproduced verbatim even when it sits beyond the
1315
+ // discussion budget, keeping the gate and the worker-visible brief on one
1316
+ // lane (#608).
1317
+ const comments = opts.comments ?? [];
1106
1318
  return renderBrief(template, {
1107
1319
  ISSUE_NUMBER: String(r.issue.number),
1108
1320
  ISSUE_TITLE: r.issue.title,
@@ -1111,8 +1323,16 @@ export async function buildBrief(
1111
1323
  BRANCH: branch,
1112
1324
  WORKTREE: worktree,
1113
1325
  ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
1114
- ISSUE_COMMENTS: renderDiscussion(opts.comments ?? []),
1326
+ ISSUE_COMMENTS: renderDiscussion(
1327
+ comments,
1328
+ opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments)),
1329
+ ),
1115
1330
  GATES: gatesBlock(r.repo),
1331
+ // The guarded shell suites and their sanctioned parse-only check, derived
1332
+ // from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
1333
+ // (#687). Non-empty whenever the guard has paths, so the placeholder line
1334
+ // always renders to a line.
1335
+ SHARED_HOST_NOTICE: sharedHostBriefNotice(),
1116
1336
  // Empty for a repo with no `graphProject`, and empty means *nothing*: the
1117
1337
  // placeholder sits flush against the next list item in the template, so an
1118
1338
  // unconfigured render leaves no blank line where a hint would have gone.
@@ -1681,7 +1901,12 @@ function orphanResumeVerdict(
1681
1901
  return { kind: "resume", prior };
1682
1902
  }
1683
1903
 
1684
- export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1904
+ export async function handleIssue(
1905
+ d: Deps,
1906
+ r: Routed,
1907
+ attempt: number,
1908
+ admittedLane?: FileLane,
1909
+ ): Promise<void> {
1685
1910
  const { project, caps, tracker, store } = d;
1686
1911
  const issue = r.issue.number;
1687
1912
  const branch = branchName(r.issue);
@@ -1824,6 +2049,24 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1824
2049
  return;
1825
2050
  }
1826
2051
 
2052
+ // The claim-side of the pause fence (#651 review #3): the tick's pause
2053
+ // gate sits above routing, so a tick that passed that gate before an
2054
+ // operator wrote the freeze can still be mid-routing when the fence lands
2055
+ // — it would claim after the setup barrier's acknowledgement and before
2056
+ // its first mutation. The claim itself re-checks the pause and refuses,
2057
+ // writing the durable admission acknowledgement the barrier waits for.
2058
+ // A claim observed under the fence is the last admission boundary there
2059
+ // is: nothing may create a run row beside a held fleet.
2060
+ if (isPaused(d.project.name)) {
2061
+ try {
2062
+ writeAdmissionAck(d.project.name);
2063
+ } catch (err) {
2064
+ log(`admission acknowledgement write failed: ${errText(err)}`);
2065
+ }
2066
+ log(`#${issue} not claimed: dispatch is paused at claim time`);
2067
+ return;
2068
+ }
2069
+
1827
2070
  // Claim on the STORE first, before anything that can fail. The run row —
1828
2071
  // not the label — is the crash-safe guard against double dispatch: rows
1829
2072
  // are local, written before any network call, and the startup orphan
@@ -2043,10 +2286,15 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2043
2286
  // The continuation notice replaces the brief for a resumed attempt (#536).
2044
2287
  // The original brief is already in the resumed transcript; re-sending it is
2045
2288
  // how a resumed worker ends up re-doing the work it just did. Everything
2046
- // below the brief is fresh-dispatch-only, exactly as today.
2289
+ // below the brief is fresh-dispatch-only, exactly as today — except the
2290
+ // file lane admission enforced for this continuation, which rides the
2291
+ // notice itself: the retained transcript's brief may show an earlier
2292
+ // declaration, and a worker continuing under a stale lane is the collision
2293
+ // the gate exists to stop (#608).
2047
2294
  let brief: string;
2048
2295
  if (resuming !== undefined) {
2049
- brief = ORPHAN_RESUME_PROMPT;
2296
+ brief =
2297
+ ORPHAN_RESUME_PROMPT + (admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
2050
2298
  } else {
2051
2299
  // The discussion is rendered at dispatch so a worker never depends on a
2052
2300
  // runtime `gh` read to see the orchestrator's grooming (#517). The read is
@@ -2068,6 +2316,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2068
2316
  ? { salvagedSha: priorSalvage }
2069
2317
  : {}),
2070
2318
  comments,
2319
+ lane: admittedLane,
2071
2320
  });
2072
2321
  }
2073
2322
  if (await settleStopBeforeSession()) return;
@@ -2535,6 +2784,749 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2535
2784
  }
2536
2785
  }
2537
2786
 
2787
+ // ------------------------------------------------------------ review revisions
2788
+
2789
+ /**
2790
+ * The daemon half of the review-revision transport (#677): wake every pending
2791
+ * revision the orchestrator's verb recorded, on the tick, exactly like
2792
+ * admission — the verb has no handle on the worker machinery, and the CLI path
2793
+ * must behave like the embedded one. Bounded by the same worker slots, so a
2794
+ * queued revision can never push the fleet past `maxConcurrentWorkers`; the
2795
+ * remaining rows stay pending for the next tick.
2796
+ *
2797
+ * The claim (`pushed-green` → `running`) is synchronous and atomic per row;
2798
+ * only the resumed worker runs in the pool. A row that moved since the verb
2799
+ * recorded it — merged after all, settled, re-claimed — fails the claim and is
2800
+ * settled `skipped` rather than woken on stale identity.
2801
+ */
2802
+ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
2803
+ const pending = d.store.pendingReviewRevisions(d.project.name);
2804
+ if (pending.length === 0) return;
2805
+ const slots = Math.max(0, d.caps.maxConcurrentWorkers - d.store.liveRuns(d.project.name).length);
2806
+ const batch = pending.slice(0, slots);
2807
+ const launches: Promise<void>[] = [];
2808
+ for (const revision of batch) {
2809
+ if (d.drain?.draining === true) {
2810
+ log(
2811
+ `review revisions held: the daemon is draining (${batch.length - launches.length} of ${batch.length} not launched)`,
2812
+ );
2813
+ break;
2814
+ }
2815
+ if (!d.store.claimRunForReview(revision.runId)) {
2816
+ d.store.settleReviewRevision(revision.id, "skipped", Date.now());
2817
+ log(
2818
+ `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer pushed-green`,
2819
+ );
2820
+ continue;
2821
+ }
2822
+ d.store.markReviewRevisionDispatched(revision.id, Date.now());
2823
+ log(`#${revision.issue} review round ${revision.round} → resuming run ${revision.runId}`);
2824
+ launches.push(handleReviewRevision(d, revision));
2825
+ }
2826
+ if (pool !== undefined) {
2827
+ for (const launch of launches) pool.launch(launch);
2828
+ return;
2829
+ }
2830
+ // `--once`: like admissions, the tick waits for the workers it launched.
2831
+ await Promise.allSettled(launches);
2832
+ }
2833
+
2834
+ /**
2835
+ * Resume one review-revision worker: the run row is already claimed
2836
+ * (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
2837
+ * OMP session — same session directory, `resume: true` at the harness, and the
2838
+ * `sessionFile` lineage compared at dispatch time — on the SAME branch and PR,
2839
+ * then settles the row back the way {@link handleIssue} settles any worker.
2840
+ *
2841
+ * The run row is reused, never cloned: same attempt number, same prUrl, same
2842
+ * transcript. That is what keeps a revision round off both budgets — the round
2843
+ * is a state transition of one already-green row, green → running → green, and
2844
+ * no new row exists for either counter to count. Only a revision worker that
2845
+ * genuinely settles terminal falls through to the ordinary failure handling,
2846
+ * budgets and all, exactly as today.
2847
+ */
2848
+ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionRecord): Promise<void> {
2849
+ const { project, caps, tracker, store } = d;
2850
+ const run = store.getRun(revision.runId);
2851
+ if (run === undefined || run.state !== "running") {
2852
+ log(
2853
+ `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${run?.state ?? "gone"} — ` +
2854
+ "the row moved between the dispatch claim and the resume.",
2855
+ );
2856
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2857
+ return;
2858
+ }
2859
+ const issue = run.issue;
2860
+ const branch = run.branch;
2861
+ const runId = run.id;
2862
+ const inProgress = project.stateLabels.inProgress;
2863
+ const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
2864
+ if (repo === undefined) {
2865
+ log(`#${issue} review round ${revision.round} cannot dispatch: repo ${run.repo} is no longer routed`);
2866
+ store.updateRun(runId, { state: "pushed-green", endedAt: Date.now() });
2867
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2868
+ return;
2869
+ }
2870
+ const mirrorPath = mirrorPathFor(repo, project.mirrorRoot);
2871
+
2872
+ // The session being resumed is the one the pushed-green row recorded. A
2873
+ // revision without that transcript cannot continue the same session — the
2874
+ // exact fake this feature exists to prevent — so fail loudly instead of
2875
+ // silently starting a fresh one.
2876
+ const priorSessionFile = run.sessionFile;
2877
+ if (priorSessionFile === undefined || !existsSync(priorSessionFile)) {
2878
+ log(
2879
+ `#${issue} review round ${revision.round} cannot resume: the recorded transcript ` +
2880
+ `${priorSessionFile ?? "(none)"} is gone`,
2881
+ );
2882
+ store.updateRun(runId, {
2883
+ state: "pushed-green",
2884
+ endedAt: Date.now(),
2885
+ lastError: `review round ${revision.round} could not resume: the run's transcript is gone`,
2886
+ });
2887
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2888
+ await safeEscalate(d, {
2889
+ tier: 1,
2890
+ project: project.name,
2891
+ issue,
2892
+ runId,
2893
+ summary: `#${issue} review round ${revision.round} could not resume — the transcript is missing`,
2894
+ detail: [
2895
+ revision.prUrl,
2896
+ "",
2897
+ "The PR is still open and green; the run row was returned to pushed-green.",
2898
+ "Resume the manual path (close + unblock + continuation) or merge it as it stands.",
2899
+ ].join("\n"),
2900
+ });
2901
+ return;
2902
+ }
2903
+ const sessionDir = dirname(priorSessionFile);
2904
+
2905
+ let worktreePath: string | undefined;
2906
+ let runRepo: RunRepoRef | undefined;
2907
+ let turnLimit: TurnLimitController | undefined;
2908
+ let workerControl: WorkerControlSlot | undefined;
2909
+ let workerSessionInstalled = false;
2910
+ let verbListener: VerbListener | undefined;
2911
+ const repoSlug = githubRepo(repo.cloneUrl);
2912
+
2913
+ /**
2914
+ * Publishes the run's branch on the privileged side: run repo → mirror →
2915
+ * GitHub, fast-forward only — the same route `handleIssue` uses, so the
2916
+ * revised head the worker pushed through its own `conductor_push` is
2917
+ * re-verified against the remote before the tree dies.
2918
+ */
2919
+ const publish: RunPublisher = async () => {
2920
+ if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
2921
+ return pushRunBranch(project, runRepo);
2922
+ };
2923
+
2924
+ const settleStopBeforeSession = async (): Promise<boolean> => {
2925
+ const reason = workerControl?.requestedStop();
2926
+ if (reason === undefined || workerSessionInstalled) return false;
2927
+ turnLimit?.close();
2928
+ turnLimit = undefined;
2929
+ recordOperatorStop(store, {
2930
+ project: project.name,
2931
+ issue,
2932
+ runId,
2933
+ inProgress,
2934
+ reason,
2935
+ patch: {
2936
+ endedAt: Date.now(),
2937
+ report: ["Operator stopped the review revision before its session started.", `Reason: ${reason}`].join("\n"),
2938
+ },
2939
+ });
2940
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2941
+ log(`#${issue} review round ${revision.round} stopped by operator before its session started: ${reason}`);
2942
+ return true;
2943
+ };
2944
+
2945
+ const settleDrainBeforeSession = async (): Promise<boolean> => {
2946
+ if (d.drain?.draining !== true || workerSessionInstalled) return false;
2947
+ turnLimit?.close();
2948
+ turnLimit = undefined;
2949
+ // A shutdown that interrupts the wake is the same situation as the
2950
+ // provisioning-failure branch below, not an operator stop — nobody
2951
+ // stopped anything, the PR is still green and still open, and only the
2952
+ // wake failed; `settleStopBeforeSession` immediately above covers the
2953
+ // case where an operator genuinely did. So mirror that branch: restore
2954
+ // `pushed-green` naming the shutdown, settle the revision `skipped` so
2955
+ // the round is not double-counted, and leave the in-progress label alone
2956
+ // — the ordinary settle sweep releases it in the same breath it
2957
+ // terminalises the row when the PR resolves. Stopping the row here would
2958
+ // strand it: `settlePushedGreen` sweeps only pushed-* rows and
2959
+ // classification excludes `stopped`, so the issue would carry neither
2960
+ // label and nothing would ever revisit the still-green PR.
2961
+ store.updateRun(runId, {
2962
+ state: "pushed-green",
2963
+ endedAt: Date.now(),
2964
+ lastError: "daemon shutdown began after the review revision claim; the round was not launched",
2965
+ });
2966
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2967
+ log(`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim`);
2968
+ return true;
2969
+ };
2970
+
2971
+ // Live-run controls, opened before any provisioning so the operator stop
2972
+ // and shutdown fences cover the whole wake window, exactly as they do for a
2973
+ // fresh claim in `handleIssue` (#374). The original run's entries were
2974
+ // closed at its settle, so reopening by issue is safe.
2975
+ turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
2976
+ workerControl = d.workerControls.open(project.name, issue, runId);
2977
+ if (await settleStopBeforeSession()) return;
2978
+ if (await settleDrainBeforeSession()) return;
2979
+
2980
+ try {
2981
+ // Reattach the run's own branch at the same per-issue path the run used:
2982
+ // the worktree was removed at the pushed-green settle and the branch was
2983
+ // published, so provisioning is the same continuation reattach as a
2984
+ // normal re-claim. A provisioning failure restores the row — the PR is
2985
+ // still green and still open, and only the wake failed — and says so.
2986
+ try {
2987
+ const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
2988
+ worktreePath = provisioned.path;
2989
+ runRepo = { repo, runRepoPath: worktreePath, branch };
2990
+ } catch (err) {
2991
+ const detail = errText(err);
2992
+ log(`#${issue} review round ${revision.round} cannot provision ${branch}: ${detail}`);
2993
+ // An operator stop that landed during provisioning wins over restoration —
2994
+ // it is a newer, explicit action, and the fences above already honoured
2995
+ // one that landed earlier.
2996
+ const pendingStop = workerControl?.requestedStop();
2997
+ if (pendingStop === undefined) {
2998
+ store.updateRun(runId, {
2999
+ state: "pushed-green",
3000
+ endedAt: Date.now(),
3001
+ lastError: `review round ${revision.round} could not provision the worktree: ${detail}`,
3002
+ });
3003
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3004
+ await safeEscalate(d, {
3005
+ tier: 1,
3006
+ project: project.name,
3007
+ issue,
3008
+ runId,
3009
+ summary: `#${issue} review round ${revision.round} could not be dispatched — worktree provisioning failed`,
3010
+ detail: [revision.prUrl, "", detail].join("\n"),
3011
+ });
3012
+ } else {
3013
+ recordOperatorStop(store, {
3014
+ project: project.name,
3015
+ issue,
3016
+ runId,
3017
+ inProgress,
3018
+ reason: pendingStop,
3019
+ patch: {
3020
+ endedAt: Date.now(),
3021
+ report: [
3022
+ `Operator stopped the review revision before its session started: ${pendingStop}`,
3023
+ `(worktree provisioning also failed: ${detail})`,
3024
+ ].join("\n"),
3025
+ },
3026
+ });
3027
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3028
+ }
3029
+ return;
3030
+ }
3031
+ if (await settleStopBeforeSession()) return;
3032
+ if (await settleDrainBeforeSession()) return;
3033
+
3034
+ // The fleet-owned omp settings overlay, staged under the SAME session
3035
+ // directory the transcript lives in — a resumed session loads it exactly
3036
+ // as the original run did (#537).
3037
+ const ompSettingsFile = materializeOmpSettings(project, sessionDir);
3038
+
3039
+ // The run's mutation channel (#126): same run id, same issue, same repo —
3040
+ // so `conductor_push` publishes the run's own branch and nothing else.
3041
+ verbListener = await listenVerbChannel(
3042
+ verbDeps(d),
3043
+ {
3044
+ kind: "run",
3045
+ path: verbSocketPath(ensureVerbSocketDir(stateDir()), `run-${String(issue)}`),
3046
+ project: project.name,
3047
+ role: "worker",
3048
+ runId,
3049
+ issue,
3050
+ repo,
3051
+ runRepoPath: worktreePath,
3052
+ branch,
3053
+ },
3054
+ { ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
3055
+ );
3056
+ if (await settleStopBeforeSession()) return;
3057
+ if (await settleDrainBeforeSession()) return;
3058
+
3059
+ store.updateRun(runId, { worktree: worktreePath });
3060
+
3061
+ // The row's own counters are cumulative across the revision: the revision
3062
+ // worker meters its own session from zero, so its deltas are added to the
3063
+ // totals the run already recorded.
3064
+ const baseTurns = run.turns;
3065
+ const baseSpend = run.spendUsd;
3066
+
3067
+ log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
3068
+
3069
+ let result: WorkerResult;
3070
+ try {
3071
+ result = await runWorker({
3072
+ brief: renderReviewRevisionPrompt(revision.findings, revision.round),
3073
+ cwd: worktreePath,
3074
+ caps,
3075
+ ...(repoSlug === undefined ? {} : { repoSlug }),
3076
+ maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
3077
+ onPauseControl: (control) => {
3078
+ workerSessionInstalled = true;
3079
+ workerControl?.install(control);
3080
+ },
3081
+ sessionDir,
3082
+ // The same-session half of the transport: `continueRecent` on the run's
3083
+ // own session directory, never a fresh `run-<uuid>` transcript.
3084
+ resume: true,
3085
+ socketPath: join(sessionDir, "ipc.sock"),
3086
+ verbSocketPath: verbListener.path,
3087
+ onSpawn: (pid) => {
3088
+ verbListener?.bindPid(pid);
3089
+ },
3090
+ onChildLog: (line) => {
3091
+ log(`#${issue} ${line}`);
3092
+ },
3093
+ // The continuation stays on the model the green run used (#286).
3094
+ ...(run.model === undefined ? {} : { model: run.model }),
3095
+ ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
3096
+ releaseGrants: resolveReleaseGrants(project),
3097
+ onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
3098
+ onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
3099
+ onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
3100
+ onKilled: () => {
3101
+ turnLimit?.close();
3102
+ turnLimit = undefined;
3103
+ },
3104
+ onSessionFile: (f) => {
3105
+ store.updateRun(runId, { sessionFile: f });
3106
+ // Same-file lineage is the proof the resume happened: a revision
3107
+ // that opened a different transcript fell back to a fresh session.
3108
+ if (f !== priorSessionFile) {
3109
+ log(
3110
+ `#${issue} review round ${revision.round} resume fell back to a fresh session: opened ${f} ` +
3111
+ `instead of ${priorSessionFile}`,
3112
+ );
3113
+ }
3114
+ },
3115
+ maySpawn: () => d.drain?.draining !== true,
3116
+ }, d.workerDeps);
3117
+ } finally {
3118
+ turnLimit?.close();
3119
+ turnLimit = undefined;
3120
+ }
3121
+
3122
+ if (result.modelFallbackMessage !== undefined) {
3123
+ log(`#${issue} review round ${revision.round} model fallback: ${result.modelFallbackMessage}`);
3124
+ }
3125
+
3126
+ const verified: { state: RunState; reason?: string } =
3127
+ result.state === "pushed-green"
3128
+ ? await verifyPushedGreenClaim(tracker, result)
3129
+ : { state: result.state };
3130
+ const state = verified.state;
3131
+
3132
+ const settlement =
3133
+ state === "pushed-green" || state === "pushed-pending" || state === "merged"
3134
+ ? undefined
3135
+ : await settleWorktree({
3136
+ issue,
3137
+ attempt: run.attempt,
3138
+ ending:
3139
+ state === "blocked"
3140
+ ? "blocked for an operator decision"
3141
+ : state === "stopped"
3142
+ ? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
3143
+ : endedBy(result.killedBy),
3144
+ worktree: worktreePath,
3145
+ branch,
3146
+ publish,
3147
+ ...(state === "failed" || state === "killed"
3148
+ ? ({ tree: "keep" } as const)
3149
+ : ({ tree: "remove", mirrorPath } as const)),
3150
+ });
3151
+ if (settlement === undefined) {
3152
+ const published = await publish(branch);
3153
+ if (!published.ok) {
3154
+ log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
3155
+ }
3156
+ await removeWorktree(mirrorPath, worktreePath);
3157
+ }
3158
+
3159
+ // The settlement report names the review round first (#692): a run that
3160
+ // reaches green again after a round must read as a revision outcome, not
3161
+ // as a fresh run's first push, and a round that failed must still say the
3162
+ // round — the state and classification below carry the failure itself.
3163
+ const finalReport = [
3164
+ `review round ${revision.round}: ${state}`,
3165
+ ...(verified.reason === undefined ? [] : ["", verified.reason]),
3166
+ result.report,
3167
+ ].join("\n");
3168
+
3169
+ const terminalPatch: Partial<RunRecord> = {
3170
+ endedAt: Date.now(),
3171
+ turns: baseTurns + result.turns,
3172
+ spendUsd: baseSpend + result.spendUsd,
3173
+ provider429Count: result.provider429Count,
3174
+ ...(result.model === undefined ? {} : { resolvedModel: result.model }),
3175
+ ...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
3176
+ retryFallbacks: result.retryFallbacks,
3177
+ retryFallbackSucceeded: result.retryFallbackSucceeded,
3178
+ modelRecoveries: result.modelRecoveries,
3179
+ autoRetryCount: result.autoRetryCount,
3180
+ autoCompactionCount: result.autoCompactionCount,
3181
+ // The worker only reports these when it actually established them; a
3182
+ // failure whose report named no PR must not wipe what the row already
3183
+ // owns (#468).
3184
+ ...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
3185
+ ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
3186
+ sessionFile: result.sessionFile,
3187
+ report: finalReport,
3188
+ ...settlement?.patch,
3189
+ };
3190
+ if (state === "stopped") {
3191
+ recordOperatorStop(store, {
3192
+ project: project.name,
3193
+ issue,
3194
+ runId,
3195
+ inProgress,
3196
+ reason: result.stoppedReason ?? "no reason recorded",
3197
+ patch: terminalPatch,
3198
+ });
3199
+ } else {
3200
+ const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
3201
+ const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
3202
+ const providerTransient =
3203
+ providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
3204
+ const lastError = completionLastError(providerCredit, providerTransient, verified.reason, sessionErr);
3205
+ store.updateRun(runId, {
3206
+ ...terminalPatch,
3207
+ state,
3208
+ ...(lastError === undefined ? {} : { lastError }),
3209
+ });
3210
+ }
3211
+
3212
+ const outcome: ReviewRevisionOutcome =
3213
+ state === "pushed-green" ? "revised" : state === "pushed-pending" ? "pending" : "failed";
3214
+ store.settleReviewRevision(revision.id, outcome, Date.now());
3215
+ log(`#${issue} review round ${revision.round} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
3216
+
3217
+ // Post-settle outcomes, mirroring handleIssue: a failed or killed revision
3218
+ // worker falls through to exactly the same label, requeue and escalation
3219
+ // handling as any failed run — budgets included — and a blocked one pages
3220
+ // the orchestrator for a decision.
3221
+ if (state === "stopped") {
3222
+ log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
3223
+ } else if (state === "blocked") {
3224
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
3225
+ await safeEscalate(d, {
3226
+ tier: 1,
3227
+ project: project.name,
3228
+ issue,
3229
+ runId,
3230
+ summary: `#${issue} is blocked on review round ${revision.round} and needs a decision`,
3231
+ detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
3232
+ });
3233
+ } else if (state === "failed" || state === "killed") {
3234
+ const continuation = store.continuationsFor(project.name, issue);
3235
+ const continueTurns = shouldContinueAfterTurnsCap({
3236
+ killedBy: result.killedBy,
3237
+ prUrl: result.prUrl,
3238
+ headSha: result.headSha,
3239
+ salvageSha: settlement?.patch?.salvageSha,
3240
+ continuation,
3241
+ maxContinuations: caps.maxContinuationsPerIssue,
3242
+ });
3243
+ if (continueTurns) {
3244
+ store.enqueueLabelOps(project.name, [
3245
+ { issue, op: "remove", label: inProgress },
3246
+ { issue, op: "add", label: project.queueLabel },
3247
+ ]);
3248
+ log(
3249
+ `#${issue} review round ${revision.round} turns-cap, continuation ` +
3250
+ `${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
3251
+ );
3252
+ await safeEscalate(d, {
3253
+ tier: 1,
3254
+ project: project.name,
3255
+ issue,
3256
+ runId,
3257
+ summary: `#${issue} hit the turns cap on review round ${revision.round} — auto-requeued for continuation`,
3258
+ detail: [
3259
+ `${result.prUrl ?? "(no PR URL)"}`,
3260
+ "",
3261
+ "The queue label is back on; the next tick should reattach the branch",
3262
+ "and open a continuation brief. No failed label was applied.",
3263
+ "",
3264
+ result.report,
3265
+ ].join("\n"),
3266
+ });
3267
+ } else {
3268
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
3269
+ await safeEscalate(d, {
3270
+ tier: 1,
3271
+ project: project.name,
3272
+ issue,
3273
+ runId,
3274
+ summary: result.killedBy
3275
+ ? `#${issue} was killed on review round ${revision.round} by the ${result.killedBy} cap`
3276
+ : `#${issue} failed on review round ${revision.round}`,
3277
+ detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
3278
+ });
3279
+ }
3280
+ }
3281
+ } catch (err) {
3282
+ // A crash anywhere after the worker ran settles like any run's dispatch
3283
+ // error: the row is terminal, the tree is kept, and the issue is relabelled
3284
+ // so nobody silently re-claims it.
3285
+ turnLimit?.close();
3286
+ turnLimit = undefined;
3287
+ const detail = errText(err);
3288
+ log(`#${issue} review round ${revision.round} errored: ${detail}`);
3289
+ const settlement =
3290
+ worktreePath === undefined
3291
+ ? undefined
3292
+ : await settleWorktree({
3293
+ issue,
3294
+ attempt: run.attempt,
3295
+ ending: "killed by a dispatch error",
3296
+ worktree: worktreePath,
3297
+ branch,
3298
+ publish,
3299
+ tree: "keep",
3300
+ });
3301
+ store.updateRun(runId, {
3302
+ state: "failed",
3303
+ endedAt: Date.now(),
3304
+ lastError: detail,
3305
+ ...settlement?.patch,
3306
+ });
3307
+ store.settleReviewRevision(revision.id, "failed", Date.now());
3308
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
3309
+ await safeEscalate(d, {
3310
+ tier: 1,
3311
+ project: project.name,
3312
+ issue,
3313
+ runId,
3314
+ summary: `#${issue} review round ${revision.round} could not be dispatched`,
3315
+ detail: [
3316
+ `${revision.prUrl}`,
3317
+ "",
3318
+ detail,
3319
+ ...(settlement?.lines ?? []),
3320
+ ].join("\n"),
3321
+ });
3322
+ } finally {
3323
+ turnLimit?.close();
3324
+ if (verbListener !== undefined) {
3325
+ try {
3326
+ await verbListener.close();
3327
+ } catch (err) {
3328
+ log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
3329
+ }
3330
+ verbListener = undefined;
3331
+ }
3332
+ workerControl?.close();
3333
+ workerControl = undefined;
3334
+ }
3335
+ }
3336
+
3337
+ /**
3338
+ * Restart recovery for in-flight review revisions (#692): a revision the
3339
+ * previous daemon claimed (`pushed-green` → `running`) and lost when it died
3340
+ * is restored to the exact state the orchestrator's verb left it in — same
3341
+ * run, same PR, same findings, same round, same session file — so the next
3342
+ * dispatch pass resumes the SAME OMP session rather than leaving the run to
3343
+ * the failure classifier.
3344
+ *
3345
+ * Runs AFTER the startup orphan sweep, on purpose: the sweep has already
3346
+ * salvaged the crashed revision's worktree to the branch (so no work is lost)
3347
+ * and marked the row `orphaned` — and this reconcile then restores that row
3348
+ * to `pushed-green` and re-queues the round before the first tick, so the
3349
+ * failure classifier never sees it. That ordering matters twice over: an
3350
+ * orphaned row would charge the continuation budget for a round that must
3351
+ * stay off it (#677), and the `review_revisions` row is the only place the
3352
+ * findings and round live. A revision the previous daemon never claimed (its
3353
+ * run is still `pushed-green`, its row still pending) needs no restoration —
3354
+ * the ordinary dispatch pass wakes it on the next tick, exactly as #677
3355
+ * already guarantees.
3356
+ *
3357
+ * A round whose run cannot be restored — the run is gone, settled terminal,
3358
+ * or its repo is no longer routed — fails closed instead: the revision is
3359
+ * settled `skipped` with the findings still on the row, ONE escalation names
3360
+ * what could not be restored, and no worker is started against a PR that may
3361
+ * already have one.
3362
+ */
3363
+ export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRevisionRecord[]> {
3364
+ const { project, store } = d;
3365
+ const revisions = store.unsettledReviewRevisions(project.name);
3366
+ if (revisions.length === 0) return [];
3367
+ const recovered: ReviewRevisionRecord[] = [];
3368
+ for (const revision of revisions) {
3369
+ const run = store.getRun(revision.runId);
3370
+ if (run === undefined) {
3371
+ log(
3372
+ `#${revision.issue} review round ${revision.round} cannot be restored across the restart: ` +
3373
+ `run ${revision.runId} no longer exists`,
3374
+ );
3375
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3376
+ await safeEscalate(d, {
3377
+ tier: 1,
3378
+ project: project.name,
3379
+ issue: revision.issue,
3380
+ runId: revision.runId,
3381
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3382
+ detail: [
3383
+ revision.prUrl,
3384
+ "",
3385
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3386
+ `the run row ${revision.runId} no longer exists.`,
3387
+ "The round was settled skipped; the findings stay on this revision's durable row.",
3388
+ "Re-review the PR (it is still open and green) or merge it as it stands.",
3389
+ ].join("\n"),
3390
+ });
3391
+ continue;
3392
+ }
3393
+ // A revision the previous daemon never claimed: the run is still a
3394
+ // settled green row and the round is still pending, so the ordinary
3395
+ // dispatch pass wakes it on the next tick untouched.
3396
+ if (run.state === "pushed-green") continue;
3397
+ // The run a previous daemon claimed for this round and died on: the
3398
+ // orphan sweep just marked it `orphaned` (salvaging the tree to the
3399
+ // branch), so restore it to the reviewable state the verb recorded.
3400
+ const crashed =
3401
+ run.state === "claimed" || run.state === "running" || run.state === "orphaned";
3402
+ if (!crashed) {
3403
+ log(
3404
+ `#${revision.issue} review round ${revision.round} cannot be restored across the restart: run ` +
3405
+ `${revision.runId} is ${run.state}`,
3406
+ );
3407
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3408
+ await safeEscalate(d, {
3409
+ tier: 1,
3410
+ project: project.name,
3411
+ issue: revision.issue,
3412
+ runId: revision.runId,
3413
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3414
+ detail: [
3415
+ revision.prUrl,
3416
+ "",
3417
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3418
+ `the run row is ${run.state}, not a state a review round resumes from.`,
3419
+ "The round was settled skipped; the findings stay on this revision's durable row.",
3420
+ "Re-review the PR (it is still open and green) or merge it as it stands.",
3421
+ ].join("\n"),
3422
+ });
3423
+ continue;
3424
+ }
3425
+ try {
3426
+ // The crashed revision's worktree — salvaged and published by the orphan
3427
+ // sweep that just ran, so the branch in the mirror has every committed
3428
+ // byte — must be gone before the next dispatch, or `addRunRepo` refuses
3429
+ // the reattach as possibly holding a previous attempt's work (#692).
3430
+ // `settleWorktree` with `tree: "remove"` re-attempts the salvage instead
3431
+ // of trusting it (a tree whose salvage failed is the only copy of work
3432
+ // and is NEVER removed), and refuses the removal if that re-attempt
3433
+ // fails — the fail-closed half of this recovery.
3434
+ if (run.worktree !== "" && existsSync(run.worktree)) {
3435
+ const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
3436
+ if (repo === undefined) {
3437
+ log(
3438
+ `#${revision.issue} review round ${revision.round} cannot clear worktree ${run.worktree}: ` +
3439
+ `repo ${run.repo} is no longer routed — the tree may hold the only copy of work`,
3440
+ );
3441
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3442
+ await safeEscalate(d, {
3443
+ tier: 1,
3444
+ project: project.name,
3445
+ issue: revision.issue,
3446
+ runId: revision.runId,
3447
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3448
+ detail: [
3449
+ revision.prUrl,
3450
+ "",
3451
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3452
+ `repo ${run.repo} is no longer routed, so the crashed revision's worktree cannot be published and removed.`,
3453
+ `The worktree is retained for recovery: ${run.worktree}`,
3454
+ "Recover it by hand (it may hold the round's last work), then re-review or merge the PR.",
3455
+ ].join("\n"),
3456
+ });
3457
+ continue;
3458
+ }
3459
+ const settlement = await settleWorktree({
3460
+ issue: run.issue,
3461
+ attempt: run.attempt,
3462
+ ending: "interrupted by a daemon restart during a review round",
3463
+ worktree: run.worktree,
3464
+ branch: run.branch,
3465
+ publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
3466
+ tree: "remove",
3467
+ mirrorPath: mirrorPathFor(repo, project.mirrorRoot),
3468
+ });
3469
+ if (settlement.retained) {
3470
+ log(
3471
+ `#${revision.issue} review round ${revision.round} worktree ${run.worktree} retained: ` +
3472
+ "its salvage failed, so the tree is the only copy of work and will not be removed",
3473
+ );
3474
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3475
+ await safeEscalate(d, {
3476
+ tier: 1,
3477
+ project: project.name,
3478
+ issue: revision.issue,
3479
+ runId: revision.runId,
3480
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3481
+ detail: [
3482
+ revision.prUrl,
3483
+ "",
3484
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3485
+ "the crashed revision's worktree could not be salvaged, so it is the only copy of work and was retained.",
3486
+ `Recover the tree by hand: ${run.worktree}`,
3487
+ "Then re-review the PR (still open and green) or merge it as it stands.",
3488
+ ].join("\n"),
3489
+ });
3490
+ continue;
3491
+ }
3492
+ }
3493
+ } catch (err) {
3494
+ // A settleWorktree throw (mirror unreachable, git refused) must not take
3495
+ // the whole startup reconcile down with one run's tree: fail this round
3496
+ // closed and let the next one try.
3497
+ const detail = errText(err);
3498
+ log(`#${revision.issue} review round ${revision.round} restore errored: ${detail}`);
3499
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3500
+ await safeEscalate(d, {
3501
+ tier: 1,
3502
+ project: project.name,
3503
+ issue: revision.issue,
3504
+ runId: revision.runId,
3505
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3506
+ detail: [revision.prUrl, "", detail].join("\n"),
3507
+ });
3508
+ continue;
3509
+ }
3510
+ // The exact state the verb recorded: a pushed-green row plus a pending
3511
+ // revision whose durable row still carries the findings, reviewed head,
3512
+ // round and target session. The next tick's dispatch pass claims and
3513
+ // wakes it exactly like a revision that was never dispatched.
3514
+ store.updateRun(revision.runId, {
3515
+ state: "pushed-green",
3516
+ endedAt: Date.now(),
3517
+ worktree: "",
3518
+ lastError: `review round ${revision.round} was restored across a daemon restart; the round is re-queued`,
3519
+ });
3520
+ store.requeueReviewRevision(revision.id);
3521
+ log(
3522
+ `#${revision.issue} review round ${revision.round} restored across restart: run ${revision.runId} ` +
3523
+ "is pushed-green again and the round is re-queued — the next dispatch pass resumes the same session",
3524
+ );
3525
+ recovered.push(revision);
3526
+ }
3527
+ return recovered;
3528
+ }
3529
+
2538
3530
  // ------------------------------------------------------------------- settlement
2539
3531
 
2540
3532
  /** What a resolved PR turns its `pushed-green` row into. */
@@ -3277,6 +4269,7 @@ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
3277
4269
  "parent-lookup-error",
3278
4270
  "open-pr-lookup-error",
3279
4271
  "issue-state-lookup-error",
4272
+ "critical-base-verify-error",
3280
4273
  ]);
3281
4274
 
3282
4275
  /** Groups transient decisions into the bounded record exposed by status. */
@@ -3632,6 +4625,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3632
4625
  // so `last dispatch` keeps moving while the fleet is deliberately parked. A
3633
4626
  // frozen clock is then a stall report, not a hold (#497).
3634
4627
  if (isPaused(d.project.name)) {
4628
+ // The held pass is the daemon-side admission acknowledgement: the daemon
4629
+ // has reached its admission boundary under the fence and claims nothing.
4630
+ // Written durably so the setup barrier can prove the fence was observed
4631
+ // (#651 review #3) — a pause cannot acknowledge itself.
4632
+ try {
4633
+ writeAdmissionAck(d.project.name);
4634
+ } catch (err) {
4635
+ log(`admission acknowledgement write failed: ${errText(err)}`);
4636
+ }
3635
4637
  d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
3636
4638
  return;
3637
4639
  }
@@ -3694,6 +4696,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3694
4696
  return;
3695
4697
  }
3696
4698
 
4699
+ // Review revisions are dispatch, not admission (#677): the orchestrator's
4700
+ // verb recorded them durably and the run row already occupies its issue, so
4701
+ // they bypass the ready queue and the claim path entirely — this pass wakes
4702
+ // them (bounded by the same worker slots) BEFORE the queue's capacity gate
4703
+ // reads live runs, so a claimed revision counts against capacity exactly like
4704
+ // the worker it is about to spawn.
4705
+ try {
4706
+ await dispatchReviewRevisions(d, workers);
4707
+ } catch (err) {
4708
+ log(`review revision dispatch failed: ${errText(err)}`);
4709
+ }
4710
+
3697
4711
  // route() filters the queue through isEligible() itself, so anything already
3698
4712
  // carrying a state label is gone before it gets here.
3699
4713
  const ready = await d.tracker.listReady();
@@ -3817,7 +4831,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3817
4831
  log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
3818
4832
  await dispatchAdmissions(
3819
4833
  pass.admitted,
3820
- (a) => handleIssue(d, a.r, a.attempt),
4834
+ (a) => handleIssue(d, a.r, a.attempt, a.lane),
3821
4835
  workers,
3822
4836
  );
3823
4837
 
@@ -4213,6 +5227,15 @@ export interface StatusSnapshot {
4213
5227
  releaseGrants: ResolvedGrants;
4214
5228
  /** Occupied issues: live workers plus green PRs awaiting a human merge. */
4215
5229
  activeRuns: RunRecord[];
5230
+ /**
5231
+ * runId → live review-revision round, for runs whose revision worker is
5232
+ * currently dispatched (#692). Derived from the durable `review_revisions`
5233
+ * rows, so the rendered `review-revision N` state rests on the same facts
5234
+ * the restart recovery reads — never on a label or a guess. A plain object
5235
+ * (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
5236
+ * it byte for byte.
5237
+ */
5238
+ reviewRounds?: Readonly<Record<string, number>>;
4216
5239
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
4217
5240
  * only copy of work the daemon could not save. */
4218
5241
  salvagedRuns: RunRecord[];
@@ -4302,6 +5325,13 @@ export function statusSnapshotFromStore(
4302
5325
  // Read once: the provenance read touches the filesystem, and the renderer
4303
5326
  // should never pay for it twice per status.
4304
5327
  const reason = pauseProvenance(p.name)?.reason;
5328
+ // The live review-revision rounds, read from the same durable rows the
5329
+ // restart recovery uses: a run whose revision is dispatched is read as
5330
+ // `review-revision N` while its worker is live (#692).
5331
+ const reviewRounds: Record<string, number> = {};
5332
+ for (const revision of store.unsettledReviewRevisions(p.name)) {
5333
+ if (revision.dispatchedAt !== undefined) reviewRounds[revision.runId] = revision.round;
5334
+ }
4305
5335
  return {
4306
5336
  project: p.name,
4307
5337
  configPath: configPath(),
@@ -4313,6 +5343,7 @@ export function statusSnapshotFromStore(
4313
5343
  caps,
4314
5344
  releaseGrants: resolveReleaseGrants(p),
4315
5345
  activeRuns: store.activeRuns(p.name),
5346
+ reviewRounds,
4316
5347
  salvagedRuns: store.salvagedRuns(p.name),
4317
5348
  turnOverrides: store.listTurnOverrides(p.name),
4318
5349
  openReports: store.openReports(p.name),
@@ -4758,7 +5789,10 @@ export async function classifyAndRecover(d: Deps): Promise<number> {
4758
5789
  // table can tell an infra outage (#177) from a real test failure by
4759
5790
  // the log's own words. First failure wins; a log that cannot be
4760
5791
  // fetched is left undefined and classification stays conservative.
4761
- const firstFailure = facts.checks.find((c) => c.state === "failure" && c.link !== undefined);
5792
+ // GitHub reports check states as `FAILURE` while the classifier reads
5793
+ // them lowercased — normalise so the live seam and the table agree on
5794
+ // which check is the failure whose log we pull (#177).
5795
+ const firstFailure = facts.checks.find((c) => normalise(c.state) === "failure" && c.link !== undefined);
4762
5796
  if (firstFailure?.link !== undefined) {
4763
5797
  facts.failingLog = await tracker.checkLog(firstFailure.link);
4764
5798
  }
@@ -4791,6 +5825,154 @@ export async function classifyAndRecover(d: Deps): Promise<number> {
4791
5825
  return settled;
4792
5826
  }
4793
5827
 
5828
+ /**
5829
+ * How many settled `ci-deterministic` rows one reconciliation pass may
5830
+ * re-examine beyond the persisted review cursor. Each candidate costs GitHub
5831
+ * calls to re-fetch the evidence its check log carried, so a fleet with a
5832
+ * long misclassified history works through it over several daemon starts
5833
+ * rather than spending one boot's budget on all of it (#638).
5834
+ */
5835
+ const HISTORICAL_INFRA_BATCH = 20;
5836
+
5837
+ /** How many workflow runs for the head commit one row's evidence pass reads —
5838
+ * and the ceiling past which the row is refused as undecided rather than
5839
+ * decided from a prefix of its head-pinned runs (review #654). */
5840
+ const HISTORICAL_INFRA_RUNS = 3;
5841
+
5842
+ /** How many attempts of one workflow run may be read for the failed log — and
5843
+ * the ceiling past which the run is refused as undecided rather than read as
5844
+ * a prefix that could hide a later attempt's real failure (review #654). A
5845
+ * failed job rerun to green leaves the failure in an earlier attempt; the
5846
+ * bound keeps one pathological run from costing the whole pass. */
5847
+ const HISTORICAL_INFRA_ATTEMPTS = 5;
5848
+
5849
+ /**
5850
+ * Repair settled `ci-deterministic` rows whose re-fetched check log carries a
5851
+ * closed infrastructure signature (#638). The forward classifier now names
5852
+ * the codeload setup 429 of #177 `ci-infra`, but a row already classified
5853
+ * `ci-deterministic`/`escalate` never re-enters the classification sweep, so
5854
+ * the old verdict charges an implementation attempt forever. Re-fetching the
5855
+ * head-pinned workflow-run attempt logs through the tracker, matching the
5856
+ * classifier's own closed signature list, and reclassifying `failureClass`
5857
+ * alone returns the attempt without re-animating a months-old run into
5858
+ * recovery.
5859
+ *
5860
+ * Bounded, idempotent and resumable: one batch per call, reclassifications
5861
+ * only, and the store's update is guarded by the row still reading
5862
+ * `ci-deterministic`, so a second pass touches nothing it already repaired.
5863
+ * The batch resumes below a persisted review cursor, so each row is evaluated
5864
+ * once rather than rescanning the newest non-matches forever and starving
5865
+ * older repairable rows. A row whose evidence could not be read is a
5866
+ * no-mutation refusal: the cursor never advances past it, so the next pass
5867
+ * asks again; a row that was read and shown *not* to be infrastructure
5868
+ * advances the cursor. Every repair requires *all* gathered failed logs to
5869
+ * carry an infra signature — a setup 429 in one attempt must not waive a
5870
+ * compile/test failure in a sibling attempt.
5871
+ *
5872
+ * Returns how many rows it repaired, for the boot log.
5873
+ */
5874
+ export async function reconcileHistoricalInfra(d: Deps): Promise<number> {
5875
+ const { project, store } = d;
5876
+ const version = infraSignatureVersion();
5877
+ const persisted = store.historicalInfraCursor(project.name);
5878
+ // A cursor stamped by an older signature list is stale: the classifier now
5879
+ // recognises more evidence, so the pass restarts from the newest row rather
5880
+ // than skipping past newly repairable history (#638).
5881
+ const cursor =
5882
+ persisted !== undefined && persisted.classifierVersion === version
5883
+ ? { startedAt: persisted.startedAt, rowid: persisted.rowid }
5884
+ : undefined;
5885
+ const candidates = store.historicalInfraCandidates(project.name, HISTORICAL_INFRA_BATCH, cursor);
5886
+ let repaired = 0;
5887
+ let lastDecided: { startedAt: number; rowid: number } | undefined;
5888
+ for (const run of candidates) {
5889
+ const chunks = await historicalInfraEvidence(d, run);
5890
+ // Unreachable evidence is undecided exactly like a fresh classifier is: a
5891
+ // no-mutation refusal the next pass asks again, and the cursor stops here
5892
+ // so the row is re-offered rather than being skipped past.
5893
+ if (chunks === undefined) break;
5894
+ lastDecided = { startedAt: run.startedAt, rowid: run.rowid };
5895
+ // The head's runs were read and none holds a failing log — determinately
5896
+ // not infrastructure, so an old misclassified verdict stays charged.
5897
+ if (chunks.length === 0) continue;
5898
+ // Mixed guard: anything gathered that is not itself a closed infra
5899
+ // signature (a compile/test failure in a sibling attempt, a product 429)
5900
+ // refuses the whole row. One setup 429 is not permission to waive a real
5901
+ // implementation failure.
5902
+ if (!chunks.every((chunk) => infraLogSignature(chunk) !== undefined)) continue;
5903
+ if (store.reclassifyInfra(run.id)) {
5904
+ repaired += 1;
5905
+ // The every-guard above guarantees a signature and a first chunk; the
5906
+ // non-null assertions make the same fact readable to the type checker.
5907
+ log(
5908
+ `#${run.issue} repaired historical ${run.failureClass ?? "ci-deterministic"} → ci-infra (run ${run.id}): ` +
5909
+ `"${infraLogSignature(chunks[0]!)}" in the failed check log`,
5910
+ );
5911
+ }
5912
+ }
5913
+ if (lastDecided !== undefined) {
5914
+ store.setHistoricalInfraCursor(
5915
+ project.name,
5916
+ lastDecided.startedAt,
5917
+ lastDecided.rowid,
5918
+ version,
5919
+ );
5920
+ }
5921
+ return repaired;
5922
+ }
5923
+
5924
+ /**
5925
+ * The failed check logs of a settled `ci-deterministic` row, head-pinned to
5926
+ * the exact commit the row ran against, or `undefined` when the evidence is
5927
+ * unreachable (#638). Only the workflow-run history at `run.headSha` is
5928
+ * evidence: the PR's *current* check rollup is not, because a later push's
5929
+ * checks must never classify an earlier run's row. `workflowRunsAt` is itself
5930
+ * head-scoped, and each run's failed attempts are read through the tracker's
5931
+ * guarded runner (call/refusal accounting and the rate-limit breaker apply).
5932
+ *
5933
+ * Returns the per-failed-job failed logs gathered across the head's runs (one
5934
+ * chunk per failed job, full and untruncated), or `undefined` when any read
5935
+ * could not be made — the run register, an attempt's job register, a failed
5936
+ * job's log, or the head-run list itself — or when a bounded evidence set
5937
+ * exceeded its limit (more head-pinned runs than `HISTORICAL_INFRA_RUNS`, or
5938
+ * more attempts than `HISTORICAL_INFRA_ATTEMPTS`). Undefined is a no-mutation
5939
+ * refusal the next pass asks again; the cursor never advances past it. `[]`
5940
+ * means the head's runs were read and none holds a failing log — a
5941
+ * determinately non-infrastructure answer.
5942
+ */
5943
+ async function historicalInfraEvidence(d: Deps, run: RunRecord): Promise<string[] | undefined> {
5944
+ // Without a head SHA there is no safe way to pin evidence to this row; a
5945
+ // head-less row is determinately not a repair candidate, so it advances the
5946
+ // cursor rather than stalling the pass.
5947
+ if (run.headSha === undefined) return [];
5948
+ const target = d.project.routing.repos[run.repo];
5949
+ const repoIdentity = target === undefined ? undefined : githubRepo(target.cloneUrl);
5950
+ if (repoIdentity === undefined) return [];
5951
+ let runs: WorkflowRun[] | undefined;
5952
+ try {
5953
+ runs = await d.tracker.workflowRunsAt(repoIdentity, run.headSha);
5954
+ } catch {
5955
+ return undefined;
5956
+ }
5957
+ if (runs === undefined) return undefined;
5958
+ // Refuse rather than decide from the first `HISTORICAL_INFRA_RUNS`: three
5959
+ // setup-429 runs must not waive a row whose fourth head-pinned run carries
5960
+ // the real compile/test failure (review #654). Undecided means the cursor
5961
+ // never advances past this row, so the next pass asks again.
5962
+ if (runs.length > HISTORICAL_INFRA_RUNS) return undefined;
5963
+ const chunks: string[] = [];
5964
+ for (const wf of runs) {
5965
+ const logs = await d.tracker.runFailedAttemptLogs(
5966
+ repoIdentity,
5967
+ wf.url,
5968
+ HISTORICAL_INFRA_ATTEMPTS,
5969
+ );
5970
+ if (logs === undefined) return undefined;
5971
+ chunks.push(...logs);
5972
+ }
5973
+ return chunks;
5974
+ }
5975
+
4794
5976
  /** Performs the one action a class names. Never chooses one of its own. */
4795
5977
  async function recoverRun(
4796
5978
  d: Deps,
@@ -5450,12 +6632,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5450
6632
  log(projects.length === 1 ? message : `[${project.name}] ${message}`);
5451
6633
  };
5452
6634
  const caps = resolveCaps(project, cfg.defaults);
6635
+ // One transient-server-error breaker per project (#642): admission's
6636
+ // GraphQL checks and the orchestrator mutation commands share it, so a 503
6637
+ // observed by either side gates both instead of one provider outage being
6638
+ // re-asked by every candidate AND every mutation.
6639
+ const graphqlBreaker = new GraphqlBreaker();
5453
6640
  const tracker = makeTracker(project, undefined, {
5454
6641
  onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
5455
6642
  onRefusal: (at) => store.recordGhRefusal?.(at),
5456
6643
  onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
5457
- });
5458
- const verbActions = githubVerbActions(project);
6644
+ }, { graphqlBreaker });
6645
+ const verbActions = githubVerbActions(project, undefined, undefined, graphqlBreaker);
5459
6646
 
5460
6647
  if (alive === undefined || alive.pid === process.pid) {
5461
6648
  const orphanPublisher = (run: RunRecord): RunPublisher => {
@@ -5604,6 +6791,26 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5604
6791
  verbActions,
5605
6792
  };
5606
6793
  runtimeDeps = d;
6794
+ // Review-revision restart recovery (#692): a revision the previous daemon
6795
+ // claimed and lost — its run is `running`/`orphaned` by the orphan sweep
6796
+ // above — is restored to `pushed-green` and re-queued so this process's
6797
+ // first dispatch pass resumes the exact session. Runs under the same
6798
+ // dead-daemon guard as `reconcileOrphanedRuns`, and after it, so the
6799
+ // revision's worktree has already been salvaged to the branch before it is
6800
+ // cleared for a fresh reattach. A round that cannot be restored was
6801
+ // settled and escalated by the reconcile itself.
6802
+ if (alive === undefined || alive.pid === process.pid) {
6803
+ try {
6804
+ for (const revision of await reconcileCrashedReviewRevisions(d)) {
6805
+ projectLog(
6806
+ `#${revision.issue} review round ${revision.round} recovered across restart — ` +
6807
+ `run ${revision.runId} is pushed-green again and awaits its next dispatch pass`,
6808
+ );
6809
+ }
6810
+ } catch (err) {
6811
+ projectLog(`review revision restart recovery failed: ${errText(err)}`);
6812
+ }
6813
+ }
5607
6814
  // Startup reconciliation: close an incident carried over from a previous
5608
6815
  // process when the orchestrator is up (one recovery notice), or open one
5609
6816
  // when it failed to start (one down page). A daemon restarted while still
@@ -5625,6 +6832,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5625
6832
  });
5626
6833
  }
5627
6834
 
6835
+ // Historical budget repair (#638): one bounded pass per daemon process,
6836
+ // before the first tick, so admission and `unblock` read the repaired counts
6837
+ // from the moment the daemon starts. A classifier fix changes future
6838
+ // verdicts but not rows already settled under the old one; re-fetching their
6839
+ // failed check logs and reclassifying the exact closed infrastructure
6840
+ // signature returns those attempts without re-animating the runs. Best-effort
6841
+ // per project — one unreachable project must not stop the rest from booting.
6842
+ for (const runtime of runtimes) {
6843
+ try {
6844
+ await reconcileHistoricalInfra(runtime.d);
6845
+ } catch (err) {
6846
+ log(`historical infra reconciliation failed: ${errText(err)}`);
6847
+ }
6848
+ }
6849
+
5628
6850
  if (o.once) {
5629
6851
  try {
5630
6852
  for (const runtime of runtimes) await tick(runtime.d, workers);