omp-conductor 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/REFERENCE.md +12 -8
  2. package/package.json +1 -1
  3. package/schema/config.schema.json +40 -1
  4. package/src/admission.ts +263 -44
  5. package/src/ask.ts +39 -3
  6. package/src/availability.ts +27 -1
  7. package/src/backups.ts +2 -2
  8. package/src/briefs/orchestrator.md +1 -0
  9. package/src/briefs/worker.md +38 -19
  10. package/src/command-help.ts +8 -1
  11. package/src/command-manifest.ts +5 -2
  12. package/src/commands/arm.ts +6 -3
  13. package/src/commands/message.ts +32 -4
  14. package/src/commands/watch.ts +62 -3
  15. package/src/config-schema.ts +53 -0
  16. package/src/config.ts +97 -1
  17. package/src/daemon.ts +1479 -1483
  18. package/src/decisions.ts +51 -6
  19. package/src/depends-on.ts +261 -1
  20. package/src/diff-flags.ts +350 -0
  21. package/src/digest-schedule.ts +37 -0
  22. package/src/doctor.ts +310 -22
  23. package/src/escalate.ts +560 -57
  24. package/src/failure-class.ts +71 -15
  25. package/src/fleet.ts +189 -34
  26. package/src/gitops.ts +103 -24
  27. package/src/graph-health.ts +20 -7
  28. package/src/graph.ts +313 -68
  29. package/src/lifecycle.ts +43 -7
  30. package/src/omp.ts +42 -0
  31. package/src/orchestrator-tick.ts +430 -162
  32. package/src/release-policy.ts +177 -5
  33. package/src/routing.ts +11 -3
  34. package/src/session-host.ts +16 -0
  35. package/src/settlement.ts +1728 -0
  36. package/src/setup-host.ts +193 -4
  37. package/src/setup-install.ts +91 -30
  38. package/src/setup-wizard.ts +1257 -78
  39. package/src/setup.ts +153 -6
  40. package/src/status-render.ts +36 -4
  41. package/src/store.ts +411 -17
  42. package/src/tracker/github.ts +607 -12
  43. package/src/types.ts +331 -5
  44. package/src/upgrade.ts +50 -19
  45. package/src/verbs/actions.ts +66 -18
  46. package/src/verbs/protocol.ts +45 -0
  47. package/src/verbs/server.ts +270 -13
  48. package/src/worker.ts +239 -6
  49. package/src/worktree.ts +115 -8
  50. package/systemd/omp-conductor-recover.sh +73 -0
  51. package/systemd/recover-unit-test.sh +61 -0
package/src/daemon.ts CHANGED
@@ -8,30 +8,35 @@
8
8
  * here and enforced before anything is claimed.
9
9
  */
10
10
  import { createHash } from "node:crypto";
11
- import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
11
+ import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
12
12
  import { dirname, join, relative } from "node:path";
13
13
  import {
14
14
  configPath,
15
+ dbBackupDirFor,
15
16
  findProject,
16
17
  loadConfig,
17
18
  resolveCaps,
18
19
  resolveReleaseGrants,
20
+ resolveReview,
19
21
  stateDir,
20
22
  } from "./config.ts";
21
23
  import { availabilityState, type AvailabilityState } from "./availability.ts";
22
24
  import {
23
- UNREADABLE_TREE_FLAG,
24
- analyseSettlement,
25
- deriveChangedLine,
26
25
  formatSettlementFlags,
27
26
  settlementFlagSummary,
28
27
  withDerivedChangedLine,
29
28
  } from "./diff-flags.ts";
30
- import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
29
+ import {
30
+ dbSnapshotDue,
31
+ dbSnapshotMarkerKey,
32
+ digestScheduleState,
33
+ localDayKey,
34
+ type DigestScheduleState,
35
+ } from "./digest-schedule.ts";
31
36
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
32
37
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
33
38
  import { graphHint } from "./graph.ts";
34
- import { acquireOnceLease, healthCheck, livingDaemon } from "./lifecycle.ts";
39
+ import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
35
40
  import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
36
41
  import { runDoctor } from "./doctor.ts";
37
42
  import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
@@ -52,34 +57,64 @@ import {
52
57
  } from "./reports.ts";
53
58
  import {
54
59
  recordReleaseBlock,
60
+ sharedHostBriefNotice,
55
61
  type GateShape,
56
62
  type ReleaseBlockContext,
57
63
  } from "./release-policy.ts";
58
- import { branchName, effectiveLabels, route } from "./routing.ts";
64
+ import { branchName, effectiveLabels, isEligible, route } from "./routing.ts";
59
65
  import type { Routed, UnroutableReason } from "./routing.ts";
60
- import { admitCandidates, hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
61
- import type { Admission, AdmissionHold } from "./admission.ts";
66
+ import {
67
+ admitCandidates,
68
+ effectiveLane,
69
+ hasContinuationBudget,
70
+ hasFailedAttemptBudget,
71
+ laneEcho,
72
+ } from "./admission.ts";
73
+ import type { Admission, AdmissionHold, FileLane } from "./admission.ts";
62
74
  import { log, errText, safeEscalate } from "./log.ts";
75
+ import {
76
+ adoptSalvagedPrs,
77
+ classifyAndRecover,
78
+ collectSettlementFlags,
79
+ formatQuarantinedRuns,
80
+ formatSalvagedRuns,
81
+ reactToProviderCredit,
82
+ readSessionError,
83
+ reconcileOrphanedRuns,
84
+ reconcileStaleLabels,
85
+ recordOperatorStop,
86
+ settlePushedGreen,
87
+ settleWorktree,
88
+ swapToQueue,
89
+ } from "./settlement.ts";
63
90
  import { materializeOmpSettings } from "./omp-settings.ts";
64
91
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
65
- import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
92
+ import { infraLogSignature, infraSignatureVersion, providerCreditRefusal, providerTransientFault } from "./failure-class.ts";
66
93
  import {
67
94
  DEFAULT_MODEL_FALLBACK_THRESHOLD,
68
95
  fallbackClause,
69
- formatModelsTried,
70
- modelsTried,
71
96
  providerFailureFacts,
72
97
  resolveDispatchModel,
73
98
  } from "./model-fallback.ts";
74
99
  import { projectLabels } from "./label-projection.ts";
75
- import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
76
- import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
100
+ import {
101
+ DB_SNAPSHOT_RETENTION,
102
+ ACTIVE_STATES,
103
+ dbPath,
104
+ LIVE_STATES,
105
+ openStore,
106
+ pruneDbSnapshots,
107
+ snapshotDb,
108
+ utcDay,
109
+ } from "./store.ts";
110
+ import { GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
77
111
  import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
78
112
  import type {
79
113
  BaseFreeze,
80
114
  BaseHealth,
81
115
  AdmissionHoldReason,
82
116
  Caps,
117
+ ConductorConfig,
83
118
  DispatchSummary,
84
119
  DigestBacklog,
85
120
  Escalation,
@@ -92,11 +127,15 @@ import type {
92
127
  PrState,
93
128
  ProjectConfig,
94
129
  ReadyIssue,
130
+ ReportingPolicy,
95
131
  RepoTarget,
96
132
  ReportRecord,
97
133
  ResolvedGrants,
98
134
  FailureClass,
99
135
  RecoveryAction,
136
+ ReviewPolicy,
137
+ ReviewRevisionOutcome,
138
+ ReviewRevisionRecord,
100
139
  RunRecord,
101
140
  RunState,
102
141
  SettlementFlag,
@@ -104,6 +143,7 @@ import type {
104
143
  Tracker,
105
144
  VerbLedgerEntry,
106
145
  TurnOverride,
146
+ WorkflowRun,
107
147
  } from "./types.ts";
108
148
  import {
109
149
  type KilledBy,
@@ -113,6 +153,7 @@ import {
113
153
  type RunWorkerDeps,
114
154
  ORPHAN_RESUME_PROMPT,
115
155
  renderBrief,
156
+ renderReviewRevisionPrompt,
116
157
  runWorker,
117
158
  } from "./worker.ts";
118
159
  import {
@@ -130,7 +171,6 @@ import { githubVerbActions } from "./verbs/actions.ts";
130
171
  import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
131
172
  import {
132
173
  listenVerbChannel,
133
- PR_LOOKUP_WINDOW_MS,
134
174
  type VerbActions,
135
175
  type VerbDeps,
136
176
  type VerbListener,
@@ -170,26 +210,7 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
170
210
  * report an operator is waiting on must not sit in the outbox for the length of
171
211
  * a poll interval, and delivery is owed even while claiming is paused (#123). */
172
212
  const REPORT_DELIVERY_INTERVAL_MS = 30_000;
173
- /** A dispatch-infra run (turn-0 git failure) is requeued so the next tick
174
- * retries — but only a bounded number of times. Three strikes for one issue
175
- * means the mirror itself is broken, not unlucky, and the sweep escalates
176
- * instead of burning a turn-0 run per tick forever (#168, #177). */
177
- const DISPATCH_INFRA_MAX_STRIKES = 3;
178
- /** A provider-transient requeue (stream stalled mid-run) is retried, but only a
179
- * bounded number of times: three aborted streams for one issue means the
180
- * provider itself is degraded, not unlucky, and the sweep escalates to a
181
- * human instead of requeueing into a down provider forever (#220). */
182
- const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
183
- /** A provider-capacity requeue (sustained in-session rate limiting) is retried,
184
- * but only a bounded number of times: three throttled runs for one issue mean
185
- * the provider is at capacity, not unlucky, and the sweep escalates to a human
186
- * instead of requeueing into a throttled provider forever (#573). The issue's
187
- * own chain moves onto its next model per strike (via {@link FAILOVER_CLASSES}),
188
- * so a bounded chain is exhaustible; this caps the unbounded no-chain case. */
189
- const PROVIDER_CAPACITY_MAX_STRIKES = 3;
190
- /** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
191
- * are maintenance, but a backlog must not turn one tick into an API burst. */
192
- const SALVAGED_PR_ADOPTION_BATCH = 10;
213
+
193
214
  const DEFAULT_PORT = 8787;
194
215
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
195
216
 
@@ -289,7 +310,9 @@ interface Deps {
289
310
  /**
290
311
  * Reads one active run's file lane for the admission file-lane interlock
291
312
  * (#555): the union of its uncommitted worktree changes and its branch-vs-base
292
- * diff. Wired by `runDaemon` to the mirror/worktree-backed
313
+ * diff. Reconciliation is not occupancy: while the run merges its base, the
314
+ * probe reports only files it has actually diverged on, never the files the
315
+ * merge merely staged (#684). Wired by `runDaemon` to the mirror/worktree-backed
293
316
  * {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert
294
317
  * (no lane is ever known occupied), which is the issue's "fail open": the
295
318
  * gate adds holds, it never refuses a well-formed issue for lack of this
@@ -303,6 +326,13 @@ interface Deps {
303
326
  * admission fails a routed cross-repo prerequisite closed.
304
327
  */
305
328
  probeIssueIn?: (repo: string, issue: number) => Promise<IssueSnapshot | undefined>;
329
+ /**
330
+ * Reads one issue's BODY in a repository the admission tracker is not bound
331
+ * to — the dependency-graph cycle pass (#421). Wired by `runDaemon` to a
332
+ * repo-scoped tracker; a test injects a fake. Absent, a routed reachable
333
+ * body fails that branch closed rather than synthesising a cycle.
334
+ */
335
+ probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
306
336
  }
307
337
 
308
338
  /**
@@ -610,6 +640,135 @@ export function setPaused(
610
640
  }
611
641
  }
612
642
 
643
+ // ----------------------------------------------------------------- admission
644
+ // acknowledgement (#651, review #3)
645
+ //
646
+ // The setup-fence's "acknowledgement" must come from the daemon itself, not a
647
+ // second synchronous worker count: a tick that passed its pause gate before
648
+ // the fence landed can sit in awaited tracker/routing/admission work and claim
649
+ // after the count and before setup mutates. Two places make that impossible
650
+ // and record it durably:
651
+ //
652
+ // - the tick's pause gate re-acknowledges every held pass (the daemon has
653
+ // reached its admission boundary and claims nothing);
654
+ // - the claim itself re-checks the pause immediately before creating the run
655
+ // row, so a tick already past its gate when the fence landed refuses at
656
+ // the claim — and writes the acknowledgement — instead of admitting.
657
+ //
658
+ // The acknowledgement file names the exact pause instance observed and the
659
+ // daemon generation that observed it, so the setup barrier can prove the
660
+ // acknowledged fence is the fence it froze and the acknowledgement belongs to
661
+ // the daemon it began with.
662
+
663
+ /** The durable admission acknowledgement a daemon writes when it observes a
664
+ * pause fence at an admission boundary. */
665
+ export interface AdmissionAckRecord {
666
+ /** The pause instance observed (source token, reason, creation instant). */
667
+ pause: { source: string; reason?: string; since: number };
668
+ /** The daemon generation that observed it — {@link daemonGeneration}. */
669
+ daemon: string;
670
+ /** When the daemon observed the fence. */
671
+ observedAt: number;
672
+ }
673
+
674
+ /** The acknowledgement file path. One per host: the fence is host-global. */
675
+ export function admissionAckPath(): string {
676
+ return join(stateDir(), "admission-ack.json");
677
+ }
678
+
679
+ /**
680
+ * The generation identity of the running daemon, or `undefined` when nothing
681
+ * provably runs. The live pidfile record wins (its pid and boot instant
682
+ * identify the exact instance, #377); a record-less ACTIVE unit is still a
683
+ * running daemon and its MainPID is the generation that lets a fence spot a
684
+ * supervisor restart (#651 review #2). Shared by the ack writer and the setup
685
+ * barrier's identity so both sides of the acknowledgement name the same
686
+ * instance by the same rule.
687
+ */
688
+ export function daemonGeneration(): string | undefined {
689
+ const daemon = livingDaemon();
690
+ if (daemon !== undefined) return `${daemon.pid}@${daemon.startedAt}`;
691
+ const ownership = probeUnit(SYSTEMD_UNIT);
692
+ return ownership.kind === "active" ? `systemd:${ownership.pid}` : undefined;
693
+ }
694
+
695
+ /** Reads the current admission acknowledgement, if one is readable. A corrupt
696
+ * or unreadable file is absence — the barrier fails closed on the absence. */
697
+ export function readAdmissionAck(): AdmissionAckRecord | undefined {
698
+ let parsed: unknown;
699
+ try {
700
+ parsed = JSON.parse(readFileSync(admissionAckPath(), "utf8"));
701
+ } catch {
702
+ return undefined;
703
+ }
704
+ if (parsed === null || typeof parsed !== "object") return undefined;
705
+ const r = parsed as Record<string, unknown>;
706
+ const pause = r["pause"];
707
+ const daemon = r["daemon"];
708
+ const observedAt = r["observedAt"];
709
+ if (pause === null || typeof pause !== "object") return undefined;
710
+ const p = pause as Record<string, unknown>;
711
+ const source = p["source"];
712
+ const since = p["since"];
713
+ const reason = p["reason"];
714
+ if (typeof source !== "string" || source.length === 0) return undefined;
715
+ if (typeof since !== "number" || !Number.isFinite(since)) return undefined;
716
+ if (reason !== undefined && typeof reason !== "string") return undefined;
717
+ if (typeof daemon !== "string" || daemon.length === 0) return undefined;
718
+ if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return undefined;
719
+ return { pause: { source, since, ...(reason === undefined ? {} : { reason }) }, daemon, observedAt };
720
+ }
721
+
722
+ /**
723
+ * Writes the admission acknowledgement for the pause the daemon just observed.
724
+ * The observed instance is the most restrictive fence in force: a host-global
725
+ * sentinel gates every project and is the pause a host-wide transaction (setup,
726
+ * an all-projects hold) froze, so it wins over a project-scoped sentinel;
727
+ * otherwise the project's own sentinel is the effective one for the daemon's
728
+ * claims, matching {@link pauseInstance}. Recording the project sentinel while
729
+ * a global fence is also in force would name a narrower, older hold instead of
730
+ * the fence that actually gates the host — and a barrier that froze the global
731
+ * sentinel could then never match its own acknowledgement (#651 review #4). A
732
+ * no-op when no pause is readable: there is nothing to acknowledge. The
733
+ * generation is the writing daemon's own (the record it owns, or its
734
+ * supervised MainPID).
735
+ */
736
+ export function writeAdmissionAck(project: string): void {
737
+ // `pauseInstance()` reads only the global sentinel; while it is in force it
738
+ // is the fence that gates every project, and the durable record must name it
739
+ // rather than a per-project hold that predates it.
740
+ const pause = pauseInstance() ?? pauseInstance(project);
741
+ if (pause === undefined) return;
742
+ const generation = daemonGeneration();
743
+ if (generation === undefined) return;
744
+ const path = admissionAckPath();
745
+ mkdirSync(dirname(path), { recursive: true });
746
+ const record: AdmissionAckRecord = { pause, daemon: generation, observedAt: Date.now() };
747
+ const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
748
+ try {
749
+ writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
750
+ renameSync(tmp, path);
751
+ } catch (err) {
752
+ rmSync(tmp, { force: true });
753
+ throw err;
754
+ }
755
+ }
756
+
757
+ /** Wake the daemon's dispatch loop to prompt a pass (best-effort; used by the
758
+ * setup barrier so a live daemon acknowledges the fence without waiting for
759
+ * its next five-minute tick). A refused/failed wake just lengthens the wait;
760
+ * the barrier's deadline is what bounds it. */
761
+ export async function wakeDaemon(port: number): Promise<void> {
762
+ try {
763
+ await fetch(`http://127.0.0.1:${port}/wake`, {
764
+ method: "POST",
765
+ signal: AbortSignal.timeout(1_500),
766
+ });
767
+ } catch {
768
+ // Best-effort by contract: the caller's bounded wait is the backstop.
769
+ }
770
+ }
771
+
613
772
  // ----------------------------------------------------------- package integrity
614
773
 
615
774
  /** Enough differing paths to tell a deploy from a tamper at a glance; the full
@@ -775,16 +934,29 @@ const DISCUSSION_CHARS_BUDGET = 8_000;
775
934
  * the #517 failure. An empty comment list renders nothing, so a commentless
776
935
  * issue's brief stays byte-identical to what this package has always shipped.
777
936
  */
778
- function renderDiscussion(comments: IssueComment[] | "unread"): string {
937
+ function renderDiscussion(comments: IssueComment[] | "unread", lane?: FileLane): string {
779
938
  if (comments === "unread") {
780
- return [
939
+ const lines = [
781
940
  "## Discussion",
782
941
  "",
783
942
  "_The issue's comments could not be read at dispatch time. The live read below is_",
784
943
  "_the only path to them; if it prints nothing, that is a failed read, not an_",
785
944
  "_absence of discussion._",
786
945
  "",
787
- ].join("\n");
946
+ ];
947
+ // The lane admission enforced is carried through dispatch, so even an
948
+ // unreadable thread cannot hide it from the worker (#608): the gate's
949
+ // effective declaration renders from the admission snapshot, not from the
950
+ // read that just failed. A body declaration needs no note — the body
951
+ // always renders.
952
+ if (lane !== undefined && lane.at !== "body") {
953
+ lines.push(
954
+ "_The effective file lane below was enforced at admission — it supersedes any earlier declaration._",
955
+ lane.source,
956
+ "",
957
+ );
958
+ }
959
+ return lines.join("\n");
788
960
  }
789
961
  if (comments.length === 0) return "";
790
962
  const total = comments.length;
@@ -812,9 +984,67 @@ function renderDiscussion(comments: IssueComment[] | "unread"): string {
812
984
  "",
813
985
  );
814
986
  }
987
+ // The file-lane gate resolves its effective lane across the whole thread,
988
+ // not just what this budget renders, so a winning declaration beyond the
989
+ // budget would otherwise control admission while the worker never sees it
990
+ // (#608). Reproduce the winning declaration verbatim here — `at` records
991
+ // the comment it came from — so the rendered brief and the gate agree on
992
+ // the same lane, and the note itself re-parses through the same
993
+ // `File lane:` grammar. The note is suppressed only when the winning
994
+ // declaration is provably visible in the rendered thread (the body always
995
+ // renders, so a body winner needs none); a carried admission lane whose
996
+ // comment the dispatch re-read shifted or dropped still renders here,
997
+ // whether or not the thread was truncated.
998
+ if (
999
+ lane !== undefined &&
1000
+ lane.at !== "body" &&
1001
+ !comments.slice(0, shown).some((c) => c.body.includes(lane.source))
1002
+ ) {
1003
+ lines.push(
1004
+ `_The effective file lane was declared in comment ${lane.at + 1} — it supersedes any earlier declaration._`,
1005
+ lane.source,
1006
+ "",
1007
+ );
1008
+ }
815
1009
  return lines.join("\n");
816
1010
  }
817
1011
 
1012
+ /**
1013
+ * The parsed file lane as a brief section (#724): the file list the gate will
1014
+ * enforce, or the explicit fail-open note — the same `laneEcho` the promotion
1015
+ * verb prints, so the author and the worker read one parse. Deliberately only
1016
+ * the parse, never the source line: the prose already renders in the body or
1017
+ * the discussion, and in #720's case the prose is exactly what looked
1018
+ * reasonable to a human while parsed greedily.
1019
+ */
1020
+ function laneBlock(lane: FileLane | undefined): string {
1021
+ const echo = laneEcho(lane);
1022
+ const body = lane === undefined ? `_${echo}_` : `\`${echo}\``;
1023
+ return ["## File lane (as parsed)", "", body, "", ""].join("\n");
1024
+ }
1025
+
1026
+ /**
1027
+ * What an orphan-resumed worker is told about the file lane on top of the
1028
+ * continuation notice (#608). The original brief already in the transcript
1029
+ * may show an earlier declaration, while admission enforces the current one —
1030
+ * replaying the whole brief would re-do the work, but continuing under a stale
1031
+ * lane is the collision the gate exists to stop. The declaration is rendered
1032
+ * verbatim, so it re-parses through the same `File lane:` grammar every
1033
+ * surfaced declaration uses.
1034
+ */
1035
+ function resumeLaneBlock(lane: FileLane): string {
1036
+ return [
1037
+ "",
1038
+ "The file lane admission enforced for this continuation is below. It supersedes",
1039
+ "any lane declaration in the brief already in your transcript:",
1040
+ "",
1041
+ lane.source,
1042
+ "",
1043
+ `Parsed files: ${laneEcho(lane)}.`,
1044
+ "",
1045
+ ].join("\n");
1046
+ }
1047
+
818
1048
  /**
819
1049
  * Record a state-label swap for projection (#201). The add enqueues before the
820
1050
  * remove — the reverse order would leave a window where the issue carries no
@@ -828,211 +1058,6 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
828
1058
  { issue, op: "remove", label: from },
829
1059
  ]);
830
1060
  }
831
- /**
832
- * Persist the operator-stop transition before releasing its live controller.
833
- * The row is terminal first, then its in-progress label is removed through the
834
- * same durable projection outbox as every other lifecycle transition.
835
- */
836
- export function recordOperatorStop(
837
- store: Pick<Store, "updateRun" | "enqueueLabelOps">,
838
- args: {
839
- project: string;
840
- issue: number;
841
- runId: string;
842
- inProgress: string;
843
- reason: string;
844
- patch: Partial<RunRecord>;
845
- },
846
- ): void {
847
- store.updateRun(args.runId, {
848
- ...args.patch,
849
- state: "stopped",
850
- lastError: `operator stopped: ${args.reason}`,
851
- });
852
- store.enqueueLabelOps(args.project, [
853
- { issue: args.issue, op: "remove", label: args.inProgress },
854
- ]);
855
- }
856
-
857
-
858
- /**
859
- * The escalator throws when no transport is configured or Telegram rejects, and
860
- * only records the dedup marker on success. A page that cannot be delivered
861
- * must not take the tick down with it — log it and let the next tick retry.
862
- *
863
- * Returns whether it actually went out, because "page once" and "page once
864
- * *successfully*" are different promises: a caller that latches a once-only
865
- * gate on the attempt turns one failed delivery into permanent silence about a
866
- * condition that is still true.
867
- */
868
-
869
- async function reactToProviderCredit(
870
- d: Deps,
871
- issue: number,
872
- message: string,
873
- sessionFile: string | undefined,
874
- ): Promise<void> {
875
- const { project } = d;
876
- const alreadyPaused = isPaused(project.name);
877
- if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message }, project.name);
878
- log(
879
- `#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
880
- );
881
- // Fleet-scoped and run-independent on purpose. The notification ledger
882
- // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
883
- // carrying no run or attempt pages once for the fleet, not once per run.
884
- await safeEscalate(d, {
885
- tier: 2,
886
- category: "fleet-stopped",
887
- project: project.name,
888
- issue: NO_ISSUE,
889
- summary: `Model provider refused for credit — ${project.name} is paused`,
890
- detail: [
891
- message,
892
- "",
893
- "No implementation attempt was charged: this is a billing state, not a",
894
- "failed implementation. Each affected issue keeps its queue label and",
895
- "re-dispatches on `omp-conductor resume` once the provider has credit.",
896
- `Session: ${sessionFile ?? "(no transcript)"}`,
897
- ].join("\n"),
898
- });
899
- }
900
-
901
- /**
902
- * What a salvage attempt contributes to the escalation: where the work went, or
903
- * that it went nowhere. Split from the effects below for the same reason
904
- * `checkIntegrity` is — this wording is the whole thing a human acts on, so it
905
- * is worth a test holding it, and the sha in it is the only pointer to work
906
- * that no longer has any other copy.
907
- *
908
- * `retained` is not cosmetic. These lines used to promise a tree "kept for
909
- * inspection" unconditionally, which was true only because salvage ran only on
910
- * the paths that keep one. A blocked run's tree is removed the moment its work
911
- * is safely on the branch, and sending an operator to a path this process just
912
- * deleted is the same class of mistake as #118 itself.
913
- */
914
- export function salvageLines(
915
- outcome: SalvageOutcome,
916
- worktree: string,
917
- retained: boolean,
918
- ): string[] {
919
- const fate = retained
920
- ? `Worktree kept for inspection: ${worktree}`
921
- : `Worktree removed: ${worktree}`;
922
-
923
- if (outcome.kind === "nothing") return [`${fate} — nothing uncommitted to salvage`];
924
-
925
- if (outcome.kind === "failed") {
926
- return [
927
- `WIP SALVAGE FAILED: ${outcome.error}`,
928
- `Uncommitted work in ${worktree} is the only copy of it, so the tree was kept.`,
929
- "This issue is held out of dispatch until the tree is recovered by hand and",
930
- "`omp-conductor unblock <n> --force` records that you accepted it.",
931
- ];
932
- }
933
-
934
- const where =
935
- `WIP committed to ${outcome.branch} @ ${outcome.sha}` +
936
- (outcome.pushed
937
- ? " and pushed — the work outlives this worktree"
938
- : ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`);
939
- // Manifest belongs in the escalation too: opening the commit is how the
940
- // orchestrator talked itself into scrubbing a worker tree (#38).
941
- const n = outcome.files.length;
942
- const count = `${n} file${n === 1 ? "" : "s"}`;
943
- const manifest =
944
- outcome.newPaths.length === 0
945
- ? `${count} (all modifications to tracked paths)`
946
- : `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
947
- outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
948
- }`;
949
- return [where, manifest, fate];
950
- }
951
-
952
- /** Everything a settled run has to record and say about its worktree. */
953
- export interface WorktreeSettlement {
954
- outcome: SalvageOutcome;
955
- /** Whether the tree still exists now the run is over. */
956
- retained: boolean;
957
- /** Escalation lines naming where the work went. */
958
- lines: string[];
959
- /** Row fields recording the durable ref, or the failure that blocks a re-claim. */
960
- patch: Pick<RunRecord, "salvageSha" | "salvageError">;
961
- }
962
-
963
- /**
964
- * Decides what becomes of a finished run's worktree: save the work, then keep
965
- * or remove the tree, then say which.
966
- *
967
- * One function because the two halves are one decision and splitting them is
968
- * how #118 happened — the removal at the end of dispatch had no idea whether
969
- * anything had been saved, and the salvage at the top of the failure branch had
970
- * no idea the blocked branch fell through to a `--force` removal.
971
- *
972
- * A salvage that *fails* retains the tree whatever the caller asked for. There
973
- * was real work, git refused to commit it, and the tree is now the only copy in
974
- * existence: deleting it on schedule would be the data loss this whole path
975
- * exists to prevent. The issue is held out of dispatch until an operator says
976
- * otherwise, because the next attempt's `worktree remove --force` would finish
977
- * the job (see `admitCandidates`).
978
- *
979
- * Exported so a test can drive the real decision against a real git tree.
980
- */
981
- export async function settleWorktree(
982
- args: {
983
- issue: number;
984
- attempt: number;
985
- /** Clause for the commit subject: "killed by the turns cap", "blocked …". */
986
- ending: string;
987
- worktree: string;
988
- /** The run's branch, so the pre-removal publish names the right ref. */
989
- branch: string;
990
- /**
991
- * Publishes the run branch on the privileged side. Required rather than
992
- * optional: the run's commits live in a repository of its own,
993
- * so a removal that did not publish first would delete the only copy —
994
- * which is #121's data loss with one extra step. `undefined` is a visible
995
- * decision at the call site, never an omission.
996
- */
997
- publish: RunPublisher | undefined;
998
- } & (
999
- | /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
1000
- { tree: "keep" }
1001
- | { tree: "remove"; mirrorPath: string }
1002
- ),
1003
- ): Promise<WorktreeSettlement> {
1004
- const { issue, attempt, ending, worktree, branch, publish } = args;
1005
- const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
1006
- const retained = args.tree === "keep" || outcome.kind === "failed";
1007
- if (!retained && args.tree === "remove") {
1008
- // Before the removal, always — not only when salvage found something. A run
1009
- // that *committed* and could not publish has its work in its own repository
1010
- // and nowhere else, and salvage never sees a committed tree because it is
1011
- // clean. The mirror fetch inside `publish` is what preserves it; the push
1012
- // to GitHub can fail (no network, protected ref) and the work still lives.
1013
- const published = await publish?.(branch);
1014
- if (published !== undefined && !published.ok) {
1015
- log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
1016
- }
1017
- await removeWorktree(args.mirrorPath, worktree);
1018
- }
1019
-
1020
- const lines = salvageLines(outcome, worktree, retained);
1021
- log(`#${issue} salvage: ${lines.join(" ")}`);
1022
- return {
1023
- outcome,
1024
- retained,
1025
- lines,
1026
- patch:
1027
- outcome.kind === "salvaged"
1028
- ? { salvageSha: outcome.sha }
1029
- : outcome.kind === "failed"
1030
- ? { salvageError: outcome.error }
1031
- : {},
1032
- };
1033
- }
1034
-
1035
-
1036
1061
  /**
1037
1062
  * How a run's end is named — in the salvage commit, and to whoever reads it.
1038
1063
  * The whole clause, not a bare reason: a graceful block was not killed by
@@ -1066,6 +1091,15 @@ export async function buildBrief(
1066
1091
  * when the tracker refused at dispatch — rare, but it must never read as
1067
1092
  * "no comments" (the #517 failure mode). Absent means an empty list. */
1068
1093
  comments?: IssueComment[] | "unread";
1094
+ /**
1095
+ * The effective file lane admission resolved for this candidate (#608).
1096
+ * When carried, the brief renders exactly this value — the gate's own
1097
+ * snapshot — instead of recomputing from the dispatch-time comment read,
1098
+ * so a changed or failed second read can neither hide nor reword the
1099
+ * lane admission enforced. Absent (unit-level callers), the lane is
1100
+ * resolved from the rendered thread itself.
1101
+ */
1102
+ lane?: FileLane;
1069
1103
  } = {},
1070
1104
  ): Promise<string> {
1071
1105
  // Read per dispatch rather than caching: editing the brief then takes effect
@@ -1103,6 +1137,18 @@ export async function buildBrief(
1103
1137
  "",
1104
1138
  ].join("\n")
1105
1139
  : "";
1140
+ // The comments as rendered, and the effective file lane the brief must show
1141
+ // — the admission-carried value when dispatch has one, else the lane the
1142
+ // thread itself resolves to — passed to the renderer so a winning
1143
+ // declaration can be reproduced verbatim even when it sits beyond the
1144
+ // discussion budget, keeping the gate and the worker-visible brief on one
1145
+ // lane (#608).
1146
+ const comments = opts.comments ?? [];
1147
+ // The effective lane admission resolved (or resolves) for this candidate:
1148
+ // the carried admission snapshot when dispatch has one, else the thread
1149
+ // itself. Both the discussion renderer and the parsed-lane section draw on
1150
+ // the same value, so the brief shows one lane on every surface (#608, #724).
1151
+ const lane = opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments));
1106
1152
  return renderBrief(template, {
1107
1153
  ISSUE_NUMBER: String(r.issue.number),
1108
1154
  ISSUE_TITLE: r.issue.title,
@@ -1111,11 +1157,18 @@ export async function buildBrief(
1111
1157
  BRANCH: branch,
1112
1158
  WORKTREE: worktree,
1113
1159
  ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
1114
- ISSUE_COMMENTS: renderDiscussion(opts.comments ?? []),
1160
+ ISSUE_COMMENTS: renderDiscussion(comments, lane),
1161
+ FILE_LANE: laneBlock(lane),
1115
1162
  GATES: gatesBlock(r.repo),
1116
- // Empty for a repo with no `graphProject`, and empty means *nothing*: the
1117
- // placeholder sits flush against the next list item in the template, so an
1118
- // unconfigured render leaves no blank line where a hint would have gone.
1163
+ // The guarded shell suites and their sanctioned parse-only check, derived
1164
+ // from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
1165
+ // (#687). Non-empty whenever the guard has paths, so the placeholder line
1166
+ // always renders to a line.
1167
+ SHARED_HOST_NOTICE: sharedHostBriefNotice(),
1168
+ // The brief's code-graph paragraph: the exact `project` key for a
1169
+ // configured repo, or an explicit "no graph" statement for an
1170
+ // unconfigured one — never silence, because a worker that knows there is
1171
+ // no graph stops looking for it.
1119
1172
  GRAPH_HINT: graphHint(r.repo),
1120
1173
  CONTINUATION: continuation,
1121
1174
  });
@@ -1124,178 +1177,6 @@ export async function buildBrief(
1124
1177
  // ------------------------------------------------------------------- one issue
1125
1178
 
1126
1179
 
1127
- /** The failure classes `countContinuations` deliberately does not charge — the
1128
- * inverted copy of its exclusions, kept beside the breakdown that consumes it
1129
- * so the two can only drift together (#439). `orphan-clean` is absent on
1130
- * purpose: daemon orphans consume the continuation budget, which is exactly
1131
- * why the requeue side has to respect the ceiling instead of racing it. */
1132
- const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
1133
- "admin-kill": true,
1134
- "settlement-stuck": true,
1135
- "env-start-failure": true,
1136
- "dispatch-infra": true,
1137
- "provider-credit": true,
1138
- "provider-transient": true,
1139
- "provider-capacity": true,
1140
- };
1141
-
1142
- /** How one issue spent its continuation budget, grouped by failure class —
1143
- * the exact rows `continuationsFor` charges, so an exhaustion escalation
1144
- * reports the same budget it says is spent. `unclassified` groups rows that
1145
- * charged before the class was written (a pre-upgrade NULL). */
1146
- function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
1147
- const perClass = new Map<string, number>();
1148
- for (const r of runs) {
1149
- const chargedAsKilledOrOrphaned =
1150
- (r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
1151
- (r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
1152
- const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
1153
- if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
1154
- const cls = r.failureClass ?? "unclassified";
1155
- perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
1156
- }
1157
- return perClass;
1158
- }
1159
-
1160
- /** The newest attempt's preserved work, if any, so an exhaustion escalation can
1161
- * say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
1162
- * three artifacts a run can leave, and all null on a branch nothing reached. */
1163
- function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
1164
- const newestFirst = [...runs].reverse();
1165
- return newestFirst.find(
1166
- (r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
1167
- );
1168
- }
1169
-
1170
- /** The fenced-block info string that marks an exhaustion postmortem comment, so
1171
- * a grooming scout re-slicing the issue can find and parse the whole block by
1172
- * grepping for it. */
1173
- export const POSTMORTEM_MARKER = "conductor-postmortem";
1174
-
1175
- /** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
1176
- * digest shape for naming what the exhaustion was. Groups every row by its
1177
- * failure class, whether or not it charged the continuation budget, because
1178
- * the postmortem tells the whole story and not just the budget half (#290). */
1179
- export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
1180
- const perClass = new Map<string, number>();
1181
- for (const r of runs) {
1182
- const cls = r.failureClass ?? "unclassified";
1183
- perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
1184
- }
1185
- if (perClass.size === 0) return "unclassified";
1186
- return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
1187
- }
1188
-
1189
- /** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
1190
- export function humanDuration(ms: number): string {
1191
- const seconds = Math.max(0, Math.round(ms / 1_000));
1192
- if (seconds < 60) return `${seconds}s`;
1193
- const minutes = Math.round(seconds / 60);
1194
- if (minutes < 60) return `${minutes}m`;
1195
- const hours = Math.floor(minutes / 60);
1196
- const rest = minutes % 60;
1197
- return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
1198
- }
1199
-
1200
- /** Flatten and bound a run's last error to one greppable table line. */
1201
- export function oneLineBrief(text: string | undefined): string | undefined {
1202
- if (text === undefined || text.trim() === "") return undefined;
1203
- const flat = text.replace(/\s+/g, " ").trim();
1204
- return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
1205
- }
1206
-
1207
- /**
1208
- * The exhaustion postmortem block: one greppable fenced block covering every
1209
- * attempt in the chain — continuation rows included — with per-attempt turns,
1210
- * wall clock, failure class and a one-line last error, the explicit salvage
1211
- * state, the spend total and the transcript paths for local inspection.
1212
- *
1213
- * Pure so the tests hold the shape, not the transport: the writer below owns
1214
- * the once-only guarantee, this owns what "once" looks like.
1215
- */
1216
- export function formatExhaustionPostmortem(args: {
1217
- issue: number;
1218
- runs: readonly RunRecord[];
1219
- reason: string;
1220
- }): string {
1221
- const { issue, runs, reason } = args;
1222
- const artifact = newestContinuableRun(runs);
1223
- const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
1224
- const spend = `$${totalSpend.toFixed(2)}`;
1225
- const attemptLines = runs.map((r) => {
1226
- const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
1227
- const cls = r.failureClass ?? "unclassified";
1228
- const error = oneLineBrief(r.lastError) ?? "—";
1229
- return (
1230
- ` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
1231
- `${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
1232
- );
1233
- });
1234
- const salvage =
1235
- artifact === undefined
1236
- ? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
1237
- : `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
1238
- `${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
1239
- return [
1240
- `\`\`\`${POSTMORTEM_MARKER}`,
1241
- `#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
1242
- reason,
1243
- "",
1244
- `Attempts (${runs.length} total, ${spend} spend):`,
1245
- ...attemptLines,
1246
- salvage,
1247
- `Spend total: ${spend} across ${runs.length} attempts.`,
1248
- "Transcripts:",
1249
- ...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
1250
- "```",
1251
- ].join("\n");
1252
- }
1253
-
1254
- /** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
1255
- function postmortemDedupeKey(project: string, issue: number): string {
1256
- return `${project}:postmortem:${issue}`;
1257
- }
1258
-
1259
- /**
1260
- * The exhaustion postmortem: written exactly once per issue, at the point the
1261
- * continuation budget is spent and the issue is settled toward a human.
1262
- *
1263
- * The comment and the material event are both gated by the store's notification
1264
- * ledger — the same idempotence guard the escalator uses — so re-settling an
1265
- * already-postmortemed issue posts nothing and records nothing. A body-string
1266
- * match on the issue would be the wrong guard: an issue re-scoped and re-run
1267
- * would still carry the old block, and the guarantee asked of this is "decided
1268
- * once", not "deduped against what is already written".
1269
- *
1270
- * The digest must name the exhaustion even if the comment write fails, so the
1271
- * material event is recorded before the write and unconditionally (the ledger
1272
- * is append-only, and the gate above already ran once). The comment failure is
1273
- * logged rather than taking the sweep down with it.
1274
- */
1275
- async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
1276
- const { project, tracker, store } = d;
1277
- const key = postmortemDedupeKey(project.name, run.issue);
1278
- if (store.wasNotified(key)) return;
1279
- const runs = store.runsForIssue(project.name, run.issue);
1280
- const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
1281
- const occurredAt = Date.now();
1282
- store.recordMaterialEvent({
1283
- project: project.name,
1284
- category: "exhaustion",
1285
- summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
1286
- evidence: body,
1287
- occurredAt,
1288
- recordedAt: occurredAt,
1289
- });
1290
- try {
1291
- await tracker.comment(run.issue, body);
1292
- store.markNotified(key);
1293
- log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
1294
- } catch (err) {
1295
- log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
1296
- }
1297
- }
1298
-
1299
1180
  export type ExtendTurnLimitResult =
1300
1181
  | { kind: "extended"; runId: string; maxTurns: number }
1301
1182
  | { kind: "not-increase"; runId: string; maxTurns: number }
@@ -1499,45 +1380,6 @@ export async function verifyPushedGreenClaim(
1499
1380
  };
1500
1381
  }
1501
1382
 
1502
- /** What the settlement audit of one green run produced: the advisory flags
1503
- * (test weakening only — the file-list disclosure is derived, not flagged),
1504
- * whether the diff was cut short, and the `changed:` line composed from the
1505
- * PR's own diff. */
1506
- export interface SettlementAuditResult {
1507
- flags: SettlementFlag[];
1508
- truncated: boolean;
1509
- /** The `changed:` file list derived from the PR's diff, present whenever the
1510
- * diff could be read. Absent means the tree could not be read, and `flags`
1511
- * then carries exactly {@link UNREADABLE_TREE_FLAG}. */
1512
- changedLine?: string;
1513
- }
1514
-
1515
- /**
1516
- * Audit a worker's own account of its work against the pull request it pushed.
1517
- *
1518
- * The thin half of the split #85 established: this fetches, {@link
1519
- * analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
1520
- * shares none of its authority — that function decides a run's state, this one
1521
- * cannot, by construction. It returns evidence and the caller appends it.
1522
- *
1523
- * A diff that cannot be read is a finding (`changed-line-missing`), never an
1524
- * empty flag list: nothing was derived and nothing was checked, and that must
1525
- * not read as a clean bill.
1526
- */
1527
- export async function collectSettlementFlags(
1528
- tracker: Pick<Tracker, "prDiff">,
1529
- claim: { prUrl?: string; issueText: string },
1530
- ): Promise<SettlementAuditResult> {
1531
- if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
1532
- const diff = await tracker.prDiff(claim.prUrl);
1533
- if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
1534
- return {
1535
- flags: analyseSettlement({ issueText: claim.issueText, diff }),
1536
- truncated: diff.truncated,
1537
- changedLine: deriveChangedLine(diff),
1538
- };
1539
- }
1540
-
1541
1383
  /**
1542
1384
  * One attempt at one issue, from claim to terminal state. Everything is inside
1543
1385
  * a single try/catch so that a bad issue costs its own run and nothing else.
@@ -1681,7 +1523,12 @@ function orphanResumeVerdict(
1681
1523
  return { kind: "resume", prior };
1682
1524
  }
1683
1525
 
1684
- export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1526
+ export async function handleIssue(
1527
+ d: Deps,
1528
+ r: Routed,
1529
+ attempt: number,
1530
+ admittedLane?: FileLane,
1531
+ ): Promise<void> {
1685
1532
  const { project, caps, tracker, store } = d;
1686
1533
  const issue = r.issue.number;
1687
1534
  const branch = branchName(r.issue);
@@ -1824,6 +1671,24 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1824
1671
  return;
1825
1672
  }
1826
1673
 
1674
+ // The claim-side of the pause fence (#651 review #3): the tick's pause
1675
+ // gate sits above routing, so a tick that passed that gate before an
1676
+ // operator wrote the freeze can still be mid-routing when the fence lands
1677
+ // — it would claim after the setup barrier's acknowledgement and before
1678
+ // its first mutation. The claim itself re-checks the pause and refuses,
1679
+ // writing the durable admission acknowledgement the barrier waits for.
1680
+ // A claim observed under the fence is the last admission boundary there
1681
+ // is: nothing may create a run row beside a held fleet.
1682
+ if (isPaused(d.project.name)) {
1683
+ try {
1684
+ writeAdmissionAck(d.project.name);
1685
+ } catch (err) {
1686
+ log(`admission acknowledgement write failed: ${errText(err)}`);
1687
+ }
1688
+ log(`#${issue} not claimed: dispatch is paused at claim time`);
1689
+ return;
1690
+ }
1691
+
1827
1692
  // Claim on the STORE first, before anything that can fail. The run row —
1828
1693
  // not the label — is the crash-safe guard against double dispatch: rows
1829
1694
  // are local, written before any network call, and the startup orphan
@@ -1899,6 +1764,10 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1899
1764
  spendUsd: 0,
1900
1765
  maxTurns: caps.workerMaxTurns,
1901
1766
  startedAt: Date.now(),
1767
+ // #567: the orphan-clean attempt this claim continues, when the verdict
1768
+ // above fired. `undefined` for a fresh dispatch — the store maps that to
1769
+ // NULL, so a fresh row simply never carries the field.
1770
+ resumedFromRunId: resuming?.id,
1902
1771
  });
1903
1772
  // #434: carry a terminal predecessor's open PR onto the continuation row.
1904
1773
  // The claim itself stays synchronous — `handleIssue` claims at the top of
@@ -2043,10 +1912,15 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2043
1912
  // The continuation notice replaces the brief for a resumed attempt (#536).
2044
1913
  // The original brief is already in the resumed transcript; re-sending it is
2045
1914
  // how a resumed worker ends up re-doing the work it just did. Everything
2046
- // below the brief is fresh-dispatch-only, exactly as today.
1915
+ // below the brief is fresh-dispatch-only, exactly as today — except the
1916
+ // file lane admission enforced for this continuation, which rides the
1917
+ // notice itself: the retained transcript's brief may show an earlier
1918
+ // declaration, and a worker continuing under a stale lane is the collision
1919
+ // the gate exists to stop (#608).
2047
1920
  let brief: string;
2048
1921
  if (resuming !== undefined) {
2049
- brief = ORPHAN_RESUME_PROMPT;
1922
+ brief =
1923
+ ORPHAN_RESUME_PROMPT + (admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
2050
1924
  } else {
2051
1925
  // The discussion is rendered at dispatch so a worker never depends on a
2052
1926
  // runtime `gh` read to see the orchestrator's grooming (#517). The read is
@@ -2068,6 +1942,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2068
1942
  ? { salvagedSha: priorSalvage }
2069
1943
  : {}),
2070
1944
  comments,
1945
+ lane: admittedLane,
2071
1946
  });
2072
1947
  }
2073
1948
  if (await settleStopBeforeSession()) return;
@@ -2183,6 +2058,9 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2183
2058
  ? await collectSettlementFlags(tracker, {
2184
2059
  prUrl: result.prUrl,
2185
2060
  issueText: `${r.issue.title}\n${r.issue.body}`,
2061
+ // The claimed-proof check compares the PR's Verified commands
2062
+ // against what this run's session actually recorded.
2063
+ sessionFile: result.sessionFile,
2186
2064
  })
2187
2065
  : undefined;
2188
2066
  if (result.state === "pushed-green" && audit?.truncated) {
@@ -2369,7 +2247,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2369
2247
  });
2370
2248
 
2371
2249
  if (providerCredit !== undefined) {
2372
- await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
2250
+ await reactToProviderCredit({ project: d.project, escalate: (e) => d.escalate(e), isPaused, setPaused }, issue, providerCredit, result.sessionFile);
2373
2251
  swapToQueue(d, issue, inProgress);
2374
2252
  } else if (continueTurns) {
2375
2253
  // Requeue as one ordered pair: the in-progress removal before the
@@ -2535,141 +2413,771 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
2535
2413
  }
2536
2414
  }
2537
2415
 
2538
- // ------------------------------------------------------------------- settlement
2539
-
2540
- /** What a resolved PR turns its `pushed-green` row into. */
2541
- export interface Settlement {
2542
- state: "merged" | "failed";
2543
- /** The log line after `#<n> settled: `, and — for a rejection — the row's own
2544
- * `lastError`, because a `failed` row whose worker succeeded has to say so. */
2545
- reason: string;
2546
- }
2547
-
2548
- /**
2549
- * What one `pushed-green` row becomes now its PR has an answer, or undefined to
2550
- * leave the row exactly as it is.
2551
- *
2552
- * A `pushed-green` row is the only one nothing ever revisited: the worker is
2553
- * finished, `reconcileOrphanedRuns` only settles rows that held a process, and
2554
- * `merged` went unwritten from day one. So they accumulated — three of them on
2555
- * the reference fleet on 2026-08-07, every PR merged and every issue closed,
2556
- * with `/healthz` still reporting three active runs and their issues
2557
- * permanently unclaimable, because the busy set *is* the active set (#18).
2558
- *
2559
- * The mapping, and why each answer is the only honest one:
2560
- *
2561
- * - `merged` — the work landed. That is what `merged` was reserved for.
2562
- * - `closed` — a human read the work and said no. Leaving it `pushed-green`
2563
- * forever is a lie; `failed` records that it did not land and releases the
2564
- * busy guard, so an issue a human re-queues can be attempted again. A row
2565
- * that had reached `pushed-green` or `pushed-pending` is classified
2566
- * `returned-for-revision` at settlement. A review decision asks for another
2567
- * implementation pass, not a failure, so it consumes the continuation budget
2568
- * instead of the failed-attempt budget.
2569
- * - `open`, and undefined — nothing changes. Undefined is "could not tell": a
2570
- * flaky network, a revoked token, a deleted PR. Settling on it would record a
2571
- * merge that never happened, and the next tick asks again for free. An
2572
- * ambiguous answer must never settle a row.
2573
- */
2574
- export function settlementFor(pr: PrState | undefined, prUrl: string): Settlement | undefined {
2575
- if (pr === "merged") return { state: "merged", reason: `${prUrl} merged` };
2576
- if (pr === "closed") return { state: "failed", reason: `${prUrl} closed without merging` };
2577
- return undefined;
2578
- }
2416
+ // ------------------------------------------------------------ review revisions
2579
2417
 
2580
2418
  /**
2581
- * Records the in-progress label's release as a projection op (#201).
2419
+ * The daemon half of the review-revision transport (#677): wake every pending
2420
+ * revision the orchestrator's verb recorded, on the tick, exactly like
2421
+ * admission — the verb has no handle on the worker machinery, and the CLI path
2422
+ * must behave like the embedded one. Bounded by the same worker slots, so a
2423
+ * queued revision can never push the fleet past `maxConcurrentWorkers`; the
2424
+ * remaining rows stay pending for the next tick.
2582
2425
  *
2583
- * Settlement used to write only half of what it knew. On 2026-08-09 that cost
2584
- * the reference fleet two issues in one night: veltro#331 settled to `failed`
2585
- * at 23:53Z once the orchestrator closed veltro#332 unmerged, and veltro#344
2586
- * settled to `merged` at 05:54Z once chad#452 squash-merged. Both rows left the
2587
- * active set correctly; an authoritative `gh issue view` on each afterwards
2588
- * still showed `agent:in-progress` — permanently unclaimable with no supported
2589
- * way back (#18).
2590
- *
2591
- * The outbox makes the row transition and the label one fact again: the
2592
- * removal is enqueued in the same breath as the row is terminalised, the
2593
- * projector applies it with unbounded retry, and while it is pending the
2594
- * eligibility overlay treats the label as already gone. A tracker that refuses
2595
- * the write (403, rate limit) can no longer strand the row — that is `#184`
2596
- * and `#198` closed. No `pushed-*` row is ever written terminal with its
2597
- * label release owed but unrecorded, because enqueueing is a local store write
2598
- * that cannot fail on the tracker.
2599
- *
2600
- * Synchronous. The op is durable the moment this returns.
2426
+ * The claim (`pushed-green` `running`) is synchronous and atomic per row;
2427
+ * only the resumed worker runs in the pool. A row that moved since the verb
2428
+ * recorded it merged after all, settled, re-claimed — fails the claim and is
2429
+ * settled `skipped` rather than woken on stale identity.
2601
2430
  */
2602
- export function releaseInProgress(
2603
- d: Pick<Deps, "project" | "store">,
2604
- issue: number,
2605
- why: string,
2606
- ): void {
2607
- const label = d.project.stateLabels.inProgress;
2608
- d.store.enqueueLabelOps(d.project.name, [{ issue, op: "remove", label }]);
2609
- log(`#${issue} released ${label} (queued): ${why}`);
2431
+ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
2432
+ const pending = d.store.pendingReviewRevisions(d.project.name);
2433
+ if (pending.length === 0) return;
2434
+ const slots = Math.max(0, d.caps.maxConcurrentWorkers - d.store.liveRuns(d.project.name).length);
2435
+ const batch = pending.slice(0, slots);
2436
+ const launches: Promise<void>[] = [];
2437
+ for (const revision of batch) {
2438
+ if (d.drain?.draining === true) {
2439
+ log(
2440
+ `review revisions held: the daemon is draining (${batch.length - launches.length} of ${batch.length} not launched)`,
2441
+ );
2442
+ break;
2443
+ }
2444
+ if (!d.store.claimRunForReview(revision.runId)) {
2445
+ d.store.settleReviewRevision(revision.id, "skipped", Date.now());
2446
+ log(
2447
+ `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer pushed-green`,
2448
+ );
2449
+ continue;
2450
+ }
2451
+ d.store.markReviewRevisionDispatched(revision.id, Date.now());
2452
+ log(`#${revision.issue} review round ${revision.round} → resuming run ${revision.runId}`);
2453
+ launches.push(handleReviewRevision(d, revision));
2454
+ }
2455
+ if (pool !== undefined) {
2456
+ for (const launch of launches) pool.launch(launch);
2457
+ return;
2458
+ }
2459
+ // `--once`: like admissions, the tick waits for the workers it launched.
2460
+ await Promise.allSettled(launches);
2610
2461
  }
2611
2462
 
2612
2463
  /**
2613
- * Asks the tracker about every `pushed-green` PR and settles the ones that
2614
- * resolved.
2464
+ * Resume one review-revision worker: the run row is already claimed
2465
+ * (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
2466
+ * OMP session — same session directory, `resume: true` at the harness, and the
2467
+ * `sessionFile` lineage compared at dispatch time — on the SAME branch and PR,
2468
+ * then settles the row back the way {@link handleIssue} settles any worker.
2615
2469
  *
2616
- * Effects at the call site, decision in {@link settlementFor} the same split
2617
- * as `checkIntegrity`/`watchOrchestrator`. Exported like `admitCandidates`
2618
- * rather than kept private, because half of what has to hold is about the sweep
2619
- * and not the mapping: that a row without a PR costs no API call, and that one
2620
- * unreachable PR does not stop the others from settling.
2621
- *
2622
- * The label is released too, which reverses what this function first promised.
2623
- * It used to leave tracker labels alone exactly as {@link reconcileOrphanedRuns}
2624
- * does, reasoning that a merge closes the issue anyway and that deciding what an
2625
- * issue's labels should say next is the orchestrator's drain duty. There turned
2626
- * out to be no such path: on 2026-08-09 two settled rows left their issues
2627
- * carrying `agent:in-progress` forever, with the brief forbidding the
2628
- * orchestrator from touching it and `unblock` declining to (see
2629
- * {@link releaseInProgress}). The row transition and the label are one fact, and
2630
- * writing half of it is the whole of that bug.
2631
- *
2632
- * Releasing it is safe here specifically because of what these rows are. A
2633
- * `pushed-green` or `pushed-pending` row has no process behind it — its worker
2634
- * exited and its worktree is gone — so a terminal answer about its PR proves no
2635
- * worker owns the issue, and the duplicate-dispatch interlock the label exists
2636
- * for is spent. {@link reconcileOrphanedRuns} still leaves labels alone for the
2637
- * opposite reason: an orphaned `running` row is work nobody has read yet. And
2638
- * the brief's rule stays absolute, because this is a daemon-owned write through
2639
- * the same Tracker port the dispatcher claimed the issue with — orphan detection
2640
- * is only trustworthy while every state label on the tracker came from this
2641
- * package.
2642
- *
2643
- * The two writes are ordered label-then-row, and the order is load-bearing. This
2644
- * sweep is the only thing that revisits a `pushed-*` row, so the terminal state
2645
- * is also the row's exit from it: written first, a tracker that then failed on
2646
- * the label would leave `agent:in-progress` with nothing left to retry it — #18
2647
- * exactly, in the last window able to reach it. Writing the label first makes
2648
- * failure cost a repeated `gh` call on the next tick instead, and the row stays
2649
- * in the busy set throughout, so no second worker can be sent at the issue while
2650
- * it waits.
2470
+ * The run row is reused, never cloned: same attempt number, same prUrl, same
2471
+ * transcript. That is what keeps a revision round off both budgets — the round
2472
+ * is a state transition of one already-green row, green running green, and
2473
+ * no new row exists for either counter to count. Only a revision worker that
2474
+ * genuinely settles terminal falls through to the ordinary failure handling,
2475
+ * budgets and all, exactly as today.
2651
2476
  */
2652
- const BASE_CHECK_BATCH = 20;
2653
- const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
2654
- const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
2655
-
2656
- const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
2477
+ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionRecord): Promise<void> {
2478
+ const { project, caps, tracker, store } = d;
2479
+ const run = store.getRun(revision.runId);
2480
+ if (run === undefined || run.state !== "running") {
2481
+ log(
2482
+ `#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${run?.state ?? "gone"} — ` +
2483
+ "the row moved between the dispatch claim and the resume.",
2484
+ );
2485
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2486
+ return;
2487
+ }
2488
+ const issue = run.issue;
2489
+ const branch = run.branch;
2490
+ const runId = run.id;
2491
+ const inProgress = project.stateLabels.inProgress;
2492
+ const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
2493
+ if (repo === undefined) {
2494
+ log(`#${issue} review round ${revision.round} cannot dispatch: repo ${run.repo} is no longer routed`);
2495
+ store.updateRun(runId, { state: "pushed-green", endedAt: Date.now() });
2496
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2497
+ return;
2498
+ }
2499
+ const mirrorPath = mirrorPathFor(repo, project.mirrorRoot);
2500
+
2501
+ // The session being resumed is the one the pushed-green row recorded. A
2502
+ // revision without that transcript cannot continue the same session — the
2503
+ // exact fake this feature exists to prevent — so fail loudly instead of
2504
+ // silently starting a fresh one.
2505
+ const priorSessionFile = run.sessionFile;
2506
+ if (priorSessionFile === undefined || !existsSync(priorSessionFile)) {
2507
+ log(
2508
+ `#${issue} review round ${revision.round} cannot resume: the recorded transcript ` +
2509
+ `${priorSessionFile ?? "(none)"} is gone`,
2510
+ );
2511
+ store.updateRun(runId, {
2512
+ state: "pushed-green",
2513
+ endedAt: Date.now(),
2514
+ lastError: `review round ${revision.round} could not resume: the run's transcript is gone`,
2515
+ });
2516
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2517
+ await safeEscalate(d, {
2518
+ tier: 1,
2519
+ project: project.name,
2520
+ issue,
2521
+ runId,
2522
+ summary: `#${issue} review round ${revision.round} could not resume — the transcript is missing`,
2523
+ detail: [
2524
+ revision.prUrl,
2525
+ "",
2526
+ "The PR is still open and green; the run row was returned to pushed-green.",
2527
+ "Resume the manual path (close + unblock + continuation) or merge it as it stands.",
2528
+ ].join("\n"),
2529
+ });
2530
+ return;
2531
+ }
2532
+ const sessionDir = dirname(priorSessionFile);
2657
2533
 
2658
- const FAILING_WORKFLOW_CONCLUSIONS = new Set([
2659
- "failure",
2660
- "cancelled",
2661
- "timed_out",
2662
- "action_required",
2663
- "startup_failure",
2664
- "stale",
2665
- ]);
2534
+ let worktreePath: string | undefined;
2535
+ let runRepo: RunRepoRef | undefined;
2536
+ let turnLimit: TurnLimitController | undefined;
2537
+ let workerControl: WorkerControlSlot | undefined;
2538
+ let workerSessionInstalled = false;
2539
+ let verbListener: VerbListener | undefined;
2540
+ const repoSlug = githubRepo(repo.cloneUrl);
2666
2541
 
2667
- function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
2668
- const flags = run.settlementFlags ?? [];
2669
- return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
2670
- ? flags
2671
- : [...flags, flag];
2672
- }
2542
+ /**
2543
+ * Publishes the run's branch on the privileged side: run repo → mirror →
2544
+ * GitHub, fast-forward only the same route `handleIssue` uses, so the
2545
+ * revised head the worker pushed through its own `conductor_push` is
2546
+ * re-verified against the remote before the tree dies.
2547
+ */
2548
+ const publish: RunPublisher = async () => {
2549
+ if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
2550
+ return pushRunBranch(project, runRepo);
2551
+ };
2552
+
2553
+ const settleStopBeforeSession = async (): Promise<boolean> => {
2554
+ const reason = workerControl?.requestedStop();
2555
+ if (reason === undefined || workerSessionInstalled) return false;
2556
+ turnLimit?.close();
2557
+ turnLimit = undefined;
2558
+ recordOperatorStop(store, {
2559
+ project: project.name,
2560
+ issue,
2561
+ runId,
2562
+ inProgress,
2563
+ reason,
2564
+ patch: {
2565
+ endedAt: Date.now(),
2566
+ report: ["Operator stopped the review revision before its session started.", `Reason: ${reason}`].join("\n"),
2567
+ },
2568
+ });
2569
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2570
+ log(`#${issue} review round ${revision.round} stopped by operator before its session started: ${reason}`);
2571
+ return true;
2572
+ };
2573
+
2574
+ const settleDrainBeforeSession = async (): Promise<boolean> => {
2575
+ if (d.drain?.draining !== true || workerSessionInstalled) return false;
2576
+ turnLimit?.close();
2577
+ turnLimit = undefined;
2578
+ // A shutdown that interrupts the wake is the same situation as the
2579
+ // provisioning-failure branch below, not an operator stop — nobody
2580
+ // stopped anything, the PR is still green and still open, and only the
2581
+ // wake failed; `settleStopBeforeSession` immediately above covers the
2582
+ // case where an operator genuinely did. So mirror that branch: restore
2583
+ // `pushed-green` naming the shutdown, settle the revision `skipped` so
2584
+ // the round is not double-counted, and leave the in-progress label alone
2585
+ // — the ordinary settle sweep releases it in the same breath it
2586
+ // terminalises the row when the PR resolves. Stopping the row here would
2587
+ // strand it: `settlePushedGreen` sweeps only pushed-* rows and
2588
+ // classification excludes `stopped`, so the issue would carry neither
2589
+ // label and nothing would ever revisit the still-green PR.
2590
+ store.updateRun(runId, {
2591
+ state: "pushed-green",
2592
+ endedAt: Date.now(),
2593
+ lastError: "daemon shutdown began after the review revision claim; the round was not launched",
2594
+ });
2595
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2596
+ log(`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim`);
2597
+ return true;
2598
+ };
2599
+
2600
+ // Live-run controls, opened before any provisioning so the operator stop
2601
+ // and shutdown fences cover the whole wake window, exactly as they do for a
2602
+ // fresh claim in `handleIssue` (#374). The original run's entries were
2603
+ // closed at its settle, so reopening by issue is safe.
2604
+ turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
2605
+ workerControl = d.workerControls.open(project.name, issue, runId);
2606
+ if (await settleStopBeforeSession()) return;
2607
+ if (await settleDrainBeforeSession()) return;
2608
+
2609
+ try {
2610
+ // Reattach the run's own branch at the same per-issue path the run used:
2611
+ // the worktree was removed at the pushed-green settle and the branch was
2612
+ // published, so provisioning is the same continuation reattach as a
2613
+ // normal re-claim. A provisioning failure restores the row — the PR is
2614
+ // still green and still open, and only the wake failed — and says so.
2615
+ try {
2616
+ const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
2617
+ worktreePath = provisioned.path;
2618
+ runRepo = { repo, runRepoPath: worktreePath, branch };
2619
+ } catch (err) {
2620
+ const detail = errText(err);
2621
+ log(`#${issue} review round ${revision.round} cannot provision ${branch}: ${detail}`);
2622
+ // An operator stop that landed during provisioning wins over restoration —
2623
+ // it is a newer, explicit action, and the fences above already honoured
2624
+ // one that landed earlier.
2625
+ const pendingStop = workerControl?.requestedStop();
2626
+ if (pendingStop === undefined) {
2627
+ store.updateRun(runId, {
2628
+ state: "pushed-green",
2629
+ endedAt: Date.now(),
2630
+ lastError: `review round ${revision.round} could not provision the worktree: ${detail}`,
2631
+ });
2632
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2633
+ await safeEscalate(d, {
2634
+ tier: 1,
2635
+ project: project.name,
2636
+ issue,
2637
+ runId,
2638
+ summary: `#${issue} review round ${revision.round} could not be dispatched — worktree provisioning failed`,
2639
+ detail: [revision.prUrl, "", detail].join("\n"),
2640
+ });
2641
+ } else {
2642
+ recordOperatorStop(store, {
2643
+ project: project.name,
2644
+ issue,
2645
+ runId,
2646
+ inProgress,
2647
+ reason: pendingStop,
2648
+ patch: {
2649
+ endedAt: Date.now(),
2650
+ report: [
2651
+ `Operator stopped the review revision before its session started: ${pendingStop}`,
2652
+ `(worktree provisioning also failed: ${detail})`,
2653
+ ].join("\n"),
2654
+ },
2655
+ });
2656
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
2657
+ }
2658
+ return;
2659
+ }
2660
+ if (await settleStopBeforeSession()) return;
2661
+ if (await settleDrainBeforeSession()) return;
2662
+
2663
+ // The fleet-owned omp settings overlay, staged under the SAME session
2664
+ // directory the transcript lives in — a resumed session loads it exactly
2665
+ // as the original run did (#537).
2666
+ const ompSettingsFile = materializeOmpSettings(project, sessionDir);
2667
+
2668
+ // The run's mutation channel (#126): same run id, same issue, same repo —
2669
+ // so `conductor_push` publishes the run's own branch and nothing else.
2670
+ verbListener = await listenVerbChannel(
2671
+ verbDeps(d),
2672
+ {
2673
+ kind: "run",
2674
+ path: verbSocketPath(ensureVerbSocketDir(stateDir()), `run-${String(issue)}`),
2675
+ project: project.name,
2676
+ role: "worker",
2677
+ runId,
2678
+ issue,
2679
+ repo,
2680
+ runRepoPath: worktreePath,
2681
+ branch,
2682
+ },
2683
+ { ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
2684
+ );
2685
+ if (await settleStopBeforeSession()) return;
2686
+ if (await settleDrainBeforeSession()) return;
2687
+
2688
+ store.updateRun(runId, { worktree: worktreePath });
2689
+
2690
+ // The row's own counters are cumulative across the revision: the revision
2691
+ // worker meters its own session from zero, so its deltas are added to the
2692
+ // totals the run already recorded.
2693
+ const baseTurns = run.turns;
2694
+ const baseSpend = run.spendUsd;
2695
+
2696
+ log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
2697
+
2698
+ let result: WorkerResult;
2699
+ try {
2700
+ result = await runWorker({
2701
+ brief: renderReviewRevisionPrompt(revision.findings, revision.round),
2702
+ cwd: worktreePath,
2703
+ caps,
2704
+ ...(repoSlug === undefined ? {} : { repoSlug }),
2705
+ maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
2706
+ onPauseControl: (control) => {
2707
+ workerSessionInstalled = true;
2708
+ workerControl?.install(control);
2709
+ },
2710
+ sessionDir,
2711
+ // The same-session half of the transport: `continueRecent` on the run's
2712
+ // own session directory, never a fresh `run-<uuid>` transcript.
2713
+ resume: true,
2714
+ socketPath: join(sessionDir, "ipc.sock"),
2715
+ verbSocketPath: verbListener.path,
2716
+ onSpawn: (pid) => {
2717
+ verbListener?.bindPid(pid);
2718
+ },
2719
+ onChildLog: (line) => {
2720
+ log(`#${issue} ${line}`);
2721
+ },
2722
+ // The continuation stays on the model the green run used (#286).
2723
+ ...(run.model === undefined ? {} : { model: run.model }),
2724
+ ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
2725
+ releaseGrants: resolveReleaseGrants(project),
2726
+ onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
2727
+ onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
2728
+ onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
2729
+ onKilled: () => {
2730
+ turnLimit?.close();
2731
+ turnLimit = undefined;
2732
+ },
2733
+ onSessionFile: (f) => {
2734
+ store.updateRun(runId, { sessionFile: f });
2735
+ // Same-file lineage is the proof the resume happened: a revision
2736
+ // that opened a different transcript fell back to a fresh session.
2737
+ if (f !== priorSessionFile) {
2738
+ log(
2739
+ `#${issue} review round ${revision.round} resume fell back to a fresh session: opened ${f} ` +
2740
+ `instead of ${priorSessionFile}`,
2741
+ );
2742
+ }
2743
+ },
2744
+ maySpawn: () => d.drain?.draining !== true,
2745
+ }, d.workerDeps);
2746
+ } finally {
2747
+ turnLimit?.close();
2748
+ turnLimit = undefined;
2749
+ }
2750
+
2751
+ if (result.modelFallbackMessage !== undefined) {
2752
+ log(`#${issue} review round ${revision.round} model fallback: ${result.modelFallbackMessage}`);
2753
+ }
2754
+
2755
+ const verified: { state: RunState; reason?: string } =
2756
+ result.state === "pushed-green"
2757
+ ? await verifyPushedGreenClaim(tracker, result)
2758
+ : { state: result.state };
2759
+ const state = verified.state;
2760
+
2761
+ const settlement =
2762
+ state === "pushed-green" || state === "pushed-pending" || state === "merged"
2763
+ ? undefined
2764
+ : await settleWorktree({
2765
+ issue,
2766
+ attempt: run.attempt,
2767
+ ending:
2768
+ state === "blocked"
2769
+ ? "blocked for an operator decision"
2770
+ : state === "stopped"
2771
+ ? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
2772
+ : endedBy(result.killedBy),
2773
+ worktree: worktreePath,
2774
+ branch,
2775
+ publish,
2776
+ ...(state === "failed" || state === "killed"
2777
+ ? ({ tree: "keep" } as const)
2778
+ : ({ tree: "remove", mirrorPath } as const)),
2779
+ });
2780
+ if (settlement === undefined) {
2781
+ const published = await publish(branch);
2782
+ if (!published.ok) {
2783
+ log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
2784
+ }
2785
+ await removeWorktree(mirrorPath, worktreePath);
2786
+ }
2787
+
2788
+ // The settlement report names the review round first (#692): a run that
2789
+ // reaches green again after a round must read as a revision outcome, not
2790
+ // as a fresh run's first push, and a round that failed must still say the
2791
+ // round — the state and classification below carry the failure itself.
2792
+ const finalReport = [
2793
+ `review round ${revision.round}: ${state}`,
2794
+ ...(verified.reason === undefined ? [] : ["", verified.reason]),
2795
+ result.report,
2796
+ ].join("\n");
2797
+
2798
+ const terminalPatch: Partial<RunRecord> = {
2799
+ endedAt: Date.now(),
2800
+ turns: baseTurns + result.turns,
2801
+ spendUsd: baseSpend + result.spendUsd,
2802
+ provider429Count: result.provider429Count,
2803
+ ...(result.model === undefined ? {} : { resolvedModel: result.model }),
2804
+ ...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
2805
+ retryFallbacks: result.retryFallbacks,
2806
+ retryFallbackSucceeded: result.retryFallbackSucceeded,
2807
+ modelRecoveries: result.modelRecoveries,
2808
+ autoRetryCount: result.autoRetryCount,
2809
+ autoCompactionCount: result.autoCompactionCount,
2810
+ // The worker only reports these when it actually established them; a
2811
+ // failure whose report named no PR must not wipe what the row already
2812
+ // owns (#468).
2813
+ ...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
2814
+ ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
2815
+ sessionFile: result.sessionFile,
2816
+ report: finalReport,
2817
+ ...settlement?.patch,
2818
+ };
2819
+ if (state === "stopped") {
2820
+ recordOperatorStop(store, {
2821
+ project: project.name,
2822
+ issue,
2823
+ runId,
2824
+ inProgress,
2825
+ reason: result.stoppedReason ?? "no reason recorded",
2826
+ patch: terminalPatch,
2827
+ });
2828
+ } else {
2829
+ const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
2830
+ const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
2831
+ const providerTransient =
2832
+ providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
2833
+ const lastError = completionLastError(providerCredit, providerTransient, verified.reason, sessionErr);
2834
+ store.updateRun(runId, {
2835
+ ...terminalPatch,
2836
+ state,
2837
+ ...(lastError === undefined ? {} : { lastError }),
2838
+ });
2839
+ }
2840
+
2841
+ const outcome: ReviewRevisionOutcome =
2842
+ state === "pushed-green" ? "revised" : state === "pushed-pending" ? "pending" : "failed";
2843
+ store.settleReviewRevision(revision.id, outcome, Date.now());
2844
+ log(`#${issue} review round ${revision.round} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
2845
+
2846
+ // Post-settle outcomes, mirroring handleIssue: a failed or killed revision
2847
+ // worker falls through to exactly the same label, requeue and escalation
2848
+ // handling as any failed run — budgets included — and a blocked one pages
2849
+ // the orchestrator for a decision.
2850
+ if (state === "stopped") {
2851
+ log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
2852
+ } else if (state === "blocked") {
2853
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
2854
+ await safeEscalate(d, {
2855
+ tier: 1,
2856
+ project: project.name,
2857
+ issue,
2858
+ runId,
2859
+ summary: `#${issue} is blocked on review round ${revision.round} and needs a decision`,
2860
+ detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
2861
+ });
2862
+ } else if (state === "failed" || state === "killed") {
2863
+ const continuation = store.continuationsFor(project.name, issue);
2864
+ const continueTurns = shouldContinueAfterTurnsCap({
2865
+ killedBy: result.killedBy,
2866
+ prUrl: result.prUrl,
2867
+ headSha: result.headSha,
2868
+ salvageSha: settlement?.patch?.salvageSha,
2869
+ continuation,
2870
+ maxContinuations: caps.maxContinuationsPerIssue,
2871
+ });
2872
+ if (continueTurns) {
2873
+ store.enqueueLabelOps(project.name, [
2874
+ { issue, op: "remove", label: inProgress },
2875
+ { issue, op: "add", label: project.queueLabel },
2876
+ ]);
2877
+ log(
2878
+ `#${issue} review round ${revision.round} turns-cap, continuation ` +
2879
+ `${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
2880
+ );
2881
+ await safeEscalate(d, {
2882
+ tier: 1,
2883
+ project: project.name,
2884
+ issue,
2885
+ runId,
2886
+ summary: `#${issue} hit the turns cap on review round ${revision.round} — auto-requeued for continuation`,
2887
+ detail: [
2888
+ `${result.prUrl ?? "(no PR URL)"}`,
2889
+ "",
2890
+ "The queue label is back on; the next tick should reattach the branch",
2891
+ "and open a continuation brief. No failed label was applied.",
2892
+ "",
2893
+ result.report,
2894
+ ].join("\n"),
2895
+ });
2896
+ } else {
2897
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
2898
+ await safeEscalate(d, {
2899
+ tier: 1,
2900
+ project: project.name,
2901
+ issue,
2902
+ runId,
2903
+ summary: result.killedBy
2904
+ ? `#${issue} was killed on review round ${revision.round} by the ${result.killedBy} cap`
2905
+ : `#${issue} failed on review round ${revision.round}`,
2906
+ detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
2907
+ });
2908
+ }
2909
+ }
2910
+ } catch (err) {
2911
+ // A crash anywhere after the worker ran settles like any run's dispatch
2912
+ // error: the row is terminal, the tree is kept, and the issue is relabelled
2913
+ // so nobody silently re-claims it.
2914
+ turnLimit?.close();
2915
+ turnLimit = undefined;
2916
+ const detail = errText(err);
2917
+ log(`#${issue} review round ${revision.round} errored: ${detail}`);
2918
+ const settlement =
2919
+ worktreePath === undefined
2920
+ ? undefined
2921
+ : await settleWorktree({
2922
+ issue,
2923
+ attempt: run.attempt,
2924
+ ending: "killed by a dispatch error",
2925
+ worktree: worktreePath,
2926
+ branch,
2927
+ publish,
2928
+ tree: "keep",
2929
+ });
2930
+ store.updateRun(runId, {
2931
+ state: "failed",
2932
+ endedAt: Date.now(),
2933
+ lastError: detail,
2934
+ ...settlement?.patch,
2935
+ });
2936
+ store.settleReviewRevision(revision.id, "failed", Date.now());
2937
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
2938
+ await safeEscalate(d, {
2939
+ tier: 1,
2940
+ project: project.name,
2941
+ issue,
2942
+ runId,
2943
+ summary: `#${issue} review round ${revision.round} could not be dispatched`,
2944
+ detail: [
2945
+ `${revision.prUrl}`,
2946
+ "",
2947
+ detail,
2948
+ ...(settlement?.lines ?? []),
2949
+ ].join("\n"),
2950
+ });
2951
+ } finally {
2952
+ turnLimit?.close();
2953
+ if (verbListener !== undefined) {
2954
+ try {
2955
+ await verbListener.close();
2956
+ } catch (err) {
2957
+ log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
2958
+ }
2959
+ verbListener = undefined;
2960
+ }
2961
+ workerControl?.close();
2962
+ workerControl = undefined;
2963
+ }
2964
+ }
2965
+
2966
+ /**
2967
+ * Restart recovery for in-flight review revisions (#692): a revision the
2968
+ * previous daemon claimed (`pushed-green` → `running`) and lost when it died
2969
+ * is restored to the exact state the orchestrator's verb left it in — same
2970
+ * run, same PR, same findings, same round, same session file — so the next
2971
+ * dispatch pass resumes the SAME OMP session rather than leaving the run to
2972
+ * the failure classifier.
2973
+ *
2974
+ * Runs AFTER the startup orphan sweep, on purpose: the sweep has already
2975
+ * salvaged the crashed revision's worktree to the branch (so no work is lost)
2976
+ * and marked the row `orphaned` — and this reconcile then restores that row
2977
+ * to `pushed-green` and re-queues the round before the first tick, so the
2978
+ * failure classifier never sees it. That ordering matters twice over: an
2979
+ * orphaned row would charge the continuation budget for a round that must
2980
+ * stay off it (#677), and the `review_revisions` row is the only place the
2981
+ * findings and round live. A revision the previous daemon never claimed (its
2982
+ * run is still `pushed-green`, its row still pending) needs no restoration —
2983
+ * the ordinary dispatch pass wakes it on the next tick, exactly as #677
2984
+ * already guarantees.
2985
+ *
2986
+ * A round whose run cannot be restored — the run is gone, settled terminal,
2987
+ * or its repo is no longer routed — fails closed instead: the revision is
2988
+ * settled `skipped` with the findings still on the row, ONE escalation names
2989
+ * what could not be restored, and no worker is started against a PR that may
2990
+ * already have one.
2991
+ */
2992
+ export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRevisionRecord[]> {
2993
+ const { project, store } = d;
2994
+ const revisions = store.unsettledReviewRevisions(project.name);
2995
+ if (revisions.length === 0) return [];
2996
+ const recovered: ReviewRevisionRecord[] = [];
2997
+ for (const revision of revisions) {
2998
+ const run = store.getRun(revision.runId);
2999
+ if (run === undefined) {
3000
+ log(
3001
+ `#${revision.issue} review round ${revision.round} cannot be restored across the restart: ` +
3002
+ `run ${revision.runId} no longer exists`,
3003
+ );
3004
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3005
+ await safeEscalate(d, {
3006
+ tier: 1,
3007
+ project: project.name,
3008
+ issue: revision.issue,
3009
+ runId: revision.runId,
3010
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3011
+ detail: [
3012
+ revision.prUrl,
3013
+ "",
3014
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3015
+ `the run row ${revision.runId} no longer exists.`,
3016
+ "The round was settled skipped; the findings stay on this revision's durable row.",
3017
+ "Re-review the PR (it is still open and green) or merge it as it stands.",
3018
+ ].join("\n"),
3019
+ });
3020
+ continue;
3021
+ }
3022
+ // A revision the previous daemon never claimed: the run is still a
3023
+ // settled green row and the round is still pending, so the ordinary
3024
+ // dispatch pass wakes it on the next tick untouched.
3025
+ if (run.state === "pushed-green") continue;
3026
+ // The run a previous daemon claimed for this round and died on: the
3027
+ // orphan sweep just marked it `orphaned` (salvaging the tree to the
3028
+ // branch), so restore it to the reviewable state the verb recorded.
3029
+ const crashed =
3030
+ run.state === "claimed" || run.state === "running" || run.state === "orphaned";
3031
+ if (!crashed) {
3032
+ log(
3033
+ `#${revision.issue} review round ${revision.round} cannot be restored across the restart: run ` +
3034
+ `${revision.runId} is ${run.state}`,
3035
+ );
3036
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3037
+ await safeEscalate(d, {
3038
+ tier: 1,
3039
+ project: project.name,
3040
+ issue: revision.issue,
3041
+ runId: revision.runId,
3042
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3043
+ detail: [
3044
+ revision.prUrl,
3045
+ "",
3046
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3047
+ `the run row is ${run.state}, not a state a review round resumes from.`,
3048
+ "The round was settled skipped; the findings stay on this revision's durable row.",
3049
+ "Re-review the PR (it is still open and green) or merge it as it stands.",
3050
+ ].join("\n"),
3051
+ });
3052
+ continue;
3053
+ }
3054
+ try {
3055
+ // The crashed revision's worktree — salvaged and published by the orphan
3056
+ // sweep that just ran, so the branch in the mirror has every committed
3057
+ // byte — must be gone before the next dispatch, or `addRunRepo` refuses
3058
+ // the reattach as possibly holding a previous attempt's work (#692).
3059
+ // `settleWorktree` with `tree: "remove"` re-attempts the salvage instead
3060
+ // of trusting it (a tree whose salvage failed is the only copy of work
3061
+ // and is NEVER removed), and refuses the removal if that re-attempt
3062
+ // fails — the fail-closed half of this recovery.
3063
+ if (run.worktree !== "" && existsSync(run.worktree)) {
3064
+ const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
3065
+ if (repo === undefined) {
3066
+ log(
3067
+ `#${revision.issue} review round ${revision.round} cannot clear worktree ${run.worktree}: ` +
3068
+ `repo ${run.repo} is no longer routed — the tree may hold the only copy of work`,
3069
+ );
3070
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3071
+ await safeEscalate(d, {
3072
+ tier: 1,
3073
+ project: project.name,
3074
+ issue: revision.issue,
3075
+ runId: revision.runId,
3076
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3077
+ detail: [
3078
+ revision.prUrl,
3079
+ "",
3080
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3081
+ `repo ${run.repo} is no longer routed, so the crashed revision's worktree cannot be published and removed.`,
3082
+ `The worktree is retained for recovery: ${run.worktree}`,
3083
+ "Recover it by hand (it may hold the round's last work), then re-review or merge the PR.",
3084
+ ].join("\n"),
3085
+ });
3086
+ continue;
3087
+ }
3088
+ const settlement = await settleWorktree({
3089
+ issue: run.issue,
3090
+ attempt: run.attempt,
3091
+ ending: "interrupted by a daemon restart during a review round",
3092
+ worktree: run.worktree,
3093
+ branch: run.branch,
3094
+ publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
3095
+ tree: "remove",
3096
+ mirrorPath: mirrorPathFor(repo, project.mirrorRoot),
3097
+ });
3098
+ if (settlement.retained) {
3099
+ log(
3100
+ `#${revision.issue} review round ${revision.round} worktree ${run.worktree} retained: ` +
3101
+ "its salvage failed, so the tree is the only copy of work and will not be removed",
3102
+ );
3103
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3104
+ await safeEscalate(d, {
3105
+ tier: 1,
3106
+ project: project.name,
3107
+ issue: revision.issue,
3108
+ runId: revision.runId,
3109
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3110
+ detail: [
3111
+ revision.prUrl,
3112
+ "",
3113
+ "The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
3114
+ "the crashed revision's worktree could not be salvaged, so it is the only copy of work and was retained.",
3115
+ `Recover the tree by hand: ${run.worktree}`,
3116
+ "Then re-review the PR (still open and green) or merge it as it stands.",
3117
+ ].join("\n"),
3118
+ });
3119
+ continue;
3120
+ }
3121
+ }
3122
+ } catch (err) {
3123
+ // A settleWorktree throw (mirror unreachable, git refused) must not take
3124
+ // the whole startup reconcile down with one run's tree: fail this round
3125
+ // closed and let the next one try.
3126
+ const detail = errText(err);
3127
+ log(`#${revision.issue} review round ${revision.round} restore errored: ${detail}`);
3128
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3129
+ await safeEscalate(d, {
3130
+ tier: 1,
3131
+ project: project.name,
3132
+ issue: revision.issue,
3133
+ runId: revision.runId,
3134
+ summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
3135
+ detail: [revision.prUrl, "", detail].join("\n"),
3136
+ });
3137
+ continue;
3138
+ }
3139
+ // The exact state the verb recorded: a pushed-green row plus a pending
3140
+ // revision whose durable row still carries the findings, reviewed head,
3141
+ // round and target session. The next tick's dispatch pass claims and
3142
+ // wakes it exactly like a revision that was never dispatched.
3143
+ store.updateRun(revision.runId, {
3144
+ state: "pushed-green",
3145
+ endedAt: Date.now(),
3146
+ worktree: "",
3147
+ lastError: `review round ${revision.round} was restored across a daemon restart; the round is re-queued`,
3148
+ });
3149
+ store.requeueReviewRevision(revision.id);
3150
+ log(
3151
+ `#${revision.issue} review round ${revision.round} restored across restart: run ${revision.runId} ` +
3152
+ "is pushed-green again and the round is re-queued — the next dispatch pass resumes the same session",
3153
+ );
3154
+ recovered.push(revision);
3155
+ }
3156
+ return recovered;
3157
+ }
3158
+
3159
+ // ------------------------------------------------------------------- settlement
3160
+ const BASE_CHECK_BATCH = 20;
3161
+ const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
3162
+ const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
3163
+
3164
+ const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
3165
+
3166
+ const FAILING_WORKFLOW_CONCLUSIONS = new Set([
3167
+ "failure",
3168
+ "cancelled",
3169
+ "timed_out",
3170
+ "action_required",
3171
+ "startup_failure",
3172
+ "stale",
3173
+ ]);
3174
+
3175
+ function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
3176
+ const flags = run.settlementFlags ?? [];
3177
+ return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
3178
+ ? flags
3179
+ : [...flags, flag];
3180
+ }
2673
3181
 
2674
3182
  /**
2675
3183
  * Observe Actions on exact merge commits for up to one day. A running workflow
@@ -2956,220 +3464,6 @@ export async function watchBaseHealth(
2956
3464
  }
2957
3465
  }
2958
3466
 
2959
- export async function settlePushedGreen(
2960
- d: Pick<Deps, "project" | "tracker" | "store">,
2961
- ): Promise<number> {
2962
- const { project, tracker, store } = d;
2963
- // Runs the sweep resolved by terminalising the row. Every terminal write
2964
- // below increments it; the tick attributes it as `settled` on the pass's
2965
- // dispatch record, so a held pass is visible as work done, not just as a
2966
- // clock that moved (#497).
2967
- let settled = 0;
2968
- // Filtered from the active set rather than asked for with a new query: active
2969
- // is live workers plus these, so the list is bounded by the worker cap plus
2970
- // the number of PRs awaiting a merge — a handful, by construction. A fleet
2971
- // where that is not a handful has a merge problem, not a dispatch one.
2972
- const pending = store
2973
- .activeRuns(project.name)
2974
- .filter((r) => r.state === "pushed-green" || r.state === "pushed-pending");
2975
-
2976
- for (const run of pending) {
2977
- // Nothing to ask about. A pushed result requires a PR, so a malformed row
2978
- // must not buy a `gh` call every five minutes forever.
2979
- if (run.prUrl === undefined) continue;
2980
-
2981
- let pr: PrState | undefined;
2982
- try {
2983
- pr = await tracker.prState(run.prUrl);
2984
- } catch (err) {
2985
- // Per row, like admission's held candidate. The GitHub adapter already
2986
- // answers undefined instead of throwing, so this catch is the port's
2987
- // contract rather than that adapter's behaviour — and a tracker that does
2988
- // throw must cost its own row, not the whole sweep.
2989
- log(`#${run.issue} not settled: PR state lookup failed (${errText(err)}) — retrying next tick`);
2990
- continue;
2991
- }
2992
-
2993
- const settlement = settlementFor(pr, run.prUrl);
2994
- if (settlement !== undefined) {
2995
- // A mediated merge enters a second, bounded observation phase. Record the
2996
- // exact merge commit before the row leaves the active set; if GitHub cannot
2997
- // supply it yet, retry this settlement next tick rather than create a
2998
- // merged row whose base result can never be attributed.
2999
- let merged: MergedPrInfo | undefined;
3000
- if (settlement.state === "merged") {
3001
- try {
3002
- merged = await tracker.mergedPrInfo(run.prUrl);
3003
- } catch (err) {
3004
- log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
3005
- continue;
3006
- }
3007
- if (merged === undefined) {
3008
- log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
3009
- continue;
3010
- }
3011
- }
3012
-
3013
- // The label removal and the terminal row are one fact again (#201): the
3014
- // release is enqueued — a durable local write that cannot fail on the
3015
- // tracker — in the same breath as the row is terminalised.
3016
- releaseInProgress(d, run.issue, settlement.reason);
3017
- const patch: Partial<RunRecord> = {
3018
- state: settlement.state,
3019
- endedAt: Date.now(),
3020
- ...(merged === undefined
3021
- ? {}
3022
- : {
3023
- mergeSha: merged.mergeSha,
3024
- baseRef: merged.baseRef,
3025
- baseCheck: "pending",
3026
- }),
3027
- };
3028
- if (settlement.state === "failed") {
3029
- patch.lastError = settlement.reason;
3030
- patch.failureClass = "returned-for-revision";
3031
- patch.recoveryAction = "none";
3032
- }
3033
- store.updateRun(run.id, patch);
3034
- settled += 1;
3035
- log(`#${run.issue} settled: ${settlement.reason}`);
3036
- continue;
3037
- }
3038
-
3039
- if (run.state !== "pushed-pending" || pr !== "open" || run.headSha === undefined) continue;
3040
- let verification;
3041
- try {
3042
- verification = await tracker.verifyPr(run.prUrl, run.headSha);
3043
- } catch (err) {
3044
- log(`#${run.issue} checks not settled (${errText(err)}) — retrying next tick`);
3045
- continue;
3046
- }
3047
- if (verification === undefined) continue;
3048
- if (verification.status === "green") {
3049
- store.updateRun(run.id, { state: "pushed-green", lastError: null });
3050
- log(`#${run.issue} checks settled: ${verification.reason}`);
3051
- } else if (verification.status === "failed") {
3052
- // Equally terminal, so the release is enqueued before the row writes,
3053
- // for the same reason as the settlement branch above (see there). The
3054
- // green branch releases nothing — that row is still awaiting a merge,
3055
- // and its live PR is exactly the work the label must keep guarding.
3056
- releaseInProgress(d, run.issue, verification.reason);
3057
- store.updateRun(run.id, { state: "failed", lastError: verification.reason });
3058
- settled += 1;
3059
- log(`#${run.issue} checks failed: ${verification.reason}`);
3060
- } else {
3061
- store.updateRun(run.id, { lastError: verification.reason });
3062
- }
3063
- }
3064
- return settled;
3065
- }
3066
-
3067
- const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
3068
- failed: true,
3069
- killed: true,
3070
- orphaned: true,
3071
- blocked: true,
3072
- };
3073
-
3074
- /**
3075
- * Reattaches a recovered PR to the terminal run that owns it (#245).
3076
- *
3077
- * A worker can fail before its completion report records `prUrl`, then have its
3078
- * dirty tree committed and pushed by salvage. If that branch already has a PR,
3079
- * the orchestrator otherwise has no policy-compliant path to inspect or merge
3080
- * it: ownership is store-backed. Adoption is deliberately stricter than
3081
- * admission. The tracker query proves the PR closes this run's issue; exact
3082
- * branch and canonical repository matches prove it is this run's recovered
3083
- * work, not an unrelated closer. Missing identity is refusal, never a guess.
3084
- *
3085
- * The newest run per issue is inspected, at most ten per tick and only inside
3086
- * the same 30-day window as mediated PR verbs. The cursor advances through the
3087
- * full eligible set so persistent non-matches cannot starve older recovered
3088
- * work. Successful adoption is idempotent because the row gains `prUrl`;
3089
- * non-matches are logged once per daemon process.
3090
- */
3091
- const rejectedSalvagedPrRuns = new Set<string>();
3092
- const salvagedPrCursor = new Map<string, string>();
3093
-
3094
- export async function adoptSalvagedPrs(
3095
- d: Pick<Deps, "project" | "tracker" | "store">,
3096
- now = Date.now(),
3097
- ): Promise<void> {
3098
- const { project, tracker, store } = d;
3099
- const eligible = store
3100
- .recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
3101
- .filter(
3102
- (run) =>
3103
- ADOPTABLE_PR_STATES[run.state] === true &&
3104
- run.prUrl === undefined &&
3105
- run.branch.trim() !== "",
3106
- );
3107
- const previous = salvagedPrCursor.get(project.name);
3108
- const previousIndex =
3109
- previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
3110
- const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
3111
- const candidates = Array.from(
3112
- { length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
3113
- (_, offset) => eligible[(start + offset) % eligible.length]!,
3114
- );
3115
- const last = candidates.at(-1);
3116
- if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
3117
-
3118
- for (const run of candidates) {
3119
- const repo = project.routing.repos[run.repo];
3120
- const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
3121
- let closers: OpenCloser[];
3122
- try {
3123
- closers = await tracker.openClosersFor(run.issue);
3124
- } catch (err) {
3125
- log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
3126
- continue;
3127
- }
3128
-
3129
- const closer = closers.find(
3130
- (candidate) =>
3131
- candidate.headRefName !== "" &&
3132
- candidate.headRefName === run.branch &&
3133
- repo !== undefined &&
3134
- candidate.repo !== "" &&
3135
- candidate.repo === repoIdentity,
3136
- );
3137
- if (closer === undefined) {
3138
- if (!rejectedSalvagedPrRuns.has(run.id)) {
3139
- const observed = closers[0];
3140
- const reason =
3141
- observed === undefined
3142
- ? "no open closing PR"
3143
- : observed.headRefName === ""
3144
- ? "closer has no head branch identity"
3145
- : observed.headRefName !== run.branch
3146
- ? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
3147
- : observed.repo === ""
3148
- ? "closer has no repository identity"
3149
- : `closer repository ${observed.repo} does not match routed repository`;
3150
- log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
3151
- rejectedSalvagedPrRuns.add(run.id);
3152
- }
3153
- continue;
3154
- }
3155
-
3156
- const flag: SettlementFlag = {
3157
- kind: "pr-adopted",
3158
- file: "(recovery)",
3159
- detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
3160
- };
3161
- store.updateRun(run.id, {
3162
- prUrl: closer.url,
3163
- settlementFlags: [...(run.settlementFlags ?? []), flag],
3164
- });
3165
- rejectedSalvagedPrRuns.delete(run.id);
3166
- log(
3167
- `#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
3168
- (run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
3169
- );
3170
- }
3171
- }
3172
-
3173
3467
  /** Canonical `owner/repo` identity from a configured network clone URL. */
3174
3468
  function githubRepo(cloneUrl: string): string | undefined {
3175
3469
  const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
@@ -3197,7 +3491,7 @@ type CleanupRetainedWorktree = (
3197
3491
  * dirty or uniquely unpushed work.
3198
3492
  */
3199
3493
  export async function cleanupRetainedRuns(
3200
- d: Pick<Deps, "project" | "tracker" | "store">,
3494
+ d: Pick<Deps, "project" | "tracker" | "store" | "escalate">,
3201
3495
  queuedIssues: ReadonlySet<number>,
3202
3496
  cursor: RetainedCleanupCursor,
3203
3497
  cleanup: CleanupRetainedWorktree = cleanupRetainedWorktree,
@@ -3262,9 +3556,35 @@ export async function cleanupRetainedRuns(
3262
3556
 
3263
3557
  const outcome = await cleanup(mirrorPathFor(repo, project.mirrorRoot), run.worktree, run.branch);
3264
3558
  if (outcome.kind === "removed") {
3265
- store.updateRun(run.id, { worktree: "" });
3559
+ store.updateRun(run.id, {
3560
+ worktree: "",
3561
+ ...(run.quarantineDetail === undefined ? {} : { quarantineDetail: null }),
3562
+ });
3266
3563
  log(`#${run.issue} retained worktree reaped: ${run.worktree} (${run.branch})`);
3564
+ } else if (outcome.reason === "quarantined") {
3565
+ // A tree whose object store cannot be made sound is potentially
3566
+ // stranded work: the daemon refuses to fetch into it, so its commits
3567
+ // cannot be verified against any remote. The row records the condition
3568
+ // so the status snapshot can name the tree, and the escalation ledger
3569
+ // dedupes on the stable summary below — a pass that keeps seeing the
3570
+ // same broken tree reports it once, never once per dispatch pass.
3571
+ store.updateRun(run.id, { quarantineDetail: outcome.detail });
3572
+ await safeEscalate(d, {
3573
+ tier: 1,
3574
+ project: project.name,
3575
+ issue: run.issue,
3576
+ summary: `#${run.issue} quarantined retained worktree — potentially stranded work`,
3577
+ detail: `${run.worktree} (${run.branch})\n${outcome.detail}`,
3578
+ });
3267
3579
  } else {
3580
+ // Any other retained reason means the tree is back under ordinary
3581
+ // retention: its alternates were repaired (or never needed it), so the
3582
+ // quarantine — if one was marked — is over and the snapshot must not
3583
+ // keep naming it as quarantined.
3584
+ if (run.quarantineDetail !== undefined) {
3585
+ store.updateRun(run.id, { quarantineDetail: null });
3586
+ log(`#${run.issue} retained worktree no longer quarantined: ${outcome.detail}`);
3587
+ }
3268
3588
  log(`#${run.issue} retained worktree kept (${outcome.reason}): ${outcome.detail}`);
3269
3589
  }
3270
3590
  }
@@ -3277,6 +3597,7 @@ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
3277
3597
  "parent-lookup-error",
3278
3598
  "open-pr-lookup-error",
3279
3599
  "issue-state-lookup-error",
3600
+ "critical-base-verify-error",
3280
3601
  ]);
3281
3602
 
3282
3603
  /** Groups transient decisions into the bounded record exposed by status. */
@@ -3471,10 +3792,63 @@ async function verifyPendingUpgradeTick(
3471
3792
  );
3472
3793
  }
3473
3794
  } catch (err) {
3474
- // A torn journal must not take the tick down: the operator still reads it
3475
- // through `status`, and the next tick retries the same pass.
3476
- log(`upgrade verification pass failed: ${errText(err)}`);
3795
+ // A torn journal must not take the tick down: the operator still reads it
3796
+ // through `status`, and the next tick retries the same pass.
3797
+ log(`upgrade verification pass failed: ${errText(err)}`);
3798
+ }
3799
+ }
3800
+
3801
+ /**
3802
+ * The daemon's conductor.db snapshot cadence (#289) — the ledger's only
3803
+ * durable copy, taken on the digest-aligned {@link dbSnapshotDue} window:
3804
+ * once per local day, at/after the digest's configured `at` when it has one.
3805
+ *
3806
+ * Host-global, like the store itself: one snapshot per day for every project,
3807
+ * so whichever project's daemon wins the day first publishes it and the
3808
+ * others no-op on the same durable marker. The marker is written through the
3809
+ * store's existing notification ledger (a one-row idempotence guard, the same
3810
+ * primitive escalations use not to act twice) *after* the snapshot is
3811
+ * published — a crash between publish and mark re-snapshots the next tick
3812
+ * instead of skipping the day, and a restored store that predates today's
3813
+ * marker takes a fresh snapshot on the next tick.
3814
+ *
3815
+ * Returns true when a snapshot was published. A store that does not exist
3816
+ * yet is a silent no-op: there is nothing to back up, and `doctor`'s
3817
+ * `db-backup` probe already treats that as a pass.
3818
+ */
3819
+ export function runDbSnapshotCadence(args: {
3820
+ digestPolicy: ReportingPolicy["digest"];
3821
+ store: Pick<Store, "wasNotified" | "markNotified">;
3822
+ source: string;
3823
+ backupDir: string;
3824
+ now?: number;
3825
+ keep?: number;
3826
+ log?: (line: string) => void;
3827
+ }): boolean {
3828
+ const { digestPolicy, store, source, backupDir } = args;
3829
+ // No store yet (first boot before any run persisted) → nothing to back up;
3830
+ // `doctor`'s `db-backup` probe already treats that as a pass.
3831
+ if (!existsSync(source)) return false;
3832
+ const now = args.now ?? Date.now();
3833
+ const today = localDayKey(now, digestPolicy.timezone);
3834
+ const alreadyToday = store.wasNotified(dbSnapshotMarkerKey(today));
3835
+ if (!dbSnapshotDue({ digest: digestPolicy }, alreadyToday ? today : undefined, now)) {
3836
+ return false;
3837
+ }
3838
+ const published = snapshotDb(source, backupDir, now);
3839
+ (args.log ?? log)(`conductor.db snapshot published: ${published}`);
3840
+ store.markNotified(dbSnapshotMarkerKey(today));
3841
+ try {
3842
+ const removed = pruneDbSnapshots(backupDir, args.keep ?? DB_SNAPSHOT_RETENTION);
3843
+ if (removed > 0) {
3844
+ (args.log ?? log)(`db snapshot retention pruned ${removed} file(s) beyond the retained ${args.keep ?? DB_SNAPSHOT_RETENTION}`);
3845
+ }
3846
+ } catch (err) {
3847
+ // The snapshot is already published and marked; an over-bound directory
3848
+ // costs disk until the next due day retries the prune, never the backup.
3849
+ (args.log ?? log)(`db snapshot retention prune failed: ${errText(err)}`);
3477
3850
  }
3851
+ return true;
3478
3852
  }
3479
3853
 
3480
3854
  export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
@@ -3482,9 +3856,13 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3482
3856
  // (#170). Re-resolve the project and its caps at the tick boundary so a tick
3483
3857
  // and every run it admits see one consistent snapshot; a failed read keeps
3484
3858
  // the boot values rather than wedging the tick, and the next tick tries
3485
- // again.
3859
+ // again. The same reloaded config feeds the db-snapshot backup dir: when it
3860
+ // is unreadable the cadence falls back to the state-root default, matching
3861
+ // `doctor`'s probe of an absent field.
3862
+ let reloadedConfig: ConductorConfig | undefined;
3486
3863
  try {
3487
3864
  const cfg = loadConfig();
3865
+ reloadedConfig = cfg;
3488
3866
  const fresh = findProject(cfg, d.project.name);
3489
3867
  const freshCaps = resolveCaps(fresh, cfg.defaults);
3490
3868
  if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
@@ -3566,6 +3944,25 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3566
3944
  log(`salvaged PR adoption sweep failed: ${errText(err)}`);
3567
3945
  }
3568
3946
 
3947
+ // The conductor.db snapshot cadence (#289) is durability, not dispatch:
3948
+ // the verb ledger, the decision rows and the run history have no other
3949
+ // copy, so they are snapshotted once per day on the digest-aligned window.
3950
+ // Above the pause gate on purpose — a parked fleet still accumulates ledger
3951
+ // rows and still deserves a backup — and a failure costs the day's snapshot
3952
+ // placeholder, never the tick: failed cadence steps are logged and retried
3953
+ // by the five-minute loop, exactly like the sweeps above it.
3954
+ try {
3955
+ runDbSnapshotCadence({
3956
+ digestPolicy: (d.project.reporting ?? DEFAULT_REPORT_POLICY).digest,
3957
+ store: d.store,
3958
+ source: dbPath(),
3959
+ backupDir: dbBackupDirFor(reloadedConfig),
3960
+ log,
3961
+ });
3962
+ } catch (err) {
3963
+ log(`db snapshot cadence failed: ${errText(err)}`);
3964
+ }
3965
+
3569
3966
  // Immediately after settlement and before any routing, so a class is on the
3570
3967
  // row before the next dispatch decision reads its budgets (#132). Above the
3571
3968
  // pause gate deliberately: classification and label reconciliation are
@@ -3575,7 +3972,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3575
3972
  // Each is guarded on its own: a tracker that fails mid-classification must not
3576
3973
  // stop the label reconcile, and neither may stop the tick.
3577
3974
  try {
3578
- settled += await classifyAndRecover(d);
3975
+ settled += await classifyAndRecover({
3976
+ project: d.project,
3977
+ caps: d.caps,
3978
+ tracker: d.tracker,
3979
+ store: d.store,
3980
+ escalate: (e) => d.escalate(e),
3981
+ isPaused,
3982
+ setPaused,
3983
+ });
3579
3984
  } catch (err) {
3580
3985
  log(`classification sweep failed: ${errText(err)}`);
3581
3986
  }
@@ -3632,6 +4037,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3632
4037
  // so `last dispatch` keeps moving while the fleet is deliberately parked. A
3633
4038
  // frozen clock is then a stall report, not a hold (#497).
3634
4039
  if (isPaused(d.project.name)) {
4040
+ // The held pass is the daemon-side admission acknowledgement: the daemon
4041
+ // has reached its admission boundary under the fence and claims nothing.
4042
+ // Written durably so the setup barrier can prove the fence was observed
4043
+ // (#651 review #3) — a pause cannot acknowledge itself.
4044
+ try {
4045
+ writeAdmissionAck(d.project.name);
4046
+ } catch (err) {
4047
+ log(`admission acknowledgement write failed: ${errText(err)}`);
4048
+ }
3635
4049
  d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
3636
4050
  return;
3637
4051
  }
@@ -3694,6 +4108,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3694
4108
  return;
3695
4109
  }
3696
4110
 
4111
+ // Review revisions are dispatch, not admission (#677): the orchestrator's
4112
+ // verb recorded them durably and the run row already occupies its issue, so
4113
+ // they bypass the ready queue and the claim path entirely — this pass wakes
4114
+ // them (bounded by the same worker slots) BEFORE the queue's capacity gate
4115
+ // reads live runs, so a claimed revision counts against capacity exactly like
4116
+ // the worker it is about to spawn.
4117
+ try {
4118
+ await dispatchReviewRevisions(d, workers);
4119
+ } catch (err) {
4120
+ log(`review revision dispatch failed: ${errText(err)}`);
4121
+ }
4122
+
3697
4123
  // route() filters the queue through isEligible() itself, so anything already
3698
4124
  // carrying a state label is gone before it gets here.
3699
4125
  const ready = await d.tracker.listReady();
@@ -3713,14 +4139,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3713
4139
  return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
3714
4140
  });
3715
4141
  const { routed, unroutable } = route(effective, project);
3716
- // Ready issues that route() dropped without a reason carry a state label
3717
- // claimed by a live or settling run — so the claimed count is subtraction
3718
- // rather than a routed-state signal (#228).
3719
- const claimed = effective.length - routed.length - unroutable.length;
3720
- const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
3721
- issue: u.issue.number,
3722
- reason: `unroutable:${u.reason}`,
3723
- }));
4142
+ // route() drops a candidate for two reasons, only one of which is a claim
4143
+ // question. A lifecycle state label (agent:in-progress/blocked/failed)
4144
+ // marks a run-owned issue: it is genuinely in flight while its newest run
4145
+ // is live or settling, and a terminal newest run — or no run at all — means
4146
+ // the label is residual, the footprint of a settled run nobody cleared,
4147
+ // which is Duty 1 reconciliation work, not occupied capacity. A missing
4148
+ // queue label instead marks an issue the outbox is withdrawing (a pending
4149
+ // queue-label removal, e.g. releaseQueueLabel after a merge the PR did not
4150
+ // close), which belongs in none of the three populations. So `claimed` is
4151
+ // defined from actual ownership over genuinely lifecycle-labelled candidates
4152
+ // rather than as the residual of routing (#228, #611).
4153
+ const stateLabels = new Set(Object.values(project.stateLabels));
4154
+ const dropped = effective.filter(
4155
+ (issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
4156
+ );
4157
+ let claimed = 0;
4158
+ const lifecycleHolds: AdmissionHold[] = [];
4159
+ for (const issue of dropped) {
4160
+ const newest = store.latestRun(project.name, issue.number);
4161
+ if (newest !== undefined && ACTIVE_STATES.includes(newest.state)) {
4162
+ claimed += 1;
4163
+ } else {
4164
+ lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
4165
+ }
4166
+ }
4167
+ const routingHolds: AdmissionHold[] = [
4168
+ ...unroutable.map(
4169
+ (u): AdmissionHold => ({ issue: u.issue.number, reason: `unroutable:${u.reason}` }),
4170
+ ),
4171
+ ...lifecycleHolds,
4172
+ ];
3724
4173
  const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
3725
4174
  store.recordDispatch(
3726
4175
  project.name,
@@ -3817,7 +4266,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3817
4266
  log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
3818
4267
  await dispatchAdmissions(
3819
4268
  pass.admitted,
3820
- (a) => handleIssue(d, a.r, a.attempt),
4269
+ (a) => handleIssue(d, a.r, a.attempt, a.lane),
3821
4270
  workers,
3822
4271
  );
3823
4272
 
@@ -4211,11 +4660,33 @@ export interface StatusSnapshot {
4211
4660
  * the config or the brief.
4212
4661
  */
4213
4662
  releaseGrants: ResolvedGrants;
4663
+ /**
4664
+ * The effective review policy (#678). On the snapshot for the reason
4665
+ * `releaseGrants` is: the orchestrator's Duty 1 has to act on it every
4666
+ * tick, and a stale level or ceiling sitting only in a config file nobody
4667
+ * opens is exactly the drift this field exists to surface. The loader always
4668
+ * materialises it, so this is never absent on a real daemon.
4669
+ */
4670
+ review: ReviewPolicy;
4214
4671
  /** Occupied issues: live workers plus green PRs awaiting a human merge. */
4215
4672
  activeRuns: RunRecord[];
4673
+ /**
4674
+ * runId → live review-revision round, for runs whose revision worker is
4675
+ * currently dispatched (#692). Derived from the durable `review_revisions`
4676
+ * rows, so the rendered `review-revision N` state rests on the same facts
4677
+ * the restart recovery reads — never on a label or a guess. A plain object
4678
+ * (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
4679
+ * it byte for byte.
4680
+ */
4681
+ reviewRounds?: Readonly<Record<string, number>>;
4216
4682
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
4217
4683
  * only copy of work the daemon could not save. */
4218
4684
  salvagedRuns: RunRecord[];
4685
+ /** Retained trees whose object store could not be made sound, so the daemon
4686
+ * refused to fetch into them and their commits cannot be verified against
4687
+ * any remote (#737). Distinguished from ordinary retention on purpose —
4688
+ * quarantine is "potentially stranded work", not routine housekeeping. */
4689
+ quarantinedRuns: RunRecord[];
4219
4690
  /** One-shot issue ceilings waiting for the next claim. */
4220
4691
  turnOverrides: TurnOverride[];
4221
4692
  /** Reports the operator has not provably received: pending, in-flight with an
@@ -4302,6 +4773,13 @@ export function statusSnapshotFromStore(
4302
4773
  // Read once: the provenance read touches the filesystem, and the renderer
4303
4774
  // should never pay for it twice per status.
4304
4775
  const reason = pauseProvenance(p.name)?.reason;
4776
+ // The live review-revision rounds, read from the same durable rows the
4777
+ // restart recovery uses: a run whose revision is dispatched is read as
4778
+ // `review-revision N` while its worker is live (#692).
4779
+ const reviewRounds: Record<string, number> = {};
4780
+ for (const revision of store.unsettledReviewRevisions(p.name)) {
4781
+ if (revision.dispatchedAt !== undefined) reviewRounds[revision.runId] = revision.round;
4782
+ }
4305
4783
  return {
4306
4784
  project: p.name,
4307
4785
  configPath: configPath(),
@@ -4312,8 +4790,11 @@ export function statusSnapshotFromStore(
4312
4790
  digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
4313
4791
  caps,
4314
4792
  releaseGrants: resolveReleaseGrants(p),
4793
+ review: resolveReview(p),
4315
4794
  activeRuns: store.activeRuns(p.name),
4795
+ reviewRounds,
4316
4796
  salvagedRuns: store.salvagedRuns(p.name),
4797
+ quarantinedRuns: store.quarantinedRuns(p.name),
4317
4798
  turnOverrides: store.listTurnOverrides(p.name),
4318
4799
  openReports: store.openReports(p.name),
4319
4800
  digestBacklog: store.digestBacklog(p.name),
@@ -4387,32 +4868,6 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
4387
4868
  return lines.join("\n");
4388
4869
  }
4389
4870
 
4390
- /**
4391
- * The WIP block: every issue whose newest attempt left work behind, and
4392
- * whether that work is safe.
4393
- *
4394
- * Blocked runs used to be invisible here, which is exactly how #118 stayed
4395
- * invisible for a full attempt cycle — the operator saw a blocked issue and had
4396
- * no way to tell "stopped with 34 uncommitted files" from "stopped clean".
4397
- * A preserved line is informational; an UNSALVAGED line is an alarm, and it
4398
- * names the directory because that directory is the work.
4399
- */
4400
- export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
4401
- if (runs.length === 0) return [];
4402
- const lines = ["", "wip"];
4403
- for (const r of runs) {
4404
- lines.push(
4405
- r.salvageError !== undefined && r.salvageAckAt === undefined
4406
- ? ` #${r.issue} UNSALVAGED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
4407
- `only copy, dispatch held (${r.salvageError})`
4408
- : r.salvageError !== undefined
4409
- ? ` #${r.issue} accepted as lost attempt ${r.attempt} (${r.salvageError})`
4410
- : ` #${r.issue} preserved ${r.salvageSha ?? "?"} on ${r.branch} (attempt ${r.attempt}, ${r.state})`,
4411
- );
4412
- }
4413
- return lines;
4414
- }
4415
-
4416
4871
  /**
4417
4872
  * The grant table, named shape by shape.
4418
4873
  *
@@ -4430,6 +4885,7 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
4430
4885
  ...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
4431
4886
  ];
4432
4887
  }
4888
+
4433
4889
  export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
4434
4890
  return rows.map((row) => {
4435
4891
  const head = row.headSha.slice(0, 8);
@@ -4519,6 +4975,7 @@ export function formatStatus(s: StatusSnapshot): string {
4519
4975
  lines.push(...formatBaseHealth(s.baseHealth));
4520
4976
  lines.push(...formatFreezes(s.freezes));
4521
4977
  lines.push(...formatSalvagedRuns(s.salvagedRuns));
4978
+ lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
4522
4979
  lines.push(...formatOpenReports(s.openReports));
4523
4980
  lines.push(...formatVerbLedger(s.verbLedger));
4524
4981
  // Deploy hint: a restart while workers are live orphans them (salvage runs
@@ -4592,61 +5049,6 @@ export function prepareConductor(project?: string): void {
4592
5049
  setPaused(true, { source: "setup" }, project);
4593
5050
  }
4594
5051
 
4595
- /** Bounded per tick: each row costs tracker calls to gather facts for. */
4596
- const CLASSIFY_BATCH = 20;
4597
-
4598
- /** Tool calls quoted as evidence for a run that spun to its turn cap. */
4599
- const SPIN_EVIDENCE_CALLS = 10;
4600
-
4601
- /**
4602
- * The last few tool names a transcript recorded, newest last.
4603
- *
4604
- * `turn-cap-spinning` escalates rather than requeueing, and the acceptance
4605
- * criterion is that the escalation carries evidence of what the worker was doing
4606
- * when it hit the cap — otherwise the orchestrator opens the transcript and
4607
- * re-derives it, which is the manual triage this whole sweep removes.
4608
- */
4609
- export function lastToolCalls(sessionFile: string | undefined, limit = SPIN_EVIDENCE_CALLS): string[] {
4610
- if (sessionFile === undefined) return [];
4611
- let text: string;
4612
- try {
4613
- text = readFileSync(sessionFile, "utf8");
4614
- } catch {
4615
- return [];
4616
- }
4617
- const names: string[] = [];
4618
- for (const line of text.split("\n")) {
4619
- if (line.length === 0) continue;
4620
- let row: unknown;
4621
- try {
4622
- row = JSON.parse(line) as unknown;
4623
- } catch {
4624
- continue;
4625
- }
4626
- if (row === null || typeof row !== "object") continue;
4627
- const rec = row as { readonly [key: string]: unknown };
4628
- // Both shapes the harness has written: a top-level tool event, and a tool
4629
- // block inside an assistant message.
4630
- const direct = rec["toolName"];
4631
- if (typeof direct === "string") {
4632
- names.push(direct);
4633
- continue;
4634
- }
4635
- const message = rec["message"];
4636
- if (message === null || typeof message !== "object") continue;
4637
- const content = (message as { readonly [key: string]: unknown })["content"];
4638
- if (!Array.isArray(content)) continue;
4639
- for (const part of content) {
4640
- if (part === null || typeof part !== "object") continue;
4641
- const p = part as { readonly [key: string]: unknown };
4642
- if (p["type"] !== "tool_use") continue;
4643
- const name = p["name"];
4644
- if (typeof name === "string") names.push(name);
4645
- }
4646
- }
4647
- return names.slice(-limit);
4648
- }
4649
-
4650
5052
  /** A provider refusal a session recorded before dying, or undefined for none. */
4651
5053
  export interface SessionError {
4652
5054
  status?: number;
@@ -4663,608 +5065,154 @@ export function completionLastError(
4663
5065
  }
4664
5066
 
4665
5067
  /**
4666
- * The last error a transcript recorded, or undefined when it recorded none.
4667
- *
4668
- * The harness writes `{"stopReason":"error","errorStatus":402,"errorId":402,
4669
- * "errorMessage":"402 This request requires more credits, ..."}`. The daemon
4670
- * read none of it, so three runs died `unknown` with an empty `lastError` and
4671
- * charged an attempt each for a billing state (#220).
4672
- *
4673
- * Scanned newest-first: a session that recovered from an early error and then
4674
- * died of something else must report the something else, and a session that
4675
- * recovered from its only error and finished cleanly reports the error anyway
4676
- * because there is no terminal verdict to outrank it (#220).
5068
+ * How many settled `ci-deterministic` rows one reconciliation pass may
5069
+ * re-examine beyond the persisted review cursor. Each candidate costs GitHub
5070
+ * calls to re-fetch the evidence its check log carried, so a fleet with a
5071
+ * long misclassified history works through it over several daemon starts
5072
+ * rather than spending one boot's budget on all of it (#638).
4677
5073
  */
4678
- export function readSessionError(sessionFile: string | undefined): SessionError | undefined {
4679
- if (sessionFile === undefined) return undefined;
4680
- let text: string;
4681
- try {
4682
- text = readFileSync(sessionFile, "utf8");
4683
- } catch {
4684
- return undefined;
4685
- }
4686
- const lines = text.split("\n");
4687
- for (let i = lines.length - 1; i >= 0; i -= 1) {
4688
- const line = lines[i];
4689
- if (line === undefined || line.length === 0) continue;
4690
- let row: unknown;
4691
- try {
4692
- row = JSON.parse(line) as unknown;
4693
- } catch {
4694
- continue;
4695
- }
4696
- if (row === null || typeof row !== "object") continue;
4697
- const rec = row as { readonly [key: string]: unknown };
4698
- if (rec["stopReason"] !== "error") continue;
4699
- const message = rec["errorMessage"];
4700
- if (typeof message !== "string" || message.trim() === "") continue;
4701
- const status = rec["errorStatus"];
4702
- return {
4703
- ...(typeof status === "number" && Number.isFinite(status) ? { status } : {}),
4704
- message: message.trim(),
4705
- };
4706
- }
4707
- return undefined;
4708
- }
5074
+ const HISTORICAL_INFRA_BATCH = 20;
5075
+
5076
+ /** How many workflow runs for the head commit one row's evidence pass reads —
5077
+ * and the ceiling past which the row is refused as undecided rather than
5078
+ * decided from a prefix of its head-pinned runs (review #654). */
5079
+ const HISTORICAL_INFRA_RUNS = 3;
5080
+
5081
+ /** How many attempts of one workflow run may be read for the failed log — and
5082
+ * the ceiling past which the run is refused as undecided rather than read as
5083
+ * a prefix that could hide a later attempt's real failure (review #654). A
5084
+ * failed job rerun to green leaves the failure in an earlier attempt; the
5085
+ * bound keeps one pathological run from costing the whole pass. */
5086
+ const HISTORICAL_INFRA_ATTEMPTS = 5;
4709
5087
 
4710
5088
  /**
4711
- * Classify every unclassified terminal run, persist the verdict, and perform the
4712
- * one recovery its class names (#132).
5089
+ * Repair settled `ci-deterministic` rows whose re-fetched check log carries a
5090
+ * closed infrastructure signature (#638). The forward classifier now names
5091
+ * the codeload setup 429 of #177 `ci-infra`, but a row already classified
5092
+ * `ci-deterministic`/`escalate` never re-enters the classification sweep, so
5093
+ * the old verdict charges an implementation attempt forever. Re-fetching the
5094
+ * head-pinned workflow-run attempt logs through the tracker, matching the
5095
+ * classifier's own closed signature list, and reclassifying `failureClass`
5096
+ * alone returns the attempt without re-animating a months-old run into
5097
+ * recovery.
4713
5098
  *
4714
- * Half this fleet's spend produced no merged PR, and every one of those runs
4715
- * ended at a human who re-derived the same triage by hand and then threw the
4716
- * conclusion away. The mechanical classes a cancelled runner, a kill from a
4717
- * daemon restart, a green PR whose base moved, a row whose PR had already merged
4718
- * need no judgement at all; the genuinely human ones are worth a person's
4719
- * attention only if they arrive with their evidence already gathered.
5099
+ * Bounded, idempotent and resumable: one batch per call, reclassifications
5100
+ * only, and the store's update is guarded by the row still reading
5101
+ * `ci-deterministic`, so a second pass touches nothing it already repaired.
5102
+ * The batch resumes below a persisted review cursor, so each row is evaluated
5103
+ * once rather than rescanning the newest non-matches forever and starving
5104
+ * older repairable rows. A row whose evidence could not be read is a
5105
+ * no-mutation refusal: the cursor never advances past it, so the next pass
5106
+ * asks again; a row that was read and shown *not* to be infrastructure
5107
+ * advances the cursor. Every repair requires *all* gathered failed logs to
5108
+ * carry an infra signature — a setup 429 in one attempt must not waive a
5109
+ * compile/test failure in a sibling attempt.
4720
5110
  *
4721
- * Facts are fetched per row and only the ones that row needs: a `killed` row
4722
- * costs nothing, a `failed` row with a PR costs a state read and a check read.
4723
- * A `pushed-green` row that classifies to nothing is left completely untouched —
4724
- * it is healthy, and writing a class onto it would take it out of this sweep for
4725
- * good.
5111
+ * Returns how many rows it repaired, for the boot log.
4726
5112
  */
4727
- export async function classifyAndRecover(d: Deps): Promise<number> {
4728
- const { project, caps, tracker, store } = d;
4729
- // Settle recoveries, counted for the pass's dispatch record — a row whose PR
4730
- // merged is settled here when the settle sweep could not establish identity
4731
- // (#497). Every other exit returns 0.
4732
- let settled = 0;
4733
- for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
4734
- const facts: ClassifyFacts = {};
4735
- let classifiedRun = run;
4736
- if (run.state === "failed" || run.state === "killed") {
4737
- const sessionError = readSessionError(run.sessionFile);
4738
- if (sessionError !== undefined) {
4739
- if (run.lastError === undefined || run.lastError === sessionError.message) {
4740
- facts.sessionError = sessionError;
4741
- }
4742
- if (run.lastError === undefined) {
4743
- store.updateRun(run.id, { lastError: sessionError.message });
4744
- classifiedRun = { ...run, lastError: sessionError.message };
4745
- }
4746
- }
4747
- }
4748
- try {
4749
- if (run.prUrl !== undefined) {
4750
- const pr = await tracker.prState(run.prUrl);
4751
- if (pr !== undefined) facts.pr = pr;
4752
- if (run.state === "pushed-green" && facts.pr === "open") {
4753
- facts.mergeable = await tracker.mergeable(run.prUrl);
4754
- }
4755
- if (run.state === "failed" && facts.pr === "open") {
4756
- facts.checks = await tracker.checkConclusions(run.prUrl);
4757
- // When a check failed with a reachable log, pull its tail so the
4758
- // table can tell an infra outage (#177) from a real test failure by
4759
- // the log's own words. First failure wins; a log that cannot be
4760
- // fetched is left undefined and classification stays conservative.
4761
- const firstFailure = facts.checks.find((c) => c.state === "failure" && c.link !== undefined);
4762
- if (firstFailure?.link !== undefined) {
4763
- facts.failingLog = await tracker.checkLog(firstFailure.link);
4764
- }
4765
- }
4766
- }
4767
- } catch (err) {
4768
- // Per row, like every other sweep here: one unreachable PR must not stop
4769
- // the rest from being classified. The next tick asks again for free.
4770
- log(`#${run.issue} not classified: fact gathering failed (${errText(err)}) — retrying next tick`);
4771
- continue;
4772
- }
4773
-
4774
- const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
4775
-
4776
- // A healthy green PR is not a failure of any class. Leaving the row
4777
- // unclassified is what keeps it eligible for the sweep on the tick where its
4778
- // base does move under it.
4779
- if (run.state === "pushed-green" && cls === "unknown") continue;
4780
-
4781
- const retry = run.failureClass !== undefined;
4782
- store.updateRun(run.id, { failureClass: cls, recoveryAction: recovery });
4783
- log(
4784
- retry
4785
- ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
4786
- : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
4787
- );
4788
- if (recovery === "settle") settled += 1;
4789
- await recoverRun(d, classifiedRun, cls, recovery, evidence);
4790
- }
4791
- return settled;
4792
- }
4793
-
4794
- /** Performs the one action a class names. Never chooses one of its own. */
4795
- async function recoverRun(
4796
- d: Deps,
4797
- run: RunRecord,
4798
- cls: FailureClass,
4799
- recovery: RecoveryAction,
4800
- evidence: string,
4801
- ): Promise<void> {
4802
- const { project, caps, tracker, store } = d;
4803
- const inProgress = project.stateLabels.inProgress;
4804
-
4805
- if (recovery === "settle") {
4806
- // Enqueue the release with the terminal write (see `settlePushedGreen`):
4807
- // the outbox keeps the label and the row one fact, so a tracker refusal
4808
- // can no longer strand `agent:in-progress` with nothing left to retry it
4809
- // (#18, #201).
4810
- releaseInProgress(d, run.issue, `PR merged: ${evidence}`);
4811
- store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
4812
- log(`#${run.issue} settled from ${cls}: ${evidence}`);
4813
- return;
4814
- }
4815
-
4816
- if (recovery === "continue") {
4817
- // Two classes recover by continuing, and only one of them has anything left
4818
- // to do here.
4819
- //
4820
- // `turn-cap-progress` was already handed back by the completion path, which
4821
- // swapped its labels and left the branch retained. There is nothing to
4822
- // perform, and writing anything would be actively wrong: overwriting
4823
- // `lastError` with a rebase brief tells the continuation worker to rebase a
4824
- // run that simply ran out of turns, and re-swapping labels the completion
4825
- // path already swapped is a pair of no-op `gh` calls. Record-only, so the
4826
- // sweep stops re-offering it.
4827
- if (cls === "turn-cap-progress") {
4828
- store.updateRun(run.id, { recoveredAt: Date.now() });
4829
- log(`#${run.issue} already continuing from ${cls}: ${evidence}`);
4830
- return;
4831
- }
4832
-
4833
- // `wall-clock-cap-progress`: the completion path only auto-continues
4834
- // turn-cap kills, so a wall-clock kill with work to show reaches this
4835
- // sweep still holding the failed label. Swap it for the queue — the branch
4836
- // is retained, so the next dispatch reattaches it and briefs a resume from
4837
- // the recorded work. Killed rows gathered no tracker facts, so the
4838
- // issue-open guard mirrors the requeue path's. The continuation gate is
4839
- // the same one the turns path applies at kill time
4840
- // (`shouldContinueAfterTurnsCap`): the row is already charged, and once
4841
- // the ceiling is spent, handing back the queue label would offer a
4842
- // candidate admission can never accept — only the failed label comes off,
4843
- // and the exhaustion reaches a human (#490, #348).
4844
- if (cls === "wall-clock-cap-progress") {
4845
- const state = await tracker.issueState(run.issue).catch(() => undefined);
4846
- if (state !== "open") {
4847
- log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
4848
- return;
4849
- }
4850
- const continuation = store.continuationsFor(project.name, run.issue);
4851
- if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
4852
- swapToQueue(d, run.issue, project.stateLabels.failed);
4853
- store.updateRun(run.id, { recoveredAt: Date.now() });
4854
- log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
4855
- } else {
4856
- store.enqueueLabelOps(project.name, [
4857
- { issue: run.issue, op: "remove", label: project.stateLabels.failed },
4858
- ]);
4859
- store.updateRun(run.id, { recoveredAt: Date.now() });
4860
- await safeEscalate(d, {
4861
- tier: 1,
4862
- project: project.name,
4863
- issue: run.issue,
4864
- summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
4865
- detail: [
4866
- `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
4867
- evidence,
4868
- `Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
4869
- "The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
4870
- ].join("\n"),
4871
- });
4872
- await postExhaustionPostmortem(
4873
- d,
4874
- run,
4875
- `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
4876
- );
4877
- log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
4878
- }
4879
- return;
4880
- }
4881
-
4882
- // `merge-conflict`: the branch is retained and its PR is open, so #50's
4883
- // continuation guard admits it and the next tick briefs a rebase.
4884
- //
4885
- // The outbox makes the retry contract one-sided: the swap is enqueued — a
4886
- // durable local write that cannot fail on the tracker — before
4887
- // `recoveredAt` is written, so the row can never again be taken out of
4888
- // `runsNeedingClassification` with its label swap still owed. That was
4889
- // the defect 0.4.4 claimed to have fixed and did not, for this one
4890
- // recovery; the projector retries until the tracker takes the swap, and
4891
- // while it is pending the eligibility overlay keeps the issue coherent
4892
- // (#201).
4893
- swapToQueue(d, run.issue, inProgress);
4894
- store.updateRun(run.id, {
4895
- state: "killed",
4896
- lastError:
4897
- "merge-conflict: base moved under a green PR — rebase, regenerate recorded artifacts, push without force",
4898
- recoveredAt: Date.now(),
4899
- });
4900
- log(`#${run.issue} requeued for a rebase continuation: ${evidence}`);
4901
- return;
4902
- }
4903
-
4904
- if (recovery === "requeue") {
4905
- if (cls === "provider-credit") {
4906
- await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
4907
- }
4908
- // A dispatch-infra requeue that keeps landing on the same issue means the
4909
- // mirror for its repo is persistently broken — a ref-lock that retry already
4910
- // exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
4911
- // with no chance of success, so after a bounded number of strikes this
4912
- // escalates to a human instead (#168, #177).
4913
- if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
4914
- await safeEscalate(d, {
4915
- tier: 1,
4916
- project: project.name,
4917
- issue: run.issue,
4918
- summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
4919
- detail: [
4920
- `The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
4921
- "The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
4922
- ].join("\n"),
4923
- });
4924
- store.updateRun(run.id, { recoveredAt: Date.now() });
4925
- log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
4926
- return;
4927
- }
4928
- // Same bound for provider-transient: an issue whose stream keeps stalling
4929
- // mid-run is requeued free (no attempt, no continuation charged) — but a
4930
- // provider that aborts three times for one issue is down, and a human has
4931
- // to check its status before hand-requeueing (#220). The escalation names
4932
- // every model the chain tried, so a merged branch built on a different
4933
- // model is attributable (#286).
4934
- if (
4935
- cls === "provider-transient" &&
4936
- store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
4937
- ) {
4938
- const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
4939
- await safeEscalate(d, {
4940
- tier: 1,
4941
- project: project.name,
4942
- issue: run.issue,
4943
- summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
4944
- detail: [
4945
- `The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
4946
- ...(tried === ""
4947
- ? []
4948
- : [`Models tried: ${tried}.`]),
4949
- "Check provider status before requeueing by hand.",
4950
- ].join("\n"),
4951
- });
4952
- store.updateRun(run.id, { recoveredAt: Date.now() });
4953
- log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
4954
- return;
4955
- }
4956
- // Same bound for provider-capacity: a run the provider throttled into the
4957
- // ground is requeued free (no attempt charged) — but a provider that
4958
- // throttles the same issue three times is at capacity, and a human has to
4959
- // check its status before hand-requeueing (#573). On a chain-configured
4960
- // project each requeue already moved the next attempt to the next chain
4961
- // model, so this escalation is what catches the no-chain case and the
4962
- // exhausted chain; it names every model the chain tried.
4963
- if (
4964
- cls === "provider-capacity" &&
4965
- store.classCountFor(project.name, run.issue, "provider-capacity") >= PROVIDER_CAPACITY_MAX_STRIKES
4966
- ) {
4967
- const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
4968
- await safeEscalate(d, {
4969
- tier: 1,
4970
- project: project.name,
4971
- issue: run.issue,
4972
- summary: `[provider-capacity] #${run.issue}: the model provider is throttling this run into the ground — ${evidence}`,
4973
- detail: [
4974
- `The provider answered #${run.issue} with sustained in-session rate limits ${PROVIDER_CAPACITY_MAX_STRIKES} times in a row; the harness retried each and was exhausted.`,
4975
- ...(tried === ""
4976
- ? []
4977
- : [`Models tried: ${tried}.`]),
4978
- "Check the provider's rate-limit status (and its throughput-oriented routes) before requeueing by hand.",
4979
- ].join("\n"),
4980
- });
4981
- store.updateRun(run.id, { recoveredAt: Date.now() });
4982
- log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
4983
- return;
4984
- }
4985
- // Only when the tracker still shows this issue as ours to hand back. An
4986
- // issue that is closed, or has no state label, was resolved by another route
4987
- // and requeueing it would dispatch work nobody asked for.
4988
- const state = await tracker.issueState(run.issue).catch(() => undefined);
4989
- if (state !== "open") {
4990
- log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
4991
- return;
4992
- }
4993
- // A clean orphan whose queue label is already absent is a deliberate
4994
- // withdrawal — the operator took the issue out of the queue (e.g. so a
4995
- // daemon restart could not re-dispatch work known to be unsafe) — and
4996
- // recovery must not recreate that intent (#423). Read the live label set;
4997
- // when the queue label is gone, release the dispatcher-owned in-progress
4998
- // label and stop, leaving the operator's withdrawal to survive recovery.
4999
- if (cls === "orphan-clean") {
5000
- const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
5001
- if (snapshot === undefined) {
5002
- log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
5003
- return;
5004
- }
5005
- if (!snapshot.labels.includes(project.queueLabel)) {
5006
- store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
5007
- store.updateRun(run.id, { recoveredAt: Date.now() });
5008
- log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
5009
- return;
5010
- }
5011
- // #439: an orphan-clean row charges the continuation budget, so once
5012
- // `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
5013
- // holds the issue on — requeueing re-adds a queue label for a candidate the
5014
- // dispatcher can never admit. Stop handing it back and hold it instead: the
5015
- // queue label comes off (so admission never re-holds on every dispatch),
5016
- // the in-progress label is released, and a single diagnosis escalates once.
5017
- // `orphan-clean` is deliberately NOT a global exclusion from the budget (a
5018
- // worker that genuinely keeps dying mid-work must still be bounded); this is
5019
- // the missing ceiling check this path never had (#348's invariant).
5020
- const continuations = store.continuationsFor(project.name, run.issue);
5021
- if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
5022
- const runs = store.runsForIssue(project.name, run.issue);
5023
- const breakdown = continuationBreakdown(runs);
5024
- const artifact = newestContinuableRun(runs);
5025
- const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
5026
- const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
5027
- const artifactLine =
5028
- artifact === undefined
5029
- ? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
5030
- : `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
5031
- store.enqueueLabelOps(project.name, [
5032
- { issue: run.issue, op: "remove", label: inProgress },
5033
- { issue: run.issue, op: "remove", label: project.queueLabel },
5034
- ]);
5035
- store.updateRun(run.id, { recoveredAt: Date.now() });
5036
- await safeEscalate(d, {
5037
- tier: 1,
5038
- project: project.name,
5039
- issue: run.issue,
5040
- summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
5041
- detail: [
5042
- `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5043
- `Continuations by failure class: ${classLine}.`,
5044
- onlyDaemonStops
5045
- ? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
5046
- : "Continuations span real work — inspect what each attempt left behind before continuing.",
5047
- artifactLine,
5048
- "What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
5049
- ].join("\n"),
5050
- });
5051
- await postExhaustionPostmortem(
5052
- d,
5053
- run,
5054
- `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5055
- );
5056
- log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
5057
- return;
5058
- }
5059
- }
5060
- const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
5061
- swapToQueue(d, run.issue, label);
5062
- store.updateRun(run.id, { recoveredAt: Date.now() });
5063
- log(`#${run.issue} requeued from ${cls}: ${evidence}`);
5064
- return;
5065
- }
5066
-
5067
- if (recovery === "rerun-checks") {
5068
- if (run.prUrl === undefined) return;
5069
- try {
5070
- await tracker.rerunFailedChecks(run.prUrl);
5071
- } catch (err) {
5072
- log(`#${run.issue} check re-run failed (${errText(err)}) — retrying next tick`);
5073
- return;
5074
- }
5075
- // Back to pending rather than green: the existing settle sweep re-verifies
5076
- // it against the recorded head on a later tick, so nothing here has to guess
5077
- // whether the re-run passed.
5078
- store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
5079
- log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
5080
- return;
5081
- }
5082
-
5083
- if (recovery === "escalate") {
5084
- const detail = [evidence];
5085
- if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
5086
- const calls = lastToolCalls(run.sessionFile);
5087
- detail.push(
5088
- calls.length === 0
5089
- ? "transcript unreadable — no tool calls could be recovered"
5090
- : `Last ${calls.length} tool calls: ${calls.join(" → ")}`,
5113
+ export async function reconcileHistoricalInfra(d: Deps): Promise<number> {
5114
+ const { project, store } = d;
5115
+ const version = infraSignatureVersion();
5116
+ const persisted = store.historicalInfraCursor(project.name);
5117
+ // A cursor stamped by an older signature list is stale: the classifier now
5118
+ // recognises more evidence, so the pass restarts from the newest row rather
5119
+ // than skipping past newly repairable history (#638).
5120
+ const cursor =
5121
+ persisted !== undefined && persisted.classifierVersion === version
5122
+ ? { startedAt: persisted.startedAt, rowid: persisted.rowid }
5123
+ : undefined;
5124
+ const candidates = store.historicalInfraCandidates(project.name, HISTORICAL_INFRA_BATCH, cursor);
5125
+ let repaired = 0;
5126
+ let lastDecided: { startedAt: number; rowid: number } | undefined;
5127
+ for (const run of candidates) {
5128
+ const chunks = await historicalInfraEvidence(d, run);
5129
+ // Unreachable evidence is undecided exactly like a fresh classifier is: a
5130
+ // no-mutation refusal the next pass asks again, and the cursor stops here
5131
+ // so the row is re-offered rather than being skipped past.
5132
+ if (chunks === undefined) break;
5133
+ lastDecided = { startedAt: run.startedAt, rowid: run.rowid };
5134
+ // The head's runs were read and none holds a failing log — determinately
5135
+ // not infrastructure, so an old misclassified verdict stays charged.
5136
+ if (chunks.length === 0) continue;
5137
+ // Mixed guard: anything gathered that is not itself a closed infra
5138
+ // signature (a compile/test failure in a sibling attempt, a product 429)
5139
+ // refuses the whole row. One setup 429 is not permission to waive a real
5140
+ // implementation failure.
5141
+ if (!chunks.every((chunk) => infraLogSignature(chunk) !== undefined)) continue;
5142
+ if (store.reclassifyInfra(run.id)) {
5143
+ repaired += 1;
5144
+ // The every-guard above guarantees a signature and a first chunk; the
5145
+ // non-null assertions make the same fact readable to the type checker.
5146
+ log(
5147
+ `#${run.issue} repaired historical ${run.failureClass ?? "ci-deterministic"} ci-infra (run ${run.id}): ` +
5148
+ `"${infraLogSignature(chunks[0]!)}" in the failed check log`,
5091
5149
  );
5092
5150
  }
5093
- if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
5094
- // #172: an unwritten transcript is "the run died before it flushed", not a
5095
- // link to a file the operator will open and find missing.
5096
- detail.push(
5097
- run.sessionFile === undefined
5098
- ? "Session: (no transcript)"
5099
- : existsSync(run.sessionFile)
5100
- ? `Session: ${run.sessionFile}`
5101
- : `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
5151
+ }
5152
+ if (lastDecided !== undefined) {
5153
+ store.setHistoricalInfraCursor(
5154
+ project.name,
5155
+ lastDecided.startedAt,
5156
+ lastDecided.rowid,
5157
+ version,
5102
5158
  );
5103
- // The class and the run are in the summary, which is what the notifications
5104
- // ledger dedupes on — so one class escalates once per run rather than every
5105
- // five minutes.
5106
- await safeEscalate(d, {
5107
- tier: 1,
5108
- project: project.name,
5109
- issue: run.issue,
5110
- runId: run.id,
5111
- summary: `[${cls}] #${run.issue} attempt ${run.attempt}: ${evidence}`,
5112
- detail: detail.join("\n"),
5113
- });
5114
- // The hand-off IS the recovery for these classes: there is nothing else this
5115
- // package can do, and leaving the row unrecovered would re-escalate forever.
5116
- store.updateRun(run.id, { recoveredAt: Date.now() });
5117
- return;
5118
5159
  }
5119
-
5120
- // `hold` (orphan-dirty) and `none`: recorded, nothing performed. The existing
5121
- // unsalvaged-WIP admission hold already fails dispatch closed until an
5122
- // operator acknowledges the tree, which is the only safe move when the
5123
- // worktree holds the only copy of real work.
5124
- }
5125
-
5126
- /**
5127
- * Enqueue a state-label → queue-label swap for projection (#201).
5128
- *
5129
- * The swap is two ops in id order — remove first, then add — which is the
5130
- * atomicity the projector guarantees: the issue never sits newly eligible
5131
- * without a queue label on its way back, and the add never lands before the
5132
- * remove when GitHub fails between them. Enqueueing is a durable local write
5133
- * that cannot fail on the tracker, so the caller records its recovery
5134
- * immediately and the projector retries the swap until the tracker takes it —
5135
- * that closes the 0.4.4 hole where a refused label swap stranded the row
5136
- * permanently under a log line promising a retry.
5137
- */
5138
- function swapToQueue(d: Pick<Deps, "project" | "store">, issue: number, label: string): void {
5139
- d.store.enqueueLabelOps(d.project.name, [
5140
- { issue, op: "remove", label },
5141
- { issue, op: "add", label: d.project.queueLabel },
5142
- ]);
5160
+ return repaired;
5143
5161
  }
5144
5162
 
5145
- /** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
5146
- const RECONCILE_LIMIT = 50;
5147
-
5148
5163
  /**
5149
- * Clear state labels from issues that no longer need them (#132's `superseded`).
5150
- *
5151
- * Two structural signals, both cheap and both observed on this fleet: an issue
5152
- * that is closed but still carries `agent:*`, and an open issue carrying
5153
- * `failed` whose sub-issues have all closed. On 2026-08-09 four issues (#307,
5154
- * #297, #140, #82) carried `agent:failed` while every one of them was already
5155
- * complete the label was residue of a turns-cap kill from two days earlier,
5156
- * and nothing in the loop ever revisited it. The board counted four phantom
5157
- * failures while the genuinely stuck issues were invisible.
5164
+ * The failed check logs of a settled `ci-deterministic` row, head-pinned to
5165
+ * the exact commit the row ran against, or `undefined` when the evidence is
5166
+ * unreachable (#638). Only the workflow-run history at `run.headSha` is
5167
+ * evidence: the PR's *current* check rollup is not, because a later push's
5168
+ * checks must never classify an earlier run's row. `workflowRunsAt` is itself
5169
+ * head-scoped, and each run's failed attempts are read through the tracker's
5170
+ * guarded runner (call/refusal accounting and the rate-limit breaker apply).
5158
5171
  *
5159
- * Positive evidence only. A tracker that cannot list answers empty, and an empty
5160
- * answer removes nothing: a reconcile that guessed would strip the interlock
5161
- * that keeps two workers off one issue.
5172
+ * Returns the per-failed-job failed logs gathered across the head's runs (one
5173
+ * chunk per failed job, full and untruncated), or `undefined` when any read
5174
+ * could not be made the run register, an attempt's job register, a failed
5175
+ * job's log, or the head-run list itself — or when a bounded evidence set
5176
+ * exceeded its limit (more head-pinned runs than `HISTORICAL_INFRA_RUNS`, or
5177
+ * more attempts than `HISTORICAL_INFRA_ATTEMPTS`). Undefined is a no-mutation
5178
+ * refusal the next pass asks again; the cursor never advances past it. `[]`
5179
+ * means the head's runs were read and none holds a failing log — a
5180
+ * determinately non-infrastructure answer.
5162
5181
  */
5163
- export async function reconcileStaleLabels(d: Deps): Promise<void> {
5164
- const { project, tracker, store } = d;
5165
- const labels = [project.stateLabels.failed, project.stateLabels.blocked, project.stateLabels.inProgress];
5166
-
5167
- for (const label of labels) {
5168
- const carrying = await tracker.listLabeled(label, RECONCILE_LIMIT).catch(() => []);
5169
- for (const issue of carrying) {
5170
- if (issue.state === "closed") {
5171
- // Never retain an `agent:*` label on a closed issue: the work is done by
5172
- // some route, and the label only makes the board lie about it. Enqueue
5173
- // rather than call — a refused write must not lose the decision; the
5174
- // projector retries the removal until the tracker takes it (#201).
5175
- store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
5176
- log(`#${issue.number} reconciled: closed issue no longer carries ${label} (queued)`);
5177
- continue;
5178
- }
5179
-
5180
- if (label !== project.stateLabels.failed) continue;
5181
- const children = await tracker.childrenOf(issue.number).catch(() => []);
5182
- if (children.length === 0 || children.some((c) => c.state !== "closed")) continue;
5183
-
5184
- const key = `${project.name}:superseded:${issue.number}`;
5185
- if (store.wasNotified(key)) continue;
5186
- const list = children.map((c) => `#${c.number}`).join(", ");
5187
- store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
5188
- try {
5189
- await tracker.comment(
5190
- issue.number,
5191
- `superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
5192
- `acceptance criteria are met on the default branch.`,
5193
- );
5194
- store.markNotified(key);
5195
- log(`#${issue.number} reconciled: superseded by ${list}`);
5196
- } catch (err) {
5197
- // The label removal is already queued and will land regardless; the
5198
- // comment is the only half that can fail here (#201).
5199
- log(`#${issue.number} could not comment the superseded note (${errText(err)}) — retrying next tick`);
5200
- }
5201
- }
5182
+ async function historicalInfraEvidence(d: Deps, run: RunRecord): Promise<string[] | undefined> {
5183
+ // Without a head SHA there is no safe way to pin evidence to this row; a
5184
+ // head-less row is determinately not a repair candidate, so it advances the
5185
+ // cursor rather than stalling the pass.
5186
+ if (run.headSha === undefined) return [];
5187
+ const target = d.project.routing.repos[run.repo];
5188
+ const repoIdentity = target === undefined ? undefined : githubRepo(target.cloneUrl);
5189
+ if (repoIdentity === undefined) return [];
5190
+ let runs: WorkflowRun[] | undefined;
5191
+ try {
5192
+ runs = await d.tracker.workflowRunsAt(repoIdentity, run.headSha);
5193
+ } catch {
5194
+ return undefined;
5202
5195
  }
5203
- }
5204
-
5205
- /**
5206
- * Settles `claimed`/`running` rows left by a dead daemon process and, before
5207
- * marking each one `orphaned`, salvages any dirty worktree.
5208
- *
5209
- * Found live after a host restart killed two workers mid-run, and again on
5210
- * every package deploy that restarted while workers were live (#35): without
5211
- * the salvage call the next attempt's `worktree remove --force` destroyed
5212
- * uncommitted edits that had no other copy. Cap-kills already salvaged (#27);
5213
- * this is the same call site for the restart path.
5214
- *
5215
- * Only the rows change. The issue keeps its in-progress label — that label is
5216
- * the crash guard against double-dispatch, and deciding what a dead worker's
5217
- * remains are worth (an open PR? a salvaged sha? a clean tree?) is the
5218
- * orchestrator's drain-duty judgement, not something to automate here. The
5219
- * rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
5220
- * still escalates instead of retrying forever.
5221
- *
5222
- * `pushed-green` rows are deliberately left alone: they hold no process — they
5223
- * are finished work waiting on a human merge, and they must keep occupying the
5224
- * issue so a second attempt cannot land on a live PR. What eventually settles
5225
- * them is {@link settlePushedGreen}, on the tick, by asking the tracker what
5226
- * became of the PR — the one question a restart cannot answer by inference.
5227
- */
5228
- export async function reconcileOrphanedRuns(
5229
- store: Store,
5230
- project: string,
5231
- /**
5232
- * Resolves the privileged publisher for one orphaned run. Optional because a
5233
- * test driving the row transitions has no repo to publish to; production
5234
- * always passes it, and without it a salvaged WIP commit stays local — which
5235
- * is the half of #121 that reaches a human.
5236
- */
5237
- publish?: (run: RunRecord) => RunPublisher,
5238
- ): Promise<RunRecord[]> {
5239
- // Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
5240
- // a process dying — it is finished work waiting on a human merge.
5241
- const stale = store.liveRuns(project);
5242
- const endedAt = Date.now();
5243
- for (const r of stale) {
5244
- // Salvage before the row flips: the worktree path is on the record, and
5245
- // salvageWip is a no-op for a missing/clean tree. The clause matches the
5246
- // cap-kill wording so triage reads the same either way, and the tree is
5247
- // kept because an orphan's remains are the orchestrator's drain-duty call.
5248
- const settlement =
5249
- r.worktree === ""
5250
- ? undefined
5251
- : await settleWorktree({
5252
- issue: r.issue,
5253
- attempt: r.attempt,
5254
- ending: "killed by a daemon restart",
5255
- worktree: r.worktree,
5256
- branch: r.branch,
5257
- // An orphan's tree is kept, so its commits are not about to be
5258
- // deleted — but a WIP salvage still has to reach GitHub, which is
5259
- // #121's whole point and is now the daemon's hop to make.
5260
- publish: publish?.(r),
5261
- tree: "keep",
5262
- });
5263
- store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
5196
+ if (runs === undefined) return undefined;
5197
+ // Refuse rather than decide from the first `HISTORICAL_INFRA_RUNS`: three
5198
+ // setup-429 runs must not waive a row whose fourth head-pinned run carries
5199
+ // the real compile/test failure (review #654). Undecided means the cursor
5200
+ // never advances past this row, so the next pass asks again.
5201
+ if (runs.length > HISTORICAL_INFRA_RUNS) return undefined;
5202
+ const chunks: string[] = [];
5203
+ for (const wf of runs) {
5204
+ const logs = await d.tracker.runFailedAttemptLogs(
5205
+ repoIdentity,
5206
+ wf.url,
5207
+ HISTORICAL_INFRA_ATTEMPTS,
5208
+ );
5209
+ if (logs === undefined) return undefined;
5210
+ chunks.push(...logs);
5264
5211
  }
5265
- return stale;
5212
+ return chunks;
5266
5213
  }
5267
5214
 
5215
+
5268
5216
  // ------------------------------------------------------------------- the daemon
5269
5217
 
5270
5218
  interface ProjectRuntime {
@@ -5450,12 +5398,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5450
5398
  log(projects.length === 1 ? message : `[${project.name}] ${message}`);
5451
5399
  };
5452
5400
  const caps = resolveCaps(project, cfg.defaults);
5401
+ // One transient-server-error breaker per project (#642): admission's
5402
+ // GraphQL checks and the orchestrator mutation commands share it, so a 503
5403
+ // observed by either side gates both instead of one provider outage being
5404
+ // re-asked by every candidate AND every mutation.
5405
+ const graphqlBreaker = new GraphqlBreaker();
5453
5406
  const tracker = makeTracker(project, undefined, {
5454
5407
  onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
5455
5408
  onRefusal: (at) => store.recordGhRefusal?.(at),
5456
5409
  onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
5457
- });
5458
- const verbActions = githubVerbActions(project);
5410
+ }, { graphqlBreaker });
5411
+ const verbActions = githubVerbActions(project, undefined, undefined, graphqlBreaker);
5459
5412
 
5460
5413
  if (alive === undefined || alive.pid === process.pid) {
5461
5414
  const orphanPublisher = (run: RunRecord): RunPublisher => {
@@ -5600,10 +5553,38 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5600
5553
  onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
5601
5554
  onRefusal: (at) => store.recordGhRefusal?.(at),
5602
5555
  }).issueSnapshot(issue),
5556
+ // The body twin of `probeIssueIn`, for the dependency-graph cycle pass
5557
+ // (#421): reads a reachable routed prerequisite's body through the same
5558
+ // per-repo tracker/credential/accounting seams.
5559
+ probeBodyIn: (ownerRepo, issue) =>
5560
+ makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, {
5561
+ onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
5562
+ onRefusal: (at) => store.recordGhRefusal?.(at),
5563
+ }).issueBody(issue),
5603
5564
  ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
5604
5565
  verbActions,
5605
5566
  };
5606
5567
  runtimeDeps = d;
5568
+ // Review-revision restart recovery (#692): a revision the previous daemon
5569
+ // claimed and lost — its run is `running`/`orphaned` by the orphan sweep
5570
+ // above — is restored to `pushed-green` and re-queued so this process's
5571
+ // first dispatch pass resumes the exact session. Runs under the same
5572
+ // dead-daemon guard as `reconcileOrphanedRuns`, and after it, so the
5573
+ // revision's worktree has already been salvaged to the branch before it is
5574
+ // cleared for a fresh reattach. A round that cannot be restored was
5575
+ // settled and escalated by the reconcile itself.
5576
+ if (alive === undefined || alive.pid === process.pid) {
5577
+ try {
5578
+ for (const revision of await reconcileCrashedReviewRevisions(d)) {
5579
+ projectLog(
5580
+ `#${revision.issue} review round ${revision.round} recovered across restart — ` +
5581
+ `run ${revision.runId} is pushed-green again and awaits its next dispatch pass`,
5582
+ );
5583
+ }
5584
+ } catch (err) {
5585
+ projectLog(`review revision restart recovery failed: ${errText(err)}`);
5586
+ }
5587
+ }
5607
5588
  // Startup reconciliation: close an incident carried over from a previous
5608
5589
  // process when the orchestrator is up (one recovery notice), or open one
5609
5590
  // when it failed to start (one down page). A daemon restarted while still
@@ -5625,6 +5606,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5625
5606
  });
5626
5607
  }
5627
5608
 
5609
+ // Historical budget repair (#638): one bounded pass per daemon process,
5610
+ // before the first tick, so admission and `unblock` read the repaired counts
5611
+ // from the moment the daemon starts. A classifier fix changes future
5612
+ // verdicts but not rows already settled under the old one; re-fetching their
5613
+ // failed check logs and reclassifying the exact closed infrastructure
5614
+ // signature returns those attempts without re-animating the runs. Best-effort
5615
+ // per project — one unreachable project must not stop the rest from booting.
5616
+ for (const runtime of runtimes) {
5617
+ try {
5618
+ await reconcileHistoricalInfra(runtime.d);
5619
+ } catch (err) {
5620
+ log(`historical infra reconciliation failed: ${errText(err)}`);
5621
+ }
5622
+ }
5623
+
5628
5624
  if (o.once) {
5629
5625
  try {
5630
5626
  for (const runtime of runtimes) await tick(runtime.d, workers);