omp-conductor 0.15.10 → 0.15.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +273 -2544
  2. package/REFERENCE.md +2680 -0
  3. package/package.json +3 -2
  4. package/schema/config.schema.json +11 -23
  5. package/src/arm-challenge.ts +112 -0
  6. package/src/ask.ts +434 -0
  7. package/src/board.ts +81 -15
  8. package/src/brief-upgrade.ts +114 -8
  9. package/src/briefs/orchestrator.md +113 -34
  10. package/src/briefs/policy.md +33 -8
  11. package/src/briefs/worker.md +18 -9
  12. package/src/chain-check.ts +1 -1
  13. package/src/check-trailing-newlines.ts +82 -0
  14. package/src/cli.ts +225 -1406
  15. package/src/commands/arm.ts +21 -0
  16. package/src/commands/board.ts +23 -0
  17. package/src/commands/brief-upgrade.ts +186 -0
  18. package/src/commands/context.ts +150 -0
  19. package/src/commands/daemon.ts +71 -0
  20. package/src/commands/dashboard.ts +74 -0
  21. package/src/commands/decision.ts +103 -0
  22. package/src/commands/disarm.ts +21 -0
  23. package/src/commands/doctor.ts +100 -0
  24. package/src/commands/event.ts +62 -0
  25. package/src/commands/extend.ts +64 -0
  26. package/src/commands/friction.ts +56 -0
  27. package/src/commands/help.ts +9 -0
  28. package/src/commands/hold.ts +26 -0
  29. package/src/commands/intake.ts +155 -0
  30. package/src/commands/ledger.ts +69 -0
  31. package/src/commands/message.ts +96 -0
  32. package/src/commands/report.ts +206 -0
  33. package/src/commands/restart.ts +76 -0
  34. package/src/commands/resume.ts +58 -0
  35. package/src/commands/setup.ts +143 -0
  36. package/src/commands/start.ts +23 -0
  37. package/src/commands/stats.ts +131 -0
  38. package/src/commands/status.ts +48 -0
  39. package/src/commands/stop.ts +51 -0
  40. package/src/commands/tail.ts +109 -0
  41. package/src/commands/unblock.ts +39 -0
  42. package/src/commands/upgrade-install.ts +31 -0
  43. package/src/commands/upgrade-rollback.ts +32 -0
  44. package/src/commands/upgrade.ts +25 -0
  45. package/src/commands/verb.ts +83 -0
  46. package/src/commands/version.ts +30 -0
  47. package/src/commands/worker.ts +100 -0
  48. package/src/config-schema.ts +47 -1
  49. package/src/config.ts +45 -4
  50. package/src/daemon.ts +972 -101
  51. package/src/dashboard/app.js +459 -0
  52. package/src/dashboard/index.html +61 -0
  53. package/src/dashboard/server.ts +481 -0
  54. package/src/dashboard/style.css +348 -0
  55. package/src/decisions.ts +39 -14
  56. package/src/diff-flags.ts +131 -241
  57. package/src/doctor.ts +932 -0
  58. package/src/escalate.ts +2 -2
  59. package/src/failure-class.ts +66 -3
  60. package/src/fleet.ts +58 -1
  61. package/src/gitops.ts +157 -0
  62. package/src/graph-health.ts +1 -1
  63. package/src/label-projection.ts +1 -1
  64. package/src/lifecycle.ts +198 -2
  65. package/src/model-fallback.ts +177 -0
  66. package/src/notices.ts +9 -0
  67. package/src/omp.ts +93 -13
  68. package/src/orchestrator-tick.ts +414 -20
  69. package/src/orchestrator.ts +4 -4
  70. package/src/privileged.ts +10 -0
  71. package/src/release-policy.ts +342 -30
  72. package/src/reports.ts +19 -5
  73. package/src/session-host.ts +11 -5
  74. package/src/setup-host.ts +663 -25
  75. package/src/setup-install.ts +292 -28
  76. package/src/setup-wizard.ts +255 -74
  77. package/src/setup.ts +156 -35
  78. package/src/stats.ts +331 -0
  79. package/src/store.ts +219 -22
  80. package/src/tracker/github.ts +74 -4
  81. package/src/types.ts +215 -31
  82. package/src/unblock.ts +55 -11
  83. package/src/upgrade-journal.ts +220 -0
  84. package/src/upgrade-verify.ts +506 -0
  85. package/src/upgrade.ts +385 -58
  86. package/src/verbs/actions.ts +73 -1
  87. package/src/verbs/protocol.ts +29 -4
  88. package/src/verbs/server.ts +183 -20
  89. package/src/worker.ts +3 -3
  90. package/systemd/omp-conductor-recover.sh +433 -0
  91. package/systemd/omp-conductor.service.example +14 -3
  92. package/systemd/recover-unit-test.sh +428 -0
package/src/daemon.ts CHANGED
@@ -20,16 +20,23 @@ import {
20
20
  } from "./config.ts";
21
21
  import { availabilityState, type AvailabilityState } from "./availability.ts";
22
22
  import {
23
+ UNREADABLE_TREE_FLAG,
23
24
  analyseSettlement,
25
+ deriveChangedLine,
24
26
  formatSettlementFlags,
25
27
  settlementFlagSummary,
28
+ withDerivedChangedLine,
26
29
  } from "./diff-flags.ts";
27
30
  import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
28
31
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
29
32
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
30
33
  import { graphHint } from "./graph.ts";
31
- import { livingDaemon } from "./lifecycle.ts";
34
+ import { acquireOnceLease, healthCheck, livingDaemon } from "./lifecycle.ts";
32
35
  import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
36
+ import { runDoctor } from "./doctor.ts";
37
+ import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
38
+ import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
39
+ import { fleetLayers } from "./fleet.ts";
33
40
  import { startOrchestrator } from "./orchestrator.ts";
34
41
  import type { OrchestratorHandle } from "./orchestrator.ts";
35
42
  import {
@@ -40,12 +47,21 @@ import {
40
47
  } from "./reports.ts";
41
48
  import {
42
49
  recordReleaseBlock,
50
+ type GateShape,
43
51
  type ReleaseBlockContext,
44
52
  } from "./release-policy.ts";
45
53
  import { branchName, effectiveLabels, route } from "./routing.ts";
46
54
  import type { Routed, UnroutableReason } from "./routing.ts";
47
55
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
48
56
  import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
57
+ import {
58
+ DEFAULT_MODEL_FALLBACK_THRESHOLD,
59
+ fallbackClause,
60
+ formatModelsTried,
61
+ modelsTried,
62
+ providerFailureFacts,
63
+ resolveDispatchModel,
64
+ } from "./model-fallback.ts";
49
65
  import { projectLabels } from "./label-projection.ts";
50
66
  import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
51
67
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
@@ -57,6 +73,7 @@ import type {
57
73
  DispatchSummary,
58
74
  DigestBacklog,
59
75
  Escalation,
76
+ IssueComment,
60
77
  IssueSnapshot,
61
78
  MergedPrInfo,
62
79
  OpenCloser,
@@ -115,7 +132,14 @@ import {
115
132
  } from "./verbs/socket.ts";
116
133
  import { homedir } from "node:os";
117
134
 
118
- import { pushRunBranch, readBaseChain, type RunRepoRef } from "./gitops.ts";
135
+ import {
136
+ probeCriticalBase,
137
+ pushRunBranch,
138
+ readBaseChain,
139
+ type CriticalBaseProbe,
140
+ type CriticalBaseVerdict,
141
+ type RunRepoRef,
142
+ } from "./gitops.ts";
119
143
  import {
120
144
  planUsageLine,
121
145
  readPlanUsage,
@@ -219,6 +243,21 @@ interface Deps {
219
243
  * the real one.
220
244
  */
221
245
  verbActions?: VerbActions;
246
+ /**
247
+ * The fleet-installs-itself verification pass (#486), run on the first tick
248
+ * after a detached upgrade. The default is the journal-gated
249
+ * {@link verifyPendingUpgradeTick}; a test injects its own to drive the tick
250
+ * without touching the host's packages, journal or outbox.
251
+ */
252
+ upgradeVerifier?: (d: Deps) => Promise<void>;
253
+ /**
254
+ * Answers whether a preserved continuation branch contains every configured
255
+ * critical-base marker, for the stale-base admission hold (#428). Wired by
256
+ * `runDaemon` to the mirror-backed {@link probeCriticalBase}; a test injects
257
+ * a fake. Absent, the stale-base gate fails closed whenever a project names
258
+ * a marker (a safety interlock must not silently weaken).
259
+ */
260
+ probeCriticalBase?: CriticalBaseProbe;
222
261
  }
223
262
 
224
263
  /**
@@ -668,6 +707,64 @@ function acceptanceCriteria(issue: ReadyIssue): string {
668
707
  : "_The issue body is empty. Read the issue and its comments, and escalate if it is genuinely underspecified._";
669
708
  }
670
709
 
710
+ /**
711
+ * How much of a comment thread one brief may hold. Grooming notes are short,
712
+ * and a thread that outgrows this is truncated with an explicit marker rather
713
+ * than a silently dropped tail — the disappearance this fix exists to prevent.
714
+ */
715
+ const DISCUSSION_CHARS_BUDGET = 8_000;
716
+
717
+ /**
718
+ * The issue's comments as a brief section: attributed, numbered, oldest
719
+ * first, so a later correction visibly supersedes an earlier note. Rendered
720
+ * beside the body at dispatch, because the orchestrator's grooming is posted
721
+ * as comments and a worker must never depend on a runtime read to see it.
722
+ *
723
+ * `"unread"` is not "empty": when the tracker refused at dispatch the section
724
+ * says so, instead of silently reading as "no discussion" — which is exactly
725
+ * the #517 failure. An empty comment list renders nothing, so a commentless
726
+ * issue's brief stays byte-identical to what this package has always shipped.
727
+ */
728
+ function renderDiscussion(comments: IssueComment[] | "unread"): string {
729
+ if (comments === "unread") {
730
+ return [
731
+ "## Discussion",
732
+ "",
733
+ "_The issue's comments could not be read at dispatch time. The live read below is_",
734
+ "_the only path to them; if it prints nothing, that is a failed read, not an_",
735
+ "_absence of discussion._",
736
+ "",
737
+ ].join("\n");
738
+ }
739
+ if (comments.length === 0) return "";
740
+ const total = comments.length;
741
+ const lines = [
742
+ "## Discussion",
743
+ "",
744
+ `${total} comment${total === 1 ? "" : "s"} on the issue at dispatch, oldest first — a later`,
745
+ "comment supersedes an earlier one.",
746
+ "",
747
+ ];
748
+ let chars = lines.join("\n").length;
749
+ let shown = 0;
750
+ for (const comment of comments) {
751
+ const block = `**@${comment.author} — comment ${shown + 1}:**\n\n${comment.body}\n\n`;
752
+ if (shown > 0 && chars + block.length > DISCUSSION_CHARS_BUDGET) break;
753
+ lines.push(`**@${comment.author} — comment ${shown + 1}:**`, "", comment.body, "");
754
+ chars += block.length;
755
+ shown += 1;
756
+ }
757
+ if (shown < total) {
758
+ const omitted = total - shown;
759
+ lines.push(
760
+ `… ${omitted} comment${omitted === 1 ? "" : "s"} omitted — the discussion exceeded ${DISCUSSION_CHARS_BUDGET} characters.`,
761
+ "Read the issue for the tail.",
762
+ "",
763
+ );
764
+ }
765
+ return lines.join("\n");
766
+ }
767
+
671
768
  /**
672
769
  * Record a state-label swap for projection (#201). The add enqueues before the
673
770
  * remove — the reverse order would leave a window where the issue carries no
@@ -920,7 +1017,15 @@ export async function buildBrief(
920
1017
  r: Routed,
921
1018
  branch: string,
922
1019
  worktree: string,
923
- opts: { continuation?: boolean; defaultBranch?: string; salvagedSha?: string } = {},
1020
+ opts: {
1021
+ continuation?: boolean;
1022
+ defaultBranch?: string;
1023
+ salvagedSha?: string;
1024
+ /** The issue's comments, rendered into the Discussion section. `"unread"`
1025
+ * when the tracker refused at dispatch — rare, but it must never read as
1026
+ * "no comments" (the #517 failure mode). Absent means an empty list. */
1027
+ comments?: IssueComment[] | "unread";
1028
+ } = {},
924
1029
  ): Promise<string> {
925
1030
  // Read per dispatch rather than caching: editing the brief then takes effect
926
1031
  // on the next issue instead of needing a daemon restart.
@@ -965,6 +1070,7 @@ export async function buildBrief(
965
1070
  BRANCH: branch,
966
1071
  WORKTREE: worktree,
967
1072
  ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
1073
+ ISSUE_COMMENTS: renderDiscussion(opts.comments ?? []),
968
1074
  GATES: gatesBlock(r.repo),
969
1075
  // Empty for a repo with no `graphProject`, and empty means *nothing*: the
970
1076
  // placeholder sits flush against the next list item in the template, so an
@@ -981,6 +1087,185 @@ export function hasContinuationBudget(stops: number, maxContinuations: number):
981
1087
  return stops <= maxContinuations;
982
1088
  }
983
1089
 
1090
+ /** True while unspent failed-implementation attempts remain. This is the
1091
+ * dispatcher's admission gate: once every `maxAttemptsPerIssue` slot is
1092
+ * spent, the issue is held as `failed-attempts` forever, and the `unblock`
1093
+ * verb withholds the queue label on the same predicate (#348). */
1094
+ export function hasFailedAttemptBudget(failures: number, maxAttempts: number): boolean {
1095
+ return failures < maxAttempts;
1096
+ }
1097
+
1098
+ /** The failure classes `countContinuations` deliberately does not charge — the
1099
+ * inverted copy of its exclusions, kept beside the breakdown that consumes it
1100
+ * so the two can only drift together (#439). `orphan-clean` is absent on
1101
+ * purpose: daemon orphans consume the continuation budget, which is exactly
1102
+ * why the requeue side has to respect the ceiling instead of racing it. */
1103
+ const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
1104
+ "admin-kill": true,
1105
+ "settlement-stuck": true,
1106
+ "env-start-failure": true,
1107
+ "dispatch-infra": true,
1108
+ "provider-credit": true,
1109
+ "provider-transient": true,
1110
+ };
1111
+
1112
+ /** How one issue spent its continuation budget, grouped by failure class —
1113
+ * the exact rows `continuationsFor` charges, so an exhaustion escalation
1114
+ * reports the same budget it says is spent. `unclassified` groups rows that
1115
+ * charged before the class was written (a pre-upgrade NULL). */
1116
+ function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
1117
+ const perClass = new Map<string, number>();
1118
+ for (const r of runs) {
1119
+ const chargedAsKilledOrOrphaned =
1120
+ (r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
1121
+ (r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
1122
+ const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
1123
+ if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
1124
+ const cls = r.failureClass ?? "unclassified";
1125
+ perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
1126
+ }
1127
+ return perClass;
1128
+ }
1129
+
1130
+ /** The newest attempt's preserved work, if any, so an exhaustion escalation can
1131
+ * say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
1132
+ * three artifacts a run can leave, and all null on a branch nothing reached. */
1133
+ function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
1134
+ const newestFirst = [...runs].reverse();
1135
+ return newestFirst.find(
1136
+ (r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
1137
+ );
1138
+ }
1139
+
1140
+ /** The fenced-block info string that marks an exhaustion postmortem comment, so
1141
+ * a grooming scout re-slicing the issue can find and parse the whole block by
1142
+ * grepping for it. */
1143
+ export const POSTMORTEM_MARKER = "conductor-postmortem";
1144
+
1145
+ /** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
1146
+ * digest shape for naming what the exhaustion was. Groups every row by its
1147
+ * failure class, whether or not it charged the continuation budget, because
1148
+ * the postmortem tells the whole story and not just the budget half (#290). */
1149
+ export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
1150
+ const perClass = new Map<string, number>();
1151
+ for (const r of runs) {
1152
+ const cls = r.failureClass ?? "unclassified";
1153
+ perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
1154
+ }
1155
+ if (perClass.size === 0) return "unclassified";
1156
+ return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
1157
+ }
1158
+
1159
+ /** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
1160
+ export function humanDuration(ms: number): string {
1161
+ const seconds = Math.max(0, Math.round(ms / 1_000));
1162
+ if (seconds < 60) return `${seconds}s`;
1163
+ const minutes = Math.round(seconds / 60);
1164
+ if (minutes < 60) return `${minutes}m`;
1165
+ const hours = Math.floor(minutes / 60);
1166
+ const rest = minutes % 60;
1167
+ return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
1168
+ }
1169
+
1170
+ /** Flatten and bound a run's last error to one greppable table line. */
1171
+ export function oneLineBrief(text: string | undefined): string | undefined {
1172
+ if (text === undefined || text.trim() === "") return undefined;
1173
+ const flat = text.replace(/\s+/g, " ").trim();
1174
+ return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
1175
+ }
1176
+
1177
+ /**
1178
+ * The exhaustion postmortem block: one greppable fenced block covering every
1179
+ * attempt in the chain — continuation rows included — with per-attempt turns,
1180
+ * wall clock, failure class and a one-line last error, the explicit salvage
1181
+ * state, the spend total and the transcript paths for local inspection.
1182
+ *
1183
+ * Pure so the tests hold the shape, not the transport: the writer below owns
1184
+ * the once-only guarantee, this owns what "once" looks like.
1185
+ */
1186
+ export function formatExhaustionPostmortem(args: {
1187
+ issue: number;
1188
+ runs: readonly RunRecord[];
1189
+ reason: string;
1190
+ }): string {
1191
+ const { issue, runs, reason } = args;
1192
+ const artifact = newestContinuableRun(runs);
1193
+ const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
1194
+ const spend = `$${totalSpend.toFixed(2)}`;
1195
+ const attemptLines = runs.map((r) => {
1196
+ const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
1197
+ const cls = r.failureClass ?? "unclassified";
1198
+ const error = oneLineBrief(r.lastError) ?? "—";
1199
+ return (
1200
+ ` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
1201
+ `${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
1202
+ );
1203
+ });
1204
+ const salvage =
1205
+ artifact === undefined
1206
+ ? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
1207
+ : `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
1208
+ `${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
1209
+ return [
1210
+ `\`\`\`${POSTMORTEM_MARKER}`,
1211
+ `#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
1212
+ reason,
1213
+ "",
1214
+ `Attempts (${runs.length} total, ${spend} spend):`,
1215
+ ...attemptLines,
1216
+ salvage,
1217
+ `Spend total: ${spend} across ${runs.length} attempts.`,
1218
+ "Transcripts:",
1219
+ ...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
1220
+ "```",
1221
+ ].join("\n");
1222
+ }
1223
+
1224
+ /** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
1225
+ function postmortemDedupeKey(project: string, issue: number): string {
1226
+ return `${project}:postmortem:${issue}`;
1227
+ }
1228
+
1229
+ /**
1230
+ * The exhaustion postmortem: written exactly once per issue, at the point the
1231
+ * continuation budget is spent and the issue is settled toward a human.
1232
+ *
1233
+ * The comment and the material event are both gated by the store's notification
1234
+ * ledger — the same idempotence guard the escalator uses — so re-settling an
1235
+ * already-postmortemed issue posts nothing and records nothing. A body-string
1236
+ * match on the issue would be the wrong guard: an issue re-scoped and re-run
1237
+ * would still carry the old block, and the guarantee asked of this is "decided
1238
+ * once", not "deduped against what is already written".
1239
+ *
1240
+ * The digest must name the exhaustion even if the comment write fails, so the
1241
+ * material event is recorded before the write and unconditionally (the ledger
1242
+ * is append-only, and the gate above already ran once). The comment failure is
1243
+ * logged rather than taking the sweep down with it.
1244
+ */
1245
+ async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
1246
+ const { project, tracker, store } = d;
1247
+ const key = postmortemDedupeKey(project.name, run.issue);
1248
+ if (store.wasNotified(key)) return;
1249
+ const runs = store.runsForIssue(project.name, run.issue);
1250
+ const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
1251
+ const occurredAt = Date.now();
1252
+ store.recordMaterialEvent({
1253
+ project: project.name,
1254
+ category: "exhaustion",
1255
+ summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
1256
+ evidence: body,
1257
+ occurredAt,
1258
+ recordedAt: occurredAt,
1259
+ });
1260
+ try {
1261
+ await tracker.comment(run.issue, body);
1262
+ store.markNotified(key);
1263
+ log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
1264
+ } catch (err) {
1265
+ log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
1266
+ }
1267
+ }
1268
+
984
1269
  export type ExtendTurnLimitResult =
985
1270
  | { kind: "extended"; runId: string; maxTurns: number }
986
1271
  | { kind: "not-increase"; runId: string; maxTurns: number }
@@ -1184,6 +1469,19 @@ export async function verifyPushedGreenClaim(
1184
1469
  };
1185
1470
  }
1186
1471
 
1472
+ /** What the settlement audit of one green run produced: the advisory flags
1473
+ * (test weakening only — the file-list disclosure is derived, not flagged),
1474
+ * whether the diff was cut short, and the `changed:` line composed from the
1475
+ * PR's own diff. */
1476
+ export interface SettlementAuditResult {
1477
+ flags: SettlementFlag[];
1478
+ truncated: boolean;
1479
+ /** The `changed:` file list derived from the PR's diff, present whenever the
1480
+ * diff could be read. Absent means the tree could not be read, and `flags`
1481
+ * then carries exactly {@link UNREADABLE_TREE_FLAG}. */
1482
+ changedLine?: string;
1483
+ }
1484
+
1187
1485
  /**
1188
1486
  * Audit a worker's own account of its work against the pull request it pushed.
1189
1487
  *
@@ -1192,27 +1490,21 @@ export async function verifyPushedGreenClaim(
1192
1490
  * shares none of its authority — that function decides a run's state, this one
1193
1491
  * cannot, by construction. It returns evidence and the caller appends it.
1194
1492
  *
1195
- * Returns undefined when the audit could not run at all: no PR to read, or a
1196
- * tracker that could not produce a diff. Undefined is *not* an empty flag list.
1197
- * An empty list means "read the whole diff, found nothing"; undefined means
1198
- * "did not look", and the report says so rather than presenting silence as a
1199
- * clean bill.
1493
+ * A diff that cannot be read is a finding (`changed-line-missing`), never an
1494
+ * empty flag list: nothing was derived and nothing was checked, and that must
1495
+ * not read as a clean bill.
1200
1496
  */
1201
1497
  export async function collectSettlementFlags(
1202
1498
  tracker: Pick<Tracker, "prDiff">,
1203
- claim: { prUrl?: string; report: string; priorReports?: readonly string[]; issueText: string },
1204
- ): Promise<{ flags: SettlementFlag[]; truncated: boolean } | undefined> {
1205
- if (claim.prUrl === undefined) return undefined;
1499
+ claim: { prUrl?: string; issueText: string },
1500
+ ): Promise<SettlementAuditResult> {
1501
+ if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
1206
1502
  const diff = await tracker.prDiff(claim.prUrl);
1207
- if (diff === undefined) return undefined;
1503
+ if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
1208
1504
  return {
1209
- flags: analyseSettlement({
1210
- report: claim.report,
1211
- issueText: claim.issueText,
1212
- diff,
1213
- priorReports: claim.priorReports,
1214
- }),
1505
+ flags: analyseSettlement({ issueText: claim.issueText, diff }),
1215
1506
  truncated: diff.truncated,
1507
+ changedLine: deriveChangedLine(diff),
1216
1508
  };
1217
1509
  }
1218
1510
 
@@ -1258,7 +1550,7 @@ export function workerReleaseBlockRecorder(
1258
1550
  issue: number,
1259
1551
  runId: string,
1260
1552
  root = stateDir(),
1261
- ): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
1553
+ ): (shape: GateShape, context: ReleaseBlockContext) => void {
1262
1554
  return (shape, context) =>
1263
1555
  recordReleaseBlock(
1264
1556
  project,
@@ -1273,6 +1565,48 @@ export function workerReleaseBlockRecorder(
1273
1565
  );
1274
1566
  }
1275
1567
 
1568
+ /**
1569
+ * The exact PR a new claim inherits from its terminal predecessor (#434).
1570
+ *
1571
+ * `recentRuns` shows only the newest attempt per issue, and the mediated verbs
1572
+ * resolve a PR's owner through the store — so the moment an orphan-clean
1573
+ * requeue creates its continuation row, the predecessor's green PR (recorded
1574
+ * on the older row) stops resolving and every mediated verb refuses it as "not
1575
+ * a pull request any run opened". Seeding the continuation row with the same
1576
+ * PR at claim time closes that window: ownership never depends on a worker
1577
+ * rediscovering and re-reporting the PR.
1578
+ *
1579
+ * The checks are deliberately the exact counterpart of the store query's scope:
1580
+ * the claim's project/issue/repo/branch must all match the predecessor, the
1581
+ * predecessor must be terminal (a live row is still doing its own work, and a
1582
+ * claim beside one is already a double dispatch), and the tracker must confirm
1583
+ * the PR is still open — a closed or merged predecessor PR is settled by the
1584
+ * sweep against the row that owns it, never carried onto a fresh attempt. An
1585
+ * unreadable tracker fails closed: the new row simply starts without PR
1586
+ * ownership, exactly as before #434, and adoption or the worker's own
1587
+ * pr_create recovers the link.
1588
+ */
1589
+ export async function inheritedPrForContinuation(
1590
+ tracker: Tracker,
1591
+ prior: RunRecord | undefined,
1592
+ claim: { project: string; issue: number; repo: string; branch: string },
1593
+ ): Promise<{ prUrl: string; headSha?: string } | undefined> {
1594
+ if (prior === undefined || prior.prUrl === undefined) return undefined;
1595
+ // Terminal only. A live predecessor owns its PR outright and is not done
1596
+ // with it; inheriting would launder a double dispatch's row.
1597
+ if (prior.state === "claimed" || prior.state === "running") return undefined;
1598
+ if (prior.project !== claim.project || prior.issue !== claim.issue) return undefined;
1599
+ if (prior.repo !== claim.repo || prior.branch !== claim.branch) return undefined;
1600
+ let state: PrState | undefined;
1601
+ try {
1602
+ state = await tracker.prState(prior.prUrl);
1603
+ } catch {
1604
+ return undefined;
1605
+ }
1606
+ if (state !== "open") return undefined;
1607
+ return { prUrl: prior.prUrl, ...(prior.headSha === undefined ? {} : { headSha: prior.headSha }) };
1608
+ }
1609
+
1276
1610
  export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1277
1611
  const { project, caps, tracker, store } = d;
1278
1612
  const issue = r.issue.number;
@@ -1429,7 +1763,13 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1429
1763
  // (#201).
1430
1764
  // Read before this attempt's own row exists, so `latestRun` still means the
1431
1765
  // attempt whose work this one inherits.
1432
- const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
1766
+ const prior = store.latestRun(project.name, issue);
1767
+ const priorSalvage = prior?.salvageSha;
1768
+ // The failover decision is made from the store the same way admission is:
1769
+ // counts of this issue's own rows, so the switch is sticky per chain and a
1770
+ // fresh issue always starts on the primary (#286). Read before the claim
1771
+ // writes this attempt's row, which would otherwise break the streak.
1772
+ const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
1433
1773
  run = store.createRun({
1434
1774
  project: project.name,
1435
1775
  issue,
@@ -1443,6 +1783,27 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1443
1783
  maxTurns: caps.workerMaxTurns,
1444
1784
  startedAt: Date.now(),
1445
1785
  });
1786
+ // #434: carry a terminal predecessor's open PR onto the continuation row.
1787
+ // The claim itself stays synchronous — `handleIssue` claims at the top of
1788
+ // its dispatch burst, the window the shutdown fence and its tests rely on
1789
+ // (#374) — so the tuple-exact, tracker-verified inheritance lands here,
1790
+ // still before any provisioning, network call or verb socket. The new row
1791
+ // is `recentRuns`' newest attempt for the issue the moment it exists, so
1792
+ // without this a requeue hides the green PR the orphaned attempt opened
1793
+ // until a worker happens to rediscover it.
1794
+ const inherited = await inheritedPrForContinuation(tracker, prior, {
1795
+ project: project.name,
1796
+ issue,
1797
+ repo: r.repo.name,
1798
+ branch,
1799
+ });
1800
+ if (inherited !== undefined && prior !== undefined) {
1801
+ store.updateRun(run.id, {
1802
+ prUrl: inherited.prUrl,
1803
+ ...(inherited.headSha === undefined ? {} : { headSha: inherited.headSha }),
1804
+ });
1805
+ log(`#${issue} inherited PR ${inherited.prUrl} from attempt ${prior.attempt} (${prior.state})`);
1806
+ }
1446
1807
  const maxTurns = run.maxTurns;
1447
1808
  const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
1448
1809
  const runId = run.id;
@@ -1522,17 +1883,50 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1522
1883
 
1523
1884
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
1524
1885
 
1886
+ // Where this attempt goes, and (when the failover fired) the clause that
1887
+ // makes it attributable: with no `modelFallbacks` configured this is the
1888
+ // primary model — or none, for an unconfigured project — and today's
1889
+ // dispatch is byte for byte what it has always been.
1890
+ const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
1891
+ const choice = resolveDispatchModel({
1892
+ workerModel: project.workerModel,
1893
+ modelFallbacks: project.modelFallbacks,
1894
+ threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
1895
+ streak: chainFacts.streak,
1896
+ });
1897
+ // Recorded before the launch, so even a run killed mid-flight leaves the
1898
+ // model it chose on its row. Only a chain-configured project writes the
1899
+ // column: absent `modelFallbacks` must preserve today's rows byte for byte.
1900
+ if (chainConfigured && choice.model !== undefined) {
1901
+ store.updateRun(runId, { model: choice.model });
1902
+ }
1903
+ const clause = fallbackClause(choice, chainFacts, project.workerModel);
1904
+
1525
1905
  log(
1526
- `#${issue} attempt ${attempt} → ${r.repo.name} ${branch}` +
1906
+ `#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
1527
1907
  (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
1528
1908
  );
1529
1909
 
1910
+ // The discussion is rendered at dispatch so a worker never depends on a
1911
+ // runtime `gh` read to see the orchestrator's grooming (#517). The read is
1912
+ // best-effort, but its failure is not silent: an unreadable tracker names
1913
+ // itself in the brief's Discussion section instead of reading as "no
1914
+ // comments" — the exact confusion this fix removes.
1915
+ let comments: IssueComment[] | "unread";
1916
+ try {
1917
+ comments = await tracker.listComments(issue);
1918
+ } catch (err) {
1919
+ log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
1920
+ comments = "unread";
1921
+ }
1922
+
1530
1923
  const brief = await buildBrief(project, r, branch, worktreePath, {
1531
1924
  continuation: provisioned.reattached,
1532
1925
  defaultBranch: r.repo.defaultBranch,
1533
1926
  ...(provisioned.reattached && priorSalvage !== undefined
1534
1927
  ? { salvagedSha: priorSalvage }
1535
1928
  : {}),
1929
+ comments,
1536
1930
  });
1537
1931
  if (await settleStopBeforeSession()) return;
1538
1932
  if (await settleDrainBeforeSession()) return;
@@ -1567,7 +1961,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1567
1961
  onChildLog: (line) => {
1568
1962
  log(`#${issue} ${line}`);
1569
1963
  },
1570
- ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
1964
+ ...(choice.model === undefined ? {} : { model: choice.model }),
1571
1965
  releaseGrants: resolveReleaseGrants(project),
1572
1966
  onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
1573
1967
  onTurn: (n) => store.updateRun(runId, { turns: n }),
@@ -1618,39 +2012,37 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1618
2012
  // The other half of not believing a worker about its own run (#128). The
1619
2013
  // claim being audited is `state: pushed-green`, so the audit runs on the
1620
2014
  // worker's claim rather than on what verification made of it: a claim that
1621
- // GitHub then contradicted is exactly the report whose `changed:` line is
1622
- // worth reading twice. Advisory throughout — nothing below reads `flags`
1623
- // when deciding `state`, and the ordering here makes that checkable.
2015
+ // GitHub then contradicted is exactly the report whose details matter
2016
+ // most. Advisory throughout — nothing below reads `flags` when deciding
2017
+ // `state`, and the ordering here makes that checkable.
1624
2018
  const audit =
1625
2019
  result.state === "pushed-green"
1626
2020
  ? await collectSettlementFlags(tracker, {
1627
2021
  prUrl: result.prUrl,
1628
- report: result.report,
1629
- // Earlier attempts' reports pool as disclosure coverage (#199). The
1630
- // current row has not settled yet, so this is exactly the prior set;
1631
- // the strict `< attempt` guard keeps an already-persisted copy of
1632
- // this attempt out just in case. Pre-#199 rows have NULL reports and
1633
- // contribute nothing — today's behaviour.
1634
- priorReports: store
1635
- .attemptReports(project.name, issue)
1636
- .filter((r) => r.attempt < attempt)
1637
- .map((r) => r.report),
1638
2022
  issueText: `${r.issue.title}\n${r.issue.body}`,
1639
2023
  })
1640
2024
  : undefined;
1641
- if (result.state === "pushed-green") {
1642
- if (audit === undefined) log(`#${issue} settlement audit skipped: no readable PR diff`);
1643
- else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
2025
+ if (result.state === "pushed-green" && audit?.truncated) {
2026
+ log(`#${issue} settlement audit read a truncated PR diff`);
1644
2027
  }
1645
2028
  const auditLines =
1646
2029
  audit === undefined
1647
2030
  ? []
1648
- : formatSettlementFlags(audit.flags, { truncated: audit.truncated, attempts: attempt });
2031
+ : formatSettlementFlags(audit.flags, { truncated: audit.truncated });
2032
+ // The report's file list is derived from the PR's own diff, not retyped
2033
+ // from the worker's memory of the session: the narrative stays the
2034
+ // worker's, the disclosure becomes the diff's (#488). A diff that could
2035
+ // not be read leaves the worker's text untouched and the audit's
2036
+ // `changed-line-missing` flag says so.
2037
+ const settlementReport =
2038
+ audit?.changedLine === undefined
2039
+ ? result.report
2040
+ : withDerivedChangedLine(result.report, audit.changedLine);
1649
2041
 
1650
2042
  const finalReport = [
1651
2043
  ...(verified.reason === undefined ? [] : [verified.reason, ""]),
1652
2044
  ...(auditLines.length === 0 ? [] : [...auditLines, ""]),
1653
- result.report,
2045
+ settlementReport,
1654
2046
  ].join("\n");
1655
2047
 
1656
2048
  // What becomes of the tree, decided once, before any label or page. A
@@ -1695,12 +2087,18 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1695
2087
  endedAt: Date.now(),
1696
2088
  turns: result.turns,
1697
2089
  spendUsd: result.spendUsd,
1698
- prUrl: result.prUrl,
1699
- headSha: result.headSha,
2090
+ // The worker only reports these when it actually established them; a kill
2091
+ // or a settle whose report named no PR must not wipe what a verb recorded
2092
+ // earlier in the same run (#468). The sink in `updateRun` skips undefined
2093
+ // too, but a settle should not even build such a patch.
2094
+ ...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
2095
+ ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
1700
2096
  sessionFile: result.sessionFile,
1701
- // Every terminal state persists the worker's report, not just a green
1702
- // push: a stopped attempt's partial report is still part of its audit trail.
1703
- report: result.report,
2097
+ // Every terminal state persists the worker's report — with the `changed:`
2098
+ // file list derived from the PR's diff where one could be read — not
2099
+ // just a green push: a stopped attempt's partial report is still part of
2100
+ // its audit trail.
2101
+ report: settlementReport,
1704
2102
  ...settlement?.patch,
1705
2103
  ...(audit === undefined || audit.flags.length === 0
1706
2104
  ? {}
@@ -2275,8 +2673,13 @@ export async function watchBaseHealth(
2275
2673
 
2276
2674
  export async function settlePushedGreen(
2277
2675
  d: Pick<Deps, "project" | "tracker" | "store">,
2278
- ): Promise<void> {
2676
+ ): Promise<number> {
2279
2677
  const { project, tracker, store } = d;
2678
+ // Runs the sweep resolved by terminalising the row. Every terminal write
2679
+ // below increments it; the tick attributes it as `settled` on the pass's
2680
+ // dispatch record, so a held pass is visible as work done, not just as a
2681
+ // clock that moved (#497).
2682
+ let settled = 0;
2280
2683
  // Filtered from the active set rather than asked for with a new query: active
2281
2684
  // is live workers plus these, so the list is bounded by the worker cap plus
2282
2685
  // the number of PRs awaiting a merge — a handful, by construction. A fleet
@@ -2343,6 +2746,7 @@ export async function settlePushedGreen(
2343
2746
  patch.recoveryAction = "none";
2344
2747
  }
2345
2748
  store.updateRun(run.id, patch);
2749
+ settled += 1;
2346
2750
  log(`#${run.issue} settled: ${settlement.reason}`);
2347
2751
  continue;
2348
2752
  }
@@ -2357,7 +2761,7 @@ export async function settlePushedGreen(
2357
2761
  }
2358
2762
  if (verification === undefined) continue;
2359
2763
  if (verification.status === "green") {
2360
- store.updateRun(run.id, { state: "pushed-green", lastError: undefined });
2764
+ store.updateRun(run.id, { state: "pushed-green", lastError: null });
2361
2765
  log(`#${run.issue} checks settled: ${verification.reason}`);
2362
2766
  } else if (verification.status === "failed") {
2363
2767
  // Equally terminal, so the release is enqueued before the row writes,
@@ -2366,11 +2770,13 @@ export async function settlePushedGreen(
2366
2770
  // and its live PR is exactly the work the label must keep guarding.
2367
2771
  releaseInProgress(d, run.issue, verification.reason);
2368
2772
  store.updateRun(run.id, { state: "failed", lastError: verification.reason });
2773
+ settled += 1;
2369
2774
  log(`#${run.issue} checks failed: ${verification.reason}`);
2370
2775
  } else {
2371
2776
  store.updateRun(run.id, { lastError: verification.reason });
2372
2777
  }
2373
2778
  }
2779
+ return settled;
2374
2780
  }
2375
2781
 
2376
2782
  const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
@@ -2612,6 +3018,7 @@ export function summarizeDispatch(
2612
3018
  admitted: number,
2613
3019
  holds: readonly AdmissionHold[],
2614
3020
  completedAt = Date.now(),
3021
+ settled = 0,
2615
3022
  ): DispatchSummary {
2616
3023
  const groups = new Map<AdmissionHoldReason, { count: number; issues: number[] }>();
2617
3024
  for (const hold of holds) {
@@ -2630,6 +3037,27 @@ export function summarizeDispatch(
2630
3037
  holds: [...groups]
2631
3038
  .sort(([a], [b]) => a.localeCompare(b))
2632
3039
  .map(([reason, group]) => ({ reason, ...group })),
3040
+ settled,
3041
+ };
3042
+ }
3043
+
3044
+ /**
3045
+ * The record of a held pass (#497): the loop ran — settling and reconciling
3046
+ * above the pause gate — but routing never happened and nothing was admitted.
3047
+ * `ready`/`routed`/`claimed` stay 0 because they were never computed; the
3048
+ * `paused` flag is what keeps a reader from reading them as an empty queue.
3049
+ */
3050
+ export function summarizeHeldPass(settled: number, completedAt = Date.now()): DispatchSummary {
3051
+ return {
3052
+ completedAt,
3053
+ ready: 0,
3054
+ routed: 0,
3055
+ claimed: 0,
3056
+ admitted: 0,
3057
+ degraded: false,
3058
+ holds: [],
3059
+ settled,
3060
+ paused: true,
2633
3061
  };
2634
3062
  }
2635
3063
 
@@ -2717,7 +3145,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
2717
3145
  * that grows a field has no business breaking these tests.
2718
3146
  */
2719
3147
  export async function admitCandidates(
2720
- d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage">,
3148
+ d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "probeCriticalBase">,
2721
3149
  routed: Routed[],
2722
3150
  slots: number,
2723
3151
  ): Promise<AdmissionPass> {
@@ -2837,7 +3265,7 @@ export async function admitCandidates(
2837
3265
 
2838
3266
  const priorRuns = store.attemptsFor(project.name, issue);
2839
3267
  const failures = store.failuresFor(project.name, issue);
2840
- if (failures >= caps.maxAttemptsPerIssue) {
3268
+ if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) {
2841
3269
  hold(issue, "failed-attempts");
2842
3270
  await safeEscalate(d, {
2843
3271
  tier: 1,
@@ -2897,6 +3325,81 @@ export async function admitCandidates(
2897
3325
  continue;
2898
3326
  }
2899
3327
 
3328
+ // #428 half (a): a preserved continuation that predates a configured
3329
+ // critical-base/safety marker must not be reattached. A base safety fix
3330
+ // protects only branches forked after it landed — a continuation forked
3331
+ // before it still carries the dangerous test/runtime code, and re-running
3332
+ // it on the shared host is what SIGTERMed the production daemon. Fail
3333
+ // closed: only a probe that proves every marker is in the reattach
3334
+ // source's ancestry admits, and a project that names a marker but has no
3335
+ // probe wired (never happens outside tests) holds. Both the hold and the
3336
+ // escalation are durable across restart and orphan recovery because this
3337
+ // gate runs every admission pass; the branch is re-admitted automatically
3338
+ // once the operator updates it to contain the marker, without losing work.
3339
+ const markers = project.criticalBase ?? [];
3340
+ if (markers.length > 0) {
3341
+ const branch = branchName(r.issue);
3342
+ let verdict: CriticalBaseVerdict;
3343
+ if (d.probeCriticalBase === undefined) {
3344
+ verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" };
3345
+ } else {
3346
+ try {
3347
+ verdict = await d.probeCriticalBase(r.repo, markers, branch);
3348
+ } catch (err) {
3349
+ verdict = { state: "unknown", error: errText(err) };
3350
+ }
3351
+ }
3352
+ if (verdict.state === "stale") {
3353
+ hold(issue, "stale-base");
3354
+ log(
3355
+ `#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`,
3356
+ );
3357
+ await safeEscalate(d, {
3358
+ tier: 1,
3359
+ project: project.name,
3360
+ issue,
3361
+ summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`,
3362
+ detail: [
3363
+ r.issue.title,
3364
+ r.issue.url,
3365
+ `The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`,
3366
+ ...(verdict.range.length > 0
3367
+ ? [`Base commits the branch is missing: ${verdict.range.join(", ")}`]
3368
+ : []),
3369
+ "Recovery: merge current base into the branch so it contains the marker, and the next",
3370
+ "admission pass re-admits it automatically without losing the branch's work; or review",
3371
+ "the branch by hand and clear the hold once the fix is present.",
3372
+ ].join("\n"),
3373
+ });
3374
+ continue;
3375
+ }
3376
+ if (verdict.state === "unknown") {
3377
+ // Fail closed: a branch that cannot be *proven* to contain the marker
3378
+ // is refused, and the reason names the unverifiable marker so the
3379
+ // operator can fix the fetch or the marker rather than guess.
3380
+ hold(issue, "stale-base");
3381
+ log(
3382
+ `#${issue} held (stale-base): continuation branch ${branch} could not be verified ` +
3383
+ `against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`,
3384
+ );
3385
+ await safeEscalate(d, {
3386
+ tier: 1,
3387
+ project: project.name,
3388
+ issue,
3389
+ summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (stale-base)`,
3390
+ detail: [
3391
+ r.issue.title,
3392
+ r.issue.url,
3393
+ `The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`,
3394
+ "Recovery: merge current base into the branch so it contains the marker, and the next",
3395
+ "admission pass re-admits it automatically without losing the branch's work; or review",
3396
+ "the branch by hand and clear the hold once the fix is present.",
3397
+ ].join("\n"),
3398
+ });
3399
+ continue;
3400
+ }
3401
+ }
3402
+
2900
3403
  // Soft concurrency per epic, per repository: at most one in-flight child of
2901
3404
  // a given parent in each repo. Children of one epic in *different* repos
2902
3405
  // parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
@@ -3103,6 +3606,70 @@ export function wakeOrchestratorForMetConditions(
3103
3606
  }
3104
3607
  }
3105
3608
 
3609
+
3610
+ /**
3611
+ * The first-tick verification for a fleet-initiated upgrade (#486).
3612
+ *
3613
+ * The detached install unit journals its progress and stops, leaving dispatch
3614
+ * paused; whichever daemon comes back after the restart runs this on its
3615
+ * first tick — above every gate that a pause or an integrity mismatch could
3616
+ * close, because the install's own pause is the thing under verification.
3617
+ * A day with no journal costs one file read; only a pending request pays for
3618
+ * the version/health/ticks/pane/doctor checks and the outbox rows.
3619
+ */
3620
+ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): UpgradeVerifyDeps {
3621
+ return {
3622
+ projectName: d.project.name,
3623
+ journal: (entry) => appendJournal(stateDir(), entry),
3624
+ run: runCommand,
3625
+ layers: (project) => fleetLayers(project),
3626
+ health: healthCheck,
3627
+ doctor: (name) => runDoctor(name),
3628
+ enqueue: (draft) => {
3629
+ try {
3630
+ d.store.enqueueReport(draft);
3631
+ } catch (err) {
3632
+ log(`upgrade verification report could not be queued: ${errText(err)}`);
3633
+ }
3634
+ },
3635
+ pause: pauseInstance,
3636
+ resume: (project) => setPaused(false, undefined, project),
3637
+ launchRollback: async (version) => {
3638
+ const launched = await launchTransientUnit(
3639
+ (argv) => runCommand(argv[0]!, argv.slice(1)),
3640
+ process.env,
3641
+ { kind: "rollback", version },
3642
+ );
3643
+ return launched.ok === true ? { ok: true, unit: launched.unit } : { ok: false, stderr: launched.stderr };
3644
+ },
3645
+ log,
3646
+ now: () => Date.now(),
3647
+ };
3648
+ }
3649
+
3650
+ async function verifyPendingUpgradeTick(
3651
+ d: Deps,
3652
+ run = verifyPendingUpgrade,
3653
+ ): Promise<void> {
3654
+ // Cheap pre-test: no journal, no pending request, nothing to touch.
3655
+ if (readUpgradeJournal(stateDir()).length === 0) return;
3656
+ const verifyDeps = upgradeVerifyDepsFor(d);
3657
+ try {
3658
+ const outcome = await run(verifyDeps);
3659
+ if (outcome.handled !== "none" && outcome.handled !== "already-closed") {
3660
+ log(
3661
+ `upgrade journal handled: ${outcome.handled}` +
3662
+ `${outcome.version === undefined ? "" : ` for omp-conductor@${outcome.version}`}` +
3663
+ `${outcome.detail === undefined ? "" : ` — ${outcome.detail}`}`,
3664
+ );
3665
+ }
3666
+ } catch (err) {
3667
+ // A torn journal must not take the tick down: the operator still reads it
3668
+ // through `status`, and the next tick retries the same pass.
3669
+ log(`upgrade verification pass failed: ${errText(err)}`);
3670
+ }
3671
+ }
3672
+
3106
3673
  export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3107
3674
  // A config edit takes effect on the next tick, not the next daemon restart
3108
3675
  // (#170). Re-resolve the project and its caps at the tick boundary so a tick
@@ -3124,6 +3691,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3124
3691
  d.deliveryPolicyValid = false;
3125
3692
  }
3126
3693
 
3694
+ // The fleet-installs-itself transaction (#486): the detached install unit
3695
+ // paused dispatch and stopped; the first tick after the restart verifies
3696
+ // against its journal and closes or rolls back. Above the pause gate on
3697
+ // purpose — the install's own pause is exactly the state being verified.
3698
+ try {
3699
+ await (d.upgradeVerifier ?? verifyPendingUpgradeTick)(d);
3700
+ } catch (err) {
3701
+ log(`upgrade journal pass failed: ${errText(err)}`);
3702
+ }
3703
+
3127
3704
  // Availability-held notices are already durable. Once the freshly reloaded
3128
3705
  // policy opens (or newly allows their category), atomically hand a bounded
3129
3706
  // batch to the report outbox. Pauses do not suppress delivery.
@@ -3150,8 +3727,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3150
3727
  // Settlement is maintenance, not dispatch. Run it before every gate that can
3151
3728
  // stop claiming — pause, integrity, spend, and capacity — so status converges
3152
3729
  // while the fleet is parked or workers are still active. Resident workers run
3153
- // through the pool without blocking this five-minute tick.
3154
- await settlePushedGreen(d);
3730
+ // through the pool without blocking this five-minute tick. The return value
3731
+ // is what this pass settled; a held pass records it so the paused state reads
3732
+ // as work done, not just a clock that moved (#497).
3733
+ let settled = await settlePushedGreen(d);
3155
3734
  try {
3156
3735
  await watchMergedBase(d);
3157
3736
  } catch (err) {
@@ -3177,7 +3756,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3177
3756
  // Each is guarded on its own: a tracker that fails mid-classification must not
3178
3757
  // stop the label reconcile, and neither may stop the tick.
3179
3758
  try {
3180
- await classifyAndRecover(d);
3759
+ settled += await classifyAndRecover(d);
3181
3760
  } catch (err) {
3182
3761
  log(`classification sweep failed: ${errText(err)}`);
3183
3762
  }
@@ -3228,8 +3807,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3228
3807
  });
3229
3808
 
3230
3809
  // A paused fleet claims nothing. Checked first so pausing takes effect on the
3231
- // next tick without signalling the process.
3232
- if (isPaused(d.project.name)) return;
3810
+ // next tick without signalling the process. But the pass still ran, and the
3811
+ // operator has to be able to see it: record it as a held pass — the work the
3812
+ // maintenance sweeps settled above this gate, and nothing admitted below it —
3813
+ // so `last dispatch` keeps moving while the fleet is deliberately parked. A
3814
+ // frozen clock is then a stall report, not a hold (#497).
3815
+ if (isPaused(d.project.name)) {
3816
+ d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
3817
+ return;
3818
+ }
3233
3819
 
3234
3820
  const { project, caps, store } = d;
3235
3821
 
@@ -3319,7 +3905,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3319
3905
  const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
3320
3906
  store.recordDispatch(
3321
3907
  project.name,
3322
- summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
3908
+ summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled),
3323
3909
  );
3324
3910
  };
3325
3911
 
@@ -3577,7 +4163,7 @@ export async function turnLimitResponse(
3577
4163
  export async function workerControlResponse(
3578
4164
  req: Request,
3579
4165
  project: string,
3580
- store: Pick<Store, "latestRun">,
4166
+ store: Pick<Store, "latestRun" | "updateRun">,
3581
4167
  registry: WorkerControlRegistry,
3582
4168
  ): Promise<Response | undefined> {
3583
4169
  const url = new URL(req.url);
@@ -3671,6 +4257,28 @@ export async function workerControlResponse(
3671
4257
  state: latest.state,
3672
4258
  });
3673
4259
  }
4260
+ // A live-state row whose controller is absent from this process is a ghost
4261
+ // (#431): a session controller lives in the daemon that launched it, and once
4262
+ // ownership is exclusive (the once lease makes a drill hold the same pidfile
4263
+ // as the long dispatcher), a row that is live but has no controller in *this*
4264
+ // owner is a row whose owner already died — not work another live daemon is
4265
+ // still writing to. Refuse nothing: recover it as an orphan (worktree kept
4266
+ // for triage, labels untouched, live slot freed) so the issue can be
4267
+ // dispatched again and the row stops counting as a live worker.
4268
+ if (action === "stop") {
4269
+ store.updateRun(latest.id, {
4270
+ state: "orphaned",
4271
+ endedAt: Date.now(),
4272
+ lastError: reason,
4273
+ });
4274
+ return Response.json({
4275
+ outcome: "stopped",
4276
+ runId: latest.id,
4277
+ state: "orphaned",
4278
+ reason: reason!,
4279
+ worktree: latest.worktree,
4280
+ });
4281
+ }
3674
4282
  return Response.json(
3675
4283
  {
3676
4284
  error:
@@ -3683,7 +4291,7 @@ export async function workerControlResponse(
3683
4291
 
3684
4292
  export interface DaemonHttpProjectDeps {
3685
4293
  project: string;
3686
- store: Pick<Store, "latestRun" | "setTurnOverride">;
4294
+ store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun">;
3687
4295
  caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
3688
4296
  }
3689
4297
 
@@ -3692,21 +4300,32 @@ export interface DaemonHttpDeps {
3692
4300
  turnLimits: TurnLimitRegistry;
3693
4301
  workerControls: WorkerControlRegistry;
3694
4302
  health: () => DaemonHealth;
4303
+ /** Request an immediate dispatch pass (`resume`'s wake, #380). */
4304
+ wake: () => void;
3695
4305
  }
3696
4306
 
3697
4307
  /**
3698
4308
  * The whole HTTP surface, in one named function so a test can pin what is *not*
3699
4309
  * on it.
3700
4310
  *
3701
- * Three route families: the health read, turn-limit control, and live-worker
3702
- * pause/resume/stop control. Everything else is 404. The controls mutate only
3703
- * daemon-owned live sessions; tracker and repository mutations stay on the
3704
- * authenticated per-run channel described at the `Bun.serve` call (#126).
4311
+ * Four route families: the health read, the dispatch wake, turn-limit control,
4312
+ * and live-worker pause/resume/stop control. Everything else is 404. The
4313
+ * controls mutate only daemon-owned live sessions; tracker and repository
4314
+ * mutations stay on the authenticated per-run channel described at the
4315
+ * `Bun.serve` call (#126).
3705
4316
  */
3706
4317
  export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
3707
4318
  const url = new URL(req.url);
3708
4319
  if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
3709
4320
 
4321
+ if (req.method === "POST" && url.pathname === "/wake") {
4322
+ // A resume wake: daemon-wide and loopback-only, like /healthz. The pass it
4323
+ // produces re-reads every project's pause file, so a project that is still
4324
+ // held stays held — the wake shortens the wait, it never bypasses a gate.
4325
+ d.wake();
4326
+ return new Response(null, { status: 204 });
4327
+ }
4328
+
3710
4329
  let selected = d.projects[0];
3711
4330
  if (
3712
4331
  req.method === "PUT" &&
@@ -3903,21 +4522,31 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
3903
4522
  if (summary === undefined) return "last dispatch (none recorded)";
3904
4523
  const lines = [
3905
4524
  `last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
3906
- ` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
3907
- ` admitted ${summary.admitted}`,
3908
4525
  ];
3909
- if (summary.holds.length === 0) {
3910
- lines.push(" held 0");
4526
+ if (summary.paused === true) {
4527
+ // A held pass never routed the queue: printing zero candidates here would
4528
+ // read as "the queue was empty". Say what the pass was and attribute what
4529
+ // it did instead (#497).
4530
+ lines.push(` pass held — nothing admitted`);
3911
4531
  } else {
3912
- lines.push(" held");
3913
- for (const hold of summary.holds) {
3914
- const sample =
3915
- hold.issues.length === 0
3916
- ? ""
3917
- : ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
3918
- lines.push(` ${hold.reason} ${hold.count}${sample}`);
4532
+ lines.push(
4533
+ ` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
4534
+ ` admitted ${summary.admitted}`,
4535
+ );
4536
+ if (summary.holds.length === 0) {
4537
+ lines.push(" held 0");
4538
+ } else {
4539
+ lines.push(" held");
4540
+ for (const hold of summary.holds) {
4541
+ const sample =
4542
+ hold.issues.length === 0
4543
+ ? ""
4544
+ : ` (#${hold.issues.join(", #")}${hold.count > hold.issues.length ? ", …" : ""})`;
4545
+ lines.push(` ${hold.reason} ${hold.count}${sample}`);
4546
+ }
3919
4547
  }
3920
4548
  }
4549
+ if ((summary.settled ?? 0) > 0) lines.push(` settled ${summary.settled}`);
3921
4550
  return lines.join("\n");
3922
4551
  }
3923
4552
 
@@ -4237,8 +4866,12 @@ export function readSessionError(sessionFile: string | undefined): SessionError
4237
4866
  * it is healthy, and writing a class onto it would take it out of this sweep for
4238
4867
  * good.
4239
4868
  */
4240
- export async function classifyAndRecover(d: Deps): Promise<void> {
4241
- const { project, tracker, store } = d;
4869
+ export async function classifyAndRecover(d: Deps): Promise<number> {
4870
+ const { project, caps, tracker, store } = d;
4871
+ // Settle recoveries, counted for the pass's dispatch record — a row whose PR
4872
+ // merged is settled here when the settle sweep could not establish identity
4873
+ // (#497). Every other exit returns 0.
4874
+ let settled = 0;
4242
4875
  for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
4243
4876
  const facts: ClassifyFacts = {};
4244
4877
  let classifiedRun = run;
@@ -4280,7 +4913,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
4280
4913
  continue;
4281
4914
  }
4282
4915
 
4283
- const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
4916
+ const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
4284
4917
 
4285
4918
  // A healthy green PR is not a failure of any class. Leaving the row
4286
4919
  // unclassified is what keeps it eligible for the sweep on the tick where its
@@ -4294,8 +4927,10 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
4294
4927
  ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
4295
4928
  : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
4296
4929
  );
4930
+ if (recovery === "settle") settled += 1;
4297
4931
  await recoverRun(d, classifiedRun, cls, recovery, evidence);
4298
4932
  }
4933
+ return settled;
4299
4934
  }
4300
4935
 
4301
4936
  /** Performs the one action a class names. Never chooses one of its own. */
@@ -4306,7 +4941,7 @@ async function recoverRun(
4306
4941
  recovery: RecoveryAction,
4307
4942
  evidence: string,
4308
4943
  ): Promise<void> {
4309
- const { project, tracker, store } = d;
4944
+ const { project, caps, tracker, store } = d;
4310
4945
  const inProgress = project.stateLabels.inProgress;
4311
4946
 
4312
4947
  if (recovery === "settle") {
@@ -4337,6 +4972,55 @@ async function recoverRun(
4337
4972
  return;
4338
4973
  }
4339
4974
 
4975
+ // `wall-clock-cap-progress`: the completion path only auto-continues
4976
+ // turn-cap kills, so a wall-clock kill with work to show reaches this
4977
+ // sweep still holding the failed label. Swap it for the queue — the branch
4978
+ // is retained, so the next dispatch reattaches it and briefs a resume from
4979
+ // the recorded work. Killed rows gathered no tracker facts, so the
4980
+ // issue-open guard mirrors the requeue path's. The continuation gate is
4981
+ // the same one the turns path applies at kill time
4982
+ // (`shouldContinueAfterTurnsCap`): the row is already charged, and once
4983
+ // the ceiling is spent, handing back the queue label would offer a
4984
+ // candidate admission can never accept — only the failed label comes off,
4985
+ // and the exhaustion reaches a human (#490, #348).
4986
+ if (cls === "wall-clock-cap-progress") {
4987
+ const state = await tracker.issueState(run.issue).catch(() => undefined);
4988
+ if (state !== "open") {
4989
+ log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
4990
+ return;
4991
+ }
4992
+ const continuation = store.continuationsFor(project.name, run.issue);
4993
+ if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
4994
+ swapToQueue(d, run.issue, project.stateLabels.failed);
4995
+ store.updateRun(run.id, { recoveredAt: Date.now() });
4996
+ log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
4997
+ } else {
4998
+ store.enqueueLabelOps(project.name, [
4999
+ { issue: run.issue, op: "remove", label: project.stateLabels.failed },
5000
+ ]);
5001
+ store.updateRun(run.id, { recoveredAt: Date.now() });
5002
+ await safeEscalate(d, {
5003
+ tier: 1,
5004
+ project: project.name,
5005
+ issue: run.issue,
5006
+ summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
5007
+ detail: [
5008
+ `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5009
+ evidence,
5010
+ `Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
5011
+ "The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
5012
+ ].join("\n"),
5013
+ });
5014
+ await postExhaustionPostmortem(
5015
+ d,
5016
+ run,
5017
+ `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5018
+ );
5019
+ log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
5020
+ }
5021
+ return;
5022
+ }
5023
+
4340
5024
  // `merge-conflict`: the branch is retained and its PR is open, so #50's
4341
5025
  // continuation guard admits it and the next tick briefs a rebase.
4342
5026
  //
@@ -4386,11 +5070,14 @@ async function recoverRun(
4386
5070
  // Same bound for provider-transient: an issue whose stream keeps stalling
4387
5071
  // mid-run is requeued free (no attempt, no continuation charged) — but a
4388
5072
  // provider that aborts three times for one issue is down, and a human has
4389
- // to check its status before hand-requeueing (#220).
5073
+ // to check its status before hand-requeueing (#220). The escalation names
5074
+ // every model the chain tried, so a merged branch built on a different
5075
+ // model is attributable (#286).
4390
5076
  if (
4391
5077
  cls === "provider-transient" &&
4392
5078
  store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
4393
5079
  ) {
5080
+ const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
4394
5081
  await safeEscalate(d, {
4395
5082
  tier: 1,
4396
5083
  project: project.name,
@@ -4398,6 +5085,9 @@ async function recoverRun(
4398
5085
  summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
4399
5086
  detail: [
4400
5087
  `The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
5088
+ ...(tried === ""
5089
+ ? []
5090
+ : [`Models tried: ${tried}.`]),
4401
5091
  "Check provider status before requeueing by hand.",
4402
5092
  ].join("\n"),
4403
5093
  });
@@ -4413,6 +5103,73 @@ async function recoverRun(
4413
5103
  log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
4414
5104
  return;
4415
5105
  }
5106
+ // A clean orphan whose queue label is already absent is a deliberate
5107
+ // withdrawal — the operator took the issue out of the queue (e.g. so a
5108
+ // daemon restart could not re-dispatch work known to be unsafe) — and
5109
+ // recovery must not recreate that intent (#423). Read the live label set;
5110
+ // when the queue label is gone, release the dispatcher-owned in-progress
5111
+ // label and stop, leaving the operator's withdrawal to survive recovery.
5112
+ if (cls === "orphan-clean") {
5113
+ const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
5114
+ if (snapshot === undefined) {
5115
+ log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
5116
+ return;
5117
+ }
5118
+ if (!snapshot.labels.includes(project.queueLabel)) {
5119
+ store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
5120
+ store.updateRun(run.id, { recoveredAt: Date.now() });
5121
+ log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
5122
+ return;
5123
+ }
5124
+ // #439: an orphan-clean row charges the continuation budget, so once
5125
+ // `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
5126
+ // holds the issue on — requeueing re-adds a queue label for a candidate the
5127
+ // dispatcher can never admit. Stop handing it back and hold it instead: the
5128
+ // queue label comes off (so admission never re-holds on every dispatch),
5129
+ // the in-progress label is released, and a single diagnosis escalates once.
5130
+ // `orphan-clean` is deliberately NOT a global exclusion from the budget (a
5131
+ // worker that genuinely keeps dying mid-work must still be bounded); this is
5132
+ // the missing ceiling check this path never had (#348's invariant).
5133
+ const continuations = store.continuationsFor(project.name, run.issue);
5134
+ if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
5135
+ const runs = store.runsForIssue(project.name, run.issue);
5136
+ const breakdown = continuationBreakdown(runs);
5137
+ const artifact = newestContinuableRun(runs);
5138
+ const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
5139
+ const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
5140
+ const artifactLine =
5141
+ artifact === undefined
5142
+ ? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
5143
+ : `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
5144
+ store.enqueueLabelOps(project.name, [
5145
+ { issue: run.issue, op: "remove", label: inProgress },
5146
+ { issue: run.issue, op: "remove", label: project.queueLabel },
5147
+ ]);
5148
+ store.updateRun(run.id, { recoveredAt: Date.now() });
5149
+ await safeEscalate(d, {
5150
+ tier: 1,
5151
+ project: project.name,
5152
+ issue: run.issue,
5153
+ summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
5154
+ detail: [
5155
+ `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5156
+ `Continuations by failure class: ${classLine}.`,
5157
+ onlyDaemonStops
5158
+ ? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
5159
+ : "Continuations span real work — inspect what each attempt left behind before continuing.",
5160
+ artifactLine,
5161
+ "What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
5162
+ ].join("\n"),
5163
+ });
5164
+ await postExhaustionPostmortem(
5165
+ d,
5166
+ run,
5167
+ `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
5168
+ );
5169
+ log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
5170
+ return;
5171
+ }
5172
+ }
4416
5173
  const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
4417
5174
  swapToQueue(d, run.issue, label);
4418
5175
  store.updateRun(run.id, { recoveredAt: Date.now() });
@@ -4431,14 +5188,14 @@ async function recoverRun(
4431
5188
  // Back to pending rather than green: the existing settle sweep re-verifies
4432
5189
  // it against the recorded head on a later tick, so nothing here has to guess
4433
5190
  // whether the re-run passed.
4434
- store.updateRun(run.id, { state: "pushed-pending", lastError: undefined, recoveredAt: Date.now() });
5191
+ store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
4435
5192
  log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
4436
5193
  return;
4437
5194
  }
4438
5195
 
4439
5196
  if (recovery === "escalate") {
4440
5197
  const detail = [evidence];
4441
- if (cls === "turn-cap-spinning") {
5198
+ if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
4442
5199
  const calls = lastToolCalls(run.sessionFile);
4443
5200
  detail.push(
4444
5201
  calls.length === 0
@@ -4669,7 +5426,112 @@ function orchestratorStandingOrders(project: ProjectConfig): {
4669
5426
  };
4670
5427
  }
4671
5428
 
5429
+ /**
5430
+ * Pacing for the resident dispatch loop, interruptible so an operator action —
5431
+ * `resume` — does not wait out the full interval (#380).
5432
+ *
5433
+ * `requestWake` is sticky until a pass absorbs it: while the long sleep is in
5434
+ * flight it is cut short, and when a pass is mid-flight the loop skips the
5435
+ * *following* sleep. Either way the next pass starts without the interval
5436
+ * wait. The single boolean coalesces any number of near-simultaneous wakes
5437
+ * into one prompt pass, and the strictly sequential loop means a wake can
5438
+ * never start a pass on top of one already running.
5439
+ */
5440
+ export interface DispatchPace {
5441
+ /** Ask for a prompt pass; cuts an in-flight sleep short. Coalesces. */
5442
+ requestWake(): void;
5443
+ /** Absorb a pending request — true when the pass that just finished should
5444
+ * skip its sleep. */
5445
+ takeWake(): boolean;
5446
+ /** The long interval sleep; resolves early when {@link requestWake} fires. */
5447
+ sleep(ms: number): Promise<void>;
5448
+ }
5449
+
5450
+ export function createDispatchPace(): DispatchPace {
5451
+ let pendingWake = false;
5452
+ let interruptSleep: (() => void) | undefined;
5453
+ return {
5454
+ requestWake() {
5455
+ pendingWake = true;
5456
+ interruptSleep?.();
5457
+ },
5458
+ takeWake() {
5459
+ const pending = pendingWake;
5460
+ pendingWake = false;
5461
+ return pending;
5462
+ },
5463
+ async sleep(ms) {
5464
+ // A wake that landed in the gap between the loop's takeWake check and
5465
+ // this call is absorbed here: skip the interval, and consume the flag
5466
+ // so the pass that follows it is the only prompt one.
5467
+ if (pendingWake) {
5468
+ pendingWake = false;
5469
+ return;
5470
+ }
5471
+ await new Promise<void>((resolve) => {
5472
+ const timer = setTimeout(resolve, ms);
5473
+ interruptSleep = () => {
5474
+ // The interruption *is* the absorption: the pass that follows the
5475
+ // sleep is the prompt pass this wake asked for, so the loop sleeps
5476
+ // normally after it instead of running a second immediate pass.
5477
+ pendingWake = false;
5478
+ clearTimeout(timer);
5479
+ resolve();
5480
+ };
5481
+ // A wake can land between the loop's takeWake check and this
5482
+ // constructor; resolve immediately instead of letting it wait out
5483
+ // the full interval.
5484
+ if (pendingWake) interruptSleep();
5485
+ });
5486
+ interruptSleep = undefined;
5487
+ },
5488
+ };
5489
+ }
5490
+
5491
+ export interface DispatchLoopOptions {
5492
+ /** One dispatch pass over every configured project runtime. */
5493
+ tick(): Promise<void>;
5494
+ /** True once a stop is requested; checked between passes. */
5495
+ shouldStop(): boolean;
5496
+ pace: DispatchPace;
5497
+ /** The scheduled interval between passes (`TICK_INTERVAL_MS` in production). */
5498
+ tickIntervalMs: number;
5499
+ }
5500
+
5501
+ /**
5502
+ * The resident daemon's dispatch loop: one pass immediately, then a
5503
+ * `tickIntervalMs` sleep the {@link DispatchPace} can cut short. Exported so
5504
+ * the resume-wake behaviour can be pinned deterministically without a live
5505
+ * daemon (#380).
5506
+ */
5507
+ export async function runDispatchLoop(o: DispatchLoopOptions): Promise<void> {
5508
+ const { tick, shouldStop, pace, tickIntervalMs } = o;
5509
+ while (!shouldStop()) {
5510
+ await tick();
5511
+ if (shouldStop()) break;
5512
+ // A wake that landed while the pass was in flight skips the following
5513
+ // sleep: that pass made its admission decisions against pre-resume state,
5514
+ // and the resumed project must not wait a full interval for the next one.
5515
+ if (pace.takeWake()) continue;
5516
+ await pace.sleep(tickIntervalMs);
5517
+ }
5518
+ }
5519
+
4672
5520
  export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
5521
+ // A `--once` tick is still a dispatcher: it settles rows, projects labels,
5522
+ // admits and launches workers, so it must hold the same exclusive daemon/
5523
+ // state ownership as the long-running dispatcher for its whole lifecycle
5524
+ // (#431). This is the one choke point every once tick passes through (both
5525
+ // `daemon --once` and the `setup host` smoke), so ownership is acquired here
5526
+ // — before any settlement, label, admission, worktree or worker mutation —
5527
+ // and refused (having done none of those) while another live daemon owns the
5528
+ // project or another live `--once` holds the lease.
5529
+ const onceLease = o.once ? acquireOnceLease({ port: o.port, project: o.project }) : undefined;
5530
+ if (onceLease !== undefined && !onceLease.ok) {
5531
+ throw new Error(
5532
+ `another daemon is alive (pid ${onceLease.pid}); stop it first — a --once tick refuses to run beside it`,
5533
+ );
5534
+ }
4673
5535
  const cfg = loadConfig();
4674
5536
  const projects = o.project === undefined ? cfg.projects : [findProject(cfg, o.project)];
4675
5537
  const store = openStore(dbPath());
@@ -4824,6 +5686,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4824
5686
  integrity,
4825
5687
  stall: { paged: false },
4826
5688
  cleanup: { next: 0 },
5689
+ probeCriticalBase: (repo, markers, branch) =>
5690
+ probeCriticalBase(project, repo, branch, markers),
4827
5691
  ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
4828
5692
  verbActions,
4829
5693
  };
@@ -4843,6 +5707,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4843
5707
  await workers.drain();
4844
5708
  for (const runtime of runtimes) await runtime.outbox.deliverDue();
4845
5709
  } finally {
5710
+ // Drop exclusivity only while it is still ours; a manager that claimed
5711
+ // the record mid-tick is left alone. A lease acquired but never released
5712
+ // (config/lease failure before this branch) is stale recovery's job.
5713
+ if (onceLease !== undefined && onceLease.ok) onceLease.lease.release();
4846
5714
  for (const runtime of runtimes.toReversed()) {
4847
5715
  await runtime.orchestrator?.dispose();
4848
5716
  await runtime.orchestratorVerbs?.close();
@@ -4891,7 +5759,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4891
5759
  }, REPORT_DELIVERY_INTERVAL_MS);
4892
5760
 
4893
5761
  let stopping = false;
4894
- let wake: (() => void) | undefined;
5762
+ // The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
5763
+ // sleep. A wake request is sticky and coalescing — several resumes in quick
5764
+ // succession produce one prompt pass, never an overlapping one.
5765
+ const pace = createDispatchPace();
4895
5766
  const stop = (): void => {
4896
5767
  if (stopping) return;
4897
5768
  stopping = true;
@@ -4901,7 +5772,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4901
5772
  // wait for (#374).
4902
5773
  drain.draining = true;
4903
5774
  log("shutting down after active workers finish");
4904
- wake?.();
5775
+ pace.requestWake();
4905
5776
  };
4906
5777
  process.on("SIGINT", stop);
4907
5778
  process.on("SIGTERM", stop);
@@ -4921,6 +5792,10 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4921
5792
  })),
4922
5793
  turnLimits,
4923
5794
  workerControls,
5795
+ wake: () => {
5796
+ log("dispatch wake requested — an immediate pass is starting");
5797
+ pace.requestWake();
5798
+ },
4924
5799
  health: () =>
4925
5800
  daemonHealth(
4926
5801
  runtimes.map((runtime) =>
@@ -4942,27 +5817,23 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4942
5817
  );
4943
5818
 
4944
5819
  try {
4945
- while (!stopping) {
4946
- for (const runtime of runtimes) {
4947
- try {
4948
- await tick(runtime.d, workers);
4949
- } catch (err) {
4950
- log(
4951
- `${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
4952
- `tick failed: ${errText(err)}`,
4953
- );
5820
+ await runDispatchLoop({
5821
+ tick: async () => {
5822
+ for (const runtime of runtimes) {
5823
+ try {
5824
+ await tick(runtime.d, workers);
5825
+ } catch (err) {
5826
+ log(
5827
+ `${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` +
5828
+ `tick failed: ${errText(err)}`,
5829
+ );
5830
+ }
4954
5831
  }
4955
- }
4956
- if (stopping) break;
4957
- await new Promise<void>((resolve) => {
4958
- const timer = setTimeout(resolve, TICK_INTERVAL_MS);
4959
- wake = () => {
4960
- clearTimeout(timer);
4961
- resolve();
4962
- };
4963
- });
4964
- wake = undefined;
4965
- }
5832
+ },
5833
+ shouldStop: () => stopping,
5834
+ pace,
5835
+ tickIntervalMs: TICK_INTERVAL_MS,
5836
+ });
4966
5837
  } finally {
4967
5838
  process.off("SIGINT", stop);
4968
5839
  process.off("SIGTERM", stop);