omp-conductor 0.17.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -12,22 +12,27 @@ import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync
12
12
  import { dirname, join, relative } from "node:path";
13
13
  import {
14
14
  configPath,
15
+ dbBackupDirFor,
15
16
  findProject,
16
17
  loadConfig,
17
18
  resolveCaps,
18
19
  resolveReleaseGrants,
20
+ resolveReview,
19
21
  stateDir,
20
22
  } from "./config.ts";
21
23
  import { availabilityState, type AvailabilityState } from "./availability.ts";
22
24
  import {
23
- UNREADABLE_TREE_FLAG,
24
- analyseSettlement,
25
- deriveChangedLine,
26
25
  formatSettlementFlags,
27
26
  settlementFlagSummary,
28
27
  withDerivedChangedLine,
29
28
  } from "./diff-flags.ts";
30
- import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
29
+ import {
30
+ dbSnapshotDue,
31
+ dbSnapshotMarkerKey,
32
+ digestScheduleState,
33
+ localDayKey,
34
+ type DigestScheduleState,
35
+ } from "./digest-schedule.ts";
31
36
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
32
37
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
33
38
  import { graphHint } from "./graph.ts";
@@ -56,29 +61,52 @@ import {
56
61
  type GateShape,
57
62
  type ReleaseBlockContext,
58
63
  } from "./release-policy.ts";
59
- import { branchName, effectiveLabels, route } from "./routing.ts";
64
+ import { branchName, effectiveLabels, isEligible, route } from "./routing.ts";
60
65
  import type { Routed, UnroutableReason } from "./routing.ts";
61
66
  import {
62
67
  admitCandidates,
63
68
  effectiveLane,
64
69
  hasContinuationBudget,
65
70
  hasFailedAttemptBudget,
71
+ laneEcho,
66
72
  } from "./admission.ts";
67
73
  import type { Admission, AdmissionHold, FileLane } from "./admission.ts";
68
74
  import { log, errText, safeEscalate } from "./log.ts";
75
+ import {
76
+ adoptSalvagedPrs,
77
+ classifyAndRecover,
78
+ collectSettlementFlags,
79
+ formatQuarantinedRuns,
80
+ formatSalvagedRuns,
81
+ reactToProviderCredit,
82
+ readSessionError,
83
+ reconcileOrphanedRuns,
84
+ reconcileStaleLabels,
85
+ recordOperatorStop,
86
+ settlePushedGreen,
87
+ settleWorktree,
88
+ swapToQueue,
89
+ } from "./settlement.ts";
69
90
  import { materializeOmpSettings } from "./omp-settings.ts";
70
91
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
71
- import { classifyRun, infraLogSignature, infraSignatureVersion, normalise, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
92
+ import { infraLogSignature, infraSignatureVersion, providerCreditRefusal, providerTransientFault } from "./failure-class.ts";
72
93
  import {
73
94
  DEFAULT_MODEL_FALLBACK_THRESHOLD,
74
95
  fallbackClause,
75
- formatModelsTried,
76
- modelsTried,
77
96
  providerFailureFacts,
78
97
  resolveDispatchModel,
79
98
  } from "./model-fallback.ts";
80
99
  import { projectLabels } from "./label-projection.ts";
81
- import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
100
+ import {
101
+ DB_SNAPSHOT_RETENTION,
102
+ ACTIVE_STATES,
103
+ dbPath,
104
+ LIVE_STATES,
105
+ openStore,
106
+ pruneDbSnapshots,
107
+ snapshotDb,
108
+ utcDay,
109
+ } from "./store.ts";
82
110
  import { GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
83
111
  import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
84
112
  import type {
@@ -86,6 +114,7 @@ import type {
86
114
  BaseHealth,
87
115
  AdmissionHoldReason,
88
116
  Caps,
117
+ ConductorConfig,
89
118
  DispatchSummary,
90
119
  DigestBacklog,
91
120
  Escalation,
@@ -98,11 +127,13 @@ import type {
98
127
  PrState,
99
128
  ProjectConfig,
100
129
  ReadyIssue,
130
+ ReportingPolicy,
101
131
  RepoTarget,
102
132
  ReportRecord,
103
133
  ResolvedGrants,
104
134
  FailureClass,
105
135
  RecoveryAction,
136
+ ReviewPolicy,
106
137
  ReviewRevisionOutcome,
107
138
  ReviewRevisionRecord,
108
139
  RunRecord,
@@ -140,7 +171,6 @@ import { githubVerbActions } from "./verbs/actions.ts";
140
171
  import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
141
172
  import {
142
173
  listenVerbChannel,
143
- PR_LOOKUP_WINDOW_MS,
144
174
  type VerbActions,
145
175
  type VerbDeps,
146
176
  type VerbListener,
@@ -180,26 +210,7 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
180
210
  * report an operator is waiting on must not sit in the outbox for the length of
181
211
  * a poll interval, and delivery is owed even while claiming is paused (#123). */
182
212
  const REPORT_DELIVERY_INTERVAL_MS = 30_000;
183
- /** A dispatch-infra run (turn-0 git failure) is requeued so the next tick
184
- * retries — but only a bounded number of times. Three strikes for one issue
185
- * means the mirror itself is broken, not unlucky, and the sweep escalates
186
- * instead of burning a turn-0 run per tick forever (#168, #177). */
187
- const DISPATCH_INFRA_MAX_STRIKES = 3;
188
- /** A provider-transient requeue (stream stalled mid-run) is retried, but only a
189
- * bounded number of times: three aborted streams for one issue means the
190
- * provider itself is degraded, not unlucky, and the sweep escalates to a
191
- * human instead of requeueing into a down provider forever (#220). */
192
- const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
193
- /** A provider-capacity requeue (sustained in-session rate limiting) is retried,
194
- * but only a bounded number of times: three throttled runs for one issue mean
195
- * the provider is at capacity, not unlucky, and the sweep escalates to a human
196
- * instead of requeueing into a throttled provider forever (#573). The issue's
197
- * own chain moves onto its next model per strike (via {@link FAILOVER_CLASSES}),
198
- * so a bounded chain is exhaustible; this caps the unbounded no-chain case. */
199
- const PROVIDER_CAPACITY_MAX_STRIKES = 3;
200
- /** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
201
- * are maintenance, but a backlog must not turn one tick into an API burst. */
202
- const SALVAGED_PR_ADOPTION_BATCH = 10;
213
+
203
214
  const DEFAULT_PORT = 8787;
204
215
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
205
216
 
@@ -315,6 +326,13 @@ interface Deps {
315
326
  * admission fails a routed cross-repo prerequisite closed.
316
327
  */
317
328
  probeIssueIn?: (repo: string, issue: number) => Promise<IssueSnapshot | undefined>;
329
+ /**
330
+ * Reads one issue's BODY in a repository the admission tracker is not bound
331
+ * to — the dependency-graph cycle pass (#421). Wired by `runDaemon` to a
332
+ * repo-scoped tracker; a test injects a fake. Absent, a routed reachable
333
+ * body fails that branch closed rather than synthesising a cycle.
334
+ */
335
+ probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
318
336
  }
319
337
 
320
338
  /**
@@ -991,6 +1009,20 @@ function renderDiscussion(comments: IssueComment[] | "unread", lane?: FileLane):
991
1009
  return lines.join("\n");
992
1010
  }
993
1011
 
1012
+ /**
1013
+ * The parsed file lane as a brief section (#724): the file list the gate will
1014
+ * enforce, or the explicit fail-open note — the same `laneEcho` the promotion
1015
+ * verb prints, so the author and the worker read one parse. Deliberately only
1016
+ * the parse, never the source line: the prose already renders in the body or
1017
+ * the discussion, and in #720's case the prose is exactly what looked
1018
+ * reasonable to a human while parsed greedily.
1019
+ */
1020
+ function laneBlock(lane: FileLane | undefined): string {
1021
+ const echo = laneEcho(lane);
1022
+ const body = lane === undefined ? `_${echo}_` : `\`${echo}\``;
1023
+ return ["## File lane (as parsed)", "", body, "", ""].join("\n");
1024
+ }
1025
+
994
1026
  /**
995
1027
  * What an orphan-resumed worker is told about the file lane on top of the
996
1028
  * continuation notice (#608). The original brief already in the transcript
@@ -1008,6 +1040,8 @@ function resumeLaneBlock(lane: FileLane): string {
1008
1040
  "",
1009
1041
  lane.source,
1010
1042
  "",
1043
+ `Parsed files: ${laneEcho(lane)}.`,
1044
+ "",
1011
1045
  ].join("\n");
1012
1046
  }
1013
1047
 
@@ -1024,211 +1058,6 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
1024
1058
  { issue, op: "remove", label: from },
1025
1059
  ]);
1026
1060
  }
1027
- /**
1028
- * Persist the operator-stop transition before releasing its live controller.
1029
- * The row is terminal first, then its in-progress label is removed through the
1030
- * same durable projection outbox as every other lifecycle transition.
1031
- */
1032
- export function recordOperatorStop(
1033
- store: Pick<Store, "updateRun" | "enqueueLabelOps">,
1034
- args: {
1035
- project: string;
1036
- issue: number;
1037
- runId: string;
1038
- inProgress: string;
1039
- reason: string;
1040
- patch: Partial<RunRecord>;
1041
- },
1042
- ): void {
1043
- store.updateRun(args.runId, {
1044
- ...args.patch,
1045
- state: "stopped",
1046
- lastError: `operator stopped: ${args.reason}`,
1047
- });
1048
- store.enqueueLabelOps(args.project, [
1049
- { issue: args.issue, op: "remove", label: args.inProgress },
1050
- ]);
1051
- }
1052
-
1053
-
1054
- /**
1055
- * The escalator throws when no transport is configured or Telegram rejects, and
1056
- * only records the dedup marker on success. A page that cannot be delivered
1057
- * must not take the tick down with it — log it and let the next tick retry.
1058
- *
1059
- * Returns whether it actually went out, because "page once" and "page once
1060
- * *successfully*" are different promises: a caller that latches a once-only
1061
- * gate on the attempt turns one failed delivery into permanent silence about a
1062
- * condition that is still true.
1063
- */
1064
-
1065
- async function reactToProviderCredit(
1066
- d: Deps,
1067
- issue: number,
1068
- message: string,
1069
- sessionFile: string | undefined,
1070
- ): Promise<void> {
1071
- const { project } = d;
1072
- const alreadyPaused = isPaused(project.name);
1073
- if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message }, project.name);
1074
- log(
1075
- `#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
1076
- );
1077
- // Fleet-scoped and run-independent on purpose. The notification ledger
1078
- // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
1079
- // carrying no run or attempt pages once for the fleet, not once per run.
1080
- await safeEscalate(d, {
1081
- tier: 2,
1082
- category: "fleet-stopped",
1083
- project: project.name,
1084
- issue: NO_ISSUE,
1085
- summary: `Model provider refused for credit — ${project.name} is paused`,
1086
- detail: [
1087
- message,
1088
- "",
1089
- "No implementation attempt was charged: this is a billing state, not a",
1090
- "failed implementation. Each affected issue keeps its queue label and",
1091
- "re-dispatches on `omp-conductor resume` once the provider has credit.",
1092
- `Session: ${sessionFile ?? "(no transcript)"}`,
1093
- ].join("\n"),
1094
- });
1095
- }
1096
-
1097
- /**
1098
- * What a salvage attempt contributes to the escalation: where the work went, or
1099
- * that it went nowhere. Split from the effects below for the same reason
1100
- * `checkIntegrity` is — this wording is the whole thing a human acts on, so it
1101
- * is worth a test holding it, and the sha in it is the only pointer to work
1102
- * that no longer has any other copy.
1103
- *
1104
- * `retained` is not cosmetic. These lines used to promise a tree "kept for
1105
- * inspection" unconditionally, which was true only because salvage ran only on
1106
- * the paths that keep one. A blocked run's tree is removed the moment its work
1107
- * is safely on the branch, and sending an operator to a path this process just
1108
- * deleted is the same class of mistake as #118 itself.
1109
- */
1110
- export function salvageLines(
1111
- outcome: SalvageOutcome,
1112
- worktree: string,
1113
- retained: boolean,
1114
- ): string[] {
1115
- const fate = retained
1116
- ? `Worktree kept for inspection: ${worktree}`
1117
- : `Worktree removed: ${worktree}`;
1118
-
1119
- if (outcome.kind === "nothing") return [`${fate} — nothing uncommitted to salvage`];
1120
-
1121
- if (outcome.kind === "failed") {
1122
- return [
1123
- `WIP SALVAGE FAILED: ${outcome.error}`,
1124
- `Uncommitted work in ${worktree} is the only copy of it, so the tree was kept.`,
1125
- "This issue is held out of dispatch until the tree is recovered by hand and",
1126
- "`omp-conductor unblock <n> --force` records that you accepted it.",
1127
- ];
1128
- }
1129
-
1130
- const where =
1131
- `WIP committed to ${outcome.branch} @ ${outcome.sha}` +
1132
- (outcome.pushed
1133
- ? " and pushed — the work outlives this worktree"
1134
- : ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`);
1135
- // Manifest belongs in the escalation too: opening the commit is how the
1136
- // orchestrator talked itself into scrubbing a worker tree (#38).
1137
- const n = outcome.files.length;
1138
- const count = `${n} file${n === 1 ? "" : "s"}`;
1139
- const manifest =
1140
- outcome.newPaths.length === 0
1141
- ? `${count} (all modifications to tracked paths)`
1142
- : `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
1143
- outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
1144
- }`;
1145
- return [where, manifest, fate];
1146
- }
1147
-
1148
- /** Everything a settled run has to record and say about its worktree. */
1149
- export interface WorktreeSettlement {
1150
- outcome: SalvageOutcome;
1151
- /** Whether the tree still exists now the run is over. */
1152
- retained: boolean;
1153
- /** Escalation lines naming where the work went. */
1154
- lines: string[];
1155
- /** Row fields recording the durable ref, or the failure that blocks a re-claim. */
1156
- patch: Pick<RunRecord, "salvageSha" | "salvageError">;
1157
- }
1158
-
1159
- /**
1160
- * Decides what becomes of a finished run's worktree: save the work, then keep
1161
- * or remove the tree, then say which.
1162
- *
1163
- * One function because the two halves are one decision and splitting them is
1164
- * how #118 happened — the removal at the end of dispatch had no idea whether
1165
- * anything had been saved, and the salvage at the top of the failure branch had
1166
- * no idea the blocked branch fell through to a `--force` removal.
1167
- *
1168
- * A salvage that *fails* retains the tree whatever the caller asked for. There
1169
- * was real work, git refused to commit it, and the tree is now the only copy in
1170
- * existence: deleting it on schedule would be the data loss this whole path
1171
- * exists to prevent. The issue is held out of dispatch until an operator says
1172
- * otherwise, because the next attempt's `worktree remove --force` would finish
1173
- * the job (see `admitCandidates`).
1174
- *
1175
- * Exported so a test can drive the real decision against a real git tree.
1176
- */
1177
- export async function settleWorktree(
1178
- args: {
1179
- issue: number;
1180
- attempt: number;
1181
- /** Clause for the commit subject: "killed by the turns cap", "blocked …". */
1182
- ending: string;
1183
- worktree: string;
1184
- /** The run's branch, so the pre-removal publish names the right ref. */
1185
- branch: string;
1186
- /**
1187
- * Publishes the run branch on the privileged side. Required rather than
1188
- * optional: the run's commits live in a repository of its own,
1189
- * so a removal that did not publish first would delete the only copy —
1190
- * which is #121's data loss with one extra step. `undefined` is a visible
1191
- * decision at the call site, never an omission.
1192
- */
1193
- publish: RunPublisher | undefined;
1194
- } & (
1195
- | /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
1196
- { tree: "keep" }
1197
- | { tree: "remove"; mirrorPath: string }
1198
- ),
1199
- ): Promise<WorktreeSettlement> {
1200
- const { issue, attempt, ending, worktree, branch, publish } = args;
1201
- const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
1202
- const retained = args.tree === "keep" || outcome.kind === "failed";
1203
- if (!retained && args.tree === "remove") {
1204
- // Before the removal, always — not only when salvage found something. A run
1205
- // that *committed* and could not publish has its work in its own repository
1206
- // and nowhere else, and salvage never sees a committed tree because it is
1207
- // clean. The mirror fetch inside `publish` is what preserves it; the push
1208
- // to GitHub can fail (no network, protected ref) and the work still lives.
1209
- const published = await publish?.(branch);
1210
- if (published !== undefined && !published.ok) {
1211
- log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
1212
- }
1213
- await removeWorktree(args.mirrorPath, worktree);
1214
- }
1215
-
1216
- const lines = salvageLines(outcome, worktree, retained);
1217
- log(`#${issue} salvage: ${lines.join(" ")}`);
1218
- return {
1219
- outcome,
1220
- retained,
1221
- lines,
1222
- patch:
1223
- outcome.kind === "salvaged"
1224
- ? { salvageSha: outcome.sha }
1225
- : outcome.kind === "failed"
1226
- ? { salvageError: outcome.error }
1227
- : {},
1228
- };
1229
- }
1230
-
1231
-
1232
1061
  /**
1233
1062
  * How a run's end is named — in the salvage commit, and to whoever reads it.
1234
1063
  * The whole clause, not a bare reason: a graceful block was not killed by
@@ -1315,6 +1144,11 @@ export async function buildBrief(
1315
1144
  // discussion budget, keeping the gate and the worker-visible brief on one
1316
1145
  // lane (#608).
1317
1146
  const comments = opts.comments ?? [];
1147
+ // The effective lane admission resolved (or resolves) for this candidate:
1148
+ // the carried admission snapshot when dispatch has one, else the thread
1149
+ // itself. Both the discussion renderer and the parsed-lane section draw on
1150
+ // the same value, so the brief shows one lane on every surface (#608, #724).
1151
+ const lane = opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments));
1318
1152
  return renderBrief(template, {
1319
1153
  ISSUE_NUMBER: String(r.issue.number),
1320
1154
  ISSUE_TITLE: r.issue.title,
@@ -1323,19 +1157,18 @@ export async function buildBrief(
1323
1157
  BRANCH: branch,
1324
1158
  WORKTREE: worktree,
1325
1159
  ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
1326
- ISSUE_COMMENTS: renderDiscussion(
1327
- comments,
1328
- opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments)),
1329
- ),
1160
+ ISSUE_COMMENTS: renderDiscussion(comments, lane),
1161
+ FILE_LANE: laneBlock(lane),
1330
1162
  GATES: gatesBlock(r.repo),
1331
1163
  // The guarded shell suites and their sanctioned parse-only check, derived
1332
1164
  // from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
1333
1165
  // (#687). Non-empty whenever the guard has paths, so the placeholder line
1334
1166
  // always renders to a line.
1335
1167
  SHARED_HOST_NOTICE: sharedHostBriefNotice(),
1336
- // Empty for a repo with no `graphProject`, and empty means *nothing*: the
1337
- // placeholder sits flush against the next list item in the template, so an
1338
- // unconfigured render leaves no blank line where a hint would have gone.
1168
+ // The brief's code-graph paragraph: the exact `project` key for a
1169
+ // configured repo, or an explicit "no graph" statement for an
1170
+ // unconfigured one never silence, because a worker that knows there is
1171
+ // no graph stops looking for it.
1339
1172
  GRAPH_HINT: graphHint(r.repo),
1340
1173
  CONTINUATION: continuation,
1341
1174
  });
@@ -1344,178 +1177,6 @@ export async function buildBrief(
1344
1177
  // ------------------------------------------------------------------- one issue
1345
1178
 
1346
1179
 
1347
- /** The failure classes `countContinuations` deliberately does not charge — the
1348
- * inverted copy of its exclusions, kept beside the breakdown that consumes it
1349
- * so the two can only drift together (#439). `orphan-clean` is absent on
1350
- * purpose: daemon orphans consume the continuation budget, which is exactly
1351
- * why the requeue side has to respect the ceiling instead of racing it. */
1352
- const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
1353
- "admin-kill": true,
1354
- "settlement-stuck": true,
1355
- "env-start-failure": true,
1356
- "dispatch-infra": true,
1357
- "provider-credit": true,
1358
- "provider-transient": true,
1359
- "provider-capacity": true,
1360
- };
1361
-
1362
- /** How one issue spent its continuation budget, grouped by failure class —
1363
- * the exact rows `continuationsFor` charges, so an exhaustion escalation
1364
- * reports the same budget it says is spent. `unclassified` groups rows that
1365
- * charged before the class was written (a pre-upgrade NULL). */
1366
- function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
1367
- const perClass = new Map<string, number>();
1368
- for (const r of runs) {
1369
- const chargedAsKilledOrOrphaned =
1370
- (r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
1371
- (r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
1372
- const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
1373
- if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
1374
- const cls = r.failureClass ?? "unclassified";
1375
- perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
1376
- }
1377
- return perClass;
1378
- }
1379
-
1380
- /** The newest attempt's preserved work, if any, so an exhaustion escalation can
1381
- * say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
1382
- * three artifacts a run can leave, and all null on a branch nothing reached. */
1383
- function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
1384
- const newestFirst = [...runs].reverse();
1385
- return newestFirst.find(
1386
- (r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
1387
- );
1388
- }
1389
-
1390
- /** The fenced-block info string that marks an exhaustion postmortem comment, so
1391
- * a grooming scout re-slicing the issue can find and parse the whole block by
1392
- * grepping for it. */
1393
- export const POSTMORTEM_MARKER = "conductor-postmortem";
1394
-
1395
- /** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
1396
- * digest shape for naming what the exhaustion was. Groups every row by its
1397
- * failure class, whether or not it charged the continuation budget, because
1398
- * the postmortem tells the whole story and not just the budget half (#290). */
1399
- export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
1400
- const perClass = new Map<string, number>();
1401
- for (const r of runs) {
1402
- const cls = r.failureClass ?? "unclassified";
1403
- perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
1404
- }
1405
- if (perClass.size === 0) return "unclassified";
1406
- return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
1407
- }
1408
-
1409
- /** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
1410
- export function humanDuration(ms: number): string {
1411
- const seconds = Math.max(0, Math.round(ms / 1_000));
1412
- if (seconds < 60) return `${seconds}s`;
1413
- const minutes = Math.round(seconds / 60);
1414
- if (minutes < 60) return `${minutes}m`;
1415
- const hours = Math.floor(minutes / 60);
1416
- const rest = minutes % 60;
1417
- return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
1418
- }
1419
-
1420
- /** Flatten and bound a run's last error to one greppable table line. */
1421
- export function oneLineBrief(text: string | undefined): string | undefined {
1422
- if (text === undefined || text.trim() === "") return undefined;
1423
- const flat = text.replace(/\s+/g, " ").trim();
1424
- return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
1425
- }
1426
-
1427
- /**
1428
- * The exhaustion postmortem block: one greppable fenced block covering every
1429
- * attempt in the chain — continuation rows included — with per-attempt turns,
1430
- * wall clock, failure class and a one-line last error, the explicit salvage
1431
- * state, the spend total and the transcript paths for local inspection.
1432
- *
1433
- * Pure so the tests hold the shape, not the transport: the writer below owns
1434
- * the once-only guarantee, this owns what "once" looks like.
1435
- */
1436
- export function formatExhaustionPostmortem(args: {
1437
- issue: number;
1438
- runs: readonly RunRecord[];
1439
- reason: string;
1440
- }): string {
1441
- const { issue, runs, reason } = args;
1442
- const artifact = newestContinuableRun(runs);
1443
- const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
1444
- const spend = `$${totalSpend.toFixed(2)}`;
1445
- const attemptLines = runs.map((r) => {
1446
- const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
1447
- const cls = r.failureClass ?? "unclassified";
1448
- const error = oneLineBrief(r.lastError) ?? "—";
1449
- return (
1450
- ` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
1451
- `${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
1452
- );
1453
- });
1454
- const salvage =
1455
- artifact === undefined
1456
- ? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
1457
- : `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
1458
- `${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
1459
- return [
1460
- `\`\`\`${POSTMORTEM_MARKER}`,
1461
- `#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
1462
- reason,
1463
- "",
1464
- `Attempts (${runs.length} total, ${spend} spend):`,
1465
- ...attemptLines,
1466
- salvage,
1467
- `Spend total: ${spend} across ${runs.length} attempts.`,
1468
- "Transcripts:",
1469
- ...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
1470
- "```",
1471
- ].join("\n");
1472
- }
1473
-
1474
- /** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
1475
- function postmortemDedupeKey(project: string, issue: number): string {
1476
- return `${project}:postmortem:${issue}`;
1477
- }
1478
-
1479
- /**
1480
- * The exhaustion postmortem: written exactly once per issue, at the point the
1481
- * continuation budget is spent and the issue is settled toward a human.
1482
- *
1483
- * The comment and the material event are both gated by the store's notification
1484
- * ledger — the same idempotence guard the escalator uses — so re-settling an
1485
- * already-postmortemed issue posts nothing and records nothing. A body-string
1486
- * match on the issue would be the wrong guard: an issue re-scoped and re-run
1487
- * would still carry the old block, and the guarantee asked of this is "decided
1488
- * once", not "deduped against what is already written".
1489
- *
1490
- * The digest must name the exhaustion even if the comment write fails, so the
1491
- * material event is recorded before the write and unconditionally (the ledger
1492
- * is append-only, and the gate above already ran once). The comment failure is
1493
- * logged rather than taking the sweep down with it.
1494
- */
1495
- async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
1496
- const { project, tracker, store } = d;
1497
- const key = postmortemDedupeKey(project.name, run.issue);
1498
- if (store.wasNotified(key)) return;
1499
- const runs = store.runsForIssue(project.name, run.issue);
1500
- const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
1501
- const occurredAt = Date.now();
1502
- store.recordMaterialEvent({
1503
- project: project.name,
1504
- category: "exhaustion",
1505
- summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
1506
- evidence: body,
1507
- occurredAt,
1508
- recordedAt: occurredAt,
1509
- });
1510
- try {
1511
- await tracker.comment(run.issue, body);
1512
- store.markNotified(key);
1513
- log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
1514
- } catch (err) {
1515
- log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
1516
- }
1517
- }
1518
-
1519
1180
  export type ExtendTurnLimitResult =
1520
1181
  | { kind: "extended"; runId: string; maxTurns: number }
1521
1182
  | { kind: "not-increase"; runId: string; maxTurns: number }
@@ -1719,45 +1380,6 @@ export async function verifyPushedGreenClaim(
1719
1380
  };
1720
1381
  }
1721
1382
 
1722
- /** What the settlement audit of one green run produced: the advisory flags
1723
- * (test weakening only — the file-list disclosure is derived, not flagged),
1724
- * whether the diff was cut short, and the `changed:` line composed from the
1725
- * PR's own diff. */
1726
- export interface SettlementAuditResult {
1727
- flags: SettlementFlag[];
1728
- truncated: boolean;
1729
- /** The `changed:` file list derived from the PR's diff, present whenever the
1730
- * diff could be read. Absent means the tree could not be read, and `flags`
1731
- * then carries exactly {@link UNREADABLE_TREE_FLAG}. */
1732
- changedLine?: string;
1733
- }
1734
-
1735
- /**
1736
- * Audit a worker's own account of its work against the pull request it pushed.
1737
- *
1738
- * The thin half of the split #85 established: this fetches, {@link
1739
- * analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
1740
- * shares none of its authority — that function decides a run's state, this one
1741
- * cannot, by construction. It returns evidence and the caller appends it.
1742
- *
1743
- * A diff that cannot be read is a finding (`changed-line-missing`), never an
1744
- * empty flag list: nothing was derived and nothing was checked, and that must
1745
- * not read as a clean bill.
1746
- */
1747
- export async function collectSettlementFlags(
1748
- tracker: Pick<Tracker, "prDiff">,
1749
- claim: { prUrl?: string; issueText: string },
1750
- ): Promise<SettlementAuditResult> {
1751
- if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
1752
- const diff = await tracker.prDiff(claim.prUrl);
1753
- if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
1754
- return {
1755
- flags: analyseSettlement({ issueText: claim.issueText, diff }),
1756
- truncated: diff.truncated,
1757
- changedLine: deriveChangedLine(diff),
1758
- };
1759
- }
1760
-
1761
1383
  /**
1762
1384
  * One attempt at one issue, from claim to terminal state. Everything is inside
1763
1385
  * a single try/catch so that a bad issue costs its own run and nothing else.
@@ -2142,6 +1764,10 @@ export async function handleIssue(
2142
1764
  spendUsd: 0,
2143
1765
  maxTurns: caps.workerMaxTurns,
2144
1766
  startedAt: Date.now(),
1767
+ // #567: the orphan-clean attempt this claim continues, when the verdict
1768
+ // above fired. `undefined` for a fresh dispatch — the store maps that to
1769
+ // NULL, so a fresh row simply never carries the field.
1770
+ resumedFromRunId: resuming?.id,
2145
1771
  });
2146
1772
  // #434: carry a terminal predecessor's open PR onto the continuation row.
2147
1773
  // The claim itself stays synchronous — `handleIssue` claims at the top of
@@ -2432,6 +2058,9 @@ export async function handleIssue(
2432
2058
  ? await collectSettlementFlags(tracker, {
2433
2059
  prUrl: result.prUrl,
2434
2060
  issueText: `${r.issue.title}\n${r.issue.body}`,
2061
+ // The claimed-proof check compares the PR's Verified commands
2062
+ // against what this run's session actually recorded.
2063
+ sessionFile: result.sessionFile,
2435
2064
  })
2436
2065
  : undefined;
2437
2066
  if (result.state === "pushed-green" && audit?.truncated) {
@@ -2618,7 +2247,7 @@ export async function handleIssue(
2618
2247
  });
2619
2248
 
2620
2249
  if (providerCredit !== undefined) {
2621
- await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
2250
+ await reactToProviderCredit({ project: d.project, escalate: (e) => d.escalate(e), isPaused, setPaused }, issue, providerCredit, result.sessionFile);
2622
2251
  swapToQueue(d, issue, inProgress);
2623
2252
  } else if (continueTurns) {
2624
2253
  // Requeue as one ordered pair: the in-progress removal before the
@@ -3528,119 +3157,6 @@ export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRe
3528
3157
  }
3529
3158
 
3530
3159
  // ------------------------------------------------------------------- settlement
3531
-
3532
- /** What a resolved PR turns its `pushed-green` row into. */
3533
- export interface Settlement {
3534
- state: "merged" | "failed";
3535
- /** The log line after `#<n> settled: `, and — for a rejection — the row's own
3536
- * `lastError`, because a `failed` row whose worker succeeded has to say so. */
3537
- reason: string;
3538
- }
3539
-
3540
- /**
3541
- * What one `pushed-green` row becomes now its PR has an answer, or undefined to
3542
- * leave the row exactly as it is.
3543
- *
3544
- * A `pushed-green` row is the only one nothing ever revisited: the worker is
3545
- * finished, `reconcileOrphanedRuns` only settles rows that held a process, and
3546
- * `merged` went unwritten from day one. So they accumulated — three of them on
3547
- * the reference fleet on 2026-08-07, every PR merged and every issue closed,
3548
- * with `/healthz` still reporting three active runs and their issues
3549
- * permanently unclaimable, because the busy set *is* the active set (#18).
3550
- *
3551
- * The mapping, and why each answer is the only honest one:
3552
- *
3553
- * - `merged` — the work landed. That is what `merged` was reserved for.
3554
- * - `closed` — a human read the work and said no. Leaving it `pushed-green`
3555
- * forever is a lie; `failed` records that it did not land and releases the
3556
- * busy guard, so an issue a human re-queues can be attempted again. A row
3557
- * that had reached `pushed-green` or `pushed-pending` is classified
3558
- * `returned-for-revision` at settlement. A review decision asks for another
3559
- * implementation pass, not a failure, so it consumes the continuation budget
3560
- * instead of the failed-attempt budget.
3561
- * - `open`, and undefined — nothing changes. Undefined is "could not tell": a
3562
- * flaky network, a revoked token, a deleted PR. Settling on it would record a
3563
- * merge that never happened, and the next tick asks again for free. An
3564
- * ambiguous answer must never settle a row.
3565
- */
3566
- export function settlementFor(pr: PrState | undefined, prUrl: string): Settlement | undefined {
3567
- if (pr === "merged") return { state: "merged", reason: `${prUrl} merged` };
3568
- if (pr === "closed") return { state: "failed", reason: `${prUrl} closed without merging` };
3569
- return undefined;
3570
- }
3571
-
3572
- /**
3573
- * Records the in-progress label's release as a projection op (#201).
3574
- *
3575
- * Settlement used to write only half of what it knew. On 2026-08-09 that cost
3576
- * the reference fleet two issues in one night: veltro#331 settled to `failed`
3577
- * at 23:53Z once the orchestrator closed veltro#332 unmerged, and veltro#344
3578
- * settled to `merged` at 05:54Z once chad#452 squash-merged. Both rows left the
3579
- * active set correctly; an authoritative `gh issue view` on each afterwards
3580
- * still showed `agent:in-progress` — permanently unclaimable with no supported
3581
- * way back (#18).
3582
- *
3583
- * The outbox makes the row transition and the label one fact again: the
3584
- * removal is enqueued in the same breath as the row is terminalised, the
3585
- * projector applies it with unbounded retry, and while it is pending the
3586
- * eligibility overlay treats the label as already gone. A tracker that refuses
3587
- * the write (403, rate limit) can no longer strand the row — that is `#184`
3588
- * and `#198` closed. No `pushed-*` row is ever written terminal with its
3589
- * label release owed but unrecorded, because enqueueing is a local store write
3590
- * that cannot fail on the tracker.
3591
- *
3592
- * Synchronous. The op is durable the moment this returns.
3593
- */
3594
- export function releaseInProgress(
3595
- d: Pick<Deps, "project" | "store">,
3596
- issue: number,
3597
- why: string,
3598
- ): void {
3599
- const label = d.project.stateLabels.inProgress;
3600
- d.store.enqueueLabelOps(d.project.name, [{ issue, op: "remove", label }]);
3601
- log(`#${issue} released ${label} (queued): ${why}`);
3602
- }
3603
-
3604
- /**
3605
- * Asks the tracker about every `pushed-green` PR and settles the ones that
3606
- * resolved.
3607
- *
3608
- * Effects at the call site, decision in {@link settlementFor} — the same split
3609
- * as `checkIntegrity`/`watchOrchestrator`. Exported like `admitCandidates`
3610
- * rather than kept private, because half of what has to hold is about the sweep
3611
- * and not the mapping: that a row without a PR costs no API call, and that one
3612
- * unreachable PR does not stop the others from settling.
3613
- *
3614
- * The label is released too, which reverses what this function first promised.
3615
- * It used to leave tracker labels alone exactly as {@link reconcileOrphanedRuns}
3616
- * does, reasoning that a merge closes the issue anyway and that deciding what an
3617
- * issue's labels should say next is the orchestrator's drain duty. There turned
3618
- * out to be no such path: on 2026-08-09 two settled rows left their issues
3619
- * carrying `agent:in-progress` forever, with the brief forbidding the
3620
- * orchestrator from touching it and `unblock` declining to (see
3621
- * {@link releaseInProgress}). The row transition and the label are one fact, and
3622
- * writing half of it is the whole of that bug.
3623
- *
3624
- * Releasing it is safe here specifically because of what these rows are. A
3625
- * `pushed-green` or `pushed-pending` row has no process behind it — its worker
3626
- * exited and its worktree is gone — so a terminal answer about its PR proves no
3627
- * worker owns the issue, and the duplicate-dispatch interlock the label exists
3628
- * for is spent. {@link reconcileOrphanedRuns} still leaves labels alone for the
3629
- * opposite reason: an orphaned `running` row is work nobody has read yet. And
3630
- * the brief's rule stays absolute, because this is a daemon-owned write through
3631
- * the same Tracker port the dispatcher claimed the issue with — orphan detection
3632
- * is only trustworthy while every state label on the tracker came from this
3633
- * package.
3634
- *
3635
- * The two writes are ordered label-then-row, and the order is load-bearing. This
3636
- * sweep is the only thing that revisits a `pushed-*` row, so the terminal state
3637
- * is also the row's exit from it: written first, a tracker that then failed on
3638
- * the label would leave `agent:in-progress` with nothing left to retry it — #18
3639
- * exactly, in the last window able to reach it. Writing the label first makes
3640
- * failure cost a repeated `gh` call on the next tick instead, and the row stays
3641
- * in the busy set throughout, so no second worker can be sent at the issue while
3642
- * it waits.
3643
- */
3644
3160
  const BASE_CHECK_BATCH = 20;
3645
3161
  const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
3646
3162
  const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
@@ -3948,220 +3464,6 @@ export async function watchBaseHealth(
3948
3464
  }
3949
3465
  }
3950
3466
 
3951
- export async function settlePushedGreen(
3952
- d: Pick<Deps, "project" | "tracker" | "store">,
3953
- ): Promise<number> {
3954
- const { project, tracker, store } = d;
3955
- // Runs the sweep resolved by terminalising the row. Every terminal write
3956
- // below increments it; the tick attributes it as `settled` on the pass's
3957
- // dispatch record, so a held pass is visible as work done, not just as a
3958
- // clock that moved (#497).
3959
- let settled = 0;
3960
- // Filtered from the active set rather than asked for with a new query: active
3961
- // is live workers plus these, so the list is bounded by the worker cap plus
3962
- // the number of PRs awaiting a merge — a handful, by construction. A fleet
3963
- // where that is not a handful has a merge problem, not a dispatch one.
3964
- const pending = store
3965
- .activeRuns(project.name)
3966
- .filter((r) => r.state === "pushed-green" || r.state === "pushed-pending");
3967
-
3968
- for (const run of pending) {
3969
- // Nothing to ask about. A pushed result requires a PR, so a malformed row
3970
- // must not buy a `gh` call every five minutes forever.
3971
- if (run.prUrl === undefined) continue;
3972
-
3973
- let pr: PrState | undefined;
3974
- try {
3975
- pr = await tracker.prState(run.prUrl);
3976
- } catch (err) {
3977
- // Per row, like admission's held candidate. The GitHub adapter already
3978
- // answers undefined instead of throwing, so this catch is the port's
3979
- // contract rather than that adapter's behaviour — and a tracker that does
3980
- // throw must cost its own row, not the whole sweep.
3981
- log(`#${run.issue} not settled: PR state lookup failed (${errText(err)}) — retrying next tick`);
3982
- continue;
3983
- }
3984
-
3985
- const settlement = settlementFor(pr, run.prUrl);
3986
- if (settlement !== undefined) {
3987
- // A mediated merge enters a second, bounded observation phase. Record the
3988
- // exact merge commit before the row leaves the active set; if GitHub cannot
3989
- // supply it yet, retry this settlement next tick rather than create a
3990
- // merged row whose base result can never be attributed.
3991
- let merged: MergedPrInfo | undefined;
3992
- if (settlement.state === "merged") {
3993
- try {
3994
- merged = await tracker.mergedPrInfo(run.prUrl);
3995
- } catch (err) {
3996
- log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
3997
- continue;
3998
- }
3999
- if (merged === undefined) {
4000
- log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
4001
- continue;
4002
- }
4003
- }
4004
-
4005
- // The label removal and the terminal row are one fact again (#201): the
4006
- // release is enqueued — a durable local write that cannot fail on the
4007
- // tracker — in the same breath as the row is terminalised.
4008
- releaseInProgress(d, run.issue, settlement.reason);
4009
- const patch: Partial<RunRecord> = {
4010
- state: settlement.state,
4011
- endedAt: Date.now(),
4012
- ...(merged === undefined
4013
- ? {}
4014
- : {
4015
- mergeSha: merged.mergeSha,
4016
- baseRef: merged.baseRef,
4017
- baseCheck: "pending",
4018
- }),
4019
- };
4020
- if (settlement.state === "failed") {
4021
- patch.lastError = settlement.reason;
4022
- patch.failureClass = "returned-for-revision";
4023
- patch.recoveryAction = "none";
4024
- }
4025
- store.updateRun(run.id, patch);
4026
- settled += 1;
4027
- log(`#${run.issue} settled: ${settlement.reason}`);
4028
- continue;
4029
- }
4030
-
4031
- if (run.state !== "pushed-pending" || pr !== "open" || run.headSha === undefined) continue;
4032
- let verification;
4033
- try {
4034
- verification = await tracker.verifyPr(run.prUrl, run.headSha);
4035
- } catch (err) {
4036
- log(`#${run.issue} checks not settled (${errText(err)}) — retrying next tick`);
4037
- continue;
4038
- }
4039
- if (verification === undefined) continue;
4040
- if (verification.status === "green") {
4041
- store.updateRun(run.id, { state: "pushed-green", lastError: null });
4042
- log(`#${run.issue} checks settled: ${verification.reason}`);
4043
- } else if (verification.status === "failed") {
4044
- // Equally terminal, so the release is enqueued before the row writes,
4045
- // for the same reason as the settlement branch above (see there). The
4046
- // green branch releases nothing — that row is still awaiting a merge,
4047
- // and its live PR is exactly the work the label must keep guarding.
4048
- releaseInProgress(d, run.issue, verification.reason);
4049
- store.updateRun(run.id, { state: "failed", lastError: verification.reason });
4050
- settled += 1;
4051
- log(`#${run.issue} checks failed: ${verification.reason}`);
4052
- } else {
4053
- store.updateRun(run.id, { lastError: verification.reason });
4054
- }
4055
- }
4056
- return settled;
4057
- }
4058
-
4059
- const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
4060
- failed: true,
4061
- killed: true,
4062
- orphaned: true,
4063
- blocked: true,
4064
- };
4065
-
4066
- /**
4067
- * Reattaches a recovered PR to the terminal run that owns it (#245).
4068
- *
4069
- * A worker can fail before its completion report records `prUrl`, then have its
4070
- * dirty tree committed and pushed by salvage. If that branch already has a PR,
4071
- * the orchestrator otherwise has no policy-compliant path to inspect or merge
4072
- * it: ownership is store-backed. Adoption is deliberately stricter than
4073
- * admission. The tracker query proves the PR closes this run's issue; exact
4074
- * branch and canonical repository matches prove it is this run's recovered
4075
- * work, not an unrelated closer. Missing identity is refusal, never a guess.
4076
- *
4077
- * The newest run per issue is inspected, at most ten per tick and only inside
4078
- * the same 30-day window as mediated PR verbs. The cursor advances through the
4079
- * full eligible set so persistent non-matches cannot starve older recovered
4080
- * work. Successful adoption is idempotent because the row gains `prUrl`;
4081
- * non-matches are logged once per daemon process.
4082
- */
4083
- const rejectedSalvagedPrRuns = new Set<string>();
4084
- const salvagedPrCursor = new Map<string, string>();
4085
-
4086
- export async function adoptSalvagedPrs(
4087
- d: Pick<Deps, "project" | "tracker" | "store">,
4088
- now = Date.now(),
4089
- ): Promise<void> {
4090
- const { project, tracker, store } = d;
4091
- const eligible = store
4092
- .recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
4093
- .filter(
4094
- (run) =>
4095
- ADOPTABLE_PR_STATES[run.state] === true &&
4096
- run.prUrl === undefined &&
4097
- run.branch.trim() !== "",
4098
- );
4099
- const previous = salvagedPrCursor.get(project.name);
4100
- const previousIndex =
4101
- previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
4102
- const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
4103
- const candidates = Array.from(
4104
- { length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
4105
- (_, offset) => eligible[(start + offset) % eligible.length]!,
4106
- );
4107
- const last = candidates.at(-1);
4108
- if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
4109
-
4110
- for (const run of candidates) {
4111
- const repo = project.routing.repos[run.repo];
4112
- const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
4113
- let closers: OpenCloser[];
4114
- try {
4115
- closers = await tracker.openClosersFor(run.issue);
4116
- } catch (err) {
4117
- log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
4118
- continue;
4119
- }
4120
-
4121
- const closer = closers.find(
4122
- (candidate) =>
4123
- candidate.headRefName !== "" &&
4124
- candidate.headRefName === run.branch &&
4125
- repo !== undefined &&
4126
- candidate.repo !== "" &&
4127
- candidate.repo === repoIdentity,
4128
- );
4129
- if (closer === undefined) {
4130
- if (!rejectedSalvagedPrRuns.has(run.id)) {
4131
- const observed = closers[0];
4132
- const reason =
4133
- observed === undefined
4134
- ? "no open closing PR"
4135
- : observed.headRefName === ""
4136
- ? "closer has no head branch identity"
4137
- : observed.headRefName !== run.branch
4138
- ? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
4139
- : observed.repo === ""
4140
- ? "closer has no repository identity"
4141
- : `closer repository ${observed.repo} does not match routed repository`;
4142
- log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
4143
- rejectedSalvagedPrRuns.add(run.id);
4144
- }
4145
- continue;
4146
- }
4147
-
4148
- const flag: SettlementFlag = {
4149
- kind: "pr-adopted",
4150
- file: "(recovery)",
4151
- detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
4152
- };
4153
- store.updateRun(run.id, {
4154
- prUrl: closer.url,
4155
- settlementFlags: [...(run.settlementFlags ?? []), flag],
4156
- });
4157
- rejectedSalvagedPrRuns.delete(run.id);
4158
- log(
4159
- `#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
4160
- (run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
4161
- );
4162
- }
4163
- }
4164
-
4165
3467
  /** Canonical `owner/repo` identity from a configured network clone URL. */
4166
3468
  function githubRepo(cloneUrl: string): string | undefined {
4167
3469
  const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
@@ -4189,7 +3491,7 @@ type CleanupRetainedWorktree = (
4189
3491
  * dirty or uniquely unpushed work.
4190
3492
  */
4191
3493
  export async function cleanupRetainedRuns(
4192
- d: Pick<Deps, "project" | "tracker" | "store">,
3494
+ d: Pick<Deps, "project" | "tracker" | "store" | "escalate">,
4193
3495
  queuedIssues: ReadonlySet<number>,
4194
3496
  cursor: RetainedCleanupCursor,
4195
3497
  cleanup: CleanupRetainedWorktree = cleanupRetainedWorktree,
@@ -4254,9 +3556,35 @@ export async function cleanupRetainedRuns(
4254
3556
 
4255
3557
  const outcome = await cleanup(mirrorPathFor(repo, project.mirrorRoot), run.worktree, run.branch);
4256
3558
  if (outcome.kind === "removed") {
4257
- store.updateRun(run.id, { worktree: "" });
3559
+ store.updateRun(run.id, {
3560
+ worktree: "",
3561
+ ...(run.quarantineDetail === undefined ? {} : { quarantineDetail: null }),
3562
+ });
4258
3563
  log(`#${run.issue} retained worktree reaped: ${run.worktree} (${run.branch})`);
3564
+ } else if (outcome.reason === "quarantined") {
3565
+ // A tree whose object store cannot be made sound is potentially
3566
+ // stranded work: the daemon refuses to fetch into it, so its commits
3567
+ // cannot be verified against any remote. The row records the condition
3568
+ // so the status snapshot can name the tree, and the escalation ledger
3569
+ // dedupes on the stable summary below — a pass that keeps seeing the
3570
+ // same broken tree reports it once, never once per dispatch pass.
3571
+ store.updateRun(run.id, { quarantineDetail: outcome.detail });
3572
+ await safeEscalate(d, {
3573
+ tier: 1,
3574
+ project: project.name,
3575
+ issue: run.issue,
3576
+ summary: `#${run.issue} quarantined retained worktree — potentially stranded work`,
3577
+ detail: `${run.worktree} (${run.branch})\n${outcome.detail}`,
3578
+ });
4259
3579
  } else {
3580
+ // Any other retained reason means the tree is back under ordinary
3581
+ // retention: its alternates were repaired (or never needed it), so the
3582
+ // quarantine — if one was marked — is over and the snapshot must not
3583
+ // keep naming it as quarantined.
3584
+ if (run.quarantineDetail !== undefined) {
3585
+ store.updateRun(run.id, { quarantineDetail: null });
3586
+ log(`#${run.issue} retained worktree no longer quarantined: ${outcome.detail}`);
3587
+ }
4260
3588
  log(`#${run.issue} retained worktree kept (${outcome.reason}): ${outcome.detail}`);
4261
3589
  }
4262
3590
  }
@@ -4470,14 +3798,71 @@ async function verifyPendingUpgradeTick(
4470
3798
  }
4471
3799
  }
4472
3800
 
3801
+ /**
3802
+ * The daemon's conductor.db snapshot cadence (#289) — the ledger's only
3803
+ * durable copy, taken on the digest-aligned {@link dbSnapshotDue} window:
3804
+ * once per local day, at/after the digest's configured `at` when it has one.
3805
+ *
3806
+ * Host-global, like the store itself: one snapshot per day for every project,
3807
+ * so whichever project's daemon wins the day first publishes it and the
3808
+ * others no-op on the same durable marker. The marker is written through the
3809
+ * store's existing notification ledger (a one-row idempotence guard, the same
3810
+ * primitive escalations use not to act twice) *after* the snapshot is
3811
+ * published — a crash between publish and mark re-snapshots the next tick
3812
+ * instead of skipping the day, and a restored store that predates today's
3813
+ * marker takes a fresh snapshot on the next tick.
3814
+ *
3815
+ * Returns true when a snapshot was published. A store that does not exist
3816
+ * yet is a silent no-op: there is nothing to back up, and `doctor`'s
3817
+ * `db-backup` probe already treats that as a pass.
3818
+ */
3819
+ export function runDbSnapshotCadence(args: {
3820
+ digestPolicy: ReportingPolicy["digest"];
3821
+ store: Pick<Store, "wasNotified" | "markNotified">;
3822
+ source: string;
3823
+ backupDir: string;
3824
+ now?: number;
3825
+ keep?: number;
3826
+ log?: (line: string) => void;
3827
+ }): boolean {
3828
+ const { digestPolicy, store, source, backupDir } = args;
3829
+ // No store yet (first boot before any run persisted) → nothing to back up;
3830
+ // `doctor`'s `db-backup` probe already treats that as a pass.
3831
+ if (!existsSync(source)) return false;
3832
+ const now = args.now ?? Date.now();
3833
+ const today = localDayKey(now, digestPolicy.timezone);
3834
+ const alreadyToday = store.wasNotified(dbSnapshotMarkerKey(today));
3835
+ if (!dbSnapshotDue({ digest: digestPolicy }, alreadyToday ? today : undefined, now)) {
3836
+ return false;
3837
+ }
3838
+ const published = snapshotDb(source, backupDir, now);
3839
+ (args.log ?? log)(`conductor.db snapshot published: ${published}`);
3840
+ store.markNotified(dbSnapshotMarkerKey(today));
3841
+ try {
3842
+ const removed = pruneDbSnapshots(backupDir, args.keep ?? DB_SNAPSHOT_RETENTION);
3843
+ if (removed > 0) {
3844
+ (args.log ?? log)(`db snapshot retention pruned ${removed} file(s) beyond the retained ${args.keep ?? DB_SNAPSHOT_RETENTION}`);
3845
+ }
3846
+ } catch (err) {
3847
+ // The snapshot is already published and marked; an over-bound directory
3848
+ // costs disk until the next due day retries the prune, never the backup.
3849
+ (args.log ?? log)(`db snapshot retention prune failed: ${errText(err)}`);
3850
+ }
3851
+ return true;
3852
+ }
3853
+
4473
3854
  export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4474
3855
  // A config edit takes effect on the next tick, not the next daemon restart
4475
3856
  // (#170). Re-resolve the project and its caps at the tick boundary so a tick
4476
3857
  // and every run it admits see one consistent snapshot; a failed read keeps
4477
3858
  // the boot values rather than wedging the tick, and the next tick tries
4478
- // again.
3859
+ // again. The same reloaded config feeds the db-snapshot backup dir: when it
3860
+ // is unreadable the cadence falls back to the state-root default, matching
3861
+ // `doctor`'s probe of an absent field.
3862
+ let reloadedConfig: ConductorConfig | undefined;
4479
3863
  try {
4480
3864
  const cfg = loadConfig();
3865
+ reloadedConfig = cfg;
4481
3866
  const fresh = findProject(cfg, d.project.name);
4482
3867
  const freshCaps = resolveCaps(fresh, cfg.defaults);
4483
3868
  if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
@@ -4559,6 +3944,25 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4559
3944
  log(`salvaged PR adoption sweep failed: ${errText(err)}`);
4560
3945
  }
4561
3946
 
3947
+ // The conductor.db snapshot cadence (#289) is durability, not dispatch:
3948
+ // the verb ledger, the decision rows and the run history have no other
3949
+ // copy, so they are snapshotted once per day on the digest-aligned window.
3950
+ // Above the pause gate on purpose — a parked fleet still accumulates ledger
3951
+ // rows and still deserves a backup — and a failure costs the day's snapshot
3952
+ // placeholder, never the tick: failed cadence steps are logged and retried
3953
+ // by the five-minute loop, exactly like the sweeps above it.
3954
+ try {
3955
+ runDbSnapshotCadence({
3956
+ digestPolicy: (d.project.reporting ?? DEFAULT_REPORT_POLICY).digest,
3957
+ store: d.store,
3958
+ source: dbPath(),
3959
+ backupDir: dbBackupDirFor(reloadedConfig),
3960
+ log,
3961
+ });
3962
+ } catch (err) {
3963
+ log(`db snapshot cadence failed: ${errText(err)}`);
3964
+ }
3965
+
4562
3966
  // Immediately after settlement and before any routing, so a class is on the
4563
3967
  // row before the next dispatch decision reads its budgets (#132). Above the
4564
3968
  // pause gate deliberately: classification and label reconciliation are
@@ -4568,7 +3972,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4568
3972
  // Each is guarded on its own: a tracker that fails mid-classification must not
4569
3973
  // stop the label reconcile, and neither may stop the tick.
4570
3974
  try {
4571
- settled += await classifyAndRecover(d);
3975
+ settled += await classifyAndRecover({
3976
+ project: d.project,
3977
+ caps: d.caps,
3978
+ tracker: d.tracker,
3979
+ store: d.store,
3980
+ escalate: (e) => d.escalate(e),
3981
+ isPaused,
3982
+ setPaused,
3983
+ });
4572
3984
  } catch (err) {
4573
3985
  log(`classification sweep failed: ${errText(err)}`);
4574
3986
  }
@@ -4727,14 +4139,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4727
4139
  return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
4728
4140
  });
4729
4141
  const { routed, unroutable } = route(effective, project);
4730
- // Ready issues that route() dropped without a reason carry a state label
4731
- // claimed by a live or settling run — so the claimed count is subtraction
4732
- // rather than a routed-state signal (#228).
4733
- const claimed = effective.length - routed.length - unroutable.length;
4734
- const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
4735
- issue: u.issue.number,
4736
- reason: `unroutable:${u.reason}`,
4737
- }));
4142
+ // route() drops a candidate for two reasons, only one of which is a claim
4143
+ // question. A lifecycle state label (agent:in-progress/blocked/failed)
4144
+ // marks a run-owned issue: it is genuinely in flight while its newest run
4145
+ // is live or settling, and a terminal newest run — or no run at all — means
4146
+ // the label is residual, the footprint of a settled run nobody cleared,
4147
+ // which is Duty 1 reconciliation work, not occupied capacity. A missing
4148
+ // queue label instead marks an issue the outbox is withdrawing (a pending
4149
+ // queue-label removal, e.g. releaseQueueLabel after a merge the PR did not
4150
+ // close), which belongs in none of the three populations. So `claimed` is
4151
+ // defined from actual ownership over genuinely lifecycle-labelled candidates
4152
+ // rather than as the residual of routing (#228, #611).
4153
+ const stateLabels = new Set(Object.values(project.stateLabels));
4154
+ const dropped = effective.filter(
4155
+ (issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
4156
+ );
4157
+ let claimed = 0;
4158
+ const lifecycleHolds: AdmissionHold[] = [];
4159
+ for (const issue of dropped) {
4160
+ const newest = store.latestRun(project.name, issue.number);
4161
+ if (newest !== undefined && ACTIVE_STATES.includes(newest.state)) {
4162
+ claimed += 1;
4163
+ } else {
4164
+ lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
4165
+ }
4166
+ }
4167
+ const routingHolds: AdmissionHold[] = [
4168
+ ...unroutable.map(
4169
+ (u): AdmissionHold => ({ issue: u.issue.number, reason: `unroutable:${u.reason}` }),
4170
+ ),
4171
+ ...lifecycleHolds,
4172
+ ];
4738
4173
  const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
4739
4174
  store.recordDispatch(
4740
4175
  project.name,
@@ -5225,6 +4660,14 @@ export interface StatusSnapshot {
5225
4660
  * the config or the brief.
5226
4661
  */
5227
4662
  releaseGrants: ResolvedGrants;
4663
+ /**
4664
+ * The effective review policy (#678). On the snapshot for the reason
4665
+ * `releaseGrants` is: the orchestrator's Duty 1 has to act on it every
4666
+ * tick, and a stale level or ceiling sitting only in a config file nobody
4667
+ * opens is exactly the drift this field exists to surface. The loader always
4668
+ * materialises it, so this is never absent on a real daemon.
4669
+ */
4670
+ review: ReviewPolicy;
5228
4671
  /** Occupied issues: live workers plus green PRs awaiting a human merge. */
5229
4672
  activeRuns: RunRecord[];
5230
4673
  /**
@@ -5239,6 +4682,11 @@ export interface StatusSnapshot {
5239
4682
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
5240
4683
  * only copy of work the daemon could not save. */
5241
4684
  salvagedRuns: RunRecord[];
4685
+ /** Retained trees whose object store could not be made sound, so the daemon
4686
+ * refused to fetch into them and their commits cannot be verified against
4687
+ * any remote (#737). Distinguished from ordinary retention on purpose —
4688
+ * quarantine is "potentially stranded work", not routine housekeeping. */
4689
+ quarantinedRuns: RunRecord[];
5242
4690
  /** One-shot issue ceilings waiting for the next claim. */
5243
4691
  turnOverrides: TurnOverride[];
5244
4692
  /** Reports the operator has not provably received: pending, in-flight with an
@@ -5342,9 +4790,11 @@ export function statusSnapshotFromStore(
5342
4790
  digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
5343
4791
  caps,
5344
4792
  releaseGrants: resolveReleaseGrants(p),
4793
+ review: resolveReview(p),
5345
4794
  activeRuns: store.activeRuns(p.name),
5346
4795
  reviewRounds,
5347
4796
  salvagedRuns: store.salvagedRuns(p.name),
4797
+ quarantinedRuns: store.quarantinedRuns(p.name),
5348
4798
  turnOverrides: store.listTurnOverrides(p.name),
5349
4799
  openReports: store.openReports(p.name),
5350
4800
  digestBacklog: store.digestBacklog(p.name),
@@ -5418,32 +4868,6 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
5418
4868
  return lines.join("\n");
5419
4869
  }
5420
4870
 
5421
- /**
5422
- * The WIP block: every issue whose newest attempt left work behind, and
5423
- * whether that work is safe.
5424
- *
5425
- * Blocked runs used to be invisible here, which is exactly how #118 stayed
5426
- * invisible for a full attempt cycle — the operator saw a blocked issue and had
5427
- * no way to tell "stopped with 34 uncommitted files" from "stopped clean".
5428
- * A preserved line is informational; an UNSALVAGED line is an alarm, and it
5429
- * names the directory because that directory is the work.
5430
- */
5431
- export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
5432
- if (runs.length === 0) return [];
5433
- const lines = ["", "wip"];
5434
- for (const r of runs) {
5435
- lines.push(
5436
- r.salvageError !== undefined && r.salvageAckAt === undefined
5437
- ? ` #${r.issue} UNSALVAGED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
5438
- `only copy, dispatch held (${r.salvageError})`
5439
- : r.salvageError !== undefined
5440
- ? ` #${r.issue} accepted as lost attempt ${r.attempt} (${r.salvageError})`
5441
- : ` #${r.issue} preserved ${r.salvageSha ?? "?"} on ${r.branch} (attempt ${r.attempt}, ${r.state})`,
5442
- );
5443
- }
5444
- return lines;
5445
- }
5446
-
5447
4871
  /**
5448
4872
  * The grant table, named shape by shape.
5449
4873
  *
@@ -5461,6 +4885,7 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
5461
4885
  ...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
5462
4886
  ];
5463
4887
  }
4888
+
5464
4889
  export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
5465
4890
  return rows.map((row) => {
5466
4891
  const head = row.headSha.slice(0, 8);
@@ -5550,6 +4975,7 @@ export function formatStatus(s: StatusSnapshot): string {
5550
4975
  lines.push(...formatBaseHealth(s.baseHealth));
5551
4976
  lines.push(...formatFreezes(s.freezes));
5552
4977
  lines.push(...formatSalvagedRuns(s.salvagedRuns));
4978
+ lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
5553
4979
  lines.push(...formatOpenReports(s.openReports));
5554
4980
  lines.push(...formatVerbLedger(s.verbLedger));
5555
4981
  // Deploy hint: a restart while workers are live orphans them (salvage runs
@@ -5623,61 +5049,6 @@ export function prepareConductor(project?: string): void {
5623
5049
  setPaused(true, { source: "setup" }, project);
5624
5050
  }
5625
5051
 
5626
- /** Bounded per tick: each row costs tracker calls to gather facts for. */
5627
- const CLASSIFY_BATCH = 20;
5628
-
5629
- /** Tool calls quoted as evidence for a run that spun to its turn cap. */
5630
- const SPIN_EVIDENCE_CALLS = 10;
5631
-
5632
- /**
5633
- * The last few tool names a transcript recorded, newest last.
5634
- *
5635
- * `turn-cap-spinning` escalates rather than requeueing, and the acceptance
5636
- * criterion is that the escalation carries evidence of what the worker was doing
5637
- * when it hit the cap — otherwise the orchestrator opens the transcript and
5638
- * re-derives it, which is the manual triage this whole sweep removes.
5639
- */
5640
- export function lastToolCalls(sessionFile: string | undefined, limit = SPIN_EVIDENCE_CALLS): string[] {
5641
- if (sessionFile === undefined) return [];
5642
- let text: string;
5643
- try {
5644
- text = readFileSync(sessionFile, "utf8");
5645
- } catch {
5646
- return [];
5647
- }
5648
- const names: string[] = [];
5649
- for (const line of text.split("\n")) {
5650
- if (line.length === 0) continue;
5651
- let row: unknown;
5652
- try {
5653
- row = JSON.parse(line) as unknown;
5654
- } catch {
5655
- continue;
5656
- }
5657
- if (row === null || typeof row !== "object") continue;
5658
- const rec = row as { readonly [key: string]: unknown };
5659
- // Both shapes the harness has written: a top-level tool event, and a tool
5660
- // block inside an assistant message.
5661
- const direct = rec["toolName"];
5662
- if (typeof direct === "string") {
5663
- names.push(direct);
5664
- continue;
5665
- }
5666
- const message = rec["message"];
5667
- if (message === null || typeof message !== "object") continue;
5668
- const content = (message as { readonly [key: string]: unknown })["content"];
5669
- if (!Array.isArray(content)) continue;
5670
- for (const part of content) {
5671
- if (part === null || typeof part !== "object") continue;
5672
- const p = part as { readonly [key: string]: unknown };
5673
- if (p["type"] !== "tool_use") continue;
5674
- const name = p["name"];
5675
- if (typeof name === "string") names.push(name);
5676
- }
5677
- }
5678
- return names.slice(-limit);
5679
- }
5680
-
5681
5052
  /** A provider refusal a session recorded before dying, or undefined for none. */
5682
5053
  export interface SessionError {
5683
5054
  status?: number;
@@ -5693,138 +5064,6 @@ export function completionLastError(
5693
5064
  return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
5694
5065
  }
5695
5066
 
5696
- /**
5697
- * The last error a transcript recorded, or undefined when it recorded none.
5698
- *
5699
- * The harness writes `{"stopReason":"error","errorStatus":402,"errorId":402,
5700
- * "errorMessage":"402 This request requires more credits, ..."}`. The daemon
5701
- * read none of it, so three runs died `unknown` with an empty `lastError` and
5702
- * charged an attempt each for a billing state (#220).
5703
- *
5704
- * Scanned newest-first: a session that recovered from an early error and then
5705
- * died of something else must report the something else, and a session that
5706
- * recovered from its only error and finished cleanly reports the error anyway
5707
- * because there is no terminal verdict to outrank it (#220).
5708
- */
5709
- export function readSessionError(sessionFile: string | undefined): SessionError | undefined {
5710
- if (sessionFile === undefined) return undefined;
5711
- let text: string;
5712
- try {
5713
- text = readFileSync(sessionFile, "utf8");
5714
- } catch {
5715
- return undefined;
5716
- }
5717
- const lines = text.split("\n");
5718
- for (let i = lines.length - 1; i >= 0; i -= 1) {
5719
- const line = lines[i];
5720
- if (line === undefined || line.length === 0) continue;
5721
- let row: unknown;
5722
- try {
5723
- row = JSON.parse(line) as unknown;
5724
- } catch {
5725
- continue;
5726
- }
5727
- if (row === null || typeof row !== "object") continue;
5728
- const rec = row as { readonly [key: string]: unknown };
5729
- if (rec["stopReason"] !== "error") continue;
5730
- const message = rec["errorMessage"];
5731
- if (typeof message !== "string" || message.trim() === "") continue;
5732
- const status = rec["errorStatus"];
5733
- return {
5734
- ...(typeof status === "number" && Number.isFinite(status) ? { status } : {}),
5735
- message: message.trim(),
5736
- };
5737
- }
5738
- return undefined;
5739
- }
5740
-
5741
- /**
5742
- * Classify every unclassified terminal run, persist the verdict, and perform the
5743
- * one recovery its class names (#132).
5744
- *
5745
- * Half this fleet's spend produced no merged PR, and every one of those runs
5746
- * ended at a human who re-derived the same triage by hand and then threw the
5747
- * conclusion away. The mechanical classes — a cancelled runner, a kill from a
5748
- * daemon restart, a green PR whose base moved, a row whose PR had already merged
5749
- * — need no judgement at all; the genuinely human ones are worth a person's
5750
- * attention only if they arrive with their evidence already gathered.
5751
- *
5752
- * Facts are fetched per row and only the ones that row needs: a `killed` row
5753
- * costs nothing, a `failed` row with a PR costs a state read and a check read.
5754
- * A `pushed-green` row that classifies to nothing is left completely untouched —
5755
- * it is healthy, and writing a class onto it would take it out of this sweep for
5756
- * good.
5757
- */
5758
- export async function classifyAndRecover(d: Deps): Promise<number> {
5759
- const { project, caps, tracker, store } = d;
5760
- // Settle recoveries, counted for the pass's dispatch record — a row whose PR
5761
- // merged is settled here when the settle sweep could not establish identity
5762
- // (#497). Every other exit returns 0.
5763
- let settled = 0;
5764
- for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
5765
- const facts: ClassifyFacts = {};
5766
- let classifiedRun = run;
5767
- if (run.state === "failed" || run.state === "killed") {
5768
- const sessionError = readSessionError(run.sessionFile);
5769
- if (sessionError !== undefined) {
5770
- if (run.lastError === undefined || run.lastError === sessionError.message) {
5771
- facts.sessionError = sessionError;
5772
- }
5773
- if (run.lastError === undefined) {
5774
- store.updateRun(run.id, { lastError: sessionError.message });
5775
- classifiedRun = { ...run, lastError: sessionError.message };
5776
- }
5777
- }
5778
- }
5779
- try {
5780
- if (run.prUrl !== undefined) {
5781
- const pr = await tracker.prState(run.prUrl);
5782
- if (pr !== undefined) facts.pr = pr;
5783
- if (run.state === "pushed-green" && facts.pr === "open") {
5784
- facts.mergeable = await tracker.mergeable(run.prUrl);
5785
- }
5786
- if (run.state === "failed" && facts.pr === "open") {
5787
- facts.checks = await tracker.checkConclusions(run.prUrl);
5788
- // When a check failed with a reachable log, pull its tail so the
5789
- // table can tell an infra outage (#177) from a real test failure by
5790
- // the log's own words. First failure wins; a log that cannot be
5791
- // fetched is left undefined and classification stays conservative.
5792
- // GitHub reports check states as `FAILURE` while the classifier reads
5793
- // them lowercased — normalise so the live seam and the table agree on
5794
- // which check is the failure whose log we pull (#177).
5795
- const firstFailure = facts.checks.find((c) => normalise(c.state) === "failure" && c.link !== undefined);
5796
- if (firstFailure?.link !== undefined) {
5797
- facts.failingLog = await tracker.checkLog(firstFailure.link);
5798
- }
5799
- }
5800
- }
5801
- } catch (err) {
5802
- // Per row, like every other sweep here: one unreachable PR must not stop
5803
- // the rest from being classified. The next tick asks again for free.
5804
- log(`#${run.issue} not classified: fact gathering failed (${errText(err)}) — retrying next tick`);
5805
- continue;
5806
- }
5807
-
5808
- const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
5809
-
5810
- // A healthy green PR is not a failure of any class. Leaving the row
5811
- // unclassified is what keeps it eligible for the sweep on the tick where its
5812
- // base does move under it.
5813
- if (run.state === "pushed-green" && cls === "unknown") continue;
5814
-
5815
- const retry = run.failureClass !== undefined;
5816
- store.updateRun(run.id, { failureClass: cls, recoveryAction: recovery });
5817
- log(
5818
- retry
5819
- ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
5820
- : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
5821
- );
5822
- if (recovery === "settle") settled += 1;
5823
- await recoverRun(d, classifiedRun, cls, recovery, evidence);
5824
- }
5825
- return settled;
5826
- }
5827
-
5828
5067
  /**
5829
5068
  * How many settled `ci-deterministic` rows one reconciliation pass may
5830
5069
  * re-examine beyond the persisted review cursor. Each candidate costs GitHub
@@ -5973,479 +5212,6 @@ async function historicalInfraEvidence(d: Deps, run: RunRecord): Promise<string[
5973
5212
  return chunks;
5974
5213
  }
5975
5214
 
5976
- /** Performs the one action a class names. Never chooses one of its own. */
5977
- async function recoverRun(
5978
- d: Deps,
5979
- run: RunRecord,
5980
- cls: FailureClass,
5981
- recovery: RecoveryAction,
5982
- evidence: string,
5983
- ): Promise<void> {
5984
- const { project, caps, tracker, store } = d;
5985
- const inProgress = project.stateLabels.inProgress;
5986
-
5987
- if (recovery === "settle") {
5988
- // Enqueue the release with the terminal write (see `settlePushedGreen`):
5989
- // the outbox keeps the label and the row one fact, so a tracker refusal
5990
- // can no longer strand `agent:in-progress` with nothing left to retry it
5991
- // (#18, #201).
5992
- releaseInProgress(d, run.issue, `PR merged: ${evidence}`);
5993
- store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
5994
- log(`#${run.issue} settled from ${cls}: ${evidence}`);
5995
- return;
5996
- }
5997
-
5998
- if (recovery === "continue") {
5999
- // Two classes recover by continuing, and only one of them has anything left
6000
- // to do here.
6001
- //
6002
- // `turn-cap-progress` was already handed back by the completion path, which
6003
- // swapped its labels and left the branch retained. There is nothing to
6004
- // perform, and writing anything would be actively wrong: overwriting
6005
- // `lastError` with a rebase brief tells the continuation worker to rebase a
6006
- // run that simply ran out of turns, and re-swapping labels the completion
6007
- // path already swapped is a pair of no-op `gh` calls. Record-only, so the
6008
- // sweep stops re-offering it.
6009
- if (cls === "turn-cap-progress") {
6010
- store.updateRun(run.id, { recoveredAt: Date.now() });
6011
- log(`#${run.issue} already continuing from ${cls}: ${evidence}`);
6012
- return;
6013
- }
6014
-
6015
- // `wall-clock-cap-progress`: the completion path only auto-continues
6016
- // turn-cap kills, so a wall-clock kill with work to show reaches this
6017
- // sweep still holding the failed label. Swap it for the queue — the branch
6018
- // is retained, so the next dispatch reattaches it and briefs a resume from
6019
- // the recorded work. Killed rows gathered no tracker facts, so the
6020
- // issue-open guard mirrors the requeue path's. The continuation gate is
6021
- // the same one the turns path applies at kill time
6022
- // (`shouldContinueAfterTurnsCap`): the row is already charged, and once
6023
- // the ceiling is spent, handing back the queue label would offer a
6024
- // candidate admission can never accept — only the failed label comes off,
6025
- // and the exhaustion reaches a human (#490, #348).
6026
- if (cls === "wall-clock-cap-progress") {
6027
- const state = await tracker.issueState(run.issue).catch(() => undefined);
6028
- if (state !== "open") {
6029
- log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
6030
- return;
6031
- }
6032
- const continuation = store.continuationsFor(project.name, run.issue);
6033
- if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
6034
- swapToQueue(d, run.issue, project.stateLabels.failed);
6035
- store.updateRun(run.id, { recoveredAt: Date.now() });
6036
- log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
6037
- } else {
6038
- store.enqueueLabelOps(project.name, [
6039
- { issue: run.issue, op: "remove", label: project.stateLabels.failed },
6040
- ]);
6041
- store.updateRun(run.id, { recoveredAt: Date.now() });
6042
- await safeEscalate(d, {
6043
- tier: 1,
6044
- project: project.name,
6045
- issue: run.issue,
6046
- summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
6047
- detail: [
6048
- `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
6049
- evidence,
6050
- `Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
6051
- "The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
6052
- ].join("\n"),
6053
- });
6054
- await postExhaustionPostmortem(
6055
- d,
6056
- run,
6057
- `Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
6058
- );
6059
- log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
6060
- }
6061
- return;
6062
- }
6063
-
6064
- // `merge-conflict`: the branch is retained and its PR is open, so #50's
6065
- // continuation guard admits it and the next tick briefs a rebase.
6066
- //
6067
- // The outbox makes the retry contract one-sided: the swap is enqueued — a
6068
- // durable local write that cannot fail on the tracker — before
6069
- // `recoveredAt` is written, so the row can never again be taken out of
6070
- // `runsNeedingClassification` with its label swap still owed. That was
6071
- // the defect 0.4.4 claimed to have fixed and did not, for this one
6072
- // recovery; the projector retries until the tracker takes the swap, and
6073
- // while it is pending the eligibility overlay keeps the issue coherent
6074
- // (#201).
6075
- swapToQueue(d, run.issue, inProgress);
6076
- store.updateRun(run.id, {
6077
- state: "killed",
6078
- lastError:
6079
- "merge-conflict: base moved under a green PR — rebase, regenerate recorded artifacts, push without force",
6080
- recoveredAt: Date.now(),
6081
- });
6082
- log(`#${run.issue} requeued for a rebase continuation: ${evidence}`);
6083
- return;
6084
- }
6085
-
6086
- if (recovery === "requeue") {
6087
- if (cls === "provider-credit") {
6088
- await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
6089
- }
6090
- // A dispatch-infra requeue that keeps landing on the same issue means the
6091
- // mirror for its repo is persistently broken — a ref-lock that retry already
6092
- // exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
6093
- // with no chance of success, so after a bounded number of strikes this
6094
- // escalates to a human instead (#168, #177).
6095
- if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
6096
- await safeEscalate(d, {
6097
- tier: 1,
6098
- project: project.name,
6099
- issue: run.issue,
6100
- summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
6101
- detail: [
6102
- `The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
6103
- "The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
6104
- ].join("\n"),
6105
- });
6106
- store.updateRun(run.id, { recoveredAt: Date.now() });
6107
- log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
6108
- return;
6109
- }
6110
- // Same bound for provider-transient: an issue whose stream keeps stalling
6111
- // mid-run is requeued free (no attempt, no continuation charged) — but a
6112
- // provider that aborts three times for one issue is down, and a human has
6113
- // to check its status before hand-requeueing (#220). The escalation names
6114
- // every model the chain tried, so a merged branch built on a different
6115
- // model is attributable (#286).
6116
- if (
6117
- cls === "provider-transient" &&
6118
- store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
6119
- ) {
6120
- const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
6121
- await safeEscalate(d, {
6122
- tier: 1,
6123
- project: project.name,
6124
- issue: run.issue,
6125
- summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
6126
- detail: [
6127
- `The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
6128
- ...(tried === ""
6129
- ? []
6130
- : [`Models tried: ${tried}.`]),
6131
- "Check provider status before requeueing by hand.",
6132
- ].join("\n"),
6133
- });
6134
- store.updateRun(run.id, { recoveredAt: Date.now() });
6135
- log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
6136
- return;
6137
- }
6138
- // Same bound for provider-capacity: a run the provider throttled into the
6139
- // ground is requeued free (no attempt charged) — but a provider that
6140
- // throttles the same issue three times is at capacity, and a human has to
6141
- // check its status before hand-requeueing (#573). On a chain-configured
6142
- // project each requeue already moved the next attempt to the next chain
6143
- // model, so this escalation is what catches the no-chain case and the
6144
- // exhausted chain; it names every model the chain tried.
6145
- if (
6146
- cls === "provider-capacity" &&
6147
- store.classCountFor(project.name, run.issue, "provider-capacity") >= PROVIDER_CAPACITY_MAX_STRIKES
6148
- ) {
6149
- const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
6150
- await safeEscalate(d, {
6151
- tier: 1,
6152
- project: project.name,
6153
- issue: run.issue,
6154
- summary: `[provider-capacity] #${run.issue}: the model provider is throttling this run into the ground — ${evidence}`,
6155
- detail: [
6156
- `The provider answered #${run.issue} with sustained in-session rate limits ${PROVIDER_CAPACITY_MAX_STRIKES} times in a row; the harness retried each and was exhausted.`,
6157
- ...(tried === ""
6158
- ? []
6159
- : [`Models tried: ${tried}.`]),
6160
- "Check the provider's rate-limit status (and its throughput-oriented routes) before requeueing by hand.",
6161
- ].join("\n"),
6162
- });
6163
- store.updateRun(run.id, { recoveredAt: Date.now() });
6164
- log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
6165
- return;
6166
- }
6167
- // Only when the tracker still shows this issue as ours to hand back. An
6168
- // issue that is closed, or has no state label, was resolved by another route
6169
- // and requeueing it would dispatch work nobody asked for.
6170
- const state = await tracker.issueState(run.issue).catch(() => undefined);
6171
- if (state !== "open") {
6172
- log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
6173
- return;
6174
- }
6175
- // A clean orphan whose queue label is already absent is a deliberate
6176
- // withdrawal — the operator took the issue out of the queue (e.g. so a
6177
- // daemon restart could not re-dispatch work known to be unsafe) — and
6178
- // recovery must not recreate that intent (#423). Read the live label set;
6179
- // when the queue label is gone, release the dispatcher-owned in-progress
6180
- // label and stop, leaving the operator's withdrawal to survive recovery.
6181
- if (cls === "orphan-clean") {
6182
- const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
6183
- if (snapshot === undefined) {
6184
- log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
6185
- return;
6186
- }
6187
- if (!snapshot.labels.includes(project.queueLabel)) {
6188
- store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
6189
- store.updateRun(run.id, { recoveredAt: Date.now() });
6190
- log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
6191
- return;
6192
- }
6193
- // #439: an orphan-clean row charges the continuation budget, so once
6194
- // `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
6195
- // holds the issue on — requeueing re-adds a queue label for a candidate the
6196
- // dispatcher can never admit. Stop handing it back and hold it instead: the
6197
- // queue label comes off (so admission never re-holds on every dispatch),
6198
- // the in-progress label is released, and a single diagnosis escalates once.
6199
- // `orphan-clean` is deliberately NOT a global exclusion from the budget (a
6200
- // worker that genuinely keeps dying mid-work must still be bounded); this is
6201
- // the missing ceiling check this path never had (#348's invariant).
6202
- const continuations = store.continuationsFor(project.name, run.issue);
6203
- if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
6204
- const runs = store.runsForIssue(project.name, run.issue);
6205
- const breakdown = continuationBreakdown(runs);
6206
- const artifact = newestContinuableRun(runs);
6207
- const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
6208
- const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
6209
- const artifactLine =
6210
- artifact === undefined
6211
- ? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
6212
- : `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
6213
- store.enqueueLabelOps(project.name, [
6214
- { issue: run.issue, op: "remove", label: inProgress },
6215
- { issue: run.issue, op: "remove", label: project.queueLabel },
6216
- ]);
6217
- store.updateRun(run.id, { recoveredAt: Date.now() });
6218
- await safeEscalate(d, {
6219
- tier: 1,
6220
- project: project.name,
6221
- issue: run.issue,
6222
- summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
6223
- detail: [
6224
- `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
6225
- `Continuations by failure class: ${classLine}.`,
6226
- onlyDaemonStops
6227
- ? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
6228
- : "Continuations span real work — inspect what each attempt left behind before continuing.",
6229
- artifactLine,
6230
- "What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
6231
- ].join("\n"),
6232
- });
6233
- await postExhaustionPostmortem(
6234
- d,
6235
- run,
6236
- `Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
6237
- );
6238
- log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
6239
- return;
6240
- }
6241
- }
6242
- const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
6243
- swapToQueue(d, run.issue, label);
6244
- store.updateRun(run.id, { recoveredAt: Date.now() });
6245
- log(`#${run.issue} requeued from ${cls}: ${evidence}`);
6246
- return;
6247
- }
6248
-
6249
- if (recovery === "rerun-checks") {
6250
- if (run.prUrl === undefined) return;
6251
- try {
6252
- await tracker.rerunFailedChecks(run.prUrl);
6253
- } catch (err) {
6254
- log(`#${run.issue} check re-run failed (${errText(err)}) — retrying next tick`);
6255
- return;
6256
- }
6257
- // Back to pending rather than green: the existing settle sweep re-verifies
6258
- // it against the recorded head on a later tick, so nothing here has to guess
6259
- // whether the re-run passed.
6260
- store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
6261
- log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
6262
- return;
6263
- }
6264
-
6265
- if (recovery === "escalate") {
6266
- const detail = [evidence];
6267
- if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
6268
- const calls = lastToolCalls(run.sessionFile);
6269
- detail.push(
6270
- calls.length === 0
6271
- ? "transcript unreadable — no tool calls could be recovered"
6272
- : `Last ${calls.length} tool calls: ${calls.join(" → ")}`,
6273
- );
6274
- }
6275
- if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
6276
- // #172: an unwritten transcript is "the run died before it flushed", not a
6277
- // link to a file the operator will open and find missing.
6278
- detail.push(
6279
- run.sessionFile === undefined
6280
- ? "Session: (no transcript)"
6281
- : existsSync(run.sessionFile)
6282
- ? `Session: ${run.sessionFile}`
6283
- : `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
6284
- );
6285
- // The class and the run are in the summary, which is what the notifications
6286
- // ledger dedupes on — so one class escalates once per run rather than every
6287
- // five minutes.
6288
- await safeEscalate(d, {
6289
- tier: 1,
6290
- project: project.name,
6291
- issue: run.issue,
6292
- runId: run.id,
6293
- summary: `[${cls}] #${run.issue} attempt ${run.attempt}: ${evidence}`,
6294
- detail: detail.join("\n"),
6295
- });
6296
- // The hand-off IS the recovery for these classes: there is nothing else this
6297
- // package can do, and leaving the row unrecovered would re-escalate forever.
6298
- store.updateRun(run.id, { recoveredAt: Date.now() });
6299
- return;
6300
- }
6301
-
6302
- // `hold` (orphan-dirty) and `none`: recorded, nothing performed. The existing
6303
- // unsalvaged-WIP admission hold already fails dispatch closed until an
6304
- // operator acknowledges the tree, which is the only safe move when the
6305
- // worktree holds the only copy of real work.
6306
- }
6307
-
6308
- /**
6309
- * Enqueue a state-label → queue-label swap for projection (#201).
6310
- *
6311
- * The swap is two ops in id order — remove first, then add — which is the
6312
- * atomicity the projector guarantees: the issue never sits newly eligible
6313
- * without a queue label on its way back, and the add never lands before the
6314
- * remove when GitHub fails between them. Enqueueing is a durable local write
6315
- * that cannot fail on the tracker, so the caller records its recovery
6316
- * immediately and the projector retries the swap until the tracker takes it —
6317
- * that closes the 0.4.4 hole where a refused label swap stranded the row
6318
- * permanently under a log line promising a retry.
6319
- */
6320
- function swapToQueue(d: Pick<Deps, "project" | "store">, issue: number, label: string): void {
6321
- d.store.enqueueLabelOps(d.project.name, [
6322
- { issue, op: "remove", label },
6323
- { issue, op: "add", label: d.project.queueLabel },
6324
- ]);
6325
- }
6326
-
6327
- /** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
6328
- const RECONCILE_LIMIT = 50;
6329
-
6330
- /**
6331
- * Clear state labels from issues that no longer need them (#132's `superseded`).
6332
- *
6333
- * Two structural signals, both cheap and both observed on this fleet: an issue
6334
- * that is closed but still carries `agent:*`, and an open issue carrying
6335
- * `failed` whose sub-issues have all closed. On 2026-08-09 four issues (#307,
6336
- * #297, #140, #82) carried `agent:failed` while every one of them was already
6337
- * complete — the label was residue of a turns-cap kill from two days earlier,
6338
- * and nothing in the loop ever revisited it. The board counted four phantom
6339
- * failures while the genuinely stuck issues were invisible.
6340
- *
6341
- * Positive evidence only. A tracker that cannot list answers empty, and an empty
6342
- * answer removes nothing: a reconcile that guessed would strip the interlock
6343
- * that keeps two workers off one issue.
6344
- */
6345
- export async function reconcileStaleLabels(d: Deps): Promise<void> {
6346
- const { project, tracker, store } = d;
6347
- const labels = [project.stateLabels.failed, project.stateLabels.blocked, project.stateLabels.inProgress];
6348
-
6349
- for (const label of labels) {
6350
- const carrying = await tracker.listLabeled(label, RECONCILE_LIMIT).catch(() => []);
6351
- for (const issue of carrying) {
6352
- if (issue.state === "closed") {
6353
- // Never retain an `agent:*` label on a closed issue: the work is done by
6354
- // some route, and the label only makes the board lie about it. Enqueue
6355
- // rather than call — a refused write must not lose the decision; the
6356
- // projector retries the removal until the tracker takes it (#201).
6357
- store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
6358
- log(`#${issue.number} reconciled: closed issue no longer carries ${label} (queued)`);
6359
- continue;
6360
- }
6361
-
6362
- if (label !== project.stateLabels.failed) continue;
6363
- const children = await tracker.childrenOf(issue.number).catch(() => []);
6364
- if (children.length === 0 || children.some((c) => c.state !== "closed")) continue;
6365
-
6366
- const key = `${project.name}:superseded:${issue.number}`;
6367
- if (store.wasNotified(key)) continue;
6368
- const list = children.map((c) => `#${c.number}`).join(", ");
6369
- store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
6370
- try {
6371
- await tracker.comment(
6372
- issue.number,
6373
- `superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
6374
- `acceptance criteria are met on the default branch.`,
6375
- );
6376
- store.markNotified(key);
6377
- log(`#${issue.number} reconciled: superseded by ${list}`);
6378
- } catch (err) {
6379
- // The label removal is already queued and will land regardless; the
6380
- // comment is the only half that can fail here (#201).
6381
- log(`#${issue.number} could not comment the superseded note (${errText(err)}) — retrying next tick`);
6382
- }
6383
- }
6384
- }
6385
- }
6386
-
6387
- /**
6388
- * Settles `claimed`/`running` rows left by a dead daemon process and, before
6389
- * marking each one `orphaned`, salvages any dirty worktree.
6390
- *
6391
- * Found live after a host restart killed two workers mid-run, and again on
6392
- * every package deploy that restarted while workers were live (#35): without
6393
- * the salvage call the next attempt's `worktree remove --force` destroyed
6394
- * uncommitted edits that had no other copy. Cap-kills already salvaged (#27);
6395
- * this is the same call site for the restart path.
6396
- *
6397
- * Only the rows change. The issue keeps its in-progress label — that label is
6398
- * the crash guard against double-dispatch, and deciding what a dead worker's
6399
- * remains are worth (an open PR? a salvaged sha? a clean tree?) is the
6400
- * orchestrator's drain-duty judgement, not something to automate here. The
6401
- * rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
6402
- * still escalates instead of retrying forever.
6403
- *
6404
- * `pushed-green` rows are deliberately left alone: they hold no process — they
6405
- * are finished work waiting on a human merge, and they must keep occupying the
6406
- * issue so a second attempt cannot land on a live PR. What eventually settles
6407
- * them is {@link settlePushedGreen}, on the tick, by asking the tracker what
6408
- * became of the PR — the one question a restart cannot answer by inference.
6409
- */
6410
- export async function reconcileOrphanedRuns(
6411
- store: Store,
6412
- project: string,
6413
- /**
6414
- * Resolves the privileged publisher for one orphaned run. Optional because a
6415
- * test driving the row transitions has no repo to publish to; production
6416
- * always passes it, and without it a salvaged WIP commit stays local — which
6417
- * is the half of #121 that reaches a human.
6418
- */
6419
- publish?: (run: RunRecord) => RunPublisher,
6420
- ): Promise<RunRecord[]> {
6421
- // Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
6422
- // a process dying — it is finished work waiting on a human merge.
6423
- const stale = store.liveRuns(project);
6424
- const endedAt = Date.now();
6425
- for (const r of stale) {
6426
- // Salvage before the row flips: the worktree path is on the record, and
6427
- // salvageWip is a no-op for a missing/clean tree. The clause matches the
6428
- // cap-kill wording so triage reads the same either way, and the tree is
6429
- // kept because an orphan's remains are the orchestrator's drain-duty call.
6430
- const settlement =
6431
- r.worktree === ""
6432
- ? undefined
6433
- : await settleWorktree({
6434
- issue: r.issue,
6435
- attempt: r.attempt,
6436
- ending: "killed by a daemon restart",
6437
- worktree: r.worktree,
6438
- branch: r.branch,
6439
- // An orphan's tree is kept, so its commits are not about to be
6440
- // deleted — but a WIP salvage still has to reach GitHub, which is
6441
- // #121's whole point and is now the daemon's hop to make.
6442
- publish: publish?.(r),
6443
- tree: "keep",
6444
- });
6445
- store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
6446
- }
6447
- return stale;
6448
- }
6449
5215
 
6450
5216
  // ------------------------------------------------------------------- the daemon
6451
5217
 
@@ -6787,6 +5553,14 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6787
5553
  onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
6788
5554
  onRefusal: (at) => store.recordGhRefusal?.(at),
6789
5555
  }).issueSnapshot(issue),
5556
+ // The body twin of `probeIssueIn`, for the dependency-graph cycle pass
5557
+ // (#421): reads a reachable routed prerequisite's body through the same
5558
+ // per-repo tracker/credential/accounting seams.
5559
+ probeBodyIn: (ownerRepo, issue) =>
5560
+ makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, {
5561
+ onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
5562
+ onRefusal: (at) => store.recordGhRefusal?.(at),
5563
+ }).issueBody(issue),
6790
5564
  ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
6791
5565
  verbActions,
6792
5566
  };