omp-conductor 0.19.7 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +10 -1
- package/agents/to-spec.md +76 -9
- package/package.json +1 -1
- package/schema/config.schema.json +4 -0
- package/src/arm-challenge.ts +204 -85
- package/src/ask.ts +130 -615
- package/src/board.ts +7 -1
- package/src/brief-upgrade.ts +24 -0
- package/src/briefs/console.md +253 -0
- package/src/briefs/correction.md +203 -0
- package/src/briefs/orchestrator.md +167 -97
- package/src/briefs/policy.md +19 -16
- package/src/briefs/to-spec.md +76 -9
- package/src/briefs/worker.md +50 -16
- package/src/cli.ts +4 -0
- package/src/command-manifest.ts +54 -8
- package/src/commands/arm.ts +113 -49
- package/src/commands/console.ts +70 -0
- package/src/commands/context.ts +2 -0
- package/src/commands/epic.ts +132 -0
- package/src/commands/extend.ts +9 -1
- package/src/commands/intake.ts +44 -14
- package/src/commands/stats.ts +19 -4
- package/src/commands/worker.ts +9 -1
- package/src/config-schema.ts +13 -0
- package/src/config.ts +27 -0
- package/src/daemon/ack.ts +159 -0
- package/src/daemon/admission-pass.ts +135 -0
- package/src/daemon/brief.ts +461 -0
- package/src/daemon/deps.ts +539 -0
- package/src/daemon/dispatch.ts +1779 -0
- package/src/daemon/drain.ts +185 -0
- package/src/daemon/groom-pass.ts +412 -0
- package/src/daemon/http.ts +417 -0
- package/src/daemon/integrity.ts +108 -0
- package/src/daemon/panes.ts +180 -0
- package/src/daemon/review.ts +1888 -0
- package/src/daemon/runtime.ts +736 -0
- package/src/daemon/settle-pass.ts +589 -0
- package/src/daemon/supervision.ts +438 -0
- package/src/daemon/tick.ts +968 -0
- package/src/daemon/views.ts +751 -0
- package/src/daemon.ts +105 -7923
- package/src/dashboard/app.js +58 -0
- package/src/dashboard/controls.ts +22 -3
- package/src/dashboard/server.ts +4 -0
- package/src/diff-flags.ts +24 -3
- package/src/failure-class.ts +75 -1
- package/src/fleet.ts +290 -164
- package/src/groom.ts +461 -0
- package/src/http-token.ts +142 -0
- package/src/knowledge.ts +229 -0
- package/src/mining.ts +316 -0
- package/src/orchestrator-tick.ts +428 -1681
- package/src/ready-gate.ts +267 -0
- package/src/settlement.ts +72 -6
- package/src/setup-host.ts +32 -9
- package/src/setup-wizard.ts +55 -7
- package/src/setup.ts +229 -3
- package/src/stats.ts +257 -2
- package/src/status-render.ts +158 -7
- package/src/store.ts +604 -26
- package/src/to-spec.ts +194 -21
- package/src/tracker/github.ts +50 -0
- package/src/types.ts +416 -15
- package/src/verbs/protocol.ts +28 -0
- package/src/verbs/server.ts +330 -39
- package/src/wake.ts +19 -2
- package/src/worker.ts +456 -1
|
@@ -0,0 +1,1888 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The review lane: what happens to a run after a reviewer says no.
|
|
3
|
+
*
|
|
4
|
+
* Three passes with one subject — revisions, the fresh-correction lane (#1047)
|
|
5
|
+
* and ceiling adjudication — kept together because they share the decision this
|
|
6
|
+
* module exists to make: *how* to relaunch. `resolveReviewLaunch` reads the
|
|
7
|
+
* prior session's health, the model it actually ran on and the round already
|
|
8
|
+
* recorded, and answers resume-or-refresh once, durably, for every caller. Split
|
|
9
|
+
* across three files that answer would be re-derived three times, which is the
|
|
10
|
+
* shape #1035 was.
|
|
11
|
+
*
|
|
12
|
+
* It launches through `dispatch.ts` rather than beside it: a correction is
|
|
13
|
+
* another launch of the same run against the same branch and PR, so the claim,
|
|
14
|
+
* label-swap and settle rules are borrowed, never restated. The dependency runs
|
|
15
|
+
* one way only — nothing in the dispatch path knows this module exists.
|
|
16
|
+
*/
|
|
17
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
18
|
+
import { dirname, join } from "node:path";
|
|
19
|
+
import { hasContinuationBudget, runSpendAllowanceUsd } from "../admission.ts";
|
|
20
|
+
import { resolveReleaseGrants, resolveReview, stateDir } from "../config.ts";
|
|
21
|
+
import { classifyRun, providerCreditRefusal, providerTransientFault, reviewRoundNeverWorked } from "../failure-class.ts";
|
|
22
|
+
import { pushRunBranch, type RunRepoRef } from "../gitops.ts";
|
|
23
|
+
import { errText, log, safeEscalate } from "../log.ts";
|
|
24
|
+
import { DEFAULT_MODEL_FALLBACK_THRESHOLD, providerFailureFacts, resolveDispatchModel } from "../model-fallback.ts";
|
|
25
|
+
import { materializeOmpSettings } from "../omp-settings.ts";
|
|
26
|
+
import { harnessLogDir, readSessionError, readTerminalEvidence, recordOperatorStop, recordWorkerDiscoveries, settleWorktree, swapToQueue } from "../settlement.ts";
|
|
27
|
+
import { GhPrMissingError } from "../tracker/github.ts";
|
|
28
|
+
import { REVIEW_ROUND_INFRA_MAX_RETRIES, type PrDiff, type PrState, type PrVerification, type RepoTarget, type ReviewAdjudicationRecord, type ReviewCorrectionProvenance, type ReviewLaunchMode, type ReviewRevisionOutcome, type ReviewRevisionRecord, type RunRecord, type RunState } from "../types.ts";
|
|
29
|
+
import { isHeadMismatch, listenVerbChannel, type VerbListener } from "../verbs/server.ts";
|
|
30
|
+
import { ensureVerbSocketDir, verbSocketPath } from "../verbs/socket.ts";
|
|
31
|
+
import { renderAdjudicationBrief, renderReviewRevisionPrompt, runAdjudicator, runCorrectionWorker, runWorker, type AdjudicationRound, type CorrectionEvidence, type WorkerPauseControl, type WorkerResult } from "../worker.ts";
|
|
32
|
+
import { addRunRepo, mirrorPathFor, removeWorktree, type RunPublisher } from "../worktree.ts";
|
|
33
|
+
import type { WorkerPool } from "./admission-pass.ts";
|
|
34
|
+
import { workerGateGroups } from "./brief.ts";
|
|
35
|
+
import { completionLastError, exhaustedSessionReason, githubRepo, knowledgeRepoKey, recordDaemonSessionSpend, repoSlug, verbDeps, type Deps, type TurnLimitController, type WorkerControlSlot } from "./deps.ts";
|
|
36
|
+
import { endedBy, shouldContinueAfterTurnsCap, swapLabel, verifyPushedGreenClaim, workerReleaseBlockRecorder } from "./dispatch.ts";
|
|
37
|
+
import { wakeOrchestratorForBlockedRun } from "./supervision.ts";
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The daemon half of the review-revision transport (#677): wake every pending
|
|
41
|
+
* revision the orchestrator's verb recorded, on the tick, exactly like
|
|
42
|
+
* admission — the verb has no handle on the worker machinery, and the CLI path
|
|
43
|
+
* must behave like the embedded one. Bounded by the same worker slots, so a
|
|
44
|
+
* queued revision can never push the fleet past `maxConcurrentWorkers`; the
|
|
45
|
+
* remaining rows stay pending for the next tick.
|
|
46
|
+
*
|
|
47
|
+
* The claim (`pushed-green` → `running`) is synchronous and atomic per row;
|
|
48
|
+
* only the resumed worker runs in the pool. A row that moved since the verb
|
|
49
|
+
* recorded it — merged after all, settled, re-claimed — fails the claim and is
|
|
50
|
+
* settled `skipped` rather than woken on stale identity.
|
|
51
|
+
*/
|
|
52
|
+
export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
53
|
+
const pending = d.store.pendingReviewRevisions(d.project.name);
|
|
54
|
+
if (pending.length === 0) return;
|
|
55
|
+
const slots = Math.max(0, d.caps.maxConcurrentWorkers - d.store.liveRuns(d.project.name).length);
|
|
56
|
+
const batch = pending.slice(0, slots);
|
|
57
|
+
const launches: Promise<void>[] = [];
|
|
58
|
+
for (const revision of batch) {
|
|
59
|
+
if (d.drain?.draining === true) {
|
|
60
|
+
log(
|
|
61
|
+
`review revisions held: the daemon is draining (${batch.length - launches.length} of ${batch.length} not launched)`,
|
|
62
|
+
);
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
// A capped/failed run has no pushed-green settle sweep keeping its row
|
|
66
|
+
// honest (#795 review round 1): nothing transitions a `failed` / `killed`
|
|
67
|
+
// row when its PR merges or closes, so a PR that changed after the verb
|
|
68
|
+
// recorded the round would otherwise be claimed and a worker resumed
|
|
69
|
+
// against a dead PR. The settled-green origin keeps its own interlock
|
|
70
|
+
// (the settle sweep flips the row and the claim below refuses it), so
|
|
71
|
+
// only terminal-origin rounds re-read the reviewed PR fact here: the
|
|
72
|
+
// round dispatches only while the PR is still open at the exact reviewed
|
|
73
|
+
// head.
|
|
74
|
+
//
|
|
75
|
+
// The skip decision is decisive-fact only (review round 2): a definitively
|
|
76
|
+
// missing PR (`GhPrMissingError` — a corroborated 404, #779), a PR that
|
|
77
|
+
// is definitively not open, and a head that has definitively MOVED are
|
|
78
|
+
// settled skipped, exactly like any other row that moved. Everything
|
|
79
|
+
// transient or unreadable — a bare 404 that could not be corroborated, an
|
|
80
|
+
// undefined answer, a pending check run, a same-head red check that a
|
|
81
|
+
// rerun may clear, a throwing tracker — stays PENDING for the next tick:
|
|
82
|
+
// settling it would permanently discard the orchestrator's findings and
|
|
83
|
+
// consume the round.
|
|
84
|
+
// The row as it stands BEFORE the claim. Read whole rather than by state:
|
|
85
|
+
// the review claim clears the terminal classification (#795), so the facts
|
|
86
|
+
// #1047's launch decision rests on are only readable here.
|
|
87
|
+
const originRun = d.store.getRun(revision.runId);
|
|
88
|
+
const origin = originRun?.state;
|
|
89
|
+
if (origin === "failed" || origin === "killed") {
|
|
90
|
+
let prState: PrState | undefined;
|
|
91
|
+
try {
|
|
92
|
+
prState = await d.tracker.prState(revision.prUrl);
|
|
93
|
+
} catch (err) {
|
|
94
|
+
if (err instanceof GhPrMissingError) {
|
|
95
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
96
|
+
log(
|
|
97
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} does not exist — the reviewed PR is gone`,
|
|
98
|
+
);
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
log(
|
|
102
|
+
`#${revision.issue} review round ${revision.round} held: PR state re-check failed (${errText(err)}) — retrying next tick`,
|
|
103
|
+
);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (prState === undefined) {
|
|
107
|
+
log(
|
|
108
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} state could not be re-read — retrying next tick`,
|
|
109
|
+
);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (prState !== "open") {
|
|
113
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
114
|
+
log(
|
|
115
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} is ${prState}, not open — a review round resumes an open PR only`,
|
|
116
|
+
);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
let verification: PrVerification | undefined;
|
|
120
|
+
try {
|
|
121
|
+
verification = await d.tracker.verifyPr(revision.prUrl, revision.headSha);
|
|
122
|
+
} catch (err) {
|
|
123
|
+
log(
|
|
124
|
+
`#${revision.issue} review round ${revision.round} held: the reviewed-head check failed (${errText(err)}) — retrying next tick`,
|
|
125
|
+
);
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
if (verification === undefined || verification.status === "pending") {
|
|
129
|
+
log(
|
|
130
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
131
|
+
`${verification?.status ?? "unverifiable"} — retrying next tick`,
|
|
132
|
+
);
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
if (verification.status === "failed" && isHeadMismatch(verification.reason)) {
|
|
136
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
137
|
+
log(
|
|
138
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} no longer stands at ` +
|
|
139
|
+
`${revision.headSha} (${verification.reason}) — the reviewed head moved`,
|
|
140
|
+
);
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
// Anything else — a same-head red check a rerun may clear, an unknown
|
|
144
|
+
// verdict — is not definitive: the round stays pending for the next
|
|
145
|
+
// tick rather than discarding the findings.
|
|
146
|
+
if (verification.status !== "green") {
|
|
147
|
+
log(
|
|
148
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
149
|
+
`${verification.status} — retrying next tick`,
|
|
150
|
+
);
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
// #1047: decide the transport from the pre-claim row, and write it once.
|
|
155
|
+
// Before the claim on purpose — the claim clears exactly the columns the
|
|
156
|
+
// diagnosis reads — and durable on purpose, so a restart, an infra retry
|
|
157
|
+
// and a second racing pass all honour this one decision instead of
|
|
158
|
+
// re-deciding against a row the claim has already rewritten.
|
|
159
|
+
const launch =
|
|
160
|
+
originRun === undefined ? undefined : resolveReviewLaunch(d, revision, originRun);
|
|
161
|
+
if (!d.store.claimRunForReview(revision.runId)) {
|
|
162
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
163
|
+
log(
|
|
164
|
+
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer pushed-green`,
|
|
165
|
+
);
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
d.store.markReviewRevisionDispatched(revision.id, Date.now());
|
|
169
|
+
log(
|
|
170
|
+
`#${revision.issue} review round ${revision.round} → ${launch?.launchMode ?? "resume-original"} on run ` +
|
|
171
|
+
`${revision.runId} (model requested ${launch?.requestedModel ?? "harness default"}): ` +
|
|
172
|
+
`${launch?.reason ?? "the run row vanished before the decision"}`,
|
|
173
|
+
);
|
|
174
|
+
launches.push(handleReviewRevision(d, revision, launch));
|
|
175
|
+
}
|
|
176
|
+
if (pool !== undefined) {
|
|
177
|
+
for (const launch of launches) pool.launch(launch);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
// `--once`: like admissions, the tick waits for the workers it launched.
|
|
181
|
+
await Promise.allSettled(launches);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* The turn ceiling for one adjudication (#932).
|
|
186
|
+
*
|
|
187
|
+
* Small on purpose, and not the worker cap: a verdict is a read and an answer,
|
|
188
|
+
* not an implementation. A session still going after this is not deciding — and
|
|
189
|
+
* the ceiling exists so a stuck adjudicator costs a bounded amount rather than a
|
|
190
|
+
* worker's whole budget, on work that has already spent its review rounds.
|
|
191
|
+
*/
|
|
192
|
+
export const ADJUDICATION_MAX_TURNS = 40;
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Launch the review-ceiling adjudications this project owes (#932).
|
|
196
|
+
*
|
|
197
|
+
* Deliberately its own pass, next to {@link dispatchReviewRevisions} but never
|
|
198
|
+
* inside it: a revision resumes an implementation worker to change code, an
|
|
199
|
+
* adjudication opens a fresh read-only session to decide about code. Folding
|
|
200
|
+
* them together is how "escalate the automation" quietly becomes "resume the
|
|
201
|
+
* same worker with a different label", which is #873's named silent fake.
|
|
202
|
+
*
|
|
203
|
+
* Only `pending` rows are launched. A `running` row belongs to a live
|
|
204
|
+
* adjudicator — or to one a restart killed, which the startup sweep settles —
|
|
205
|
+
* so a restart re-reads it and never launches a second adjudicator for one head.
|
|
206
|
+
*/
|
|
207
|
+
export async function dispatchReviewAdjudications(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
208
|
+
const open = d.store.openReviewAdjudications(d.project.name).filter((a) => a.state === "pending");
|
|
209
|
+
if (open.length === 0) return;
|
|
210
|
+
const launches: Promise<void>[] = [];
|
|
211
|
+
for (const adjudication of open) {
|
|
212
|
+
if (d.drain?.draining === true) {
|
|
213
|
+
log(`adjudications held: the daemon is draining (${open.length - launches.length} not launched)`);
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
// The head is re-read before anything launches. A verdict describes the diff
|
|
217
|
+
// it read, so adjudicating a head the PR has since moved past would produce
|
|
218
|
+
// a decision about code nobody is merging — settled `stale-head` with both
|
|
219
|
+
// heads named, never launched.
|
|
220
|
+
let live: string | undefined;
|
|
221
|
+
try {
|
|
222
|
+
live = await d.tracker.prHead(adjudication.prUrl);
|
|
223
|
+
} catch (err) {
|
|
224
|
+
log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head unreadable (${errText(err)})`);
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
if (live === undefined) {
|
|
228
|
+
log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head could not be read — retrying next tick`);
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
|
|
232
|
+
d.store.settleReviewAdjudication(
|
|
233
|
+
adjudication.id,
|
|
234
|
+
"stale-head",
|
|
235
|
+
`recorded ${adjudication.headSha}, live ${live} — the pull request moved before this adjudication launched`,
|
|
236
|
+
Date.now(),
|
|
237
|
+
);
|
|
238
|
+
log(`#${adjudication.issue} adjudication stale-head: recorded ${adjudication.headSha}, live ${live}`);
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
// The claim IS the single-flight guard: the transition is the row's own
|
|
242
|
+
// WHERE clause, so two dispatch passes racing one adjudication cannot both
|
|
243
|
+
// launch — the loser changes nothing and skips.
|
|
244
|
+
const review = resolveReview(d.project);
|
|
245
|
+
const provenance = { role: adjudication.role, model: adjudication.role, resolvedAt: Date.now() };
|
|
246
|
+
if (!d.store.markReviewAdjudicationRunning(adjudication.id, provenance, Date.now())) {
|
|
247
|
+
log(`#${adjudication.issue} adjudication skipped: another pass already claimed it`);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
log(`#${adjudication.issue} adjudication → launching role ${adjudication.role} at ${adjudication.headSha}`);
|
|
251
|
+
launches.push(handleReviewAdjudication(d, adjudication, review.maxRounds));
|
|
252
|
+
}
|
|
253
|
+
if (pool !== undefined) {
|
|
254
|
+
for (const launch of launches) pool.launch(launch);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
await Promise.allSettled(launches);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Assemble one adjudication's evidence and run it to a durable verdict (#932).
|
|
262
|
+
*
|
|
263
|
+
* Every fact comes from a durable row or a live tracker read — never from the
|
|
264
|
+
* worker's own report, which is the claim under adjudication. The issue body is
|
|
265
|
+
* carried verbatim because the acceptance criteria ARE the standard being
|
|
266
|
+
* judged, and every prior round travels WITH its settled outcome: a finding
|
|
267
|
+
* without its outcome reads as an outstanding complaint even when the worker
|
|
268
|
+
* fixed it, which is how an adjudicator rejects work that was already corrected.
|
|
269
|
+
*/
|
|
270
|
+
export async function handleReviewAdjudication(
|
|
271
|
+
d: Deps,
|
|
272
|
+
adjudication: ReviewAdjudicationRecord,
|
|
273
|
+
maxRounds: number,
|
|
274
|
+
): Promise<void> {
|
|
275
|
+
const issue = adjudication.issue;
|
|
276
|
+
const [title, body, diff, verification] = await Promise.all([
|
|
277
|
+
d.tracker.getIssue(issue).then(
|
|
278
|
+
(row) => row?.title,
|
|
279
|
+
() => undefined,
|
|
280
|
+
),
|
|
281
|
+
d.tracker.issueBody(issue).then(
|
|
282
|
+
(text) => text,
|
|
283
|
+
() => undefined,
|
|
284
|
+
),
|
|
285
|
+
d.tracker.prDiff(adjudication.prUrl).then(
|
|
286
|
+
(parsed) => parsed,
|
|
287
|
+
() => undefined,
|
|
288
|
+
),
|
|
289
|
+
d.tracker.verifyPr(adjudication.prUrl, adjudication.headSha).then(
|
|
290
|
+
(result) => result,
|
|
291
|
+
() => undefined,
|
|
292
|
+
),
|
|
293
|
+
]);
|
|
294
|
+
// Prior rounds, oldest first, keyed by PR so a continuation that inherited the
|
|
295
|
+
// PR cannot hide its predecessor's findings. The ceiling call's own findings
|
|
296
|
+
// are appended as the last entry — they are the reason this adjudication
|
|
297
|
+
// exists, and the durable row is where they were kept precisely so this brief
|
|
298
|
+
// could carry them.
|
|
299
|
+
const rounds: AdjudicationRound[] = d.store
|
|
300
|
+
.reviewRevisionsForPr(d.project.name, adjudication.prUrl)
|
|
301
|
+
.map((revision) => ({
|
|
302
|
+
round: revision.round,
|
|
303
|
+
findings: revision.findings,
|
|
304
|
+
...(revision.outcome === undefined ? {} : { outcome: revision.outcome }),
|
|
305
|
+
}));
|
|
306
|
+
if (adjudication.findings !== undefined) {
|
|
307
|
+
rounds.push({
|
|
308
|
+
round: rounds.length + 1,
|
|
309
|
+
findings: adjudication.findings,
|
|
310
|
+
outcome: "escalated to this adjudication — no worker round was available",
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
const brief = renderAdjudicationBrief({
|
|
314
|
+
issue,
|
|
315
|
+
...(title === undefined ? {} : { issueTitle: title }),
|
|
316
|
+
issueBody: body ?? "(the issue body could not be read — judge the diff against the findings below)",
|
|
317
|
+
prUrl: adjudication.prUrl,
|
|
318
|
+
headSha: adjudication.headSha,
|
|
319
|
+
checks:
|
|
320
|
+
verification === undefined
|
|
321
|
+
? "(the check state could not be read)"
|
|
322
|
+
: `${verification.status} — ${verification.reason}`,
|
|
323
|
+
diff:
|
|
324
|
+
diff === undefined
|
|
325
|
+
? "(the diff could not be read — say so in your reasons rather than guessing)"
|
|
326
|
+
: renderDiffForAdjudication(diff),
|
|
327
|
+
rounds,
|
|
328
|
+
maxRounds,
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
const sessionDir = join(stateDir(), "sessions", `adjudication-${adjudication.id}`);
|
|
332
|
+
mkdirSync(sessionDir, { recursive: true });
|
|
333
|
+
const ompSettingsFile = materializeOmpSettings(d.project, sessionDir);
|
|
334
|
+
const result = await (d.runAdjudicatorImpl ?? runAdjudicator)({
|
|
335
|
+
brief,
|
|
336
|
+
// No worktree: an adjudicator has no branch, and everything it needs is in
|
|
337
|
+
// the brief. The session directory doubles as its confined cwd.
|
|
338
|
+
cwd: sessionDir,
|
|
339
|
+
sessionDir,
|
|
340
|
+
model: adjudication.role,
|
|
341
|
+
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
342
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
343
|
+
maxTurns: ADJUDICATION_MAX_TURNS,
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
// Above every verdict branch below — a stale-head refusal and an unresolvable
|
|
347
|
+
// role cost as much as a clear, and all three used to be free as far as
|
|
348
|
+
// attribution was concerned.
|
|
349
|
+
recordDaemonSessionSpend(d, {
|
|
350
|
+
role: "adjudicator",
|
|
351
|
+
issue,
|
|
352
|
+
...(adjudication.role === undefined ? {} : { model: adjudication.role }),
|
|
353
|
+
...(result.model === undefined ? {} : { resolvedModel: result.model }),
|
|
354
|
+
turns: result.turns,
|
|
355
|
+
spendUsd: result.spendUsd,
|
|
356
|
+
at: Date.now(),
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
// What actually ran it, named in the evidence when it differs from the role
|
|
360
|
+
// that was asked for: the provenance recorded at claim time is conductor's own
|
|
361
|
+
// resolution (the role it passed to OMP), and the model that wrote the
|
|
362
|
+
// messages is a later observation only this result carries.
|
|
363
|
+
const ran =
|
|
364
|
+
result.model === undefined
|
|
365
|
+
? ""
|
|
366
|
+
: `adjudicated by ${result.model}${result.provider === undefined ? "" : ` (${result.provider})`}: `;
|
|
367
|
+
if (result.verdict === undefined) {
|
|
368
|
+
// A session that never took a turn did not run: the role could not be
|
|
369
|
+
// resolved or the harness could not start, and the remedy is configuration
|
|
370
|
+
// rather than a retry. One that ran and produced nothing usable is a
|
|
371
|
+
// failure. Neither is ever a clear.
|
|
372
|
+
const state = result.turns === 0 ? "unavailable-model" : "failed";
|
|
373
|
+
d.store.settleReviewAdjudication(
|
|
374
|
+
adjudication.id,
|
|
375
|
+
state,
|
|
376
|
+
`${ran}${result.report === "" ? "no verdict and no output" : result.report}`,
|
|
377
|
+
Date.now(),
|
|
378
|
+
);
|
|
379
|
+
log(`#${issue} adjudication ${state} at ${adjudication.headSha}`);
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
// A verdict that names a different head judged something else. Fail closed:
|
|
383
|
+
// the one-shot is spent on the head it was opened for, and nothing merges on
|
|
384
|
+
// the strength of a decision about another diff.
|
|
385
|
+
if (
|
|
386
|
+
result.verdict.headSha !== undefined &&
|
|
387
|
+
result.verdict.headSha.toLowerCase() !== adjudication.headSha.toLowerCase()
|
|
388
|
+
) {
|
|
389
|
+
d.store.settleReviewAdjudication(
|
|
390
|
+
adjudication.id,
|
|
391
|
+
"stale-head",
|
|
392
|
+
`${ran}the verdict names head ${result.verdict.headSha}, not the adjudicated ${adjudication.headSha}: ${result.verdict.reasons}`,
|
|
393
|
+
Date.now(),
|
|
394
|
+
);
|
|
395
|
+
log(`#${issue} adjudication stale-head: verdict named ${result.verdict.headSha}`);
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
d.store.settleReviewAdjudication(
|
|
399
|
+
adjudication.id,
|
|
400
|
+
result.verdict.verdict,
|
|
401
|
+
`${ran}${result.verdict.reasons}`,
|
|
402
|
+
Date.now(),
|
|
403
|
+
);
|
|
404
|
+
log(`#${issue} adjudication ${result.verdict.verdict} at ${adjudication.headSha}`);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Apply the terminal dispositions an adjudication's verdict implies (#876).
|
|
409
|
+
*
|
|
410
|
+
* The verdict is a decision; this is what is DONE about it, and the two are
|
|
411
|
+
* separate rows for exactly that reason. Recording `cleared` while leaving the
|
|
412
|
+
* merge gate refusing, or `rejected` while leaving the PR open to block the
|
|
413
|
+
* issue forever, is the issue's named silent fake — so what this pass changes is
|
|
414
|
+
* tracker and gate state, and the disposition text is only its receipt.
|
|
415
|
+
*
|
|
416
|
+
* Idempotent by that receipt: a row whose `disposition` is already recorded is
|
|
417
|
+
* skipped, so a restart, a retried tick or a partial failure cannot close one PR
|
|
418
|
+
* twice. Nothing here is destructive before its explanation is durable.
|
|
419
|
+
*
|
|
420
|
+
* Neither branch opens a review round, launches another adjudicator, or asks an
|
|
421
|
+
* operator for anything. Neither touches an attempt or continuation budget
|
|
422
|
+
* either: a rejected PR closes, and the ORDINARY settle sweep charges that
|
|
423
|
+
* closure exactly as it charges any PR closed without merging — which is what
|
|
424
|
+
* keeps this composable with #815's separate corrective-worker path instead of
|
|
425
|
+
* resetting its counters.
|
|
426
|
+
*/
|
|
427
|
+
export async function applyAdjudicationDispositions(d: Deps): Promise<void> {
|
|
428
|
+
const { project, store } = d;
|
|
429
|
+
// Terminal rows only, and only those nothing has disposed of yet. Read from
|
|
430
|
+
// the PRs still in flight: a settled adjudication whose PR is long gone needs
|
|
431
|
+
// no disposition, and scanning history would re-litigate closed work.
|
|
432
|
+
const pending = store
|
|
433
|
+
.activeRuns(project.name)
|
|
434
|
+
.flatMap((run) => (run.prUrl === undefined ? [] : store.reviewAdjudicationsForPr(project.name, run.prUrl)))
|
|
435
|
+
.filter((row) => row.settledAt !== undefined && row.disposition === undefined);
|
|
436
|
+
for (const adjudication of pending) {
|
|
437
|
+
if (adjudication.state === "cleared") {
|
|
438
|
+
// The clear re-enters the ORDINARY merge path, and it does so through the
|
|
439
|
+
// mechanism that path already reads: the #888 gate refuses a head carrying
|
|
440
|
+
// unresolved review evidence, and #913's clearance is the recorded
|
|
441
|
+
// disposition that settles it. Reusing it means no second gate, no
|
|
442
|
+
// adjudication-shaped exception inside `conductor_pr_merge`, and one
|
|
443
|
+
// audit trail for "this head was cleared, by whom, and why".
|
|
444
|
+
//
|
|
445
|
+
// The head is re-read first: a clearance is head-scoped and clears
|
|
446
|
+
// backwards only, so recording one against a head the PR has moved past
|
|
447
|
+
// would bless a diff nobody adjudicated.
|
|
448
|
+
let live: string | undefined;
|
|
449
|
+
try {
|
|
450
|
+
live = await d.tracker.prHead(adjudication.prUrl);
|
|
451
|
+
} catch (err) {
|
|
452
|
+
log(`#${adjudication.issue} adjudication clearance held: head unreadable (${errText(err)})`);
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
if (live === undefined) {
|
|
456
|
+
log(`#${adjudication.issue} adjudication clearance held: head could not be read — retrying next tick`);
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
|
|
460
|
+
// Visible and non-destructive: the clearance is not recorded, the row
|
|
461
|
+
// says why, and the PR keeps whatever gate state it has.
|
|
462
|
+
store.recordReviewAdjudicationDisposition(
|
|
463
|
+
adjudication.id,
|
|
464
|
+
`no clearance recorded: the head moved from ${adjudication.headSha} to ${live} after the verdict`,
|
|
465
|
+
);
|
|
466
|
+
log(`#${adjudication.issue} adjudication clearance skipped: head moved to ${live}`);
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
store.recordReviewClearance({
|
|
470
|
+
project: project.name,
|
|
471
|
+
prUrl: adjudication.prUrl,
|
|
472
|
+
headSha: adjudication.headSha,
|
|
473
|
+
by: `adjudication:${adjudication.role}`,
|
|
474
|
+
reason: `cleared by the review-ceiling adjudication: ${adjudication.evidence ?? "no evidence recorded"}`,
|
|
475
|
+
});
|
|
476
|
+
store.recordReviewAdjudicationDisposition(
|
|
477
|
+
adjudication.id,
|
|
478
|
+
`recorded a review clearance at ${adjudication.headSha}; the PR is on the ordinary merge path`,
|
|
479
|
+
);
|
|
480
|
+
log(`#${adjudication.issue} adjudication cleared → merge path open at ${adjudication.headSha}`);
|
|
481
|
+
continue;
|
|
482
|
+
}
|
|
483
|
+
// Every non-clear terminal state disposes the same way, and that is
|
|
484
|
+
// deliberate: `rejected`, `failed`, `unavailable-model` and `stale-head` all
|
|
485
|
+
// mean "this head is not merging and no further round is coming", and the
|
|
486
|
+
// blocking artefact is identical in each case — an open PR that occupies the
|
|
487
|
+
// issue. Only the recorded reason differs.
|
|
488
|
+
const findings = store
|
|
489
|
+
.reviewRevisionsForPr(project.name, adjudication.prUrl)
|
|
490
|
+
.map((revision) => `### Review round ${revision.round} (${revision.outcome ?? "never settled"})
|
|
491
|
+
|
|
492
|
+
${revision.findings}`);
|
|
493
|
+
const comment = [
|
|
494
|
+
`## Adjudication: ${adjudication.state}`,
|
|
495
|
+
"",
|
|
496
|
+
`The review-round ceiling was reached on ${adjudication.headSha}, and the \`${adjudication.role}\` adjudicator ` +
|
|
497
|
+
`did not clear it. Closing this pull request so the issue is not blocked by work that cannot merge — ` +
|
|
498
|
+
"nothing here asks anyone to repair it by hand.",
|
|
499
|
+
"",
|
|
500
|
+
"### Adjudicator's reasons",
|
|
501
|
+
"",
|
|
502
|
+
adjudication.evidence ?? "(none recorded)",
|
|
503
|
+
...(findings.length === 0 ? [] : ["", "## Preserved review findings", "", ...findings]),
|
|
504
|
+
"",
|
|
505
|
+
"The branch itself is untouched and remains in the repository.",
|
|
506
|
+
].join("\n");
|
|
507
|
+
const closed = await d.verbActions?.closePr(adjudication.prUrl, comment);
|
|
508
|
+
if (closed === undefined) {
|
|
509
|
+
log(`#${adjudication.issue} adjudication disposition held: no action surface is wired`);
|
|
510
|
+
continue;
|
|
511
|
+
}
|
|
512
|
+
if (!closed.ok) {
|
|
513
|
+
// Retryable, and no receipt written: the next tick tries again rather than
|
|
514
|
+
// leaving a rejected PR open with a row that claims it was handled.
|
|
515
|
+
log(`#${adjudication.issue} adjudication disposition held: could not close ${adjudication.prUrl} — ${closed.stderr}`);
|
|
516
|
+
continue;
|
|
517
|
+
}
|
|
518
|
+
store.recordReviewAdjudicationDisposition(
|
|
519
|
+
adjudication.id,
|
|
520
|
+
`closed ${adjudication.prUrl} with the adjudicator's reasons and every preserved finding; ` +
|
|
521
|
+
"the settle sweep releases the issue",
|
|
522
|
+
);
|
|
523
|
+
log(`#${adjudication.issue} adjudication ${adjudication.state} → closed ${adjudication.prUrl}`);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/** The diff as one text, from the structured read the settlement audit uses.
|
|
528
|
+
* `renderAdjudicationBrief` bounds the total; this only flattens it, and says
|
|
529
|
+
* when the adapter itself had to cut the read short — an adjudicator must know
|
|
530
|
+
* its silence covers only what it saw. */
|
|
531
|
+
export function renderDiffForAdjudication(diff: PrDiff): string {
|
|
532
|
+
const files = diff.files.map((file) =>
|
|
533
|
+
[
|
|
534
|
+
`--- ${file.status} ${file.previousPath === undefined ? "" : `${file.previousPath} → `}${file.path}`,
|
|
535
|
+
file.hunks ?? "(no textual hunks: binary, mode-only or pure rename)",
|
|
536
|
+
].join("\n"),
|
|
537
|
+
);
|
|
538
|
+
return [...(diff.truncated ? ["[the tracker truncated this diff read]"] : []), ...files].join("\n\n");
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// ============================== review-correction launch selection (#1047)
|
|
542
|
+
//
|
|
543
|
+
// Why a review round has a *decision* at all, rather than just resuming:
|
|
544
|
+
// #1035. A capped implementation session (Ox, `model-empty-stop` at its turn
|
|
545
|
+
// ceiling) kept receiving correction rounds by `resume: true` after the fleet
|
|
546
|
+
// had already moved its worker model to DeepSeek. Every round replayed an
|
|
547
|
+
// exhausted transcript under the model nobody dispatches any more, the PR
|
|
548
|
+
// never moved, and fleet status said only `review-revision N`.
|
|
549
|
+
//
|
|
550
|
+
// So dispatch chooses, once per round, between two transports on ONE durable
|
|
551
|
+
// identity:
|
|
552
|
+
//
|
|
553
|
+
// - `resume-original` — the cheap path, unchanged: `continueRecent` on the
|
|
554
|
+
// run's own transcript, on the model the green run used. A healthy bounded
|
|
555
|
+
// round must keep this; it is what makes small corrections nearly free.
|
|
556
|
+
// - `fresh-correction` — #1046's transport: a blank session on the SAME
|
|
557
|
+
// branch, worktree, PR and reviewed head, briefed from durable evidence,
|
|
558
|
+
// launched on the model the fleet resolves TODAY.
|
|
559
|
+
//
|
|
560
|
+
// Two rules make the choice trustworthy rather than clever:
|
|
561
|
+
//
|
|
562
|
+
// 1. It is made from the run row BEFORE the review claim, because the claim
|
|
563
|
+
// deliberately clears `failureClass`/`recoveryAction`/`recoveredAt` so the
|
|
564
|
+
// resumed execution classifies fresh (#795). The diagnosis this decision
|
|
565
|
+
// rests on lives in exactly those columns, so reading them after the claim
|
|
566
|
+
// would read a blank row and always resume — the bug wearing a decision.
|
|
567
|
+
// 2. It is written ONCE per round (#1045) and re-read on every later claim,
|
|
568
|
+
// retry and restart. A round that was decided `fresh-correction` and then
|
|
569
|
+
// infra-killed relaunches fresh; it never quietly reverts because the row
|
|
570
|
+
// it was diagnosed from has since been rewritten.
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* The worker model a dispatch would resolve for this issue right now — the
|
|
574
|
+
* same ladder {@link handleIssue} walks: this chain's one model escalation
|
|
575
|
+
* (#807) outranks the project default, and the provider failover chain (#286)
|
|
576
|
+
* layers on top of whichever is primary.
|
|
577
|
+
*
|
|
578
|
+
* Deliberately NOT re-parsing the issue's `Model:` declaration: that value is
|
|
579
|
+
* an admission-time read of the issue thread, and a correction round reads no
|
|
580
|
+
* comments. The durable escalation marker — the only per-issue override that
|
|
581
|
+
* outranks config — is read, so an escalated chain still corrects on its
|
|
582
|
+
* stronger tier.
|
|
583
|
+
*/
|
|
584
|
+
export function currentWorkerModel(d: Pick<Deps, "project" | "store">, issue: number): string | undefined {
|
|
585
|
+
const { project, store } = d;
|
|
586
|
+
const facts = providerFailureFacts(store.runsForIssue(project.name, issue));
|
|
587
|
+
const escalation = store.modelEscalation(project.name, issue);
|
|
588
|
+
return resolveDispatchModel({
|
|
589
|
+
workerModel: escalation?.model ?? project.workerModel,
|
|
590
|
+
modelFallbacks: project.modelFallbacks,
|
|
591
|
+
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
592
|
+
streak: facts.streak,
|
|
593
|
+
}).model;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/** One round's launch decision, as dispatch made it or as the store returns
|
|
597
|
+
* it. `reason` is the clause a human reads and the fresh correction's brief
|
|
598
|
+
* carries; it is derived, never persisted, because the durable columns are
|
|
599
|
+
* the facts and this is the sentence about them. */
|
|
600
|
+
export interface ReviewLaunchDecision {
|
|
601
|
+
launchMode: ReviewLaunchMode;
|
|
602
|
+
requestedModel?: string;
|
|
603
|
+
originSessionRef?: string;
|
|
604
|
+
reason: string;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* Choose the transport for one PENDING round from the pre-claim run row
|
|
609
|
+
* (#1047).
|
|
610
|
+
*
|
|
611
|
+
* Fresh on any of three diagnosed facts, and resume otherwise:
|
|
612
|
+
*
|
|
613
|
+
* 1. the session is exhausted ({@link exhaustedSessionReason});
|
|
614
|
+
* 2. the transcript the round names is gone, so there is nothing to resume —
|
|
615
|
+
* which used to escalate the round away by hand;
|
|
616
|
+
* 3. the model moved: the run dispatched on a selector the fleet no longer
|
|
617
|
+
* resolves. Compared selector-to-selector only. `RunRecord.resolvedModel`
|
|
618
|
+
* is the harness's own resolution and lives in a different namespace from
|
|
619
|
+
* a configured selector (`@fast` resolves to a provider/model string), so
|
|
620
|
+
* comparing across the two would invent a model change on every role-alias
|
|
621
|
+
* project. A row with no recorded selector therefore cannot prove a
|
|
622
|
+
* configured change and keeps the cheap path — never a guess.
|
|
623
|
+
*/
|
|
624
|
+
export function selectReviewLaunch(
|
|
625
|
+
d: Pick<Deps, "project" | "store">,
|
|
626
|
+
revision: ReviewRevisionRecord,
|
|
627
|
+
run: RunRecord,
|
|
628
|
+
): ReviewLaunchDecision {
|
|
629
|
+
const originSessionRef = revision.sessionFile ?? run.sessionFile;
|
|
630
|
+
const resume = (reason: string): ReviewLaunchDecision => ({
|
|
631
|
+
launchMode: "resume-original",
|
|
632
|
+
...(run.model === undefined ? {} : { requestedModel: run.model }),
|
|
633
|
+
...(originSessionRef === undefined ? {} : { originSessionRef }),
|
|
634
|
+
reason,
|
|
635
|
+
});
|
|
636
|
+
const fresh = (reason: string): ReviewLaunchDecision => {
|
|
637
|
+
const requestedModel = currentWorkerModel(d, revision.issue);
|
|
638
|
+
return {
|
|
639
|
+
launchMode: "fresh-correction",
|
|
640
|
+
...(requestedModel === undefined ? {} : { requestedModel }),
|
|
641
|
+
...(originSessionRef === undefined ? {} : { originSessionRef }),
|
|
642
|
+
reason,
|
|
643
|
+
};
|
|
644
|
+
};
|
|
645
|
+
|
|
646
|
+
const exhausted = exhaustedSessionReason(run);
|
|
647
|
+
if (exhausted !== undefined) {
|
|
648
|
+
return fresh(`the implementation session cannot be resumed: ${exhausted}`);
|
|
649
|
+
}
|
|
650
|
+
if (originSessionRef === undefined || !existsSync(originSessionRef)) {
|
|
651
|
+
return fresh(
|
|
652
|
+
`the implementation transcript ${originSessionRef ?? "(none recorded)"} is gone, so there is no session to continue`,
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
const current = currentWorkerModel(d, revision.issue);
|
|
656
|
+
if (run.model !== undefined && current !== undefined && run.model !== current) {
|
|
657
|
+
return fresh(
|
|
658
|
+
`the fleet's worker model changed since the reviewed run: it ran on ${run.model}, dispatch now resolves ${current}`,
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
return resume("the reviewed session is bounded and healthy");
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
/**
|
|
665
|
+
* The decision for one round: the recorded one when there is one, otherwise
|
|
666
|
+
* this pass's selection, written once (#1045, #1047).
|
|
667
|
+
*
|
|
668
|
+
* The write is the arbiter, not this function: `recordReviewRevisionLaunch`
|
|
669
|
+
* refuses a contradictory second decision, and a refusal means another pass
|
|
670
|
+
* already decided — so the refused caller re-reads and follows the recorded
|
|
671
|
+
* decision rather than launching its own. That is what makes a restart, a
|
|
672
|
+
* dispatch retry and two racing passes all produce ONE correction.
|
|
673
|
+
*/
|
|
674
|
+
export function resolveReviewLaunch(
|
|
675
|
+
d: Pick<Deps, "project" | "store">,
|
|
676
|
+
revision: ReviewRevisionRecord,
|
|
677
|
+
run: RunRecord,
|
|
678
|
+
): ReviewLaunchDecision {
|
|
679
|
+
const recorded = d.store.reviewRevisionLaunch(d.project.name, revision.id);
|
|
680
|
+
if (recorded?.launchMode !== undefined) return fromRecordedLaunch(recorded);
|
|
681
|
+
const chosen = selectReviewLaunch(d, revision, run);
|
|
682
|
+
const wrote = d.store.recordReviewRevisionLaunch(d.project.name, revision.id, {
|
|
683
|
+
launchMode: chosen.launchMode,
|
|
684
|
+
...(chosen.requestedModel === undefined ? {} : { requestedModel: chosen.requestedModel }),
|
|
685
|
+
...(chosen.originSessionRef === undefined ? {} : { originSessionRef: chosen.originSessionRef }),
|
|
686
|
+
at: Date.now(),
|
|
687
|
+
});
|
|
688
|
+
if (wrote) return chosen;
|
|
689
|
+
const winner = d.store.reviewRevisionLaunch(d.project.name, revision.id);
|
|
690
|
+
if (winner?.launchMode === undefined) {
|
|
691
|
+
// No row, or a row the store refuses to decide: the round is not
|
|
692
|
+
// dispatchable as a correction, and the honest fallback is the transport
|
|
693
|
+
// that changes nothing. The dispatch pass's own claim already refuses a
|
|
694
|
+
// round whose run moved, so this is reachable only for a row that vanished
|
|
695
|
+
// between the two reads.
|
|
696
|
+
return {
|
|
697
|
+
...chosen,
|
|
698
|
+
launchMode: "resume-original",
|
|
699
|
+
reason: `${chosen.reason} — but no launch decision could be recorded for this round`,
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
return fromRecordedLaunch(winner);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/** The durable provenance read back as a decision. The reason names the store
|
|
706
|
+
* as its source, so a log line for a re-read decision cannot be mistaken for
|
|
707
|
+
* a fresh diagnosis of the row as it stands now. */
|
|
708
|
+
export function fromRecordedLaunch(recorded: ReviewCorrectionProvenance): ReviewLaunchDecision {
|
|
709
|
+
const launchMode = recorded.launchMode ?? "resume-original";
|
|
710
|
+
return {
|
|
711
|
+
launchMode,
|
|
712
|
+
...(recorded.requestedModel === undefined ? {} : { requestedModel: recorded.requestedModel }),
|
|
713
|
+
...(recorded.originSessionRef === undefined ? {} : { originSessionRef: recorded.originSessionRef }),
|
|
714
|
+
reason: `the decision already recorded for this round (${launchMode})`,
|
|
715
|
+
};
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* Everything a fresh correction is told, assembled from durable rows and live
|
|
720
|
+
* reads only (#1046, #1047).
|
|
721
|
+
*
|
|
722
|
+
* The shape of {@link handleReviewAdjudication}'s evidence gathering, and for
|
|
723
|
+
* the same reason: a session with no transcript has to be handed the facts, and
|
|
724
|
+
* the one source that must NOT be trusted is the worker's own prior report —
|
|
725
|
+
* that report is the claim the findings contradict. So the issue body is
|
|
726
|
+
* verbatim (the acceptance criteria are the standard), the diff and the check
|
|
727
|
+
* state come from the tracker at the exact reviewed head, and every prior round
|
|
728
|
+
* travels with its settled outcome so a fixed finding does not read as an
|
|
729
|
+
* outstanding one.
|
|
730
|
+
*
|
|
731
|
+
* Every read is individually best-effort: a tracker that refuses one fact
|
|
732
|
+
* degrades that line to a named absence rather than losing the round. Losing
|
|
733
|
+
* the round is worse — the findings are already durable, the PR is already
|
|
734
|
+
* green, and the correction is the only thing that can move either.
|
|
735
|
+
*/
|
|
736
|
+
export async function correctionEvidenceFor(
|
|
737
|
+
d: Deps,
|
|
738
|
+
revision: ReviewRevisionRecord,
|
|
739
|
+
run: RunRecord,
|
|
740
|
+
repo: RepoTarget,
|
|
741
|
+
args: { branch: string; worktree: string; reason: string },
|
|
742
|
+
): Promise<CorrectionEvidence> {
|
|
743
|
+
const issue = revision.issue;
|
|
744
|
+
const [title, body, diff, verification] = await Promise.all([
|
|
745
|
+
d.tracker.getIssue(issue).then(
|
|
746
|
+
(row) => row?.title,
|
|
747
|
+
() => undefined,
|
|
748
|
+
),
|
|
749
|
+
d.tracker.issueBody(issue).then(
|
|
750
|
+
(text) => text,
|
|
751
|
+
() => undefined,
|
|
752
|
+
),
|
|
753
|
+
d.tracker.prDiff(revision.prUrl).then(
|
|
754
|
+
(parsed) => parsed,
|
|
755
|
+
() => undefined,
|
|
756
|
+
),
|
|
757
|
+
d.tracker.verifyPr(revision.prUrl, revision.headSha).then(
|
|
758
|
+
(result) => result,
|
|
759
|
+
() => undefined,
|
|
760
|
+
),
|
|
761
|
+
]);
|
|
762
|
+
// Earlier rounds on this PR, oldest first, with what became of each. This
|
|
763
|
+
// round's own findings are carried separately and verbatim, so they are
|
|
764
|
+
// excluded here rather than repeated.
|
|
765
|
+
const priorRounds: AdjudicationRound[] = d.store
|
|
766
|
+
.reviewRevisionsForPr(d.project.name, revision.prUrl)
|
|
767
|
+
.filter((prior) => prior.round < revision.round)
|
|
768
|
+
.sort((a, b) => a.round - b.round)
|
|
769
|
+
.map((prior) => ({
|
|
770
|
+
round: prior.round,
|
|
771
|
+
findings: prior.findings,
|
|
772
|
+
...(prior.outcome === undefined ? {} : { outcome: prior.outcome }),
|
|
773
|
+
}));
|
|
774
|
+
const review = resolveReview(d.project);
|
|
775
|
+
return {
|
|
776
|
+
repo: knowledgeRepoKey(repo),
|
|
777
|
+
issue,
|
|
778
|
+
...(title === undefined ? {} : { issueTitle: title }),
|
|
779
|
+
issueBody: body ?? "(the issue body could not be read — the findings below are your contract)",
|
|
780
|
+
prUrl: revision.prUrl,
|
|
781
|
+
headSha: revision.headSha,
|
|
782
|
+
branch: args.branch,
|
|
783
|
+
worktree: args.worktree,
|
|
784
|
+
checks:
|
|
785
|
+
verification === undefined
|
|
786
|
+
? "(the check state at this head could not be read)"
|
|
787
|
+
: `${verification.status} — ${verification.reason}`,
|
|
788
|
+
diff:
|
|
789
|
+
diff === undefined
|
|
790
|
+
? "(the diff could not be read — read it from the worktree before you change anything)"
|
|
791
|
+
: renderDiffForAdjudication(diff),
|
|
792
|
+
round: revision.round,
|
|
793
|
+
maxRounds: review.maxRounds,
|
|
794
|
+
findings: revision.findings,
|
|
795
|
+
priorRounds,
|
|
796
|
+
reason: args.reason,
|
|
797
|
+
// The gates this worker may actually run (#1042): the CI-owned ones are
|
|
798
|
+
// not proof it can produce, and commanding one buys a refusal.
|
|
799
|
+
proofCommands: workerGateGroups(repo).local.map((gate) => `${gate.cmd} (from ${gate.cwd})`),
|
|
800
|
+
// `knowledge` is deliberately omitted: the transport reads the overlay for
|
|
801
|
+
// `evidence.repo` itself, so one reader owns the cap and the heading.
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* Run one review-correction round on an already-claimed run row
|
|
807
|
+
* (`pushed-green` → `running` by the dispatch pass), in whichever of the two
|
|
808
|
+
* transports #1047's decision names — and settle the row the way
|
|
809
|
+
* {@link handleIssue} settles any worker.
|
|
810
|
+
*
|
|
811
|
+
* `resume-original` resumes the SAME OMP session: same session directory,
|
|
812
|
+
* `resume: true` at the harness, the `sessionFile` lineage compared afterwards,
|
|
813
|
+
* on the same branch and PR. `fresh-correction` opens a blank session on that
|
|
814
|
+
* same branch, worktree, PR and reviewed head, briefed from durable evidence
|
|
815
|
+
* instead of a transcript (#1046), on the model the fleet resolves today.
|
|
816
|
+
*
|
|
817
|
+
* Both share one identity, and that is the point of the whole lane: the run row
|
|
818
|
+
* is reused, never cloned — same attempt number, same charges, same prUrl, same
|
|
819
|
+
* head, same lease — so a correction can never read as a second attempt or open
|
|
820
|
+
* a second PR. A revision round stays off both budgets because it is a state
|
|
821
|
+
* transition of one already-green row, green → running → green, and no new row
|
|
822
|
+
* exists for either counter to count. Only a correction worker that genuinely
|
|
823
|
+
* settles terminal falls through to the ordinary failure handling, budgets and
|
|
824
|
+
* all, exactly as today.
|
|
825
|
+
*
|
|
826
|
+
* `launch` is passed by the dispatch pass, which decided it from the pre-claim
|
|
827
|
+
* row. Absent (a direct caller, a test), the decision is resolved here from the
|
|
828
|
+
* store — the recorded one when one exists, so a re-entry after a restart still
|
|
829
|
+
* cannot re-decide.
|
|
830
|
+
*/
|
|
831
|
+
export async function handleReviewRevision(
|
|
832
|
+
d: Deps,
|
|
833
|
+
revision: ReviewRevisionRecord,
|
|
834
|
+
launch?: ReviewLaunchDecision,
|
|
835
|
+
): Promise<void> {
|
|
836
|
+
const { project, caps, tracker, store } = d;
|
|
837
|
+
const run = store.getRun(revision.runId);
|
|
838
|
+
if (run === undefined || run.state !== "running") {
|
|
839
|
+
log(
|
|
840
|
+
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${run?.state ?? "gone"} — ` +
|
|
841
|
+
"the row moved between the dispatch claim and the resume.",
|
|
842
|
+
);
|
|
843
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
844
|
+
return;
|
|
845
|
+
}
|
|
846
|
+
const issue = run.issue;
|
|
847
|
+
const branch = run.branch;
|
|
848
|
+
const runId = run.id;
|
|
849
|
+
const inProgress = project.stateLabels.inProgress;
|
|
850
|
+
const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
|
|
851
|
+
if (repo === undefined) {
|
|
852
|
+
log(`#${issue} review round ${revision.round} cannot dispatch: repo ${run.repo} is no longer routed`);
|
|
853
|
+
store.updateRun(runId, { state: "pushed-green", endedAt: Date.now() });
|
|
854
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
855
|
+
return;
|
|
856
|
+
}
|
|
857
|
+
const mirrorPath = mirrorPathFor(repo, project.mirrorRoot);
|
|
858
|
+
|
|
859
|
+
// The transport for this round (#1047). The dispatch pass decided it from the
|
|
860
|
+
// pre-claim row; a direct caller gets the recorded decision, or — for a round
|
|
861
|
+
// nobody has decided — a selection made from the row as it stands, which
|
|
862
|
+
// after the claim can only ever be more conservative.
|
|
863
|
+
const decision = launch ?? resolveReviewLaunch(d, revision, run);
|
|
864
|
+
const fresh = decision.launchMode === "fresh-correction";
|
|
865
|
+
|
|
866
|
+
// The session a resumed round continues is the one the pushed-green row
|
|
867
|
+
// recorded. A resume without that transcript cannot continue the same session
|
|
868
|
+
// — the exact fake this feature exists to prevent — so fail loudly instead of
|
|
869
|
+
// silently starting a fresh one. A `fresh-correction` round is not that case:
|
|
870
|
+
// it never claimed to continue anything, its evidence comes from durable rows
|
|
871
|
+
// and the live PR, and a missing transcript is one of the facts that selected
|
|
872
|
+
// it (#1047). Escalating there would page a human about the absence of a
|
|
873
|
+
// thing this round does not use.
|
|
874
|
+
const priorSessionFile = run.sessionFile;
|
|
875
|
+
if (!fresh && (priorSessionFile === undefined || !existsSync(priorSessionFile))) {
|
|
876
|
+
log(
|
|
877
|
+
`#${issue} review round ${revision.round} cannot resume: the recorded transcript ` +
|
|
878
|
+
`${priorSessionFile ?? "(none)"} is gone`,
|
|
879
|
+
);
|
|
880
|
+
store.updateRun(runId, {
|
|
881
|
+
state: "pushed-green",
|
|
882
|
+
endedAt: Date.now(),
|
|
883
|
+
lastError: `review round ${revision.round} could not resume: the run's transcript is gone`,
|
|
884
|
+
});
|
|
885
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
886
|
+
await safeEscalate(d, {
|
|
887
|
+
tier: 1,
|
|
888
|
+
project: project.name,
|
|
889
|
+
issue,
|
|
890
|
+
runId,
|
|
891
|
+
summary: `#${issue} review round ${revision.round} could not resume — the transcript is missing`,
|
|
892
|
+
detail: [
|
|
893
|
+
revision.prUrl,
|
|
894
|
+
"",
|
|
895
|
+
"The PR is still open and green; the run row was returned to pushed-green.",
|
|
896
|
+
"Resume the manual path (close + unblock + continuation) or merge it as it stands.",
|
|
897
|
+
].join("\n"),
|
|
898
|
+
});
|
|
899
|
+
return;
|
|
900
|
+
}
|
|
901
|
+
// The run's own session directory either way: a resumed round continues the
|
|
902
|
+
// newest transcript in it, and a fresh correction writes its own beside that
|
|
903
|
+
// one — the lineage of one run stays in one place, and the next round's
|
|
904
|
+
// `continueRecent` finds the correction rather than the transcript it
|
|
905
|
+
// replaced. Only one session per row ever runs at a time, so the shared
|
|
906
|
+
// control socket and staged settings are still one session's.
|
|
907
|
+
const sessionDir =
|
|
908
|
+
priorSessionFile === undefined
|
|
909
|
+
? join(stateDir(), "sessions", `run-${String(runId)}`)
|
|
910
|
+
: dirname(priorSessionFile);
|
|
911
|
+
mkdirSync(sessionDir, { recursive: true });
|
|
912
|
+
|
|
913
|
+
let worktreePath: string | undefined;
|
|
914
|
+
let runRepo: RunRepoRef | undefined;
|
|
915
|
+
let turnLimit: TurnLimitController | undefined;
|
|
916
|
+
let workerControl: WorkerControlSlot | undefined;
|
|
917
|
+
let workerSessionInstalled = false;
|
|
918
|
+
let verbListener: VerbListener | undefined;
|
|
919
|
+
const repoSlug = githubRepo(repo.cloneUrl);
|
|
920
|
+
|
|
921
|
+
/**
|
|
922
|
+
* Publishes the run's branch on the privileged side: run repo → mirror →
|
|
923
|
+
* GitHub, fast-forward only — the same route `handleIssue` uses, so the
|
|
924
|
+
* revised head the worker pushed through its own `conductor_push` is
|
|
925
|
+
* re-verified against the remote before the tree dies.
|
|
926
|
+
*/
|
|
927
|
+
const publish: RunPublisher = async () => {
|
|
928
|
+
if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
|
|
929
|
+
return pushRunBranch(project, runRepo);
|
|
930
|
+
};
|
|
931
|
+
|
|
932
|
+
const settleStopBeforeSession = async (): Promise<boolean> => {
|
|
933
|
+
const reason = workerControl?.requestedStop();
|
|
934
|
+
if (reason === undefined || workerSessionInstalled) return false;
|
|
935
|
+
turnLimit?.close();
|
|
936
|
+
turnLimit = undefined;
|
|
937
|
+
recordOperatorStop(store, {
|
|
938
|
+
project: project.name,
|
|
939
|
+
issue,
|
|
940
|
+
runId,
|
|
941
|
+
inProgress,
|
|
942
|
+
reason,
|
|
943
|
+
patch: {
|
|
944
|
+
endedAt: Date.now(),
|
|
945
|
+
report: ["Operator stopped the review revision before its session started.", `Reason: ${reason}`].join("\n"),
|
|
946
|
+
},
|
|
947
|
+
});
|
|
948
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
949
|
+
log(`#${issue} review round ${revision.round} stopped by operator before its session started: ${reason}`);
|
|
950
|
+
return true;
|
|
951
|
+
};
|
|
952
|
+
|
|
953
|
+
const settleDrainBeforeSession = async (): Promise<boolean> => {
|
|
954
|
+
if (d.drain?.draining !== true || workerSessionInstalled) return false;
|
|
955
|
+
turnLimit?.close();
|
|
956
|
+
turnLimit = undefined;
|
|
957
|
+
// A shutdown that interrupts the wake is the same situation as the
|
|
958
|
+
// provisioning-failure branch below, not an operator stop — nobody
|
|
959
|
+
// stopped anything, the PR is still green and still open, and only the
|
|
960
|
+
// wake failed; `settleStopBeforeSession` immediately above covers the
|
|
961
|
+
// case where an operator genuinely did. So mirror that branch: restore
|
|
962
|
+
// `pushed-green` naming the shutdown, and leave the in-progress label
|
|
963
|
+
// alone — the ordinary settle sweep releases it in the same breath it
|
|
964
|
+
// terminalises the row when the PR resolves. Stopping the row here would
|
|
965
|
+
// strand it: `settlePushedGreen` sweeps only pushed-* rows and
|
|
966
|
+
// classification excludes `stopped`, so the issue would carry neither
|
|
967
|
+
// label and nothing would ever revisit the still-green PR.
|
|
968
|
+
//
|
|
969
|
+
// The round itself is RE-QUEUED rather than settled `skipped` (#903). A
|
|
970
|
+
// daemon shutdown is the plainest infrastructure kill there is — no
|
|
971
|
+
// session started, so no round was worked — and a settled row consumed
|
|
972
|
+
// one of the PR's rounds through `latestReviewRound`, which is how an
|
|
973
|
+
// upgrade or a restart used to cost a review round it never spent. The
|
|
974
|
+
// retry is counted and bounded exactly like any other infra kill, so a
|
|
975
|
+
// host that shuts down mid-wake forever still escalates.
|
|
976
|
+
store.updateRun(runId, {
|
|
977
|
+
state: "pushed-green",
|
|
978
|
+
endedAt: Date.now(),
|
|
979
|
+
lastError: "daemon shutdown began after the review revision claim; the round was not launched",
|
|
980
|
+
});
|
|
981
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
982
|
+
if (retry.kind === "exhausted") {
|
|
983
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
984
|
+
log(
|
|
985
|
+
`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim, and ` +
|
|
986
|
+
`its infra retries are exhausted (${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settled skipped`,
|
|
987
|
+
);
|
|
988
|
+
return true;
|
|
989
|
+
}
|
|
990
|
+
log(
|
|
991
|
+
`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim — ` +
|
|
992
|
+
`re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
993
|
+
);
|
|
994
|
+
return true;
|
|
995
|
+
};
|
|
996
|
+
|
|
997
|
+
// Live-run controls, opened before any provisioning so the operator stop
|
|
998
|
+
// and shutdown fences cover the whole wake window, exactly as they do for a
|
|
999
|
+
// fresh claim in `handleIssue` (#374). The original run's entries were
|
|
1000
|
+
// closed at its settle, so reopening by issue is safe.
|
|
1001
|
+
//
|
|
1002
|
+
// #747: the round gets its own bounded allowance instead of the original
|
|
1003
|
+
// attempt's remainder. The resumed session continues the run's own counter
|
|
1004
|
+
// — the attempt's spent turns travel with the session — so a ceiling of
|
|
1005
|
+
// `run.maxTurns` would leave the round only `run.maxTurns - run.turns`,
|
|
1006
|
+
// and returning a PR from a near-cap run could not finish. Open at the
|
|
1007
|
+
// run's ceiling, then raise it by the round's own allowance — the run's
|
|
1008
|
+
// ceiling again, capped by the project's hard ceiling — through the same
|
|
1009
|
+
// override path `omp-conductor extend` uses, so the raise is persisted on
|
|
1010
|
+
// the row (the run's budget stays explainable in status) while `turns`
|
|
1011
|
+
// keeps accumulating the run's true cost: nothing is reset.
|
|
1012
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
|
|
1013
|
+
const revisionAllowance = Math.min(run.maxTurns, caps.workerMaxTurnsCeiling);
|
|
1014
|
+
const revisionCeiling = run.turns + revisionAllowance;
|
|
1015
|
+
const revisionRaise = d.turnLimits.extend(project.name, issue, revisionCeiling);
|
|
1016
|
+
if (revisionRaise.kind === "extended") {
|
|
1017
|
+
log(
|
|
1018
|
+
`#${issue} review round ${revision.round} turn ceiling ${run.maxTurns} → ${revisionCeiling} ` +
|
|
1019
|
+
`(+${revisionAllowance} round allowance on ${run.turns} spent)`,
|
|
1020
|
+
);
|
|
1021
|
+
}
|
|
1022
|
+
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
1023
|
+
if (await settleStopBeforeSession()) return;
|
|
1024
|
+
if (await settleDrainBeforeSession()) return;
|
|
1025
|
+
|
|
1026
|
+
try {
|
|
1027
|
+
// Reattach the run's own branch at the same per-issue path the run used:
|
|
1028
|
+
// a `pushed-green` settle removed the worktree, so provisioning is the
|
|
1029
|
+
// same continuation reattach as a normal re-claim. A capped/failed run's
|
|
1030
|
+
// tree was RETAINED at its terminal settle (`tree: "keep"`, #795) — the
|
|
1031
|
+
// reviewed PR is green, so the branch already holds the work and the kept
|
|
1032
|
+
// tree holds nothing the branch does not, unless its salvage failed. Clear
|
|
1033
|
+
// it the way restart recovery does (#692): re-attempt the salvage through
|
|
1034
|
+
// `tree: "remove"`, and refuse the removal if that still fails — the tree
|
|
1035
|
+
// is then the only copy of work and is never destroyed. A provisioning
|
|
1036
|
+
// failure restores the row — the PR is still green and still open, and
|
|
1037
|
+
// only the wake failed — and says so.
|
|
1038
|
+
try {
|
|
1039
|
+
if (run.worktree !== "" && existsSync(run.worktree)) {
|
|
1040
|
+
const keptSettlement = await settleWorktree({
|
|
1041
|
+
issue: run.issue,
|
|
1042
|
+
attempt: run.attempt,
|
|
1043
|
+
ending: `review round ${revision.round} resume`,
|
|
1044
|
+
worktree: run.worktree,
|
|
1045
|
+
branch: run.branch,
|
|
1046
|
+
publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
|
|
1047
|
+
tree: "remove",
|
|
1048
|
+
mirrorPath,
|
|
1049
|
+
});
|
|
1050
|
+
if (keptSettlement.retained) {
|
|
1051
|
+
log(
|
|
1052
|
+
`#${issue} review round ${revision.round} retained worktree ${run.worktree}: ` +
|
|
1053
|
+
"its salvage failed, so the tree is the only copy of work and will not be removed",
|
|
1054
|
+
);
|
|
1055
|
+
store.updateRun(runId, {
|
|
1056
|
+
state: "pushed-green",
|
|
1057
|
+
endedAt: Date.now(),
|
|
1058
|
+
lastError: `review round ${revision.round} could not clear the run's retained worktree: ${keptSettlement.lines.join(" ")}`,
|
|
1059
|
+
});
|
|
1060
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1061
|
+
await safeEscalate(d, {
|
|
1062
|
+
tier: 1,
|
|
1063
|
+
project: project.name,
|
|
1064
|
+
issue,
|
|
1065
|
+
runId,
|
|
1066
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched — the run's retained worktree is the only copy of work`,
|
|
1067
|
+
detail: [revision.prUrl, "", ...keptSettlement.lines].join("\n"),
|
|
1068
|
+
});
|
|
1069
|
+
return;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
|
|
1073
|
+
worktreePath = provisioned.path;
|
|
1074
|
+
runRepo = { repo, runRepoPath: worktreePath, branch };
|
|
1075
|
+
} catch (err) {
|
|
1076
|
+
const detail = errText(err);
|
|
1077
|
+
log(`#${issue} review round ${revision.round} cannot provision ${branch}: ${detail}`);
|
|
1078
|
+
// An operator stop that landed during provisioning wins over restoration —
|
|
1079
|
+
// it is a newer, explicit action, and the fences above already honoured
|
|
1080
|
+
// one that landed earlier.
|
|
1081
|
+
const pendingStop = workerControl?.requestedStop();
|
|
1082
|
+
if (pendingStop === undefined) {
|
|
1083
|
+
store.updateRun(runId, {
|
|
1084
|
+
state: "pushed-green",
|
|
1085
|
+
endedAt: Date.now(),
|
|
1086
|
+
lastError: `review round ${revision.round} could not provision the worktree: ${detail}`,
|
|
1087
|
+
});
|
|
1088
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1089
|
+
await safeEscalate(d, {
|
|
1090
|
+
tier: 1,
|
|
1091
|
+
project: project.name,
|
|
1092
|
+
issue,
|
|
1093
|
+
runId,
|
|
1094
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched — worktree provisioning failed`,
|
|
1095
|
+
detail: [revision.prUrl, "", detail].join("\n"),
|
|
1096
|
+
});
|
|
1097
|
+
} else {
|
|
1098
|
+
recordOperatorStop(store, {
|
|
1099
|
+
project: project.name,
|
|
1100
|
+
issue,
|
|
1101
|
+
runId,
|
|
1102
|
+
inProgress,
|
|
1103
|
+
reason: pendingStop,
|
|
1104
|
+
patch: {
|
|
1105
|
+
endedAt: Date.now(),
|
|
1106
|
+
report: [
|
|
1107
|
+
`Operator stopped the review revision before its session started: ${pendingStop}`,
|
|
1108
|
+
`(worktree provisioning also failed: ${detail})`,
|
|
1109
|
+
].join("\n"),
|
|
1110
|
+
},
|
|
1111
|
+
});
|
|
1112
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1113
|
+
}
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
if (await settleStopBeforeSession()) return;
|
|
1117
|
+
if (await settleDrainBeforeSession()) return;
|
|
1118
|
+
|
|
1119
|
+
// The fleet-owned omp settings overlay, staged under the SAME session
|
|
1120
|
+
// directory the transcript lives in — a resumed session loads it exactly
|
|
1121
|
+
// as the original run did (#537).
|
|
1122
|
+
const ompSettingsFile = materializeOmpSettings(project, sessionDir);
|
|
1123
|
+
|
|
1124
|
+
// The run's mutation channel (#126): same run id, same issue, same repo —
|
|
1125
|
+
// so `conductor_push` publishes the run's own branch and nothing else.
|
|
1126
|
+
verbListener = await listenVerbChannel(
|
|
1127
|
+
verbDeps(d),
|
|
1128
|
+
{
|
|
1129
|
+
kind: "run",
|
|
1130
|
+
path: verbSocketPath(ensureVerbSocketDir(stateDir()), `run-${String(issue)}`),
|
|
1131
|
+
project: project.name,
|
|
1132
|
+
role: "worker",
|
|
1133
|
+
runId,
|
|
1134
|
+
issue,
|
|
1135
|
+
repo,
|
|
1136
|
+
runRepoPath: worktreePath,
|
|
1137
|
+
branch,
|
|
1138
|
+
},
|
|
1139
|
+
{
|
|
1140
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
1141
|
+
},
|
|
1142
|
+
);
|
|
1143
|
+
if (await settleStopBeforeSession()) return;
|
|
1144
|
+
if (await settleDrainBeforeSession()) return;
|
|
1145
|
+
|
|
1146
|
+
// A review revision is a new worker process on the same run row. Clear the
|
|
1147
|
+
// prior process evidence before launch so a failed wake cannot inherit an
|
|
1148
|
+
// earlier round's harness route.
|
|
1149
|
+
store.updateRun(runId, { worktree: worktreePath, workerPid: null, terminalEvidence: null });
|
|
1150
|
+
|
|
1151
|
+
// The row's own counters are cumulative across the revision: the revision
|
|
1152
|
+
// worker meters its own session from zero, so its deltas are added to the
|
|
1153
|
+
// totals the run already recorded.
|
|
1154
|
+
const baseTurns = run.turns;
|
|
1155
|
+
const baseSpend = run.spendUsd;
|
|
1156
|
+
const baseOutputTokens = run.outputTokens ?? 0;
|
|
1157
|
+
const baseReasoningTokens = run.reasoningTokens ?? 0;
|
|
1158
|
+
|
|
1159
|
+
log(
|
|
1160
|
+
`#${issue} review round ${revision.round} → ${decision.launchMode} on ${branch} ` +
|
|
1161
|
+
`(${fresh ? "fresh session" : `resuming ${priorSessionFile ?? "(none)"}`}, model requested ` +
|
|
1162
|
+
`${decision.requestedModel ?? "harness default"}): ${decision.reason}`,
|
|
1163
|
+
);
|
|
1164
|
+
|
|
1165
|
+
let workerStartedAt: number | undefined;
|
|
1166
|
+
let result: WorkerResult;
|
|
1167
|
+
try {
|
|
1168
|
+
const runAllowanceUsd = runSpendAllowanceUsd(caps);
|
|
1169
|
+
// Everything except the brief, the transcript decision and the model is
|
|
1170
|
+
// shared by both transports, and shared by construction rather than by
|
|
1171
|
+
// two lists that have to be kept equal: a correction that metered its
|
|
1172
|
+
// turns, spend, tokens or pauses differently from a resume would be a
|
|
1173
|
+
// second accounting path for one run row, which is exactly what #1047
|
|
1174
|
+
// forbids.
|
|
1175
|
+
const common = {
|
|
1176
|
+
cwd: worktreePath,
|
|
1177
|
+
caps,
|
|
1178
|
+
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
1179
|
+
maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
|
|
1180
|
+
// A review round meters its own session from zero and gets the same
|
|
1181
|
+
// allowance as any run (#851) — the round is real work with a real
|
|
1182
|
+
// cost, and an unbounded round would reopen the hole for any PR that
|
|
1183
|
+
// was ever returned for revision.
|
|
1184
|
+
...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
|
|
1185
|
+
onPauseControl: (control: WorkerPauseControl) => {
|
|
1186
|
+
workerSessionInstalled = true;
|
|
1187
|
+
workerControl?.install(control);
|
|
1188
|
+
},
|
|
1189
|
+
sessionDir,
|
|
1190
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
1191
|
+
verbSocketPath: verbListener.path,
|
|
1192
|
+
onSpawn: (pid: number) => {
|
|
1193
|
+
workerStartedAt = Date.now();
|
|
1194
|
+
verbListener?.bindPid(pid);
|
|
1195
|
+
store.updateRun(runId, { workerPid: pid });
|
|
1196
|
+
},
|
|
1197
|
+
onChildLog: (line: string) => {
|
|
1198
|
+
log(`#${issue} ${line}`);
|
|
1199
|
+
},
|
|
1200
|
+
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
1201
|
+
releaseGrants: resolveReleaseGrants(project),
|
|
1202
|
+
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
1203
|
+
onTurn: (n: number) => store.updateRun(runId, { turns: baseTurns + n }),
|
|
1204
|
+
onSpend: (usd: number) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
|
|
1205
|
+
// A round runs on the SAME run, so its tokens add to the attempt's
|
|
1206
|
+
// totals exactly as its turns and spend do (#518) — resetting them would
|
|
1207
|
+
// make a long deliberation look like a fresh one.
|
|
1208
|
+
onTokens: (tokens: { output: number; reasoning: number }) =>
|
|
1209
|
+
store.updateRun(runId, {
|
|
1210
|
+
outputTokens: baseOutputTokens + tokens.output,
|
|
1211
|
+
reasoningTokens: baseReasoningTokens + tokens.reasoning,
|
|
1212
|
+
}),
|
|
1213
|
+
onKilled: () => {
|
|
1214
|
+
turnLimit?.close();
|
|
1215
|
+
turnLimit = undefined;
|
|
1216
|
+
},
|
|
1217
|
+
maySpawn: () => d.drain?.draining !== true,
|
|
1218
|
+
};
|
|
1219
|
+
if (fresh) {
|
|
1220
|
+
// #1046's transport: no `resume` — the option does not exist on it — a
|
|
1221
|
+
// brief built from durable evidence instead of a transcript, and the
|
|
1222
|
+
// model dispatch resolved TODAY rather than the one the rejected run
|
|
1223
|
+
// ran on. Same branch, same worktree, same PR, same reviewed head.
|
|
1224
|
+
const evidence = await correctionEvidenceFor(d, revision, run, repo, {
|
|
1225
|
+
branch,
|
|
1226
|
+
worktree: worktreePath,
|
|
1227
|
+
reason: decision.reason,
|
|
1228
|
+
});
|
|
1229
|
+
const correction = await runCorrectionWorker(
|
|
1230
|
+
{
|
|
1231
|
+
...common,
|
|
1232
|
+
evidence,
|
|
1233
|
+
...(decision.originSessionRef === undefined
|
|
1234
|
+
? {}
|
|
1235
|
+
: { originSessionRef: decision.originSessionRef }),
|
|
1236
|
+
...(decision.requestedModel === undefined ? {} : { model: decision.requestedModel }),
|
|
1237
|
+
onSessionFile: (f) => {
|
|
1238
|
+
store.updateRun(runId, { sessionFile: f });
|
|
1239
|
+
},
|
|
1240
|
+
},
|
|
1241
|
+
d.workerDeps,
|
|
1242
|
+
);
|
|
1243
|
+
result = correction;
|
|
1244
|
+
// The lineage is recorded only when the transport confirms it opened a
|
|
1245
|
+
// session of its own: `freshSession: false` means the harness handed
|
|
1246
|
+
// back the transcript we deliberately did not ask to continue, and
|
|
1247
|
+
// recording that as a correction lineage is #1046's named silent fake.
|
|
1248
|
+
if (correction.freshSession && correction.sessionRef !== undefined) {
|
|
1249
|
+
store.recordReviewCorrectionSession(
|
|
1250
|
+
project.name,
|
|
1251
|
+
revision.id,
|
|
1252
|
+
correction.sessionRef,
|
|
1253
|
+
correction.model,
|
|
1254
|
+
);
|
|
1255
|
+
log(
|
|
1256
|
+
`#${issue} review round ${revision.round} fresh correction session ${correction.sessionRef} ` +
|
|
1257
|
+
`ran on ${correction.model ?? "an unreported model"}` +
|
|
1258
|
+
`${correction.provider === undefined ? "" : ` (${correction.provider})`} ` +
|
|
1259
|
+
`(requested ${decision.requestedModel ?? "harness default"})`,
|
|
1260
|
+
);
|
|
1261
|
+
} else {
|
|
1262
|
+
log(
|
|
1263
|
+
`#${issue} review round ${revision.round} fresh correction did NOT open its own session ` +
|
|
1264
|
+
`(${correction.sessionRef ?? "no transcript reported"}): no correction lineage recorded`,
|
|
1265
|
+
);
|
|
1266
|
+
}
|
|
1267
|
+
} else {
|
|
1268
|
+
result = await runWorker(
|
|
1269
|
+
{
|
|
1270
|
+
...common,
|
|
1271
|
+
brief: renderReviewRevisionPrompt(revision.findings, revision.round),
|
|
1272
|
+
// The same-session half of the transport: `continueRecent` on the
|
|
1273
|
+
// run's own session directory, never a fresh `run-<uuid>` transcript.
|
|
1274
|
+
resume: true,
|
|
1275
|
+
// The continuation stays on the model the green run used (#286);
|
|
1276
|
+
// a resume onto a NEW model string is the diagnosed #1035 failure,
|
|
1277
|
+
// and the fresh transport above is how a model change is honoured.
|
|
1278
|
+
...(run.model === undefined ? {} : { model: run.model }),
|
|
1279
|
+
onSessionFile: (f) => {
|
|
1280
|
+
store.updateRun(runId, { sessionFile: f });
|
|
1281
|
+
// Same-file lineage is the proof the resume happened: a revision
|
|
1282
|
+
// that opened a different transcript fell back to a fresh session.
|
|
1283
|
+
if (f !== priorSessionFile) {
|
|
1284
|
+
log(
|
|
1285
|
+
`#${issue} review round ${revision.round} resume fell back to a fresh session: opened ${f} ` +
|
|
1286
|
+
`instead of ${priorSessionFile}`,
|
|
1287
|
+
);
|
|
1288
|
+
}
|
|
1289
|
+
},
|
|
1290
|
+
},
|
|
1291
|
+
d.workerDeps,
|
|
1292
|
+
);
|
|
1293
|
+
}
|
|
1294
|
+
} finally {
|
|
1295
|
+
turnLimit?.close();
|
|
1296
|
+
turnLimit = undefined;
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
if (result.modelFallbackMessage !== undefined) {
|
|
1300
|
+
log(`#${issue} review round ${revision.round} model fallback: ${result.modelFallbackMessage}`);
|
|
1301
|
+
}
|
|
1302
|
+
// What this round learned about the repo, into the fleet overlay the next
|
|
1303
|
+
// brief renders. A correction round is where the sharpest discoveries come
|
|
1304
|
+
// from — the worker has just been told what it got wrong — and a knowledge
|
|
1305
|
+
// write can never fail a settlement.
|
|
1306
|
+
recordWorkerDiscoveries(knowledgeRepoKey(repo), issue, result.discoveries);
|
|
1307
|
+
|
|
1308
|
+
const verified: { state: RunState; reason?: string } =
|
|
1309
|
+
result.state === "pushed-green"
|
|
1310
|
+
? await verifyPushedGreenClaim(tracker, result, {
|
|
1311
|
+
project: project.name,
|
|
1312
|
+
issue,
|
|
1313
|
+
runId,
|
|
1314
|
+
branch,
|
|
1315
|
+
repo: repoSlug ?? repo.name,
|
|
1316
|
+
store,
|
|
1317
|
+
})
|
|
1318
|
+
: { state: result.state };
|
|
1319
|
+
const state = verified.state;
|
|
1320
|
+
|
|
1321
|
+
const settlement =
|
|
1322
|
+
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
1323
|
+
? undefined
|
|
1324
|
+
: await settleWorktree({
|
|
1325
|
+
issue,
|
|
1326
|
+
attempt: run.attempt,
|
|
1327
|
+
ending:
|
|
1328
|
+
state === "blocked"
|
|
1329
|
+
? "blocked for an operator decision"
|
|
1330
|
+
: state === "stopped"
|
|
1331
|
+
? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
|
|
1332
|
+
: endedBy(result.killedBy),
|
|
1333
|
+
worktree: worktreePath,
|
|
1334
|
+
branch,
|
|
1335
|
+
publish,
|
|
1336
|
+
...(state === "failed" || state === "killed"
|
|
1337
|
+
? ({ tree: "keep" } as const)
|
|
1338
|
+
: ({ tree: "remove", mirrorPath } as const)),
|
|
1339
|
+
});
|
|
1340
|
+
if (settlement === undefined) {
|
|
1341
|
+
const published = await publish(branch);
|
|
1342
|
+
if (!published.ok) {
|
|
1343
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
1344
|
+
}
|
|
1345
|
+
await removeWorktree(mirrorPath, worktreePath);
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
// The settlement report names the review round first (#692): a run that
|
|
1349
|
+
// reaches green again after a round must read as a revision outcome, not
|
|
1350
|
+
// as a fresh run's first push, and a round that failed must still say the
|
|
1351
|
+
// round — the state and classification below carry the failure itself.
|
|
1352
|
+
const finalReport = [
|
|
1353
|
+
`review round ${revision.round}: ${state}`,
|
|
1354
|
+
...(verified.reason === undefined ? [] : ["", verified.reason]),
|
|
1355
|
+
result.report,
|
|
1356
|
+
].join("\n");
|
|
1357
|
+
const latestRun = store.getRun(runId) ?? run;
|
|
1358
|
+
const terminalEvidence =
|
|
1359
|
+
state === "failed" || state === "killed"
|
|
1360
|
+
? readTerminalEvidence(
|
|
1361
|
+
{
|
|
1362
|
+
sessionFile: result.sessionFile,
|
|
1363
|
+
startedAt: workerStartedAt ?? latestRun.startedAt,
|
|
1364
|
+
...(latestRun.workerPid === undefined ? {} : { workerPid: latestRun.workerPid }),
|
|
1365
|
+
},
|
|
1366
|
+
d.harnessLogDir ?? harnessLogDir(),
|
|
1367
|
+
)
|
|
1368
|
+
: undefined;
|
|
1369
|
+
|
|
1370
|
+
const terminalPatch: Partial<RunRecord> = {
|
|
1371
|
+
endedAt: Date.now(),
|
|
1372
|
+
turns: baseTurns + result.turns,
|
|
1373
|
+
spendUsd: baseSpend + result.spendUsd,
|
|
1374
|
+
provider429Count: result.provider429Count,
|
|
1375
|
+
...(result.model === undefined ? {} : { resolvedModel: result.model }),
|
|
1376
|
+
...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
|
|
1377
|
+
retryFallbacks: result.retryFallbacks,
|
|
1378
|
+
retryFallbackSucceeded: result.retryFallbackSucceeded,
|
|
1379
|
+
modelRecoveries: result.modelRecoveries,
|
|
1380
|
+
autoRetryCount: result.autoRetryCount,
|
|
1381
|
+
autoCompactionCount: result.autoCompactionCount,
|
|
1382
|
+
// The worker only reports these when it actually established them; a
|
|
1383
|
+
// failure whose report named no PR must not wipe what the row already
|
|
1384
|
+
// owns (#468).
|
|
1385
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
1386
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
1387
|
+
sessionFile: result.sessionFile,
|
|
1388
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
1389
|
+
...(terminalEvidence === undefined ? {} : { terminalEvidence }),
|
|
1390
|
+
report: finalReport,
|
|
1391
|
+
...settlement?.patch,
|
|
1392
|
+
};
|
|
1393
|
+
const terminalClassification =
|
|
1394
|
+
terminalEvidence === undefined
|
|
1395
|
+
? undefined
|
|
1396
|
+
: classifyRun(
|
|
1397
|
+
{
|
|
1398
|
+
...latestRun,
|
|
1399
|
+
...terminalPatch,
|
|
1400
|
+
state,
|
|
1401
|
+
terminalEvidence,
|
|
1402
|
+
},
|
|
1403
|
+
{},
|
|
1404
|
+
caps,
|
|
1405
|
+
);
|
|
1406
|
+
if (terminalClassification?.cls === "model-empty-stop") {
|
|
1407
|
+
const recoveredAt = Date.now();
|
|
1408
|
+
store.updateRun(runId, {
|
|
1409
|
+
...terminalPatch,
|
|
1410
|
+
state: "failed",
|
|
1411
|
+
failureClass: terminalClassification.cls,
|
|
1412
|
+
recoveryAction: terminalClassification.recovery,
|
|
1413
|
+
recoveredAt,
|
|
1414
|
+
});
|
|
1415
|
+
const continuations = store.continuationsFor(project.name, issue);
|
|
1416
|
+
if (hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
|
|
1417
|
+
// Keep the same review row, round, findings, session, branch and PR.
|
|
1418
|
+
// Restore the exact pushed-green origin state while moving this stop
|
|
1419
|
+
// into the explicit continuation counter; the next claim must not
|
|
1420
|
+
// depend on re-verifying a failed-state origin at an unchanged head.
|
|
1421
|
+
const chargedRun = store.getRun(runId) ?? latestRun;
|
|
1422
|
+
store.updateRun(runId, {
|
|
1423
|
+
...terminalPatch,
|
|
1424
|
+
state: "pushed-green",
|
|
1425
|
+
failureClass: null,
|
|
1426
|
+
recoveryAction: null,
|
|
1427
|
+
recoveredAt: null,
|
|
1428
|
+
continuationCharges: (chargedRun.continuationCharges ?? 0) + 1,
|
|
1429
|
+
workerPid: null,
|
|
1430
|
+
lastError: terminalClassification.evidence,
|
|
1431
|
+
});
|
|
1432
|
+
store.requeueReviewRevision(revision.id);
|
|
1433
|
+
log(
|
|
1434
|
+
`#${issue} review round ${revision.round} re-queued after a provider empty-stop ` +
|
|
1435
|
+
`(${continuations}/${caps.maxContinuationsPerIssue} continuations): ${terminalClassification.evidence}`,
|
|
1436
|
+
);
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
store.settleReviewRevision(revision.id, "failed", recoveredAt);
|
|
1441
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1442
|
+
const model = result.model ?? latestRun.resolvedModel ?? latestRun.model ?? "configured model";
|
|
1443
|
+
const provider = result.provider ?? latestRun.resolvedProvider ?? "provider";
|
|
1444
|
+
await safeEscalate(d, {
|
|
1445
|
+
tier: 1,
|
|
1446
|
+
project: project.name,
|
|
1447
|
+
issue,
|
|
1448
|
+
runId,
|
|
1449
|
+
summary: `#${issue} review round ${revision.round}: the model kept returning empty responses`,
|
|
1450
|
+
detail:
|
|
1451
|
+
`${provider}/${model} ended ${continuations} review session(s) with empty responses until ` +
|
|
1452
|
+
`the harness retry cap. The same review round was preserved while continuation budget remained; ` +
|
|
1453
|
+
`that budget is now exhausted at ${caps.maxContinuationsPerIssue}. No implementation failure or ` +
|
|
1454
|
+
`additional review round was charged.\n\n${terminalClassification.evidence}`,
|
|
1455
|
+
});
|
|
1456
|
+
log(`#${issue} review round ${revision.round} provider empty-stops exhausted the continuation budget`);
|
|
1457
|
+
return;
|
|
1458
|
+
}
|
|
1459
|
+
// #903: an infrastructure kill spends no review round.
|
|
1460
|
+
//
|
|
1461
|
+
// A dispatch that dies before the resumed session takes a turn — a host
|
|
1462
|
+
// permission fault, a spawn failure, a harness that could not start — did
|
|
1463
|
+
// none of the work a round exists to do, yet the pre-#903 path settled the
|
|
1464
|
+
// row `failed` with no retry. During the 2026-08-21/22 outage that turned
|
|
1465
|
+
// one broken mount into a review-ceiling deadlock: three such deaths
|
|
1466
|
+
// consumed the PR's three rounds, `conductor_pr_review` then refused any
|
|
1467
|
+
// further round, and the findings could never be addressed by a worker at
|
|
1468
|
+
// all. Nothing in the loop could recover it.
|
|
1469
|
+
//
|
|
1470
|
+
// So the round is returned to the pending set instead: the same row, the
|
|
1471
|
+
// same round number, the same findings, and the run restored to the exact
|
|
1472
|
+
// `pushed-green` state the verb recorded — which is what the restart
|
|
1473
|
+
// reconcile already does for a round a dying daemon interrupted. The next
|
|
1474
|
+
// dispatch pass resumes the same session. Bounded by
|
|
1475
|
+
// REVIEW_ROUND_INFRA_MAX_RETRIES and counted durably on the row, so a
|
|
1476
|
+
// permanently broken host escalates through the ordinary failed path
|
|
1477
|
+
// instead of retrying every tick forever.
|
|
1478
|
+
const infraKill =
|
|
1479
|
+
state === "failed" || state === "killed"
|
|
1480
|
+
? reviewRoundNeverWorked({
|
|
1481
|
+
turns: result.turns,
|
|
1482
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
1483
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
1484
|
+
...(settlement?.patch?.salvageSha === undefined
|
|
1485
|
+
? {}
|
|
1486
|
+
: { salvageSha: settlement.patch.salvageSha }),
|
|
1487
|
+
})
|
|
1488
|
+
: undefined;
|
|
1489
|
+
if (infraKill !== undefined) {
|
|
1490
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
1491
|
+
if (retry.kind === "requeued") {
|
|
1492
|
+
const detail =
|
|
1493
|
+
`review round ${revision.round} was killed before its session took a turn ` +
|
|
1494
|
+
`(${infraKill}); the round is re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`;
|
|
1495
|
+
store.updateRun(runId, {
|
|
1496
|
+
...terminalPatch,
|
|
1497
|
+
state: "pushed-green",
|
|
1498
|
+
// The transcript the round must still resume is the one the run
|
|
1499
|
+
// recorded, not whatever this dead dispatch opened: a session that
|
|
1500
|
+
// never took a turn has no lineage of its own to inherit.
|
|
1501
|
+
sessionFile: priorSessionFile,
|
|
1502
|
+
report: [`review round ${revision.round}: dispatch killed before turn 1`, "", detail].join("\n"),
|
|
1503
|
+
lastError: detail,
|
|
1504
|
+
});
|
|
1505
|
+
// No label swap and no escalation: nothing failed that a human can
|
|
1506
|
+
// act on yet, the issue keeps its in-progress label, and the next
|
|
1507
|
+
// dispatch pass picks the round up on its own.
|
|
1508
|
+
log(`#${issue} ${detail}`);
|
|
1509
|
+
return;
|
|
1510
|
+
}
|
|
1511
|
+
log(
|
|
1512
|
+
`#${issue} review round ${revision.round} exhausted its infra retries ` +
|
|
1513
|
+
`(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settling it the ordinary way`,
|
|
1514
|
+
);
|
|
1515
|
+
}
|
|
1516
|
+
if (state === "stopped") {
|
|
1517
|
+
recordOperatorStop(store, {
|
|
1518
|
+
project: project.name,
|
|
1519
|
+
issue,
|
|
1520
|
+
runId,
|
|
1521
|
+
inProgress,
|
|
1522
|
+
reason: result.stoppedReason ?? "no reason recorded",
|
|
1523
|
+
patch: terminalPatch,
|
|
1524
|
+
});
|
|
1525
|
+
} else {
|
|
1526
|
+
const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
|
|
1527
|
+
const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
|
|
1528
|
+
const providerTransient =
|
|
1529
|
+
providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
|
|
1530
|
+
const lastError = completionLastError(providerCredit, providerTransient, verified.reason, sessionErr);
|
|
1531
|
+
store.updateRun(runId, {
|
|
1532
|
+
...terminalPatch,
|
|
1533
|
+
state,
|
|
1534
|
+
...(lastError === undefined ? {} : { lastError }),
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
const outcome: ReviewRevisionOutcome =
|
|
1539
|
+
state === "pushed-green" ? "revised" : state === "pushed-pending" ? "pending" : "failed";
|
|
1540
|
+
store.settleReviewRevision(revision.id, outcome, Date.now());
|
|
1541
|
+
log(`#${issue} review round ${revision.round} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
1542
|
+
|
|
1543
|
+
// Post-settle outcomes, mirroring handleIssue: a failed or killed revision
|
|
1544
|
+
// worker falls through to exactly the same label, requeue and escalation
|
|
1545
|
+
// handling as any failed run — budgets included — and a blocked one pages
|
|
1546
|
+
// the orchestrator for a decision.
|
|
1547
|
+
if (state === "stopped") {
|
|
1548
|
+
log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
|
|
1549
|
+
} else if (state === "blocked") {
|
|
1550
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
1551
|
+
wakeOrchestratorForBlockedRun(project.name, issue);
|
|
1552
|
+
await safeEscalate(d, {
|
|
1553
|
+
tier: 1,
|
|
1554
|
+
project: project.name,
|
|
1555
|
+
issue,
|
|
1556
|
+
runId,
|
|
1557
|
+
summary: `#${issue} is blocked on review round ${revision.round} and needs a decision`,
|
|
1558
|
+
detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
|
|
1559
|
+
});
|
|
1560
|
+
} else if (state === "failed" || state === "killed") {
|
|
1561
|
+
const continuation = store.continuationsFor(project.name, issue);
|
|
1562
|
+
const continueTurns = shouldContinueAfterTurnsCap({
|
|
1563
|
+
killedBy: result.killedBy,
|
|
1564
|
+
prUrl: result.prUrl,
|
|
1565
|
+
headSha: result.headSha,
|
|
1566
|
+
salvageSha: settlement?.patch?.salvageSha,
|
|
1567
|
+
continuation,
|
|
1568
|
+
maxContinuations: caps.maxContinuationsPerIssue,
|
|
1569
|
+
});
|
|
1570
|
+
if (continueTurns) {
|
|
1571
|
+
// The same chokepoint the implementation path uses: one ordered pair
|
|
1572
|
+
// plus the dispatch wake, so a capped review round is picked up on an
|
|
1573
|
+
// immediate pass (#201, #1041).
|
|
1574
|
+
await swapToQueue(d, issue, inProgress);
|
|
1575
|
+
log(
|
|
1576
|
+
`#${issue} review round ${revision.round} turns-cap, continuation ` +
|
|
1577
|
+
`${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
|
|
1578
|
+
);
|
|
1579
|
+
await safeEscalate(d, {
|
|
1580
|
+
tier: 1,
|
|
1581
|
+
project: project.name,
|
|
1582
|
+
issue,
|
|
1583
|
+
runId,
|
|
1584
|
+
summary: `#${issue} hit the turns cap on review round ${revision.round} — auto-requeued for continuation`,
|
|
1585
|
+
detail: [
|
|
1586
|
+
`${result.prUrl ?? "(no PR URL)"}`,
|
|
1587
|
+
"",
|
|
1588
|
+
"The queue label is back on; the next tick should reattach the branch",
|
|
1589
|
+
"and open a continuation brief. No failed label was applied.",
|
|
1590
|
+
"",
|
|
1591
|
+
result.report,
|
|
1592
|
+
].join("\n"),
|
|
1593
|
+
});
|
|
1594
|
+
} else {
|
|
1595
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1596
|
+
await safeEscalate(d, {
|
|
1597
|
+
tier: 1,
|
|
1598
|
+
project: project.name,
|
|
1599
|
+
issue,
|
|
1600
|
+
runId,
|
|
1601
|
+
summary: result.killedBy
|
|
1602
|
+
? `#${issue} was killed on review round ${revision.round} by the ${result.killedBy} cap`
|
|
1603
|
+
: `#${issue} failed on review round ${revision.round}`,
|
|
1604
|
+
detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
|
|
1605
|
+
});
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
} catch (err) {
|
|
1609
|
+
// A crash anywhere after the worker ran settles like any run's dispatch
|
|
1610
|
+
// error: the row is terminal, the tree is kept, and the issue is relabelled
|
|
1611
|
+
// so nobody silently re-claims it.
|
|
1612
|
+
turnLimit?.close();
|
|
1613
|
+
turnLimit = undefined;
|
|
1614
|
+
const detail = errText(err);
|
|
1615
|
+
log(`#${issue} review round ${revision.round} errored: ${detail}`);
|
|
1616
|
+
const settlement =
|
|
1617
|
+
worktreePath === undefined
|
|
1618
|
+
? undefined
|
|
1619
|
+
: await settleWorktree({
|
|
1620
|
+
issue,
|
|
1621
|
+
attempt: run.attempt,
|
|
1622
|
+
ending: "killed by a dispatch error",
|
|
1623
|
+
worktree: worktreePath,
|
|
1624
|
+
branch,
|
|
1625
|
+
publish,
|
|
1626
|
+
tree: "keep",
|
|
1627
|
+
});
|
|
1628
|
+
// A crash BEFORE the worker session was installed spent no round (#903):
|
|
1629
|
+
// nothing resumed, nothing was reviewed, and the round is re-queued for
|
|
1630
|
+
// the next pass instead of consuming one of the PR's three. Once the
|
|
1631
|
+
// session is installed the round belongs to the worker, and a crash after
|
|
1632
|
+
// that settles the ordinary way below — including the exhausted case, so
|
|
1633
|
+
// a dispatch that keeps throwing still reaches a human.
|
|
1634
|
+
if (!workerSessionInstalled) {
|
|
1635
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
1636
|
+
if (retry.kind === "requeued") {
|
|
1637
|
+
store.updateRun(runId, {
|
|
1638
|
+
state: "pushed-green",
|
|
1639
|
+
endedAt: Date.now(),
|
|
1640
|
+
sessionFile: priorSessionFile,
|
|
1641
|
+
lastError:
|
|
1642
|
+
`review round ${revision.round} could not be dispatched (${detail}); the round is re-queued, ` +
|
|
1643
|
+
`retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
1644
|
+
...settlement?.patch,
|
|
1645
|
+
});
|
|
1646
|
+
log(
|
|
1647
|
+
`#${issue} review round ${revision.round} re-queued after a pre-session dispatch error — ` +
|
|
1648
|
+
`retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
1649
|
+
);
|
|
1650
|
+
return;
|
|
1651
|
+
}
|
|
1652
|
+
log(
|
|
1653
|
+
`#${issue} review round ${revision.round} exhausted its infra retries ` +
|
|
1654
|
+
`(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — failing the round`,
|
|
1655
|
+
);
|
|
1656
|
+
}
|
|
1657
|
+
store.updateRun(runId, {
|
|
1658
|
+
state: "failed",
|
|
1659
|
+
endedAt: Date.now(),
|
|
1660
|
+
lastError: detail,
|
|
1661
|
+
...settlement?.patch,
|
|
1662
|
+
});
|
|
1663
|
+
store.settleReviewRevision(revision.id, "failed", Date.now());
|
|
1664
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1665
|
+
await safeEscalate(d, {
|
|
1666
|
+
tier: 1,
|
|
1667
|
+
project: project.name,
|
|
1668
|
+
issue,
|
|
1669
|
+
runId,
|
|
1670
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched`,
|
|
1671
|
+
detail: [
|
|
1672
|
+
`${revision.prUrl}`,
|
|
1673
|
+
"",
|
|
1674
|
+
detail,
|
|
1675
|
+
...(settlement?.lines ?? []),
|
|
1676
|
+
].join("\n"),
|
|
1677
|
+
});
|
|
1678
|
+
} finally {
|
|
1679
|
+
turnLimit?.close();
|
|
1680
|
+
if (verbListener !== undefined) {
|
|
1681
|
+
try {
|
|
1682
|
+
await verbListener.close();
|
|
1683
|
+
} catch (err) {
|
|
1684
|
+
log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
|
|
1685
|
+
}
|
|
1686
|
+
verbListener = undefined;
|
|
1687
|
+
}
|
|
1688
|
+
workerControl?.close();
|
|
1689
|
+
workerControl = undefined;
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
/**
|
|
1694
|
+
* Restart recovery for in-flight review revisions (#692): a revision the
|
|
1695
|
+
* previous daemon claimed (`pushed-green` → `running`) and lost when it died
|
|
1696
|
+
* is restored to the exact state the orchestrator's verb left it in — same
|
|
1697
|
+
* run, same PR, same findings, same round, same session file — so the next
|
|
1698
|
+
* dispatch pass resumes the SAME OMP session rather than leaving the run to
|
|
1699
|
+
* the failure classifier.
|
|
1700
|
+
*
|
|
1701
|
+
* Runs AFTER the startup orphan sweep, on purpose: the sweep has already
|
|
1702
|
+
* salvaged the crashed revision's worktree to the branch (so no work is lost)
|
|
1703
|
+
* and marked the row `orphaned` — and this reconcile then restores that row
|
|
1704
|
+
* to `pushed-green` and re-queues the round before the first tick, so the
|
|
1705
|
+
* failure classifier never sees it. That ordering matters twice over: an
|
|
1706
|
+
* orphaned row would charge the continuation budget for a round that must
|
|
1707
|
+
* stay off it (#677), and the `review_revisions` row is the only place the
|
|
1708
|
+
* findings and round live. A revision the previous daemon never claimed (its
|
|
1709
|
+
* run is still `pushed-green`, its row still pending) needs no restoration —
|
|
1710
|
+
* the ordinary dispatch pass wakes it on the next tick, exactly as #677
|
|
1711
|
+
* already guarantees.
|
|
1712
|
+
*
|
|
1713
|
+
* A round whose run cannot be restored — the run is gone, settled terminal,
|
|
1714
|
+
* or its repo is no longer routed — fails closed instead: the revision is
|
|
1715
|
+
* settled `skipped` with the findings still on the row, ONE escalation names
|
|
1716
|
+
* what could not be restored, and no worker is started against a PR that may
|
|
1717
|
+
* already have one.
|
|
1718
|
+
*/
|
|
1719
|
+
export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRevisionRecord[]> {
|
|
1720
|
+
const { project, store } = d;
|
|
1721
|
+
const revisions = store.unsettledReviewRevisions(project.name);
|
|
1722
|
+
if (revisions.length === 0) return [];
|
|
1723
|
+
const recovered: ReviewRevisionRecord[] = [];
|
|
1724
|
+
for (const revision of revisions) {
|
|
1725
|
+
const run = store.getRun(revision.runId);
|
|
1726
|
+
if (run === undefined) {
|
|
1727
|
+
log(
|
|
1728
|
+
`#${revision.issue} review round ${revision.round} cannot be restored across the restart: ` +
|
|
1729
|
+
`run ${revision.runId} no longer exists`,
|
|
1730
|
+
);
|
|
1731
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1732
|
+
await safeEscalate(d, {
|
|
1733
|
+
tier: 1,
|
|
1734
|
+
project: project.name,
|
|
1735
|
+
issue: revision.issue,
|
|
1736
|
+
runId: revision.runId,
|
|
1737
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
1738
|
+
detail: [
|
|
1739
|
+
revision.prUrl,
|
|
1740
|
+
"",
|
|
1741
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
1742
|
+
`the run row ${revision.runId} no longer exists.`,
|
|
1743
|
+
"The round was settled skipped; the findings stay on this revision's durable row.",
|
|
1744
|
+
"Re-review the PR (it is still open and green) or merge it as it stands.",
|
|
1745
|
+
].join("\n"),
|
|
1746
|
+
});
|
|
1747
|
+
continue;
|
|
1748
|
+
}
|
|
1749
|
+
// A revision the previous daemon never claimed: the run is still a
|
|
1750
|
+
// settled green row and the round is still pending, so the ordinary
|
|
1751
|
+
// dispatch pass wakes it on the next tick untouched. Same for a
|
|
1752
|
+
// capped/failed run whose review round is pending (#795): a `failed` /
|
|
1753
|
+
// `killed` row is terminal too — no process died on this round, the
|
|
1754
|
+
// revision was never claimed, and the ordinary dispatch pass claims it
|
|
1755
|
+
// exactly like a pending round on a pushed-green row.
|
|
1756
|
+
if (run.state === "pushed-green" || run.state === "failed" || run.state === "killed") continue;
|
|
1757
|
+
// The run a previous daemon claimed for this round and died on: the
|
|
1758
|
+
// orphan sweep just marked it `orphaned` (salvaging the tree to the
|
|
1759
|
+
// branch), so restore it to the reviewable state the verb recorded.
|
|
1760
|
+
const crashed =
|
|
1761
|
+
run.state === "claimed" || run.state === "running" || run.state === "orphaned";
|
|
1762
|
+
if (!crashed) {
|
|
1763
|
+
log(
|
|
1764
|
+
`#${revision.issue} review round ${revision.round} cannot be restored across the restart: run ` +
|
|
1765
|
+
`${revision.runId} is ${run.state}`,
|
|
1766
|
+
);
|
|
1767
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1768
|
+
await safeEscalate(d, {
|
|
1769
|
+
tier: 1,
|
|
1770
|
+
project: project.name,
|
|
1771
|
+
issue: revision.issue,
|
|
1772
|
+
runId: revision.runId,
|
|
1773
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
1774
|
+
detail: [
|
|
1775
|
+
revision.prUrl,
|
|
1776
|
+
"",
|
|
1777
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
1778
|
+
`the run row is ${run.state}, not a state a review round resumes from.`,
|
|
1779
|
+
"The round was settled skipped; the findings stay on this revision's durable row.",
|
|
1780
|
+
"Re-review the PR (it is still open and green) or merge it as it stands.",
|
|
1781
|
+
].join("\n"),
|
|
1782
|
+
});
|
|
1783
|
+
continue;
|
|
1784
|
+
}
|
|
1785
|
+
try {
|
|
1786
|
+
// The crashed revision's worktree — salvaged and published by the orphan
|
|
1787
|
+
// sweep that just ran, so the branch in the mirror has every committed
|
|
1788
|
+
// byte — must be gone before the next dispatch, or `addRunRepo` refuses
|
|
1789
|
+
// the reattach as possibly holding a previous attempt's work (#692).
|
|
1790
|
+
// `settleWorktree` with `tree: "remove"` re-attempts the salvage instead
|
|
1791
|
+
// of trusting it (a tree whose salvage failed is the only copy of work
|
|
1792
|
+
// and is NEVER removed), and refuses the removal if that re-attempt
|
|
1793
|
+
// fails — the fail-closed half of this recovery.
|
|
1794
|
+
if (run.worktree !== "" && existsSync(run.worktree)) {
|
|
1795
|
+
const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
|
|
1796
|
+
if (repo === undefined) {
|
|
1797
|
+
log(
|
|
1798
|
+
`#${revision.issue} review round ${revision.round} cannot clear worktree ${run.worktree}: ` +
|
|
1799
|
+
`repo ${run.repo} is no longer routed — the tree may hold the only copy of work`,
|
|
1800
|
+
);
|
|
1801
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1802
|
+
await safeEscalate(d, {
|
|
1803
|
+
tier: 1,
|
|
1804
|
+
project: project.name,
|
|
1805
|
+
issue: revision.issue,
|
|
1806
|
+
runId: revision.runId,
|
|
1807
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
1808
|
+
detail: [
|
|
1809
|
+
revision.prUrl,
|
|
1810
|
+
"",
|
|
1811
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
1812
|
+
`repo ${run.repo} is no longer routed, so the crashed revision's worktree cannot be published and removed.`,
|
|
1813
|
+
`The worktree is retained for recovery: ${run.worktree}`,
|
|
1814
|
+
"Recover it by hand (it may hold the round's last work), then re-review or merge the PR.",
|
|
1815
|
+
].join("\n"),
|
|
1816
|
+
});
|
|
1817
|
+
continue;
|
|
1818
|
+
}
|
|
1819
|
+
const settlement = await settleWorktree({
|
|
1820
|
+
issue: run.issue,
|
|
1821
|
+
attempt: run.attempt,
|
|
1822
|
+
ending: "interrupted by a daemon restart during a review round",
|
|
1823
|
+
worktree: run.worktree,
|
|
1824
|
+
branch: run.branch,
|
|
1825
|
+
publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
|
|
1826
|
+
tree: "remove",
|
|
1827
|
+
mirrorPath: mirrorPathFor(repo, project.mirrorRoot),
|
|
1828
|
+
});
|
|
1829
|
+
if (settlement.retained) {
|
|
1830
|
+
log(
|
|
1831
|
+
`#${revision.issue} review round ${revision.round} worktree ${run.worktree} retained: ` +
|
|
1832
|
+
"its salvage failed, so the tree is the only copy of work and will not be removed",
|
|
1833
|
+
);
|
|
1834
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1835
|
+
await safeEscalate(d, {
|
|
1836
|
+
tier: 1,
|
|
1837
|
+
project: project.name,
|
|
1838
|
+
issue: revision.issue,
|
|
1839
|
+
runId: revision.runId,
|
|
1840
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
1841
|
+
detail: [
|
|
1842
|
+
revision.prUrl,
|
|
1843
|
+
"",
|
|
1844
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
1845
|
+
"the crashed revision's worktree could not be salvaged, so it is the only copy of work and was retained.",
|
|
1846
|
+
`Recover the tree by hand: ${run.worktree}`,
|
|
1847
|
+
"Then re-review the PR (still open and green) or merge it as it stands.",
|
|
1848
|
+
].join("\n"),
|
|
1849
|
+
});
|
|
1850
|
+
continue;
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
} catch (err) {
|
|
1854
|
+
// A settleWorktree throw (mirror unreachable, git refused) must not take
|
|
1855
|
+
// the whole startup reconcile down with one run's tree: fail this round
|
|
1856
|
+
// closed and let the next one try.
|
|
1857
|
+
const detail = errText(err);
|
|
1858
|
+
log(`#${revision.issue} review round ${revision.round} restore errored: ${detail}`);
|
|
1859
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
1860
|
+
await safeEscalate(d, {
|
|
1861
|
+
tier: 1,
|
|
1862
|
+
project: project.name,
|
|
1863
|
+
issue: revision.issue,
|
|
1864
|
+
runId: revision.runId,
|
|
1865
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
1866
|
+
detail: [revision.prUrl, "", detail].join("\n"),
|
|
1867
|
+
});
|
|
1868
|
+
continue;
|
|
1869
|
+
}
|
|
1870
|
+
// The exact state the verb recorded: a pushed-green row plus a pending
|
|
1871
|
+
// revision whose durable row still carries the findings, reviewed head,
|
|
1872
|
+
// round and target session. The next tick's dispatch pass claims and
|
|
1873
|
+
// wakes it exactly like a revision that was never dispatched.
|
|
1874
|
+
store.updateRun(revision.runId, {
|
|
1875
|
+
state: "pushed-green",
|
|
1876
|
+
endedAt: Date.now(),
|
|
1877
|
+
worktree: "",
|
|
1878
|
+
lastError: `review round ${revision.round} was restored across a daemon restart; the round is re-queued`,
|
|
1879
|
+
});
|
|
1880
|
+
store.requeueReviewRevision(revision.id);
|
|
1881
|
+
log(
|
|
1882
|
+
`#${revision.issue} review round ${revision.round} restored across restart: run ${revision.runId} ` +
|
|
1883
|
+
"is pushed-green again and the round is re-queued — the next dispatch pass resumes the same session",
|
|
1884
|
+
);
|
|
1885
|
+
recovered.push(revision);
|
|
1886
|
+
}
|
|
1887
|
+
return recovered;
|
|
1888
|
+
}
|