omp-conductor 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +11 -7
- package/package.json +1 -1
- package/schema/config.schema.json +24 -1
- package/src/admission.ts +104 -1
- package/src/ask.ts +39 -3
- package/src/backups.ts +2 -2
- package/src/briefs/orchestrator.md +1 -0
- package/src/briefs/worker.md +36 -19
- package/src/command-help.ts +8 -1
- package/src/command-manifest.ts +5 -2
- package/src/commands/watch.ts +62 -3
- package/src/config-schema.ts +33 -0
- package/src/config.ts +60 -1
- package/src/daemon.ts +253 -1479
- package/src/decisions.ts +51 -6
- package/src/depends-on.ts +261 -1
- package/src/diff-flags.ts +350 -0
- package/src/digest-schedule.ts +37 -0
- package/src/failure-class.ts +15 -2
- package/src/fleet.ts +20 -2
- package/src/graph-health.ts +20 -7
- package/src/graph.ts +313 -68
- package/src/lifecycle.ts +36 -5
- package/src/omp.ts +42 -0
- package/src/orchestrator-tick.ts +104 -12
- package/src/routing.ts +11 -3
- package/src/session-host.ts +16 -0
- package/src/settlement.ts +1728 -0
- package/src/setup-install.ts +91 -30
- package/src/setup-wizard.ts +75 -0
- package/src/setup.ts +93 -3
- package/src/status-render.ts +31 -9
- package/src/store.ts +78 -5
- package/src/tracker/github.ts +46 -0
- package/src/types.ts +127 -3
- package/src/verbs/server.ts +58 -2
- package/src/worker.ts +214 -7
- package/src/worktree.ts +115 -8
|
@@ -0,0 +1,1728 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The settlement half of the dispatcher — push-result sweeps, orphan
|
|
3
|
+
* reconciliation, worktree settlement and failure classification recovery
|
|
4
|
+
* (`daemon.ts` composes; this owns the region).
|
|
5
|
+
*
|
|
6
|
+
* Extracted from `daemon.ts` (TerrifiedBug/conductor#705) so the second most
|
|
7
|
+
* contested region of the composition root is a lane of its own: a settlement
|
|
8
|
+
* diff must not have to touch the composition root. This is a move, not a
|
|
9
|
+
* rewrite — behaviour is byte-identical; a later slice reworks the seams.
|
|
10
|
+
*
|
|
11
|
+
* The module is a leaf: it never imports `daemon.ts`, and `daemon.ts` never
|
|
12
|
+
* re-exports it. What it needs from the daemon that only the daemon owns —
|
|
13
|
+
* the pause-sentinel ops used by the provider-credit recovery — arrives as
|
|
14
|
+
* fields of the structural {@link SettlementDeps}, wired at the call sites.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
18
|
+
import { log, errText, safeEscalate } from "./log.ts";
|
|
19
|
+
import { hasContinuationBudget } from "./admission.ts";
|
|
20
|
+
import {
|
|
21
|
+
UNREADABLE_TREE_FLAG,
|
|
22
|
+
analyseSettlement,
|
|
23
|
+
deriveChangedLine,
|
|
24
|
+
} from "./diff-flags.ts";
|
|
25
|
+
import { classifyRun, normalise, type ClassifyFacts } from "./failure-class.ts";
|
|
26
|
+
import { formatModelsTried, modelsTried } from "./model-fallback.ts";
|
|
27
|
+
import { PR_LOOKUP_WINDOW_MS } from "./verbs/server.ts";
|
|
28
|
+
import {
|
|
29
|
+
removeWorktree,
|
|
30
|
+
salvageWip,
|
|
31
|
+
type RunPublisher,
|
|
32
|
+
type SalvageOutcome,
|
|
33
|
+
} from "./worktree.ts";
|
|
34
|
+
import type {
|
|
35
|
+
Caps,
|
|
36
|
+
Escalation,
|
|
37
|
+
FailureClass,
|
|
38
|
+
MergedPrInfo,
|
|
39
|
+
OpenCloser,
|
|
40
|
+
PrState,
|
|
41
|
+
ProjectConfig,
|
|
42
|
+
RecoveryAction,
|
|
43
|
+
RunRecord,
|
|
44
|
+
RunState,
|
|
45
|
+
SettlementFlag,
|
|
46
|
+
Store,
|
|
47
|
+
Tracker,
|
|
48
|
+
} from "./types.ts";
|
|
49
|
+
|
|
50
|
+
/** Fleet-wide escalations still need an issue number in the payload; 0 is the
|
|
51
|
+
* sentinel that reads as "no issue" in every renderer. One per module —
|
|
52
|
+
* `admission.ts`, `daemon.ts` and this module each hold their own copy. */
|
|
53
|
+
const NO_ISSUE = 0;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The slice of the daemon's `Deps` settlement reads. Defined here rather than
|
|
57
|
+
* imported from `daemon.ts` (that would be the cycle this leaf exists to
|
|
58
|
+
* avoid); the composition root wires it at the call sites, and
|
|
59
|
+
* `daemon.test.ts`'s shared fake satisfies it (its `deps()` helper gained the
|
|
60
|
+
* two pause-op passthroughs below when this module landed).
|
|
61
|
+
*
|
|
62
|
+
* `isPaused`/`setPaused` are the pause-sentinel ops, owned by the daemon
|
|
63
|
+
* (the pause cluster has a long list of other consumers that import them from
|
|
64
|
+
* there); the composition root wires them in. Keeping them off the object is
|
|
65
|
+
* what lets this module stay a leaf.
|
|
66
|
+
*/
|
|
67
|
+
export interface SettlementDeps {
|
|
68
|
+
project: ProjectConfig;
|
|
69
|
+
caps: Caps;
|
|
70
|
+
tracker: Tracker;
|
|
71
|
+
store: Store;
|
|
72
|
+
escalate(e: Escalation): Promise<void>;
|
|
73
|
+
/** Whether the pause sentinel is set, daemon-owned. */
|
|
74
|
+
isPaused(project?: string): boolean;
|
|
75
|
+
/** Writes the pause sentinel, daemon-owned. */
|
|
76
|
+
setPaused(
|
|
77
|
+
paused: boolean,
|
|
78
|
+
provenance: { source: string; reason?: string },
|
|
79
|
+
project?: string,
|
|
80
|
+
): void;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** What the settlement audit of one green run produced: the advisory flags
|
|
84
|
+
* (test weakening only — the file-list disclosure is derived, not flagged),
|
|
85
|
+
* whether the diff was cut short, and the `changed:` line composed from the
|
|
86
|
+
* PR's own diff. */
|
|
87
|
+
export interface SettlementAuditResult {
|
|
88
|
+
flags: SettlementFlag[];
|
|
89
|
+
truncated: boolean;
|
|
90
|
+
/** The `changed:` file list derived from the PR's diff, present whenever the
|
|
91
|
+
* diff could be read. Absent means the tree could not be read, and `flags`
|
|
92
|
+
* then carries exactly {@link UNREADABLE_TREE_FLAG}. */
|
|
93
|
+
changedLine?: string;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Audit a worker's own account of its work against the pull request it pushed.
|
|
98
|
+
*
|
|
99
|
+
* The thin half of the split #85 established: this fetches, {@link
|
|
100
|
+
* analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
|
|
101
|
+
* shares none of its authority — that function decides a run's state, this one
|
|
102
|
+
* cannot, by construction. It returns evidence and the caller appends it.
|
|
103
|
+
*
|
|
104
|
+
* A diff that cannot be read is a finding (`changed-line-missing`), never an
|
|
105
|
+
* empty flag list: nothing was derived and nothing was checked, and that must
|
|
106
|
+
* not read as a clean bill.
|
|
107
|
+
*/
|
|
108
|
+
export async function collectSettlementFlags(
|
|
109
|
+
tracker: Pick<Tracker, "prDiff" | "prBody">,
|
|
110
|
+
claim: { prUrl?: string; issueText: string; sessionFile?: string },
|
|
111
|
+
): Promise<SettlementAuditResult> {
|
|
112
|
+
if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
113
|
+
// The diff and the claimed-proof body read in parallel; both are advisory,
|
|
114
|
+
// and an unreadable either one costs that half of the audit and never a
|
|
115
|
+
// run's state.
|
|
116
|
+
const [diff, prBody] = await Promise.all([
|
|
117
|
+
tracker.prDiff(claim.prUrl),
|
|
118
|
+
tracker.prBody(claim.prUrl),
|
|
119
|
+
]);
|
|
120
|
+
if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
121
|
+
// Absent session/transcript stays silent — no command can be checked —
|
|
122
|
+
// never "every claim is verified", the same "could not read" posture as
|
|
123
|
+
// `changed-line-missing`.
|
|
124
|
+
let transcript: string | undefined;
|
|
125
|
+
if (claim.sessionFile !== undefined) {
|
|
126
|
+
try {
|
|
127
|
+
transcript = readFileSync(claim.sessionFile, "utf8");
|
|
128
|
+
} catch {
|
|
129
|
+
transcript = undefined;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
flags: analyseSettlement({
|
|
134
|
+
issueText: claim.issueText,
|
|
135
|
+
diff,
|
|
136
|
+
...(prBody === undefined ? {} : { prBody }),
|
|
137
|
+
...(transcript === undefined ? {} : { transcript }),
|
|
138
|
+
}),
|
|
139
|
+
truncated: diff.truncated,
|
|
140
|
+
changedLine: deriveChangedLine(diff),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
// ------------------------------------------------------------------- pushed-green settlement
|
|
147
|
+
|
|
148
|
+
/** What a resolved PR turns its `pushed-green` row into. */
|
|
149
|
+
export interface Settlement {
|
|
150
|
+
state: "merged" | "failed";
|
|
151
|
+
/** The log line after `#<n> settled: `, and — for a rejection — the row's own
|
|
152
|
+
* `lastError`, because a `failed` row whose worker succeeded has to say so. */
|
|
153
|
+
reason: string;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* What one `pushed-green` row becomes now its PR has an answer, or undefined to
|
|
158
|
+
* leave the row exactly as it is.
|
|
159
|
+
*
|
|
160
|
+
* A `pushed-green` row is the only one nothing ever revisited: the worker is
|
|
161
|
+
* finished, `reconcileOrphanedRuns` only settles rows that held a process, and
|
|
162
|
+
* `merged` went unwritten from day one. So they accumulated — three of them on
|
|
163
|
+
* the reference fleet on 2026-08-07, every PR merged and every issue closed,
|
|
164
|
+
* with `/healthz` still reporting three active runs and their issues
|
|
165
|
+
* permanently unclaimable, because the busy set *is* the active set (#18).
|
|
166
|
+
*
|
|
167
|
+
* The mapping, and why each answer is the only honest one:
|
|
168
|
+
*
|
|
169
|
+
* - `merged` — the work landed. That is what `merged` was reserved for.
|
|
170
|
+
* - `closed` — a human read the work and said no. Leaving it `pushed-green`
|
|
171
|
+
* forever is a lie; `failed` records that it did not land and releases the
|
|
172
|
+
* busy guard, so an issue a human re-queues can be attempted again. A row
|
|
173
|
+
* that had reached `pushed-green` or `pushed-pending` is classified
|
|
174
|
+
* `returned-for-revision` at settlement. A review decision asks for another
|
|
175
|
+
* implementation pass, not a failure, so it consumes the continuation budget
|
|
176
|
+
* instead of the failed-attempt budget.
|
|
177
|
+
* - `open`, and undefined — nothing changes. Undefined is "could not tell": a
|
|
178
|
+
* flaky network, a revoked token, a deleted PR. Settling on it would record a
|
|
179
|
+
* merge that never happened, and the next tick asks again for free. An
|
|
180
|
+
* ambiguous answer must never settle a row.
|
|
181
|
+
*/
|
|
182
|
+
export function settlementFor(pr: PrState | undefined, prUrl: string): Settlement | undefined {
|
|
183
|
+
if (pr === "merged") return { state: "merged", reason: `${prUrl} merged` };
|
|
184
|
+
if (pr === "closed") return { state: "failed", reason: `${prUrl} closed without merging` };
|
|
185
|
+
return undefined;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Records the in-progress label's release as a projection op (#201).
|
|
190
|
+
*
|
|
191
|
+
* Settlement used to write only half of what it knew. On 2026-08-09 that cost
|
|
192
|
+
* the reference fleet two issues in one night: veltro#331 settled to `failed`
|
|
193
|
+
* at 23:53Z once the orchestrator closed veltro#332 unmerged, and veltro#344
|
|
194
|
+
* settled to `merged` at 05:54Z once chad#452 squash-merged. Both rows left the
|
|
195
|
+
* active set correctly; an authoritative `gh issue view` on each afterwards
|
|
196
|
+
* still showed `agent:in-progress` — permanently unclaimable with no supported
|
|
197
|
+
* way back (#18).
|
|
198
|
+
*
|
|
199
|
+
* The outbox makes the row transition and the label one fact again: the
|
|
200
|
+
* removal is enqueued in the same breath as the row is terminalised, the
|
|
201
|
+
* projector applies it with unbounded retry, and while it is pending the
|
|
202
|
+
* eligibility overlay treats the label as already gone. A tracker that refuses
|
|
203
|
+
* the write (403, rate limit) can no longer strand the row — that is `#184`
|
|
204
|
+
* and `#198` closed. No `pushed-*` row is ever written terminal with its
|
|
205
|
+
* label release owed but unrecorded, because enqueueing is a local store write
|
|
206
|
+
* that cannot fail on the tracker.
|
|
207
|
+
*
|
|
208
|
+
* Synchronous. The op is durable the moment this returns.
|
|
209
|
+
*/
|
|
210
|
+
export function releaseInProgress(
|
|
211
|
+
d: Pick<SettlementDeps, "project" | "store">,
|
|
212
|
+
issue: number,
|
|
213
|
+
why: string,
|
|
214
|
+
): void {
|
|
215
|
+
const label = d.project.stateLabels.inProgress;
|
|
216
|
+
d.store.enqueueLabelOps(d.project.name, [{ issue, op: "remove", label }]);
|
|
217
|
+
log(`#${issue} released ${label} (queued): ${why}`);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Records the queue label's release as a projection op (#607).
|
|
222
|
+
*
|
|
223
|
+
* A settled merge means the work landed. Whether the merge closed the issue or
|
|
224
|
+
* the PR carried no closing keyword and left it open, the issue must not stay
|
|
225
|
+
* claimable — and the queue label is the claim gate. Before this function
|
|
226
|
+
* existed nothing ever took it off a settled issue: the claim path only adds
|
|
227
|
+
* the in-progress label, so `ready-for-agent` rode along through the run and
|
|
228
|
+
* outlived the merge. By 2026-08-17 the closed-issue backlog still carrying it
|
|
229
|
+
* exceeded GitHub's 50-result listing page on this fleet's tracker.
|
|
230
|
+
*
|
|
231
|
+
* Same outbox discipline as {@link releaseInProgress}: the removal is a
|
|
232
|
+
* durable local write enqueued in the same breath as the terminal row, the
|
|
233
|
+
* projector applies it with unbounded retry, and while it is pending the
|
|
234
|
+
* eligibility overlay treats the label as already gone. A removal that races a
|
|
235
|
+
* human's own removal lands as a no-op on the tracker (REST DELETE of an
|
|
236
|
+
* absent label is idempotent), so it can never resurrect a label the operator
|
|
237
|
+
* took off.
|
|
238
|
+
*
|
|
239
|
+
* Synchronous. The op is durable the moment this returns.
|
|
240
|
+
*/
|
|
241
|
+
export function releaseQueueLabel(
|
|
242
|
+
d: Pick<SettlementDeps, "project" | "store">,
|
|
243
|
+
issue: number,
|
|
244
|
+
why: string,
|
|
245
|
+
): void {
|
|
246
|
+
const label = d.project.queueLabel;
|
|
247
|
+
d.store.enqueueLabelOps(d.project.name, [{ issue, op: "remove", label }]);
|
|
248
|
+
log(`#${issue} released ${label} (queued): ${why}`);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Asks the tracker about every `pushed-green` PR and settles the ones that
|
|
254
|
+
* resolved.
|
|
255
|
+
*
|
|
256
|
+
* Effects at the call site, decision in {@link settlementFor} — the same split
|
|
257
|
+
* as `checkIntegrity`/`watchOrchestrator`. Exported like `admitCandidates`
|
|
258
|
+
* rather than kept private, because half of what has to hold is about the sweep
|
|
259
|
+
* and not the mapping: that a row without a PR costs no API call, and that one
|
|
260
|
+
* unreachable PR does not stop the others from settling.
|
|
261
|
+
*
|
|
262
|
+
* The label is released too, which reverses what this function first promised.
|
|
263
|
+
* It used to leave tracker labels alone exactly as {@link reconcileOrphanedRuns}
|
|
264
|
+
* does, reasoning that a merge closes the issue anyway and that deciding what an
|
|
265
|
+
* issue's labels should say next is the orchestrator's drain duty. There turned
|
|
266
|
+
* out to be no such path: on 2026-08-09 two settled rows left their issues
|
|
267
|
+
* carrying `agent:in-progress` forever, with the brief forbidding the
|
|
268
|
+
* orchestrator from touching it and `unblock` declining to (see
|
|
269
|
+
* {@link releaseInProgress}). The row transition and the label are one fact, and
|
|
270
|
+
* writing half of it is the whole of that bug.
|
|
271
|
+
*
|
|
272
|
+
* Releasing it is safe here specifically because of what these rows are. A
|
|
273
|
+
* `pushed-green` or `pushed-pending` row has no process behind it — its worker
|
|
274
|
+
* exited and its worktree is gone — so a terminal answer about its PR proves no
|
|
275
|
+
* worker owns the issue, and the duplicate-dispatch interlock the label exists
|
|
276
|
+
* for is spent. {@link reconcileOrphanedRuns} still leaves labels alone for the
|
|
277
|
+
* opposite reason: an orphaned `running` row is work nobody has read yet. And
|
|
278
|
+
* the brief's rule stays absolute, because this is a daemon-owned write through
|
|
279
|
+
* the same Tracker port the dispatcher claimed the issue with — orphan detection
|
|
280
|
+
* is only trustworthy while every state label on the tracker came from this
|
|
281
|
+
* package.
|
|
282
|
+
*
|
|
283
|
+
* The two writes are ordered label-then-row, and the order is load-bearing. This
|
|
284
|
+
* sweep is the only thing that revisits a `pushed-*` row, so the terminal state
|
|
285
|
+
* is also the row's exit from it: written first, a tracker that then failed on
|
|
286
|
+
* the label would leave `agent:in-progress` with nothing left to retry it — #18
|
|
287
|
+
* exactly, in the last window able to reach it. Writing the label first makes
|
|
288
|
+
* failure cost a repeated `gh` call on the next tick instead, and the row stays
|
|
289
|
+
* in the busy set throughout, so no second worker can be sent at the issue while
|
|
290
|
+
* it waits.
|
|
291
|
+
*/
|
|
292
|
+
export async function settlePushedGreen(
|
|
293
|
+
d: Pick<SettlementDeps, "project" | "tracker" | "store">,
|
|
294
|
+
): Promise<number> {
|
|
295
|
+
const { project, tracker, store } = d;
|
|
296
|
+
// Runs the sweep resolved by terminalising the row. Every terminal write
|
|
297
|
+
// below increments it; the tick attributes it as `settled` on the pass's
|
|
298
|
+
// dispatch record, so a held pass is visible as work done, not just as a
|
|
299
|
+
// clock that moved (#497).
|
|
300
|
+
let settled = 0;
|
|
301
|
+
// Filtered from the active set rather than asked for with a new query: active
|
|
302
|
+
// is live workers plus these, so the list is bounded by the worker cap plus
|
|
303
|
+
// the number of PRs awaiting a merge — a handful, by construction. A fleet
|
|
304
|
+
// where that is not a handful has a merge problem, not a dispatch one.
|
|
305
|
+
const pending = store
|
|
306
|
+
.activeRuns(project.name)
|
|
307
|
+
.filter((r) => r.state === "pushed-green" || r.state === "pushed-pending");
|
|
308
|
+
|
|
309
|
+
for (const run of pending) {
|
|
310
|
+
// Nothing to ask about. A pushed result requires a PR, so a malformed row
|
|
311
|
+
// must not buy a `gh` call every five minutes forever.
|
|
312
|
+
if (run.prUrl === undefined) continue;
|
|
313
|
+
|
|
314
|
+
let pr: PrState | undefined;
|
|
315
|
+
try {
|
|
316
|
+
pr = await tracker.prState(run.prUrl);
|
|
317
|
+
} catch (err) {
|
|
318
|
+
// Per row, like admission's held candidate. The GitHub adapter already
|
|
319
|
+
// answers undefined instead of throwing, so this catch is the port's
|
|
320
|
+
// contract rather than that adapter's behaviour — and a tracker that does
|
|
321
|
+
// throw must cost its own row, not the whole sweep.
|
|
322
|
+
log(`#${run.issue} not settled: PR state lookup failed (${errText(err)}) — retrying next tick`);
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
const settlement = settlementFor(pr, run.prUrl);
|
|
327
|
+
if (settlement !== undefined) {
|
|
328
|
+
// A mediated merge enters a second, bounded observation phase. Record the
|
|
329
|
+
// exact merge commit before the row leaves the active set; if GitHub cannot
|
|
330
|
+
// supply it yet, retry this settlement next tick rather than create a
|
|
331
|
+
// merged row whose base result can never be attributed.
|
|
332
|
+
let merged: MergedPrInfo | undefined;
|
|
333
|
+
if (settlement.state === "merged") {
|
|
334
|
+
try {
|
|
335
|
+
merged = await tracker.mergedPrInfo(run.prUrl);
|
|
336
|
+
} catch (err) {
|
|
337
|
+
log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
if (merged === undefined) {
|
|
341
|
+
log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// The label removal and the terminal row are one fact again (#201): the
|
|
347
|
+
// release is enqueued — a durable local write that cannot fail on the
|
|
348
|
+
// tracker — in the same breath as the row is terminalised.
|
|
349
|
+
releaseInProgress(d, run.issue, settlement.reason);
|
|
350
|
+
if (settlement.state === "merged") {
|
|
351
|
+
// #607: merged work is done — whether the merge closed the issue or
|
|
352
|
+
// the PR carried no closing keyword and left it open — so the claim
|
|
353
|
+
// gate comes off too, through the same durable projection. A failed
|
|
354
|
+
// settlement (closed without merging) keeps the queue label: the
|
|
355
|
+
// issue is meant to return to the queue for another attempt.
|
|
356
|
+
releaseQueueLabel(d, run.issue, settlement.reason);
|
|
357
|
+
}
|
|
358
|
+
const patch: Partial<RunRecord> = {
|
|
359
|
+
state: settlement.state,
|
|
360
|
+
endedAt: Date.now(),
|
|
361
|
+
...(merged === undefined
|
|
362
|
+
? {}
|
|
363
|
+
: {
|
|
364
|
+
mergeSha: merged.mergeSha,
|
|
365
|
+
baseRef: merged.baseRef,
|
|
366
|
+
baseCheck: "pending",
|
|
367
|
+
}),
|
|
368
|
+
};
|
|
369
|
+
if (settlement.state === "failed") {
|
|
370
|
+
patch.lastError = settlement.reason;
|
|
371
|
+
patch.failureClass = "returned-for-revision";
|
|
372
|
+
patch.recoveryAction = "none";
|
|
373
|
+
}
|
|
374
|
+
store.updateRun(run.id, patch);
|
|
375
|
+
settled += 1;
|
|
376
|
+
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (run.state !== "pushed-pending" || pr !== "open" || run.headSha === undefined) continue;
|
|
381
|
+
let verification;
|
|
382
|
+
try {
|
|
383
|
+
verification = await tracker.verifyPr(run.prUrl, run.headSha);
|
|
384
|
+
} catch (err) {
|
|
385
|
+
log(`#${run.issue} checks not settled (${errText(err)}) — retrying next tick`);
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
if (verification === undefined) continue;
|
|
389
|
+
if (verification.status === "green") {
|
|
390
|
+
store.updateRun(run.id, { state: "pushed-green", lastError: null });
|
|
391
|
+
log(`#${run.issue} checks settled: ${verification.reason}`);
|
|
392
|
+
} else if (verification.status === "failed") {
|
|
393
|
+
// Equally terminal, so the release is enqueued before the row writes,
|
|
394
|
+
// for the same reason as the settlement branch above (see there). The
|
|
395
|
+
// green branch releases nothing — that row is still awaiting a merge,
|
|
396
|
+
// and its live PR is exactly the work the label must keep guarding.
|
|
397
|
+
releaseInProgress(d, run.issue, verification.reason);
|
|
398
|
+
store.updateRun(run.id, { state: "failed", lastError: verification.reason });
|
|
399
|
+
settled += 1;
|
|
400
|
+
log(`#${run.issue} checks failed: ${verification.reason}`);
|
|
401
|
+
} else {
|
|
402
|
+
store.updateRun(run.id, { lastError: verification.reason });
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
return settled;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
// ------------------------------------------------------------------- salvaged-PR adoption
|
|
410
|
+
|
|
411
|
+
const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
|
|
412
|
+
failed: true,
|
|
413
|
+
killed: true,
|
|
414
|
+
orphaned: true,
|
|
415
|
+
blocked: true,
|
|
416
|
+
};
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Reattaches a recovered PR to the terminal run that owns it (#245).
|
|
420
|
+
*
|
|
421
|
+
* A worker can fail before its completion report records `prUrl`, then have its
|
|
422
|
+
* dirty tree committed and pushed by salvage. If that branch already has a PR,
|
|
423
|
+
* the orchestrator otherwise has no policy-compliant path to inspect or merge
|
|
424
|
+
* it: ownership is store-backed. Adoption is deliberately stricter than
|
|
425
|
+
* admission. The tracker query proves the PR closes this run's issue; exact
|
|
426
|
+
* branch and canonical repository matches prove it is this run's recovered
|
|
427
|
+
* work, not an unrelated closer. Missing identity is refusal, never a guess.
|
|
428
|
+
*
|
|
429
|
+
* The newest run per issue is inspected, at most ten per tick and only inside
|
|
430
|
+
* the same 30-day window as mediated PR verbs. The cursor advances through the
|
|
431
|
+
* full eligible set so persistent non-matches cannot starve older recovered
|
|
432
|
+
* work. Successful adoption is idempotent because the row gains `prUrl`;
|
|
433
|
+
* non-matches are logged once per daemon process.
|
|
434
|
+
*/
|
|
435
|
+
const rejectedSalvagedPrRuns = new Set<string>();
|
|
436
|
+
const salvagedPrCursor = new Map<string, string>();
|
|
437
|
+
|
|
438
|
+
export async function adoptSalvagedPrs(
|
|
439
|
+
d: Pick<SettlementDeps, "project" | "tracker" | "store">,
|
|
440
|
+
now = Date.now(),
|
|
441
|
+
): Promise<void> {
|
|
442
|
+
const { project, tracker, store } = d;
|
|
443
|
+
const eligible = store
|
|
444
|
+
.recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
|
|
445
|
+
.filter(
|
|
446
|
+
(run) =>
|
|
447
|
+
ADOPTABLE_PR_STATES[run.state] === true &&
|
|
448
|
+
run.prUrl === undefined &&
|
|
449
|
+
run.branch.trim() !== "",
|
|
450
|
+
);
|
|
451
|
+
const previous = salvagedPrCursor.get(project.name);
|
|
452
|
+
const previousIndex =
|
|
453
|
+
previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
|
|
454
|
+
const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
|
|
455
|
+
const candidates = Array.from(
|
|
456
|
+
{ length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
|
|
457
|
+
(_, offset) => eligible[(start + offset) % eligible.length]!,
|
|
458
|
+
);
|
|
459
|
+
const last = candidates.at(-1);
|
|
460
|
+
if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
|
|
461
|
+
|
|
462
|
+
for (const run of candidates) {
|
|
463
|
+
const repo = project.routing.repos[run.repo];
|
|
464
|
+
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
465
|
+
let closers: OpenCloser[];
|
|
466
|
+
try {
|
|
467
|
+
closers = await tracker.openClosersFor(run.issue);
|
|
468
|
+
} catch (err) {
|
|
469
|
+
log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
const closer = closers.find(
|
|
474
|
+
(candidate) =>
|
|
475
|
+
candidate.headRefName !== "" &&
|
|
476
|
+
candidate.headRefName === run.branch &&
|
|
477
|
+
repo !== undefined &&
|
|
478
|
+
candidate.repo !== "" &&
|
|
479
|
+
candidate.repo === repoIdentity,
|
|
480
|
+
);
|
|
481
|
+
if (closer === undefined) {
|
|
482
|
+
if (!rejectedSalvagedPrRuns.has(run.id)) {
|
|
483
|
+
const observed = closers[0];
|
|
484
|
+
const reason =
|
|
485
|
+
observed === undefined
|
|
486
|
+
? "no open closing PR"
|
|
487
|
+
: observed.headRefName === ""
|
|
488
|
+
? "closer has no head branch identity"
|
|
489
|
+
: observed.headRefName !== run.branch
|
|
490
|
+
? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
|
|
491
|
+
: observed.repo === ""
|
|
492
|
+
? "closer has no repository identity"
|
|
493
|
+
: `closer repository ${observed.repo} does not match routed repository`;
|
|
494
|
+
log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
|
|
495
|
+
rejectedSalvagedPrRuns.add(run.id);
|
|
496
|
+
}
|
|
497
|
+
continue;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
const flag: SettlementFlag = {
|
|
501
|
+
kind: "pr-adopted",
|
|
502
|
+
file: "(recovery)",
|
|
503
|
+
detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
|
|
504
|
+
};
|
|
505
|
+
store.updateRun(run.id, {
|
|
506
|
+
prUrl: closer.url,
|
|
507
|
+
settlementFlags: [...(run.settlementFlags ?? []), flag],
|
|
508
|
+
});
|
|
509
|
+
rejectedSalvagedPrRuns.delete(run.id);
|
|
510
|
+
log(
|
|
511
|
+
`#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
|
|
512
|
+
(run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/** Canonical `owner/repo` identity from a configured network clone URL. */
|
|
518
|
+
function githubRepo(cloneUrl: string): string | undefined {
|
|
519
|
+
const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
|
|
520
|
+
const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
|
|
521
|
+
normalized,
|
|
522
|
+
);
|
|
523
|
+
return match?.[1];
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
// ------------------------------------------------------------------- worktree settlement
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* Persist the operator-stop transition before releasing its live controller.
|
|
531
|
+
* The row is terminal first, then its in-progress label is removed through the
|
|
532
|
+
* same durable projection outbox as every other lifecycle transition.
|
|
533
|
+
*/
|
|
534
|
+
export function recordOperatorStop(
|
|
535
|
+
store: Pick<Store, "updateRun" | "enqueueLabelOps">,
|
|
536
|
+
args: {
|
|
537
|
+
project: string;
|
|
538
|
+
issue: number;
|
|
539
|
+
runId: string;
|
|
540
|
+
inProgress: string;
|
|
541
|
+
reason: string;
|
|
542
|
+
patch: Partial<RunRecord>;
|
|
543
|
+
},
|
|
544
|
+
): void {
|
|
545
|
+
store.updateRun(args.runId, {
|
|
546
|
+
...args.patch,
|
|
547
|
+
state: "stopped",
|
|
548
|
+
lastError: `operator stopped: ${args.reason}`,
|
|
549
|
+
});
|
|
550
|
+
store.enqueueLabelOps(args.project, [
|
|
551
|
+
{ issue: args.issue, op: "remove", label: args.inProgress },
|
|
552
|
+
]);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/**
|
|
556
|
+
* What a salvage attempt contributes to the escalation: where the work went, or
|
|
557
|
+
* that it went nowhere. Split from the effects below for the same reason
|
|
558
|
+
* `checkIntegrity` is — this wording is the whole thing a human acts on, so it
|
|
559
|
+
* is worth a test holding it, and the sha in it is the only pointer to work
|
|
560
|
+
* that no longer has any other copy.
|
|
561
|
+
*
|
|
562
|
+
* `retained` is not cosmetic. These lines used to promise a tree "kept for
|
|
563
|
+
* inspection" unconditionally, which was true only because salvage ran only on
|
|
564
|
+
* the paths that keep one. A blocked run's tree is removed the moment its work
|
|
565
|
+
* is safely on the branch, and sending an operator to a path this process just
|
|
566
|
+
* deleted is the same class of mistake as #118 itself.
|
|
567
|
+
*/
|
|
568
|
+
export function salvageLines(
|
|
569
|
+
outcome: SalvageOutcome,
|
|
570
|
+
worktree: string,
|
|
571
|
+
retained: boolean,
|
|
572
|
+
): string[] {
|
|
573
|
+
const fate = retained
|
|
574
|
+
? `Worktree kept for inspection: ${worktree}`
|
|
575
|
+
: `Worktree removed: ${worktree}`;
|
|
576
|
+
|
|
577
|
+
if (outcome.kind === "nothing") return [`${fate} — nothing uncommitted to salvage`];
|
|
578
|
+
|
|
579
|
+
if (outcome.kind === "failed") {
|
|
580
|
+
return [
|
|
581
|
+
`WIP SALVAGE FAILED: ${outcome.error}`,
|
|
582
|
+
`Uncommitted work in ${worktree} is the only copy of it, so the tree was kept.`,
|
|
583
|
+
"This issue is held out of dispatch until the tree is recovered by hand and",
|
|
584
|
+
"`omp-conductor unblock <n> --force` records that you accepted it.",
|
|
585
|
+
];
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
const where =
|
|
589
|
+
`WIP committed to ${outcome.branch} @ ${outcome.sha}` +
|
|
590
|
+
(outcome.pushed
|
|
591
|
+
? " and pushed — the work outlives this worktree"
|
|
592
|
+
: ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`);
|
|
593
|
+
// Manifest belongs in the escalation too: opening the commit is how the
|
|
594
|
+
// orchestrator talked itself into scrubbing a worker tree (#38).
|
|
595
|
+
const n = outcome.files.length;
|
|
596
|
+
const count = `${n} file${n === 1 ? "" : "s"}`;
|
|
597
|
+
const manifest =
|
|
598
|
+
outcome.newPaths.length === 0
|
|
599
|
+
? `${count} (all modifications to tracked paths)`
|
|
600
|
+
: `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
|
|
601
|
+
outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
|
|
602
|
+
}`;
|
|
603
|
+
return [where, manifest, fate];
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/** Everything a settled run has to record and say about its worktree. */
|
|
607
|
+
export interface WorktreeSettlement {
|
|
608
|
+
outcome: SalvageOutcome;
|
|
609
|
+
/** Whether the tree still exists now the run is over. */
|
|
610
|
+
retained: boolean;
|
|
611
|
+
/** Escalation lines naming where the work went. */
|
|
612
|
+
lines: string[];
|
|
613
|
+
/** Row fields recording the durable ref, or the failure that blocks a re-claim. */
|
|
614
|
+
patch: Pick<RunRecord, "salvageSha" | "salvageError">;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Decides what becomes of a finished run's worktree: save the work, then keep
|
|
619
|
+
* or remove the tree, then say which.
|
|
620
|
+
*
|
|
621
|
+
* One function because the two halves are one decision and splitting them is
|
|
622
|
+
* how #118 happened — the removal at the end of dispatch had no idea whether
|
|
623
|
+
* anything had been saved, and the salvage at the top of the failure branch had
|
|
624
|
+
* no idea the blocked branch fell through to a `--force` removal.
|
|
625
|
+
*
|
|
626
|
+
* A salvage that *fails* retains the tree whatever the caller asked for. There
|
|
627
|
+
* was real work, git refused to commit it, and the tree is now the only copy in
|
|
628
|
+
* existence: deleting it on schedule would be the data loss this whole path
|
|
629
|
+
* exists to prevent. The issue is held out of dispatch until an operator says
|
|
630
|
+
* otherwise, because the next attempt's `worktree remove --force` would finish
|
|
631
|
+
* the job (see `admitCandidates`).
|
|
632
|
+
*
|
|
633
|
+
* Exported so a test can drive the real decision against a real git tree.
|
|
634
|
+
*/
|
|
635
|
+
export async function settleWorktree(
|
|
636
|
+
args: {
|
|
637
|
+
issue: number;
|
|
638
|
+
attempt: number;
|
|
639
|
+
/** Clause for the commit subject: "killed by the turns cap", "blocked …". */
|
|
640
|
+
ending: string;
|
|
641
|
+
worktree: string;
|
|
642
|
+
/** The run's branch, so the pre-removal publish names the right ref. */
|
|
643
|
+
branch: string;
|
|
644
|
+
/**
|
|
645
|
+
* Publishes the run branch on the privileged side. Required rather than
|
|
646
|
+
* optional: the run's commits live in a repository of its own,
|
|
647
|
+
* so a removal that did not publish first would delete the only copy —
|
|
648
|
+
* which is #121's data loss with one extra step. `undefined` is a visible
|
|
649
|
+
* decision at the call site, never an omission.
|
|
650
|
+
*/
|
|
651
|
+
publish: RunPublisher | undefined;
|
|
652
|
+
} & (
|
|
653
|
+
| /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
|
|
654
|
+
{ tree: "keep" }
|
|
655
|
+
| { tree: "remove"; mirrorPath: string }
|
|
656
|
+
),
|
|
657
|
+
): Promise<WorktreeSettlement> {
|
|
658
|
+
const { issue, attempt, ending, worktree, branch, publish } = args;
|
|
659
|
+
const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
|
|
660
|
+
const retained = args.tree === "keep" || outcome.kind === "failed";
|
|
661
|
+
if (!retained && args.tree === "remove") {
|
|
662
|
+
// Before the removal, always — not only when salvage found something. A run
|
|
663
|
+
// that *committed* and could not publish has its work in its own repository
|
|
664
|
+
// and nowhere else, and salvage never sees a committed tree because it is
|
|
665
|
+
// clean. The mirror fetch inside `publish` is what preserves it; the push
|
|
666
|
+
// to GitHub can fail (no network, protected ref) and the work still lives.
|
|
667
|
+
const published = await publish?.(branch);
|
|
668
|
+
if (published !== undefined && !published.ok) {
|
|
669
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
670
|
+
}
|
|
671
|
+
await removeWorktree(args.mirrorPath, worktree);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
const lines = salvageLines(outcome, worktree, retained);
|
|
675
|
+
log(`#${issue} salvage: ${lines.join(" ")}`);
|
|
676
|
+
return {
|
|
677
|
+
outcome,
|
|
678
|
+
retained,
|
|
679
|
+
lines,
|
|
680
|
+
patch:
|
|
681
|
+
outcome.kind === "salvaged"
|
|
682
|
+
? { salvageSha: outcome.sha }
|
|
683
|
+
: outcome.kind === "failed"
|
|
684
|
+
? { salvageError: outcome.error }
|
|
685
|
+
: {},
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
// ------------------------------------------------------------------- orphan & stale-label reconciliation
|
|
691
|
+
|
|
692
|
+
/** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
|
|
693
|
+
const RECONCILE_LIMIT = 50;
|
|
694
|
+
|
|
695
|
+
/**
|
|
696
|
+
* Clear labels from issues that no longer need them (#132's `superseded`).
|
|
697
|
+
*
|
|
698
|
+
* Three structural signals, all cheap and all observed on this fleet: an issue
|
|
699
|
+
* that is closed but still carries an `agent:*` state label, a closed issue
|
|
700
|
+
* still carrying the queue label — the claim gate on work that can never be
|
|
701
|
+
* claimed again, which the merge path only recently learned to remove itself
|
|
702
|
+
* (#607), leaving a 50+-issue backlog — and an open issue carrying `failed`
|
|
703
|
+
* whose sub-issues have all closed. On 2026-08-09 four issues (#307,
|
|
704
|
+
* #297, #140, #82) carried `agent:failed` while every one of them was already
|
|
705
|
+
* complete — the label was residue of a turns-cap kill from two days earlier,
|
|
706
|
+
* and nothing in the loop ever revisited it. The board counted four phantom
|
|
707
|
+
* failures while the genuinely stuck issues were invisible.
|
|
708
|
+
*
|
|
709
|
+
* The queue label is swept off closed issues only: an open issue carrying it
|
|
710
|
+
* is claimable work by definition, and a reconcile that guessed would strip
|
|
711
|
+
* the interlock that keeps two workers off one issue.
|
|
712
|
+
*
|
|
713
|
+
* Positive evidence only. A tracker that cannot list answers empty, and an empty
|
|
714
|
+
* answer removes nothing: a reconcile that guessed would strip the interlock
|
|
715
|
+
* that keeps two workers off one issue.
|
|
716
|
+
*/
|
|
717
|
+
export async function reconcileStaleLabels(d: Pick<SettlementDeps, "project" | "tracker" | "store">): Promise<void> {
|
|
718
|
+
const { project, tracker, store } = d;
|
|
719
|
+
const labels = [
|
|
720
|
+
project.stateLabels.failed,
|
|
721
|
+
project.stateLabels.blocked,
|
|
722
|
+
project.stateLabels.inProgress,
|
|
723
|
+
project.queueLabel,
|
|
724
|
+
];
|
|
725
|
+
|
|
726
|
+
for (const label of labels) {
|
|
727
|
+
const carrying = await tracker.listLabeled(label, RECONCILE_LIMIT).catch(() => []);
|
|
728
|
+
for (const issue of carrying) {
|
|
729
|
+
if (issue.state === "closed") {
|
|
730
|
+
// Never retain an `agent:*` label on a closed issue: the work is done by
|
|
731
|
+
// some route, and the label only makes the board lie about it. Enqueue
|
|
732
|
+
// rather than call — a refused write must not lose the decision; the
|
|
733
|
+
// projector retries the removal until the tracker takes it (#201).
|
|
734
|
+
store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
|
|
735
|
+
log(`#${issue.number} reconciled: closed issue no longer carries ${label} (queued)`);
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
if (label !== project.stateLabels.failed) continue;
|
|
740
|
+
const children = await tracker.childrenOf(issue.number).catch(() => []);
|
|
741
|
+
if (children.length === 0 || children.some((c) => c.state !== "closed")) continue;
|
|
742
|
+
|
|
743
|
+
const key = `${project.name}:superseded:${issue.number}`;
|
|
744
|
+
if (store.wasNotified(key)) continue;
|
|
745
|
+
const list = children.map((c) => `#${c.number}`).join(", ");
|
|
746
|
+
store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
|
|
747
|
+
try {
|
|
748
|
+
await tracker.comment(
|
|
749
|
+
issue.number,
|
|
750
|
+
`superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
|
|
751
|
+
`acceptance criteria are met on the default branch.`,
|
|
752
|
+
);
|
|
753
|
+
store.markNotified(key);
|
|
754
|
+
log(`#${issue.number} reconciled: superseded by ${list}`);
|
|
755
|
+
} catch (err) {
|
|
756
|
+
// The label removal is already queued and will land regardless; the
|
|
757
|
+
// comment is the only half that can fail here (#201).
|
|
758
|
+
log(`#${issue.number} could not comment the superseded note (${errText(err)}) — retrying next tick`);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
/**
|
|
765
|
+
* Settles `claimed`/`running` rows left by a dead daemon process and, before
|
|
766
|
+
* marking each one `orphaned`, salvages any dirty worktree.
|
|
767
|
+
*
|
|
768
|
+
* Found live after a host restart killed two workers mid-run, and again on
|
|
769
|
+
* every package deploy that restarted while workers were live (#35): without
|
|
770
|
+
* the salvage call the next attempt's `worktree remove --force` destroyed
|
|
771
|
+
* uncommitted edits that had no other copy. Cap-kills already salvaged (#27);
|
|
772
|
+
* this is the same call site for the restart path.
|
|
773
|
+
*
|
|
774
|
+
* Only the rows change. The issue keeps its in-progress label — that label is
|
|
775
|
+
* the crash guard against double-dispatch, and deciding what a dead worker's
|
|
776
|
+
* remains are worth (an open PR? a salvaged sha? a clean tree?) is the
|
|
777
|
+
* orchestrator's drain-duty judgement, not something to automate here. The
|
|
778
|
+
* rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
|
|
779
|
+
* still escalates instead of retrying forever.
|
|
780
|
+
*
|
|
781
|
+
* `pushed-green` rows are deliberately left alone: they hold no process — they
|
|
782
|
+
* are finished work waiting on a human merge, and they must keep occupying the
|
|
783
|
+
* issue so a second attempt cannot land on a live PR. What eventually settles
|
|
784
|
+
* them is {@link settlePushedGreen}, on the tick, by asking the tracker what
|
|
785
|
+
* became of the PR — the one question a restart cannot answer by inference.
|
|
786
|
+
*/
|
|
787
|
+
export async function reconcileOrphanedRuns(
|
|
788
|
+
store: Store,
|
|
789
|
+
project: string,
|
|
790
|
+
/**
|
|
791
|
+
* Resolves the privileged publisher for one orphaned run. Optional because a
|
|
792
|
+
* test driving the row transitions has no repo to publish to; production
|
|
793
|
+
* always passes it, and without it a salvaged WIP commit stays local — which
|
|
794
|
+
* is the half of #121 that reaches a human.
|
|
795
|
+
*/
|
|
796
|
+
publish?: (run: RunRecord) => RunPublisher,
|
|
797
|
+
): Promise<RunRecord[]> {
|
|
798
|
+
// Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
|
|
799
|
+
// a process dying — it is finished work waiting on a human merge.
|
|
800
|
+
const stale = store.liveRuns(project);
|
|
801
|
+
const endedAt = Date.now();
|
|
802
|
+
for (const r of stale) {
|
|
803
|
+
// Salvage before the row flips: the worktree path is on the record, and
|
|
804
|
+
// salvageWip is a no-op for a missing/clean tree. The clause matches the
|
|
805
|
+
// cap-kill wording so triage reads the same either way, and the tree is
|
|
806
|
+
// kept because an orphan's remains are the orchestrator's drain-duty call.
|
|
807
|
+
const settlement =
|
|
808
|
+
r.worktree === ""
|
|
809
|
+
? undefined
|
|
810
|
+
: await settleWorktree({
|
|
811
|
+
issue: r.issue,
|
|
812
|
+
attempt: r.attempt,
|
|
813
|
+
ending: "killed by a daemon restart",
|
|
814
|
+
worktree: r.worktree,
|
|
815
|
+
branch: r.branch,
|
|
816
|
+
// An orphan's tree is kept, so its commits are not about to be
|
|
817
|
+
// deleted — but a WIP salvage still has to reach GitHub, which is
|
|
818
|
+
// #121's whole point and is now the daemon's hop to make.
|
|
819
|
+
publish: publish?.(r),
|
|
820
|
+
tree: "keep",
|
|
821
|
+
});
|
|
822
|
+
store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
|
|
823
|
+
}
|
|
824
|
+
return stale;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
// ------------------------------------------------------------------- classification & recovery
|
|
831
|
+
|
|
832
|
+
/** A dispatch-infra run (turn-0 git failure) is requeued so the next tick
|
|
833
|
+
* retries — but only a bounded number of times. Three strikes for one issue
|
|
834
|
+
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
835
|
+
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
836
|
+
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
837
|
+
/** A provider-transient requeue (stream stalled mid-run) is retried, but only a
|
|
838
|
+
* bounded number of times: three aborted streams for one issue means the
|
|
839
|
+
* provider itself is degraded, not unlucky, and the sweep escalates to a
|
|
840
|
+
* human instead of requeueing into a down provider forever (#220). */
|
|
841
|
+
const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
|
|
842
|
+
/** A provider-capacity requeue (sustained in-session rate limiting) is retried,
|
|
843
|
+
* but only a bounded number of times: three throttled runs for one issue mean
|
|
844
|
+
* the provider is at capacity, not unlucky, and the sweep escalates to a human
|
|
845
|
+
* instead of requeueing into a throttled provider forever (#573). The issue's
|
|
846
|
+
* own chain moves onto its next model per strike (via {@link FAILOVER_CLASSES}),
|
|
847
|
+
* so a bounded chain is exhaustible; this caps the unbounded no-chain case. */
|
|
848
|
+
const PROVIDER_CAPACITY_MAX_STRIKES = 3;
|
|
849
|
+
/** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
|
|
850
|
+
* are maintenance, but a backlog must not turn one tick into an API burst. */
|
|
851
|
+
const SALVAGED_PR_ADOPTION_BATCH = 10;
|
|
852
|
+
|
|
853
|
+
/** The failure classes `countContinuations` deliberately does not charge — the
|
|
854
|
+
* inverted copy of its exclusions, kept beside the breakdown that consumes it
|
|
855
|
+
* so the two can only drift together (#439). `orphan-clean` is absent on
|
|
856
|
+
* purpose: daemon orphans consume the continuation budget, which is exactly
|
|
857
|
+
* why the requeue side has to respect the ceiling instead of racing it. */
|
|
858
|
+
const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
|
|
859
|
+
"admin-kill": true,
|
|
860
|
+
"settlement-stuck": true,
|
|
861
|
+
"env-start-failure": true,
|
|
862
|
+
"dispatch-infra": true,
|
|
863
|
+
"provider-credit": true,
|
|
864
|
+
"provider-transient": true,
|
|
865
|
+
"provider-capacity": true,
|
|
866
|
+
};
|
|
867
|
+
|
|
868
|
+
/** How one issue spent its continuation budget, grouped by failure class —
|
|
869
|
+
* the exact rows `continuationsFor` charges, so an exhaustion escalation
|
|
870
|
+
* reports the same budget it says is spent. `unclassified` groups rows that
|
|
871
|
+
* charged before the class was written (a pre-upgrade NULL). */
|
|
872
|
+
function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
|
|
873
|
+
const perClass = new Map<string, number>();
|
|
874
|
+
for (const r of runs) {
|
|
875
|
+
const chargedAsKilledOrOrphaned =
|
|
876
|
+
(r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
|
|
877
|
+
(r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
|
|
878
|
+
const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
|
|
879
|
+
if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
|
|
880
|
+
const cls = r.failureClass ?? "unclassified";
|
|
881
|
+
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
882
|
+
}
|
|
883
|
+
return perClass;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/** The newest attempt's preserved work, if any, so an exhaustion escalation can
|
|
887
|
+
* say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
|
|
888
|
+
* three artifacts a run can leave, and all null on a branch nothing reached. */
|
|
889
|
+
function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
|
|
890
|
+
const newestFirst = [...runs].reverse();
|
|
891
|
+
return newestFirst.find(
|
|
892
|
+
(r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
|
|
893
|
+
);
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
/** The fenced-block info string that marks an exhaustion postmortem comment, so
|
|
897
|
+
* a grooming scout re-slicing the issue can find and parse the whole block by
|
|
898
|
+
* grepping for it. */
|
|
899
|
+
export const POSTMORTEM_MARKER = "conductor-postmortem";
|
|
900
|
+
|
|
901
|
+
/** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
|
|
902
|
+
* digest shape for naming what the exhaustion was. Groups every row by its
|
|
903
|
+
* failure class, whether or not it charged the continuation budget, because
|
|
904
|
+
* the postmortem tells the whole story and not just the budget half (#290). */
|
|
905
|
+
export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
|
|
906
|
+
const perClass = new Map<string, number>();
|
|
907
|
+
for (const r of runs) {
|
|
908
|
+
const cls = r.failureClass ?? "unclassified";
|
|
909
|
+
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
910
|
+
}
|
|
911
|
+
if (perClass.size === 0) return "unclassified";
|
|
912
|
+
return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
|
|
916
|
+
export function humanDuration(ms: number): string {
|
|
917
|
+
const seconds = Math.max(0, Math.round(ms / 1_000));
|
|
918
|
+
if (seconds < 60) return `${seconds}s`;
|
|
919
|
+
const minutes = Math.round(seconds / 60);
|
|
920
|
+
if (minutes < 60) return `${minutes}m`;
|
|
921
|
+
const hours = Math.floor(minutes / 60);
|
|
922
|
+
const rest = minutes % 60;
|
|
923
|
+
return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
/** Flatten and bound a run's last error to one greppable table line. */
|
|
927
|
+
export function oneLineBrief(text: string | undefined): string | undefined {
|
|
928
|
+
if (text === undefined || text.trim() === "") return undefined;
|
|
929
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
930
|
+
return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/**
|
|
934
|
+
* The exhaustion postmortem block: one greppable fenced block covering every
|
|
935
|
+
* attempt in the chain — continuation rows included — with per-attempt turns,
|
|
936
|
+
* wall clock, failure class and a one-line last error, the explicit salvage
|
|
937
|
+
* state, the spend total and the transcript paths for local inspection.
|
|
938
|
+
*
|
|
939
|
+
* Pure so the tests hold the shape, not the transport: the writer below owns
|
|
940
|
+
* the once-only guarantee, this owns what "once" looks like.
|
|
941
|
+
*/
|
|
942
|
+
export function formatExhaustionPostmortem(args: {
|
|
943
|
+
issue: number;
|
|
944
|
+
runs: readonly RunRecord[];
|
|
945
|
+
reason: string;
|
|
946
|
+
}): string {
|
|
947
|
+
const { issue, runs, reason } = args;
|
|
948
|
+
const artifact = newestContinuableRun(runs);
|
|
949
|
+
const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
|
|
950
|
+
const spend = `$${totalSpend.toFixed(2)}`;
|
|
951
|
+
const attemptLines = runs.map((r) => {
|
|
952
|
+
const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
|
|
953
|
+
const cls = r.failureClass ?? "unclassified";
|
|
954
|
+
const error = oneLineBrief(r.lastError) ?? "—";
|
|
955
|
+
return (
|
|
956
|
+
` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
|
|
957
|
+
`${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
|
|
958
|
+
);
|
|
959
|
+
});
|
|
960
|
+
const salvage =
|
|
961
|
+
artifact === undefined
|
|
962
|
+
? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
|
|
963
|
+
: `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
|
|
964
|
+
`${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
|
|
965
|
+
return [
|
|
966
|
+
`\`\`\`${POSTMORTEM_MARKER}`,
|
|
967
|
+
`#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
968
|
+
reason,
|
|
969
|
+
"",
|
|
970
|
+
`Attempts (${runs.length} total, ${spend} spend):`,
|
|
971
|
+
...attemptLines,
|
|
972
|
+
salvage,
|
|
973
|
+
`Spend total: ${spend} across ${runs.length} attempts.`,
|
|
974
|
+
"Transcripts:",
|
|
975
|
+
...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
|
|
976
|
+
"```",
|
|
977
|
+
].join("\n");
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
/** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
|
|
981
|
+
function postmortemDedupeKey(project: string, issue: number): string {
|
|
982
|
+
return `${project}:postmortem:${issue}`;
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/**
|
|
986
|
+
* The exhaustion postmortem: written exactly once per issue, at the point the
|
|
987
|
+
* continuation budget is spent and the issue is settled toward a human.
|
|
988
|
+
*
|
|
989
|
+
* The comment and the material event are both gated by the store's notification
|
|
990
|
+
* ledger — the same idempotence guard the escalator uses — so re-settling an
|
|
991
|
+
* already-postmortemed issue posts nothing and records nothing. A body-string
|
|
992
|
+
* match on the issue would be the wrong guard: an issue re-scoped and re-run
|
|
993
|
+
* would still carry the old block, and the guarantee asked of this is "decided
|
|
994
|
+
* once", not "deduped against what is already written".
|
|
995
|
+
*
|
|
996
|
+
* The digest must name the exhaustion even if the comment write fails, so the
|
|
997
|
+
* material event is recorded before the write and unconditionally (the ledger
|
|
998
|
+
* is append-only, and the gate above already ran once). The comment failure is
|
|
999
|
+
* logged rather than taking the sweep down with it.
|
|
1000
|
+
*/
|
|
1001
|
+
async function postExhaustionPostmortem(d: Pick<SettlementDeps, "project" | "tracker" | "store">, run: RunRecord, reason: string): Promise<void> {
|
|
1002
|
+
const { project, tracker, store } = d;
|
|
1003
|
+
const key = postmortemDedupeKey(project.name, run.issue);
|
|
1004
|
+
if (store.wasNotified(key)) return;
|
|
1005
|
+
const runs = store.runsForIssue(project.name, run.issue);
|
|
1006
|
+
const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
|
|
1007
|
+
const occurredAt = Date.now();
|
|
1008
|
+
store.recordMaterialEvent({
|
|
1009
|
+
project: project.name,
|
|
1010
|
+
category: "exhaustion",
|
|
1011
|
+
summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1012
|
+
evidence: body,
|
|
1013
|
+
occurredAt,
|
|
1014
|
+
recordedAt: occurredAt,
|
|
1015
|
+
});
|
|
1016
|
+
try {
|
|
1017
|
+
await tracker.comment(run.issue, body);
|
|
1018
|
+
store.markNotified(key);
|
|
1019
|
+
log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
|
|
1020
|
+
} catch (err) {
|
|
1021
|
+
log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* The escalator throws when no transport is configured or Telegram rejects, and
|
|
1027
|
+
* only records the dedup marker on success. A page that cannot be delivered
|
|
1028
|
+
* must not take the tick down with it — log it and let the next tick retry.
|
|
1029
|
+
*
|
|
1030
|
+
* Returns whether it actually went out, because "page once" and "page once
|
|
1031
|
+
* *successfully*" are different promises: a caller that latches a once-only
|
|
1032
|
+
* gate on the attempt turns one failed delivery into permanent silence about a
|
|
1033
|
+
* condition that is still true.
|
|
1034
|
+
*/
|
|
1035
|
+
|
|
1036
|
+
export async function reactToProviderCredit(
|
|
1037
|
+
d: Pick<SettlementDeps, "project" | "escalate" | "isPaused" | "setPaused">,
|
|
1038
|
+
issue: number,
|
|
1039
|
+
message: string,
|
|
1040
|
+
sessionFile: string | undefined,
|
|
1041
|
+
): Promise<void> {
|
|
1042
|
+
const { project } = d;
|
|
1043
|
+
const alreadyPaused = d.isPaused(project.name);
|
|
1044
|
+
if (!alreadyPaused) d.setPaused(true, { source: "provider-credit", reason: message }, project.name);
|
|
1045
|
+
log(
|
|
1046
|
+
`#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
|
|
1047
|
+
);
|
|
1048
|
+
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
1049
|
+
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
1050
|
+
// carrying no run or attempt pages once for the fleet, not once per run.
|
|
1051
|
+
await safeEscalate(d, {
|
|
1052
|
+
tier: 2,
|
|
1053
|
+
category: "fleet-stopped",
|
|
1054
|
+
project: project.name,
|
|
1055
|
+
issue: NO_ISSUE,
|
|
1056
|
+
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
1057
|
+
detail: [
|
|
1058
|
+
message,
|
|
1059
|
+
"",
|
|
1060
|
+
"No implementation attempt was charged: this is a billing state, not a",
|
|
1061
|
+
"failed implementation. Each affected issue keeps its queue label and",
|
|
1062
|
+
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
1063
|
+
`Session: ${sessionFile ?? "(no transcript)"}`,
|
|
1064
|
+
].join("\n"),
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
/** Bounded per tick: each row costs tracker calls to gather facts for. */
|
|
1069
|
+
const CLASSIFY_BATCH = 20;
|
|
1070
|
+
|
|
1071
|
+
/** Tool calls quoted as evidence for a run that spun to its turn cap. */
|
|
1072
|
+
const SPIN_EVIDENCE_CALLS = 10;
|
|
1073
|
+
|
|
1074
|
+
/**
|
|
1075
|
+
* The last few tool names a transcript recorded, newest last.
|
|
1076
|
+
*
|
|
1077
|
+
* `turn-cap-spinning` escalates rather than requeueing, and the acceptance
|
|
1078
|
+
* criterion is that the escalation carries evidence of what the worker was doing
|
|
1079
|
+
* when it hit the cap — otherwise the orchestrator opens the transcript and
|
|
1080
|
+
* re-derives it, which is the manual triage this whole sweep removes.
|
|
1081
|
+
*/
|
|
1082
|
+
export function lastToolCalls(sessionFile: string | undefined, limit = SPIN_EVIDENCE_CALLS): string[] {
|
|
1083
|
+
if (sessionFile === undefined) return [];
|
|
1084
|
+
let text: string;
|
|
1085
|
+
try {
|
|
1086
|
+
text = readFileSync(sessionFile, "utf8");
|
|
1087
|
+
} catch {
|
|
1088
|
+
return [];
|
|
1089
|
+
}
|
|
1090
|
+
const names: string[] = [];
|
|
1091
|
+
for (const line of text.split("\n")) {
|
|
1092
|
+
if (line.length === 0) continue;
|
|
1093
|
+
let row: unknown;
|
|
1094
|
+
try {
|
|
1095
|
+
row = JSON.parse(line) as unknown;
|
|
1096
|
+
} catch {
|
|
1097
|
+
continue;
|
|
1098
|
+
}
|
|
1099
|
+
if (row === null || typeof row !== "object") continue;
|
|
1100
|
+
const rec = row as { readonly [key: string]: unknown };
|
|
1101
|
+
// Both shapes the harness has written: a top-level tool event, and a tool
|
|
1102
|
+
// block inside an assistant message.
|
|
1103
|
+
const direct = rec["toolName"];
|
|
1104
|
+
if (typeof direct === "string") {
|
|
1105
|
+
names.push(direct);
|
|
1106
|
+
continue;
|
|
1107
|
+
}
|
|
1108
|
+
const message = rec["message"];
|
|
1109
|
+
if (message === null || typeof message !== "object") continue;
|
|
1110
|
+
const content = (message as { readonly [key: string]: unknown })["content"];
|
|
1111
|
+
if (!Array.isArray(content)) continue;
|
|
1112
|
+
for (const part of content) {
|
|
1113
|
+
if (part === null || typeof part !== "object") continue;
|
|
1114
|
+
const p = part as { readonly [key: string]: unknown };
|
|
1115
|
+
if (p["type"] !== "tool_use") continue;
|
|
1116
|
+
const name = p["name"];
|
|
1117
|
+
if (typeof name === "string") names.push(name);
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
return names.slice(-limit);
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
/**
|
|
1124
|
+
* The last error a transcript recorded, or undefined when it recorded none.
|
|
1125
|
+
*
|
|
1126
|
+
* The harness writes `{"stopReason":"error","errorStatus":402,"errorId":402,
|
|
1127
|
+
* "errorMessage":"402 This request requires more credits, ..."}`. The daemon
|
|
1128
|
+
* read none of it, so three runs died `unknown` with an empty `lastError` and
|
|
1129
|
+
* charged an attempt each for a billing state (#220).
|
|
1130
|
+
*
|
|
1131
|
+
* Scanned newest-first: a session that recovered from an early error and then
|
|
1132
|
+
* died of something else must report the something else, and a session that
|
|
1133
|
+
* recovered from its only error and finished cleanly reports the error anyway
|
|
1134
|
+
* because there is no terminal verdict to outrank it (#220).
|
|
1135
|
+
*/
|
|
1136
|
+
/** Transcript error shape — the daemon owns the same interface for
|
|
1137
|
+
* `completionLastError`; declared inline here so this leaf never imports it.
|
|
1138
|
+
*
|
|
1139
|
+
* `kind: "provider-stream"` marks a record the harness itself attributed to a
|
|
1140
|
+
* provider abort (see {@link assistantStopError}), so the classifier can trust
|
|
1141
|
+
* the attribution without matching any vendor's wording (#743). */
|
|
1142
|
+
export type TranscriptError = {
|
|
1143
|
+
status?: number;
|
|
1144
|
+
message: string;
|
|
1145
|
+
kind?: "provider-stream";
|
|
1146
|
+
};
|
|
1147
|
+
|
|
1148
|
+
/**
|
|
1149
|
+
* The provider's own message in a synthetic tool-result record, or undefined
|
|
1150
|
+
* when the record is not the harness attributing a mid-stream provider abort
|
|
1151
|
+
* (#743).
|
|
1152
|
+
*
|
|
1153
|
+
* The harness writes this when the provider dies while the assistant is still
|
|
1154
|
+
* producing and a pending tool call never runs:
|
|
1155
|
+
*
|
|
1156
|
+
* {"type":"message","message":{"role":"toolResult","isError":true,
|
|
1157
|
+
* "toolName":"edit","content":[{"type":"text","text":"Tool call was not
|
|
1158
|
+
* executed because the provider stream ended with an error before the tool
|
|
1159
|
+
* could run: …"}],"details":{"__synthetic":true,
|
|
1160
|
+
* "source":"assistant_stop_error","executed":false,
|
|
1161
|
+
* "upstreamError":"server_error: Upstream error from DeepInfra:
|
|
1162
|
+
* Exception: Response payload is not completed"}}}
|
|
1163
|
+
*
|
|
1164
|
+
* The `source` field is the harness's own marker, so recognition survives any
|
|
1165
|
+
* provider's wording; `upstreamError` (falling back to the record's text) is
|
|
1166
|
+
* the provider's own prose, carried as evidence.
|
|
1167
|
+
*
|
|
1168
|
+
* Deliberately narrower than "any isError toolResult": the marker is what makes
|
|
1169
|
+
* the attribution structural. An `isError` tool result without it is a tool
|
|
1170
|
+
* call that failed on its own — the session's work, not a provider fault — and
|
|
1171
|
+
* must keep falling through.
|
|
1172
|
+
*/
|
|
1173
|
+
function assistantStopError(rec: { readonly [key: string]: unknown }): string | undefined {
|
|
1174
|
+
if (rec["type"] !== "message") return undefined;
|
|
1175
|
+
const message = rec["message"];
|
|
1176
|
+
if (message === null || typeof message !== "object") return undefined;
|
|
1177
|
+
const msg = message as { readonly [key: string]: unknown };
|
|
1178
|
+
if (msg["role"] !== "toolResult" || msg["isError"] !== true) return undefined;
|
|
1179
|
+
const details = msg["details"];
|
|
1180
|
+
if (details === null || typeof details !== "object") return undefined;
|
|
1181
|
+
const detail = details as { readonly [key: string]: unknown };
|
|
1182
|
+
if (detail["source"] !== "assistant_stop_error") return undefined;
|
|
1183
|
+
const upstream = detail["upstreamError"];
|
|
1184
|
+
if (typeof upstream === "string" && upstream.trim() !== "") return upstream.trim();
|
|
1185
|
+
const content = msg["content"];
|
|
1186
|
+
if (!Array.isArray(content)) return undefined;
|
|
1187
|
+
for (const part of content) {
|
|
1188
|
+
if (part === null || typeof part !== "object") continue;
|
|
1189
|
+
const p = part as { readonly [key: string]: unknown };
|
|
1190
|
+
if (p["type"] !== "text" || typeof p["text"] !== "string") continue;
|
|
1191
|
+
if (p["text"].trim() !== "") return p["text"].trim();
|
|
1192
|
+
}
|
|
1193
|
+
return undefined;
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
export function readSessionError(sessionFile: string | undefined): TranscriptError | undefined {
|
|
1197
|
+
if (sessionFile === undefined) return undefined;
|
|
1198
|
+
let text: string;
|
|
1199
|
+
try {
|
|
1200
|
+
text = readFileSync(sessionFile, "utf8");
|
|
1201
|
+
} catch {
|
|
1202
|
+
return undefined;
|
|
1203
|
+
}
|
|
1204
|
+
const lines = text.split("\n");
|
|
1205
|
+
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
1206
|
+
const line = lines[i];
|
|
1207
|
+
if (line === undefined || line.length === 0) continue;
|
|
1208
|
+
let row: unknown;
|
|
1209
|
+
try {
|
|
1210
|
+
row = JSON.parse(line) as unknown;
|
|
1211
|
+
} catch {
|
|
1212
|
+
continue;
|
|
1213
|
+
}
|
|
1214
|
+
if (row === null || typeof row !== "object") continue;
|
|
1215
|
+
const rec = row as { readonly [key: string]: unknown };
|
|
1216
|
+
// A provider abort mid-stream lands as a synthetic tool-result record, not
|
|
1217
|
+
// a `stopReason:error` row: the pending tool call "was not executed because
|
|
1218
|
+
// the provider stream ended with an error". The harness's own `source`
|
|
1219
|
+
// marker is the attribution — no vendor prose to match — and the provider's
|
|
1220
|
+
// own text rides along as evidence (#743).
|
|
1221
|
+
if (rec["type"] === "message") {
|
|
1222
|
+
const providerText = assistantStopError(rec);
|
|
1223
|
+
if (providerText !== undefined) {
|
|
1224
|
+
return { message: providerText, kind: "provider-stream" };
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
if (rec["stopReason"] !== "error") continue;
|
|
1228
|
+
const message = rec["errorMessage"];
|
|
1229
|
+
if (typeof message !== "string" || message.trim() === "") continue;
|
|
1230
|
+
const status = rec["errorStatus"];
|
|
1231
|
+
return {
|
|
1232
|
+
...(typeof status === "number" && Number.isFinite(status) ? { status } : {}),
|
|
1233
|
+
message: message.trim(),
|
|
1234
|
+
};
|
|
1235
|
+
}
|
|
1236
|
+
return undefined;
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
/**
|
|
1240
|
+
* Classify every unclassified terminal run, persist the verdict, and perform the
|
|
1241
|
+
* one recovery its class names (#132).
|
|
1242
|
+
*
|
|
1243
|
+
* Half this fleet's spend produced no merged PR, and every one of those runs
|
|
1244
|
+
* ended at a human who re-derived the same triage by hand and then threw the
|
|
1245
|
+
* conclusion away. The mechanical classes — a cancelled runner, a kill from a
|
|
1246
|
+
* daemon restart, a green PR whose base moved, a row whose PR had already merged
|
|
1247
|
+
* — need no judgement at all; the genuinely human ones are worth a person's
|
|
1248
|
+
* attention only if they arrive with their evidence already gathered.
|
|
1249
|
+
*
|
|
1250
|
+
* Facts are fetched per row and only the ones that row needs: a `killed` row
|
|
1251
|
+
* costs nothing, a `failed` row with a PR costs a state read and a check read.
|
|
1252
|
+
* A `pushed-green` row that classifies to nothing is left completely untouched —
|
|
1253
|
+
* it is healthy, and writing a class onto it would take it out of this sweep for
|
|
1254
|
+
* good.
|
|
1255
|
+
*/
|
|
1256
|
+
export async function classifyAndRecover(d: SettlementDeps): Promise<number> {
|
|
1257
|
+
const { project, caps, tracker, store } = d;
|
|
1258
|
+
// Settle recoveries, counted for the pass's dispatch record — a row whose PR
|
|
1259
|
+
// merged is settled here when the settle sweep could not establish identity
|
|
1260
|
+
// (#497). Every other exit returns 0.
|
|
1261
|
+
let settled = 0;
|
|
1262
|
+
for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
|
|
1263
|
+
const facts: ClassifyFacts = {};
|
|
1264
|
+
let classifiedRun = run;
|
|
1265
|
+
if (run.state === "failed" || run.state === "killed") {
|
|
1266
|
+
const sessionError = readSessionError(run.sessionFile);
|
|
1267
|
+
if (sessionError !== undefined) {
|
|
1268
|
+
if (run.lastError === undefined || run.lastError === sessionError.message) {
|
|
1269
|
+
facts.sessionError = sessionError;
|
|
1270
|
+
}
|
|
1271
|
+
if (run.lastError === undefined) {
|
|
1272
|
+
store.updateRun(run.id, { lastError: sessionError.message });
|
|
1273
|
+
classifiedRun = { ...run, lastError: sessionError.message };
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
try {
|
|
1278
|
+
if (run.prUrl !== undefined) {
|
|
1279
|
+
const pr = await tracker.prState(run.prUrl);
|
|
1280
|
+
if (pr !== undefined) facts.pr = pr;
|
|
1281
|
+
if (run.state === "pushed-green" && facts.pr === "open") {
|
|
1282
|
+
facts.mergeable = await tracker.mergeable(run.prUrl);
|
|
1283
|
+
}
|
|
1284
|
+
if (run.state === "failed" && facts.pr === "open") {
|
|
1285
|
+
facts.checks = await tracker.checkConclusions(run.prUrl);
|
|
1286
|
+
// When a check failed with a reachable log, pull its tail so the
|
|
1287
|
+
// table can tell an infra outage (#177) from a real test failure by
|
|
1288
|
+
// the log's own words. First failure wins; a log that cannot be
|
|
1289
|
+
// fetched is left undefined and classification stays conservative.
|
|
1290
|
+
// GitHub reports check states as `FAILURE` while the classifier reads
|
|
1291
|
+
// them lowercased — normalise so the live seam and the table agree on
|
|
1292
|
+
// which check is the failure whose log we pull (#177).
|
|
1293
|
+
const firstFailure = facts.checks.find((c) => normalise(c.state) === "failure" && c.link !== undefined);
|
|
1294
|
+
if (firstFailure?.link !== undefined) {
|
|
1295
|
+
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
} catch (err) {
|
|
1300
|
+
// Per row, like every other sweep here: one unreachable PR must not stop
|
|
1301
|
+
// the rest from being classified. The next tick asks again for free.
|
|
1302
|
+
log(`#${run.issue} not classified: fact gathering failed (${errText(err)}) — retrying next tick`);
|
|
1303
|
+
continue;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
|
|
1307
|
+
|
|
1308
|
+
// A healthy green PR is not a failure of any class. Leaving the row
|
|
1309
|
+
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
1310
|
+
// base does move under it.
|
|
1311
|
+
if (run.state === "pushed-green" && cls === "unknown") continue;
|
|
1312
|
+
|
|
1313
|
+
const retry = run.failureClass !== undefined;
|
|
1314
|
+
store.updateRun(run.id, { failureClass: cls, recoveryAction: recovery });
|
|
1315
|
+
log(
|
|
1316
|
+
retry
|
|
1317
|
+
? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
|
|
1318
|
+
: `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
|
|
1319
|
+
);
|
|
1320
|
+
if (recovery === "settle") settled += 1;
|
|
1321
|
+
await recoverRun(d, classifiedRun, cls, recovery, evidence);
|
|
1322
|
+
}
|
|
1323
|
+
return settled;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/** Performs the one action a class names. Never chooses one of its own. */
|
|
1327
|
+
async function recoverRun(
|
|
1328
|
+
d: SettlementDeps,
|
|
1329
|
+
run: RunRecord,
|
|
1330
|
+
cls: FailureClass,
|
|
1331
|
+
recovery: RecoveryAction,
|
|
1332
|
+
evidence: string,
|
|
1333
|
+
): Promise<void> {
|
|
1334
|
+
const { project, caps, tracker, store } = d;
|
|
1335
|
+
const inProgress = project.stateLabels.inProgress;
|
|
1336
|
+
|
|
1337
|
+
if (recovery === "settle") {
|
|
1338
|
+
// Enqueue the release with the terminal write (see `settlePushedGreen`):
|
|
1339
|
+
// the outbox keeps the label and the row one fact, so a tracker refusal
|
|
1340
|
+
// can no longer strand `agent:in-progress` with nothing left to retry it
|
|
1341
|
+
// (#18, #201).
|
|
1342
|
+
releaseInProgress(d, run.issue, `PR merged: ${evidence}`);
|
|
1343
|
+
// #607: a recovered row whose PR is observed merged is settled work, so
|
|
1344
|
+
// the claim gate comes off with the same projection as the sweep does.
|
|
1345
|
+
releaseQueueLabel(d, run.issue, `PR merged: ${evidence}`);
|
|
1346
|
+
store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
|
|
1347
|
+
log(`#${run.issue} settled from ${cls}: ${evidence}`);
|
|
1348
|
+
return;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
if (recovery === "continue") {
|
|
1352
|
+
// Two classes recover by continuing, and only one of them has anything left
|
|
1353
|
+
// to do here.
|
|
1354
|
+
//
|
|
1355
|
+
// `turn-cap-progress` was already handed back by the completion path, which
|
|
1356
|
+
// swapped its labels and left the branch retained. There is nothing to
|
|
1357
|
+
// perform, and writing anything would be actively wrong: overwriting
|
|
1358
|
+
// `lastError` with a rebase brief tells the continuation worker to rebase a
|
|
1359
|
+
// run that simply ran out of turns, and re-swapping labels the completion
|
|
1360
|
+
// path already swapped is a pair of no-op `gh` calls. Record-only, so the
|
|
1361
|
+
// sweep stops re-offering it.
|
|
1362
|
+
if (cls === "turn-cap-progress") {
|
|
1363
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1364
|
+
log(`#${run.issue} already continuing from ${cls}: ${evidence}`);
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
// `wall-clock-cap-progress`: the completion path only auto-continues
|
|
1369
|
+
// turn-cap kills, so a wall-clock kill with work to show reaches this
|
|
1370
|
+
// sweep still holding the failed label. Swap it for the queue — the branch
|
|
1371
|
+
// is retained, so the next dispatch reattaches it and briefs a resume from
|
|
1372
|
+
// the recorded work. Killed rows gathered no tracker facts, so the
|
|
1373
|
+
// issue-open guard mirrors the requeue path's. The continuation gate is
|
|
1374
|
+
// the same one the turns path applies at kill time
|
|
1375
|
+
// (`shouldContinueAfterTurnsCap`): the row is already charged, and once
|
|
1376
|
+
// the ceiling is spent, handing back the queue label would offer a
|
|
1377
|
+
// candidate admission can never accept — only the failed label comes off,
|
|
1378
|
+
// and the exhaustion reaches a human (#490, #348).
|
|
1379
|
+
if (cls === "wall-clock-cap-progress") {
|
|
1380
|
+
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
1381
|
+
if (state !== "open") {
|
|
1382
|
+
log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
const continuation = store.continuationsFor(project.name, run.issue);
|
|
1386
|
+
if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
|
|
1387
|
+
swapToQueue(d, run.issue, project.stateLabels.failed);
|
|
1388
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1389
|
+
log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
|
|
1390
|
+
} else {
|
|
1391
|
+
store.enqueueLabelOps(project.name, [
|
|
1392
|
+
{ issue: run.issue, op: "remove", label: project.stateLabels.failed },
|
|
1393
|
+
]);
|
|
1394
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1395
|
+
await safeEscalate(d, {
|
|
1396
|
+
tier: 1,
|
|
1397
|
+
project: project.name,
|
|
1398
|
+
issue: run.issue,
|
|
1399
|
+
summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
|
|
1400
|
+
detail: [
|
|
1401
|
+
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
1402
|
+
evidence,
|
|
1403
|
+
`Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
|
|
1404
|
+
"The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
|
|
1405
|
+
].join("\n"),
|
|
1406
|
+
});
|
|
1407
|
+
await postExhaustionPostmortem(
|
|
1408
|
+
d,
|
|
1409
|
+
run,
|
|
1410
|
+
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
1411
|
+
);
|
|
1412
|
+
log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
|
|
1413
|
+
}
|
|
1414
|
+
return;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
// `merge-conflict`: the branch is retained and its PR is open, so #50's
|
|
1418
|
+
// continuation guard admits it and the next tick briefs a rebase.
|
|
1419
|
+
//
|
|
1420
|
+
// The outbox makes the retry contract one-sided: the swap is enqueued — a
|
|
1421
|
+
// durable local write that cannot fail on the tracker — before
|
|
1422
|
+
// `recoveredAt` is written, so the row can never again be taken out of
|
|
1423
|
+
// `runsNeedingClassification` with its label swap still owed. That was
|
|
1424
|
+
// the defect 0.4.4 claimed to have fixed and did not, for this one
|
|
1425
|
+
// recovery; the projector retries until the tracker takes the swap, and
|
|
1426
|
+
// while it is pending the eligibility overlay keeps the issue coherent
|
|
1427
|
+
// (#201).
|
|
1428
|
+
swapToQueue(d, run.issue, inProgress);
|
|
1429
|
+
store.updateRun(run.id, {
|
|
1430
|
+
state: "killed",
|
|
1431
|
+
lastError:
|
|
1432
|
+
"merge-conflict: base moved under a green PR — rebase, regenerate recorded artifacts, push without force",
|
|
1433
|
+
recoveredAt: Date.now(),
|
|
1434
|
+
});
|
|
1435
|
+
log(`#${run.issue} requeued for a rebase continuation: ${evidence}`);
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
if (recovery === "requeue") {
|
|
1440
|
+
if (cls === "provider-credit") {
|
|
1441
|
+
await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
|
|
1442
|
+
}
|
|
1443
|
+
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
1444
|
+
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
1445
|
+
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
1446
|
+
// with no chance of success, so after a bounded number of strikes this
|
|
1447
|
+
// escalates to a human instead (#168, #177).
|
|
1448
|
+
if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
|
|
1449
|
+
await safeEscalate(d, {
|
|
1450
|
+
tier: 1,
|
|
1451
|
+
project: project.name,
|
|
1452
|
+
issue: run.issue,
|
|
1453
|
+
summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
|
|
1454
|
+
detail: [
|
|
1455
|
+
`The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
|
|
1456
|
+
"The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
|
|
1457
|
+
].join("\n"),
|
|
1458
|
+
});
|
|
1459
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1460
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
1461
|
+
return;
|
|
1462
|
+
}
|
|
1463
|
+
// Same bound for provider-transient: an issue whose stream keeps stalling
|
|
1464
|
+
// mid-run is requeued free (no attempt, no continuation charged) — but a
|
|
1465
|
+
// provider that aborts three times for one issue is down, and a human has
|
|
1466
|
+
// to check its status before hand-requeueing (#220). The escalation names
|
|
1467
|
+
// every model the chain tried, so a merged branch built on a different
|
|
1468
|
+
// model is attributable (#286).
|
|
1469
|
+
if (
|
|
1470
|
+
cls === "provider-transient" &&
|
|
1471
|
+
store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
|
|
1472
|
+
) {
|
|
1473
|
+
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
1474
|
+
await safeEscalate(d, {
|
|
1475
|
+
tier: 1,
|
|
1476
|
+
project: project.name,
|
|
1477
|
+
issue: run.issue,
|
|
1478
|
+
summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
|
|
1479
|
+
detail: [
|
|
1480
|
+
`The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
|
|
1481
|
+
...(tried === ""
|
|
1482
|
+
? []
|
|
1483
|
+
: [`Models tried: ${tried}.`]),
|
|
1484
|
+
"Check provider status before requeueing by hand.",
|
|
1485
|
+
].join("\n"),
|
|
1486
|
+
});
|
|
1487
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1488
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
1489
|
+
return;
|
|
1490
|
+
}
|
|
1491
|
+
// Same bound for provider-capacity: a run the provider throttled into the
|
|
1492
|
+
// ground is requeued free (no attempt charged) — but a provider that
|
|
1493
|
+
// throttles the same issue three times is at capacity, and a human has to
|
|
1494
|
+
// check its status before hand-requeueing (#573). On a chain-configured
|
|
1495
|
+
// project each requeue already moved the next attempt to the next chain
|
|
1496
|
+
// model, so this escalation is what catches the no-chain case and the
|
|
1497
|
+
// exhausted chain; it names every model the chain tried.
|
|
1498
|
+
if (
|
|
1499
|
+
cls === "provider-capacity" &&
|
|
1500
|
+
store.classCountFor(project.name, run.issue, "provider-capacity") >= PROVIDER_CAPACITY_MAX_STRIKES
|
|
1501
|
+
) {
|
|
1502
|
+
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
1503
|
+
await safeEscalate(d, {
|
|
1504
|
+
tier: 1,
|
|
1505
|
+
project: project.name,
|
|
1506
|
+
issue: run.issue,
|
|
1507
|
+
summary: `[provider-capacity] #${run.issue}: the model provider is throttling this run into the ground — ${evidence}`,
|
|
1508
|
+
detail: [
|
|
1509
|
+
`The provider answered #${run.issue} with sustained in-session rate limits ${PROVIDER_CAPACITY_MAX_STRIKES} times in a row; the harness retried each and was exhausted.`,
|
|
1510
|
+
...(tried === ""
|
|
1511
|
+
? []
|
|
1512
|
+
: [`Models tried: ${tried}.`]),
|
|
1513
|
+
"Check the provider's rate-limit status (and its throughput-oriented routes) before requeueing by hand.",
|
|
1514
|
+
].join("\n"),
|
|
1515
|
+
});
|
|
1516
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1517
|
+
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
1518
|
+
return;
|
|
1519
|
+
}
|
|
1520
|
+
// Only when the tracker still shows this issue as ours to hand back. An
|
|
1521
|
+
// issue that is closed, or has no state label, was resolved by another route
|
|
1522
|
+
// and requeueing it would dispatch work nobody asked for.
|
|
1523
|
+
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
1524
|
+
if (state !== "open") {
|
|
1525
|
+
log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
1526
|
+
return;
|
|
1527
|
+
}
|
|
1528
|
+
// A clean orphan whose queue label is already absent is a deliberate
|
|
1529
|
+
// withdrawal — the operator took the issue out of the queue (e.g. so a
|
|
1530
|
+
// daemon restart could not re-dispatch work known to be unsafe) — and
|
|
1531
|
+
// recovery must not recreate that intent (#423). Read the live label set;
|
|
1532
|
+
// when the queue label is gone, release the dispatcher-owned in-progress
|
|
1533
|
+
// label and stop, leaving the operator's withdrawal to survive recovery.
|
|
1534
|
+
if (cls === "orphan-clean") {
|
|
1535
|
+
const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
|
|
1536
|
+
if (snapshot === undefined) {
|
|
1537
|
+
log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
if (!snapshot.labels.includes(project.queueLabel)) {
|
|
1541
|
+
store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
|
|
1542
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1543
|
+
log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
|
|
1544
|
+
return;
|
|
1545
|
+
}
|
|
1546
|
+
// #439: an orphan-clean row charges the continuation budget, so once
|
|
1547
|
+
// `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
|
|
1548
|
+
// holds the issue on — requeueing re-adds a queue label for a candidate the
|
|
1549
|
+
// dispatcher can never admit. Stop handing it back and hold it instead: the
|
|
1550
|
+
// queue label comes off (so admission never re-holds on every dispatch),
|
|
1551
|
+
// the in-progress label is released, and a single diagnosis escalates once.
|
|
1552
|
+
// `orphan-clean` is deliberately NOT a global exclusion from the budget (a
|
|
1553
|
+
// worker that genuinely keeps dying mid-work must still be bounded); this is
|
|
1554
|
+
// the missing ceiling check this path never had (#348's invariant).
|
|
1555
|
+
const continuations = store.continuationsFor(project.name, run.issue);
|
|
1556
|
+
if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
|
|
1557
|
+
const runs = store.runsForIssue(project.name, run.issue);
|
|
1558
|
+
const breakdown = continuationBreakdown(runs);
|
|
1559
|
+
const artifact = newestContinuableRun(runs);
|
|
1560
|
+
const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
|
|
1561
|
+
const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
|
|
1562
|
+
const artifactLine =
|
|
1563
|
+
artifact === undefined
|
|
1564
|
+
? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
|
|
1565
|
+
: `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
|
|
1566
|
+
store.enqueueLabelOps(project.name, [
|
|
1567
|
+
{ issue: run.issue, op: "remove", label: inProgress },
|
|
1568
|
+
{ issue: run.issue, op: "remove", label: project.queueLabel },
|
|
1569
|
+
]);
|
|
1570
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1571
|
+
await safeEscalate(d, {
|
|
1572
|
+
tier: 1,
|
|
1573
|
+
project: project.name,
|
|
1574
|
+
issue: run.issue,
|
|
1575
|
+
summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
|
|
1576
|
+
detail: [
|
|
1577
|
+
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
1578
|
+
`Continuations by failure class: ${classLine}.`,
|
|
1579
|
+
onlyDaemonStops
|
|
1580
|
+
? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
|
|
1581
|
+
: "Continuations span real work — inspect what each attempt left behind before continuing.",
|
|
1582
|
+
artifactLine,
|
|
1583
|
+
"What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
|
|
1584
|
+
].join("\n"),
|
|
1585
|
+
});
|
|
1586
|
+
await postExhaustionPostmortem(
|
|
1587
|
+
d,
|
|
1588
|
+
run,
|
|
1589
|
+
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
1590
|
+
);
|
|
1591
|
+
log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
|
|
1592
|
+
return;
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
|
|
1596
|
+
swapToQueue(d, run.issue, label);
|
|
1597
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1598
|
+
log(`#${run.issue} requeued from ${cls}: ${evidence}`);
|
|
1599
|
+
return;
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
if (recovery === "rerun-checks") {
|
|
1603
|
+
if (run.prUrl === undefined) return;
|
|
1604
|
+
try {
|
|
1605
|
+
await tracker.rerunFailedChecks(run.prUrl);
|
|
1606
|
+
} catch (err) {
|
|
1607
|
+
log(`#${run.issue} check re-run failed (${errText(err)}) — retrying next tick`);
|
|
1608
|
+
return;
|
|
1609
|
+
}
|
|
1610
|
+
// Back to pending rather than green: the existing settle sweep re-verifies
|
|
1611
|
+
// it against the recorded head on a later tick, so nothing here has to guess
|
|
1612
|
+
// whether the re-run passed.
|
|
1613
|
+
store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
|
|
1614
|
+
log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
|
|
1615
|
+
return;
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
if (recovery === "escalate") {
|
|
1619
|
+
const detail = [evidence];
|
|
1620
|
+
if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
|
|
1621
|
+
const calls = lastToolCalls(run.sessionFile);
|
|
1622
|
+
detail.push(
|
|
1623
|
+
calls.length === 0
|
|
1624
|
+
? "transcript unreadable — no tool calls could be recovered"
|
|
1625
|
+
: `Last ${calls.length} tool calls: ${calls.join(" → ")}`,
|
|
1626
|
+
);
|
|
1627
|
+
}
|
|
1628
|
+
if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
|
|
1629
|
+
// #172: an unwritten transcript is "the run died before it flushed", not a
|
|
1630
|
+
// link to a file the operator will open and find missing.
|
|
1631
|
+
detail.push(
|
|
1632
|
+
run.sessionFile === undefined
|
|
1633
|
+
? "Session: (no transcript)"
|
|
1634
|
+
: existsSync(run.sessionFile)
|
|
1635
|
+
? `Session: ${run.sessionFile}`
|
|
1636
|
+
: `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
|
|
1637
|
+
);
|
|
1638
|
+
// The class and the run are in the summary, which is what the notifications
|
|
1639
|
+
// ledger dedupes on — so one class escalates once per run rather than every
|
|
1640
|
+
// five minutes.
|
|
1641
|
+
await safeEscalate(d, {
|
|
1642
|
+
tier: 1,
|
|
1643
|
+
project: project.name,
|
|
1644
|
+
issue: run.issue,
|
|
1645
|
+
runId: run.id,
|
|
1646
|
+
summary: `[${cls}] #${run.issue} attempt ${run.attempt}: ${evidence}`,
|
|
1647
|
+
detail: detail.join("\n"),
|
|
1648
|
+
});
|
|
1649
|
+
// The hand-off IS the recovery for these classes: there is nothing else this
|
|
1650
|
+
// package can do, and leaving the row unrecovered would re-escalate forever.
|
|
1651
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
1652
|
+
return;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
// `hold` (orphan-dirty) and `none`: recorded, nothing performed. The existing
|
|
1656
|
+
// unsalvaged-WIP admission hold already fails dispatch closed until an
|
|
1657
|
+
// operator acknowledges the tree, which is the only safe move when the
|
|
1658
|
+
// worktree holds the only copy of real work.
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
/**
|
|
1662
|
+
* Enqueue a state-label → queue-label swap for projection (#201).
|
|
1663
|
+
*
|
|
1664
|
+
* The swap is two ops in id order — remove first, then add — which is the
|
|
1665
|
+
* atomicity the projector guarantees: the issue never sits newly eligible
|
|
1666
|
+
* without a queue label on its way back, and the add never lands before the
|
|
1667
|
+
* remove when GitHub fails between them. Enqueueing is a durable local write
|
|
1668
|
+
* that cannot fail on the tracker, so the caller records its recovery
|
|
1669
|
+
* immediately and the projector retries the swap until the tracker takes it —
|
|
1670
|
+
* that closes the 0.4.4 hole where a refused label swap stranded the row
|
|
1671
|
+
* permanently under a log line promising a retry.
|
|
1672
|
+
*/
|
|
1673
|
+
export function swapToQueue(d: Pick<SettlementDeps, "project" | "store">, issue: number, label: string): void {
|
|
1674
|
+
d.store.enqueueLabelOps(d.project.name, [
|
|
1675
|
+
{ issue, op: "remove", label },
|
|
1676
|
+
{ issue, op: "add", label: d.project.queueLabel },
|
|
1677
|
+
]);
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
|
|
1681
|
+
// ------------------------------------------------------------------- status rendering of salvaged runs
|
|
1682
|
+
|
|
1683
|
+
/**
|
|
1684
|
+
* The WIP block: every issue whose newest attempt left work behind, and
|
|
1685
|
+
* whether that work is safe.
|
|
1686
|
+
*
|
|
1687
|
+
* Blocked runs used to be invisible here, which is exactly how #118 stayed
|
|
1688
|
+
* invisible for a full attempt cycle — the operator saw a blocked issue and had
|
|
1689
|
+
* no way to tell "stopped with 34 uncommitted files" from "stopped clean".
|
|
1690
|
+
* A preserved line is informational; an UNSALVAGED line is an alarm, and it
|
|
1691
|
+
* names the directory because that directory is the work.
|
|
1692
|
+
*/
|
|
1693
|
+
export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
|
|
1694
|
+
if (runs.length === 0) return [];
|
|
1695
|
+
const lines = ["", "wip"];
|
|
1696
|
+
for (const r of runs) {
|
|
1697
|
+
lines.push(
|
|
1698
|
+
r.salvageError !== undefined && r.salvageAckAt === undefined
|
|
1699
|
+
? ` #${r.issue} UNSALVAGED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
|
|
1700
|
+
`only copy, dispatch held (${r.salvageError})`
|
|
1701
|
+
: r.salvageError !== undefined
|
|
1702
|
+
? ` #${r.issue} accepted as lost attempt ${r.attempt} (${r.salvageError})`
|
|
1703
|
+
: ` #${r.issue} preserved ${r.salvageSha ?? "?"} on ${r.branch} (attempt ${r.attempt}, ${r.state})`,
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
return lines;
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
/**
|
|
1710
|
+
* The quarantined block (#737): retained worktrees whose object store could
|
|
1711
|
+
* not be made sound, so the daemon refused to fetch into them and their
|
|
1712
|
+
* commits cannot be verified against any remote. Distinct from the `wip` block
|
|
1713
|
+
* on purpose — quarantine is not work the exit left behind, it is a tree the
|
|
1714
|
+
* daemon is refusing to trust, and "potentially stranded" outranks any
|
|
1715
|
+
* wording that reads like routine retention. The line names the directory
|
|
1716
|
+
* because the directory is the work.
|
|
1717
|
+
*/
|
|
1718
|
+
export function formatQuarantinedRuns(runs: readonly RunRecord[]): string[] {
|
|
1719
|
+
if (runs.length === 0) return [];
|
|
1720
|
+
const lines = ["", "quarantined"];
|
|
1721
|
+
for (const r of runs) {
|
|
1722
|
+
lines.push(
|
|
1723
|
+
` #${r.issue} QUARANTINED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
|
|
1724
|
+
(r.quarantineDetail ?? "object store could not be made sound"),
|
|
1725
|
+
);
|
|
1726
|
+
}
|
|
1727
|
+
return lines;
|
|
1728
|
+
}
|