omp-conductor 0.19.6 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +27 -2
- package/agents/to-spec.md +76 -9
- package/package.json +1 -1
- package/schema/config.schema.json +4 -0
- package/src/arm-challenge.ts +204 -85
- package/src/ask.ts +130 -615
- package/src/board.ts +7 -1
- package/src/brief-upgrade.ts +24 -0
- package/src/briefs/console.md +253 -0
- package/src/briefs/correction.md +203 -0
- package/src/briefs/orchestrator.md +167 -97
- package/src/briefs/policy.md +19 -16
- package/src/briefs/to-spec.md +76 -9
- package/src/briefs/worker.md +50 -16
- package/src/cli.ts +4 -0
- package/src/command-manifest.ts +54 -8
- package/src/commands/arm.ts +113 -49
- package/src/commands/console.ts +70 -0
- package/src/commands/context.ts +2 -0
- package/src/commands/epic.ts +132 -0
- package/src/commands/extend.ts +9 -1
- package/src/commands/intake.ts +44 -14
- package/src/commands/stats.ts +19 -4
- package/src/commands/worker.ts +9 -1
- package/src/config-schema.ts +13 -0
- package/src/config.ts +27 -0
- package/src/daemon/ack.ts +159 -0
- package/src/daemon/admission-pass.ts +135 -0
- package/src/daemon/brief.ts +461 -0
- package/src/daemon/deps.ts +539 -0
- package/src/daemon/dispatch.ts +1779 -0
- package/src/daemon/drain.ts +185 -0
- package/src/daemon/groom-pass.ts +412 -0
- package/src/daemon/http.ts +417 -0
- package/src/daemon/integrity.ts +108 -0
- package/src/daemon/panes.ts +180 -0
- package/src/daemon/review.ts +1888 -0
- package/src/daemon/runtime.ts +736 -0
- package/src/daemon/settle-pass.ts +589 -0
- package/src/daemon/supervision.ts +438 -0
- package/src/daemon/tick.ts +968 -0
- package/src/daemon/views.ts +751 -0
- package/src/daemon.ts +105 -7832
- package/src/dashboard/app.js +58 -0
- package/src/dashboard/controls.ts +22 -3
- package/src/dashboard/server.ts +4 -0
- package/src/diff-flags.ts +24 -3
- package/src/doctor.ts +17 -12
- package/src/escalate.ts +39 -21
- package/src/failure-class.ts +75 -1
- package/src/fleet.ts +1218 -304
- package/src/groom.ts +461 -0
- package/src/http-token.ts +142 -0
- package/src/knowledge.ts +229 -0
- package/src/mining.ts +316 -0
- package/src/orchestrator-tick.ts +428 -1681
- package/src/ready-gate.ts +267 -0
- package/src/settlement.ts +72 -6
- package/src/setup-host.ts +32 -9
- package/src/setup-wizard.ts +55 -7
- package/src/setup.ts +229 -3
- package/src/stats.ts +257 -2
- package/src/status-render.ts +158 -7
- package/src/store.ts +646 -26
- package/src/to-spec.ts +194 -21
- package/src/tracker/github.ts +50 -0
- package/src/types.ts +435 -15
- package/src/verbs/protocol.ts +28 -0
- package/src/verbs/server.ts +384 -12
- package/src/wake.ts +19 -2
- package/src/worker.ts +456 -1
|
@@ -0,0 +1,1779 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Taking one admitted issue to a running session, and closing the books when it
|
|
3
|
+
* ends: the claim, the lease, the worktree, the launch, the settle.
|
|
4
|
+
*
|
|
5
|
+
* This is the module the whole dispatcher exists for, and it is drawn around a
|
|
6
|
+
* single run's life rather than around a tick: `handleIssue` is called once per
|
|
7
|
+
* admitted issue and owns every irreversible step in between, so the ordering
|
|
8
|
+
* that makes a crash recoverable — provenance before label, label before
|
|
9
|
+
* launch, lease before work — is readable in one place.
|
|
10
|
+
*
|
|
11
|
+
* The two in-process registries live here for the same reason. A turn-limit
|
|
12
|
+
* controller and a worker-control slot are handles on a *running* session; they
|
|
13
|
+
* are created by `runDaemon`, held by the HTTP surface, and consumed here, and
|
|
14
|
+
* the only place that knows when a slot begins and ends is the launch path.
|
|
15
|
+
*
|
|
16
|
+
* The review-correction lane (`review.ts`) relaunches through the same helpers —
|
|
17
|
+
* `swapLabel`, `endedBy`, `verifyPushedGreenClaim`, `shouldContinueAfterTurnsCap`,
|
|
18
|
+
* `workerReleaseBlockRecorder` — and imports them from here. That direction is
|
|
19
|
+
* deliberate and one-way: a correction is another launch of the same run, so it
|
|
20
|
+
* borrows the launch path's rules rather than restating them.
|
|
21
|
+
*/
|
|
22
|
+
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
23
|
+
import { dirname, join } from "node:path";
|
|
24
|
+
import { hasContinuationBudget, runSpendAllowanceUsd } from "../admission.ts";
|
|
25
|
+
import { resolveReleaseGrants, stateDir } from "../config.ts";
|
|
26
|
+
import { oauthFenceVerdict } from "../credential-class.ts";
|
|
27
|
+
import { formatSettlementFlags, withDerivedChangedLine } from "../diff-flags.ts";
|
|
28
|
+
import { providerCreditRefusal, providerTransientFault } from "../failure-class.ts";
|
|
29
|
+
import { openWorkerPane, reportWorkerPaneState, retireWorkerPane, type WorkerPaneOutcome } from "../fleet.ts";
|
|
30
|
+
import { pushRunBranch, type RunRepoRef } from "../gitops.ts";
|
|
31
|
+
import { errText, log, safeEscalate } from "../log.ts";
|
|
32
|
+
import { DEFAULT_MODEL_FALLBACK_THRESHOLD, fallbackClause, providerFailureFacts, resolveDispatchModel } from "../model-fallback.ts";
|
|
33
|
+
import { materializeOmpSettings } from "../omp-settings.ts";
|
|
34
|
+
import { isPaused, setPaused } from "../pause.ts";
|
|
35
|
+
import { recordReleaseBlock, type GateShape, type ReleaseBlockContext } from "../release-policy.ts";
|
|
36
|
+
import { reliabilitySettlementLine } from "../reports.ts";
|
|
37
|
+
import { branchName, type Routed } from "../routing.ts";
|
|
38
|
+
import { collectSettlementFlags, reactToProviderCredit, readSessionError, recordOperatorStop, recordWorkerDiscoveries, settleWorktree, swapToQueue } from "../settlement.ts";
|
|
39
|
+
import type { EffectiveModel, FileLane, IssueComment, PrState, RunRecord, RunState, Store, Tracker } from "../types.ts";
|
|
40
|
+
import { listenVerbChannel, type VerbListener } from "../verbs/server.ts";
|
|
41
|
+
import { ensureVerbSocketDir, verbSocketPath } from "../verbs/socket.ts";
|
|
42
|
+
import { ORPHAN_RESUME_PROMPT, runWorker, type KilledBy, type WorkerPauseControl, type WorkerPausePhase, type WorkerResult } from "../worker.ts";
|
|
43
|
+
import { addRunRepo, mirrorPathFor, removeWorktree, worktreePathFor, type RunPublisher } from "../worktree.ts";
|
|
44
|
+
import { writeAdmissionAck } from "./ack.ts";
|
|
45
|
+
import { buildBrief, renderAnsweredBlockPrompt, renderDiscussion, resumeLaneBlock } from "./brief.ts";
|
|
46
|
+
import { completionLastError, exhaustedSessionReason, githubRepo, knowledgeRepoKey, repoSlug, verbDeps, type Deps, type TurnLimitController, type TurnLimitRegistry, type WorkerControlRegistry, type WorkerControlSlot } from "./deps.ts";
|
|
47
|
+
import { readDrain } from "./drain.ts";
|
|
48
|
+
import { workspaceOwnership } from "./panes.ts";
|
|
49
|
+
import { wakeOrchestratorForBlockedRun } from "./supervision.ts";
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Record a state-label swap for projection (#201). The add enqueues before the
|
|
53
|
+
* remove — the reverse order would leave a window where the issue carries no
|
|
54
|
+
* state label at all, which is exactly the shape `isEligible` treats as fresh
|
|
55
|
+
* work. Synchronous: enqueueing is a local store write and cannot fail on the
|
|
56
|
+
* tracker; the projector applies the pair in order and retries on refusal.
|
|
57
|
+
*/
|
|
58
|
+
export function swapLabel(store: Store, projectName: string, issue: number, from: string, to: string): void {
|
|
59
|
+
store.enqueueLabelOps(projectName, [
|
|
60
|
+
{ issue, op: "add", label: to },
|
|
61
|
+
{ issue, op: "remove", label: from },
|
|
62
|
+
]);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* How a run's end is named — in the salvage commit, and to whoever reads it.
|
|
66
|
+
* The whole clause, not a bare reason: a graceful block was not killed by
|
|
67
|
+
* anything, and the commit subject is read during recovery.
|
|
68
|
+
*/
|
|
69
|
+
export function endedBy(killedBy: KilledBy | undefined): string {
|
|
70
|
+
if (killedBy === "turns") return "killed by the turns cap";
|
|
71
|
+
if (killedBy === "wallclock") return "killed by the wall-clock cap";
|
|
72
|
+
if (killedBy === "spend") return "killed by the per-run spend cap";
|
|
73
|
+
return "killed by a failed run";
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Authoritative live-run turn controls. Persistence happens synchronously
|
|
78
|
+
* before the in-memory ceiling changes, so no turn event can interleave.
|
|
79
|
+
*/
|
|
80
|
+
export function createTurnLimitRegistry(
|
|
81
|
+
persist: (runId: string, maxTurns: number) => void,
|
|
82
|
+
): TurnLimitRegistry {
|
|
83
|
+
const active = new Map<string, { runId: string; maxTurns: number }>();
|
|
84
|
+
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
85
|
+
return {
|
|
86
|
+
open(project, issue, runId, maxTurns) {
|
|
87
|
+
const k = key(project, issue);
|
|
88
|
+
if (active.has(k)) throw new Error(`#${issue} already has a live turn controller`);
|
|
89
|
+
const entry = { runId, maxTurns };
|
|
90
|
+
active.set(k, entry);
|
|
91
|
+
return {
|
|
92
|
+
maxTurns: () => entry.maxTurns,
|
|
93
|
+
close: () => {
|
|
94
|
+
if (active.get(k) === entry) active.delete(k);
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
},
|
|
98
|
+
extend(project, issue, maxTurns) {
|
|
99
|
+
if (!Number.isSafeInteger(maxTurns) || maxTurns < 1) {
|
|
100
|
+
throw new RangeError(`turn ceiling must be a positive integer, got ${maxTurns}`);
|
|
101
|
+
}
|
|
102
|
+
const entry = active.get(key(project, issue));
|
|
103
|
+
if (entry === undefined) return { kind: "not-active" };
|
|
104
|
+
if (maxTurns <= entry.maxTurns) {
|
|
105
|
+
return { kind: "not-increase", runId: entry.runId, maxTurns: entry.maxTurns };
|
|
106
|
+
}
|
|
107
|
+
persist(entry.runId, maxTurns);
|
|
108
|
+
entry.maxTurns = maxTurns;
|
|
109
|
+
return { kind: "extended", runId: entry.runId, maxTurns };
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Authoritative controls for sessions owned by this daemon process. */
|
|
115
|
+
export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
116
|
+
interface Entry {
|
|
117
|
+
project: string;
|
|
118
|
+
issue: number;
|
|
119
|
+
runId: string;
|
|
120
|
+
control?: WorkerPauseControl;
|
|
121
|
+
stopReason?: string;
|
|
122
|
+
stopError?: string;
|
|
123
|
+
finished: PromiseWithResolvers<void>;
|
|
124
|
+
onPhase?: (phase: WorkerPausePhase) => void;
|
|
125
|
+
/** Who asked for the live pause, and when — cleared on resume (#997). */
|
|
126
|
+
pausedBy?: { source: string; at: number };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const active = new Map<string, Entry>();
|
|
130
|
+
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
131
|
+
return {
|
|
132
|
+
open(project, issue, runId, onPhase) {
|
|
133
|
+
const k = key(project, issue);
|
|
134
|
+
if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
|
|
135
|
+
const entry: Entry = {
|
|
136
|
+
project,
|
|
137
|
+
issue,
|
|
138
|
+
runId,
|
|
139
|
+
finished: Promise.withResolvers<void>(),
|
|
140
|
+
...(onPhase === undefined ? {} : { onPhase }),
|
|
141
|
+
};
|
|
142
|
+
active.set(k, entry);
|
|
143
|
+
return {
|
|
144
|
+
install: (control) => {
|
|
145
|
+
if (active.get(k) !== entry) return;
|
|
146
|
+
entry.control = control;
|
|
147
|
+
if (entry.stopReason === undefined) return;
|
|
148
|
+
try {
|
|
149
|
+
control.stop(entry.stopReason);
|
|
150
|
+
} catch (err) {
|
|
151
|
+
entry.stopError = err instanceof Error ? err.message : String(err);
|
|
152
|
+
}
|
|
153
|
+
},
|
|
154
|
+
requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
|
|
155
|
+
close: () => {
|
|
156
|
+
if (active.get(k) === entry) active.delete(k);
|
|
157
|
+
entry.finished.resolve();
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
},
|
|
161
|
+
async pause(project, issue, source) {
|
|
162
|
+
const entry = active.get(key(project, issue));
|
|
163
|
+
if (entry?.control === undefined) return { kind: "not-active" };
|
|
164
|
+
try {
|
|
165
|
+
await entry.control.pause();
|
|
166
|
+
const phase = entry.control.phase();
|
|
167
|
+
entry.pausedBy = { source, at: Date.now() };
|
|
168
|
+
entry.onPhase?.(phase);
|
|
169
|
+
return { kind: "ok", runId: entry.runId, phase };
|
|
170
|
+
} catch (err) {
|
|
171
|
+
return {
|
|
172
|
+
kind: "refused",
|
|
173
|
+
runId: entry.runId,
|
|
174
|
+
error: err instanceof Error ? err.message : String(err),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
},
|
|
178
|
+
resume(project, issue) {
|
|
179
|
+
const entry = active.get(key(project, issue));
|
|
180
|
+
if (entry?.control === undefined) return { kind: "not-active" };
|
|
181
|
+
try {
|
|
182
|
+
entry.control.resume();
|
|
183
|
+
const phase = entry.control.phase();
|
|
184
|
+
delete entry.pausedBy;
|
|
185
|
+
entry.onPhase?.(phase);
|
|
186
|
+
return { kind: "ok", runId: entry.runId, phase };
|
|
187
|
+
} catch (err) {
|
|
188
|
+
return {
|
|
189
|
+
kind: "refused",
|
|
190
|
+
runId: entry.runId,
|
|
191
|
+
error: err instanceof Error ? err.message : String(err),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
},
|
|
195
|
+
async stop(project, issue, reason) {
|
|
196
|
+
const entry = active.get(key(project, issue));
|
|
197
|
+
if (entry === undefined) return { kind: "not-active" };
|
|
198
|
+
if (entry.stopReason === undefined) {
|
|
199
|
+
entry.stopReason = reason;
|
|
200
|
+
if (entry.control !== undefined) {
|
|
201
|
+
try {
|
|
202
|
+
entry.control.stop(reason);
|
|
203
|
+
} catch (err) {
|
|
204
|
+
entry.stopReason = undefined;
|
|
205
|
+
return {
|
|
206
|
+
kind: "refused",
|
|
207
|
+
runId: entry.runId,
|
|
208
|
+
error: err instanceof Error ? err.message : String(err),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
await entry.finished.promise;
|
|
214
|
+
if (entry.stopError !== undefined) {
|
|
215
|
+
return { kind: "refused", runId: entry.runId, error: entry.stopError };
|
|
216
|
+
}
|
|
217
|
+
return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
|
|
218
|
+
},
|
|
219
|
+
snapshot(project) {
|
|
220
|
+
const workers: {
|
|
221
|
+
issue: number;
|
|
222
|
+
runId: string;
|
|
223
|
+
phase: WorkerPausePhase;
|
|
224
|
+
source?: string;
|
|
225
|
+
pausedAtMs?: number;
|
|
226
|
+
}[] = [];
|
|
227
|
+
for (const entry of active.values()) {
|
|
228
|
+
if (entry.project !== project || entry.control === undefined) continue;
|
|
229
|
+
const phase = entry.control.phase();
|
|
230
|
+
if (phase === "running") continue;
|
|
231
|
+
workers.push({
|
|
232
|
+
issue: entry.issue,
|
|
233
|
+
runId: entry.runId,
|
|
234
|
+
phase,
|
|
235
|
+
...(entry.pausedBy === undefined
|
|
236
|
+
? {}
|
|
237
|
+
: { source: entry.pausedBy.source, pausedAtMs: entry.pausedBy.at }),
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
return workers;
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* The daemon half of a worker's `pushed-green` claim (#85's contact with
|
|
247
|
+
* reality; #782's publication attribution).
|
|
248
|
+
*
|
|
249
|
+
* A yield is a transport, not proof of GitHub side effects: `claim` is
|
|
250
|
+
* caller-supplied text and the worker adapter turns it into a green result
|
|
251
|
+
* without the daemon ever having seen a mediated publication verb for it — the
|
|
252
|
+
* shape that let the #777 incident guess a non-existent PR. The tracker read
|
|
253
|
+
* at the end proves the PR is open and green at the exact head; the evidence
|
|
254
|
+
* gate here proves that PR is *this run's own mediated work* first.
|
|
255
|
+
*
|
|
256
|
+
* Two things are asked, in cost order:
|
|
257
|
+
*
|
|
258
|
+
* - the claimed URL must equal the one this run's row records, which is never
|
|
259
|
+
* a worker-supplied string at the moment a live run is verified —
|
|
260
|
+
* conductor_pr_create (create or adoption) writes it on the run, the daemon
|
|
261
|
+
* seeds it at claim from a terminal predecessor on the same branch (#434),
|
|
262
|
+
* and orchestrator-only recovery writes it for settled runs;
|
|
263
|
+
* - and the claimed head must have a mediated publisher: an allowed
|
|
264
|
+
* `conductor_push` on this exact run that published it on this run's branch
|
|
265
|
+
* (the verb only ever pushes `refs/heads/<branch>`), an allowed
|
|
266
|
+
* `conductor_pr_create` on this run for this exact PR, or — for a run that
|
|
267
|
+
* published nothing new — the live tip of this run's own branch.
|
|
268
|
+
*
|
|
269
|
+
* That last path is the one the row cannot supply. It used to be served by
|
|
270
|
+
* comparing the claim against `run.prUrl`/`run.headSha`, and both are
|
|
271
|
+
* worker-tainted upstream: terminal settlement writes a worker's reported pair
|
|
272
|
+
* onto the row *before* any verification, and continuation inheritance
|
|
273
|
+
* validates only the predecessor's metadata and open state — so a turn-capped
|
|
274
|
+
* attempt that reported someone else's real green PR had that pair inherited
|
|
275
|
+
* and re-presented as its own evidence. Asking the tracker for the branch tip
|
|
276
|
+
* removes the worker from the loop entirely: `refs/heads/<branch>` is a ref
|
|
277
|
+
* only a mediated `conductor_push` or a mediated `conductor_pr_update_branch`
|
|
278
|
+
* can move, so its live commit is daemon provenance no reported string can
|
|
279
|
+
* forge. It is also why a legitimate mediated base-branch update now passes:
|
|
280
|
+
* that server-side merge creates a head no `conductor_push` ever published and
|
|
281
|
+
* the row still carries the older one, which the recorded-pair test rejected.
|
|
282
|
+
*
|
|
283
|
+
* Binding to the *exact current run* is what makes an old attempt, a different
|
|
284
|
+
* branch, or #806's orchestrator-only settled-run recovery invisible here: the
|
|
285
|
+
* ledger query is run-scoped by `runId`, and recoveries store no runId at all.
|
|
286
|
+
* That query asks for the run's complete history rather than the ledger's
|
|
287
|
+
* newest rows — publication evidence sits at the *start* of a run, and a
|
|
288
|
+
* review-revision round or a burst of refused mutations pushed it past the
|
|
289
|
+
* default page, turning a verified push into a definitive false failure.
|
|
290
|
+
*
|
|
291
|
+
* A claim that fails this gate is a definitive failure — never a retryable
|
|
292
|
+
* `pushed-pending` — because a guessed URL is not something a later tick is
|
|
293
|
+
* waiting on. A claim the gate could not *read* is the opposite: an unreadable
|
|
294
|
+
* branch tip is #781's transient outage, so it stays retryable rather than
|
|
295
|
+
* burning an attempt on a flaky read.
|
|
296
|
+
*/
|
|
297
|
+
export async function verifyPushedGreenClaim(
|
|
298
|
+
tracker: Pick<Tracker, "verifyPr" | "branchHead">,
|
|
299
|
+
claim: Pick<WorkerResult, "prUrl" | "headSha">,
|
|
300
|
+
publication: {
|
|
301
|
+
project: string;
|
|
302
|
+
issue: number;
|
|
303
|
+
runId: string;
|
|
304
|
+
/** The branch conductor routed this run onto. `conductor_push` publishes
|
|
305
|
+
* exactly `refs/heads/<branch>` and refuses any other ref. */
|
|
306
|
+
branch: string;
|
|
307
|
+
/** The identity `tracker.branchHead` reads the live tip with: the
|
|
308
|
+
* canonical `owner/repo` when the routed clone URL carries one, else the
|
|
309
|
+
* routed repository name. A tracker that cannot resolve it answers
|
|
310
|
+
* undefined, which stays retryable rather than definitive. */
|
|
311
|
+
repo: string;
|
|
312
|
+
store: Pick<Store, "verbLedger" | "getRun">;
|
|
313
|
+
},
|
|
314
|
+
): Promise<{
|
|
315
|
+
state: "pushed-green" | "pushed-pending" | "failed";
|
|
316
|
+
reason?: string;
|
|
317
|
+
}> {
|
|
318
|
+
const { prUrl, headSha } = claim;
|
|
319
|
+
if (prUrl === undefined || headSha === undefined) {
|
|
320
|
+
return { state: "failed", reason: "Worker did not report a PR URL and observed head SHA" };
|
|
321
|
+
}
|
|
322
|
+
// Exact current run, never "some publication for the issue": the query is
|
|
323
|
+
// scoped to this runId plus project/issue, so a previous attempt's verbs and
|
|
324
|
+
// the orchestrator's recovery verbs are invisible here. Unbounded on
|
|
325
|
+
// purpose — see the note above about evidence ageing off the newest page.
|
|
326
|
+
const ledger = publication.store.verbLedger(publication.project, {
|
|
327
|
+
runId: publication.runId,
|
|
328
|
+
issue: publication.issue,
|
|
329
|
+
limit: Number.MAX_SAFE_INTEGER,
|
|
330
|
+
});
|
|
331
|
+
const run = publication.store.getRun(publication.runId);
|
|
332
|
+
const branchRef = `refs/heads/${publication.branch}`;
|
|
333
|
+
if (run?.prUrl !== prUrl) {
|
|
334
|
+
return {
|
|
335
|
+
state: "failed",
|
|
336
|
+
reason:
|
|
337
|
+
"Pushed-green claim has no mediated publication evidence: the claimed PR is not this run's " +
|
|
338
|
+
"recorded PR",
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
const pushedThisHead = ledger.some(
|
|
342
|
+
(entry) =>
|
|
343
|
+
entry.decision === "allowed" &&
|
|
344
|
+
entry.verb === "conductor_push" &&
|
|
345
|
+
entry.sha === headSha &&
|
|
346
|
+
entry.detail.includes(branchRef),
|
|
347
|
+
);
|
|
348
|
+
// Bound to the claimed PR, not merely to "a create happened": both allowed
|
|
349
|
+
// details name the URL they produced (`opened <url> …`, `adopted <url> …`),
|
|
350
|
+
// so an unrelated create on this run cannot vouch for another PR.
|
|
351
|
+
const createdHere = ledger.some(
|
|
352
|
+
(entry) =>
|
|
353
|
+
entry.decision === "allowed" &&
|
|
354
|
+
entry.verb === "conductor_pr_create" &&
|
|
355
|
+
entry.detail.includes(prUrl),
|
|
356
|
+
);
|
|
357
|
+
|
|
358
|
+
if (!pushedThisHead && !createdHere) {
|
|
359
|
+
// Nothing this run published carries the claimed head, so the only
|
|
360
|
+
// remaining evidence is the branch itself: a continuation or review round
|
|
361
|
+
// that pushed nothing, or a head a mediated base-branch update produced.
|
|
362
|
+
let tip: string | undefined;
|
|
363
|
+
try {
|
|
364
|
+
tip = await tracker.branchHead(publication.repo, publication.branch);
|
|
365
|
+
} catch {
|
|
366
|
+
tip = undefined;
|
|
367
|
+
}
|
|
368
|
+
if (tip === undefined) {
|
|
369
|
+
return {
|
|
370
|
+
state: "pushed-pending",
|
|
371
|
+
reason: `Live head of ${branchRef} unavailable; retrying`,
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
if (tip !== headSha) {
|
|
375
|
+
return {
|
|
376
|
+
state: "failed",
|
|
377
|
+
reason:
|
|
378
|
+
"Pushed-green claim has no mediated publication evidence: this run's conductor_push / " +
|
|
379
|
+
`conductor_pr_create ledger does not cover the claimed head, and ${branchRef} is at ${tip}, ` +
|
|
380
|
+
"not the claimed head",
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
const verification = await tracker.verifyPr(prUrl, headSha);
|
|
386
|
+
if (verification === undefined) {
|
|
387
|
+
return { state: "pushed-pending", reason: "GitHub PR verification unavailable; retrying" };
|
|
388
|
+
}
|
|
389
|
+
if (verification.status === "green") return { state: "pushed-green" };
|
|
390
|
+
return {
|
|
391
|
+
state: verification.status === "pending" ? "pushed-pending" : "failed",
|
|
392
|
+
reason: verification.reason,
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Whether a turns-cap kill is handed straight back to the queue.
|
|
398
|
+
*
|
|
399
|
+
* A turns cap consumes the independent continuation budget rather than an
|
|
400
|
+
* implementation-failure attempt, so a run that ran out of turns mid-work is
|
|
401
|
+
* requeued and the next tick reattaches its branch (#50).
|
|
402
|
+
*
|
|
403
|
+
* The artifact test is what keeps that from becoming a loop. A run that reached
|
|
404
|
+
* the ceiling with no PR, no observed head and no salvage commit produced
|
|
405
|
+
* nothing to continue *from*, and requeueing it spends another continuation on
|
|
406
|
+
* the same spin — the seven rows on this fleet that burned $109 producing no
|
|
407
|
+
* merged PR. That shape is `turn-cap-spinning`, which #132 says must escalate
|
|
408
|
+
* with evidence instead; requeueing it here would settle the question before the
|
|
409
|
+
* classifier ever saw the row, and the queue label would already be back on.
|
|
410
|
+
*
|
|
411
|
+
* Pure so the rule is testable without a worker session — the same split as
|
|
412
|
+
* {@link settlementFor}.
|
|
413
|
+
*/
|
|
414
|
+
export function shouldContinueAfterTurnsCap(f: {
|
|
415
|
+
killedBy?: string;
|
|
416
|
+
prUrl?: string;
|
|
417
|
+
headSha?: string;
|
|
418
|
+
salvageSha?: string;
|
|
419
|
+
continuation: number;
|
|
420
|
+
maxContinuations: number;
|
|
421
|
+
}): boolean {
|
|
422
|
+
if (f.killedBy !== "turns") return false;
|
|
423
|
+
const hasArtifacts = f.prUrl !== undefined || f.headSha !== undefined || f.salvageSha !== undefined;
|
|
424
|
+
if (!hasArtifacts) return false;
|
|
425
|
+
return hasContinuationBudget(f.continuation, f.maxContinuations);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/** Bind a worker's run identity to every release-policy block it emits. */
|
|
429
|
+
export function workerReleaseBlockRecorder(
|
|
430
|
+
project: string,
|
|
431
|
+
issue: number,
|
|
432
|
+
runId: string,
|
|
433
|
+
root = stateDir(),
|
|
434
|
+
): (shape: GateShape, context: ReleaseBlockContext) => void {
|
|
435
|
+
return (shape, context) =>
|
|
436
|
+
recordReleaseBlock(
|
|
437
|
+
project,
|
|
438
|
+
"worker",
|
|
439
|
+
shape,
|
|
440
|
+
{
|
|
441
|
+
...context,
|
|
442
|
+
issue,
|
|
443
|
+
runId,
|
|
444
|
+
},
|
|
445
|
+
root,
|
|
446
|
+
);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* The exact PR a new claim inherits from its terminal predecessor (#434).
|
|
451
|
+
*
|
|
452
|
+
* `recentRuns` shows only the newest attempt per issue, and the mediated verbs
|
|
453
|
+
* resolve a PR's owner through the store — so the moment an orphan-clean
|
|
454
|
+
* requeue creates its continuation row, the predecessor's green PR (recorded
|
|
455
|
+
* on the older row) stops resolving and every mediated verb refuses it as "not
|
|
456
|
+
* a pull request any run opened". Seeding the continuation row with the same
|
|
457
|
+
* PR at claim time closes that window: ownership never depends on a worker
|
|
458
|
+
* rediscovering and re-reporting the PR.
|
|
459
|
+
*
|
|
460
|
+
* The checks are deliberately the exact counterpart of the store query's scope:
|
|
461
|
+
* the claim's project/issue/repo/branch must all match the predecessor, the
|
|
462
|
+
* predecessor must be terminal (a live row is still doing its own work, and a
|
|
463
|
+
* claim beside one is already a double dispatch), and the tracker must confirm
|
|
464
|
+
* the PR is still open — a closed or merged predecessor PR is settled by the
|
|
465
|
+
* sweep against the row that owns it, never carried onto a fresh attempt. An
|
|
466
|
+
* unreadable tracker fails closed: the new row simply starts without PR
|
|
467
|
+
* ownership, exactly as before #434, and adoption or the worker's own
|
|
468
|
+
* pr_create recovers the link.
|
|
469
|
+
*/
|
|
470
|
+
export async function inheritedPrForContinuation(
|
|
471
|
+
tracker: Tracker,
|
|
472
|
+
prior: RunRecord | undefined,
|
|
473
|
+
claim: { project: string; issue: number; repo: string; branch: string },
|
|
474
|
+
): Promise<{ prUrl: string; headSha?: string } | undefined> {
|
|
475
|
+
if (prior === undefined || prior.prUrl === undefined) return undefined;
|
|
476
|
+
// Terminal only. A live predecessor owns its PR outright and is not done
|
|
477
|
+
// with it; inheriting would launder a double dispatch's row.
|
|
478
|
+
if (prior.state === "claimed" || prior.state === "running") return undefined;
|
|
479
|
+
if (prior.project !== claim.project || prior.issue !== claim.issue) return undefined;
|
|
480
|
+
if (prior.repo !== claim.repo || prior.branch !== claim.branch) return undefined;
|
|
481
|
+
let state: PrState | undefined;
|
|
482
|
+
try {
|
|
483
|
+
state = await tracker.prState(prior.prUrl);
|
|
484
|
+
} catch {
|
|
485
|
+
return undefined;
|
|
486
|
+
}
|
|
487
|
+
if (state !== "open") return undefined;
|
|
488
|
+
return { prUrl: prior.prUrl, ...(prior.headSha === undefined ? {} : { headSha: prior.headSha }) };
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
/**
|
|
492
|
+
* Which continuation, if any, may resume its predecessor's session instead of
|
|
493
|
+
* dispatching a cold brief.
|
|
494
|
+
*
|
|
495
|
+
* Two shapes qualify, and they resume differently:
|
|
496
|
+
*
|
|
497
|
+
* - `orphan-clean` (#536): a daemon death interrupted a healthy worker.
|
|
498
|
+
* `reconcileOrphanedRuns` KEEPS the worktree (salvage commit included) and
|
|
499
|
+
* the transcript is file-backed, so the next attempt gets its own memory
|
|
500
|
+
* back whole: the same worktree, the same session directory, `resume: true`.
|
|
501
|
+
* The old dispatch built a fresh `run-<uuid>` session and re-read the repo
|
|
502
|
+
* from zero — the exact rediscovery orphan-clean spent turns on in #535.
|
|
503
|
+
* - `answered-block` (Phase 3 inner loop): the worker asked a question, parked,
|
|
504
|
+
* and an operator answered. Its tree was removed at the blocked settle
|
|
505
|
+
* because the branch already held the work, so this one REATTACHES the branch
|
|
506
|
+
* the ordinary way and resumes only the session — the transcript is the
|
|
507
|
+
* expensive part, and today's cold brief throws away a whole investigation
|
|
508
|
+
* to deliver one answer.
|
|
509
|
+
*
|
|
510
|
+
* Everything else dispatches cold, and that is not laziness: a cap-killed or
|
|
511
|
+
* otherwise failed worker was killed for cause, and orphan-dirty is held
|
|
512
|
+
* precisely because the worktree is the only copy.
|
|
513
|
+
*
|
|
514
|
+
* An answered block additionally has to pass #1047's health test. Resuming a
|
|
515
|
+
* session that is out of turns or that the provider abandoned buys a session
|
|
516
|
+
* that dies the same way with the answer unread, which is the #1035 shape in a
|
|
517
|
+
* different lane. The orphan-clean case deliberately keeps its own rule set: it
|
|
518
|
+
* is defined by an interruption rather than an ending, and #536's contract is
|
|
519
|
+
* that such a run continues.
|
|
520
|
+
*
|
|
521
|
+
* The checks here are what the daemon can prove cheaply before the claim (the
|
|
522
|
+
* transcript is present and non-empty; the worktree, where the mode needs one);
|
|
523
|
+
* the harness's own `continueRecent` is the backstop, and a corrupt transcript
|
|
524
|
+
* it silently falls back from is surfaced loudly by the `sessionFile` lineage
|
|
525
|
+
* compare at the dispatch site rather than left quiet.
|
|
526
|
+
*/
|
|
527
|
+
export type ResumeMode = "orphan-clean" | "answered-block";
|
|
528
|
+
|
|
529
|
+
export function continuationResumeVerdict(
|
|
530
|
+
prior: RunRecord | undefined,
|
|
531
|
+
): { kind: "resume"; prior: RunRecord; mode: ResumeMode } | { kind: "fresh"; reason?: string } {
|
|
532
|
+
const mode: ResumeMode | undefined =
|
|
533
|
+
prior === undefined
|
|
534
|
+
? undefined
|
|
535
|
+
: prior.state === "orphaned" && prior.failureClass === "orphan-clean"
|
|
536
|
+
? "orphan-clean"
|
|
537
|
+
: prior.state === "blocked"
|
|
538
|
+
? "answered-block"
|
|
539
|
+
: undefined;
|
|
540
|
+
if (prior === undefined || mode === undefined) return { kind: "fresh" };
|
|
541
|
+
const what = mode === "orphan-clean" ? "the orphaned attempt" : "the blocked attempt";
|
|
542
|
+
if (prior.sessionFile === undefined) {
|
|
543
|
+
return { kind: "fresh", reason: `${what} recorded no transcript` };
|
|
544
|
+
}
|
|
545
|
+
try {
|
|
546
|
+
if (!existsSync(prior.sessionFile)) {
|
|
547
|
+
return { kind: "fresh", reason: `transcript ${prior.sessionFile} is gone` };
|
|
548
|
+
}
|
|
549
|
+
if (statSync(prior.sessionFile).size === 0) {
|
|
550
|
+
return { kind: "fresh", reason: `transcript ${prior.sessionFile} is empty` };
|
|
551
|
+
}
|
|
552
|
+
} catch (err) {
|
|
553
|
+
return { kind: "fresh", reason: `transcript ${prior.sessionFile} is unreadable (${errText(err)})` };
|
|
554
|
+
}
|
|
555
|
+
if (mode === "orphan-clean" && (prior.worktree === "" || !existsSync(prior.worktree))) {
|
|
556
|
+
return { kind: "fresh", reason: `worktree ${prior.worktree} is gone` };
|
|
557
|
+
}
|
|
558
|
+
if (mode === "answered-block") {
|
|
559
|
+
const exhausted = exhaustedSessionReason(prior);
|
|
560
|
+
if (exhausted !== undefined) {
|
|
561
|
+
return { kind: "fresh", reason: `${what}'s session cannot continue: ${exhausted}` };
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
return { kind: "resume", prior, mode };
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* One attempt at one issue, from claim to terminal state. Everything is inside
|
|
569
|
+
* a single try/catch so that a bad issue costs its own run and nothing else.
|
|
570
|
+
*/
|
|
571
|
+
export async function handleIssue(
|
|
572
|
+
d: Deps,
|
|
573
|
+
r: Routed,
|
|
574
|
+
attempt: number,
|
|
575
|
+
admittedLane?: FileLane,
|
|
576
|
+
admittedModel?: EffectiveModel,
|
|
577
|
+
): Promise<void> {
|
|
578
|
+
const { project, caps, tracker, store } = d;
|
|
579
|
+
const issue = r.issue.number;
|
|
580
|
+
const branch = branchName(r.issue);
|
|
581
|
+
const inProgress = project.stateLabels.inProgress;
|
|
582
|
+
|
|
583
|
+
let claimed = false;
|
|
584
|
+
let run: RunRecord | undefined;
|
|
585
|
+
// Hoisted out of the try so the catch path can still name and save the tree:
|
|
586
|
+
// a crash mid-dispatch is one of the ends whose uncommitted work has to be
|
|
587
|
+
// salvaged too, and it is the path least likely to have committed first.
|
|
588
|
+
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
589
|
+
let worktreePath: string | undefined;
|
|
590
|
+
let turnLimit: TurnLimitController | undefined;
|
|
591
|
+
let workerControl: WorkerControlSlot | undefined;
|
|
592
|
+
let workerSessionInstalled = false;
|
|
593
|
+
// The run's Herdr representation (#840), hoisted for the same reason the verb
|
|
594
|
+
// listener is: whoever tears the run down has to hand the pane's lifecycle
|
|
595
|
+
// authority back, and a representation that outlives its child is a worker the
|
|
596
|
+
// workspace still shows as live.
|
|
597
|
+
let workerPane: Extract<WorkerPaneOutcome, { kind: "tracked" }> | undefined;
|
|
598
|
+
// Herdr orders lifecycle reports by `seq`, so this run owns one counter and
|
|
599
|
+
// every report takes the next value (#842). A shared or restarted counter would
|
|
600
|
+
// let a late report overwrite a newer state — the pane would then show `working`
|
|
601
|
+
// for a worker that has already blocked.
|
|
602
|
+
let paneSeq = 0;
|
|
603
|
+
const nextPaneSeq = (): number => (paneSeq += 1);
|
|
604
|
+
/**
|
|
605
|
+
* The injected Herdr runner, when this daemon carries one (#1035); every
|
|
606
|
+
* representation call below shares it so tests see one argv stream. The
|
|
607
|
+
* ownership adapter is always present in production — the store-backed
|
|
608
|
+
* half of workspace discovery, so what a Herdr restart forgets, the
|
|
609
|
+
* durable row remembers (#1035 review).
|
|
610
|
+
*/
|
|
611
|
+
const herdr = {
|
|
612
|
+
...(d.herdrRun === undefined ? {} : { run: d.herdrRun }),
|
|
613
|
+
ownership: workspaceOwnership(d.store),
|
|
614
|
+
};
|
|
615
|
+
/**
|
|
616
|
+
* Project one authoritative transition onto the pane (#842).
|
|
617
|
+
*
|
|
618
|
+
* The input is always a durable run transition or a typed session event —
|
|
619
|
+
* never the pane's own output, which is display and nothing more. A failure is
|
|
620
|
+
* logged, not raised: the workspace lagging is not worth failing a run over.
|
|
621
|
+
*/
|
|
622
|
+
const projectPaneState = (state: "working" | "idle" | "blocked" | "unknown", message?: string): void => {
|
|
623
|
+
if (workerPane === undefined) return;
|
|
624
|
+
const reported = reportWorkerPaneState(workerPane.paneId, workerPane.label, state, {
|
|
625
|
+
...herdr,
|
|
626
|
+
seq: nextPaneSeq(),
|
|
627
|
+
...(message === undefined ? {} : { message }),
|
|
628
|
+
});
|
|
629
|
+
if (!reported.ok) log(`#${issue} herdr pane state ${state} not reported: ${reported.reason}`);
|
|
630
|
+
};
|
|
631
|
+
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
632
|
+
// the catch and finally paths have to publish the branch.
|
|
633
|
+
let runRepo: RunRepoRef | undefined;
|
|
634
|
+
// #126: the run's own verb socket. Hoisted like the run repo, because the catch
|
|
635
|
+
// and finally paths have to close it — a socket outliving its run is a
|
|
636
|
+
// channel nobody is authenticating any more.
|
|
637
|
+
let verbListener: VerbListener | undefined;
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* Publishes the run's branch on the privileged side: run repo → mirror →
|
|
641
|
+
* GitHub, fast-forward only. The dispatcher performs every network git
|
|
642
|
+
* operation for a run so the settlement record and the branch cannot
|
|
643
|
+
* disagree (#126); this is the only route a worker's commits take out, and it
|
|
644
|
+
* is also what stops a per-run repository from being the *only* copy when the
|
|
645
|
+
* tree is removed.
|
|
646
|
+
*/
|
|
647
|
+
const publish: RunPublisher = async () => {
|
|
648
|
+
if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
|
|
649
|
+
return pushRunBranch(project, runRepo);
|
|
650
|
+
};
|
|
651
|
+
|
|
652
|
+
/**
|
|
653
|
+
* A stop can arrive after the run is claimed but before `runWorker` exposes
|
|
654
|
+
* its controller. Settle that run here instead of turning an operator action
|
|
655
|
+
* into a dispatch failure or making the HTTP request wait for a session that
|
|
656
|
+
* will never exist.
|
|
657
|
+
*/
|
|
658
|
+
const settleStopBeforeSession = async (): Promise<boolean> => {
|
|
659
|
+
const reason = workerControl?.requestedStop();
|
|
660
|
+
if (reason === undefined || run === undefined || workerSessionInstalled) return false;
|
|
661
|
+
turnLimit?.close();
|
|
662
|
+
turnLimit = undefined;
|
|
663
|
+
const settlement =
|
|
664
|
+
worktreePath === undefined
|
|
665
|
+
? undefined
|
|
666
|
+
: await settleWorktree({
|
|
667
|
+
issue,
|
|
668
|
+
attempt,
|
|
669
|
+
ending: `stopped by the operator: ${reason}`,
|
|
670
|
+
worktree: worktreePath,
|
|
671
|
+
branch,
|
|
672
|
+
publish,
|
|
673
|
+
tree: "remove",
|
|
674
|
+
mirrorPath,
|
|
675
|
+
});
|
|
676
|
+
recordOperatorStop(store, {
|
|
677
|
+
project: project.name,
|
|
678
|
+
issue,
|
|
679
|
+
runId: run.id,
|
|
680
|
+
inProgress,
|
|
681
|
+
reason,
|
|
682
|
+
patch: {
|
|
683
|
+
endedAt: Date.now(),
|
|
684
|
+
turns: run.turns,
|
|
685
|
+
spendUsd: run.spendUsd,
|
|
686
|
+
worktree: worktreePath ?? run.worktree,
|
|
687
|
+
report: [
|
|
688
|
+
"Operator stopped the run before its worker session started.",
|
|
689
|
+
`Reason: ${reason}`,
|
|
690
|
+
...(settlement?.lines ?? []),
|
|
691
|
+
].join("\n"),
|
|
692
|
+
...settlement?.patch,
|
|
693
|
+
},
|
|
694
|
+
});
|
|
695
|
+
log(`#${issue} stopped by operator before its worker session started: ${reason}`);
|
|
696
|
+
return true;
|
|
697
|
+
};
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* The daemon-wide stop fence can land after the claim but before `runWorker`
|
|
701
|
+
* exposes its controller — the same window `settleStopBeforeSession` closes
|
|
702
|
+
* for a per-run operator stop, entered from the shutdown path instead of the
|
|
703
|
+
* pause control. Settle identically (close the claim, never launch) so the
|
|
704
|
+
* drain waits only for workers admitted before the stop request, and the
|
|
705
|
+
* shutdown cannot extend its own workload (#374).
|
|
706
|
+
*/
|
|
707
|
+
const settleDrainBeforeSession = async (): Promise<boolean> => {
|
|
708
|
+
if (d.drain?.draining !== true || run === undefined || workerSessionInstalled) return false;
|
|
709
|
+
turnLimit?.close();
|
|
710
|
+
turnLimit = undefined;
|
|
711
|
+
const settlement =
|
|
712
|
+
worktreePath === undefined
|
|
713
|
+
? undefined
|
|
714
|
+
: await settleWorktree({
|
|
715
|
+
issue,
|
|
716
|
+
attempt,
|
|
717
|
+
ending: "daemon shutdown began while the run was being prepared",
|
|
718
|
+
worktree: worktreePath,
|
|
719
|
+
branch,
|
|
720
|
+
publish,
|
|
721
|
+
tree: "remove",
|
|
722
|
+
mirrorPath,
|
|
723
|
+
});
|
|
724
|
+
recordOperatorStop(store, {
|
|
725
|
+
project: project.name,
|
|
726
|
+
issue,
|
|
727
|
+
runId: run.id,
|
|
728
|
+
inProgress,
|
|
729
|
+
reason: "daemon shutdown began after the claim",
|
|
730
|
+
patch: {
|
|
731
|
+
endedAt: Date.now(),
|
|
732
|
+
turns: run.turns,
|
|
733
|
+
spendUsd: run.spendUsd,
|
|
734
|
+
worktree: worktreePath ?? run.worktree,
|
|
735
|
+
report: [
|
|
736
|
+
"Daemon shutdown began while this run was still being prepared; the worker",
|
|
737
|
+
"never started and the claim was closed so the shutdown could complete.",
|
|
738
|
+
...(settlement?.lines ?? []),
|
|
739
|
+
].join("\n"),
|
|
740
|
+
...settlement?.patch,
|
|
741
|
+
},
|
|
742
|
+
});
|
|
743
|
+
log(`#${issue} not launched: daemon shutdown began after the claim`);
|
|
744
|
+
return true;
|
|
745
|
+
};
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* The launch half of the credential-class fence (#852).
|
|
749
|
+
*
|
|
750
|
+
* Admission already held every candidate while a required provider billed to
|
|
751
|
+
* the wrong credential, but a grant can be disabled in the window *between*
|
|
752
|
+
* that check and this spawn — a refresh failing, an operator revoking, a token
|
|
753
|
+
* expiring — and that window is exactly what the incident's acceptance
|
|
754
|
+
* criterion names. So the same question is asked again here, as the last thing
|
|
755
|
+
* before the session exists, and the run is closed rather than launched.
|
|
756
|
+
*
|
|
757
|
+
* Settled the same way the stop and drain fences settle: the claim closes, the
|
|
758
|
+
* tree is salvaged, nothing is launched. It is deliberately NOT a failed
|
|
759
|
+
* attempt against the issue's budget — the issue is not what is wrong, the
|
|
760
|
+
* host's credentials are, and charging an attempt for it would exhaust an
|
|
761
|
+
* issue while the operator re-authenticates.
|
|
762
|
+
*/
|
|
763
|
+
const settleCredentialClassBeforeSession = async (): Promise<boolean> => {
|
|
764
|
+
const required = project.requireOauthProviders ?? [];
|
|
765
|
+
if (required.length === 0 || run === undefined || workerSessionInstalled) return false;
|
|
766
|
+
const fence = await oauthFenceVerdict(
|
|
767
|
+
required,
|
|
768
|
+
d.probeCredentialClass ??
|
|
769
|
+
(async (provider) => ({
|
|
770
|
+
ok: false,
|
|
771
|
+
reason: `no credential probe is wired for ${provider}`,
|
|
772
|
+
})),
|
|
773
|
+
);
|
|
774
|
+
if (fence.ok) return false;
|
|
775
|
+
turnLimit?.close();
|
|
776
|
+
turnLimit = undefined;
|
|
777
|
+
const settlement =
|
|
778
|
+
worktreePath === undefined
|
|
779
|
+
? undefined
|
|
780
|
+
: await settleWorktree({
|
|
781
|
+
issue,
|
|
782
|
+
attempt,
|
|
783
|
+
ending: "not launched: a required subscription credential is unavailable",
|
|
784
|
+
worktree: worktreePath,
|
|
785
|
+
branch,
|
|
786
|
+
publish,
|
|
787
|
+
tree: "remove",
|
|
788
|
+
mirrorPath,
|
|
789
|
+
});
|
|
790
|
+
recordOperatorStop(store, {
|
|
791
|
+
project: project.name,
|
|
792
|
+
issue,
|
|
793
|
+
runId: run.id,
|
|
794
|
+
inProgress,
|
|
795
|
+
reason: `required subscription credential unavailable: ${fence.reason}`,
|
|
796
|
+
patch: {
|
|
797
|
+
endedAt: Date.now(),
|
|
798
|
+
turns: run.turns,
|
|
799
|
+
spendUsd: run.spendUsd,
|
|
800
|
+
worktree: worktreePath ?? run.worktree,
|
|
801
|
+
report: [
|
|
802
|
+
"This run was closed instead of launched: a provider this project requires to bill",
|
|
803
|
+
"to its subscription would have billed to something else.",
|
|
804
|
+
fence.reason,
|
|
805
|
+
...(settlement?.lines ?? []),
|
|
806
|
+
].join("\n"),
|
|
807
|
+
...settlement?.patch,
|
|
808
|
+
},
|
|
809
|
+
});
|
|
810
|
+
log(`#${issue} not launched: ${fence.reason}`);
|
|
811
|
+
return true;
|
|
812
|
+
};
|
|
813
|
+
|
|
814
|
+
try {
|
|
815
|
+
// The claim-side of the stop fence (#374): the run row is the boundary the
|
|
816
|
+
// shutdown drain waits on, so the claim itself refuses once the daemon is
|
|
817
|
+
// draining. The tick re-checks before the dispatch tail; this covers the
|
|
818
|
+
// same fence for any caller that reaches `handleIssue` without one, and
|
|
819
|
+
// makes "no post-stop run rows" a property of the claim, not of its caller.
|
|
820
|
+
if (d.drain?.draining === true) {
|
|
821
|
+
log(`#${issue} not claimed: daemon is draining`);
|
|
822
|
+
return;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
// The claim-side of the pause fence (#651 review #3): the tick's pause
|
|
826
|
+
// gate sits above routing, so a tick that passed that gate before an
|
|
827
|
+
// operator wrote the freeze can still be mid-routing when the fence lands
|
|
828
|
+
// — it would claim after the setup barrier's acknowledgement and before
|
|
829
|
+
// its first mutation. The claim itself re-checks the pause and refuses,
|
|
830
|
+
// writing the durable admission acknowledgement the barrier waits for.
|
|
831
|
+
// A claim observed under the fence is the last admission boundary there
|
|
832
|
+
// is: nothing may create a run row beside a held fleet.
|
|
833
|
+
if (isPaused(d.project.name)) {
|
|
834
|
+
try {
|
|
835
|
+
writeAdmissionAck(d.project.name);
|
|
836
|
+
} catch (err) {
|
|
837
|
+
log(`admission acknowledgement write failed: ${errText(err)}`);
|
|
838
|
+
}
|
|
839
|
+
log(`#${issue} not claimed: dispatch is paused at claim time`);
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
// The claim-side of the project drain fence (#484): the tick's gate sits
|
|
844
|
+
// above routing, so a drain created after that gate can still land
|
|
845
|
+
// mid-pass — the claim re-checks and refuses while a fresh drain is in
|
|
846
|
+
// force. An invalid record fails closed the same way. The read here is
|
|
847
|
+
// observational on purpose: the claim refuses but must not consume the
|
|
848
|
+
// marker, or the first refused claim would unbind the rest of its own
|
|
849
|
+
// pass; the tick's consumeDrain removes it on the next pass, so no claim
|
|
850
|
+
// path can ever be blocked permanently by it.
|
|
851
|
+
const claimDrain = readDrain(d.project.name);
|
|
852
|
+
if (claimDrain.kind === "active" || claimDrain.kind === "error") {
|
|
853
|
+
log(
|
|
854
|
+
claimDrain.kind === "error"
|
|
855
|
+
? `#${issue} not claimed: drain record invalid (${claimDrain.problem})`
|
|
856
|
+
: `#${issue} not claimed: project drain in effect`,
|
|
857
|
+
);
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
// Claim on the STORE first, before anything that can fail. The run row —
|
|
862
|
+
// not the label — is the crash-safe guard against double dispatch: rows
|
|
863
|
+
// are local, written before any network call, and the startup orphan
|
|
864
|
+
// sweep marks process-less claimed/running rows `orphaned`, so a daemon
|
|
865
|
+
// dying mid-claim leaves a row a human can triage instead of a label only
|
|
866
|
+
// the orchestrator may touch. The in-progress label is a write-behind
|
|
867
|
+
// projection of that row: enqueued here, flushed by the post-admission
|
|
868
|
+
// projection pass in the same tick on the healthy path, and — while
|
|
869
|
+
// pending — kept effective off the ready set by the routing overlay, so a
|
|
870
|
+
// crash between the row and its label cannot double-dispatch either
|
|
871
|
+
// (#201).
|
|
872
|
+
// Read before this attempt's own row exists, so `latestRun` still means the
|
|
873
|
+
// attempt whose work this one inherits.
|
|
874
|
+
const prior = store.latestRun(project.name, issue);
|
|
875
|
+
const priorSalvage = prior?.salvageSha;
|
|
876
|
+
// The failover decision is made from the store the same way admission is:
|
|
877
|
+
// counts of this issue's own rows, so the switch is sticky per chain and a
|
|
878
|
+
// fresh issue always starts on the primary (#286). Read before the claim
|
|
879
|
+
// writes this attempt's row, which would otherwise break the streak.
|
|
880
|
+
const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
|
|
881
|
+
// Resolved before the claim, not after provisioning: the model is also
|
|
882
|
+
// part of the #536 resume decision, which has to be made before the
|
|
883
|
+
// dispatch shape (fresh provision vs kept worktree) is chosen. With no
|
|
884
|
+
// `modelFallbacks` configured this is the primary model — or none, for an
|
|
885
|
+
// unconfigured project — and today's dispatch is byte for byte what it
|
|
886
|
+
// has always been.
|
|
887
|
+
const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
|
|
888
|
+
// A `Model:` declaration admission resolved for this candidate (#535) is
|
|
889
|
+
// this issue's workerModel: the orchestrator names a tier when it
|
|
890
|
+
// promotes, and dispatch launches on that selector exactly as if the
|
|
891
|
+
// project's `workerModel` were the declared value. Absent a declaration,
|
|
892
|
+
// today's `project.workerModel` is unchanged, and the failover chain
|
|
893
|
+
// (#286) keeps its semantics in both cases — it is the same resolution,
|
|
894
|
+
// one different primary.
|
|
895
|
+
const declaredModel = admittedModel?.model;
|
|
896
|
+
// #807: a chain that already bought its one model escalation dispatches on
|
|
897
|
+
// that stronger selector, outranking both the declaration and the project
|
|
898
|
+
// default — the settlement that wrote the marker did so precisely because
|
|
899
|
+
// the previous tier spun to a cap with nothing to show. Read from the
|
|
900
|
+
// store, so it is sticky to this issue's chain and a fresh issue (no
|
|
901
|
+
// marker) resolves exactly as it always has. The provider chain still
|
|
902
|
+
// layers on top: an escalated run whose provider then aborts fails over
|
|
903
|
+
// normally, because escalation only ever changes the primary.
|
|
904
|
+
const escalation = store.modelEscalation(project.name, issue);
|
|
905
|
+
const primaryModel = escalation?.model ?? declaredModel ?? project.workerModel;
|
|
906
|
+
const choice = resolveDispatchModel({
|
|
907
|
+
workerModel: primaryModel,
|
|
908
|
+
modelFallbacks: project.modelFallbacks,
|
|
909
|
+
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
910
|
+
streak: chainFacts.streak,
|
|
911
|
+
});
|
|
912
|
+
const clause =
|
|
913
|
+
fallbackClause(choice, chainFacts, primaryModel) ??
|
|
914
|
+
(escalation === undefined
|
|
915
|
+
? undefined
|
|
916
|
+
: `on ${escalation.model} — this chain's one model escalation, after ${escalation.failureClass}`);
|
|
917
|
+
|
|
918
|
+
// Whether this attempt continues its predecessor's session instead of
|
|
919
|
+
// reading the repo from zero (#536, and the answered-block half of the
|
|
920
|
+
// Phase 3 inner loop). Decided here, before this attempt's row exists,
|
|
921
|
+
// because `prior` is still the attempt whose work this one inherits and the
|
|
922
|
+
// whole dispatch shape follows the verdict.
|
|
923
|
+
let resuming: RunRecord | undefined;
|
|
924
|
+
let resumeMode: ResumeMode | undefined;
|
|
925
|
+
const verdict = continuationResumeVerdict(prior);
|
|
926
|
+
if (verdict.kind === "resume") {
|
|
927
|
+
const resumePrior = verdict.prior;
|
|
928
|
+
// The continuation must stay on the model the interrupted session was
|
|
929
|
+
// using; a chain that now resolves differently dispatches fresh rather
|
|
930
|
+
// than quietly continuing on another model, which would smear one
|
|
931
|
+
// attempt's work across two models (#286 attribution) — and, on the
|
|
932
|
+
// review side, is the diagnosed #1035 failure itself.
|
|
933
|
+
if (resumePrior.model !== undefined && choice.model !== undefined && resumePrior.model !== choice.model) {
|
|
934
|
+
log(
|
|
935
|
+
`#${issue} attempt ${attempt} not resumed: attempt ${resumePrior.attempt} ran on ${resumePrior.model} but dispatch ` +
|
|
936
|
+
`now resolves ${choice.model} — fresh dispatch`,
|
|
937
|
+
);
|
|
938
|
+
} else {
|
|
939
|
+
resuming = resumePrior;
|
|
940
|
+
resumeMode = verdict.mode;
|
|
941
|
+
log(
|
|
942
|
+
`#${issue} attempt ${attempt} continuing session of attempt ${resumePrior.attempt} (${verdict.mode}) → ` +
|
|
943
|
+
`transcript ${resumePrior.sessionFile}, ` +
|
|
944
|
+
`${verdict.mode === "orphan-clean" ? "worktree kept" : "branch reattached"}`,
|
|
945
|
+
);
|
|
946
|
+
}
|
|
947
|
+
} else if (verdict.reason !== undefined) {
|
|
948
|
+
log(`#${issue} attempt ${attempt} not resumed: ${verdict.reason} — fresh dispatch`);
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
run = store.createRun({
|
|
952
|
+
project: project.name,
|
|
953
|
+
issue,
|
|
954
|
+
repo: r.repo.name,
|
|
955
|
+
branch,
|
|
956
|
+
worktree: "",
|
|
957
|
+
state: "claimed",
|
|
958
|
+
attempt,
|
|
959
|
+
turns: 0,
|
|
960
|
+
spendUsd: 0,
|
|
961
|
+
maxTurns: caps.workerMaxTurns,
|
|
962
|
+
startedAt: Date.now(),
|
|
963
|
+
// #567: the orphan-clean attempt this claim continues, when the verdict
|
|
964
|
+
// above fired. `undefined` for a fresh dispatch — the store maps that to
|
|
965
|
+
// NULL, so a fresh row simply never carries the field.
|
|
966
|
+
resumedFromRunId: resuming?.id,
|
|
967
|
+
// #744: the file-lane declaration admission resolved for this candidate
|
|
968
|
+
// is persisted on the row, so lane occupancy survives across dispatch
|
|
969
|
+
// passes — a later pass knows what this run *intends* to touch, not only
|
|
970
|
+
// what it has touched so far. This is the exact `Admission.lane` the gate
|
|
971
|
+
// enforced and the brief rendered, never a re-parse. Absent for a run
|
|
972
|
+
// with no declaration (fail open), exactly as it was admitted.
|
|
973
|
+
lane: admittedLane,
|
|
974
|
+
// #851: the allowance admission reserved for this run, recorded on the
|
|
975
|
+
// row before anything launches. Derived from the same caps the gate
|
|
976
|
+
// read, so the reservation the pass enforced and the one the row holds
|
|
977
|
+
// are one value; `undefined` (NULL) when the fleet has no spend cap at
|
|
978
|
+
// all, which is the truth rather than "reserved nothing".
|
|
979
|
+
spendReservedUsd: runSpendAllowanceUsd(caps) ?? undefined,
|
|
980
|
+
});
|
|
981
|
+
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
982
|
+
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
983
|
+
// its dispatch burst, the window the shutdown fence and its tests rely on
|
|
984
|
+
// (#374) — so the tuple-exact, tracker-verified inheritance lands here,
|
|
985
|
+
// still before any provisioning, network call or verb socket. The new row
|
|
986
|
+
// is `recentRuns`' newest attempt for the issue the moment it exists, so
|
|
987
|
+
// without this a requeue hides the green PR the orphaned attempt opened
|
|
988
|
+
// until a worker happens to rediscover it.
|
|
989
|
+
const inherited = await inheritedPrForContinuation(tracker, prior, {
|
|
990
|
+
project: project.name,
|
|
991
|
+
issue,
|
|
992
|
+
repo: r.repo.name,
|
|
993
|
+
branch,
|
|
994
|
+
});
|
|
995
|
+
if (inherited !== undefined && prior !== undefined) {
|
|
996
|
+
store.updateRun(run.id, {
|
|
997
|
+
prUrl: inherited.prUrl,
|
|
998
|
+
...(inherited.headSha === undefined ? {} : { headSha: inherited.headSha }),
|
|
999
|
+
});
|
|
1000
|
+
log(`#${issue} inherited PR ${inherited.prUrl} from attempt ${prior.attempt} (${prior.state})`);
|
|
1001
|
+
}
|
|
1002
|
+
const maxTurns = run.maxTurns;
|
|
1003
|
+
const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
|
|
1004
|
+
const runId = run.id;
|
|
1005
|
+
if (turnOverride !== undefined) {
|
|
1006
|
+
log(
|
|
1007
|
+
`#${issue} claimed with turn override ${turnOverride} ` +
|
|
1008
|
+
`(base ${caps.workerMaxTurns})`,
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
1012
|
+
claimed = true;
|
|
1013
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
|
|
1014
|
+
// The pane follows the phase the registry actually reached (#842) — never a
|
|
1015
|
+
// guess made at the call site, and never the pane's own output read back.
|
|
1016
|
+
workerControl = d.workerControls.open(project.name, issue, runId, (phase) => {
|
|
1017
|
+
projectPaneState(
|
|
1018
|
+
phase === "running" ? "working" : "idle",
|
|
1019
|
+
phase === "running" ? "resumed" : `${phase} by the operator`,
|
|
1020
|
+
);
|
|
1021
|
+
});
|
|
1022
|
+
if (await settleStopBeforeSession()) return;
|
|
1023
|
+
if (await settleDrainBeforeSession()) return;
|
|
1024
|
+
// The credential fence, twice, for the same reason the stop fence is
|
|
1025
|
+
// consulted more than once (#852). Here it is the cheap one: a run whose
|
|
1026
|
+
// required subscription credential is already unavailable is closed before
|
|
1027
|
+
// anything clones a mirror or cuts a worktree, because provisioning for a
|
|
1028
|
+
// launch that cannot happen is pure waste. The second call, immediately
|
|
1029
|
+
// before the spawn, is the one that closes the admission-to-session window.
|
|
1030
|
+
if (await settleCredentialClassBeforeSession()) return;
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
1034
|
+
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
1035
|
+
// to be cleared first. Both helpers are pure path math, and removeWorktree
|
|
1036
|
+
// tolerates a mirror or tree that is not there yet. An ORPHAN-CLEAN resume
|
|
1037
|
+
// skips the whole dance: the orphaned run's tree is the work to continue
|
|
1038
|
+
// (its salvage commit is already on the branch), and re-cloning it from
|
|
1039
|
+
// the mirror would be exactly the rediscovery this feature exists to skip.
|
|
1040
|
+
// An answered-block resume does NOT skip it: that run's tree was removed at
|
|
1041
|
+
// its blocked settle because the branch already held the work, so it
|
|
1042
|
+
// reattaches the branch exactly as any continuation does and resumes only
|
|
1043
|
+
// the session.
|
|
1044
|
+
let provisioned: Awaited<ReturnType<typeof addRunRepo>> | undefined;
|
|
1045
|
+
if (resuming !== undefined && resumeMode === "orphan-clean") {
|
|
1046
|
+
worktreePath = resuming.worktree;
|
|
1047
|
+
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
1048
|
+
} else {
|
|
1049
|
+
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
1050
|
+
if (await settleStopBeforeSession()) return;
|
|
1051
|
+
if (await settleDrainBeforeSession()) return;
|
|
1052
|
+
const provisionedTree = await addRunRepo(
|
|
1053
|
+
r.repo,
|
|
1054
|
+
project.mirrorRoot,
|
|
1055
|
+
project.workspaceRoot,
|
|
1056
|
+
issue,
|
|
1057
|
+
branch,
|
|
1058
|
+
);
|
|
1059
|
+
provisioned = provisionedTree;
|
|
1060
|
+
worktreePath = provisionedTree.path;
|
|
1061
|
+
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
1062
|
+
}
|
|
1063
|
+
if (await settleStopBeforeSession()) return;
|
|
1064
|
+
if (await settleDrainBeforeSession()) return;
|
|
1065
|
+
|
|
1066
|
+
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
1067
|
+
// directory and learns the real path back from the result. Inventing one
|
|
1068
|
+
// here would put a file that never gets written into an escalation.
|
|
1069
|
+
//
|
|
1070
|
+
// Per run rather than one shared directory, so one run's transcript cannot
|
|
1071
|
+
// be truncated or replaced by the next — except for a resumed attempt,
|
|
1072
|
+
// which deliberately reuses the interrupted session's directory so the
|
|
1073
|
+
// SDK's `continueRecent(cwd, dir)` picks up that transcript and keeps
|
|
1074
|
+
// writing it.
|
|
1075
|
+
const runTreeRoot = stateDir();
|
|
1076
|
+
const sessionDir =
|
|
1077
|
+
resuming === undefined
|
|
1078
|
+
? join(runTreeRoot, "sessions", `run-${String(runId)}`)
|
|
1079
|
+
: dirname(resuming.sessionFile!);
|
|
1080
|
+
mkdirSync(sessionDir, { recursive: true });
|
|
1081
|
+
|
|
1082
|
+
// The fleet-owned omp settings overlay (#537): the project's `ompSettings`
|
|
1083
|
+
// map (plus the retry keys derived from `modelFallbacks`, #539's staging
|
|
1084
|
+
// half) materialised to YAML under the run's session directory — never
|
|
1085
|
+
// inside the worktree, whose diff is the PR a worker ships. Rewritten on
|
|
1086
|
+
// every attempt, so a config edit takes effect on the next dispatch and a
|
|
1087
|
+
// resumed attempt reuses the kept session dir with the *current* config.
|
|
1088
|
+
// Absent `ompSettings` and `modelFallbacks`, no file is written, nothing
|
|
1089
|
+
// is passed, and dispatch is byte-for-byte today's.
|
|
1090
|
+
const ompSettingsFile = materializeOmpSettings(project, sessionDir);
|
|
1091
|
+
|
|
1092
|
+
// ---- the run's mutation channel (#126) -------------------------------
|
|
1093
|
+
// A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
|
|
1094
|
+
// per-run *directory*: a directory owned by the run principal would hand
|
|
1095
|
+
// back the power to unlink a sibling's socket and bind an impostor in its
|
|
1096
|
+
// place, which is the whole thing the layout buys. `listenVerbChannel`
|
|
1097
|
+
// validates every component before binding and throws rather than
|
|
1098
|
+
// degrading, so a tampered path refuses this dispatch instead of running
|
|
1099
|
+
// the worker with a channel nobody can vouch for.
|
|
1100
|
+
verbListener = await listenVerbChannel(
|
|
1101
|
+
verbDeps(d),
|
|
1102
|
+
{
|
|
1103
|
+
kind: "run",
|
|
1104
|
+
// `runTreeRoot`, not the private state dir: the slot has to *connect*
|
|
1105
|
+
// to this socket, and a 0700 ancestor makes it unreachable however
|
|
1106
|
+
// correct the socket's own mode is. `ensureVerbSocketDir` still owns
|
|
1107
|
+
// the parent and still runs its anti-rebind checks there (#126).
|
|
1108
|
+
path: verbSocketPath(ensureVerbSocketDir(runTreeRoot), `run-${String(issue)}`),
|
|
1109
|
+
project: project.name,
|
|
1110
|
+
role: "worker",
|
|
1111
|
+
runId,
|
|
1112
|
+
issue,
|
|
1113
|
+
repo: r.repo,
|
|
1114
|
+
runRepoPath: worktreePath,
|
|
1115
|
+
branch,
|
|
1116
|
+
},
|
|
1117
|
+
{
|
|
1118
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
1119
|
+
},
|
|
1120
|
+
);
|
|
1121
|
+
if (await settleStopBeforeSession()) return;
|
|
1122
|
+
if (await settleDrainBeforeSession()) return;
|
|
1123
|
+
|
|
1124
|
+
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
1125
|
+
|
|
1126
|
+
// Where this attempt goes, and (when the failover fired) the clause that
|
|
1127
|
+
// makes it attributable. `chainConfigured`/`choice`/`clause` were resolved
|
|
1128
|
+
// before the claim — the resume verdict had to be made before the dispatch
|
|
1129
|
+
// shape was chosen (#536) — so only the record write lives here.
|
|
1130
|
+
// Recorded before the launch, so even a run killed mid-flight leaves the
|
|
1131
|
+
// model it chose on its row. Only a chain-configured project writes the
|
|
1132
|
+
// column: absent `modelFallbacks` must preserve today's rows byte for byte.
|
|
1133
|
+
// An escalated chain (#807) writes it too — the selector it dispatched on
|
|
1134
|
+
// is the whole provenance of that recovery, and its harness-resolved model
|
|
1135
|
+
// lands beside it in `resolvedModel` when the run settles.
|
|
1136
|
+
if ((chainConfigured || escalation !== undefined) && choice.model !== undefined) {
|
|
1137
|
+
store.updateRun(runId, { model: choice.model });
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
log(
|
|
1141
|
+
`#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
|
|
1142
|
+
(provisioned?.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
1143
|
+
);
|
|
1144
|
+
|
|
1145
|
+
// The continuation notice replaces the brief for a resumed attempt (#536).
|
|
1146
|
+
// The original brief is already in the resumed transcript; re-sending it is
|
|
1147
|
+
// how a resumed worker ends up re-doing the work it just did. Everything
|
|
1148
|
+
// below the brief is fresh-dispatch-only, exactly as today — except the
|
|
1149
|
+
// file lane admission enforced for this continuation, which rides the
|
|
1150
|
+
// notice itself: the retained transcript's brief may show an earlier
|
|
1151
|
+
// declaration, and a worker continuing under a stale lane is the collision
|
|
1152
|
+
// the gate exists to stop (#608).
|
|
1153
|
+
//
|
|
1154
|
+
// An answered block resumes for a different reason and so carries something
|
|
1155
|
+
// different: the investigation is in the transcript, but the ANSWER it
|
|
1156
|
+
// parked for is not — it arrived as a comment afterwards. So the discussion
|
|
1157
|
+
// travels with the notice, rendered by the same budgeted renderer the cold
|
|
1158
|
+
// brief uses, and an unreadable thread degrades to a named absence rather
|
|
1159
|
+
// than silently reading as "nobody answered".
|
|
1160
|
+
let brief: string;
|
|
1161
|
+
if (resuming !== undefined && resumeMode === "answered-block") {
|
|
1162
|
+
let answered: IssueComment[] | "unread";
|
|
1163
|
+
try {
|
|
1164
|
+
answered = await tracker.listComments(issue);
|
|
1165
|
+
} catch (err) {
|
|
1166
|
+
log(`#${issue} issue comments unreadable at dispatch; the continuation will say so: ${errText(err)}`);
|
|
1167
|
+
answered = "unread";
|
|
1168
|
+
}
|
|
1169
|
+
brief =
|
|
1170
|
+
renderAnsweredBlockPrompt(issue, renderDiscussion(answered, admittedLane)) +
|
|
1171
|
+
(admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
|
|
1172
|
+
} else if (resuming !== undefined) {
|
|
1173
|
+
brief =
|
|
1174
|
+
ORPHAN_RESUME_PROMPT + (admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
|
|
1175
|
+
} else {
|
|
1176
|
+
// The discussion is rendered at dispatch so a worker never depends on a
|
|
1177
|
+
// runtime `gh` read to see the orchestrator's grooming (#517). The read is
|
|
1178
|
+
// best-effort, but its failure is not silent: an unreadable tracker names
|
|
1179
|
+
// itself in the brief's Discussion section instead of reading as "no
|
|
1180
|
+
// comments" — the exact confusion this fix removes.
|
|
1181
|
+
let comments: IssueComment[] | "unread";
|
|
1182
|
+
try {
|
|
1183
|
+
comments = await tracker.listComments(issue);
|
|
1184
|
+
} catch (err) {
|
|
1185
|
+
log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
|
|
1186
|
+
comments = "unread";
|
|
1187
|
+
}
|
|
1188
|
+
|
|
1189
|
+
brief = await buildBrief(project, r, branch, worktreePath, {
|
|
1190
|
+
continuation: provisioned?.reattached === true,
|
|
1191
|
+
defaultBranch: r.repo.defaultBranch,
|
|
1192
|
+
...(provisioned?.reattached === true && priorSalvage !== undefined
|
|
1193
|
+
? { salvagedSha: priorSalvage }
|
|
1194
|
+
: {}),
|
|
1195
|
+
comments,
|
|
1196
|
+
lane: admittedLane,
|
|
1197
|
+
model: admittedModel,
|
|
1198
|
+
host: d.host,
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
if (await settleStopBeforeSession()) return;
|
|
1202
|
+
if (await settleDrainBeforeSession()) return;
|
|
1203
|
+
// Last, so it is the newest fact anything has about the credentials — a
|
|
1204
|
+
// grant disabled since admission is caught here rather than paid for.
|
|
1205
|
+
if (await settleCredentialClassBeforeSession()) return;
|
|
1206
|
+
|
|
1207
|
+
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
let result: WorkerResult;
|
|
1211
|
+
try {
|
|
1212
|
+
const runAllowanceUsd = runSpendAllowanceUsd(caps);
|
|
1213
|
+
result = await runWorker({
|
|
1214
|
+
brief,
|
|
1215
|
+
cwd: worktreePath,
|
|
1216
|
+
caps,
|
|
1217
|
+
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
1218
|
+
maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
|
|
1219
|
+
// #851: the allowance this run reserved at admission is also its live
|
|
1220
|
+
// ceiling. Without it the reservation would only bound how many runs
|
|
1221
|
+
// start, not what one of them spends, and the day's total would be
|
|
1222
|
+
// unbounded again.
|
|
1223
|
+
...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
|
|
1224
|
+
onPauseControl: (control) => {
|
|
1225
|
+
workerSessionInstalled = true;
|
|
1226
|
+
workerControl?.install(control);
|
|
1227
|
+
},
|
|
1228
|
+
sessionDir,
|
|
1229
|
+
...(resuming === undefined ? {} : { resume: true }),
|
|
1230
|
+
// The session's control socket, under the daemon's own state directory —
|
|
1231
|
+
// a child process of the daemon reaches it directly.
|
|
1232
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
1233
|
+
verbSocketPath: verbListener.path,
|
|
1234
|
+
// Names this run's channel the instant the child exists. Until this
|
|
1235
|
+
// fires the channel refuses every connection, because a session that
|
|
1236
|
+
// reached it first would be one nobody had identified — and every
|
|
1237
|
+
// session shares this daemon's uid, so the socket alone cannot tell
|
|
1238
|
+
// them apart (#163).
|
|
1239
|
+
onSpawn: (pid) => {
|
|
1240
|
+
verbListener?.bindPid(pid);
|
|
1241
|
+
},
|
|
1242
|
+
// The workspace representation of this exact child (#840). Best effort by
|
|
1243
|
+
// construction: a worker that cannot be *shown* is still a worker, so a
|
|
1244
|
+
// failure is logged with its reason and the run proceeds. What a failure
|
|
1245
|
+
// should mean for the launch — fail closed, or a named degraded state —
|
|
1246
|
+
// is #841's policy and is deliberately not decided here. Nothing is
|
|
1247
|
+
// claimed silently: either the pane id is logged, or the reason is.
|
|
1248
|
+
pane: {
|
|
1249
|
+
open: (pid) => {
|
|
1250
|
+
// Adopt before creating (#842): a run that already carries a pane
|
|
1251
|
+
// identity gets that pane reported against, never a second one. The
|
|
1252
|
+
// durable row is what makes this survive the process that opened it,
|
|
1253
|
+
// and duplicate-prevention is exactly what it buys — a re-entered
|
|
1254
|
+
// launch for one run must not leave two panes claiming it.
|
|
1255
|
+
const recorded = store.getRun(runId);
|
|
1256
|
+
if (recorded?.paneId !== undefined && recorded.paneLabel !== undefined) {
|
|
1257
|
+
workerPane = { kind: "tracked", paneId: recorded.paneId, label: recorded.paneLabel, pid };
|
|
1258
|
+
store.updateRun(runId, { workerPid: pid });
|
|
1259
|
+
const state = reportWorkerPaneState(recorded.paneId, recorded.paneLabel, "working", {
|
|
1260
|
+
...herdr,
|
|
1261
|
+
seq: nextPaneSeq(),
|
|
1262
|
+
});
|
|
1263
|
+
log(
|
|
1264
|
+
state.ok
|
|
1265
|
+
? `#${issue} herdr pane ${recorded.paneId} re-reported for pid ${pid}`
|
|
1266
|
+
: `#${issue} herdr pane ${recorded.paneId} could not be re-reported: ${state.reason}`,
|
|
1267
|
+
);
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
1270
|
+
const outcome = openWorkerPane(
|
|
1271
|
+
{
|
|
1272
|
+
project: project.name,
|
|
1273
|
+
issue,
|
|
1274
|
+
attempt,
|
|
1275
|
+
runId,
|
|
1276
|
+
pid,
|
|
1277
|
+
...(recorded?.sessionFile === undefined ? {} : { sessionFile: recorded.sessionFile }),
|
|
1278
|
+
},
|
|
1279
|
+
herdr,
|
|
1280
|
+
);
|
|
1281
|
+
if (outcome.kind === "tracked") {
|
|
1282
|
+
workerPane = outcome;
|
|
1283
|
+
// Durable before it is announced: a pane the store does not know
|
|
1284
|
+
// about is a pane a restart cannot reconcile.
|
|
1285
|
+
store.updateRun(runId, {
|
|
1286
|
+
workerPid: pid,
|
|
1287
|
+
paneId: outcome.paneId,
|
|
1288
|
+
paneLabel: outcome.label,
|
|
1289
|
+
// Cleared, because there is now a pane: a leftover reason beside a
|
|
1290
|
+
// tracked run would keep `status` saying degraded forever (#841).
|
|
1291
|
+
paneUnavailable: null,
|
|
1292
|
+
});
|
|
1293
|
+
log(`#${issue} herdr pane ${outcome.paneId} (${outcome.label}) tracks pid ${pid}`);
|
|
1294
|
+
} else {
|
|
1295
|
+
// Named, never silent (#841): the run proceeds untracked, and the
|
|
1296
|
+
// reason is durable so `status` can say the fleet is running blind
|
|
1297
|
+
// rather than looking identical to a fleet with no workers.
|
|
1298
|
+
store.updateRun(runId, { workerPid: pid, paneUnavailable: outcome.reason });
|
|
1299
|
+
log(`#${issue} no herdr pane: ${outcome.reason}`);
|
|
1300
|
+
}
|
|
1301
|
+
},
|
|
1302
|
+
release: () => {
|
|
1303
|
+
if (workerPane === undefined) return;
|
|
1304
|
+
// Retirement, not release alone (#1035): a settled run keeps no
|
|
1305
|
+
// representation. The close reaches only the follower; the
|
|
1306
|
+
// authoritative child was never in the pane to be signalled.
|
|
1307
|
+
const retired = retireWorkerPane(workerPane.paneId, workerPane.label, {
|
|
1308
|
+
...herdr,
|
|
1309
|
+
seq: nextPaneSeq(),
|
|
1310
|
+
});
|
|
1311
|
+
if (!retired.ok) log(`#${issue} herdr pane retirement failed: ${retired.reason}`);
|
|
1312
|
+
workerPane = undefined;
|
|
1313
|
+
},
|
|
1314
|
+
},
|
|
1315
|
+
onChildLog: (line) => {
|
|
1316
|
+
log(`#${issue} ${line}`);
|
|
1317
|
+
},
|
|
1318
|
+
...(choice.model === undefined ? {} : { model: choice.model }),
|
|
1319
|
+
// The fleet-owned omp settings overlay (#537): the staged YAML the
|
|
1320
|
+
// session loads through `Settings.init({ configFiles: [<path>] })` —
|
|
1321
|
+
// the project's `ompSettings` map plus the within-run failover keys
|
|
1322
|
+
// #581 staged directly (the project's own chain, not an empty default).
|
|
1323
|
+
// Absent both, nothing is staged, today's dispatch byte for byte.
|
|
1324
|
+
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
1325
|
+
releaseGrants: resolveReleaseGrants(project),
|
|
1326
|
+
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
1327
|
+
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
1328
|
+
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
1329
|
+
// #518: recorded live, not at settlement — the moment this explains is
|
|
1330
|
+
// forty minutes before the wall-clock cap fires, so a number only a
|
|
1331
|
+
// finished run carries would answer the question too late.
|
|
1332
|
+
onTokens: (tokens) =>
|
|
1333
|
+
store.updateRun(runId, { outputTokens: tokens.output, reasoningTokens: tokens.reasoning }),
|
|
1334
|
+
onKilled: () => {
|
|
1335
|
+
turnLimit?.close();
|
|
1336
|
+
turnLimit = undefined;
|
|
1337
|
+
},
|
|
1338
|
+
// Recorded the moment the session opens its transcript, not when the run
|
|
1339
|
+
// ends: `omp-conductor tail` resolves an issue to a file through this row,
|
|
1340
|
+
// and a path written at completion is a path nobody can follow live. The
|
|
1341
|
+
// completion-time update below writes the same value again, harmlessly.
|
|
1342
|
+
onSessionFile: (f) => {
|
|
1343
|
+
store.updateRun(runId, { sessionFile: f });
|
|
1344
|
+
// #536: the harness's own `continueRecent` can still fall back to a
|
|
1345
|
+
// blank session (corrupt transcript, nothing to continue) — and a
|
|
1346
|
+
// blank session in the "resumed" worktree is indistinguishable from
|
|
1347
|
+
// today's dispatch unless the downgrade is named. Same-file lineage
|
|
1348
|
+
// is the proof the resume happened: the resumed run must keep
|
|
1349
|
+
// writing the orphaned attempt's transcript.
|
|
1350
|
+
if (resuming !== undefined && f !== resuming.sessionFile) {
|
|
1351
|
+
log(
|
|
1352
|
+
`#${issue} attempt ${attempt} resume fell back to a fresh session: opened ${f} ` +
|
|
1353
|
+
`instead of the orphaned attempt's ${resuming.sessionFile}`,
|
|
1354
|
+
);
|
|
1355
|
+
}
|
|
1356
|
+
},
|
|
1357
|
+
// The last fence (#374): every pre-launch settle check above has
|
|
1358
|
+
// passed, but the stop can still land while the session socket is
|
|
1359
|
+
// binding inside `createSession`. This gate is re-checked there,
|
|
1360
|
+
// immediately before the child spawn, and closes the run as stopped
|
|
1361
|
+
// instead of launching a worker the shutdown would have to wait for.
|
|
1362
|
+
maySpawn: () => d.drain?.draining !== true,
|
|
1363
|
+
}, d.workerDeps);
|
|
1364
|
+
} finally {
|
|
1365
|
+
// This is the authoritative settlement edge for `extend`: close before
|
|
1366
|
+
// PR verification or terminal row writes can leave stale `running` state.
|
|
1367
|
+
turnLimit?.close();
|
|
1368
|
+
turnLimit = undefined;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
// A configured model the harness could not honour means this run was done by
|
|
1372
|
+
// a different model than the operator chose. Logged per run, because it is
|
|
1373
|
+
// the only place that fact is still attached to the issue it affected.
|
|
1374
|
+
if (result.modelFallbackMessage !== undefined) {
|
|
1375
|
+
log(`#${issue} model fallback: ${result.modelFallbackMessage}`);
|
|
1376
|
+
}
|
|
1377
|
+
// What this run learned about the repo, into the fleet overlay the next
|
|
1378
|
+
// brief renders: the entry point it actually had to find, the gate that
|
|
1379
|
+
// really proved the change, the fake that looked like a test. Reported by
|
|
1380
|
+
// the worker in its own settlement, so nothing is inferred — and a
|
|
1381
|
+
// knowledge write can never fail a settlement.
|
|
1382
|
+
recordWorkerDiscoveries(knowledgeRepoKey(r.repo), issue, result.discoveries);
|
|
1383
|
+
|
|
1384
|
+
const verified: { state: RunState; reason?: string } =
|
|
1385
|
+
result.state === "pushed-green"
|
|
1386
|
+
? await verifyPushedGreenClaim(tracker, result, {
|
|
1387
|
+
project: project.name,
|
|
1388
|
+
issue,
|
|
1389
|
+
runId,
|
|
1390
|
+
branch,
|
|
1391
|
+
repo: repoSlug ?? r.repo.name,
|
|
1392
|
+
store,
|
|
1393
|
+
})
|
|
1394
|
+
: { state: result.state };
|
|
1395
|
+
const state = verified.state;
|
|
1396
|
+
|
|
1397
|
+
// Read before the row is written so `lastError` carries the provider's own
|
|
1398
|
+
// message — it names the fix and the URL, which no classification can.
|
|
1399
|
+
const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
|
|
1400
|
+
const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
|
|
1401
|
+
const providerTransient =
|
|
1402
|
+
providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
|
|
1403
|
+
|
|
1404
|
+
// The other half of not believing a worker about its own run (#128). The
|
|
1405
|
+
// claim being audited is `state: pushed-green`, so the audit runs on the
|
|
1406
|
+
// worker's claim rather than on what verification made of it: a claim that
|
|
1407
|
+
// GitHub then contradicted is exactly the report whose details matter
|
|
1408
|
+
// most. Advisory throughout — nothing below reads `flags` when deciding
|
|
1409
|
+
// `state`, and the ordering here makes that checkable.
|
|
1410
|
+
const audit =
|
|
1411
|
+
result.state === "pushed-green"
|
|
1412
|
+
? await collectSettlementFlags(tracker, {
|
|
1413
|
+
prUrl: result.prUrl,
|
|
1414
|
+
issueText: `${r.issue.title}\n${r.issue.body}`,
|
|
1415
|
+
// The claimed-proof check compares the PR's Verified commands
|
|
1416
|
+
// against what this run's session actually recorded.
|
|
1417
|
+
sessionFile: result.sessionFile,
|
|
1418
|
+
// The effective file lane admission resolved for this run at
|
|
1419
|
+
// dispatch — the same value the brief rendered, pre-dispatch
|
|
1420
|
+
// comment declarations included (#608, #744). The audit flags a
|
|
1421
|
+
// diff that escapes it, so a widened lane is named on evidence
|
|
1422
|
+
// rather than found by reading the PR's file list by hand (#739).
|
|
1423
|
+
lane: admittedLane,
|
|
1424
|
+
})
|
|
1425
|
+
: undefined;
|
|
1426
|
+
if (result.state === "pushed-green" && audit?.truncated) {
|
|
1427
|
+
log(`#${issue} settlement audit read a truncated PR diff`);
|
|
1428
|
+
}
|
|
1429
|
+
const auditLines =
|
|
1430
|
+
audit === undefined
|
|
1431
|
+
? []
|
|
1432
|
+
: formatSettlementFlags(audit.flags, { truncated: audit.truncated });
|
|
1433
|
+
// The report's file list is derived from the PR's own diff, not retyped
|
|
1434
|
+
// from the worker's memory of the session: the narrative stays the
|
|
1435
|
+
// worker's, the disclosure becomes the diff's (#488). A diff that could
|
|
1436
|
+
// not be read leaves the worker's text untouched and the audit's
|
|
1437
|
+
// `changed-line-missing` flag says so.
|
|
1438
|
+
// The within-run reliability sentence, appended where a human reads it
|
|
1439
|
+
// (#584): which model the run finished on and whether it swapped
|
|
1440
|
+
// mid-flight. Empty (undefined) for a clean run, so a run that never
|
|
1441
|
+
// retried, swapped or compacted keeps today's settlement report byte for
|
|
1442
|
+
// byte — the "additive" claim asserted rather than assumed.
|
|
1443
|
+
const reliabilityLine = reliabilitySettlementLine({
|
|
1444
|
+
resolvedModel: result.model,
|
|
1445
|
+
resolvedProvider: result.provider,
|
|
1446
|
+
retryFallbacks: result.retryFallbacks,
|
|
1447
|
+
retryFallbackSucceeded: result.retryFallbackSucceeded,
|
|
1448
|
+
modelRecoveries: result.modelRecoveries,
|
|
1449
|
+
autoRetryCount: result.autoRetryCount,
|
|
1450
|
+
autoCompactionCount: result.autoCompactionCount,
|
|
1451
|
+
});
|
|
1452
|
+
|
|
1453
|
+
const settlementReport = [
|
|
1454
|
+
audit?.changedLine === undefined
|
|
1455
|
+
? result.report
|
|
1456
|
+
: withDerivedChangedLine(result.report, audit.changedLine),
|
|
1457
|
+
...(reliabilityLine === undefined ? [] : ["", reliabilityLine]),
|
|
1458
|
+
].join("\n");
|
|
1459
|
+
|
|
1460
|
+
const finalReport = [
|
|
1461
|
+
...(verified.reason === undefined ? [] : [verified.reason, ""]),
|
|
1462
|
+
...(auditLines.length === 0 ? [] : [...auditLines, ""]),
|
|
1463
|
+
settlementReport,
|
|
1464
|
+
].join("\n");
|
|
1465
|
+
|
|
1466
|
+
// What becomes of the tree, decided once, before any label or page. A
|
|
1467
|
+
// `pushed-*` run is the only end that does not salvage: its deliverable is
|
|
1468
|
+
// already on a remote branch, whatever is left loose in the tree is by the
|
|
1469
|
+
// worker's own account not part of it, and appending a WIP commit would
|
|
1470
|
+
// turn the green PR this daemon just verified red. Every other end may
|
|
1471
|
+
// contain work, so it is salvaged before the tree's final fate is decided.
|
|
1472
|
+
const settlement =
|
|
1473
|
+
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
1474
|
+
? undefined
|
|
1475
|
+
: await settleWorktree({
|
|
1476
|
+
issue,
|
|
1477
|
+
attempt,
|
|
1478
|
+
ending:
|
|
1479
|
+
state === "blocked"
|
|
1480
|
+
? "blocked for an operator decision"
|
|
1481
|
+
: state === "stopped"
|
|
1482
|
+
? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
|
|
1483
|
+
: endedBy(result.killedBy),
|
|
1484
|
+
worktree: worktreePath,
|
|
1485
|
+
branch,
|
|
1486
|
+
publish,
|
|
1487
|
+
...(state === "failed" || state === "killed"
|
|
1488
|
+
? ({ tree: "keep" } as const)
|
|
1489
|
+
: ({ tree: "remove", mirrorPath } as const)),
|
|
1490
|
+
});
|
|
1491
|
+
if (settlement === undefined) {
|
|
1492
|
+
// A `pushed-*` end does not salvage — its deliverable is already on a
|
|
1493
|
+
// remote branch — but its repository still goes away here, so the branch
|
|
1494
|
+
// is published first for the same reason `settleWorktree` does it: the
|
|
1495
|
+
// run repo is no longer a view of the mirror, and anything it holds that
|
|
1496
|
+
// never reached the mirror dies with the directory.
|
|
1497
|
+
const published = await publish(branch);
|
|
1498
|
+
if (!published.ok) {
|
|
1499
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
1500
|
+
}
|
|
1501
|
+
await removeWorktree(mirrorPath, worktreePath);
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
const terminalPatch: Partial<RunRecord> = {
|
|
1505
|
+
endedAt: Date.now(),
|
|
1506
|
+
turns: result.turns,
|
|
1507
|
+
spendUsd: result.spendUsd,
|
|
1508
|
+
// The count of in-session provider 429s the worker metered live, so a
|
|
1509
|
+
// run the provider throttled into the ground carries its own diagnosis
|
|
1510
|
+
// instead of landing `unknown` — the classifier reads it straight off
|
|
1511
|
+
// this column (#573).
|
|
1512
|
+
provider429Count: result.provider429Count,
|
|
1513
|
+
// The within-run harness reliability surface #581 collected, persisted
|
|
1514
|
+
// now that settlement owns the row (#584). The resolved model/provider
|
|
1515
|
+
// only travel when some assistant message carried them (the run recorded
|
|
1516
|
+
// no model, or the worker never established one); the count fields always
|
|
1517
|
+
// travel, 0 for a clean run, so an absent column can never be read as a
|
|
1518
|
+
// quiet fleet. Written for every terminal state, clean or not.
|
|
1519
|
+
...(result.model === undefined ? {} : { resolvedModel: result.model }),
|
|
1520
|
+
...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
|
|
1521
|
+
retryFallbacks: result.retryFallbacks,
|
|
1522
|
+
retryFallbackSucceeded: result.retryFallbackSucceeded,
|
|
1523
|
+
modelRecoveries: result.modelRecoveries,
|
|
1524
|
+
autoRetryCount: result.autoRetryCount,
|
|
1525
|
+
autoCompactionCount: result.autoCompactionCount,
|
|
1526
|
+
// The worker only reports these when it actually established them; a kill
|
|
1527
|
+
// or a settle whose report named no PR must not wipe what a verb recorded
|
|
1528
|
+
// earlier in the same run (#468). The sink in `updateRun` skips undefined
|
|
1529
|
+
// too, but a settle should not even build such a patch.
|
|
1530
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
1531
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
1532
|
+
sessionFile: result.sessionFile,
|
|
1533
|
+
// The code-graph session observation (#726): what the run's own session
|
|
1534
|
+
// registry held at start, persisted with the rest of the run's facts.
|
|
1535
|
+
// Absent only when the session surface did not record one.
|
|
1536
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
1537
|
+
// Every terminal state persists the worker's report — with the `changed:`
|
|
1538
|
+
// file list derived from the PR's diff where one could be read — not
|
|
1539
|
+
// just a green push: a stopped attempt's partial report is still part of
|
|
1540
|
+
// its audit trail.
|
|
1541
|
+
report: settlementReport,
|
|
1542
|
+
...settlement?.patch,
|
|
1543
|
+
...(audit === undefined || audit.flags.length === 0
|
|
1544
|
+
? {}
|
|
1545
|
+
: { settlementFlags: audit.flags }),
|
|
1546
|
+
};
|
|
1547
|
+
if (state === "stopped") {
|
|
1548
|
+
recordOperatorStop(store, {
|
|
1549
|
+
project: project.name,
|
|
1550
|
+
issue,
|
|
1551
|
+
runId,
|
|
1552
|
+
inProgress,
|
|
1553
|
+
reason: result.stoppedReason ?? "no reason recorded",
|
|
1554
|
+
patch: terminalPatch,
|
|
1555
|
+
});
|
|
1556
|
+
} else {
|
|
1557
|
+
const lastError = completionLastError(
|
|
1558
|
+
providerCredit,
|
|
1559
|
+
providerTransient,
|
|
1560
|
+
verified.reason,
|
|
1561
|
+
sessionErr,
|
|
1562
|
+
);
|
|
1563
|
+
store.updateRun(runId, {
|
|
1564
|
+
...terminalPatch,
|
|
1565
|
+
state,
|
|
1566
|
+
...(lastError === undefined ? {} : { lastError }),
|
|
1567
|
+
});
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
const salvaged = settlement?.lines ?? [];
|
|
1571
|
+
|
|
1572
|
+
// The run's reliability news, surfaced where the tick digest reads it
|
|
1573
|
+
// (#584): the digest is model-authored but consumes the store's material
|
|
1574
|
+
// ledger, so a run that swapped mid-flight or rode out a throttled
|
|
1575
|
+
// provider lands one event the digest can name. Clean runs record nothing
|
|
1576
|
+
// here, so a quiet fleet's digest is unchanged.
|
|
1577
|
+
if (reliabilityLine !== undefined) {
|
|
1578
|
+
store.recordMaterialEvent({
|
|
1579
|
+
project: project.name,
|
|
1580
|
+
category: "reliability",
|
|
1581
|
+
summary: `#${issue} ${reliabilityLine}`,
|
|
1582
|
+
evidence: `${r.issue.title}\n${r.issue.url}\n\n${reliabilityLine}`,
|
|
1583
|
+
occurredAt: Date.now(),
|
|
1584
|
+
recordedAt: Date.now(),
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
if (state === "stopped") {
|
|
1589
|
+
log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
|
|
1590
|
+
} else if (state === "blocked") {
|
|
1591
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
1592
|
+
wakeOrchestratorForBlockedRun(project.name, issue);
|
|
1593
|
+
await safeEscalate(d, {
|
|
1594
|
+
tier: 1,
|
|
1595
|
+
project: project.name,
|
|
1596
|
+
issue,
|
|
1597
|
+
runId,
|
|
1598
|
+
summary: `#${issue} is blocked on attempt ${attempt} and needs a decision`,
|
|
1599
|
+
detail: [`${r.issue.title}`, r.issue.url, "", ...salvaged, "", result.report].join("\n"),
|
|
1600
|
+
});
|
|
1601
|
+
} else if (state === "failed" || state === "killed") {
|
|
1602
|
+
// The row is already `killed`, so this count includes the segment that
|
|
1603
|
+
// just ended.
|
|
1604
|
+
const continuation = store.continuationsFor(project.name, issue);
|
|
1605
|
+
const continueTurns = shouldContinueAfterTurnsCap({
|
|
1606
|
+
killedBy: result.killedBy,
|
|
1607
|
+
prUrl: result.prUrl,
|
|
1608
|
+
headSha: result.headSha,
|
|
1609
|
+
salvageSha: settlement?.patch?.salvageSha,
|
|
1610
|
+
continuation,
|
|
1611
|
+
maxContinuations: caps.maxContinuationsPerIssue,
|
|
1612
|
+
});
|
|
1613
|
+
|
|
1614
|
+
if (providerCredit !== undefined) {
|
|
1615
|
+
await reactToProviderCredit({ project: d.project, escalate: (e) => d.escalate(e), isPaused, setPaused }, issue, providerCredit, result.sessionFile);
|
|
1616
|
+
await swapToQueue(d, issue, inProgress);
|
|
1617
|
+
} else if (continueTurns) {
|
|
1618
|
+
// One ordered pair through the shared chokepoint: the in-progress
|
|
1619
|
+
// removal before the queue add, exactly the order the projector will
|
|
1620
|
+
// apply them in (#201) — and the wake that comes with it, so a
|
|
1621
|
+
// salvaged continuation is claimed on an immediate pass instead of
|
|
1622
|
+
// waiting out the interval (#1041).
|
|
1623
|
+
await swapToQueue(d, issue, inProgress);
|
|
1624
|
+
log(
|
|
1625
|
+
`#${issue} turns-cap on run ${attempt}, continuation ` +
|
|
1626
|
+
`${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
|
|
1627
|
+
);
|
|
1628
|
+
await safeEscalate(d, {
|
|
1629
|
+
tier: 1,
|
|
1630
|
+
project: project.name,
|
|
1631
|
+
issue,
|
|
1632
|
+
runId,
|
|
1633
|
+
summary: `#${issue} hit the turns cap on attempt ${attempt} — auto-requeued for continuation`,
|
|
1634
|
+
detail: [
|
|
1635
|
+
`${r.issue.title}`,
|
|
1636
|
+
r.issue.url,
|
|
1637
|
+
...salvaged,
|
|
1638
|
+
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
1639
|
+
"",
|
|
1640
|
+
"The queue label is back on; the next tick should reattach the branch",
|
|
1641
|
+
"and open a continuation brief. No failed label was applied.",
|
|
1642
|
+
"",
|
|
1643
|
+
result.report,
|
|
1644
|
+
].join("\n"),
|
|
1645
|
+
});
|
|
1646
|
+
} else {
|
|
1647
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1648
|
+
await safeEscalate(d, {
|
|
1649
|
+
tier: 1,
|
|
1650
|
+
project: project.name,
|
|
1651
|
+
issue,
|
|
1652
|
+
runId,
|
|
1653
|
+
// The dedup key includes the summary, so the attempt number is what
|
|
1654
|
+
// lets a genuine second failure page again while a tick that keeps
|
|
1655
|
+
// seeing the same dead issue stays quiet.
|
|
1656
|
+
summary: result.killedBy
|
|
1657
|
+
? `#${issue} was killed on attempt ${attempt} by the ${result.killedBy} cap`
|
|
1658
|
+
: `#${issue} failed on attempt ${attempt}`,
|
|
1659
|
+
detail: [
|
|
1660
|
+
`${r.issue.title}`,
|
|
1661
|
+
r.issue.url,
|
|
1662
|
+
...salvaged,
|
|
1663
|
+
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
1664
|
+
"",
|
|
1665
|
+
finalReport,
|
|
1666
|
+
].join("\n"),
|
|
1667
|
+
});
|
|
1668
|
+
}
|
|
1669
|
+
} else {
|
|
1670
|
+
// A verified or still-pending PR keeps the in-progress label until its
|
|
1671
|
+
// checks or merge settle, preventing another worker from duplicating it.
|
|
1672
|
+
log(`#${issue} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
1673
|
+
// A green run's report is the only one nothing ever delivers: it raises no
|
|
1674
|
+
// escalation, and the row it settles into says nothing about what the
|
|
1675
|
+
// worker wrote. That is fine while the report is believed — and it is
|
|
1676
|
+
// exactly wrong once something has stopped believing it. A flagged
|
|
1677
|
+
// settlement is the one case where a `pushed-green` run has news, and it
|
|
1678
|
+
// has it *now*, while the PR is still open and nobody has merged it (#128).
|
|
1679
|
+
//
|
|
1680
|
+
// Tier 1, because the judgement is the orchestrator's: it can read the
|
|
1681
|
+
// diff, it can ask, and it can promote to a human. The state is
|
|
1682
|
+
// deliberately unchanged either way — the run below this line settles
|
|
1683
|
+
// `pushed-green` whether it was flagged or not.
|
|
1684
|
+
//
|
|
1685
|
+
// Findings only. A diff too large to read in full is noted in the log and
|
|
1686
|
+
// in the report, but it is not news for a human: paging "0 flags, could
|
|
1687
|
+
// not read it all" is exactly the kind of line that teaches an
|
|
1688
|
+
// orchestrator to skim past this escalation the next time it carries one.
|
|
1689
|
+
const found = audit?.flags ?? [];
|
|
1690
|
+
if (found.length > 0) {
|
|
1691
|
+
await safeEscalate(d, {
|
|
1692
|
+
tier: 1,
|
|
1693
|
+
project: project.name,
|
|
1694
|
+
issue,
|
|
1695
|
+
runId,
|
|
1696
|
+
summary:
|
|
1697
|
+
`#${issue} settled ${state} on attempt ${attempt} with ` +
|
|
1698
|
+
`${found.length} settlement audit flag(s)`,
|
|
1699
|
+
detail: [
|
|
1700
|
+
`${r.issue.title}`,
|
|
1701
|
+
r.issue.url,
|
|
1702
|
+
result.prUrl ?? "(no PR URL)",
|
|
1703
|
+
"",
|
|
1704
|
+
"The run is NOT blocked and its state is unchanged. These are advisory",
|
|
1705
|
+
"findings about the worker's own account of its work — judge them, then",
|
|
1706
|
+
"merge, ask, or close as you would have anyway.",
|
|
1707
|
+
"",
|
|
1708
|
+
finalReport,
|
|
1709
|
+
].join("\n"),
|
|
1710
|
+
});
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
} catch (err) {
|
|
1714
|
+
// Dispatch setup can fail after the controller opens but before runWorker's
|
|
1715
|
+
// inner settlement guard exists. Latch it before any terminal write or await.
|
|
1716
|
+
turnLimit?.close();
|
|
1717
|
+
turnLimit = undefined;
|
|
1718
|
+
if (await settleStopBeforeSession()) return;
|
|
1719
|
+
const detail = errText(err);
|
|
1720
|
+
log(`#${issue} errored: ${detail}`);
|
|
1721
|
+
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
1722
|
+
// copy of real work. Nothing else on this path so much as looks at it.
|
|
1723
|
+
const settlement =
|
|
1724
|
+
worktreePath === undefined
|
|
1725
|
+
? undefined
|
|
1726
|
+
: await settleWorktree({
|
|
1727
|
+
issue,
|
|
1728
|
+
attempt,
|
|
1729
|
+
ending: "killed by a dispatch error",
|
|
1730
|
+
worktree: worktreePath,
|
|
1731
|
+
branch,
|
|
1732
|
+
publish,
|
|
1733
|
+
tree: "keep",
|
|
1734
|
+
});
|
|
1735
|
+
if (run) {
|
|
1736
|
+
store.updateRun(run.id, {
|
|
1737
|
+
state: "failed",
|
|
1738
|
+
endedAt: Date.now(),
|
|
1739
|
+
lastError: detail,
|
|
1740
|
+
...settlement?.patch,
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
if (claimed) {
|
|
1744
|
+
// Leaving the issue stuck as in-progress would hide it from both the
|
|
1745
|
+
// queue and the human, so relabel even on the error path.
|
|
1746
|
+
// The failure-path relabel cannot throw: it is a local outbox write, and
|
|
1747
|
+
// the projector retries until the tracker takes it (#201).
|
|
1748
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
const salvaged = settlement?.lines ?? [];
|
|
1752
|
+
await safeEscalate(d, {
|
|
1753
|
+
tier: 1,
|
|
1754
|
+
project: project.name,
|
|
1755
|
+
issue,
|
|
1756
|
+
runId: run?.id,
|
|
1757
|
+
summary: `#${issue} could not be dispatched on attempt ${attempt}`,
|
|
1758
|
+
detail: salvaged.length === 0 ? detail : [detail, "", ...salvaged].join("\n"),
|
|
1759
|
+
});
|
|
1760
|
+
// The worktree, if one was created, is deliberately left in place: this is
|
|
1761
|
+
// a failure path, and whatever it still held is now a commit on the branch.
|
|
1762
|
+
} finally {
|
|
1763
|
+
turnLimit?.close();
|
|
1764
|
+
// The run is over, so its channel is too. Closed here rather than beside
|
|
1765
|
+
// the session so the crash path closes it as well: a listener left bound
|
|
1766
|
+
// after its run settled is a socket whose `run-not-live` check is the only
|
|
1767
|
+
// thing standing between a stale child and a push.
|
|
1768
|
+
if (verbListener !== undefined) {
|
|
1769
|
+
try {
|
|
1770
|
+
await verbListener.close();
|
|
1771
|
+
} catch (err) {
|
|
1772
|
+
log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
|
|
1773
|
+
}
|
|
1774
|
+
verbListener = undefined;
|
|
1775
|
+
}
|
|
1776
|
+
workerControl?.close();
|
|
1777
|
+
workerControl = undefined;
|
|
1778
|
+
}
|
|
1779
|
+
}
|