omp-conductor 0.19.6 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/REFERENCE.md +27 -2
  2. package/agents/to-spec.md +76 -9
  3. package/package.json +1 -1
  4. package/schema/config.schema.json +4 -0
  5. package/src/arm-challenge.ts +204 -85
  6. package/src/ask.ts +130 -615
  7. package/src/board.ts +7 -1
  8. package/src/brief-upgrade.ts +24 -0
  9. package/src/briefs/console.md +253 -0
  10. package/src/briefs/correction.md +203 -0
  11. package/src/briefs/orchestrator.md +167 -97
  12. package/src/briefs/policy.md +19 -16
  13. package/src/briefs/to-spec.md +76 -9
  14. package/src/briefs/worker.md +50 -16
  15. package/src/cli.ts +4 -0
  16. package/src/command-manifest.ts +54 -8
  17. package/src/commands/arm.ts +113 -49
  18. package/src/commands/console.ts +70 -0
  19. package/src/commands/context.ts +2 -0
  20. package/src/commands/epic.ts +132 -0
  21. package/src/commands/extend.ts +9 -1
  22. package/src/commands/intake.ts +44 -14
  23. package/src/commands/stats.ts +19 -4
  24. package/src/commands/worker.ts +9 -1
  25. package/src/config-schema.ts +13 -0
  26. package/src/config.ts +27 -0
  27. package/src/daemon/ack.ts +159 -0
  28. package/src/daemon/admission-pass.ts +135 -0
  29. package/src/daemon/brief.ts +461 -0
  30. package/src/daemon/deps.ts +539 -0
  31. package/src/daemon/dispatch.ts +1779 -0
  32. package/src/daemon/drain.ts +185 -0
  33. package/src/daemon/groom-pass.ts +412 -0
  34. package/src/daemon/http.ts +417 -0
  35. package/src/daemon/integrity.ts +108 -0
  36. package/src/daemon/panes.ts +180 -0
  37. package/src/daemon/review.ts +1888 -0
  38. package/src/daemon/runtime.ts +736 -0
  39. package/src/daemon/settle-pass.ts +589 -0
  40. package/src/daemon/supervision.ts +438 -0
  41. package/src/daemon/tick.ts +968 -0
  42. package/src/daemon/views.ts +751 -0
  43. package/src/daemon.ts +105 -7832
  44. package/src/dashboard/app.js +58 -0
  45. package/src/dashboard/controls.ts +22 -3
  46. package/src/dashboard/server.ts +4 -0
  47. package/src/diff-flags.ts +24 -3
  48. package/src/doctor.ts +17 -12
  49. package/src/escalate.ts +39 -21
  50. package/src/failure-class.ts +75 -1
  51. package/src/fleet.ts +1218 -304
  52. package/src/groom.ts +461 -0
  53. package/src/http-token.ts +142 -0
  54. package/src/knowledge.ts +229 -0
  55. package/src/mining.ts +316 -0
  56. package/src/orchestrator-tick.ts +428 -1681
  57. package/src/ready-gate.ts +267 -0
  58. package/src/settlement.ts +72 -6
  59. package/src/setup-host.ts +32 -9
  60. package/src/setup-wizard.ts +55 -7
  61. package/src/setup.ts +229 -3
  62. package/src/stats.ts +257 -2
  63. package/src/status-render.ts +158 -7
  64. package/src/store.ts +646 -26
  65. package/src/to-spec.ts +194 -21
  66. package/src/tracker/github.ts +50 -0
  67. package/src/types.ts +435 -15
  68. package/src/verbs/protocol.ts +28 -0
  69. package/src/verbs/server.ts +384 -12
  70. package/src/wake.ts +19 -2
  71. package/src/worker.ts +456 -1
@@ -0,0 +1,1779 @@
1
+ /**
2
+ * Taking one admitted issue to a running session, and closing the books when it
3
+ * ends: the claim, the lease, the worktree, the launch, the settle.
4
+ *
5
+ * This is the module the whole dispatcher exists for, and it is drawn around a
6
+ * single run's life rather than around a tick: `handleIssue` is called once per
7
+ * admitted issue and owns every irreversible step in between, so the ordering
8
+ * that makes a crash recoverable — provenance before label, label before
9
+ * launch, lease before work — is readable in one place.
10
+ *
11
+ * The two in-process registries live here for the same reason. A turn-limit
12
+ * controller and a worker-control slot are handles on a *running* session; they
13
+ * are created by `runDaemon`, held by the HTTP surface, and consumed here, and
14
+ * the only place that knows when a slot begins and ends is the launch path.
15
+ *
16
+ * The review-correction lane (`review.ts`) relaunches through the same helpers —
17
+ * `swapLabel`, `endedBy`, `verifyPushedGreenClaim`, `shouldContinueAfterTurnsCap`,
18
+ * `workerReleaseBlockRecorder` — and imports them from here. That direction is
19
+ * deliberate and one-way: a correction is another launch of the same run, so it
20
+ * borrows the launch path's rules rather than restating them.
21
+ */
22
+ import { existsSync, mkdirSync, statSync } from "node:fs";
23
+ import { dirname, join } from "node:path";
24
+ import { hasContinuationBudget, runSpendAllowanceUsd } from "../admission.ts";
25
+ import { resolveReleaseGrants, stateDir } from "../config.ts";
26
+ import { oauthFenceVerdict } from "../credential-class.ts";
27
+ import { formatSettlementFlags, withDerivedChangedLine } from "../diff-flags.ts";
28
+ import { providerCreditRefusal, providerTransientFault } from "../failure-class.ts";
29
+ import { openWorkerPane, reportWorkerPaneState, retireWorkerPane, type WorkerPaneOutcome } from "../fleet.ts";
30
+ import { pushRunBranch, type RunRepoRef } from "../gitops.ts";
31
+ import { errText, log, safeEscalate } from "../log.ts";
32
+ import { DEFAULT_MODEL_FALLBACK_THRESHOLD, fallbackClause, providerFailureFacts, resolveDispatchModel } from "../model-fallback.ts";
33
+ import { materializeOmpSettings } from "../omp-settings.ts";
34
+ import { isPaused, setPaused } from "../pause.ts";
35
+ import { recordReleaseBlock, type GateShape, type ReleaseBlockContext } from "../release-policy.ts";
36
+ import { reliabilitySettlementLine } from "../reports.ts";
37
+ import { branchName, type Routed } from "../routing.ts";
38
+ import { collectSettlementFlags, reactToProviderCredit, readSessionError, recordOperatorStop, recordWorkerDiscoveries, settleWorktree, swapToQueue } from "../settlement.ts";
39
+ import type { EffectiveModel, FileLane, IssueComment, PrState, RunRecord, RunState, Store, Tracker } from "../types.ts";
40
+ import { listenVerbChannel, type VerbListener } from "../verbs/server.ts";
41
+ import { ensureVerbSocketDir, verbSocketPath } from "../verbs/socket.ts";
42
+ import { ORPHAN_RESUME_PROMPT, runWorker, type KilledBy, type WorkerPauseControl, type WorkerPausePhase, type WorkerResult } from "../worker.ts";
43
+ import { addRunRepo, mirrorPathFor, removeWorktree, worktreePathFor, type RunPublisher } from "../worktree.ts";
44
+ import { writeAdmissionAck } from "./ack.ts";
45
+ import { buildBrief, renderAnsweredBlockPrompt, renderDiscussion, resumeLaneBlock } from "./brief.ts";
46
+ import { completionLastError, exhaustedSessionReason, githubRepo, knowledgeRepoKey, repoSlug, verbDeps, type Deps, type TurnLimitController, type TurnLimitRegistry, type WorkerControlRegistry, type WorkerControlSlot } from "./deps.ts";
47
+ import { readDrain } from "./drain.ts";
48
+ import { workspaceOwnership } from "./panes.ts";
49
+ import { wakeOrchestratorForBlockedRun } from "./supervision.ts";
50
+
51
+ /**
52
+ * Record a state-label swap for projection (#201). The add enqueues before the
53
+ * remove — the reverse order would leave a window where the issue carries no
54
+ * state label at all, which is exactly the shape `isEligible` treats as fresh
55
+ * work. Synchronous: enqueueing is a local store write and cannot fail on the
56
+ * tracker; the projector applies the pair in order and retries on refusal.
57
+ */
58
+ export function swapLabel(store: Store, projectName: string, issue: number, from: string, to: string): void {
59
+ store.enqueueLabelOps(projectName, [
60
+ { issue, op: "add", label: to },
61
+ { issue, op: "remove", label: from },
62
+ ]);
63
+ }
64
+ /**
65
+ * How a run's end is named — in the salvage commit, and to whoever reads it.
66
+ * The whole clause, not a bare reason: a graceful block was not killed by
67
+ * anything, and the commit subject is read during recovery.
68
+ */
69
+ export function endedBy(killedBy: KilledBy | undefined): string {
70
+ if (killedBy === "turns") return "killed by the turns cap";
71
+ if (killedBy === "wallclock") return "killed by the wall-clock cap";
72
+ if (killedBy === "spend") return "killed by the per-run spend cap";
73
+ return "killed by a failed run";
74
+ }
75
+
76
+ /**
77
+ * Authoritative live-run turn controls. Persistence happens synchronously
78
+ * before the in-memory ceiling changes, so no turn event can interleave.
79
+ */
80
+ export function createTurnLimitRegistry(
81
+ persist: (runId: string, maxTurns: number) => void,
82
+ ): TurnLimitRegistry {
83
+ const active = new Map<string, { runId: string; maxTurns: number }>();
84
+ const key = (project: string, issue: number): string => `${project}\0${issue}`;
85
+ return {
86
+ open(project, issue, runId, maxTurns) {
87
+ const k = key(project, issue);
88
+ if (active.has(k)) throw new Error(`#${issue} already has a live turn controller`);
89
+ const entry = { runId, maxTurns };
90
+ active.set(k, entry);
91
+ return {
92
+ maxTurns: () => entry.maxTurns,
93
+ close: () => {
94
+ if (active.get(k) === entry) active.delete(k);
95
+ },
96
+ };
97
+ },
98
+ extend(project, issue, maxTurns) {
99
+ if (!Number.isSafeInteger(maxTurns) || maxTurns < 1) {
100
+ throw new RangeError(`turn ceiling must be a positive integer, got ${maxTurns}`);
101
+ }
102
+ const entry = active.get(key(project, issue));
103
+ if (entry === undefined) return { kind: "not-active" };
104
+ if (maxTurns <= entry.maxTurns) {
105
+ return { kind: "not-increase", runId: entry.runId, maxTurns: entry.maxTurns };
106
+ }
107
+ persist(entry.runId, maxTurns);
108
+ entry.maxTurns = maxTurns;
109
+ return { kind: "extended", runId: entry.runId, maxTurns };
110
+ },
111
+ };
112
+ }
113
+
114
+ /** Authoritative controls for sessions owned by this daemon process. */
115
+ export function createWorkerControlRegistry(): WorkerControlRegistry {
116
+ interface Entry {
117
+ project: string;
118
+ issue: number;
119
+ runId: string;
120
+ control?: WorkerPauseControl;
121
+ stopReason?: string;
122
+ stopError?: string;
123
+ finished: PromiseWithResolvers<void>;
124
+ onPhase?: (phase: WorkerPausePhase) => void;
125
+ /** Who asked for the live pause, and when — cleared on resume (#997). */
126
+ pausedBy?: { source: string; at: number };
127
+ }
128
+
129
+ const active = new Map<string, Entry>();
130
+ const key = (project: string, issue: number): string => `${project}\0${issue}`;
131
+ return {
132
+ open(project, issue, runId, onPhase) {
133
+ const k = key(project, issue);
134
+ if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
135
+ const entry: Entry = {
136
+ project,
137
+ issue,
138
+ runId,
139
+ finished: Promise.withResolvers<void>(),
140
+ ...(onPhase === undefined ? {} : { onPhase }),
141
+ };
142
+ active.set(k, entry);
143
+ return {
144
+ install: (control) => {
145
+ if (active.get(k) !== entry) return;
146
+ entry.control = control;
147
+ if (entry.stopReason === undefined) return;
148
+ try {
149
+ control.stop(entry.stopReason);
150
+ } catch (err) {
151
+ entry.stopError = err instanceof Error ? err.message : String(err);
152
+ }
153
+ },
154
+ requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
155
+ close: () => {
156
+ if (active.get(k) === entry) active.delete(k);
157
+ entry.finished.resolve();
158
+ },
159
+ };
160
+ },
161
+ async pause(project, issue, source) {
162
+ const entry = active.get(key(project, issue));
163
+ if (entry?.control === undefined) return { kind: "not-active" };
164
+ try {
165
+ await entry.control.pause();
166
+ const phase = entry.control.phase();
167
+ entry.pausedBy = { source, at: Date.now() };
168
+ entry.onPhase?.(phase);
169
+ return { kind: "ok", runId: entry.runId, phase };
170
+ } catch (err) {
171
+ return {
172
+ kind: "refused",
173
+ runId: entry.runId,
174
+ error: err instanceof Error ? err.message : String(err),
175
+ };
176
+ }
177
+ },
178
+ resume(project, issue) {
179
+ const entry = active.get(key(project, issue));
180
+ if (entry?.control === undefined) return { kind: "not-active" };
181
+ try {
182
+ entry.control.resume();
183
+ const phase = entry.control.phase();
184
+ delete entry.pausedBy;
185
+ entry.onPhase?.(phase);
186
+ return { kind: "ok", runId: entry.runId, phase };
187
+ } catch (err) {
188
+ return {
189
+ kind: "refused",
190
+ runId: entry.runId,
191
+ error: err instanceof Error ? err.message : String(err),
192
+ };
193
+ }
194
+ },
195
+ async stop(project, issue, reason) {
196
+ const entry = active.get(key(project, issue));
197
+ if (entry === undefined) return { kind: "not-active" };
198
+ if (entry.stopReason === undefined) {
199
+ entry.stopReason = reason;
200
+ if (entry.control !== undefined) {
201
+ try {
202
+ entry.control.stop(reason);
203
+ } catch (err) {
204
+ entry.stopReason = undefined;
205
+ return {
206
+ kind: "refused",
207
+ runId: entry.runId,
208
+ error: err instanceof Error ? err.message : String(err),
209
+ };
210
+ }
211
+ }
212
+ }
213
+ await entry.finished.promise;
214
+ if (entry.stopError !== undefined) {
215
+ return { kind: "refused", runId: entry.runId, error: entry.stopError };
216
+ }
217
+ return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
218
+ },
219
+ snapshot(project) {
220
+ const workers: {
221
+ issue: number;
222
+ runId: string;
223
+ phase: WorkerPausePhase;
224
+ source?: string;
225
+ pausedAtMs?: number;
226
+ }[] = [];
227
+ for (const entry of active.values()) {
228
+ if (entry.project !== project || entry.control === undefined) continue;
229
+ const phase = entry.control.phase();
230
+ if (phase === "running") continue;
231
+ workers.push({
232
+ issue: entry.issue,
233
+ runId: entry.runId,
234
+ phase,
235
+ ...(entry.pausedBy === undefined
236
+ ? {}
237
+ : { source: entry.pausedBy.source, pausedAtMs: entry.pausedBy.at }),
238
+ });
239
+ }
240
+ return workers;
241
+ },
242
+ };
243
+ }
244
+
245
+ /**
246
+ * The daemon half of a worker's `pushed-green` claim (#85's contact with
247
+ * reality; #782's publication attribution).
248
+ *
249
+ * A yield is a transport, not proof of GitHub side effects: `claim` is
250
+ * caller-supplied text and the worker adapter turns it into a green result
251
+ * without the daemon ever having seen a mediated publication verb for it — the
252
+ * shape that let the #777 incident guess a non-existent PR. The tracker read
253
+ * at the end proves the PR is open and green at the exact head; the evidence
254
+ * gate here proves that PR is *this run's own mediated work* first.
255
+ *
256
+ * Two things are asked, in cost order:
257
+ *
258
+ * - the claimed URL must equal the one this run's row records, which is never
259
+ * a worker-supplied string at the moment a live run is verified —
260
+ * conductor_pr_create (create or adoption) writes it on the run, the daemon
261
+ * seeds it at claim from a terminal predecessor on the same branch (#434),
262
+ * and orchestrator-only recovery writes it for settled runs;
263
+ * - and the claimed head must have a mediated publisher: an allowed
264
+ * `conductor_push` on this exact run that published it on this run's branch
265
+ * (the verb only ever pushes `refs/heads/<branch>`), an allowed
266
+ * `conductor_pr_create` on this run for this exact PR, or — for a run that
267
+ * published nothing new — the live tip of this run's own branch.
268
+ *
269
+ * That last path is the one the row cannot supply. It used to be served by
270
+ * comparing the claim against `run.prUrl`/`run.headSha`, and both are
271
+ * worker-tainted upstream: terminal settlement writes a worker's reported pair
272
+ * onto the row *before* any verification, and continuation inheritance
273
+ * validates only the predecessor's metadata and open state — so a turn-capped
274
+ * attempt that reported someone else's real green PR had that pair inherited
275
+ * and re-presented as its own evidence. Asking the tracker for the branch tip
276
+ * removes the worker from the loop entirely: `refs/heads/<branch>` is a ref
277
+ * only a mediated `conductor_push` or a mediated `conductor_pr_update_branch`
278
+ * can move, so its live commit is daemon provenance no reported string can
279
+ * forge. It is also why a legitimate mediated base-branch update now passes:
280
+ * that server-side merge creates a head no `conductor_push` ever published and
281
+ * the row still carries the older one, which the recorded-pair test rejected.
282
+ *
283
+ * Binding to the *exact current run* is what makes an old attempt, a different
284
+ * branch, or #806's orchestrator-only settled-run recovery invisible here: the
285
+ * ledger query is run-scoped by `runId`, and recoveries store no runId at all.
286
+ * That query asks for the run's complete history rather than the ledger's
287
+ * newest rows — publication evidence sits at the *start* of a run, and a
288
+ * review-revision round or a burst of refused mutations pushed it past the
289
+ * default page, turning a verified push into a definitive false failure.
290
+ *
291
+ * A claim that fails this gate is a definitive failure — never a retryable
292
+ * `pushed-pending` — because a guessed URL is not something a later tick is
293
+ * waiting on. A claim the gate could not *read* is the opposite: an unreadable
294
+ * branch tip is #781's transient outage, so it stays retryable rather than
295
+ * burning an attempt on a flaky read.
296
+ */
297
+ export async function verifyPushedGreenClaim(
298
+ tracker: Pick<Tracker, "verifyPr" | "branchHead">,
299
+ claim: Pick<WorkerResult, "prUrl" | "headSha">,
300
+ publication: {
301
+ project: string;
302
+ issue: number;
303
+ runId: string;
304
+ /** The branch conductor routed this run onto. `conductor_push` publishes
305
+ * exactly `refs/heads/<branch>` and refuses any other ref. */
306
+ branch: string;
307
+ /** The identity `tracker.branchHead` reads the live tip with: the
308
+ * canonical `owner/repo` when the routed clone URL carries one, else the
309
+ * routed repository name. A tracker that cannot resolve it answers
310
+ * undefined, which stays retryable rather than definitive. */
311
+ repo: string;
312
+ store: Pick<Store, "verbLedger" | "getRun">;
313
+ },
314
+ ): Promise<{
315
+ state: "pushed-green" | "pushed-pending" | "failed";
316
+ reason?: string;
317
+ }> {
318
+ const { prUrl, headSha } = claim;
319
+ if (prUrl === undefined || headSha === undefined) {
320
+ return { state: "failed", reason: "Worker did not report a PR URL and observed head SHA" };
321
+ }
322
+ // Exact current run, never "some publication for the issue": the query is
323
+ // scoped to this runId plus project/issue, so a previous attempt's verbs and
324
+ // the orchestrator's recovery verbs are invisible here. Unbounded on
325
+ // purpose — see the note above about evidence ageing off the newest page.
326
+ const ledger = publication.store.verbLedger(publication.project, {
327
+ runId: publication.runId,
328
+ issue: publication.issue,
329
+ limit: Number.MAX_SAFE_INTEGER,
330
+ });
331
+ const run = publication.store.getRun(publication.runId);
332
+ const branchRef = `refs/heads/${publication.branch}`;
333
+ if (run?.prUrl !== prUrl) {
334
+ return {
335
+ state: "failed",
336
+ reason:
337
+ "Pushed-green claim has no mediated publication evidence: the claimed PR is not this run's " +
338
+ "recorded PR",
339
+ };
340
+ }
341
+ const pushedThisHead = ledger.some(
342
+ (entry) =>
343
+ entry.decision === "allowed" &&
344
+ entry.verb === "conductor_push" &&
345
+ entry.sha === headSha &&
346
+ entry.detail.includes(branchRef),
347
+ );
348
+ // Bound to the claimed PR, not merely to "a create happened": both allowed
349
+ // details name the URL they produced (`opened <url> …`, `adopted <url> …`),
350
+ // so an unrelated create on this run cannot vouch for another PR.
351
+ const createdHere = ledger.some(
352
+ (entry) =>
353
+ entry.decision === "allowed" &&
354
+ entry.verb === "conductor_pr_create" &&
355
+ entry.detail.includes(prUrl),
356
+ );
357
+
358
+ if (!pushedThisHead && !createdHere) {
359
+ // Nothing this run published carries the claimed head, so the only
360
+ // remaining evidence is the branch itself: a continuation or review round
361
+ // that pushed nothing, or a head a mediated base-branch update produced.
362
+ let tip: string | undefined;
363
+ try {
364
+ tip = await tracker.branchHead(publication.repo, publication.branch);
365
+ } catch {
366
+ tip = undefined;
367
+ }
368
+ if (tip === undefined) {
369
+ return {
370
+ state: "pushed-pending",
371
+ reason: `Live head of ${branchRef} unavailable; retrying`,
372
+ };
373
+ }
374
+ if (tip !== headSha) {
375
+ return {
376
+ state: "failed",
377
+ reason:
378
+ "Pushed-green claim has no mediated publication evidence: this run's conductor_push / " +
379
+ `conductor_pr_create ledger does not cover the claimed head, and ${branchRef} is at ${tip}, ` +
380
+ "not the claimed head",
381
+ };
382
+ }
383
+ }
384
+
385
+ const verification = await tracker.verifyPr(prUrl, headSha);
386
+ if (verification === undefined) {
387
+ return { state: "pushed-pending", reason: "GitHub PR verification unavailable; retrying" };
388
+ }
389
+ if (verification.status === "green") return { state: "pushed-green" };
390
+ return {
391
+ state: verification.status === "pending" ? "pushed-pending" : "failed",
392
+ reason: verification.reason,
393
+ };
394
+ }
395
+
396
+ /**
397
+ * Whether a turns-cap kill is handed straight back to the queue.
398
+ *
399
+ * A turns cap consumes the independent continuation budget rather than an
400
+ * implementation-failure attempt, so a run that ran out of turns mid-work is
401
+ * requeued and the next tick reattaches its branch (#50).
402
+ *
403
+ * The artifact test is what keeps that from becoming a loop. A run that reached
404
+ * the ceiling with no PR, no observed head and no salvage commit produced
405
+ * nothing to continue *from*, and requeueing it spends another continuation on
406
+ * the same spin — the seven rows on this fleet that burned $109 producing no
407
+ * merged PR. That shape is `turn-cap-spinning`, which #132 says must escalate
408
+ * with evidence instead; requeueing it here would settle the question before the
409
+ * classifier ever saw the row, and the queue label would already be back on.
410
+ *
411
+ * Pure so the rule is testable without a worker session — the same split as
412
+ * {@link settlementFor}.
413
+ */
414
+ export function shouldContinueAfterTurnsCap(f: {
415
+ killedBy?: string;
416
+ prUrl?: string;
417
+ headSha?: string;
418
+ salvageSha?: string;
419
+ continuation: number;
420
+ maxContinuations: number;
421
+ }): boolean {
422
+ if (f.killedBy !== "turns") return false;
423
+ const hasArtifacts = f.prUrl !== undefined || f.headSha !== undefined || f.salvageSha !== undefined;
424
+ if (!hasArtifacts) return false;
425
+ return hasContinuationBudget(f.continuation, f.maxContinuations);
426
+ }
427
+
428
+ /** Bind a worker's run identity to every release-policy block it emits. */
429
+ export function workerReleaseBlockRecorder(
430
+ project: string,
431
+ issue: number,
432
+ runId: string,
433
+ root = stateDir(),
434
+ ): (shape: GateShape, context: ReleaseBlockContext) => void {
435
+ return (shape, context) =>
436
+ recordReleaseBlock(
437
+ project,
438
+ "worker",
439
+ shape,
440
+ {
441
+ ...context,
442
+ issue,
443
+ runId,
444
+ },
445
+ root,
446
+ );
447
+ }
448
+
449
+ /**
450
+ * The exact PR a new claim inherits from its terminal predecessor (#434).
451
+ *
452
+ * `recentRuns` shows only the newest attempt per issue, and the mediated verbs
453
+ * resolve a PR's owner through the store — so the moment an orphan-clean
454
+ * requeue creates its continuation row, the predecessor's green PR (recorded
455
+ * on the older row) stops resolving and every mediated verb refuses it as "not
456
+ * a pull request any run opened". Seeding the continuation row with the same
457
+ * PR at claim time closes that window: ownership never depends on a worker
458
+ * rediscovering and re-reporting the PR.
459
+ *
460
+ * The checks are deliberately the exact counterpart of the store query's scope:
461
+ * the claim's project/issue/repo/branch must all match the predecessor, the
462
+ * predecessor must be terminal (a live row is still doing its own work, and a
463
+ * claim beside one is already a double dispatch), and the tracker must confirm
464
+ * the PR is still open — a closed or merged predecessor PR is settled by the
465
+ * sweep against the row that owns it, never carried onto a fresh attempt. An
466
+ * unreadable tracker fails closed: the new row simply starts without PR
467
+ * ownership, exactly as before #434, and adoption or the worker's own
468
+ * pr_create recovers the link.
469
+ */
470
+ export async function inheritedPrForContinuation(
471
+ tracker: Tracker,
472
+ prior: RunRecord | undefined,
473
+ claim: { project: string; issue: number; repo: string; branch: string },
474
+ ): Promise<{ prUrl: string; headSha?: string } | undefined> {
475
+ if (prior === undefined || prior.prUrl === undefined) return undefined;
476
+ // Terminal only. A live predecessor owns its PR outright and is not done
477
+ // with it; inheriting would launder a double dispatch's row.
478
+ if (prior.state === "claimed" || prior.state === "running") return undefined;
479
+ if (prior.project !== claim.project || prior.issue !== claim.issue) return undefined;
480
+ if (prior.repo !== claim.repo || prior.branch !== claim.branch) return undefined;
481
+ let state: PrState | undefined;
482
+ try {
483
+ state = await tracker.prState(prior.prUrl);
484
+ } catch {
485
+ return undefined;
486
+ }
487
+ if (state !== "open") return undefined;
488
+ return { prUrl: prior.prUrl, ...(prior.headSha === undefined ? {} : { headSha: prior.headSha }) };
489
+ }
490
+
491
+ /**
492
+ * Which continuation, if any, may resume its predecessor's session instead of
493
+ * dispatching a cold brief.
494
+ *
495
+ * Two shapes qualify, and they resume differently:
496
+ *
497
+ * - `orphan-clean` (#536): a daemon death interrupted a healthy worker.
498
+ * `reconcileOrphanedRuns` KEEPS the worktree (salvage commit included) and
499
+ * the transcript is file-backed, so the next attempt gets its own memory
500
+ * back whole: the same worktree, the same session directory, `resume: true`.
501
+ * The old dispatch built a fresh `run-<uuid>` session and re-read the repo
502
+ * from zero — the exact rediscovery orphan-clean spent turns on in #535.
503
+ * - `answered-block` (Phase 3 inner loop): the worker asked a question, parked,
504
+ * and an operator answered. Its tree was removed at the blocked settle
505
+ * because the branch already held the work, so this one REATTACHES the branch
506
+ * the ordinary way and resumes only the session — the transcript is the
507
+ * expensive part, and today's cold brief throws away a whole investigation
508
+ * to deliver one answer.
509
+ *
510
+ * Everything else dispatches cold, and that is not laziness: a cap-killed or
511
+ * otherwise failed worker was killed for cause, and orphan-dirty is held
512
+ * precisely because the worktree is the only copy.
513
+ *
514
+ * An answered block additionally has to pass #1047's health test. Resuming a
515
+ * session that is out of turns or that the provider abandoned buys a session
516
+ * that dies the same way with the answer unread, which is the #1035 shape in a
517
+ * different lane. The orphan-clean case deliberately keeps its own rule set: it
518
+ * is defined by an interruption rather than an ending, and #536's contract is
519
+ * that such a run continues.
520
+ *
521
+ * The checks here are what the daemon can prove cheaply before the claim (the
522
+ * transcript is present and non-empty; the worktree, where the mode needs one);
523
+ * the harness's own `continueRecent` is the backstop, and a corrupt transcript
524
+ * it silently falls back from is surfaced loudly by the `sessionFile` lineage
525
+ * compare at the dispatch site rather than left quiet.
526
+ */
527
+ export type ResumeMode = "orphan-clean" | "answered-block";
528
+
529
+ export function continuationResumeVerdict(
530
+ prior: RunRecord | undefined,
531
+ ): { kind: "resume"; prior: RunRecord; mode: ResumeMode } | { kind: "fresh"; reason?: string } {
532
+ const mode: ResumeMode | undefined =
533
+ prior === undefined
534
+ ? undefined
535
+ : prior.state === "orphaned" && prior.failureClass === "orphan-clean"
536
+ ? "orphan-clean"
537
+ : prior.state === "blocked"
538
+ ? "answered-block"
539
+ : undefined;
540
+ if (prior === undefined || mode === undefined) return { kind: "fresh" };
541
+ const what = mode === "orphan-clean" ? "the orphaned attempt" : "the blocked attempt";
542
+ if (prior.sessionFile === undefined) {
543
+ return { kind: "fresh", reason: `${what} recorded no transcript` };
544
+ }
545
+ try {
546
+ if (!existsSync(prior.sessionFile)) {
547
+ return { kind: "fresh", reason: `transcript ${prior.sessionFile} is gone` };
548
+ }
549
+ if (statSync(prior.sessionFile).size === 0) {
550
+ return { kind: "fresh", reason: `transcript ${prior.sessionFile} is empty` };
551
+ }
552
+ } catch (err) {
553
+ return { kind: "fresh", reason: `transcript ${prior.sessionFile} is unreadable (${errText(err)})` };
554
+ }
555
+ if (mode === "orphan-clean" && (prior.worktree === "" || !existsSync(prior.worktree))) {
556
+ return { kind: "fresh", reason: `worktree ${prior.worktree} is gone` };
557
+ }
558
+ if (mode === "answered-block") {
559
+ const exhausted = exhaustedSessionReason(prior);
560
+ if (exhausted !== undefined) {
561
+ return { kind: "fresh", reason: `${what}'s session cannot continue: ${exhausted}` };
562
+ }
563
+ }
564
+ return { kind: "resume", prior, mode };
565
+ }
566
+
567
+ /**
568
+ * One attempt at one issue, from claim to terminal state. Everything is inside
569
+ * a single try/catch so that a bad issue costs its own run and nothing else.
570
+ */
571
+ export async function handleIssue(
572
+ d: Deps,
573
+ r: Routed,
574
+ attempt: number,
575
+ admittedLane?: FileLane,
576
+ admittedModel?: EffectiveModel,
577
+ ): Promise<void> {
578
+ const { project, caps, tracker, store } = d;
579
+ const issue = r.issue.number;
580
+ const branch = branchName(r.issue);
581
+ const inProgress = project.stateLabels.inProgress;
582
+
583
+ let claimed = false;
584
+ let run: RunRecord | undefined;
585
+ // Hoisted out of the try so the catch path can still name and save the tree:
586
+ // a crash mid-dispatch is one of the ends whose uncommitted work has to be
587
+ // salvaged too, and it is the path least likely to have committed first.
588
+ const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
589
+ let worktreePath: string | undefined;
590
+ let turnLimit: TurnLimitController | undefined;
591
+ let workerControl: WorkerControlSlot | undefined;
592
+ let workerSessionInstalled = false;
593
+ // The run's Herdr representation (#840), hoisted for the same reason the verb
594
+ // listener is: whoever tears the run down has to hand the pane's lifecycle
595
+ // authority back, and a representation that outlives its child is a worker the
596
+ // workspace still shows as live.
597
+ let workerPane: Extract<WorkerPaneOutcome, { kind: "tracked" }> | undefined;
598
+ // Herdr orders lifecycle reports by `seq`, so this run owns one counter and
599
+ // every report takes the next value (#842). A shared or restarted counter would
600
+ // let a late report overwrite a newer state — the pane would then show `working`
601
+ // for a worker that has already blocked.
602
+ let paneSeq = 0;
603
+ const nextPaneSeq = (): number => (paneSeq += 1);
604
+ /**
605
+ * The injected Herdr runner, when this daemon carries one (#1035); every
606
+ * representation call below shares it so tests see one argv stream. The
607
+ * ownership adapter is always present in production — the store-backed
608
+ * half of workspace discovery, so what a Herdr restart forgets, the
609
+ * durable row remembers (#1035 review).
610
+ */
611
+ const herdr = {
612
+ ...(d.herdrRun === undefined ? {} : { run: d.herdrRun }),
613
+ ownership: workspaceOwnership(d.store),
614
+ };
615
+ /**
616
+ * Project one authoritative transition onto the pane (#842).
617
+ *
618
+ * The input is always a durable run transition or a typed session event —
619
+ * never the pane's own output, which is display and nothing more. A failure is
620
+ * logged, not raised: the workspace lagging is not worth failing a run over.
621
+ */
622
+ const projectPaneState = (state: "working" | "idle" | "blocked" | "unknown", message?: string): void => {
623
+ if (workerPane === undefined) return;
624
+ const reported = reportWorkerPaneState(workerPane.paneId, workerPane.label, state, {
625
+ ...herdr,
626
+ seq: nextPaneSeq(),
627
+ ...(message === undefined ? {} : { message }),
628
+ });
629
+ if (!reported.ok) log(`#${issue} herdr pane state ${state} not reported: ${reported.reason}`);
630
+ };
631
+ // The run's own repository. Hoisted for the same reason `worktreePath` is —
632
+ // the catch and finally paths have to publish the branch.
633
+ let runRepo: RunRepoRef | undefined;
634
+ // #126: the run's own verb socket. Hoisted like the run repo, because the catch
635
+ // and finally paths have to close it — a socket outliving its run is a
636
+ // channel nobody is authenticating any more.
637
+ let verbListener: VerbListener | undefined;
638
+
639
+ /**
640
+ * Publishes the run's branch on the privileged side: run repo → mirror →
641
+ * GitHub, fast-forward only. The dispatcher performs every network git
642
+ * operation for a run so the settlement record and the branch cannot
643
+ * disagree (#126); this is the only route a worker's commits take out, and it
644
+ * is also what stops a per-run repository from being the *only* copy when the
645
+ * tree is removed.
646
+ */
647
+ const publish: RunPublisher = async () => {
648
+ if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
649
+ return pushRunBranch(project, runRepo);
650
+ };
651
+
652
+ /**
653
+ * A stop can arrive after the run is claimed but before `runWorker` exposes
654
+ * its controller. Settle that run here instead of turning an operator action
655
+ * into a dispatch failure or making the HTTP request wait for a session that
656
+ * will never exist.
657
+ */
658
+ const settleStopBeforeSession = async (): Promise<boolean> => {
659
+ const reason = workerControl?.requestedStop();
660
+ if (reason === undefined || run === undefined || workerSessionInstalled) return false;
661
+ turnLimit?.close();
662
+ turnLimit = undefined;
663
+ const settlement =
664
+ worktreePath === undefined
665
+ ? undefined
666
+ : await settleWorktree({
667
+ issue,
668
+ attempt,
669
+ ending: `stopped by the operator: ${reason}`,
670
+ worktree: worktreePath,
671
+ branch,
672
+ publish,
673
+ tree: "remove",
674
+ mirrorPath,
675
+ });
676
+ recordOperatorStop(store, {
677
+ project: project.name,
678
+ issue,
679
+ runId: run.id,
680
+ inProgress,
681
+ reason,
682
+ patch: {
683
+ endedAt: Date.now(),
684
+ turns: run.turns,
685
+ spendUsd: run.spendUsd,
686
+ worktree: worktreePath ?? run.worktree,
687
+ report: [
688
+ "Operator stopped the run before its worker session started.",
689
+ `Reason: ${reason}`,
690
+ ...(settlement?.lines ?? []),
691
+ ].join("\n"),
692
+ ...settlement?.patch,
693
+ },
694
+ });
695
+ log(`#${issue} stopped by operator before its worker session started: ${reason}`);
696
+ return true;
697
+ };
698
+
699
+ /**
700
+ * The daemon-wide stop fence can land after the claim but before `runWorker`
701
+ * exposes its controller — the same window `settleStopBeforeSession` closes
702
+ * for a per-run operator stop, entered from the shutdown path instead of the
703
+ * pause control. Settle identically (close the claim, never launch) so the
704
+ * drain waits only for workers admitted before the stop request, and the
705
+ * shutdown cannot extend its own workload (#374).
706
+ */
707
+ const settleDrainBeforeSession = async (): Promise<boolean> => {
708
+ if (d.drain?.draining !== true || run === undefined || workerSessionInstalled) return false;
709
+ turnLimit?.close();
710
+ turnLimit = undefined;
711
+ const settlement =
712
+ worktreePath === undefined
713
+ ? undefined
714
+ : await settleWorktree({
715
+ issue,
716
+ attempt,
717
+ ending: "daemon shutdown began while the run was being prepared",
718
+ worktree: worktreePath,
719
+ branch,
720
+ publish,
721
+ tree: "remove",
722
+ mirrorPath,
723
+ });
724
+ recordOperatorStop(store, {
725
+ project: project.name,
726
+ issue,
727
+ runId: run.id,
728
+ inProgress,
729
+ reason: "daemon shutdown began after the claim",
730
+ patch: {
731
+ endedAt: Date.now(),
732
+ turns: run.turns,
733
+ spendUsd: run.spendUsd,
734
+ worktree: worktreePath ?? run.worktree,
735
+ report: [
736
+ "Daemon shutdown began while this run was still being prepared; the worker",
737
+ "never started and the claim was closed so the shutdown could complete.",
738
+ ...(settlement?.lines ?? []),
739
+ ].join("\n"),
740
+ ...settlement?.patch,
741
+ },
742
+ });
743
+ log(`#${issue} not launched: daemon shutdown began after the claim`);
744
+ return true;
745
+ };
746
+
747
+ /**
748
+ * The launch half of the credential-class fence (#852).
749
+ *
750
+ * Admission already held every candidate while a required provider billed to
751
+ * the wrong credential, but a grant can be disabled in the window *between*
752
+ * that check and this spawn — a refresh failing, an operator revoking, a token
753
+ * expiring — and that window is exactly what the incident's acceptance
754
+ * criterion names. So the same question is asked again here, as the last thing
755
+ * before the session exists, and the run is closed rather than launched.
756
+ *
757
+ * Settled the same way the stop and drain fences settle: the claim closes, the
758
+ * tree is salvaged, nothing is launched. It is deliberately NOT a failed
759
+ * attempt against the issue's budget — the issue is not what is wrong, the
760
+ * host's credentials are, and charging an attempt for it would exhaust an
761
+ * issue while the operator re-authenticates.
762
+ */
763
+ const settleCredentialClassBeforeSession = async (): Promise<boolean> => {
764
+ const required = project.requireOauthProviders ?? [];
765
+ if (required.length === 0 || run === undefined || workerSessionInstalled) return false;
766
+ const fence = await oauthFenceVerdict(
767
+ required,
768
+ d.probeCredentialClass ??
769
+ (async (provider) => ({
770
+ ok: false,
771
+ reason: `no credential probe is wired for ${provider}`,
772
+ })),
773
+ );
774
+ if (fence.ok) return false;
775
+ turnLimit?.close();
776
+ turnLimit = undefined;
777
+ const settlement =
778
+ worktreePath === undefined
779
+ ? undefined
780
+ : await settleWorktree({
781
+ issue,
782
+ attempt,
783
+ ending: "not launched: a required subscription credential is unavailable",
784
+ worktree: worktreePath,
785
+ branch,
786
+ publish,
787
+ tree: "remove",
788
+ mirrorPath,
789
+ });
790
+ recordOperatorStop(store, {
791
+ project: project.name,
792
+ issue,
793
+ runId: run.id,
794
+ inProgress,
795
+ reason: `required subscription credential unavailable: ${fence.reason}`,
796
+ patch: {
797
+ endedAt: Date.now(),
798
+ turns: run.turns,
799
+ spendUsd: run.spendUsd,
800
+ worktree: worktreePath ?? run.worktree,
801
+ report: [
802
+ "This run was closed instead of launched: a provider this project requires to bill",
803
+ "to its subscription would have billed to something else.",
804
+ fence.reason,
805
+ ...(settlement?.lines ?? []),
806
+ ].join("\n"),
807
+ ...settlement?.patch,
808
+ },
809
+ });
810
+ log(`#${issue} not launched: ${fence.reason}`);
811
+ return true;
812
+ };
813
+
814
+ try {
815
+ // The claim-side of the stop fence (#374): the run row is the boundary the
816
+ // shutdown drain waits on, so the claim itself refuses once the daemon is
817
+ // draining. The tick re-checks before the dispatch tail; this covers the
818
+ // same fence for any caller that reaches `handleIssue` without one, and
819
+ // makes "no post-stop run rows" a property of the claim, not of its caller.
820
+ if (d.drain?.draining === true) {
821
+ log(`#${issue} not claimed: daemon is draining`);
822
+ return;
823
+ }
824
+
825
+ // The claim-side of the pause fence (#651 review #3): the tick's pause
826
+ // gate sits above routing, so a tick that passed that gate before an
827
+ // operator wrote the freeze can still be mid-routing when the fence lands
828
+ // — it would claim after the setup barrier's acknowledgement and before
829
+ // its first mutation. The claim itself re-checks the pause and refuses,
830
+ // writing the durable admission acknowledgement the barrier waits for.
831
+ // A claim observed under the fence is the last admission boundary there
832
+ // is: nothing may create a run row beside a held fleet.
833
+ if (isPaused(d.project.name)) {
834
+ try {
835
+ writeAdmissionAck(d.project.name);
836
+ } catch (err) {
837
+ log(`admission acknowledgement write failed: ${errText(err)}`);
838
+ }
839
+ log(`#${issue} not claimed: dispatch is paused at claim time`);
840
+ return;
841
+ }
842
+
843
+ // The claim-side of the project drain fence (#484): the tick's gate sits
844
+ // above routing, so a drain created after that gate can still land
845
+ // mid-pass — the claim re-checks and refuses while a fresh drain is in
846
+ // force. An invalid record fails closed the same way. The read here is
847
+ // observational on purpose: the claim refuses but must not consume the
848
+ // marker, or the first refused claim would unbind the rest of its own
849
+ // pass; the tick's consumeDrain removes it on the next pass, so no claim
850
+ // path can ever be blocked permanently by it.
851
+ const claimDrain = readDrain(d.project.name);
852
+ if (claimDrain.kind === "active" || claimDrain.kind === "error") {
853
+ log(
854
+ claimDrain.kind === "error"
855
+ ? `#${issue} not claimed: drain record invalid (${claimDrain.problem})`
856
+ : `#${issue} not claimed: project drain in effect`,
857
+ );
858
+ return;
859
+ }
860
+
861
+ // Claim on the STORE first, before anything that can fail. The run row —
862
+ // not the label — is the crash-safe guard against double dispatch: rows
863
+ // are local, written before any network call, and the startup orphan
864
+ // sweep marks process-less claimed/running rows `orphaned`, so a daemon
865
+ // dying mid-claim leaves a row a human can triage instead of a label only
866
+ // the orchestrator may touch. The in-progress label is a write-behind
867
+ // projection of that row: enqueued here, flushed by the post-admission
868
+ // projection pass in the same tick on the healthy path, and — while
869
+ // pending — kept effective off the ready set by the routing overlay, so a
870
+ // crash between the row and its label cannot double-dispatch either
871
+ // (#201).
872
+ // Read before this attempt's own row exists, so `latestRun` still means the
873
+ // attempt whose work this one inherits.
874
+ const prior = store.latestRun(project.name, issue);
875
+ const priorSalvage = prior?.salvageSha;
876
+ // The failover decision is made from the store the same way admission is:
877
+ // counts of this issue's own rows, so the switch is sticky per chain and a
878
+ // fresh issue always starts on the primary (#286). Read before the claim
879
+ // writes this attempt's row, which would otherwise break the streak.
880
+ const chainFacts = providerFailureFacts(store.runsForIssue(project.name, issue));
881
+ // Resolved before the claim, not after provisioning: the model is also
882
+ // part of the #536 resume decision, which has to be made before the
883
+ // dispatch shape (fresh provision vs kept worktree) is chosen. With no
884
+ // `modelFallbacks` configured this is the primary model — or none, for an
885
+ // unconfigured project — and today's dispatch is byte for byte what it
886
+ // has always been.
887
+ const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
888
+ // A `Model:` declaration admission resolved for this candidate (#535) is
889
+ // this issue's workerModel: the orchestrator names a tier when it
890
+ // promotes, and dispatch launches on that selector exactly as if the
891
+ // project's `workerModel` were the declared value. Absent a declaration,
892
+ // today's `project.workerModel` is unchanged, and the failover chain
893
+ // (#286) keeps its semantics in both cases — it is the same resolution,
894
+ // one different primary.
895
+ const declaredModel = admittedModel?.model;
896
+ // #807: a chain that already bought its one model escalation dispatches on
897
+ // that stronger selector, outranking both the declaration and the project
898
+ // default — the settlement that wrote the marker did so precisely because
899
+ // the previous tier spun to a cap with nothing to show. Read from the
900
+ // store, so it is sticky to this issue's chain and a fresh issue (no
901
+ // marker) resolves exactly as it always has. The provider chain still
902
+ // layers on top: an escalated run whose provider then aborts fails over
903
+ // normally, because escalation only ever changes the primary.
904
+ const escalation = store.modelEscalation(project.name, issue);
905
+ const primaryModel = escalation?.model ?? declaredModel ?? project.workerModel;
906
+ const choice = resolveDispatchModel({
907
+ workerModel: primaryModel,
908
+ modelFallbacks: project.modelFallbacks,
909
+ threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
910
+ streak: chainFacts.streak,
911
+ });
912
+ const clause =
913
+ fallbackClause(choice, chainFacts, primaryModel) ??
914
+ (escalation === undefined
915
+ ? undefined
916
+ : `on ${escalation.model} — this chain's one model escalation, after ${escalation.failureClass}`);
917
+
918
+ // Whether this attempt continues its predecessor's session instead of
919
+ // reading the repo from zero (#536, and the answered-block half of the
920
+ // Phase 3 inner loop). Decided here, before this attempt's row exists,
921
+ // because `prior` is still the attempt whose work this one inherits and the
922
+ // whole dispatch shape follows the verdict.
923
+ let resuming: RunRecord | undefined;
924
+ let resumeMode: ResumeMode | undefined;
925
+ const verdict = continuationResumeVerdict(prior);
926
+ if (verdict.kind === "resume") {
927
+ const resumePrior = verdict.prior;
928
+ // The continuation must stay on the model the interrupted session was
929
+ // using; a chain that now resolves differently dispatches fresh rather
930
+ // than quietly continuing on another model, which would smear one
931
+ // attempt's work across two models (#286 attribution) — and, on the
932
+ // review side, is the diagnosed #1035 failure itself.
933
+ if (resumePrior.model !== undefined && choice.model !== undefined && resumePrior.model !== choice.model) {
934
+ log(
935
+ `#${issue} attempt ${attempt} not resumed: attempt ${resumePrior.attempt} ran on ${resumePrior.model} but dispatch ` +
936
+ `now resolves ${choice.model} — fresh dispatch`,
937
+ );
938
+ } else {
939
+ resuming = resumePrior;
940
+ resumeMode = verdict.mode;
941
+ log(
942
+ `#${issue} attempt ${attempt} continuing session of attempt ${resumePrior.attempt} (${verdict.mode}) → ` +
943
+ `transcript ${resumePrior.sessionFile}, ` +
944
+ `${verdict.mode === "orphan-clean" ? "worktree kept" : "branch reattached"}`,
945
+ );
946
+ }
947
+ } else if (verdict.reason !== undefined) {
948
+ log(`#${issue} attempt ${attempt} not resumed: ${verdict.reason} — fresh dispatch`);
949
+ }
950
+
951
+ run = store.createRun({
952
+ project: project.name,
953
+ issue,
954
+ repo: r.repo.name,
955
+ branch,
956
+ worktree: "",
957
+ state: "claimed",
958
+ attempt,
959
+ turns: 0,
960
+ spendUsd: 0,
961
+ maxTurns: caps.workerMaxTurns,
962
+ startedAt: Date.now(),
963
+ // #567: the orphan-clean attempt this claim continues, when the verdict
964
+ // above fired. `undefined` for a fresh dispatch — the store maps that to
965
+ // NULL, so a fresh row simply never carries the field.
966
+ resumedFromRunId: resuming?.id,
967
+ // #744: the file-lane declaration admission resolved for this candidate
968
+ // is persisted on the row, so lane occupancy survives across dispatch
969
+ // passes — a later pass knows what this run *intends* to touch, not only
970
+ // what it has touched so far. This is the exact `Admission.lane` the gate
971
+ // enforced and the brief rendered, never a re-parse. Absent for a run
972
+ // with no declaration (fail open), exactly as it was admitted.
973
+ lane: admittedLane,
974
+ // #851: the allowance admission reserved for this run, recorded on the
975
+ // row before anything launches. Derived from the same caps the gate
976
+ // read, so the reservation the pass enforced and the one the row holds
977
+ // are one value; `undefined` (NULL) when the fleet has no spend cap at
978
+ // all, which is the truth rather than "reserved nothing".
979
+ spendReservedUsd: runSpendAllowanceUsd(caps) ?? undefined,
980
+ });
981
+ // #434: carry a terminal predecessor's open PR onto the continuation row.
982
+ // The claim itself stays synchronous — `handleIssue` claims at the top of
983
+ // its dispatch burst, the window the shutdown fence and its tests rely on
984
+ // (#374) — so the tuple-exact, tracker-verified inheritance lands here,
985
+ // still before any provisioning, network call or verb socket. The new row
986
+ // is `recentRuns`' newest attempt for the issue the moment it exists, so
987
+ // without this a requeue hides the green PR the orphaned attempt opened
988
+ // until a worker happens to rediscover it.
989
+ const inherited = await inheritedPrForContinuation(tracker, prior, {
990
+ project: project.name,
991
+ issue,
992
+ repo: r.repo.name,
993
+ branch,
994
+ });
995
+ if (inherited !== undefined && prior !== undefined) {
996
+ store.updateRun(run.id, {
997
+ prUrl: inherited.prUrl,
998
+ ...(inherited.headSha === undefined ? {} : { headSha: inherited.headSha }),
999
+ });
1000
+ log(`#${issue} inherited PR ${inherited.prUrl} from attempt ${prior.attempt} (${prior.state})`);
1001
+ }
1002
+ const maxTurns = run.maxTurns;
1003
+ const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
1004
+ const runId = run.id;
1005
+ if (turnOverride !== undefined) {
1006
+ log(
1007
+ `#${issue} claimed with turn override ${turnOverride} ` +
1008
+ `(base ${caps.workerMaxTurns})`,
1009
+ );
1010
+ }
1011
+ store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
1012
+ claimed = true;
1013
+ turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
1014
+ // The pane follows the phase the registry actually reached (#842) — never a
1015
+ // guess made at the call site, and never the pane's own output read back.
1016
+ workerControl = d.workerControls.open(project.name, issue, runId, (phase) => {
1017
+ projectPaneState(
1018
+ phase === "running" ? "working" : "idle",
1019
+ phase === "running" ? "resumed" : `${phase} by the operator`,
1020
+ );
1021
+ });
1022
+ if (await settleStopBeforeSession()) return;
1023
+ if (await settleDrainBeforeSession()) return;
1024
+ // The credential fence, twice, for the same reason the stop fence is
1025
+ // consulted more than once (#852). Here it is the cheap one: a run whose
1026
+ // required subscription credential is already unavailable is closed before
1027
+ // anything clones a mirror or cuts a worktree, because provisioning for a
1028
+ // launch that cannot happen is pure waste. The second call, immediately
1029
+ // before the spawn, is the one that closes the admission-to-session window.
1030
+ if (await settleCredentialClassBeforeSession()) return;
1031
+
1032
+
1033
+ // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
1034
+ // an existing path, so a retry — or a tree kept from a failed attempt — has
1035
+ // to be cleared first. Both helpers are pure path math, and removeWorktree
1036
+ // tolerates a mirror or tree that is not there yet. An ORPHAN-CLEAN resume
1037
+ // skips the whole dance: the orphaned run's tree is the work to continue
1038
+ // (its salvage commit is already on the branch), and re-cloning it from
1039
+ // the mirror would be exactly the rediscovery this feature exists to skip.
1040
+ // An answered-block resume does NOT skip it: that run's tree was removed at
1041
+ // its blocked settle because the branch already held the work, so it
1042
+ // reattaches the branch exactly as any continuation does and resumes only
1043
+ // the session.
1044
+ let provisioned: Awaited<ReturnType<typeof addRunRepo>> | undefined;
1045
+ if (resuming !== undefined && resumeMode === "orphan-clean") {
1046
+ worktreePath = resuming.worktree;
1047
+ runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
1048
+ } else {
1049
+ await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
1050
+ if (await settleStopBeforeSession()) return;
1051
+ if (await settleDrainBeforeSession()) return;
1052
+ const provisionedTree = await addRunRepo(
1053
+ r.repo,
1054
+ project.mirrorRoot,
1055
+ project.workspaceRoot,
1056
+ issue,
1057
+ branch,
1058
+ );
1059
+ provisioned = provisionedTree;
1060
+ worktreePath = provisionedTree.path;
1061
+ runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
1062
+ }
1063
+ if (await settleStopBeforeSession()) return;
1064
+ if (await settleDrainBeforeSession()) return;
1065
+
1066
+ // The SDK names the transcript itself, so the daemon supplies the parent
1067
+ // directory and learns the real path back from the result. Inventing one
1068
+ // here would put a file that never gets written into an escalation.
1069
+ //
1070
+ // Per run rather than one shared directory, so one run's transcript cannot
1071
+ // be truncated or replaced by the next — except for a resumed attempt,
1072
+ // which deliberately reuses the interrupted session's directory so the
1073
+ // SDK's `continueRecent(cwd, dir)` picks up that transcript and keeps
1074
+ // writing it.
1075
+ const runTreeRoot = stateDir();
1076
+ const sessionDir =
1077
+ resuming === undefined
1078
+ ? join(runTreeRoot, "sessions", `run-${String(runId)}`)
1079
+ : dirname(resuming.sessionFile!);
1080
+ mkdirSync(sessionDir, { recursive: true });
1081
+
1082
+ // The fleet-owned omp settings overlay (#537): the project's `ompSettings`
1083
+ // map (plus the retry keys derived from `modelFallbacks`, #539's staging
1084
+ // half) materialised to YAML under the run's session directory — never
1085
+ // inside the worktree, whose diff is the PR a worker ships. Rewritten on
1086
+ // every attempt, so a config edit takes effect on the next dispatch and a
1087
+ // resumed attempt reuses the kept session dir with the *current* config.
1088
+ // Absent `ompSettings` and `modelFallbacks`, no file is written, nothing
1089
+ // is passed, and dispatch is byte-for-byte today's.
1090
+ const ompSettingsFile = materializeOmpSettings(project, sessionDir);
1091
+
1092
+ // ---- the run's mutation channel (#126) -------------------------------
1093
+ // A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
1094
+ // per-run *directory*: a directory owned by the run principal would hand
1095
+ // back the power to unlink a sibling's socket and bind an impostor in its
1096
+ // place, which is the whole thing the layout buys. `listenVerbChannel`
1097
+ // validates every component before binding and throws rather than
1098
+ // degrading, so a tampered path refuses this dispatch instead of running
1099
+ // the worker with a channel nobody can vouch for.
1100
+ verbListener = await listenVerbChannel(
1101
+ verbDeps(d),
1102
+ {
1103
+ kind: "run",
1104
+ // `runTreeRoot`, not the private state dir: the slot has to *connect*
1105
+ // to this socket, and a 0700 ancestor makes it unreachable however
1106
+ // correct the socket's own mode is. `ensureVerbSocketDir` still owns
1107
+ // the parent and still runs its anti-rebind checks there (#126).
1108
+ path: verbSocketPath(ensureVerbSocketDir(runTreeRoot), `run-${String(issue)}`),
1109
+ project: project.name,
1110
+ role: "worker",
1111
+ runId,
1112
+ issue,
1113
+ repo: r.repo,
1114
+ runRepoPath: worktreePath,
1115
+ branch,
1116
+ },
1117
+ {
1118
+ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
1119
+ },
1120
+ );
1121
+ if (await settleStopBeforeSession()) return;
1122
+ if (await settleDrainBeforeSession()) return;
1123
+
1124
+ store.updateRun(runId, { worktree: worktreePath, state: "running" });
1125
+
1126
+ // Where this attempt goes, and (when the failover fired) the clause that
1127
+ // makes it attributable. `chainConfigured`/`choice`/`clause` were resolved
1128
+ // before the claim — the resume verdict had to be made before the dispatch
1129
+ // shape was chosen (#536) — so only the record write lives here.
1130
+ // Recorded before the launch, so even a run killed mid-flight leaves the
1131
+ // model it chose on its row. Only a chain-configured project writes the
1132
+ // column: absent `modelFallbacks` must preserve today's rows byte for byte.
1133
+ // An escalated chain (#807) writes it too — the selector it dispatched on
1134
+ // is the whole provenance of that recovery, and its harness-resolved model
1135
+ // lands beside it in `resolvedModel` when the run settles.
1136
+ if ((chainConfigured || escalation !== undefined) && choice.model !== undefined) {
1137
+ store.updateRun(runId, { model: choice.model });
1138
+ }
1139
+
1140
+ log(
1141
+ `#${issue} attempt ${attempt}${clause === undefined ? "" : ` ${clause}`} → ${r.repo.name} ${branch}` +
1142
+ (provisioned?.reattached ? " (continuation: reattached existing branch)" : ""),
1143
+ );
1144
+
1145
+ // The continuation notice replaces the brief for a resumed attempt (#536).
1146
+ // The original brief is already in the resumed transcript; re-sending it is
1147
+ // how a resumed worker ends up re-doing the work it just did. Everything
1148
+ // below the brief is fresh-dispatch-only, exactly as today — except the
1149
+ // file lane admission enforced for this continuation, which rides the
1150
+ // notice itself: the retained transcript's brief may show an earlier
1151
+ // declaration, and a worker continuing under a stale lane is the collision
1152
+ // the gate exists to stop (#608).
1153
+ //
1154
+ // An answered block resumes for a different reason and so carries something
1155
+ // different: the investigation is in the transcript, but the ANSWER it
1156
+ // parked for is not — it arrived as a comment afterwards. So the discussion
1157
+ // travels with the notice, rendered by the same budgeted renderer the cold
1158
+ // brief uses, and an unreadable thread degrades to a named absence rather
1159
+ // than silently reading as "nobody answered".
1160
+ let brief: string;
1161
+ if (resuming !== undefined && resumeMode === "answered-block") {
1162
+ let answered: IssueComment[] | "unread";
1163
+ try {
1164
+ answered = await tracker.listComments(issue);
1165
+ } catch (err) {
1166
+ log(`#${issue} issue comments unreadable at dispatch; the continuation will say so: ${errText(err)}`);
1167
+ answered = "unread";
1168
+ }
1169
+ brief =
1170
+ renderAnsweredBlockPrompt(issue, renderDiscussion(answered, admittedLane)) +
1171
+ (admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
1172
+ } else if (resuming !== undefined) {
1173
+ brief =
1174
+ ORPHAN_RESUME_PROMPT + (admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
1175
+ } else {
1176
+ // The discussion is rendered at dispatch so a worker never depends on a
1177
+ // runtime `gh` read to see the orchestrator's grooming (#517). The read is
1178
+ // best-effort, but its failure is not silent: an unreadable tracker names
1179
+ // itself in the brief's Discussion section instead of reading as "no
1180
+ // comments" — the exact confusion this fix removes.
1181
+ let comments: IssueComment[] | "unread";
1182
+ try {
1183
+ comments = await tracker.listComments(issue);
1184
+ } catch (err) {
1185
+ log(`#${issue} issue comments unreadable at dispatch; the brief will say so: ${errText(err)}`);
1186
+ comments = "unread";
1187
+ }
1188
+
1189
+ brief = await buildBrief(project, r, branch, worktreePath, {
1190
+ continuation: provisioned?.reattached === true,
1191
+ defaultBranch: r.repo.defaultBranch,
1192
+ ...(provisioned?.reattached === true && priorSalvage !== undefined
1193
+ ? { salvagedSha: priorSalvage }
1194
+ : {}),
1195
+ comments,
1196
+ lane: admittedLane,
1197
+ model: admittedModel,
1198
+ host: d.host,
1199
+ });
1200
+ }
1201
+ if (await settleStopBeforeSession()) return;
1202
+ if (await settleDrainBeforeSession()) return;
1203
+ // Last, so it is the newest fact anything has about the credentials — a
1204
+ // grant disabled since admission is caught here rather than paid for.
1205
+ if (await settleCredentialClassBeforeSession()) return;
1206
+
1207
+ const repoSlug = githubRepo(r.repo.cloneUrl);
1208
+
1209
+
1210
+ let result: WorkerResult;
1211
+ try {
1212
+ const runAllowanceUsd = runSpendAllowanceUsd(caps);
1213
+ result = await runWorker({
1214
+ brief,
1215
+ cwd: worktreePath,
1216
+ caps,
1217
+ ...(repoSlug === undefined ? {} : { repoSlug }),
1218
+ maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
1219
+ // #851: the allowance this run reserved at admission is also its live
1220
+ // ceiling. Without it the reservation would only bound how many runs
1221
+ // start, not what one of them spends, and the day's total would be
1222
+ // unbounded again.
1223
+ ...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
1224
+ onPauseControl: (control) => {
1225
+ workerSessionInstalled = true;
1226
+ workerControl?.install(control);
1227
+ },
1228
+ sessionDir,
1229
+ ...(resuming === undefined ? {} : { resume: true }),
1230
+ // The session's control socket, under the daemon's own state directory —
1231
+ // a child process of the daemon reaches it directly.
1232
+ socketPath: join(sessionDir, "ipc.sock"),
1233
+ verbSocketPath: verbListener.path,
1234
+ // Names this run's channel the instant the child exists. Until this
1235
+ // fires the channel refuses every connection, because a session that
1236
+ // reached it first would be one nobody had identified — and every
1237
+ // session shares this daemon's uid, so the socket alone cannot tell
1238
+ // them apart (#163).
1239
+ onSpawn: (pid) => {
1240
+ verbListener?.bindPid(pid);
1241
+ },
1242
+ // The workspace representation of this exact child (#840). Best effort by
1243
+ // construction: a worker that cannot be *shown* is still a worker, so a
1244
+ // failure is logged with its reason and the run proceeds. What a failure
1245
+ // should mean for the launch — fail closed, or a named degraded state —
1246
+ // is #841's policy and is deliberately not decided here. Nothing is
1247
+ // claimed silently: either the pane id is logged, or the reason is.
1248
+ pane: {
1249
+ open: (pid) => {
1250
+ // Adopt before creating (#842): a run that already carries a pane
1251
+ // identity gets that pane reported against, never a second one. The
1252
+ // durable row is what makes this survive the process that opened it,
1253
+ // and duplicate-prevention is exactly what it buys — a re-entered
1254
+ // launch for one run must not leave two panes claiming it.
1255
+ const recorded = store.getRun(runId);
1256
+ if (recorded?.paneId !== undefined && recorded.paneLabel !== undefined) {
1257
+ workerPane = { kind: "tracked", paneId: recorded.paneId, label: recorded.paneLabel, pid };
1258
+ store.updateRun(runId, { workerPid: pid });
1259
+ const state = reportWorkerPaneState(recorded.paneId, recorded.paneLabel, "working", {
1260
+ ...herdr,
1261
+ seq: nextPaneSeq(),
1262
+ });
1263
+ log(
1264
+ state.ok
1265
+ ? `#${issue} herdr pane ${recorded.paneId} re-reported for pid ${pid}`
1266
+ : `#${issue} herdr pane ${recorded.paneId} could not be re-reported: ${state.reason}`,
1267
+ );
1268
+ return;
1269
+ }
1270
+ const outcome = openWorkerPane(
1271
+ {
1272
+ project: project.name,
1273
+ issue,
1274
+ attempt,
1275
+ runId,
1276
+ pid,
1277
+ ...(recorded?.sessionFile === undefined ? {} : { sessionFile: recorded.sessionFile }),
1278
+ },
1279
+ herdr,
1280
+ );
1281
+ if (outcome.kind === "tracked") {
1282
+ workerPane = outcome;
1283
+ // Durable before it is announced: a pane the store does not know
1284
+ // about is a pane a restart cannot reconcile.
1285
+ store.updateRun(runId, {
1286
+ workerPid: pid,
1287
+ paneId: outcome.paneId,
1288
+ paneLabel: outcome.label,
1289
+ // Cleared, because there is now a pane: a leftover reason beside a
1290
+ // tracked run would keep `status` saying degraded forever (#841).
1291
+ paneUnavailable: null,
1292
+ });
1293
+ log(`#${issue} herdr pane ${outcome.paneId} (${outcome.label}) tracks pid ${pid}`);
1294
+ } else {
1295
+ // Named, never silent (#841): the run proceeds untracked, and the
1296
+ // reason is durable so `status` can say the fleet is running blind
1297
+ // rather than looking identical to a fleet with no workers.
1298
+ store.updateRun(runId, { workerPid: pid, paneUnavailable: outcome.reason });
1299
+ log(`#${issue} no herdr pane: ${outcome.reason}`);
1300
+ }
1301
+ },
1302
+ release: () => {
1303
+ if (workerPane === undefined) return;
1304
+ // Retirement, not release alone (#1035): a settled run keeps no
1305
+ // representation. The close reaches only the follower; the
1306
+ // authoritative child was never in the pane to be signalled.
1307
+ const retired = retireWorkerPane(workerPane.paneId, workerPane.label, {
1308
+ ...herdr,
1309
+ seq: nextPaneSeq(),
1310
+ });
1311
+ if (!retired.ok) log(`#${issue} herdr pane retirement failed: ${retired.reason}`);
1312
+ workerPane = undefined;
1313
+ },
1314
+ },
1315
+ onChildLog: (line) => {
1316
+ log(`#${issue} ${line}`);
1317
+ },
1318
+ ...(choice.model === undefined ? {} : { model: choice.model }),
1319
+ // The fleet-owned omp settings overlay (#537): the staged YAML the
1320
+ // session loads through `Settings.init({ configFiles: [<path>] })` —
1321
+ // the project's `ompSettings` map plus the within-run failover keys
1322
+ // #581 staged directly (the project's own chain, not an empty default).
1323
+ // Absent both, nothing is staged, today's dispatch byte for byte.
1324
+ ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
1325
+ releaseGrants: resolveReleaseGrants(project),
1326
+ onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
1327
+ onTurn: (n) => store.updateRun(runId, { turns: n }),
1328
+ onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
1329
+ // #518: recorded live, not at settlement — the moment this explains is
1330
+ // forty minutes before the wall-clock cap fires, so a number only a
1331
+ // finished run carries would answer the question too late.
1332
+ onTokens: (tokens) =>
1333
+ store.updateRun(runId, { outputTokens: tokens.output, reasoningTokens: tokens.reasoning }),
1334
+ onKilled: () => {
1335
+ turnLimit?.close();
1336
+ turnLimit = undefined;
1337
+ },
1338
+ // Recorded the moment the session opens its transcript, not when the run
1339
+ // ends: `omp-conductor tail` resolves an issue to a file through this row,
1340
+ // and a path written at completion is a path nobody can follow live. The
1341
+ // completion-time update below writes the same value again, harmlessly.
1342
+ onSessionFile: (f) => {
1343
+ store.updateRun(runId, { sessionFile: f });
1344
+ // #536: the harness's own `continueRecent` can still fall back to a
1345
+ // blank session (corrupt transcript, nothing to continue) — and a
1346
+ // blank session in the "resumed" worktree is indistinguishable from
1347
+ // today's dispatch unless the downgrade is named. Same-file lineage
1348
+ // is the proof the resume happened: the resumed run must keep
1349
+ // writing the orphaned attempt's transcript.
1350
+ if (resuming !== undefined && f !== resuming.sessionFile) {
1351
+ log(
1352
+ `#${issue} attempt ${attempt} resume fell back to a fresh session: opened ${f} ` +
1353
+ `instead of the orphaned attempt's ${resuming.sessionFile}`,
1354
+ );
1355
+ }
1356
+ },
1357
+ // The last fence (#374): every pre-launch settle check above has
1358
+ // passed, but the stop can still land while the session socket is
1359
+ // binding inside `createSession`. This gate is re-checked there,
1360
+ // immediately before the child spawn, and closes the run as stopped
1361
+ // instead of launching a worker the shutdown would have to wait for.
1362
+ maySpawn: () => d.drain?.draining !== true,
1363
+ }, d.workerDeps);
1364
+ } finally {
1365
+ // This is the authoritative settlement edge for `extend`: close before
1366
+ // PR verification or terminal row writes can leave stale `running` state.
1367
+ turnLimit?.close();
1368
+ turnLimit = undefined;
1369
+ }
1370
+
1371
+ // A configured model the harness could not honour means this run was done by
1372
+ // a different model than the operator chose. Logged per run, because it is
1373
+ // the only place that fact is still attached to the issue it affected.
1374
+ if (result.modelFallbackMessage !== undefined) {
1375
+ log(`#${issue} model fallback: ${result.modelFallbackMessage}`);
1376
+ }
1377
+ // What this run learned about the repo, into the fleet overlay the next
1378
+ // brief renders: the entry point it actually had to find, the gate that
1379
+ // really proved the change, the fake that looked like a test. Reported by
1380
+ // the worker in its own settlement, so nothing is inferred — and a
1381
+ // knowledge write can never fail a settlement.
1382
+ recordWorkerDiscoveries(knowledgeRepoKey(r.repo), issue, result.discoveries);
1383
+
1384
+ const verified: { state: RunState; reason?: string } =
1385
+ result.state === "pushed-green"
1386
+ ? await verifyPushedGreenClaim(tracker, result, {
1387
+ project: project.name,
1388
+ issue,
1389
+ runId,
1390
+ branch,
1391
+ repo: repoSlug ?? r.repo.name,
1392
+ store,
1393
+ })
1394
+ : { state: result.state };
1395
+ const state = verified.state;
1396
+
1397
+ // Read before the row is written so `lastError` carries the provider's own
1398
+ // message — it names the fix and the URL, which no classification can.
1399
+ const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
1400
+ const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
1401
+ const providerTransient =
1402
+ providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
1403
+
1404
+ // The other half of not believing a worker about its own run (#128). The
1405
+ // claim being audited is `state: pushed-green`, so the audit runs on the
1406
+ // worker's claim rather than on what verification made of it: a claim that
1407
+ // GitHub then contradicted is exactly the report whose details matter
1408
+ // most. Advisory throughout — nothing below reads `flags` when deciding
1409
+ // `state`, and the ordering here makes that checkable.
1410
+ const audit =
1411
+ result.state === "pushed-green"
1412
+ ? await collectSettlementFlags(tracker, {
1413
+ prUrl: result.prUrl,
1414
+ issueText: `${r.issue.title}\n${r.issue.body}`,
1415
+ // The claimed-proof check compares the PR's Verified commands
1416
+ // against what this run's session actually recorded.
1417
+ sessionFile: result.sessionFile,
1418
+ // The effective file lane admission resolved for this run at
1419
+ // dispatch — the same value the brief rendered, pre-dispatch
1420
+ // comment declarations included (#608, #744). The audit flags a
1421
+ // diff that escapes it, so a widened lane is named on evidence
1422
+ // rather than found by reading the PR's file list by hand (#739).
1423
+ lane: admittedLane,
1424
+ })
1425
+ : undefined;
1426
+ if (result.state === "pushed-green" && audit?.truncated) {
1427
+ log(`#${issue} settlement audit read a truncated PR diff`);
1428
+ }
1429
+ const auditLines =
1430
+ audit === undefined
1431
+ ? []
1432
+ : formatSettlementFlags(audit.flags, { truncated: audit.truncated });
1433
+ // The report's file list is derived from the PR's own diff, not retyped
1434
+ // from the worker's memory of the session: the narrative stays the
1435
+ // worker's, the disclosure becomes the diff's (#488). A diff that could
1436
+ // not be read leaves the worker's text untouched and the audit's
1437
+ // `changed-line-missing` flag says so.
1438
+ // The within-run reliability sentence, appended where a human reads it
1439
+ // (#584): which model the run finished on and whether it swapped
1440
+ // mid-flight. Empty (undefined) for a clean run, so a run that never
1441
+ // retried, swapped or compacted keeps today's settlement report byte for
1442
+ // byte — the "additive" claim asserted rather than assumed.
1443
+ const reliabilityLine = reliabilitySettlementLine({
1444
+ resolvedModel: result.model,
1445
+ resolvedProvider: result.provider,
1446
+ retryFallbacks: result.retryFallbacks,
1447
+ retryFallbackSucceeded: result.retryFallbackSucceeded,
1448
+ modelRecoveries: result.modelRecoveries,
1449
+ autoRetryCount: result.autoRetryCount,
1450
+ autoCompactionCount: result.autoCompactionCount,
1451
+ });
1452
+
1453
+ const settlementReport = [
1454
+ audit?.changedLine === undefined
1455
+ ? result.report
1456
+ : withDerivedChangedLine(result.report, audit.changedLine),
1457
+ ...(reliabilityLine === undefined ? [] : ["", reliabilityLine]),
1458
+ ].join("\n");
1459
+
1460
+ const finalReport = [
1461
+ ...(verified.reason === undefined ? [] : [verified.reason, ""]),
1462
+ ...(auditLines.length === 0 ? [] : [...auditLines, ""]),
1463
+ settlementReport,
1464
+ ].join("\n");
1465
+
1466
+ // What becomes of the tree, decided once, before any label or page. A
1467
+ // `pushed-*` run is the only end that does not salvage: its deliverable is
1468
+ // already on a remote branch, whatever is left loose in the tree is by the
1469
+ // worker's own account not part of it, and appending a WIP commit would
1470
+ // turn the green PR this daemon just verified red. Every other end may
1471
+ // contain work, so it is salvaged before the tree's final fate is decided.
1472
+ const settlement =
1473
+ state === "pushed-green" || state === "pushed-pending" || state === "merged"
1474
+ ? undefined
1475
+ : await settleWorktree({
1476
+ issue,
1477
+ attempt,
1478
+ ending:
1479
+ state === "blocked"
1480
+ ? "blocked for an operator decision"
1481
+ : state === "stopped"
1482
+ ? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
1483
+ : endedBy(result.killedBy),
1484
+ worktree: worktreePath,
1485
+ branch,
1486
+ publish,
1487
+ ...(state === "failed" || state === "killed"
1488
+ ? ({ tree: "keep" } as const)
1489
+ : ({ tree: "remove", mirrorPath } as const)),
1490
+ });
1491
+ if (settlement === undefined) {
1492
+ // A `pushed-*` end does not salvage — its deliverable is already on a
1493
+ // remote branch — but its repository still goes away here, so the branch
1494
+ // is published first for the same reason `settleWorktree` does it: the
1495
+ // run repo is no longer a view of the mirror, and anything it holds that
1496
+ // never reached the mirror dies with the directory.
1497
+ const published = await publish(branch);
1498
+ if (!published.ok) {
1499
+ log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
1500
+ }
1501
+ await removeWorktree(mirrorPath, worktreePath);
1502
+ }
1503
+
1504
+ const terminalPatch: Partial<RunRecord> = {
1505
+ endedAt: Date.now(),
1506
+ turns: result.turns,
1507
+ spendUsd: result.spendUsd,
1508
+ // The count of in-session provider 429s the worker metered live, so a
1509
+ // run the provider throttled into the ground carries its own diagnosis
1510
+ // instead of landing `unknown` — the classifier reads it straight off
1511
+ // this column (#573).
1512
+ provider429Count: result.provider429Count,
1513
+ // The within-run harness reliability surface #581 collected, persisted
1514
+ // now that settlement owns the row (#584). The resolved model/provider
1515
+ // only travel when some assistant message carried them (the run recorded
1516
+ // no model, or the worker never established one); the count fields always
1517
+ // travel, 0 for a clean run, so an absent column can never be read as a
1518
+ // quiet fleet. Written for every terminal state, clean or not.
1519
+ ...(result.model === undefined ? {} : { resolvedModel: result.model }),
1520
+ ...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
1521
+ retryFallbacks: result.retryFallbacks,
1522
+ retryFallbackSucceeded: result.retryFallbackSucceeded,
1523
+ modelRecoveries: result.modelRecoveries,
1524
+ autoRetryCount: result.autoRetryCount,
1525
+ autoCompactionCount: result.autoCompactionCount,
1526
+ // The worker only reports these when it actually established them; a kill
1527
+ // or a settle whose report named no PR must not wipe what a verb recorded
1528
+ // earlier in the same run (#468). The sink in `updateRun` skips undefined
1529
+ // too, but a settle should not even build such a patch.
1530
+ ...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
1531
+ ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
1532
+ sessionFile: result.sessionFile,
1533
+ // The code-graph session observation (#726): what the run's own session
1534
+ // registry held at start, persisted with the rest of the run's facts.
1535
+ // Absent only when the session surface did not record one.
1536
+ ...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
1537
+ // Every terminal state persists the worker's report — with the `changed:`
1538
+ // file list derived from the PR's diff where one could be read — not
1539
+ // just a green push: a stopped attempt's partial report is still part of
1540
+ // its audit trail.
1541
+ report: settlementReport,
1542
+ ...settlement?.patch,
1543
+ ...(audit === undefined || audit.flags.length === 0
1544
+ ? {}
1545
+ : { settlementFlags: audit.flags }),
1546
+ };
1547
+ if (state === "stopped") {
1548
+ recordOperatorStop(store, {
1549
+ project: project.name,
1550
+ issue,
1551
+ runId,
1552
+ inProgress,
1553
+ reason: result.stoppedReason ?? "no reason recorded",
1554
+ patch: terminalPatch,
1555
+ });
1556
+ } else {
1557
+ const lastError = completionLastError(
1558
+ providerCredit,
1559
+ providerTransient,
1560
+ verified.reason,
1561
+ sessionErr,
1562
+ );
1563
+ store.updateRun(runId, {
1564
+ ...terminalPatch,
1565
+ state,
1566
+ ...(lastError === undefined ? {} : { lastError }),
1567
+ });
1568
+ }
1569
+
1570
+ const salvaged = settlement?.lines ?? [];
1571
+
1572
+ // The run's reliability news, surfaced where the tick digest reads it
1573
+ // (#584): the digest is model-authored but consumes the store's material
1574
+ // ledger, so a run that swapped mid-flight or rode out a throttled
1575
+ // provider lands one event the digest can name. Clean runs record nothing
1576
+ // here, so a quiet fleet's digest is unchanged.
1577
+ if (reliabilityLine !== undefined) {
1578
+ store.recordMaterialEvent({
1579
+ project: project.name,
1580
+ category: "reliability",
1581
+ summary: `#${issue} ${reliabilityLine}`,
1582
+ evidence: `${r.issue.title}\n${r.issue.url}\n\n${reliabilityLine}`,
1583
+ occurredAt: Date.now(),
1584
+ recordedAt: Date.now(),
1585
+ });
1586
+ }
1587
+
1588
+ if (state === "stopped") {
1589
+ log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
1590
+ } else if (state === "blocked") {
1591
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
1592
+ wakeOrchestratorForBlockedRun(project.name, issue);
1593
+ await safeEscalate(d, {
1594
+ tier: 1,
1595
+ project: project.name,
1596
+ issue,
1597
+ runId,
1598
+ summary: `#${issue} is blocked on attempt ${attempt} and needs a decision`,
1599
+ detail: [`${r.issue.title}`, r.issue.url, "", ...salvaged, "", result.report].join("\n"),
1600
+ });
1601
+ } else if (state === "failed" || state === "killed") {
1602
+ // The row is already `killed`, so this count includes the segment that
1603
+ // just ended.
1604
+ const continuation = store.continuationsFor(project.name, issue);
1605
+ const continueTurns = shouldContinueAfterTurnsCap({
1606
+ killedBy: result.killedBy,
1607
+ prUrl: result.prUrl,
1608
+ headSha: result.headSha,
1609
+ salvageSha: settlement?.patch?.salvageSha,
1610
+ continuation,
1611
+ maxContinuations: caps.maxContinuationsPerIssue,
1612
+ });
1613
+
1614
+ if (providerCredit !== undefined) {
1615
+ await reactToProviderCredit({ project: d.project, escalate: (e) => d.escalate(e), isPaused, setPaused }, issue, providerCredit, result.sessionFile);
1616
+ await swapToQueue(d, issue, inProgress);
1617
+ } else if (continueTurns) {
1618
+ // One ordered pair through the shared chokepoint: the in-progress
1619
+ // removal before the queue add, exactly the order the projector will
1620
+ // apply them in (#201) — and the wake that comes with it, so a
1621
+ // salvaged continuation is claimed on an immediate pass instead of
1622
+ // waiting out the interval (#1041).
1623
+ await swapToQueue(d, issue, inProgress);
1624
+ log(
1625
+ `#${issue} turns-cap on run ${attempt}, continuation ` +
1626
+ `${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
1627
+ );
1628
+ await safeEscalate(d, {
1629
+ tier: 1,
1630
+ project: project.name,
1631
+ issue,
1632
+ runId,
1633
+ summary: `#${issue} hit the turns cap on attempt ${attempt} — auto-requeued for continuation`,
1634
+ detail: [
1635
+ `${r.issue.title}`,
1636
+ r.issue.url,
1637
+ ...salvaged,
1638
+ `Session: ${result.sessionFile ?? "(no transcript)"}`,
1639
+ "",
1640
+ "The queue label is back on; the next tick should reattach the branch",
1641
+ "and open a continuation brief. No failed label was applied.",
1642
+ "",
1643
+ result.report,
1644
+ ].join("\n"),
1645
+ });
1646
+ } else {
1647
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
1648
+ await safeEscalate(d, {
1649
+ tier: 1,
1650
+ project: project.name,
1651
+ issue,
1652
+ runId,
1653
+ // The dedup key includes the summary, so the attempt number is what
1654
+ // lets a genuine second failure page again while a tick that keeps
1655
+ // seeing the same dead issue stays quiet.
1656
+ summary: result.killedBy
1657
+ ? `#${issue} was killed on attempt ${attempt} by the ${result.killedBy} cap`
1658
+ : `#${issue} failed on attempt ${attempt}`,
1659
+ detail: [
1660
+ `${r.issue.title}`,
1661
+ r.issue.url,
1662
+ ...salvaged,
1663
+ `Session: ${result.sessionFile ?? "(no transcript)"}`,
1664
+ "",
1665
+ finalReport,
1666
+ ].join("\n"),
1667
+ });
1668
+ }
1669
+ } else {
1670
+ // A verified or still-pending PR keeps the in-progress label until its
1671
+ // checks or merge settle, preventing another worker from duplicating it.
1672
+ log(`#${issue} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
1673
+ // A green run's report is the only one nothing ever delivers: it raises no
1674
+ // escalation, and the row it settles into says nothing about what the
1675
+ // worker wrote. That is fine while the report is believed — and it is
1676
+ // exactly wrong once something has stopped believing it. A flagged
1677
+ // settlement is the one case where a `pushed-green` run has news, and it
1678
+ // has it *now*, while the PR is still open and nobody has merged it (#128).
1679
+ //
1680
+ // Tier 1, because the judgement is the orchestrator's: it can read the
1681
+ // diff, it can ask, and it can promote to a human. The state is
1682
+ // deliberately unchanged either way — the run below this line settles
1683
+ // `pushed-green` whether it was flagged or not.
1684
+ //
1685
+ // Findings only. A diff too large to read in full is noted in the log and
1686
+ // in the report, but it is not news for a human: paging "0 flags, could
1687
+ // not read it all" is exactly the kind of line that teaches an
1688
+ // orchestrator to skim past this escalation the next time it carries one.
1689
+ const found = audit?.flags ?? [];
1690
+ if (found.length > 0) {
1691
+ await safeEscalate(d, {
1692
+ tier: 1,
1693
+ project: project.name,
1694
+ issue,
1695
+ runId,
1696
+ summary:
1697
+ `#${issue} settled ${state} on attempt ${attempt} with ` +
1698
+ `${found.length} settlement audit flag(s)`,
1699
+ detail: [
1700
+ `${r.issue.title}`,
1701
+ r.issue.url,
1702
+ result.prUrl ?? "(no PR URL)",
1703
+ "",
1704
+ "The run is NOT blocked and its state is unchanged. These are advisory",
1705
+ "findings about the worker's own account of its work — judge them, then",
1706
+ "merge, ask, or close as you would have anyway.",
1707
+ "",
1708
+ finalReport,
1709
+ ].join("\n"),
1710
+ });
1711
+ }
1712
+ }
1713
+ } catch (err) {
1714
+ // Dispatch setup can fail after the controller opens but before runWorker's
1715
+ // inner settlement guard exists. Latch it before any terminal write or await.
1716
+ turnLimit?.close();
1717
+ turnLimit = undefined;
1718
+ if (await settleStopBeforeSession()) return;
1719
+ const detail = errText(err);
1720
+ log(`#${issue} errored: ${detail}`);
1721
+ // A crash lands anywhere, including mid-edit in a tree holding the only
1722
+ // copy of real work. Nothing else on this path so much as looks at it.
1723
+ const settlement =
1724
+ worktreePath === undefined
1725
+ ? undefined
1726
+ : await settleWorktree({
1727
+ issue,
1728
+ attempt,
1729
+ ending: "killed by a dispatch error",
1730
+ worktree: worktreePath,
1731
+ branch,
1732
+ publish,
1733
+ tree: "keep",
1734
+ });
1735
+ if (run) {
1736
+ store.updateRun(run.id, {
1737
+ state: "failed",
1738
+ endedAt: Date.now(),
1739
+ lastError: detail,
1740
+ ...settlement?.patch,
1741
+ });
1742
+ }
1743
+ if (claimed) {
1744
+ // Leaving the issue stuck as in-progress would hide it from both the
1745
+ // queue and the human, so relabel even on the error path.
1746
+ // The failure-path relabel cannot throw: it is a local outbox write, and
1747
+ // the projector retries until the tracker takes it (#201).
1748
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
1749
+ }
1750
+
1751
+ const salvaged = settlement?.lines ?? [];
1752
+ await safeEscalate(d, {
1753
+ tier: 1,
1754
+ project: project.name,
1755
+ issue,
1756
+ runId: run?.id,
1757
+ summary: `#${issue} could not be dispatched on attempt ${attempt}`,
1758
+ detail: salvaged.length === 0 ? detail : [detail, "", ...salvaged].join("\n"),
1759
+ });
1760
+ // The worktree, if one was created, is deliberately left in place: this is
1761
+ // a failure path, and whatever it still held is now a commit on the branch.
1762
+ } finally {
1763
+ turnLimit?.close();
1764
+ // The run is over, so its channel is too. Closed here rather than beside
1765
+ // the session so the crash path closes it as well: a listener left bound
1766
+ // after its run settled is a socket whose `run-not-live` check is the only
1767
+ // thing standing between a stale child and a push.
1768
+ if (verbListener !== undefined) {
1769
+ try {
1770
+ await verbListener.close();
1771
+ } catch (err) {
1772
+ log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
1773
+ }
1774
+ verbListener = undefined;
1775
+ }
1776
+ workerControl?.close();
1777
+ workerControl = undefined;
1778
+ }
1779
+ }