omp-conductor 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -31,13 +31,16 @@ import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
31
31
  import { startOrchestrator } from "./orchestrator.ts";
32
32
  import type { OrchestratorHandle } from "./orchestrator.ts";
33
33
  import { createReportOutbox, formatOpenReports } from "./reports.ts";
34
- import { recordReleaseBlock } from "./release-policy.ts";
34
+ import {
35
+ recordReleaseBlock,
36
+ type ReleaseBlockContext,
37
+ } from "./release-policy.ts";
35
38
  import { branchName, effectiveLabels, route } from "./routing.ts";
36
39
  import type { Routed, UnroutableReason } from "./routing.ts";
37
40
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
38
41
  import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
39
42
  import { projectLabels } from "./label-projection.ts";
40
- import { dbPath, openStore, utcDay } from "./store.ts";
43
+ import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
41
44
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
42
45
  import { RELEASE_SHAPES } from "./types.ts";
43
46
  import type {
@@ -45,7 +48,10 @@ import type {
45
48
  Caps,
46
49
  DispatchSummary,
47
50
  Escalation,
51
+ IssueSnapshot,
52
+ MergedPrInfo,
48
53
  OpenCloser,
54
+ ReleaseShape,
49
55
  PrState,
50
56
  ProjectConfig,
51
57
  ReadyIssue,
@@ -60,12 +66,14 @@ import type {
60
66
  Store,
61
67
  Tracker,
62
68
  VerbLedgerEntry,
69
+ TurnOverride,
63
70
  } from "./types.ts";
64
71
  import {
65
72
  type KilledBy,
66
73
  type WorkerPauseControl,
67
74
  type WorkerPausePhase,
68
75
  type WorkerResult,
76
+ type RunWorkerDeps,
69
77
  renderBrief,
70
78
  runWorker,
71
79
  } from "./worker.ts";
@@ -82,8 +90,13 @@ import {
82
90
  } from "./worktree.ts";
83
91
  import { githubVerbActions } from "./verbs/actions.ts";
84
92
  import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
85
- import { listenVerbChannel } from "./verbs/server.ts";
86
- import type { VerbActions, VerbDeps, VerbListener } from "./verbs/server.ts";
93
+ import {
94
+ listenVerbChannel,
95
+ PR_LOOKUP_WINDOW_MS,
96
+ type VerbActions,
97
+ type VerbDeps,
98
+ type VerbListener,
99
+ } from "./verbs/server.ts";
87
100
  import {
88
101
  ensureVerbSocketDir,
89
102
  peerCredentialReader,
@@ -120,6 +133,9 @@ const DISPATCH_INFRA_MAX_STRIKES = 3;
120
133
  * provider itself is degraded, not unlucky, and the sweep escalates to a
121
134
  * human instead of requeueing into a down provider forever (#220). */
122
135
  const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
136
+ /** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
137
+ * are maintenance, but a backlog must not turn one tick into an API burst. */
138
+ const SALVAGED_PR_ADOPTION_BATCH = 10;
123
139
  const DEFAULT_PORT = 8787;
124
140
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
125
141
 
@@ -157,6 +173,8 @@ interface Deps {
157
173
  escalate(e: Escalation): Promise<void>;
158
174
  turnLimits: TurnLimitRegistry;
159
175
  workerControls: WorkerControlRegistry;
176
+ /** Session seam for lifecycle integration tests; production uses the real harness. */
177
+ workerDeps?: RunWorkerDeps;
160
178
  integrity: IntegrityGate;
161
179
  stall: StallGate;
162
180
  cleanup?: RetainedCleanupCursor;
@@ -212,18 +230,25 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
212
230
 
213
231
  // ------------------------------------------------------- orchestrator liveness
214
232
 
233
+ /** No more than one reminder per hour while the same marker remains. */
234
+ const STALL_RENOTIFY_MS = 60 * 60_000;
235
+
215
236
  /**
216
- * Whether the wedged-orchestrator page has already gone out for the stall
217
- * currently on disk. One page per episode: the marker persists until a tick is
218
- * consumed, so paging per five minutes would be paging forever.
237
+ * Whether the wedged-orchestrator page has gone out, and when it last went out,
238
+ * for the stall currently on disk.
219
239
  */
220
240
  export interface StallGate {
221
241
  paged: boolean;
242
+ lastPagedAt?: number;
222
243
  }
223
244
 
224
245
  export interface StallVerdict {
225
246
  /** The marker's own line, when one is there. */
226
247
  since?: string;
248
+ /** Count written by the stalled tick producer, when its marker is readable. */
249
+ unconsumedTicks?: number;
250
+ /** Whole hours elapsed since the marker timestamp, when it is parseable. */
251
+ stalledHours?: number;
227
252
  page: boolean;
228
253
  }
229
254
 
@@ -246,51 +271,84 @@ export interface StallVerdict {
246
271
  *
247
272
  * Resets when the marker disappears, so a second stall days later pages again.
248
273
  */
249
- export function checkStall(gate: StallGate, marker: string): StallVerdict {
274
+ export function checkStall(gate: StallGate, marker: string, now = Date.now()): StallVerdict {
250
275
  if (!existsSync(marker)) {
251
276
  gate.paged = false;
277
+ delete gate.lastPagedAt;
252
278
  return { page: false };
253
279
  }
254
- const page = !gate.paged;
280
+ const page =
281
+ !gate.paged ||
282
+ gate.lastPagedAt === undefined ||
283
+ now - gate.lastPagedAt >= STALL_RENOTIFY_MS;
255
284
  let since: string | undefined;
285
+ let unconsumedTicks: number | undefined;
286
+ let stalledHours: number | undefined;
256
287
  try {
257
288
  const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
258
- if (body !== undefined && body !== "") since = body;
289
+ if (body !== undefined && body !== "") {
290
+ since = body;
291
+ const ticks = /\b(\d+) ticks queued unconsumed\b/.exec(body)?.[1];
292
+ if (ticks !== undefined) unconsumedTicks = Number(ticks);
293
+ const startedAt = Date.parse(body.split(/\s+/, 1)[0] ?? "");
294
+ if (Number.isFinite(startedAt)) {
295
+ stalledHours = Math.max(0, Math.floor((now - startedAt) / STALL_RENOTIFY_MS));
296
+ }
297
+ }
259
298
  } catch {
260
- // An unreadable marker still means stalled; the timestamp is a nicety.
299
+ // An unreadable marker still means stalled; its evidence is a nicety.
261
300
  }
262
- return { ...(since === undefined ? {} : { since }), page };
301
+ return {
302
+ ...(since === undefined ? {} : { since }),
303
+ ...(unconsumedTicks === undefined ? {} : { unconsumedTicks }),
304
+ ...(stalledHours === undefined ? {} : { stalledHours }),
305
+ page,
306
+ };
263
307
  }
264
308
 
265
309
  /**
266
- * Pages tier 2 once when the orchestrator session stops draining its queue.
310
+ * Pages tier 2 when the orchestrator session stops draining its queue.
267
311
  *
268
312
  * Deliberately does not restart anything. A wedge lands mid-turn, this process
269
313
  * cannot tell a half-applied edit from an idle loop, and killing the session
270
314
  * could destroy work an operator would rather read first — the same refusal to
271
315
  * guess that the recovery plugin is built on.
272
316
  */
273
- async function watchOrchestrator(d: Deps): Promise<void> {
317
+ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
274
318
  const marker = join(stateDir(), STALL_MARKER_FILE);
275
- const verdict = checkStall(d.stall, marker);
319
+ const repeat = d.stall.paged;
320
+ const verdict = checkStall(d.stall, marker, now);
276
321
  if (!verdict.page) return;
277
322
 
323
+ const greenRuns = d.store.activeRuns(d.project.name).filter((run) => run.state === "pushed-green");
324
+ const repeatSuffix = repeat
325
+ ? ` — still stalled (${
326
+ verdict.stalledHours === undefined
327
+ ? `${new Date(now).toISOString().slice(0, 13)}Z`
328
+ : `${verdict.stalledHours}h`
329
+ })`
330
+ : "";
331
+
278
332
  log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
279
333
  const delivered = await safeEscalate(d, {
280
334
  tier: 2,
281
335
  category: "confirmed-failure",
336
+ urgent: true,
282
337
  project: d.project.name,
283
338
  issue: NO_ISSUE,
284
339
  // Keyed on the marker's own timestamp, not the date. The dedup ledger keys
285
340
  // on this summary, and two wedges in one day is not a hypothetical — the
286
341
  // failure mode is a session that gets stuck, gets restarted, and gets stuck
287
- // again on the same cause an hour later. A day-keyed summary would report
288
- // the first and silently swallow every one after it.
342
+ // again on the same cause an hour later. The hourly suffix makes bounded
343
+ // reminders distinct without allowing every five-minute tick through.
289
344
  summary:
290
345
  `Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
291
- `it has stopped reading its queue (${d.project.name})`,
346
+ `it has stopped reading its queue (${d.project.name})${repeatSuffix}`,
292
347
  detail: [
293
348
  verdict.since ?? "Marker present with no readable timestamp.",
349
+ `Unconsumed ticks: ${verdict.unconsumedTicks ?? "unknown"}`,
350
+ `Open green worker PRs: ${greenRuns.length}`,
351
+ ...greenRuns.map((run) => `- ${run.prUrl ?? `#${run.issue} (PR URL unavailable)`}`),
294
352
  `Marker: ${marker}`,
295
353
  "",
296
354
  "Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
@@ -302,7 +360,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
302
360
  "Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
303
361
  ].join("\n"),
304
362
  });
305
- markPaged(d.stall, delivered);
363
+ markPaged(d.stall, delivered, now);
306
364
  }
307
365
 
308
366
  /**
@@ -319,10 +377,10 @@ export function isPaused(): boolean {
319
377
  * The epoch-ms timestamp at which the current pause began, read from the same
320
378
  * sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
321
379
  * `undefined` when the fleet is not paused, or when the file's first line does
322
- * not parse as a date a legacy/blank sentinel keeps today's refuse-everything
323
- * behavior, because a run admitted before an *unknown* pause cannot be proven
324
- * innocent. {@link isPaused} is the authority on *whether*; this answers
325
- * *since when*.
380
+ * not parse as a date. A legacy/blank sentinel keeps completion mutations
381
+ * fail-closed because a run admitted before an *unknown* pause cannot be proven
382
+ * innocent. {@link isPaused} is the authority on *whether*; this answers *since
383
+ * when*.
326
384
  */
327
385
  export function pausedAt(): number | undefined {
328
386
  const f = join(stateDir(), "paused");
@@ -333,8 +391,8 @@ export function pausedAt(): number | undefined {
333
391
  const t = Date.parse(first);
334
392
  return Number.isNaN(t) ? undefined : t;
335
393
  } catch {
336
- // Unreadable sentinel (permissions, corruption): fail closed like an
337
- // unparseable line refuse mutations while the pause is unprovable.
394
+ // Unreadable sentinel (permissions, corruption): fail completion mutations
395
+ // closed like an unparseable line while the pause time is unprovable.
338
396
  return undefined;
339
397
  }
340
398
  }
@@ -359,8 +417,8 @@ export function pauseProvenance(): { source: string; reason?: string } | undefin
359
417
  const reason = match[2];
360
418
  return { source, ...(reason === undefined ? {} : { reason }) };
361
419
  } catch {
362
- // Unreadable sentinel: no provenance to name, and the refuse-everything
363
- // posture of an unknown pause is unchanged.
420
+ // Unreadable sentinel: no provenance to name. Completion mutations still
421
+ // fail closed because the pause time is unknown.
364
422
  return undefined;
365
423
  }
366
424
  }
@@ -471,9 +529,17 @@ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>
471
529
  return { diff, pause: true, page: !gate.paged };
472
530
  }
473
531
 
474
- /** Latch a once-only page, after delivery is confirmed and never before. */
475
- export function markPaged(gate: { paged: boolean }, delivered: boolean): void {
476
- if (delivered) gate.paged = true;
532
+ /** Latch a page after delivery is confirmed, and never before. */
533
+ export function markPaged(gate: StallGate, delivered: boolean, now: number): void;
534
+ export function markPaged(gate: IntegrityGate, delivered: boolean): void;
535
+ export function markPaged(
536
+ gate: { paged: boolean; lastPagedAt?: number },
537
+ delivered: boolean,
538
+ now?: number,
539
+ ): void {
540
+ if (!delivered) return;
541
+ gate.paged = true;
542
+ if (now !== undefined) gate.lastPagedAt = now;
477
543
  }
478
544
 
479
545
  // ---------------------------------------------------------------------- helpers
@@ -543,6 +609,32 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
543
609
  { issue, op: "remove", label: from },
544
610
  ]);
545
611
  }
612
+ /**
613
+ * Persist the operator-stop transition before releasing its live controller.
614
+ * The row is terminal first, then its in-progress label is removed through the
615
+ * same durable projection outbox as every other lifecycle transition.
616
+ */
617
+ export function recordOperatorStop(
618
+ store: Pick<Store, "updateRun" | "enqueueLabelOps">,
619
+ args: {
620
+ project: string;
621
+ issue: number;
622
+ runId: string;
623
+ inProgress: string;
624
+ reason: string;
625
+ patch: Partial<RunRecord>;
626
+ },
627
+ ): void {
628
+ store.updateRun(args.runId, {
629
+ ...args.patch,
630
+ state: "stopped",
631
+ lastError: `operator stopped: ${args.reason}`,
632
+ });
633
+ store.enqueueLabelOps(args.project, [
634
+ { issue: args.issue, op: "remove", label: args.inProgress },
635
+ ]);
636
+ }
637
+
546
638
 
547
639
  /**
548
640
  * The escalator throws when no transport is configured or Telegram rejects, and
@@ -564,6 +656,38 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
564
656
  }
565
657
  }
566
658
 
659
+ async function reactToProviderCredit(
660
+ d: Deps,
661
+ issue: number,
662
+ message: string,
663
+ sessionFile: string | undefined,
664
+ ): Promise<void> {
665
+ const { project } = d;
666
+ const alreadyPaused = isPaused();
667
+ if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
668
+ log(
669
+ `#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
670
+ );
671
+ // Fleet-scoped and run-independent on purpose. The notification ledger
672
+ // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
673
+ // carrying no run or attempt pages once for the fleet, not once per run.
674
+ await safeEscalate(d, {
675
+ tier: 2,
676
+ category: "fleet-stopped",
677
+ project: project.name,
678
+ issue: NO_ISSUE,
679
+ summary: `Model provider refused for credit — ${project.name} is paused`,
680
+ detail: [
681
+ message,
682
+ "",
683
+ "No implementation attempt was charged: this is a billing state, not a",
684
+ "failed implementation. Each affected issue keeps its queue label and",
685
+ "re-dispatches on `omp-conductor resume` once the provider has credit.",
686
+ `Session: ${sessionFile ?? "(no transcript)"}`,
687
+ ].join("\n"),
688
+ });
689
+ }
690
+
567
691
  /**
568
692
  * What a salvage attempt contributes to the escalation: where the work went, or
569
693
  * that it went nowhere. Split from the effects below for the same reason
@@ -840,11 +964,14 @@ export function createTurnLimitRegistry(
840
964
 
841
965
  export type WorkerControlResult =
842
966
  | { kind: "ok"; runId: string; phase: WorkerPausePhase }
967
+ | { kind: "stopped"; runId: string; reason: string }
843
968
  | { kind: "refused"; runId: string; error: string }
844
969
  | { kind: "not-active" };
845
970
 
846
971
  export interface WorkerControlSlot {
847
972
  install(control: WorkerPauseControl): void;
973
+ /** Stop accepted before the session controller exists. */
974
+ requestedStop(): string | undefined;
848
975
  close(): void;
849
976
  }
850
977
 
@@ -852,34 +979,51 @@ export interface WorkerControlRegistry {
852
979
  open(project: string, issue: number, runId: string): WorkerControlSlot;
853
980
  pause(project: string, issue: number): Promise<WorkerControlResult>;
854
981
  resume(project: string, issue: number): WorkerControlResult;
982
+ stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
855
983
  /** Live runs whose phase is not `running` — what /healthz and the board show. */
856
984
  snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
857
985
  }
858
986
 
859
987
  /** Authoritative controls for sessions owned by this daemon process. */
860
988
  export function createWorkerControlRegistry(): WorkerControlRegistry {
861
- const active = new Map<
862
- string,
863
- { project: string; issue: number; runId: string; control?: WorkerPauseControl }
864
- >();
989
+ interface Entry {
990
+ project: string;
991
+ issue: number;
992
+ runId: string;
993
+ control?: WorkerPauseControl;
994
+ stopReason?: string;
995
+ stopError?: string;
996
+ finished: PromiseWithResolvers<void>;
997
+ }
998
+
999
+ const active = new Map<string, Entry>();
865
1000
  const key = (project: string, issue: number): string => `${project}\0${issue}`;
866
1001
  return {
867
1002
  open(project, issue, runId) {
868
1003
  const k = key(project, issue);
869
1004
  if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
870
- const entry = { project, issue, runId } as {
871
- project: string;
872
- issue: number;
873
- runId: string;
874
- control?: WorkerPauseControl;
1005
+ const entry: Entry = {
1006
+ project,
1007
+ issue,
1008
+ runId,
1009
+ finished: Promise.withResolvers<void>(),
875
1010
  };
876
1011
  active.set(k, entry);
877
1012
  return {
878
1013
  install: (control) => {
879
- if (active.get(k) === entry) entry.control = control;
1014
+ if (active.get(k) !== entry) return;
1015
+ entry.control = control;
1016
+ if (entry.stopReason === undefined) return;
1017
+ try {
1018
+ control.stop(entry.stopReason);
1019
+ } catch (err) {
1020
+ entry.stopError = err instanceof Error ? err.message : String(err);
1021
+ }
880
1022
  },
1023
+ requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
881
1024
  close: () => {
882
1025
  if (active.get(k) === entry) active.delete(k);
1026
+ entry.finished.resolve();
883
1027
  },
884
1028
  };
885
1029
  },
@@ -911,6 +1055,30 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
911
1055
  };
912
1056
  }
913
1057
  },
1058
+ async stop(project, issue, reason) {
1059
+ const entry = active.get(key(project, issue));
1060
+ if (entry === undefined) return { kind: "not-active" };
1061
+ if (entry.stopReason === undefined) {
1062
+ entry.stopReason = reason;
1063
+ if (entry.control !== undefined) {
1064
+ try {
1065
+ entry.control.stop(reason);
1066
+ } catch (err) {
1067
+ entry.stopReason = undefined;
1068
+ return {
1069
+ kind: "refused",
1070
+ runId: entry.runId,
1071
+ error: err instanceof Error ? err.message : String(err),
1072
+ };
1073
+ }
1074
+ }
1075
+ }
1076
+ await entry.finished.promise;
1077
+ if (entry.stopError !== undefined) {
1078
+ return { kind: "refused", runId: entry.runId, error: entry.stopError };
1079
+ }
1080
+ return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
1081
+ },
914
1082
  snapshot(project) {
915
1083
  const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
916
1084
  for (const entry of active.values()) {
@@ -1012,7 +1180,28 @@ export function shouldContinueAfterTurnsCap(f: {
1012
1180
  return hasContinuationBudget(f.continuation, f.maxContinuations);
1013
1181
  }
1014
1182
 
1015
- async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1183
+ /** Bind a worker's run identity to every release-policy block it emits. */
1184
+ export function workerReleaseBlockRecorder(
1185
+ project: string,
1186
+ issue: number,
1187
+ runId: string,
1188
+ root = stateDir(),
1189
+ ): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
1190
+ return (shape, context) =>
1191
+ recordReleaseBlock(
1192
+ project,
1193
+ "worker",
1194
+ shape,
1195
+ {
1196
+ ...context,
1197
+ issue,
1198
+ runId,
1199
+ },
1200
+ root,
1201
+ );
1202
+ }
1203
+
1204
+ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1016
1205
  const { project, caps, tracker, store } = d;
1017
1206
  const issue = r.issue.number;
1018
1207
  const branch = branchName(r.issue);
@@ -1027,6 +1216,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1027
1216
  let worktreePath: string | undefined;
1028
1217
  let turnLimit: TurnLimitController | undefined;
1029
1218
  let workerControl: WorkerControlSlot | undefined;
1219
+ let workerSessionInstalled = false;
1030
1220
  // The run's own repository. Hoisted for the same reason `worktreePath` is —
1031
1221
  // the catch and finally paths have to publish the branch.
1032
1222
  let runRepo: RunRepoRef | undefined;
@@ -1048,6 +1238,53 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1048
1238
  return pushRunBranch(project, runRepo);
1049
1239
  };
1050
1240
 
1241
+ /**
1242
+ * A stop can arrive after the run is claimed but before `runWorker` exposes
1243
+ * its controller. Settle that run here instead of turning an operator action
1244
+ * into a dispatch failure or making the HTTP request wait for a session that
1245
+ * will never exist.
1246
+ */
1247
+ const settleStopBeforeSession = async (): Promise<boolean> => {
1248
+ const reason = workerControl?.requestedStop();
1249
+ if (reason === undefined || run === undefined || workerSessionInstalled) return false;
1250
+ turnLimit?.close();
1251
+ turnLimit = undefined;
1252
+ const settlement =
1253
+ worktreePath === undefined
1254
+ ? undefined
1255
+ : await settleWorktree({
1256
+ issue,
1257
+ attempt,
1258
+ ending: `stopped by the operator: ${reason}`,
1259
+ worktree: worktreePath,
1260
+ branch,
1261
+ publish,
1262
+ tree: "remove",
1263
+ mirrorPath,
1264
+ });
1265
+ recordOperatorStop(store, {
1266
+ project: project.name,
1267
+ issue,
1268
+ runId: run.id,
1269
+ inProgress,
1270
+ reason,
1271
+ patch: {
1272
+ endedAt: Date.now(),
1273
+ turns: run.turns,
1274
+ spendUsd: run.spendUsd,
1275
+ worktree: worktreePath ?? run.worktree,
1276
+ report: [
1277
+ "Operator stopped the run before its worker session started.",
1278
+ `Reason: ${reason}`,
1279
+ ...(settlement?.lines ?? []),
1280
+ ].join("\n"),
1281
+ ...settlement?.patch,
1282
+ },
1283
+ });
1284
+ log(`#${issue} stopped by operator before its worker session started: ${reason}`);
1285
+ return true;
1286
+ };
1287
+
1051
1288
  try {
1052
1289
  // Claim on the STORE first, before anything that can fail. The run row —
1053
1290
  // not the label — is the crash-safe guard against double dispatch: rows
@@ -1063,8 +1300,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1063
1300
  // Read before this attempt's own row exists, so `latestRun` still means the
1064
1301
  // attempt whose work this one inherits.
1065
1302
  const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
1066
-
1067
-
1068
1303
  run = store.createRun({
1069
1304
  project: project.name,
1070
1305
  issue,
@@ -1078,11 +1313,20 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1078
1313
  maxTurns: caps.workerMaxTurns,
1079
1314
  startedAt: Date.now(),
1080
1315
  });
1316
+ const maxTurns = run.maxTurns;
1317
+ const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
1081
1318
  const runId = run.id;
1319
+ if (turnOverride !== undefined) {
1320
+ log(
1321
+ `#${issue} claimed with turn override ${turnOverride} ` +
1322
+ `(base ${caps.workerMaxTurns})`,
1323
+ );
1324
+ }
1082
1325
  store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
1083
1326
  claimed = true;
1084
- turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
1327
+ turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
1085
1328
  workerControl = d.workerControls.open(project.name, issue, runId);
1329
+ if (await settleStopBeforeSession()) return;
1086
1330
 
1087
1331
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
1088
1332
  // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -1091,6 +1335,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1091
1335
  // first attempt. addRunRepo does its own ensureMirror; calling it here too
1092
1336
  // would cost a second network fetch per attempt.
1093
1337
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
1338
+ if (await settleStopBeforeSession()) return;
1094
1339
  const provisioned = await addRunRepo(
1095
1340
  r.repo,
1096
1341
  project.mirrorRoot,
@@ -1100,6 +1345,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1100
1345
  );
1101
1346
  worktreePath = provisioned.path;
1102
1347
  runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
1348
+ if (await settleStopBeforeSession()) return;
1103
1349
 
1104
1350
  // The SDK names the transcript itself, so the daemon supplies the parent
1105
1351
  // directory and learns the real path back from the result. Inventing one
@@ -1138,6 +1384,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1138
1384
  },
1139
1385
  { ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
1140
1386
  );
1387
+ if (await settleStopBeforeSession()) return;
1141
1388
 
1142
1389
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
1143
1390
 
@@ -1146,20 +1393,26 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1146
1393
  (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
1147
1394
  );
1148
1395
 
1396
+ const brief = await buildBrief(project, r, branch, worktreePath, {
1397
+ continuation: provisioned.reattached,
1398
+ defaultBranch: r.repo.defaultBranch,
1399
+ ...(provisioned.reattached && priorSalvage !== undefined
1400
+ ? { salvagedSha: priorSalvage }
1401
+ : {}),
1402
+ });
1403
+ if (await settleStopBeforeSession()) return;
1404
+
1149
1405
  let result: WorkerResult;
1150
1406
  try {
1151
1407
  result = await runWorker({
1152
- brief: await buildBrief(project, r, branch, worktreePath, {
1153
- continuation: provisioned.reattached,
1154
- defaultBranch: r.repo.defaultBranch,
1155
- ...(provisioned.reattached && priorSalvage !== undefined
1156
- ? { salvagedSha: priorSalvage }
1157
- : {}),
1158
- }),
1408
+ brief,
1159
1409
  cwd: worktreePath,
1160
1410
  caps,
1161
- maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
1162
- onPauseControl: (control) => workerControl?.install(control),
1411
+ maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
1412
+ onPauseControl: (control) => {
1413
+ workerSessionInstalled = true;
1414
+ workerControl?.install(control);
1415
+ },
1163
1416
  sessionDir,
1164
1417
  // The session's control socket, under the daemon's own state directory —
1165
1418
  // a child process of the daemon reaches it directly.
@@ -1178,28 +1431,24 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1178
1431
  },
1179
1432
  ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
1180
1433
  releaseGrants: resolveReleaseGrants(project),
1181
- onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
1434
+ onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
1182
1435
  onTurn: (n) => store.updateRun(runId, { turns: n }),
1183
1436
  onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
1184
1437
  onKilled: () => {
1185
1438
  turnLimit?.close();
1186
1439
  turnLimit = undefined;
1187
- workerControl?.close();
1188
- workerControl = undefined;
1189
1440
  },
1190
1441
  // Recorded the moment the session opens its transcript, not when the run
1191
1442
  // ends: `omp-conductor tail` resolves an issue to a file through this row,
1192
1443
  // and a path written at completion is a path nobody can follow live. The
1193
1444
  // completion-time update below writes the same value again, harmlessly.
1194
1445
  onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
1195
- });
1446
+ }, d.workerDeps);
1196
1447
  } finally {
1197
1448
  // This is the authoritative settlement edge for `extend`: close before
1198
1449
  // PR verification or terminal row writes can leave stale `running` state.
1199
1450
  turnLimit?.close();
1200
1451
  turnLimit = undefined;
1201
- workerControl?.close();
1202
- workerControl = undefined;
1203
1452
  }
1204
1453
 
1205
1454
  // A configured model the harness could not honour means this run was done by
@@ -1264,8 +1513,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1264
1513
  // `pushed-*` run is the only end that does not salvage: its deliverable is
1265
1514
  // already on a remote branch, whatever is left loose in the tree is by the
1266
1515
  // worker's own account not part of it, and appending a WIP commit would
1267
- // turn the green PR this daemon just verified red. Every other end is
1268
- // continuable, so its tree is treated as work.
1516
+ // turn the green PR this daemon just verified red. Every other end may
1517
+ // contain work, so it is salvaged before the tree's final fate is decided.
1269
1518
  const settlement =
1270
1519
  state === "pushed-green" || state === "pushed-pending" || state === "merged"
1271
1520
  ? undefined
@@ -1273,7 +1522,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1273
1522
  issue,
1274
1523
  attempt,
1275
1524
  ending:
1276
- state === "blocked" ? "blocked for an operator decision" : endedBy(result.killedBy),
1525
+ state === "blocked"
1526
+ ? "blocked for an operator decision"
1527
+ : state === "stopped"
1528
+ ? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
1529
+ : endedBy(result.killedBy),
1277
1530
  worktree: worktreePath,
1278
1531
  branch,
1279
1532
  publish,
@@ -1294,8 +1547,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1294
1547
  await removeWorktree(mirrorPath, worktreePath);
1295
1548
  }
1296
1549
 
1297
- store.updateRun(runId, {
1298
- state,
1550
+ const terminalPatch: Partial<RunRecord> = {
1299
1551
  endedAt: Date.now(),
1300
1552
  turns: result.turns,
1301
1553
  spendUsd: result.spendUsd,
@@ -1303,23 +1555,41 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1303
1555
  headSha: result.headSha,
1304
1556
  sessionFile: result.sessionFile,
1305
1557
  // Every terminal state persists the worker's report, not just a green
1306
- // push: the report of a killed attempt is exactly the one a later
1307
- // continuation must pool its disclosures from (#199).
1558
+ // push: a stopped attempt's partial report is still part of its audit trail.
1308
1559
  report: result.report,
1309
- ...(providerCredit !== undefined || providerTransient !== undefined
1310
- ? { lastError: providerCredit ?? providerTransient }
1311
- : verified.reason === undefined
1312
- ? {}
1313
- : { lastError: verified.reason }),
1314
1560
  ...settlement?.patch,
1315
1561
  ...(audit === undefined || audit.flags.length === 0
1316
1562
  ? {}
1317
1563
  : { settlementFlags: audit.flags }),
1318
- });
1564
+ };
1565
+ if (state === "stopped") {
1566
+ recordOperatorStop(store, {
1567
+ project: project.name,
1568
+ issue,
1569
+ runId,
1570
+ inProgress,
1571
+ reason: result.stoppedReason ?? "no reason recorded",
1572
+ patch: terminalPatch,
1573
+ });
1574
+ } else {
1575
+ const lastError = completionLastError(
1576
+ providerCredit,
1577
+ providerTransient,
1578
+ verified.reason,
1579
+ sessionErr,
1580
+ );
1581
+ store.updateRun(runId, {
1582
+ ...terminalPatch,
1583
+ state,
1584
+ ...(lastError === undefined ? {} : { lastError }),
1585
+ });
1586
+ }
1319
1587
 
1320
1588
  const salvaged = settlement?.lines ?? [];
1321
1589
 
1322
- if (state === "blocked") {
1590
+ if (state === "stopped") {
1591
+ log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
1592
+ } else if (state === "blocked") {
1323
1593
  swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
1324
1594
  await safeEscalate(d, {
1325
1595
  tier: 1,
@@ -1343,31 +1613,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1343
1613
  });
1344
1614
 
1345
1615
  if (providerCredit !== undefined) {
1346
- // Pause here rather than at classification: the sweep runs on the tick,
1347
- // and three issues each burned an attempt in the fifteen minutes between
1348
- // the first 402 and a human noticing (#220).
1349
- setPaused(true, { source: "provider-credit", reason: providerCredit });
1350
- log(`#${issue} provider refused for credit — dispatch paused: ${providerCredit}`);
1351
- swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
1352
- // Fleet-scoped and run-independent on purpose. The notification ledger
1353
- // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
1354
- // carrying no run or attempt is what makes this page once for the fleet
1355
- // instead of once per affected run.
1356
- await safeEscalate(d, {
1357
- tier: 2,
1358
- category: "fleet-stopped",
1359
- project: project.name,
1360
- issue: NO_ISSUE,
1361
- summary: `Model provider refused for credit — ${project.name} is paused`,
1362
- detail: [
1363
- providerCredit,
1364
- "",
1365
- "No implementation attempt was charged: this is a billing state, not a",
1366
- "failed implementation. Each affected issue keeps its queue label and",
1367
- "re-dispatches on `omp-conductor resume` once the provider has credit.",
1368
- `Session: ${result.sessionFile ?? "(no transcript)"}`,
1369
- ].join("\n"),
1370
- });
1616
+ await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
1617
+ swapToQueue(d, issue, inProgress);
1371
1618
  } else if (continueTurns) {
1372
1619
  // Requeue as one ordered pair: the in-progress removal before the
1373
1620
  // queue add, exactly the order the projector will apply them in (#201).
@@ -1469,8 +1716,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1469
1716
  // inner settlement guard exists. Latch it before any terminal write or await.
1470
1717
  turnLimit?.close();
1471
1718
  turnLimit = undefined;
1472
- workerControl?.close();
1473
- workerControl = undefined;
1719
+ if (await settleStopBeforeSession()) return;
1474
1720
  const detail = errText(err);
1475
1721
  log(`#${issue} errored: ${detail}`);
1476
1722
  // A crash lands anywhere, including mid-edit in a tree holding the only
@@ -1516,8 +1762,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1516
1762
  // a failure path, and whatever it still held is now a commit on the branch.
1517
1763
  } finally {
1518
1764
  turnLimit?.close();
1519
- workerControl?.close();
1520
- workerControl = undefined;
1521
1765
  // The run is over, so its channel is too. Closed here rather than beside
1522
1766
  // the session so the crash path closes it as well: a listener left bound
1523
1767
  // after its run settled is a socket whose `run-not-live` check is the only
@@ -1530,6 +1774,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1530
1774
  }
1531
1775
  verbListener = undefined;
1532
1776
  }
1777
+ workerControl?.close();
1778
+ workerControl = undefined;
1533
1779
  }
1534
1780
  }
1535
1781
 
@@ -1647,6 +1893,129 @@ export function releaseInProgress(
1647
1893
  * in the busy set throughout, so no second worker can be sent at the issue while
1648
1894
  * it waits.
1649
1895
  */
1896
+ const BASE_CHECK_BATCH = 20;
1897
+ const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
1898
+ const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
1899
+
1900
+ const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
1901
+
1902
+ const FAILING_WORKFLOW_CONCLUSIONS = new Set([
1903
+ "failure",
1904
+ "cancelled",
1905
+ "timed_out",
1906
+ "action_required",
1907
+ "startup_failure",
1908
+ "stale",
1909
+ ]);
1910
+
1911
+ function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
1912
+ const flags = run.settlementFlags ?? [];
1913
+ return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
1914
+ ? flags
1915
+ : [...flags, flag];
1916
+ }
1917
+
1918
+ /**
1919
+ * Observe Actions on exact merge commits for up to one day. A running workflow
1920
+ * stays quiet and pending; a failure becomes durable evidence on the merged row
1921
+ * and pages exactly once because the row leaves `pending` before delivery.
1922
+ */
1923
+ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "store" | "escalate">): Promise<void> {
1924
+ const now = Date.now();
1925
+ for (const run of d.store.runsNeedingBaseCheck(d.project.name, BASE_CHECK_BATCH)) {
1926
+ if (
1927
+ run.endedAt === undefined ||
1928
+ now - run.endedAt > BASE_CHECK_WINDOW_MS ||
1929
+ run.mergeSha === undefined ||
1930
+ run.baseRef === undefined
1931
+ ) {
1932
+ d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
1933
+ log(`#${run.issue} base check unknown: merge identity is absent or older than 24h`);
1934
+ continue;
1935
+ }
1936
+
1937
+ const repo = d.project.routing.repos[run.repo];
1938
+ const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
1939
+ if (repoIdentity === undefined) {
1940
+ d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
1941
+ log(`#${run.issue} base check unknown: routed repository ${run.repo} has no GitHub identity`);
1942
+ continue;
1943
+ }
1944
+
1945
+ let workflows;
1946
+ try {
1947
+ workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha);
1948
+ } catch (err) {
1949
+ log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
1950
+ continue;
1951
+ }
1952
+ if (workflows === undefined || workflows.length === 0) {
1953
+ log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
1954
+ continue;
1955
+ }
1956
+ if (workflows.some((workflow) => workflow.status !== "completed")) continue;
1957
+
1958
+ const failed = workflows.find(
1959
+ (workflow) =>
1960
+ workflow.conclusion !== undefined &&
1961
+ FAILING_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
1962
+ );
1963
+ if (failed === undefined) {
1964
+ const unknown = workflows.find(
1965
+ (workflow) =>
1966
+ workflow.conclusion === undefined ||
1967
+ !SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
1968
+ );
1969
+ if (unknown !== undefined) {
1970
+ log(
1971
+ `#${run.issue} base check has unrecognised completed conclusion ` +
1972
+ `${JSON.stringify(unknown.conclusion)} for ${unknown.name} — retrying next tick`,
1973
+ );
1974
+ continue;
1975
+ }
1976
+ d.store.updateRun(run.id, { baseCheck: "green", baseCheckAt: now });
1977
+ log(`#${run.issue} base ${run.baseRef} green at ${run.mergeSha}`);
1978
+ continue;
1979
+ }
1980
+
1981
+ let previous;
1982
+ try {
1983
+ previous = await d.tracker.previousWorkflowRun(
1984
+ repoIdentity,
1985
+ failed.workflowId,
1986
+ run.baseRef,
1987
+ failed.createdAt,
1988
+ );
1989
+ } catch (err) {
1990
+ log(`#${run.issue} previous ${failed.name} run unavailable (${errText(err)}) — retrying next tick`);
1991
+ continue;
1992
+ }
1993
+ if (previous === undefined || (previous !== null && previous.status !== "completed")) continue;
1994
+ const preexisting =
1995
+ previous !== null &&
1996
+ previous.conclusion !== undefined &&
1997
+ FAILING_WORKFLOW_CONCLUSIONS.has(previous.conclusion);
1998
+ const detail =
1999
+ `${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
2000
+ (preexisting ? " (already red before this merge)" : "");
2001
+ const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
2002
+ const delivered = await safeEscalate(d, {
2003
+ tier: 1,
2004
+ project: d.project.name,
2005
+ issue: run.issue,
2006
+ runId: run.id,
2007
+ summary: `Base branch ${run.baseRef} is red after merge`,
2008
+ detail,
2009
+ });
2010
+ if (!delivered) continue;
2011
+ d.store.updateRun(run.id, {
2012
+ baseCheck: preexisting ? "red-preexisting" : "red",
2013
+ baseCheckAt: now,
2014
+ settlementFlags: appendSettlementFlag(run, flag),
2015
+ });
2016
+ }
2017
+ }
2018
+
1650
2019
  export async function settlePushedGreen(
1651
2020
  d: Pick<Deps, "project" | "tracker" | "store">,
1652
2021
  ): Promise<void> {
@@ -1678,16 +2047,39 @@ export async function settlePushedGreen(
1678
2047
 
1679
2048
  const settlement = settlementFor(pr, run.prUrl);
1680
2049
  if (settlement !== undefined) {
2050
+ // A mediated merge enters a second, bounded observation phase. Record the
2051
+ // exact merge commit before the row leaves the active set; if GitHub cannot
2052
+ // supply it yet, retry this settlement next tick rather than create a
2053
+ // merged row whose base result can never be attributed.
2054
+ let merged: MergedPrInfo | undefined;
2055
+ if (settlement.state === "merged") {
2056
+ try {
2057
+ merged = await tracker.mergedPrInfo(run.prUrl);
2058
+ } catch (err) {
2059
+ log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
2060
+ continue;
2061
+ }
2062
+ if (merged === undefined) {
2063
+ log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
2064
+ continue;
2065
+ }
2066
+ }
2067
+
1681
2068
  // The label removal and the terminal row are one fact again (#201): the
1682
2069
  // release is enqueued — a durable local write that cannot fail on the
1683
- // tracker — in the same breath as the row is terminalised, so there is
1684
- // no window in which a row beyond every later tick still owes its label.
1685
- // The projector applies it with retry; while pending, the eligibility
1686
- // overlay treats the label as already gone, so #18's
1687
- // permanent-`agent:in-progress` cannot re-form even when GitHub refuses
1688
- // the write.
2070
+ // tracker — in the same breath as the row is terminalised.
1689
2071
  releaseInProgress(d, run.issue, settlement.reason);
1690
- const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
2072
+ const patch: Partial<RunRecord> = {
2073
+ state: settlement.state,
2074
+ endedAt: Date.now(),
2075
+ ...(merged === undefined
2076
+ ? {}
2077
+ : {
2078
+ mergeSha: merged.mergeSha,
2079
+ baseRef: merged.baseRef,
2080
+ baseCheck: "pending",
2081
+ }),
2082
+ };
1691
2083
  if (settlement.state === "failed") patch.lastError = settlement.reason;
1692
2084
  store.updateRun(run.id, patch);
1693
2085
  log(`#${run.issue} settled: ${settlement.reason}`);
@@ -1720,6 +2112,121 @@ export async function settlePushedGreen(
1720
2112
  }
1721
2113
  }
1722
2114
 
2115
+ const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
2116
+ failed: true,
2117
+ killed: true,
2118
+ orphaned: true,
2119
+ blocked: true,
2120
+ };
2121
+
2122
+ /**
2123
+ * Reattaches a recovered PR to the terminal run that owns it (#245).
2124
+ *
2125
+ * A worker can fail before its completion report records `prUrl`, then have its
2126
+ * dirty tree committed and pushed by salvage. If that branch already has a PR,
2127
+ * the orchestrator otherwise has no policy-compliant path to inspect or merge
2128
+ * it: ownership is store-backed. Adoption is deliberately stricter than
2129
+ * admission. The tracker query proves the PR closes this run's issue; exact
2130
+ * branch and canonical repository matches prove it is this run's recovered
2131
+ * work, not an unrelated closer. Missing identity is refusal, never a guess.
2132
+ *
2133
+ * The newest run per issue is inspected, at most ten per tick and only inside
2134
+ * the same 30-day window as mediated PR verbs. The cursor advances through the
2135
+ * full eligible set so persistent non-matches cannot starve older recovered
2136
+ * work. Successful adoption is idempotent because the row gains `prUrl`;
2137
+ * non-matches are logged once per daemon process.
2138
+ */
2139
+ const rejectedSalvagedPrRuns = new Set<string>();
2140
+ const salvagedPrCursor = new Map<string, string>();
2141
+
2142
+ export async function adoptSalvagedPrs(
2143
+ d: Pick<Deps, "project" | "tracker" | "store">,
2144
+ now = Date.now(),
2145
+ ): Promise<void> {
2146
+ const { project, tracker, store } = d;
2147
+ const eligible = store
2148
+ .recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
2149
+ .filter(
2150
+ (run) =>
2151
+ ADOPTABLE_PR_STATES[run.state] === true &&
2152
+ run.prUrl === undefined &&
2153
+ run.branch.trim() !== "",
2154
+ );
2155
+ const previous = salvagedPrCursor.get(project.name);
2156
+ const previousIndex =
2157
+ previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
2158
+ const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
2159
+ const candidates = Array.from(
2160
+ { length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
2161
+ (_, offset) => eligible[(start + offset) % eligible.length]!,
2162
+ );
2163
+ const last = candidates.at(-1);
2164
+ if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
2165
+
2166
+ for (const run of candidates) {
2167
+ const repo = project.routing.repos[run.repo];
2168
+ const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
2169
+ let closers: OpenCloser[];
2170
+ try {
2171
+ closers = await tracker.openClosersFor(run.issue);
2172
+ } catch (err) {
2173
+ log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
2174
+ continue;
2175
+ }
2176
+
2177
+ const closer = closers.find(
2178
+ (candidate) =>
2179
+ candidate.headRefName !== "" &&
2180
+ candidate.headRefName === run.branch &&
2181
+ repo !== undefined &&
2182
+ candidate.repo !== "" &&
2183
+ candidate.repo === repoIdentity,
2184
+ );
2185
+ if (closer === undefined) {
2186
+ if (!rejectedSalvagedPrRuns.has(run.id)) {
2187
+ const observed = closers[0];
2188
+ const reason =
2189
+ observed === undefined
2190
+ ? "no open closing PR"
2191
+ : observed.headRefName === ""
2192
+ ? "closer has no head branch identity"
2193
+ : observed.headRefName !== run.branch
2194
+ ? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
2195
+ : observed.repo === ""
2196
+ ? "closer has no repository identity"
2197
+ : `closer repository ${observed.repo} does not match routed repository`;
2198
+ log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
2199
+ rejectedSalvagedPrRuns.add(run.id);
2200
+ }
2201
+ continue;
2202
+ }
2203
+
2204
+ const flag: SettlementFlag = {
2205
+ kind: "pr-adopted",
2206
+ file: "(recovery)",
2207
+ detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
2208
+ };
2209
+ store.updateRun(run.id, {
2210
+ prUrl: closer.url,
2211
+ settlementFlags: [...(run.settlementFlags ?? []), flag],
2212
+ });
2213
+ rejectedSalvagedPrRuns.delete(run.id);
2214
+ log(
2215
+ `#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
2216
+ (run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
2217
+ );
2218
+ }
2219
+ }
2220
+
2221
+ /** Canonical `owner/repo` identity from a configured network clone URL. */
2222
+ function githubRepo(cloneUrl: string): string | undefined {
2223
+ const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
2224
+ const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
2225
+ normalized,
2226
+ );
2227
+ return match?.[1];
2228
+ }
2229
+
1723
2230
  const RETAINED_CLEANUP_BATCH = 10;
1724
2231
 
1725
2232
  export interface RetainedCleanupCursor {
@@ -1833,6 +2340,7 @@ const HOLD_SAMPLE_SIZE = 5;
1833
2340
  const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
1834
2341
  "parent-lookup-error",
1835
2342
  "open-pr-lookup-error",
2343
+ "issue-state-lookup-error",
1836
2344
  ]);
1837
2345
 
1838
2346
  /** Groups transient decisions into the bounded record exposed by status. */
@@ -2217,6 +2725,32 @@ export async function admitCandidates(
2217
2725
  log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
2218
2726
  }
2219
2727
 
2728
+ // The queue comes from GitHub's eventually-consistent search index. Re-read
2729
+ // state and labels directly at the last possible moment so a just-closed or
2730
+ // explicitly dequeued issue cannot turn a stale candidate into another
2731
+ // attempt (#247).
2732
+ let snapshot: IssueSnapshot | undefined;
2733
+ try {
2734
+ snapshot = await tracker.issueSnapshot(issue);
2735
+ } catch {
2736
+ snapshot = undefined;
2737
+ }
2738
+ if (snapshot === undefined) {
2739
+ hold(issue, "issue-state-lookup-error");
2740
+ log(`#${issue} held: issue snapshot check failed — retrying next tick`);
2741
+ continue;
2742
+ }
2743
+ if (snapshot.state === "closed") {
2744
+ hold(issue, "issue-closed");
2745
+ log(`#${issue} skipped: issue is closed (search index lag)`);
2746
+ continue;
2747
+ }
2748
+ if (!snapshot.labels.includes(project.queueLabel)) {
2749
+ hold(issue, "issue-dequeued");
2750
+ log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
2751
+ continue;
2752
+ }
2753
+
2220
2754
  admitted.push({ r, attempt: priorRuns + 1 });
2221
2755
  liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
2222
2756
  if (parent !== undefined) {
@@ -2305,6 +2839,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2305
2839
  // while the fleet is parked or workers are still active. Resident workers run
2306
2840
  // through the pool without blocking this five-minute tick.
2307
2841
  await settlePushedGreen(d);
2842
+ try {
2843
+ await watchMergedBase(d);
2844
+ } catch (err) {
2845
+ log(`base-branch check sweep failed: ${errText(err)}`);
2846
+ }
2847
+ try {
2848
+ await adoptSalvagedPrs(d);
2849
+ } catch (err) {
2850
+ log(`salvaged PR adoption sweep failed: ${errText(err)}`);
2851
+ }
2308
2852
 
2309
2853
  // Immediately after settlement and before any routing, so a class is on the
2310
2854
  // row before the next dispatch decision reads its budgets (#132). Above the
@@ -2548,6 +3092,8 @@ export interface DaemonHealthSnapshot {
2548
3092
  paused: boolean;
2549
3093
  activeRuns: number;
2550
3094
  project: string;
3095
+ /** One-shot issue ceilings waiting for the next claim. */
3096
+ turnOverrides: TurnOverride[];
2551
3097
  /** Resident set of this daemon; workers are in-process omp sessions. */
2552
3098
  rssBytes: number;
2553
3099
  dispatch?: DispatchSummary;
@@ -2569,6 +3115,7 @@ export function daemonHealthSnapshot(
2569
3115
  ok: true,
2570
3116
  paused,
2571
3117
  activeRuns: store.activeRuns(project).length,
3118
+ turnOverrides: store.listTurnOverrides(project),
2572
3119
  project,
2573
3120
  rssBytes,
2574
3121
  ...(dispatch === undefined ? {} : { dispatch }),
@@ -2577,11 +3124,19 @@ export function daemonHealthSnapshot(
2577
3124
  };
2578
3125
  }
2579
3126
 
3127
+ const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
3128
+ "failed",
3129
+ "killed",
3130
+ "orphaned",
3131
+ "blocked",
3132
+ ]);
3133
+
2580
3134
  export async function turnLimitResponse(
2581
3135
  req: Request,
2582
3136
  project: string,
2583
- store: Pick<Store, "latestRun">,
3137
+ store: Pick<Store, "latestRun" | "setTurnOverride">,
2584
3138
  registry: TurnLimitRegistry,
3139
+ caps: Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">,
2585
3140
  ): Promise<Response | undefined> {
2586
3141
  const url = new URL(req.url);
2587
3142
  const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
@@ -2615,6 +3170,16 @@ export async function turnLimitResponse(
2615
3170
  }
2616
3171
 
2617
3172
  const issue = Number(match[1]);
3173
+ if ((maxTurns as number) > caps.workerMaxTurnsCeiling) {
3174
+ return Response.json(
3175
+ {
3176
+ error:
3177
+ `#${issue} turn budget ${maxTurns as number} exceeds the ` +
3178
+ `${caps.workerMaxTurnsCeiling}-turn caps.workerMaxTurnsCeiling`,
3179
+ },
3180
+ { status: 422 },
3181
+ );
3182
+ }
2618
3183
  const outcome = registry.extend(project, issue, maxTurns as number);
2619
3184
  if (outcome.kind === "extended") return Response.json(outcome);
2620
3185
  if (outcome.kind === "not-increase") {
@@ -2628,14 +3193,33 @@ export async function turnLimitResponse(
2628
3193
  if (latest === undefined) {
2629
3194
  return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
2630
3195
  }
2631
- return Response.json(
2632
- {
2633
- error:
2634
- `#${issue} has no live worker controller; its session already settled ` +
2635
- `or belongs to another daemon (stored state: ${latest.state})`,
2636
- },
2637
- { status: 409 },
2638
- );
3196
+ if (!TURN_OVERRIDE_STATES.has(latest.state)) {
3197
+ return Response.json(
3198
+ {
3199
+ error:
3200
+ `#${issue} has no live worker controller; its session already settled ` +
3201
+ `or belongs to another daemon (stored state: ${latest.state})`,
3202
+ },
3203
+ { status: 409 },
3204
+ );
3205
+ }
3206
+ if ((maxTurns as number) <= caps.workerMaxTurns) {
3207
+ return Response.json(
3208
+ {
3209
+ error:
3210
+ `#${issue} next-attempt turn budget must exceed the ` +
3211
+ `${caps.workerMaxTurns}-turn caps.workerMaxTurns base`,
3212
+ },
3213
+ { status: 409 },
3214
+ );
3215
+ }
3216
+ store.setTurnOverride(project, issue, maxTurns as number);
3217
+ return Response.json({
3218
+ kind: "next-attempt",
3219
+ issue,
3220
+ nextAttemptMaxTurns: maxTurns,
3221
+ baseMaxTurns: caps.workerMaxTurns,
3222
+ });
2639
3223
  }
2640
3224
 
2641
3225
  export async function workerControlResponse(
@@ -2645,7 +3229,7 @@ export async function workerControlResponse(
2645
3229
  registry: WorkerControlRegistry,
2646
3230
  ): Promise<Response | undefined> {
2647
3231
  const url = new URL(req.url);
2648
- const match = /^\/runs\/(\d+)\/(pause|resume)$/.exec(url.pathname);
3232
+ const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
2649
3233
  if (req.method !== "PUT" || match === null) return undefined;
2650
3234
  if (!req.headers.get("content-type")?.startsWith("application/json")) {
2651
3235
  return Response.json({ error: "content-type must be application/json" }, { status: 415 });
@@ -2670,23 +3254,71 @@ export async function workerControlResponse(
2670
3254
  { status: 409 },
2671
3255
  );
2672
3256
  }
3257
+ const action = match[2] as "pause" | "resume" | "stop";
3258
+ let reason: string | undefined;
3259
+ if (action === "stop") {
3260
+ const rawReason = Reflect.get(body, "reason");
3261
+ if (typeof rawReason !== "string" || rawReason.trim() === "") {
3262
+ return Response.json({ error: "reason must be a non-empty string" }, { status: 400 });
3263
+ }
3264
+ reason = rawReason.trim().replace(/\s+/g, " ");
3265
+ if (reason.length > 500) {
3266
+ return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
3267
+ }
3268
+ }
3269
+
2673
3270
 
2674
3271
  const issue = Number(match[1]);
2675
3272
  const outcome =
2676
- match[2] === "pause"
3273
+ action === "pause"
2677
3274
  ? await registry.pause(project, issue)
2678
- : registry.resume(project, issue);
3275
+ : action === "resume"
3276
+ ? registry.resume(project, issue)
3277
+ : await registry.stop(project, issue, reason!);
2679
3278
  if (outcome.kind === "ok") {
2680
3279
  return Response.json({ runId: outcome.runId, phase: outcome.phase });
2681
3280
  }
2682
3281
  if (outcome.kind === "refused") {
2683
3282
  return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
2684
3283
  }
3284
+ if (outcome.kind === "stopped") {
3285
+ const stopped = store.latestRun(project, issue);
3286
+ if (stopped === undefined || stopped.id !== outcome.runId) {
3287
+ return Response.json(
3288
+ { error: `#${issue} stopped, but its terminal run record is unavailable` },
3289
+ { status: 500 },
3290
+ );
3291
+ }
3292
+ if (stopped.state !== "stopped") {
3293
+ return Response.json({
3294
+ outcome: "already-terminal",
3295
+ runId: stopped.id,
3296
+ state: stopped.state,
3297
+ });
3298
+ }
3299
+ return Response.json({
3300
+ outcome: "stopped",
3301
+ runId: stopped.id,
3302
+ state: stopped.state,
3303
+ reason: outcome.reason,
3304
+ ...(stopped.salvageSha === undefined ? {} : { salvageSha: stopped.salvageSha }),
3305
+ ...(stopped.salvageError === undefined ? {} : { salvageError: stopped.salvageError }),
3306
+ worktree: stopped.worktree,
3307
+ });
3308
+ }
3309
+
2685
3310
 
2686
3311
  const latest = store.latestRun(project, issue);
2687
3312
  if (latest === undefined) {
2688
3313
  return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
2689
3314
  }
3315
+ if (action === "stop" && !LIVE_STATES.includes(latest.state)) {
3316
+ return Response.json({
3317
+ outcome: "already-terminal",
3318
+ runId: latest.id,
3319
+ state: latest.state,
3320
+ });
3321
+ }
2690
3322
  return Response.json(
2691
3323
  {
2692
3324
  error:
@@ -2699,7 +3331,8 @@ export async function workerControlResponse(
2699
3331
 
2700
3332
  export interface DaemonHttpDeps {
2701
3333
  project: string;
2702
- store: Pick<Store, "latestRun">;
3334
+ store: Pick<Store, "latestRun" | "setTurnOverride">;
3335
+ caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
2703
3336
  turnLimits: TurnLimitRegistry;
2704
3337
  workerControls: WorkerControlRegistry;
2705
3338
  health: () => DaemonHealthSnapshot;
@@ -2709,15 +3342,15 @@ export interface DaemonHttpDeps {
2709
3342
  * The whole HTTP surface, in one named function so a test can pin what is *not*
2710
3343
  * on it.
2711
3344
  *
2712
- * Three route families: the health read, turn-limit control, and worker
2713
- * pause/resume control. Everything else is 404. The controls mutate only
3345
+ * Three route families: the health read, turn-limit control, and live-worker
3346
+ * pause/resume/stop control. Everything else is 404. The controls mutate only
2714
3347
  * daemon-owned live sessions; tracker and repository mutations stay on the
2715
3348
  * authenticated per-run channel described at the `Bun.serve` call (#126).
2716
3349
  */
2717
3350
  export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
2718
3351
  const url = new URL(req.url);
2719
3352
  if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
2720
- const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
3353
+ const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits, d.caps());
2721
3354
  if (turnLimit !== undefined) return turnLimit;
2722
3355
  const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
2723
3356
  return workerControl ?? new Response("not found\n", { status: 404 });
@@ -2747,6 +3380,8 @@ export interface StatusSnapshot {
2747
3380
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
2748
3381
  * only copy of work the daemon could not save. */
2749
3382
  salvagedRuns: RunRecord[];
3383
+ /** One-shot issue ceilings waiting for the next claim. */
3384
+ turnOverrides: TurnOverride[];
2750
3385
  /** Reports the operator has not provably received: pending, in-flight with an
2751
3386
  * unknown outcome, or written off. An empty list is the only honest way to
2752
3387
  * say "everything authored this cycle actually went out" (#123). */
@@ -2758,6 +3393,8 @@ export interface StatusSnapshot {
2758
3393
  * config does not let it, and an operator who has to know to go looking is an
2759
3394
  * operator who finds out from the tracker instead.
2760
3395
  */
3396
+ /** Newest post-merge base verdict per routed repository within seven days. */
3397
+ baseChecks: RunRecord[];
2761
3398
  verbLedger: VerbLedgerEntry[];
2762
3399
  /** Runs backed by a worker process — the number capacity compares against. */
2763
3400
  liveWorkers: number;
@@ -2821,6 +3458,7 @@ export function statusSnapshotFromStore(
2821
3458
  releaseGrants: resolveReleaseGrants(p),
2822
3459
  activeRuns: store.activeRuns(p.name),
2823
3460
  salvagedRuns: store.salvagedRuns(p.name),
3461
+ turnOverrides: store.listTurnOverrides(p.name),
2824
3462
  openReports: store.openReports(p.name),
2825
3463
  verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
2826
3464
  liveWorkers: store.liveRuns(p.name).length,
@@ -2835,6 +3473,7 @@ export function statusSnapshotFromStore(
2835
3473
  ...(labelOpsPending === 0 || oldestLabelOpAt === undefined
2836
3474
  ? {}
2837
3475
  : { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
3476
+ baseChecks: store.latestBaseChecks(p.name, Date.now() - BASE_STATUS_WINDOW_MS),
2838
3477
  };
2839
3478
  }
2840
3479
 
@@ -2917,6 +3556,21 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
2917
3556
  ...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
2918
3557
  ];
2919
3558
  }
3559
+ export function formatBaseChecks(runs: readonly RunRecord[]): string[] {
3560
+ return runs.flatMap((run) => {
3561
+ if (run.baseCheck === undefined) return [];
3562
+ const branch = run.baseRef ?? "?";
3563
+ const flag = run.settlementFlags?.find((candidate) => candidate.kind === "base-branch-red");
3564
+ const verdict =
3565
+ run.baseCheck === "red"
3566
+ ? `RED — ${flag?.detail ?? "workflow failed after merge"}`
3567
+ : run.baseCheck === "red-preexisting"
3568
+ ? `red before merge${flag === undefined ? "" : ` — ${flag.detail}`}`
3569
+ : run.baseCheck;
3570
+ return [`base ${run.repo}/${branch} ${verdict}`];
3571
+ });
3572
+ }
3573
+
2920
3574
 
2921
3575
  export function formatStatus(s: StatusSnapshot): string {
2922
3576
  const lines = [
@@ -2935,6 +3589,9 @@ export function formatStatus(s: StatusSnapshot): string {
2935
3589
  // fleet (#110).
2936
3590
  ` plan usage ${planUsageLine(s.planUsage)}`,
2937
3591
  ` new worker turns ${s.caps.workerMaxTurns}`,
3592
+ ...s.turnOverrides.map(
3593
+ ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
3594
+ ),
2938
3595
  ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
2939
3596
  ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
2940
3597
  ` continuations ${s.caps.maxContinuationsPerIssue}`,
@@ -2961,6 +3618,7 @@ export function formatStatus(s: StatusSnapshot): string {
2961
3618
  if (flagged !== undefined) lines.push(` ${flagged}`);
2962
3619
  }
2963
3620
  }
3621
+ lines.push(...formatBaseChecks(s.baseChecks));
2964
3622
  lines.push(...formatSalvagedRuns(s.salvagedRuns));
2965
3623
  lines.push(...formatOpenReports(s.openReports));
2966
3624
  lines.push(...formatVerbLedger(s.verbLedger));
@@ -3096,6 +3754,15 @@ export interface SessionError {
3096
3754
  message: string;
3097
3755
  }
3098
3756
 
3757
+ export function completionLastError(
3758
+ providerCredit: string | undefined,
3759
+ providerTransient: string | undefined,
3760
+ verifiedReason: string | undefined,
3761
+ sessionErr: SessionError | undefined,
3762
+ ): string | undefined {
3763
+ return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
3764
+ }
3765
+
3099
3766
  /**
3100
3767
  * The last error a transcript recorded, or undefined when it recorded none.
3101
3768
  *
@@ -3162,6 +3829,19 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3162
3829
  const { project, tracker, store } = d;
3163
3830
  for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
3164
3831
  const facts: ClassifyFacts = {};
3832
+ let classifiedRun = run;
3833
+ if (run.state === "failed" || run.state === "killed") {
3834
+ const sessionError = readSessionError(run.sessionFile);
3835
+ if (sessionError !== undefined) {
3836
+ if (run.lastError === undefined || run.lastError === sessionError.message) {
3837
+ facts.sessionError = sessionError;
3838
+ }
3839
+ if (run.lastError === undefined) {
3840
+ store.updateRun(run.id, { lastError: sessionError.message });
3841
+ classifiedRun = { ...run, lastError: sessionError.message };
3842
+ }
3843
+ }
3844
+ }
3165
3845
  try {
3166
3846
  if (run.prUrl !== undefined) {
3167
3847
  const pr = await tracker.prState(run.prUrl);
@@ -3188,7 +3868,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3188
3868
  continue;
3189
3869
  }
3190
3870
 
3191
- const { cls, recovery, evidence } = classifyRun(run, facts);
3871
+ const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
3192
3872
 
3193
3873
  // A healthy green PR is not a failure of any class. Leaving the row
3194
3874
  // unclassified is what keeps it eligible for the sweep on the tick where its
@@ -3202,7 +3882,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3202
3882
  ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
3203
3883
  : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
3204
3884
  );
3205
- await recoverRun(d, run, cls, recovery, evidence);
3885
+ await recoverRun(d, classifiedRun, cls, recovery, evidence);
3206
3886
  }
3207
3887
  }
3208
3888
 
@@ -3268,6 +3948,9 @@ async function recoverRun(
3268
3948
  }
3269
3949
 
3270
3950
  if (recovery === "requeue") {
3951
+ if (cls === "provider-credit") {
3952
+ await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
3953
+ }
3271
3954
  // A dispatch-infra requeue that keeps landing on the same issue means the
3272
3955
  // mirror for its repo is persistently broken — a ref-lock that retry already
3273
3956
  // exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
@@ -3682,7 +4365,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3682
4365
  onChildLog: (line) => {
3683
4366
  log(`orchestrator ${line}`);
3684
4367
  },
3685
- onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
4368
+ onReleaseBlocked: (shape, context) =>
4369
+ recordReleaseBlock(project.name, "orchestrator", shape, context),
3686
4370
  });
3687
4371
  const transcript = orchestrator.sessionFile();
3688
4372
  log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
@@ -3821,9 +4505,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3821
4505
  // carries no credential of any kind, and it cannot tell one caller from
3822
4506
  // another: `127.0.0.1` is not an identity. The turn-limit and worker
3823
4507
  // pause/resume controls trust a body-supplied `project`, which is exactly the
3824
- // shape "identity from the payload" takes when nobody is watching. Those
3825
- // controls are tolerable only because they are bounded, live-run-local, and
3826
- // reversible.
4508
+ // shape "identity from the payload" takes when nobody is watching. They stay
4509
+ // tolerable only because every effect is bounded: pauses are reversible,
4510
+ // live extensions touch one controller, and a persisted next-attempt
4511
+ // override can only raise the project base up to its configured ceiling and
4512
+ // is consumed by one claim.
3827
4513
  //
3828
4514
  // A merge, a push, a release or a label is none of those things. Do not add
3829
4515
  // one here, and do not add "just a small one" behind a shared secret either:
@@ -3838,6 +4524,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3838
4524
  daemonHttpResponse(req, {
3839
4525
  project: project.name,
3840
4526
  store,
4527
+ caps: () => d.caps,
3841
4528
  turnLimits,
3842
4529
  workerControls,
3843
4530
  health: () =>