omp-conductor 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -18,11 +18,13 @@ import {
18
18
  resolveReleaseGrants,
19
19
  stateDir,
20
20
  } from "./config.ts";
21
+ import { availabilityState, type AvailabilityState } from "./availability.ts";
21
22
  import {
22
23
  analyseSettlement,
23
24
  formatSettlementFlags,
24
25
  settlementFlagSummary,
25
26
  } from "./diff-flags.ts";
27
+ import { digestScheduleState, type DigestScheduleState } from "./digest-schedule.ts";
26
28
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
27
29
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
28
30
  import { graphHint } from "./graph.ts";
@@ -30,22 +32,34 @@ import { livingDaemon } from "./lifecycle.ts";
30
32
  import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
31
33
  import { startOrchestrator } from "./orchestrator.ts";
32
34
  import type { OrchestratorHandle } from "./orchestrator.ts";
33
- import { createReportOutbox, formatOpenReports } from "./reports.ts";
34
- import { recordReleaseBlock } from "./release-policy.ts";
35
+ import {
36
+ createReportOutbox,
37
+ enqueueAvailableHeldNotices,
38
+ formatOpenReports,
39
+ } from "./reports.ts";
40
+ import {
41
+ recordReleaseBlock,
42
+ type ReleaseBlockContext,
43
+ } from "./release-policy.ts";
35
44
  import { branchName, effectiveLabels, route } from "./routing.ts";
36
45
  import type { Routed, UnroutableReason } from "./routing.ts";
37
46
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
38
47
  import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
39
48
  import { projectLabels } from "./label-projection.ts";
40
- import { dbPath, openStore, utcDay } from "./store.ts";
49
+ import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
41
50
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
42
- import { RELEASE_SHAPES } from "./types.ts";
51
+ import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
43
52
  import type {
53
+ BaseHealth,
44
54
  AdmissionHoldReason,
45
55
  Caps,
46
56
  DispatchSummary,
57
+ DigestBacklog,
47
58
  Escalation,
59
+ IssueSnapshot,
60
+ MergedPrInfo,
48
61
  OpenCloser,
62
+ ReleaseShape,
49
63
  PrState,
50
64
  ProjectConfig,
51
65
  ReadyIssue,
@@ -60,12 +74,14 @@ import type {
60
74
  Store,
61
75
  Tracker,
62
76
  VerbLedgerEntry,
77
+ TurnOverride,
63
78
  } from "./types.ts";
64
79
  import {
65
80
  type KilledBy,
66
81
  type WorkerPauseControl,
67
82
  type WorkerPausePhase,
68
83
  type WorkerResult,
84
+ type RunWorkerDeps,
69
85
  renderBrief,
70
86
  runWorker,
71
87
  } from "./worker.ts";
@@ -82,8 +98,13 @@ import {
82
98
  } from "./worktree.ts";
83
99
  import { githubVerbActions } from "./verbs/actions.ts";
84
100
  import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
85
- import { listenVerbChannel } from "./verbs/server.ts";
86
- import type { VerbActions, VerbDeps, VerbListener } from "./verbs/server.ts";
101
+ import {
102
+ listenVerbChannel,
103
+ PR_LOOKUP_WINDOW_MS,
104
+ type VerbActions,
105
+ type VerbDeps,
106
+ type VerbListener,
107
+ } from "./verbs/server.ts";
87
108
  import {
88
109
  ensureVerbSocketDir,
89
110
  peerCredentialReader,
@@ -120,6 +141,9 @@ const DISPATCH_INFRA_MAX_STRIKES = 3;
120
141
  * provider itself is degraded, not unlucky, and the sweep escalates to a
121
142
  * human instead of requeueing into a down provider forever (#220). */
122
143
  const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
144
+ /** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
145
+ * are maintenance, but a backlog must not turn one tick into an API burst. */
146
+ const SALVAGED_PR_ADOPTION_BATCH = 10;
123
147
  const DEFAULT_PORT = 8787;
124
148
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
125
149
 
@@ -151,12 +175,17 @@ interface Deps {
151
175
  caps: Caps;
152
176
  tracker: Tracker;
153
177
  store: Store;
178
+ /** False after a live config reload fails; autonomous delivery then holds
179
+ * fail-closed until a later tick validates the config again. */
180
+ deliveryPolicyValid?: boolean;
154
181
  /** Provider-reported plan allowance, cached with a TTL. Resolved once at
155
182
  * startup like every other dep so a tick cannot swap its own meter. */
156
183
  usage: UsageSource;
157
184
  escalate(e: Escalation): Promise<void>;
158
185
  turnLimits: TurnLimitRegistry;
159
186
  workerControls: WorkerControlRegistry;
187
+ /** Session seam for lifecycle integration tests; production uses the real harness. */
188
+ workerDeps?: RunWorkerDeps;
160
189
  integrity: IntegrityGate;
161
190
  stall: StallGate;
162
191
  cleanup?: RetainedCleanupCursor;
@@ -212,18 +241,25 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
212
241
 
213
242
  // ------------------------------------------------------- orchestrator liveness
214
243
 
244
+ /** No more than one reminder per hour while the same marker remains. */
245
+ const STALL_RENOTIFY_MS = 60 * 60_000;
246
+
215
247
  /**
216
- * Whether the wedged-orchestrator page has already gone out for the stall
217
- * currently on disk. One page per episode: the marker persists until a tick is
218
- * consumed, so paging per five minutes would be paging forever.
248
+ * Whether the wedged-orchestrator page has gone out, and when it last went out,
249
+ * for the stall currently on disk.
219
250
  */
220
251
  export interface StallGate {
221
252
  paged: boolean;
253
+ lastPagedAt?: number;
222
254
  }
223
255
 
224
256
  export interface StallVerdict {
225
257
  /** The marker's own line, when one is there. */
226
258
  since?: string;
259
+ /** Count written by the stalled tick producer, when its marker is readable. */
260
+ unconsumedTicks?: number;
261
+ /** Whole hours elapsed since the marker timestamp, when it is parseable. */
262
+ stalledHours?: number;
227
263
  page: boolean;
228
264
  }
229
265
 
@@ -246,51 +282,84 @@ export interface StallVerdict {
246
282
  *
247
283
  * Resets when the marker disappears, so a second stall days later pages again.
248
284
  */
249
- export function checkStall(gate: StallGate, marker: string): StallVerdict {
285
+ export function checkStall(gate: StallGate, marker: string, now = Date.now()): StallVerdict {
250
286
  if (!existsSync(marker)) {
251
287
  gate.paged = false;
288
+ delete gate.lastPagedAt;
252
289
  return { page: false };
253
290
  }
254
- const page = !gate.paged;
291
+ const page =
292
+ !gate.paged ||
293
+ gate.lastPagedAt === undefined ||
294
+ now - gate.lastPagedAt >= STALL_RENOTIFY_MS;
255
295
  let since: string | undefined;
296
+ let unconsumedTicks: number | undefined;
297
+ let stalledHours: number | undefined;
256
298
  try {
257
299
  const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
258
- if (body !== undefined && body !== "") since = body;
300
+ if (body !== undefined && body !== "") {
301
+ since = body;
302
+ const ticks = /\b(\d+) ticks queued unconsumed\b/.exec(body)?.[1];
303
+ if (ticks !== undefined) unconsumedTicks = Number(ticks);
304
+ const startedAt = Date.parse(body.split(/\s+/, 1)[0] ?? "");
305
+ if (Number.isFinite(startedAt)) {
306
+ stalledHours = Math.max(0, Math.floor((now - startedAt) / STALL_RENOTIFY_MS));
307
+ }
308
+ }
259
309
  } catch {
260
- // An unreadable marker still means stalled; the timestamp is a nicety.
310
+ // An unreadable marker still means stalled; its evidence is a nicety.
261
311
  }
262
- return { ...(since === undefined ? {} : { since }), page };
312
+ return {
313
+ ...(since === undefined ? {} : { since }),
314
+ ...(unconsumedTicks === undefined ? {} : { unconsumedTicks }),
315
+ ...(stalledHours === undefined ? {} : { stalledHours }),
316
+ page,
317
+ };
263
318
  }
264
319
 
265
320
  /**
266
- * Pages tier 2 once when the orchestrator session stops draining its queue.
321
+ * Pages tier 2 when the orchestrator session stops draining its queue.
267
322
  *
268
323
  * Deliberately does not restart anything. A wedge lands mid-turn, this process
269
324
  * cannot tell a half-applied edit from an idle loop, and killing the session
270
325
  * could destroy work an operator would rather read first — the same refusal to
271
326
  * guess that the recovery plugin is built on.
272
327
  */
273
- async function watchOrchestrator(d: Deps): Promise<void> {
328
+ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
274
329
  const marker = join(stateDir(), STALL_MARKER_FILE);
275
- const verdict = checkStall(d.stall, marker);
330
+ const repeat = d.stall.paged;
331
+ const verdict = checkStall(d.stall, marker, now);
276
332
  if (!verdict.page) return;
277
333
 
334
+ const greenRuns = d.store.activeRuns(d.project.name).filter((run) => run.state === "pushed-green");
335
+ const repeatSuffix = repeat
336
+ ? ` — still stalled (${
337
+ verdict.stalledHours === undefined
338
+ ? `${new Date(now).toISOString().slice(0, 13)}Z`
339
+ : `${verdict.stalledHours}h`
340
+ })`
341
+ : "";
342
+
278
343
  log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
279
344
  const delivered = await safeEscalate(d, {
280
345
  tier: 2,
281
346
  category: "confirmed-failure",
347
+ urgent: true,
282
348
  project: d.project.name,
283
349
  issue: NO_ISSUE,
284
350
  // Keyed on the marker's own timestamp, not the date. The dedup ledger keys
285
351
  // on this summary, and two wedges in one day is not a hypothetical — the
286
352
  // failure mode is a session that gets stuck, gets restarted, and gets stuck
287
- // again on the same cause an hour later. A day-keyed summary would report
288
- // the first and silently swallow every one after it.
353
+ // again on the same cause an hour later. The hourly suffix makes bounded
354
+ // reminders distinct without allowing every five-minute tick through.
289
355
  summary:
290
356
  `Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
291
- `it has stopped reading its queue (${d.project.name})`,
357
+ `it has stopped reading its queue (${d.project.name})${repeatSuffix}`,
292
358
  detail: [
293
359
  verdict.since ?? "Marker present with no readable timestamp.",
360
+ `Unconsumed ticks: ${verdict.unconsumedTicks ?? "unknown"}`,
361
+ `Open green worker PRs: ${greenRuns.length}`,
362
+ ...greenRuns.map((run) => `- ${run.prUrl ?? `#${run.issue} (PR URL unavailable)`}`),
294
363
  `Marker: ${marker}`,
295
364
  "",
296
365
  "Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
@@ -302,7 +371,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
302
371
  "Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
303
372
  ].join("\n"),
304
373
  });
305
- markPaged(d.stall, delivered);
374
+ markPaged(d.stall, delivered, now);
306
375
  }
307
376
 
308
377
  /**
@@ -319,10 +388,10 @@ export function isPaused(): boolean {
319
388
  * The epoch-ms timestamp at which the current pause began, read from the same
320
389
  * sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
321
390
  * `undefined` when the fleet is not paused, or when the file's first line does
322
- * not parse as a date a legacy/blank sentinel keeps today's refuse-everything
323
- * behavior, because a run admitted before an *unknown* pause cannot be proven
324
- * innocent. {@link isPaused} is the authority on *whether*; this answers
325
- * *since when*.
391
+ * not parse as a date. A legacy/blank sentinel keeps completion mutations
392
+ * fail-closed because a run admitted before an *unknown* pause cannot be proven
393
+ * innocent. {@link isPaused} is the authority on *whether*; this answers *since
394
+ * when*.
326
395
  */
327
396
  export function pausedAt(): number | undefined {
328
397
  const f = join(stateDir(), "paused");
@@ -333,8 +402,8 @@ export function pausedAt(): number | undefined {
333
402
  const t = Date.parse(first);
334
403
  return Number.isNaN(t) ? undefined : t;
335
404
  } catch {
336
- // Unreadable sentinel (permissions, corruption): fail closed like an
337
- // unparseable line refuse mutations while the pause is unprovable.
405
+ // Unreadable sentinel (permissions, corruption): fail completion mutations
406
+ // closed like an unparseable line while the pause time is unprovable.
338
407
  return undefined;
339
408
  }
340
409
  }
@@ -359,8 +428,8 @@ export function pauseProvenance(): { source: string; reason?: string } | undefin
359
428
  const reason = match[2];
360
429
  return { source, ...(reason === undefined ? {} : { reason }) };
361
430
  } catch {
362
- // Unreadable sentinel: no provenance to name, and the refuse-everything
363
- // posture of an unknown pause is unchanged.
431
+ // Unreadable sentinel: no provenance to name. Completion mutations still
432
+ // fail closed because the pause time is unknown.
364
433
  return undefined;
365
434
  }
366
435
  }
@@ -471,9 +540,17 @@ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>
471
540
  return { diff, pause: true, page: !gate.paged };
472
541
  }
473
542
 
474
- /** Latch a once-only page, after delivery is confirmed and never before. */
475
- export function markPaged(gate: { paged: boolean }, delivered: boolean): void {
476
- if (delivered) gate.paged = true;
543
+ /** Latch a page after delivery is confirmed, and never before. */
544
+ export function markPaged(gate: StallGate, delivered: boolean, now: number): void;
545
+ export function markPaged(gate: IntegrityGate, delivered: boolean): void;
546
+ export function markPaged(
547
+ gate: { paged: boolean; lastPagedAt?: number },
548
+ delivered: boolean,
549
+ now?: number,
550
+ ): void {
551
+ if (!delivered) return;
552
+ gate.paged = true;
553
+ if (now !== undefined) gate.lastPagedAt = now;
477
554
  }
478
555
 
479
556
  // ---------------------------------------------------------------------- helpers
@@ -543,6 +620,32 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
543
620
  { issue, op: "remove", label: from },
544
621
  ]);
545
622
  }
623
+ /**
624
+ * Persist the operator-stop transition before releasing its live controller.
625
+ * The row is terminal first, then its in-progress label is removed through the
626
+ * same durable projection outbox as every other lifecycle transition.
627
+ */
628
+ export function recordOperatorStop(
629
+ store: Pick<Store, "updateRun" | "enqueueLabelOps">,
630
+ args: {
631
+ project: string;
632
+ issue: number;
633
+ runId: string;
634
+ inProgress: string;
635
+ reason: string;
636
+ patch: Partial<RunRecord>;
637
+ },
638
+ ): void {
639
+ store.updateRun(args.runId, {
640
+ ...args.patch,
641
+ state: "stopped",
642
+ lastError: `operator stopped: ${args.reason}`,
643
+ });
644
+ store.enqueueLabelOps(args.project, [
645
+ { issue: args.issue, op: "remove", label: args.inProgress },
646
+ ]);
647
+ }
648
+
546
649
 
547
650
  /**
548
651
  * The escalator throws when no transport is configured or Telegram rejects, and
@@ -564,6 +667,38 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
564
667
  }
565
668
  }
566
669
 
670
+ async function reactToProviderCredit(
671
+ d: Deps,
672
+ issue: number,
673
+ message: string,
674
+ sessionFile: string | undefined,
675
+ ): Promise<void> {
676
+ const { project } = d;
677
+ const alreadyPaused = isPaused();
678
+ if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
679
+ log(
680
+ `#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
681
+ );
682
+ // Fleet-scoped and run-independent on purpose. The notification ledger
683
+ // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
684
+ // carrying no run or attempt pages once for the fleet, not once per run.
685
+ await safeEscalate(d, {
686
+ tier: 2,
687
+ category: "fleet-stopped",
688
+ project: project.name,
689
+ issue: NO_ISSUE,
690
+ summary: `Model provider refused for credit — ${project.name} is paused`,
691
+ detail: [
692
+ message,
693
+ "",
694
+ "No implementation attempt was charged: this is a billing state, not a",
695
+ "failed implementation. Each affected issue keeps its queue label and",
696
+ "re-dispatches on `omp-conductor resume` once the provider has credit.",
697
+ `Session: ${sessionFile ?? "(no transcript)"}`,
698
+ ].join("\n"),
699
+ });
700
+ }
701
+
567
702
  /**
568
703
  * What a salvage attempt contributes to the escalation: where the work went, or
569
704
  * that it went nowhere. Split from the effects below for the same reason
@@ -840,11 +975,14 @@ export function createTurnLimitRegistry(
840
975
 
841
976
  export type WorkerControlResult =
842
977
  | { kind: "ok"; runId: string; phase: WorkerPausePhase }
978
+ | { kind: "stopped"; runId: string; reason: string }
843
979
  | { kind: "refused"; runId: string; error: string }
844
980
  | { kind: "not-active" };
845
981
 
846
982
  export interface WorkerControlSlot {
847
983
  install(control: WorkerPauseControl): void;
984
+ /** Stop accepted before the session controller exists. */
985
+ requestedStop(): string | undefined;
848
986
  close(): void;
849
987
  }
850
988
 
@@ -852,34 +990,51 @@ export interface WorkerControlRegistry {
852
990
  open(project: string, issue: number, runId: string): WorkerControlSlot;
853
991
  pause(project: string, issue: number): Promise<WorkerControlResult>;
854
992
  resume(project: string, issue: number): WorkerControlResult;
993
+ stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
855
994
  /** Live runs whose phase is not `running` — what /healthz and the board show. */
856
995
  snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
857
996
  }
858
997
 
859
998
  /** Authoritative controls for sessions owned by this daemon process. */
860
999
  export function createWorkerControlRegistry(): WorkerControlRegistry {
861
- const active = new Map<
862
- string,
863
- { project: string; issue: number; runId: string; control?: WorkerPauseControl }
864
- >();
1000
+ interface Entry {
1001
+ project: string;
1002
+ issue: number;
1003
+ runId: string;
1004
+ control?: WorkerPauseControl;
1005
+ stopReason?: string;
1006
+ stopError?: string;
1007
+ finished: PromiseWithResolvers<void>;
1008
+ }
1009
+
1010
+ const active = new Map<string, Entry>();
865
1011
  const key = (project: string, issue: number): string => `${project}\0${issue}`;
866
1012
  return {
867
1013
  open(project, issue, runId) {
868
1014
  const k = key(project, issue);
869
1015
  if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
870
- const entry = { project, issue, runId } as {
871
- project: string;
872
- issue: number;
873
- runId: string;
874
- control?: WorkerPauseControl;
1016
+ const entry: Entry = {
1017
+ project,
1018
+ issue,
1019
+ runId,
1020
+ finished: Promise.withResolvers<void>(),
875
1021
  };
876
1022
  active.set(k, entry);
877
1023
  return {
878
1024
  install: (control) => {
879
- if (active.get(k) === entry) entry.control = control;
1025
+ if (active.get(k) !== entry) return;
1026
+ entry.control = control;
1027
+ if (entry.stopReason === undefined) return;
1028
+ try {
1029
+ control.stop(entry.stopReason);
1030
+ } catch (err) {
1031
+ entry.stopError = err instanceof Error ? err.message : String(err);
1032
+ }
880
1033
  },
1034
+ requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
881
1035
  close: () => {
882
1036
  if (active.get(k) === entry) active.delete(k);
1037
+ entry.finished.resolve();
883
1038
  },
884
1039
  };
885
1040
  },
@@ -911,6 +1066,30 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
911
1066
  };
912
1067
  }
913
1068
  },
1069
+ async stop(project, issue, reason) {
1070
+ const entry = active.get(key(project, issue));
1071
+ if (entry === undefined) return { kind: "not-active" };
1072
+ if (entry.stopReason === undefined) {
1073
+ entry.stopReason = reason;
1074
+ if (entry.control !== undefined) {
1075
+ try {
1076
+ entry.control.stop(reason);
1077
+ } catch (err) {
1078
+ entry.stopReason = undefined;
1079
+ return {
1080
+ kind: "refused",
1081
+ runId: entry.runId,
1082
+ error: err instanceof Error ? err.message : String(err),
1083
+ };
1084
+ }
1085
+ }
1086
+ }
1087
+ await entry.finished.promise;
1088
+ if (entry.stopError !== undefined) {
1089
+ return { kind: "refused", runId: entry.runId, error: entry.stopError };
1090
+ }
1091
+ return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
1092
+ },
914
1093
  snapshot(project) {
915
1094
  const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
916
1095
  for (const entry of active.values()) {
@@ -1012,7 +1191,28 @@ export function shouldContinueAfterTurnsCap(f: {
1012
1191
  return hasContinuationBudget(f.continuation, f.maxContinuations);
1013
1192
  }
1014
1193
 
1015
- async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1194
+ /** Bind a worker's run identity to every release-policy block it emits. */
1195
+ export function workerReleaseBlockRecorder(
1196
+ project: string,
1197
+ issue: number,
1198
+ runId: string,
1199
+ root = stateDir(),
1200
+ ): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
1201
+ return (shape, context) =>
1202
+ recordReleaseBlock(
1203
+ project,
1204
+ "worker",
1205
+ shape,
1206
+ {
1207
+ ...context,
1208
+ issue,
1209
+ runId,
1210
+ },
1211
+ root,
1212
+ );
1213
+ }
1214
+
1215
+ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1016
1216
  const { project, caps, tracker, store } = d;
1017
1217
  const issue = r.issue.number;
1018
1218
  const branch = branchName(r.issue);
@@ -1027,6 +1227,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1027
1227
  let worktreePath: string | undefined;
1028
1228
  let turnLimit: TurnLimitController | undefined;
1029
1229
  let workerControl: WorkerControlSlot | undefined;
1230
+ let workerSessionInstalled = false;
1030
1231
  // The run's own repository. Hoisted for the same reason `worktreePath` is —
1031
1232
  // the catch and finally paths have to publish the branch.
1032
1233
  let runRepo: RunRepoRef | undefined;
@@ -1048,6 +1249,53 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1048
1249
  return pushRunBranch(project, runRepo);
1049
1250
  };
1050
1251
 
1252
+ /**
1253
+ * A stop can arrive after the run is claimed but before `runWorker` exposes
1254
+ * its controller. Settle that run here instead of turning an operator action
1255
+ * into a dispatch failure or making the HTTP request wait for a session that
1256
+ * will never exist.
1257
+ */
1258
+ const settleStopBeforeSession = async (): Promise<boolean> => {
1259
+ const reason = workerControl?.requestedStop();
1260
+ if (reason === undefined || run === undefined || workerSessionInstalled) return false;
1261
+ turnLimit?.close();
1262
+ turnLimit = undefined;
1263
+ const settlement =
1264
+ worktreePath === undefined
1265
+ ? undefined
1266
+ : await settleWorktree({
1267
+ issue,
1268
+ attempt,
1269
+ ending: `stopped by the operator: ${reason}`,
1270
+ worktree: worktreePath,
1271
+ branch,
1272
+ publish,
1273
+ tree: "remove",
1274
+ mirrorPath,
1275
+ });
1276
+ recordOperatorStop(store, {
1277
+ project: project.name,
1278
+ issue,
1279
+ runId: run.id,
1280
+ inProgress,
1281
+ reason,
1282
+ patch: {
1283
+ endedAt: Date.now(),
1284
+ turns: run.turns,
1285
+ spendUsd: run.spendUsd,
1286
+ worktree: worktreePath ?? run.worktree,
1287
+ report: [
1288
+ "Operator stopped the run before its worker session started.",
1289
+ `Reason: ${reason}`,
1290
+ ...(settlement?.lines ?? []),
1291
+ ].join("\n"),
1292
+ ...settlement?.patch,
1293
+ },
1294
+ });
1295
+ log(`#${issue} stopped by operator before its worker session started: ${reason}`);
1296
+ return true;
1297
+ };
1298
+
1051
1299
  try {
1052
1300
  // Claim on the STORE first, before anything that can fail. The run row —
1053
1301
  // not the label — is the crash-safe guard against double dispatch: rows
@@ -1063,8 +1311,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1063
1311
  // Read before this attempt's own row exists, so `latestRun` still means the
1064
1312
  // attempt whose work this one inherits.
1065
1313
  const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
1066
-
1067
-
1068
1314
  run = store.createRun({
1069
1315
  project: project.name,
1070
1316
  issue,
@@ -1078,11 +1324,20 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1078
1324
  maxTurns: caps.workerMaxTurns,
1079
1325
  startedAt: Date.now(),
1080
1326
  });
1327
+ const maxTurns = run.maxTurns;
1328
+ const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
1081
1329
  const runId = run.id;
1330
+ if (turnOverride !== undefined) {
1331
+ log(
1332
+ `#${issue} claimed with turn override ${turnOverride} ` +
1333
+ `(base ${caps.workerMaxTurns})`,
1334
+ );
1335
+ }
1082
1336
  store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
1083
1337
  claimed = true;
1084
- turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
1338
+ turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
1085
1339
  workerControl = d.workerControls.open(project.name, issue, runId);
1340
+ if (await settleStopBeforeSession()) return;
1086
1341
 
1087
1342
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
1088
1343
  // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -1091,6 +1346,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1091
1346
  // first attempt. addRunRepo does its own ensureMirror; calling it here too
1092
1347
  // would cost a second network fetch per attempt.
1093
1348
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
1349
+ if (await settleStopBeforeSession()) return;
1094
1350
  const provisioned = await addRunRepo(
1095
1351
  r.repo,
1096
1352
  project.mirrorRoot,
@@ -1100,6 +1356,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1100
1356
  );
1101
1357
  worktreePath = provisioned.path;
1102
1358
  runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
1359
+ if (await settleStopBeforeSession()) return;
1103
1360
 
1104
1361
  // The SDK names the transcript itself, so the daemon supplies the parent
1105
1362
  // directory and learns the real path back from the result. Inventing one
@@ -1138,6 +1395,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1138
1395
  },
1139
1396
  { ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
1140
1397
  );
1398
+ if (await settleStopBeforeSession()) return;
1141
1399
 
1142
1400
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
1143
1401
 
@@ -1146,20 +1404,29 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1146
1404
  (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
1147
1405
  );
1148
1406
 
1407
+ const brief = await buildBrief(project, r, branch, worktreePath, {
1408
+ continuation: provisioned.reattached,
1409
+ defaultBranch: r.repo.defaultBranch,
1410
+ ...(provisioned.reattached && priorSalvage !== undefined
1411
+ ? { salvagedSha: priorSalvage }
1412
+ : {}),
1413
+ });
1414
+ if (await settleStopBeforeSession()) return;
1415
+
1416
+ const repoSlug = githubRepo(r.repo.cloneUrl);
1417
+
1149
1418
  let result: WorkerResult;
1150
1419
  try {
1151
1420
  result = await runWorker({
1152
- brief: await buildBrief(project, r, branch, worktreePath, {
1153
- continuation: provisioned.reattached,
1154
- defaultBranch: r.repo.defaultBranch,
1155
- ...(provisioned.reattached && priorSalvage !== undefined
1156
- ? { salvagedSha: priorSalvage }
1157
- : {}),
1158
- }),
1421
+ brief,
1159
1422
  cwd: worktreePath,
1160
1423
  caps,
1161
- maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
1162
- onPauseControl: (control) => workerControl?.install(control),
1424
+ ...(repoSlug === undefined ? {} : { repoSlug }),
1425
+ maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
1426
+ onPauseControl: (control) => {
1427
+ workerSessionInstalled = true;
1428
+ workerControl?.install(control);
1429
+ },
1163
1430
  sessionDir,
1164
1431
  // The session's control socket, under the daemon's own state directory —
1165
1432
  // a child process of the daemon reaches it directly.
@@ -1178,28 +1445,24 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1178
1445
  },
1179
1446
  ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
1180
1447
  releaseGrants: resolveReleaseGrants(project),
1181
- onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
1448
+ onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
1182
1449
  onTurn: (n) => store.updateRun(runId, { turns: n }),
1183
1450
  onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
1184
1451
  onKilled: () => {
1185
1452
  turnLimit?.close();
1186
1453
  turnLimit = undefined;
1187
- workerControl?.close();
1188
- workerControl = undefined;
1189
1454
  },
1190
1455
  // Recorded the moment the session opens its transcript, not when the run
1191
1456
  // ends: `omp-conductor tail` resolves an issue to a file through this row,
1192
1457
  // and a path written at completion is a path nobody can follow live. The
1193
1458
  // completion-time update below writes the same value again, harmlessly.
1194
1459
  onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
1195
- });
1460
+ }, d.workerDeps);
1196
1461
  } finally {
1197
1462
  // This is the authoritative settlement edge for `extend`: close before
1198
1463
  // PR verification or terminal row writes can leave stale `running` state.
1199
1464
  turnLimit?.close();
1200
1465
  turnLimit = undefined;
1201
- workerControl?.close();
1202
- workerControl = undefined;
1203
1466
  }
1204
1467
 
1205
1468
  // A configured model the harness could not honour means this run was done by
@@ -1264,8 +1527,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1264
1527
  // `pushed-*` run is the only end that does not salvage: its deliverable is
1265
1528
  // already on a remote branch, whatever is left loose in the tree is by the
1266
1529
  // worker's own account not part of it, and appending a WIP commit would
1267
- // turn the green PR this daemon just verified red. Every other end is
1268
- // continuable, so its tree is treated as work.
1530
+ // turn the green PR this daemon just verified red. Every other end may
1531
+ // contain work, so it is salvaged before the tree's final fate is decided.
1269
1532
  const settlement =
1270
1533
  state === "pushed-green" || state === "pushed-pending" || state === "merged"
1271
1534
  ? undefined
@@ -1273,7 +1536,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1273
1536
  issue,
1274
1537
  attempt,
1275
1538
  ending:
1276
- state === "blocked" ? "blocked for an operator decision" : endedBy(result.killedBy),
1539
+ state === "blocked"
1540
+ ? "blocked for an operator decision"
1541
+ : state === "stopped"
1542
+ ? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
1543
+ : endedBy(result.killedBy),
1277
1544
  worktree: worktreePath,
1278
1545
  branch,
1279
1546
  publish,
@@ -1294,8 +1561,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1294
1561
  await removeWorktree(mirrorPath, worktreePath);
1295
1562
  }
1296
1563
 
1297
- store.updateRun(runId, {
1298
- state,
1564
+ const terminalPatch: Partial<RunRecord> = {
1299
1565
  endedAt: Date.now(),
1300
1566
  turns: result.turns,
1301
1567
  spendUsd: result.spendUsd,
@@ -1303,23 +1569,41 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1303
1569
  headSha: result.headSha,
1304
1570
  sessionFile: result.sessionFile,
1305
1571
  // Every terminal state persists the worker's report, not just a green
1306
- // push: the report of a killed attempt is exactly the one a later
1307
- // continuation must pool its disclosures from (#199).
1572
+ // push: a stopped attempt's partial report is still part of its audit trail.
1308
1573
  report: result.report,
1309
- ...(providerCredit !== undefined || providerTransient !== undefined
1310
- ? { lastError: providerCredit ?? providerTransient }
1311
- : verified.reason === undefined
1312
- ? {}
1313
- : { lastError: verified.reason }),
1314
1574
  ...settlement?.patch,
1315
1575
  ...(audit === undefined || audit.flags.length === 0
1316
1576
  ? {}
1317
1577
  : { settlementFlags: audit.flags }),
1318
- });
1578
+ };
1579
+ if (state === "stopped") {
1580
+ recordOperatorStop(store, {
1581
+ project: project.name,
1582
+ issue,
1583
+ runId,
1584
+ inProgress,
1585
+ reason: result.stoppedReason ?? "no reason recorded",
1586
+ patch: terminalPatch,
1587
+ });
1588
+ } else {
1589
+ const lastError = completionLastError(
1590
+ providerCredit,
1591
+ providerTransient,
1592
+ verified.reason,
1593
+ sessionErr,
1594
+ );
1595
+ store.updateRun(runId, {
1596
+ ...terminalPatch,
1597
+ state,
1598
+ ...(lastError === undefined ? {} : { lastError }),
1599
+ });
1600
+ }
1319
1601
 
1320
1602
  const salvaged = settlement?.lines ?? [];
1321
1603
 
1322
- if (state === "blocked") {
1604
+ if (state === "stopped") {
1605
+ log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
1606
+ } else if (state === "blocked") {
1323
1607
  swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
1324
1608
  await safeEscalate(d, {
1325
1609
  tier: 1,
@@ -1343,31 +1627,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1343
1627
  });
1344
1628
 
1345
1629
  if (providerCredit !== undefined) {
1346
- // Pause here rather than at classification: the sweep runs on the tick,
1347
- // and three issues each burned an attempt in the fifteen minutes between
1348
- // the first 402 and a human noticing (#220).
1349
- setPaused(true, { source: "provider-credit", reason: providerCredit });
1350
- log(`#${issue} provider refused for credit — dispatch paused: ${providerCredit}`);
1351
- swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
1352
- // Fleet-scoped and run-independent on purpose. The notification ledger
1353
- // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
1354
- // carrying no run or attempt is what makes this page once for the fleet
1355
- // instead of once per affected run.
1356
- await safeEscalate(d, {
1357
- tier: 2,
1358
- category: "fleet-stopped",
1359
- project: project.name,
1360
- issue: NO_ISSUE,
1361
- summary: `Model provider refused for credit — ${project.name} is paused`,
1362
- detail: [
1363
- providerCredit,
1364
- "",
1365
- "No implementation attempt was charged: this is a billing state, not a",
1366
- "failed implementation. Each affected issue keeps its queue label and",
1367
- "re-dispatches on `omp-conductor resume` once the provider has credit.",
1368
- `Session: ${result.sessionFile ?? "(no transcript)"}`,
1369
- ].join("\n"),
1370
- });
1630
+ await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
1631
+ swapToQueue(d, issue, inProgress);
1371
1632
  } else if (continueTurns) {
1372
1633
  // Requeue as one ordered pair: the in-progress removal before the
1373
1634
  // queue add, exactly the order the projector will apply them in (#201).
@@ -1469,8 +1730,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1469
1730
  // inner settlement guard exists. Latch it before any terminal write or await.
1470
1731
  turnLimit?.close();
1471
1732
  turnLimit = undefined;
1472
- workerControl?.close();
1473
- workerControl = undefined;
1733
+ if (await settleStopBeforeSession()) return;
1474
1734
  const detail = errText(err);
1475
1735
  log(`#${issue} errored: ${detail}`);
1476
1736
  // A crash lands anywhere, including mid-edit in a tree holding the only
@@ -1516,8 +1776,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1516
1776
  // a failure path, and whatever it still held is now a commit on the branch.
1517
1777
  } finally {
1518
1778
  turnLimit?.close();
1519
- workerControl?.close();
1520
- workerControl = undefined;
1521
1779
  // The run is over, so its channel is too. Closed here rather than beside
1522
1780
  // the session so the crash path closes it as well: a listener left bound
1523
1781
  // after its run settled is a socket whose `run-not-live` check is the only
@@ -1530,6 +1788,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1530
1788
  }
1531
1789
  verbListener = undefined;
1532
1790
  }
1791
+ workerControl?.close();
1792
+ workerControl = undefined;
1533
1793
  }
1534
1794
  }
1535
1795
 
@@ -1558,12 +1818,12 @@ export interface Settlement {
1558
1818
  *
1559
1819
  * - `merged` — the work landed. That is what `merged` was reserved for.
1560
1820
  * - `closed` — a human read the work and said no. Leaving it `pushed-green`
1561
- * strands the issue forever behind a PR nobody will ever merge, and calling it
1562
- * `merged` is simply a lie about work that does not exist on the base branch.
1563
- * `failed` is true the attempt did not land — and it releases the busy guard,
1564
- * so an issue a human re-queues can be attempted again. The attempt counter is
1565
- * untouched either way: this row was a real attempt, and pretending otherwise
1566
- * would let a rejected issue cycle past `maxAttemptsPerIssue`.
1821
+ * forever is a lie; `failed` records that it did not land and releases the
1822
+ * busy guard, so an issue a human re-queues can be attempted again. A row
1823
+ * that had reached `pushed-green` or `pushed-pending` is classified
1824
+ * `returned-for-revision` at settlement. A review decision asks for another
1825
+ * implementation pass, not a failure, so it consumes the continuation budget
1826
+ * instead of the failed-attempt budget.
1567
1827
  * - `open`, and undefined — nothing changes. Undefined is "could not tell": a
1568
1828
  * flaky network, a revoked token, a deleted PR. Settling on it would record a
1569
1829
  * merge that never happened, and the next tick asks again for free. An
@@ -1647,6 +1907,242 @@ export function releaseInProgress(
1647
1907
  * in the busy set throughout, so no second worker can be sent at the issue while
1648
1908
  * it waits.
1649
1909
  */
1910
+ const BASE_CHECK_BATCH = 20;
1911
+ const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
1912
+ const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
1913
+
1914
+ const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
1915
+
1916
+ const FAILING_WORKFLOW_CONCLUSIONS = new Set([
1917
+ "failure",
1918
+ "cancelled",
1919
+ "timed_out",
1920
+ "action_required",
1921
+ "startup_failure",
1922
+ "stale",
1923
+ ]);
1924
+
1925
+ function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
1926
+ const flags = run.settlementFlags ?? [];
1927
+ return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
1928
+ ? flags
1929
+ : [...flags, flag];
1930
+ }
1931
+
1932
+ /**
1933
+ * Observe Actions on exact merge commits for up to one day. A running workflow
1934
+ * stays quiet and pending; a failure becomes durable evidence on the merged row
1935
+ * and pages exactly once because the row leaves `pending` before delivery.
1936
+ */
1937
+ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "store" | "escalate">): Promise<void> {
1938
+ const now = Date.now();
1939
+ for (const run of d.store.runsNeedingBaseCheck(d.project.name, BASE_CHECK_BATCH)) {
1940
+ if (
1941
+ run.endedAt === undefined ||
1942
+ now - run.endedAt > BASE_CHECK_WINDOW_MS ||
1943
+ run.mergeSha === undefined ||
1944
+ run.baseRef === undefined
1945
+ ) {
1946
+ d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
1947
+ log(`#${run.issue} base check unknown: merge identity is absent or older than 24h`);
1948
+ continue;
1949
+ }
1950
+
1951
+ const repo = d.project.routing.repos[run.repo];
1952
+ const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
1953
+ if (repoIdentity === undefined) {
1954
+ d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
1955
+ log(`#${run.issue} base check unknown: routed repository ${run.repo} has no GitHub identity`);
1956
+ continue;
1957
+ }
1958
+
1959
+ let workflows;
1960
+ try {
1961
+ workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha, {
1962
+ event: "push",
1963
+ branch: run.baseRef,
1964
+ });
1965
+ } catch (err) {
1966
+ log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
1967
+ continue;
1968
+ }
1969
+ if (workflows === undefined) {
1970
+ log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
1971
+ continue;
1972
+ }
1973
+ if (workflows.length === 0) {
1974
+ log(
1975
+ `#${run.issue} base check: no push-triggered run yet for ${run.mergeSha} — retrying next tick`,
1976
+ );
1977
+ continue;
1978
+ }
1979
+ if (workflows.some((workflow) => workflow.status !== "completed")) continue;
1980
+
1981
+ const failed = workflows.find(
1982
+ (workflow) =>
1983
+ workflow.conclusion !== undefined &&
1984
+ FAILING_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
1985
+ );
1986
+ if (failed === undefined) {
1987
+ const unknown = workflows.find(
1988
+ (workflow) =>
1989
+ workflow.conclusion === undefined ||
1990
+ !SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
1991
+ );
1992
+ if (unknown !== undefined) {
1993
+ log(
1994
+ `#${run.issue} base check has unrecognised completed conclusion ` +
1995
+ `${JSON.stringify(unknown.conclusion)} for ${unknown.name} — retrying next tick`,
1996
+ );
1997
+ continue;
1998
+ }
1999
+ d.store.updateRun(run.id, { baseCheck: "green", baseCheckAt: now });
2000
+ log(`#${run.issue} base ${run.baseRef} green at ${run.mergeSha}`);
2001
+ continue;
2002
+ }
2003
+
2004
+ let previous;
2005
+ try {
2006
+ previous = await d.tracker.previousWorkflowRun(
2007
+ repoIdentity,
2008
+ failed.workflowId,
2009
+ run.baseRef,
2010
+ failed.createdAt,
2011
+ );
2012
+ } catch (err) {
2013
+ log(`#${run.issue} previous ${failed.name} run unavailable (${errText(err)}) — retrying next tick`);
2014
+ continue;
2015
+ }
2016
+ if (previous === undefined || (previous !== null && previous.status !== "completed")) continue;
2017
+ const preexisting =
2018
+ previous !== null &&
2019
+ previous.conclusion !== undefined &&
2020
+ FAILING_WORKFLOW_CONCLUSIONS.has(previous.conclusion);
2021
+ const detail =
2022
+ `${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
2023
+ (preexisting ? " (already red before this merge)" : "");
2024
+ const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
2025
+ const delivered = await safeEscalate(d, {
2026
+ tier: 1,
2027
+ project: d.project.name,
2028
+ issue: run.issue,
2029
+ runId: run.id,
2030
+ summary: `Base branch ${run.baseRef} is red after merge`,
2031
+ detail,
2032
+ });
2033
+ if (!delivered) continue;
2034
+ d.store.updateRun(run.id, {
2035
+ baseCheck: preexisting ? "red-preexisting" : "red",
2036
+ baseCheckAt: now,
2037
+ settlementFlags: appendSettlementFlag(run, flag),
2038
+ });
2039
+ }
2040
+ }
2041
+
2042
+ /**
2043
+ * Refresh current base-branch health at each recently merged repository's live
2044
+ * head. This is status and release-gate evidence only; the per-merge audit
2045
+ * above remains the sole path that attributes and escalates a regression.
2046
+ */
2047
+ export async function watchBaseHealth(
2048
+ d: Pick<Deps, "project" | "tracker" | "store">,
2049
+ ): Promise<void> {
2050
+ const now = Date.now();
2051
+ const previousByRepo = new Map(
2052
+ d.store.baseHealth(d.project.name).map((row) => [row.repo, row] as const),
2053
+ );
2054
+ for (const { repo, baseRef } of d.store.mergedRepoBranches(
2055
+ d.project.name,
2056
+ now - BASE_STATUS_WINDOW_MS,
2057
+ )) {
2058
+ const target = d.project.routing.repos[repo];
2059
+ if (target === undefined) {
2060
+ log(`base health skipped: routed repository ${repo} is no longer configured`);
2061
+ continue;
2062
+ }
2063
+ const identity = githubRepo(target.cloneUrl);
2064
+ if (identity === undefined) {
2065
+ log(`base health skipped: routed repository ${repo} has no GitHub identity`);
2066
+ continue;
2067
+ }
2068
+ const branch = baseRef ?? target.defaultBranch;
2069
+
2070
+ let head: string | undefined;
2071
+ try {
2072
+ head = await d.tracker.branchHead(identity, branch);
2073
+ } catch (err) {
2074
+ log(`base ${repo}/${branch} head unavailable (${errText(err)}) — keeping previous health`);
2075
+ continue;
2076
+ }
2077
+ if (head === undefined) {
2078
+ log(`base ${repo}/${branch} head unavailable — keeping previous health`);
2079
+ continue;
2080
+ }
2081
+
2082
+ const previous = previousByRepo.get(repo);
2083
+ if (
2084
+ previous?.branch === branch &&
2085
+ previous.headSha === head &&
2086
+ (previous.verdict === "green" || previous.verdict === "red")
2087
+ ) {
2088
+ continue;
2089
+ }
2090
+
2091
+ let runs;
2092
+ try {
2093
+ runs = await d.tracker.workflowRunsAt(identity, head, { event: "push", branch });
2094
+ } catch (err) {
2095
+ log(`base ${repo}/${branch} workflows unavailable (${errText(err)}) — keeping previous health`);
2096
+ continue;
2097
+ }
2098
+ if (runs === undefined) {
2099
+ log(`base ${repo}/${branch} workflows unavailable — keeping previous health`);
2100
+ continue;
2101
+ }
2102
+
2103
+ let verdict: BaseHealth["verdict"];
2104
+ let detail: string | undefined;
2105
+ if (runs.length === 0) {
2106
+ verdict = "unknown";
2107
+ detail = `no push-triggered workflow run for ${head.slice(0, 8)}`;
2108
+ } else if (runs.some((run) => run.status !== "completed")) {
2109
+ verdict = "pending";
2110
+ } else {
2111
+ const failed = runs.find(
2112
+ (run) =>
2113
+ run.conclusion !== undefined &&
2114
+ FAILING_WORKFLOW_CONCLUSIONS.has(run.conclusion),
2115
+ );
2116
+ if (failed !== undefined) {
2117
+ verdict = "red";
2118
+ detail = `${failed.name} failed at ${head.slice(0, 8)} — ${failed.url}`;
2119
+ } else if (
2120
+ runs.some(
2121
+ (run) =>
2122
+ run.conclusion === undefined ||
2123
+ !SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(run.conclusion),
2124
+ )
2125
+ ) {
2126
+ verdict = "pending";
2127
+ } else {
2128
+ verdict = "green";
2129
+ }
2130
+ }
2131
+
2132
+ const health: BaseHealth = {
2133
+ repo,
2134
+ branch,
2135
+ headSha: head,
2136
+ verdict,
2137
+ runsCount: runs.length,
2138
+ checkedAt: now,
2139
+ ...(detail === undefined ? {} : { detail }),
2140
+ };
2141
+ d.store.upsertBaseHealth(d.project.name, health);
2142
+ previousByRepo.set(repo, health);
2143
+ }
2144
+ }
2145
+
1650
2146
  export async function settlePushedGreen(
1651
2147
  d: Pick<Deps, "project" | "tracker" | "store">,
1652
2148
  ): Promise<void> {
@@ -1678,17 +2174,44 @@ export async function settlePushedGreen(
1678
2174
 
1679
2175
  const settlement = settlementFor(pr, run.prUrl);
1680
2176
  if (settlement !== undefined) {
2177
+ // A mediated merge enters a second, bounded observation phase. Record the
2178
+ // exact merge commit before the row leaves the active set; if GitHub cannot
2179
+ // supply it yet, retry this settlement next tick rather than create a
2180
+ // merged row whose base result can never be attributed.
2181
+ let merged: MergedPrInfo | undefined;
2182
+ if (settlement.state === "merged") {
2183
+ try {
2184
+ merged = await tracker.mergedPrInfo(run.prUrl);
2185
+ } catch (err) {
2186
+ log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
2187
+ continue;
2188
+ }
2189
+ if (merged === undefined) {
2190
+ log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
2191
+ continue;
2192
+ }
2193
+ }
2194
+
1681
2195
  // The label removal and the terminal row are one fact again (#201): the
1682
2196
  // release is enqueued — a durable local write that cannot fail on the
1683
- // tracker — in the same breath as the row is terminalised, so there is
1684
- // no window in which a row beyond every later tick still owes its label.
1685
- // The projector applies it with retry; while pending, the eligibility
1686
- // overlay treats the label as already gone, so #18's
1687
- // permanent-`agent:in-progress` cannot re-form even when GitHub refuses
1688
- // the write.
2197
+ // tracker — in the same breath as the row is terminalised.
1689
2198
  releaseInProgress(d, run.issue, settlement.reason);
1690
- const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
1691
- if (settlement.state === "failed") patch.lastError = settlement.reason;
2199
+ const patch: Partial<RunRecord> = {
2200
+ state: settlement.state,
2201
+ endedAt: Date.now(),
2202
+ ...(merged === undefined
2203
+ ? {}
2204
+ : {
2205
+ mergeSha: merged.mergeSha,
2206
+ baseRef: merged.baseRef,
2207
+ baseCheck: "pending",
2208
+ }),
2209
+ };
2210
+ if (settlement.state === "failed") {
2211
+ patch.lastError = settlement.reason;
2212
+ patch.failureClass = "returned-for-revision";
2213
+ patch.recoveryAction = "none";
2214
+ }
1692
2215
  store.updateRun(run.id, patch);
1693
2216
  log(`#${run.issue} settled: ${settlement.reason}`);
1694
2217
  continue;
@@ -1720,6 +2243,121 @@ export async function settlePushedGreen(
1720
2243
  }
1721
2244
  }
1722
2245
 
2246
+ const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
2247
+ failed: true,
2248
+ killed: true,
2249
+ orphaned: true,
2250
+ blocked: true,
2251
+ };
2252
+
2253
+ /**
2254
+ * Reattaches a recovered PR to the terminal run that owns it (#245).
2255
+ *
2256
+ * A worker can fail before its completion report records `prUrl`, then have its
2257
+ * dirty tree committed and pushed by salvage. If that branch already has a PR,
2258
+ * the orchestrator otherwise has no policy-compliant path to inspect or merge
2259
+ * it: ownership is store-backed. Adoption is deliberately stricter than
2260
+ * admission. The tracker query proves the PR closes this run's issue; exact
2261
+ * branch and canonical repository matches prove it is this run's recovered
2262
+ * work, not an unrelated closer. Missing identity is refusal, never a guess.
2263
+ *
2264
+ * The newest run per issue is inspected, at most ten per tick and only inside
2265
+ * the same 30-day window as mediated PR verbs. The cursor advances through the
2266
+ * full eligible set so persistent non-matches cannot starve older recovered
2267
+ * work. Successful adoption is idempotent because the row gains `prUrl`;
2268
+ * non-matches are logged once per daemon process.
2269
+ */
2270
+ const rejectedSalvagedPrRuns = new Set<string>();
2271
+ const salvagedPrCursor = new Map<string, string>();
2272
+
2273
+ export async function adoptSalvagedPrs(
2274
+ d: Pick<Deps, "project" | "tracker" | "store">,
2275
+ now = Date.now(),
2276
+ ): Promise<void> {
2277
+ const { project, tracker, store } = d;
2278
+ const eligible = store
2279
+ .recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
2280
+ .filter(
2281
+ (run) =>
2282
+ ADOPTABLE_PR_STATES[run.state] === true &&
2283
+ run.prUrl === undefined &&
2284
+ run.branch.trim() !== "",
2285
+ );
2286
+ const previous = salvagedPrCursor.get(project.name);
2287
+ const previousIndex =
2288
+ previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
2289
+ const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
2290
+ const candidates = Array.from(
2291
+ { length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
2292
+ (_, offset) => eligible[(start + offset) % eligible.length]!,
2293
+ );
2294
+ const last = candidates.at(-1);
2295
+ if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
2296
+
2297
+ for (const run of candidates) {
2298
+ const repo = project.routing.repos[run.repo];
2299
+ const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
2300
+ let closers: OpenCloser[];
2301
+ try {
2302
+ closers = await tracker.openClosersFor(run.issue);
2303
+ } catch (err) {
2304
+ log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
2305
+ continue;
2306
+ }
2307
+
2308
+ const closer = closers.find(
2309
+ (candidate) =>
2310
+ candidate.headRefName !== "" &&
2311
+ candidate.headRefName === run.branch &&
2312
+ repo !== undefined &&
2313
+ candidate.repo !== "" &&
2314
+ candidate.repo === repoIdentity,
2315
+ );
2316
+ if (closer === undefined) {
2317
+ if (!rejectedSalvagedPrRuns.has(run.id)) {
2318
+ const observed = closers[0];
2319
+ const reason =
2320
+ observed === undefined
2321
+ ? "no open closing PR"
2322
+ : observed.headRefName === ""
2323
+ ? "closer has no head branch identity"
2324
+ : observed.headRefName !== run.branch
2325
+ ? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
2326
+ : observed.repo === ""
2327
+ ? "closer has no repository identity"
2328
+ : `closer repository ${observed.repo} does not match routed repository`;
2329
+ log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
2330
+ rejectedSalvagedPrRuns.add(run.id);
2331
+ }
2332
+ continue;
2333
+ }
2334
+
2335
+ const flag: SettlementFlag = {
2336
+ kind: "pr-adopted",
2337
+ file: "(recovery)",
2338
+ detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
2339
+ };
2340
+ store.updateRun(run.id, {
2341
+ prUrl: closer.url,
2342
+ settlementFlags: [...(run.settlementFlags ?? []), flag],
2343
+ });
2344
+ rejectedSalvagedPrRuns.delete(run.id);
2345
+ log(
2346
+ `#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
2347
+ (run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
2348
+ );
2349
+ }
2350
+ }
2351
+
2352
+ /** Canonical `owner/repo` identity from a configured network clone URL. */
2353
+ function githubRepo(cloneUrl: string): string | undefined {
2354
+ const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
2355
+ const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
2356
+ normalized,
2357
+ );
2358
+ return match?.[1];
2359
+ }
2360
+
1723
2361
  const RETAINED_CLEANUP_BATCH = 10;
1724
2362
 
1725
2363
  export interface RetainedCleanupCursor {
@@ -1833,6 +2471,7 @@ const HOLD_SAMPLE_SIZE = 5;
1833
2471
  const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
1834
2472
  "parent-lookup-error",
1835
2473
  "open-pr-lookup-error",
2474
+ "issue-state-lookup-error",
1836
2475
  ]);
1837
2476
 
1838
2477
  /** Groups transient decisions into the bounded record exposed by status. */
@@ -2217,6 +2856,32 @@ export async function admitCandidates(
2217
2856
  log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
2218
2857
  }
2219
2858
 
2859
+ // The queue comes from GitHub's eventually-consistent search index. Re-read
2860
+ // state and labels directly at the last possible moment so a just-closed or
2861
+ // explicitly dequeued issue cannot turn a stale candidate into another
2862
+ // attempt (#247).
2863
+ let snapshot: IssueSnapshot | undefined;
2864
+ try {
2865
+ snapshot = await tracker.issueSnapshot(issue);
2866
+ } catch {
2867
+ snapshot = undefined;
2868
+ }
2869
+ if (snapshot === undefined) {
2870
+ hold(issue, "issue-state-lookup-error");
2871
+ log(`#${issue} held: issue snapshot check failed — retrying next tick`);
2872
+ continue;
2873
+ }
2874
+ if (snapshot.state === "closed") {
2875
+ hold(issue, "issue-closed");
2876
+ log(`#${issue} skipped: issue is closed (search index lag)`);
2877
+ continue;
2878
+ }
2879
+ if (!snapshot.labels.includes(project.queueLabel)) {
2880
+ hold(issue, "issue-dequeued");
2881
+ log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
2882
+ continue;
2883
+ }
2884
+
2220
2885
  admitted.push({ r, attempt: priorRuns + 1 });
2221
2886
  liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
2222
2887
  if (parent !== undefined) {
@@ -2289,8 +2954,26 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2289
2954
  }
2290
2955
  d.project = fresh;
2291
2956
  d.caps = freshCaps;
2957
+ d.deliveryPolicyValid = true;
2292
2958
  } catch (err) {
2293
- log(`config reload failed (${errText(err)}) — continuing with the values loaded at boot`);
2959
+ log(`config reload failed (${errText(err)}) — retaining boot values but blocking autonomous delivery`);
2960
+ d.deliveryPolicyValid = false;
2961
+ }
2962
+
2963
+ // Availability-held notices are already durable. Once the freshly reloaded
2964
+ // policy opens (or newly allows their category), atomically hand a bounded
2965
+ // batch to the report outbox. Pauses do not suppress delivery.
2966
+ if (d.deliveryPolicyValid !== false) {
2967
+ try {
2968
+ const catchUp = enqueueAvailableHeldNotices(d.project, d.store, Date.now());
2969
+ if (catchUp !== undefined && !catchUp.deduped) {
2970
+ log(`availability catch-up ${catchUp.report.id} queued for ${d.project.name}`);
2971
+ }
2972
+ } catch (err) {
2973
+ // The daily digest may have claimed the same rows from another process
2974
+ // between selection and association. Either way the ledger still owns them.
2975
+ log(`availability catch-up handoff deferred (${errText(err)}) — retrying next tick`);
2976
+ }
2294
2977
  }
2295
2978
 
2296
2979
  // Before the pause check, deliberately. This one is not about dispatch: the
@@ -2305,6 +2988,21 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2305
2988
  // while the fleet is parked or workers are still active. Resident workers run
2306
2989
  // through the pool without blocking this five-minute tick.
2307
2990
  await settlePushedGreen(d);
2991
+ try {
2992
+ await watchMergedBase(d);
2993
+ } catch (err) {
2994
+ log(`base-branch check sweep failed: ${errText(err)}`);
2995
+ }
2996
+ try {
2997
+ await watchBaseHealth(d);
2998
+ } catch (err) {
2999
+ log(`current base-health sweep failed: ${errText(err)}`);
3000
+ }
3001
+ try {
3002
+ await adoptSalvagedPrs(d);
3003
+ } catch (err) {
3004
+ log(`salvaged PR adoption sweep failed: ${errText(err)}`);
3005
+ }
2308
3006
 
2309
3007
  // Immediately after settlement and before any routing, so a class is on the
2310
3008
  // row before the next dispatch decision reads its budgets (#132). Above the
@@ -2548,6 +3246,8 @@ export interface DaemonHealthSnapshot {
2548
3246
  paused: boolean;
2549
3247
  activeRuns: number;
2550
3248
  project: string;
3249
+ /** One-shot issue ceilings waiting for the next claim. */
3250
+ turnOverrides: TurnOverride[];
2551
3251
  /** Resident set of this daemon; workers are in-process omp sessions. */
2552
3252
  rssBytes: number;
2553
3253
  dispatch?: DispatchSummary;
@@ -2569,6 +3269,7 @@ export function daemonHealthSnapshot(
2569
3269
  ok: true,
2570
3270
  paused,
2571
3271
  activeRuns: store.activeRuns(project).length,
3272
+ turnOverrides: store.listTurnOverrides(project),
2572
3273
  project,
2573
3274
  rssBytes,
2574
3275
  ...(dispatch === undefined ? {} : { dispatch }),
@@ -2577,11 +3278,19 @@ export function daemonHealthSnapshot(
2577
3278
  };
2578
3279
  }
2579
3280
 
3281
+ const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
3282
+ "failed",
3283
+ "killed",
3284
+ "orphaned",
3285
+ "blocked",
3286
+ ]);
3287
+
2580
3288
  export async function turnLimitResponse(
2581
3289
  req: Request,
2582
3290
  project: string,
2583
- store: Pick<Store, "latestRun">,
3291
+ store: Pick<Store, "latestRun" | "setTurnOverride">,
2584
3292
  registry: TurnLimitRegistry,
3293
+ caps: Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">,
2585
3294
  ): Promise<Response | undefined> {
2586
3295
  const url = new URL(req.url);
2587
3296
  const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
@@ -2615,6 +3324,16 @@ export async function turnLimitResponse(
2615
3324
  }
2616
3325
 
2617
3326
  const issue = Number(match[1]);
3327
+ if ((maxTurns as number) > caps.workerMaxTurnsCeiling) {
3328
+ return Response.json(
3329
+ {
3330
+ error:
3331
+ `#${issue} turn budget ${maxTurns as number} exceeds the ` +
3332
+ `${caps.workerMaxTurnsCeiling}-turn caps.workerMaxTurnsCeiling`,
3333
+ },
3334
+ { status: 422 },
3335
+ );
3336
+ }
2618
3337
  const outcome = registry.extend(project, issue, maxTurns as number);
2619
3338
  if (outcome.kind === "extended") return Response.json(outcome);
2620
3339
  if (outcome.kind === "not-increase") {
@@ -2628,14 +3347,33 @@ export async function turnLimitResponse(
2628
3347
  if (latest === undefined) {
2629
3348
  return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
2630
3349
  }
2631
- return Response.json(
2632
- {
2633
- error:
2634
- `#${issue} has no live worker controller; its session already settled ` +
2635
- `or belongs to another daemon (stored state: ${latest.state})`,
2636
- },
2637
- { status: 409 },
2638
- );
3350
+ if (!TURN_OVERRIDE_STATES.has(latest.state)) {
3351
+ return Response.json(
3352
+ {
3353
+ error:
3354
+ `#${issue} has no live worker controller; its session already settled ` +
3355
+ `or belongs to another daemon (stored state: ${latest.state})`,
3356
+ },
3357
+ { status: 409 },
3358
+ );
3359
+ }
3360
+ if ((maxTurns as number) <= caps.workerMaxTurns) {
3361
+ return Response.json(
3362
+ {
3363
+ error:
3364
+ `#${issue} next-attempt turn budget must exceed the ` +
3365
+ `${caps.workerMaxTurns}-turn caps.workerMaxTurns base`,
3366
+ },
3367
+ { status: 409 },
3368
+ );
3369
+ }
3370
+ store.setTurnOverride(project, issue, maxTurns as number);
3371
+ return Response.json({
3372
+ kind: "next-attempt",
3373
+ issue,
3374
+ nextAttemptMaxTurns: maxTurns,
3375
+ baseMaxTurns: caps.workerMaxTurns,
3376
+ });
2639
3377
  }
2640
3378
 
2641
3379
  export async function workerControlResponse(
@@ -2645,7 +3383,7 @@ export async function workerControlResponse(
2645
3383
  registry: WorkerControlRegistry,
2646
3384
  ): Promise<Response | undefined> {
2647
3385
  const url = new URL(req.url);
2648
- const match = /^\/runs\/(\d+)\/(pause|resume)$/.exec(url.pathname);
3386
+ const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
2649
3387
  if (req.method !== "PUT" || match === null) return undefined;
2650
3388
  if (!req.headers.get("content-type")?.startsWith("application/json")) {
2651
3389
  return Response.json({ error: "content-type must be application/json" }, { status: 415 });
@@ -2670,23 +3408,71 @@ export async function workerControlResponse(
2670
3408
  { status: 409 },
2671
3409
  );
2672
3410
  }
3411
+ const action = match[2] as "pause" | "resume" | "stop";
3412
+ let reason: string | undefined;
3413
+ if (action === "stop") {
3414
+ const rawReason = Reflect.get(body, "reason");
3415
+ if (typeof rawReason !== "string" || rawReason.trim() === "") {
3416
+ return Response.json({ error: "reason must be a non-empty string" }, { status: 400 });
3417
+ }
3418
+ reason = rawReason.trim().replace(/\s+/g, " ");
3419
+ if (reason.length > 500) {
3420
+ return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
3421
+ }
3422
+ }
3423
+
2673
3424
 
2674
3425
  const issue = Number(match[1]);
2675
3426
  const outcome =
2676
- match[2] === "pause"
3427
+ action === "pause"
2677
3428
  ? await registry.pause(project, issue)
2678
- : registry.resume(project, issue);
3429
+ : action === "resume"
3430
+ ? registry.resume(project, issue)
3431
+ : await registry.stop(project, issue, reason!);
2679
3432
  if (outcome.kind === "ok") {
2680
3433
  return Response.json({ runId: outcome.runId, phase: outcome.phase });
2681
3434
  }
2682
3435
  if (outcome.kind === "refused") {
2683
3436
  return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
2684
3437
  }
3438
+ if (outcome.kind === "stopped") {
3439
+ const stopped = store.latestRun(project, issue);
3440
+ if (stopped === undefined || stopped.id !== outcome.runId) {
3441
+ return Response.json(
3442
+ { error: `#${issue} stopped, but its terminal run record is unavailable` },
3443
+ { status: 500 },
3444
+ );
3445
+ }
3446
+ if (stopped.state !== "stopped") {
3447
+ return Response.json({
3448
+ outcome: "already-terminal",
3449
+ runId: stopped.id,
3450
+ state: stopped.state,
3451
+ });
3452
+ }
3453
+ return Response.json({
3454
+ outcome: "stopped",
3455
+ runId: stopped.id,
3456
+ state: stopped.state,
3457
+ reason: outcome.reason,
3458
+ ...(stopped.salvageSha === undefined ? {} : { salvageSha: stopped.salvageSha }),
3459
+ ...(stopped.salvageError === undefined ? {} : { salvageError: stopped.salvageError }),
3460
+ worktree: stopped.worktree,
3461
+ });
3462
+ }
3463
+
2685
3464
 
2686
3465
  const latest = store.latestRun(project, issue);
2687
3466
  if (latest === undefined) {
2688
3467
  return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
2689
3468
  }
3469
+ if (action === "stop" && !LIVE_STATES.includes(latest.state)) {
3470
+ return Response.json({
3471
+ outcome: "already-terminal",
3472
+ runId: latest.id,
3473
+ state: latest.state,
3474
+ });
3475
+ }
2690
3476
  return Response.json(
2691
3477
  {
2692
3478
  error:
@@ -2699,7 +3485,8 @@ export async function workerControlResponse(
2699
3485
 
2700
3486
  export interface DaemonHttpDeps {
2701
3487
  project: string;
2702
- store: Pick<Store, "latestRun">;
3488
+ store: Pick<Store, "latestRun" | "setTurnOverride">;
3489
+ caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
2703
3490
  turnLimits: TurnLimitRegistry;
2704
3491
  workerControls: WorkerControlRegistry;
2705
3492
  health: () => DaemonHealthSnapshot;
@@ -2709,15 +3496,15 @@ export interface DaemonHttpDeps {
2709
3496
  * The whole HTTP surface, in one named function so a test can pin what is *not*
2710
3497
  * on it.
2711
3498
  *
2712
- * Three route families: the health read, turn-limit control, and worker
2713
- * pause/resume control. Everything else is 404. The controls mutate only
3499
+ * Three route families: the health read, turn-limit control, and live-worker
3500
+ * pause/resume/stop control. Everything else is 404. The controls mutate only
2714
3501
  * daemon-owned live sessions; tracker and repository mutations stay on the
2715
3502
  * authenticated per-run channel described at the `Bun.serve` call (#126).
2716
3503
  */
2717
3504
  export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
2718
3505
  const url = new URL(req.url);
2719
3506
  if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
2720
- const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
3507
+ const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits, d.caps());
2721
3508
  if (turnLimit !== undefined) return turnLimit;
2722
3509
  const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
2723
3510
  return workerControl ?? new Response("not found\n", { status: 404 });
@@ -2734,6 +3521,10 @@ export interface StatusSnapshot {
2734
3521
  * reading like a mistake (#220).
2735
3522
  */
2736
3523
  pauseReason?: string;
3524
+ /** Mechanical operator availability at the moment this snapshot was read. */
3525
+ availability?: AvailabilityState;
3526
+ /** Next digest opportunity under the same predicate that gates submission. */
3527
+ digestSchedule?: DigestScheduleState;
2737
3528
  caps: Caps;
2738
3529
  /**
2739
3530
  * The effective per-shape release grants. On the snapshot rather than re-read
@@ -2747,10 +3538,15 @@ export interface StatusSnapshot {
2747
3538
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
2748
3539
  * only copy of work the daemon could not save. */
2749
3540
  salvagedRuns: RunRecord[];
3541
+ /** One-shot issue ceilings waiting for the next claim. */
3542
+ turnOverrides: TurnOverride[];
2750
3543
  /** Reports the operator has not provably received: pending, in-flight with an
2751
3544
  * unknown outcome, or written off. An empty list is the only honest way to
2752
3545
  * say "everything authored this cycle actually went out" (#123). */
2753
3546
  openReports: ReportRecord[];
3547
+ /** Ordinary outcomes and deferred escalations not yet associated with an
3548
+ * accepted digest report. */
3549
+ digestBacklog: DigestBacklog;
2754
3550
  /**
2755
3551
  * The most recent conductor-verb calls and how the daemon decided them
2756
3552
  * (#126). On `status` rather than only behind `omp-conductor ledger` because
@@ -2758,6 +3554,8 @@ export interface StatusSnapshot {
2758
3554
  * config does not let it, and an operator who has to know to go looking is an
2759
3555
  * operator who finds out from the tracker instead.
2760
3556
  */
3557
+ /** Current live-head push-workflow verdict per recently merged repository. */
3558
+ baseHealth: BaseHealth[];
2761
3559
  verbLedger: VerbLedgerEntry[];
2762
3560
  /** Runs backed by a worker process — the number capacity compares against. */
2763
3561
  liveWorkers: number;
@@ -2804,6 +3602,10 @@ export function statusSnapshotFromStore(
2804
3602
  store: Store,
2805
3603
  planUsage?: PlanUsageStatus,
2806
3604
  ): StatusSnapshot {
3605
+ const now = Date.now();
3606
+ const lastDigestKey = store.lastDigestDedupeKey(p.name);
3607
+ const lastDigestDay =
3608
+ lastDigestKey === undefined ? undefined : lastDigestKey.slice("digest:".length);
2807
3609
  const since = startOfToday();
2808
3610
  const dispatch = store.latestDispatch(p.name);
2809
3611
  const labelOpsPending = store.countPendingLabelOps(p.name);
@@ -2817,11 +3619,15 @@ export function statusSnapshotFromStore(
2817
3619
  stateDir: stateDir(),
2818
3620
  paused: isPaused(),
2819
3621
  ...(reason === undefined ? {} : { pauseReason: reason }),
3622
+ availability: availabilityState(p.reporting, now),
3623
+ digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
2820
3624
  caps,
2821
3625
  releaseGrants: resolveReleaseGrants(p),
2822
3626
  activeRuns: store.activeRuns(p.name),
2823
3627
  salvagedRuns: store.salvagedRuns(p.name),
3628
+ turnOverrides: store.listTurnOverrides(p.name),
2824
3629
  openReports: store.openReports(p.name),
3630
+ digestBacklog: store.digestBacklog(p.name),
2825
3631
  verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
2826
3632
  liveWorkers: store.liveRuns(p.name).length,
2827
3633
  runsToday: store.runsStartedSince(p.name, since),
@@ -2830,11 +3636,12 @@ export function statusSnapshotFromStore(
2830
3636
  ...(planUsage === undefined ? {} : { planUsage }),
2831
3637
  // Written by the tracker's hooks rather than polled, so the renderer does
2832
3638
  // not re-read GitHub to know it is being refused (#198).
2833
- ghRefusals: store.ghRefusalsSince?.(Date.now() - 5 * 60_000),
3639
+ ghRefusals: store.ghRefusalsSince?.(now - 5 * 60_000),
2834
3640
  ghCallsToday: store.ghCallsToday?.(utcDay()),
2835
3641
  ...(labelOpsPending === 0 || oldestLabelOpAt === undefined
2836
3642
  ? {}
2837
- : { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
3643
+ : { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
3644
+ baseHealth: store.baseHealth(p.name),
2838
3645
  };
2839
3646
  }
2840
3647
 
@@ -2917,6 +3724,25 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
2917
3724
  ...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
2918
3725
  ];
2919
3726
  }
3727
+ export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
3728
+ return rows.map((row) => {
3729
+ const head = row.headSha.slice(0, 8);
3730
+ if (row.verdict === "green") {
3731
+ return `base ${row.repo}/${row.branch} green (${row.runsCount} run(s)) at ${head}`;
3732
+ }
3733
+ if (row.verdict === "red") {
3734
+ return `base ${row.repo}/${row.branch} RED — ${row.detail ?? `workflow failed at ${head}`}`;
3735
+ }
3736
+ if (row.verdict === "pending") {
3737
+ return `base ${row.repo}/${row.branch} pending (${row.runsCount} run(s)) at ${head}`;
3738
+ }
3739
+ return (
3740
+ `base ${row.repo}/${row.branch} unknown — ` +
3741
+ (row.detail ?? `no push-triggered workflow run for ${head}`)
3742
+ );
3743
+ });
3744
+ }
3745
+
2920
3746
 
2921
3747
  export function formatStatus(s: StatusSnapshot): string {
2922
3748
  const lines = [
@@ -2935,6 +3761,9 @@ export function formatStatus(s: StatusSnapshot): string {
2935
3761
  // fleet (#110).
2936
3762
  ` plan usage ${planUsageLine(s.planUsage)}`,
2937
3763
  ` new worker turns ${s.caps.workerMaxTurns}`,
3764
+ ...s.turnOverrides.map(
3765
+ ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
3766
+ ),
2938
3767
  ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
2939
3768
  ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
2940
3769
  ` continuations ${s.caps.maxContinuationsPerIssue}`,
@@ -2961,6 +3790,7 @@ export function formatStatus(s: StatusSnapshot): string {
2961
3790
  if (flagged !== undefined) lines.push(` ${flagged}`);
2962
3791
  }
2963
3792
  }
3793
+ lines.push(...formatBaseHealth(s.baseHealth));
2964
3794
  lines.push(...formatSalvagedRuns(s.salvagedRuns));
2965
3795
  lines.push(...formatOpenReports(s.openReports));
2966
3796
  lines.push(...formatVerbLedger(s.verbLedger));
@@ -3096,6 +3926,15 @@ export interface SessionError {
3096
3926
  message: string;
3097
3927
  }
3098
3928
 
3929
+ export function completionLastError(
3930
+ providerCredit: string | undefined,
3931
+ providerTransient: string | undefined,
3932
+ verifiedReason: string | undefined,
3933
+ sessionErr: SessionError | undefined,
3934
+ ): string | undefined {
3935
+ return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
3936
+ }
3937
+
3099
3938
  /**
3100
3939
  * The last error a transcript recorded, or undefined when it recorded none.
3101
3940
  *
@@ -3162,6 +4001,19 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3162
4001
  const { project, tracker, store } = d;
3163
4002
  for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
3164
4003
  const facts: ClassifyFacts = {};
4004
+ let classifiedRun = run;
4005
+ if (run.state === "failed" || run.state === "killed") {
4006
+ const sessionError = readSessionError(run.sessionFile);
4007
+ if (sessionError !== undefined) {
4008
+ if (run.lastError === undefined || run.lastError === sessionError.message) {
4009
+ facts.sessionError = sessionError;
4010
+ }
4011
+ if (run.lastError === undefined) {
4012
+ store.updateRun(run.id, { lastError: sessionError.message });
4013
+ classifiedRun = { ...run, lastError: sessionError.message };
4014
+ }
4015
+ }
4016
+ }
3165
4017
  try {
3166
4018
  if (run.prUrl !== undefined) {
3167
4019
  const pr = await tracker.prState(run.prUrl);
@@ -3188,7 +4040,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3188
4040
  continue;
3189
4041
  }
3190
4042
 
3191
- const { cls, recovery, evidence } = classifyRun(run, facts);
4043
+ const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
3192
4044
 
3193
4045
  // A healthy green PR is not a failure of any class. Leaving the row
3194
4046
  // unclassified is what keeps it eligible for the sweep on the tick where its
@@ -3202,7 +4054,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3202
4054
  ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
3203
4055
  : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
3204
4056
  );
3205
- await recoverRun(d, run, cls, recovery, evidence);
4057
+ await recoverRun(d, classifiedRun, cls, recovery, evidence);
3206
4058
  }
3207
4059
  }
3208
4060
 
@@ -3268,6 +4120,9 @@ async function recoverRun(
3268
4120
  }
3269
4121
 
3270
4122
  if (recovery === "requeue") {
4123
+ if (cls === "provider-credit") {
4124
+ await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
4125
+ }
3271
4126
  // A dispatch-infra requeue that keeps landing on the same issue means the
3272
4127
  // mirror for its repo is persistently broken — a ref-lock that retry already
3273
4128
  // exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
@@ -3682,7 +4537,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3682
4537
  onChildLog: (line) => {
3683
4538
  log(`orchestrator ${line}`);
3684
4539
  },
3685
- onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
4540
+ onReleaseBlocked: (shape, context) =>
4541
+ recordReleaseBlock(project.name, "orchestrator", shape, context),
3686
4542
  });
3687
4543
  const transcript = orchestrator.sessionFile();
3688
4544
  log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
@@ -3696,7 +4552,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3696
4552
  }
3697
4553
  }
3698
4554
 
3699
- const escalator = createEscalator(project, tracker, store, orchestrator);
4555
+ let runtimeDeps: Deps | undefined;
4556
+ const currentProject = (): ProjectConfig => runtimeDeps?.project ?? project;
4557
+ const deliveryPolicyValid = (): boolean => runtimeDeps?.deliveryPolicyValid === true;
4558
+ const escalator = createEscalator(
4559
+ currentProject,
4560
+ tracker,
4561
+ store,
4562
+ orchestrator,
4563
+ Date.now,
4564
+ deliveryPolicyValid,
4565
+ );
3700
4566
 
3701
4567
  // Report delivery is the daemon's, not the model's (#123). Built beside the
3702
4568
  // escalator because a report nobody can deliver pages through it, and driven
@@ -3704,10 +4570,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3704
4570
  // but still owes its operator the report it was handed, and five minutes is a
3705
4571
  // long time to sit on a page.
3706
4572
  const outbox = createReportOutbox({
3707
- project,
4573
+ project: currentProject,
3708
4574
  store,
3709
4575
  escalate: (e) => escalator.escalate(e),
3710
4576
  log,
4577
+ deliveryAllowed: deliveryPolicyValid,
3711
4578
  });
3712
4579
 
3713
4580
  // Every row still `sending` when a daemon boots belonged to a process that is
@@ -3733,6 +4600,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3733
4600
  caps,
3734
4601
  tracker,
3735
4602
  store,
4603
+ // Fail closed until the first tick re-reads and validates the live config.
4604
+ deliveryPolicyValid: false,
3736
4605
  // Process-wide, so a `status` served off this daemon's own HTTP surface
3737
4606
  // reuses the tick's reading instead of shelling out again.
3738
4607
  usage: sharedUsageSource(),
@@ -3747,6 +4616,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3747
4616
  ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
3748
4617
  verbActions,
3749
4618
  };
4619
+ runtimeDeps = d;
3750
4620
 
3751
4621
  if (o.once) {
3752
4622
  try {
@@ -3821,9 +4691,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3821
4691
  // carries no credential of any kind, and it cannot tell one caller from
3822
4692
  // another: `127.0.0.1` is not an identity. The turn-limit and worker
3823
4693
  // pause/resume controls trust a body-supplied `project`, which is exactly the
3824
- // shape "identity from the payload" takes when nobody is watching. Those
3825
- // controls are tolerable only because they are bounded, live-run-local, and
3826
- // reversible.
4694
+ // shape "identity from the payload" takes when nobody is watching. They stay
4695
+ // tolerable only because every effect is bounded: pauses are reversible,
4696
+ // live extensions touch one controller, and a persisted next-attempt
4697
+ // override can only raise the project base up to its configured ceiling and
4698
+ // is consumed by one claim.
3827
4699
  //
3828
4700
  // A merge, a push, a release or a label is none of those things. Do not add
3829
4701
  // one here, and do not add "just a small one" behind a shared secret either:
@@ -3838,6 +4710,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3838
4710
  daemonHttpResponse(req, {
3839
4711
  project: project.name,
3840
4712
  store,
4713
+ caps: () => d.caps,
3841
4714
  turnLimits,
3842
4715
  workerControls,
3843
4716
  health: () =>