omp-conductor 0.10.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -31,13 +31,16 @@ import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
31
31
  import { startOrchestrator } from "./orchestrator.ts";
32
32
  import type { OrchestratorHandle } from "./orchestrator.ts";
33
33
  import { createReportOutbox, formatOpenReports } from "./reports.ts";
34
- import { recordReleaseBlock } from "./release-policy.ts";
34
+ import {
35
+ recordReleaseBlock,
36
+ type ReleaseBlockContext,
37
+ } from "./release-policy.ts";
35
38
  import { branchName, effectiveLabels, route } from "./routing.ts";
36
39
  import type { Routed, UnroutableReason } from "./routing.ts";
37
40
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
38
- import { classifyRun, providerCreditRefusal, type ClassifyFacts } from "./failure-class.ts";
41
+ import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
39
42
  import { projectLabels } from "./label-projection.ts";
40
- import { dbPath, openStore, utcDay } from "./store.ts";
43
+ import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
41
44
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
42
45
  import { RELEASE_SHAPES } from "./types.ts";
43
46
  import type {
@@ -45,7 +48,10 @@ import type {
45
48
  Caps,
46
49
  DispatchSummary,
47
50
  Escalation,
51
+ IssueSnapshot,
52
+ MergedPrInfo,
48
53
  OpenCloser,
54
+ ReleaseShape,
49
55
  PrState,
50
56
  ProjectConfig,
51
57
  ReadyIssue,
@@ -60,8 +66,17 @@ import type {
60
66
  Store,
61
67
  Tracker,
62
68
  VerbLedgerEntry,
69
+ TurnOverride,
63
70
  } from "./types.ts";
64
- import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
71
+ import {
72
+ type KilledBy,
73
+ type WorkerPauseControl,
74
+ type WorkerPausePhase,
75
+ type WorkerResult,
76
+ type RunWorkerDeps,
77
+ renderBrief,
78
+ runWorker,
79
+ } from "./worker.ts";
65
80
  import {
66
81
  addRunRepo,
67
82
  cleanupRetainedWorktree,
@@ -75,8 +90,13 @@ import {
75
90
  } from "./worktree.ts";
76
91
  import { githubVerbActions } from "./verbs/actions.ts";
77
92
  import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
78
- import { listenVerbChannel } from "./verbs/server.ts";
79
- import type { VerbActions, VerbDeps, VerbListener } from "./verbs/server.ts";
93
+ import {
94
+ listenVerbChannel,
95
+ PR_LOOKUP_WINDOW_MS,
96
+ type VerbActions,
97
+ type VerbDeps,
98
+ type VerbListener,
99
+ } from "./verbs/server.ts";
80
100
  import {
81
101
  ensureVerbSocketDir,
82
102
  peerCredentialReader,
@@ -86,7 +106,7 @@ import {
86
106
  } from "./verbs/socket.ts";
87
107
  import { homedir } from "node:os";
88
108
 
89
- import { pushRunBranch, type RunRepoRef } from "./gitops.ts";
109
+ import { pushRunBranch, readBaseChain, type RunRepoRef } from "./gitops.ts";
90
110
  import {
91
111
  planUsageLine,
92
112
  readPlanUsage,
@@ -108,6 +128,14 @@ const REPORT_DELIVERY_INTERVAL_MS = 30_000;
108
128
  * means the mirror itself is broken, not unlucky, and the sweep escalates
109
129
  * instead of burning a turn-0 run per tick forever (#168, #177). */
110
130
  const DISPATCH_INFRA_MAX_STRIKES = 3;
131
+ /** A provider-transient requeue (stream stalled mid-run) is retried, but only a
132
+ * bounded number of times: three aborted streams for one issue means the
133
+ * provider itself is degraded, not unlucky, and the sweep escalates to a
134
+ * human instead of requeueing into a down provider forever (#220). */
135
+ const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
136
+ /** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
137
+ * are maintenance, but a backlog must not turn one tick into an API burst. */
138
+ const SALVAGED_PR_ADOPTION_BATCH = 10;
111
139
  const DEFAULT_PORT = 8787;
112
140
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
113
141
 
@@ -144,6 +172,9 @@ interface Deps {
144
172
  usage: UsageSource;
145
173
  escalate(e: Escalation): Promise<void>;
146
174
  turnLimits: TurnLimitRegistry;
175
+ workerControls: WorkerControlRegistry;
176
+ /** Session seam for lifecycle integration tests; production uses the real harness. */
177
+ workerDeps?: RunWorkerDeps;
147
178
  integrity: IntegrityGate;
148
179
  stall: StallGate;
149
180
  cleanup?: RetainedCleanupCursor;
@@ -190,6 +221,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
190
221
  pausedAt,
191
222
  log,
192
223
  now: () => Date.now(),
224
+ chain: { readBaseChain },
193
225
  };
194
226
  }
195
227
 
@@ -198,18 +230,25 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
198
230
 
199
231
  // ------------------------------------------------------- orchestrator liveness
200
232
 
233
+ /** No more than one reminder per hour while the same marker remains. */
234
+ const STALL_RENOTIFY_MS = 60 * 60_000;
235
+
201
236
  /**
202
- * Whether the wedged-orchestrator page has already gone out for the stall
203
- * currently on disk. One page per episode: the marker persists until a tick is
204
- * consumed, so paging per five minutes would be paging forever.
237
+ * Whether the wedged-orchestrator page has gone out, and when it last went out,
238
+ * for the stall currently on disk.
205
239
  */
206
240
  export interface StallGate {
207
241
  paged: boolean;
242
+ lastPagedAt?: number;
208
243
  }
209
244
 
210
245
  export interface StallVerdict {
211
246
  /** The marker's own line, when one is there. */
212
247
  since?: string;
248
+ /** Count written by the stalled tick producer, when its marker is readable. */
249
+ unconsumedTicks?: number;
250
+ /** Whole hours elapsed since the marker timestamp, when it is parseable. */
251
+ stalledHours?: number;
213
252
  page: boolean;
214
253
  }
215
254
 
@@ -232,50 +271,84 @@ export interface StallVerdict {
232
271
  *
233
272
  * Resets when the marker disappears, so a second stall days later pages again.
234
273
  */
235
- export function checkStall(gate: StallGate, marker: string): StallVerdict {
274
+ export function checkStall(gate: StallGate, marker: string, now = Date.now()): StallVerdict {
236
275
  if (!existsSync(marker)) {
237
276
  gate.paged = false;
277
+ delete gate.lastPagedAt;
238
278
  return { page: false };
239
279
  }
240
- const page = !gate.paged;
280
+ const page =
281
+ !gate.paged ||
282
+ gate.lastPagedAt === undefined ||
283
+ now - gate.lastPagedAt >= STALL_RENOTIFY_MS;
241
284
  let since: string | undefined;
285
+ let unconsumedTicks: number | undefined;
286
+ let stalledHours: number | undefined;
242
287
  try {
243
288
  const body = readFileSync(marker, "utf8").split("\n")[0]?.trim();
244
- if (body !== undefined && body !== "") since = body;
289
+ if (body !== undefined && body !== "") {
290
+ since = body;
291
+ const ticks = /\b(\d+) ticks queued unconsumed\b/.exec(body)?.[1];
292
+ if (ticks !== undefined) unconsumedTicks = Number(ticks);
293
+ const startedAt = Date.parse(body.split(/\s+/, 1)[0] ?? "");
294
+ if (Number.isFinite(startedAt)) {
295
+ stalledHours = Math.max(0, Math.floor((now - startedAt) / STALL_RENOTIFY_MS));
296
+ }
297
+ }
245
298
  } catch {
246
- // An unreadable marker still means stalled; the timestamp is a nicety.
299
+ // An unreadable marker still means stalled; its evidence is a nicety.
247
300
  }
248
- return { ...(since === undefined ? {} : { since }), page };
301
+ return {
302
+ ...(since === undefined ? {} : { since }),
303
+ ...(unconsumedTicks === undefined ? {} : { unconsumedTicks }),
304
+ ...(stalledHours === undefined ? {} : { stalledHours }),
305
+ page,
306
+ };
249
307
  }
250
308
 
251
309
  /**
252
- * Pages tier 2 once when the orchestrator session stops draining its queue.
310
+ * Pages tier 2 when the orchestrator session stops draining its queue.
253
311
  *
254
312
  * Deliberately does not restart anything. A wedge lands mid-turn, this process
255
313
  * cannot tell a half-applied edit from an idle loop, and killing the session
256
314
  * could destroy work an operator would rather read first — the same refusal to
257
315
  * guess that the recovery plugin is built on.
258
316
  */
259
- async function watchOrchestrator(d: Deps): Promise<void> {
317
+ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
260
318
  const marker = join(stateDir(), STALL_MARKER_FILE);
261
- const verdict = checkStall(d.stall, marker);
319
+ const repeat = d.stall.paged;
320
+ const verdict = checkStall(d.stall, marker, now);
262
321
  if (!verdict.page) return;
263
322
 
323
+ const greenRuns = d.store.activeRuns(d.project.name).filter((run) => run.state === "pushed-green");
324
+ const repeatSuffix = repeat
325
+ ? ` — still stalled (${
326
+ verdict.stalledHours === undefined
327
+ ? `${new Date(now).toISOString().slice(0, 13)}Z`
328
+ : `${verdict.stalledHours}h`
329
+ })`
330
+ : "";
331
+
264
332
  log(`ERROR: the orchestrator session is not draining its queue — ${verdict.since ?? "no timestamp"}`);
265
333
  const delivered = await safeEscalate(d, {
266
334
  tier: 2,
335
+ category: "confirmed-failure",
336
+ urgent: true,
267
337
  project: d.project.name,
268
338
  issue: NO_ISSUE,
269
339
  // Keyed on the marker's own timestamp, not the date. The dedup ledger keys
270
340
  // on this summary, and two wedges in one day is not a hypothetical — the
271
341
  // failure mode is a session that gets stuck, gets restarted, and gets stuck
272
- // again on the same cause an hour later. A day-keyed summary would report
273
- // the first and silently swallow every one after it.
342
+ // again on the same cause an hour later. The hourly suffix makes bounded
343
+ // reminders distinct without allowing every five-minute tick through.
274
344
  summary:
275
345
  `Orchestrator session wedged (${verdict.since ?? `marker at ${marker}`}) — ` +
276
- `it has stopped reading its queue (${d.project.name})`,
346
+ `it has stopped reading its queue (${d.project.name})${repeatSuffix}`,
277
347
  detail: [
278
348
  verdict.since ?? "Marker present with no readable timestamp.",
349
+ `Unconsumed ticks: ${verdict.unconsumedTicks ?? "unknown"}`,
350
+ `Open green worker PRs: ${greenRuns.length}`,
351
+ ...greenRuns.map((run) => `- ${run.prUrl ?? `#${run.issue} (PR URL unavailable)`}`),
279
352
  `Marker: ${marker}`,
280
353
  "",
281
354
  "Its process and its herdr agent label are both healthy, which is why nothing else noticed:",
@@ -287,7 +360,7 @@ async function watchOrchestrator(d: Deps): Promise<void> {
287
360
  "Dispatch is unaffected: workers keep running. What stops is drain, groom, report and merge.",
288
361
  ].join("\n"),
289
362
  });
290
- markPaged(d.stall, delivered);
363
+ markPaged(d.stall, delivered, now);
291
364
  }
292
365
 
293
366
  /**
@@ -304,10 +377,10 @@ export function isPaused(): boolean {
304
377
  * The epoch-ms timestamp at which the current pause began, read from the same
305
378
  * sentinel file {@link setPaused} writes (`<stateDir()>/paused`). Returns
306
379
  * `undefined` when the fleet is not paused, or when the file's first line does
307
- * not parse as a date a legacy/blank sentinel keeps today's refuse-everything
308
- * behavior, because a run admitted before an *unknown* pause cannot be proven
309
- * innocent. {@link isPaused} is the authority on *whether*; this answers
310
- * *since when*.
380
+ * not parse as a date. A legacy/blank sentinel keeps completion mutations
381
+ * fail-closed because a run admitted before an *unknown* pause cannot be proven
382
+ * innocent. {@link isPaused} is the authority on *whether*; this answers *since
383
+ * when*.
311
384
  */
312
385
  export function pausedAt(): number | undefined {
313
386
  const f = join(stateDir(), "paused");
@@ -318,8 +391,8 @@ export function pausedAt(): number | undefined {
318
391
  const t = Date.parse(first);
319
392
  return Number.isNaN(t) ? undefined : t;
320
393
  } catch {
321
- // Unreadable sentinel (permissions, corruption): fail closed like an
322
- // unparseable line refuse mutations while the pause is unprovable.
394
+ // Unreadable sentinel (permissions, corruption): fail completion mutations
395
+ // closed like an unparseable line while the pause time is unprovable.
323
396
  return undefined;
324
397
  }
325
398
  }
@@ -344,8 +417,8 @@ export function pauseProvenance(): { source: string; reason?: string } | undefin
344
417
  const reason = match[2];
345
418
  return { source, ...(reason === undefined ? {} : { reason }) };
346
419
  } catch {
347
- // Unreadable sentinel: no provenance to name, and the refuse-everything
348
- // posture of an unknown pause is unchanged.
420
+ // Unreadable sentinel: no provenance to name. Completion mutations still
421
+ // fail closed because the pause time is unknown.
349
422
  return undefined;
350
423
  }
351
424
  }
@@ -456,9 +529,17 @@ export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>
456
529
  return { diff, pause: true, page: !gate.paged };
457
530
  }
458
531
 
459
- /** Latch a once-only page, after delivery is confirmed and never before. */
460
- export function markPaged(gate: { paged: boolean }, delivered: boolean): void {
461
- if (delivered) gate.paged = true;
532
+ /** Latch a page after delivery is confirmed, and never before. */
533
+ export function markPaged(gate: StallGate, delivered: boolean, now: number): void;
534
+ export function markPaged(gate: IntegrityGate, delivered: boolean): void;
535
+ export function markPaged(
536
+ gate: { paged: boolean; lastPagedAt?: number },
537
+ delivered: boolean,
538
+ now?: number,
539
+ ): void {
540
+ if (!delivered) return;
541
+ gate.paged = true;
542
+ if (now !== undefined) gate.lastPagedAt = now;
462
543
  }
463
544
 
464
545
  // ---------------------------------------------------------------------- helpers
@@ -528,6 +609,32 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
528
609
  { issue, op: "remove", label: from },
529
610
  ]);
530
611
  }
612
+ /**
613
+ * Persist the operator-stop transition before releasing its live controller.
614
+ * The row is terminal first, then its in-progress label is removed through the
615
+ * same durable projection outbox as every other lifecycle transition.
616
+ */
617
+ export function recordOperatorStop(
618
+ store: Pick<Store, "updateRun" | "enqueueLabelOps">,
619
+ args: {
620
+ project: string;
621
+ issue: number;
622
+ runId: string;
623
+ inProgress: string;
624
+ reason: string;
625
+ patch: Partial<RunRecord>;
626
+ },
627
+ ): void {
628
+ store.updateRun(args.runId, {
629
+ ...args.patch,
630
+ state: "stopped",
631
+ lastError: `operator stopped: ${args.reason}`,
632
+ });
633
+ store.enqueueLabelOps(args.project, [
634
+ { issue: args.issue, op: "remove", label: args.inProgress },
635
+ ]);
636
+ }
637
+
531
638
 
532
639
  /**
533
640
  * The escalator throws when no transport is configured or Telegram rejects, and
@@ -549,6 +656,38 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
549
656
  }
550
657
  }
551
658
 
659
+ async function reactToProviderCredit(
660
+ d: Deps,
661
+ issue: number,
662
+ message: string,
663
+ sessionFile: string | undefined,
664
+ ): Promise<void> {
665
+ const { project } = d;
666
+ const alreadyPaused = isPaused();
667
+ if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message });
668
+ log(
669
+ `#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
670
+ );
671
+ // Fleet-scoped and run-independent on purpose. The notification ledger
672
+ // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
673
+ // carrying no run or attempt pages once for the fleet, not once per run.
674
+ await safeEscalate(d, {
675
+ tier: 2,
676
+ category: "fleet-stopped",
677
+ project: project.name,
678
+ issue: NO_ISSUE,
679
+ summary: `Model provider refused for credit — ${project.name} is paused`,
680
+ detail: [
681
+ message,
682
+ "",
683
+ "No implementation attempt was charged: this is a billing state, not a",
684
+ "failed implementation. Each affected issue keeps its queue label and",
685
+ "re-dispatches on `omp-conductor resume` once the provider has credit.",
686
+ `Session: ${sessionFile ?? "(no transcript)"}`,
687
+ ].join("\n"),
688
+ });
689
+ }
690
+
552
691
  /**
553
692
  * What a salvage attempt contributes to the escalation: where the work went, or
554
693
  * that it went nowhere. Split from the effects below for the same reason
@@ -823,6 +962,135 @@ export function createTurnLimitRegistry(
823
962
  };
824
963
  }
825
964
 
965
+ export type WorkerControlResult =
966
+ | { kind: "ok"; runId: string; phase: WorkerPausePhase }
967
+ | { kind: "stopped"; runId: string; reason: string }
968
+ | { kind: "refused"; runId: string; error: string }
969
+ | { kind: "not-active" };
970
+
971
+ export interface WorkerControlSlot {
972
+ install(control: WorkerPauseControl): void;
973
+ /** Stop accepted before the session controller exists. */
974
+ requestedStop(): string | undefined;
975
+ close(): void;
976
+ }
977
+
978
+ export interface WorkerControlRegistry {
979
+ open(project: string, issue: number, runId: string): WorkerControlSlot;
980
+ pause(project: string, issue: number): Promise<WorkerControlResult>;
981
+ resume(project: string, issue: number): WorkerControlResult;
982
+ stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
983
+ /** Live runs whose phase is not `running` — what /healthz and the board show. */
984
+ snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase }[];
985
+ }
986
+
987
+ /** Authoritative controls for sessions owned by this daemon process. */
988
+ export function createWorkerControlRegistry(): WorkerControlRegistry {
989
+ interface Entry {
990
+ project: string;
991
+ issue: number;
992
+ runId: string;
993
+ control?: WorkerPauseControl;
994
+ stopReason?: string;
995
+ stopError?: string;
996
+ finished: PromiseWithResolvers<void>;
997
+ }
998
+
999
+ const active = new Map<string, Entry>();
1000
+ const key = (project: string, issue: number): string => `${project}\0${issue}`;
1001
+ return {
1002
+ open(project, issue, runId) {
1003
+ const k = key(project, issue);
1004
+ if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
1005
+ const entry: Entry = {
1006
+ project,
1007
+ issue,
1008
+ runId,
1009
+ finished: Promise.withResolvers<void>(),
1010
+ };
1011
+ active.set(k, entry);
1012
+ return {
1013
+ install: (control) => {
1014
+ if (active.get(k) !== entry) return;
1015
+ entry.control = control;
1016
+ if (entry.stopReason === undefined) return;
1017
+ try {
1018
+ control.stop(entry.stopReason);
1019
+ } catch (err) {
1020
+ entry.stopError = err instanceof Error ? err.message : String(err);
1021
+ }
1022
+ },
1023
+ requestedStop: () => active.get(k) === entry ? entry.stopReason : undefined,
1024
+ close: () => {
1025
+ if (active.get(k) === entry) active.delete(k);
1026
+ entry.finished.resolve();
1027
+ },
1028
+ };
1029
+ },
1030
+ async pause(project, issue) {
1031
+ const entry = active.get(key(project, issue));
1032
+ if (entry?.control === undefined) return { kind: "not-active" };
1033
+ try {
1034
+ await entry.control.pause();
1035
+ return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
1036
+ } catch (err) {
1037
+ return {
1038
+ kind: "refused",
1039
+ runId: entry.runId,
1040
+ error: err instanceof Error ? err.message : String(err),
1041
+ };
1042
+ }
1043
+ },
1044
+ resume(project, issue) {
1045
+ const entry = active.get(key(project, issue));
1046
+ if (entry?.control === undefined) return { kind: "not-active" };
1047
+ try {
1048
+ entry.control.resume();
1049
+ return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
1050
+ } catch (err) {
1051
+ return {
1052
+ kind: "refused",
1053
+ runId: entry.runId,
1054
+ error: err instanceof Error ? err.message : String(err),
1055
+ };
1056
+ }
1057
+ },
1058
+ async stop(project, issue, reason) {
1059
+ const entry = active.get(key(project, issue));
1060
+ if (entry === undefined) return { kind: "not-active" };
1061
+ if (entry.stopReason === undefined) {
1062
+ entry.stopReason = reason;
1063
+ if (entry.control !== undefined) {
1064
+ try {
1065
+ entry.control.stop(reason);
1066
+ } catch (err) {
1067
+ entry.stopReason = undefined;
1068
+ return {
1069
+ kind: "refused",
1070
+ runId: entry.runId,
1071
+ error: err instanceof Error ? err.message : String(err),
1072
+ };
1073
+ }
1074
+ }
1075
+ }
1076
+ await entry.finished.promise;
1077
+ if (entry.stopError !== undefined) {
1078
+ return { kind: "refused", runId: entry.runId, error: entry.stopError };
1079
+ }
1080
+ return { kind: "stopped", runId: entry.runId, reason: entry.stopReason! };
1081
+ },
1082
+ snapshot(project) {
1083
+ const workers: { issue: number; runId: string; phase: WorkerPausePhase }[] = [];
1084
+ for (const entry of active.values()) {
1085
+ if (entry.project !== project || entry.control === undefined) continue;
1086
+ const phase = entry.control.phase();
1087
+ if (phase !== "running") workers.push({ issue: entry.issue, runId: entry.runId, phase });
1088
+ }
1089
+ return workers;
1090
+ },
1091
+ };
1092
+ }
1093
+
826
1094
  export async function verifyPushedGreenClaim(
827
1095
  tracker: Pick<Tracker, "verifyPr">,
828
1096
  claim: Pick<WorkerResult, "prUrl" | "headSha">,
@@ -912,7 +1180,28 @@ export function shouldContinueAfterTurnsCap(f: {
912
1180
  return hasContinuationBudget(f.continuation, f.maxContinuations);
913
1181
  }
914
1182
 
915
- async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1183
+ /** Bind a worker's run identity to every release-policy block it emits. */
1184
+ export function workerReleaseBlockRecorder(
1185
+ project: string,
1186
+ issue: number,
1187
+ runId: string,
1188
+ root = stateDir(),
1189
+ ): (shape: ReleaseShape, context: ReleaseBlockContext) => void {
1190
+ return (shape, context) =>
1191
+ recordReleaseBlock(
1192
+ project,
1193
+ "worker",
1194
+ shape,
1195
+ {
1196
+ ...context,
1197
+ issue,
1198
+ runId,
1199
+ },
1200
+ root,
1201
+ );
1202
+ }
1203
+
1204
+ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
916
1205
  const { project, caps, tracker, store } = d;
917
1206
  const issue = r.issue.number;
918
1207
  const branch = branchName(r.issue);
@@ -926,6 +1215,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
926
1215
  const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
927
1216
  let worktreePath: string | undefined;
928
1217
  let turnLimit: TurnLimitController | undefined;
1218
+ let workerControl: WorkerControlSlot | undefined;
1219
+ let workerSessionInstalled = false;
929
1220
  // The run's own repository. Hoisted for the same reason `worktreePath` is —
930
1221
  // the catch and finally paths have to publish the branch.
931
1222
  let runRepo: RunRepoRef | undefined;
@@ -947,6 +1238,53 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
947
1238
  return pushRunBranch(project, runRepo);
948
1239
  };
949
1240
 
1241
+ /**
1242
+ * A stop can arrive after the run is claimed but before `runWorker` exposes
1243
+ * its controller. Settle that run here instead of turning an operator action
1244
+ * into a dispatch failure or making the HTTP request wait for a session that
1245
+ * will never exist.
1246
+ */
1247
+ const settleStopBeforeSession = async (): Promise<boolean> => {
1248
+ const reason = workerControl?.requestedStop();
1249
+ if (reason === undefined || run === undefined || workerSessionInstalled) return false;
1250
+ turnLimit?.close();
1251
+ turnLimit = undefined;
1252
+ const settlement =
1253
+ worktreePath === undefined
1254
+ ? undefined
1255
+ : await settleWorktree({
1256
+ issue,
1257
+ attempt,
1258
+ ending: `stopped by the operator: ${reason}`,
1259
+ worktree: worktreePath,
1260
+ branch,
1261
+ publish,
1262
+ tree: "remove",
1263
+ mirrorPath,
1264
+ });
1265
+ recordOperatorStop(store, {
1266
+ project: project.name,
1267
+ issue,
1268
+ runId: run.id,
1269
+ inProgress,
1270
+ reason,
1271
+ patch: {
1272
+ endedAt: Date.now(),
1273
+ turns: run.turns,
1274
+ spendUsd: run.spendUsd,
1275
+ worktree: worktreePath ?? run.worktree,
1276
+ report: [
1277
+ "Operator stopped the run before its worker session started.",
1278
+ `Reason: ${reason}`,
1279
+ ...(settlement?.lines ?? []),
1280
+ ].join("\n"),
1281
+ ...settlement?.patch,
1282
+ },
1283
+ });
1284
+ log(`#${issue} stopped by operator before its worker session started: ${reason}`);
1285
+ return true;
1286
+ };
1287
+
950
1288
  try {
951
1289
  // Claim on the STORE first, before anything that can fail. The run row —
952
1290
  // not the label — is the crash-safe guard against double dispatch: rows
@@ -962,8 +1300,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
962
1300
  // Read before this attempt's own row exists, so `latestRun` still means the
963
1301
  // attempt whose work this one inherits.
964
1302
  const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
965
-
966
-
967
1303
  run = store.createRun({
968
1304
  project: project.name,
969
1305
  issue,
@@ -977,10 +1313,20 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
977
1313
  maxTurns: caps.workerMaxTurns,
978
1314
  startedAt: Date.now(),
979
1315
  });
1316
+ const maxTurns = run.maxTurns;
1317
+ const turnOverride = maxTurns > caps.workerMaxTurns ? maxTurns : undefined;
980
1318
  const runId = run.id;
1319
+ if (turnOverride !== undefined) {
1320
+ log(
1321
+ `#${issue} claimed with turn override ${turnOverride} ` +
1322
+ `(base ${caps.workerMaxTurns})`,
1323
+ );
1324
+ }
981
1325
  store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
982
1326
  claimed = true;
983
- turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
1327
+ turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
1328
+ workerControl = d.workerControls.open(project.name, issue, runId);
1329
+ if (await settleStopBeforeSession()) return;
984
1330
 
985
1331
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
986
1332
  // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -989,6 +1335,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
989
1335
  // first attempt. addRunRepo does its own ensureMirror; calling it here too
990
1336
  // would cost a second network fetch per attempt.
991
1337
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
1338
+ if (await settleStopBeforeSession()) return;
992
1339
  const provisioned = await addRunRepo(
993
1340
  r.repo,
994
1341
  project.mirrorRoot,
@@ -998,6 +1345,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
998
1345
  );
999
1346
  worktreePath = provisioned.path;
1000
1347
  runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
1348
+ if (await settleStopBeforeSession()) return;
1001
1349
 
1002
1350
  // The SDK names the transcript itself, so the daemon supplies the parent
1003
1351
  // directory and learns the real path back from the result. Inventing one
@@ -1036,6 +1384,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1036
1384
  },
1037
1385
  { ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
1038
1386
  );
1387
+ if (await settleStopBeforeSession()) return;
1039
1388
 
1040
1389
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
1041
1390
 
@@ -1044,19 +1393,26 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1044
1393
  (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
1045
1394
  );
1046
1395
 
1396
+ const brief = await buildBrief(project, r, branch, worktreePath, {
1397
+ continuation: provisioned.reattached,
1398
+ defaultBranch: r.repo.defaultBranch,
1399
+ ...(provisioned.reattached && priorSalvage !== undefined
1400
+ ? { salvagedSha: priorSalvage }
1401
+ : {}),
1402
+ });
1403
+ if (await settleStopBeforeSession()) return;
1404
+
1047
1405
  let result: WorkerResult;
1048
1406
  try {
1049
1407
  result = await runWorker({
1050
- brief: await buildBrief(project, r, branch, worktreePath, {
1051
- continuation: provisioned.reattached,
1052
- defaultBranch: r.repo.defaultBranch,
1053
- ...(provisioned.reattached && priorSalvage !== undefined
1054
- ? { salvagedSha: priorSalvage }
1055
- : {}),
1056
- }),
1408
+ brief,
1057
1409
  cwd: worktreePath,
1058
1410
  caps,
1059
- maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
1411
+ maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
1412
+ onPauseControl: (control) => {
1413
+ workerSessionInstalled = true;
1414
+ workerControl?.install(control);
1415
+ },
1060
1416
  sessionDir,
1061
1417
  // The session's control socket, under the daemon's own state directory —
1062
1418
  // a child process of the daemon reaches it directly.
@@ -1075,7 +1431,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1075
1431
  },
1076
1432
  ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
1077
1433
  releaseGrants: resolveReleaseGrants(project),
1078
- onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
1434
+ onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
1079
1435
  onTurn: (n) => store.updateRun(runId, { turns: n }),
1080
1436
  onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
1081
1437
  onKilled: () => {
@@ -1087,7 +1443,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1087
1443
  // and a path written at completion is a path nobody can follow live. The
1088
1444
  // completion-time update below writes the same value again, harmlessly.
1089
1445
  onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
1090
- });
1446
+ }, d.workerDeps);
1091
1447
  } finally {
1092
1448
  // This is the authoritative settlement edge for `extend`: close before
1093
1449
  // PR verification or terminal row writes can leave stale `running` state.
@@ -1110,13 +1466,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1110
1466
 
1111
1467
  // Read before the row is written so `lastError` carries the provider's own
1112
1468
  // message — it names the fix and the URL, which no classification can.
1113
- const providerCredit =
1114
- state === "failed" || state === "killed"
1115
- ? (() => {
1116
- const err = readSessionError(result.sessionFile);
1117
- return err === undefined ? undefined : providerCreditRefusal(err);
1118
- })()
1119
- : undefined;
1469
+ const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
1470
+ const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
1471
+ const providerTransient =
1472
+ providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
1120
1473
 
1121
1474
  // The other half of not believing a worker about its own run (#128). The
1122
1475
  // claim being audited is `state: pushed-green`, so the audit runs on the
@@ -1160,8 +1513,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1160
1513
  // `pushed-*` run is the only end that does not salvage: its deliverable is
1161
1514
  // already on a remote branch, whatever is left loose in the tree is by the
1162
1515
  // worker's own account not part of it, and appending a WIP commit would
1163
- // turn the green PR this daemon just verified red. Every other end is
1164
- // continuable, so its tree is treated as work.
1516
+ // turn the green PR this daemon just verified red. Every other end may
1517
+ // contain work, so it is salvaged before the tree's final fate is decided.
1165
1518
  const settlement =
1166
1519
  state === "pushed-green" || state === "pushed-pending" || state === "merged"
1167
1520
  ? undefined
@@ -1169,7 +1522,11 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1169
1522
  issue,
1170
1523
  attempt,
1171
1524
  ending:
1172
- state === "blocked" ? "blocked for an operator decision" : endedBy(result.killedBy),
1525
+ state === "blocked"
1526
+ ? "blocked for an operator decision"
1527
+ : state === "stopped"
1528
+ ? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
1529
+ : endedBy(result.killedBy),
1173
1530
  worktree: worktreePath,
1174
1531
  branch,
1175
1532
  publish,
@@ -1190,8 +1547,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1190
1547
  await removeWorktree(mirrorPath, worktreePath);
1191
1548
  }
1192
1549
 
1193
- store.updateRun(runId, {
1194
- state,
1550
+ const terminalPatch: Partial<RunRecord> = {
1195
1551
  endedAt: Date.now(),
1196
1552
  turns: result.turns,
1197
1553
  spendUsd: result.spendUsd,
@@ -1199,23 +1555,41 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1199
1555
  headSha: result.headSha,
1200
1556
  sessionFile: result.sessionFile,
1201
1557
  // Every terminal state persists the worker's report, not just a green
1202
- // push: the report of a killed attempt is exactly the one a later
1203
- // continuation must pool its disclosures from (#199).
1558
+ // push: a stopped attempt's partial report is still part of its audit trail.
1204
1559
  report: result.report,
1205
- ...(providerCredit !== undefined
1206
- ? { lastError: providerCredit }
1207
- : verified.reason === undefined
1208
- ? {}
1209
- : { lastError: verified.reason }),
1210
1560
  ...settlement?.patch,
1211
1561
  ...(audit === undefined || audit.flags.length === 0
1212
1562
  ? {}
1213
1563
  : { settlementFlags: audit.flags }),
1214
- });
1564
+ };
1565
+ if (state === "stopped") {
1566
+ recordOperatorStop(store, {
1567
+ project: project.name,
1568
+ issue,
1569
+ runId,
1570
+ inProgress,
1571
+ reason: result.stoppedReason ?? "no reason recorded",
1572
+ patch: terminalPatch,
1573
+ });
1574
+ } else {
1575
+ const lastError = completionLastError(
1576
+ providerCredit,
1577
+ providerTransient,
1578
+ verified.reason,
1579
+ sessionErr,
1580
+ );
1581
+ store.updateRun(runId, {
1582
+ ...terminalPatch,
1583
+ state,
1584
+ ...(lastError === undefined ? {} : { lastError }),
1585
+ });
1586
+ }
1215
1587
 
1216
1588
  const salvaged = settlement?.lines ?? [];
1217
1589
 
1218
- if (state === "blocked") {
1590
+ if (state === "stopped") {
1591
+ log(`#${issue} stopped by operator on attempt ${attempt}: ${result.stoppedReason}`);
1592
+ } else if (state === "blocked") {
1219
1593
  swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
1220
1594
  await safeEscalate(d, {
1221
1595
  tier: 1,
@@ -1239,30 +1613,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1239
1613
  });
1240
1614
 
1241
1615
  if (providerCredit !== undefined) {
1242
- // Pause here rather than at classification: the sweep runs on the tick,
1243
- // and three issues each burned an attempt in the fifteen minutes between
1244
- // the first 402 and a human noticing (#220).
1245
- setPaused(true, { source: "provider-credit", reason: providerCredit });
1246
- log(`#${issue} provider refused for credit — dispatch paused: ${providerCredit}`);
1247
- swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
1248
- // Fleet-scoped and run-independent on purpose. The notification ledger
1249
- // dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
1250
- // carrying no run or attempt is what makes this page once for the fleet
1251
- // instead of once per affected run.
1252
- await safeEscalate(d, {
1253
- tier: 2,
1254
- project: project.name,
1255
- issue: NO_ISSUE,
1256
- summary: `Model provider refused for credit — ${project.name} is paused`,
1257
- detail: [
1258
- providerCredit,
1259
- "",
1260
- "No implementation attempt was charged: this is a billing state, not a",
1261
- "failed implementation. Each affected issue keeps its queue label and",
1262
- "re-dispatches on `omp-conductor resume` once the provider has credit.",
1263
- `Session: ${result.sessionFile ?? "(no transcript)"}`,
1264
- ].join("\n"),
1265
- });
1616
+ await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
1617
+ swapToQueue(d, issue, inProgress);
1266
1618
  } else if (continueTurns) {
1267
1619
  // Requeue as one ordered pair: the in-progress removal before the
1268
1620
  // queue add, exactly the order the projector will apply them in (#201).
@@ -1364,6 +1716,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1364
1716
  // inner settlement guard exists. Latch it before any terminal write or await.
1365
1717
  turnLimit?.close();
1366
1718
  turnLimit = undefined;
1719
+ if (await settleStopBeforeSession()) return;
1367
1720
  const detail = errText(err);
1368
1721
  log(`#${issue} errored: ${detail}`);
1369
1722
  // A crash lands anywhere, including mid-edit in a tree holding the only
@@ -1421,6 +1774,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1421
1774
  }
1422
1775
  verbListener = undefined;
1423
1776
  }
1777
+ workerControl?.close();
1778
+ workerControl = undefined;
1424
1779
  }
1425
1780
  }
1426
1781
 
@@ -1538,6 +1893,129 @@ export function releaseInProgress(
1538
1893
  * in the busy set throughout, so no second worker can be sent at the issue while
1539
1894
  * it waits.
1540
1895
  */
1896
+ const BASE_CHECK_BATCH = 20;
1897
+ const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
1898
+ const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
1899
+
1900
+ const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
1901
+
1902
+ const FAILING_WORKFLOW_CONCLUSIONS = new Set([
1903
+ "failure",
1904
+ "cancelled",
1905
+ "timed_out",
1906
+ "action_required",
1907
+ "startup_failure",
1908
+ "stale",
1909
+ ]);
1910
+
1911
+ function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
1912
+ const flags = run.settlementFlags ?? [];
1913
+ return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
1914
+ ? flags
1915
+ : [...flags, flag];
1916
+ }
1917
+
1918
+ /**
1919
+ * Observe Actions on exact merge commits for up to one day. A running workflow
1920
+ * stays quiet and pending; a failure becomes durable evidence on the merged row
1921
+ * and pages exactly once because the row leaves `pending` before delivery.
1922
+ */
1923
+ export async function watchMergedBase(d: Pick<Deps, "project" | "tracker" | "store" | "escalate">): Promise<void> {
1924
+ const now = Date.now();
1925
+ for (const run of d.store.runsNeedingBaseCheck(d.project.name, BASE_CHECK_BATCH)) {
1926
+ if (
1927
+ run.endedAt === undefined ||
1928
+ now - run.endedAt > BASE_CHECK_WINDOW_MS ||
1929
+ run.mergeSha === undefined ||
1930
+ run.baseRef === undefined
1931
+ ) {
1932
+ d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
1933
+ log(`#${run.issue} base check unknown: merge identity is absent or older than 24h`);
1934
+ continue;
1935
+ }
1936
+
1937
+ const repo = d.project.routing.repos[run.repo];
1938
+ const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
1939
+ if (repoIdentity === undefined) {
1940
+ d.store.updateRun(run.id, { baseCheck: "unknown", baseCheckAt: now });
1941
+ log(`#${run.issue} base check unknown: routed repository ${run.repo} has no GitHub identity`);
1942
+ continue;
1943
+ }
1944
+
1945
+ let workflows;
1946
+ try {
1947
+ workflows = await d.tracker.workflowRunsAt(repoIdentity, run.mergeSha);
1948
+ } catch (err) {
1949
+ log(`#${run.issue} base check unavailable (${errText(err)}) — retrying next tick`);
1950
+ continue;
1951
+ }
1952
+ if (workflows === undefined || workflows.length === 0) {
1953
+ log(`#${run.issue} base check unavailable for ${run.mergeSha} — retrying next tick`);
1954
+ continue;
1955
+ }
1956
+ if (workflows.some((workflow) => workflow.status !== "completed")) continue;
1957
+
1958
+ const failed = workflows.find(
1959
+ (workflow) =>
1960
+ workflow.conclusion !== undefined &&
1961
+ FAILING_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
1962
+ );
1963
+ if (failed === undefined) {
1964
+ const unknown = workflows.find(
1965
+ (workflow) =>
1966
+ workflow.conclusion === undefined ||
1967
+ !SUCCESSFUL_WORKFLOW_CONCLUSIONS.has(workflow.conclusion),
1968
+ );
1969
+ if (unknown !== undefined) {
1970
+ log(
1971
+ `#${run.issue} base check has unrecognised completed conclusion ` +
1972
+ `${JSON.stringify(unknown.conclusion)} for ${unknown.name} — retrying next tick`,
1973
+ );
1974
+ continue;
1975
+ }
1976
+ d.store.updateRun(run.id, { baseCheck: "green", baseCheckAt: now });
1977
+ log(`#${run.issue} base ${run.baseRef} green at ${run.mergeSha}`);
1978
+ continue;
1979
+ }
1980
+
1981
+ let previous;
1982
+ try {
1983
+ previous = await d.tracker.previousWorkflowRun(
1984
+ repoIdentity,
1985
+ failed.workflowId,
1986
+ run.baseRef,
1987
+ failed.createdAt,
1988
+ );
1989
+ } catch (err) {
1990
+ log(`#${run.issue} previous ${failed.name} run unavailable (${errText(err)}) — retrying next tick`);
1991
+ continue;
1992
+ }
1993
+ if (previous === undefined || (previous !== null && previous.status !== "completed")) continue;
1994
+ const preexisting =
1995
+ previous !== null &&
1996
+ previous.conclusion !== undefined &&
1997
+ FAILING_WORKFLOW_CONCLUSIONS.has(previous.conclusion);
1998
+ const detail =
1999
+ `${failed.name} failed at ${run.mergeSha} — ${failed.url}` +
2000
+ (preexisting ? " (already red before this merge)" : "");
2001
+ const flag: SettlementFlag = { kind: "base-branch-red", file: "(base branch)", detail };
2002
+ const delivered = await safeEscalate(d, {
2003
+ tier: 1,
2004
+ project: d.project.name,
2005
+ issue: run.issue,
2006
+ runId: run.id,
2007
+ summary: `Base branch ${run.baseRef} is red after merge`,
2008
+ detail,
2009
+ });
2010
+ if (!delivered) continue;
2011
+ d.store.updateRun(run.id, {
2012
+ baseCheck: preexisting ? "red-preexisting" : "red",
2013
+ baseCheckAt: now,
2014
+ settlementFlags: appendSettlementFlag(run, flag),
2015
+ });
2016
+ }
2017
+ }
2018
+
1541
2019
  export async function settlePushedGreen(
1542
2020
  d: Pick<Deps, "project" | "tracker" | "store">,
1543
2021
  ): Promise<void> {
@@ -1569,16 +2047,39 @@ export async function settlePushedGreen(
1569
2047
 
1570
2048
  const settlement = settlementFor(pr, run.prUrl);
1571
2049
  if (settlement !== undefined) {
2050
+ // A mediated merge enters a second, bounded observation phase. Record the
2051
+ // exact merge commit before the row leaves the active set; if GitHub cannot
2052
+ // supply it yet, retry this settlement next tick rather than create a
2053
+ // merged row whose base result can never be attributed.
2054
+ let merged: MergedPrInfo | undefined;
2055
+ if (settlement.state === "merged") {
2056
+ try {
2057
+ merged = await tracker.mergedPrInfo(run.prUrl);
2058
+ } catch (err) {
2059
+ log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
2060
+ continue;
2061
+ }
2062
+ if (merged === undefined) {
2063
+ log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
2064
+ continue;
2065
+ }
2066
+ }
2067
+
1572
2068
  // The label removal and the terminal row are one fact again (#201): the
1573
2069
  // release is enqueued — a durable local write that cannot fail on the
1574
- // tracker — in the same breath as the row is terminalised, so there is
1575
- // no window in which a row beyond every later tick still owes its label.
1576
- // The projector applies it with retry; while pending, the eligibility
1577
- // overlay treats the label as already gone, so #18's
1578
- // permanent-`agent:in-progress` cannot re-form even when GitHub refuses
1579
- // the write.
2070
+ // tracker — in the same breath as the row is terminalised.
1580
2071
  releaseInProgress(d, run.issue, settlement.reason);
1581
- const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
2072
+ const patch: Partial<RunRecord> = {
2073
+ state: settlement.state,
2074
+ endedAt: Date.now(),
2075
+ ...(merged === undefined
2076
+ ? {}
2077
+ : {
2078
+ mergeSha: merged.mergeSha,
2079
+ baseRef: merged.baseRef,
2080
+ baseCheck: "pending",
2081
+ }),
2082
+ };
1582
2083
  if (settlement.state === "failed") patch.lastError = settlement.reason;
1583
2084
  store.updateRun(run.id, patch);
1584
2085
  log(`#${run.issue} settled: ${settlement.reason}`);
@@ -1611,6 +2112,121 @@ export async function settlePushedGreen(
1611
2112
  }
1612
2113
  }
1613
2114
 
2115
+ const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
2116
+ failed: true,
2117
+ killed: true,
2118
+ orphaned: true,
2119
+ blocked: true,
2120
+ };
2121
+
2122
+ /**
2123
+ * Reattaches a recovered PR to the terminal run that owns it (#245).
2124
+ *
2125
+ * A worker can fail before its completion report records `prUrl`, then have its
2126
+ * dirty tree committed and pushed by salvage. If that branch already has a PR,
2127
+ * the orchestrator otherwise has no policy-compliant path to inspect or merge
2128
+ * it: ownership is store-backed. Adoption is deliberately stricter than
2129
+ * admission. The tracker query proves the PR closes this run's issue; exact
2130
+ * branch and canonical repository matches prove it is this run's recovered
2131
+ * work, not an unrelated closer. Missing identity is refusal, never a guess.
2132
+ *
2133
+ * The newest run per issue is inspected, at most ten per tick and only inside
2134
+ * the same 30-day window as mediated PR verbs. The cursor advances through the
2135
+ * full eligible set so persistent non-matches cannot starve older recovered
2136
+ * work. Successful adoption is idempotent because the row gains `prUrl`;
2137
+ * non-matches are logged once per daemon process.
2138
+ */
2139
+ const rejectedSalvagedPrRuns = new Set<string>();
2140
+ const salvagedPrCursor = new Map<string, string>();
2141
+
2142
+ export async function adoptSalvagedPrs(
2143
+ d: Pick<Deps, "project" | "tracker" | "store">,
2144
+ now = Date.now(),
2145
+ ): Promise<void> {
2146
+ const { project, tracker, store } = d;
2147
+ const eligible = store
2148
+ .recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
2149
+ .filter(
2150
+ (run) =>
2151
+ ADOPTABLE_PR_STATES[run.state] === true &&
2152
+ run.prUrl === undefined &&
2153
+ run.branch.trim() !== "",
2154
+ );
2155
+ const previous = salvagedPrCursor.get(project.name);
2156
+ const previousIndex =
2157
+ previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
2158
+ const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
2159
+ const candidates = Array.from(
2160
+ { length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
2161
+ (_, offset) => eligible[(start + offset) % eligible.length]!,
2162
+ );
2163
+ const last = candidates.at(-1);
2164
+ if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
2165
+
2166
+ for (const run of candidates) {
2167
+ const repo = project.routing.repos[run.repo];
2168
+ const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
2169
+ let closers: OpenCloser[];
2170
+ try {
2171
+ closers = await tracker.openClosersFor(run.issue);
2172
+ } catch (err) {
2173
+ log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
2174
+ continue;
2175
+ }
2176
+
2177
+ const closer = closers.find(
2178
+ (candidate) =>
2179
+ candidate.headRefName !== "" &&
2180
+ candidate.headRefName === run.branch &&
2181
+ repo !== undefined &&
2182
+ candidate.repo !== "" &&
2183
+ candidate.repo === repoIdentity,
2184
+ );
2185
+ if (closer === undefined) {
2186
+ if (!rejectedSalvagedPrRuns.has(run.id)) {
2187
+ const observed = closers[0];
2188
+ const reason =
2189
+ observed === undefined
2190
+ ? "no open closing PR"
2191
+ : observed.headRefName === ""
2192
+ ? "closer has no head branch identity"
2193
+ : observed.headRefName !== run.branch
2194
+ ? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
2195
+ : observed.repo === ""
2196
+ ? "closer has no repository identity"
2197
+ : `closer repository ${observed.repo} does not match routed repository`;
2198
+ log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
2199
+ rejectedSalvagedPrRuns.add(run.id);
2200
+ }
2201
+ continue;
2202
+ }
2203
+
2204
+ const flag: SettlementFlag = {
2205
+ kind: "pr-adopted",
2206
+ file: "(recovery)",
2207
+ detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
2208
+ };
2209
+ store.updateRun(run.id, {
2210
+ prUrl: closer.url,
2211
+ settlementFlags: [...(run.settlementFlags ?? []), flag],
2212
+ });
2213
+ rejectedSalvagedPrRuns.delete(run.id);
2214
+ log(
2215
+ `#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
2216
+ (run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
2217
+ );
2218
+ }
2219
+ }
2220
+
2221
+ /** Canonical `owner/repo` identity from a configured network clone URL. */
2222
+ function githubRepo(cloneUrl: string): string | undefined {
2223
+ const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
2224
+ const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
2225
+ normalized,
2226
+ );
2227
+ return match?.[1];
2228
+ }
2229
+
1614
2230
  const RETAINED_CLEANUP_BATCH = 10;
1615
2231
 
1616
2232
  export interface RetainedCleanupCursor {
@@ -1724,12 +2340,14 @@ const HOLD_SAMPLE_SIZE = 5;
1724
2340
  const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
1725
2341
  "parent-lookup-error",
1726
2342
  "open-pr-lookup-error",
2343
+ "issue-state-lookup-error",
1727
2344
  ]);
1728
2345
 
1729
2346
  /** Groups transient decisions into the bounded record exposed by status. */
1730
2347
  export function summarizeDispatch(
1731
2348
  ready: number,
1732
2349
  routed: number,
2350
+ claimed: number,
1733
2351
  admitted: number,
1734
2352
  holds: readonly AdmissionHold[],
1735
2353
  completedAt = Date.now(),
@@ -1745,6 +2363,7 @@ export function summarizeDispatch(
1745
2363
  completedAt,
1746
2364
  ready,
1747
2365
  routed,
2366
+ claimed,
1748
2367
  admitted,
1749
2368
  degraded: holds.some((hold) => DEGRADED_HOLDS.has(hold.reason)),
1750
2369
  holds: [...groups]
@@ -1792,6 +2411,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
1792
2411
  return {
1793
2412
  ...base,
1794
2413
  tier: 2,
2414
+ category: "fleet-stopped",
1795
2415
  // Dated: a meter that breaks again next month is a new incident, not a
1796
2416
  // repeat of this one.
1797
2417
  summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
@@ -1810,6 +2430,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
1810
2430
  return {
1811
2431
  ...base,
1812
2432
  tier: 2,
2433
+ category: "fleet-stopped",
1813
2434
  summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
1814
2435
  detail: [
1815
2436
  plan.detail,
@@ -2104,6 +2725,32 @@ export async function admitCandidates(
2104
2725
  log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
2105
2726
  }
2106
2727
 
2728
+ // The queue comes from GitHub's eventually-consistent search index. Re-read
2729
+ // state and labels directly at the last possible moment so a just-closed or
2730
+ // explicitly dequeued issue cannot turn a stale candidate into another
2731
+ // attempt (#247).
2732
+ let snapshot: IssueSnapshot | undefined;
2733
+ try {
2734
+ snapshot = await tracker.issueSnapshot(issue);
2735
+ } catch {
2736
+ snapshot = undefined;
2737
+ }
2738
+ if (snapshot === undefined) {
2739
+ hold(issue, "issue-state-lookup-error");
2740
+ log(`#${issue} held: issue snapshot check failed — retrying next tick`);
2741
+ continue;
2742
+ }
2743
+ if (snapshot.state === "closed") {
2744
+ hold(issue, "issue-closed");
2745
+ log(`#${issue} skipped: issue is closed (search index lag)`);
2746
+ continue;
2747
+ }
2748
+ if (!snapshot.labels.includes(project.queueLabel)) {
2749
+ hold(issue, "issue-dequeued");
2750
+ log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`);
2751
+ continue;
2752
+ }
2753
+
2107
2754
  admitted.push({ r, attempt: priorRuns + 1 });
2108
2755
  liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
2109
2756
  if (parent !== undefined) {
@@ -2192,6 +2839,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2192
2839
  // while the fleet is parked or workers are still active. Resident workers run
2193
2840
  // through the pool without blocking this five-minute tick.
2194
2841
  await settlePushedGreen(d);
2842
+ try {
2843
+ await watchMergedBase(d);
2844
+ } catch (err) {
2845
+ log(`base-branch check sweep failed: ${errText(err)}`);
2846
+ }
2847
+ try {
2848
+ await adoptSalvagedPrs(d);
2849
+ } catch (err) {
2850
+ log(`salvaged PR adoption sweep failed: ${errText(err)}`);
2851
+ }
2195
2852
 
2196
2853
  // Immediately after settlement and before any routing, so a class is on the
2197
2854
  // row before the next dispatch decision reads its budgets (#132). Above the
@@ -2284,6 +2941,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2284
2941
  if (integrity.page) {
2285
2942
  const delivered = await safeEscalate(d, {
2286
2943
  tier: 2,
2944
+ category: "fleet-stopped",
2287
2945
  project: project.name,
2288
2946
  issue: NO_ISSUE,
2289
2947
  // Dated for the same reason the spend cap is: the dedup key is the
@@ -2327,6 +2985,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2327
2985
  return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
2328
2986
  });
2329
2987
  const { routed, unroutable } = route(effective, project);
2988
+ // Ready issues that route() dropped without a reason carry a state label —
2989
+ // claimed by a live or settling run — so the claimed count is subtraction
2990
+ // rather than a routed-state signal (#228).
2991
+ const claimed = effective.length - routed.length - unroutable.length;
2330
2992
  const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
2331
2993
  issue: u.issue.number,
2332
2994
  reason: `unroutable:${u.reason}`,
@@ -2334,7 +2996,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2334
2996
  const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
2335
2997
  store.recordDispatch(
2336
2998
  project.name,
2337
- summarizeDispatch(ready.length, routed.length, admitted, holds),
2999
+ summarizeDispatch(ready.length, routed.length, claimed, admitted, holds),
2338
3000
  );
2339
3001
  };
2340
3002
 
@@ -2368,6 +3030,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2368
3030
  setPaused(true, { source: "spend-cap", reason: `daily spend reached $${caps.dailySpendUsd}` });
2369
3031
  await safeEscalate(d, {
2370
3032
  tier: 2,
3033
+ category: "fleet-stopped",
2371
3034
  project: project.name,
2372
3035
  issue: NO_ISSUE,
2373
3036
  // Dated so the same cap pages again tomorrow, but only once per day.
@@ -2429,10 +3092,14 @@ export interface DaemonHealthSnapshot {
2429
3092
  paused: boolean;
2430
3093
  activeRuns: number;
2431
3094
  project: string;
3095
+ /** One-shot issue ceilings waiting for the next claim. */
3096
+ turnOverrides: TurnOverride[];
2432
3097
  /** Resident set of this daemon; workers are in-process omp sessions. */
2433
3098
  rssBytes: number;
2434
3099
  dispatch?: DispatchSummary;
2435
3100
  codeGraph?: CodeGraphHealth;
3101
+ /** Live workers in a non-running pause phase; absent/empty = nothing paused. */
3102
+ workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
2436
3103
  }
2437
3104
 
2438
3105
  export function daemonHealthSnapshot(
@@ -2441,24 +3108,35 @@ export function daemonHealthSnapshot(
2441
3108
  paused = isPaused(),
2442
3109
  rssBytes = process.memoryUsage().rss,
2443
3110
  codeGraph?: CodeGraphHealth,
3111
+ workerControls?: WorkerControlRegistry,
2444
3112
  ): DaemonHealthSnapshot {
2445
3113
  const dispatch = store.latestDispatch(project);
2446
3114
  return {
2447
3115
  ok: true,
2448
3116
  paused,
2449
3117
  activeRuns: store.activeRuns(project).length,
3118
+ turnOverrides: store.listTurnOverrides(project),
2450
3119
  project,
2451
3120
  rssBytes,
2452
3121
  ...(dispatch === undefined ? {} : { dispatch }),
2453
3122
  ...(codeGraph?.configured === true ? { codeGraph } : {}),
3123
+ ...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
2454
3124
  };
2455
3125
  }
2456
3126
 
3127
+ const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
3128
+ "failed",
3129
+ "killed",
3130
+ "orphaned",
3131
+ "blocked",
3132
+ ]);
3133
+
2457
3134
  export async function turnLimitResponse(
2458
3135
  req: Request,
2459
3136
  project: string,
2460
- store: Pick<Store, "latestRun">,
3137
+ store: Pick<Store, "latestRun" | "setTurnOverride">,
2461
3138
  registry: TurnLimitRegistry,
3139
+ caps: Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">,
2462
3140
  ): Promise<Response | undefined> {
2463
3141
  const url = new URL(req.url);
2464
3142
  const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
@@ -2492,6 +3170,16 @@ export async function turnLimitResponse(
2492
3170
  }
2493
3171
 
2494
3172
  const issue = Number(match[1]);
3173
+ if ((maxTurns as number) > caps.workerMaxTurnsCeiling) {
3174
+ return Response.json(
3175
+ {
3176
+ error:
3177
+ `#${issue} turn budget ${maxTurns as number} exceeds the ` +
3178
+ `${caps.workerMaxTurnsCeiling}-turn caps.workerMaxTurnsCeiling`,
3179
+ },
3180
+ { status: 422 },
3181
+ );
3182
+ }
2495
3183
  const outcome = registry.extend(project, issue, maxTurns as number);
2496
3184
  if (outcome.kind === "extended") return Response.json(outcome);
2497
3185
  if (outcome.kind === "not-increase") {
@@ -2505,6 +3193,132 @@ export async function turnLimitResponse(
2505
3193
  if (latest === undefined) {
2506
3194
  return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
2507
3195
  }
3196
+ if (!TURN_OVERRIDE_STATES.has(latest.state)) {
3197
+ return Response.json(
3198
+ {
3199
+ error:
3200
+ `#${issue} has no live worker controller; its session already settled ` +
3201
+ `or belongs to another daemon (stored state: ${latest.state})`,
3202
+ },
3203
+ { status: 409 },
3204
+ );
3205
+ }
3206
+ if ((maxTurns as number) <= caps.workerMaxTurns) {
3207
+ return Response.json(
3208
+ {
3209
+ error:
3210
+ `#${issue} next-attempt turn budget must exceed the ` +
3211
+ `${caps.workerMaxTurns}-turn caps.workerMaxTurns base`,
3212
+ },
3213
+ { status: 409 },
3214
+ );
3215
+ }
3216
+ store.setTurnOverride(project, issue, maxTurns as number);
3217
+ return Response.json({
3218
+ kind: "next-attempt",
3219
+ issue,
3220
+ nextAttemptMaxTurns: maxTurns,
3221
+ baseMaxTurns: caps.workerMaxTurns,
3222
+ });
3223
+ }
3224
+
3225
+ export async function workerControlResponse(
3226
+ req: Request,
3227
+ project: string,
3228
+ store: Pick<Store, "latestRun">,
3229
+ registry: WorkerControlRegistry,
3230
+ ): Promise<Response | undefined> {
3231
+ const url = new URL(req.url);
3232
+ const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
3233
+ if (req.method !== "PUT" || match === null) return undefined;
3234
+ if (!req.headers.get("content-type")?.startsWith("application/json")) {
3235
+ return Response.json({ error: "content-type must be application/json" }, { status: 415 });
3236
+ }
3237
+
3238
+ let body: unknown;
3239
+ try {
3240
+ body = await req.json();
3241
+ } catch {
3242
+ return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
3243
+ }
3244
+ if (body === null || typeof body !== "object") {
3245
+ return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
3246
+ }
3247
+ const requestedProject = Reflect.get(body, "project");
3248
+ if (typeof requestedProject !== "string" || requestedProject.length === 0) {
3249
+ return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
3250
+ }
3251
+ if (requestedProject !== project) {
3252
+ return Response.json(
3253
+ { error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
3254
+ { status: 409 },
3255
+ );
3256
+ }
3257
+ const action = match[2] as "pause" | "resume" | "stop";
3258
+ let reason: string | undefined;
3259
+ if (action === "stop") {
3260
+ const rawReason = Reflect.get(body, "reason");
3261
+ if (typeof rawReason !== "string" || rawReason.trim() === "") {
3262
+ return Response.json({ error: "reason must be a non-empty string" }, { status: 400 });
3263
+ }
3264
+ reason = rawReason.trim().replace(/\s+/g, " ");
3265
+ if (reason.length > 500) {
3266
+ return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
3267
+ }
3268
+ }
3269
+
3270
+
3271
+ const issue = Number(match[1]);
3272
+ const outcome =
3273
+ action === "pause"
3274
+ ? await registry.pause(project, issue)
3275
+ : action === "resume"
3276
+ ? registry.resume(project, issue)
3277
+ : await registry.stop(project, issue, reason!);
3278
+ if (outcome.kind === "ok") {
3279
+ return Response.json({ runId: outcome.runId, phase: outcome.phase });
3280
+ }
3281
+ if (outcome.kind === "refused") {
3282
+ return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
3283
+ }
3284
+ if (outcome.kind === "stopped") {
3285
+ const stopped = store.latestRun(project, issue);
3286
+ if (stopped === undefined || stopped.id !== outcome.runId) {
3287
+ return Response.json(
3288
+ { error: `#${issue} stopped, but its terminal run record is unavailable` },
3289
+ { status: 500 },
3290
+ );
3291
+ }
3292
+ if (stopped.state !== "stopped") {
3293
+ return Response.json({
3294
+ outcome: "already-terminal",
3295
+ runId: stopped.id,
3296
+ state: stopped.state,
3297
+ });
3298
+ }
3299
+ return Response.json({
3300
+ outcome: "stopped",
3301
+ runId: stopped.id,
3302
+ state: stopped.state,
3303
+ reason: outcome.reason,
3304
+ ...(stopped.salvageSha === undefined ? {} : { salvageSha: stopped.salvageSha }),
3305
+ ...(stopped.salvageError === undefined ? {} : { salvageError: stopped.salvageError }),
3306
+ worktree: stopped.worktree,
3307
+ });
3308
+ }
3309
+
3310
+
3311
+ const latest = store.latestRun(project, issue);
3312
+ if (latest === undefined) {
3313
+ return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
3314
+ }
3315
+ if (action === "stop" && !LIVE_STATES.includes(latest.state)) {
3316
+ return Response.json({
3317
+ outcome: "already-terminal",
3318
+ runId: latest.id,
3319
+ state: latest.state,
3320
+ });
3321
+ }
2508
3322
  return Response.json(
2509
3323
  {
2510
3324
  error:
@@ -2517,8 +3331,10 @@ export async function turnLimitResponse(
2517
3331
 
2518
3332
  export interface DaemonHttpDeps {
2519
3333
  project: string;
2520
- store: Pick<Store, "latestRun">;
3334
+ store: Pick<Store, "latestRun" | "setTurnOverride">;
3335
+ caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
2521
3336
  turnLimits: TurnLimitRegistry;
3337
+ workerControls: WorkerControlRegistry;
2522
3338
  health: () => DaemonHealthSnapshot;
2523
3339
  }
2524
3340
 
@@ -2526,17 +3342,18 @@ export interface DaemonHttpDeps {
2526
3342
  * The whole HTTP surface, in one named function so a test can pin what is *not*
2527
3343
  * on it.
2528
3344
  *
2529
- * Two routes: a health read, and the turn-limit control. Everything else is
2530
- * 404, and that is the contract see the prohibition at the `Bun.serve` call
2531
- * for why no mutation may be added here. Extracted from the serve callback
2532
- * precisely so "the daemon's HTTP port exposes no mutation route" is something
2533
- * a test asserts rather than something a reviewer has to notice (#126).
3345
+ * Three route families: the health read, turn-limit control, and live-worker
3346
+ * pause/resume/stop control. Everything else is 404. The controls mutate only
3347
+ * daemon-owned live sessions; tracker and repository mutations stay on the
3348
+ * authenticated per-run channel described at the `Bun.serve` call (#126).
2534
3349
  */
2535
3350
  export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
2536
3351
  const url = new URL(req.url);
2537
3352
  if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
2538
- const control = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
2539
- return control ?? new Response("not found\n", { status: 404 });
3353
+ const turnLimit = await turnLimitResponse(req, d.project, d.store, d.turnLimits, d.caps());
3354
+ if (turnLimit !== undefined) return turnLimit;
3355
+ const workerControl = await workerControlResponse(req, d.project, d.store, d.workerControls);
3356
+ return workerControl ?? new Response("not found\n", { status: 404 });
2540
3357
  }
2541
3358
 
2542
3359
  export interface StatusSnapshot {
@@ -2563,6 +3380,8 @@ export interface StatusSnapshot {
2563
3380
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
2564
3381
  * only copy of work the daemon could not save. */
2565
3382
  salvagedRuns: RunRecord[];
3383
+ /** One-shot issue ceilings waiting for the next claim. */
3384
+ turnOverrides: TurnOverride[];
2566
3385
  /** Reports the operator has not provably received: pending, in-flight with an
2567
3386
  * unknown outcome, or written off. An empty list is the only honest way to
2568
3387
  * say "everything authored this cycle actually went out" (#123). */
@@ -2574,6 +3393,8 @@ export interface StatusSnapshot {
2574
3393
  * config does not let it, and an operator who has to know to go looking is an
2575
3394
  * operator who finds out from the tracker instead.
2576
3395
  */
3396
+ /** Newest post-merge base verdict per routed repository within seven days. */
3397
+ baseChecks: RunRecord[];
2577
3398
  verbLedger: VerbLedgerEntry[];
2578
3399
  /** Runs backed by a worker process — the number capacity compares against. */
2579
3400
  liveWorkers: number;
@@ -2637,6 +3458,7 @@ export function statusSnapshotFromStore(
2637
3458
  releaseGrants: resolveReleaseGrants(p),
2638
3459
  activeRuns: store.activeRuns(p.name),
2639
3460
  salvagedRuns: store.salvagedRuns(p.name),
3461
+ turnOverrides: store.listTurnOverrides(p.name),
2640
3462
  openReports: store.openReports(p.name),
2641
3463
  verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
2642
3464
  liveWorkers: store.liveRuns(p.name).length,
@@ -2651,6 +3473,7 @@ export function statusSnapshotFromStore(
2651
3473
  ...(labelOpsPending === 0 || oldestLabelOpAt === undefined
2652
3474
  ? {}
2653
3475
  : { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
3476
+ baseChecks: store.latestBaseChecks(p.name, Date.now() - BASE_STATUS_WINDOW_MS),
2654
3477
  };
2655
3478
  }
2656
3479
 
@@ -2672,7 +3495,7 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
2672
3495
  if (summary === undefined) return "last dispatch (none recorded)";
2673
3496
  const lines = [
2674
3497
  `last dispatch ${new Date(summary.completedAt).toISOString()}${summary.degraded ? " DEGRADED" : ""}`,
2675
- ` candidates ${summary.ready} ready / ${summary.routed} routed`,
3498
+ ` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
2676
3499
  ` admitted ${summary.admitted}`,
2677
3500
  ];
2678
3501
  if (summary.holds.length === 0) {
@@ -2733,6 +3556,21 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
2733
3556
  ...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
2734
3557
  ];
2735
3558
  }
3559
+ export function formatBaseChecks(runs: readonly RunRecord[]): string[] {
3560
+ return runs.flatMap((run) => {
3561
+ if (run.baseCheck === undefined) return [];
3562
+ const branch = run.baseRef ?? "?";
3563
+ const flag = run.settlementFlags?.find((candidate) => candidate.kind === "base-branch-red");
3564
+ const verdict =
3565
+ run.baseCheck === "red"
3566
+ ? `RED — ${flag?.detail ?? "workflow failed after merge"}`
3567
+ : run.baseCheck === "red-preexisting"
3568
+ ? `red before merge${flag === undefined ? "" : ` — ${flag.detail}`}`
3569
+ : run.baseCheck;
3570
+ return [`base ${run.repo}/${branch} ${verdict}`];
3571
+ });
3572
+ }
3573
+
2736
3574
 
2737
3575
  export function formatStatus(s: StatusSnapshot): string {
2738
3576
  const lines = [
@@ -2751,6 +3589,9 @@ export function formatStatus(s: StatusSnapshot): string {
2751
3589
  // fleet (#110).
2752
3590
  ` plan usage ${planUsageLine(s.planUsage)}`,
2753
3591
  ` new worker turns ${s.caps.workerMaxTurns}`,
3592
+ ...s.turnOverrides.map(
3593
+ ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
3594
+ ),
2754
3595
  ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
2755
3596
  ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
2756
3597
  ` continuations ${s.caps.maxContinuationsPerIssue}`,
@@ -2777,6 +3618,7 @@ export function formatStatus(s: StatusSnapshot): string {
2777
3618
  if (flagged !== undefined) lines.push(` ${flagged}`);
2778
3619
  }
2779
3620
  }
3621
+ lines.push(...formatBaseChecks(s.baseChecks));
2780
3622
  lines.push(...formatSalvagedRuns(s.salvagedRuns));
2781
3623
  lines.push(...formatOpenReports(s.openReports));
2782
3624
  lines.push(...formatVerbLedger(s.verbLedger));
@@ -2912,6 +3754,15 @@ export interface SessionError {
2912
3754
  message: string;
2913
3755
  }
2914
3756
 
3757
+ export function completionLastError(
3758
+ providerCredit: string | undefined,
3759
+ providerTransient: string | undefined,
3760
+ verifiedReason: string | undefined,
3761
+ sessionErr: SessionError | undefined,
3762
+ ): string | undefined {
3763
+ return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
3764
+ }
3765
+
2915
3766
  /**
2916
3767
  * The last error a transcript recorded, or undefined when it recorded none.
2917
3768
  *
@@ -2978,6 +3829,19 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
2978
3829
  const { project, tracker, store } = d;
2979
3830
  for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
2980
3831
  const facts: ClassifyFacts = {};
3832
+ let classifiedRun = run;
3833
+ if (run.state === "failed" || run.state === "killed") {
3834
+ const sessionError = readSessionError(run.sessionFile);
3835
+ if (sessionError !== undefined) {
3836
+ if (run.lastError === undefined || run.lastError === sessionError.message) {
3837
+ facts.sessionError = sessionError;
3838
+ }
3839
+ if (run.lastError === undefined) {
3840
+ store.updateRun(run.id, { lastError: sessionError.message });
3841
+ classifiedRun = { ...run, lastError: sessionError.message };
3842
+ }
3843
+ }
3844
+ }
2981
3845
  try {
2982
3846
  if (run.prUrl !== undefined) {
2983
3847
  const pr = await tracker.prState(run.prUrl);
@@ -3004,7 +3868,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3004
3868
  continue;
3005
3869
  }
3006
3870
 
3007
- const { cls, recovery, evidence } = classifyRun(run, facts);
3871
+ const { cls, recovery, evidence } = classifyRun(classifiedRun, facts);
3008
3872
 
3009
3873
  // A healthy green PR is not a failure of any class. Leaving the row
3010
3874
  // unclassified is what keeps it eligible for the sweep on the tick where its
@@ -3018,7 +3882,7 @@ export async function classifyAndRecover(d: Deps): Promise<void> {
3018
3882
  ? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
3019
3883
  : `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
3020
3884
  );
3021
- await recoverRun(d, run, cls, recovery, evidence);
3885
+ await recoverRun(d, classifiedRun, cls, recovery, evidence);
3022
3886
  }
3023
3887
  }
3024
3888
 
@@ -3084,6 +3948,9 @@ async function recoverRun(
3084
3948
  }
3085
3949
 
3086
3950
  if (recovery === "requeue") {
3951
+ if (cls === "provider-credit") {
3952
+ await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
3953
+ }
3087
3954
  // A dispatch-infra requeue that keeps landing on the same issue means the
3088
3955
  // mirror for its repo is persistently broken — a ref-lock that retry already
3089
3956
  // exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
@@ -3104,6 +3971,28 @@ async function recoverRun(
3104
3971
  log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
3105
3972
  return;
3106
3973
  }
3974
+ // Same bound for provider-transient: an issue whose stream keeps stalling
3975
+ // mid-run is requeued free (no attempt, no continuation charged) — but a
3976
+ // provider that aborts three times for one issue is down, and a human has
3977
+ // to check its status before hand-requeueing (#220).
3978
+ if (
3979
+ cls === "provider-transient" &&
3980
+ store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
3981
+ ) {
3982
+ await safeEscalate(d, {
3983
+ tier: 1,
3984
+ project: project.name,
3985
+ issue: run.issue,
3986
+ summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
3987
+ detail: [
3988
+ `The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
3989
+ "Check provider status before requeueing by hand.",
3990
+ ].join("\n"),
3991
+ });
3992
+ store.updateRun(run.id, { recoveredAt: Date.now() });
3993
+ log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
3994
+ return;
3995
+ }
3107
3996
  // Only when the tracker still shows this issue as ours to hand back. An
3108
3997
  // issue that is closed, or has no state label, was resolved by another route
3109
3998
  // and requeueing it would dispatch work nobody asked for.
@@ -3476,7 +4365,8 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3476
4365
  onChildLog: (line) => {
3477
4366
  log(`orchestrator ${line}`);
3478
4367
  },
3479
- onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
4368
+ onReleaseBlocked: (shape, context) =>
4369
+ recordReleaseBlock(project.name, "orchestrator", shape, context),
3480
4370
  });
3481
4371
  const transcript = orchestrator.sessionFile();
3482
4372
  log(`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}`);
@@ -3521,6 +4411,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3521
4411
  const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
3522
4412
  store.updateRun(runId, { maxTurns });
3523
4413
  });
4414
+ const workerControls = createWorkerControlRegistry();
3524
4415
  const d: Deps = {
3525
4416
  project,
3526
4417
  caps,
@@ -3531,6 +4422,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3531
4422
  usage: sharedUsageSource(),
3532
4423
  escalate: (e) => escalator.escalate(e),
3533
4424
  turnLimits,
4425
+ workerControls,
3534
4426
  integrity,
3535
4427
  // Fresh per daemon run, like the integrity gate: a restart is entitled to
3536
4428
  // page again about a stall that is still on disk.
@@ -3607,14 +4499,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3607
4499
  process.on("SIGINT", stop);
3608
4500
  process.on("SIGTERM", stop);
3609
4501
 
3610
- // ── NO MUTATION ROUTE BELONGS ON THIS PORT ────────────────────────────────
4502
+ // ── NO TRACKER OR REPOSITORY MUTATION BELONGS ON THIS PORT ───────────────
3611
4503
  //
3612
4504
  // This is unauthenticated loopback TCP. Every local user can reach it, it
3613
4505
  // carries no credential of any kind, and it cannot tell one caller from
3614
- // another: `127.0.0.1` is not an identity. `turnLimitResponse` already shows
3615
- // what that costs — it trusts a body-supplied `project`, which is exactly the
3616
- // shape "identity from the payload" takes when nobody is watching, and it is
3617
- // tolerable only because raising a turn ceiling is bounded and reversible.
4506
+ // another: `127.0.0.1` is not an identity. The turn-limit and worker
4507
+ // pause/resume controls trust a body-supplied `project`, which is exactly the
4508
+ // shape "identity from the payload" takes when nobody is watching. They stay
4509
+ // tolerable only because every effect is bounded: pauses are reversible,
4510
+ // live extensions touch one controller, and a persisted next-attempt
4511
+ // override can only raise the project base up to its configured ceiling and
4512
+ // is consumed by one claim.
3618
4513
  //
3619
4514
  // A merge, a push, a release or a label is none of those things. Do not add
3620
4515
  // one here, and do not add "just a small one" behind a shared secret either:
@@ -3629,8 +4524,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3629
4524
  daemonHttpResponse(req, {
3630
4525
  project: project.name,
3631
4526
  store,
4527
+ caps: () => d.caps,
3632
4528
  turnLimits,
3633
- health: () => daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph),
4529
+ workerControls,
4530
+ health: () =>
4531
+ daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph, workerControls),
3634
4532
  }),
3635
4533
  });
3636
4534
  log(`serving /healthz on :${server.port}, project ${project.name}`);