omp-conductor 0.18.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +105 -40
  2. package/REFERENCE.md +865 -30
  3. package/package.json +1 -1
  4. package/schema/config.schema.json +26 -0
  5. package/src/admission.ts +212 -26
  6. package/src/ask.ts +288 -1
  7. package/src/briefs/orchestrator.md +6 -5
  8. package/src/cli.ts +5 -1
  9. package/src/command-help.ts +9 -1
  10. package/src/command-manifest.ts +36 -3
  11. package/src/commands/arm.ts +5 -1
  12. package/src/commands/context.ts +2 -0
  13. package/src/commands/message.ts +26 -2
  14. package/src/commands/reconcile-units.ts +104 -0
  15. package/src/commands/release-composition.ts +232 -0
  16. package/src/commands/resume.ts +2 -27
  17. package/src/commands/setup.ts +101 -16
  18. package/src/commands/stats.ts +11 -30
  19. package/src/commands/tail.ts +31 -1
  20. package/src/commands/upgrade.ts +20 -3
  21. package/src/commands/verb.ts +2 -1
  22. package/src/config-schema.ts +19 -0
  23. package/src/config.ts +80 -0
  24. package/src/credential-class.ts +366 -0
  25. package/src/daemon.ts +1218 -288
  26. package/src/dashboard/app.js +504 -2
  27. package/src/dashboard/controls.ts +336 -0
  28. package/src/dashboard/index.html +30 -0
  29. package/src/dashboard/server.ts +271 -30
  30. package/src/dashboard/style.css +116 -0
  31. package/src/dashboard/transcript.ts +173 -0
  32. package/src/doctor.ts +377 -20
  33. package/src/failure-class.ts +59 -0
  34. package/src/fleet.ts +497 -15
  35. package/src/host.ts +6 -130
  36. package/src/omp.ts +29 -0
  37. package/src/orchestrator-tick.ts +343 -88
  38. package/src/pause.ts +233 -0
  39. package/src/settlement.ts +159 -2
  40. package/src/setup-answers.ts +97 -0
  41. package/src/setup-host.ts +321 -1155
  42. package/src/setup-install.ts +204 -27
  43. package/src/setup-wizard.ts +111 -50
  44. package/src/setup.ts +33 -0
  45. package/src/spend-telemetry.ts +117 -0
  46. package/src/stats.ts +35 -0
  47. package/src/status-render.ts +348 -19
  48. package/src/store.ts +1229 -55
  49. package/src/telegram-freshness.ts +269 -0
  50. package/src/to-spec.ts +27 -0
  51. package/src/types.ts +697 -4
  52. package/src/unblock.ts +22 -0
  53. package/src/unit-reconcile.ts +303 -0
  54. package/src/upgrade-verify.ts +8 -1
  55. package/src/upgrade.ts +299 -12
  56. package/src/verbs/actions.ts +124 -10
  57. package/src/verbs/protocol.ts +70 -2
  58. package/src/verbs/server.ts +447 -8
  59. package/src/wake.ts +48 -0
  60. package/src/worker.ts +403 -3
package/src/daemon.ts CHANGED
@@ -39,11 +39,35 @@ import { graphHint } from "./graph.ts";
39
39
  import { hostConstraintsNotice } from "./host.ts";
40
40
  import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
41
41
  import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
42
- import { runDoctor } from "./doctor.ts";
42
+ import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS, runDoctor } from "./doctor.ts";
43
+ import { judgeSpendTelemetry, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
44
+ import {
45
+ clearPauseIfUnchanged,
46
+ isPaused,
47
+ pausedAt,
48
+ pausedPath,
49
+ pauseInstance,
50
+ pauseInstanceAt,
51
+ pauseProvenance,
52
+ pauseSourceToken,
53
+ setPaused,
54
+ } from "./pause.ts";
43
55
  import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
44
56
  import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
57
+ import { inspectSurfaces, type InstalledSurfaces } from "./upgrade.ts";
58
+ import { checkTelegramFreshness, type TelegramFreshness } from "./telegram-freshness.ts";
45
59
  import { processStartTimeMs } from "./upgrade-verify.ts";
46
- import { fleetLayers, herdrPaneOmpStarts, resolveHerdrSession } from "./fleet.ts";
60
+ import {
61
+ fleetLayers,
62
+ herdrPaneOmpStarts,
63
+ openWorkerPane,
64
+ reconcileWorkerPanes,
65
+ releaseOrphanedWorkerPane,
66
+ releaseWorkerPane,
67
+ reportWorkerPaneState,
68
+ resolveHerdrSession,
69
+ type WorkerPaneOutcome,
70
+ } from "./fleet.ts";
47
71
  import { startOrchestrator } from "./orchestrator.ts";
48
72
  import type { OrchestratorHandle } from "./orchestrator.ts";
49
73
  import {
@@ -71,11 +95,18 @@ import {
71
95
  effectiveModel,
72
96
  hasContinuationBudget,
73
97
  hasFailedAttemptBudget,
98
+ runSpendAllowanceUsd,
99
+ startOfToday,
74
100
  laneEcho,
75
101
  } from "./admission.ts";
76
102
  import type { Admission, AdmissionHold } from "./admission.ts";
77
103
  import type { EffectiveModel, FileLane } from "./types.ts";
78
104
  import { log, errText, safeEscalate } from "./log.ts";
105
+ import {
106
+ oauthFenceVerdict,
107
+ probeCredentialClass,
108
+ type CredentialClassProbeResult,
109
+ } from "./credential-class.ts";
79
110
  import {
80
111
  adoptSalvagedPrs,
81
112
  classifyAndRecover,
@@ -85,6 +116,7 @@ import {
85
116
  reactToProviderCredit,
86
117
  readSessionError,
87
118
  reconcileOrphanedRuns,
119
+ reconcileGroomingClosures,
88
120
  reconcileStaleLabels,
89
121
  recordOperatorStop,
90
122
  settlePushedGreen,
@@ -93,7 +125,13 @@ import {
93
125
  } from "./settlement.ts";
94
126
  import { materializeOmpSettings } from "./omp-settings.ts";
95
127
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
96
- import { infraLogSignature, infraSignatureVersion, providerCreditRefusal, providerTransientFault } from "./failure-class.ts";
128
+ import {
129
+ infraLogSignature,
130
+ infraSignatureVersion,
131
+ providerCreditRefusal,
132
+ providerTransientFault,
133
+ reviewRoundNeverWorked,
134
+ } from "./failure-class.ts";
97
135
  import {
98
136
  DEFAULT_MODEL_FALLBACK_THRESHOLD,
99
137
  fallbackClause,
@@ -112,10 +150,11 @@ import {
112
150
  utcDay,
113
151
  } from "./store.ts";
114
152
  import { GhPrMissingError, GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
115
- import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
153
+ import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES, REVIEW_ROUND_INFRA_MAX_RETRIES } from "./types.ts";
116
154
  import type {
117
155
  BaseFreeze,
118
156
  BaseHealth,
157
+ InstallSurfaceObservation,
119
158
  AdmissionHoldReason,
120
159
  Caps,
121
160
  ConductorConfig,
@@ -125,6 +164,8 @@ import type {
125
164
  InterruptCategory,
126
165
  IssueComment,
127
166
  IssueSnapshot,
167
+ PrDiff,
168
+ ReviewAdjudicationRecord,
128
169
  MergedPrInfo,
129
170
  OpenCloser,
130
171
  ReleaseShape,
@@ -141,6 +182,7 @@ import type {
141
182
  FailureClass,
142
183
  HostConstraints,
143
184
  RecoveryAction,
185
+ ReviewHeadBlocker,
144
186
  ReviewPolicy,
145
187
  ReviewRevisionOutcome,
146
188
  ReviewRevisionRecord,
@@ -154,14 +196,19 @@ import type {
154
196
  WorkflowRun,
155
197
  } from "./types.ts";
156
198
  import {
199
+ type AdjudicationRound,
200
+ type AdjudicatorOpts,
201
+ type AdjudicationResult,
157
202
  type KilledBy,
158
203
  type WorkerPauseControl,
159
204
  type WorkerPausePhase,
160
205
  type WorkerResult,
161
206
  type RunWorkerDeps,
162
207
  ORPHAN_RESUME_PROMPT,
208
+ renderAdjudicationBrief,
163
209
  renderBrief,
164
210
  renderReviewRevisionPrompt,
211
+ runAdjudicator,
165
212
  runWorker,
166
213
  } from "./worker.ts";
167
214
  import {
@@ -332,6 +379,17 @@ interface Deps {
332
379
  * probe the way `criticalBase` does.
333
380
  */
334
381
  probeWorktreeLane?: RunLaneProbe;
382
+ /**
383
+ * Reads one provider's current credential class from the harness, out of
384
+ * process (#852). Wired by `runDaemon` to {@link probeCredentialClass}; a test
385
+ * injects a fake.
386
+ *
387
+ * Absent, both fences refuse a project that declares `requireOauthProviders` —
388
+ * this one fails CLOSED, unlike `probeWorktreeLane`, because an unverified
389
+ * credential class costs exactly what a wrong one costs. A project declaring
390
+ * nothing never calls it, so an unwired test dispatches as it always has.
391
+ */
392
+ probeCredentialClass?: (provider: string) => Promise<CredentialClassProbeResult>;
335
393
  /**
336
394
  * Reads one issue's tracker state in a repository the admission tracker is
337
395
  * not bound to — the cross-repo Depends-on interlock (#420). Wired by
@@ -346,6 +404,24 @@ interface Deps {
346
404
  * body fails that branch closed rather than synthesising a cycle.
347
405
  */
348
406
  probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
407
+ /**
408
+ * Reads the three install identities this host carries (#919). Wired by
409
+ * `runDaemon` to `inspectSurfaces`, the same seam `doctor` and `upgrade`
410
+ * read, so nothing re-implements the probe. A test injects its own; absent,
411
+ * the pass records nothing and every cheap surface honestly says "not
412
+ * observed yet" rather than claiming agreement.
413
+ */
414
+ probeInstallSurfaces?: () => Promise<InstalledSurfaces>;
415
+ /** The `omp-telegram` install/daemon/published triple, read on the same
416
+ * periodic pass as the surfaces above (#961). */
417
+ probeTelegramFreshness?: () => Promise<TelegramFreshness>;
418
+ /**
419
+ * Runs one review-ceiling adjudication (#932). Production is
420
+ * {@link runAdjudicator}; a test injects a fake so the assertion can be the
421
+ * launch arguments and the assembled brief — which is what this pass actually
422
+ * produces — rather than a stored flag.
423
+ */
424
+ runAdjudicatorImpl?: (opts: AdjudicatorOpts) => Promise<AdjudicationResult>;
349
425
  }
350
426
 
351
427
  /**
@@ -375,6 +451,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
375
451
  log,
376
452
  now: () => Date.now(),
377
453
  chain: { readBaseChain },
454
+ lane: { probeRunLane },
378
455
  };
379
456
  }
380
457
 
@@ -516,180 +593,21 @@ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void
516
593
  markPaged(d.stall, delivered, now);
517
594
  }
518
595
 
519
- /**
520
- * Pause is a file rather than process state on purpose: `omp-conductor hold`
521
- * runs in a different process from the daemon, and a flag
522
- * on disk needs no IPC and survives a restart. A daemon that crashed while
523
- * paused comes back paused.
524
- */
525
- export function pausedPath(project?: string): string {
526
- return join(stateDir(), project === undefined ? "paused" : `paused-${project}`);
527
- }
528
-
529
- function activePausePaths(project?: string): string[] {
530
- const paths = project === undefined ? [pausedPath()] : [pausedPath(project), pausedPath()];
531
- return paths.filter((path) => existsSync(path));
532
- }
533
-
534
- export function isPaused(project?: string): boolean {
535
- return activePausePaths(project).length !== 0;
536
- }
537
-
538
- /**
539
- * The epoch-ms timestamp at which the current pause began. A project pause also
540
- * observes the legacy bare sentinel, which pauses every project. If any active
541
- * sentinel is unreadable or unparseable, the timestamp is unknown so callers
542
- * continue to fail closed.
543
- */
544
- export function pausedAt(project?: string): number | undefined {
545
- const paths = activePausePaths(project);
546
- if (paths.length === 0) return undefined;
547
- const times: number[] = [];
548
- for (const path of paths) {
549
- try {
550
- const first = readFileSync(path, "utf8").split("\n")[0]?.trim();
551
- if (first === undefined || first === "") return undefined;
552
- const time = Date.parse(first);
553
- if (Number.isNaN(time)) return undefined;
554
- times.push(time);
555
- } catch {
556
- return undefined;
557
- }
558
- }
559
- return Math.min(...times);
560
- }
561
-
562
- /**
563
- * Who paused the project and why. Per-project provenance wins when both its
564
- * sentinel and the legacy all-project sentinel are active.
565
- */
566
- export function pauseProvenance(
567
- project?: string,
568
- ): { source: string; reason?: string } | undefined {
569
- const paths =
570
- project === undefined
571
- ? [pausedPath()]
572
- : [pausedPath(project), pausedPath()];
573
- const path = paths.find((candidate) => existsSync(candidate));
574
- if (path === undefined) return undefined;
575
- try {
576
- const second = readFileSync(path, "utf8").split("\n")[1];
577
- if (second === undefined) return undefined;
578
- const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(second.trim());
579
- if (match === null) return undefined;
580
- const source = match[1]!;
581
- const reason = match[2];
582
- return { source, ...(reason === undefined ? {} : { reason }) };
583
- } catch {
584
- return undefined;
585
- }
586
- }
587
-
588
- /**
589
- * One pause sentinel FILE read as an instance identity: who set it, why, and
590
- * the creation instant, all from that exact file. An unreadable or malformed
591
- * file is undefined — the caller may treat it as absence.
592
- */
593
- function pauseInstanceAt(
594
- path: string,
595
- ): { source: string; reason?: string; since: number } | undefined {
596
- try {
597
- const [line1, line2] = readFileSync(path, "utf8").split("\n");
598
- const since = Date.parse(line1?.trim() ?? "");
599
- if (!Number.isFinite(since)) return undefined;
600
- if (line2 === undefined) return undefined;
601
- const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
602
- if (match === null) return undefined;
603
- const source = match[1]!;
604
- const reason = match[2];
605
- return { source, since, ...(reason === undefined ? {} : { reason }) };
606
- } catch {
607
- return undefined;
608
- }
609
- }
610
-
611
- /**
612
- * One pause sentinel read as a single identity: who set it, why, and the
613
- * creation instant, all from the SAME file that was selected. Unlike pairing
614
- * {@link pauseProvenance} with {@link pausedAt} — which can describe
615
- * different files when a project pause coexists with the legacy global
616
- * sentinel, letting a stale global timestamp mask a recreated project pause —
617
- * this reads provenance and timestamp from one sentinel, so a caller can prove
618
- * "the pause I set still exists" instead of "some pause with the same labels
619
- * still exists" (#377). Per-project sentinel wins, like {@link pauseProvenance}.
620
- */
621
- export function pauseInstance(
622
- project?: string,
623
- ): { source: string; reason?: string; since: number } | undefined {
624
- const paths =
625
- project === undefined
626
- ? [pausedPath()]
627
- : [pausedPath(project), pausedPath()];
628
- const path = paths.find((candidate) => existsSync(candidate));
629
- if (path === undefined) return undefined;
630
- return pauseInstanceAt(path);
631
- }
632
-
633
- /**
634
- * Compare-and-clear one pause sentinel (#780 review): remove `path` only while
635
- * it still holds exactly the `expected` instance — same source, same reason,
636
- * same creation instant — as read by {@link pauseInstance}. A hold or pause
637
- * that replaced or recreated the sentinel between the read and the clear is a
638
- * newer instance (writes always re-stamp `since`), so it is never destroyed:
639
- * returning false keeps the newer fence in force. Scoped strictly to one
640
- * caller-provided path, so auto-expiry can clear the per-project spend-cap
641
- * sentinel without ever touching the legacy global sentinel.
642
- */
643
- export function clearPauseIfUnchanged(
644
- path: string,
645
- expected: { source: string; reason?: string; since: number },
646
- ): boolean {
647
- const current = pauseInstanceAt(path);
648
- if (current === undefined) return false;
649
- if (
650
- current.source !== expected.source ||
651
- current.since !== expected.since ||
652
- current.reason !== expected.reason
653
- ) {
654
- return false;
655
- }
656
- rmSync(path, { force: true });
657
- return true;
658
- }
659
-
660
- /**
661
- * The pause sentinel's `source=` token, proven from a verb. The sentinel's
662
- * source line is read back as a single `\S+` token (see {@link pauseInstance}
663
- * and {@link pauseProvenance}), so a verb that contains a space (`setup host`)
664
- * is unrepresentable verbatim and must be encoded before it reaches disk —
665
- * otherwise the fence cannot prove its own pause and refuses forever (#552).
666
- * Spaces become `-`; the human-readable verb is preserved in the sentinel's
667
- * `reason=` instead.
668
- */
669
- export function pauseSourceToken(verb: string): string {
670
- return verb.trim().replace(/\s+/g, "-");
671
- }
672
-
673
- export function setPaused(
674
- v: boolean,
675
- why?: { source: string; reason?: string },
676
- project?: string,
677
- ): void {
678
- const path = pausedPath(project);
679
- if (v) {
680
- mkdirSync(dirname(path), { recursive: true });
681
- const line1 = `${new Date().toISOString()}\n`;
682
- if (why === undefined) {
683
- writeFileSync(path, line1);
684
- } else {
685
- const reason = why.reason === undefined ? "" : ` reason="${why.reason.replaceAll('"', "")}"`;
686
- writeFileSync(path, `${line1}source=${why.source}${reason}\n`);
687
- }
688
- } else {
689
- rmSync(path, { force: true });
690
- if (project !== undefined) rmSync(pausedPath(), { force: true });
691
- }
692
- }
596
+ // ------------------------------------------------------------------- pause
597
+ // The sentinel itself lives in `pause.ts` (#938): this module imports
598
+ // `doctor.ts`, so a health check that needs to read a fence cannot import back
599
+ // here. Re-exported so every existing `from "./daemon.ts"` import still
600
+ // resolves the move is structural, not a change of surface.
601
+ export {
602
+ clearPauseIfUnchanged,
603
+ isPaused,
604
+ pausedAt,
605
+ pausedPath,
606
+ pauseInstance,
607
+ pauseProvenance,
608
+ pauseSourceToken,
609
+ setPaused,
610
+ };
693
611
 
694
612
  // ------------------------------------------------------------------- drain
695
613
  // (#484 slice 1) A project drain is a durable, self-expiring admission fence:
@@ -1090,19 +1008,6 @@ export function markPaged(
1090
1008
 
1091
1009
  // ---------------------------------------------------------------------- helpers
1092
1010
 
1093
- /**
1094
- * Local midnight, matching how a human reads "today".
1095
- *
1096
- * ponytail: a rolling 24h window would be fairer to a run that started at
1097
- * 23:50, but midnight is what someone checking a morning spend report expects.
1098
- * Upgrade path is a `capWindow: "day" | "rolling24h"` config key.
1099
- */
1100
- function startOfToday(): number {
1101
- const d = new Date();
1102
- d.setHours(0, 0, 0, 0);
1103
- return d.getTime();
1104
- }
1105
-
1106
1011
  /**
1107
1012
  * `owner/repo` for `gh`, derived from the clone URL.
1108
1013
  *
@@ -1300,6 +1205,7 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
1300
1205
  function endedBy(killedBy: KilledBy | undefined): string {
1301
1206
  if (killedBy === "turns") return "killed by the turns cap";
1302
1207
  if (killedBy === "wallclock") return "killed by the wall-clock cap";
1208
+ if (killedBy === "spend") return "killed by the per-run spend cap";
1303
1209
  return "killed by a failed run";
1304
1210
  }
1305
1211
 
@@ -1505,7 +1411,19 @@ export interface WorkerControlSlot {
1505
1411
  }
1506
1412
 
1507
1413
  export interface WorkerControlRegistry {
1508
- open(project: string, issue: number, runId: string): WorkerControlSlot;
1414
+ /**
1415
+ * `onPhase` is called after a pause/resume/stop that actually changed this
1416
+ * worker's phase (#842) — the authoritative transition, from the same control
1417
+ * that performed it. It exists so a surface outside this registry (the run's
1418
+ * Herdr representation) can follow the phase without polling and without
1419
+ * inventing a second notion of "paused".
1420
+ */
1421
+ open(
1422
+ project: string,
1423
+ issue: number,
1424
+ runId: string,
1425
+ onPhase?: (phase: WorkerPausePhase) => void,
1426
+ ): WorkerControlSlot;
1509
1427
  pause(project: string, issue: number): Promise<WorkerControlResult>;
1510
1428
  resume(project: string, issue: number): WorkerControlResult;
1511
1429
  stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
@@ -1523,12 +1441,13 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1523
1441
  stopReason?: string;
1524
1442
  stopError?: string;
1525
1443
  finished: PromiseWithResolvers<void>;
1444
+ onPhase?: (phase: WorkerPausePhase) => void;
1526
1445
  }
1527
1446
 
1528
1447
  const active = new Map<string, Entry>();
1529
1448
  const key = (project: string, issue: number): string => `${project}\0${issue}`;
1530
1449
  return {
1531
- open(project, issue, runId) {
1450
+ open(project, issue, runId, onPhase) {
1532
1451
  const k = key(project, issue);
1533
1452
  if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
1534
1453
  const entry: Entry = {
@@ -1536,6 +1455,7 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1536
1455
  issue,
1537
1456
  runId,
1538
1457
  finished: Promise.withResolvers<void>(),
1458
+ ...(onPhase === undefined ? {} : { onPhase }),
1539
1459
  };
1540
1460
  active.set(k, entry);
1541
1461
  return {
@@ -1561,7 +1481,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1561
1481
  if (entry?.control === undefined) return { kind: "not-active" };
1562
1482
  try {
1563
1483
  await entry.control.pause();
1564
- return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
1484
+ const phase = entry.control.phase();
1485
+ entry.onPhase?.(phase);
1486
+ return { kind: "ok", runId: entry.runId, phase };
1565
1487
  } catch (err) {
1566
1488
  return {
1567
1489
  kind: "refused",
@@ -1575,7 +1497,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1575
1497
  if (entry?.control === undefined) return { kind: "not-active" };
1576
1498
  try {
1577
1499
  entry.control.resume();
1578
- return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
1500
+ const phase = entry.control.phase();
1501
+ entry.onPhase?.(phase);
1502
+ return { kind: "ok", runId: entry.runId, phase };
1579
1503
  } catch (err) {
1580
1504
  return {
1581
1505
  kind: "refused",
@@ -1936,6 +1860,32 @@ export async function handleIssue(
1936
1860
  let turnLimit: TurnLimitController | undefined;
1937
1861
  let workerControl: WorkerControlSlot | undefined;
1938
1862
  let workerSessionInstalled = false;
1863
+ // The run's Herdr representation (#840), hoisted for the same reason the verb
1864
+ // listener is: whoever tears the run down has to hand the pane's lifecycle
1865
+ // authority back, and a representation that outlives its child is a worker the
1866
+ // workspace still shows as live.
1867
+ let workerPane: Extract<WorkerPaneOutcome, { kind: "tracked" }> | undefined;
1868
+ // Herdr orders lifecycle reports by `seq`, so this run owns one counter and
1869
+ // every report takes the next value (#842). A shared or restarted counter would
1870
+ // let a late report overwrite a newer state — the pane would then show `working`
1871
+ // for a worker that has already blocked.
1872
+ let paneSeq = 0;
1873
+ const nextPaneSeq = (): number => (paneSeq += 1);
1874
+ /**
1875
+ * Project one authoritative transition onto the pane (#842).
1876
+ *
1877
+ * The input is always a durable run transition or a typed session event —
1878
+ * never the pane's own output, which is display and nothing more. A failure is
1879
+ * logged, not raised: the workspace lagging is not worth failing a run over.
1880
+ */
1881
+ const projectPaneState = (state: "working" | "idle" | "blocked" | "unknown", message?: string): void => {
1882
+ if (workerPane === undefined) return;
1883
+ const reported = reportWorkerPaneState(workerPane.paneId, workerPane.label, state, {
1884
+ seq: nextPaneSeq(),
1885
+ ...(message === undefined ? {} : { message }),
1886
+ });
1887
+ if (!reported.ok) log(`#${issue} herdr pane state ${state} not reported: ${reported.reason}`);
1888
+ };
1939
1889
  // The run's own repository. Hoisted for the same reason `worktreePath` is —
1940
1890
  // the catch and finally paths have to publish the branch.
1941
1891
  let runRepo: RunRepoRef | undefined;
@@ -2052,6 +2002,73 @@ export async function handleIssue(
2052
2002
  return true;
2053
2003
  };
2054
2004
 
2005
+ /**
2006
+ * The launch half of the credential-class fence (#852).
2007
+ *
2008
+ * Admission already held every candidate while a required provider billed to
2009
+ * the wrong credential, but a grant can be disabled in the window *between*
2010
+ * that check and this spawn — a refresh failing, an operator revoking, a token
2011
+ * expiring — and that window is exactly what the incident's acceptance
2012
+ * criterion names. So the same question is asked again here, as the last thing
2013
+ * before the session exists, and the run is closed rather than launched.
2014
+ *
2015
+ * Settled the same way the stop and drain fences settle: the claim closes, the
2016
+ * tree is salvaged, nothing is launched. It is deliberately NOT a failed
2017
+ * attempt against the issue's budget — the issue is not what is wrong, the
2018
+ * host's credentials are, and charging an attempt for it would exhaust an
2019
+ * issue while the operator re-authenticates.
2020
+ */
2021
+ const settleCredentialClassBeforeSession = async (): Promise<boolean> => {
2022
+ const required = project.requireOauthProviders ?? [];
2023
+ if (required.length === 0 || run === undefined || workerSessionInstalled) return false;
2024
+ const fence = await oauthFenceVerdict(
2025
+ required,
2026
+ d.probeCredentialClass ??
2027
+ (async (provider) => ({
2028
+ ok: false,
2029
+ reason: `no credential probe is wired for ${provider}`,
2030
+ })),
2031
+ );
2032
+ if (fence.ok) return false;
2033
+ turnLimit?.close();
2034
+ turnLimit = undefined;
2035
+ const settlement =
2036
+ worktreePath === undefined
2037
+ ? undefined
2038
+ : await settleWorktree({
2039
+ issue,
2040
+ attempt,
2041
+ ending: "not launched: a required subscription credential is unavailable",
2042
+ worktree: worktreePath,
2043
+ branch,
2044
+ publish,
2045
+ tree: "remove",
2046
+ mirrorPath,
2047
+ });
2048
+ recordOperatorStop(store, {
2049
+ project: project.name,
2050
+ issue,
2051
+ runId: run.id,
2052
+ inProgress,
2053
+ reason: `required subscription credential unavailable: ${fence.reason}`,
2054
+ patch: {
2055
+ endedAt: Date.now(),
2056
+ turns: run.turns,
2057
+ spendUsd: run.spendUsd,
2058
+ worktree: worktreePath ?? run.worktree,
2059
+ report: [
2060
+ "This run was closed instead of launched: a provider this project requires to bill",
2061
+ "to its subscription would have billed to something else.",
2062
+ fence.reason,
2063
+ ...(settlement?.lines ?? []),
2064
+ ].join("\n"),
2065
+ ...settlement?.patch,
2066
+ },
2067
+ });
2068
+ log(`#${issue} not launched: ${fence.reason}`);
2069
+ return true;
2070
+ };
2071
+
2055
2072
  try {
2056
2073
  // The claim-side of the stop fence (#374): the run row is the boundary the
2057
2074
  // shutdown drain waits on, so the claim itself refuses once the daemon is
@@ -2134,13 +2151,27 @@ export async function handleIssue(
2134
2151
  // (#286) keeps its semantics in both cases — it is the same resolution,
2135
2152
  // one different primary.
2136
2153
  const declaredModel = admittedModel?.model;
2154
+ // #807: a chain that already bought its one model escalation dispatches on
2155
+ // that stronger selector, outranking both the declaration and the project
2156
+ // default — the settlement that wrote the marker did so precisely because
2157
+ // the previous tier spun to a cap with nothing to show. Read from the
2158
+ // store, so it is sticky to this issue's chain and a fresh issue (no
2159
+ // marker) resolves exactly as it always has. The provider chain still
2160
+ // layers on top: an escalated run whose provider then aborts fails over
2161
+ // normally, because escalation only ever changes the primary.
2162
+ const escalation = store.modelEscalation(project.name, issue);
2163
+ const primaryModel = escalation?.model ?? declaredModel ?? project.workerModel;
2137
2164
  const choice = resolveDispatchModel({
2138
- workerModel: declaredModel ?? project.workerModel,
2165
+ workerModel: primaryModel,
2139
2166
  modelFallbacks: project.modelFallbacks,
2140
2167
  threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
2141
2168
  streak: chainFacts.streak,
2142
2169
  });
2143
- const clause = fallbackClause(choice, chainFacts, declaredModel ?? project.workerModel);
2170
+ const clause =
2171
+ fallbackClause(choice, chainFacts, primaryModel) ??
2172
+ (escalation === undefined
2173
+ ? undefined
2174
+ : `on ${escalation.model} — this chain's one model escalation, after ${escalation.failureClass}`);
2144
2175
 
2145
2176
  // #536: an orphan-clean requeue resumes the interrupted session instead of
2146
2177
  // re-reading the repo from zero. Decided here, before this attempt's row
@@ -2193,6 +2224,12 @@ export async function handleIssue(
2193
2224
  // enforced and the brief rendered, never a re-parse. Absent for a run
2194
2225
  // with no declaration (fail open), exactly as it was admitted.
2195
2226
  lane: admittedLane,
2227
+ // #851: the allowance admission reserved for this run, recorded on the
2228
+ // row before anything launches. Derived from the same caps the gate
2229
+ // read, so the reservation the pass enforced and the one the row holds
2230
+ // are one value; `undefined` (NULL) when the fleet has no spend cap at
2231
+ // all, which is the truth rather than "reserved nothing".
2232
+ spendReservedUsd: runSpendAllowanceUsd(caps) ?? undefined,
2196
2233
  });
2197
2234
  // #434: carry a terminal predecessor's open PR onto the continuation row.
2198
2235
  // The claim itself stays synchronous — `handleIssue` claims at the top of
@@ -2227,9 +2264,23 @@ export async function handleIssue(
2227
2264
  store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
2228
2265
  claimed = true;
2229
2266
  turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
2230
- workerControl = d.workerControls.open(project.name, issue, runId);
2267
+ // The pane follows the phase the registry actually reached (#842) never a
2268
+ // guess made at the call site, and never the pane's own output read back.
2269
+ workerControl = d.workerControls.open(project.name, issue, runId, (phase) => {
2270
+ projectPaneState(
2271
+ phase === "running" ? "working" : "idle",
2272
+ phase === "running" ? "resumed" : `${phase} by the operator`,
2273
+ );
2274
+ });
2231
2275
  if (await settleStopBeforeSession()) return;
2232
2276
  if (await settleDrainBeforeSession()) return;
2277
+ // The credential fence, twice, for the same reason the stop fence is
2278
+ // consulted more than once (#852). Here it is the cheap one: a run whose
2279
+ // required subscription credential is already unavailable is closed before
2280
+ // anything clones a mirror or cuts a worktree, because provisioning for a
2281
+ // launch that cannot happen is pure waste. The second call, immediately
2282
+ // before the spawn, is the one that closes the admission-to-session window.
2283
+ if (await settleCredentialClassBeforeSession()) return;
2233
2284
 
2234
2285
 
2235
2286
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
@@ -2328,7 +2379,10 @@ export async function handleIssue(
2328
2379
  // Recorded before the launch, so even a run killed mid-flight leaves the
2329
2380
  // model it chose on its row. Only a chain-configured project writes the
2330
2381
  // column: absent `modelFallbacks` must preserve today's rows byte for byte.
2331
- if (chainConfigured && choice.model !== undefined) {
2382
+ // An escalated chain (#807) writes it too the selector it dispatched on
2383
+ // is the whole provenance of that recovery, and its harness-resolved model
2384
+ // lands beside it in `resolvedModel` when the run settles.
2385
+ if ((chainConfigured || escalation !== undefined) && choice.model !== undefined) {
2332
2386
  store.updateRun(runId, { model: choice.model });
2333
2387
  }
2334
2388
 
@@ -2377,18 +2431,27 @@ export async function handleIssue(
2377
2431
  }
2378
2432
  if (await settleStopBeforeSession()) return;
2379
2433
  if (await settleDrainBeforeSession()) return;
2434
+ // Last, so it is the newest fact anything has about the credentials — a
2435
+ // grant disabled since admission is caught here rather than paid for.
2436
+ if (await settleCredentialClassBeforeSession()) return;
2380
2437
 
2381
2438
  const repoSlug = githubRepo(r.repo.cloneUrl);
2382
2439
 
2383
2440
 
2384
2441
  let result: WorkerResult;
2385
2442
  try {
2443
+ const runAllowanceUsd = runSpendAllowanceUsd(caps);
2386
2444
  result = await runWorker({
2387
2445
  brief,
2388
2446
  cwd: worktreePath,
2389
2447
  caps,
2390
2448
  ...(repoSlug === undefined ? {} : { repoSlug }),
2391
2449
  maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
2450
+ // #851: the allowance this run reserved at admission is also its live
2451
+ // ceiling. Without it the reservation would only bound how many runs
2452
+ // start, not what one of them spends, and the day's total would be
2453
+ // unbounded again.
2454
+ ...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
2392
2455
  onPauseControl: (control) => {
2393
2456
  workerSessionInstalled = true;
2394
2457
  workerControl?.install(control);
@@ -2407,6 +2470,69 @@ export async function handleIssue(
2407
2470
  onSpawn: (pid) => {
2408
2471
  verbListener?.bindPid(pid);
2409
2472
  },
2473
+ // The workspace representation of this exact child (#840). Best effort by
2474
+ // construction: a worker that cannot be *shown* is still a worker, so a
2475
+ // failure is logged with its reason and the run proceeds. What a failure
2476
+ // should mean for the launch — fail closed, or a named degraded state —
2477
+ // is #841's policy and is deliberately not decided here. Nothing is
2478
+ // claimed silently: either the pane id is logged, or the reason is.
2479
+ pane: {
2480
+ open: (pid) => {
2481
+ // Adopt before creating (#842): a run that already carries a pane
2482
+ // identity gets that pane reported against, never a second one. The
2483
+ // durable row is what makes this survive the process that opened it,
2484
+ // and duplicate-prevention is exactly what it buys — a re-entered
2485
+ // launch for one run must not leave two panes claiming it.
2486
+ const recorded = store.getRun(runId);
2487
+ if (recorded?.paneId !== undefined && recorded.paneLabel !== undefined) {
2488
+ workerPane = { kind: "tracked", paneId: recorded.paneId, label: recorded.paneLabel, pid };
2489
+ store.updateRun(runId, { workerPid: pid });
2490
+ const state = reportWorkerPaneState(recorded.paneId, recorded.paneLabel, "working", {
2491
+ seq: nextPaneSeq(),
2492
+ });
2493
+ log(
2494
+ state.ok
2495
+ ? `#${issue} herdr pane ${recorded.paneId} re-reported for pid ${pid}`
2496
+ : `#${issue} herdr pane ${recorded.paneId} could not be re-reported: ${state.reason}`,
2497
+ );
2498
+ return;
2499
+ }
2500
+ const outcome = openWorkerPane({
2501
+ project: project.name,
2502
+ issue,
2503
+ attempt,
2504
+ runId,
2505
+ pid,
2506
+ ...(recorded?.sessionFile === undefined ? {} : { sessionFile: recorded.sessionFile }),
2507
+ });
2508
+ if (outcome.kind === "tracked") {
2509
+ workerPane = outcome;
2510
+ // Durable before it is announced: a pane the store does not know
2511
+ // about is a pane a restart cannot reconcile.
2512
+ store.updateRun(runId, {
2513
+ workerPid: pid,
2514
+ paneId: outcome.paneId,
2515
+ paneLabel: outcome.label,
2516
+ // Cleared, because there is now a pane: a leftover reason beside a
2517
+ // tracked run would keep `status` saying degraded forever (#841).
2518
+ paneUnavailable: null,
2519
+ });
2520
+ log(`#${issue} herdr pane ${outcome.paneId} (${outcome.label}) tracks pid ${pid}`);
2521
+ } else {
2522
+ // Named, never silent (#841): the run proceeds untracked, and the
2523
+ // reason is durable so `status` can say the fleet is running blind
2524
+ // rather than looking identical to a fleet with no workers.
2525
+ store.updateRun(runId, { workerPid: pid, paneUnavailable: outcome.reason });
2526
+ log(`#${issue} no herdr pane: ${outcome.reason}`);
2527
+ }
2528
+ },
2529
+ release: () => {
2530
+ if (workerPane === undefined) return;
2531
+ const released = releaseWorkerPane(workerPane.paneId, workerPane.label, { seq: nextPaneSeq() });
2532
+ if (!released.ok) log(`#${issue} herdr pane release failed: ${released.reason}`);
2533
+ workerPane = undefined;
2534
+ },
2535
+ },
2410
2536
  onChildLog: (line) => {
2411
2537
  log(`#${issue} ${line}`);
2412
2538
  },
@@ -2421,6 +2547,11 @@ export async function handleIssue(
2421
2547
  onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
2422
2548
  onTurn: (n) => store.updateRun(runId, { turns: n }),
2423
2549
  onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
2550
+ // #518: recorded live, not at settlement — the moment this explains is
2551
+ // forty minutes before the wall-clock cap fires, so a number only a
2552
+ // finished run carries would answer the question too late.
2553
+ onTokens: (tokens) =>
2554
+ store.updateRun(runId, { outputTokens: tokens.output, reasoningTokens: tokens.reasoning }),
2424
2555
  onKilled: () => {
2425
2556
  turnLimit?.close();
2426
2557
  turnLimit = undefined;
@@ -2876,6 +3007,114 @@ export async function handleIssue(
2876
3007
  * recorded it — merged after all, settled, re-claimed — fails the claim and is
2877
3008
  * settled `skipped` rather than woken on stale identity.
2878
3009
  */
3010
+ /**
3011
+ * Record what this host has installed, once per dispatch pass (#919).
3012
+ *
3013
+ * Here rather than at render time, and this is the whole design of the slice:
3014
+ * the read costs three subprocesses, and calling it from the tick took the
3015
+ * tick's own suite from 8.4s to 83.4s while spawning three children every
3016
+ * fifteen minutes to answer a question that changes only when someone installs
3017
+ * something. The dispatch pass is already async and already spawns `gh`, so one
3018
+ * read per pass is free by comparison, and every cheap surface then reads a row.
3019
+ *
3020
+ * Advisory throughout: a probe that throws (no `herdr` on PATH, a `$PATH`
3021
+ * without `omp`) leaves the previous observation in place and logs. A stale
3022
+ * observation is still the truth about the last time anyone could look, and
3023
+ * losing a dispatch pass over a version string would be absurd.
3024
+ */
3025
+ /**
3026
+ * Make the workspace agree with the live run set, once per pass (#841).
3027
+ *
3028
+ * Runs on every dispatch pass rather than only at startup, because the thing it
3029
+ * repairs — a Herdr restart — is not a conductor event and announces itself
3030
+ * nowhere. It is idempotent by construction: a reconciled fleet reports `intact`
3031
+ * for every live worker and finds no stale panes, so repeated passes converge
3032
+ * instead of accumulating panes or churning them.
3033
+ *
3034
+ * Nothing here can stop a worker. The only mutations are creating a pane, and
3035
+ * releasing one whose run is not live; the authoritative child is never
3036
+ * signalled, and no pane is ever closed.
3037
+ */
3038
+ function reconcilePanes(d: Deps, project: string, log: (message: string) => void): void {
3039
+ const live = d.store.liveRuns(project);
3040
+ const result = reconcileWorkerPanes(
3041
+ live.map((run) => ({
3042
+ runId: run.id,
3043
+ issue: run.issue,
3044
+ attempt: run.attempt,
3045
+ project,
3046
+ ...(run.workerPid === undefined ? {} : { pid: run.workerPid }),
3047
+ ...(run.paneId === undefined ? {} : { paneId: run.paneId }),
3048
+ ...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
3049
+ ...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
3050
+ })),
3051
+ );
3052
+ if (!result.ok) {
3053
+ // An unreadable workspace is not evidence that anything is stale, so nothing
3054
+ // is released and no run is relabelled — but a fleet that cannot see its own
3055
+ // panes says so, on every pass, rather than going quiet.
3056
+ log(`worker panes not reconciled: ${result.reason}`);
3057
+ return;
3058
+ }
3059
+ for (const outcome of result.outcomes) {
3060
+ switch (outcome.kind) {
3061
+ case "intact":
3062
+ break;
3063
+ case "reassociated":
3064
+ d.store.updateRun(outcome.runId, {
3065
+ paneId: outcome.paneId,
3066
+ paneLabel: outcome.label,
3067
+ paneUnavailable: null,
3068
+ });
3069
+ log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
3070
+ break;
3071
+ case "untracked":
3072
+ d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
3073
+ log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
3074
+ break;
3075
+ case "stale-released":
3076
+ log(`released stale herdr pane ${outcome.paneId} (run ${outcome.runId || "unidentified"} is not live)`);
3077
+ break;
3078
+ case "stale-release-failed":
3079
+ log(`stale herdr pane ${outcome.paneId} could not be released: ${outcome.reason}`);
3080
+ break;
3081
+ }
3082
+ }
3083
+ }
3084
+
3085
+ /** The issue a run id belongs to, for a log line a human reads. */
3086
+ function runIssue(live: readonly RunRecord[], runId: string): string {
3087
+ return String(live.find((run) => run.id === runId)?.issue ?? "?");
3088
+ }
3089
+
3090
+ async function recordInstallSurfaces(d: Deps): Promise<void> {
3091
+ if (d.probeInstallSurfaces === undefined) return;
3092
+ try {
3093
+ const surfaces = await d.probeInstallSurfaces();
3094
+ // The telegram peer's versions ride along on the same pass (#961). Read
3095
+ // here rather than in `status` because `status` runs every tick and this
3096
+ // touches the npm registry: one periodic read, rendered from the store as
3097
+ // often as anyone looks. A read that throws leaves the fields absent, which
3098
+ // renders as unverified — never as agreement.
3099
+ const telegram =
3100
+ d.probeTelegramFreshness === undefined ? undefined : await d.probeTelegramFreshness();
3101
+ const tgInstalled = telegram?.surfaces.installed;
3102
+ const tgDaemon = telegram?.surfaces.daemon;
3103
+ const tgPublished = telegram?.surfaces.published;
3104
+ d.store.recordInstallSurfaces({
3105
+ at: Date.now(),
3106
+ cliVersion: surfaces.cliVersion,
3107
+ ...(surfaces.ompVersion === undefined ? {} : { ompVersion: surfaces.ompVersion }),
3108
+ ...(surfaces.herdrSource === undefined ? {} : { herdrSource: surfaces.herdrSource }),
3109
+ ...(tgInstalled?.kind === "version" ? { telegramInstalled: tgInstalled.version } : {}),
3110
+ ...(tgDaemon?.kind === "version" ? { telegramDaemon: tgDaemon.version } : {}),
3111
+ ...(tgPublished?.kind === "version" ? { telegramPublished: tgPublished.version } : {}),
3112
+ });
3113
+ } catch (err) {
3114
+ log(`install-surface read skipped: ${errText(err)}`);
3115
+ }
3116
+ }
3117
+
2879
3118
  export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
2880
3119
  const pending = d.store.pendingReviewRevisions(d.project.name);
2881
3120
  if (pending.length === 0) return;
@@ -2993,6 +3232,350 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
2993
3232
  await Promise.allSettled(launches);
2994
3233
  }
2995
3234
 
3235
+ /**
3236
+ * Launch the review-ceiling adjudications this project owes (#932).
3237
+ *
3238
+ * Deliberately its own pass, next to {@link dispatchReviewRevisions} but never
3239
+ * inside it: a revision resumes an implementation worker to change code, an
3240
+ * adjudication opens a fresh read-only session to decide about code. Folding
3241
+ * them together is how "escalate the automation" quietly becomes "resume the
3242
+ * same worker with a different label", which is #873's named silent fake.
3243
+ *
3244
+ * Only `pending` rows are launched. A `running` row belongs to a live
3245
+ * adjudicator — or to one a restart killed, which the startup sweep settles —
3246
+ * so a restart re-reads it and never launches a second adjudicator for one head.
3247
+ */
3248
+ /**
3249
+ * The turn ceiling for one adjudication (#932).
3250
+ *
3251
+ * Small on purpose, and not the worker cap: a verdict is a read and an answer,
3252
+ * not an implementation. A session still going after this is not deciding — and
3253
+ * the ceiling exists so a stuck adjudicator costs a bounded amount rather than a
3254
+ * worker's whole budget, on work that has already spent its review rounds.
3255
+ */
3256
+ export const ADJUDICATION_MAX_TURNS = 40;
3257
+
3258
+ export async function dispatchReviewAdjudications(d: Deps, pool?: WorkerPool): Promise<void> {
3259
+ const open = d.store.openReviewAdjudications(d.project.name).filter((a) => a.state === "pending");
3260
+ if (open.length === 0) return;
3261
+ const launches: Promise<void>[] = [];
3262
+ for (const adjudication of open) {
3263
+ if (d.drain?.draining === true) {
3264
+ log(`adjudications held: the daemon is draining (${open.length - launches.length} not launched)`);
3265
+ break;
3266
+ }
3267
+ // The head is re-read before anything launches. A verdict describes the diff
3268
+ // it read, so adjudicating a head the PR has since moved past would produce
3269
+ // a decision about code nobody is merging — settled `stale-head` with both
3270
+ // heads named, never launched.
3271
+ let live: string | undefined;
3272
+ try {
3273
+ live = await d.tracker.prHead(adjudication.prUrl);
3274
+ } catch (err) {
3275
+ log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head unreadable (${errText(err)})`);
3276
+ continue;
3277
+ }
3278
+ if (live === undefined) {
3279
+ log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head could not be read — retrying next tick`);
3280
+ continue;
3281
+ }
3282
+ if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
3283
+ d.store.settleReviewAdjudication(
3284
+ adjudication.id,
3285
+ "stale-head",
3286
+ `recorded ${adjudication.headSha}, live ${live} — the pull request moved before this adjudication launched`,
3287
+ Date.now(),
3288
+ );
3289
+ log(`#${adjudication.issue} adjudication stale-head: recorded ${adjudication.headSha}, live ${live}`);
3290
+ continue;
3291
+ }
3292
+ // The claim IS the single-flight guard: the transition is the row's own
3293
+ // WHERE clause, so two dispatch passes racing one adjudication cannot both
3294
+ // launch — the loser changes nothing and skips.
3295
+ const review = resolveReview(d.project);
3296
+ const provenance = { role: adjudication.role, model: adjudication.role, resolvedAt: Date.now() };
3297
+ if (!d.store.markReviewAdjudicationRunning(adjudication.id, provenance, Date.now())) {
3298
+ log(`#${adjudication.issue} adjudication skipped: another pass already claimed it`);
3299
+ continue;
3300
+ }
3301
+ log(`#${adjudication.issue} adjudication → launching role ${adjudication.role} at ${adjudication.headSha}`);
3302
+ launches.push(handleReviewAdjudication(d, adjudication, review.maxRounds));
3303
+ }
3304
+ if (pool !== undefined) {
3305
+ for (const launch of launches) pool.launch(launch);
3306
+ return;
3307
+ }
3308
+ await Promise.allSettled(launches);
3309
+ }
3310
+
3311
+ /**
3312
+ * Assemble one adjudication's evidence and run it to a durable verdict (#932).
3313
+ *
3314
+ * Every fact comes from a durable row or a live tracker read — never from the
3315
+ * worker's own report, which is the claim under adjudication. The issue body is
3316
+ * carried verbatim because the acceptance criteria ARE the standard being
3317
+ * judged, and every prior round travels WITH its settled outcome: a finding
3318
+ * without its outcome reads as an outstanding complaint even when the worker
3319
+ * fixed it, which is how an adjudicator rejects work that was already corrected.
3320
+ */
3321
+ export async function handleReviewAdjudication(
3322
+ d: Deps,
3323
+ adjudication: ReviewAdjudicationRecord,
3324
+ maxRounds: number,
3325
+ ): Promise<void> {
3326
+ const issue = adjudication.issue;
3327
+ const [title, body, diff, verification] = await Promise.all([
3328
+ d.tracker.getIssue(issue).then(
3329
+ (row) => row?.title,
3330
+ () => undefined,
3331
+ ),
3332
+ d.tracker.issueBody(issue).then(
3333
+ (text) => text,
3334
+ () => undefined,
3335
+ ),
3336
+ d.tracker.prDiff(adjudication.prUrl).then(
3337
+ (parsed) => parsed,
3338
+ () => undefined,
3339
+ ),
3340
+ d.tracker.verifyPr(adjudication.prUrl, adjudication.headSha).then(
3341
+ (result) => result,
3342
+ () => undefined,
3343
+ ),
3344
+ ]);
3345
+ // Prior rounds, oldest first, keyed by PR so a continuation that inherited the
3346
+ // PR cannot hide its predecessor's findings. The ceiling call's own findings
3347
+ // are appended as the last entry — they are the reason this adjudication
3348
+ // exists, and the durable row is where they were kept precisely so this brief
3349
+ // could carry them.
3350
+ const rounds: AdjudicationRound[] = d.store
3351
+ .reviewRevisionsForPr(d.project.name, adjudication.prUrl)
3352
+ .map((revision) => ({
3353
+ round: revision.round,
3354
+ findings: revision.findings,
3355
+ ...(revision.outcome === undefined ? {} : { outcome: revision.outcome }),
3356
+ }));
3357
+ if (adjudication.findings !== undefined) {
3358
+ rounds.push({
3359
+ round: rounds.length + 1,
3360
+ findings: adjudication.findings,
3361
+ outcome: "escalated to this adjudication — no worker round was available",
3362
+ });
3363
+ }
3364
+ const brief = renderAdjudicationBrief({
3365
+ issue,
3366
+ ...(title === undefined ? {} : { issueTitle: title }),
3367
+ issueBody: body ?? "(the issue body could not be read — judge the diff against the findings below)",
3368
+ prUrl: adjudication.prUrl,
3369
+ headSha: adjudication.headSha,
3370
+ checks:
3371
+ verification === undefined
3372
+ ? "(the check state could not be read)"
3373
+ : `${verification.status} — ${verification.reason}`,
3374
+ diff:
3375
+ diff === undefined
3376
+ ? "(the diff could not be read — say so in your reasons rather than guessing)"
3377
+ : renderDiffForAdjudication(diff),
3378
+ rounds,
3379
+ maxRounds,
3380
+ });
3381
+
3382
+ const sessionDir = join(stateDir(), "sessions", `adjudication-${adjudication.id}`);
3383
+ mkdirSync(sessionDir, { recursive: true });
3384
+ const ompSettingsFile = materializeOmpSettings(d.project, sessionDir);
3385
+ const result = await (d.runAdjudicatorImpl ?? runAdjudicator)({
3386
+ brief,
3387
+ // No worktree: an adjudicator has no branch, and everything it needs is in
3388
+ // the brief. The session directory doubles as its confined cwd.
3389
+ cwd: sessionDir,
3390
+ sessionDir,
3391
+ model: adjudication.role,
3392
+ ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
3393
+ socketPath: join(sessionDir, "ipc.sock"),
3394
+ maxTurns: ADJUDICATION_MAX_TURNS,
3395
+ });
3396
+
3397
+ // What actually ran it, named in the evidence when it differs from the role
3398
+ // that was asked for: the provenance recorded at claim time is conductor's own
3399
+ // resolution (the role it passed to OMP), and the model that wrote the
3400
+ // messages is a later observation only this result carries.
3401
+ const ran =
3402
+ result.model === undefined
3403
+ ? ""
3404
+ : `adjudicated by ${result.model}${result.provider === undefined ? "" : ` (${result.provider})`}: `;
3405
+ if (result.verdict === undefined) {
3406
+ // A session that never took a turn did not run: the role could not be
3407
+ // resolved or the harness could not start, and the remedy is configuration
3408
+ // rather than a retry. One that ran and produced nothing usable is a
3409
+ // failure. Neither is ever a clear.
3410
+ const state = result.turns === 0 ? "unavailable-model" : "failed";
3411
+ d.store.settleReviewAdjudication(
3412
+ adjudication.id,
3413
+ state,
3414
+ `${ran}${result.report === "" ? "no verdict and no output" : result.report}`,
3415
+ Date.now(),
3416
+ );
3417
+ log(`#${issue} adjudication ${state} at ${adjudication.headSha}`);
3418
+ return;
3419
+ }
3420
+ // A verdict that names a different head judged something else. Fail closed:
3421
+ // the one-shot is spent on the head it was opened for, and nothing merges on
3422
+ // the strength of a decision about another diff.
3423
+ if (
3424
+ result.verdict.headSha !== undefined &&
3425
+ result.verdict.headSha.toLowerCase() !== adjudication.headSha.toLowerCase()
3426
+ ) {
3427
+ d.store.settleReviewAdjudication(
3428
+ adjudication.id,
3429
+ "stale-head",
3430
+ `${ran}the verdict names head ${result.verdict.headSha}, not the adjudicated ${adjudication.headSha}: ${result.verdict.reasons}`,
3431
+ Date.now(),
3432
+ );
3433
+ log(`#${issue} adjudication stale-head: verdict named ${result.verdict.headSha}`);
3434
+ return;
3435
+ }
3436
+ d.store.settleReviewAdjudication(
3437
+ adjudication.id,
3438
+ result.verdict.verdict,
3439
+ `${ran}${result.verdict.reasons}`,
3440
+ Date.now(),
3441
+ );
3442
+ log(`#${issue} adjudication ${result.verdict.verdict} at ${adjudication.headSha}`);
3443
+ }
3444
+
3445
+ /**
3446
+ * Apply the terminal dispositions an adjudication's verdict implies (#876).
3447
+ *
3448
+ * The verdict is a decision; this is what is DONE about it, and the two are
3449
+ * separate rows for exactly that reason. Recording `cleared` while leaving the
3450
+ * merge gate refusing, or `rejected` while leaving the PR open to block the
3451
+ * issue forever, is the issue's named silent fake — so what this pass changes is
3452
+ * tracker and gate state, and the disposition text is only its receipt.
3453
+ *
3454
+ * Idempotent by that receipt: a row whose `disposition` is already recorded is
3455
+ * skipped, so a restart, a retried tick or a partial failure cannot close one PR
3456
+ * twice. Nothing here is destructive before its explanation is durable.
3457
+ *
3458
+ * Neither branch opens a review round, launches another adjudicator, or asks an
3459
+ * operator for anything. Neither touches an attempt or continuation budget
3460
+ * either: a rejected PR closes, and the ORDINARY settle sweep charges that
3461
+ * closure exactly as it charges any PR closed without merging — which is what
3462
+ * keeps this composable with #815's separate corrective-worker path instead of
3463
+ * resetting its counters.
3464
+ */
3465
+ export async function applyAdjudicationDispositions(d: Deps): Promise<void> {
3466
+ const { project, store } = d;
3467
+ // Terminal rows only, and only those nothing has disposed of yet. Read from
3468
+ // the PRs still in flight: a settled adjudication whose PR is long gone needs
3469
+ // no disposition, and scanning history would re-litigate closed work.
3470
+ const pending = store
3471
+ .activeRuns(project.name)
3472
+ .flatMap((run) => (run.prUrl === undefined ? [] : store.reviewAdjudicationsForPr(project.name, run.prUrl)))
3473
+ .filter((row) => row.settledAt !== undefined && row.disposition === undefined);
3474
+ for (const adjudication of pending) {
3475
+ if (adjudication.state === "cleared") {
3476
+ // The clear re-enters the ORDINARY merge path, and it does so through the
3477
+ // mechanism that path already reads: the #888 gate refuses a head carrying
3478
+ // unresolved review evidence, and #913's clearance is the recorded
3479
+ // disposition that settles it. Reusing it means no second gate, no
3480
+ // adjudication-shaped exception inside `conductor_pr_merge`, and one
3481
+ // audit trail for "this head was cleared, by whom, and why".
3482
+ //
3483
+ // The head is re-read first: a clearance is head-scoped and clears
3484
+ // backwards only, so recording one against a head the PR has moved past
3485
+ // would bless a diff nobody adjudicated.
3486
+ let live: string | undefined;
3487
+ try {
3488
+ live = await d.tracker.prHead(adjudication.prUrl);
3489
+ } catch (err) {
3490
+ log(`#${adjudication.issue} adjudication clearance held: head unreadable (${errText(err)})`);
3491
+ continue;
3492
+ }
3493
+ if (live === undefined) {
3494
+ log(`#${adjudication.issue} adjudication clearance held: head could not be read — retrying next tick`);
3495
+ continue;
3496
+ }
3497
+ if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
3498
+ // Visible and non-destructive: the clearance is not recorded, the row
3499
+ // says why, and the PR keeps whatever gate state it has.
3500
+ store.recordReviewAdjudicationDisposition(
3501
+ adjudication.id,
3502
+ `no clearance recorded: the head moved from ${adjudication.headSha} to ${live} after the verdict`,
3503
+ );
3504
+ log(`#${adjudication.issue} adjudication clearance skipped: head moved to ${live}`);
3505
+ continue;
3506
+ }
3507
+ store.recordReviewClearance({
3508
+ project: project.name,
3509
+ prUrl: adjudication.prUrl,
3510
+ headSha: adjudication.headSha,
3511
+ by: `adjudication:${adjudication.role}`,
3512
+ reason: `cleared by the review-ceiling adjudication: ${adjudication.evidence ?? "no evidence recorded"}`,
3513
+ });
3514
+ store.recordReviewAdjudicationDisposition(
3515
+ adjudication.id,
3516
+ `recorded a review clearance at ${adjudication.headSha}; the PR is on the ordinary merge path`,
3517
+ );
3518
+ log(`#${adjudication.issue} adjudication cleared → merge path open at ${adjudication.headSha}`);
3519
+ continue;
3520
+ }
3521
+ // Every non-clear terminal state disposes the same way, and that is
3522
+ // deliberate: `rejected`, `failed`, `unavailable-model` and `stale-head` all
3523
+ // mean "this head is not merging and no further round is coming", and the
3524
+ // blocking artefact is identical in each case — an open PR that occupies the
3525
+ // issue. Only the recorded reason differs.
3526
+ const findings = store
3527
+ .reviewRevisionsForPr(project.name, adjudication.prUrl)
3528
+ .map((revision) => `### Review round ${revision.round} (${revision.outcome ?? "never settled"})
3529
+
3530
+ ${revision.findings}`);
3531
+ const comment = [
3532
+ `## Adjudication: ${adjudication.state}`,
3533
+ "",
3534
+ `The review-round ceiling was reached on ${adjudication.headSha}, and the \`${adjudication.role}\` adjudicator ` +
3535
+ `did not clear it. Closing this pull request so the issue is not blocked by work that cannot merge — ` +
3536
+ "nothing here asks anyone to repair it by hand.",
3537
+ "",
3538
+ "### Adjudicator's reasons",
3539
+ "",
3540
+ adjudication.evidence ?? "(none recorded)",
3541
+ ...(findings.length === 0 ? [] : ["", "## Preserved review findings", "", ...findings]),
3542
+ "",
3543
+ "The branch itself is untouched and remains in the repository.",
3544
+ ].join("\n");
3545
+ const closed = await d.verbActions?.closePr(adjudication.prUrl, comment);
3546
+ if (closed === undefined) {
3547
+ log(`#${adjudication.issue} adjudication disposition held: no action surface is wired`);
3548
+ continue;
3549
+ }
3550
+ if (!closed.ok) {
3551
+ // Retryable, and no receipt written: the next tick tries again rather than
3552
+ // leaving a rejected PR open with a row that claims it was handled.
3553
+ log(`#${adjudication.issue} adjudication disposition held: could not close ${adjudication.prUrl} — ${closed.stderr}`);
3554
+ continue;
3555
+ }
3556
+ store.recordReviewAdjudicationDisposition(
3557
+ adjudication.id,
3558
+ `closed ${adjudication.prUrl} with the adjudicator's reasons and every preserved finding; ` +
3559
+ "the settle sweep releases the issue",
3560
+ );
3561
+ log(`#${adjudication.issue} adjudication ${adjudication.state} → closed ${adjudication.prUrl}`);
3562
+ }
3563
+ }
3564
+
3565
+ /** The diff as one text, from the structured read the settlement audit uses.
3566
+ * `renderAdjudicationBrief` bounds the total; this only flattens it, and says
3567
+ * when the adapter itself had to cut the read short — an adjudicator must know
3568
+ * its silence covers only what it saw. */
3569
+ function renderDiffForAdjudication(diff: PrDiff): string {
3570
+ const files = diff.files.map((file) =>
3571
+ [
3572
+ `--- ${file.status} ${file.previousPath === undefined ? "" : `${file.previousPath} → `}${file.path}`,
3573
+ file.hunks ?? "(no textual hunks: binary, mode-only or pure rename)",
3574
+ ].join("\n"),
3575
+ );
3576
+ return [...(diff.truncated ? ["[the tracker truncated this diff read]"] : []), ...files].join("\n\n");
3577
+ }
3578
+
2996
3579
  /**
2997
3580
  * Resume one review-revision worker: the run row is already claimed
2998
3581
  * (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
@@ -3113,20 +3696,38 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3113
3696
  // stopped anything, the PR is still green and still open, and only the
3114
3697
  // wake failed; `settleStopBeforeSession` immediately above covers the
3115
3698
  // case where an operator genuinely did. So mirror that branch: restore
3116
- // `pushed-green` naming the shutdown, settle the revision `skipped` so
3117
- // the round is not double-counted, and leave the in-progress label alone
3118
- // — the ordinary settle sweep releases it in the same breath it
3699
+ // `pushed-green` naming the shutdown, and leave the in-progress label
3700
+ // alone — the ordinary settle sweep releases it in the same breath it
3119
3701
  // terminalises the row when the PR resolves. Stopping the row here would
3120
3702
  // strand it: `settlePushedGreen` sweeps only pushed-* rows and
3121
3703
  // classification excludes `stopped`, so the issue would carry neither
3122
3704
  // label and nothing would ever revisit the still-green PR.
3705
+ //
3706
+ // The round itself is RE-QUEUED rather than settled `skipped` (#903). A
3707
+ // daemon shutdown is the plainest infrastructure kill there is — no
3708
+ // session started, so no round was worked — and a settled row consumed
3709
+ // one of the PR's rounds through `latestReviewRound`, which is how an
3710
+ // upgrade or a restart used to cost a review round it never spent. The
3711
+ // retry is counted and bounded exactly like any other infra kill, so a
3712
+ // host that shuts down mid-wake forever still escalates.
3123
3713
  store.updateRun(runId, {
3124
3714
  state: "pushed-green",
3125
3715
  endedAt: Date.now(),
3126
3716
  lastError: "daemon shutdown began after the review revision claim; the round was not launched",
3127
3717
  });
3128
- store.settleReviewRevision(revision.id, "skipped", Date.now());
3129
- log(`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim`);
3718
+ const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
3719
+ if (retry.kind === "exhausted") {
3720
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3721
+ log(
3722
+ `#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim, and ` +
3723
+ `its infra retries are exhausted (${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settled skipped`,
3724
+ );
3725
+ return true;
3726
+ }
3727
+ log(
3728
+ `#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim — ` +
3729
+ `re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
3730
+ );
3130
3731
  return true;
3131
3732
  };
3132
3733
 
@@ -3286,18 +3887,26 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3286
3887
  // totals the run already recorded.
3287
3888
  const baseTurns = run.turns;
3288
3889
  const baseSpend = run.spendUsd;
3890
+ const baseOutputTokens = run.outputTokens ?? 0;
3891
+ const baseReasoningTokens = run.reasoningTokens ?? 0;
3289
3892
 
3290
3893
  log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
3291
3894
 
3292
3895
 
3293
3896
  let result: WorkerResult;
3294
3897
  try {
3898
+ const runAllowanceUsd = runSpendAllowanceUsd(caps);
3295
3899
  result = await runWorker({
3296
3900
  brief: renderReviewRevisionPrompt(revision.findings, revision.round),
3297
3901
  cwd: worktreePath,
3298
3902
  caps,
3299
3903
  ...(repoSlug === undefined ? {} : { repoSlug }),
3300
3904
  maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
3905
+ // A review round meters its own session from zero and gets the same
3906
+ // allowance as any run (#851) — the round is real work with a real
3907
+ // cost, and an unbounded round would reopen the hole for any PR that
3908
+ // was ever returned for revision.
3909
+ ...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
3301
3910
  onPauseControl: (control) => {
3302
3911
  workerSessionInstalled = true;
3303
3912
  workerControl?.install(control);
@@ -3321,6 +3930,14 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3321
3930
  onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
3322
3931
  onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
3323
3932
  onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
3933
+ // A revision resumes the SAME run, so its tokens add to the attempt's
3934
+ // totals exactly as its turns and spend do (#518) — resetting them would
3935
+ // make a long deliberation look like a fresh one.
3936
+ onTokens: (tokens) =>
3937
+ store.updateRun(runId, {
3938
+ outputTokens: baseOutputTokens + tokens.output,
3939
+ reasoningTokens: baseReasoningTokens + tokens.reasoning,
3940
+ }),
3324
3941
  onKilled: () => {
3325
3942
  turnLimit?.close();
3326
3943
  turnLimit = undefined;
@@ -3419,6 +4036,63 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3419
4036
  report: finalReport,
3420
4037
  ...settlement?.patch,
3421
4038
  };
4039
+ // #903: an infrastructure kill spends no review round.
4040
+ //
4041
+ // A dispatch that dies before the resumed session takes a turn — a host
4042
+ // permission fault, a spawn failure, a harness that could not start — did
4043
+ // none of the work a round exists to do, yet the pre-#903 path settled the
4044
+ // row `failed` with no retry. During the 2026-08-21/22 outage that turned
4045
+ // one broken mount into a review-ceiling deadlock: three such deaths
4046
+ // consumed the PR's three rounds, `conductor_pr_review` then refused any
4047
+ // further round, and the findings could never be addressed by a worker at
4048
+ // all. Nothing in the loop could recover it.
4049
+ //
4050
+ // So the round is returned to the pending set instead: the same row, the
4051
+ // same round number, the same findings, and the run restored to the exact
4052
+ // `pushed-green` state the verb recorded — which is what the restart
4053
+ // reconcile already does for a round a dying daemon interrupted. The next
4054
+ // dispatch pass resumes the same session. Bounded by
4055
+ // REVIEW_ROUND_INFRA_MAX_RETRIES and counted durably on the row, so a
4056
+ // permanently broken host escalates through the ordinary failed path
4057
+ // instead of retrying every tick forever.
4058
+ const infraKill =
4059
+ state === "failed" || state === "killed"
4060
+ ? reviewRoundNeverWorked({
4061
+ turns: result.turns,
4062
+ ...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
4063
+ ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
4064
+ ...(settlement?.patch?.salvageSha === undefined
4065
+ ? {}
4066
+ : { salvageSha: settlement.patch.salvageSha }),
4067
+ })
4068
+ : undefined;
4069
+ if (infraKill !== undefined) {
4070
+ const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
4071
+ if (retry.kind === "requeued") {
4072
+ const detail =
4073
+ `review round ${revision.round} was killed before its session took a turn ` +
4074
+ `(${infraKill}); the round is re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`;
4075
+ store.updateRun(runId, {
4076
+ ...terminalPatch,
4077
+ state: "pushed-green",
4078
+ // The transcript the round must still resume is the one the run
4079
+ // recorded, not whatever this dead dispatch opened: a session that
4080
+ // never took a turn has no lineage of its own to inherit.
4081
+ sessionFile: priorSessionFile,
4082
+ report: [`review round ${revision.round}: dispatch killed before turn 1`, "", detail].join("\n"),
4083
+ lastError: detail,
4084
+ });
4085
+ // No label swap and no escalation: nothing failed that a human can
4086
+ // act on yet, the issue keeps its in-progress label, and the next
4087
+ // dispatch pass picks the round up on its own.
4088
+ log(`#${issue} ${detail}`);
4089
+ return;
4090
+ }
4091
+ log(
4092
+ `#${issue} review round ${revision.round} exhausted its infra retries ` +
4093
+ `(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settling it the ordinary way`,
4094
+ );
4095
+ }
3422
4096
  if (state === "stopped") {
3423
4097
  recordOperatorStop(store, {
3424
4098
  project: project.name,
@@ -3530,6 +4204,35 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3530
4204
  publish,
3531
4205
  tree: "keep",
3532
4206
  });
4207
+ // A crash BEFORE the worker session was installed spent no round (#903):
4208
+ // nothing resumed, nothing was reviewed, and the round is re-queued for
4209
+ // the next pass instead of consuming one of the PR's three. Once the
4210
+ // session is installed the round belongs to the worker, and a crash after
4211
+ // that settles the ordinary way below — including the exhausted case, so
4212
+ // a dispatch that keeps throwing still reaches a human.
4213
+ if (!workerSessionInstalled) {
4214
+ const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
4215
+ if (retry.kind === "requeued") {
4216
+ store.updateRun(runId, {
4217
+ state: "pushed-green",
4218
+ endedAt: Date.now(),
4219
+ sessionFile: priorSessionFile,
4220
+ lastError:
4221
+ `review round ${revision.round} could not be dispatched (${detail}); the round is re-queued, ` +
4222
+ `retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
4223
+ ...settlement?.patch,
4224
+ });
4225
+ log(
4226
+ `#${issue} review round ${revision.round} re-queued after a pre-session dispatch error — ` +
4227
+ `retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
4228
+ );
4229
+ return;
4230
+ }
4231
+ log(
4232
+ `#${issue} review round ${revision.round} exhausted its infra retries ` +
4233
+ `(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — failing the round`,
4234
+ );
4235
+ }
3533
4236
  store.updateRun(runId, {
3534
4237
  state: "failed",
3535
4238
  endedAt: Date.now(),
@@ -4279,16 +4982,26 @@ export interface WorkerPool {
4279
4982
  drain(): Promise<void>;
4280
4983
  }
4281
4984
 
4282
- /** Keeps background workers alive without making the five-minute tick await them. */
4283
- export function createWorkerPool(): WorkerPool {
4985
+ /**
4986
+ * Keeps background workers alive without making the five-minute tick await them.
4987
+ *
4988
+ * `onSettled` fires as each worker leaves the pool, whichever way it ended
4989
+ * (#878): that is the instant a slot frees, and without it queued work waited
4990
+ * for the next scheduled pass — up to five minutes of idle capacity that reads,
4991
+ * from outside, exactly like a stalled queue. It only *prompts* a pass; every
4992
+ * hold, drain, lane, budget and routing gate is re-evaluated by that pass as
4993
+ * usual.
4994
+ */
4995
+ export function createWorkerPool(onSettled?: () => void): WorkerPool {
4284
4996
  const active = new Set<Promise<void>>();
4285
4997
  return {
4286
4998
  launch(work) {
4287
4999
  active.add(work);
4288
- void work.then(
4289
- () => active.delete(work),
4290
- () => active.delete(work),
4291
- );
5000
+ const settled = (): void => {
5001
+ active.delete(work);
5002
+ onSettled?.();
5003
+ };
5004
+ void work.then(settled, settled);
4292
5005
  },
4293
5006
  activeCount: () => active.size,
4294
5007
  async drain() {
@@ -4609,6 +5322,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4609
5322
  } catch (err) {
4610
5323
  log(`label reconcile failed: ${errText(err)}`);
4611
5324
  }
5325
+ try {
5326
+ // Same phase, same reason as the label reconcile above: a durable row the
5327
+ // tracker has moved past (#964). A separate call because that one iterates
5328
+ // labelled issues, and a promotable issue carries no label.
5329
+ await reconcileGroomingClosures(d);
5330
+ } catch (err) {
5331
+ log(`grooming closure reconcile failed: ${errText(err)}`);
5332
+ }
4612
5333
 
4613
5334
  // Drain the label projection outbox (#201). The maintenance phases above may
4614
5335
  // have enqueued ops (settlement releases, recovery requeues, reconciles);
@@ -4802,6 +5523,28 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4802
5523
  log(`review revision dispatch failed: ${errText(err)}`);
4803
5524
  }
4804
5525
 
5526
+ // Adjudications are dispatch too (#932), and their own pass: an adjudication
5527
+ // occupies no worker slot and no issue — it opens a fresh read-only session
5528
+ // about a PR whose run has already settled — so it is neither bounded by the
5529
+ // worker cap nor allowed to consume a review round. Placed after revisions so
5530
+ // a PR with a live revision is never adjudicated in the same tick it is being
5531
+ // corrected in; the head re-read inside the pass is the durable guard.
5532
+ try {
5533
+ await dispatchReviewAdjudications(d, workers);
5534
+ } catch (err) {
5535
+ log(`adjudication dispatch failed: ${errText(err)}`);
5536
+ }
5537
+
5538
+ // And what the verdicts imply (#876). Its own pass, after dispatch, because a
5539
+ // disposition acts on a SETTLED adjudication: running it before dispatch would
5540
+ // simply be a tick late, and running it inside dispatch would tie a tracker
5541
+ // mutation to the launch that produced the verdict.
5542
+ try {
5543
+ await applyAdjudicationDispositions(d);
5544
+ } catch (err) {
5545
+ log(`adjudication dispositions failed: ${errText(err)}`);
5546
+ }
5547
+
4805
5548
  // route() filters the queue through isEligible() itself, so anything already
4806
5549
  // carrying a state label is gone before it gets here.
4807
5550
  const ready = await d.tracker.listReady();
@@ -4820,7 +5563,65 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4820
5563
  const pending = store.pendingLabelOpsFor(project.name, issue.number);
4821
5564
  return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
4822
5565
  });
4823
- const { routed, unroutable } = route(effective, project);
5566
+ // Post-unblock stale-list revalidation (#891). `listReady` is a
5567
+ // label-FILTERED search, and GitHub's search index is eventually consistent:
5568
+ // measured 2026-08-22T09:18Z, an `unblock` cleared `agent:failed` and
5569
+ // restored the queue label, `gh issue view` returned the clean label set
5570
+ // immediately, and the very next pass still saw the stale labels, held #889
5571
+ // as `stale-lifecycle` and left the fleet at 0/5 with four spare slots. The
5572
+ // next scheduled pass usually recovers — this is throughput, not correctness
5573
+ // — but an operator-driven unblock that cannot refill the fleet has not done
5574
+ // what it says.
5575
+ //
5576
+ // So a lifecycle label read from the LIST is a suspicion, not a verdict: the
5577
+ // exact per-issue read decides. Bounded to candidates that would otherwise be
5578
+ // dropped for a lifecycle label — never the whole queue — and further to those
5579
+ // whose newest run is not active (a genuinely in-flight issue needs no
5580
+ // revalidation; its own row is the authority) and that are not operator-parked
5581
+ // (a park is a decision, not a stale label).
5582
+ //
5583
+ // An unreadable exact read keeps the stale labels, so the issue stays out of
5584
+ // this pass, and is held as `issue-state-lookup-error` rather than
5585
+ // `stale-lifecycle`: "the tracker could not say" is not evidence of a
5586
+ // residual label, and it must not summon Duty 1 to reconcile a label nobody
5587
+ // has read.
5588
+ const stateLabelSet = new Set(Object.values(project.stateLabels));
5589
+ const suspect = effective.filter(
5590
+ (issue) =>
5591
+ !isEligible(issue, project) &&
5592
+ issue.labels.some((l) => stateLabelSet.has(l)) &&
5593
+ !issue.labels.includes(project.stateLabels.backlog) &&
5594
+ !ACTIVE_STATES.includes(store.latestRun(project.name, issue.number)?.state ?? "merged"),
5595
+ );
5596
+ const revalidated = new Map<number, readonly string[]>();
5597
+ const unreadable = new Set<number>();
5598
+ for (const issue of suspect) {
5599
+ let snapshot: IssueSnapshot | undefined;
5600
+ try {
5601
+ snapshot = await d.tracker.issueSnapshot(issue.number);
5602
+ } catch {
5603
+ snapshot = undefined;
5604
+ }
5605
+ if (snapshot === undefined) {
5606
+ unreadable.add(issue.number);
5607
+ log(`#${issue.number} lifecycle labels could not be revalidated — holding this pass`);
5608
+ continue;
5609
+ }
5610
+ // A closed issue is nobody's candidate; leave the list labels alone and let
5611
+ // the ordinary gates speak.
5612
+ if (snapshot.state === "closed") continue;
5613
+ if (snapshot.labels.join("\u0000") === issue.labels.join("\u0000")) continue;
5614
+ revalidated.set(issue.number, snapshot.labels);
5615
+ log(`#${issue.number} lifecycle labels revalidated: the queue list was stale`);
5616
+ }
5617
+ const verified =
5618
+ revalidated.size === 0
5619
+ ? effective
5620
+ : effective.map((issue) => {
5621
+ const live = revalidated.get(issue.number);
5622
+ return live === undefined ? issue : { ...issue, labels: [...live] };
5623
+ });
5624
+ const { routed, unroutable } = route(verified, project);
4824
5625
  // route() drops a candidate for two reasons, only one of which is a claim
4825
5626
  // question. A lifecycle state label (agent:in-progress/blocked/failed)
4826
5627
  // marks a run-owned issue: it is genuinely in flight while its newest run
@@ -4832,9 +5633,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4832
5633
  // close), which belongs in none of the three populations. So `claimed` is
4833
5634
  // defined from actual ownership over genuinely lifecycle-labelled candidates
4834
5635
  // rather than as the residual of routing (#228, #611).
4835
- const stateLabels = new Set(Object.values(project.stateLabels));
4836
- const dropped = effective.filter(
4837
- (issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
5636
+ // Every population below reads the VERIFIED list, so a candidate the exact
5637
+ // read cleared is not simultaneously admitted and counted as held (#891).
5638
+ const dropped = verified.filter(
5639
+ (issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabelSet.has(l)),
4838
5640
  );
4839
5641
  // The operator's park label is not a stale lifecycle label: the queue query
4840
5642
  // still returns a parked-and-queued issue, route() drops it as ineligible,
@@ -4843,7 +5645,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4843
5645
  // to reconcile a deliberate decision. Parked is its own population, derived
4844
5646
  // from the same isEligible read the gate uses, so the number status renders
4845
5647
  // cannot disagree with what admission would hold (#507).
4846
- const parkedCandidates = effective.filter(
5648
+ const parkedCandidates = verified.filter(
4847
5649
  (issue) => !isEligible(issue, project) && issue.labels.includes(project.stateLabels.backlog),
4848
5650
  );
4849
5651
  const parkedNumbers = new Set(parkedCandidates.map((issue) => issue.number));
@@ -4858,6 +5660,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4858
5660
  // A park never kills a live run; a parked issue with no run is inventory
4859
5661
  // the operator is deliberately holding, not reconciliation work.
4860
5662
  parked += 1;
5663
+ } else if (unreadable.has(issue.number)) {
5664
+ // Fail closed, and say which read failed: a tracker that could not answer
5665
+ // is not evidence of a residual label (#891).
5666
+ lifecycleHolds.push({
5667
+ issue: issue.number,
5668
+ reason: "issue-state-lookup-error",
5669
+ detail: "lifecycle labels could not be revalidated against the exact issue read",
5670
+ });
4861
5671
  } else {
4862
5672
  lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
4863
5673
  }
@@ -4959,6 +5769,8 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4959
5769
  return;
4960
5770
  }
4961
5771
  recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
5772
+ await recordInstallSurfaces(d);
5773
+ reconcilePanes(d, project.name, log);
4962
5774
 
4963
5775
  if (pass.admitted.length === 0) return;
4964
5776
 
@@ -5452,8 +6264,34 @@ export interface StatusSnapshot {
5452
6264
  * materialises it, so this is never absent on a real daemon.
5453
6265
  */
5454
6266
  review: ReviewPolicy;
5455
- /** Occupied issues: live workers plus green PRs awaiting a human merge. */
6267
+ /** Occupied issues: live workers plus green PRs awaiting a human merge. Both
6268
+ * lifecycle kinds, told apart by {@link StatusSnapshot.leasedRunIds}. */
5456
6269
  activeRuns: RunRecord[];
6270
+ /**
6271
+ * The subset of {@link StatusSnapshot.activeRuns} holding an **active
6272
+ * mutation lease** (#898): a live worker, or a dispatched unsettled review
6273
+ * revision. Everything else in `activeRuns` is a worker-free preserved
6274
+ * artifact — durable work awaiting review, merge or recovery, with nothing
6275
+ * writing to it.
6276
+ *
6277
+ * Carried as ids rather than a second run list so the two renderings cannot
6278
+ * disagree about the rows themselves, and derived from `Store.leasedRuns` —
6279
+ * the same query admission's file-lane gate and `conductor_pr_recover` read,
6280
+ * so what status calls a lease is exactly what those two enforce (#899,
6281
+ * #925). A plain array so the dashboard's JSON round-trip preserves it.
6282
+ */
6283
+ leasedRunIds?: readonly string[];
6284
+ /**
6285
+ * The review-ceiling adjudications an operator needs to see (#874): every
6286
+ * non-terminal one, plus the terminal one for each active run's PR head — so a
6287
+ * cleared or rejected verdict stays visible for exactly as long as the PR it
6288
+ * decided is still in flight, and disappears with it rather than accumulating.
6289
+ *
6290
+ * Keyed by nothing: the records carry their own `prUrl` and `headSha`, which is
6291
+ * how the renderer matches them to a run without inventing a second identity
6292
+ * for a PR.
6293
+ */
6294
+ reviewAdjudications?: readonly ReviewAdjudicationRecord[];
5457
6295
  /**
5458
6296
  * runId → live review-revision round, for runs whose revision worker is
5459
6297
  * currently dispatched (#692). Derived from the durable `review_revisions`
@@ -5462,7 +6300,22 @@ export interface StatusSnapshot {
5462
6300
  * (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
5463
6301
  * it byte for byte.
5464
6302
  */
5465
- reviewRounds?: Readonly<Record<string, number>>;
6303
+ /**
6304
+ * The live review round per run: its number AND the instant it was dispatched
6305
+ * (#802). The instant is what makes the line honest — a resumed session's
6306
+ * `turns` and `startedAt` are cumulative over the whole attempt, so without a
6307
+ * phase boundary a fresh revision reads as though it had been running for
6308
+ * hours.
6309
+ */
6310
+ reviewRounds?: Readonly<Record<string, { round: number; dispatchedAt: number }>>;
6311
+ /**
6312
+ * runId → the durable review evidence blocking a merge of the run's
6313
+ * pushed-green PR at its exact recorded head (#888). Derived from the same
6314
+ * `review_revisions` rows `conductor_pr_merge` refuses on, so status can
6315
+ * never present a green PR as merge-ready while the privileged verb will
6316
+ * refuse it. An absent entry means nothing blocks that run's head.
6317
+ */
6318
+ mergeBlockers?: Readonly<Record<string, ReviewHeadBlocker>>;
5466
6319
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
5467
6320
  * only copy of work the daemon could not save. */
5468
6321
  salvagedRuns: RunRecord[];
@@ -5496,6 +6349,20 @@ export interface StatusSnapshot {
5496
6349
  liveWorkers: number;
5497
6350
  runsToday: number;
5498
6351
  spendTodayUsd: number;
6352
+ /** The install identities the last dispatch pass recorded (#919). Absent
6353
+ * means no pass has looked yet, which the renderer says out loud rather
6354
+ * than presenting as agreement. */
6355
+ installSurfaces?: InstallSurfaceObservation;
6356
+ /** What live runs have reserved out of today's budget but not yet spent
6357
+ * (#851). Optional so a caller that built a snapshot before reservations
6358
+ * existed renders no figure rather than a fabricated zero. */
6359
+ reservedSpendUsd?: number;
6360
+ /**
6361
+ * Whether the figure above is built on runs that actually reported cost
6362
+ * (#970). Absent when the caller did not judge it; the renderer then says
6363
+ * nothing rather than implying the telemetry is sound.
6364
+ */
6365
+ spendTelemetry?: SpendTelemetryVerdict;
5499
6366
  dispatch?: DispatchSummary;
5500
6367
  /**
5501
6368
  * Latest plan-allowance verdict, when the caller read one. Optional because
@@ -5570,12 +6437,50 @@ export function statusSnapshotFromStore(
5570
6437
  // (#776 review #2).
5571
6438
  const live = store.liveRuns(p.name);
5572
6439
  const active = store.activeRuns(p.name);
6440
+ // Which of those active rows something is actually writing through (#898).
6441
+ // The same query admission's file-lane gate and `conductor_pr_recover` read,
6442
+ // so status cannot call a row a lease that those two treat as released, or
6443
+ // vice versa (#899, #925).
6444
+ const leasedRunIds = store.leasedRuns(p.name).map((r) => r.id);
6445
+ // Adjudications worth rendering (#874): the open ones always, plus a terminal
6446
+ // verdict for a head an active run is still sitting on. Bounded by the active
6447
+ // set rather than by a history window, so a fleet with a thousand settled
6448
+ // adjudications renders the handful that still describe live work.
6449
+ const openAdjudications = store.openReviewAdjudications(p.name);
6450
+ const adjudicationIds = new Set(openAdjudications.map((a) => a.id));
6451
+ const reviewAdjudications = [...openAdjudications];
6452
+ for (const r of active) {
6453
+ if (r.prUrl === undefined || r.headSha === undefined) continue;
6454
+ const decided = store.reviewAdjudicationForHead(p.name, r.prUrl, r.headSha);
6455
+ if (decided !== undefined && !adjudicationIds.has(decided.id)) {
6456
+ adjudicationIds.add(decided.id);
6457
+ reviewAdjudications.push(decided);
6458
+ }
6459
+ }
5573
6460
  // The live review-revision rounds, read from the same durable rows the
5574
6461
  // restart recovery uses: a run whose revision is dispatched is read as
5575
6462
  // `review-revision N` while its worker is live (#692).
5576
- const reviewRounds: Record<string, number> = {};
6463
+ const reviewRounds: Record<string, { round: number; dispatchedAt: number }> = {};
5577
6464
  for (const revision of store.unsettledReviewRevisions(p.name)) {
5578
- if (revision.dispatchedAt !== undefined) reviewRounds[revision.runId] = revision.round;
6465
+ if (revision.dispatchedAt !== undefined) {
6466
+ reviewRounds[revision.runId] = { round: revision.round, dispatchedAt: revision.dispatchedAt };
6467
+ }
6468
+ }
6469
+ // The exact-head merge blockers (#888), read from the same durable rows the
6470
+ // merge verb consults: a pushed-green run whose PR stands at a head carrying
6471
+ // unresolved review evidence — a round queued there, crashed mid-review
6472
+ // there, or settled `failed` there.
6473
+ const mergeBlockers: Record<string, ReviewHeadBlocker> = {};
6474
+ for (const r of active) {
6475
+ if (r.prUrl === undefined || r.headSha === undefined) continue;
6476
+ const blocker = store.mergeBlockingReviews(p.name, r.prUrl, r.headSha)[0];
6477
+ if (blocker !== undefined) {
6478
+ mergeBlockers[r.id] = {
6479
+ round: blocker.round,
6480
+ state:
6481
+ blocker.settledAt !== undefined ? "failed" : blocker.dispatchedAt !== undefined ? "crashed" : "pending",
6482
+ };
6483
+ }
5579
6484
  }
5580
6485
  return {
5581
6486
  project: p.name,
@@ -5600,6 +6505,8 @@ export function statusSnapshotFromStore(
5600
6505
  releaseGrants: resolveReleaseGrants(p),
5601
6506
  review: resolveReview(p),
5602
6507
  activeRuns: active,
6508
+ leasedRunIds,
6509
+ reviewAdjudications,
5603
6510
  reviewRounds,
5604
6511
  salvagedRuns: store.salvagedRuns(p.name),
5605
6512
  quarantinedRuns: store.quarantinedRuns(p.name),
@@ -5610,6 +6517,19 @@ export function statusSnapshotFromStore(
5610
6517
  liveWorkers: live.length,
5611
6518
  runsToday: store.runsStartedSince(p.name, since),
5612
6519
  spendTodayUsd: store.spendSince(p.name, since),
6520
+ reservedSpendUsd: store.reservedSpendUsd(p.name),
6521
+ // Judged from the store on the same read as the figure above (#970), so the
6522
+ // row that qualifies the spend number cannot disagree with it, and nothing
6523
+ // is probed at render time.
6524
+ spendTelemetry: judgeSpendTelemetry(
6525
+ store.recentSpendSamples?.(p.name, SPEND_SAMPLE_ROWS) ?? [],
6526
+ SPEND_SAMPLE_RUNS,
6527
+ ),
6528
+ // Read, never probed: the recorded row is the whole point (#919).
6529
+ ...(() => {
6530
+ const observed = store.installSurfaces();
6531
+ return observed === undefined ? {} : { installSurfaces: observed };
6532
+ })(),
5613
6533
  ...(dispatch === undefined ? {} : { dispatch }),
5614
6534
  ...(planUsage === undefined ? {} : { planUsage }),
5615
6535
  // Written by the tracker's hooks rather than polled, so the renderer does
@@ -5621,6 +6541,7 @@ export function statusSnapshotFromStore(
5621
6541
  : { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
5622
6542
  baseHealth: store.baseHealth(p.name),
5623
6543
  freezes: store.freezes(p.name),
6544
+ mergeBlockers,
5624
6545
  ...(orchestratorDown === undefined ? {} : { orchestratorDown }),
5625
6546
  };
5626
6547
  }
@@ -5674,8 +6595,15 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
5674
6595
  lines.push(` ${hold.reason} ${hold.count}${sample}`);
5675
6596
  // A `file-lane` hold groups several issues, each blocked by a different
5676
6597
  // file and holder; the grouped line says how many, this says which.
5677
- if ((hold.details?.length ?? 0) > 0) {
5678
- lines.push(` ${hold.details!.join(" | ")}`);
6598
+ //
6599
+ // Deduplicated for display only, never in the record: a fleet-wide hold
6600
+ // (`credential-class`, #852) gives every held candidate the *same*
6601
+ // sentence, and printing one reason five times reads as five problems.
6602
+ // The persisted details stay index-aligned with `issues` — a reader that
6603
+ // needs "which issue got which detail" still has it.
6604
+ const details = [...new Set(hold.details ?? [])];
6605
+ if (details.length > 0) {
6606
+ lines.push(` ${details.join(" | ")}`);
5679
6607
  }
5680
6608
  }
5681
6609
  }
@@ -5741,71 +6669,6 @@ export function formatFreezes(freezes: readonly BaseFreeze[]): string[] {
5741
6669
  }
5742
6670
 
5743
6671
 
5744
- export function formatStatus(s: StatusSnapshot): string {
5745
- const lines = [
5746
- `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
5747
- `config ${s.configPath}`,
5748
- `state ${s.stateDir}`,
5749
- "",
5750
- ...(s.orchestratorDown === undefined ? [] : formatOrchestratorDown(s.orchestratorDown)),
5751
- "caps",
5752
- ` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
5753
- ` issues today ${s.runsToday}`,
5754
- s.caps.dailySpendUsd === null
5755
- ? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
5756
- : ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
5757
- // Its own row, never folded into the spend row: they are two independent
5758
- // controls, and an operator has to be able to see which one stopped the
5759
- // fleet (#110).
5760
- ` plan usage ${planUsageLine(s.planUsage)}`,
5761
- ` new worker turns ${s.caps.workerMaxTurns}`,
5762
- ...s.turnOverrides.map(
5763
- ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
5764
- ),
5765
- ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
5766
- ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
5767
- ` continuations ${s.caps.maxContinuationsPerIssue}`,
5768
- "",
5769
- ...formatReleaseGrants(s.releaseGrants),
5770
- "",
5771
- formatDispatchSummary(s.dispatch),
5772
- "",
5773
- ];
5774
- if (s.activeRuns.length === 0) {
5775
- lines.push("active runs (none)");
5776
- } else {
5777
- lines.push("active runs");
5778
- for (const r of s.activeRuns) {
5779
- lines.push(
5780
- ` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
5781
- `${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
5782
- (r.prUrl ? ` ${r.prUrl}` : ""),
5783
- );
5784
- // Its escalation was deduplicated the moment it was delivered, so this
5785
- // line is the only place a flagged run stays visible while its PR waits
5786
- // for a merge — which is precisely the window the flag is about (#128).
5787
- const flagged = settlementFlagSummary(r.settlementFlags);
5788
- if (flagged !== undefined) lines.push(` ${flagged}`);
5789
- }
5790
- }
5791
- lines.push(...formatBaseHealth(s.baseHealth));
5792
- lines.push(...formatFreezes(s.freezes));
5793
- lines.push(...formatSalvagedRuns(s.salvagedRuns));
5794
- lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
5795
- lines.push(...formatOpenReports(s.openReports));
5796
- lines.push(...formatVerbLedger(s.verbLedger));
5797
- // Deploy hint: a restart while workers are live orphans them (salvage runs
5798
- // first — #35). Prefer pause + drain to zero live workers when you can wait.
5799
- if (s.liveWorkers > 0) {
5800
- lines.push(
5801
- "",
5802
- `deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
5803
- `pause and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
5804
- );
5805
- }
5806
- return lines.join("\n");
5807
- }
5808
-
5809
6672
  export interface QueuePreview {
5810
6673
  project: string;
5811
6674
  configPath: string;
@@ -6194,7 +7057,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6194
7057
  store.updateRun(runId, { maxTurns });
6195
7058
  });
6196
7059
  const workerControls = createWorkerControlRegistry();
6197
- const workers = createWorkerPool();
7060
+ // Created before the pool so a worker's completion can poke the loop it
7061
+ // shares with the interval sleep (#878).
7062
+ const pace = createDispatchPace();
7063
+ const workers = createWorkerPool(() => {
7064
+ // Sticky and coalescing by construction: several workers settling together
7065
+ // produce one pass, and a pass already running absorbs the request rather
7066
+ // than overlapping. No second owner is created and no gate is skipped.
7067
+ pace.requestWake();
7068
+ });
6198
7069
  const alive = livingDaemon();
6199
7070
  const runtimes: ProjectRuntime[] = [];
6200
7071
 
@@ -6241,6 +7112,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6241
7112
  : pushRunBranch(project, { repo, runRepoPath: run.worktree, branch });
6242
7113
  };
6243
7114
  for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
7115
+ // The workspace outlives the process, so a restart inherits panes whose
7116
+ // workers are gone (#842). A session-host child dies with the daemon that
7117
+ // owned its socket, so an orphaned row's pane is authoritatively dead
7118
+ // whatever its recorded pid says — pids are reused, and matching one
7119
+ // would be the mechanism that makes a stranger's process read as a live
7120
+ // worker. Release (never close) so lifecycle authority goes back to
7121
+ // Herdr and the pane stops claiming a worker conductor no longer runs.
7122
+ const orphanPane = releaseOrphanedWorkerPane(run);
7123
+ if (orphanPane.kind === "released") {
7124
+ projectLog(
7125
+ `#${run.issue} herdr pane ${orphanPane.paneId} released: its worker died with the previous daemon`,
7126
+ );
7127
+ } else if (orphanPane.kind === "failed") {
7128
+ projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} release failed: ${orphanPane.reason}`);
7129
+ }
6244
7130
  projectLog(
6245
7131
  `#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` +
6246
7132
  `worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` +
@@ -6365,7 +7251,27 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6365
7251
  cleanup: { next: 0 },
6366
7252
  probeCriticalBase: (repo, markers, branch) =>
6367
7253
  probeCriticalBase(project, repo, branch, markers),
7254
+ // The same seam `doctor` and `upgrade` read (#904/#919), never a second
7255
+ // implementation. `readHerdr` stays on: a host that runs herdr is the
7256
+ // case where a stale recovery pin matters, and a host without it answers
7257
+ // "absent", which no cheap surface treats as a fault.
7258
+ probeInstallSurfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
7259
+ probeTelegramFreshness: () =>
7260
+ // The module's own reader, so the daemon, `doctor` and `status` cannot
7261
+ // come to disagree about what is installed (#961).
7262
+ checkTelegramFreshness({
7263
+ run: async (cmd, args) => {
7264
+ const r = await runCommand(cmd, args);
7265
+ return { code: r.code, stdout: r.stdout };
7266
+ },
7267
+ }),
6368
7268
  probeWorktreeLane: (input) => probeRunLane(input),
7269
+ // The credential-class fence's reader (#852): the runnable probe module,
7270
+ // spawned against the live credential store. Wired once here so the
7271
+ // admission gate and the launch fence ask the same question through the
7272
+ // same transport — two readers would eventually disagree, and the one that
7273
+ // disagreed by passing is the expensive one.
7274
+ probeCredentialClass: (provider) => probeCredentialClass(provider),
6369
7275
  // A cross-repo Depends-on prerequisite reads through the same GitHub
6370
7276
  // credential/accounting seams as the project tracker — a fresh tracker
6371
7277
  // scoped to the referenced repo, reusing the daemon's gh hooks so API
@@ -6407,6 +7313,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6407
7313
  projectLog(`review revision restart recovery failed: ${errText(err)}`);
6408
7314
  }
6409
7315
  }
7316
+ // Adjudication restart recovery (#932). An adjudication is `running` only
7317
+ // while a session it launched is alive, and that session died with the
7318
+ // previous process — nothing resumes it, because an adjudicator has no
7319
+ // branch, no worktree and no transcript worth continuing: its whole output
7320
+ // is a verdict it never produced.
7321
+ //
7322
+ // So it settles `failed`, which is honest and terminal, rather than being
7323
+ // left `running` forever (a row nothing would ever touch again) or silently
7324
+ // re-queued (a second launch for one head, which the one-shot exists to
7325
+ // prevent). The one shot for that head is spent, and #876's disposition path
7326
+ // reads `failed` exactly as it reads any other non-clear verdict.
7327
+ if (alive === undefined || alive.pid === process.pid) {
7328
+ for (const adjudication of store.openReviewAdjudications(project.name)) {
7329
+ if (adjudication.state !== "running") continue;
7330
+ store.settleReviewAdjudication(
7331
+ adjudication.id,
7332
+ "failed",
7333
+ "the daemon restarted while this adjudication was running; no verdict was produced",
7334
+ Date.now(),
7335
+ );
7336
+ projectLog(
7337
+ `#${adjudication.issue} adjudication failed across restart — its session died with the previous daemon`,
7338
+ );
7339
+ }
7340
+ }
6410
7341
  // Startup reconciliation: close an incident carried over from a previous
6411
7342
  // process when the orchestrator is up (one recovery notice), or open one
6412
7343
  // when it failed to start (one down page). A daemon restarted while still
@@ -6504,7 +7435,6 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6504
7435
  // The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
6505
7436
  // sleep. A wake request is sticky and coalescing — several resumes in quick
6506
7437
  // succession produce one prompt pass, never an overlapping one.
6507
- const pace = createDispatchPace();
6508
7438
  const stop = (): void => {
6509
7439
  if (stopping) return;
6510
7440
  stopping = true;