omp-conductor 0.18.1 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +106 -41
  2. package/REFERENCE.md +866 -31
  3. package/agents/to-spec.md +6 -2
  4. package/package.json +1 -1
  5. package/schema/config.schema.json +32 -1
  6. package/src/admission.ts +212 -26
  7. package/src/arm-challenge.ts +250 -57
  8. package/src/ask.ts +288 -1
  9. package/src/briefs/orchestrator.md +27 -13
  10. package/src/briefs/to-spec.md +6 -2
  11. package/src/cli.ts +127 -2
  12. package/src/command-help.ts +9 -1
  13. package/src/command-manifest.ts +52 -8
  14. package/src/commands/arm.ts +6 -2
  15. package/src/commands/context.ts +2 -0
  16. package/src/commands/intake.ts +4 -19
  17. package/src/commands/message.ts +26 -2
  18. package/src/commands/reconcile-units.ts +104 -0
  19. package/src/commands/release-composition.ts +232 -0
  20. package/src/commands/resume.ts +2 -27
  21. package/src/commands/setup.ts +101 -16
  22. package/src/commands/stats.ts +11 -30
  23. package/src/commands/tail.ts +31 -1
  24. package/src/commands/upgrade.ts +20 -3
  25. package/src/commands/verb.ts +2 -1
  26. package/src/commands/watch.ts +4 -17
  27. package/src/config-schema.ts +38 -6
  28. package/src/config.ts +103 -8
  29. package/src/credential-class.ts +366 -0
  30. package/src/daemon.ts +1368 -529
  31. package/src/dashboard/app.js +504 -2
  32. package/src/dashboard/controls.ts +336 -0
  33. package/src/dashboard/index.html +30 -0
  34. package/src/dashboard/server.ts +271 -30
  35. package/src/dashboard/style.css +116 -0
  36. package/src/dashboard/transcript.ts +173 -0
  37. package/src/decisions.ts +19 -11
  38. package/src/doctor.ts +431 -148
  39. package/src/escalate.ts +22 -11
  40. package/src/failure-class.ts +59 -0
  41. package/src/fleet.ts +587 -230
  42. package/src/host.ts +6 -455
  43. package/src/omp-settings.ts +19 -0
  44. package/src/omp.ts +40 -56
  45. package/src/orchestrator-tick.ts +564 -121
  46. package/src/pause.ts +233 -0
  47. package/src/session-host.ts +6 -41
  48. package/src/settlement.ts +159 -2
  49. package/src/setup-answers.ts +97 -0
  50. package/src/setup-host.ts +343 -1160
  51. package/src/setup-install.ts +204 -27
  52. package/src/setup-wizard.ts +252 -51
  53. package/src/setup.ts +87 -4
  54. package/src/spend-telemetry.ts +117 -0
  55. package/src/stats.ts +35 -0
  56. package/src/status-render.ts +485 -19
  57. package/src/store.ts +1229 -55
  58. package/src/telegram-freshness.ts +269 -0
  59. package/src/to-spec.ts +50 -2
  60. package/src/types.ts +759 -10
  61. package/src/unblock.ts +22 -0
  62. package/src/unit-reconcile.ts +303 -0
  63. package/src/upgrade-verify.ts +8 -1
  64. package/src/upgrade.ts +299 -12
  65. package/src/verbs/actions.ts +124 -10
  66. package/src/verbs/protocol.ts +70 -2
  67. package/src/verbs/server.ts +485 -11
  68. package/src/wake.ts +48 -0
  69. package/src/worker.ts +401 -14
package/src/daemon.ts CHANGED
@@ -36,20 +36,38 @@ import {
36
36
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
37
37
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
38
38
  import { graphHint } from "./graph.ts";
39
- import {
40
- hostConstraintsNotice,
41
- resolveWorkerIdentity,
42
- WORKER_ACCOUNT,
43
- type WorkerIdentity,
44
- type WorkerIdentityResolution,
45
- } from "./host.ts";
39
+ import { hostConstraintsNotice } from "./host.ts";
46
40
  import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
47
41
  import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
48
- import { runDoctor } from "./doctor.ts";
42
+ import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS, runDoctor } from "./doctor.ts";
43
+ import { judgeSpendTelemetry, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
44
+ import {
45
+ clearPauseIfUnchanged,
46
+ isPaused,
47
+ pausedAt,
48
+ pausedPath,
49
+ pauseInstance,
50
+ pauseInstanceAt,
51
+ pauseProvenance,
52
+ pauseSourceToken,
53
+ setPaused,
54
+ } from "./pause.ts";
49
55
  import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
50
56
  import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
57
+ import { inspectSurfaces, type InstalledSurfaces } from "./upgrade.ts";
58
+ import { checkTelegramFreshness, type TelegramFreshness } from "./telegram-freshness.ts";
51
59
  import { processStartTimeMs } from "./upgrade-verify.ts";
52
- import { fleetLayers, herdrPaneOmpStarts, resolveHerdrSession } from "./fleet.ts";
60
+ import {
61
+ fleetLayers,
62
+ herdrPaneOmpStarts,
63
+ openWorkerPane,
64
+ reconcileWorkerPanes,
65
+ releaseOrphanedWorkerPane,
66
+ releaseWorkerPane,
67
+ reportWorkerPaneState,
68
+ resolveHerdrSession,
69
+ type WorkerPaneOutcome,
70
+ } from "./fleet.ts";
53
71
  import { startOrchestrator } from "./orchestrator.ts";
54
72
  import type { OrchestratorHandle } from "./orchestrator.ts";
55
73
  import {
@@ -77,11 +95,18 @@ import {
77
95
  effectiveModel,
78
96
  hasContinuationBudget,
79
97
  hasFailedAttemptBudget,
98
+ runSpendAllowanceUsd,
99
+ startOfToday,
80
100
  laneEcho,
81
101
  } from "./admission.ts";
82
102
  import type { Admission, AdmissionHold } from "./admission.ts";
83
103
  import type { EffectiveModel, FileLane } from "./types.ts";
84
104
  import { log, errText, safeEscalate } from "./log.ts";
105
+ import {
106
+ oauthFenceVerdict,
107
+ probeCredentialClass,
108
+ type CredentialClassProbeResult,
109
+ } from "./credential-class.ts";
85
110
  import {
86
111
  adoptSalvagedPrs,
87
112
  classifyAndRecover,
@@ -91,6 +116,7 @@ import {
91
116
  reactToProviderCredit,
92
117
  readSessionError,
93
118
  reconcileOrphanedRuns,
119
+ reconcileGroomingClosures,
94
120
  reconcileStaleLabels,
95
121
  recordOperatorStop,
96
122
  settlePushedGreen,
@@ -99,7 +125,13 @@ import {
99
125
  } from "./settlement.ts";
100
126
  import { materializeOmpSettings } from "./omp-settings.ts";
101
127
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
102
- import { infraLogSignature, infraSignatureVersion, providerCreditRefusal, providerTransientFault } from "./failure-class.ts";
128
+ import {
129
+ infraLogSignature,
130
+ infraSignatureVersion,
131
+ providerCreditRefusal,
132
+ providerTransientFault,
133
+ reviewRoundNeverWorked,
134
+ } from "./failure-class.ts";
103
135
  import {
104
136
  DEFAULT_MODEL_FALLBACK_THRESHOLD,
105
137
  fallbackClause,
@@ -118,10 +150,11 @@ import {
118
150
  utcDay,
119
151
  } from "./store.ts";
120
152
  import { GhPrMissingError, GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
121
- import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
153
+ import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES, REVIEW_ROUND_INFRA_MAX_RETRIES } from "./types.ts";
122
154
  import type {
123
155
  BaseFreeze,
124
156
  BaseHealth,
157
+ InstallSurfaceObservation,
125
158
  AdmissionHoldReason,
126
159
  Caps,
127
160
  ConductorConfig,
@@ -131,6 +164,8 @@ import type {
131
164
  InterruptCategory,
132
165
  IssueComment,
133
166
  IssueSnapshot,
167
+ PrDiff,
168
+ ReviewAdjudicationRecord,
134
169
  MergedPrInfo,
135
170
  OpenCloser,
136
171
  ReleaseShape,
@@ -147,6 +182,7 @@ import type {
147
182
  FailureClass,
148
183
  HostConstraints,
149
184
  RecoveryAction,
185
+ ReviewHeadBlocker,
150
186
  ReviewPolicy,
151
187
  ReviewRevisionOutcome,
152
188
  ReviewRevisionRecord,
@@ -160,14 +196,19 @@ import type {
160
196
  WorkflowRun,
161
197
  } from "./types.ts";
162
198
  import {
199
+ type AdjudicationRound,
200
+ type AdjudicatorOpts,
201
+ type AdjudicationResult,
163
202
  type KilledBy,
164
203
  type WorkerPauseControl,
165
204
  type WorkerPausePhase,
166
205
  type WorkerResult,
167
206
  type RunWorkerDeps,
168
207
  ORPHAN_RESUME_PROMPT,
208
+ renderAdjudicationBrief,
169
209
  renderBrief,
170
210
  renderReviewRevisionPrompt,
211
+ runAdjudicator,
171
212
  runWorker,
172
213
  } from "./worker.ts";
173
214
  import {
@@ -286,33 +327,6 @@ interface Deps {
286
327
  workerControls: WorkerControlRegistry;
287
328
  /** Session seam for lifecycle integration tests; production uses the real harness. */
288
329
  workerDeps?: RunWorkerDeps;
289
- /**
290
- * Resolves the dedicated worker identity (#798) — the account worker sessions
291
- * run under, with its uid/gid/home, the transition launcher and a live harness
292
- * binding (#828).
293
- *
294
- * Called **at every worker launch**, never cached: every input is host state
295
- * that can change under a running daemon. A reboot can start this service
296
- * before systemd has mounted the harness binding, and an operator can install
297
- * the account or re-run `setup host` at any time — a verdict taken once at
298
- * startup would hold the whole fleet closed until somebody thought to restart
299
- * the daemon, which is the outage #828 exists to end rather than relocate.
300
- * Resolution is a handful of `stat` calls and one `/etc/passwd` read; a
301
- * dispatch does far more than that before it reaches this gate.
302
- *
303
- * Every worker dispatch refuses to launch on an unresolved identity (fail
304
- * closed — an unbound worker is indistinguishable from an operator shell),
305
- * and this is how the dispatcher secures the run's sockets and grants the
306
- * run's paths to the account. Absent entirely, dispatch fails closed naming
307
- * the missing account.
308
- */
309
- workerIdentity?: () => WorkerIdentityResolution;
310
- /**
311
- * Re-owns a run's working paths (worktree + session dir) under the worker
312
- * identity before the session launches. Wired by `runDaemon` to the
313
- * recursive chown; a test fixture leaves it unset so no test chowns.
314
- */
315
- grantWorkerPaths?: (identity: WorkerIdentity, worktreePath: string, sessionDir: string) => void;
316
330
  integrity: IntegrityGate;
317
331
  stall: StallGate;
318
332
  /**
@@ -365,6 +379,17 @@ interface Deps {
365
379
  * probe the way `criticalBase` does.
366
380
  */
367
381
  probeWorktreeLane?: RunLaneProbe;
382
+ /**
383
+ * Reads one provider's current credential class from the harness, out of
384
+ * process (#852). Wired by `runDaemon` to {@link probeCredentialClass}; a test
385
+ * injects a fake.
386
+ *
387
+ * Absent, both fences refuse a project that declares `requireOauthProviders` —
388
+ * this one fails CLOSED, unlike `probeWorktreeLane`, because an unverified
389
+ * credential class costs exactly what a wrong one costs. A project declaring
390
+ * nothing never calls it, so an unwired test dispatches as it always has.
391
+ */
392
+ probeCredentialClass?: (provider: string) => Promise<CredentialClassProbeResult>;
368
393
  /**
369
394
  * Reads one issue's tracker state in a repository the admission tracker is
370
395
  * not bound to — the cross-repo Depends-on interlock (#420). Wired by
@@ -379,6 +404,24 @@ interface Deps {
379
404
  * body fails that branch closed rather than synthesising a cycle.
380
405
  */
381
406
  probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
407
+ /**
408
+ * Reads the three install identities this host carries (#919). Wired by
409
+ * `runDaemon` to `inspectSurfaces`, the same seam `doctor` and `upgrade`
410
+ * read, so nothing re-implements the probe. A test injects its own; absent,
411
+ * the pass records nothing and every cheap surface honestly says "not
412
+ * observed yet" rather than claiming agreement.
413
+ */
414
+ probeInstallSurfaces?: () => Promise<InstalledSurfaces>;
415
+ /** The `omp-telegram` install/daemon/published triple, read on the same
416
+ * periodic pass as the surfaces above (#961). */
417
+ probeTelegramFreshness?: () => Promise<TelegramFreshness>;
418
+ /**
419
+ * Runs one review-ceiling adjudication (#932). Production is
420
+ * {@link runAdjudicator}; a test injects a fake so the assertion can be the
421
+ * launch arguments and the assembled brief — which is what this pass actually
422
+ * produces — rather than a stored flag.
423
+ */
424
+ runAdjudicatorImpl?: (opts: AdjudicatorOpts) => Promise<AdjudicationResult>;
382
425
  }
383
426
 
384
427
  /**
@@ -408,6 +451,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
408
451
  log,
409
452
  now: () => Date.now(),
410
453
  chain: { readBaseChain },
454
+ lane: { probeRunLane },
411
455
  };
412
456
  }
413
457
 
@@ -500,121 +544,6 @@ export function checkStall(gate: StallGate, marker: string, now = Date.now()): S
500
544
  * could destroy work an operator would rather read first — the same refusal to
501
545
  * guess that the recovery plugin is built on.
502
546
  */
503
- // --------------------------------------------------------- worker identity (#798) --
504
-
505
- /**
506
- * Re-own a run's working paths (worktree, session dir) under the worker
507
- * identity before the session launches. The daemon runs this as root; the
508
- * worker identity is granted its own run paths by ownership, never by
509
- * loosened modes on the daemon's.
510
- *
511
- * The walk never re-owns through a path a worker-uid process could
512
- * re-resolve: every ownership change is `fchownSync` on a descriptor opened
513
- * with `O_NOFOLLOW`, and every descriptor is verified — via `/proc/self/fd`'s
514
- * kernel-resolved path — to still sit under the directory the walk opened.
515
- * A directory entry raced into a symlink therefore either fails `O_NOFOLLOW`
516
- * at the final component or resolves outside the verified parent and is
517
- * skipped; it can never carry the chown to an external target. Symlinks
518
- * within the tree are left untouched (the worker manages entries through its
519
- * parent directories, and git recreates links on checkout), as are anything
520
- * unopenable — fifos, sockets, devices — which git never creates. A missing
521
- * or racing entry is not this dispatch's problem — the next dispatch re-owns
522
- * whatever survives; a tree already owned by the worker identity — the
523
- * resume and review-revision re-ownership of a tree a prior dispatch granted
524
- * — is the only tree a worker-uid process could have modified, and is
525
- * skipped outright rather than walked.
526
- */
527
- export function chownRecursive(root: string, uid: number, gid: number): void {
528
- try {
529
- const current = lstatSync(root);
530
- if (current.uid === uid && current.gid === gid) return;
531
- } catch {
532
- // A missing or racing root is the caller's own existence check; the next
533
- // dispatch re-owns whatever survives.
534
- return;
535
- }
536
- const rootFd = openNoFollowDir(root);
537
- try {
538
- walkDir(rootFd, root, uid, gid);
539
- } finally {
540
- try {
541
- closeSync(rootFd);
542
- } catch {
543
- // Already closed by a raced-away walk; nothing to do.
544
- }
545
- }
546
- }
547
-
548
- /** Open a directory without following a final-component symlink. */
549
- function openNoFollowDir(path: string): number {
550
- return openSync(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
551
- }
552
-
553
- /** The kernel-resolved path of an open descriptor: what the object actually is. */
554
- function fdRealPath(fd: number): string | undefined {
555
- try {
556
- return readlinkSync(`/proc/self/fd/${fd}`);
557
- } catch {
558
- return undefined;
559
- }
560
- }
561
-
562
- function walkDir(dirFd: number, dirPath: string, uid: number, gid: number): void {
563
- try {
564
- fchownSync(dirFd, uid, gid);
565
- } catch {
566
- // Raced away; the next dispatch re-owns what survives.
567
- }
568
- const dirReal = fdRealPath(dirFd);
569
- if (dirReal === undefined) return;
570
- let entries: Dirent[];
571
- try {
572
- entries = readdirSync(dirPath, { withFileTypes: true });
573
- } catch {
574
- return;
575
- }
576
- for (const entry of entries) {
577
- const childPath = join(dirPath, entry.name);
578
- try {
579
- if (entry.isDirectory()) {
580
- const childFd = openNoFollowDir(childPath);
581
- try {
582
- // The entry is genuinely beneath the directory this fd owns only
583
- // when the kernel resolves the opened object to a path under it.
584
- // Anything else — a name swapped to a symlink, a foreign listing
585
- // read through a replaced parent — is refused, never re-resolved.
586
- const childReal = fdRealPath(childFd);
587
- if (childReal === undefined || !childReal.startsWith(`${dirReal}/`)) continue;
588
- walkDir(childFd, childPath, uid, gid);
589
- } finally {
590
- try {
591
- closeSync(childFd);
592
- } catch {
593
- // Raced away; nothing to close.
594
- }
595
- }
596
- continue;
597
- }
598
- // Symlinks stay untouched (never followed, never chowned), and so do
599
- // entries a read-only open cannot name safely (fifos, sockets, devices).
600
- if (entry.isSymbolicLink() || !entry.isFile()) continue;
601
- const fd = openSync(childPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
602
- try {
603
- const real = fdRealPath(fd);
604
- if (real !== undefined && real.startsWith(`${dirReal}/`)) fchownSync(fd, uid, gid);
605
- } finally {
606
- try {
607
- closeSync(fd);
608
- } catch {
609
- // Raced away; nothing to close.
610
- }
611
- }
612
- } catch {
613
- // O_NOFOLLOW refusal (a symlink swapped onto the name), or gone: nothing to chown.
614
- }
615
- }
616
- }
617
-
618
547
  export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
619
548
  const marker = join(stateDir(), STALL_MARKER_FILE);
620
549
  const repeat = d.stall.paged;
@@ -664,180 +593,21 @@ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void
664
593
  markPaged(d.stall, delivered, now);
665
594
  }
666
595
 
667
- /**
668
- * Pause is a file rather than process state on purpose: `omp-conductor hold`
669
- * runs in a different process from the daemon, and a flag
670
- * on disk needs no IPC and survives a restart. A daemon that crashed while
671
- * paused comes back paused.
672
- */
673
- export function pausedPath(project?: string): string {
674
- return join(stateDir(), project === undefined ? "paused" : `paused-${project}`);
675
- }
676
-
677
- function activePausePaths(project?: string): string[] {
678
- const paths = project === undefined ? [pausedPath()] : [pausedPath(project), pausedPath()];
679
- return paths.filter((path) => existsSync(path));
680
- }
681
-
682
- export function isPaused(project?: string): boolean {
683
- return activePausePaths(project).length !== 0;
684
- }
685
-
686
- /**
687
- * The epoch-ms timestamp at which the current pause began. A project pause also
688
- * observes the legacy bare sentinel, which pauses every project. If any active
689
- * sentinel is unreadable or unparseable, the timestamp is unknown so callers
690
- * continue to fail closed.
691
- */
692
- export function pausedAt(project?: string): number | undefined {
693
- const paths = activePausePaths(project);
694
- if (paths.length === 0) return undefined;
695
- const times: number[] = [];
696
- for (const path of paths) {
697
- try {
698
- const first = readFileSync(path, "utf8").split("\n")[0]?.trim();
699
- if (first === undefined || first === "") return undefined;
700
- const time = Date.parse(first);
701
- if (Number.isNaN(time)) return undefined;
702
- times.push(time);
703
- } catch {
704
- return undefined;
705
- }
706
- }
707
- return Math.min(...times);
708
- }
709
-
710
- /**
711
- * Who paused the project and why. Per-project provenance wins when both its
712
- * sentinel and the legacy all-project sentinel are active.
713
- */
714
- export function pauseProvenance(
715
- project?: string,
716
- ): { source: string; reason?: string } | undefined {
717
- const paths =
718
- project === undefined
719
- ? [pausedPath()]
720
- : [pausedPath(project), pausedPath()];
721
- const path = paths.find((candidate) => existsSync(candidate));
722
- if (path === undefined) return undefined;
723
- try {
724
- const second = readFileSync(path, "utf8").split("\n")[1];
725
- if (second === undefined) return undefined;
726
- const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(second.trim());
727
- if (match === null) return undefined;
728
- const source = match[1]!;
729
- const reason = match[2];
730
- return { source, ...(reason === undefined ? {} : { reason }) };
731
- } catch {
732
- return undefined;
733
- }
734
- }
735
-
736
- /**
737
- * One pause sentinel FILE read as an instance identity: who set it, why, and
738
- * the creation instant, all from that exact file. An unreadable or malformed
739
- * file is undefined — the caller may treat it as absence.
740
- */
741
- function pauseInstanceAt(
742
- path: string,
743
- ): { source: string; reason?: string; since: number } | undefined {
744
- try {
745
- const [line1, line2] = readFileSync(path, "utf8").split("\n");
746
- const since = Date.parse(line1?.trim() ?? "");
747
- if (!Number.isFinite(since)) return undefined;
748
- if (line2 === undefined) return undefined;
749
- const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
750
- if (match === null) return undefined;
751
- const source = match[1]!;
752
- const reason = match[2];
753
- return { source, since, ...(reason === undefined ? {} : { reason }) };
754
- } catch {
755
- return undefined;
756
- }
757
- }
758
-
759
- /**
760
- * One pause sentinel read as a single identity: who set it, why, and the
761
- * creation instant, all from the SAME file that was selected. Unlike pairing
762
- * {@link pauseProvenance} with {@link pausedAt} — which can describe
763
- * different files when a project pause coexists with the legacy global
764
- * sentinel, letting a stale global timestamp mask a recreated project pause —
765
- * this reads provenance and timestamp from one sentinel, so a caller can prove
766
- * "the pause I set still exists" instead of "some pause with the same labels
767
- * still exists" (#377). Per-project sentinel wins, like {@link pauseProvenance}.
768
- */
769
- export function pauseInstance(
770
- project?: string,
771
- ): { source: string; reason?: string; since: number } | undefined {
772
- const paths =
773
- project === undefined
774
- ? [pausedPath()]
775
- : [pausedPath(project), pausedPath()];
776
- const path = paths.find((candidate) => existsSync(candidate));
777
- if (path === undefined) return undefined;
778
- return pauseInstanceAt(path);
779
- }
780
-
781
- /**
782
- * Compare-and-clear one pause sentinel (#780 review): remove `path` only while
783
- * it still holds exactly the `expected` instance — same source, same reason,
784
- * same creation instant — as read by {@link pauseInstance}. A hold or pause
785
- * that replaced or recreated the sentinel between the read and the clear is a
786
- * newer instance (writes always re-stamp `since`), so it is never destroyed:
787
- * returning false keeps the newer fence in force. Scoped strictly to one
788
- * caller-provided path, so auto-expiry can clear the per-project spend-cap
789
- * sentinel without ever touching the legacy global sentinel.
790
- */
791
- export function clearPauseIfUnchanged(
792
- path: string,
793
- expected: { source: string; reason?: string; since: number },
794
- ): boolean {
795
- const current = pauseInstanceAt(path);
796
- if (current === undefined) return false;
797
- if (
798
- current.source !== expected.source ||
799
- current.since !== expected.since ||
800
- current.reason !== expected.reason
801
- ) {
802
- return false;
803
- }
804
- rmSync(path, { force: true });
805
- return true;
806
- }
807
-
808
- /**
809
- * The pause sentinel's `source=` token, proven from a verb. The sentinel's
810
- * source line is read back as a single `\S+` token (see {@link pauseInstance}
811
- * and {@link pauseProvenance}), so a verb that contains a space (`setup host`)
812
- * is unrepresentable verbatim and must be encoded before it reaches disk —
813
- * otherwise the fence cannot prove its own pause and refuses forever (#552).
814
- * Spaces become `-`; the human-readable verb is preserved in the sentinel's
815
- * `reason=` instead.
816
- */
817
- export function pauseSourceToken(verb: string): string {
818
- return verb.trim().replace(/\s+/g, "-");
819
- }
820
-
821
- export function setPaused(
822
- v: boolean,
823
- why?: { source: string; reason?: string },
824
- project?: string,
825
- ): void {
826
- const path = pausedPath(project);
827
- if (v) {
828
- mkdirSync(dirname(path), { recursive: true });
829
- const line1 = `${new Date().toISOString()}\n`;
830
- if (why === undefined) {
831
- writeFileSync(path, line1);
832
- } else {
833
- const reason = why.reason === undefined ? "" : ` reason="${why.reason.replaceAll('"', "")}"`;
834
- writeFileSync(path, `${line1}source=${why.source}${reason}\n`);
835
- }
836
- } else {
837
- rmSync(path, { force: true });
838
- if (project !== undefined) rmSync(pausedPath(), { force: true });
839
- }
840
- }
596
+ // ------------------------------------------------------------------- pause
597
+ // The sentinel itself lives in `pause.ts` (#938): this module imports
598
+ // `doctor.ts`, so a health check that needs to read a fence cannot import back
599
+ // here. Re-exported so every existing `from "./daemon.ts"` import still
600
+ // resolves the move is structural, not a change of surface.
601
+ export {
602
+ clearPauseIfUnchanged,
603
+ isPaused,
604
+ pausedAt,
605
+ pausedPath,
606
+ pauseInstance,
607
+ pauseProvenance,
608
+ pauseSourceToken,
609
+ setPaused,
610
+ };
841
611
 
842
612
  // ------------------------------------------------------------------- drain
843
613
  // (#484 slice 1) A project drain is a durable, self-expiring admission fence:
@@ -1238,19 +1008,6 @@ export function markPaged(
1238
1008
 
1239
1009
  // ---------------------------------------------------------------------- helpers
1240
1010
 
1241
- /**
1242
- * Local midnight, matching how a human reads "today".
1243
- *
1244
- * ponytail: a rolling 24h window would be fairer to a run that started at
1245
- * 23:50, but midnight is what someone checking a morning spend report expects.
1246
- * Upgrade path is a `capWindow: "day" | "rolling24h"` config key.
1247
- */
1248
- function startOfToday(): number {
1249
- const d = new Date();
1250
- d.setHours(0, 0, 0, 0);
1251
- return d.getTime();
1252
- }
1253
-
1254
1011
  /**
1255
1012
  * `owner/repo` for `gh`, derived from the clone URL.
1256
1013
  *
@@ -1448,6 +1205,7 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
1448
1205
  function endedBy(killedBy: KilledBy | undefined): string {
1449
1206
  if (killedBy === "turns") return "killed by the turns cap";
1450
1207
  if (killedBy === "wallclock") return "killed by the wall-clock cap";
1208
+ if (killedBy === "spend") return "killed by the per-run spend cap";
1451
1209
  return "killed by a failed run";
1452
1210
  }
1453
1211
 
@@ -1653,7 +1411,19 @@ export interface WorkerControlSlot {
1653
1411
  }
1654
1412
 
1655
1413
  export interface WorkerControlRegistry {
1656
- open(project: string, issue: number, runId: string): WorkerControlSlot;
1414
+ /**
1415
+ * `onPhase` is called after a pause/resume/stop that actually changed this
1416
+ * worker's phase (#842) — the authoritative transition, from the same control
1417
+ * that performed it. It exists so a surface outside this registry (the run's
1418
+ * Herdr representation) can follow the phase without polling and without
1419
+ * inventing a second notion of "paused".
1420
+ */
1421
+ open(
1422
+ project: string,
1423
+ issue: number,
1424
+ runId: string,
1425
+ onPhase?: (phase: WorkerPausePhase) => void,
1426
+ ): WorkerControlSlot;
1657
1427
  pause(project: string, issue: number): Promise<WorkerControlResult>;
1658
1428
  resume(project: string, issue: number): WorkerControlResult;
1659
1429
  stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
@@ -1671,12 +1441,13 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1671
1441
  stopReason?: string;
1672
1442
  stopError?: string;
1673
1443
  finished: PromiseWithResolvers<void>;
1444
+ onPhase?: (phase: WorkerPausePhase) => void;
1674
1445
  }
1675
1446
 
1676
1447
  const active = new Map<string, Entry>();
1677
1448
  const key = (project: string, issue: number): string => `${project}\0${issue}`;
1678
1449
  return {
1679
- open(project, issue, runId) {
1450
+ open(project, issue, runId, onPhase) {
1680
1451
  const k = key(project, issue);
1681
1452
  if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
1682
1453
  const entry: Entry = {
@@ -1684,6 +1455,7 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1684
1455
  issue,
1685
1456
  runId,
1686
1457
  finished: Promise.withResolvers<void>(),
1458
+ ...(onPhase === undefined ? {} : { onPhase }),
1687
1459
  };
1688
1460
  active.set(k, entry);
1689
1461
  return {
@@ -1709,7 +1481,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1709
1481
  if (entry?.control === undefined) return { kind: "not-active" };
1710
1482
  try {
1711
1483
  await entry.control.pause();
1712
- return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
1484
+ const phase = entry.control.phase();
1485
+ entry.onPhase?.(phase);
1486
+ return { kind: "ok", runId: entry.runId, phase };
1713
1487
  } catch (err) {
1714
1488
  return {
1715
1489
  kind: "refused",
@@ -1723,7 +1497,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1723
1497
  if (entry?.control === undefined) return { kind: "not-active" };
1724
1498
  try {
1725
1499
  entry.control.resume();
1726
- return { kind: "ok", runId: entry.runId, phase: entry.control.phase() };
1500
+ const phase = entry.control.phase();
1501
+ entry.onPhase?.(phase);
1502
+ return { kind: "ok", runId: entry.runId, phase };
1727
1503
  } catch (err) {
1728
1504
  return {
1729
1505
  kind: "refused",
@@ -1768,17 +1544,147 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
1768
1544
  };
1769
1545
  }
1770
1546
 
1547
+ /**
1548
+ * The daemon half of a worker's `pushed-green` claim (#85's contact with
1549
+ * reality; #782's publication attribution).
1550
+ *
1551
+ * A yield is a transport, not proof of GitHub side effects: `claim` is
1552
+ * caller-supplied text and the worker adapter turns it into a green result
1553
+ * without the daemon ever having seen a mediated publication verb for it — the
1554
+ * shape that let the #777 incident guess a non-existent PR. The tracker read
1555
+ * at the end proves the PR is open and green at the exact head; the evidence
1556
+ * gate here proves that PR is *this run's own mediated work* first.
1557
+ *
1558
+ * Two things are asked, in cost order:
1559
+ *
1560
+ * - the claimed URL must equal the one this run's row records, which is never
1561
+ * a worker-supplied string at the moment a live run is verified —
1562
+ * conductor_pr_create (create or adoption) writes it on the run, the daemon
1563
+ * seeds it at claim from a terminal predecessor on the same branch (#434),
1564
+ * and orchestrator-only recovery writes it for settled runs;
1565
+ * - and the claimed head must have a mediated publisher: an allowed
1566
+ * `conductor_push` on this exact run that published it on this run's branch
1567
+ * (the verb only ever pushes `refs/heads/<branch>`), an allowed
1568
+ * `conductor_pr_create` on this run for this exact PR, or — for a run that
1569
+ * published nothing new — the live tip of this run's own branch.
1570
+ *
1571
+ * That last path is the one the row cannot supply. It used to be served by
1572
+ * comparing the claim against `run.prUrl`/`run.headSha`, and both are
1573
+ * worker-tainted upstream: terminal settlement writes a worker's reported pair
1574
+ * onto the row *before* any verification, and continuation inheritance
1575
+ * validates only the predecessor's metadata and open state — so a turn-capped
1576
+ * attempt that reported someone else's real green PR had that pair inherited
1577
+ * and re-presented as its own evidence. Asking the tracker for the branch tip
1578
+ * removes the worker from the loop entirely: `refs/heads/<branch>` is a ref
1579
+ * only a mediated `conductor_push` or a mediated `conductor_pr_update_branch`
1580
+ * can move, so its live commit is daemon provenance no reported string can
1581
+ * forge. It is also why a legitimate mediated base-branch update now passes:
1582
+ * that server-side merge creates a head no `conductor_push` ever published and
1583
+ * the row still carries the older one, which the recorded-pair test rejected.
1584
+ *
1585
+ * Binding to the *exact current run* is what makes an old attempt, a different
1586
+ * branch, or #806's orchestrator-only settled-run recovery invisible here: the
1587
+ * ledger query is run-scoped by `runId`, and recoveries store no runId at all.
1588
+ * That query asks for the run's complete history rather than the ledger's
1589
+ * newest rows — publication evidence sits at the *start* of a run, and a
1590
+ * review-revision round or a burst of refused mutations pushed it past the
1591
+ * default page, turning a verified push into a definitive false failure.
1592
+ *
1593
+ * A claim that fails this gate is a definitive failure — never a retryable
1594
+ * `pushed-pending` — because a guessed URL is not something a later tick is
1595
+ * waiting on. A claim the gate could not *read* is the opposite: an unreadable
1596
+ * branch tip is #781's transient outage, so it stays retryable rather than
1597
+ * burning an attempt on a flaky read.
1598
+ */
1771
1599
  export async function verifyPushedGreenClaim(
1772
- tracker: Pick<Tracker, "verifyPr">,
1600
+ tracker: Pick<Tracker, "verifyPr" | "branchHead">,
1773
1601
  claim: Pick<WorkerResult, "prUrl" | "headSha">,
1602
+ publication: {
1603
+ project: string;
1604
+ issue: number;
1605
+ runId: string;
1606
+ /** The branch conductor routed this run onto. `conductor_push` publishes
1607
+ * exactly `refs/heads/<branch>` and refuses any other ref. */
1608
+ branch: string;
1609
+ /** The identity `tracker.branchHead` reads the live tip with: the
1610
+ * canonical `owner/repo` when the routed clone URL carries one, else the
1611
+ * routed repository name. A tracker that cannot resolve it answers
1612
+ * undefined, which stays retryable rather than definitive. */
1613
+ repo: string;
1614
+ store: Pick<Store, "verbLedger" | "getRun">;
1615
+ },
1774
1616
  ): Promise<{
1775
1617
  state: "pushed-green" | "pushed-pending" | "failed";
1776
1618
  reason?: string;
1777
1619
  }> {
1778
- if (claim.prUrl === undefined || claim.headSha === undefined) {
1620
+ const { prUrl, headSha } = claim;
1621
+ if (prUrl === undefined || headSha === undefined) {
1779
1622
  return { state: "failed", reason: "Worker did not report a PR URL and observed head SHA" };
1780
1623
  }
1781
- const verification = await tracker.verifyPr(claim.prUrl, claim.headSha);
1624
+ // Exact current run, never "some publication for the issue": the query is
1625
+ // scoped to this runId plus project/issue, so a previous attempt's verbs and
1626
+ // the orchestrator's recovery verbs are invisible here. Unbounded on
1627
+ // purpose — see the note above about evidence ageing off the newest page.
1628
+ const ledger = publication.store.verbLedger(publication.project, {
1629
+ runId: publication.runId,
1630
+ issue: publication.issue,
1631
+ limit: Number.MAX_SAFE_INTEGER,
1632
+ });
1633
+ const run = publication.store.getRun(publication.runId);
1634
+ const branchRef = `refs/heads/${publication.branch}`;
1635
+ if (run?.prUrl !== prUrl) {
1636
+ return {
1637
+ state: "failed",
1638
+ reason:
1639
+ "Pushed-green claim has no mediated publication evidence: the claimed PR is not this run's " +
1640
+ "recorded PR",
1641
+ };
1642
+ }
1643
+ const pushedThisHead = ledger.some(
1644
+ (entry) =>
1645
+ entry.decision === "allowed" &&
1646
+ entry.verb === "conductor_push" &&
1647
+ entry.sha === headSha &&
1648
+ entry.detail.includes(branchRef),
1649
+ );
1650
+ // Bound to the claimed PR, not merely to "a create happened": both allowed
1651
+ // details name the URL they produced (`opened <url> …`, `adopted <url> …`),
1652
+ // so an unrelated create on this run cannot vouch for another PR.
1653
+ const createdHere = ledger.some(
1654
+ (entry) =>
1655
+ entry.decision === "allowed" &&
1656
+ entry.verb === "conductor_pr_create" &&
1657
+ entry.detail.includes(prUrl),
1658
+ );
1659
+
1660
+ if (!pushedThisHead && !createdHere) {
1661
+ // Nothing this run published carries the claimed head, so the only
1662
+ // remaining evidence is the branch itself: a continuation or review round
1663
+ // that pushed nothing, or a head a mediated base-branch update produced.
1664
+ let tip: string | undefined;
1665
+ try {
1666
+ tip = await tracker.branchHead(publication.repo, publication.branch);
1667
+ } catch {
1668
+ tip = undefined;
1669
+ }
1670
+ if (tip === undefined) {
1671
+ return {
1672
+ state: "pushed-pending",
1673
+ reason: `Live head of ${branchRef} unavailable; retrying`,
1674
+ };
1675
+ }
1676
+ if (tip !== headSha) {
1677
+ return {
1678
+ state: "failed",
1679
+ reason:
1680
+ "Pushed-green claim has no mediated publication evidence: this run's conductor_push / " +
1681
+ `conductor_pr_create ledger does not cover the claimed head, and ${branchRef} is at ${tip}, ` +
1682
+ "not the claimed head",
1683
+ };
1684
+ }
1685
+ }
1686
+
1687
+ const verification = await tracker.verifyPr(prUrl, headSha);
1782
1688
  if (verification === undefined) {
1783
1689
  return { state: "pushed-pending", reason: "GitHub PR verification unavailable; retrying" };
1784
1690
  }
@@ -1932,33 +1838,6 @@ function orphanResumeVerdict(
1932
1838
  return { kind: "resume", prior };
1933
1839
  }
1934
1840
 
1935
- /**
1936
- * The worker identity for one launch, resolved now — or a throw naming the host
1937
- * change that is missing (#798/#828).
1938
- *
1939
- * Every input is host state a running daemon does not control: the account can
1940
- * be created after startup, and systemd can bring this service up before it has
1941
- * mounted the harness binding a worker resolves through. So the verdict is taken
1942
- * per launch. A daemon that cached one at startup would hold the whole fleet
1943
- * closed on a boot race until somebody restarted it by hand — the outage #828
1944
- * exists to end, not to relocate.
1945
- *
1946
- * The throw lands in the caller's dispatch catch, which settles the run failed
1947
- * with this reason and escalates. It classifies as a start failure, so a host
1948
- * fault charges the issue no implementation attempt.
1949
- */
1950
- function launchIdentity(d: Pick<Deps, "workerIdentity">, launching: string): WorkerIdentity {
1951
- const resolution: WorkerIdentityResolution = d.workerIdentity?.() ?? {
1952
- ok: false,
1953
- reason: `the ${WORKER_ACCOUNT} account is not installed on this host`,
1954
- };
1955
- if (resolution.ok) return resolution.identity;
1956
- throw new Error(
1957
- `worker identity unavailable: ${resolution.reason} — refusing to launch an unbound ${launching}; ` +
1958
- "run `omp-conductor setup host` to install the dedicated worker identity",
1959
- );
1960
- }
1961
-
1962
1841
  export async function handleIssue(
1963
1842
  d: Deps,
1964
1843
  r: Routed,
@@ -1981,6 +1860,32 @@ export async function handleIssue(
1981
1860
  let turnLimit: TurnLimitController | undefined;
1982
1861
  let workerControl: WorkerControlSlot | undefined;
1983
1862
  let workerSessionInstalled = false;
1863
+ // The run's Herdr representation (#840), hoisted for the same reason the verb
1864
+ // listener is: whoever tears the run down has to hand the pane's lifecycle
1865
+ // authority back, and a representation that outlives its child is a worker the
1866
+ // workspace still shows as live.
1867
+ let workerPane: Extract<WorkerPaneOutcome, { kind: "tracked" }> | undefined;
1868
+ // Herdr orders lifecycle reports by `seq`, so this run owns one counter and
1869
+ // every report takes the next value (#842). A shared or restarted counter would
1870
+ // let a late report overwrite a newer state — the pane would then show `working`
1871
+ // for a worker that has already blocked.
1872
+ let paneSeq = 0;
1873
+ const nextPaneSeq = (): number => (paneSeq += 1);
1874
+ /**
1875
+ * Project one authoritative transition onto the pane (#842).
1876
+ *
1877
+ * The input is always a durable run transition or a typed session event —
1878
+ * never the pane's own output, which is display and nothing more. A failure is
1879
+ * logged, not raised: the workspace lagging is not worth failing a run over.
1880
+ */
1881
+ const projectPaneState = (state: "working" | "idle" | "blocked" | "unknown", message?: string): void => {
1882
+ if (workerPane === undefined) return;
1883
+ const reported = reportWorkerPaneState(workerPane.paneId, workerPane.label, state, {
1884
+ seq: nextPaneSeq(),
1885
+ ...(message === undefined ? {} : { message }),
1886
+ });
1887
+ if (!reported.ok) log(`#${issue} herdr pane state ${state} not reported: ${reported.reason}`);
1888
+ };
1984
1889
  // The run's own repository. Hoisted for the same reason `worktreePath` is —
1985
1890
  // the catch and finally paths have to publish the branch.
1986
1891
  let runRepo: RunRepoRef | undefined;
@@ -2097,6 +2002,73 @@ export async function handleIssue(
2097
2002
  return true;
2098
2003
  };
2099
2004
 
2005
+ /**
2006
+ * The launch half of the credential-class fence (#852).
2007
+ *
2008
+ * Admission already held every candidate while a required provider billed to
2009
+ * the wrong credential, but a grant can be disabled in the window *between*
2010
+ * that check and this spawn — a refresh failing, an operator revoking, a token
2011
+ * expiring — and that window is exactly what the incident's acceptance
2012
+ * criterion names. So the same question is asked again here, as the last thing
2013
+ * before the session exists, and the run is closed rather than launched.
2014
+ *
2015
+ * Settled the same way the stop and drain fences settle: the claim closes, the
2016
+ * tree is salvaged, nothing is launched. It is deliberately NOT a failed
2017
+ * attempt against the issue's budget — the issue is not what is wrong, the
2018
+ * host's credentials are, and charging an attempt for it would exhaust an
2019
+ * issue while the operator re-authenticates.
2020
+ */
2021
+ const settleCredentialClassBeforeSession = async (): Promise<boolean> => {
2022
+ const required = project.requireOauthProviders ?? [];
2023
+ if (required.length === 0 || run === undefined || workerSessionInstalled) return false;
2024
+ const fence = await oauthFenceVerdict(
2025
+ required,
2026
+ d.probeCredentialClass ??
2027
+ (async (provider) => ({
2028
+ ok: false,
2029
+ reason: `no credential probe is wired for ${provider}`,
2030
+ })),
2031
+ );
2032
+ if (fence.ok) return false;
2033
+ turnLimit?.close();
2034
+ turnLimit = undefined;
2035
+ const settlement =
2036
+ worktreePath === undefined
2037
+ ? undefined
2038
+ : await settleWorktree({
2039
+ issue,
2040
+ attempt,
2041
+ ending: "not launched: a required subscription credential is unavailable",
2042
+ worktree: worktreePath,
2043
+ branch,
2044
+ publish,
2045
+ tree: "remove",
2046
+ mirrorPath,
2047
+ });
2048
+ recordOperatorStop(store, {
2049
+ project: project.name,
2050
+ issue,
2051
+ runId: run.id,
2052
+ inProgress,
2053
+ reason: `required subscription credential unavailable: ${fence.reason}`,
2054
+ patch: {
2055
+ endedAt: Date.now(),
2056
+ turns: run.turns,
2057
+ spendUsd: run.spendUsd,
2058
+ worktree: worktreePath ?? run.worktree,
2059
+ report: [
2060
+ "This run was closed instead of launched: a provider this project requires to bill",
2061
+ "to its subscription would have billed to something else.",
2062
+ fence.reason,
2063
+ ...(settlement?.lines ?? []),
2064
+ ].join("\n"),
2065
+ ...settlement?.patch,
2066
+ },
2067
+ });
2068
+ log(`#${issue} not launched: ${fence.reason}`);
2069
+ return true;
2070
+ };
2071
+
2100
2072
  try {
2101
2073
  // The claim-side of the stop fence (#374): the run row is the boundary the
2102
2074
  // shutdown drain waits on, so the claim itself refuses once the daemon is
@@ -2179,13 +2151,27 @@ export async function handleIssue(
2179
2151
  // (#286) keeps its semantics in both cases — it is the same resolution,
2180
2152
  // one different primary.
2181
2153
  const declaredModel = admittedModel?.model;
2154
+ // #807: a chain that already bought its one model escalation dispatches on
2155
+ // that stronger selector, outranking both the declaration and the project
2156
+ // default — the settlement that wrote the marker did so precisely because
2157
+ // the previous tier spun to a cap with nothing to show. Read from the
2158
+ // store, so it is sticky to this issue's chain and a fresh issue (no
2159
+ // marker) resolves exactly as it always has. The provider chain still
2160
+ // layers on top: an escalated run whose provider then aborts fails over
2161
+ // normally, because escalation only ever changes the primary.
2162
+ const escalation = store.modelEscalation(project.name, issue);
2163
+ const primaryModel = escalation?.model ?? declaredModel ?? project.workerModel;
2182
2164
  const choice = resolveDispatchModel({
2183
- workerModel: declaredModel ?? project.workerModel,
2165
+ workerModel: primaryModel,
2184
2166
  modelFallbacks: project.modelFallbacks,
2185
2167
  threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
2186
2168
  streak: chainFacts.streak,
2187
2169
  });
2188
- const clause = fallbackClause(choice, chainFacts, declaredModel ?? project.workerModel);
2170
+ const clause =
2171
+ fallbackClause(choice, chainFacts, primaryModel) ??
2172
+ (escalation === undefined
2173
+ ? undefined
2174
+ : `on ${escalation.model} — this chain's one model escalation, after ${escalation.failureClass}`);
2189
2175
 
2190
2176
  // #536: an orphan-clean requeue resumes the interrupted session instead of
2191
2177
  // re-reading the repo from zero. Decided here, before this attempt's row
@@ -2238,6 +2224,12 @@ export async function handleIssue(
2238
2224
  // enforced and the brief rendered, never a re-parse. Absent for a run
2239
2225
  // with no declaration (fail open), exactly as it was admitted.
2240
2226
  lane: admittedLane,
2227
+ // #851: the allowance admission reserved for this run, recorded on the
2228
+ // row before anything launches. Derived from the same caps the gate
2229
+ // read, so the reservation the pass enforced and the one the row holds
2230
+ // are one value; `undefined` (NULL) when the fleet has no spend cap at
2231
+ // all, which is the truth rather than "reserved nothing".
2232
+ spendReservedUsd: runSpendAllowanceUsd(caps) ?? undefined,
2241
2233
  });
2242
2234
  // #434: carry a terminal predecessor's open PR onto the continuation row.
2243
2235
  // The claim itself stays synchronous — `handleIssue` claims at the top of
@@ -2272,17 +2264,24 @@ export async function handleIssue(
2272
2264
  store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
2273
2265
  claimed = true;
2274
2266
  turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
2275
- workerControl = d.workerControls.open(project.name, issue, runId);
2267
+ // The pane follows the phase the registry actually reached (#842) never a
2268
+ // guess made at the call site, and never the pane's own output read back.
2269
+ workerControl = d.workerControls.open(project.name, issue, runId, (phase) => {
2270
+ projectPaneState(
2271
+ phase === "running" ? "working" : "idle",
2272
+ phase === "running" ? "resumed" : `${phase} by the operator`,
2273
+ );
2274
+ });
2276
2275
  if (await settleStopBeforeSession()) return;
2277
2276
  if (await settleDrainBeforeSession()) return;
2277
+ // The credential fence, twice, for the same reason the stop fence is
2278
+ // consulted more than once (#852). Here it is the cheap one: a run whose
2279
+ // required subscription credential is already unavailable is closed before
2280
+ // anything clones a mirror or cuts a worktree, because provisioning for a
2281
+ // launch that cannot happen is pure waste. The second call, immediately
2282
+ // before the spawn, is the one that closes the admission-to-session window.
2283
+ if (await settleCredentialClassBeforeSession()) return;
2278
2284
 
2279
- // The worker identity is this run's launch gate (#798): a worker session
2280
- // that cannot be launched under the dedicated unprivileged account is
2281
- // indistinguishable from an operator shell, so dispatch refuses before any
2282
- // tree or session is created. Resolved here rather than read off a startup
2283
- // verdict, so a host that gained its account — or its harness binding
2284
- // (#828) — after the daemon came up dispatches on the next tick.
2285
- const identity = launchIdentity(d, "worker session");
2286
2285
 
2287
2286
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
2288
2287
  // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -2366,10 +2365,6 @@ export async function handleIssue(
2366
2365
  },
2367
2366
  {
2368
2367
  ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
2369
- // A worker-run channel is secured to the worker identity (#798): the
2370
- // child that must connect to it runs as that uid, and the peer verdict
2371
- // expects that uid on the wire rather than the daemon's.
2372
- channelOwner: { uid: identity.uid, gid: identity.gid },
2373
2368
  },
2374
2369
  );
2375
2370
  if (await settleStopBeforeSession()) return;
@@ -2384,7 +2379,10 @@ export async function handleIssue(
2384
2379
  // Recorded before the launch, so even a run killed mid-flight leaves the
2385
2380
  // model it chose on its row. Only a chain-configured project writes the
2386
2381
  // column: absent `modelFallbacks` must preserve today's rows byte for byte.
2387
- if (chainConfigured && choice.model !== undefined) {
2382
+ // An escalated chain (#807) writes it too the selector it dispatched on
2383
+ // is the whole provenance of that recovery, and its harness-resolved model
2384
+ // lands beside it in `resolvedModel` when the run settles.
2385
+ if ((chainConfigured || escalation !== undefined) && choice.model !== undefined) {
2388
2386
  store.updateRun(runId, { model: choice.model });
2389
2387
  }
2390
2388
 
@@ -2433,24 +2431,27 @@ export async function handleIssue(
2433
2431
  }
2434
2432
  if (await settleStopBeforeSession()) return;
2435
2433
  if (await settleDrainBeforeSession()) return;
2434
+ // Last, so it is the newest fact anything has about the credentials — a
2435
+ // grant disabled since admission is caught here rather than paid for.
2436
+ if (await settleCredentialClassBeforeSession()) return;
2436
2437
 
2437
2438
  const repoSlug = githubRepo(r.repo.cloneUrl);
2438
2439
 
2439
- // The run's working paths are granted to the worker identity by ownership
2440
- // (#798) — the worktree the session edits and the session directory it
2441
- // writes its transcript and settings into. The chown lands here, after
2442
- // provisioning and the verb socket, so the session starts on a tree it
2443
- // owns; the worker identity never inherits anything from root's trees.
2444
- d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
2445
2440
 
2446
2441
  let result: WorkerResult;
2447
2442
  try {
2443
+ const runAllowanceUsd = runSpendAllowanceUsd(caps);
2448
2444
  result = await runWorker({
2449
2445
  brief,
2450
2446
  cwd: worktreePath,
2451
2447
  caps,
2452
2448
  ...(repoSlug === undefined ? {} : { repoSlug }),
2453
2449
  maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
2450
+ // #851: the allowance this run reserved at admission is also its live
2451
+ // ceiling. Without it the reservation would only bound how many runs
2452
+ // start, not what one of them spends, and the day's total would be
2453
+ // unbounded again.
2454
+ ...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
2454
2455
  onPauseControl: (control) => {
2455
2456
  workerSessionInstalled = true;
2456
2457
  workerControl?.install(control);
@@ -2469,10 +2470,72 @@ export async function handleIssue(
2469
2470
  onSpawn: (pid) => {
2470
2471
  verbListener?.bindPid(pid);
2471
2472
  },
2473
+ // The workspace representation of this exact child (#840). Best effort by
2474
+ // construction: a worker that cannot be *shown* is still a worker, so a
2475
+ // failure is logged with its reason and the run proceeds. What a failure
2476
+ // should mean for the launch — fail closed, or a named degraded state —
2477
+ // is #841's policy and is deliberately not decided here. Nothing is
2478
+ // claimed silently: either the pane id is logged, or the reason is.
2479
+ pane: {
2480
+ open: (pid) => {
2481
+ // Adopt before creating (#842): a run that already carries a pane
2482
+ // identity gets that pane reported against, never a second one. The
2483
+ // durable row is what makes this survive the process that opened it,
2484
+ // and duplicate-prevention is exactly what it buys — a re-entered
2485
+ // launch for one run must not leave two panes claiming it.
2486
+ const recorded = store.getRun(runId);
2487
+ if (recorded?.paneId !== undefined && recorded.paneLabel !== undefined) {
2488
+ workerPane = { kind: "tracked", paneId: recorded.paneId, label: recorded.paneLabel, pid };
2489
+ store.updateRun(runId, { workerPid: pid });
2490
+ const state = reportWorkerPaneState(recorded.paneId, recorded.paneLabel, "working", {
2491
+ seq: nextPaneSeq(),
2492
+ });
2493
+ log(
2494
+ state.ok
2495
+ ? `#${issue} herdr pane ${recorded.paneId} re-reported for pid ${pid}`
2496
+ : `#${issue} herdr pane ${recorded.paneId} could not be re-reported: ${state.reason}`,
2497
+ );
2498
+ return;
2499
+ }
2500
+ const outcome = openWorkerPane({
2501
+ project: project.name,
2502
+ issue,
2503
+ attempt,
2504
+ runId,
2505
+ pid,
2506
+ ...(recorded?.sessionFile === undefined ? {} : { sessionFile: recorded.sessionFile }),
2507
+ });
2508
+ if (outcome.kind === "tracked") {
2509
+ workerPane = outcome;
2510
+ // Durable before it is announced: a pane the store does not know
2511
+ // about is a pane a restart cannot reconcile.
2512
+ store.updateRun(runId, {
2513
+ workerPid: pid,
2514
+ paneId: outcome.paneId,
2515
+ paneLabel: outcome.label,
2516
+ // Cleared, because there is now a pane: a leftover reason beside a
2517
+ // tracked run would keep `status` saying degraded forever (#841).
2518
+ paneUnavailable: null,
2519
+ });
2520
+ log(`#${issue} herdr pane ${outcome.paneId} (${outcome.label}) tracks pid ${pid}`);
2521
+ } else {
2522
+ // Named, never silent (#841): the run proceeds untracked, and the
2523
+ // reason is durable so `status` can say the fleet is running blind
2524
+ // rather than looking identical to a fleet with no workers.
2525
+ store.updateRun(runId, { workerPid: pid, paneUnavailable: outcome.reason });
2526
+ log(`#${issue} no herdr pane: ${outcome.reason}`);
2527
+ }
2528
+ },
2529
+ release: () => {
2530
+ if (workerPane === undefined) return;
2531
+ const released = releaseWorkerPane(workerPane.paneId, workerPane.label, { seq: nextPaneSeq() });
2532
+ if (!released.ok) log(`#${issue} herdr pane release failed: ${released.reason}`);
2533
+ workerPane = undefined;
2534
+ },
2535
+ },
2472
2536
  onChildLog: (line) => {
2473
2537
  log(`#${issue} ${line}`);
2474
2538
  },
2475
- workerIdentity: identity,
2476
2539
  ...(choice.model === undefined ? {} : { model: choice.model }),
2477
2540
  // The fleet-owned omp settings overlay (#537): the staged YAML the
2478
2541
  // session loads through `Settings.init({ configFiles: [<path>] })` —
@@ -2484,6 +2547,11 @@ export async function handleIssue(
2484
2547
  onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
2485
2548
  onTurn: (n) => store.updateRun(runId, { turns: n }),
2486
2549
  onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
2550
+ // #518: recorded live, not at settlement — the moment this explains is
2551
+ // forty minutes before the wall-clock cap fires, so a number only a
2552
+ // finished run carries would answer the question too late.
2553
+ onTokens: (tokens) =>
2554
+ store.updateRun(runId, { outputTokens: tokens.output, reasoningTokens: tokens.reasoning }),
2487
2555
  onKilled: () => {
2488
2556
  turnLimit?.close();
2489
2557
  turnLimit = undefined;
@@ -2530,7 +2598,14 @@ export async function handleIssue(
2530
2598
 
2531
2599
  const verified: { state: RunState; reason?: string } =
2532
2600
  result.state === "pushed-green"
2533
- ? await verifyPushedGreenClaim(tracker, result)
2601
+ ? await verifyPushedGreenClaim(tracker, result, {
2602
+ project: project.name,
2603
+ issue,
2604
+ runId,
2605
+ branch,
2606
+ repo: repoSlug ?? r.repo.name,
2607
+ store,
2608
+ })
2534
2609
  : { state: result.state };
2535
2610
  const state = verified.state;
2536
2611
 
@@ -2932,6 +3007,114 @@ export async function handleIssue(
2932
3007
  * recorded it — merged after all, settled, re-claimed — fails the claim and is
2933
3008
  * settled `skipped` rather than woken on stale identity.
2934
3009
  */
3010
+ /**
3011
+ * Record what this host has installed, once per dispatch pass (#919).
3012
+ *
3013
+ * Here rather than at render time, and this is the whole design of the slice:
3014
+ * the read costs three subprocesses, and calling it from the tick took the
3015
+ * tick's own suite from 8.4s to 83.4s while spawning three children every
3016
+ * fifteen minutes to answer a question that changes only when someone installs
3017
+ * something. The dispatch pass is already async and already spawns `gh`, so one
3018
+ * read per pass is free by comparison, and every cheap surface then reads a row.
3019
+ *
3020
+ * Advisory throughout: a probe that throws (no `herdr` on PATH, a `$PATH`
3021
+ * without `omp`) leaves the previous observation in place and logs. A stale
3022
+ * observation is still the truth about the last time anyone could look, and
3023
+ * losing a dispatch pass over a version string would be absurd.
3024
+ */
3025
+ /**
3026
+ * Make the workspace agree with the live run set, once per pass (#841).
3027
+ *
3028
+ * Runs on every dispatch pass rather than only at startup, because the thing it
3029
+ * repairs — a Herdr restart — is not a conductor event and announces itself
3030
+ * nowhere. It is idempotent by construction: a reconciled fleet reports `intact`
3031
+ * for every live worker and finds no stale panes, so repeated passes converge
3032
+ * instead of accumulating panes or churning them.
3033
+ *
3034
+ * Nothing here can stop a worker. The only mutations are creating a pane, and
3035
+ * releasing one whose run is not live; the authoritative child is never
3036
+ * signalled, and no pane is ever closed.
3037
+ */
3038
+ function reconcilePanes(d: Deps, project: string, log: (message: string) => void): void {
3039
+ const live = d.store.liveRuns(project);
3040
+ const result = reconcileWorkerPanes(
3041
+ live.map((run) => ({
3042
+ runId: run.id,
3043
+ issue: run.issue,
3044
+ attempt: run.attempt,
3045
+ project,
3046
+ ...(run.workerPid === undefined ? {} : { pid: run.workerPid }),
3047
+ ...(run.paneId === undefined ? {} : { paneId: run.paneId }),
3048
+ ...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
3049
+ ...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
3050
+ })),
3051
+ );
3052
+ if (!result.ok) {
3053
+ // An unreadable workspace is not evidence that anything is stale, so nothing
3054
+ // is released and no run is relabelled — but a fleet that cannot see its own
3055
+ // panes says so, on every pass, rather than going quiet.
3056
+ log(`worker panes not reconciled: ${result.reason}`);
3057
+ return;
3058
+ }
3059
+ for (const outcome of result.outcomes) {
3060
+ switch (outcome.kind) {
3061
+ case "intact":
3062
+ break;
3063
+ case "reassociated":
3064
+ d.store.updateRun(outcome.runId, {
3065
+ paneId: outcome.paneId,
3066
+ paneLabel: outcome.label,
3067
+ paneUnavailable: null,
3068
+ });
3069
+ log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
3070
+ break;
3071
+ case "untracked":
3072
+ d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
3073
+ log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
3074
+ break;
3075
+ case "stale-released":
3076
+ log(`released stale herdr pane ${outcome.paneId} (run ${outcome.runId || "unidentified"} is not live)`);
3077
+ break;
3078
+ case "stale-release-failed":
3079
+ log(`stale herdr pane ${outcome.paneId} could not be released: ${outcome.reason}`);
3080
+ break;
3081
+ }
3082
+ }
3083
+ }
3084
+
3085
+ /** The issue a run id belongs to, for a log line a human reads. */
3086
+ function runIssue(live: readonly RunRecord[], runId: string): string {
3087
+ return String(live.find((run) => run.id === runId)?.issue ?? "?");
3088
+ }
3089
+
3090
+ async function recordInstallSurfaces(d: Deps): Promise<void> {
3091
+ if (d.probeInstallSurfaces === undefined) return;
3092
+ try {
3093
+ const surfaces = await d.probeInstallSurfaces();
3094
+ // The telegram peer's versions ride along on the same pass (#961). Read
3095
+ // here rather than in `status` because `status` runs every tick and this
3096
+ // touches the npm registry: one periodic read, rendered from the store as
3097
+ // often as anyone looks. A read that throws leaves the fields absent, which
3098
+ // renders as unverified — never as agreement.
3099
+ const telegram =
3100
+ d.probeTelegramFreshness === undefined ? undefined : await d.probeTelegramFreshness();
3101
+ const tgInstalled = telegram?.surfaces.installed;
3102
+ const tgDaemon = telegram?.surfaces.daemon;
3103
+ const tgPublished = telegram?.surfaces.published;
3104
+ d.store.recordInstallSurfaces({
3105
+ at: Date.now(),
3106
+ cliVersion: surfaces.cliVersion,
3107
+ ...(surfaces.ompVersion === undefined ? {} : { ompVersion: surfaces.ompVersion }),
3108
+ ...(surfaces.herdrSource === undefined ? {} : { herdrSource: surfaces.herdrSource }),
3109
+ ...(tgInstalled?.kind === "version" ? { telegramInstalled: tgInstalled.version } : {}),
3110
+ ...(tgDaemon?.kind === "version" ? { telegramDaemon: tgDaemon.version } : {}),
3111
+ ...(tgPublished?.kind === "version" ? { telegramPublished: tgPublished.version } : {}),
3112
+ });
3113
+ } catch (err) {
3114
+ log(`install-surface read skipped: ${errText(err)}`);
3115
+ }
3116
+ }
3117
+
2935
3118
  export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
2936
3119
  const pending = d.store.pendingReviewRevisions(d.project.name);
2937
3120
  if (pending.length === 0) return;
@@ -3049,6 +3232,350 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
3049
3232
  await Promise.allSettled(launches);
3050
3233
  }
3051
3234
 
3235
+ /**
3236
+ * Launch the review-ceiling adjudications this project owes (#932).
3237
+ *
3238
+ * Deliberately its own pass, next to {@link dispatchReviewRevisions} but never
3239
+ * inside it: a revision resumes an implementation worker to change code, an
3240
+ * adjudication opens a fresh read-only session to decide about code. Folding
3241
+ * them together is how "escalate the automation" quietly becomes "resume the
3242
+ * same worker with a different label", which is #873's named silent fake.
3243
+ *
3244
+ * Only `pending` rows are launched. A `running` row belongs to a live
3245
+ * adjudicator — or to one a restart killed, which the startup sweep settles —
3246
+ * so a restart re-reads it and never launches a second adjudicator for one head.
3247
+ */
3248
+ /**
3249
+ * The turn ceiling for one adjudication (#932).
3250
+ *
3251
+ * Small on purpose, and not the worker cap: a verdict is a read and an answer,
3252
+ * not an implementation. A session still going after this is not deciding — and
3253
+ * the ceiling exists so a stuck adjudicator costs a bounded amount rather than a
3254
+ * worker's whole budget, on work that has already spent its review rounds.
3255
+ */
3256
+ export const ADJUDICATION_MAX_TURNS = 40;
3257
+
3258
+ export async function dispatchReviewAdjudications(d: Deps, pool?: WorkerPool): Promise<void> {
3259
+ const open = d.store.openReviewAdjudications(d.project.name).filter((a) => a.state === "pending");
3260
+ if (open.length === 0) return;
3261
+ const launches: Promise<void>[] = [];
3262
+ for (const adjudication of open) {
3263
+ if (d.drain?.draining === true) {
3264
+ log(`adjudications held: the daemon is draining (${open.length - launches.length} not launched)`);
3265
+ break;
3266
+ }
3267
+ // The head is re-read before anything launches. A verdict describes the diff
3268
+ // it read, so adjudicating a head the PR has since moved past would produce
3269
+ // a decision about code nobody is merging — settled `stale-head` with both
3270
+ // heads named, never launched.
3271
+ let live: string | undefined;
3272
+ try {
3273
+ live = await d.tracker.prHead(adjudication.prUrl);
3274
+ } catch (err) {
3275
+ log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head unreadable (${errText(err)})`);
3276
+ continue;
3277
+ }
3278
+ if (live === undefined) {
3279
+ log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head could not be read — retrying next tick`);
3280
+ continue;
3281
+ }
3282
+ if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
3283
+ d.store.settleReviewAdjudication(
3284
+ adjudication.id,
3285
+ "stale-head",
3286
+ `recorded ${adjudication.headSha}, live ${live} — the pull request moved before this adjudication launched`,
3287
+ Date.now(),
3288
+ );
3289
+ log(`#${adjudication.issue} adjudication stale-head: recorded ${adjudication.headSha}, live ${live}`);
3290
+ continue;
3291
+ }
3292
+ // The claim IS the single-flight guard: the transition is the row's own
3293
+ // WHERE clause, so two dispatch passes racing one adjudication cannot both
3294
+ // launch — the loser changes nothing and skips.
3295
+ const review = resolveReview(d.project);
3296
+ const provenance = { role: adjudication.role, model: adjudication.role, resolvedAt: Date.now() };
3297
+ if (!d.store.markReviewAdjudicationRunning(adjudication.id, provenance, Date.now())) {
3298
+ log(`#${adjudication.issue} adjudication skipped: another pass already claimed it`);
3299
+ continue;
3300
+ }
3301
+ log(`#${adjudication.issue} adjudication → launching role ${adjudication.role} at ${adjudication.headSha}`);
3302
+ launches.push(handleReviewAdjudication(d, adjudication, review.maxRounds));
3303
+ }
3304
+ if (pool !== undefined) {
3305
+ for (const launch of launches) pool.launch(launch);
3306
+ return;
3307
+ }
3308
+ await Promise.allSettled(launches);
3309
+ }
3310
+
3311
+ /**
3312
+ * Assemble one adjudication's evidence and run it to a durable verdict (#932).
3313
+ *
3314
+ * Every fact comes from a durable row or a live tracker read — never from the
3315
+ * worker's own report, which is the claim under adjudication. The issue body is
3316
+ * carried verbatim because the acceptance criteria ARE the standard being
3317
+ * judged, and every prior round travels WITH its settled outcome: a finding
3318
+ * without its outcome reads as an outstanding complaint even when the worker
3319
+ * fixed it, which is how an adjudicator rejects work that was already corrected.
3320
+ */
3321
+ export async function handleReviewAdjudication(
3322
+ d: Deps,
3323
+ adjudication: ReviewAdjudicationRecord,
3324
+ maxRounds: number,
3325
+ ): Promise<void> {
3326
+ const issue = adjudication.issue;
3327
+ const [title, body, diff, verification] = await Promise.all([
3328
+ d.tracker.getIssue(issue).then(
3329
+ (row) => row?.title,
3330
+ () => undefined,
3331
+ ),
3332
+ d.tracker.issueBody(issue).then(
3333
+ (text) => text,
3334
+ () => undefined,
3335
+ ),
3336
+ d.tracker.prDiff(adjudication.prUrl).then(
3337
+ (parsed) => parsed,
3338
+ () => undefined,
3339
+ ),
3340
+ d.tracker.verifyPr(adjudication.prUrl, adjudication.headSha).then(
3341
+ (result) => result,
3342
+ () => undefined,
3343
+ ),
3344
+ ]);
3345
+ // Prior rounds, oldest first, keyed by PR so a continuation that inherited the
3346
+ // PR cannot hide its predecessor's findings. The ceiling call's own findings
3347
+ // are appended as the last entry — they are the reason this adjudication
3348
+ // exists, and the durable row is where they were kept precisely so this brief
3349
+ // could carry them.
3350
+ const rounds: AdjudicationRound[] = d.store
3351
+ .reviewRevisionsForPr(d.project.name, adjudication.prUrl)
3352
+ .map((revision) => ({
3353
+ round: revision.round,
3354
+ findings: revision.findings,
3355
+ ...(revision.outcome === undefined ? {} : { outcome: revision.outcome }),
3356
+ }));
3357
+ if (adjudication.findings !== undefined) {
3358
+ rounds.push({
3359
+ round: rounds.length + 1,
3360
+ findings: adjudication.findings,
3361
+ outcome: "escalated to this adjudication — no worker round was available",
3362
+ });
3363
+ }
3364
+ const brief = renderAdjudicationBrief({
3365
+ issue,
3366
+ ...(title === undefined ? {} : { issueTitle: title }),
3367
+ issueBody: body ?? "(the issue body could not be read — judge the diff against the findings below)",
3368
+ prUrl: adjudication.prUrl,
3369
+ headSha: adjudication.headSha,
3370
+ checks:
3371
+ verification === undefined
3372
+ ? "(the check state could not be read)"
3373
+ : `${verification.status} — ${verification.reason}`,
3374
+ diff:
3375
+ diff === undefined
3376
+ ? "(the diff could not be read — say so in your reasons rather than guessing)"
3377
+ : renderDiffForAdjudication(diff),
3378
+ rounds,
3379
+ maxRounds,
3380
+ });
3381
+
3382
+ const sessionDir = join(stateDir(), "sessions", `adjudication-${adjudication.id}`);
3383
+ mkdirSync(sessionDir, { recursive: true });
3384
+ const ompSettingsFile = materializeOmpSettings(d.project, sessionDir);
3385
+ const result = await (d.runAdjudicatorImpl ?? runAdjudicator)({
3386
+ brief,
3387
+ // No worktree: an adjudicator has no branch, and everything it needs is in
3388
+ // the brief. The session directory doubles as its confined cwd.
3389
+ cwd: sessionDir,
3390
+ sessionDir,
3391
+ model: adjudication.role,
3392
+ ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
3393
+ socketPath: join(sessionDir, "ipc.sock"),
3394
+ maxTurns: ADJUDICATION_MAX_TURNS,
3395
+ });
3396
+
3397
+ // What actually ran it, named in the evidence when it differs from the role
3398
+ // that was asked for: the provenance recorded at claim time is conductor's own
3399
+ // resolution (the role it passed to OMP), and the model that wrote the
3400
+ // messages is a later observation only this result carries.
3401
+ const ran =
3402
+ result.model === undefined
3403
+ ? ""
3404
+ : `adjudicated by ${result.model}${result.provider === undefined ? "" : ` (${result.provider})`}: `;
3405
+ if (result.verdict === undefined) {
3406
+ // A session that never took a turn did not run: the role could not be
3407
+ // resolved or the harness could not start, and the remedy is configuration
3408
+ // rather than a retry. One that ran and produced nothing usable is a
3409
+ // failure. Neither is ever a clear.
3410
+ const state = result.turns === 0 ? "unavailable-model" : "failed";
3411
+ d.store.settleReviewAdjudication(
3412
+ adjudication.id,
3413
+ state,
3414
+ `${ran}${result.report === "" ? "no verdict and no output" : result.report}`,
3415
+ Date.now(),
3416
+ );
3417
+ log(`#${issue} adjudication ${state} at ${adjudication.headSha}`);
3418
+ return;
3419
+ }
3420
+ // A verdict that names a different head judged something else. Fail closed:
3421
+ // the one-shot is spent on the head it was opened for, and nothing merges on
3422
+ // the strength of a decision about another diff.
3423
+ if (
3424
+ result.verdict.headSha !== undefined &&
3425
+ result.verdict.headSha.toLowerCase() !== adjudication.headSha.toLowerCase()
3426
+ ) {
3427
+ d.store.settleReviewAdjudication(
3428
+ adjudication.id,
3429
+ "stale-head",
3430
+ `${ran}the verdict names head ${result.verdict.headSha}, not the adjudicated ${adjudication.headSha}: ${result.verdict.reasons}`,
3431
+ Date.now(),
3432
+ );
3433
+ log(`#${issue} adjudication stale-head: verdict named ${result.verdict.headSha}`);
3434
+ return;
3435
+ }
3436
+ d.store.settleReviewAdjudication(
3437
+ adjudication.id,
3438
+ result.verdict.verdict,
3439
+ `${ran}${result.verdict.reasons}`,
3440
+ Date.now(),
3441
+ );
3442
+ log(`#${issue} adjudication ${result.verdict.verdict} at ${adjudication.headSha}`);
3443
+ }
3444
+
3445
+ /**
3446
+ * Apply the terminal dispositions an adjudication's verdict implies (#876).
3447
+ *
3448
+ * The verdict is a decision; this is what is DONE about it, and the two are
3449
+ * separate rows for exactly that reason. Recording `cleared` while leaving the
3450
+ * merge gate refusing, or `rejected` while leaving the PR open to block the
3451
+ * issue forever, is the issue's named silent fake — so what this pass changes is
3452
+ * tracker and gate state, and the disposition text is only its receipt.
3453
+ *
3454
+ * Idempotent by that receipt: a row whose `disposition` is already recorded is
3455
+ * skipped, so a restart, a retried tick or a partial failure cannot close one PR
3456
+ * twice. Nothing here is destructive before its explanation is durable.
3457
+ *
3458
+ * Neither branch opens a review round, launches another adjudicator, or asks an
3459
+ * operator for anything. Neither touches an attempt or continuation budget
3460
+ * either: a rejected PR closes, and the ORDINARY settle sweep charges that
3461
+ * closure exactly as it charges any PR closed without merging — which is what
3462
+ * keeps this composable with #815's separate corrective-worker path instead of
3463
+ * resetting its counters.
3464
+ */
3465
+ export async function applyAdjudicationDispositions(d: Deps): Promise<void> {
3466
+ const { project, store } = d;
3467
+ // Terminal rows only, and only those nothing has disposed of yet. Read from
3468
+ // the PRs still in flight: a settled adjudication whose PR is long gone needs
3469
+ // no disposition, and scanning history would re-litigate closed work.
3470
+ const pending = store
3471
+ .activeRuns(project.name)
3472
+ .flatMap((run) => (run.prUrl === undefined ? [] : store.reviewAdjudicationsForPr(project.name, run.prUrl)))
3473
+ .filter((row) => row.settledAt !== undefined && row.disposition === undefined);
3474
+ for (const adjudication of pending) {
3475
+ if (adjudication.state === "cleared") {
3476
+ // The clear re-enters the ORDINARY merge path, and it does so through the
3477
+ // mechanism that path already reads: the #888 gate refuses a head carrying
3478
+ // unresolved review evidence, and #913's clearance is the recorded
3479
+ // disposition that settles it. Reusing it means no second gate, no
3480
+ // adjudication-shaped exception inside `conductor_pr_merge`, and one
3481
+ // audit trail for "this head was cleared, by whom, and why".
3482
+ //
3483
+ // The head is re-read first: a clearance is head-scoped and clears
3484
+ // backwards only, so recording one against a head the PR has moved past
3485
+ // would bless a diff nobody adjudicated.
3486
+ let live: string | undefined;
3487
+ try {
3488
+ live = await d.tracker.prHead(adjudication.prUrl);
3489
+ } catch (err) {
3490
+ log(`#${adjudication.issue} adjudication clearance held: head unreadable (${errText(err)})`);
3491
+ continue;
3492
+ }
3493
+ if (live === undefined) {
3494
+ log(`#${adjudication.issue} adjudication clearance held: head could not be read — retrying next tick`);
3495
+ continue;
3496
+ }
3497
+ if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
3498
+ // Visible and non-destructive: the clearance is not recorded, the row
3499
+ // says why, and the PR keeps whatever gate state it has.
3500
+ store.recordReviewAdjudicationDisposition(
3501
+ adjudication.id,
3502
+ `no clearance recorded: the head moved from ${adjudication.headSha} to ${live} after the verdict`,
3503
+ );
3504
+ log(`#${adjudication.issue} adjudication clearance skipped: head moved to ${live}`);
3505
+ continue;
3506
+ }
3507
+ store.recordReviewClearance({
3508
+ project: project.name,
3509
+ prUrl: adjudication.prUrl,
3510
+ headSha: adjudication.headSha,
3511
+ by: `adjudication:${adjudication.role}`,
3512
+ reason: `cleared by the review-ceiling adjudication: ${adjudication.evidence ?? "no evidence recorded"}`,
3513
+ });
3514
+ store.recordReviewAdjudicationDisposition(
3515
+ adjudication.id,
3516
+ `recorded a review clearance at ${adjudication.headSha}; the PR is on the ordinary merge path`,
3517
+ );
3518
+ log(`#${adjudication.issue} adjudication cleared → merge path open at ${adjudication.headSha}`);
3519
+ continue;
3520
+ }
3521
+ // Every non-clear terminal state disposes the same way, and that is
3522
+ // deliberate: `rejected`, `failed`, `unavailable-model` and `stale-head` all
3523
+ // mean "this head is not merging and no further round is coming", and the
3524
+ // blocking artefact is identical in each case — an open PR that occupies the
3525
+ // issue. Only the recorded reason differs.
3526
+ const findings = store
3527
+ .reviewRevisionsForPr(project.name, adjudication.prUrl)
3528
+ .map((revision) => `### Review round ${revision.round} (${revision.outcome ?? "never settled"})
3529
+
3530
+ ${revision.findings}`);
3531
+ const comment = [
3532
+ `## Adjudication: ${adjudication.state}`,
3533
+ "",
3534
+ `The review-round ceiling was reached on ${adjudication.headSha}, and the \`${adjudication.role}\` adjudicator ` +
3535
+ `did not clear it. Closing this pull request so the issue is not blocked by work that cannot merge — ` +
3536
+ "nothing here asks anyone to repair it by hand.",
3537
+ "",
3538
+ "### Adjudicator's reasons",
3539
+ "",
3540
+ adjudication.evidence ?? "(none recorded)",
3541
+ ...(findings.length === 0 ? [] : ["", "## Preserved review findings", "", ...findings]),
3542
+ "",
3543
+ "The branch itself is untouched and remains in the repository.",
3544
+ ].join("\n");
3545
+ const closed = await d.verbActions?.closePr(adjudication.prUrl, comment);
3546
+ if (closed === undefined) {
3547
+ log(`#${adjudication.issue} adjudication disposition held: no action surface is wired`);
3548
+ continue;
3549
+ }
3550
+ if (!closed.ok) {
3551
+ // Retryable, and no receipt written: the next tick tries again rather than
3552
+ // leaving a rejected PR open with a row that claims it was handled.
3553
+ log(`#${adjudication.issue} adjudication disposition held: could not close ${adjudication.prUrl} — ${closed.stderr}`);
3554
+ continue;
3555
+ }
3556
+ store.recordReviewAdjudicationDisposition(
3557
+ adjudication.id,
3558
+ `closed ${adjudication.prUrl} with the adjudicator's reasons and every preserved finding; ` +
3559
+ "the settle sweep releases the issue",
3560
+ );
3561
+ log(`#${adjudication.issue} adjudication ${adjudication.state} → closed ${adjudication.prUrl}`);
3562
+ }
3563
+ }
3564
+
3565
+ /** The diff as one text, from the structured read the settlement audit uses.
3566
+ * `renderAdjudicationBrief` bounds the total; this only flattens it, and says
3567
+ * when the adapter itself had to cut the read short — an adjudicator must know
3568
+ * its silence covers only what it saw. */
3569
+ function renderDiffForAdjudication(diff: PrDiff): string {
3570
+ const files = diff.files.map((file) =>
3571
+ [
3572
+ `--- ${file.status} ${file.previousPath === undefined ? "" : `${file.previousPath} → `}${file.path}`,
3573
+ file.hunks ?? "(no textual hunks: binary, mode-only or pure rename)",
3574
+ ].join("\n"),
3575
+ );
3576
+ return [...(diff.truncated ? ["[the tracker truncated this diff read]"] : []), ...files].join("\n\n");
3577
+ }
3578
+
3052
3579
  /**
3053
3580
  * Resume one review-revision worker: the run row is already claimed
3054
3581
  * (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
@@ -3169,20 +3696,38 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3169
3696
  // stopped anything, the PR is still green and still open, and only the
3170
3697
  // wake failed; `settleStopBeforeSession` immediately above covers the
3171
3698
  // case where an operator genuinely did. So mirror that branch: restore
3172
- // `pushed-green` naming the shutdown, settle the revision `skipped` so
3173
- // the round is not double-counted, and leave the in-progress label alone
3174
- // — the ordinary settle sweep releases it in the same breath it
3699
+ // `pushed-green` naming the shutdown, and leave the in-progress label
3700
+ // alone — the ordinary settle sweep releases it in the same breath it
3175
3701
  // terminalises the row when the PR resolves. Stopping the row here would
3176
3702
  // strand it: `settlePushedGreen` sweeps only pushed-* rows and
3177
3703
  // classification excludes `stopped`, so the issue would carry neither
3178
3704
  // label and nothing would ever revisit the still-green PR.
3705
+ //
3706
+ // The round itself is RE-QUEUED rather than settled `skipped` (#903). A
3707
+ // daemon shutdown is the plainest infrastructure kill there is — no
3708
+ // session started, so no round was worked — and a settled row consumed
3709
+ // one of the PR's rounds through `latestReviewRound`, which is how an
3710
+ // upgrade or a restart used to cost a review round it never spent. The
3711
+ // retry is counted and bounded exactly like any other infra kill, so a
3712
+ // host that shuts down mid-wake forever still escalates.
3179
3713
  store.updateRun(runId, {
3180
3714
  state: "pushed-green",
3181
3715
  endedAt: Date.now(),
3182
3716
  lastError: "daemon shutdown began after the review revision claim; the round was not launched",
3183
3717
  });
3184
- store.settleReviewRevision(revision.id, "skipped", Date.now());
3185
- log(`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim`);
3718
+ const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
3719
+ if (retry.kind === "exhausted") {
3720
+ store.settleReviewRevision(revision.id, "skipped", Date.now());
3721
+ log(
3722
+ `#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim, and ` +
3723
+ `its infra retries are exhausted (${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settled skipped`,
3724
+ );
3725
+ return true;
3726
+ }
3727
+ log(
3728
+ `#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim — ` +
3729
+ `re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
3730
+ );
3186
3731
  return true;
3187
3732
  };
3188
3733
 
@@ -3215,14 +3760,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3215
3760
  if (await settleStopBeforeSession()) return;
3216
3761
  if (await settleDrainBeforeSession()) return;
3217
3762
 
3218
- // The same launch gate as a fresh claim (#798): a revision worker is a
3219
- // worker, and an unbound one is an operator shell — refuse before any tree
3220
- // or session is created. Thrown inside the try so the ordinary dispatch
3221
- // catch settles it: the run row is returned to terminal `failed` with the
3222
- // reason, the revision row is settled with it, and the still-green PR stays
3223
- // open for a healthy retry.
3224
3763
  try {
3225
- const identity = launchIdentity(d, "review-revision worker");
3226
3764
  // Reattach the run's own branch at the same per-issue path the run used:
3227
3765
  // a `pushed-green` settle removed the worktree, so provisioning is the
3228
3766
  // same continuation reattach as a normal re-claim. A capped/failed run's
@@ -3337,7 +3875,6 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3337
3875
  },
3338
3876
  {
3339
3877
  ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
3340
- channelOwner: { uid: identity.uid, gid: identity.gid },
3341
3878
  },
3342
3879
  );
3343
3880
  if (await settleStopBeforeSession()) return;
@@ -3350,22 +3887,26 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3350
3887
  // totals the run already recorded.
3351
3888
  const baseTurns = run.turns;
3352
3889
  const baseSpend = run.spendUsd;
3890
+ const baseOutputTokens = run.outputTokens ?? 0;
3891
+ const baseReasoningTokens = run.reasoningTokens ?? 0;
3353
3892
 
3354
3893
  log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
3355
3894
 
3356
- // The revision worker gets its own tree ownership, exactly as a fresh
3357
- // claim does (#798): the resumed worktree and session directory belong to
3358
- // the worker identity before its session starts.
3359
- d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
3360
3895
 
3361
3896
  let result: WorkerResult;
3362
3897
  try {
3898
+ const runAllowanceUsd = runSpendAllowanceUsd(caps);
3363
3899
  result = await runWorker({
3364
3900
  brief: renderReviewRevisionPrompt(revision.findings, revision.round),
3365
3901
  cwd: worktreePath,
3366
3902
  caps,
3367
3903
  ...(repoSlug === undefined ? {} : { repoSlug }),
3368
3904
  maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
3905
+ // A review round meters its own session from zero and gets the same
3906
+ // allowance as any run (#851) — the round is real work with a real
3907
+ // cost, and an unbounded round would reopen the hole for any PR that
3908
+ // was ever returned for revision.
3909
+ ...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
3369
3910
  onPauseControl: (control) => {
3370
3911
  workerSessionInstalled = true;
3371
3912
  workerControl?.install(control);
@@ -3382,7 +3923,6 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3382
3923
  onChildLog: (line) => {
3383
3924
  log(`#${issue} ${line}`);
3384
3925
  },
3385
- workerIdentity: identity,
3386
3926
  // The continuation stays on the model the green run used (#286).
3387
3927
  ...(run.model === undefined ? {} : { model: run.model }),
3388
3928
  ...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
@@ -3390,6 +3930,14 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3390
3930
  onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
3391
3931
  onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
3392
3932
  onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
3933
+ // A revision resumes the SAME run, so its tokens add to the attempt's
3934
+ // totals exactly as its turns and spend do (#518) — resetting them would
3935
+ // make a long deliberation look like a fresh one.
3936
+ onTokens: (tokens) =>
3937
+ store.updateRun(runId, {
3938
+ outputTokens: baseOutputTokens + tokens.output,
3939
+ reasoningTokens: baseReasoningTokens + tokens.reasoning,
3940
+ }),
3393
3941
  onKilled: () => {
3394
3942
  turnLimit?.close();
3395
3943
  turnLimit = undefined;
@@ -3418,7 +3966,14 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3418
3966
 
3419
3967
  const verified: { state: RunState; reason?: string } =
3420
3968
  result.state === "pushed-green"
3421
- ? await verifyPushedGreenClaim(tracker, result)
3969
+ ? await verifyPushedGreenClaim(tracker, result, {
3970
+ project: project.name,
3971
+ issue,
3972
+ runId,
3973
+ branch,
3974
+ repo: repoSlug ?? repo.name,
3975
+ store,
3976
+ })
3422
3977
  : { state: result.state };
3423
3978
  const state = verified.state;
3424
3979
 
@@ -3481,6 +4036,63 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3481
4036
  report: finalReport,
3482
4037
  ...settlement?.patch,
3483
4038
  };
4039
+ // #903: an infrastructure kill spends no review round.
4040
+ //
4041
+ // A dispatch that dies before the resumed session takes a turn — a host
4042
+ // permission fault, a spawn failure, a harness that could not start — did
4043
+ // none of the work a round exists to do, yet the pre-#903 path settled the
4044
+ // row `failed` with no retry. During the 2026-08-21/22 outage that turned
4045
+ // one broken mount into a review-ceiling deadlock: three such deaths
4046
+ // consumed the PR's three rounds, `conductor_pr_review` then refused any
4047
+ // further round, and the findings could never be addressed by a worker at
4048
+ // all. Nothing in the loop could recover it.
4049
+ //
4050
+ // So the round is returned to the pending set instead: the same row, the
4051
+ // same round number, the same findings, and the run restored to the exact
4052
+ // `pushed-green` state the verb recorded — which is what the restart
4053
+ // reconcile already does for a round a dying daemon interrupted. The next
4054
+ // dispatch pass resumes the same session. Bounded by
4055
+ // REVIEW_ROUND_INFRA_MAX_RETRIES and counted durably on the row, so a
4056
+ // permanently broken host escalates through the ordinary failed path
4057
+ // instead of retrying every tick forever.
4058
+ const infraKill =
4059
+ state === "failed" || state === "killed"
4060
+ ? reviewRoundNeverWorked({
4061
+ turns: result.turns,
4062
+ ...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
4063
+ ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
4064
+ ...(settlement?.patch?.salvageSha === undefined
4065
+ ? {}
4066
+ : { salvageSha: settlement.patch.salvageSha }),
4067
+ })
4068
+ : undefined;
4069
+ if (infraKill !== undefined) {
4070
+ const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
4071
+ if (retry.kind === "requeued") {
4072
+ const detail =
4073
+ `review round ${revision.round} was killed before its session took a turn ` +
4074
+ `(${infraKill}); the round is re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`;
4075
+ store.updateRun(runId, {
4076
+ ...terminalPatch,
4077
+ state: "pushed-green",
4078
+ // The transcript the round must still resume is the one the run
4079
+ // recorded, not whatever this dead dispatch opened: a session that
4080
+ // never took a turn has no lineage of its own to inherit.
4081
+ sessionFile: priorSessionFile,
4082
+ report: [`review round ${revision.round}: dispatch killed before turn 1`, "", detail].join("\n"),
4083
+ lastError: detail,
4084
+ });
4085
+ // No label swap and no escalation: nothing failed that a human can
4086
+ // act on yet, the issue keeps its in-progress label, and the next
4087
+ // dispatch pass picks the round up on its own.
4088
+ log(`#${issue} ${detail}`);
4089
+ return;
4090
+ }
4091
+ log(
4092
+ `#${issue} review round ${revision.round} exhausted its infra retries ` +
4093
+ `(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settling it the ordinary way`,
4094
+ );
4095
+ }
3484
4096
  if (state === "stopped") {
3485
4097
  recordOperatorStop(store, {
3486
4098
  project: project.name,
@@ -3592,6 +4204,35 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
3592
4204
  publish,
3593
4205
  tree: "keep",
3594
4206
  });
4207
+ // A crash BEFORE the worker session was installed spent no round (#903):
4208
+ // nothing resumed, nothing was reviewed, and the round is re-queued for
4209
+ // the next pass instead of consuming one of the PR's three. Once the
4210
+ // session is installed the round belongs to the worker, and a crash after
4211
+ // that settles the ordinary way below — including the exhausted case, so
4212
+ // a dispatch that keeps throwing still reaches a human.
4213
+ if (!workerSessionInstalled) {
4214
+ const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
4215
+ if (retry.kind === "requeued") {
4216
+ store.updateRun(runId, {
4217
+ state: "pushed-green",
4218
+ endedAt: Date.now(),
4219
+ sessionFile: priorSessionFile,
4220
+ lastError:
4221
+ `review round ${revision.round} could not be dispatched (${detail}); the round is re-queued, ` +
4222
+ `retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
4223
+ ...settlement?.patch,
4224
+ });
4225
+ log(
4226
+ `#${issue} review round ${revision.round} re-queued after a pre-session dispatch error — ` +
4227
+ `retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
4228
+ );
4229
+ return;
4230
+ }
4231
+ log(
4232
+ `#${issue} review round ${revision.round} exhausted its infra retries ` +
4233
+ `(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — failing the round`,
4234
+ );
4235
+ }
3595
4236
  store.updateRun(runId, {
3596
4237
  state: "failed",
3597
4238
  endedAt: Date.now(),
@@ -4341,16 +4982,26 @@ export interface WorkerPool {
4341
4982
  drain(): Promise<void>;
4342
4983
  }
4343
4984
 
4344
- /** Keeps background workers alive without making the five-minute tick await them. */
4345
- export function createWorkerPool(): WorkerPool {
4985
+ /**
4986
+ * Keeps background workers alive without making the five-minute tick await them.
4987
+ *
4988
+ * `onSettled` fires as each worker leaves the pool, whichever way it ended
4989
+ * (#878): that is the instant a slot frees, and without it queued work waited
4990
+ * for the next scheduled pass — up to five minutes of idle capacity that reads,
4991
+ * from outside, exactly like a stalled queue. It only *prompts* a pass; every
4992
+ * hold, drain, lane, budget and routing gate is re-evaluated by that pass as
4993
+ * usual.
4994
+ */
4995
+ export function createWorkerPool(onSettled?: () => void): WorkerPool {
4346
4996
  const active = new Set<Promise<void>>();
4347
4997
  return {
4348
4998
  launch(work) {
4349
4999
  active.add(work);
4350
- void work.then(
4351
- () => active.delete(work),
4352
- () => active.delete(work),
4353
- );
5000
+ const settled = (): void => {
5001
+ active.delete(work);
5002
+ onSettled?.();
5003
+ };
5004
+ void work.then(settled, settled);
4354
5005
  },
4355
5006
  activeCount: () => active.size,
4356
5007
  async drain() {
@@ -4671,6 +5322,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4671
5322
  } catch (err) {
4672
5323
  log(`label reconcile failed: ${errText(err)}`);
4673
5324
  }
5325
+ try {
5326
+ // Same phase, same reason as the label reconcile above: a durable row the
5327
+ // tracker has moved past (#964). A separate call because that one iterates
5328
+ // labelled issues, and a promotable issue carries no label.
5329
+ await reconcileGroomingClosures(d);
5330
+ } catch (err) {
5331
+ log(`grooming closure reconcile failed: ${errText(err)}`);
5332
+ }
4674
5333
 
4675
5334
  // Drain the label projection outbox (#201). The maintenance phases above may
4676
5335
  // have enqueued ops (settlement releases, recovery requeues, reconciles);
@@ -4864,6 +5523,28 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4864
5523
  log(`review revision dispatch failed: ${errText(err)}`);
4865
5524
  }
4866
5525
 
5526
+ // Adjudications are dispatch too (#932), and their own pass: an adjudication
5527
+ // occupies no worker slot and no issue — it opens a fresh read-only session
5528
+ // about a PR whose run has already settled — so it is neither bounded by the
5529
+ // worker cap nor allowed to consume a review round. Placed after revisions so
5530
+ // a PR with a live revision is never adjudicated in the same tick it is being
5531
+ // corrected in; the head re-read inside the pass is the durable guard.
5532
+ try {
5533
+ await dispatchReviewAdjudications(d, workers);
5534
+ } catch (err) {
5535
+ log(`adjudication dispatch failed: ${errText(err)}`);
5536
+ }
5537
+
5538
+ // And what the verdicts imply (#876). Its own pass, after dispatch, because a
5539
+ // disposition acts on a SETTLED adjudication: running it before dispatch would
5540
+ // simply be a tick late, and running it inside dispatch would tie a tracker
5541
+ // mutation to the launch that produced the verdict.
5542
+ try {
5543
+ await applyAdjudicationDispositions(d);
5544
+ } catch (err) {
5545
+ log(`adjudication dispositions failed: ${errText(err)}`);
5546
+ }
5547
+
4867
5548
  // route() filters the queue through isEligible() itself, so anything already
4868
5549
  // carrying a state label is gone before it gets here.
4869
5550
  const ready = await d.tracker.listReady();
@@ -4882,7 +5563,65 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4882
5563
  const pending = store.pendingLabelOpsFor(project.name, issue.number);
4883
5564
  return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
4884
5565
  });
4885
- const { routed, unroutable } = route(effective, project);
5566
+ // Post-unblock stale-list revalidation (#891). `listReady` is a
5567
+ // label-FILTERED search, and GitHub's search index is eventually consistent:
5568
+ // measured 2026-08-22T09:18Z, an `unblock` cleared `agent:failed` and
5569
+ // restored the queue label, `gh issue view` returned the clean label set
5570
+ // immediately, and the very next pass still saw the stale labels, held #889
5571
+ // as `stale-lifecycle` and left the fleet at 0/5 with four spare slots. The
5572
+ // next scheduled pass usually recovers — this is throughput, not correctness
5573
+ // — but an operator-driven unblock that cannot refill the fleet has not done
5574
+ // what it says.
5575
+ //
5576
+ // So a lifecycle label read from the LIST is a suspicion, not a verdict: the
5577
+ // exact per-issue read decides. Bounded to candidates that would otherwise be
5578
+ // dropped for a lifecycle label — never the whole queue — and further to those
5579
+ // whose newest run is not active (a genuinely in-flight issue needs no
5580
+ // revalidation; its own row is the authority) and that are not operator-parked
5581
+ // (a park is a decision, not a stale label).
5582
+ //
5583
+ // An unreadable exact read keeps the stale labels, so the issue stays out of
5584
+ // this pass, and is held as `issue-state-lookup-error` rather than
5585
+ // `stale-lifecycle`: "the tracker could not say" is not evidence of a
5586
+ // residual label, and it must not summon Duty 1 to reconcile a label nobody
5587
+ // has read.
5588
+ const stateLabelSet = new Set(Object.values(project.stateLabels));
5589
+ const suspect = effective.filter(
5590
+ (issue) =>
5591
+ !isEligible(issue, project) &&
5592
+ issue.labels.some((l) => stateLabelSet.has(l)) &&
5593
+ !issue.labels.includes(project.stateLabels.backlog) &&
5594
+ !ACTIVE_STATES.includes(store.latestRun(project.name, issue.number)?.state ?? "merged"),
5595
+ );
5596
+ const revalidated = new Map<number, readonly string[]>();
5597
+ const unreadable = new Set<number>();
5598
+ for (const issue of suspect) {
5599
+ let snapshot: IssueSnapshot | undefined;
5600
+ try {
5601
+ snapshot = await d.tracker.issueSnapshot(issue.number);
5602
+ } catch {
5603
+ snapshot = undefined;
5604
+ }
5605
+ if (snapshot === undefined) {
5606
+ unreadable.add(issue.number);
5607
+ log(`#${issue.number} lifecycle labels could not be revalidated — holding this pass`);
5608
+ continue;
5609
+ }
5610
+ // A closed issue is nobody's candidate; leave the list labels alone and let
5611
+ // the ordinary gates speak.
5612
+ if (snapshot.state === "closed") continue;
5613
+ if (snapshot.labels.join("\u0000") === issue.labels.join("\u0000")) continue;
5614
+ revalidated.set(issue.number, snapshot.labels);
5615
+ log(`#${issue.number} lifecycle labels revalidated: the queue list was stale`);
5616
+ }
5617
+ const verified =
5618
+ revalidated.size === 0
5619
+ ? effective
5620
+ : effective.map((issue) => {
5621
+ const live = revalidated.get(issue.number);
5622
+ return live === undefined ? issue : { ...issue, labels: [...live] };
5623
+ });
5624
+ const { routed, unroutable } = route(verified, project);
4886
5625
  // route() drops a candidate for two reasons, only one of which is a claim
4887
5626
  // question. A lifecycle state label (agent:in-progress/blocked/failed)
4888
5627
  // marks a run-owned issue: it is genuinely in flight while its newest run
@@ -4894,9 +5633,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4894
5633
  // close), which belongs in none of the three populations. So `claimed` is
4895
5634
  // defined from actual ownership over genuinely lifecycle-labelled candidates
4896
5635
  // rather than as the residual of routing (#228, #611).
4897
- const stateLabels = new Set(Object.values(project.stateLabels));
4898
- const dropped = effective.filter(
4899
- (issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
5636
+ // Every population below reads the VERIFIED list, so a candidate the exact
5637
+ // read cleared is not simultaneously admitted and counted as held (#891).
5638
+ const dropped = verified.filter(
5639
+ (issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabelSet.has(l)),
4900
5640
  );
4901
5641
  // The operator's park label is not a stale lifecycle label: the queue query
4902
5642
  // still returns a parked-and-queued issue, route() drops it as ineligible,
@@ -4905,7 +5645,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4905
5645
  // to reconcile a deliberate decision. Parked is its own population, derived
4906
5646
  // from the same isEligible read the gate uses, so the number status renders
4907
5647
  // cannot disagree with what admission would hold (#507).
4908
- const parkedCandidates = effective.filter(
5648
+ const parkedCandidates = verified.filter(
4909
5649
  (issue) => !isEligible(issue, project) && issue.labels.includes(project.stateLabels.backlog),
4910
5650
  );
4911
5651
  const parkedNumbers = new Set(parkedCandidates.map((issue) => issue.number));
@@ -4920,6 +5660,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
4920
5660
  // A park never kills a live run; a parked issue with no run is inventory
4921
5661
  // the operator is deliberately holding, not reconciliation work.
4922
5662
  parked += 1;
5663
+ } else if (unreadable.has(issue.number)) {
5664
+ // Fail closed, and say which read failed: a tracker that could not answer
5665
+ // is not evidence of a residual label (#891).
5666
+ lifecycleHolds.push({
5667
+ issue: issue.number,
5668
+ reason: "issue-state-lookup-error",
5669
+ detail: "lifecycle labels could not be revalidated against the exact issue read",
5670
+ });
4923
5671
  } else {
4924
5672
  lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
4925
5673
  }
@@ -5021,6 +5769,8 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
5021
5769
  return;
5022
5770
  }
5023
5771
  recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
5772
+ await recordInstallSurfaces(d);
5773
+ reconcilePanes(d, project.name, log);
5024
5774
 
5025
5775
  if (pass.admitted.length === 0) return;
5026
5776
 
@@ -5514,8 +6264,34 @@ export interface StatusSnapshot {
5514
6264
  * materialises it, so this is never absent on a real daemon.
5515
6265
  */
5516
6266
  review: ReviewPolicy;
5517
- /** Occupied issues: live workers plus green PRs awaiting a human merge. */
6267
+ /** Occupied issues: live workers plus green PRs awaiting a human merge. Both
6268
+ * lifecycle kinds, told apart by {@link StatusSnapshot.leasedRunIds}. */
5518
6269
  activeRuns: RunRecord[];
6270
+ /**
6271
+ * The subset of {@link StatusSnapshot.activeRuns} holding an **active
6272
+ * mutation lease** (#898): a live worker, or a dispatched unsettled review
6273
+ * revision. Everything else in `activeRuns` is a worker-free preserved
6274
+ * artifact — durable work awaiting review, merge or recovery, with nothing
6275
+ * writing to it.
6276
+ *
6277
+ * Carried as ids rather than a second run list so the two renderings cannot
6278
+ * disagree about the rows themselves, and derived from `Store.leasedRuns` —
6279
+ * the same query admission's file-lane gate and `conductor_pr_recover` read,
6280
+ * so what status calls a lease is exactly what those two enforce (#899,
6281
+ * #925). A plain array so the dashboard's JSON round-trip preserves it.
6282
+ */
6283
+ leasedRunIds?: readonly string[];
6284
+ /**
6285
+ * The review-ceiling adjudications an operator needs to see (#874): every
6286
+ * non-terminal one, plus the terminal one for each active run's PR head — so a
6287
+ * cleared or rejected verdict stays visible for exactly as long as the PR it
6288
+ * decided is still in flight, and disappears with it rather than accumulating.
6289
+ *
6290
+ * Keyed by nothing: the records carry their own `prUrl` and `headSha`, which is
6291
+ * how the renderer matches them to a run without inventing a second identity
6292
+ * for a PR.
6293
+ */
6294
+ reviewAdjudications?: readonly ReviewAdjudicationRecord[];
5519
6295
  /**
5520
6296
  * runId → live review-revision round, for runs whose revision worker is
5521
6297
  * currently dispatched (#692). Derived from the durable `review_revisions`
@@ -5524,7 +6300,22 @@ export interface StatusSnapshot {
5524
6300
  * (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
5525
6301
  * it byte for byte.
5526
6302
  */
5527
- reviewRounds?: Readonly<Record<string, number>>;
6303
+ /**
6304
+ * The live review round per run: its number AND the instant it was dispatched
6305
+ * (#802). The instant is what makes the line honest — a resumed session's
6306
+ * `turns` and `startedAt` are cumulative over the whole attempt, so without a
6307
+ * phase boundary a fresh revision reads as though it had been running for
6308
+ * hours.
6309
+ */
6310
+ reviewRounds?: Readonly<Record<string, { round: number; dispatchedAt: number }>>;
6311
+ /**
6312
+ * runId → the durable review evidence blocking a merge of the run's
6313
+ * pushed-green PR at its exact recorded head (#888). Derived from the same
6314
+ * `review_revisions` rows `conductor_pr_merge` refuses on, so status can
6315
+ * never present a green PR as merge-ready while the privileged verb will
6316
+ * refuse it. An absent entry means nothing blocks that run's head.
6317
+ */
6318
+ mergeBlockers?: Readonly<Record<string, ReviewHeadBlocker>>;
5528
6319
  /** Newest attempts holding a preserved WIP tip, or a tree that is still the
5529
6320
  * only copy of work the daemon could not save. */
5530
6321
  salvagedRuns: RunRecord[];
@@ -5558,6 +6349,20 @@ export interface StatusSnapshot {
5558
6349
  liveWorkers: number;
5559
6350
  runsToday: number;
5560
6351
  spendTodayUsd: number;
6352
+ /** The install identities the last dispatch pass recorded (#919). Absent
6353
+ * means no pass has looked yet, which the renderer says out loud rather
6354
+ * than presenting as agreement. */
6355
+ installSurfaces?: InstallSurfaceObservation;
6356
+ /** What live runs have reserved out of today's budget but not yet spent
6357
+ * (#851). Optional so a caller that built a snapshot before reservations
6358
+ * existed renders no figure rather than a fabricated zero. */
6359
+ reservedSpendUsd?: number;
6360
+ /**
6361
+ * Whether the figure above is built on runs that actually reported cost
6362
+ * (#970). Absent when the caller did not judge it; the renderer then says
6363
+ * nothing rather than implying the telemetry is sound.
6364
+ */
6365
+ spendTelemetry?: SpendTelemetryVerdict;
5561
6366
  dispatch?: DispatchSummary;
5562
6367
  /**
5563
6368
  * Latest plan-allowance verdict, when the caller read one. Optional because
@@ -5632,12 +6437,50 @@ export function statusSnapshotFromStore(
5632
6437
  // (#776 review #2).
5633
6438
  const live = store.liveRuns(p.name);
5634
6439
  const active = store.activeRuns(p.name);
6440
+ // Which of those active rows something is actually writing through (#898).
6441
+ // The same query admission's file-lane gate and `conductor_pr_recover` read,
6442
+ // so status cannot call a row a lease that those two treat as released, or
6443
+ // vice versa (#899, #925).
6444
+ const leasedRunIds = store.leasedRuns(p.name).map((r) => r.id);
6445
+ // Adjudications worth rendering (#874): the open ones always, plus a terminal
6446
+ // verdict for a head an active run is still sitting on. Bounded by the active
6447
+ // set rather than by a history window, so a fleet with a thousand settled
6448
+ // adjudications renders the handful that still describe live work.
6449
+ const openAdjudications = store.openReviewAdjudications(p.name);
6450
+ const adjudicationIds = new Set(openAdjudications.map((a) => a.id));
6451
+ const reviewAdjudications = [...openAdjudications];
6452
+ for (const r of active) {
6453
+ if (r.prUrl === undefined || r.headSha === undefined) continue;
6454
+ const decided = store.reviewAdjudicationForHead(p.name, r.prUrl, r.headSha);
6455
+ if (decided !== undefined && !adjudicationIds.has(decided.id)) {
6456
+ adjudicationIds.add(decided.id);
6457
+ reviewAdjudications.push(decided);
6458
+ }
6459
+ }
5635
6460
  // The live review-revision rounds, read from the same durable rows the
5636
6461
  // restart recovery uses: a run whose revision is dispatched is read as
5637
6462
  // `review-revision N` while its worker is live (#692).
5638
- const reviewRounds: Record<string, number> = {};
6463
+ const reviewRounds: Record<string, { round: number; dispatchedAt: number }> = {};
5639
6464
  for (const revision of store.unsettledReviewRevisions(p.name)) {
5640
- if (revision.dispatchedAt !== undefined) reviewRounds[revision.runId] = revision.round;
6465
+ if (revision.dispatchedAt !== undefined) {
6466
+ reviewRounds[revision.runId] = { round: revision.round, dispatchedAt: revision.dispatchedAt };
6467
+ }
6468
+ }
6469
+ // The exact-head merge blockers (#888), read from the same durable rows the
6470
+ // merge verb consults: a pushed-green run whose PR stands at a head carrying
6471
+ // unresolved review evidence — a round queued there, crashed mid-review
6472
+ // there, or settled `failed` there.
6473
+ const mergeBlockers: Record<string, ReviewHeadBlocker> = {};
6474
+ for (const r of active) {
6475
+ if (r.prUrl === undefined || r.headSha === undefined) continue;
6476
+ const blocker = store.mergeBlockingReviews(p.name, r.prUrl, r.headSha)[0];
6477
+ if (blocker !== undefined) {
6478
+ mergeBlockers[r.id] = {
6479
+ round: blocker.round,
6480
+ state:
6481
+ blocker.settledAt !== undefined ? "failed" : blocker.dispatchedAt !== undefined ? "crashed" : "pending",
6482
+ };
6483
+ }
5641
6484
  }
5642
6485
  return {
5643
6486
  project: p.name,
@@ -5662,6 +6505,8 @@ export function statusSnapshotFromStore(
5662
6505
  releaseGrants: resolveReleaseGrants(p),
5663
6506
  review: resolveReview(p),
5664
6507
  activeRuns: active,
6508
+ leasedRunIds,
6509
+ reviewAdjudications,
5665
6510
  reviewRounds,
5666
6511
  salvagedRuns: store.salvagedRuns(p.name),
5667
6512
  quarantinedRuns: store.quarantinedRuns(p.name),
@@ -5672,6 +6517,19 @@ export function statusSnapshotFromStore(
5672
6517
  liveWorkers: live.length,
5673
6518
  runsToday: store.runsStartedSince(p.name, since),
5674
6519
  spendTodayUsd: store.spendSince(p.name, since),
6520
+ reservedSpendUsd: store.reservedSpendUsd(p.name),
6521
+ // Judged from the store on the same read as the figure above (#970), so the
6522
+ // row that qualifies the spend number cannot disagree with it, and nothing
6523
+ // is probed at render time.
6524
+ spendTelemetry: judgeSpendTelemetry(
6525
+ store.recentSpendSamples?.(p.name, SPEND_SAMPLE_ROWS) ?? [],
6526
+ SPEND_SAMPLE_RUNS,
6527
+ ),
6528
+ // Read, never probed: the recorded row is the whole point (#919).
6529
+ ...(() => {
6530
+ const observed = store.installSurfaces();
6531
+ return observed === undefined ? {} : { installSurfaces: observed };
6532
+ })(),
5675
6533
  ...(dispatch === undefined ? {} : { dispatch }),
5676
6534
  ...(planUsage === undefined ? {} : { planUsage }),
5677
6535
  // Written by the tracker's hooks rather than polled, so the renderer does
@@ -5683,6 +6541,7 @@ export function statusSnapshotFromStore(
5683
6541
  : { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
5684
6542
  baseHealth: store.baseHealth(p.name),
5685
6543
  freezes: store.freezes(p.name),
6544
+ mergeBlockers,
5686
6545
  ...(orchestratorDown === undefined ? {} : { orchestratorDown }),
5687
6546
  };
5688
6547
  }
@@ -5736,8 +6595,15 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
5736
6595
  lines.push(` ${hold.reason} ${hold.count}${sample}`);
5737
6596
  // A `file-lane` hold groups several issues, each blocked by a different
5738
6597
  // file and holder; the grouped line says how many, this says which.
5739
- if ((hold.details?.length ?? 0) > 0) {
5740
- lines.push(` ${hold.details!.join(" | ")}`);
6598
+ //
6599
+ // Deduplicated for display only, never in the record: a fleet-wide hold
6600
+ // (`credential-class`, #852) gives every held candidate the *same*
6601
+ // sentence, and printing one reason five times reads as five problems.
6602
+ // The persisted details stay index-aligned with `issues` — a reader that
6603
+ // needs "which issue got which detail" still has it.
6604
+ const details = [...new Set(hold.details ?? [])];
6605
+ if (details.length > 0) {
6606
+ lines.push(` ${details.join(" | ")}`);
5741
6607
  }
5742
6608
  }
5743
6609
  }
@@ -5803,71 +6669,6 @@ export function formatFreezes(freezes: readonly BaseFreeze[]): string[] {
5803
6669
  }
5804
6670
 
5805
6671
 
5806
- export function formatStatus(s: StatusSnapshot): string {
5807
- const lines = [
5808
- `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
5809
- `config ${s.configPath}`,
5810
- `state ${s.stateDir}`,
5811
- "",
5812
- ...(s.orchestratorDown === undefined ? [] : formatOrchestratorDown(s.orchestratorDown)),
5813
- "caps",
5814
- ` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
5815
- ` issues today ${s.runsToday}`,
5816
- s.caps.dailySpendUsd === null
5817
- ? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
5818
- : ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
5819
- // Its own row, never folded into the spend row: they are two independent
5820
- // controls, and an operator has to be able to see which one stopped the
5821
- // fleet (#110).
5822
- ` plan usage ${planUsageLine(s.planUsage)}`,
5823
- ` new worker turns ${s.caps.workerMaxTurns}`,
5824
- ...s.turnOverrides.map(
5825
- ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
5826
- ),
5827
- ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
5828
- ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
5829
- ` continuations ${s.caps.maxContinuationsPerIssue}`,
5830
- "",
5831
- ...formatReleaseGrants(s.releaseGrants),
5832
- "",
5833
- formatDispatchSummary(s.dispatch),
5834
- "",
5835
- ];
5836
- if (s.activeRuns.length === 0) {
5837
- lines.push("active runs (none)");
5838
- } else {
5839
- lines.push("active runs");
5840
- for (const r of s.activeRuns) {
5841
- lines.push(
5842
- ` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
5843
- `${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
5844
- (r.prUrl ? ` ${r.prUrl}` : ""),
5845
- );
5846
- // Its escalation was deduplicated the moment it was delivered, so this
5847
- // line is the only place a flagged run stays visible while its PR waits
5848
- // for a merge — which is precisely the window the flag is about (#128).
5849
- const flagged = settlementFlagSummary(r.settlementFlags);
5850
- if (flagged !== undefined) lines.push(` ${flagged}`);
5851
- }
5852
- }
5853
- lines.push(...formatBaseHealth(s.baseHealth));
5854
- lines.push(...formatFreezes(s.freezes));
5855
- lines.push(...formatSalvagedRuns(s.salvagedRuns));
5856
- lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
5857
- lines.push(...formatOpenReports(s.openReports));
5858
- lines.push(...formatVerbLedger(s.verbLedger));
5859
- // Deploy hint: a restart while workers are live orphans them (salvage runs
5860
- // first — #35). Prefer pause + drain to zero live workers when you can wait.
5861
- if (s.liveWorkers > 0) {
5862
- lines.push(
5863
- "",
5864
- `deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
5865
- `pause and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
5866
- );
5867
- }
5868
- return lines.join("\n");
5869
- }
5870
-
5871
6672
  export interface QueuePreview {
5872
6673
  project: string;
5873
6674
  configPath: string;
@@ -6250,32 +7051,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6250
7051
  const store = openStore(dbPath());
6251
7052
  const verbPeerReader = peerCredentialReader();
6252
7053
  const verbDir = ensureVerbSocketDir(stateDir());
6253
- // The worker identity (#798). Probed once here for the startup banner only —
6254
- // every dispatch re-resolves it through {@link launchIdentity}, because the
6255
- // account, setpriv and the harness binding (#828) are all host state that can
6256
- // arrive after this process did. A host that cannot establish it keeps its
6257
- // control plane running and fails each worker launch closed with the reason,
6258
- // so the operator hears a concrete host change instead of a fleet that
6259
- // silently ran workers as root.
6260
- const identityAtStartup = resolveWorkerIdentity();
6261
- if (identityAtStartup.ok) {
6262
- log(
6263
- `worker identity: ${identityAtStartup.identity.account} uid=${identityAtStartup.identity.uid} ` +
6264
- `gid=${identityAtStartup.identity.gid} home=${identityAtStartup.identity.home}`,
6265
- );
6266
- } else {
6267
- log(
6268
- "worker identity unavailable — worker dispatch fails closed until the host provides it " +
6269
- `(re-checked at every launch, so no restart is needed once it does): ${identityAtStartup.reason}`,
6270
- );
6271
- }
6272
7054
  const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
6273
7055
  const usage = sharedUsageSource();
6274
7056
  const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
6275
7057
  store.updateRun(runId, { maxTurns });
6276
7058
  });
6277
7059
  const workerControls = createWorkerControlRegistry();
6278
- const workers = createWorkerPool();
7060
+ // Created before the pool so a worker's completion can poke the loop it
7061
+ // shares with the interval sleep (#878).
7062
+ const pace = createDispatchPace();
7063
+ const workers = createWorkerPool(() => {
7064
+ // Sticky and coalescing by construction: several workers settling together
7065
+ // produce one pass, and a pass already running absorbs the request rather
7066
+ // than overlapping. No second owner is created and no gate is skipped.
7067
+ pace.requestWake();
7068
+ });
6279
7069
  const alive = livingDaemon();
6280
7070
  const runtimes: ProjectRuntime[] = [];
6281
7071
 
@@ -6322,6 +7112,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6322
7112
  : pushRunBranch(project, { repo, runRepoPath: run.worktree, branch });
6323
7113
  };
6324
7114
  for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
7115
+ // The workspace outlives the process, so a restart inherits panes whose
7116
+ // workers are gone (#842). A session-host child dies with the daemon that
7117
+ // owned its socket, so an orphaned row's pane is authoritatively dead
7118
+ // whatever its recorded pid says — pids are reused, and matching one
7119
+ // would be the mechanism that makes a stranger's process read as a live
7120
+ // worker. Release (never close) so lifecycle authority goes back to
7121
+ // Herdr and the pane stops claiming a worker conductor no longer runs.
7122
+ const orphanPane = releaseOrphanedWorkerPane(run);
7123
+ if (orphanPane.kind === "released") {
7124
+ projectLog(
7125
+ `#${run.issue} herdr pane ${orphanPane.paneId} released: its worker died with the previous daemon`,
7126
+ );
7127
+ } else if (orphanPane.kind === "failed") {
7128
+ projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} release failed: ${orphanPane.reason}`);
7129
+ }
6325
7130
  projectLog(
6326
7131
  `#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` +
6327
7132
  `worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` +
@@ -6446,17 +7251,27 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6446
7251
  cleanup: { next: 0 },
6447
7252
  probeCriticalBase: (repo, markers, branch) =>
6448
7253
  probeCriticalBase(project, repo, branch, markers),
7254
+ // The same seam `doctor` and `upgrade` read (#904/#919), never a second
7255
+ // implementation. `readHerdr` stays on: a host that runs herdr is the
7256
+ // case where a stale recovery pin matters, and a host without it answers
7257
+ // "absent", which no cheap surface treats as a fault.
7258
+ probeInstallSurfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
7259
+ probeTelegramFreshness: () =>
7260
+ // The module's own reader, so the daemon, `doctor` and `status` cannot
7261
+ // come to disagree about what is installed (#961).
7262
+ checkTelegramFreshness({
7263
+ run: async (cmd, args) => {
7264
+ const r = await runCommand(cmd, args);
7265
+ return { code: r.code, stdout: r.stdout };
7266
+ },
7267
+ }),
6449
7268
  probeWorktreeLane: (input) => probeRunLane(input),
6450
- // The live host, per launch — never the startup verdict above.
6451
- workerIdentity: () => resolveWorkerIdentity(),
6452
- // Grant each run's paths to the worker identity by ownership, before the
6453
- // session that must edit them starts (#798). The recursive chown runs as
6454
- // root from the dispatch path; the worker identity owns its checkout and
6455
- // transcript and nothing else.
6456
- grantWorkerPaths: (identity, worktreePath, sessionDir) => {
6457
- chownRecursive(worktreePath, identity.uid, identity.gid);
6458
- chownRecursive(sessionDir, identity.uid, identity.gid);
6459
- },
7269
+ // The credential-class fence's reader (#852): the runnable probe module,
7270
+ // spawned against the live credential store. Wired once here so the
7271
+ // admission gate and the launch fence ask the same question through the
7272
+ // same transport two readers would eventually disagree, and the one that
7273
+ // disagreed by passing is the expensive one.
7274
+ probeCredentialClass: (provider) => probeCredentialClass(provider),
6460
7275
  // A cross-repo Depends-on prerequisite reads through the same GitHub
6461
7276
  // credential/accounting seams as the project tracker — a fresh tracker
6462
7277
  // scoped to the referenced repo, reusing the daemon's gh hooks so API
@@ -6498,6 +7313,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6498
7313
  projectLog(`review revision restart recovery failed: ${errText(err)}`);
6499
7314
  }
6500
7315
  }
7316
+ // Adjudication restart recovery (#932). An adjudication is `running` only
7317
+ // while a session it launched is alive, and that session died with the
7318
+ // previous process — nothing resumes it, because an adjudicator has no
7319
+ // branch, no worktree and no transcript worth continuing: its whole output
7320
+ // is a verdict it never produced.
7321
+ //
7322
+ // So it settles `failed`, which is honest and terminal, rather than being
7323
+ // left `running` forever (a row nothing would ever touch again) or silently
7324
+ // re-queued (a second launch for one head, which the one-shot exists to
7325
+ // prevent). The one shot for that head is spent, and #876's disposition path
7326
+ // reads `failed` exactly as it reads any other non-clear verdict.
7327
+ if (alive === undefined || alive.pid === process.pid) {
7328
+ for (const adjudication of store.openReviewAdjudications(project.name)) {
7329
+ if (adjudication.state !== "running") continue;
7330
+ store.settleReviewAdjudication(
7331
+ adjudication.id,
7332
+ "failed",
7333
+ "the daemon restarted while this adjudication was running; no verdict was produced",
7334
+ Date.now(),
7335
+ );
7336
+ projectLog(
7337
+ `#${adjudication.issue} adjudication failed across restart — its session died with the previous daemon`,
7338
+ );
7339
+ }
7340
+ }
6501
7341
  // Startup reconciliation: close an incident carried over from a previous
6502
7342
  // process when the orchestrator is up (one recovery notice), or open one
6503
7343
  // when it failed to start (one down page). A daemon restarted while still
@@ -6595,7 +7435,6 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
6595
7435
  // The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
6596
7436
  // sleep. A wake request is sticky and coalescing — several resumes in quick
6597
7437
  // succession produce one prompt pass, never an overlapping one.
6598
- const pace = createDispatchPace();
6599
7438
  const stop = (): void => {
6600
7439
  if (stopping) return;
6601
7440
  stopping = true;