omp-conductor 0.3.24 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -8,9 +8,24 @@
8
8
  * here and enforced before anything is claimed.
9
9
  */
10
10
  import { createHash } from "node:crypto";
11
- import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
11
+ import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
12
12
  import { dirname, join, relative } from "node:path";
13
- import { configPath, findProject, loadConfig, resolveCaps, resolveReleasePolicy, stateDir } from "./config.ts";
13
+ import {
14
+ configPath,
15
+ findProject,
16
+ loadConfig,
17
+ migrateCredentialsOnDisk,
18
+ resolveCaps,
19
+ resolveCredentials,
20
+ resolveReleaseGrants,
21
+ stateDir,
22
+ sharedRoot,
23
+ } from "./config.ts";
24
+ import {
25
+ analyseSettlement,
26
+ formatSettlementFlags,
27
+ settlementFlagSummary,
28
+ } from "./diff-flags.ts";
14
29
  import { createEscalator, escalationIssueRef } from "./escalate.ts";
15
30
  import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
16
31
  import { graphHint } from "./graph.ts";
@@ -18,11 +33,13 @@ import { livingDaemon } from "./lifecycle.ts";
18
33
  import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
19
34
  import { startOrchestrator } from "./orchestrator.ts";
20
35
  import type { OrchestratorHandle } from "./orchestrator.ts";
36
+ import { createReportOutbox, formatOpenReports } from "./reports.ts";
21
37
  import { recordReleaseBlock } from "./release-policy.ts";
22
38
  import { branchName, route } from "./routing.ts";
23
39
  import type { Routed, UnroutableReason } from "./routing.ts";
24
40
  import { dbPath, openStore } from "./store.ts";
25
41
  import { makeTracker } from "./tracker/github.ts";
42
+ import { RELEASE_SHAPES } from "./types.ts";
26
43
  import type {
27
44
  AdmissionHoldReason,
28
45
  Caps,
@@ -30,30 +47,84 @@ import type {
30
47
  Escalation,
31
48
  OpenCloser,
32
49
  PrState,
50
+ CredentialIsolation,
33
51
  ProjectConfig,
34
52
  ReadyIssue,
35
53
  RepoTarget,
54
+ ReportRecord,
55
+ ResolvedGrants,
36
56
  RunRecord,
37
57
  RunState,
58
+ SettlementFlag,
38
59
  Store,
39
60
  Tracker,
61
+ VerbLedgerEntry,
40
62
  } from "./types.ts";
63
+ import { CONDUCTOR_GROUPS } from "./types.ts";
41
64
  import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
42
65
  import {
43
- addWorktree,
66
+ addRunRepo,
44
67
  cleanupRetainedWorktree,
45
68
  mirrorPathFor,
46
69
  removeWorktree,
47
70
  salvageWip,
48
71
  type RetainedWorktreeCleanup,
72
+ type RunPublisher,
49
73
  type SalvageOutcome,
50
74
  worktreePathFor,
51
75
  } from "./worktree.ts";
76
+ import { githubVerbActions } from "./verbs/actions.ts";
77
+ import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
78
+ import { listenVerbChannel } from "./verbs/server.ts";
79
+ import type { VerbActions, VerbDeps, VerbListener } from "./verbs/server.ts";
80
+ import {
81
+ ensureVerbSocketDir,
82
+ peerCredentialReader,
83
+ transportBanner,
84
+ verbSocketPath,
85
+ type PeerReader,
86
+ } from "./verbs/socket.ts";
87
+ import { homedir } from "node:os";
88
+
89
+ import {
90
+ applyRunOwnership,
91
+ applyMirrorReadAccess,
92
+ auditMcpRoots,
93
+ mirrorTraversalRefusal,
94
+ credentialReadRefusal,
95
+ spawnCaptured,
96
+ boundaryRefusal,
97
+ buildSessionBoundary,
98
+ createSlotPool,
99
+ describeBoundary,
100
+ liveDaemonCredentials,
101
+ mcpRefusal,
102
+ mechanismSatisfies,
103
+ openRunPr,
104
+ probeHost,
105
+ pushRunBranch,
106
+ verifyDaemonAccess,
107
+ type HostProbe,
108
+ type RunRepoRef,
109
+ type SessionBoundary,
110
+ type SlotPool,
111
+ } from "./credentials.ts";
112
+ import {
113
+ planUsageLine,
114
+ readPlanUsage,
115
+ sharedUsageSource,
116
+ type PlanUsageStatus,
117
+ type UsageSource,
118
+ } from "./usage.ts";
52
119
 
53
120
  /** Long enough that the tracker is not polled raw, short enough that a human
54
121
  * who labels an issue sees it picked up within a coffee break. */
55
122
  const TICK_INTERVAL_MS = 5 * 60_000;
56
123
  const GRAPH_HEALTH_INTERVAL_MS = 60_000;
124
+ /** Report delivery runs on its own timer, not the five-minute dispatch tick: a
125
+ * report an operator is waiting on must not sit in the outbox for the length of
126
+ * a poll interval, and delivery is owed even while claiming is paused (#123). */
127
+ const REPORT_DELIVERY_INTERVAL_MS = 30_000;
57
128
  const DEFAULT_PORT = 8787;
58
129
  const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
59
130
 
@@ -75,16 +146,95 @@ export interface DaemonOpts {
75
146
 
76
147
  /** Everything one tick touches, resolved once at startup so a tick never
77
148
  * re-reads config mid-flight and changes its own limits underneath itself. */
149
+ /**
150
+ * The credential boundary this fleet is running behind (#125), resolved once at
151
+ * startup like every other dep so a tick cannot change its own protection.
152
+ *
153
+ * `paged` is the same one-page-per-episode gate the integrity tripwire uses: a
154
+ * host that cannot build the boundary the operator asked for holds *every*
155
+ * candidate on *every* tick, and paging per five minutes forever is paging
156
+ * nobody reads.
157
+ */
158
+ export interface FleetBoundary {
159
+ isolation: CredentialIsolation;
160
+ probe: HostProbe;
161
+ slots: SlotPool;
162
+ paged: boolean;
163
+ }
164
+
165
+ /**
166
+ * True when the operator asked for a boundary this host cannot build.
167
+ *
168
+ * Asks {@link mechanismSatisfies} rather than testing for `none`, because the
169
+ * interesting refusal is the near miss: a host offering `group-mode` against a
170
+ * `per-run` config used to dispatch happily under the weaker boundary, which is
171
+ * the silent downgrade #125 forbids.
172
+ */
173
+ export function boundaryRefusesDispatch(boundary: FleetBoundary | undefined): boolean {
174
+ return boundary !== undefined && !mechanismSatisfies(boundary.isolation, boundary.probe.mechanism);
175
+ }
176
+
78
177
  interface Deps {
79
178
  project: ProjectConfig;
80
179
  caps: Caps;
180
+ /**
181
+ * Optional only so a test can build a `Deps` without a host probe; production
182
+ * always resolves one in `runDaemon`. Absent means the A1 shape — sessions
183
+ * are still child processes, but they run as the daemon's own user and no
184
+ * security claim is made for them.
185
+ */
186
+ boundary?: FleetBoundary;
81
187
  tracker: Tracker;
82
188
  store: Store;
189
+ /** Provider-reported plan allowance, cached with a TTL. Resolved once at
190
+ * startup like every other dep so a tick cannot swap its own meter. */
191
+ usage: UsageSource;
83
192
  escalate(e: Escalation): Promise<void>;
84
193
  turnLimits: TurnLimitRegistry;
85
194
  integrity: IntegrityGate;
86
195
  stall: StallGate;
87
196
  cleanup?: RetainedCleanupCursor;
197
+ /**
198
+ * Reads the connecting uid off a verb socket (#126). Resolved once at startup
199
+ * so the mechanism is logged before the first socket exists; absent on a host
200
+ * exposing no peer-credential call, where the `0600` socket under the
201
+ * daemon-owned `0711` parent is the whole boundary and `status` says so.
202
+ */
203
+ verbPeerReader?: PeerReader;
204
+ /**
205
+ * The privileged half the verbs call once their checks pass. Optional only so
206
+ * a test can build a `Deps` without a repository; `runDaemon` always wires
207
+ * the real one.
208
+ */
209
+ verbActions?: VerbActions;
210
+ }
211
+
212
+ /**
213
+ * The daemon-side view a verb call decides against (#126).
214
+ *
215
+ * `project` is a thunk, not the resolved value the rest of a tick uses. That is
216
+ * the whole "fail closed on an unreadable config" rule: a config read once at
217
+ * boot can never become unreadable, and it cannot pick up an operator who has
218
+ * just taken merge authority back either. Every verb pays one file read for
219
+ * the property that its answer reflects the config as it is *now*.
220
+ *
221
+ * `fleetStop` is read the same way and for the same reason — inside the verb,
222
+ * after the model decided to call it. `hold` and `halt` both set the pause
223
+ * sentinel alongside disarming ticks, so this one read covers both.
224
+ */
225
+ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbActions">): VerbDeps {
226
+ return {
227
+ project: () => findProject(loadConfig(), d.project.name),
228
+ store: d.store,
229
+ tracker: d.tracker,
230
+ actions: d.verbActions ?? githubVerbActions(d.project),
231
+ fleetStop: () =>
232
+ isPaused()
233
+ ? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
234
+ : undefined,
235
+ log,
236
+ now: () => Date.now(),
237
+ };
88
238
  }
89
239
 
90
240
  // ---------------------------------------------------------------- paths & pause
@@ -387,16 +537,30 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
387
537
  * `checkIntegrity` is — this wording is the whole thing a human acts on, so it
388
538
  * is worth a test holding it, and the sha in it is the only pointer to work
389
539
  * that no longer has any other copy.
540
+ *
541
+ * `retained` is not cosmetic. These lines used to promise a tree "kept for
542
+ * inspection" unconditionally, which was true only because salvage ran only on
543
+ * the paths that keep one. A blocked run's tree is removed the moment its work
544
+ * is safely on the branch, and sending an operator to a path this process just
545
+ * deleted is the same class of mistake as #118 itself.
390
546
  */
391
- export function salvageLines(outcome: SalvageOutcome, worktree: string): string[] {
392
- const kept = `Worktree kept for inspection: ${worktree}`;
547
+ export function salvageLines(
548
+ outcome: SalvageOutcome,
549
+ worktree: string,
550
+ retained: boolean,
551
+ ): string[] {
552
+ const fate = retained
553
+ ? `Worktree kept for inspection: ${worktree}`
554
+ : `Worktree removed: ${worktree}`;
393
555
 
394
- if (outcome.kind === "nothing") return [`${kept} — nothing uncommitted to salvage`];
556
+ if (outcome.kind === "nothing") return [`${fate} — nothing uncommitted to salvage`];
395
557
 
396
558
  if (outcome.kind === "failed") {
397
559
  return [
398
560
  `WIP SALVAGE FAILED: ${outcome.error}`,
399
- `Uncommitted work in ${worktree} is the only copy of it, and the next attempt removes that tree.`,
561
+ `Uncommitted work in ${worktree} is the only copy of it, so the tree was kept.`,
562
+ "This issue is held out of dispatch until the tree is recovered by hand and",
563
+ "`omp-conductor unblock <n> --force` records that you accepted it.",
400
564
  ];
401
565
  }
402
566
 
@@ -415,35 +579,144 @@ export function salvageLines(outcome: SalvageOutcome, worktree: string): string[
415
579
  : `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
416
580
  outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
417
581
  }`;
418
- return [where, manifest, kept];
582
+ return [where, manifest, fate];
583
+ }
584
+
585
+ /** Everything a settled run has to record and say about its worktree. */
586
+ export interface WorktreeSettlement {
587
+ outcome: SalvageOutcome;
588
+ /** Whether the tree still exists now the run is over. */
589
+ retained: boolean;
590
+ /** Escalation lines naming where the work went. */
591
+ lines: string[];
592
+ /** Row fields recording the durable ref, or the failure that blocks a re-claim. */
593
+ patch: Pick<RunRecord, "salvageSha" | "salvageError">;
419
594
  }
420
595
 
421
596
  /**
422
- * Commits and pushes whatever a dead run left uncommitted, logs the outcome and
423
- * returns the escalation lines that say where that work now lives.
597
+ * Decides what becomes of a finished run's worktree: save the work, then keep
598
+ * or remove the tree, then say which.
424
599
  *
425
- * Only ever called on a non-graceful end a cap kill, a crashed session, a
426
- * dispatch error. A `blocked` run stopped on purpose, with turns still in hand
427
- * and a brief that tells it to report rather than push, so nothing is committed
428
- * behind its back. The rest never got the chance: the kill is external and
429
- * lands mid-edit, in the tree the next attempt removes `--force`.
600
+ * One function because the two halves are one decision and splitting them is
601
+ * how #118 happened the removal at the end of dispatch had no idea whether
602
+ * anything had been saved, and the salvage at the top of the failure branch had
603
+ * no idea the blocked branch fell through to a `--force` removal.
604
+ *
605
+ * A salvage that *fails* retains the tree whatever the caller asked for. There
606
+ * was real work, git refused to commit it, and the tree is now the only copy in
607
+ * existence: deleting it on schedule would be the data loss this whole path
608
+ * exists to prevent. The issue is held out of dispatch until an operator says
609
+ * otherwise, because the next attempt's `worktree remove --force` would finish
610
+ * the job (see `admitCandidates`).
611
+ *
612
+ * Exported so a test can drive the real decision against a real git tree.
430
613
  */
431
- async function salvage(
432
- issue: number,
433
- attempt: number,
434
- reason: string,
435
- worktree: string,
436
- ): Promise<string[]> {
437
- const lines = salvageLines(await salvageWip(worktree, issue, attempt, reason), worktree);
614
+ export async function settleWorktree(
615
+ args: {
616
+ issue: number;
617
+ attempt: number;
618
+ /** Clause for the commit subject: "killed by the turns cap", "blocked …". */
619
+ ending: string;
620
+ worktree: string;
621
+ /** The run's branch, so the pre-removal publish names the right ref. */
622
+ branch: string;
623
+ /**
624
+ * Publishes the run branch on the privileged side. Required rather than
625
+ * optional: since #125 the run's commits live in a repository of its own,
626
+ * so a removal that did not publish first would delete the only copy —
627
+ * which is #121's data loss with one extra step. `undefined` is a visible
628
+ * decision at the call site, never an omission.
629
+ */
630
+ publish: RunPublisher | undefined;
631
+ } & (
632
+ | /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
633
+ { tree: "keep" }
634
+ | { tree: "remove"; mirrorPath: string }
635
+ ),
636
+ ): Promise<WorktreeSettlement> {
637
+ const { issue, attempt, ending, worktree, branch, publish } = args;
638
+ const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
639
+ const retained = args.tree === "keep" || outcome.kind === "failed";
640
+ if (!retained && args.tree === "remove") {
641
+ // Before the removal, always — not only when salvage found something. A run
642
+ // that *committed* and could not publish has its work in its own repository
643
+ // and nowhere else, and salvage never sees a committed tree because it is
644
+ // clean. The mirror fetch inside `publish` is what preserves it; the push
645
+ // to GitHub can fail (no network, protected ref) and the work still lives.
646
+ const published = await publish?.(branch);
647
+ if (published !== undefined && !published.ok) {
648
+ log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
649
+ }
650
+ await removeWorktree(args.mirrorPath, worktree);
651
+ }
652
+
653
+ const lines = salvageLines(outcome, worktree, retained);
438
654
  log(`#${issue} salvage: ${lines.join(" ")}`);
439
- return lines;
655
+ return {
656
+ outcome,
657
+ retained,
658
+ lines,
659
+ patch:
660
+ outcome.kind === "salvaged"
661
+ ? { salvageSha: outcome.sha }
662
+ : outcome.kind === "failed"
663
+ ? { salvageError: outcome.error }
664
+ : {},
665
+ };
440
666
  }
441
667
 
442
- /** How a run's end is named — in the salvage commit, and to whoever reads it. */
668
+ /**
669
+ * The probe a session gets when no boundary was resolved at all — a test, or a
670
+ * `--once` run built without one. Named rather than inlined so the "no
671
+ * mechanism, no claim" shape is one object every call site shares.
672
+ */
673
+ const NO_BOUNDARY_PROBE: HostProbe = {
674
+ mechanism: "none",
675
+ reasons: ["no host probe was run"],
676
+ residuals: [],
677
+ boundingSetDropped: false,
678
+ slots: [],
679
+ };
680
+
681
+ /**
682
+ * Credential paths the macOS profile denies outright.
683
+ *
684
+ * Derived from the operator's real home rather than from the session's
685
+ * redirected one: the point is that the *operator's* `gh` config and keys stay
686
+ * unreachable even when a session names their absolute path, which is exactly
687
+ * the probe `GH_CONFIG_DIR=<operator home>/.config/gh gh auth status` in #125.
688
+ */
689
+ function credentialDenyRoots(): string[] {
690
+ const home = homedir();
691
+ return [join(home, ".ssh"), join(home, ".config", "gh"), join(home, ".gnupg"), join(home, ".aws")];
692
+ }
693
+
694
+ function credentialDenyFiles(): string[] {
695
+ const home = homedir();
696
+ return [
697
+ join(home, ".git-credentials"),
698
+ join(home, ".npmrc"),
699
+ join(home, ".netrc"),
700
+ // Both git config locations, because a token does not only live in
701
+ // `.git-credentials`: `url.https://<token>@github.com/.insteadOf` is a
702
+ // perfectly ordinary way to wire one up, and an ordinary `.gitconfig` is
703
+ // 0644. Hardening chmods it, but hardening is not the claim — this list is
704
+ // what the empirical recheck actually asks the slot principal about, so a
705
+ // file missing from here is a credential nobody verified was out of reach.
706
+ join(home, ".gitconfig"),
707
+ join(home, ".config", "git", "config"),
708
+ ];
709
+ }
710
+
711
+ /**
712
+ * How a run's end is named — in the salvage commit, and to whoever reads it.
713
+ * The whole clause, not a bare reason: a graceful block was not killed by
714
+ * anything, and the commit subject is read during recovery.
715
+ */
443
716
  function endedBy(killedBy: KilledBy | undefined): string {
444
- if (killedBy === "turns") return "the turns cap";
445
- if (killedBy === "wallclock") return "the wall-clock cap";
446
- return "a failed run";
717
+ if (killedBy === "turns") return "killed by the turns cap";
718
+ if (killedBy === "wallclock") return "killed by the wall-clock cap";
719
+ return "killed by a failed run";
447
720
  }
448
721
 
449
722
  /**
@@ -460,7 +733,7 @@ export async function buildBrief(
460
733
  r: Routed,
461
734
  branch: string,
462
735
  worktree: string,
463
- opts: { continuation?: boolean; defaultBranch?: string } = {},
736
+ opts: { continuation?: boolean; defaultBranch?: string; salvagedSha?: string } = {},
464
737
  ): Promise<string> {
465
738
  // Read per dispatch rather than caching: editing the brief then takes effect
466
739
  // on the next issue instead of needing a daemon restart.
@@ -474,6 +747,14 @@ export async function buildBrief(
474
747
  "",
475
748
  `You are **resuming** issue #${r.issue.number}. Branch \`${branch}\` already exists`,
476
749
  "and was reattached with prior commits (and possibly a salvaged WIP tip).",
750
+ ...(opts.salvagedSha === undefined
751
+ ? []
752
+ : [
753
+ "",
754
+ `The previous attempt's uncommitted work was preserved for you as commit`,
755
+ `\`${opts.salvagedSha}\` on this branch. It is the tip you are continuing from,`,
756
+ "and it is the only copy of that work — do not reset past it or force-push over it.",
757
+ ]),
477
758
  "Before writing anything:",
478
759
  "",
479
760
  "```bash",
@@ -587,6 +868,33 @@ export async function verifyPushedGreenClaim(
587
868
  };
588
869
  }
589
870
 
871
+ /**
872
+ * Audit a worker's own account of its work against the pull request it pushed.
873
+ *
874
+ * The thin half of the split #85 established: this fetches, {@link
875
+ * analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
876
+ * shares none of its authority — that function decides a run's state, this one
877
+ * cannot, by construction. It returns evidence and the caller appends it.
878
+ *
879
+ * Returns undefined when the audit could not run at all: no PR to read, or a
880
+ * tracker that could not produce a diff. Undefined is *not* an empty flag list.
881
+ * An empty list means "read the whole diff, found nothing"; undefined means
882
+ * "did not look", and the report says so rather than presenting silence as a
883
+ * clean bill.
884
+ */
885
+ export async function collectSettlementFlags(
886
+ tracker: Pick<Tracker, "prDiff">,
887
+ claim: { prUrl?: string; report: string; issueText: string },
888
+ ): Promise<{ flags: SettlementFlag[]; truncated: boolean } | undefined> {
889
+ if (claim.prUrl === undefined) return undefined;
890
+ const diff = await tracker.prDiff(claim.prUrl);
891
+ if (diff === undefined) return undefined;
892
+ return {
893
+ flags: analyseSettlement({ report: claim.report, issueText: claim.issueText, diff }),
894
+ truncated: diff.truncated,
895
+ };
896
+ }
897
+
590
898
  /**
591
899
  * One attempt at one issue, from claim to terminal state. Everything is inside
592
900
  * a single try/catch so that a bad issue costs its own run and nothing else.
@@ -599,11 +907,35 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
599
907
 
600
908
  let claimed = false;
601
909
  let run: RunRecord | undefined;
602
- // Hoisted out of the try so the catch path can still name the tree: a crash
603
- // mid-dispatch is one of the non-graceful ends whose uncommitted work has to
604
- // be salvaged too, and it is the path least likely to have committed first.
910
+ // Hoisted out of the try so the catch path can still name and save the tree:
911
+ // a crash mid-dispatch is one of the ends whose uncommitted work has to be
912
+ // salvaged too, and it is the path least likely to have committed first.
913
+ const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
605
914
  let worktreePath: string | undefined;
606
915
  let turnLimit: TurnLimitController | undefined;
916
+ // #125: the run's own repository, the slot it holds, and the boundary its
917
+ // session runs behind. Hoisted for the same reason `worktreePath` is — the
918
+ // catch and finally paths have to release the slot and publish the branch.
919
+ let runRepo: RunRepoRef | undefined;
920
+ let slotIndex: number | undefined;
921
+ let boundary: SessionBoundary | undefined;
922
+ // #126: the run's own verb socket. Hoisted like the slot, because the catch
923
+ // and finally paths have to close it — a socket outliving its run is a
924
+ // channel nobody is authenticating any more.
925
+ let verbListener: VerbListener | undefined;
926
+ const credentials = resolveCredentials(project);
927
+
928
+ /**
929
+ * Publishes the run's branch on the privileged side: run repo → mirror →
930
+ * GitHub, fast-forward only. The worker holds no credential and never
931
+ * performs a network git operation itself (#125 item 3); this is the only
932
+ * route its commits take out, and it is also what stops a per-run repository
933
+ * from being the *only* copy when the tree is removed.
934
+ */
935
+ const publish: RunPublisher = async () => {
936
+ if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
937
+ return pushRunBranch(project, runRepo);
938
+ };
607
939
 
608
940
  try {
609
941
  // Claim on the tracker FIRST, before any local work. The label — not the
@@ -612,6 +944,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
612
944
  // issue out, and a human decides what to do with the orphan.
613
945
  await tracker.addLabel(issue, inProgress);
614
946
  claimed = true;
947
+ // Read before this attempt's own row exists, so `latestRun` still means the
948
+ // attempt whose work this one inherits.
949
+ const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
950
+
615
951
 
616
952
  run = store.createRun({
617
953
  project: project.name,
@@ -629,15 +965,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
629
965
  const runId = run.id;
630
966
  turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
631
967
 
632
- // A run's tree is <workspaceRoot>/<issue> and addWorktree refuses to reuse
968
+ // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
633
969
  // an existing path, so a retry — or a tree kept from a failed attempt — has
634
970
  // to be cleared first. Both helpers are pure path math and removeWorktree
635
971
  // tolerates a mirror or tree that is not there yet, so this is safe on a
636
- // first attempt. addWorktree does its own ensureMirror; calling it here too
972
+ // first attempt. addRunRepo does its own ensureMirror; calling it here too
637
973
  // would cost a second network fetch per attempt.
638
- const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
639
974
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
640
- const provisioned = await addWorktree(
975
+ const provisioned = await addRunRepo(
641
976
  r.repo,
642
977
  project.mirrorRoot,
643
978
  project.workspaceRoot,
@@ -645,12 +980,157 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
645
980
  branch,
646
981
  );
647
982
  worktreePath = provisioned.path;
983
+ runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
648
984
 
649
985
  // The SDK names the transcript itself, so the daemon supplies the parent
650
986
  // directory and learns the real path back from the result. Inventing one
651
987
  // here would put a file that never gets written into an escalation.
652
- const sessionDir = join(stateDir(), "sessions");
988
+ //
989
+ // Per run rather than one shared directory: under `uid-pool` the session
990
+ // writes its own transcript as its slot principal, and a directory shared
991
+ // with every other run would let each read the others'. The daemon still
992
+ // reads all of them — it owns the group.
993
+ //
994
+ // Under `uid-pool` the session runs as a *different uid*, so every tree it
995
+ // has to reach lives outside the private state directory: that one stays
996
+ // 0700 because it holds conductor.db and the WAL SQLite keeps recreating,
997
+ // and making it searchable to let a slot in would publish fleet history to
998
+ // every local account. `sandbox-exec` and `none` keep the old paths — same
999
+ // uid, nothing to traverse into (#125).
1000
+ const runTreeRoot = d.boundary?.probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
1001
+ if (runTreeRoot !== stateDir()) {
1002
+ mkdirSync(runTreeRoot, { recursive: true });
1003
+ chmodSync(runTreeRoot, 0o711);
1004
+ }
1005
+ const sessionDir = join(runTreeRoot, "sessions", `run-${String(runId)}`);
1006
+ const envRoot = join(runTreeRoot, "boundaries", `run-${String(runId)}`);
653
1007
  mkdirSync(sessionDir, { recursive: true });
1008
+
1009
+ // ---- the credential boundary for this run (#125) --------------------
1010
+ slotIndex = d.boundary?.slots.acquire();
1011
+ boundary = buildSessionBoundary({
1012
+ probe: d.boundary?.probe ?? NO_BOUNDARY_PROBE,
1013
+ isolation: d.boundary?.isolation ?? "none",
1014
+ role: "worker",
1015
+ ...(slotIndex === undefined ? {} : { slot: slotIndex }),
1016
+ envRoot,
1017
+ writeRoots: [worktreePath, sessionDir],
1018
+ // The workspace root, denied wholesale and then re-granted for this run's
1019
+ // own checkout by the trailing allow in `sandboxProfile`. On `uid-pool`
1020
+ // the modes already do this; on macOS the profile is the only thing that
1021
+ // does, because sandbox-exec does not change the uid.
1022
+ //
1023
+ // `mirrorRoot` is deliberately NOT denied. The run repository borrows its
1024
+ // objects from the mirror through git alternates, so denying it would
1025
+ // break every git command in the checkout — and a run being able to read
1026
+ // another run's objects out of that shared store is the residual this
1027
+ // design accepts and documents, not one it pretends to have closed. The
1028
+ // mirror stays unwritable, because it is not in `writeRoots`.
1029
+ denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot],
1030
+ denyReadFiles: credentialDenyFiles(),
1031
+ ...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
1032
+ });
1033
+
1034
+ if (boundary.principal !== undefined) {
1035
+ // Ownership handoff, then proof. `applyRunOwnership` makes the tree
1036
+ // `<slot>:conductor-daemon 2770`, which is also the point of no return:
1037
+ // if this daemon is not an *effective* member of that group it has just
1038
+ // locked itself out of push, salvage and reclaim, and the modes look
1039
+ // perfectly correct while it does. So it writes, reads and removes a
1040
+ // probe file rather than inspecting them (#125).
1041
+ // Both gids come from the probe's own `getent`, never from the daemon's
1042
+ // `egid`. They are different groups: the shipped unit runs
1043
+ // `User=fleet`/`Group=fleet` with `conductor-daemon` merely
1044
+ // supplementary, so `egid` is `fleet` — handing that to
1045
+ // `applyRunOwnership` would make every run repo `<slot>:fleet`, readable
1046
+ // by anyone else in `fleet`, while the probe had validated a group the
1047
+ // modes never used. A uid-pool probe always resolves both, so their
1048
+ // absence here is a contradiction rather than a default to paper over.
1049
+ const { daemonGid, runsGid } = d.boundary?.probe ?? {};
1050
+ if (daemonGid === undefined || runsGid === undefined) {
1051
+ throw new Error(
1052
+ `the credential boundary reported a per-run principal without resolving ` +
1053
+ `${CONDUCTOR_GROUPS.daemon}/${CONDUCTOR_GROUPS.runs} gids — refusing to hand the tree to a group ` +
1054
+ `nobody verified.`,
1055
+ );
1056
+ }
1057
+ applyRunOwnership(worktreePath, boundary.principal, daemonGid);
1058
+ applyRunOwnership(sessionDir, boundary.principal, daemonGid);
1059
+ // The session's own HOME, TMPDIR and gh-config tree.
1060
+ // `prepareSessionEnvRoot` creates them daemon-owned 0700, so without this
1061
+ // the slot principal cannot write its own home and the harness fails at
1062
+ // session startup — which reads like a broken install rather than a
1063
+ // permissions handoff that was one directory short.
1064
+ applyRunOwnership(envRoot, boundary.principal, daemonGid);
1065
+ // The claim, checked rather than assumed. On Linux the principal
1066
+ // separates by DAC alone, so whether the operator's credentials are
1067
+ // actually out of reach depends on modes this package never set. Ask the
1068
+ // slot itself, and refuse the dispatch if the answer is yes.
1069
+ const leaking = await credentialReadRefusal(
1070
+ spawnCaptured,
1071
+ boundary.principal,
1072
+ { boundingSet: d.boundary?.probe.boundingSetDropped ?? false },
1073
+ [...credentialDenyRoots(), ...credentialDenyFiles()],
1074
+ );
1075
+ if (leaking !== undefined) throw new Error(leaking);
1076
+ // The other half of the layout, and it is not optional: the run's git dir
1077
+ // borrows the mirror's objects through `alternates`, so a mirror the slot
1078
+ // principal cannot read makes the checkout look corrupt rather than
1079
+ // forbidden.
1080
+ //
1081
+ // Both trees are checked first, and the check REFUSES rather than
1082
+ // widening the state directory — see `mirrorTraversalRefusal`. A slot
1083
+ // that cannot reach its own checkout is just as broken as one that cannot
1084
+ // read the mirror, and the shipped defaults put both under the private
1085
+ // root.
1086
+ const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
1087
+ for (const tree of [mirrorPath, worktreePath]) {
1088
+ const unreachable = mirrorTraversalRefusal(tree, stateDir());
1089
+ if (unreachable !== undefined) throw new Error(unreachable);
1090
+ }
1091
+ applyMirrorReadAccess(mirrorPath, runsGid);
1092
+ const denied = verifyDaemonAccess(worktreePath);
1093
+ if (denied !== undefined) throw new Error(denied);
1094
+ }
1095
+
1096
+ // An MCP server carrying its own PAT re-opens the hole the principal
1097
+ // closes, so this is a refusal and not a warning. Audited here, after
1098
+ // provisioning, because both roots the harness will discover — the run's
1099
+ // own checkout and the agent principal's config root — exist only now.
1100
+ const mcp = mcpRefusal(
1101
+ auditMcpRoots({ agentHome: boundary.env["HOME"] ?? homedir(), cwd: worktreePath }),
1102
+ );
1103
+ if (mcp !== undefined) throw new Error(mcp);
1104
+
1105
+ // ---- the run's mutation channel (#126) -------------------------------
1106
+ // A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
1107
+ // per-run *directory*: a directory owned by the run principal would hand
1108
+ // back the power to unlink a sibling's socket and bind an impostor in its
1109
+ // place, which is the whole thing the layout buys. `listenVerbChannel`
1110
+ // validates every component before binding and throws rather than
1111
+ // degrading, so a tampered path refuses this dispatch instead of running
1112
+ // the worker with a channel nobody can vouch for.
1113
+ verbListener = await listenVerbChannel(
1114
+ verbDeps(d),
1115
+ {
1116
+ kind: "run",
1117
+ // `runTreeRoot`, not the private state dir: the slot has to *connect*
1118
+ // to this socket, and a 0700 ancestor makes it unreachable however
1119
+ // correct the socket's own mode is. `ensureVerbSocketDir` still owns
1120
+ // the parent and still runs its anti-rebind checks there (#126).
1121
+ path: verbSocketPath(ensureVerbSocketDir(runTreeRoot), `run-${String(issue)}`),
1122
+ project: project.name,
1123
+ role: "worker",
1124
+ runId,
1125
+ issue,
1126
+ repo: r.repo,
1127
+ runRepoPath: worktreePath,
1128
+ branch,
1129
+ ...(boundary.principal === undefined ? {} : { principal: boundary.principal }),
1130
+ },
1131
+ { ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
1132
+ );
1133
+
654
1134
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
655
1135
 
656
1136
  log(
@@ -664,13 +1144,27 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
664
1144
  brief: await buildBrief(project, r, branch, worktreePath, {
665
1145
  continuation: provisioned.reattached,
666
1146
  defaultBranch: r.repo.defaultBranch,
1147
+ ...(provisioned.reattached && priorSalvage !== undefined
1148
+ ? { salvagedSha: priorSalvage }
1149
+ : {}),
667
1150
  }),
668
1151
  cwd: worktreePath,
669
1152
  caps,
670
1153
  maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
671
1154
  sessionDir,
1155
+ boundary,
1156
+ // Inside the run's own boundary root, so a slot principal can reach it
1157
+ // by traversal. A socket under the daemon's 0700 home would be
1158
+ // unreachable to the very process it exists for (#125).
1159
+ // Derived from the hoisted envRoot for the same reason, and because
1160
+ // that tree has already been handed to the slot principal.
1161
+ socketPath: join(envRoot, "ipc.sock"),
1162
+ verbSocketPath: verbListener.path,
1163
+ onChildLog: (line) => {
1164
+ log(`#${issue} ${line}`);
1165
+ },
672
1166
  ...(project.workerModel === undefined ? {} : { model: project.workerModel }),
673
- releasePolicy: resolveReleasePolicy(project),
1167
+ releaseGrants: resolveReleaseGrants(project),
674
1168
  onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
675
1169
  onTurn: (n) => store.updateRun(runId, { turns: n }),
676
1170
  onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
@@ -689,6 +1183,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
689
1183
  // PR verification or terminal row writes can leave stale `running` state.
690
1184
  turnLimit?.close();
691
1185
  turnLimit = undefined;
1186
+ // The slot IS the principal, so holding it past the session means the
1187
+ // next run could be handed the same uid while this one's tree still
1188
+ // exists — two live runs able to write each other's checkout, which is
1189
+ // the cross-run property this whole change buys.
1190
+ if (slotIndex !== undefined) {
1191
+ d.boundary?.slots.release(slotIndex);
1192
+ slotIndex = undefined;
1193
+ }
692
1194
  }
693
1195
 
694
1196
  // A configured model the harness could not honour means this run was done by
@@ -703,8 +1205,67 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
703
1205
  ? await verifyPushedGreenClaim(tracker, result)
704
1206
  : { state: result.state };
705
1207
  const state = verified.state;
706
- const finalReport =
707
- verified.reason === undefined ? result.report : `${verified.reason}\n\n${result.report}`;
1208
+
1209
+ // The other half of not believing a worker about its own run (#128). The
1210
+ // claim being audited is `state: pushed-green`, so the audit runs on the
1211
+ // worker's claim rather than on what verification made of it: a claim that
1212
+ // GitHub then contradicted is exactly the report whose `changed:` line is
1213
+ // worth reading twice. Advisory throughout — nothing below reads `flags`
1214
+ // when deciding `state`, and the ordering here makes that checkable.
1215
+ const audit =
1216
+ result.state === "pushed-green"
1217
+ ? await collectSettlementFlags(tracker, {
1218
+ prUrl: result.prUrl,
1219
+ report: result.report,
1220
+ issueText: `${r.issue.title}\n${r.issue.body}`,
1221
+ })
1222
+ : undefined;
1223
+ if (result.state === "pushed-green") {
1224
+ if (audit === undefined) log(`#${issue} settlement audit skipped: no readable PR diff`);
1225
+ else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
1226
+ }
1227
+ const auditLines =
1228
+ audit === undefined ? [] : formatSettlementFlags(audit.flags, { truncated: audit.truncated });
1229
+
1230
+ const finalReport = [
1231
+ ...(verified.reason === undefined ? [] : [verified.reason, ""]),
1232
+ ...(auditLines.length === 0 ? [] : [...auditLines, ""]),
1233
+ result.report,
1234
+ ].join("\n");
1235
+
1236
+ // What becomes of the tree, decided once, before any label or page. A
1237
+ // `pushed-*` run is the only end that does not salvage: its deliverable is
1238
+ // already on a remote branch, whatever is left loose in the tree is by the
1239
+ // worker's own account not part of it, and appending a WIP commit would
1240
+ // turn the green PR this daemon just verified red. Every other end is
1241
+ // continuable, so its tree is treated as work.
1242
+ const settlement =
1243
+ state === "pushed-green" || state === "pushed-pending" || state === "merged"
1244
+ ? undefined
1245
+ : await settleWorktree({
1246
+ issue,
1247
+ attempt,
1248
+ ending:
1249
+ state === "blocked" ? "blocked for an operator decision" : endedBy(result.killedBy),
1250
+ worktree: worktreePath,
1251
+ branch,
1252
+ publish,
1253
+ ...(state === "failed" || state === "killed"
1254
+ ? ({ tree: "keep" } as const)
1255
+ : ({ tree: "remove", mirrorPath } as const)),
1256
+ });
1257
+ if (settlement === undefined) {
1258
+ // A `pushed-*` end does not salvage — its deliverable is already on a
1259
+ // remote branch — but its repository still goes away here, so the branch
1260
+ // is published first for the same reason `settleWorktree` does it: the
1261
+ // run repo is no longer a view of the mirror, and anything it holds that
1262
+ // never reached the mirror dies with the directory.
1263
+ const published = await publish(branch);
1264
+ if (!published.ok) {
1265
+ log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
1266
+ }
1267
+ await removeWorktree(mirrorPath, worktreePath);
1268
+ }
708
1269
 
709
1270
  store.updateRun(runId, {
710
1271
  state,
@@ -715,8 +1276,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
715
1276
  headSha: result.headSha,
716
1277
  sessionFile: result.sessionFile,
717
1278
  ...(verified.reason === undefined ? {} : { lastError: verified.reason }),
1279
+ ...settlement?.patch,
1280
+ ...(audit === undefined || audit.flags.length === 0
1281
+ ? {}
1282
+ : { settlementFlags: audit.flags }),
718
1283
  });
719
1284
 
1285
+ const salvaged = settlement?.lines ?? [];
1286
+
720
1287
  if (state === "blocked") {
721
1288
  await swapLabel(tracker, issue, inProgress, project.stateLabels.blocked);
722
1289
  await safeEscalate(d, {
@@ -725,7 +1292,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
725
1292
  issue,
726
1293
  runId,
727
1294
  summary: `#${issue} is blocked on attempt ${attempt} and needs a decision`,
728
- detail: [`${r.issue.title}`, r.issue.url, "", result.report].join("\n"),
1295
+ detail: [`${r.issue.title}`, r.issue.url, "", ...salvaged, "", result.report].join("\n"),
729
1296
  });
730
1297
  } else if (state === "failed" || state === "killed") {
731
1298
  // A turns cap consumes the independent continuation budget, not an
@@ -735,7 +1302,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
735
1302
  const continueTurns =
736
1303
  result.killedBy === "turns" &&
737
1304
  hasContinuationBudget(continuation, caps.maxContinuationsPerIssue);
738
- const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
739
1305
 
740
1306
  if (continueTurns) {
741
1307
  await tracker.removeLabel(issue, inProgress);
@@ -789,13 +1355,45 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
789
1355
  // A verified or still-pending PR keeps the in-progress label until its
790
1356
  // checks or merge settle, preventing another worker from duplicating it.
791
1357
  log(`#${issue} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
792
- }
793
-
794
- // A failed or killed tree is evidencekeep it. Anything else is just
795
- // disk, and the mirror means re-provisioning is cheap. (The kept tree is
796
- // wiped by the next attempt, not left to accumulate forever.)
797
- if (state !== "failed" && state !== "killed") {
798
- await removeWorktree(mirrorPath, worktreePath);
1358
+ // A green run's report is the only one nothing ever delivers: it raises no
1359
+ // escalation, and the row it settles into says nothing about what the
1360
+ // worker wrote. That is fine while the report is believedand it is
1361
+ // exactly wrong once something has stopped believing it. A flagged
1362
+ // settlement is the one case where a `pushed-green` run has news, and it
1363
+ // has it *now*, while the PR is still open and nobody has merged it (#128).
1364
+ //
1365
+ // Tier 1, because the judgement is the orchestrator's: it can read the
1366
+ // diff, it can ask, and it can promote to a human. The state is
1367
+ // deliberately unchanged either way — the run below this line settles
1368
+ // `pushed-green` whether it was flagged or not.
1369
+ //
1370
+ // Findings only. A diff too large to read in full is noted in the log and
1371
+ // in the report, but it is not news for a human: paging "0 flags, could
1372
+ // not read it all" is exactly the kind of line that teaches an
1373
+ // orchestrator to skim past this escalation the next time it carries one.
1374
+ const found = audit?.flags ?? [];
1375
+ if (found.length > 0) {
1376
+ await safeEscalate(d, {
1377
+ tier: 1,
1378
+ project: project.name,
1379
+ issue,
1380
+ runId,
1381
+ summary:
1382
+ `#${issue} settled ${state} on attempt ${attempt} with ` +
1383
+ `${found.length} settlement audit flag(s)`,
1384
+ detail: [
1385
+ `${r.issue.title}`,
1386
+ r.issue.url,
1387
+ result.prUrl ?? "(no PR URL)",
1388
+ "",
1389
+ "The run is NOT blocked and its state is unchanged. These are advisory",
1390
+ "findings about the worker's own account of its work — judge them, then",
1391
+ "merge, ask, or close as you would have anyway.",
1392
+ "",
1393
+ finalReport,
1394
+ ].join("\n"),
1395
+ });
1396
+ }
799
1397
  }
800
1398
  } catch (err) {
801
1399
  // Dispatch setup can fail after the controller opens but before runWorker's
@@ -804,8 +1402,34 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
804
1402
  turnLimit = undefined;
805
1403
  const detail = errText(err);
806
1404
  log(`#${issue} errored: ${detail}`);
1405
+ // Released here as well: a dispatch that failed before `runWorker` never
1406
+ // reached the `finally` above, and a leaked slot permanently shrinks the
1407
+ // pool until the daemon restarts.
1408
+ if (slotIndex !== undefined) {
1409
+ d.boundary?.slots.release(slotIndex);
1410
+ slotIndex = undefined;
1411
+ }
1412
+ // A crash lands anywhere, including mid-edit in a tree holding the only
1413
+ // copy of real work. Nothing else on this path so much as looks at it.
1414
+ const settlement =
1415
+ worktreePath === undefined
1416
+ ? undefined
1417
+ : await settleWorktree({
1418
+ issue,
1419
+ attempt,
1420
+ ending: "killed by a dispatch error",
1421
+ worktree: worktreePath,
1422
+ branch,
1423
+ publish,
1424
+ tree: "keep",
1425
+ });
807
1426
  if (run) {
808
- store.updateRun(run.id, { state: "failed", endedAt: Date.now(), lastError: detail });
1427
+ store.updateRun(run.id, {
1428
+ state: "failed",
1429
+ endedAt: Date.now(),
1430
+ lastError: detail,
1431
+ ...settlement?.patch,
1432
+ });
809
1433
  }
810
1434
  if (claimed) {
811
1435
  // Leaving the issue stuck as in-progress would hide it from both the
@@ -816,13 +1440,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
816
1440
  log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
817
1441
  }
818
1442
  }
819
- // A crash lands anywhere, including mid-edit in a tree holding the only
820
- // copy of real work. Nothing else on this path so much as looks at it.
821
- const salvaged =
822
- worktreePath === undefined
823
- ? []
824
- : await salvage(issue, attempt, "a dispatch error", worktreePath);
825
1443
 
1444
+ const salvaged = settlement?.lines ?? [];
826
1445
  await safeEscalate(d, {
827
1446
  tier: 1,
828
1447
  project: project.name,
@@ -835,6 +1454,18 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
835
1454
  // a failure path, and whatever it still held is now a commit on the branch.
836
1455
  } finally {
837
1456
  turnLimit?.close();
1457
+ // The run is over, so its channel is too. Closed here rather than beside
1458
+ // the session so the crash path closes it as well: a listener left bound
1459
+ // after its run settled is a socket whose `run-not-live` check is the only
1460
+ // thing standing between a stale child and a push.
1461
+ if (verbListener !== undefined) {
1462
+ try {
1463
+ await verbListener.close();
1464
+ } catch (err) {
1465
+ log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
1466
+ }
1467
+ verbListener = undefined;
1468
+ }
838
1469
  }
839
1470
  }
840
1471
 
@@ -1178,6 +1809,74 @@ export function summarizeDispatch(
1178
1809
  };
1179
1810
  }
1180
1811
 
1812
+ /**
1813
+ * What a held plan-usage gate says to a human, if anything.
1814
+ *
1815
+ * Three different problems hide behind one hold, and they want different
1816
+ * tiers. Reaching the threshold is the guard *working*: tier 1, because the
1817
+ * fleet resumes on its own at the provider's reset and nobody needs to get
1818
+ * out of bed. Everything else — a window nothing reports, a window that
1819
+ * resolves to two allowances, a meter that has been unreadable for half an
1820
+ * hour — is dispatch stopped with no self-recovery, which is tier 2.
1821
+ *
1822
+ * Each summary carries the fact that will change when the situation does (the
1823
+ * reset instant, the configured id, the date), because the escalation ledger
1824
+ * dedupes on the summary: a stable one pages once and then goes quiet, which
1825
+ * is right for a repeated tick and wrong for the next window.
1826
+ */
1827
+ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation | undefined {
1828
+ const base = { project, issue: NO_ISSUE };
1829
+ if (plan.state === "at-cap") {
1830
+ const window = plan.window?.id ?? plan.cap?.windowId ?? "the configured window";
1831
+ const resets =
1832
+ plan.resetsAt === undefined
1833
+ ? new Date().toISOString().slice(0, 10)
1834
+ : new Date(plan.resetsAt).toISOString();
1835
+ return {
1836
+ ...base,
1837
+ tier: 1,
1838
+ summary: `Plan allowance cap reached on ${window} — ${project} is not claiming new work (window ${resets})`,
1839
+ detail: [
1840
+ plan.detail,
1841
+ "Running workers finish normally; only new claims are held.",
1842
+ "Dispatch resumes by itself once the provider reports the window reset or usage below the threshold —",
1843
+ "no `resume` needed. Raise `caps.planUsage.maxUsedFraction` only if you mean to spend the rest.",
1844
+ ].join("\n"),
1845
+ };
1846
+ }
1847
+ if (plan.state === "blind") {
1848
+ return {
1849
+ ...base,
1850
+ tier: 2,
1851
+ // Dated: a meter that breaks again next month is a new incident, not a
1852
+ // repeat of this one.
1853
+ summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
1854
+ detail: [
1855
+ plan.detail,
1856
+ "The guard admitted work while the failure looked transient and has now stopped.",
1857
+ "Check `omp usage --json` on the fleet host, or set `caps.planUsage` to null if this fleet is unmetered.",
1858
+ ].join("\n"),
1859
+ };
1860
+ }
1861
+ if (
1862
+ plan.state === "window-missing" ||
1863
+ plan.state === "window-ambiguous" ||
1864
+ plan.state === "window-uncomparable"
1865
+ ) {
1866
+ return {
1867
+ ...base,
1868
+ tier: 2,
1869
+ summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
1870
+ detail: [
1871
+ plan.detail,
1872
+ "Run `omp usage --json` and copy an allowance `id` into `caps.planUsage.windowId`,",
1873
+ "or set `caps.planUsage` to null if this fleet is unmetered.",
1874
+ ].join("\n"),
1875
+ };
1876
+ }
1877
+ return undefined;
1878
+ }
1879
+
1181
1880
  /**
1182
1881
  * Which routed candidates get a worker this tick — in queue order, never more
1183
1882
  * than `slots` of them. Every non-admission receives a stable reason code.
@@ -1192,7 +1891,7 @@ export function summarizeDispatch(
1192
1891
  * that grows a field has no business breaking these tests.
1193
1892
  */
1194
1893
  export async function admitCandidates(
1195
- d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
1894
+ d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "boundary">,
1196
1895
  routed: Routed[],
1197
1896
  slots: number,
1198
1897
  ): Promise<AdmissionPass> {
@@ -1204,6 +1903,50 @@ export async function admitCandidates(
1204
1903
  holds.push({ issue, reason });
1205
1904
  };
1206
1905
 
1906
+ // The fail-closed the issue asks for, and it only ever fires because the
1907
+ // operator asked for a boundary by name. Placed above every other gate,
1908
+ // including the tracker calls: a fleet that must not dispatch must not spend
1909
+ // API budget discovering it, and it must not claim an issue it will refuse.
1910
+ //
1911
+ // Deliberately NOT a downgrade to group-mode. That is a different, weaker
1912
+ // claim, and silently substituting it is how an operator ends up believing
1913
+ // they have a boundary they do not have (#125).
1914
+ if (boundaryRefusesDispatch(d.boundary) && d.boundary !== undefined) {
1915
+ const refusal = boundaryRefusal(d.boundary.probe);
1916
+ for (const r of routed) hold(r.issue.number, "credential-boundary");
1917
+ log(`credential boundary holding ${String(routed.length)} candidate(s): ${refusal}`);
1918
+ if (!d.boundary.paged) {
1919
+ d.boundary.paged = true;
1920
+ await safeEscalate(d, {
1921
+ tier: 2,
1922
+ project: project.name,
1923
+ issue: NO_ISSUE,
1924
+ summary: `${project.name} cannot dispatch: credentials.isolation is "per-run" and this host offers no mechanism`,
1925
+ detail: [refusal, "", "Nothing is dispatched until this is resolved. No work has been claimed."].join("\n"),
1926
+ });
1927
+ }
1928
+ return { admitted: [], holds };
1929
+ }
1930
+
1931
+ // The plan allowance is a fleet-wide question, so it is asked once per pass
1932
+ // and answers for every candidate — unlike every gate below it, which is
1933
+ // per-issue. It sits here rather than beside the spend cap in `tick` for one
1934
+ // reason: the spend cap *pauses the daemon* and waits for a human, and a
1935
+ // weekly plan window resets by itself. A guard that demanded `resume` after
1936
+ // every rollover would cost more operator attention than the guard saves
1937
+ // (#110). Already-running workers are untouched and settle normally.
1938
+ //
1939
+ // Placed after the cheap local busy-set read and before the first tracker
1940
+ // call, so a held fleet spends no GitHub API budget discovering it is held.
1941
+ const plan = await readPlanUsage(caps.planUsage, d.usage);
1942
+ if (plan.blocking) {
1943
+ for (const r of routed) hold(r.issue.number, "plan-usage-cap");
1944
+ log(`plan usage gate holding ${String(routed.length)} candidate(s): ${plan.detail}`);
1945
+ const escalation = planUsageEscalation(project.name, plan);
1946
+ if (escalation !== undefined) await safeEscalate(d, escalation);
1947
+ return { admitted: [], holds };
1948
+ }
1949
+
1207
1950
  // parent -> blocking issue. Seeded from active runs (including pushed-green),
1208
1951
  // then extended by candidates admitted earlier in this same pass so two
1209
1952
  // siblings never both clear the gate in one tick.
@@ -1278,6 +2021,32 @@ export async function admitCandidates(
1278
2021
  continue;
1279
2022
  }
1280
2023
 
2024
+ // Fail closed on work that exists only in a run repo. `addRunRepo` clears
2025
+ // the tree at <workspaceRoot>/<issue> before it provisions, so admitting
2026
+ // this issue is what finally destroys the copy the salvage could not save
2027
+ // (#118). Nothing here can recover it — git already refused once — so the
2028
+ // only safe move is to refuse the claim and keep saying why until an
2029
+ // operator has looked and run `unblock --force`.
2030
+ const newest = store.latestRun(project.name, issue);
2031
+ if (newest?.salvageError !== undefined && newest.salvageAckAt === undefined) {
2032
+ hold(issue, "unsalvaged-wip");
2033
+ await safeEscalate(d, {
2034
+ tier: 1,
2035
+ project: project.name,
2036
+ issue,
2037
+ summary: `#${issue} is holding unsalvaged work and will not be re-claimed`,
2038
+ detail: [
2039
+ r.issue.title,
2040
+ r.issue.url,
2041
+ `Attempt ${newest.attempt} could not commit its uncommitted changes: ${newest.salvageError}`,
2042
+ `The only copy is the worktree ${newest.worktree === "" ? "(path not recorded)" : newest.worktree}.`,
2043
+ "Dispatch is held because claiming this issue removes that tree.",
2044
+ "Recover it by hand, then `omp-conductor unblock <n> --force` to release the hold.",
2045
+ ].join("\n"),
2046
+ });
2047
+ continue;
2048
+ }
2049
+
1281
2050
  // Soft concurrency per epic: at most one in-flight child of a given parent.
1282
2051
  // No parent means today's concurrent admission. Cheap local filters already
1283
2052
  // ran; this sits before the open-PR API call so a held sibling frees the
@@ -1666,25 +2435,83 @@ export async function turnLimitResponse(
1666
2435
  );
1667
2436
  }
1668
2437
 
2438
+ export interface DaemonHttpDeps {
2439
+ project: string;
2440
+ store: Pick<Store, "latestRun">;
2441
+ turnLimits: TurnLimitRegistry;
2442
+ health: () => DaemonHealthSnapshot;
2443
+ }
2444
+
2445
+ /**
2446
+ * The whole HTTP surface, in one named function so a test can pin what is *not*
2447
+ * on it.
2448
+ *
2449
+ * Two routes: a health read, and the turn-limit control. Everything else is
2450
+ * 404, and that is the contract — see the prohibition at the `Bun.serve` call
2451
+ * for why no mutation may be added here. Extracted from the serve callback
2452
+ * precisely so "the daemon's HTTP port exposes no mutation route" is something
2453
+ * a test asserts rather than something a reviewer has to notice (#126).
2454
+ */
2455
+ export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
2456
+ const url = new URL(req.url);
2457
+ if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
2458
+ const control = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
2459
+ return control ?? new Response("not found\n", { status: 404 });
2460
+ }
2461
+
1669
2462
  export interface StatusSnapshot {
1670
2463
  project: string;
1671
2464
  configPath: string;
1672
2465
  stateDir: string;
1673
2466
  paused: boolean;
1674
2467
  caps: Caps;
2468
+ /**
2469
+ * The effective per-shape release grants. On the snapshot rather than re-read
2470
+ * by each renderer because #122 began with a grant nobody had looked at in
2471
+ * weeks: `status` names them so a stale one is visible without opening either
2472
+ * the config or the brief.
2473
+ */
2474
+ releaseGrants: ResolvedGrants;
1675
2475
  /** Occupied issues: live workers plus green PRs awaiting a human merge. */
1676
2476
  activeRuns: RunRecord[];
2477
+ /** Newest attempts holding a preserved WIP tip, or a tree that is still the
2478
+ * only copy of work the daemon could not save. */
2479
+ salvagedRuns: RunRecord[];
2480
+ /** Reports the operator has not provably received: pending, in-flight with an
2481
+ * unknown outcome, or written off. An empty list is the only honest way to
2482
+ * say "everything authored this cycle actually went out" (#123). */
2483
+ openReports: ReportRecord[];
2484
+ /**
2485
+ * The most recent conductor-verb calls and how the daemon decided them
2486
+ * (#126). On `status` rather than only behind `omp-conductor ledger` because
2487
+ * a refused merge is news: it means a session tried to do something the
2488
+ * config does not let it, and an operator who has to know to go looking is an
2489
+ * operator who finds out from the tracker instead.
2490
+ */
2491
+ verbLedger: VerbLedgerEntry[];
1677
2492
  /** Runs backed by a worker process — the number capacity compares against. */
1678
2493
  liveWorkers: number;
1679
2494
  runsToday: number;
1680
2495
  spendTodayUsd: number;
1681
2496
  dispatch?: DispatchSummary;
2497
+ /**
2498
+ * Latest plan-allowance verdict, when the caller read one. Optional because
2499
+ * this snapshot is built synchronously off the store while the provider read
2500
+ * is I/O: a renderer that did not do that work must say "not read" rather
2501
+ * than print a percentage nobody measured (#110).
2502
+ */
2503
+ planUsage?: PlanUsageStatus;
1682
2504
  }
1683
2505
 
1684
2506
  /** Builds a status reading from an already-open store. Long-lived operator
1685
2507
  * surfaces use this path so a one-second refresh does not repeatedly open and
1686
2508
  * initialise SQLite connections. */
1687
- export function statusSnapshotFromStore(p: ProjectConfig, caps: Caps, store: Store): StatusSnapshot {
2509
+ export function statusSnapshotFromStore(
2510
+ p: ProjectConfig,
2511
+ caps: Caps,
2512
+ store: Store,
2513
+ planUsage?: PlanUsageStatus,
2514
+ ): StatusSnapshot {
1688
2515
  const since = startOfToday();
1689
2516
  const dispatch = store.latestDispatch(p.name);
1690
2517
  return {
@@ -1693,11 +2520,16 @@ export function statusSnapshotFromStore(p: ProjectConfig, caps: Caps, store: Sto
1693
2520
  stateDir: stateDir(),
1694
2521
  paused: isPaused(),
1695
2522
  caps,
2523
+ releaseGrants: resolveReleaseGrants(p),
1696
2524
  activeRuns: store.activeRuns(p.name),
2525
+ salvagedRuns: store.salvagedRuns(p.name),
2526
+ openReports: store.openReports(p.name),
2527
+ verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
1697
2528
  liveWorkers: store.liveRuns(p.name).length,
1698
2529
  runsToday: store.runsStartedSince(p.name, since),
1699
2530
  spendTodayUsd: store.spendSince(p.name, since),
1700
2531
  ...(dispatch === undefined ? {} : { dispatch }),
2532
+ ...(planUsage === undefined ? {} : { planUsage }),
1701
2533
  };
1702
2534
  }
1703
2535
 
@@ -1737,6 +2569,50 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
1737
2569
  return lines.join("\n");
1738
2570
  }
1739
2571
 
2572
+ /**
2573
+ * The WIP block: every issue whose newest attempt left work behind, and
2574
+ * whether that work is safe.
2575
+ *
2576
+ * Blocked runs used to be invisible here, which is exactly how #118 stayed
2577
+ * invisible for a full attempt cycle — the operator saw a blocked issue and had
2578
+ * no way to tell "stopped with 34 uncommitted files" from "stopped clean".
2579
+ * A preserved line is informational; an UNSALVAGED line is an alarm, and it
2580
+ * names the directory because that directory is the work.
2581
+ */
2582
+ export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
2583
+ if (runs.length === 0) return [];
2584
+ const lines = ["", "wip"];
2585
+ for (const r of runs) {
2586
+ lines.push(
2587
+ r.salvageError !== undefined && r.salvageAckAt === undefined
2588
+ ? ` #${r.issue} UNSALVAGED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
2589
+ `only copy, dispatch held (${r.salvageError})`
2590
+ : r.salvageError !== undefined
2591
+ ? ` #${r.issue} accepted as lost attempt ${r.attempt} (${r.salvageError})`
2592
+ : ` #${r.issue} preserved ${r.salvageSha ?? "?"} on ${r.branch} (attempt ${r.attempt}, ${r.state})`,
2593
+ );
2594
+ }
2595
+ return lines;
2596
+ }
2597
+
2598
+ /**
2599
+ * The grant table, named shape by shape.
2600
+ *
2601
+ * One renderer for both status surfaces (this one and the fleet view), because
2602
+ * #122 began with a POLICY grant that no longer matched anyone's intent and went
2603
+ * unnoticed: a stale grant has to be visible from `status` alone, and two
2604
+ * renderers would eventually show it in only one of them.
2605
+ */
2606
+ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
2607
+ const granted = RELEASE_SHAPES.filter((shape) => grants[shape] === "orchestrator");
2608
+ return [
2609
+ granted.length === 0
2610
+ ? "release no shape granted — every release/deploy tool call is blocked"
2611
+ : `release granted to the orchestrator: ${granted.join(", ")}`,
2612
+ ...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
2613
+ ];
2614
+ }
2615
+
1740
2616
  export function formatStatus(s: StatusSnapshot): string {
1741
2617
  const lines = [
1742
2618
  `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
@@ -1749,11 +2625,17 @@ export function formatStatus(s: StatusSnapshot): string {
1749
2625
  s.caps.dailySpendUsd === null
1750
2626
  ? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
1751
2627
  : ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
2628
+ // Its own row, never folded into the spend row: they are two independent
2629
+ // controls, and an operator has to be able to see which one stopped the
2630
+ // fleet (#110).
2631
+ ` plan usage ${planUsageLine(s.planUsage)}`,
1752
2632
  ` new worker turns ${s.caps.workerMaxTurns}`,
1753
2633
  ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
1754
2634
  ` failed attempts ${s.caps.maxAttemptsPerIssue}`,
1755
2635
  ` continuations ${s.caps.maxContinuationsPerIssue}`,
1756
2636
  "",
2637
+ ...formatReleaseGrants(s.releaseGrants),
2638
+ "",
1757
2639
  formatDispatchSummary(s.dispatch),
1758
2640
  "",
1759
2641
  ];
@@ -1767,8 +2649,16 @@ export function formatStatus(s: StatusSnapshot): string {
1767
2649
  `${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
1768
2650
  (r.prUrl ? ` ${r.prUrl}` : ""),
1769
2651
  );
2652
+ // Its escalation was deduplicated the moment it was delivered, so this
2653
+ // line is the only place a flagged run stays visible while its PR waits
2654
+ // for a merge — which is precisely the window the flag is about (#128).
2655
+ const flagged = settlementFlagSummary(r.settlementFlags);
2656
+ if (flagged !== undefined) lines.push(` ${flagged}`);
1770
2657
  }
1771
2658
  }
2659
+ lines.push(...formatSalvagedRuns(s.salvagedRuns));
2660
+ lines.push(...formatOpenReports(s.openReports));
2661
+ lines.push(...formatVerbLedger(s.verbLedger));
1772
2662
  // Deploy hint: a restart while workers are live orphans them (salvage runs
1773
2663
  // first — #35). Prefer pause + drain to zero live workers when you can wait.
1774
2664
  if (s.liveWorkers > 0) {
@@ -1866,6 +2756,13 @@ export function prepareConductor(): void {
1866
2756
  export async function reconcileOrphanedRuns(
1867
2757
  store: Store,
1868
2758
  project: string,
2759
+ /**
2760
+ * Resolves the privileged publisher for one orphaned run. Optional because a
2761
+ * test driving the row transitions has no repo to publish to; production
2762
+ * always passes it, and without it a salvaged WIP commit stays local — which
2763
+ * is the half of #121 that reaches a human.
2764
+ */
2765
+ publish?: (run: RunRecord) => RunPublisher,
1869
2766
  ): Promise<RunRecord[]> {
1870
2767
  // Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
1871
2768
  // a process dying — it is finished work waiting on a human merge.
@@ -1873,12 +2770,25 @@ export async function reconcileOrphanedRuns(
1873
2770
  const endedAt = Date.now();
1874
2771
  for (const r of stale) {
1875
2772
  // Salvage before the row flips: the worktree path is on the record, and
1876
- // salvageWip is a no-op for a missing/clean tree. Reason string matches
1877
- // the cap-kill wording so triage reads the same either way.
1878
- if (r.worktree !== "") {
1879
- await salvage(r.issue, r.attempt, "a daemon restart", r.worktree);
1880
- }
1881
- store.updateRun(r.id, { state: "orphaned", endedAt });
2773
+ // salvageWip is a no-op for a missing/clean tree. The clause matches the
2774
+ // cap-kill wording so triage reads the same either way, and the tree is
2775
+ // kept because an orphan's remains are the orchestrator's drain-duty call.
2776
+ const settlement =
2777
+ r.worktree === ""
2778
+ ? undefined
2779
+ : await settleWorktree({
2780
+ issue: r.issue,
2781
+ attempt: r.attempt,
2782
+ ending: "killed by a daemon restart",
2783
+ worktree: r.worktree,
2784
+ branch: r.branch,
2785
+ // An orphan's tree is kept, so its commits are not about to be
2786
+ // deleted — but a WIP salvage still has to reach GitHub, which is
2787
+ // #121's whole point and is now the daemon's hop to make.
2788
+ publish: publish?.(r),
2789
+ tree: "keep",
2790
+ });
2791
+ store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
1882
2792
  }
1883
2793
  return stale;
1884
2794
  }
@@ -1886,12 +2796,63 @@ export async function reconcileOrphanedRuns(
1886
2796
  // ------------------------------------------------------------------- the daemon
1887
2797
 
1888
2798
  export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
2799
+ // Before anything reads the config for real: a file written before #125 gains
2800
+ // an explicit `credentials.isolation` here, so the answer lives on disk
2801
+ // rather than being inherited from a default nobody chose. Non-fatal by
2802
+ // design — a config the daemon cannot rewrite still loads as `"none"` and
2803
+ // still dispatches, because an outage caused by a security feature is worse
2804
+ // than the day before it shipped.
2805
+ try {
2806
+ const migration = migrateCredentialsOnDisk();
2807
+ if (migration.migrated.length > 0) {
2808
+ log(
2809
+ `migrated ${migration.migrated.join(", ")} to an explicit credentials.isolation: "none" in ` +
2810
+ `${migration.path} — this fleet is UNPROTECTED: sessions run as this daemon's user and can reach ` +
2811
+ `its GitHub credential. Set credentials.isolation to "per-run" once the host is provisioned (README).`,
2812
+ );
2813
+ }
2814
+ } catch (err) {
2815
+ log(`could not persist the credentials migration (${errText(err)}) — continuing with isolation "none"`);
2816
+ }
2817
+
1889
2818
  const cfg = loadConfig();
1890
2819
  const project = findProject(cfg, o.project);
1891
2820
  const caps = resolveCaps(project, cfg.defaults);
1892
2821
  const store = openStore(dbPath());
1893
2822
  const tracker = makeTracker(project);
1894
2823
 
2824
+ // The host capability probe, once, at startup. It states the mechanism
2825
+ // rather than guessing, and it proves the launcher works against a trivial
2826
+ // child before reporting one available — a mechanism whose argv the host
2827
+ // rejects looks identical here and then fails every dispatch at run time.
2828
+ const credentials = resolveCredentials(project);
2829
+ const probe = await probeHost({ slots: caps.maxConcurrentWorkers });
2830
+ const boundary: FleetBoundary = {
2831
+ isolation: credentials.isolation,
2832
+ probe,
2833
+ slots: createSlotPool(caps.maxConcurrentWorkers),
2834
+ paged: false,
2835
+ };
2836
+ const described = describeBoundary(credentials.isolation, probe);
2837
+ log(`credential boundary: ${described.headline}`);
2838
+ for (const line of described.detail) log(`credential boundary: ${line}`);
2839
+
2840
+ // #126's transport, stated at startup rather than guessed at first use. The
2841
+ // banner names what this host can actually enforce — whether the kernel will
2842
+ // vouch for a caller's uid, and whether runs get distinct principals at all —
2843
+ // because "peer credentials asserted" is a claim, and a claim nobody printed
2844
+ // is one nobody can check against the host it is running on.
2845
+ const verbPeerReader = peerCredentialReader();
2846
+ const verbActions = githubVerbActions(project);
2847
+ let orchestratorVerbs: VerbListener | undefined;
2848
+ log(
2849
+ `verb transport: ${transportBanner(
2850
+ ensureVerbSocketDir(stateDir()),
2851
+ verbPeerReader,
2852
+ probe.mechanism === "uid-pool",
2853
+ )}`,
2854
+ );
2855
+
1895
2856
  // Recorded here, before a single tick runs, so that the deploy an operator
1896
2857
  // *means* to do never trips the tripwire: installing a new build and
1897
2858
  // restarting the unit re-records this from the new files. What it catches is
@@ -1906,7 +2867,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1906
2867
  // daemon must not orphan that daemon's real, live workers).
1907
2868
  const alive = livingDaemon();
1908
2869
  if (alive === undefined || alive.pid === process.pid) {
1909
- for (const r of await reconcileOrphanedRuns(store, project.name)) {
2870
+ // The orphan path salvages, and #121 requires that salvage to reach
2871
+ // GitHub. Since the run's commits now live in its own repository, the hop
2872
+ // is the daemon's: resolve the routed repo by name off the run row.
2873
+ const orphanPublisher = (r: RunRecord): RunPublisher => {
2874
+ const repo = project.routing.repos[r.repo];
2875
+ return async (branch) =>
2876
+ repo === undefined
2877
+ ? { ok: false, stderr: `run ${String(r.id)} names repo "${r.repo}", which this project no longer routes` }
2878
+ : pushRunBranch(project, { repo, runRepoPath: r.worktree, branch });
2879
+ };
2880
+ for (const r of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
1910
2881
  log(
1911
2882
  `#${r.issue} orphaned by a previous daemon (attempt ${r.attempt}, was ${r.state}, worktree ${r.worktree}) — ` +
1912
2883
  `slot freed; the ${project.stateLabels.inProgress} label stays until the orchestrator triages what the worker left`,
@@ -1919,6 +2890,9 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1919
2890
  // Standing orders. The orchestrator holds none of this file's context, so
1920
2891
  // everything it needs to act — which tracker, which labels, what the fleet
1921
2892
  // does — has to be said once, in words.
2893
+ const releaseGrants = resolveReleaseGrants(project);
2894
+ const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
2895
+ const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
1922
2896
  const brief = [
1923
2897
  `You are the omp-conductor orchestrator for project "${project.name}".`,
1924
2898
  `Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
@@ -1939,10 +2913,14 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1939
2913
  "a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
1940
2914
  : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
1941
2915
  "human merges.",
1942
- `Release tool gate: releasePolicy=${resolveReleasePolicy(project)}. ` +
1943
- (resolveReleasePolicy(project) === "none"
1944
- ? "Release and deploy tool calls are mechanically blocked."
1945
- : "Release and deploy tool calls are permitted only by the operator brief."),
2916
+ // Named shape by shape rather than as one policy word, so a stale grant is
2917
+ // legible in the transcript instead of only in the config file — the #122
2918
+ // incident began with a grant that no longer matched anyone's intent.
2919
+ `Release tool gate: ${
2920
+ grantedShapes.length === 0
2921
+ ? "every release and deploy shape is mechanically blocked for you"
2922
+ : `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
2923
+ }.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
1946
2924
  "Handle each escalation below before the next one.",
1947
2925
  ].join("\n");
1948
2926
 
@@ -1960,10 +2938,87 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1960
2938
  log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
1961
2939
  } else {
1962
2940
  try {
2941
+ // Its own principal, distinct from every worker slot. The orchestrator
2942
+ // reads the state directory and its briefs and must have no read or
2943
+ // write access to any run checkout — so `workspaceRoot` and the mirror
2944
+ // are denied outright here, on top of the #127 tool-layer jail. Two
2945
+ // layers because they fail differently: the jail refuses a structured
2946
+ // tool call, the principal refuses the syscall `bash` would make.
2947
+ // Same rule as a worker's, for the same reason: under `uid-pool` the
2948
+ // orchestrator is its own uid, so every tree it must reach lives outside
2949
+ // the 0700 private state directory. Its principal is in `conductor-runs`
2950
+ // and deliberately NOT in the daemon group, so it could not traverse
2951
+ // there even once — and widening the state directory is the one fix that
2952
+ // is not available, because that is where conductor.db lives.
2953
+ const orchTreeRoot = probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
2954
+ if (orchTreeRoot !== stateDir()) {
2955
+ mkdirSync(orchTreeRoot, { recursive: true });
2956
+ chmodSync(orchTreeRoot, 0o711);
2957
+ }
2958
+ const orchEnvRoot = join(orchTreeRoot, "boundaries", "orchestrator");
2959
+ const orchCwd = join(orchTreeRoot, "orchestrator");
2960
+ mkdirSync(orchCwd, { recursive: true });
2961
+ const orchestratorBoundary = buildSessionBoundary({
2962
+ probe,
2963
+ isolation: credentials.isolation,
2964
+ role: "orchestrator",
2965
+ envRoot: orchEnvRoot,
2966
+ writeRoots: [orchCwd],
2967
+ denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot, project.mirrorRoot],
2968
+ denyReadFiles: credentialDenyFiles(),
2969
+ ...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
2970
+ });
2971
+ if (orchestratorBoundary.principal !== undefined) {
2972
+ const gid = probe.daemonGid;
2973
+ if (gid === undefined) {
2974
+ throw new Error(
2975
+ `the orchestrator principal was allocated without a resolved ${CONDUCTOR_GROUPS.daemon} gid`,
2976
+ );
2977
+ }
2978
+ applyRunOwnership(orchEnvRoot, orchestratorBoundary.principal, gid);
2979
+ applyRunOwnership(orchCwd, orchestratorBoundary.principal, gid);
2980
+ // The same empirical gate a worker gets. The orchestrator is the
2981
+ // session with merge and release authority, so "it cannot reach the
2982
+ // credential" matters more here, not less — and it was the half that
2983
+ // was never checked. `sharedGroup: false` because it is launched with
2984
+ // no supplementary group at all, so the probe has to run under exactly
2985
+ // that identity rather than a worker's.
2986
+ const orchLeaking = await credentialReadRefusal(
2987
+ spawnCaptured,
2988
+ orchestratorBoundary.principal,
2989
+ { boundingSet: probe.boundingSetDropped, sharedGroup: false },
2990
+ [...credentialDenyRoots(), ...credentialDenyFiles()],
2991
+ );
2992
+ if (orchLeaking !== undefined) throw new Error(orchLeaking);
2993
+ }
2994
+ // A third socket, distinct from every run's, in the same daemon-owned
2995
+ // 0711 parent. This is what makes "merge authority is the orchestrator's"
2996
+ // a property of the channel: the daemon knows which session is speaking
2997
+ // because of where the connection arrived, and no payload can move a
2998
+ // worker's call onto this one (#126).
2999
+ orchestratorVerbs = await listenVerbChannel(
3000
+ verbDeps({ project, store, tracker, verbActions }),
3001
+ {
3002
+ kind: "orchestrator",
3003
+ path: verbSocketPath(ensureVerbSocketDir(orchTreeRoot), "orchestrator"),
3004
+ project: project.name,
3005
+ role: "orchestrator",
3006
+ ...(orchestratorBoundary.principal === undefined
3007
+ ? {}
3008
+ : { principal: orchestratorBoundary.principal }),
3009
+ },
3010
+ { ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
3011
+ );
1963
3012
  orchestrator = await startOrchestrator({
1964
- cwd: stateDir(),
3013
+ cwd: orchCwd,
1965
3014
  brief,
1966
- releasePolicy: resolveReleasePolicy(project),
3015
+ releaseGrants,
3016
+ boundary: orchestratorBoundary,
3017
+ socketPath: join(orchEnvRoot, "ipc.sock"),
3018
+ verbSocketPath: orchestratorVerbs.path,
3019
+ onChildLog: (line) => {
3020
+ log(`orchestrator ${line}`);
3021
+ },
1967
3022
  onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
1968
3023
  });
1969
3024
  const transcript = orchestrator.sessionFile();
@@ -1979,14 +3034,45 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1979
3034
  }
1980
3035
 
1981
3036
  const escalator = createEscalator(project, tracker, store, orchestrator);
3037
+
3038
+ // Report delivery is the daemon's, not the model's (#123). Built beside the
3039
+ // escalator because a report nobody can deliver pages through it, and driven
3040
+ // on its own timer rather than inside `tick()`: a paused fleet claims nothing
3041
+ // but still owes its operator the report it was handed, and five minutes is a
3042
+ // long time to sit on a page.
3043
+ const outbox = createReportOutbox({
3044
+ project,
3045
+ store,
3046
+ escalate: (e) => escalator.escalate(e),
3047
+ log,
3048
+ });
3049
+
3050
+ // Every row still `sending` when a daemon boots belonged to a process that is
3051
+ // gone, so its outcome will never be learned — retry it and say so in the
3052
+ // message. Guarded exactly like the orphan reconciliation above and for the
3053
+ // same reason: a row belonging to a *live* daemon is genuinely in flight, and
3054
+ // stealing it would page the operator twice for one report.
3055
+ if (alive === undefined || alive.pid === process.pid) {
3056
+ for (const r of outbox.recover(Date.now())) {
3057
+ log(
3058
+ `report ${r.id} was left mid-send by a previous daemon (attempt ${r.attempts}) — ` +
3059
+ `retrying; its message will say it may be a repeat`,
3060
+ );
3061
+ }
3062
+ }
3063
+
1982
3064
  const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
1983
3065
  store.updateRun(runId, { maxTurns });
1984
3066
  });
1985
3067
  const d: Deps = {
1986
3068
  project,
1987
3069
  caps,
3070
+ boundary,
1988
3071
  tracker,
1989
3072
  store,
3073
+ // Process-wide, so a `status` served off this daemon's own HTTP surface
3074
+ // reuses the tick's reading instead of shelling out again.
3075
+ usage: sharedUsageSource(),
1990
3076
  escalate: (e) => escalator.escalate(e),
1991
3077
  turnLimits,
1992
3078
  integrity,
@@ -1994,13 +3080,20 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1994
3080
  // page again about a stall that is still on disk.
1995
3081
  stall: { paged: false },
1996
3082
  cleanup: { next: 0 },
3083
+ ...(verbPeerReader === undefined ? {} : { verbPeerReader }),
3084
+ verbActions,
1997
3085
  };
1998
3086
 
1999
3087
  if (o.once) {
2000
3088
  try {
2001
3089
  await tick(d);
3090
+ // A single tick still owes the outbox a pass: a drill that leaves a
3091
+ // report undelivered teaches an operator the wrong thing about the
3092
+ // mechanism it is drilling.
3093
+ await outbox.deliverDue();
2002
3094
  } finally {
2003
3095
  await orchestrator?.dispose();
3096
+ await orchestratorVerbs?.close();
2004
3097
  store.close();
2005
3098
  }
2006
3099
  return;
@@ -2027,6 +3120,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
2027
3120
  refreshCodeGraph();
2028
3121
  const graphTimer = setInterval(refreshCodeGraph, GRAPH_HEALTH_INTERVAL_MS);
2029
3122
 
3123
+ // Overlap-guarded like the graph probe: a pass that is still waiting on
3124
+ // Telegram must not have a second pass started on top of it, or one report
3125
+ // would be claimed, reclaimed and sent twice by this process alone.
3126
+ let reportPass: Promise<void> | undefined;
3127
+ const drainReports = (): void => {
3128
+ if (reportPass !== undefined) return;
3129
+ reportPass = outbox
3130
+ .deliverDue()
3131
+ .then(() => {})
3132
+ .catch((err: unknown) => {
3133
+ log(`report delivery pass failed: ${errText(err)}`);
3134
+ })
3135
+ .finally(() => {
3136
+ reportPass = undefined;
3137
+ });
3138
+ };
3139
+ drainReports();
3140
+ const reportTimer = setInterval(drainReports, REPORT_DELIVERY_INTERVAL_MS);
3141
+
2030
3142
  const workers = createWorkerPool();
2031
3143
  let stopping = false;
2032
3144
  let wake: (() => void) | undefined;
@@ -2039,17 +3151,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
2039
3151
  process.on("SIGINT", stop);
2040
3152
  process.on("SIGTERM", stop);
2041
3153
 
3154
+ // ── NO MUTATION ROUTE BELONGS ON THIS PORT ────────────────────────────────
3155
+ //
3156
+ // This is unauthenticated loopback TCP. Every local user can reach it, it
3157
+ // carries no credential of any kind, and it cannot tell one caller from
3158
+ // another: `127.0.0.1` is not an identity. `turnLimitResponse` already shows
3159
+ // what that costs — it trusts a body-supplied `project`, which is exactly the
3160
+ // shape "identity from the payload" takes when nobody is watching, and it is
3161
+ // tolerable only because raising a turn ceiling is bounded and reversible.
3162
+ //
3163
+ // A merge, a push, a release or a label is none of those things. Do not add
3164
+ // one here, and do not add "just a small one" behind a shared secret either:
3165
+ // a secret readable by the process that would be attacking you is not
3166
+ // authentication. Mutations go over the per-run unix sockets in
3167
+ // `verbs/socket.ts`, where the kernel says who the caller is and the daemon
3168
+ // derives project, run and role from the channel rather than the body (#126).
2042
3169
  const server = Bun.serve({
2043
3170
  hostname: "127.0.0.1",
2044
3171
  port: o.port ?? DEFAULT_PORT,
2045
- async fetch(req) {
2046
- const url = new URL(req.url);
2047
- if (req.method === "GET" && url.pathname === "/healthz") {
2048
- return Response.json(daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph));
2049
- }
2050
- const control = await turnLimitResponse(req, project.name, store, turnLimits);
2051
- return control ?? new Response("not found\n", { status: 404 });
2052
- },
3172
+ fetch: (req) =>
3173
+ daemonHttpResponse(req, {
3174
+ project: project.name,
3175
+ store,
3176
+ turnLimits,
3177
+ health: () => daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph),
3178
+ }),
2053
3179
  });
2054
3180
  log(`serving /healthz on :${server.port}, project ${project.name}`);
2055
3181
 
@@ -2076,11 +3202,20 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
2076
3202
  process.off("SIGINT", stop);
2077
3203
  process.off("SIGTERM", stop);
2078
3204
  clearInterval(graphTimer);
3205
+ clearInterval(reportTimer);
3206
+ // Before the store closes, like the orchestrator below: a pass mid-send has
3207
+ // a `markReportDelivered` still to write, and losing that write is exactly
3208
+ // how a delivered report comes back as an ambiguous one on the next boot.
3209
+ await reportPass;
2079
3210
  await workers.drain();
2080
3211
  await server.stop(true);
2081
3212
  // Before the store closes: a queued injection that rejects on the way out
2082
3213
  // falls back to an issue comment, and that path writes the dedup marker.
2083
3214
  await orchestrator?.dispose();
3215
+ // After the session it belongs to is gone. A bound socket outliving its
3216
+ // orchestrator is a channel accepting merges for a session that no longer
3217
+ // exists.
3218
+ await orchestratorVerbs?.close();
2084
3219
  store.close();
2085
3220
  log("stopped");
2086
3221
  // A handled SIGTERM still leaves some runtimes with a non-zero default