omp-conductor 0.3.25 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +733 -30
- package/package.json +1 -1
- package/src/board.ts +24 -5
- package/src/briefs/orchestrator.md +106 -20
- package/src/briefs/policy.md +48 -27
- package/src/briefs/worker.md +82 -31
- package/src/cli.ts +155 -2
- package/src/config.ts +669 -16
- package/src/confinement.ts +506 -25
- package/src/credentials.ts +2029 -0
- package/src/daemon.ts +1000 -32
- package/src/diff-flags.ts +696 -0
- package/src/escalate.ts +136 -9
- package/src/fleet.ts +71 -3
- package/src/omp.ts +471 -15
- package/src/orchestrator-tick.ts +42 -19
- package/src/orchestrator.ts +32 -5
- package/src/plugin.ts +267 -15
- package/src/release-policy.ts +191 -29
- package/src/reports.ts +440 -0
- package/src/session-host.ts +307 -0
- package/src/setup-host.ts +19 -1
- package/src/setup.ts +207 -18
- package/src/store.ts +506 -3
- package/src/tracker/github.ts +45 -0
- package/src/types.ts +847 -10
- package/src/usage.ts +726 -0
- package/src/verbs/actions.ts +142 -0
- package/src/verbs/client.ts +207 -0
- package/src/verbs/ledger.ts +77 -0
- package/src/verbs/protocol.ts +465 -0
- package/src/verbs/server.ts +1098 -0
- package/src/verbs/socket.ts +446 -0
- package/src/worker.ts +36 -7
- package/src/worktree.ts +202 -109
- package/systemd/omp-conductor.service.example +96 -8
package/src/daemon.ts
CHANGED
|
@@ -8,9 +8,24 @@
|
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
configPath,
|
|
15
|
+
findProject,
|
|
16
|
+
loadConfig,
|
|
17
|
+
migrateCredentialsOnDisk,
|
|
18
|
+
resolveCaps,
|
|
19
|
+
resolveCredentials,
|
|
20
|
+
resolveReleaseGrants,
|
|
21
|
+
stateDir,
|
|
22
|
+
sharedRoot,
|
|
23
|
+
} from "./config.ts";
|
|
24
|
+
import {
|
|
25
|
+
analyseSettlement,
|
|
26
|
+
formatSettlementFlags,
|
|
27
|
+
settlementFlagSummary,
|
|
28
|
+
} from "./diff-flags.ts";
|
|
14
29
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
15
30
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
16
31
|
import { graphHint } from "./graph.ts";
|
|
@@ -18,11 +33,13 @@ import { livingDaemon } from "./lifecycle.ts";
|
|
|
18
33
|
import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
19
34
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
20
35
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
36
|
+
import { createReportOutbox, formatOpenReports } from "./reports.ts";
|
|
21
37
|
import { recordReleaseBlock } from "./release-policy.ts";
|
|
22
38
|
import { branchName, route } from "./routing.ts";
|
|
23
39
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
24
40
|
import { dbPath, openStore } from "./store.ts";
|
|
25
41
|
import { makeTracker } from "./tracker/github.ts";
|
|
42
|
+
import { RELEASE_SHAPES } from "./types.ts";
|
|
26
43
|
import type {
|
|
27
44
|
AdmissionHoldReason,
|
|
28
45
|
Caps,
|
|
@@ -30,30 +47,84 @@ import type {
|
|
|
30
47
|
Escalation,
|
|
31
48
|
OpenCloser,
|
|
32
49
|
PrState,
|
|
50
|
+
CredentialIsolation,
|
|
33
51
|
ProjectConfig,
|
|
34
52
|
ReadyIssue,
|
|
35
53
|
RepoTarget,
|
|
54
|
+
ReportRecord,
|
|
55
|
+
ResolvedGrants,
|
|
36
56
|
RunRecord,
|
|
37
57
|
RunState,
|
|
58
|
+
SettlementFlag,
|
|
38
59
|
Store,
|
|
39
60
|
Tracker,
|
|
61
|
+
VerbLedgerEntry,
|
|
40
62
|
} from "./types.ts";
|
|
63
|
+
import { CONDUCTOR_GROUPS } from "./types.ts";
|
|
41
64
|
import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
|
|
42
65
|
import {
|
|
43
|
-
|
|
66
|
+
addRunRepo,
|
|
44
67
|
cleanupRetainedWorktree,
|
|
45
68
|
mirrorPathFor,
|
|
46
69
|
removeWorktree,
|
|
47
70
|
salvageWip,
|
|
48
71
|
type RetainedWorktreeCleanup,
|
|
72
|
+
type RunPublisher,
|
|
49
73
|
type SalvageOutcome,
|
|
50
74
|
worktreePathFor,
|
|
51
75
|
} from "./worktree.ts";
|
|
76
|
+
import { githubVerbActions } from "./verbs/actions.ts";
|
|
77
|
+
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
78
|
+
import { listenVerbChannel } from "./verbs/server.ts";
|
|
79
|
+
import type { VerbActions, VerbDeps, VerbListener } from "./verbs/server.ts";
|
|
80
|
+
import {
|
|
81
|
+
ensureVerbSocketDir,
|
|
82
|
+
peerCredentialReader,
|
|
83
|
+
transportBanner,
|
|
84
|
+
verbSocketPath,
|
|
85
|
+
type PeerReader,
|
|
86
|
+
} from "./verbs/socket.ts";
|
|
87
|
+
import { homedir } from "node:os";
|
|
88
|
+
|
|
89
|
+
import {
|
|
90
|
+
applyRunOwnership,
|
|
91
|
+
applyMirrorReadAccess,
|
|
92
|
+
auditMcpRoots,
|
|
93
|
+
mirrorTraversalRefusal,
|
|
94
|
+
credentialReadRefusal,
|
|
95
|
+
spawnCaptured,
|
|
96
|
+
boundaryRefusal,
|
|
97
|
+
buildSessionBoundary,
|
|
98
|
+
createSlotPool,
|
|
99
|
+
describeBoundary,
|
|
100
|
+
liveDaemonCredentials,
|
|
101
|
+
mcpRefusal,
|
|
102
|
+
mechanismSatisfies,
|
|
103
|
+
openRunPr,
|
|
104
|
+
probeHost,
|
|
105
|
+
pushRunBranch,
|
|
106
|
+
verifyDaemonAccess,
|
|
107
|
+
type HostProbe,
|
|
108
|
+
type RunRepoRef,
|
|
109
|
+
type SessionBoundary,
|
|
110
|
+
type SlotPool,
|
|
111
|
+
} from "./credentials.ts";
|
|
112
|
+
import {
|
|
113
|
+
planUsageLine,
|
|
114
|
+
readPlanUsage,
|
|
115
|
+
sharedUsageSource,
|
|
116
|
+
type PlanUsageStatus,
|
|
117
|
+
type UsageSource,
|
|
118
|
+
} from "./usage.ts";
|
|
52
119
|
|
|
53
120
|
/** Long enough that the tracker is not polled raw, short enough that a human
|
|
54
121
|
* who labels an issue sees it picked up within a coffee break. */
|
|
55
122
|
const TICK_INTERVAL_MS = 5 * 60_000;
|
|
56
123
|
const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
124
|
+
/** Report delivery runs on its own timer, not the five-minute dispatch tick: a
|
|
125
|
+
* report an operator is waiting on must not sit in the outbox for the length of
|
|
126
|
+
* a poll interval, and delivery is owed even while claiming is paused (#123). */
|
|
127
|
+
const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
57
128
|
const DEFAULT_PORT = 8787;
|
|
58
129
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
59
130
|
|
|
@@ -75,16 +146,95 @@ export interface DaemonOpts {
|
|
|
75
146
|
|
|
76
147
|
/** Everything one tick touches, resolved once at startup so a tick never
|
|
77
148
|
* re-reads config mid-flight and changes its own limits underneath itself. */
|
|
149
|
+
/**
|
|
150
|
+
* The credential boundary this fleet is running behind (#125), resolved once at
|
|
151
|
+
* startup like every other dep so a tick cannot change its own protection.
|
|
152
|
+
*
|
|
153
|
+
* `paged` is the same one-page-per-episode gate the integrity tripwire uses: a
|
|
154
|
+
* host that cannot build the boundary the operator asked for holds *every*
|
|
155
|
+
* candidate on *every* tick, and paging per five minutes forever is paging
|
|
156
|
+
* nobody reads.
|
|
157
|
+
*/
|
|
158
|
+
export interface FleetBoundary {
|
|
159
|
+
isolation: CredentialIsolation;
|
|
160
|
+
probe: HostProbe;
|
|
161
|
+
slots: SlotPool;
|
|
162
|
+
paged: boolean;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* True when the operator asked for a boundary this host cannot build.
|
|
167
|
+
*
|
|
168
|
+
* Asks {@link mechanismSatisfies} rather than testing for `none`, because the
|
|
169
|
+
* interesting refusal is the near miss: a host offering `group-mode` against a
|
|
170
|
+
* `per-run` config used to dispatch happily under the weaker boundary, which is
|
|
171
|
+
* the silent downgrade #125 forbids.
|
|
172
|
+
*/
|
|
173
|
+
export function boundaryRefusesDispatch(boundary: FleetBoundary | undefined): boolean {
|
|
174
|
+
return boundary !== undefined && !mechanismSatisfies(boundary.isolation, boundary.probe.mechanism);
|
|
175
|
+
}
|
|
176
|
+
|
|
78
177
|
interface Deps {
|
|
79
178
|
project: ProjectConfig;
|
|
80
179
|
caps: Caps;
|
|
180
|
+
/**
|
|
181
|
+
* Optional only so a test can build a `Deps` without a host probe; production
|
|
182
|
+
* always resolves one in `runDaemon`. Absent means the A1 shape — sessions
|
|
183
|
+
* are still child processes, but they run as the daemon's own user and no
|
|
184
|
+
* security claim is made for them.
|
|
185
|
+
*/
|
|
186
|
+
boundary?: FleetBoundary;
|
|
81
187
|
tracker: Tracker;
|
|
82
188
|
store: Store;
|
|
189
|
+
/** Provider-reported plan allowance, cached with a TTL. Resolved once at
|
|
190
|
+
* startup like every other dep so a tick cannot swap its own meter. */
|
|
191
|
+
usage: UsageSource;
|
|
83
192
|
escalate(e: Escalation): Promise<void>;
|
|
84
193
|
turnLimits: TurnLimitRegistry;
|
|
85
194
|
integrity: IntegrityGate;
|
|
86
195
|
stall: StallGate;
|
|
87
196
|
cleanup?: RetainedCleanupCursor;
|
|
197
|
+
/**
|
|
198
|
+
* Reads the connecting uid off a verb socket (#126). Resolved once at startup
|
|
199
|
+
* so the mechanism is logged before the first socket exists; absent on a host
|
|
200
|
+
* exposing no peer-credential call, where the `0600` socket under the
|
|
201
|
+
* daemon-owned `0711` parent is the whole boundary and `status` says so.
|
|
202
|
+
*/
|
|
203
|
+
verbPeerReader?: PeerReader;
|
|
204
|
+
/**
|
|
205
|
+
* The privileged half the verbs call once their checks pass. Optional only so
|
|
206
|
+
* a test can build a `Deps` without a repository; `runDaemon` always wires
|
|
207
|
+
* the real one.
|
|
208
|
+
*/
|
|
209
|
+
verbActions?: VerbActions;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* The daemon-side view a verb call decides against (#126).
|
|
214
|
+
*
|
|
215
|
+
* `project` is a thunk, not the resolved value the rest of a tick uses. That is
|
|
216
|
+
* the whole "fail closed on an unreadable config" rule: a config read once at
|
|
217
|
+
* boot can never become unreadable, and it cannot pick up an operator who has
|
|
218
|
+
* just taken merge authority back either. Every verb pays one file read for
|
|
219
|
+
* the property that its answer reflects the config as it is *now*.
|
|
220
|
+
*
|
|
221
|
+
* `fleetStop` is read the same way and for the same reason — inside the verb,
|
|
222
|
+
* after the model decided to call it. `hold` and `halt` both set the pause
|
|
223
|
+
* sentinel alongside disarming ticks, so this one read covers both.
|
|
224
|
+
*/
|
|
225
|
+
export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbActions">): VerbDeps {
|
|
226
|
+
return {
|
|
227
|
+
project: () => findProject(loadConfig(), d.project.name),
|
|
228
|
+
store: d.store,
|
|
229
|
+
tracker: d.tracker,
|
|
230
|
+
actions: d.verbActions ?? githubVerbActions(d.project),
|
|
231
|
+
fleetStop: () =>
|
|
232
|
+
isPaused()
|
|
233
|
+
? "claiming is paused for this fleet (omp-conductor pause, hold or halt)"
|
|
234
|
+
: undefined,
|
|
235
|
+
log,
|
|
236
|
+
now: () => Date.now(),
|
|
237
|
+
};
|
|
88
238
|
}
|
|
89
239
|
|
|
90
240
|
// ---------------------------------------------------------------- paths & pause
|
|
@@ -468,16 +618,37 @@ export async function settleWorktree(
|
|
|
468
618
|
/** Clause for the commit subject: "killed by the turns cap", "blocked …". */
|
|
469
619
|
ending: string;
|
|
470
620
|
worktree: string;
|
|
621
|
+
/** The run's branch, so the pre-removal publish names the right ref. */
|
|
622
|
+
branch: string;
|
|
623
|
+
/**
|
|
624
|
+
* Publishes the run branch on the privileged side. Required rather than
|
|
625
|
+
* optional: since #125 the run's commits live in a repository of its own,
|
|
626
|
+
* so a removal that did not publish first would delete the only copy —
|
|
627
|
+
* which is #121's data loss with one extra step. `undefined` is a visible
|
|
628
|
+
* decision at the call site, never an omission.
|
|
629
|
+
*/
|
|
630
|
+
publish: RunPublisher | undefined;
|
|
471
631
|
} & (
|
|
472
632
|
| /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
|
|
473
633
|
{ tree: "keep" }
|
|
474
634
|
| { tree: "remove"; mirrorPath: string }
|
|
475
635
|
),
|
|
476
636
|
): Promise<WorktreeSettlement> {
|
|
477
|
-
const { issue, attempt, ending, worktree } = args;
|
|
478
|
-
const outcome = await salvageWip(worktree, issue, attempt, ending);
|
|
637
|
+
const { issue, attempt, ending, worktree, branch, publish } = args;
|
|
638
|
+
const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
|
|
479
639
|
const retained = args.tree === "keep" || outcome.kind === "failed";
|
|
480
|
-
if (!retained && args.tree === "remove")
|
|
640
|
+
if (!retained && args.tree === "remove") {
|
|
641
|
+
// Before the removal, always — not only when salvage found something. A run
|
|
642
|
+
// that *committed* and could not publish has its work in its own repository
|
|
643
|
+
// and nowhere else, and salvage never sees a committed tree because it is
|
|
644
|
+
// clean. The mirror fetch inside `publish` is what preserves it; the push
|
|
645
|
+
// to GitHub can fail (no network, protected ref) and the work still lives.
|
|
646
|
+
const published = await publish?.(branch);
|
|
647
|
+
if (published !== undefined && !published.ok) {
|
|
648
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
649
|
+
}
|
|
650
|
+
await removeWorktree(args.mirrorPath, worktree);
|
|
651
|
+
}
|
|
481
652
|
|
|
482
653
|
const lines = salvageLines(outcome, worktree, retained);
|
|
483
654
|
log(`#${issue} salvage: ${lines.join(" ")}`);
|
|
@@ -494,6 +665,49 @@ export async function settleWorktree(
|
|
|
494
665
|
};
|
|
495
666
|
}
|
|
496
667
|
|
|
668
|
+
/**
|
|
669
|
+
* The probe a session gets when no boundary was resolved at all — a test, or a
|
|
670
|
+
* `--once` run built without one. Named rather than inlined so the "no
|
|
671
|
+
* mechanism, no claim" shape is one object every call site shares.
|
|
672
|
+
*/
|
|
673
|
+
const NO_BOUNDARY_PROBE: HostProbe = {
|
|
674
|
+
mechanism: "none",
|
|
675
|
+
reasons: ["no host probe was run"],
|
|
676
|
+
residuals: [],
|
|
677
|
+
boundingSetDropped: false,
|
|
678
|
+
slots: [],
|
|
679
|
+
};
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Credential paths the macOS profile denies outright.
|
|
683
|
+
*
|
|
684
|
+
* Derived from the operator's real home rather than from the session's
|
|
685
|
+
* redirected one: the point is that the *operator's* `gh` config and keys stay
|
|
686
|
+
* unreachable even when a session names their absolute path, which is exactly
|
|
687
|
+
* the probe `GH_CONFIG_DIR=<operator home>/.config/gh gh auth status` in #125.
|
|
688
|
+
*/
|
|
689
|
+
function credentialDenyRoots(): string[] {
|
|
690
|
+
const home = homedir();
|
|
691
|
+
return [join(home, ".ssh"), join(home, ".config", "gh"), join(home, ".gnupg"), join(home, ".aws")];
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function credentialDenyFiles(): string[] {
|
|
695
|
+
const home = homedir();
|
|
696
|
+
return [
|
|
697
|
+
join(home, ".git-credentials"),
|
|
698
|
+
join(home, ".npmrc"),
|
|
699
|
+
join(home, ".netrc"),
|
|
700
|
+
// Both git config locations, because a token does not only live in
|
|
701
|
+
// `.git-credentials`: `url.https://<token>@github.com/.insteadOf` is a
|
|
702
|
+
// perfectly ordinary way to wire one up, and an ordinary `.gitconfig` is
|
|
703
|
+
// 0644. Hardening chmods it, but hardening is not the claim — this list is
|
|
704
|
+
// what the empirical recheck actually asks the slot principal about, so a
|
|
705
|
+
// file missing from here is a credential nobody verified was out of reach.
|
|
706
|
+
join(home, ".gitconfig"),
|
|
707
|
+
join(home, ".config", "git", "config"),
|
|
708
|
+
];
|
|
709
|
+
}
|
|
710
|
+
|
|
497
711
|
/**
|
|
498
712
|
* How a run's end is named — in the salvage commit, and to whoever reads it.
|
|
499
713
|
* The whole clause, not a bare reason: a graceful block was not killed by
|
|
@@ -654,6 +868,33 @@ export async function verifyPushedGreenClaim(
|
|
|
654
868
|
};
|
|
655
869
|
}
|
|
656
870
|
|
|
871
|
+
/**
|
|
872
|
+
* Audit a worker's own account of its work against the pull request it pushed.
|
|
873
|
+
*
|
|
874
|
+
* The thin half of the split #85 established: this fetches, {@link
|
|
875
|
+
* analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
|
|
876
|
+
* shares none of its authority — that function decides a run's state, this one
|
|
877
|
+
* cannot, by construction. It returns evidence and the caller appends it.
|
|
878
|
+
*
|
|
879
|
+
* Returns undefined when the audit could not run at all: no PR to read, or a
|
|
880
|
+
* tracker that could not produce a diff. Undefined is *not* an empty flag list.
|
|
881
|
+
* An empty list means "read the whole diff, found nothing"; undefined means
|
|
882
|
+
* "did not look", and the report says so rather than presenting silence as a
|
|
883
|
+
* clean bill.
|
|
884
|
+
*/
|
|
885
|
+
export async function collectSettlementFlags(
|
|
886
|
+
tracker: Pick<Tracker, "prDiff">,
|
|
887
|
+
claim: { prUrl?: string; report: string; issueText: string },
|
|
888
|
+
): Promise<{ flags: SettlementFlag[]; truncated: boolean } | undefined> {
|
|
889
|
+
if (claim.prUrl === undefined) return undefined;
|
|
890
|
+
const diff = await tracker.prDiff(claim.prUrl);
|
|
891
|
+
if (diff === undefined) return undefined;
|
|
892
|
+
return {
|
|
893
|
+
flags: analyseSettlement({ report: claim.report, issueText: claim.issueText, diff }),
|
|
894
|
+
truncated: diff.truncated,
|
|
895
|
+
};
|
|
896
|
+
}
|
|
897
|
+
|
|
657
898
|
/**
|
|
658
899
|
* One attempt at one issue, from claim to terminal state. Everything is inside
|
|
659
900
|
* a single try/catch so that a bad issue costs its own run and nothing else.
|
|
@@ -672,6 +913,29 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
672
913
|
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
673
914
|
let worktreePath: string | undefined;
|
|
674
915
|
let turnLimit: TurnLimitController | undefined;
|
|
916
|
+
// #125: the run's own repository, the slot it holds, and the boundary its
|
|
917
|
+
// session runs behind. Hoisted for the same reason `worktreePath` is — the
|
|
918
|
+
// catch and finally paths have to release the slot and publish the branch.
|
|
919
|
+
let runRepo: RunRepoRef | undefined;
|
|
920
|
+
let slotIndex: number | undefined;
|
|
921
|
+
let boundary: SessionBoundary | undefined;
|
|
922
|
+
// #126: the run's own verb socket. Hoisted like the slot, because the catch
|
|
923
|
+
// and finally paths have to close it — a socket outliving its run is a
|
|
924
|
+
// channel nobody is authenticating any more.
|
|
925
|
+
let verbListener: VerbListener | undefined;
|
|
926
|
+
const credentials = resolveCredentials(project);
|
|
927
|
+
|
|
928
|
+
/**
|
|
929
|
+
* Publishes the run's branch on the privileged side: run repo → mirror →
|
|
930
|
+
* GitHub, fast-forward only. The worker holds no credential and never
|
|
931
|
+
* performs a network git operation itself (#125 item 3); this is the only
|
|
932
|
+
* route its commits take out, and it is also what stops a per-run repository
|
|
933
|
+
* from being the *only* copy when the tree is removed.
|
|
934
|
+
*/
|
|
935
|
+
const publish: RunPublisher = async () => {
|
|
936
|
+
if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
|
|
937
|
+
return pushRunBranch(project, runRepo);
|
|
938
|
+
};
|
|
675
939
|
|
|
676
940
|
try {
|
|
677
941
|
// Claim on the tracker FIRST, before any local work. The label — not the
|
|
@@ -701,14 +965,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
701
965
|
const runId = run.id;
|
|
702
966
|
turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
|
|
703
967
|
|
|
704
|
-
// A run's tree is <workspaceRoot>/<issue> and
|
|
968
|
+
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
705
969
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
706
970
|
// to be cleared first. Both helpers are pure path math and removeWorktree
|
|
707
971
|
// tolerates a mirror or tree that is not there yet, so this is safe on a
|
|
708
|
-
// first attempt.
|
|
972
|
+
// first attempt. addRunRepo does its own ensureMirror; calling it here too
|
|
709
973
|
// would cost a second network fetch per attempt.
|
|
710
974
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
711
|
-
const provisioned = await
|
|
975
|
+
const provisioned = await addRunRepo(
|
|
712
976
|
r.repo,
|
|
713
977
|
project.mirrorRoot,
|
|
714
978
|
project.workspaceRoot,
|
|
@@ -716,12 +980,157 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
716
980
|
branch,
|
|
717
981
|
);
|
|
718
982
|
worktreePath = provisioned.path;
|
|
983
|
+
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
719
984
|
|
|
720
985
|
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
721
986
|
// directory and learns the real path back from the result. Inventing one
|
|
722
987
|
// here would put a file that never gets written into an escalation.
|
|
723
|
-
|
|
988
|
+
//
|
|
989
|
+
// Per run rather than one shared directory: under `uid-pool` the session
|
|
990
|
+
// writes its own transcript as its slot principal, and a directory shared
|
|
991
|
+
// with every other run would let each read the others'. The daemon still
|
|
992
|
+
// reads all of them — it owns the group.
|
|
993
|
+
//
|
|
994
|
+
// Under `uid-pool` the session runs as a *different uid*, so every tree it
|
|
995
|
+
// has to reach lives outside the private state directory: that one stays
|
|
996
|
+
// 0700 because it holds conductor.db and the WAL SQLite keeps recreating,
|
|
997
|
+
// and making it searchable to let a slot in would publish fleet history to
|
|
998
|
+
// every local account. `sandbox-exec` and `none` keep the old paths — same
|
|
999
|
+
// uid, nothing to traverse into (#125).
|
|
1000
|
+
const runTreeRoot = d.boundary?.probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
|
|
1001
|
+
if (runTreeRoot !== stateDir()) {
|
|
1002
|
+
mkdirSync(runTreeRoot, { recursive: true });
|
|
1003
|
+
chmodSync(runTreeRoot, 0o711);
|
|
1004
|
+
}
|
|
1005
|
+
const sessionDir = join(runTreeRoot, "sessions", `run-${String(runId)}`);
|
|
1006
|
+
const envRoot = join(runTreeRoot, "boundaries", `run-${String(runId)}`);
|
|
724
1007
|
mkdirSync(sessionDir, { recursive: true });
|
|
1008
|
+
|
|
1009
|
+
// ---- the credential boundary for this run (#125) --------------------
|
|
1010
|
+
slotIndex = d.boundary?.slots.acquire();
|
|
1011
|
+
boundary = buildSessionBoundary({
|
|
1012
|
+
probe: d.boundary?.probe ?? NO_BOUNDARY_PROBE,
|
|
1013
|
+
isolation: d.boundary?.isolation ?? "none",
|
|
1014
|
+
role: "worker",
|
|
1015
|
+
...(slotIndex === undefined ? {} : { slot: slotIndex }),
|
|
1016
|
+
envRoot,
|
|
1017
|
+
writeRoots: [worktreePath, sessionDir],
|
|
1018
|
+
// The workspace root, denied wholesale and then re-granted for this run's
|
|
1019
|
+
// own checkout by the trailing allow in `sandboxProfile`. On `uid-pool`
|
|
1020
|
+
// the modes already do this; on macOS the profile is the only thing that
|
|
1021
|
+
// does, because sandbox-exec does not change the uid.
|
|
1022
|
+
//
|
|
1023
|
+
// `mirrorRoot` is deliberately NOT denied. The run repository borrows its
|
|
1024
|
+
// objects from the mirror through git alternates, so denying it would
|
|
1025
|
+
// break every git command in the checkout — and a run being able to read
|
|
1026
|
+
// another run's objects out of that shared store is the residual this
|
|
1027
|
+
// design accepts and documents, not one it pretends to have closed. The
|
|
1028
|
+
// mirror stays unwritable, because it is not in `writeRoots`.
|
|
1029
|
+
denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot],
|
|
1030
|
+
denyReadFiles: credentialDenyFiles(),
|
|
1031
|
+
...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
|
|
1032
|
+
});
|
|
1033
|
+
|
|
1034
|
+
if (boundary.principal !== undefined) {
|
|
1035
|
+
// Ownership handoff, then proof. `applyRunOwnership` makes the tree
|
|
1036
|
+
// `<slot>:conductor-daemon 2770`, which is also the point of no return:
|
|
1037
|
+
// if this daemon is not an *effective* member of that group it has just
|
|
1038
|
+
// locked itself out of push, salvage and reclaim, and the modes look
|
|
1039
|
+
// perfectly correct while it does. So it writes, reads and removes a
|
|
1040
|
+
// probe file rather than inspecting them (#125).
|
|
1041
|
+
// Both gids come from the probe's own `getent`, never from the daemon's
|
|
1042
|
+
// `egid`. They are different groups: the shipped unit runs
|
|
1043
|
+
// `User=fleet`/`Group=fleet` with `conductor-daemon` merely
|
|
1044
|
+
// supplementary, so `egid` is `fleet` — handing that to
|
|
1045
|
+
// `applyRunOwnership` would make every run repo `<slot>:fleet`, readable
|
|
1046
|
+
// by anyone else in `fleet`, while the probe had validated a group the
|
|
1047
|
+
// modes never used. A uid-pool probe always resolves both, so their
|
|
1048
|
+
// absence here is a contradiction rather than a default to paper over.
|
|
1049
|
+
const { daemonGid, runsGid } = d.boundary?.probe ?? {};
|
|
1050
|
+
if (daemonGid === undefined || runsGid === undefined) {
|
|
1051
|
+
throw new Error(
|
|
1052
|
+
`the credential boundary reported a per-run principal without resolving ` +
|
|
1053
|
+
`${CONDUCTOR_GROUPS.daemon}/${CONDUCTOR_GROUPS.runs} gids — refusing to hand the tree to a group ` +
|
|
1054
|
+
`nobody verified.`,
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
applyRunOwnership(worktreePath, boundary.principal, daemonGid);
|
|
1058
|
+
applyRunOwnership(sessionDir, boundary.principal, daemonGid);
|
|
1059
|
+
// The session's own HOME, TMPDIR and gh-config tree.
|
|
1060
|
+
// `prepareSessionEnvRoot` creates them daemon-owned 0700, so without this
|
|
1061
|
+
// the slot principal cannot write its own home and the harness fails at
|
|
1062
|
+
// session startup — which reads like a broken install rather than a
|
|
1063
|
+
// permissions handoff that was one directory short.
|
|
1064
|
+
applyRunOwnership(envRoot, boundary.principal, daemonGid);
|
|
1065
|
+
// The claim, checked rather than assumed. On Linux the principal
|
|
1066
|
+
// separates by DAC alone, so whether the operator's credentials are
|
|
1067
|
+
// actually out of reach depends on modes this package never set. Ask the
|
|
1068
|
+
// slot itself, and refuse the dispatch if the answer is yes.
|
|
1069
|
+
const leaking = await credentialReadRefusal(
|
|
1070
|
+
spawnCaptured,
|
|
1071
|
+
boundary.principal,
|
|
1072
|
+
{ boundingSet: d.boundary?.probe.boundingSetDropped ?? false },
|
|
1073
|
+
[...credentialDenyRoots(), ...credentialDenyFiles()],
|
|
1074
|
+
);
|
|
1075
|
+
if (leaking !== undefined) throw new Error(leaking);
|
|
1076
|
+
// The other half of the layout, and it is not optional: the run's git dir
|
|
1077
|
+
// borrows the mirror's objects through `alternates`, so a mirror the slot
|
|
1078
|
+
// principal cannot read makes the checkout look corrupt rather than
|
|
1079
|
+
// forbidden.
|
|
1080
|
+
//
|
|
1081
|
+
// Both trees are checked first, and the check REFUSES rather than
|
|
1082
|
+
// widening the state directory — see `mirrorTraversalRefusal`. A slot
|
|
1083
|
+
// that cannot reach its own checkout is just as broken as one that cannot
|
|
1084
|
+
// read the mirror, and the shipped defaults put both under the private
|
|
1085
|
+
// root.
|
|
1086
|
+
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
1087
|
+
for (const tree of [mirrorPath, worktreePath]) {
|
|
1088
|
+
const unreachable = mirrorTraversalRefusal(tree, stateDir());
|
|
1089
|
+
if (unreachable !== undefined) throw new Error(unreachable);
|
|
1090
|
+
}
|
|
1091
|
+
applyMirrorReadAccess(mirrorPath, runsGid);
|
|
1092
|
+
const denied = verifyDaemonAccess(worktreePath);
|
|
1093
|
+
if (denied !== undefined) throw new Error(denied);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
// An MCP server carrying its own PAT re-opens the hole the principal
|
|
1097
|
+
// closes, so this is a refusal and not a warning. Audited here, after
|
|
1098
|
+
// provisioning, because both roots the harness will discover — the run's
|
|
1099
|
+
// own checkout and the agent principal's config root — exist only now.
|
|
1100
|
+
const mcp = mcpRefusal(
|
|
1101
|
+
auditMcpRoots({ agentHome: boundary.env["HOME"] ?? homedir(), cwd: worktreePath }),
|
|
1102
|
+
);
|
|
1103
|
+
if (mcp !== undefined) throw new Error(mcp);
|
|
1104
|
+
|
|
1105
|
+
// ---- the run's mutation channel (#126) -------------------------------
|
|
1106
|
+
// A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
|
|
1107
|
+
// per-run *directory*: a directory owned by the run principal would hand
|
|
1108
|
+
// back the power to unlink a sibling's socket and bind an impostor in its
|
|
1109
|
+
// place, which is the whole thing the layout buys. `listenVerbChannel`
|
|
1110
|
+
// validates every component before binding and throws rather than
|
|
1111
|
+
// degrading, so a tampered path refuses this dispatch instead of running
|
|
1112
|
+
// the worker with a channel nobody can vouch for.
|
|
1113
|
+
verbListener = await listenVerbChannel(
|
|
1114
|
+
verbDeps(d),
|
|
1115
|
+
{
|
|
1116
|
+
kind: "run",
|
|
1117
|
+
// `runTreeRoot`, not the private state dir: the slot has to *connect*
|
|
1118
|
+
// to this socket, and a 0700 ancestor makes it unreachable however
|
|
1119
|
+
// correct the socket's own mode is. `ensureVerbSocketDir` still owns
|
|
1120
|
+
// the parent and still runs its anti-rebind checks there (#126).
|
|
1121
|
+
path: verbSocketPath(ensureVerbSocketDir(runTreeRoot), `run-${String(issue)}`),
|
|
1122
|
+
project: project.name,
|
|
1123
|
+
role: "worker",
|
|
1124
|
+
runId,
|
|
1125
|
+
issue,
|
|
1126
|
+
repo: r.repo,
|
|
1127
|
+
runRepoPath: worktreePath,
|
|
1128
|
+
branch,
|
|
1129
|
+
...(boundary.principal === undefined ? {} : { principal: boundary.principal }),
|
|
1130
|
+
},
|
|
1131
|
+
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
1132
|
+
);
|
|
1133
|
+
|
|
725
1134
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
726
1135
|
|
|
727
1136
|
log(
|
|
@@ -743,8 +1152,19 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
743
1152
|
caps,
|
|
744
1153
|
maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
|
|
745
1154
|
sessionDir,
|
|
1155
|
+
boundary,
|
|
1156
|
+
// Inside the run's own boundary root, so a slot principal can reach it
|
|
1157
|
+
// by traversal. A socket under the daemon's 0700 home would be
|
|
1158
|
+
// unreachable to the very process it exists for (#125).
|
|
1159
|
+
// Derived from the hoisted envRoot for the same reason, and because
|
|
1160
|
+
// that tree has already been handed to the slot principal.
|
|
1161
|
+
socketPath: join(envRoot, "ipc.sock"),
|
|
1162
|
+
verbSocketPath: verbListener.path,
|
|
1163
|
+
onChildLog: (line) => {
|
|
1164
|
+
log(`#${issue} ${line}`);
|
|
1165
|
+
},
|
|
746
1166
|
...(project.workerModel === undefined ? {} : { model: project.workerModel }),
|
|
747
|
-
|
|
1167
|
+
releaseGrants: resolveReleaseGrants(project),
|
|
748
1168
|
onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "worker", shape),
|
|
749
1169
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
750
1170
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
@@ -763,6 +1183,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
763
1183
|
// PR verification or terminal row writes can leave stale `running` state.
|
|
764
1184
|
turnLimit?.close();
|
|
765
1185
|
turnLimit = undefined;
|
|
1186
|
+
// The slot IS the principal, so holding it past the session means the
|
|
1187
|
+
// next run could be handed the same uid while this one's tree still
|
|
1188
|
+
// exists — two live runs able to write each other's checkout, which is
|
|
1189
|
+
// the cross-run property this whole change buys.
|
|
1190
|
+
if (slotIndex !== undefined) {
|
|
1191
|
+
d.boundary?.slots.release(slotIndex);
|
|
1192
|
+
slotIndex = undefined;
|
|
1193
|
+
}
|
|
766
1194
|
}
|
|
767
1195
|
|
|
768
1196
|
// A configured model the harness could not honour means this run was done by
|
|
@@ -777,8 +1205,33 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
777
1205
|
? await verifyPushedGreenClaim(tracker, result)
|
|
778
1206
|
: { state: result.state };
|
|
779
1207
|
const state = verified.state;
|
|
780
|
-
|
|
781
|
-
|
|
1208
|
+
|
|
1209
|
+
// The other half of not believing a worker about its own run (#128). The
|
|
1210
|
+
// claim being audited is `state: pushed-green`, so the audit runs on the
|
|
1211
|
+
// worker's claim rather than on what verification made of it: a claim that
|
|
1212
|
+
// GitHub then contradicted is exactly the report whose `changed:` line is
|
|
1213
|
+
// worth reading twice. Advisory throughout — nothing below reads `flags`
|
|
1214
|
+
// when deciding `state`, and the ordering here makes that checkable.
|
|
1215
|
+
const audit =
|
|
1216
|
+
result.state === "pushed-green"
|
|
1217
|
+
? await collectSettlementFlags(tracker, {
|
|
1218
|
+
prUrl: result.prUrl,
|
|
1219
|
+
report: result.report,
|
|
1220
|
+
issueText: `${r.issue.title}\n${r.issue.body}`,
|
|
1221
|
+
})
|
|
1222
|
+
: undefined;
|
|
1223
|
+
if (result.state === "pushed-green") {
|
|
1224
|
+
if (audit === undefined) log(`#${issue} settlement audit skipped: no readable PR diff`);
|
|
1225
|
+
else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
|
|
1226
|
+
}
|
|
1227
|
+
const auditLines =
|
|
1228
|
+
audit === undefined ? [] : formatSettlementFlags(audit.flags, { truncated: audit.truncated });
|
|
1229
|
+
|
|
1230
|
+
const finalReport = [
|
|
1231
|
+
...(verified.reason === undefined ? [] : [verified.reason, ""]),
|
|
1232
|
+
...(auditLines.length === 0 ? [] : [...auditLines, ""]),
|
|
1233
|
+
result.report,
|
|
1234
|
+
].join("\n");
|
|
782
1235
|
|
|
783
1236
|
// What becomes of the tree, decided once, before any label or page. A
|
|
784
1237
|
// `pushed-*` run is the only end that does not salvage: its deliverable is
|
|
@@ -795,11 +1248,24 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
795
1248
|
ending:
|
|
796
1249
|
state === "blocked" ? "blocked for an operator decision" : endedBy(result.killedBy),
|
|
797
1250
|
worktree: worktreePath,
|
|
1251
|
+
branch,
|
|
1252
|
+
publish,
|
|
798
1253
|
...(state === "failed" || state === "killed"
|
|
799
1254
|
? ({ tree: "keep" } as const)
|
|
800
1255
|
: ({ tree: "remove", mirrorPath } as const)),
|
|
801
1256
|
});
|
|
802
|
-
if (settlement === undefined)
|
|
1257
|
+
if (settlement === undefined) {
|
|
1258
|
+
// A `pushed-*` end does not salvage — its deliverable is already on a
|
|
1259
|
+
// remote branch — but its repository still goes away here, so the branch
|
|
1260
|
+
// is published first for the same reason `settleWorktree` does it: the
|
|
1261
|
+
// run repo is no longer a view of the mirror, and anything it holds that
|
|
1262
|
+
// never reached the mirror dies with the directory.
|
|
1263
|
+
const published = await publish(branch);
|
|
1264
|
+
if (!published.ok) {
|
|
1265
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
1266
|
+
}
|
|
1267
|
+
await removeWorktree(mirrorPath, worktreePath);
|
|
1268
|
+
}
|
|
803
1269
|
|
|
804
1270
|
store.updateRun(runId, {
|
|
805
1271
|
state,
|
|
@@ -811,6 +1277,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
811
1277
|
sessionFile: result.sessionFile,
|
|
812
1278
|
...(verified.reason === undefined ? {} : { lastError: verified.reason }),
|
|
813
1279
|
...settlement?.patch,
|
|
1280
|
+
...(audit === undefined || audit.flags.length === 0
|
|
1281
|
+
? {}
|
|
1282
|
+
: { settlementFlags: audit.flags }),
|
|
814
1283
|
});
|
|
815
1284
|
|
|
816
1285
|
const salvaged = settlement?.lines ?? [];
|
|
@@ -886,6 +1355,45 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
886
1355
|
// A verified or still-pending PR keeps the in-progress label until its
|
|
887
1356
|
// checks or merge settle, preventing another worker from duplicating it.
|
|
888
1357
|
log(`#${issue} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
1358
|
+
// A green run's report is the only one nothing ever delivers: it raises no
|
|
1359
|
+
// escalation, and the row it settles into says nothing about what the
|
|
1360
|
+
// worker wrote. That is fine while the report is believed — and it is
|
|
1361
|
+
// exactly wrong once something has stopped believing it. A flagged
|
|
1362
|
+
// settlement is the one case where a `pushed-green` run has news, and it
|
|
1363
|
+
// has it *now*, while the PR is still open and nobody has merged it (#128).
|
|
1364
|
+
//
|
|
1365
|
+
// Tier 1, because the judgement is the orchestrator's: it can read the
|
|
1366
|
+
// diff, it can ask, and it can promote to a human. The state is
|
|
1367
|
+
// deliberately unchanged either way — the run below this line settles
|
|
1368
|
+
// `pushed-green` whether it was flagged or not.
|
|
1369
|
+
//
|
|
1370
|
+
// Findings only. A diff too large to read in full is noted in the log and
|
|
1371
|
+
// in the report, but it is not news for a human: paging "0 flags, could
|
|
1372
|
+
// not read it all" is exactly the kind of line that teaches an
|
|
1373
|
+
// orchestrator to skim past this escalation the next time it carries one.
|
|
1374
|
+
const found = audit?.flags ?? [];
|
|
1375
|
+
if (found.length > 0) {
|
|
1376
|
+
await safeEscalate(d, {
|
|
1377
|
+
tier: 1,
|
|
1378
|
+
project: project.name,
|
|
1379
|
+
issue,
|
|
1380
|
+
runId,
|
|
1381
|
+
summary:
|
|
1382
|
+
`#${issue} settled ${state} on attempt ${attempt} with ` +
|
|
1383
|
+
`${found.length} settlement audit flag(s)`,
|
|
1384
|
+
detail: [
|
|
1385
|
+
`${r.issue.title}`,
|
|
1386
|
+
r.issue.url,
|
|
1387
|
+
result.prUrl ?? "(no PR URL)",
|
|
1388
|
+
"",
|
|
1389
|
+
"The run is NOT blocked and its state is unchanged. These are advisory",
|
|
1390
|
+
"findings about the worker's own account of its work — judge them, then",
|
|
1391
|
+
"merge, ask, or close as you would have anyway.",
|
|
1392
|
+
"",
|
|
1393
|
+
finalReport,
|
|
1394
|
+
].join("\n"),
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
889
1397
|
}
|
|
890
1398
|
} catch (err) {
|
|
891
1399
|
// Dispatch setup can fail after the controller opens but before runWorker's
|
|
@@ -894,6 +1402,13 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
894
1402
|
turnLimit = undefined;
|
|
895
1403
|
const detail = errText(err);
|
|
896
1404
|
log(`#${issue} errored: ${detail}`);
|
|
1405
|
+
// Released here as well: a dispatch that failed before `runWorker` never
|
|
1406
|
+
// reached the `finally` above, and a leaked slot permanently shrinks the
|
|
1407
|
+
// pool until the daemon restarts.
|
|
1408
|
+
if (slotIndex !== undefined) {
|
|
1409
|
+
d.boundary?.slots.release(slotIndex);
|
|
1410
|
+
slotIndex = undefined;
|
|
1411
|
+
}
|
|
897
1412
|
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
898
1413
|
// copy of real work. Nothing else on this path so much as looks at it.
|
|
899
1414
|
const settlement =
|
|
@@ -904,6 +1419,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
904
1419
|
attempt,
|
|
905
1420
|
ending: "killed by a dispatch error",
|
|
906
1421
|
worktree: worktreePath,
|
|
1422
|
+
branch,
|
|
1423
|
+
publish,
|
|
907
1424
|
tree: "keep",
|
|
908
1425
|
});
|
|
909
1426
|
if (run) {
|
|
@@ -937,6 +1454,18 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
937
1454
|
// a failure path, and whatever it still held is now a commit on the branch.
|
|
938
1455
|
} finally {
|
|
939
1456
|
turnLimit?.close();
|
|
1457
|
+
// The run is over, so its channel is too. Closed here rather than beside
|
|
1458
|
+
// the session so the crash path closes it as well: a listener left bound
|
|
1459
|
+
// after its run settled is a socket whose `run-not-live` check is the only
|
|
1460
|
+
// thing standing between a stale child and a push.
|
|
1461
|
+
if (verbListener !== undefined) {
|
|
1462
|
+
try {
|
|
1463
|
+
await verbListener.close();
|
|
1464
|
+
} catch (err) {
|
|
1465
|
+
log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
|
|
1466
|
+
}
|
|
1467
|
+
verbListener = undefined;
|
|
1468
|
+
}
|
|
940
1469
|
}
|
|
941
1470
|
}
|
|
942
1471
|
|
|
@@ -1280,6 +1809,74 @@ export function summarizeDispatch(
|
|
|
1280
1809
|
};
|
|
1281
1810
|
}
|
|
1282
1811
|
|
|
1812
|
+
/**
|
|
1813
|
+
* What a held plan-usage gate says to a human, if anything.
|
|
1814
|
+
*
|
|
1815
|
+
* Three different problems hide behind one hold, and they want different
|
|
1816
|
+
* tiers. Reaching the threshold is the guard *working*: tier 1, because the
|
|
1817
|
+
* fleet resumes on its own at the provider's reset and nobody needs to get
|
|
1818
|
+
* out of bed. Everything else — a window nothing reports, a window that
|
|
1819
|
+
* resolves to two allowances, a meter that has been unreadable for half an
|
|
1820
|
+
* hour — is dispatch stopped with no self-recovery, which is tier 2.
|
|
1821
|
+
*
|
|
1822
|
+
* Each summary carries the fact that will change when the situation does (the
|
|
1823
|
+
* reset instant, the configured id, the date), because the escalation ledger
|
|
1824
|
+
* dedupes on the summary: a stable one pages once and then goes quiet, which
|
|
1825
|
+
* is right for a repeated tick and wrong for the next window.
|
|
1826
|
+
*/
|
|
1827
|
+
function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation | undefined {
|
|
1828
|
+
const base = { project, issue: NO_ISSUE };
|
|
1829
|
+
if (plan.state === "at-cap") {
|
|
1830
|
+
const window = plan.window?.id ?? plan.cap?.windowId ?? "the configured window";
|
|
1831
|
+
const resets =
|
|
1832
|
+
plan.resetsAt === undefined
|
|
1833
|
+
? new Date().toISOString().slice(0, 10)
|
|
1834
|
+
: new Date(plan.resetsAt).toISOString();
|
|
1835
|
+
return {
|
|
1836
|
+
...base,
|
|
1837
|
+
tier: 1,
|
|
1838
|
+
summary: `Plan allowance cap reached on ${window} — ${project} is not claiming new work (window ${resets})`,
|
|
1839
|
+
detail: [
|
|
1840
|
+
plan.detail,
|
|
1841
|
+
"Running workers finish normally; only new claims are held.",
|
|
1842
|
+
"Dispatch resumes by itself once the provider reports the window reset or usage below the threshold —",
|
|
1843
|
+
"no `resume` needed. Raise `caps.planUsage.maxUsedFraction` only if you mean to spend the rest.",
|
|
1844
|
+
].join("\n"),
|
|
1845
|
+
};
|
|
1846
|
+
}
|
|
1847
|
+
if (plan.state === "blind") {
|
|
1848
|
+
return {
|
|
1849
|
+
...base,
|
|
1850
|
+
tier: 2,
|
|
1851
|
+
// Dated: a meter that breaks again next month is a new incident, not a
|
|
1852
|
+
// repeat of this one.
|
|
1853
|
+
summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`,
|
|
1854
|
+
detail: [
|
|
1855
|
+
plan.detail,
|
|
1856
|
+
"The guard admitted work while the failure looked transient and has now stopped.",
|
|
1857
|
+
"Check `omp usage --json` on the fleet host, or set `caps.planUsage` to null if this fleet is unmetered.",
|
|
1858
|
+
].join("\n"),
|
|
1859
|
+
};
|
|
1860
|
+
}
|
|
1861
|
+
if (
|
|
1862
|
+
plan.state === "window-missing" ||
|
|
1863
|
+
plan.state === "window-ambiguous" ||
|
|
1864
|
+
plan.state === "window-uncomparable"
|
|
1865
|
+
) {
|
|
1866
|
+
return {
|
|
1867
|
+
...base,
|
|
1868
|
+
tier: 2,
|
|
1869
|
+
summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`,
|
|
1870
|
+
detail: [
|
|
1871
|
+
plan.detail,
|
|
1872
|
+
"Run `omp usage --json` and copy an allowance `id` into `caps.planUsage.windowId`,",
|
|
1873
|
+
"or set `caps.planUsage` to null if this fleet is unmetered.",
|
|
1874
|
+
].join("\n"),
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
return undefined;
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1283
1880
|
/**
|
|
1284
1881
|
* Which routed candidates get a worker this tick — in queue order, never more
|
|
1285
1882
|
* than `slots` of them. Every non-admission receives a stable reason code.
|
|
@@ -1294,7 +1891,7 @@ export function summarizeDispatch(
|
|
|
1294
1891
|
* that grows a field has no business breaking these tests.
|
|
1295
1892
|
*/
|
|
1296
1893
|
export async function admitCandidates(
|
|
1297
|
-
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate">,
|
|
1894
|
+
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage" | "boundary">,
|
|
1298
1895
|
routed: Routed[],
|
|
1299
1896
|
slots: number,
|
|
1300
1897
|
): Promise<AdmissionPass> {
|
|
@@ -1306,6 +1903,50 @@ export async function admitCandidates(
|
|
|
1306
1903
|
holds.push({ issue, reason });
|
|
1307
1904
|
};
|
|
1308
1905
|
|
|
1906
|
+
// The fail-closed the issue asks for, and it only ever fires because the
|
|
1907
|
+
// operator asked for a boundary by name. Placed above every other gate,
|
|
1908
|
+
// including the tracker calls: a fleet that must not dispatch must not spend
|
|
1909
|
+
// API budget discovering it, and it must not claim an issue it will refuse.
|
|
1910
|
+
//
|
|
1911
|
+
// Deliberately NOT a downgrade to group-mode. That is a different, weaker
|
|
1912
|
+
// claim, and silently substituting it is how an operator ends up believing
|
|
1913
|
+
// they have a boundary they do not have (#125).
|
|
1914
|
+
if (boundaryRefusesDispatch(d.boundary) && d.boundary !== undefined) {
|
|
1915
|
+
const refusal = boundaryRefusal(d.boundary.probe);
|
|
1916
|
+
for (const r of routed) hold(r.issue.number, "credential-boundary");
|
|
1917
|
+
log(`credential boundary holding ${String(routed.length)} candidate(s): ${refusal}`);
|
|
1918
|
+
if (!d.boundary.paged) {
|
|
1919
|
+
d.boundary.paged = true;
|
|
1920
|
+
await safeEscalate(d, {
|
|
1921
|
+
tier: 2,
|
|
1922
|
+
project: project.name,
|
|
1923
|
+
issue: NO_ISSUE,
|
|
1924
|
+
summary: `${project.name} cannot dispatch: credentials.isolation is "per-run" and this host offers no mechanism`,
|
|
1925
|
+
detail: [refusal, "", "Nothing is dispatched until this is resolved. No work has been claimed."].join("\n"),
|
|
1926
|
+
});
|
|
1927
|
+
}
|
|
1928
|
+
return { admitted: [], holds };
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
// The plan allowance is a fleet-wide question, so it is asked once per pass
|
|
1932
|
+
// and answers for every candidate — unlike every gate below it, which is
|
|
1933
|
+
// per-issue. It sits here rather than beside the spend cap in `tick` for one
|
|
1934
|
+
// reason: the spend cap *pauses the daemon* and waits for a human, and a
|
|
1935
|
+
// weekly plan window resets by itself. A guard that demanded `resume` after
|
|
1936
|
+
// every rollover would cost more operator attention than the guard saves
|
|
1937
|
+
// (#110). Already-running workers are untouched and settle normally.
|
|
1938
|
+
//
|
|
1939
|
+
// Placed after the cheap local busy-set read and before the first tracker
|
|
1940
|
+
// call, so a held fleet spends no GitHub API budget discovering it is held.
|
|
1941
|
+
const plan = await readPlanUsage(caps.planUsage, d.usage);
|
|
1942
|
+
if (plan.blocking) {
|
|
1943
|
+
for (const r of routed) hold(r.issue.number, "plan-usage-cap");
|
|
1944
|
+
log(`plan usage gate holding ${String(routed.length)} candidate(s): ${plan.detail}`);
|
|
1945
|
+
const escalation = planUsageEscalation(project.name, plan);
|
|
1946
|
+
if (escalation !== undefined) await safeEscalate(d, escalation);
|
|
1947
|
+
return { admitted: [], holds };
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1309
1950
|
// parent -> blocking issue. Seeded from active runs (including pushed-green),
|
|
1310
1951
|
// then extended by candidates admitted earlier in this same pass so two
|
|
1311
1952
|
// siblings never both clear the gate in one tick.
|
|
@@ -1380,7 +2021,7 @@ export async function admitCandidates(
|
|
|
1380
2021
|
continue;
|
|
1381
2022
|
}
|
|
1382
2023
|
|
|
1383
|
-
// Fail closed on work that exists only in a
|
|
2024
|
+
// Fail closed on work that exists only in a run repo. `addRunRepo` clears
|
|
1384
2025
|
// the tree at <workspaceRoot>/<issue> before it provisions, so admitting
|
|
1385
2026
|
// this issue is what finally destroys the copy the salvage could not save
|
|
1386
2027
|
// (#118). Nothing here can recover it — git already refused once — so the
|
|
@@ -1794,28 +2435,83 @@ export async function turnLimitResponse(
|
|
|
1794
2435
|
);
|
|
1795
2436
|
}
|
|
1796
2437
|
|
|
2438
|
+
export interface DaemonHttpDeps {
|
|
2439
|
+
project: string;
|
|
2440
|
+
store: Pick<Store, "latestRun">;
|
|
2441
|
+
turnLimits: TurnLimitRegistry;
|
|
2442
|
+
health: () => DaemonHealthSnapshot;
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
/**
|
|
2446
|
+
* The whole HTTP surface, in one named function so a test can pin what is *not*
|
|
2447
|
+
* on it.
|
|
2448
|
+
*
|
|
2449
|
+
* Two routes: a health read, and the turn-limit control. Everything else is
|
|
2450
|
+
* 404, and that is the contract — see the prohibition at the `Bun.serve` call
|
|
2451
|
+
* for why no mutation may be added here. Extracted from the serve callback
|
|
2452
|
+
* precisely so "the daemon's HTTP port exposes no mutation route" is something
|
|
2453
|
+
* a test asserts rather than something a reviewer has to notice (#126).
|
|
2454
|
+
*/
|
|
2455
|
+
export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
|
|
2456
|
+
const url = new URL(req.url);
|
|
2457
|
+
if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
|
|
2458
|
+
const control = await turnLimitResponse(req, d.project, d.store, d.turnLimits);
|
|
2459
|
+
return control ?? new Response("not found\n", { status: 404 });
|
|
2460
|
+
}
|
|
2461
|
+
|
|
1797
2462
|
export interface StatusSnapshot {
|
|
1798
2463
|
project: string;
|
|
1799
2464
|
configPath: string;
|
|
1800
2465
|
stateDir: string;
|
|
1801
2466
|
paused: boolean;
|
|
1802
2467
|
caps: Caps;
|
|
2468
|
+
/**
|
|
2469
|
+
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
2470
|
+
* by each renderer because #122 began with a grant nobody had looked at in
|
|
2471
|
+
* weeks: `status` names them so a stale one is visible without opening either
|
|
2472
|
+
* the config or the brief.
|
|
2473
|
+
*/
|
|
2474
|
+
releaseGrants: ResolvedGrants;
|
|
1803
2475
|
/** Occupied issues: live workers plus green PRs awaiting a human merge. */
|
|
1804
2476
|
activeRuns: RunRecord[];
|
|
1805
2477
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
1806
2478
|
* only copy of work the daemon could not save. */
|
|
1807
2479
|
salvagedRuns: RunRecord[];
|
|
2480
|
+
/** Reports the operator has not provably received: pending, in-flight with an
|
|
2481
|
+
* unknown outcome, or written off. An empty list is the only honest way to
|
|
2482
|
+
* say "everything authored this cycle actually went out" (#123). */
|
|
2483
|
+
openReports: ReportRecord[];
|
|
2484
|
+
/**
|
|
2485
|
+
* The most recent conductor-verb calls and how the daemon decided them
|
|
2486
|
+
* (#126). On `status` rather than only behind `omp-conductor ledger` because
|
|
2487
|
+
* a refused merge is news: it means a session tried to do something the
|
|
2488
|
+
* config does not let it, and an operator who has to know to go looking is an
|
|
2489
|
+
* operator who finds out from the tracker instead.
|
|
2490
|
+
*/
|
|
2491
|
+
verbLedger: VerbLedgerEntry[];
|
|
1808
2492
|
/** Runs backed by a worker process — the number capacity compares against. */
|
|
1809
2493
|
liveWorkers: number;
|
|
1810
2494
|
runsToday: number;
|
|
1811
2495
|
spendTodayUsd: number;
|
|
1812
2496
|
dispatch?: DispatchSummary;
|
|
2497
|
+
/**
|
|
2498
|
+
* Latest plan-allowance verdict, when the caller read one. Optional because
|
|
2499
|
+
* this snapshot is built synchronously off the store while the provider read
|
|
2500
|
+
* is I/O: a renderer that did not do that work must say "not read" rather
|
|
2501
|
+
* than print a percentage nobody measured (#110).
|
|
2502
|
+
*/
|
|
2503
|
+
planUsage?: PlanUsageStatus;
|
|
1813
2504
|
}
|
|
1814
2505
|
|
|
1815
2506
|
/** Builds a status reading from an already-open store. Long-lived operator
|
|
1816
2507
|
* surfaces use this path so a one-second refresh does not repeatedly open and
|
|
1817
2508
|
* initialise SQLite connections. */
|
|
1818
|
-
export function statusSnapshotFromStore(
|
|
2509
|
+
export function statusSnapshotFromStore(
|
|
2510
|
+
p: ProjectConfig,
|
|
2511
|
+
caps: Caps,
|
|
2512
|
+
store: Store,
|
|
2513
|
+
planUsage?: PlanUsageStatus,
|
|
2514
|
+
): StatusSnapshot {
|
|
1819
2515
|
const since = startOfToday();
|
|
1820
2516
|
const dispatch = store.latestDispatch(p.name);
|
|
1821
2517
|
return {
|
|
@@ -1824,12 +2520,16 @@ export function statusSnapshotFromStore(p: ProjectConfig, caps: Caps, store: Sto
|
|
|
1824
2520
|
stateDir: stateDir(),
|
|
1825
2521
|
paused: isPaused(),
|
|
1826
2522
|
caps,
|
|
2523
|
+
releaseGrants: resolveReleaseGrants(p),
|
|
1827
2524
|
activeRuns: store.activeRuns(p.name),
|
|
1828
2525
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
2526
|
+
openReports: store.openReports(p.name),
|
|
2527
|
+
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
1829
2528
|
liveWorkers: store.liveRuns(p.name).length,
|
|
1830
2529
|
runsToday: store.runsStartedSince(p.name, since),
|
|
1831
2530
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
1832
2531
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
2532
|
+
...(planUsage === undefined ? {} : { planUsage }),
|
|
1833
2533
|
};
|
|
1834
2534
|
}
|
|
1835
2535
|
|
|
@@ -1895,6 +2595,24 @@ export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
|
|
|
1895
2595
|
return lines;
|
|
1896
2596
|
}
|
|
1897
2597
|
|
|
2598
|
+
/**
|
|
2599
|
+
* The grant table, named shape by shape.
|
|
2600
|
+
*
|
|
2601
|
+
* One renderer for both status surfaces (this one and the fleet view), because
|
|
2602
|
+
* #122 began with a POLICY grant that no longer matched anyone's intent and went
|
|
2603
|
+
* unnoticed: a stale grant has to be visible from `status` alone, and two
|
|
2604
|
+
* renderers would eventually show it in only one of them.
|
|
2605
|
+
*/
|
|
2606
|
+
export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
2607
|
+
const granted = RELEASE_SHAPES.filter((shape) => grants[shape] === "orchestrator");
|
|
2608
|
+
return [
|
|
2609
|
+
granted.length === 0
|
|
2610
|
+
? "release no shape granted — every release/deploy tool call is blocked"
|
|
2611
|
+
: `release granted to the orchestrator: ${granted.join(", ")}`,
|
|
2612
|
+
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
2613
|
+
];
|
|
2614
|
+
}
|
|
2615
|
+
|
|
1898
2616
|
export function formatStatus(s: StatusSnapshot): string {
|
|
1899
2617
|
const lines = [
|
|
1900
2618
|
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
@@ -1907,11 +2625,17 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1907
2625
|
s.caps.dailySpendUsd === null
|
|
1908
2626
|
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
1909
2627
|
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
2628
|
+
// Its own row, never folded into the spend row: they are two independent
|
|
2629
|
+
// controls, and an operator has to be able to see which one stopped the
|
|
2630
|
+
// fleet (#110).
|
|
2631
|
+
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
1910
2632
|
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
1911
2633
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
1912
2634
|
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
1913
2635
|
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
1914
2636
|
"",
|
|
2637
|
+
...formatReleaseGrants(s.releaseGrants),
|
|
2638
|
+
"",
|
|
1915
2639
|
formatDispatchSummary(s.dispatch),
|
|
1916
2640
|
"",
|
|
1917
2641
|
];
|
|
@@ -1925,9 +2649,16 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1925
2649
|
`${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
1926
2650
|
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
1927
2651
|
);
|
|
2652
|
+
// Its escalation was deduplicated the moment it was delivered, so this
|
|
2653
|
+
// line is the only place a flagged run stays visible while its PR waits
|
|
2654
|
+
// for a merge — which is precisely the window the flag is about (#128).
|
|
2655
|
+
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
2656
|
+
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
1928
2657
|
}
|
|
1929
2658
|
}
|
|
1930
2659
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
2660
|
+
lines.push(...formatOpenReports(s.openReports));
|
|
2661
|
+
lines.push(...formatVerbLedger(s.verbLedger));
|
|
1931
2662
|
// Deploy hint: a restart while workers are live orphans them (salvage runs
|
|
1932
2663
|
// first — #35). Prefer pause + drain to zero live workers when you can wait.
|
|
1933
2664
|
if (s.liveWorkers > 0) {
|
|
@@ -2025,6 +2756,13 @@ export function prepareConductor(): void {
|
|
|
2025
2756
|
export async function reconcileOrphanedRuns(
|
|
2026
2757
|
store: Store,
|
|
2027
2758
|
project: string,
|
|
2759
|
+
/**
|
|
2760
|
+
* Resolves the privileged publisher for one orphaned run. Optional because a
|
|
2761
|
+
* test driving the row transitions has no repo to publish to; production
|
|
2762
|
+
* always passes it, and without it a salvaged WIP commit stays local — which
|
|
2763
|
+
* is the half of #121 that reaches a human.
|
|
2764
|
+
*/
|
|
2765
|
+
publish?: (run: RunRecord) => RunPublisher,
|
|
2028
2766
|
): Promise<RunRecord[]> {
|
|
2029
2767
|
// Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
|
|
2030
2768
|
// a process dying — it is finished work waiting on a human merge.
|
|
@@ -2043,6 +2781,11 @@ export async function reconcileOrphanedRuns(
|
|
|
2043
2781
|
attempt: r.attempt,
|
|
2044
2782
|
ending: "killed by a daemon restart",
|
|
2045
2783
|
worktree: r.worktree,
|
|
2784
|
+
branch: r.branch,
|
|
2785
|
+
// An orphan's tree is kept, so its commits are not about to be
|
|
2786
|
+
// deleted — but a WIP salvage still has to reach GitHub, which is
|
|
2787
|
+
// #121's whole point and is now the daemon's hop to make.
|
|
2788
|
+
publish: publish?.(r),
|
|
2046
2789
|
tree: "keep",
|
|
2047
2790
|
});
|
|
2048
2791
|
store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
|
|
@@ -2053,12 +2796,63 @@ export async function reconcileOrphanedRuns(
|
|
|
2053
2796
|
// ------------------------------------------------------------------- the daemon
|
|
2054
2797
|
|
|
2055
2798
|
export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
2799
|
+
// Before anything reads the config for real: a file written before #125 gains
|
|
2800
|
+
// an explicit `credentials.isolation` here, so the answer lives on disk
|
|
2801
|
+
// rather than being inherited from a default nobody chose. Non-fatal by
|
|
2802
|
+
// design — a config the daemon cannot rewrite still loads as `"none"` and
|
|
2803
|
+
// still dispatches, because an outage caused by a security feature is worse
|
|
2804
|
+
// than the day before it shipped.
|
|
2805
|
+
try {
|
|
2806
|
+
const migration = migrateCredentialsOnDisk();
|
|
2807
|
+
if (migration.migrated.length > 0) {
|
|
2808
|
+
log(
|
|
2809
|
+
`migrated ${migration.migrated.join(", ")} to an explicit credentials.isolation: "none" in ` +
|
|
2810
|
+
`${migration.path} — this fleet is UNPROTECTED: sessions run as this daemon's user and can reach ` +
|
|
2811
|
+
`its GitHub credential. Set credentials.isolation to "per-run" once the host is provisioned (README).`,
|
|
2812
|
+
);
|
|
2813
|
+
}
|
|
2814
|
+
} catch (err) {
|
|
2815
|
+
log(`could not persist the credentials migration (${errText(err)}) — continuing with isolation "none"`);
|
|
2816
|
+
}
|
|
2817
|
+
|
|
2056
2818
|
const cfg = loadConfig();
|
|
2057
2819
|
const project = findProject(cfg, o.project);
|
|
2058
2820
|
const caps = resolveCaps(project, cfg.defaults);
|
|
2059
2821
|
const store = openStore(dbPath());
|
|
2060
2822
|
const tracker = makeTracker(project);
|
|
2061
2823
|
|
|
2824
|
+
// The host capability probe, once, at startup. It states the mechanism
|
|
2825
|
+
// rather than guessing, and it proves the launcher works against a trivial
|
|
2826
|
+
// child before reporting one available — a mechanism whose argv the host
|
|
2827
|
+
// rejects looks identical here and then fails every dispatch at run time.
|
|
2828
|
+
const credentials = resolveCredentials(project);
|
|
2829
|
+
const probe = await probeHost({ slots: caps.maxConcurrentWorkers });
|
|
2830
|
+
const boundary: FleetBoundary = {
|
|
2831
|
+
isolation: credentials.isolation,
|
|
2832
|
+
probe,
|
|
2833
|
+
slots: createSlotPool(caps.maxConcurrentWorkers),
|
|
2834
|
+
paged: false,
|
|
2835
|
+
};
|
|
2836
|
+
const described = describeBoundary(credentials.isolation, probe);
|
|
2837
|
+
log(`credential boundary: ${described.headline}`);
|
|
2838
|
+
for (const line of described.detail) log(`credential boundary: ${line}`);
|
|
2839
|
+
|
|
2840
|
+
// #126's transport, stated at startup rather than guessed at first use. The
|
|
2841
|
+
// banner names what this host can actually enforce — whether the kernel will
|
|
2842
|
+
// vouch for a caller's uid, and whether runs get distinct principals at all —
|
|
2843
|
+
// because "peer credentials asserted" is a claim, and a claim nobody printed
|
|
2844
|
+
// is one nobody can check against the host it is running on.
|
|
2845
|
+
const verbPeerReader = peerCredentialReader();
|
|
2846
|
+
const verbActions = githubVerbActions(project);
|
|
2847
|
+
let orchestratorVerbs: VerbListener | undefined;
|
|
2848
|
+
log(
|
|
2849
|
+
`verb transport: ${transportBanner(
|
|
2850
|
+
ensureVerbSocketDir(stateDir()),
|
|
2851
|
+
verbPeerReader,
|
|
2852
|
+
probe.mechanism === "uid-pool",
|
|
2853
|
+
)}`,
|
|
2854
|
+
);
|
|
2855
|
+
|
|
2062
2856
|
// Recorded here, before a single tick runs, so that the deploy an operator
|
|
2063
2857
|
// *means* to do never trips the tripwire: installing a new build and
|
|
2064
2858
|
// restarting the unit re-records this from the new files. What it catches is
|
|
@@ -2073,7 +2867,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2073
2867
|
// daemon must not orphan that daemon's real, live workers).
|
|
2074
2868
|
const alive = livingDaemon();
|
|
2075
2869
|
if (alive === undefined || alive.pid === process.pid) {
|
|
2076
|
-
|
|
2870
|
+
// The orphan path salvages, and #121 requires that salvage to reach
|
|
2871
|
+
// GitHub. Since the run's commits now live in its own repository, the hop
|
|
2872
|
+
// is the daemon's: resolve the routed repo by name off the run row.
|
|
2873
|
+
const orphanPublisher = (r: RunRecord): RunPublisher => {
|
|
2874
|
+
const repo = project.routing.repos[r.repo];
|
|
2875
|
+
return async (branch) =>
|
|
2876
|
+
repo === undefined
|
|
2877
|
+
? { ok: false, stderr: `run ${String(r.id)} names repo "${r.repo}", which this project no longer routes` }
|
|
2878
|
+
: pushRunBranch(project, { repo, runRepoPath: r.worktree, branch });
|
|
2879
|
+
};
|
|
2880
|
+
for (const r of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
|
|
2077
2881
|
log(
|
|
2078
2882
|
`#${r.issue} orphaned by a previous daemon (attempt ${r.attempt}, was ${r.state}, worktree ${r.worktree}) — ` +
|
|
2079
2883
|
`slot freed; the ${project.stateLabels.inProgress} label stays until the orchestrator triages what the worker left`,
|
|
@@ -2086,6 +2890,9 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2086
2890
|
// Standing orders. The orchestrator holds none of this file's context, so
|
|
2087
2891
|
// everything it needs to act — which tracker, which labels, what the fleet
|
|
2088
2892
|
// does — has to be said once, in words.
|
|
2893
|
+
const releaseGrants = resolveReleaseGrants(project);
|
|
2894
|
+
const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
|
|
2895
|
+
const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
|
|
2089
2896
|
const brief = [
|
|
2090
2897
|
`You are the omp-conductor orchestrator for project "${project.name}".`,
|
|
2091
2898
|
`Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`,
|
|
@@ -2106,10 +2913,14 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2106
2913
|
"a time, freshness-checked against the base branch, per the Releases section of your POLICY.md."
|
|
2107
2914
|
: "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " +
|
|
2108
2915
|
"human merges.",
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2916
|
+
// Named shape by shape rather than as one policy word, so a stale grant is
|
|
2917
|
+
// legible in the transcript instead of only in the config file — the #122
|
|
2918
|
+
// incident began with a grant that no longer matched anyone's intent.
|
|
2919
|
+
`Release tool gate: ${
|
|
2920
|
+
grantedShapes.length === 0
|
|
2921
|
+
? "every release and deploy shape is mechanically blocked for you"
|
|
2922
|
+
: `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
|
|
2923
|
+
}.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
|
|
2113
2924
|
"Handle each escalation below before the next one.",
|
|
2114
2925
|
].join("\n");
|
|
2115
2926
|
|
|
@@ -2127,10 +2938,87 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2127
2938
|
log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
|
|
2128
2939
|
} else {
|
|
2129
2940
|
try {
|
|
2941
|
+
// Its own principal, distinct from every worker slot. The orchestrator
|
|
2942
|
+
// reads the state directory and its briefs and must have no read or
|
|
2943
|
+
// write access to any run checkout — so `workspaceRoot` and the mirror
|
|
2944
|
+
// are denied outright here, on top of the #127 tool-layer jail. Two
|
|
2945
|
+
// layers because they fail differently: the jail refuses a structured
|
|
2946
|
+
// tool call, the principal refuses the syscall `bash` would make.
|
|
2947
|
+
// Same rule as a worker's, for the same reason: under `uid-pool` the
|
|
2948
|
+
// orchestrator is its own uid, so every tree it must reach lives outside
|
|
2949
|
+
// the 0700 private state directory. Its principal is in `conductor-runs`
|
|
2950
|
+
// and deliberately NOT in the daemon group, so it could not traverse
|
|
2951
|
+
// there even once — and widening the state directory is the one fix that
|
|
2952
|
+
// is not available, because that is where conductor.db lives.
|
|
2953
|
+
const orchTreeRoot = probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
|
|
2954
|
+
if (orchTreeRoot !== stateDir()) {
|
|
2955
|
+
mkdirSync(orchTreeRoot, { recursive: true });
|
|
2956
|
+
chmodSync(orchTreeRoot, 0o711);
|
|
2957
|
+
}
|
|
2958
|
+
const orchEnvRoot = join(orchTreeRoot, "boundaries", "orchestrator");
|
|
2959
|
+
const orchCwd = join(orchTreeRoot, "orchestrator");
|
|
2960
|
+
mkdirSync(orchCwd, { recursive: true });
|
|
2961
|
+
const orchestratorBoundary = buildSessionBoundary({
|
|
2962
|
+
probe,
|
|
2963
|
+
isolation: credentials.isolation,
|
|
2964
|
+
role: "orchestrator",
|
|
2965
|
+
envRoot: orchEnvRoot,
|
|
2966
|
+
writeRoots: [orchCwd],
|
|
2967
|
+
denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot, project.mirrorRoot],
|
|
2968
|
+
denyReadFiles: credentialDenyFiles(),
|
|
2969
|
+
...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
|
|
2970
|
+
});
|
|
2971
|
+
if (orchestratorBoundary.principal !== undefined) {
|
|
2972
|
+
const gid = probe.daemonGid;
|
|
2973
|
+
if (gid === undefined) {
|
|
2974
|
+
throw new Error(
|
|
2975
|
+
`the orchestrator principal was allocated without a resolved ${CONDUCTOR_GROUPS.daemon} gid`,
|
|
2976
|
+
);
|
|
2977
|
+
}
|
|
2978
|
+
applyRunOwnership(orchEnvRoot, orchestratorBoundary.principal, gid);
|
|
2979
|
+
applyRunOwnership(orchCwd, orchestratorBoundary.principal, gid);
|
|
2980
|
+
// The same empirical gate a worker gets. The orchestrator is the
|
|
2981
|
+
// session with merge and release authority, so "it cannot reach the
|
|
2982
|
+
// credential" matters more here, not less — and it was the half that
|
|
2983
|
+
// was never checked. `sharedGroup: false` because it is launched with
|
|
2984
|
+
// no supplementary group at all, so the probe has to run under exactly
|
|
2985
|
+
// that identity rather than a worker's.
|
|
2986
|
+
const orchLeaking = await credentialReadRefusal(
|
|
2987
|
+
spawnCaptured,
|
|
2988
|
+
orchestratorBoundary.principal,
|
|
2989
|
+
{ boundingSet: probe.boundingSetDropped, sharedGroup: false },
|
|
2990
|
+
[...credentialDenyRoots(), ...credentialDenyFiles()],
|
|
2991
|
+
);
|
|
2992
|
+
if (orchLeaking !== undefined) throw new Error(orchLeaking);
|
|
2993
|
+
}
|
|
2994
|
+
// A third socket, distinct from every run's, in the same daemon-owned
|
|
2995
|
+
// 0711 parent. This is what makes "merge authority is the orchestrator's"
|
|
2996
|
+
// a property of the channel: the daemon knows which session is speaking
|
|
2997
|
+
// because of where the connection arrived, and no payload can move a
|
|
2998
|
+
// worker's call onto this one (#126).
|
|
2999
|
+
orchestratorVerbs = await listenVerbChannel(
|
|
3000
|
+
verbDeps({ project, store, tracker, verbActions }),
|
|
3001
|
+
{
|
|
3002
|
+
kind: "orchestrator",
|
|
3003
|
+
path: verbSocketPath(ensureVerbSocketDir(orchTreeRoot), "orchestrator"),
|
|
3004
|
+
project: project.name,
|
|
3005
|
+
role: "orchestrator",
|
|
3006
|
+
...(orchestratorBoundary.principal === undefined
|
|
3007
|
+
? {}
|
|
3008
|
+
: { principal: orchestratorBoundary.principal }),
|
|
3009
|
+
},
|
|
3010
|
+
{ ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
|
|
3011
|
+
);
|
|
2130
3012
|
orchestrator = await startOrchestrator({
|
|
2131
|
-
cwd:
|
|
3013
|
+
cwd: orchCwd,
|
|
2132
3014
|
brief,
|
|
2133
|
-
|
|
3015
|
+
releaseGrants,
|
|
3016
|
+
boundary: orchestratorBoundary,
|
|
3017
|
+
socketPath: join(orchEnvRoot, "ipc.sock"),
|
|
3018
|
+
verbSocketPath: orchestratorVerbs.path,
|
|
3019
|
+
onChildLog: (line) => {
|
|
3020
|
+
log(`orchestrator ${line}`);
|
|
3021
|
+
},
|
|
2134
3022
|
onReleaseBlocked: (shape) => recordReleaseBlock(project.name, "orchestrator", shape),
|
|
2135
3023
|
});
|
|
2136
3024
|
const transcript = orchestrator.sessionFile();
|
|
@@ -2146,14 +3034,45 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2146
3034
|
}
|
|
2147
3035
|
|
|
2148
3036
|
const escalator = createEscalator(project, tracker, store, orchestrator);
|
|
3037
|
+
|
|
3038
|
+
// Report delivery is the daemon's, not the model's (#123). Built beside the
|
|
3039
|
+
// escalator because a report nobody can deliver pages through it, and driven
|
|
3040
|
+
// on its own timer rather than inside `tick()`: a paused fleet claims nothing
|
|
3041
|
+
// but still owes its operator the report it was handed, and five minutes is a
|
|
3042
|
+
// long time to sit on a page.
|
|
3043
|
+
const outbox = createReportOutbox({
|
|
3044
|
+
project,
|
|
3045
|
+
store,
|
|
3046
|
+
escalate: (e) => escalator.escalate(e),
|
|
3047
|
+
log,
|
|
3048
|
+
});
|
|
3049
|
+
|
|
3050
|
+
// Every row still `sending` when a daemon boots belonged to a process that is
|
|
3051
|
+
// gone, so its outcome will never be learned — retry it and say so in the
|
|
3052
|
+
// message. Guarded exactly like the orphan reconciliation above and for the
|
|
3053
|
+
// same reason: a row belonging to a *live* daemon is genuinely in flight, and
|
|
3054
|
+
// stealing it would page the operator twice for one report.
|
|
3055
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
3056
|
+
for (const r of outbox.recover(Date.now())) {
|
|
3057
|
+
log(
|
|
3058
|
+
`report ${r.id} was left mid-send by a previous daemon (attempt ${r.attempts}) — ` +
|
|
3059
|
+
`retrying; its message will say it may be a repeat`,
|
|
3060
|
+
);
|
|
3061
|
+
}
|
|
3062
|
+
}
|
|
3063
|
+
|
|
2149
3064
|
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
2150
3065
|
store.updateRun(runId, { maxTurns });
|
|
2151
3066
|
});
|
|
2152
3067
|
const d: Deps = {
|
|
2153
3068
|
project,
|
|
2154
3069
|
caps,
|
|
3070
|
+
boundary,
|
|
2155
3071
|
tracker,
|
|
2156
3072
|
store,
|
|
3073
|
+
// Process-wide, so a `status` served off this daemon's own HTTP surface
|
|
3074
|
+
// reuses the tick's reading instead of shelling out again.
|
|
3075
|
+
usage: sharedUsageSource(),
|
|
2157
3076
|
escalate: (e) => escalator.escalate(e),
|
|
2158
3077
|
turnLimits,
|
|
2159
3078
|
integrity,
|
|
@@ -2161,13 +3080,20 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2161
3080
|
// page again about a stall that is still on disk.
|
|
2162
3081
|
stall: { paged: false },
|
|
2163
3082
|
cleanup: { next: 0 },
|
|
3083
|
+
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
3084
|
+
verbActions,
|
|
2164
3085
|
};
|
|
2165
3086
|
|
|
2166
3087
|
if (o.once) {
|
|
2167
3088
|
try {
|
|
2168
3089
|
await tick(d);
|
|
3090
|
+
// A single tick still owes the outbox a pass: a drill that leaves a
|
|
3091
|
+
// report undelivered teaches an operator the wrong thing about the
|
|
3092
|
+
// mechanism it is drilling.
|
|
3093
|
+
await outbox.deliverDue();
|
|
2169
3094
|
} finally {
|
|
2170
3095
|
await orchestrator?.dispose();
|
|
3096
|
+
await orchestratorVerbs?.close();
|
|
2171
3097
|
store.close();
|
|
2172
3098
|
}
|
|
2173
3099
|
return;
|
|
@@ -2194,6 +3120,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2194
3120
|
refreshCodeGraph();
|
|
2195
3121
|
const graphTimer = setInterval(refreshCodeGraph, GRAPH_HEALTH_INTERVAL_MS);
|
|
2196
3122
|
|
|
3123
|
+
// Overlap-guarded like the graph probe: a pass that is still waiting on
|
|
3124
|
+
// Telegram must not have a second pass started on top of it, or one report
|
|
3125
|
+
// would be claimed, reclaimed and sent twice by this process alone.
|
|
3126
|
+
let reportPass: Promise<void> | undefined;
|
|
3127
|
+
const drainReports = (): void => {
|
|
3128
|
+
if (reportPass !== undefined) return;
|
|
3129
|
+
reportPass = outbox
|
|
3130
|
+
.deliverDue()
|
|
3131
|
+
.then(() => {})
|
|
3132
|
+
.catch((err: unknown) => {
|
|
3133
|
+
log(`report delivery pass failed: ${errText(err)}`);
|
|
3134
|
+
})
|
|
3135
|
+
.finally(() => {
|
|
3136
|
+
reportPass = undefined;
|
|
3137
|
+
});
|
|
3138
|
+
};
|
|
3139
|
+
drainReports();
|
|
3140
|
+
const reportTimer = setInterval(drainReports, REPORT_DELIVERY_INTERVAL_MS);
|
|
3141
|
+
|
|
2197
3142
|
const workers = createWorkerPool();
|
|
2198
3143
|
let stopping = false;
|
|
2199
3144
|
let wake: (() => void) | undefined;
|
|
@@ -2206,17 +3151,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2206
3151
|
process.on("SIGINT", stop);
|
|
2207
3152
|
process.on("SIGTERM", stop);
|
|
2208
3153
|
|
|
3154
|
+
// ── NO MUTATION ROUTE BELONGS ON THIS PORT ────────────────────────────────
|
|
3155
|
+
//
|
|
3156
|
+
// This is unauthenticated loopback TCP. Every local user can reach it, it
|
|
3157
|
+
// carries no credential of any kind, and it cannot tell one caller from
|
|
3158
|
+
// another: `127.0.0.1` is not an identity. `turnLimitResponse` already shows
|
|
3159
|
+
// what that costs — it trusts a body-supplied `project`, which is exactly the
|
|
3160
|
+
// shape "identity from the payload" takes when nobody is watching, and it is
|
|
3161
|
+
// tolerable only because raising a turn ceiling is bounded and reversible.
|
|
3162
|
+
//
|
|
3163
|
+
// A merge, a push, a release or a label is none of those things. Do not add
|
|
3164
|
+
// one here, and do not add "just a small one" behind a shared secret either:
|
|
3165
|
+
// a secret readable by the process that would be attacking you is not
|
|
3166
|
+
// authentication. Mutations go over the per-run unix sockets in
|
|
3167
|
+
// `verbs/socket.ts`, where the kernel says who the caller is and the daemon
|
|
3168
|
+
// derives project, run and role from the channel rather than the body (#126).
|
|
2209
3169
|
const server = Bun.serve({
|
|
2210
3170
|
hostname: "127.0.0.1",
|
|
2211
3171
|
port: o.port ?? DEFAULT_PORT,
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
},
|
|
3172
|
+
fetch: (req) =>
|
|
3173
|
+
daemonHttpResponse(req, {
|
|
3174
|
+
project: project.name,
|
|
3175
|
+
store,
|
|
3176
|
+
turnLimits,
|
|
3177
|
+
health: () => daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph),
|
|
3178
|
+
}),
|
|
2220
3179
|
});
|
|
2221
3180
|
log(`serving /healthz on :${server.port}, project ${project.name}`);
|
|
2222
3181
|
|
|
@@ -2243,11 +3202,20 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
2243
3202
|
process.off("SIGINT", stop);
|
|
2244
3203
|
process.off("SIGTERM", stop);
|
|
2245
3204
|
clearInterval(graphTimer);
|
|
3205
|
+
clearInterval(reportTimer);
|
|
3206
|
+
// Before the store closes, like the orchestrator below: a pass mid-send has
|
|
3207
|
+
// a `markReportDelivered` still to write, and losing that write is exactly
|
|
3208
|
+
// how a delivered report comes back as an ambiguous one on the next boot.
|
|
3209
|
+
await reportPass;
|
|
2246
3210
|
await workers.drain();
|
|
2247
3211
|
await server.stop(true);
|
|
2248
3212
|
// Before the store closes: a queued injection that rejects on the way out
|
|
2249
3213
|
// falls back to an issue comment, and that path writes the dedup marker.
|
|
2250
3214
|
await orchestrator?.dispose();
|
|
3215
|
+
// After the session it belongs to is gone. A bound socket outliving its
|
|
3216
|
+
// orchestrator is a channel accepting merges for a session that no longer
|
|
3217
|
+
// exists.
|
|
3218
|
+
await orchestratorVerbs?.close();
|
|
2251
3219
|
store.close();
|
|
2252
3220
|
log("stopped");
|
|
2253
3221
|
// A handled SIGTERM still leaves some runtimes with a non-zero default
|