omp-conductor 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -269
- package/package.json +1 -1
- package/skills/conductor-onboarding/SKILL.md +0 -56
- package/skills/conductor-update/SKILL.md +4 -9
- package/src/briefs/orchestrator.md +13 -19
- package/src/briefs/worker.md +18 -16
- package/src/cli.ts +1 -48
- package/src/config.ts +7 -224
- package/src/confinement.ts +2 -3
- package/src/daemon.ts +49 -406
- package/src/failure-class.ts +44 -0
- package/src/fleet.ts +1 -85
- package/src/gitops.ts +189 -0
- package/src/omp.ts +11 -35
- package/src/orchestrator.ts +0 -4
- package/src/plugin.ts +1 -42
- package/src/session-host.ts +12 -19
- package/src/setup-host.ts +1 -157
- package/src/setup.ts +6 -41
- package/src/store.ts +36 -2
- package/src/tracker/github.ts +1 -1
- package/src/types.ts +1 -85
- package/src/upgrade.ts +0 -88
- package/src/verbs/actions.ts +1 -1
- package/src/verbs/server.ts +3 -28
- package/src/verbs/socket.ts +15 -23
- package/src/worker.ts +1 -13
- package/src/worktree.ts +4 -4
- package/systemd/omp-conductor.service.example +8 -94
- package/src/credentials.ts +0 -2029
package/src/daemon.ts
CHANGED
|
@@ -14,12 +14,9 @@ import {
|
|
|
14
14
|
configPath,
|
|
15
15
|
findProject,
|
|
16
16
|
loadConfig,
|
|
17
|
-
migrateCredentialsOnDisk,
|
|
18
17
|
resolveCaps,
|
|
19
|
-
resolveCredentials,
|
|
20
18
|
resolveReleaseGrants,
|
|
21
19
|
stateDir,
|
|
22
|
-
sharedRoot,
|
|
23
20
|
} from "./config.ts";
|
|
24
21
|
import {
|
|
25
22
|
analyseSettlement,
|
|
@@ -49,7 +46,6 @@ import type {
|
|
|
49
46
|
Escalation,
|
|
50
47
|
OpenCloser,
|
|
51
48
|
PrState,
|
|
52
|
-
CredentialIsolation,
|
|
53
49
|
ProjectConfig,
|
|
54
50
|
ReadyIssue,
|
|
55
51
|
RepoTarget,
|
|
@@ -64,7 +60,6 @@ import type {
|
|
|
64
60
|
Tracker,
|
|
65
61
|
VerbLedgerEntry,
|
|
66
62
|
} from "./types.ts";
|
|
67
|
-
import { CONDUCTOR_GROUPS, type IsolationMechanism } from "./types.ts";
|
|
68
63
|
import { type KilledBy, type WorkerResult, renderBrief, runWorker } from "./worker.ts";
|
|
69
64
|
import {
|
|
70
65
|
addRunRepo,
|
|
@@ -90,29 +85,7 @@ import {
|
|
|
90
85
|
} from "./verbs/socket.ts";
|
|
91
86
|
import { homedir } from "node:os";
|
|
92
87
|
|
|
93
|
-
import {
|
|
94
|
-
applyRunOwnership,
|
|
95
|
-
applyMirrorReadAccess,
|
|
96
|
-
auditMcpRoots,
|
|
97
|
-
mirrorTraversalRefusal,
|
|
98
|
-
credentialReadRefusal,
|
|
99
|
-
spawnCaptured,
|
|
100
|
-
boundaryRefusal,
|
|
101
|
-
buildSessionBoundary,
|
|
102
|
-
createSlotPool,
|
|
103
|
-
describeBoundary,
|
|
104
|
-
liveDaemonCredentials,
|
|
105
|
-
mcpRefusal,
|
|
106
|
-
mechanismSatisfies,
|
|
107
|
-
openRunPr,
|
|
108
|
-
probeHost,
|
|
109
|
-
pushRunBranch,
|
|
110
|
-
verifyDaemonAccess,
|
|
111
|
-
type HostProbe,
|
|
112
|
-
type RunRepoRef,
|
|
113
|
-
type SessionBoundary,
|
|
114
|
-
type SlotPool,
|
|
115
|
-
} from "./credentials.ts";
|
|
88
|
+
import { pushRunBranch, type RunRepoRef } from "./gitops.ts";
|
|
116
89
|
import {
|
|
117
90
|
planUsageLine,
|
|
118
91
|
readPlanUsage,
|
|
@@ -150,44 +123,9 @@ export interface DaemonOpts {
|
|
|
150
123
|
|
|
151
124
|
/** Everything one tick touches, resolved once at startup so a tick never
|
|
152
125
|
* re-reads config mid-flight and changes its own limits underneath itself. */
|
|
153
|
-
/**
|
|
154
|
-
* The credential boundary this fleet is running behind (#125), resolved once at
|
|
155
|
-
* startup like every other dep so a tick cannot change its own protection.
|
|
156
|
-
*
|
|
157
|
-
* `paged` is the same one-page-per-episode gate the integrity tripwire uses: a
|
|
158
|
-
* host that cannot build the boundary the operator asked for holds *every*
|
|
159
|
-
* candidate on *every* tick, and paging per five minutes forever is paging
|
|
160
|
-
* nobody reads.
|
|
161
|
-
*/
|
|
162
|
-
export interface FleetBoundary {
|
|
163
|
-
isolation: CredentialIsolation;
|
|
164
|
-
probe: HostProbe;
|
|
165
|
-
slots: SlotPool;
|
|
166
|
-
paged: boolean;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
/**
|
|
170
|
-
* True when the operator asked for a boundary this host cannot build.
|
|
171
|
-
*
|
|
172
|
-
* Asks {@link mechanismSatisfies} rather than testing for `none`, because the
|
|
173
|
-
* interesting refusal is the near miss: a host offering `group-mode` against a
|
|
174
|
-
* `per-run` config used to dispatch happily under the weaker boundary, which is
|
|
175
|
-
* the silent downgrade #125 forbids.
|
|
176
|
-
*/
|
|
177
|
-
export function boundaryRefusesDispatch(boundary: FleetBoundary | undefined): boolean {
|
|
178
|
-
return boundary !== undefined && !mechanismSatisfies(boundary.isolation, boundary.probe.mechanism);
|
|
179
|
-
}
|
|
180
|
-
|
|
181
126
|
interface Deps {
|
|
182
127
|
project: ProjectConfig;
|
|
183
128
|
caps: Caps;
|
|
184
|
-
/**
|
|
185
|
-
* Optional only so a test can build a `Deps` without a host probe; production
|
|
186
|
-
* always resolves one in `runDaemon`. Absent means the A1 shape — sessions
|
|
187
|
-
* are still child processes, but they run as the daemon's own user and no
|
|
188
|
-
* security claim is made for them.
|
|
189
|
-
*/
|
|
190
|
-
boundary?: FleetBoundary;
|
|
191
129
|
tracker: Tracker;
|
|
192
130
|
store: Store;
|
|
193
131
|
/** Provider-reported plan allowance, cached with a TTL. Resolved once at
|
|
@@ -669,48 +607,6 @@ export async function settleWorktree(
|
|
|
669
607
|
};
|
|
670
608
|
}
|
|
671
609
|
|
|
672
|
-
/**
|
|
673
|
-
* The probe a session gets when no boundary was resolved at all — a test, or a
|
|
674
|
-
* `--once` run built without one. Named rather than inlined so the "no
|
|
675
|
-
* mechanism, no claim" shape is one object every call site shares.
|
|
676
|
-
*/
|
|
677
|
-
const NO_BOUNDARY_PROBE: HostProbe = {
|
|
678
|
-
mechanism: "none",
|
|
679
|
-
reasons: ["no host probe was run"],
|
|
680
|
-
residuals: [],
|
|
681
|
-
boundingSetDropped: false,
|
|
682
|
-
slots: [],
|
|
683
|
-
};
|
|
684
|
-
|
|
685
|
-
/**
|
|
686
|
-
* Credential paths the macOS profile denies outright.
|
|
687
|
-
*
|
|
688
|
-
* Derived from the operator's real home rather than from the session's
|
|
689
|
-
* redirected one: the point is that the *operator's* `gh` config and keys stay
|
|
690
|
-
* unreachable even when a session names their absolute path, which is exactly
|
|
691
|
-
* the probe `GH_CONFIG_DIR=<operator home>/.config/gh gh auth status` in #125.
|
|
692
|
-
*/
|
|
693
|
-
function credentialDenyRoots(): string[] {
|
|
694
|
-
const home = homedir();
|
|
695
|
-
return [join(home, ".ssh"), join(home, ".config", "gh"), join(home, ".gnupg"), join(home, ".aws")];
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
function credentialDenyFiles(): string[] {
|
|
699
|
-
const home = homedir();
|
|
700
|
-
return [
|
|
701
|
-
join(home, ".git-credentials"),
|
|
702
|
-
join(home, ".npmrc"),
|
|
703
|
-
join(home, ".netrc"),
|
|
704
|
-
// Both git config locations, because a token does not only live in
|
|
705
|
-
// `.git-credentials`: `url.https://<token>@github.com/.insteadOf` is a
|
|
706
|
-
// perfectly ordinary way to wire one up, and an ordinary `.gitconfig` is
|
|
707
|
-
// 0644. Hardening chmods it, but hardening is not the claim — this list is
|
|
708
|
-
// what the empirical recheck actually asks the slot principal about, so a
|
|
709
|
-
// file missing from here is a credential nobody verified was out of reach.
|
|
710
|
-
join(home, ".gitconfig"),
|
|
711
|
-
join(home, ".config", "git", "config"),
|
|
712
|
-
];
|
|
713
|
-
}
|
|
714
610
|
|
|
715
611
|
/**
|
|
716
612
|
* How a run's end is named — in the salvage commit, and to whoever reads it.
|
|
@@ -949,24 +845,21 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
949
845
|
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
950
846
|
let worktreePath: string | undefined;
|
|
951
847
|
let turnLimit: TurnLimitController | undefined;
|
|
952
|
-
//
|
|
953
|
-
//
|
|
954
|
-
// catch and finally paths have to release the slot and publish the branch.
|
|
848
|
+
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
849
|
+
// the catch and finally paths have to publish the branch.
|
|
955
850
|
let runRepo: RunRepoRef | undefined;
|
|
956
|
-
|
|
957
|
-
let boundary: SessionBoundary | undefined;
|
|
958
|
-
// #126: the run's own verb socket. Hoisted like the slot, because the catch
|
|
851
|
+
// #126: the run's own verb socket. Hoisted like the run repo, because the catch
|
|
959
852
|
// and finally paths have to close it — a socket outliving its run is a
|
|
960
853
|
// channel nobody is authenticating any more.
|
|
961
854
|
let verbListener: VerbListener | undefined;
|
|
962
|
-
const credentials = resolveCredentials(project);
|
|
963
855
|
|
|
964
856
|
/**
|
|
965
857
|
* Publishes the run's branch on the privileged side: run repo → mirror →
|
|
966
|
-
* GitHub, fast-forward only. The
|
|
967
|
-
*
|
|
968
|
-
* route
|
|
969
|
-
* from being the *only* copy when the
|
|
858
|
+
* GitHub, fast-forward only. The dispatcher performs every network git
|
|
859
|
+
* operation for a run so the settlement record and the branch cannot
|
|
860
|
+
* disagree (#126); this is the only route a worker's commits take out, and it
|
|
861
|
+
* is also what stops a per-run repository from being the *only* copy when the
|
|
862
|
+
* tree is removed.
|
|
970
863
|
*/
|
|
971
864
|
const publish: RunPublisher = async () => {
|
|
972
865
|
if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
|
|
@@ -1022,122 +915,12 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1022
915
|
// directory and learns the real path back from the result. Inventing one
|
|
1023
916
|
// here would put a file that never gets written into an escalation.
|
|
1024
917
|
//
|
|
1025
|
-
// Per run rather than one shared directory
|
|
1026
|
-
//
|
|
1027
|
-
|
|
1028
|
-
// reads all of them — it owns the group.
|
|
1029
|
-
//
|
|
1030
|
-
// Under `uid-pool` the session runs as a *different uid*, so every tree it
|
|
1031
|
-
// has to reach lives outside the private state directory: that one stays
|
|
1032
|
-
// 0700 because it holds conductor.db and the WAL SQLite keeps recreating,
|
|
1033
|
-
// and making it searchable to let a slot in would publish fleet history to
|
|
1034
|
-
// every local account. `sandbox-exec` and `none` keep the old paths — same
|
|
1035
|
-
// uid, nothing to traverse into (#125).
|
|
1036
|
-
const runTreeRoot = d.boundary?.probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
|
|
1037
|
-
if (runTreeRoot !== stateDir()) {
|
|
1038
|
-
mkdirSync(runTreeRoot, { recursive: true });
|
|
1039
|
-
chmodSync(runTreeRoot, 0o711);
|
|
1040
|
-
}
|
|
918
|
+
// Per run rather than one shared directory, so one run's transcript cannot
|
|
919
|
+
// be truncated or replaced by the next.
|
|
920
|
+
const runTreeRoot = stateDir();
|
|
1041
921
|
const sessionDir = join(runTreeRoot, "sessions", `run-${String(runId)}`);
|
|
1042
|
-
const envRoot = join(runTreeRoot, "boundaries", `run-${String(runId)}`);
|
|
1043
922
|
mkdirSync(sessionDir, { recursive: true });
|
|
1044
923
|
|
|
1045
|
-
// ---- the credential boundary for this run (#125) --------------------
|
|
1046
|
-
slotIndex = d.boundary?.slots.acquire();
|
|
1047
|
-
boundary = buildSessionBoundary({
|
|
1048
|
-
probe: d.boundary?.probe ?? NO_BOUNDARY_PROBE,
|
|
1049
|
-
isolation: d.boundary?.isolation ?? "none",
|
|
1050
|
-
role: "worker",
|
|
1051
|
-
...(slotIndex === undefined ? {} : { slot: slotIndex }),
|
|
1052
|
-
envRoot,
|
|
1053
|
-
writeRoots: [worktreePath, sessionDir],
|
|
1054
|
-
// The workspace root, denied wholesale and then re-granted for this run's
|
|
1055
|
-
// own checkout by the trailing allow in `sandboxProfile`. On `uid-pool`
|
|
1056
|
-
// the modes already do this; on macOS the profile is the only thing that
|
|
1057
|
-
// does, because sandbox-exec does not change the uid.
|
|
1058
|
-
//
|
|
1059
|
-
// `mirrorRoot` is deliberately NOT denied. The run repository borrows its
|
|
1060
|
-
// objects from the mirror through git alternates, so denying it would
|
|
1061
|
-
// break every git command in the checkout — and a run being able to read
|
|
1062
|
-
// another run's objects out of that shared store is the residual this
|
|
1063
|
-
// design accepts and documents, not one it pretends to have closed. The
|
|
1064
|
-
// mirror stays unwritable, because it is not in `writeRoots`.
|
|
1065
|
-
denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot],
|
|
1066
|
-
denyReadFiles: credentialDenyFiles(),
|
|
1067
|
-
...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
|
|
1068
|
-
});
|
|
1069
|
-
|
|
1070
|
-
if (boundary.principal !== undefined) {
|
|
1071
|
-
// Ownership handoff, then proof. `applyRunOwnership` makes the tree
|
|
1072
|
-
// `<slot>:conductor-daemon 2770`, which is also the point of no return:
|
|
1073
|
-
// if this daemon is not an *effective* member of that group it has just
|
|
1074
|
-
// locked itself out of push, salvage and reclaim, and the modes look
|
|
1075
|
-
// perfectly correct while it does. So it writes, reads and removes a
|
|
1076
|
-
// probe file rather than inspecting them (#125).
|
|
1077
|
-
// Both gids come from the probe's own `getent`, never from the daemon's
|
|
1078
|
-
// `egid`. They are different groups: the shipped unit runs
|
|
1079
|
-
// `User=fleet`/`Group=fleet` with `conductor-daemon` merely
|
|
1080
|
-
// supplementary, so `egid` is `fleet` — handing that to
|
|
1081
|
-
// `applyRunOwnership` would make every run repo `<slot>:fleet`, readable
|
|
1082
|
-
// by anyone else in `fleet`, while the probe had validated a group the
|
|
1083
|
-
// modes never used. A uid-pool probe always resolves both, so their
|
|
1084
|
-
// absence here is a contradiction rather than a default to paper over.
|
|
1085
|
-
const { daemonGid, runsGid } = d.boundary?.probe ?? {};
|
|
1086
|
-
if (daemonGid === undefined || runsGid === undefined) {
|
|
1087
|
-
throw new Error(
|
|
1088
|
-
`the credential boundary reported a per-run principal without resolving ` +
|
|
1089
|
-
`${CONDUCTOR_GROUPS.daemon}/${CONDUCTOR_GROUPS.runs} gids — refusing to hand the tree to a group ` +
|
|
1090
|
-
`nobody verified.`,
|
|
1091
|
-
);
|
|
1092
|
-
}
|
|
1093
|
-
applyRunOwnership(worktreePath, boundary.principal, daemonGid);
|
|
1094
|
-
applyRunOwnership(sessionDir, boundary.principal, daemonGid);
|
|
1095
|
-
// The session's own HOME, TMPDIR and gh-config tree.
|
|
1096
|
-
// `prepareSessionEnvRoot` creates them daemon-owned 0700, so without this
|
|
1097
|
-
// the slot principal cannot write its own home and the harness fails at
|
|
1098
|
-
// session startup — which reads like a broken install rather than a
|
|
1099
|
-
// permissions handoff that was one directory short.
|
|
1100
|
-
applyRunOwnership(envRoot, boundary.principal, daemonGid);
|
|
1101
|
-
// The claim, checked rather than assumed. On Linux the principal
|
|
1102
|
-
// separates by DAC alone, so whether the operator's credentials are
|
|
1103
|
-
// actually out of reach depends on modes this package never set. Ask the
|
|
1104
|
-
// slot itself, and refuse the dispatch if the answer is yes.
|
|
1105
|
-
const leaking = await credentialReadRefusal(
|
|
1106
|
-
spawnCaptured,
|
|
1107
|
-
boundary.principal,
|
|
1108
|
-
{ boundingSet: d.boundary?.probe.boundingSetDropped ?? false },
|
|
1109
|
-
[...credentialDenyRoots(), ...credentialDenyFiles()],
|
|
1110
|
-
);
|
|
1111
|
-
if (leaking !== undefined) throw new Error(leaking);
|
|
1112
|
-
// The other half of the layout, and it is not optional: the run's git dir
|
|
1113
|
-
// borrows the mirror's objects through `alternates`, so a mirror the slot
|
|
1114
|
-
// principal cannot read makes the checkout look corrupt rather than
|
|
1115
|
-
// forbidden.
|
|
1116
|
-
//
|
|
1117
|
-
// Both trees are checked first, and the check REFUSES rather than
|
|
1118
|
-
// widening the state directory — see `mirrorTraversalRefusal`. A slot
|
|
1119
|
-
// that cannot reach its own checkout is just as broken as one that cannot
|
|
1120
|
-
// read the mirror, and the shipped defaults put both under the private
|
|
1121
|
-
// root.
|
|
1122
|
-
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
1123
|
-
for (const tree of [mirrorPath, worktreePath]) {
|
|
1124
|
-
const unreachable = mirrorTraversalRefusal(tree, stateDir());
|
|
1125
|
-
if (unreachable !== undefined) throw new Error(unreachable);
|
|
1126
|
-
}
|
|
1127
|
-
applyMirrorReadAccess(mirrorPath, runsGid);
|
|
1128
|
-
const denied = verifyDaemonAccess(worktreePath);
|
|
1129
|
-
if (denied !== undefined) throw new Error(denied);
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
// An MCP server carrying its own PAT re-opens the hole the principal
|
|
1133
|
-
// closes, so this is a refusal and not a warning. Audited here, after
|
|
1134
|
-
// provisioning, because both roots the harness will discover — the run's
|
|
1135
|
-
// own checkout and the agent principal's config root — exist only now.
|
|
1136
|
-
const mcp = mcpRefusal(
|
|
1137
|
-
auditMcpRoots({ agentHome: boundary.env["HOME"] ?? homedir(), cwd: worktreePath }),
|
|
1138
|
-
);
|
|
1139
|
-
if (mcp !== undefined) throw new Error(mcp);
|
|
1140
|
-
|
|
1141
924
|
// ---- the run's mutation channel (#126) -------------------------------
|
|
1142
925
|
// A shared, daemon-owned 0711 parent with one 0600 socket per run, never a
|
|
1143
926
|
// per-run *directory*: a directory owned by the run principal would hand
|
|
@@ -1162,7 +945,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1162
945
|
repo: r.repo,
|
|
1163
946
|
runRepoPath: worktreePath,
|
|
1164
947
|
branch,
|
|
1165
|
-
...(boundary.principal === undefined ? {} : { principal: boundary.principal }),
|
|
1166
948
|
},
|
|
1167
949
|
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
1168
950
|
);
|
|
@@ -1188,13 +970,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1188
970
|
caps,
|
|
1189
971
|
maxTurns: () => turnLimit?.maxTurns() ?? caps.workerMaxTurns,
|
|
1190
972
|
sessionDir,
|
|
1191
|
-
|
|
1192
|
-
//
|
|
1193
|
-
|
|
1194
|
-
// unreachable to the very process it exists for (#125).
|
|
1195
|
-
// Derived from the hoisted envRoot for the same reason, and because
|
|
1196
|
-
// that tree has already been handed to the slot principal.
|
|
1197
|
-
socketPath: join(envRoot, "ipc.sock"),
|
|
973
|
+
// The session's control socket, under the daemon's own state directory —
|
|
974
|
+
// a child process of the daemon reaches it directly.
|
|
975
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
1198
976
|
verbSocketPath: verbListener.path,
|
|
1199
977
|
onChildLog: (line) => {
|
|
1200
978
|
log(`#${issue} ${line}`);
|
|
@@ -1219,14 +997,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1219
997
|
// PR verification or terminal row writes can leave stale `running` state.
|
|
1220
998
|
turnLimit?.close();
|
|
1221
999
|
turnLimit = undefined;
|
|
1222
|
-
// The slot IS the principal, so holding it past the session means the
|
|
1223
|
-
// next run could be handed the same uid while this one's tree still
|
|
1224
|
-
// exists — two live runs able to write each other's checkout, which is
|
|
1225
|
-
// the cross-run property this whole change buys.
|
|
1226
|
-
if (slotIndex !== undefined) {
|
|
1227
|
-
d.boundary?.slots.release(slotIndex);
|
|
1228
|
-
slotIndex = undefined;
|
|
1229
|
-
}
|
|
1230
1000
|
}
|
|
1231
1001
|
|
|
1232
1002
|
// A configured model the harness could not honour means this run was done by
|
|
@@ -1442,13 +1212,6 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1442
1212
|
turnLimit = undefined;
|
|
1443
1213
|
const detail = errText(err);
|
|
1444
1214
|
log(`#${issue} errored: ${detail}`);
|
|
1445
|
-
// Released here as well: a dispatch that failed before `runWorker` never
|
|
1446
|
-
// reached the `finally` above, and a leaked slot permanently shrinks the
|
|
1447
|
-
// pool until the daemon restarts.
|
|
1448
|
-
if (slotIndex !== undefined) {
|
|
1449
|
-
d.boundary?.slots.release(slotIndex);
|
|
1450
|
-
slotIndex = undefined;
|
|
1451
|
-
}
|
|
1452
1215
|
// A crash lands anywhere, including mid-edit in a tree holding the only
|
|
1453
1216
|
// copy of real work. Nothing else on this path so much as looks at it.
|
|
1454
1217
|
const settlement =
|
|
@@ -1931,7 +1694,7 @@ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation
|
|
|
1931
1694
|
* that grows a field has no business breaking these tests.
|
|
1932
1695
|
*/
|
|
1933
1696
|
export async function admitCandidates(
|
|
1934
|
-
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage"
|
|
1697
|
+
d: Pick<Deps, "project" | "caps" | "tracker" | "store" | "escalate" | "usage">,
|
|
1935
1698
|
routed: Routed[],
|
|
1936
1699
|
slots: number,
|
|
1937
1700
|
): Promise<AdmissionPass> {
|
|
@@ -1943,31 +1706,6 @@ export async function admitCandidates(
|
|
|
1943
1706
|
holds.push({ issue, reason });
|
|
1944
1707
|
};
|
|
1945
1708
|
|
|
1946
|
-
// The fail-closed the issue asks for, and it only ever fires because the
|
|
1947
|
-
// operator asked for a boundary by name. Placed above every other gate,
|
|
1948
|
-
// including the tracker calls: a fleet that must not dispatch must not spend
|
|
1949
|
-
// API budget discovering it, and it must not claim an issue it will refuse.
|
|
1950
|
-
//
|
|
1951
|
-
// Deliberately NOT a downgrade to group-mode. That is a different, weaker
|
|
1952
|
-
// claim, and silently substituting it is how an operator ends up believing
|
|
1953
|
-
// they have a boundary they do not have (#125).
|
|
1954
|
-
if (boundaryRefusesDispatch(d.boundary) && d.boundary !== undefined) {
|
|
1955
|
-
const refusal = boundaryRefusal(d.boundary.probe);
|
|
1956
|
-
for (const r of routed) hold(r.issue.number, "credential-boundary");
|
|
1957
|
-
log(`credential boundary holding ${String(routed.length)} candidate(s): ${refusal}`);
|
|
1958
|
-
if (!d.boundary.paged) {
|
|
1959
|
-
d.boundary.paged = true;
|
|
1960
|
-
await safeEscalate(d, {
|
|
1961
|
-
tier: 2,
|
|
1962
|
-
project: project.name,
|
|
1963
|
-
issue: NO_ISSUE,
|
|
1964
|
-
summary: `${project.name} cannot dispatch: credentials.isolation is "per-run" and this host offers no mechanism`,
|
|
1965
|
-
detail: [refusal, "", "Nothing is dispatched until this is resolved. No work has been claimed."].join("\n"),
|
|
1966
|
-
});
|
|
1967
|
-
}
|
|
1968
|
-
return { admitted: [], holds };
|
|
1969
|
-
}
|
|
1970
|
-
|
|
1971
1709
|
// The plan allowance is a fleet-wide question, so it is asked once per pass
|
|
1972
1710
|
// and answers for every candidate — unlike every gate below it, which is
|
|
1973
1711
|
// per-issue. It sits here rather than beside the spend cap in `tick` for one
|
|
@@ -2433,20 +2171,6 @@ export interface DaemonHealthSnapshot {
|
|
|
2433
2171
|
rssBytes: number;
|
|
2434
2172
|
dispatch?: DispatchSummary;
|
|
2435
2173
|
codeGraph?: CodeGraphHealth;
|
|
2436
|
-
/**
|
|
2437
|
-
* The boundary THIS daemon resolved at startup.
|
|
2438
|
-
*
|
|
2439
|
-
* Published because capabilities are a property of the process, not the host:
|
|
2440
|
-
* `omp-conductor status` runs in an interactive shell with none of the unit's
|
|
2441
|
-
* ambient capabilities and no conductor groups, so re-probing there reports
|
|
2442
|
-
* `group-mode` on a fleet that is genuinely `uid-pool`. The operator's
|
|
2443
|
-
* verification step would say the boundary failed while it was working.
|
|
2444
|
-
*
|
|
2445
|
-
* It is also the only answer that survives a `daemon-reload` without a
|
|
2446
|
-
* restart: the config on disk can say anything, this is what the running
|
|
2447
|
-
* process actually got.
|
|
2448
|
-
*/
|
|
2449
|
-
boundary?: { isolation: CredentialIsolation; mechanism: IsolationMechanism; headline: string };
|
|
2450
2174
|
}
|
|
2451
2175
|
|
|
2452
2176
|
export function daemonHealthSnapshot(
|
|
@@ -2455,7 +2179,6 @@ export function daemonHealthSnapshot(
|
|
|
2455
2179
|
paused = isPaused(),
|
|
2456
2180
|
rssBytes = process.memoryUsage().rss,
|
|
2457
2181
|
codeGraph?: CodeGraphHealth,
|
|
2458
|
-
boundary?: DaemonHealthSnapshot["boundary"],
|
|
2459
2182
|
): DaemonHealthSnapshot {
|
|
2460
2183
|
const dispatch = store.latestDispatch(project);
|
|
2461
2184
|
return {
|
|
@@ -2466,7 +2189,6 @@ export function daemonHealthSnapshot(
|
|
|
2466
2189
|
rssBytes,
|
|
2467
2190
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
2468
2191
|
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
2469
|
-
...(boundary === undefined ? {} : { boundary }),
|
|
2470
2192
|
};
|
|
2471
2193
|
}
|
|
2472
2194
|
|
|
@@ -2961,15 +2683,38 @@ async function recoverRun(
|
|
|
2961
2683
|
}
|
|
2962
2684
|
|
|
2963
2685
|
if (recovery === "continue") {
|
|
2964
|
-
//
|
|
2965
|
-
//
|
|
2686
|
+
// Two classes recover by continuing, and only one of them has anything left
|
|
2687
|
+
// to do here.
|
|
2688
|
+
//
|
|
2689
|
+
// `turn-cap-progress` was already handed back by the completion path, which
|
|
2690
|
+
// swapped its labels and left the branch retained. There is nothing to
|
|
2691
|
+
// perform, and writing anything would be actively wrong: overwriting
|
|
2692
|
+
// `lastError` with a rebase brief tells the continuation worker to rebase a
|
|
2693
|
+
// run that simply ran out of turns, and re-swapping labels the completion
|
|
2694
|
+
// path already swapped is a pair of no-op `gh` calls. Record-only, so the
|
|
2695
|
+
// sweep stops re-offering it.
|
|
2696
|
+
if (cls === "turn-cap-progress") {
|
|
2697
|
+
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
2698
|
+
log(`#${run.issue} already continuing from ${cls}: ${evidence}`);
|
|
2699
|
+
return;
|
|
2700
|
+
}
|
|
2701
|
+
|
|
2702
|
+
// `merge-conflict`: the branch is retained and its PR is open, so #50's
|
|
2703
|
+
// continuation guard admits it and the next tick briefs a rebase.
|
|
2704
|
+
//
|
|
2705
|
+
// The tracker write goes FIRST, and that ordering is the whole retry
|
|
2706
|
+
// contract. Writing `recoveredAt` before the swap took the row out of
|
|
2707
|
+
// `runsNeedingClassification` — which selects on `recoveredAt IS NULL` — so a
|
|
2708
|
+
// refused label swap stranded the row permanently under a log line promising
|
|
2709
|
+
// a retry. That was the defect 0.4.4 claimed to have fixed and did not, for
|
|
2710
|
+
// this one recovery.
|
|
2711
|
+
if (!(await swapToQueue(d, run.issue, inProgress))) return;
|
|
2966
2712
|
store.updateRun(run.id, {
|
|
2967
2713
|
state: "killed",
|
|
2968
2714
|
lastError:
|
|
2969
2715
|
"merge-conflict: base moved under a green PR — rebase, regenerate recorded artifacts, push without force",
|
|
2970
2716
|
recoveredAt: Date.now(),
|
|
2971
2717
|
});
|
|
2972
|
-
if (!(await swapToQueue(d, run.issue, inProgress))) return;
|
|
2973
2718
|
log(`#${run.issue} requeued for a rebase continuation: ${evidence}`);
|
|
2974
2719
|
return;
|
|
2975
2720
|
}
|
|
@@ -3190,47 +2935,12 @@ export async function reconcileOrphanedRuns(
|
|
|
3190
2935
|
// ------------------------------------------------------------------- the daemon
|
|
3191
2936
|
|
|
3192
2937
|
export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
3193
|
-
// Before anything reads the config for real: a file written before #125 gains
|
|
3194
|
-
// an explicit `credentials.isolation` here, so the answer lives on disk
|
|
3195
|
-
// rather than being inherited from a default nobody chose. Non-fatal by
|
|
3196
|
-
// design — a config the daemon cannot rewrite still loads as `"none"` and
|
|
3197
|
-
// still dispatches, because an outage caused by a security feature is worse
|
|
3198
|
-
// than the day before it shipped.
|
|
3199
|
-
try {
|
|
3200
|
-
const migration = migrateCredentialsOnDisk();
|
|
3201
|
-
if (migration.migrated.length > 0) {
|
|
3202
|
-
log(
|
|
3203
|
-
`migrated ${migration.migrated.join(", ")} to an explicit credentials.isolation: "none" in ` +
|
|
3204
|
-
`${migration.path} — this fleet is UNPROTECTED: sessions run as this daemon's user and can reach ` +
|
|
3205
|
-
`its GitHub credential. Set credentials.isolation to "per-run" once the host is provisioned (README).`,
|
|
3206
|
-
);
|
|
3207
|
-
}
|
|
3208
|
-
} catch (err) {
|
|
3209
|
-
log(`could not persist the credentials migration (${errText(err)}) — continuing with isolation "none"`);
|
|
3210
|
-
}
|
|
3211
|
-
|
|
3212
2938
|
const cfg = loadConfig();
|
|
3213
2939
|
const project = findProject(cfg, o.project);
|
|
3214
2940
|
const caps = resolveCaps(project, cfg.defaults);
|
|
3215
2941
|
const store = openStore(dbPath());
|
|
3216
2942
|
const tracker = makeTracker(project);
|
|
3217
2943
|
|
|
3218
|
-
// The host capability probe, once, at startup. It states the mechanism
|
|
3219
|
-
// rather than guessing, and it proves the launcher works against a trivial
|
|
3220
|
-
// child before reporting one available — a mechanism whose argv the host
|
|
3221
|
-
// rejects looks identical here and then fails every dispatch at run time.
|
|
3222
|
-
const credentials = resolveCredentials(project);
|
|
3223
|
-
const probe = await probeHost({ slots: caps.maxConcurrentWorkers });
|
|
3224
|
-
const boundary: FleetBoundary = {
|
|
3225
|
-
isolation: credentials.isolation,
|
|
3226
|
-
probe,
|
|
3227
|
-
slots: createSlotPool(caps.maxConcurrentWorkers),
|
|
3228
|
-
paged: false,
|
|
3229
|
-
};
|
|
3230
|
-
const described = describeBoundary(credentials.isolation, probe);
|
|
3231
|
-
log(`credential boundary: ${described.headline}`);
|
|
3232
|
-
for (const line of described.detail) log(`credential boundary: ${line}`);
|
|
3233
|
-
|
|
3234
2944
|
// #126's transport, stated at startup rather than guessed at first use. The
|
|
3235
2945
|
// banner names what this host can actually enforce — whether the kernel will
|
|
3236
2946
|
// vouch for a caller's uid, and whether runs get distinct principals at all —
|
|
@@ -3239,13 +2949,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3239
2949
|
const verbPeerReader = peerCredentialReader();
|
|
3240
2950
|
const verbActions = githubVerbActions(project);
|
|
3241
2951
|
let orchestratorVerbs: VerbListener | undefined;
|
|
3242
|
-
log(
|
|
3243
|
-
`verb transport: ${transportBanner(
|
|
3244
|
-
ensureVerbSocketDir(stateDir()),
|
|
3245
|
-
verbPeerReader,
|
|
3246
|
-
probe.mechanism === "uid-pool",
|
|
3247
|
-
)}`,
|
|
3248
|
-
);
|
|
2952
|
+
log(`verb transport: ${transportBanner(ensureVerbSocketDir(stateDir()), verbPeerReader)}`);
|
|
3249
2953
|
|
|
3250
2954
|
// Recorded here, before a single tick runs, so that the deploy an operator
|
|
3251
2955
|
// *means* to do never trips the tripwire: installing a new build and
|
|
@@ -3332,63 +3036,12 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3332
3036
|
log("orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty");
|
|
3333
3037
|
} else {
|
|
3334
3038
|
try {
|
|
3335
|
-
//
|
|
3336
|
-
//
|
|
3337
|
-
//
|
|
3338
|
-
|
|
3339
|
-
// orchestrator gets: the tool-layer jail was deleted in 0.4.3 (#143)
|
|
3340
|
-
// because it could only be installed in sessions this daemon spawns, and
|
|
3341
|
-
// the principal refuses the syscall `bash` would make anyway, which the
|
|
3342
|
-
// jail never did. It is also conditional, and honestly so: on a host with
|
|
3343
|
-
// no isolating mechanism there is no principal and no gate, which is what
|
|
3344
|
-
// `status` reports as `unprotected`.
|
|
3345
|
-
// Same rule as a worker's, for the same reason: under `uid-pool` the
|
|
3346
|
-
// orchestrator is its own uid, so every tree it must reach lives outside
|
|
3347
|
-
// the 0700 private state directory. Its principal is in `conductor-runs`
|
|
3348
|
-
// and deliberately NOT in the daemon group, so it could not traverse
|
|
3349
|
-
// there even once — and widening the state directory is the one fix that
|
|
3350
|
-
// is not available, because that is where conductor.db lives.
|
|
3351
|
-
const orchTreeRoot = probe.mechanism === "uid-pool" ? sharedRoot() : stateDir();
|
|
3352
|
-
if (orchTreeRoot !== stateDir()) {
|
|
3353
|
-
mkdirSync(orchTreeRoot, { recursive: true });
|
|
3354
|
-
chmodSync(orchTreeRoot, 0o711);
|
|
3355
|
-
}
|
|
3356
|
-
const orchEnvRoot = join(orchTreeRoot, "boundaries", "orchestrator");
|
|
3039
|
+
// The orchestrator is a child process of this daemon running as its own
|
|
3040
|
+
// user. Nothing mechanically stops it reading a run checkout — what holds
|
|
3041
|
+
// it is its brief and the verb ledger (#143).
|
|
3042
|
+
const orchTreeRoot = stateDir();
|
|
3357
3043
|
const orchCwd = join(orchTreeRoot, "orchestrator");
|
|
3358
3044
|
mkdirSync(orchCwd, { recursive: true });
|
|
3359
|
-
const orchestratorBoundary = buildSessionBoundary({
|
|
3360
|
-
probe,
|
|
3361
|
-
isolation: credentials.isolation,
|
|
3362
|
-
role: "orchestrator",
|
|
3363
|
-
envRoot: orchEnvRoot,
|
|
3364
|
-
writeRoots: [orchCwd],
|
|
3365
|
-
denyReadRoots: [...credentialDenyRoots(), project.workspaceRoot, project.mirrorRoot],
|
|
3366
|
-
denyReadFiles: credentialDenyFiles(),
|
|
3367
|
-
...(credentials.readToken === undefined ? {} : { readToken: credentials.readToken }),
|
|
3368
|
-
});
|
|
3369
|
-
if (orchestratorBoundary.principal !== undefined) {
|
|
3370
|
-
const gid = probe.daemonGid;
|
|
3371
|
-
if (gid === undefined) {
|
|
3372
|
-
throw new Error(
|
|
3373
|
-
`the orchestrator principal was allocated without a resolved ${CONDUCTOR_GROUPS.daemon} gid`,
|
|
3374
|
-
);
|
|
3375
|
-
}
|
|
3376
|
-
applyRunOwnership(orchEnvRoot, orchestratorBoundary.principal, gid);
|
|
3377
|
-
applyRunOwnership(orchCwd, orchestratorBoundary.principal, gid);
|
|
3378
|
-
// The same empirical gate a worker gets. The orchestrator is the
|
|
3379
|
-
// session with merge and release authority, so "it cannot reach the
|
|
3380
|
-
// credential" matters more here, not less — and it was the half that
|
|
3381
|
-
// was never checked. `sharedGroup: false` because it is launched with
|
|
3382
|
-
// no supplementary group at all, so the probe has to run under exactly
|
|
3383
|
-
// that identity rather than a worker's.
|
|
3384
|
-
const orchLeaking = await credentialReadRefusal(
|
|
3385
|
-
spawnCaptured,
|
|
3386
|
-
orchestratorBoundary.principal,
|
|
3387
|
-
{ boundingSet: probe.boundingSetDropped, sharedGroup: false },
|
|
3388
|
-
[...credentialDenyRoots(), ...credentialDenyFiles()],
|
|
3389
|
-
);
|
|
3390
|
-
if (orchLeaking !== undefined) throw new Error(orchLeaking);
|
|
3391
|
-
}
|
|
3392
3045
|
// A third socket, distinct from every run's, in the same daemon-owned
|
|
3393
3046
|
// 0711 parent. This is what makes "merge authority is the orchestrator's"
|
|
3394
3047
|
// a property of the channel: the daemon knows which session is speaking
|
|
@@ -3401,9 +3054,6 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3401
3054
|
path: verbSocketPath(ensureVerbSocketDir(orchTreeRoot), "orchestrator"),
|
|
3402
3055
|
project: project.name,
|
|
3403
3056
|
role: "orchestrator",
|
|
3404
|
-
...(orchestratorBoundary.principal === undefined
|
|
3405
|
-
? {}
|
|
3406
|
-
: { principal: orchestratorBoundary.principal }),
|
|
3407
3057
|
},
|
|
3408
3058
|
{ ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) },
|
|
3409
3059
|
);
|
|
@@ -3411,8 +3061,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3411
3061
|
cwd: orchCwd,
|
|
3412
3062
|
brief,
|
|
3413
3063
|
releaseGrants,
|
|
3414
|
-
|
|
3415
|
-
socketPath: join(orchEnvRoot, "ipc.sock"),
|
|
3064
|
+
socketPath: join(orchCwd, "ipc.sock"),
|
|
3416
3065
|
verbSocketPath: orchestratorVerbs.path,
|
|
3417
3066
|
onChildLog: (line) => {
|
|
3418
3067
|
log(`orchestrator ${line}`);
|
|
@@ -3465,7 +3114,6 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3465
3114
|
const d: Deps = {
|
|
3466
3115
|
project,
|
|
3467
3116
|
caps,
|
|
3468
|
-
boundary,
|
|
3469
3117
|
tracker,
|
|
3470
3118
|
store,
|
|
3471
3119
|
// Process-wide, so a `status` served off this daemon's own HTTP surface
|
|
@@ -3572,12 +3220,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3572
3220
|
project: project.name,
|
|
3573
3221
|
store,
|
|
3574
3222
|
turnLimits,
|
|
3575
|
-
health: () =>
|
|
3576
|
-
daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph, {
|
|
3577
|
-
isolation: boundary.isolation,
|
|
3578
|
-
mechanism: boundary.probe.mechanism,
|
|
3579
|
-
headline: describeBoundary(boundary.isolation, boundary.probe).headline,
|
|
3580
|
-
}),
|
|
3223
|
+
health: () => daemonHealthSnapshot(store, project.name, undefined, undefined, codeGraph),
|
|
3581
3224
|
}),
|
|
3582
3225
|
});
|
|
3583
3226
|
log(`serving /healthz on :${server.port}, project ${project.name}`);
|