omp-conductor 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -269
- package/package.json +1 -1
- package/skills/conductor-onboarding/SKILL.md +0 -56
- package/skills/conductor-update/SKILL.md +4 -9
- package/src/briefs/orchestrator.md +13 -19
- package/src/briefs/worker.md +18 -16
- package/src/cli.ts +1 -48
- package/src/config.ts +7 -224
- package/src/confinement.ts +2 -3
- package/src/daemon.ts +49 -406
- package/src/failure-class.ts +44 -0
- package/src/fleet.ts +1 -85
- package/src/gitops.ts +189 -0
- package/src/omp.ts +11 -35
- package/src/orchestrator.ts +0 -4
- package/src/plugin.ts +1 -42
- package/src/session-host.ts +12 -19
- package/src/setup-host.ts +1 -157
- package/src/setup.ts +6 -41
- package/src/store.ts +36 -2
- package/src/tracker/github.ts +1 -1
- package/src/types.ts +1 -85
- package/src/upgrade.ts +0 -88
- package/src/verbs/actions.ts +1 -1
- package/src/verbs/server.ts +3 -28
- package/src/verbs/socket.ts +15 -23
- package/src/worker.ts +1 -13
- package/src/worktree.ts +4 -4
- package/systemd/omp-conductor.service.example +8 -94
- package/src/credentials.ts +0 -2029
package/src/setup-host.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { homedir, userInfo } from "node:os";
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
|
-
import { configPath,
|
|
4
|
+
import { configPath, stateDir } from "./config.ts";
|
|
5
5
|
import { isPaused, runDaemon, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
|
|
6
6
|
import { DEFAULT_FLEET_AGENT_NAME } from "./fleet.ts";
|
|
7
7
|
import {
|
|
@@ -110,28 +110,6 @@ export function renderDaemonService(
|
|
|
110
110
|
`Environment=${systemdQuote(`PATH=${runtime.path}`)}`,
|
|
111
111
|
`Environment=${systemdQuote(`OMP_CONDUCTOR_HOME=${runtime.conductorHome}`)}`,
|
|
112
112
|
`Environment=${systemdQuote(`OMP_TELEGRAM_STATE_DIR=${runtime.telegramStateDir}`)}`,
|
|
113
|
-
// Both of these are per-run only, and grouped so the managed set stays one
|
|
114
|
-
// idea. The shared root is baked in so a custom location survives systemd's
|
|
115
|
-
// clean environment — without it the daemon resolves the platform default
|
|
116
|
-
// while the operator provisioned somewhere else, and every dispatch is
|
|
117
|
-
// refused for a path nobody chose. It is meaningless under `none`, where
|
|
118
|
-
// nothing resolves it, and rendering it there would make every unit
|
|
119
|
-
// installed before 0.4.0 look drifted for no reason (#125).
|
|
120
|
-
//
|
|
121
|
-
// The capabilities exist to be DROPPED INTO run children by the
|
|
122
|
-
// privilege-dropping launcher, never inherited by them — see `setprivArgv`.
|
|
123
|
-
// Rendered only for `per-run`, because that is the only isolation that
|
|
124
|
-
// changes uid; granting them to a fleet that does not need them widens the
|
|
125
|
-
// daemon for nothing. Without them the probe honestly reports `group-mode`
|
|
126
|
-
// and a configured per-run fleet refuses every dispatch, which is the
|
|
127
|
-
// failure an operator following setup would otherwise hit first.
|
|
128
|
-
...(resolveCredentials(project).isolation === "per-run"
|
|
129
|
-
? [
|
|
130
|
-
`Environment=${systemdQuote(`OMP_CONDUCTOR_SHARED=${sharedRoot()}`)}`,
|
|
131
|
-
"AmbientCapabilities=CAP_SETUID CAP_SETGID CAP_CHOWN CAP_FOWNER CAP_SETPCAP",
|
|
132
|
-
"CapabilityBoundingSet=CAP_SETUID CAP_SETGID CAP_CHOWN CAP_FOWNER CAP_SETPCAP",
|
|
133
|
-
]
|
|
134
|
-
: []),
|
|
135
113
|
`WorkingDirectory=${systemdQuote(stateDir())}`,
|
|
136
114
|
`ExecStart=${command.map(systemdQuote).join(" ")}`,
|
|
137
115
|
"Restart=on-failure",
|
|
@@ -310,137 +288,3 @@ export async function runSetupSmoke(
|
|
|
310
288
|
export function installedUnitPath(): string {
|
|
311
289
|
return join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME);
|
|
312
290
|
}
|
|
313
|
-
|
|
314
|
-
/**
|
|
315
|
-
* Why the *live* systemd unit no longer matches what this version would render.
|
|
316
|
-
*
|
|
317
|
-
* `planHostRuntime` compares against the **staged** copy under the state dir,
|
|
318
|
-
* which says nothing about the file systemd booted from. That gap is not
|
|
319
|
-
* cosmetic since 0.4.0: a fleet configured `per-run` whose installed unit
|
|
320
|
-
* predates the capability grant probes down to `group-mode` and then refuses
|
|
321
|
-
* every dispatch — and `upgrade` would happily verify package identities,
|
|
322
|
-
* services, pane and ticks around it, because none of those look at the unit.
|
|
323
|
-
*
|
|
324
|
-
* Checked by directive rather than by whole-file equality on purpose. An
|
|
325
|
-
* operator is entitled to hand-tune `MemoryMax`, add `After=`, or set an
|
|
326
|
-
* `Environment=` of their own, and failing an upgrade over that would teach
|
|
327
|
-
* them to stop running upgrades. What is reported is the set of directives this
|
|
328
|
-
* configuration *requires* and the live unit lacks.
|
|
329
|
-
*
|
|
330
|
-
* `undefined` means either "no drift that matters" or "no installed unit at
|
|
331
|
-
* all" — a host running `omp-conductor start` without systemd is not broken,
|
|
332
|
-
* and must not be told it is.
|
|
333
|
-
*/
|
|
334
|
-
/**
|
|
335
|
-
* What systemd has actually loaded for the unit, as `systemctl show` reports it.
|
|
336
|
-
*
|
|
337
|
-
* Deliberately not the file on disk. A unit installed without `daemon-reload`
|
|
338
|
-
* can match the rendered text byte for byte while the manager still runs the
|
|
339
|
-
* previous configuration, and a drop-in under `…service.d/` changes what runs
|
|
340
|
-
* without touching the file at all. Both read clean from the filesystem.
|
|
341
|
-
*/
|
|
342
|
-
export interface EffectiveUnit {
|
|
343
|
-
/** `systemctl show -p AmbientCapabilities`; empty string when unset. */
|
|
344
|
-
ambientCapabilities: string;
|
|
345
|
-
capabilityBoundingSet: string;
|
|
346
|
-
/** `systemctl show -p Environment`, newline- or space-joined. */
|
|
347
|
-
environment: string;
|
|
348
|
-
needDaemonReload: boolean;
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
/** Capability names as a comparable set: systemd reports them lowercased, `cap_`-prefixed and reordered. */
|
|
352
|
-
function capabilitySet(text: string): Set<string> {
|
|
353
|
-
return new Set(
|
|
354
|
-
text
|
|
355
|
-
.split(/[\s,]+/)
|
|
356
|
-
.map((name) => name.trim().toLowerCase().replace(/^cap_/, ""))
|
|
357
|
-
.filter((name) => name.length > 0),
|
|
358
|
-
);
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
function renderedDirective(rendered: string, key: string): string | undefined {
|
|
362
|
-
for (const line of rendered.split("\n")) {
|
|
363
|
-
const trimmed = line.trim();
|
|
364
|
-
if (trimmed.startsWith(`${key}=`)) return trimmed.slice(key.length + 1);
|
|
365
|
-
}
|
|
366
|
-
return undefined;
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
/**
|
|
370
|
-
* Why what systemd loaded no longer matches what this version would render.
|
|
371
|
-
*
|
|
372
|
-
* Three asymmetries, each for a reason:
|
|
373
|
-
*
|
|
374
|
-
* - **Ambient capabilities are compared exactly, both ways.** Missing them
|
|
375
|
-
* breaks a per-run fleet; *leftover* ones are worse. Roll back to `none` and
|
|
376
|
-
* there is no `setpriv` launcher at all, so ambient capabilities on the unit
|
|
377
|
-
* are inherited straight into model-executed code, which can then `setuid()`
|
|
378
|
-
* to any account. `systemctl` reports an empty string when unset, so the
|
|
379
|
-
* rollback case is unambiguous.
|
|
380
|
-
* - **The bounding set is checked only when this version sets it.** systemd
|
|
381
|
-
* reports the full default set when a unit does not, so comparing both ways
|
|
382
|
-
* would flag every ordinary host.
|
|
383
|
-
* - **Only directives this configuration manages are considered**, so an
|
|
384
|
-
* operator's own `MemoryMax`, `After=` or drop-in never fails an upgrade.
|
|
385
|
-
*/
|
|
386
|
-
export function unitDriftReason(rendered: string, effective: EffectiveUnit | undefined): string | undefined {
|
|
387
|
-
if (effective === undefined) return undefined;
|
|
388
|
-
const problems: string[] = [];
|
|
389
|
-
|
|
390
|
-
const wantAmbient = capabilitySet(renderedDirective(rendered, "AmbientCapabilities") ?? "");
|
|
391
|
-
const haveAmbient = capabilitySet(effective.ambientCapabilities);
|
|
392
|
-
const missingAmbient = [...wantAmbient].filter((c) => !haveAmbient.has(c));
|
|
393
|
-
const extraAmbient = [...haveAmbient].filter((c) => !wantAmbient.has(c));
|
|
394
|
-
if (missingAmbient.length > 0) {
|
|
395
|
-
problems.push(`AmbientCapabilities is missing ${missingAmbient.join(", ")}`);
|
|
396
|
-
}
|
|
397
|
-
if (extraAmbient.length > 0) {
|
|
398
|
-
problems.push(
|
|
399
|
-
`AmbientCapabilities grants ${extraAmbient.join(", ")} that this configuration does not — with ` +
|
|
400
|
-
`isolation off there is no privilege-dropping launcher, so these are inherited by model-executed code`,
|
|
401
|
-
);
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
const wantBounding = renderedDirective(rendered, "CapabilityBoundingSet");
|
|
405
|
-
if (wantBounding !== undefined) {
|
|
406
|
-
const missing = [...capabilitySet(wantBounding)].filter(
|
|
407
|
-
(c) => !capabilitySet(effective.capabilityBoundingSet).has(c),
|
|
408
|
-
);
|
|
409
|
-
if (missing.length > 0) problems.push(`CapabilityBoundingSet is missing ${missing.join(", ")}`);
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
const wantShared = rendered
|
|
413
|
-
.split("\n")
|
|
414
|
-
.map((l) => l.trim())
|
|
415
|
-
.find((l) => l.includes("OMP_CONDUCTOR_SHARED="));
|
|
416
|
-
if (wantShared !== undefined) {
|
|
417
|
-
const value = wantShared.slice(wantShared.indexOf("OMP_CONDUCTOR_SHARED=")).replace(/"$/, "");
|
|
418
|
-
if (!effective.environment.includes(value)) {
|
|
419
|
-
problems.push(`Environment is missing ${value}`);
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
if (effective.needDaemonReload) {
|
|
424
|
-
problems.push("systemd reports the unit needs a daemon-reload, so what runs is not what is on disk");
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
if (problems.length === 0) return undefined;
|
|
428
|
-
return (
|
|
429
|
-
`what systemd loaded for ${STAGED_SERVICE_NAME} does not match this configuration: ${problems.join("; ")}.`
|
|
430
|
-
);
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
/** Parse `systemctl show -p …` key=value output into an {@link EffectiveUnit}. */
|
|
434
|
-
export function parseEffectiveUnit(stdout: string): EffectiveUnit {
|
|
435
|
-
const values = new Map<string, string>();
|
|
436
|
-
for (const line of stdout.split("\n")) {
|
|
437
|
-
const at = line.indexOf("=");
|
|
438
|
-
if (at > 0) values.set(line.slice(0, at).trim(), line.slice(at + 1).trim());
|
|
439
|
-
}
|
|
440
|
-
return {
|
|
441
|
-
ambientCapabilities: values.get("AmbientCapabilities") ?? "",
|
|
442
|
-
capabilityBoundingSet: values.get("CapabilityBoundingSet") ?? "",
|
|
443
|
-
environment: values.get("Environment") ?? "",
|
|
444
|
-
needDaemonReload: (values.get("NeedDaemonReload") ?? "no") === "yes",
|
|
445
|
-
};
|
|
446
|
-
}
|
package/src/setup.ts
CHANGED
|
@@ -35,11 +35,11 @@ import {
|
|
|
35
35
|
import {
|
|
36
36
|
clonePolicy,
|
|
37
37
|
configPath,
|
|
38
|
+
defaultMirrorRoot,
|
|
39
|
+
defaultWorkspaceRoot,
|
|
38
40
|
resolveCaps,
|
|
39
|
-
resolveCredentials,
|
|
40
41
|
resolvePolicy,
|
|
41
42
|
resolveReleaseGrants,
|
|
42
|
-
projectTreeBase,
|
|
43
43
|
stateDir,
|
|
44
44
|
} from "./config.ts";
|
|
45
45
|
import { graphProjectPath, graphRepos } from "./graph.ts";
|
|
@@ -51,8 +51,6 @@ import {
|
|
|
51
51
|
DEFAULT_REPORT_SCOPE,
|
|
52
52
|
DENIED_RELEASE_GRANTS,
|
|
53
53
|
RELEASE_SHAPES,
|
|
54
|
-
type CredentialConfig,
|
|
55
|
-
type CredentialIsolation,
|
|
56
54
|
type BaseFreshness,
|
|
57
55
|
type BehindBaseAction,
|
|
58
56
|
type Caps,
|
|
@@ -103,13 +101,6 @@ export interface SetupAnswers {
|
|
|
103
101
|
* leave one to be defaulted by whichever reader gets there first.
|
|
104
102
|
*/
|
|
105
103
|
policy: ProjectPolicy;
|
|
106
|
-
/**
|
|
107
|
-
* Whether model-executed code runs under its own OS principal (#125). The
|
|
108
|
-
* wizard offers `per-run` only when the host probe says this box can build
|
|
109
|
-
* the boundary, because an answer the host cannot honour is a fleet that
|
|
110
|
-
* refuses to dispatch rather than a fleet that is protected.
|
|
111
|
-
*/
|
|
112
|
-
credentials: CredentialConfig;
|
|
113
104
|
/**
|
|
114
105
|
* Whether to render `ORCHESTRATOR.md` into the project's workspace root. Not
|
|
115
106
|
* part of the config — the brief is the operator's file, and the conductor
|
|
@@ -174,13 +165,6 @@ export const SETUP_DEFAULTS = {
|
|
|
174
165
|
releaseGrants: DENIED_RELEASE_GRANTS,
|
|
175
166
|
/** The strictest reading of the prose these conditions replaced (#129). */
|
|
176
167
|
policy: DEFAULT_PROJECT_POLICY,
|
|
177
|
-
/**
|
|
178
|
-
* No boundary until an operator asks for one on a host that can build it.
|
|
179
|
-
* `none` is the honest default rather than the safe-sounding one: writing
|
|
180
|
-
* `per-run` into a fresh config on a box with no mechanism would produce a
|
|
181
|
-
* conductor that installs cleanly and then declines every issue (#125).
|
|
182
|
-
*/
|
|
183
|
-
credentials: { isolation: "none" } as CredentialConfig,
|
|
184
168
|
/** The daemon runs its own triage session unless an operator already runs one. */
|
|
185
169
|
orchestratorMode: "embedded",
|
|
186
170
|
} as const;
|
|
@@ -520,11 +504,6 @@ export async function createMissingLabels(trackerRepo: string, plan: LabelPlan[]
|
|
|
520
504
|
* with {@link answersFromProject} — the pair an amend's carry-through rests on.
|
|
521
505
|
*/
|
|
522
506
|
export function buildProject(a: SetupAnswers): ProjectConfig {
|
|
523
|
-
// Under a per-run boundary the worktrees and mirrors have to sit where a slot
|
|
524
|
-
// principal can actually traverse to them. Defaulting them under the 0700
|
|
525
|
-
// state directory would provision a repository and then be refused at
|
|
526
|
-
// dispatch, which looks like a broken install rather than a layout choice.
|
|
527
|
-
const dir = projectTreeBase(a.credentials.isolation);
|
|
528
507
|
|
|
529
508
|
const repos: Record<string, RepoTarget> = {};
|
|
530
509
|
const graphRoot = a.graphRoot?.trim();
|
|
@@ -580,14 +559,13 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
|
|
|
580
559
|
// Written out in full for the same reason: the file then says what a merge
|
|
581
560
|
// and a release require without anyone having to know a default (#129).
|
|
582
561
|
policy: clonePolicy(a.policy),
|
|
583
|
-
// Written out even when it is the default, so
|
|
584
|
-
//
|
|
585
|
-
credentials: { ...a.credentials },
|
|
562
|
+
// Written out even when it is the default, so an operator amending the
|
|
563
|
+
// volume has a line in the file to point at.
|
|
586
564
|
reporting: { scope: a.reportScope },
|
|
587
565
|
// Both under the state dir so one `rm -rf ~/.omp/conductor` is a complete
|
|
588
566
|
// uninstall, and neither can land in a repo the daemon then tries to commit.
|
|
589
|
-
workspaceRoot:
|
|
590
|
-
mirrorRoot:
|
|
567
|
+
workspaceRoot: defaultWorkspaceRoot(),
|
|
568
|
+
mirrorRoot: defaultMirrorRoot(),
|
|
591
569
|
};
|
|
592
570
|
}
|
|
593
571
|
|
|
@@ -641,7 +619,6 @@ export function defaultAnswers(projectName: string): SetupAnswers {
|
|
|
641
619
|
authority: { ...SETUP_DEFAULTS.authority },
|
|
642
620
|
releaseGrants: { ...SETUP_DEFAULTS.releaseGrants },
|
|
643
621
|
policy: clonePolicy(SETUP_DEFAULTS.policy),
|
|
644
|
-
credentials: { ...SETUP_DEFAULTS.credentials },
|
|
645
622
|
orchestratorMode: SETUP_DEFAULTS.orchestratorMode,
|
|
646
623
|
reportScope: SETUP_DEFAULT_REPORT_SCOPE,
|
|
647
624
|
writeOrchestratorBrief: false,
|
|
@@ -689,7 +666,6 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
|
|
|
689
666
|
authority: { ...p.authority },
|
|
690
667
|
releaseGrants: resolveReleaseGrants(p),
|
|
691
668
|
policy: resolvePolicy(p),
|
|
692
|
-
credentials: resolveCredentials(p),
|
|
693
669
|
orchestratorMode: p.escalation.orchestrator,
|
|
694
670
|
reportScope: p.reporting?.scope ?? DEFAULT_REPORT_SCOPE,
|
|
695
671
|
writeOrchestratorBrief: false,
|
|
@@ -1137,7 +1113,6 @@ export const AMEND_AREA_IDS = [
|
|
|
1137
1113
|
"graph",
|
|
1138
1114
|
"authority",
|
|
1139
1115
|
"policy",
|
|
1140
|
-
"credentials",
|
|
1141
1116
|
"escalation",
|
|
1142
1117
|
"reporting",
|
|
1143
1118
|
"brief",
|
|
@@ -1243,16 +1218,6 @@ export const AMEND_AREAS: {
|
|
|
1243
1218
|
);
|
|
1244
1219
|
},
|
|
1245
1220
|
},
|
|
1246
|
-
credentials: {
|
|
1247
|
-
name: "credential isolation",
|
|
1248
|
-
asks: "whether worker and orchestrator sessions run under their own OS principal, or as the daemon's own user",
|
|
1249
|
-
describe: (p) => {
|
|
1250
|
-
const c = resolveCredentials(p);
|
|
1251
|
-
return c.isolation === "per-run"
|
|
1252
|
-
? `per-run principals${c.readToken === undefined ? "" : ", with a read-scoped token"}`
|
|
1253
|
-
: "none — sessions run as the daemon's user and can reach its credentials";
|
|
1254
|
-
},
|
|
1255
|
-
},
|
|
1256
1221
|
escalation: {
|
|
1257
1222
|
name: "escalation & triage",
|
|
1258
1223
|
asks: "the tier-2 Telegram chat, whether escalations also comment, and where the orchestrator session lives",
|
package/src/store.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { dirname, join } from "node:path";
|
|
|
15
15
|
|
|
16
16
|
import { stateDir } from "./config.ts";
|
|
17
17
|
import { DECISION_TTL_MS, DEFAULT_CAPS } from "./types.ts";
|
|
18
|
+
import { startFailure } from "./failure-class.ts";
|
|
18
19
|
import type {
|
|
19
20
|
DecisionDraft,
|
|
20
21
|
FailureClass,
|
|
@@ -678,6 +679,39 @@ export function openStore(dbPath: string): Store {
|
|
|
678
679
|
}
|
|
679
680
|
}
|
|
680
681
|
|
|
682
|
+
// One-time repair, and the reason it lives here rather than in the classifier:
|
|
683
|
+
// a turn-0 environment fault that was ALREADY classified `unknown` and
|
|
684
|
+
// escalated is never revisited. `runsNeedingClassification` only offers a row
|
|
685
|
+
// whose class is NULL, or one whose recovery has not run yet — an
|
|
686
|
+
// `unknown`/`escalate` row with `recoveredAt` set satisfies neither, so no
|
|
687
|
+
// upgrade and no later tick would ever correct it.
|
|
688
|
+
//
|
|
689
|
+
// That mattered on a real fleet. The per-run `$HOME` redirect this release
|
|
690
|
+
// deletes hid the harness's own credentials, so every dispatch died before it
|
|
691
|
+
// read its issue and each one still spent an implementation attempt. Deleting
|
|
692
|
+
// the redirect stops the bleeding; it does not give back attempts already
|
|
693
|
+
// charged, and an issue sitting at its cap for a fault that was never in its
|
|
694
|
+
// code stays undispatchable forever.
|
|
695
|
+
//
|
|
696
|
+
// Narrow by construction: `turns = 0` (the session never took a turn), an
|
|
697
|
+
// error the classifier itself recognises as a start failure, and a class of
|
|
698
|
+
// exactly `unknown` — a row a human or a later release classified as anything
|
|
699
|
+
// else is left alone. Idempotent for free: afterwards those rows read
|
|
700
|
+
// `env-start-failure`, which the WHERE no longer matches.
|
|
701
|
+
const misread = db
|
|
702
|
+
.query<
|
|
703
|
+
{ id: string; lastError: string | null },
|
|
704
|
+
[]
|
|
705
|
+
>(`SELECT id, lastError FROM runs WHERE failureClass = 'unknown' AND turns = 0 AND lastError IS NOT NULL`)
|
|
706
|
+
.all()
|
|
707
|
+
.filter((row) => startFailure(row.lastError ?? undefined) !== undefined);
|
|
708
|
+
if (misread.length > 0) {
|
|
709
|
+
const reclassify = db.query<unknown, [string]>(
|
|
710
|
+
`UPDATE runs SET failureClass = 'env-start-failure' WHERE id = ?`,
|
|
711
|
+
);
|
|
712
|
+
for (const row of misread) reclassify.run(row.id);
|
|
713
|
+
}
|
|
714
|
+
|
|
681
715
|
const insertRun = db.query<unknown, SqlValue[]>(
|
|
682
716
|
`INSERT INTO runs (
|
|
683
717
|
id, project, issue, repo, branch, worktree, state, attempt, turns,
|
|
@@ -725,12 +759,12 @@ export function openStore(dbPath: string): Store {
|
|
|
725
759
|
const countFailures = db.query<{ n: number }, [string, number]>(
|
|
726
760
|
`SELECT COUNT(*) AS n FROM runs
|
|
727
761
|
WHERE project = ? AND issue = ? AND state = 'failed'
|
|
728
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck'))`,
|
|
762
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure'))`,
|
|
729
763
|
);
|
|
730
764
|
const countContinuations = db.query<{ n: number }, [string, number]>(
|
|
731
765
|
`SELECT COUNT(*) AS n FROM runs
|
|
732
766
|
WHERE project = ? AND issue = ? AND state IN ('killed', 'orphaned', 'blocked')
|
|
733
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck'))`,
|
|
767
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure'))`,
|
|
734
768
|
);
|
|
735
769
|
// Newest first, and bounded: every row this returns costs `gh` calls to gather
|
|
736
770
|
// facts for, so a fleet with a long unclassified history classifies over
|
package/src/tracker/github.ts
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
* `gh auth token`; the eleven Tracker methods above it stay untouched.
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
import { credentialedEnv } from "../
|
|
24
|
+
import { credentialedEnv } from "../gitops.ts";
|
|
25
25
|
import { parsePrDiff } from "../diff-flags.ts";
|
|
26
26
|
import type {
|
|
27
27
|
IssueState,
|
package/src/types.ts
CHANGED
|
@@ -457,76 +457,6 @@ export type ReleaseJustification = VerbJustification<ReleaseReason>;
|
|
|
457
457
|
|
|
458
458
|
export type LabelJustification = VerbJustification<LabelReason>;
|
|
459
459
|
|
|
460
|
-
/**
|
|
461
|
-
* How hard a boundary separates model-executed code from the operator's GitHub
|
|
462
|
-
* write credential (#125).
|
|
463
|
-
*
|
|
464
|
-
* Three values, ordered strongest first, because the difference between them is
|
|
465
|
-
* a difference in what anyone can honestly claim:
|
|
466
|
-
*
|
|
467
|
-
* - `per-run` — every session runs as its own OS principal that cannot read the
|
|
468
|
-
* daemon's `gh` config, keychain, `~/.ssh` or `~/.npmrc`, and cannot write a
|
|
469
|
-
* sibling run's checkout. Satisfied **only** by the `uid-pool` and
|
|
470
|
-
* `sandbox-exec` mechanisms.
|
|
471
|
-
* - `group-mode` — same uid as the daemon, cross-run separation by group and
|
|
472
|
-
* mode only. It bounds accidents and does **not** contain a determined bash
|
|
473
|
-
* escape. It exists as a config value rather than as a fallback so that an
|
|
474
|
-
* operator who takes it has said the weaker sentence out loud: a `per-run`
|
|
475
|
-
* request that quietly resolved to this would let somebody believe they had
|
|
476
|
-
* configured the full boundary and shipped the lesser one, which is precisely
|
|
477
|
-
* the "never the silent default" failure #125 exists to end.
|
|
478
|
-
* - `none` — the session runs as the daemon's own user and the env scrubbing in
|
|
479
|
-
* `credentials.ts` is *all* that stands between the model and the credential,
|
|
480
|
-
* which same-uid code defeats in one line.
|
|
481
|
-
*
|
|
482
|
-
* A request is never downgraded. If the host cannot build what was asked for,
|
|
483
|
-
* dispatch refuses and names the missing mechanism.
|
|
484
|
-
*/
|
|
485
|
-
export const CREDENTIAL_ISOLATIONS = ["per-run", "group-mode", "none"] as const;
|
|
486
|
-
|
|
487
|
-
export type CredentialIsolation = (typeof CREDENTIAL_ISOLATIONS)[number];
|
|
488
|
-
|
|
489
|
-
/**
|
|
490
|
-
* What the host can actually enforce, decided by the startup probe rather than
|
|
491
|
-
* by config. Kept separate from {@link CREDENTIAL_ISOLATIONS} on purpose: the
|
|
492
|
-
* operator asks for a boundary, the host says which one it can build, and
|
|
493
|
-
* `status` reports the difference instead of letting either side guess.
|
|
494
|
-
*
|
|
495
|
-
* - `uid-pool` — one unprivileged account per run slot, entered through a
|
|
496
|
-
* privilege-dropping launcher. The only mechanism that claims to contain a
|
|
497
|
-
* determined bash escape on Linux.
|
|
498
|
-
* - `group-mode` — same uid as the daemon, cross-run separation by group and
|
|
499
|
-
* mode only. Bounds accidents, does **not** contain an escape.
|
|
500
|
-
* - `sandbox-exec` — the macOS equivalent of `uid-pool` for a dev host.
|
|
501
|
-
* - `none` — no boundary at all.
|
|
502
|
-
*/
|
|
503
|
-
export const ISOLATION_MECHANISMS = ["uid-pool", "group-mode", "sandbox-exec", "none"] as const;
|
|
504
|
-
|
|
505
|
-
export type IsolationMechanism = (typeof ISOLATION_MECHANISMS)[number];
|
|
506
|
-
|
|
507
|
-
/**
|
|
508
|
-
* The two groups the filesystem model needs, named once so the probe, the
|
|
509
|
-
* provisioning docs and the adversarial tests cannot drift apart (#125).
|
|
510
|
-
*
|
|
511
|
-
* `daemon` holds the daemon account **only** and is what lets it fetch, salvage
|
|
512
|
-
* and reclaim every run repo. `runs` holds every slot principal and grants
|
|
513
|
-
* read-only access to the shared mirror. A slot principal in `daemon` would
|
|
514
|
-
* void the whole cross-run boundary, so the probe asserts the absence.
|
|
515
|
-
*/
|
|
516
|
-
export const CONDUCTOR_GROUPS = { daemon: "conductor-daemon", runs: "conductor-runs" } as const;
|
|
517
|
-
|
|
518
|
-
/** Per-project credential boundary settings. See {@link CREDENTIAL_ISOLATIONS}. */
|
|
519
|
-
export interface CredentialConfig {
|
|
520
|
-
isolation: CredentialIsolation;
|
|
521
|
-
/**
|
|
522
|
-
* Optional **read-scoped** token handed to sessions so they can look things
|
|
523
|
-
* up on GitHub. Absent is the default and the safe reading: a worker then
|
|
524
|
-
* works from its dispatch brief and the daemon's mediated verbs, which is
|
|
525
|
-
* why the brief must not tell it to run `gh pr view`.
|
|
526
|
-
*/
|
|
527
|
-
readToken?: string;
|
|
528
|
-
}
|
|
529
|
-
|
|
530
460
|
/**
|
|
531
461
|
* Where the session that triages escalations lives. `embedded` is the daemon's
|
|
532
462
|
* own child session; `external` means an operator already runs one — a visible
|
|
@@ -601,16 +531,6 @@ export interface ProjectConfig {
|
|
|
601
531
|
* `resolveReportScope` rather than reaching for `.scope` directly.
|
|
602
532
|
*/
|
|
603
533
|
reporting?: { scope: ReportScope };
|
|
604
|
-
/**
|
|
605
|
-
* The credential boundary this project's sessions run behind (#125).
|
|
606
|
-
*
|
|
607
|
-
* Optional only on disk, and only for one release: a config written before
|
|
608
|
-
* this key existed is migrated to an explicit `{ isolation: "none" }` and
|
|
609
|
-
* rewritten, so the operator ends up with the answer in the file rather than
|
|
610
|
-
* inheriting a silent default. Never read directly — go through
|
|
611
|
-
* `resolveCredentials`, which is where that migration is spelled once.
|
|
612
|
-
*/
|
|
613
|
-
credentials?: CredentialConfig;
|
|
614
534
|
/** Parent directory for per-run worktrees. */
|
|
615
535
|
workspaceRoot: string;
|
|
616
536
|
/** Cache of bare clones, so N runs share one fetch instead of N. */
|
|
@@ -877,6 +797,7 @@ export interface Tracker {
|
|
|
877
797
|
* answers instead of re-investigating.
|
|
878
798
|
*/
|
|
879
799
|
export const FAILURE_CLASSES = [
|
|
800
|
+
"env-start-failure",
|
|
880
801
|
"turn-cap-progress",
|
|
881
802
|
"turn-cap-spinning",
|
|
882
803
|
"admin-kill",
|
|
@@ -1000,11 +921,6 @@ export type AdmissionHoldReason =
|
|
|
1000
921
|
| "unsalvaged-wip"
|
|
1001
922
|
| "daily-spend-cap"
|
|
1002
923
|
| "plan-usage-cap"
|
|
1003
|
-
/** `credentials.isolation: "per-run"` on a host whose probe found no
|
|
1004
|
-
* mechanism. The operator asked for a boundary this host cannot build, so
|
|
1005
|
-
* dispatch refuses rather than running unprotected under a config that says
|
|
1006
|
-
* otherwise (#125). */
|
|
1007
|
-
| "credential-boundary"
|
|
1008
924
|
| "unroutable:no-repo-label"
|
|
1009
925
|
| "unroutable:multiple-repo-labels"
|
|
1010
926
|
| "unroutable:unknown-repo";
|
package/src/upgrade.ts
CHANGED
|
@@ -7,11 +7,8 @@ import { configPath, findProject, loadConfig, resolveCaps, writeConfigRaw } from
|
|
|
7
7
|
import { renderBriefForProject } from "./setup.ts";
|
|
8
8
|
import {
|
|
9
9
|
STAGED_SERVICE_NAME,
|
|
10
|
-
parseEffectiveUnit,
|
|
11
10
|
planHostRuntime,
|
|
12
|
-
unitDriftReason,
|
|
13
11
|
writeHostRuntime,
|
|
14
|
-
type EffectiveUnit,
|
|
15
12
|
} from "./setup-host.ts";
|
|
16
13
|
|
|
17
14
|
const PACKAGE = "omp-conductor";
|
|
@@ -41,15 +38,6 @@ export interface UpgradeResult {
|
|
|
41
38
|
dispatch: DispatchLayer;
|
|
42
39
|
}
|
|
43
40
|
|
|
44
|
-
/** What the unit check reads, and how it repairs what it can. */
|
|
45
|
-
export interface UnitFiles {
|
|
46
|
-
rendered: string;
|
|
47
|
-
/** What systemd loaded, not what is on disk. Undefined means no systemd here. */
|
|
48
|
-
effective: EffectiveUnit | undefined;
|
|
49
|
-
/** Writes the corrected unit to the state dir; returns the commands root must run. */
|
|
50
|
-
stage?(): readonly string[];
|
|
51
|
-
}
|
|
52
|
-
|
|
53
41
|
export interface UpgradeDeps {
|
|
54
42
|
run(command: string, args: readonly string[]): Promise<UpgradeCommandResult>;
|
|
55
43
|
snapshot(project?: string): { liveWorkers: number };
|
|
@@ -61,12 +49,6 @@ export interface UpgradeDeps {
|
|
|
61
49
|
sleep(ms: number): Promise<void>;
|
|
62
50
|
env: NodeJS.ProcessEnv;
|
|
63
51
|
log(message: string): void;
|
|
64
|
-
/**
|
|
65
|
-
* What this version would render as the unit, and what systemd actually
|
|
66
|
-
* booted from. Injected so the drift check is testable without a systemd host
|
|
67
|
-
* — and defaulted, so no production caller has to know it exists.
|
|
68
|
-
*/
|
|
69
|
-
unitFiles?(): Promise<UnitFiles | undefined> | UnitFiles | undefined;
|
|
70
52
|
}
|
|
71
53
|
|
|
72
54
|
async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
|
|
@@ -450,49 +432,6 @@ async function rollbackUpgrade(
|
|
|
450
432
|
if (failures.length > 0) throw new Error(failures.join("; "));
|
|
451
433
|
}
|
|
452
434
|
|
|
453
|
-
/**
|
|
454
|
-
* The rendered-vs-installed pair for {@link unitDriftReason}.
|
|
455
|
-
*
|
|
456
|
-
* Best-effort by design: a host with no config, no project, or no systemd is
|
|
457
|
-
* not a host with a broken unit, and an upgrade must not fail because it could
|
|
458
|
-
* not answer a question that does not apply there.
|
|
459
|
-
*/
|
|
460
|
-
async function defaultUnitFiles(deps: UpgradeDeps): Promise<UnitFiles | undefined> {
|
|
461
|
-
try {
|
|
462
|
-
const cfg = loadConfig();
|
|
463
|
-
const project = cfg.projects[0];
|
|
464
|
-
if (project === undefined) return undefined;
|
|
465
|
-
const plan = planHostRuntime(project, resolveCaps(project, cfg.defaults), telegramStateDir());
|
|
466
|
-
const shown = await deps.run("systemctl", [
|
|
467
|
-
"show",
|
|
468
|
-
STAGED_SERVICE_NAME,
|
|
469
|
-
"-p",
|
|
470
|
-
"AmbientCapabilities",
|
|
471
|
-
"-p",
|
|
472
|
-
"CapabilityBoundingSet",
|
|
473
|
-
"-p",
|
|
474
|
-
"Environment",
|
|
475
|
-
"-p",
|
|
476
|
-
"NeedDaemonReload",
|
|
477
|
-
]);
|
|
478
|
-
return {
|
|
479
|
-
rendered: plan.service.content,
|
|
480
|
-
effective: shown.code === 0 ? parseEffectiveUnit(shown.stdout) : undefined,
|
|
481
|
-
// Staging needs no privilege, so the repair is reduced to the two lines
|
|
482
|
-
// that genuinely do. Installing the unit is root's, and this command runs
|
|
483
|
-
// as the fleet user by design — so it goes as far as it can and then says
|
|
484
|
-
// exactly what is left, rather than sending the operator back through a
|
|
485
|
-
// wizard to regenerate a file it could write itself.
|
|
486
|
-
stage: () => {
|
|
487
|
-
writeHostRuntime(plan);
|
|
488
|
-
return plan.installCommands;
|
|
489
|
-
},
|
|
490
|
-
};
|
|
491
|
-
} catch {
|
|
492
|
-
return undefined;
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
|
-
|
|
496
435
|
export async function upgradeConductor(
|
|
497
436
|
options: UpgradeOptions = {},
|
|
498
437
|
overrides: Partial<UpgradeDeps> = {},
|
|
@@ -519,22 +458,6 @@ export async function upgradeConductor(
|
|
|
519
458
|
const surfaces = await inspectSurfaces(deps);
|
|
520
459
|
const brief = deps.brief(project);
|
|
521
460
|
const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
|
|
522
|
-
// Computed BEFORE the no-op decision, not inside verification, and that
|
|
523
|
-
// placement is the whole point. The upgrade that installs a version is run by
|
|
524
|
-
// the *previous* CLI, so a check living only in the new code never executes
|
|
525
|
-
// for the release that introduces it — and re-running afterwards would take
|
|
526
|
-
// the already-current early return and skip it forever. A fleet whose unit is
|
|
527
|
-
// stale is not current, whatever its package identities say.
|
|
528
|
-
const files = await (deps.unitFiles === undefined ? defaultUnitFiles(deps) : deps.unitFiles());
|
|
529
|
-
const drift = files === undefined ? undefined : unitDriftReason(files.rendered, files.effective);
|
|
530
|
-
if (drift !== undefined) {
|
|
531
|
-
const commands = files?.stage?.() ?? [];
|
|
532
|
-
throw new Error(
|
|
533
|
-
commands.length === 0
|
|
534
|
-
? drift
|
|
535
|
-
: `${drift}\n\nThe corrected unit has been staged. Run:\n${commands.map((c) => ` ${c}`).join("\n")}`,
|
|
536
|
-
);
|
|
537
|
-
}
|
|
538
461
|
if (!installNeeded && brief.current) {
|
|
539
462
|
return {
|
|
540
463
|
previousVersion: surfaces.cliVersion,
|
|
@@ -629,17 +552,6 @@ export async function upgradeConductor(
|
|
|
629
552
|
const active = await mustRun(deps, "systemctl", ["is-active", HERDR_UNIT]);
|
|
630
553
|
if (active.stdout.trim() !== "active") throw new Error(`${HERDR_UNIT} is not active after restart`);
|
|
631
554
|
}
|
|
632
|
-
// The unit is the one surface `upgrade` never looked at, and since 0.4.0 it
|
|
633
|
-
// is the difference between a working per-run fleet and one that refuses
|
|
634
|
-
// every issue. Fatal rather than a warning: the whole point of this command
|
|
635
|
-
// is that a fleet is either upgraded or explicitly left paused, and a
|
|
636
|
-
// "verified" fleet that cannot dispatch is the worse outcome.
|
|
637
|
-
// Re-read rather than trust the earlier pass: the daemon has restarted
|
|
638
|
-
// since, and an operator who installed a unit mid-upgrade should not get a
|
|
639
|
-
// green verification for a file nobody checked.
|
|
640
|
-
const after = await (deps.unitFiles === undefined ? defaultUnitFiles(deps) : deps.unitFiles());
|
|
641
|
-
const stillDrifted = after === undefined ? undefined : unitDriftReason(after.rendered, after.effective);
|
|
642
|
-
if (stillDrifted !== undefined) throw new Error(stillDrifted);
|
|
643
555
|
await waitForRecovery(deps, initial, project);
|
|
644
556
|
deps.log("verify 2/2: recovered fleet remains stable");
|
|
645
557
|
await deps.sleep(1_000);
|
package/src/verbs/actions.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* enough to actually write.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import { credentialedEnv, openRunPr, pushRunBranch, repoSlugFor } from "../
|
|
17
|
+
import { credentialedEnv, openRunPr, pushRunBranch, repoSlugFor } from "../gitops.ts";
|
|
18
18
|
import type { ProjectConfig, ReleaseShape } from "../types.ts";
|
|
19
19
|
import type { ActionOutcome, ReleaseExecution, VerbActions } from "./server.ts";
|
|
20
20
|
|