omp-conductor 0.15.5 → 0.15.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -27
- package/package.json +1 -1
- package/schema/config.schema.json +39 -0
- package/src/board.ts +6 -28
- package/src/briefs/orchestrator.md +34 -10
- package/src/briefs/policy.md +8 -3
- package/src/cli.ts +61 -1
- package/src/config-schema.ts +20 -0
- package/src/config.ts +43 -0
- package/src/fleet.ts +168 -51
- package/src/lifecycle.ts +164 -21
- package/src/orchestrator-tick.ts +96 -7
- package/src/reports.ts +47 -0
- package/src/setup-discover.ts +425 -0
- package/src/setup-host.ts +21 -2
- package/src/setup-wizard.ts +106 -10
- package/src/setup.ts +25 -1
- package/src/store.ts +5 -1
- package/src/types.ts +34 -0
- package/src/verbs/actions.ts +407 -4
- package/src/verbs/protocol.ts +2 -2
- package/src/verbs/server.ts +141 -27
package/src/config.ts
CHANGED
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
type MergePreconditions,
|
|
46
46
|
type PlanUsageCap,
|
|
47
47
|
type ProjectConfig,
|
|
48
|
+
type RecoveryMergeAuthorization,
|
|
48
49
|
type ProjectPolicy,
|
|
49
50
|
type ReleasePreconditions,
|
|
50
51
|
type ReleaseRequirement,
|
|
@@ -1044,9 +1045,33 @@ function finalizeProject(
|
|
|
1044
1045
|
const escalation = finalizeEscalation(p["escalation"] as Raw | undefined);
|
|
1045
1046
|
const authority = finalizeAuthority(p["authority"] as Raw | undefined);
|
|
1046
1047
|
const releasePolicy = finalizeReleasePolicy(p["releasePolicy"], label, problems);
|
|
1048
|
+
const strandedTagRepos =
|
|
1049
|
+
releasePolicy["git-tag"] === "orchestrator" && releasePolicy["version-bump-pr"] !== "orchestrator"
|
|
1050
|
+
? Object.values(repos).filter((repo) => repo.release !== undefined)
|
|
1051
|
+
: [];
|
|
1052
|
+
if (strandedTagRepos.length > 0) {
|
|
1053
|
+
problems.push(
|
|
1054
|
+
`${label}: releasePolicy delegates git-tag, but ${strandedTagRepos
|
|
1055
|
+
.map((repo) => `${repo.name}.release.versionFile`)
|
|
1056
|
+
.join(", ")} requires a version-change PR and version-bump-pr is not delegated — ` +
|
|
1057
|
+
`grant "version-bump-pr": "orchestrator" or keep git-tag with the human`,
|
|
1058
|
+
);
|
|
1059
|
+
}
|
|
1047
1060
|
const policy = finalizePolicy(p["policy"], label, problems);
|
|
1048
1061
|
const caps = reconcileCaps(p["caps"], `${label}: caps`, problems, legacyCaps);
|
|
1049
1062
|
const reporting = finalizeReporting(p["reporting"], label, problems);
|
|
1063
|
+
const recoveryMerges = (p["recoveryMerges"] as RecoveryMergeAuthorization[] | undefined)?.map(
|
|
1064
|
+
(entry) => ({ ...entry }),
|
|
1065
|
+
);
|
|
1066
|
+
if (recoveryMerges !== undefined) {
|
|
1067
|
+
const seen = new Set<string>();
|
|
1068
|
+
for (const entry of recoveryMerges) {
|
|
1069
|
+
if (seen.has(entry.prUrl)) {
|
|
1070
|
+
problems.push(`${label}: recoveryMerges contains duplicate prUrl ${JSON.stringify(entry.prUrl)}`);
|
|
1071
|
+
}
|
|
1072
|
+
seen.add(entry.prUrl);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1050
1075
|
const rawWorkerModel = p["workerModel"];
|
|
1051
1076
|
const workerModel =
|
|
1052
1077
|
typeof rawWorkerModel === "string" && rawWorkerModel.trim() !== "" ? rawWorkerModel : undefined;
|
|
@@ -1070,6 +1095,7 @@ function finalizeProject(
|
|
|
1070
1095
|
authority,
|
|
1071
1096
|
releasePolicy,
|
|
1072
1097
|
policy,
|
|
1098
|
+
...(recoveryMerges === undefined ? {} : { recoveryMerges }),
|
|
1073
1099
|
reporting,
|
|
1074
1100
|
workspaceRoot: expandHome(pickString(p["workspaceRoot"], defaultWorkspaceRoot())),
|
|
1075
1101
|
mirrorRoot: expandHome(pickString(p["mirrorRoot"], defaultMirrorRoot())),
|
|
@@ -1223,6 +1249,8 @@ function finalizeRepos(parsed: unknown, label: string, problems: string[]): Reco
|
|
|
1223
1249
|
if (graph !== undefined) target.graphProject = graph;
|
|
1224
1250
|
const migrations = finalizeMigrationsDir(value?.["migrations"], `${label}: routing.repos.${key}`, problems);
|
|
1225
1251
|
if (migrations !== undefined) target.migrations = { dir: migrations };
|
|
1252
|
+
const versionFile = finalizeVersionFile(value?.["release"], `${label}: routing.repos.${key}`, problems);
|
|
1253
|
+
if (versionFile !== undefined) target.release = { versionFile };
|
|
1226
1254
|
repos[key] = target;
|
|
1227
1255
|
}
|
|
1228
1256
|
return repos;
|
|
@@ -1262,6 +1290,21 @@ function finalizeMigrationsDir(parsed: unknown, label: string, problems: string[
|
|
|
1262
1290
|
return dir;
|
|
1263
1291
|
}
|
|
1264
1292
|
|
|
1293
|
+
function finalizeVersionFile(parsed: unknown, label: string, problems: string[]): string | undefined {
|
|
1294
|
+
if (parsed === undefined) return undefined;
|
|
1295
|
+
const raw = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? (parsed as Raw) : undefined;
|
|
1296
|
+
const file = raw?.["versionFile"];
|
|
1297
|
+
if (typeof file !== "string" || file.trim() === "") {
|
|
1298
|
+
problems.push(`${label}.release.versionFile must be a non-empty string`);
|
|
1299
|
+
return undefined;
|
|
1300
|
+
}
|
|
1301
|
+
if (file.startsWith("/") || file.split("/").includes("..")) {
|
|
1302
|
+
problems.push(`${label}.release.versionFile must be a repo-relative file`);
|
|
1303
|
+
return undefined;
|
|
1304
|
+
}
|
|
1305
|
+
return file;
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1265
1308
|
function finalizeGraphProject(parsed: unknown, label: string, problems: string[]): string | undefined {
|
|
1266
1309
|
if (parsed === undefined) return undefined;
|
|
1267
1310
|
if (typeof parsed !== "string" || parsed.trim() === "") {
|
package/src/fleet.ts
CHANGED
|
@@ -55,8 +55,11 @@ import {
|
|
|
55
55
|
healthCheck,
|
|
56
56
|
isAlive,
|
|
57
57
|
livingDaemon,
|
|
58
|
+
probeUnit,
|
|
59
|
+
runSystemctl,
|
|
58
60
|
stopDaemon,
|
|
59
61
|
type StopResult,
|
|
62
|
+
type UnitOwnership,
|
|
60
63
|
SYSTEMD_UNIT,
|
|
61
64
|
} from "./lifecycle.ts";
|
|
62
65
|
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
@@ -99,7 +102,7 @@ export const ARM_CHALLENGE_TIMEOUT_MS = 300_000;
|
|
|
99
102
|
const LEGACY_HERDR_SESSION = "fleet";
|
|
100
103
|
|
|
101
104
|
export const LEGACY_HERDR_SESSION_HINT =
|
|
102
|
-
'herdr session "fleet" found — rename it to "conductor" or
|
|
105
|
+
'herdr session "fleet" found — rename it to "conductor" or run omp-conductor setup host to pin HERDR_SESSION=fleet (remove this bridge next minor)';
|
|
103
106
|
|
|
104
107
|
let legacyHerdrSessionHintPrinted = false;
|
|
105
108
|
|
|
@@ -564,21 +567,7 @@ export interface HerdrStartDeps {
|
|
|
564
567
|
* systemd or without that optional unit keep the standalone daemon behaviour.
|
|
565
568
|
*/
|
|
566
569
|
export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {}): HerdrStartResult {
|
|
567
|
-
const run
|
|
568
|
-
deps.systemctl ??
|
|
569
|
-
((args: string[]) => {
|
|
570
|
-
const res = spawnSync("systemctl", args, { encoding: "utf8", timeout: 15_000, env: process.env });
|
|
571
|
-
if (res.error) {
|
|
572
|
-
const err = res.error as NodeJS.ErrnoException;
|
|
573
|
-
return {
|
|
574
|
-
ok: false,
|
|
575
|
-
stdout: "",
|
|
576
|
-
stderr: err.message,
|
|
577
|
-
...(err.code === "ENOENT" ? { missing: true } : {}),
|
|
578
|
-
};
|
|
579
|
-
}
|
|
580
|
-
return { ok: res.status === 0, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
|
|
581
|
-
});
|
|
570
|
+
const run = deps.systemctl ?? runSystemctl;
|
|
582
571
|
|
|
583
572
|
const shown = run(["show", DEFAULT_HERDR_UNIT, "--property=LoadState", "--value"]);
|
|
584
573
|
if (shown.missing) return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "no systemctl" };
|
|
@@ -1270,10 +1259,132 @@ export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()):
|
|
|
1270
1259
|
}
|
|
1271
1260
|
|
|
1272
1261
|
|
|
1262
|
+
/**
|
|
1263
|
+
* Project-scoped reading of a living daemon record + `/healthz` body.
|
|
1264
|
+
*
|
|
1265
|
+
* Board and `status` share this so a host running one daemon for several
|
|
1266
|
+
* projects cannot be called healthy for a project it does not serve (#379).
|
|
1267
|
+
* A third interpretation convention is forbidden.
|
|
1268
|
+
*/
|
|
1269
|
+
export type DaemonProjectHealth =
|
|
1270
|
+
| { kind: "stopped" }
|
|
1271
|
+
| { kind: "ok" }
|
|
1272
|
+
| { kind: "unreachable" }
|
|
1273
|
+
| { kind: "other-project"; serves?: string };
|
|
1274
|
+
|
|
1275
|
+
/**
|
|
1276
|
+
* Facts `formatFleetStatus` needs about the living daemon. Pure input — the
|
|
1277
|
+
* caller probes; the formatter never shells out (#379).
|
|
1278
|
+
*/
|
|
1279
|
+
export type FleetDaemonProbe = {
|
|
1280
|
+
project: DaemonProjectHealth;
|
|
1281
|
+
/** `/healthz` body when the probe is project-ok (rss / overlays). */
|
|
1282
|
+
body?: string;
|
|
1283
|
+
/** systemd ownership of the living record pid; omit when unprobed. */
|
|
1284
|
+
unit?: UnitOwnership;
|
|
1285
|
+
};
|
|
1286
|
+
|
|
1287
|
+
/**
|
|
1288
|
+
* Decide whether a living record + optional `/healthz` answer serve `project`.
|
|
1289
|
+
*
|
|
1290
|
+
* Mirrors the board gate: a record pinned to another project is never probed
|
|
1291
|
+
* further; a multi-project payload is ok only when `project` appears in
|
|
1292
|
+
* `projects[]` (or as the top-level single-project `project` field).
|
|
1293
|
+
*/
|
|
1294
|
+
export function classifyDaemonProjectHealth(
|
|
1295
|
+
record: { project?: string } | undefined,
|
|
1296
|
+
health: { ok: boolean; body?: string } | undefined,
|
|
1297
|
+
project: string,
|
|
1298
|
+
): DaemonProjectHealth {
|
|
1299
|
+
if (record === undefined) return { kind: "stopped" };
|
|
1300
|
+
// Record project is authoritative when set — a single-project leftover from
|
|
1301
|
+
// #376 must not be presented as this project's daemon just because its port
|
|
1302
|
+
// still answers.
|
|
1303
|
+
if (record.project !== undefined && record.project !== project) {
|
|
1304
|
+
return { kind: "other-project", serves: record.project };
|
|
1305
|
+
}
|
|
1306
|
+
if (health?.ok !== true) return { kind: "unreachable" };
|
|
1307
|
+
try {
|
|
1308
|
+
const payload = JSON.parse(health.body ?? "null") as unknown;
|
|
1309
|
+
if (payload === null || typeof payload !== "object") {
|
|
1310
|
+
return { kind: "unreachable" };
|
|
1311
|
+
}
|
|
1312
|
+
if (Reflect.get(payload, "project") === project) return { kind: "ok" };
|
|
1313
|
+
const projects = Reflect.get(payload, "projects");
|
|
1314
|
+
if (
|
|
1315
|
+
Array.isArray(projects) &&
|
|
1316
|
+
projects.some(
|
|
1317
|
+
(entry) =>
|
|
1318
|
+
entry !== null &&
|
|
1319
|
+
typeof entry === "object" &&
|
|
1320
|
+
Reflect.get(entry, "project") === project,
|
|
1321
|
+
)
|
|
1322
|
+
) {
|
|
1323
|
+
return { kind: "ok" };
|
|
1324
|
+
}
|
|
1325
|
+
return { kind: "other-project", serves: servedProjectName(payload) };
|
|
1326
|
+
} catch {
|
|
1327
|
+
return { kind: "unreachable" };
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/** Best-effort name of who a foreign `/healthz` payload is actually serving. */
|
|
1332
|
+
function servedProjectName(payload: object): string | undefined {
|
|
1333
|
+
const top = Reflect.get(payload, "project");
|
|
1334
|
+
if (typeof top === "string" && top.length > 0) return top;
|
|
1335
|
+
const projects = Reflect.get(payload, "projects");
|
|
1336
|
+
if (!Array.isArray(projects)) return undefined;
|
|
1337
|
+
for (const entry of projects) {
|
|
1338
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
1339
|
+
const name = Reflect.get(entry, "project");
|
|
1340
|
+
if (typeof name === "string" && name.length > 0) return name;
|
|
1341
|
+
}
|
|
1342
|
+
return undefined;
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
1346
|
+
if (probe === undefined) return "unprobed";
|
|
1347
|
+
switch (probe.project.kind) {
|
|
1348
|
+
case "stopped":
|
|
1349
|
+
return "stopped";
|
|
1350
|
+
case "ok":
|
|
1351
|
+
return "ok";
|
|
1352
|
+
case "unreachable":
|
|
1353
|
+
return "unreachable — the process is up but not serving";
|
|
1354
|
+
case "other-project":
|
|
1355
|
+
return probe.project.serves === undefined
|
|
1356
|
+
? "other-project"
|
|
1357
|
+
: `other-project — serves ${probe.project.serves}`;
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
/**
|
|
1362
|
+
* Unit line for a living record. The bare unit name alone used to imply
|
|
1363
|
+
* systemd ownership that was never checked (#379); every branch states the
|
|
1364
|
+
* probe result, and `unknown` is never rendered as owned.
|
|
1365
|
+
*/
|
|
1366
|
+
function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
|
|
1367
|
+
if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`;
|
|
1368
|
+
switch (unit.kind) {
|
|
1369
|
+
case "active":
|
|
1370
|
+
if (unit.pid === recordPid) {
|
|
1371
|
+
return ` unit ${SYSTEMD_UNIT} systemd-owned`;
|
|
1372
|
+
}
|
|
1373
|
+
// Live unit, different MainPID — the record pid is not systemd's.
|
|
1374
|
+
return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`;
|
|
1375
|
+
case "failed":
|
|
1376
|
+
return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`;
|
|
1377
|
+
case "inactive":
|
|
1378
|
+
return ` unit ${SYSTEMD_UNIT} inactive`;
|
|
1379
|
+
case "unknown":
|
|
1380
|
+
return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`;
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1273
1384
|
export function formatFleetStatus(
|
|
1274
1385
|
s: StatusSnapshot,
|
|
1275
1386
|
layers: FleetLayers,
|
|
1276
|
-
|
|
1387
|
+
daemon: FleetDaemonProbe | undefined = undefined,
|
|
1277
1388
|
telegram: TelegramHealth = { kind: "unprobed" },
|
|
1278
1389
|
now = Date.now(),
|
|
1279
1390
|
codeGraph: CodeGraphHealth = { configured: false },
|
|
@@ -1315,20 +1426,17 @@ export function formatFleetStatus(
|
|
|
1315
1426
|
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
1316
1427
|
daemonBlock = "daemon not running";
|
|
1317
1428
|
} else {
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
? "ok"
|
|
1323
|
-
: "unreachable — the process is up but not serving";
|
|
1324
|
-
const rss = rssBytesFromHealthz(daemonHealth?.body);
|
|
1429
|
+
// Only trust rss from a project-ok body — a foreign payload's bytes are
|
|
1430
|
+
// not this project's daemon facts (#379).
|
|
1431
|
+
const rss =
|
|
1432
|
+
daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
|
|
1325
1433
|
daemonBlock = [
|
|
1326
1434
|
"daemon",
|
|
1327
1435
|
` pid ${layers.daemon.pid}`,
|
|
1328
1436
|
` port ${layers.daemon.port ?? "?"}`,
|
|
1329
1437
|
...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
|
|
1330
|
-
` healthz ${
|
|
1331
|
-
|
|
1438
|
+
` healthz ${formatDaemonHealthz(daemon)}`,
|
|
1439
|
+
formatDaemonUnit(daemon?.unit, layers.daemon.pid),
|
|
1332
1440
|
].join("\n");
|
|
1333
1441
|
}
|
|
1334
1442
|
|
|
@@ -1504,8 +1612,11 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1504
1612
|
const layers = fleetLayers(projectName);
|
|
1505
1613
|
const project = findProject(loadConfig(), projectName);
|
|
1506
1614
|
const rec = livingDaemon();
|
|
1507
|
-
|
|
1508
|
-
|
|
1615
|
+
// Gate the port probe on record membership first — a leftover single-project
|
|
1616
|
+
// daemon from #376 must not be health-checked as if it served us (#379).
|
|
1617
|
+
const wrongRecord = rec?.project !== undefined && rec.project !== project.name;
|
|
1618
|
+
const [rawHealth, telegram, planUsage, github] = await Promise.all([
|
|
1619
|
+
rec === undefined || wrongRecord ? undefined : healthCheck(rec.port),
|
|
1509
1620
|
probeTelegramHealth(projectName),
|
|
1510
1621
|
// Read here rather than in `statusSnapshot`, which is synchronous and used
|
|
1511
1622
|
// by callers that must not shell out. An unmetered project never spawns
|
|
@@ -1516,13 +1627,27 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1516
1627
|
// broken report (#188).
|
|
1517
1628
|
fetchRateLimit(),
|
|
1518
1629
|
]);
|
|
1519
|
-
const
|
|
1630
|
+
const projectHealth = classifyDaemonProjectHealth(rec, rawHealth, project.name);
|
|
1631
|
+
// Ownership is probed, never assumed from the unit name constant (#379).
|
|
1632
|
+
const unit = rec === undefined ? undefined : probeUnit();
|
|
1633
|
+
const daemon: FleetDaemonProbe | undefined =
|
|
1634
|
+
rec === undefined
|
|
1635
|
+
? undefined
|
|
1636
|
+
: {
|
|
1637
|
+
project: projectHealth,
|
|
1638
|
+
...(projectHealth.kind === "ok" && rawHealth?.body !== undefined
|
|
1639
|
+
? { body: rawHealth.body }
|
|
1640
|
+
: {}),
|
|
1641
|
+
...(unit === undefined ? {} : { unit }),
|
|
1642
|
+
};
|
|
1643
|
+
const healthBody = projectHealth.kind === "ok" ? rawHealth?.body : undefined;
|
|
1644
|
+
const cached = codeGraphFromHealthz(healthBody, project.name);
|
|
1520
1645
|
const codeGraph = cached ?? (await probeCodeGraph(project));
|
|
1521
|
-
const workerPhases = workerPhasesFromHealthz(
|
|
1646
|
+
const workerPhases = workerPhasesFromHealthz(healthBody, project.name);
|
|
1522
1647
|
return formatFleetStatus(
|
|
1523
1648
|
{ ...s, planUsage, github },
|
|
1524
1649
|
layers,
|
|
1525
|
-
|
|
1650
|
+
daemon,
|
|
1526
1651
|
telegram,
|
|
1527
1652
|
Date.now(),
|
|
1528
1653
|
codeGraph,
|
|
@@ -1862,26 +1987,18 @@ export async function transcriptHasUserCode(path: string, code: string): Promise
|
|
|
1862
1987
|
// ---------------------------------------------------------------------------
|
|
1863
1988
|
|
|
1864
1989
|
function probeHerdrUnit(unit = DEFAULT_HERDR_UNIT): { kind: HerdrLayer; detail?: string } {
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
}
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
}
|
|
1876
|
-
|
|
1877
|
-
if (out === "active") return { kind: "active", detail: unit };
|
|
1878
|
-
if (out === "inactive" || out === "failed" || out === "dead") {
|
|
1879
|
-
return { kind: "inactive", detail: `${unit} ${out}` };
|
|
1880
|
-
}
|
|
1881
|
-
return { kind: "unknown", detail: `${unit} ${out || `exit ${String(res.status)}`}` };
|
|
1882
|
-
} catch (err) {
|
|
1883
|
-
return { kind: "unknown", detail: err instanceof Error ? err.message : String(err) };
|
|
1884
|
-
}
|
|
1990
|
+
const res = runSystemctl(["is-active", unit]);
|
|
1991
|
+
if (res.missing) return { kind: "unknown", detail: "no systemctl" };
|
|
1992
|
+
const out = res.stdout.trim();
|
|
1993
|
+
if (out === "active") return { kind: "active", detail: unit };
|
|
1994
|
+
if (out === "inactive" || out === "failed" || out === "dead") {
|
|
1995
|
+
return { kind: "inactive", detail: `${unit} ${out}` };
|
|
1996
|
+
}
|
|
1997
|
+
const error = res.stderr.trim();
|
|
1998
|
+
return {
|
|
1999
|
+
kind: "unknown",
|
|
2000
|
+
detail: `${unit} ${out || error || "systemctl is-active failed"}`,
|
|
2001
|
+
};
|
|
1885
2002
|
}
|
|
1886
2003
|
|
|
1887
2004
|
export function paneLayerFromAgents(
|
package/src/lifecycle.ts
CHANGED
|
@@ -195,18 +195,34 @@ export async function healthCheck(port: number): Promise<{ ok: boolean; body?: s
|
|
|
195
195
|
}
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
+
function healthServesProject(body: string | undefined, project: string | undefined): boolean {
|
|
199
|
+
if (project === undefined) return true;
|
|
200
|
+
if (body === undefined) return false;
|
|
201
|
+
try {
|
|
202
|
+
const parsed = JSON.parse(body) as unknown;
|
|
203
|
+
if (parsed === null || typeof parsed !== "object") return false;
|
|
204
|
+
const projects = Reflect.get(parsed, "projects");
|
|
205
|
+
if (!Array.isArray(projects)) return false;
|
|
206
|
+
return projects.some((entry) => {
|
|
207
|
+
if (typeof entry === "string") return entry === project;
|
|
208
|
+
return entry !== null && typeof entry === "object" && Reflect.get(entry, "project") === project;
|
|
209
|
+
});
|
|
210
|
+
} catch {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
198
215
|
/**
|
|
199
|
-
* Starts the daemon
|
|
200
|
-
* `/healthz`.
|
|
216
|
+
* Starts the daemon and does not return until it answers `/healthz`.
|
|
201
217
|
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
* both the pid and the endpoint, and fail loudly with the log if either says
|
|
207
|
-
* no.
|
|
218
|
+
* An installed unit owns the host-wide daemon lifecycle. `start --project X`
|
|
219
|
+
* may select the project whose health is proved, but it must not turn the unit
|
|
220
|
+
* into a detached single-project competitor (#400). Only a host proven not to
|
|
221
|
+
* have the unit keeps the standalone spawn path.
|
|
208
222
|
*/
|
|
209
|
-
export async function startDaemon(
|
|
223
|
+
export async function startDaemon(
|
|
224
|
+
o: { port?: number; project?: string; timeoutMs?: number } = {},
|
|
225
|
+
): Promise<DaemonRecord> {
|
|
210
226
|
const running = livingDaemon();
|
|
211
227
|
if (running !== undefined) {
|
|
212
228
|
throw new Error(
|
|
@@ -215,6 +231,28 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
215
231
|
);
|
|
216
232
|
}
|
|
217
233
|
|
|
234
|
+
const installation = probeUnitInstallation();
|
|
235
|
+
if (installation.kind === "unknown") {
|
|
236
|
+
throw new Error(
|
|
237
|
+
`cannot determine whether ${SYSTEMD_UNIT} is installed (${installation.reason}); ` +
|
|
238
|
+
"refusing to launch a detached daemon that could compete with the service manager",
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
if (installation.kind === "installed") {
|
|
242
|
+
const ownership = probeUnit();
|
|
243
|
+
if (ownership.kind === "unknown") {
|
|
244
|
+
throw new Error(ownershipUnknown("start", ownership.reason));
|
|
245
|
+
}
|
|
246
|
+
if (ownership.kind === "failed") {
|
|
247
|
+
return await restoreFailedUnit(o.timeoutMs, o.project);
|
|
248
|
+
}
|
|
249
|
+
const started = systemctl(["start", SYSTEMD_UNIT]);
|
|
250
|
+
if (!started.ok) {
|
|
251
|
+
throw new Error(systemctlFailure("start", started));
|
|
252
|
+
}
|
|
253
|
+
return await waitForOwnedDaemon("start", o.timeoutMs, o.project);
|
|
254
|
+
}
|
|
255
|
+
|
|
218
256
|
const port = o.port ?? DEFAULT_PORT;
|
|
219
257
|
const logFile = join(ensureRuntimeDir(), "daemon.log");
|
|
220
258
|
|
|
@@ -260,7 +298,7 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
260
298
|
// rather than nothing at all.
|
|
261
299
|
writeRecord(record);
|
|
262
300
|
|
|
263
|
-
const deadline = Date.now() + READY_TIMEOUT_MS;
|
|
301
|
+
const deadline = Date.now() + (o.timeoutMs ?? READY_TIMEOUT_MS);
|
|
264
302
|
for (;;) {
|
|
265
303
|
// Liveness first. If the child is gone, a healthy answer on that port came
|
|
266
304
|
// from somebody else's server, and reporting it as ours would be worse
|
|
@@ -270,7 +308,7 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
270
308
|
throw new Error(`daemon exited during startup${tailLog(logFile)}`);
|
|
271
309
|
}
|
|
272
310
|
const health = await healthCheck(port);
|
|
273
|
-
if (health.ok) return record;
|
|
311
|
+
if (health.ok && healthServesProject(health.body, o.project)) return record;
|
|
274
312
|
if (Date.now() >= deadline) break;
|
|
275
313
|
await sleep(READY_POLL_MS);
|
|
276
314
|
}
|
|
@@ -280,7 +318,7 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
280
318
|
await terminate(pid, STOP_TIMEOUT_MS);
|
|
281
319
|
clearRecord();
|
|
282
320
|
throw new Error(
|
|
283
|
-
`daemon did not answer http://127.0.0.1:${port}/healthz within ${Math.round(READY_TIMEOUT_MS / 1000)}s${tailLog(logFile)}`,
|
|
321
|
+
`daemon did not answer http://127.0.0.1:${port}/healthz for the requested project within ${Math.round((o.timeoutMs ?? READY_TIMEOUT_MS) / 1000)}s${tailLog(logFile)}`,
|
|
284
322
|
);
|
|
285
323
|
}
|
|
286
324
|
|
|
@@ -395,7 +433,7 @@ export async function restartDaemon(
|
|
|
395
433
|
// startDaemon() — that leaves the unit failed while handing the operator
|
|
396
434
|
// a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
|
|
397
435
|
// Any manager refusal is terminal; so is unproven ownership afterwards.
|
|
398
|
-
const record = await restoreFailedUnit(o.timeoutMs);
|
|
436
|
+
const record = await restoreFailedUnit(o.timeoutMs, o.project);
|
|
399
437
|
return { previous, record, via: "systemctl" };
|
|
400
438
|
}
|
|
401
439
|
|
|
@@ -413,7 +451,7 @@ export async function restartDaemon(
|
|
|
413
451
|
// systemctl restart returns once the new MainPID is up; the pidfile is
|
|
414
452
|
// written by the daemon itself on boot, so wait for that rather than
|
|
415
453
|
// inventing a record from the unit alone.
|
|
416
|
-
const record = await waitForOwnedDaemon("restart", o.timeoutMs);
|
|
454
|
+
const record = await waitForOwnedDaemon("restart", o.timeoutMs, o.project);
|
|
417
455
|
return { previous, record, via: "systemctl" };
|
|
418
456
|
}
|
|
419
457
|
|
|
@@ -430,7 +468,7 @@ export async function restartDaemon(
|
|
|
430
468
|
* (see {@link waitForOwnedDaemon}). Refusal or unproven ownership is terminal
|
|
431
469
|
* — never a fallthrough to the detached CLI daemon.
|
|
432
470
|
*/
|
|
433
|
-
async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
|
|
471
|
+
async function restoreFailedUnit(timeoutMs?: number, project?: string): Promise<DaemonRecord> {
|
|
434
472
|
const reset = systemctl(["reset-failed", SYSTEMD_UNIT]);
|
|
435
473
|
if (!reset.ok) {
|
|
436
474
|
throw new Error(systemctlFailure("reset-failed", reset));
|
|
@@ -439,7 +477,7 @@ async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
|
|
|
439
477
|
if (!started.ok) {
|
|
440
478
|
throw new Error(systemctlFailure("start", started));
|
|
441
479
|
}
|
|
442
|
-
return await waitForOwnedDaemon("start", timeoutMs);
|
|
480
|
+
return await waitForOwnedDaemon("start", timeoutMs, project);
|
|
443
481
|
}
|
|
444
482
|
|
|
445
483
|
/**
|
|
@@ -450,7 +488,11 @@ async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
|
|
|
450
488
|
* cannot un-fail it. Any other unproven state fails at the deadline with a
|
|
451
489
|
* diagnostic that names the mismatch, never a bare "not ready".
|
|
452
490
|
*/
|
|
453
|
-
async function waitForOwnedDaemon(
|
|
491
|
+
async function waitForOwnedDaemon(
|
|
492
|
+
verb: "restart" | "start",
|
|
493
|
+
timeoutMs?: number,
|
|
494
|
+
project?: string,
|
|
495
|
+
): Promise<DaemonRecord> {
|
|
454
496
|
const via = `systemctl ${verb} ${SYSTEMD_UNIT} returned`;
|
|
455
497
|
const inspect = `systemctl status ${SYSTEMD_UNIT}`;
|
|
456
498
|
const deadline = Date.now() + (timeoutMs ?? READY_TIMEOUT_MS);
|
|
@@ -459,7 +501,7 @@ async function waitForOwnedDaemon(verb: "restart" | "start", timeoutMs?: number)
|
|
|
459
501
|
const ownership = probeUnit();
|
|
460
502
|
if (rec !== undefined && ownership.kind === "active" && ownership.pid === rec.pid) {
|
|
461
503
|
const health = await healthCheck(rec.port);
|
|
462
|
-
if (health.ok) return rec;
|
|
504
|
+
if (health.ok && healthServesProject(health.body, project)) return rec;
|
|
463
505
|
} else if (ownership.kind === "failed") {
|
|
464
506
|
// A confirmed negative: the start did not take and the unit is failed
|
|
465
507
|
// again. Waiting longer cannot un-fail it.
|
|
@@ -482,9 +524,10 @@ async function waitForOwnedDaemon(verb: "restart" | "start", timeoutMs?: number)
|
|
|
482
524
|
}
|
|
483
525
|
if (ownership.kind === "active") {
|
|
484
526
|
if (ownership.pid === rec.pid) {
|
|
527
|
+
const projectDetail = project === undefined ? "" : ` for project ${project}`;
|
|
485
528
|
throw new Error(
|
|
486
|
-
`${via}, but the daemon never answered /healthz on :${rec.port} — ` +
|
|
487
|
-
`the service manager owns pid ${rec.pid}, but it is not serving; ` +
|
|
529
|
+
`${via}, but the daemon never answered /healthz${projectDetail} on :${rec.port} — ` +
|
|
530
|
+
`the service manager owns pid ${rec.pid}, but it is not serving the requested project; ` +
|
|
488
531
|
`check \`${inspect}\` and ${rec.logFile}`,
|
|
489
532
|
);
|
|
490
533
|
}
|
|
@@ -579,6 +622,31 @@ export function probeUnit(unit = SYSTEMD_UNIT): UnitOwnership {
|
|
|
579
622
|
return { kind: "inactive" };
|
|
580
623
|
}
|
|
581
624
|
|
|
625
|
+
type UnitInstallation =
|
|
626
|
+
| { kind: "installed" }
|
|
627
|
+
| { kind: "absent" }
|
|
628
|
+
| { kind: "unknown"; reason: string };
|
|
629
|
+
|
|
630
|
+
/**
|
|
631
|
+
* Installation is separate from ownership: an inactive installed unit and an
|
|
632
|
+
* absent unit both have MainPID 0, but only the latter permits a detached
|
|
633
|
+
* fallback.
|
|
634
|
+
*/
|
|
635
|
+
function probeUnitInstallation(unit = SYSTEMD_UNIT): UnitInstallation {
|
|
636
|
+
const ran = systemctl(["show", unit, "--property=LoadState", "--value"]);
|
|
637
|
+
if (!ran.ok) {
|
|
638
|
+
if (ran.missing) return { kind: "absent" };
|
|
639
|
+
const detail = (ran.stderr.trim() || ran.stdout.trim() || "systemctl show failed").split("\n")[0]!;
|
|
640
|
+
return { kind: "unknown", reason: detail };
|
|
641
|
+
}
|
|
642
|
+
const loadState = ran.stdout.trim();
|
|
643
|
+
if (loadState === "not-found") return { kind: "absent" };
|
|
644
|
+
if (loadState.length === 0) return { kind: "unknown", reason: "systemctl returned no LoadState" };
|
|
645
|
+
// loaded, masked, error and bad-setting all prove a manager-known unit. Let
|
|
646
|
+
// `systemctl start` provide the actionable refusal for a broken definition.
|
|
647
|
+
return { kind: "installed" };
|
|
648
|
+
}
|
|
649
|
+
|
|
582
650
|
/**
|
|
583
651
|
* The MainPID of an *active* {@link SYSTEMD_UNIT}, or `undefined` when the
|
|
584
652
|
* unit is confirmed inactive, failed, or absent, or when ownership could not
|
|
@@ -662,7 +730,14 @@ function systemctlFailure(
|
|
|
662
730
|
);
|
|
663
731
|
}
|
|
664
732
|
|
|
665
|
-
function ownershipUnknown(verb: "stop" | "restart", reason: string): string {
|
|
733
|
+
function ownershipUnknown(verb: "start" | "stop" | "restart", reason: string): string {
|
|
734
|
+
if (verb === "start") {
|
|
735
|
+
return (
|
|
736
|
+
`cannot determine whether ${SYSTEMD_UNIT} owns the daemon (${reason}) — ` +
|
|
737
|
+
`refusing to start a detached competitor while ownership is unknown; ` +
|
|
738
|
+
`retry when systemctl answers, or run \`systemctl start ${SYSTEMD_UNIT}\` yourself`
|
|
739
|
+
);
|
|
740
|
+
}
|
|
666
741
|
return (
|
|
667
742
|
`cannot determine whether ${SYSTEMD_UNIT} owns the daemon (${reason}) — ` +
|
|
668
743
|
`refusing to ${verb} via signal while ownership is unknown ` +
|
|
@@ -689,7 +764,69 @@ export type SystemctlResult = {
|
|
|
689
764
|
|
|
690
765
|
export type SystemctlFn = (args: string[]) => SystemctlResult;
|
|
691
766
|
|
|
767
|
+
interface TestSystemctlState {
|
|
768
|
+
installed?: boolean;
|
|
769
|
+
mainPid?: number;
|
|
770
|
+
activeState?: string;
|
|
771
|
+
calls?: string[][];
|
|
772
|
+
failures?: Record<string, string>;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
/**
|
|
776
|
+
* Child-process CLI tests cannot inject {@link setSystemctlForTest}. Under
|
|
777
|
+
* NODE_ENV=test they therefore get a file-backed fake manager, or a refusal
|
|
778
|
+
* when no fixture was supplied. The real binary is never reached from a test
|
|
779
|
+
* process (#399).
|
|
780
|
+
*/
|
|
781
|
+
function testSystemctl(args: string[]): SystemctlResult {
|
|
782
|
+
const path = process.env["OMP_CONDUCTOR_TEST_SYSTEMCTL_STATE"];
|
|
783
|
+
if (path === undefined) {
|
|
784
|
+
return {
|
|
785
|
+
ok: false,
|
|
786
|
+
stdout: "",
|
|
787
|
+
stderr: "real systemctl is disabled under NODE_ENV=test; no fake manager state was supplied",
|
|
788
|
+
};
|
|
789
|
+
}
|
|
790
|
+
try {
|
|
791
|
+
const state = JSON.parse(readFileSync(path, "utf8")) as TestSystemctlState;
|
|
792
|
+
const calls = Array.isArray(state.calls) ? state.calls : [];
|
|
793
|
+
calls.push(args);
|
|
794
|
+
state.calls = calls;
|
|
795
|
+
const verb = args[0] ?? "";
|
|
796
|
+
const failure = state.failures?.[verb];
|
|
797
|
+
if (failure !== undefined) {
|
|
798
|
+
writeFileSync(path, JSON.stringify(state));
|
|
799
|
+
return { ok: false, stdout: "", stderr: failure };
|
|
800
|
+
}
|
|
801
|
+
let stdout = "";
|
|
802
|
+
if (verb === "show") {
|
|
803
|
+
stdout = args.includes("--property=LoadState")
|
|
804
|
+
? `${state.installed === false ? "not-found" : "loaded"}\n`
|
|
805
|
+
: `${state.mainPid ?? 0}\n${state.activeState ?? "inactive"}\n`;
|
|
806
|
+
} else if (verb === "is-active") {
|
|
807
|
+
const activeState = state.activeState ?? "inactive";
|
|
808
|
+
stdout = `${activeState}\n`;
|
|
809
|
+
writeFileSync(path, JSON.stringify(state));
|
|
810
|
+
return { ok: activeState === "active", stdout, stderr: "" };
|
|
811
|
+
} else if (verb === "stop") {
|
|
812
|
+
state.mainPid = 0;
|
|
813
|
+
state.activeState = "inactive";
|
|
814
|
+
} else if (verb === "reset-failed") {
|
|
815
|
+
state.activeState = "inactive";
|
|
816
|
+
}
|
|
817
|
+
writeFileSync(path, JSON.stringify(state));
|
|
818
|
+
return { ok: true, stdout, stderr: "" };
|
|
819
|
+
} catch (err) {
|
|
820
|
+
return {
|
|
821
|
+
ok: false,
|
|
822
|
+
stdout: "",
|
|
823
|
+
stderr: `fake systemctl state failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
824
|
+
};
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
|
|
692
828
|
function defaultSystemctl(args: string[]): SystemctlResult {
|
|
829
|
+
if (process.env["NODE_ENV"] === "test") return testSystemctl(args);
|
|
693
830
|
try {
|
|
694
831
|
const res = spawnSync("systemctl", args, {
|
|
695
832
|
encoding: "utf8",
|
|
@@ -719,6 +856,12 @@ function defaultSystemctl(args: string[]): SystemctlResult {
|
|
|
719
856
|
|
|
720
857
|
let systemctl: SystemctlFn = defaultSystemctl;
|
|
721
858
|
|
|
859
|
+
/** Shared manager runner. Production reaches the hardcoded binary; test
|
|
860
|
+
* processes reach only the explicit fake or the fail-closed refusal. */
|
|
861
|
+
export function runSystemctl(args: string[]): SystemctlResult {
|
|
862
|
+
return systemctl(args);
|
|
863
|
+
}
|
|
864
|
+
|
|
722
865
|
/** Test-only: replace the `systemctl` runner. Pass `undefined` to restore. */
|
|
723
866
|
export function setSystemctlForTest(fn: SystemctlFn | undefined): void {
|
|
724
867
|
systemctl = fn ?? defaultSystemctl;
|