omp-conductor 0.15.6 → 0.15.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -27
- package/package.json +1 -1
- package/schema/config.schema.json +39 -0
- package/src/briefs/orchestrator.md +34 -10
- package/src/briefs/policy.md +8 -3
- package/src/cli.ts +62 -1
- package/src/config-schema.ts +20 -0
- package/src/config.ts +43 -0
- package/src/escalate.ts +69 -1
- package/src/fleet.ts +19 -39
- package/src/lifecycle.ts +164 -21
- package/src/orchestrator-tick.ts +96 -7
- package/src/reports.ts +49 -2
- package/src/setup-discover.ts +425 -0
- package/src/setup-wizard.ts +108 -34
- package/src/setup.ts +25 -1
- package/src/store.ts +5 -1
- package/src/types.ts +34 -0
- package/src/verbs/actions.ts +407 -4
- package/src/verbs/protocol.ts +2 -2
- package/src/verbs/server.ts +141 -27
package/src/config.ts
CHANGED
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
type MergePreconditions,
|
|
46
46
|
type PlanUsageCap,
|
|
47
47
|
type ProjectConfig,
|
|
48
|
+
type RecoveryMergeAuthorization,
|
|
48
49
|
type ProjectPolicy,
|
|
49
50
|
type ReleasePreconditions,
|
|
50
51
|
type ReleaseRequirement,
|
|
@@ -1044,9 +1045,33 @@ function finalizeProject(
|
|
|
1044
1045
|
const escalation = finalizeEscalation(p["escalation"] as Raw | undefined);
|
|
1045
1046
|
const authority = finalizeAuthority(p["authority"] as Raw | undefined);
|
|
1046
1047
|
const releasePolicy = finalizeReleasePolicy(p["releasePolicy"], label, problems);
|
|
1048
|
+
const strandedTagRepos =
|
|
1049
|
+
releasePolicy["git-tag"] === "orchestrator" && releasePolicy["version-bump-pr"] !== "orchestrator"
|
|
1050
|
+
? Object.values(repos).filter((repo) => repo.release !== undefined)
|
|
1051
|
+
: [];
|
|
1052
|
+
if (strandedTagRepos.length > 0) {
|
|
1053
|
+
problems.push(
|
|
1054
|
+
`${label}: releasePolicy delegates git-tag, but ${strandedTagRepos
|
|
1055
|
+
.map((repo) => `${repo.name}.release.versionFile`)
|
|
1056
|
+
.join(", ")} requires a version-change PR and version-bump-pr is not delegated — ` +
|
|
1057
|
+
`grant "version-bump-pr": "orchestrator" or keep git-tag with the human`,
|
|
1058
|
+
);
|
|
1059
|
+
}
|
|
1047
1060
|
const policy = finalizePolicy(p["policy"], label, problems);
|
|
1048
1061
|
const caps = reconcileCaps(p["caps"], `${label}: caps`, problems, legacyCaps);
|
|
1049
1062
|
const reporting = finalizeReporting(p["reporting"], label, problems);
|
|
1063
|
+
const recoveryMerges = (p["recoveryMerges"] as RecoveryMergeAuthorization[] | undefined)?.map(
|
|
1064
|
+
(entry) => ({ ...entry }),
|
|
1065
|
+
);
|
|
1066
|
+
if (recoveryMerges !== undefined) {
|
|
1067
|
+
const seen = new Set<string>();
|
|
1068
|
+
for (const entry of recoveryMerges) {
|
|
1069
|
+
if (seen.has(entry.prUrl)) {
|
|
1070
|
+
problems.push(`${label}: recoveryMerges contains duplicate prUrl ${JSON.stringify(entry.prUrl)}`);
|
|
1071
|
+
}
|
|
1072
|
+
seen.add(entry.prUrl);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1050
1075
|
const rawWorkerModel = p["workerModel"];
|
|
1051
1076
|
const workerModel =
|
|
1052
1077
|
typeof rawWorkerModel === "string" && rawWorkerModel.trim() !== "" ? rawWorkerModel : undefined;
|
|
@@ -1070,6 +1095,7 @@ function finalizeProject(
|
|
|
1070
1095
|
authority,
|
|
1071
1096
|
releasePolicy,
|
|
1072
1097
|
policy,
|
|
1098
|
+
...(recoveryMerges === undefined ? {} : { recoveryMerges }),
|
|
1073
1099
|
reporting,
|
|
1074
1100
|
workspaceRoot: expandHome(pickString(p["workspaceRoot"], defaultWorkspaceRoot())),
|
|
1075
1101
|
mirrorRoot: expandHome(pickString(p["mirrorRoot"], defaultMirrorRoot())),
|
|
@@ -1223,6 +1249,8 @@ function finalizeRepos(parsed: unknown, label: string, problems: string[]): Reco
|
|
|
1223
1249
|
if (graph !== undefined) target.graphProject = graph;
|
|
1224
1250
|
const migrations = finalizeMigrationsDir(value?.["migrations"], `${label}: routing.repos.${key}`, problems);
|
|
1225
1251
|
if (migrations !== undefined) target.migrations = { dir: migrations };
|
|
1252
|
+
const versionFile = finalizeVersionFile(value?.["release"], `${label}: routing.repos.${key}`, problems);
|
|
1253
|
+
if (versionFile !== undefined) target.release = { versionFile };
|
|
1226
1254
|
repos[key] = target;
|
|
1227
1255
|
}
|
|
1228
1256
|
return repos;
|
|
@@ -1262,6 +1290,21 @@ function finalizeMigrationsDir(parsed: unknown, label: string, problems: string[
|
|
|
1262
1290
|
return dir;
|
|
1263
1291
|
}
|
|
1264
1292
|
|
|
1293
|
+
function finalizeVersionFile(parsed: unknown, label: string, problems: string[]): string | undefined {
|
|
1294
|
+
if (parsed === undefined) return undefined;
|
|
1295
|
+
const raw = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? (parsed as Raw) : undefined;
|
|
1296
|
+
const file = raw?.["versionFile"];
|
|
1297
|
+
if (typeof file !== "string" || file.trim() === "") {
|
|
1298
|
+
problems.push(`${label}.release.versionFile must be a non-empty string`);
|
|
1299
|
+
return undefined;
|
|
1300
|
+
}
|
|
1301
|
+
if (file.startsWith("/") || file.split("/").includes("..")) {
|
|
1302
|
+
problems.push(`${label}.release.versionFile must be a repo-relative file`);
|
|
1303
|
+
return undefined;
|
|
1304
|
+
}
|
|
1305
|
+
return file;
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1265
1308
|
function finalizeGraphProject(parsed: unknown, label: string, problems: string[]): string | undefined {
|
|
1266
1309
|
if (parsed === undefined) return undefined;
|
|
1267
1310
|
if (typeof parsed !== "string" || parsed.trim() === "") {
|
package/src/escalate.ts
CHANGED
|
@@ -269,7 +269,7 @@ export function createEscalator(
|
|
|
269
269
|
// the tick re-raises it. A *report* has no such source — it exists
|
|
270
270
|
// once, in the model's head, and nothing regenerates it — which is
|
|
271
271
|
// why that one needed a ledger and this one does not.
|
|
272
|
-
await sendTelegram(token, chatId, text, { topicId: p
|
|
272
|
+
await sendTelegram(token, chatId, text, { topicId: resolveProjectTopicId(p) });
|
|
273
273
|
store.markNotified(key);
|
|
274
274
|
return;
|
|
275
275
|
}
|
|
@@ -350,6 +350,74 @@ export function readTelegramToken(): string | undefined {
|
|
|
350
350
|
return undefined;
|
|
351
351
|
}
|
|
352
352
|
|
|
353
|
+
/**
|
|
354
|
+
* The forum topic a project's Telegram actually lives in *now*.
|
|
355
|
+
*
|
|
356
|
+
* `escalation.telegramTopicId` is pinned by the wizard from the topics
|
|
357
|
+
* omp-telegram had claimed at setup time — but the bridge re-claims a pane's
|
|
358
|
+
* topic across restarts, and conductor restarts that service itself during
|
|
359
|
+
* `upgrade` and `restart`. The pinned id therefore goes stale on the fleet's
|
|
360
|
+
* own maintenance, and every topic-addressed send silently degrades to the main
|
|
361
|
+
* chat: tier-2 pages, reports, digests, arm challenges, direct messages (#407).
|
|
362
|
+
*
|
|
363
|
+
* The operator's pin still wins whenever it is live. Only a pin that is
|
|
364
|
+
* *provably* absent from the current claims is replaced, and only by the claim
|
|
365
|
+
* whose name is this project's — so a deliberately separate alerts topic is
|
|
366
|
+
* never hijacked by the pane's own thread. Unreadable bridge state changes
|
|
367
|
+
* nothing, and #318's stale-topic retry remains the last line of defence.
|
|
368
|
+
*
|
|
369
|
+
* The substitution is logged, naming the project and the claim's name. Never an
|
|
370
|
+
* id: a log line is a place these leak from.
|
|
371
|
+
*/
|
|
372
|
+
export function resolveProjectTopicId(project: ProjectConfig): number | undefined {
|
|
373
|
+
const pinned = project.escalation.telegramTopicId;
|
|
374
|
+
if (pinned === undefined) return undefined;
|
|
375
|
+
const claims = claimedTelegramTopics();
|
|
376
|
+
if (claims.length === 0) return pinned;
|
|
377
|
+
if (claims.some((claim) => claim.threadId === pinned)) return pinned;
|
|
378
|
+
const named = claims.find((claim) => claim.name === project.name);
|
|
379
|
+
if (named === undefined) return pinned;
|
|
380
|
+
warn(
|
|
381
|
+
`escalation.telegramTopicId for ${project.name} is no longer a claimed topic; ` +
|
|
382
|
+
`following omp-telegram's current "${named.name}" claim instead`,
|
|
383
|
+
);
|
|
384
|
+
return named.threadId;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* omp-telegram's live topic claims: `{ threads: { "<threadId>": { name } } }`.
|
|
389
|
+
* Borrowed exactly as the token is, and tolerant of every shape it might not
|
|
390
|
+
* be — a bridge that has never run a forum pane has no file at all.
|
|
391
|
+
*
|
|
392
|
+
* `stateDir` is passed by the setup wizard, which has already probed for the
|
|
393
|
+
* bridge; send-time callers let it resolve the same way the token does.
|
|
394
|
+
*/
|
|
395
|
+
export function claimedTelegramTopics(stateDir?: string): Array<{ threadId: number; name: string }> {
|
|
396
|
+
const override = stateDir?.trim() ?? process.env.OMP_TELEGRAM_STATE_DIR?.trim();
|
|
397
|
+
const dir = override ? override : join(homedir(), ".omp", "agent", "telegram");
|
|
398
|
+
let raw: unknown;
|
|
399
|
+
try {
|
|
400
|
+
raw = JSON.parse(readFileSync(join(dir, "threads.json"), "utf8"));
|
|
401
|
+
} catch {
|
|
402
|
+
return [];
|
|
403
|
+
}
|
|
404
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw) || !("threads" in raw)) return [];
|
|
405
|
+
const threads = raw.threads;
|
|
406
|
+
if (typeof threads !== "object" || threads === null || Array.isArray(threads)) return [];
|
|
407
|
+
const out: Array<{ threadId: number; name: string }> = [];
|
|
408
|
+
for (const [id, entry] of Object.entries(threads)) {
|
|
409
|
+
const threadId = Number(id);
|
|
410
|
+
if (!Number.isFinite(threadId) || !Number.isSafeInteger(threadId)) continue;
|
|
411
|
+
let name = id;
|
|
412
|
+
if (typeof entry === "object" && entry !== null && "name" in entry) {
|
|
413
|
+
const candidate = entry.name;
|
|
414
|
+
if (typeof candidate === "string" && candidate.trim() !== "") name = candidate.trim();
|
|
415
|
+
}
|
|
416
|
+
out.push({ threadId, name });
|
|
417
|
+
}
|
|
418
|
+
return out;
|
|
419
|
+
}
|
|
420
|
+
|
|
353
421
|
/**
|
|
354
422
|
* The one Telegram send in this package. Exported so the report outbox (#123)
|
|
355
423
|
* reuses it rather than forking it: the response handling below is load-bearing
|
package/src/fleet.ts
CHANGED
|
@@ -29,7 +29,7 @@ import { homedir } from "node:os";
|
|
|
29
29
|
import { dirname, join } from "node:path";
|
|
30
30
|
import { formatZonedMinute } from "./availability.ts";
|
|
31
31
|
import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
32
|
-
import { sendTelegram } from "./escalate.ts";
|
|
32
|
+
import { resolveProjectTopicId, sendTelegram } from "./escalate.ts";
|
|
33
33
|
import { planUsageLine, readPlanUsage, sharedUsageSource } from "./usage.ts";
|
|
34
34
|
import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
|
|
35
35
|
import { inspectBriefLayout } from "./brief-upgrade.ts";
|
|
@@ -56,6 +56,7 @@ import {
|
|
|
56
56
|
isAlive,
|
|
57
57
|
livingDaemon,
|
|
58
58
|
probeUnit,
|
|
59
|
+
runSystemctl,
|
|
59
60
|
stopDaemon,
|
|
60
61
|
type StopResult,
|
|
61
62
|
type UnitOwnership,
|
|
@@ -348,12 +349,13 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
348
349
|
`Reply to this chat with exactly:\n${code}\n` +
|
|
349
350
|
`Nothing will be dispatched until that reply is seen in the orchestrator session.`;
|
|
350
351
|
|
|
351
|
-
// Prefer the project's
|
|
352
|
-
// escalations already do (#318)
|
|
352
|
+
// Prefer the project's live forum topic so arm challenges land where
|
|
353
|
+
// escalations already do (#318), following the bridge's current claim when the
|
|
354
|
+
// pinned id has gone stale (#407). Missing project config keeps flat-chat 0.13.
|
|
353
355
|
let topicId: number | undefined;
|
|
354
356
|
if (named !== undefined) {
|
|
355
357
|
try {
|
|
356
|
-
topicId = findProject(loadConfig(), named)
|
|
358
|
+
topicId = resolveProjectTopicId(findProject(loadConfig(), named));
|
|
357
359
|
} catch {
|
|
358
360
|
/* no project config */
|
|
359
361
|
}
|
|
@@ -566,21 +568,7 @@ export interface HerdrStartDeps {
|
|
|
566
568
|
* systemd or without that optional unit keep the standalone daemon behaviour.
|
|
567
569
|
*/
|
|
568
570
|
export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {}): HerdrStartResult {
|
|
569
|
-
const run
|
|
570
|
-
deps.systemctl ??
|
|
571
|
-
((args: string[]) => {
|
|
572
|
-
const res = spawnSync("systemctl", args, { encoding: "utf8", timeout: 15_000, env: process.env });
|
|
573
|
-
if (res.error) {
|
|
574
|
-
const err = res.error as NodeJS.ErrnoException;
|
|
575
|
-
return {
|
|
576
|
-
ok: false,
|
|
577
|
-
stdout: "",
|
|
578
|
-
stderr: err.message,
|
|
579
|
-
...(err.code === "ENOENT" ? { missing: true } : {}),
|
|
580
|
-
};
|
|
581
|
-
}
|
|
582
|
-
return { ok: res.status === 0, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
|
|
583
|
-
});
|
|
571
|
+
const run = deps.systemctl ?? runSystemctl;
|
|
584
572
|
|
|
585
573
|
const shown = run(["show", DEFAULT_HERDR_UNIT, "--property=LoadState", "--value"]);
|
|
586
574
|
if (shown.missing) return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "no systemctl" };
|
|
@@ -2000,26 +1988,18 @@ export async function transcriptHasUserCode(path: string, code: string): Promise
|
|
|
2000
1988
|
// ---------------------------------------------------------------------------
|
|
2001
1989
|
|
|
2002
1990
|
function probeHerdrUnit(unit = DEFAULT_HERDR_UNIT): { kind: HerdrLayer; detail?: string } {
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
}
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
}
|
|
2014
|
-
|
|
2015
|
-
if (out === "active") return { kind: "active", detail: unit };
|
|
2016
|
-
if (out === "inactive" || out === "failed" || out === "dead") {
|
|
2017
|
-
return { kind: "inactive", detail: `${unit} ${out}` };
|
|
2018
|
-
}
|
|
2019
|
-
return { kind: "unknown", detail: `${unit} ${out || `exit ${String(res.status)}`}` };
|
|
2020
|
-
} catch (err) {
|
|
2021
|
-
return { kind: "unknown", detail: err instanceof Error ? err.message : String(err) };
|
|
2022
|
-
}
|
|
1991
|
+
const res = runSystemctl(["is-active", unit]);
|
|
1992
|
+
if (res.missing) return { kind: "unknown", detail: "no systemctl" };
|
|
1993
|
+
const out = res.stdout.trim();
|
|
1994
|
+
if (out === "active") return { kind: "active", detail: unit };
|
|
1995
|
+
if (out === "inactive" || out === "failed" || out === "dead") {
|
|
1996
|
+
return { kind: "inactive", detail: `${unit} ${out}` };
|
|
1997
|
+
}
|
|
1998
|
+
const error = res.stderr.trim();
|
|
1999
|
+
return {
|
|
2000
|
+
kind: "unknown",
|
|
2001
|
+
detail: `${unit} ${out || error || "systemctl is-active failed"}`,
|
|
2002
|
+
};
|
|
2023
2003
|
}
|
|
2024
2004
|
|
|
2025
2005
|
export function paneLayerFromAgents(
|
package/src/lifecycle.ts
CHANGED
|
@@ -195,18 +195,34 @@ export async function healthCheck(port: number): Promise<{ ok: boolean; body?: s
|
|
|
195
195
|
}
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
+
function healthServesProject(body: string | undefined, project: string | undefined): boolean {
|
|
199
|
+
if (project === undefined) return true;
|
|
200
|
+
if (body === undefined) return false;
|
|
201
|
+
try {
|
|
202
|
+
const parsed = JSON.parse(body) as unknown;
|
|
203
|
+
if (parsed === null || typeof parsed !== "object") return false;
|
|
204
|
+
const projects = Reflect.get(parsed, "projects");
|
|
205
|
+
if (!Array.isArray(projects)) return false;
|
|
206
|
+
return projects.some((entry) => {
|
|
207
|
+
if (typeof entry === "string") return entry === project;
|
|
208
|
+
return entry !== null && typeof entry === "object" && Reflect.get(entry, "project") === project;
|
|
209
|
+
});
|
|
210
|
+
} catch {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
198
215
|
/**
|
|
199
|
-
* Starts the daemon
|
|
200
|
-
* `/healthz`.
|
|
216
|
+
* Starts the daemon and does not return until it answers `/healthz`.
|
|
201
217
|
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
* both the pid and the endpoint, and fail loudly with the log if either says
|
|
207
|
-
* no.
|
|
218
|
+
* An installed unit owns the host-wide daemon lifecycle. `start --project X`
|
|
219
|
+
* may select the project whose health is proved, but it must not turn the unit
|
|
220
|
+
* into a detached single-project competitor (#400). Only a host proven not to
|
|
221
|
+
* have the unit keeps the standalone spawn path.
|
|
208
222
|
*/
|
|
209
|
-
export async function startDaemon(
|
|
223
|
+
export async function startDaemon(
|
|
224
|
+
o: { port?: number; project?: string; timeoutMs?: number } = {},
|
|
225
|
+
): Promise<DaemonRecord> {
|
|
210
226
|
const running = livingDaemon();
|
|
211
227
|
if (running !== undefined) {
|
|
212
228
|
throw new Error(
|
|
@@ -215,6 +231,28 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
215
231
|
);
|
|
216
232
|
}
|
|
217
233
|
|
|
234
|
+
const installation = probeUnitInstallation();
|
|
235
|
+
if (installation.kind === "unknown") {
|
|
236
|
+
throw new Error(
|
|
237
|
+
`cannot determine whether ${SYSTEMD_UNIT} is installed (${installation.reason}); ` +
|
|
238
|
+
"refusing to launch a detached daemon that could compete with the service manager",
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
if (installation.kind === "installed") {
|
|
242
|
+
const ownership = probeUnit();
|
|
243
|
+
if (ownership.kind === "unknown") {
|
|
244
|
+
throw new Error(ownershipUnknown("start", ownership.reason));
|
|
245
|
+
}
|
|
246
|
+
if (ownership.kind === "failed") {
|
|
247
|
+
return await restoreFailedUnit(o.timeoutMs, o.project);
|
|
248
|
+
}
|
|
249
|
+
const started = systemctl(["start", SYSTEMD_UNIT]);
|
|
250
|
+
if (!started.ok) {
|
|
251
|
+
throw new Error(systemctlFailure("start", started));
|
|
252
|
+
}
|
|
253
|
+
return await waitForOwnedDaemon("start", o.timeoutMs, o.project);
|
|
254
|
+
}
|
|
255
|
+
|
|
218
256
|
const port = o.port ?? DEFAULT_PORT;
|
|
219
257
|
const logFile = join(ensureRuntimeDir(), "daemon.log");
|
|
220
258
|
|
|
@@ -260,7 +298,7 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
260
298
|
// rather than nothing at all.
|
|
261
299
|
writeRecord(record);
|
|
262
300
|
|
|
263
|
-
const deadline = Date.now() + READY_TIMEOUT_MS;
|
|
301
|
+
const deadline = Date.now() + (o.timeoutMs ?? READY_TIMEOUT_MS);
|
|
264
302
|
for (;;) {
|
|
265
303
|
// Liveness first. If the child is gone, a healthy answer on that port came
|
|
266
304
|
// from somebody else's server, and reporting it as ours would be worse
|
|
@@ -270,7 +308,7 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
270
308
|
throw new Error(`daemon exited during startup${tailLog(logFile)}`);
|
|
271
309
|
}
|
|
272
310
|
const health = await healthCheck(port);
|
|
273
|
-
if (health.ok) return record;
|
|
311
|
+
if (health.ok && healthServesProject(health.body, o.project)) return record;
|
|
274
312
|
if (Date.now() >= deadline) break;
|
|
275
313
|
await sleep(READY_POLL_MS);
|
|
276
314
|
}
|
|
@@ -280,7 +318,7 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
280
318
|
await terminate(pid, STOP_TIMEOUT_MS);
|
|
281
319
|
clearRecord();
|
|
282
320
|
throw new Error(
|
|
283
|
-
`daemon did not answer http://127.0.0.1:${port}/healthz within ${Math.round(READY_TIMEOUT_MS / 1000)}s${tailLog(logFile)}`,
|
|
321
|
+
`daemon did not answer http://127.0.0.1:${port}/healthz for the requested project within ${Math.round((o.timeoutMs ?? READY_TIMEOUT_MS) / 1000)}s${tailLog(logFile)}`,
|
|
284
322
|
);
|
|
285
323
|
}
|
|
286
324
|
|
|
@@ -395,7 +433,7 @@ export async function restartDaemon(
|
|
|
395
433
|
// startDaemon() — that leaves the unit failed while handing the operator
|
|
396
434
|
// a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
|
|
397
435
|
// Any manager refusal is terminal; so is unproven ownership afterwards.
|
|
398
|
-
const record = await restoreFailedUnit(o.timeoutMs);
|
|
436
|
+
const record = await restoreFailedUnit(o.timeoutMs, o.project);
|
|
399
437
|
return { previous, record, via: "systemctl" };
|
|
400
438
|
}
|
|
401
439
|
|
|
@@ -413,7 +451,7 @@ export async function restartDaemon(
|
|
|
413
451
|
// systemctl restart returns once the new MainPID is up; the pidfile is
|
|
414
452
|
// written by the daemon itself on boot, so wait for that rather than
|
|
415
453
|
// inventing a record from the unit alone.
|
|
416
|
-
const record = await waitForOwnedDaemon("restart", o.timeoutMs);
|
|
454
|
+
const record = await waitForOwnedDaemon("restart", o.timeoutMs, o.project);
|
|
417
455
|
return { previous, record, via: "systemctl" };
|
|
418
456
|
}
|
|
419
457
|
|
|
@@ -430,7 +468,7 @@ export async function restartDaemon(
|
|
|
430
468
|
* (see {@link waitForOwnedDaemon}). Refusal or unproven ownership is terminal
|
|
431
469
|
* — never a fallthrough to the detached CLI daemon.
|
|
432
470
|
*/
|
|
433
|
-
async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
|
|
471
|
+
async function restoreFailedUnit(timeoutMs?: number, project?: string): Promise<DaemonRecord> {
|
|
434
472
|
const reset = systemctl(["reset-failed", SYSTEMD_UNIT]);
|
|
435
473
|
if (!reset.ok) {
|
|
436
474
|
throw new Error(systemctlFailure("reset-failed", reset));
|
|
@@ -439,7 +477,7 @@ async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
|
|
|
439
477
|
if (!started.ok) {
|
|
440
478
|
throw new Error(systemctlFailure("start", started));
|
|
441
479
|
}
|
|
442
|
-
return await waitForOwnedDaemon("start", timeoutMs);
|
|
480
|
+
return await waitForOwnedDaemon("start", timeoutMs, project);
|
|
443
481
|
}
|
|
444
482
|
|
|
445
483
|
/**
|
|
@@ -450,7 +488,11 @@ async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
|
|
|
450
488
|
* cannot un-fail it. Any other unproven state fails at the deadline with a
|
|
451
489
|
* diagnostic that names the mismatch, never a bare "not ready".
|
|
452
490
|
*/
|
|
453
|
-
async function waitForOwnedDaemon(
|
|
491
|
+
async function waitForOwnedDaemon(
|
|
492
|
+
verb: "restart" | "start",
|
|
493
|
+
timeoutMs?: number,
|
|
494
|
+
project?: string,
|
|
495
|
+
): Promise<DaemonRecord> {
|
|
454
496
|
const via = `systemctl ${verb} ${SYSTEMD_UNIT} returned`;
|
|
455
497
|
const inspect = `systemctl status ${SYSTEMD_UNIT}`;
|
|
456
498
|
const deadline = Date.now() + (timeoutMs ?? READY_TIMEOUT_MS);
|
|
@@ -459,7 +501,7 @@ async function waitForOwnedDaemon(verb: "restart" | "start", timeoutMs?: number)
|
|
|
459
501
|
const ownership = probeUnit();
|
|
460
502
|
if (rec !== undefined && ownership.kind === "active" && ownership.pid === rec.pid) {
|
|
461
503
|
const health = await healthCheck(rec.port);
|
|
462
|
-
if (health.ok) return rec;
|
|
504
|
+
if (health.ok && healthServesProject(health.body, project)) return rec;
|
|
463
505
|
} else if (ownership.kind === "failed") {
|
|
464
506
|
// A confirmed negative: the start did not take and the unit is failed
|
|
465
507
|
// again. Waiting longer cannot un-fail it.
|
|
@@ -482,9 +524,10 @@ async function waitForOwnedDaemon(verb: "restart" | "start", timeoutMs?: number)
|
|
|
482
524
|
}
|
|
483
525
|
if (ownership.kind === "active") {
|
|
484
526
|
if (ownership.pid === rec.pid) {
|
|
527
|
+
const projectDetail = project === undefined ? "" : ` for project ${project}`;
|
|
485
528
|
throw new Error(
|
|
486
|
-
`${via}, but the daemon never answered /healthz on :${rec.port} — ` +
|
|
487
|
-
`the service manager owns pid ${rec.pid}, but it is not serving; ` +
|
|
529
|
+
`${via}, but the daemon never answered /healthz${projectDetail} on :${rec.port} — ` +
|
|
530
|
+
`the service manager owns pid ${rec.pid}, but it is not serving the requested project; ` +
|
|
488
531
|
`check \`${inspect}\` and ${rec.logFile}`,
|
|
489
532
|
);
|
|
490
533
|
}
|
|
@@ -579,6 +622,31 @@ export function probeUnit(unit = SYSTEMD_UNIT): UnitOwnership {
|
|
|
579
622
|
return { kind: "inactive" };
|
|
580
623
|
}
|
|
581
624
|
|
|
625
|
+
type UnitInstallation =
|
|
626
|
+
| { kind: "installed" }
|
|
627
|
+
| { kind: "absent" }
|
|
628
|
+
| { kind: "unknown"; reason: string };
|
|
629
|
+
|
|
630
|
+
/**
|
|
631
|
+
* Installation is separate from ownership: an inactive installed unit and an
|
|
632
|
+
* absent unit both have MainPID 0, but only the latter permits a detached
|
|
633
|
+
* fallback.
|
|
634
|
+
*/
|
|
635
|
+
function probeUnitInstallation(unit = SYSTEMD_UNIT): UnitInstallation {
|
|
636
|
+
const ran = systemctl(["show", unit, "--property=LoadState", "--value"]);
|
|
637
|
+
if (!ran.ok) {
|
|
638
|
+
if (ran.missing) return { kind: "absent" };
|
|
639
|
+
const detail = (ran.stderr.trim() || ran.stdout.trim() || "systemctl show failed").split("\n")[0]!;
|
|
640
|
+
return { kind: "unknown", reason: detail };
|
|
641
|
+
}
|
|
642
|
+
const loadState = ran.stdout.trim();
|
|
643
|
+
if (loadState === "not-found") return { kind: "absent" };
|
|
644
|
+
if (loadState.length === 0) return { kind: "unknown", reason: "systemctl returned no LoadState" };
|
|
645
|
+
// loaded, masked, error and bad-setting all prove a manager-known unit. Let
|
|
646
|
+
// `systemctl start` provide the actionable refusal for a broken definition.
|
|
647
|
+
return { kind: "installed" };
|
|
648
|
+
}
|
|
649
|
+
|
|
582
650
|
/**
|
|
583
651
|
* The MainPID of an *active* {@link SYSTEMD_UNIT}, or `undefined` when the
|
|
584
652
|
* unit is confirmed inactive, failed, or absent, or when ownership could not
|
|
@@ -662,7 +730,14 @@ function systemctlFailure(
|
|
|
662
730
|
);
|
|
663
731
|
}
|
|
664
732
|
|
|
665
|
-
function ownershipUnknown(verb: "stop" | "restart", reason: string): string {
|
|
733
|
+
function ownershipUnknown(verb: "start" | "stop" | "restart", reason: string): string {
|
|
734
|
+
if (verb === "start") {
|
|
735
|
+
return (
|
|
736
|
+
`cannot determine whether ${SYSTEMD_UNIT} owns the daemon (${reason}) — ` +
|
|
737
|
+
`refusing to start a detached competitor while ownership is unknown; ` +
|
|
738
|
+
`retry when systemctl answers, or run \`systemctl start ${SYSTEMD_UNIT}\` yourself`
|
|
739
|
+
);
|
|
740
|
+
}
|
|
666
741
|
return (
|
|
667
742
|
`cannot determine whether ${SYSTEMD_UNIT} owns the daemon (${reason}) — ` +
|
|
668
743
|
`refusing to ${verb} via signal while ownership is unknown ` +
|
|
@@ -689,7 +764,69 @@ export type SystemctlResult = {
|
|
|
689
764
|
|
|
690
765
|
export type SystemctlFn = (args: string[]) => SystemctlResult;
|
|
691
766
|
|
|
767
|
+
interface TestSystemctlState {
|
|
768
|
+
installed?: boolean;
|
|
769
|
+
mainPid?: number;
|
|
770
|
+
activeState?: string;
|
|
771
|
+
calls?: string[][];
|
|
772
|
+
failures?: Record<string, string>;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
/**
|
|
776
|
+
* Child-process CLI tests cannot inject {@link setSystemctlForTest}. Under
|
|
777
|
+
* NODE_ENV=test they therefore get a file-backed fake manager, or a refusal
|
|
778
|
+
* when no fixture was supplied. The real binary is never reached from a test
|
|
779
|
+
* process (#399).
|
|
780
|
+
*/
|
|
781
|
+
function testSystemctl(args: string[]): SystemctlResult {
|
|
782
|
+
const path = process.env["OMP_CONDUCTOR_TEST_SYSTEMCTL_STATE"];
|
|
783
|
+
if (path === undefined) {
|
|
784
|
+
return {
|
|
785
|
+
ok: false,
|
|
786
|
+
stdout: "",
|
|
787
|
+
stderr: "real systemctl is disabled under NODE_ENV=test; no fake manager state was supplied",
|
|
788
|
+
};
|
|
789
|
+
}
|
|
790
|
+
try {
|
|
791
|
+
const state = JSON.parse(readFileSync(path, "utf8")) as TestSystemctlState;
|
|
792
|
+
const calls = Array.isArray(state.calls) ? state.calls : [];
|
|
793
|
+
calls.push(args);
|
|
794
|
+
state.calls = calls;
|
|
795
|
+
const verb = args[0] ?? "";
|
|
796
|
+
const failure = state.failures?.[verb];
|
|
797
|
+
if (failure !== undefined) {
|
|
798
|
+
writeFileSync(path, JSON.stringify(state));
|
|
799
|
+
return { ok: false, stdout: "", stderr: failure };
|
|
800
|
+
}
|
|
801
|
+
let stdout = "";
|
|
802
|
+
if (verb === "show") {
|
|
803
|
+
stdout = args.includes("--property=LoadState")
|
|
804
|
+
? `${state.installed === false ? "not-found" : "loaded"}\n`
|
|
805
|
+
: `${state.mainPid ?? 0}\n${state.activeState ?? "inactive"}\n`;
|
|
806
|
+
} else if (verb === "is-active") {
|
|
807
|
+
const activeState = state.activeState ?? "inactive";
|
|
808
|
+
stdout = `${activeState}\n`;
|
|
809
|
+
writeFileSync(path, JSON.stringify(state));
|
|
810
|
+
return { ok: activeState === "active", stdout, stderr: "" };
|
|
811
|
+
} else if (verb === "stop") {
|
|
812
|
+
state.mainPid = 0;
|
|
813
|
+
state.activeState = "inactive";
|
|
814
|
+
} else if (verb === "reset-failed") {
|
|
815
|
+
state.activeState = "inactive";
|
|
816
|
+
}
|
|
817
|
+
writeFileSync(path, JSON.stringify(state));
|
|
818
|
+
return { ok: true, stdout, stderr: "" };
|
|
819
|
+
} catch (err) {
|
|
820
|
+
return {
|
|
821
|
+
ok: false,
|
|
822
|
+
stdout: "",
|
|
823
|
+
stderr: `fake systemctl state failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
824
|
+
};
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
|
|
692
828
|
function defaultSystemctl(args: string[]): SystemctlResult {
|
|
829
|
+
if (process.env["NODE_ENV"] === "test") return testSystemctl(args);
|
|
693
830
|
try {
|
|
694
831
|
const res = spawnSync("systemctl", args, {
|
|
695
832
|
encoding: "utf8",
|
|
@@ -719,6 +856,12 @@ function defaultSystemctl(args: string[]): SystemctlResult {
|
|
|
719
856
|
|
|
720
857
|
let systemctl: SystemctlFn = defaultSystemctl;
|
|
721
858
|
|
|
859
|
+
/** Shared manager runner. Production reaches the hardcoded binary; test
|
|
860
|
+
* processes reach only the explicit fake or the fail-closed refusal. */
|
|
861
|
+
export function runSystemctl(args: string[]): SystemctlResult {
|
|
862
|
+
return systemctl(args);
|
|
863
|
+
}
|
|
864
|
+
|
|
722
865
|
/** Test-only: replace the `systemctl` runner. Pass `undefined` to restore. */
|
|
723
866
|
export function setSystemctlForTest(fn: SystemctlFn | undefined): void {
|
|
724
867
|
systemctl = fn ?? defaultSystemctl;
|