omp-conductor 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -40
- package/REFERENCE.md +865 -30
- package/package.json +1 -1
- package/schema/config.schema.json +26 -0
- package/src/admission.ts +212 -26
- package/src/ask.ts +288 -1
- package/src/briefs/orchestrator.md +6 -5
- package/src/cli.ts +5 -1
- package/src/command-help.ts +9 -1
- package/src/command-manifest.ts +36 -3
- package/src/commands/arm.ts +5 -1
- package/src/commands/context.ts +2 -0
- package/src/commands/message.ts +26 -2
- package/src/commands/reconcile-units.ts +104 -0
- package/src/commands/release-composition.ts +232 -0
- package/src/commands/resume.ts +2 -27
- package/src/commands/setup.ts +101 -16
- package/src/commands/stats.ts +11 -30
- package/src/commands/tail.ts +31 -1
- package/src/commands/upgrade.ts +20 -3
- package/src/commands/verb.ts +2 -1
- package/src/config-schema.ts +19 -0
- package/src/config.ts +80 -0
- package/src/credential-class.ts +366 -0
- package/src/daemon.ts +1218 -288
- package/src/dashboard/app.js +504 -2
- package/src/dashboard/controls.ts +336 -0
- package/src/dashboard/index.html +30 -0
- package/src/dashboard/server.ts +271 -30
- package/src/dashboard/style.css +116 -0
- package/src/dashboard/transcript.ts +173 -0
- package/src/doctor.ts +377 -20
- package/src/failure-class.ts +59 -0
- package/src/fleet.ts +497 -15
- package/src/host.ts +6 -130
- package/src/omp.ts +29 -0
- package/src/orchestrator-tick.ts +343 -88
- package/src/pause.ts +233 -0
- package/src/settlement.ts +159 -2
- package/src/setup-answers.ts +97 -0
- package/src/setup-host.ts +321 -1155
- package/src/setup-install.ts +204 -27
- package/src/setup-wizard.ts +111 -50
- package/src/setup.ts +33 -0
- package/src/spend-telemetry.ts +117 -0
- package/src/stats.ts +35 -0
- package/src/status-render.ts +348 -19
- package/src/store.ts +1229 -55
- package/src/telegram-freshness.ts +269 -0
- package/src/to-spec.ts +27 -0
- package/src/types.ts +697 -4
- package/src/unblock.ts +22 -0
- package/src/unit-reconcile.ts +303 -0
- package/src/upgrade-verify.ts +8 -1
- package/src/upgrade.ts +299 -12
- package/src/verbs/actions.ts +124 -10
- package/src/verbs/protocol.ts +70 -2
- package/src/verbs/server.ts +447 -8
- package/src/wake.ts +48 -0
- package/src/worker.ts +403 -3
package/src/setup-install.ts
CHANGED
|
@@ -20,7 +20,8 @@ import { platform } from "node:os";
|
|
|
20
20
|
import { join } from "node:path";
|
|
21
21
|
import { loadConfig, stateDir } from "./config.ts";
|
|
22
22
|
import { pauseInstance, pauseSourceToken, setPaused, statusSnapshot } from "./daemon.ts";
|
|
23
|
-
import { fleetLayers, DEFAULT_HERDR_UNIT } from "./fleet.ts";
|
|
23
|
+
import { fleetLayers, DEFAULT_HERDR_SESSION, DEFAULT_HERDR_UNIT } from "./fleet.ts";
|
|
24
|
+
import { spawnSync } from "node:child_process";
|
|
24
25
|
import { livingDaemon } from "./lifecycle.ts";
|
|
25
26
|
import { dbPath, openStore } from "./store.ts";
|
|
26
27
|
import {
|
|
@@ -51,8 +52,9 @@ import { runPrivileged, type PrivilegedDeps, type PrivilegedStep } from "./privi
|
|
|
51
52
|
import {
|
|
52
53
|
agentRenameVerdict,
|
|
53
54
|
checkEscalation,
|
|
55
|
+
checkSessionRestartContext,
|
|
56
|
+
herdrOwnership,
|
|
54
57
|
DEFAULT_AGENT_RENAME_DEPS,
|
|
55
|
-
defaultIdentityProbes,
|
|
56
58
|
defaultServiceRuntime,
|
|
57
59
|
planHostRuntime,
|
|
58
60
|
totalConfiguredWorkers,
|
|
@@ -62,8 +64,9 @@ import {
|
|
|
62
64
|
RECOVER_SCRIPT_INSTALL_PATH,
|
|
63
65
|
type AgentRenameDeps,
|
|
64
66
|
type EscalationDeps,
|
|
67
|
+
type HerdrOwnership,
|
|
68
|
+
type HerdrOwnershipFacts,
|
|
65
69
|
type ServiceRuntime,
|
|
66
|
-
type WorkerIdentityProbes,
|
|
67
70
|
} from "./setup-host.ts";
|
|
68
71
|
import type { WizardUi } from "./wizard-ui.ts";
|
|
69
72
|
import type { Caps, ProjectConfig, Store } from "./types.ts";
|
|
@@ -84,14 +87,6 @@ export interface InstallDeps {
|
|
|
84
87
|
escalation?: EscalationDeps;
|
|
85
88
|
/** `"linux"` gates the systemd half. Injectable so the refusal is testable. */
|
|
86
89
|
platform?: () => string;
|
|
87
|
-
/**
|
|
88
|
-
* The worker identity plan's probes (#798). `undefined` means the real
|
|
89
|
-
* host's read-only facts — production: every `setup host` installs and
|
|
90
|
-
* grants the dedicated worker account. `null` disables the identity plan
|
|
91
|
-
* entirely (the historical step list, for tests pinning it), and an object
|
|
92
|
-
* is used as given (tests pinning a specific host state).
|
|
93
|
-
*/
|
|
94
|
-
workerIdentityProbes?: WorkerIdentityProbes | null;
|
|
95
90
|
unitDir?: string;
|
|
96
91
|
/**
|
|
97
92
|
* Where the recovery playbook installs. Injectable so the idempotency gate
|
|
@@ -133,6 +128,31 @@ export interface InstallDeps {
|
|
|
133
128
|
* default is the same `herdr --session` CLI recover.sh and the tick use.
|
|
134
129
|
*/
|
|
135
130
|
agentRename?: AgentRenameDeps;
|
|
131
|
+
/**
|
|
132
|
+
* The environment the caller's context is read from (#834): `$HERDR_ENV` says
|
|
133
|
+
* this shell is a pane inside the unit the install restarts. Injectable so
|
|
134
|
+
* the refusal is testable without a live Herdr session.
|
|
135
|
+
*/
|
|
136
|
+
env?: Record<string, string | undefined>;
|
|
137
|
+
/**
|
|
138
|
+
* The live supervision facts and the one remedy for them (#893). Injectable
|
|
139
|
+
* so the ownership transfer and its readiness gate are testable without
|
|
140
|
+
* touching real systemd or a real herdr server — production reads
|
|
141
|
+
* `herdr session list --json` and `systemctl show`.
|
|
142
|
+
*/
|
|
143
|
+
supervision?: SupervisionDeps;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** The reads and the one write the #893 ownership transfer needs. */
|
|
147
|
+
export interface SupervisionDeps {
|
|
148
|
+
/** Who is serving the fleet session right now. */
|
|
149
|
+
facts: (session: string) => HerdrOwnershipFacts;
|
|
150
|
+
/** Bounded readiness polling between reads. */
|
|
151
|
+
sleep: (ms: number) => Promise<void>;
|
|
152
|
+
/** How many times the post-restart gate re-reads before giving up. */
|
|
153
|
+
attempts?: number;
|
|
154
|
+
/** Milliseconds between those reads. */
|
|
155
|
+
intervalMs?: number;
|
|
136
156
|
}
|
|
137
157
|
|
|
138
158
|
/**
|
|
@@ -199,13 +219,6 @@ export async function runHostInstall(
|
|
|
199
219
|
// happens only after consent (#510), so the prompt's "Nothing has been run
|
|
200
220
|
// yet" is true of the staged tree when it is printed, and a declined run
|
|
201
221
|
// leaves every staged file byte-identical.
|
|
202
|
-
// The worker identity plan rides the same consent (#798): production probes
|
|
203
|
-
// the real host (account present? grants current?); a test may pass its own
|
|
204
|
-
// probes, or `null` to plan the historical step list only.
|
|
205
|
-
const identityProbes: WorkerIdentityProbes | null =
|
|
206
|
-
deps.workerIdentityProbes === null
|
|
207
|
-
? null
|
|
208
|
-
: deps.workerIdentityProbes ?? defaultIdentityProbes(deps.runtime ?? defaultServiceRuntime(telegramStateDir));
|
|
209
222
|
const plan = planHostRuntime(
|
|
210
223
|
project,
|
|
211
224
|
caps,
|
|
@@ -215,7 +228,6 @@ export async function runHostInstall(
|
|
|
215
228
|
unitDir,
|
|
216
229
|
deps.recoverScriptInstallPath ?? RECOVER_SCRIPT_INSTALL_PATH,
|
|
217
230
|
deps.multiProject ?? hostMultiProject(),
|
|
218
|
-
identityProbes,
|
|
219
231
|
);
|
|
220
232
|
// No pane-shell key is planned (unusable login shell, unparseable config):
|
|
221
233
|
// the operator hears why before the consent prompt, not after an install
|
|
@@ -257,6 +269,17 @@ export async function runHostInstall(
|
|
|
257
269
|
return { kind: "installed", wrote: [] };
|
|
258
270
|
}
|
|
259
271
|
|
|
272
|
+
// The caller must survive the transaction it started (#834). Placed here on
|
|
273
|
+
// purpose: after the plan and the no-op gate — so a host with no herdr unit,
|
|
274
|
+
// and a re-run with nothing to install, are never refused for a restart that
|
|
275
|
+
// is not going to happen — and before the drain fence and the post-consent
|
|
276
|
+
// staging, so a refusal has mutated nothing at all.
|
|
277
|
+
const context = checkSessionRestartContext("setup host", plan.steps, deps.env);
|
|
278
|
+
if (context.kind === "refuse") {
|
|
279
|
+
ui.notify(context.message, "error");
|
|
280
|
+
return { kind: "refused", reason: context.message };
|
|
281
|
+
}
|
|
282
|
+
|
|
260
283
|
// argv, not shell: `installCommands` renders `sudo …` strings for humans to
|
|
261
284
|
// read, and re-parsing those into an argv is how a path with a space becomes
|
|
262
285
|
// two arguments. The plan carries the exact argv once, in
|
|
@@ -298,7 +321,47 @@ export async function runHostInstall(
|
|
|
298
321
|
attributeRestartKills(fence.scope, project?.name ?? "", deps.drain?.drainStore);
|
|
299
322
|
};
|
|
300
323
|
|
|
301
|
-
|
|
324
|
+
// Who is serving the fleet session, before anything is staged or run (#893).
|
|
325
|
+
// A `systemctl restart` that returns 0 does not prove supervision moved: an
|
|
326
|
+
// unmanaged `herdr --session <name> server` holding the session makes every
|
|
327
|
+
// start of the unit exit 1, and `Type=simple` returns before the child finds
|
|
328
|
+
// out. So the collision is resolved by stopping the unmanaged session inside
|
|
329
|
+
// this transaction, or the transaction refuses before it mutates anything.
|
|
330
|
+
const supervision = deps.supervision ?? defaultSupervisionDeps();
|
|
331
|
+
const session = plan.herdrSession ?? DEFAULT_HERDR_SESSION;
|
|
332
|
+
let transferSteps: PrivilegedStep[] = [];
|
|
333
|
+
if (plan.herdrUnit !== undefined) {
|
|
334
|
+
const ownership = herdrOwnership(supervision.facts(session));
|
|
335
|
+
if (ownership.kind === "unknown") {
|
|
336
|
+
// The remedy here stops somebody's live session, so an unreadable fact is
|
|
337
|
+
// a refusal, never an assumption. Nothing has been staged or installed.
|
|
338
|
+
const reason =
|
|
339
|
+
`refused: ${ownership.detail}, so this install cannot tell whether ${DEFAULT_HERDR_UNIT} owns the ` +
|
|
340
|
+
`fleet session or an unmanaged server does. Stopping the wrong one kills a live fleet, so nothing ` +
|
|
341
|
+
`was staged or installed. Check \`systemctl status ${DEFAULT_HERDR_UNIT}\` and ` +
|
|
342
|
+
`\`herdr session list\`, then re-run.`;
|
|
343
|
+
ui.notify(reason, "error");
|
|
344
|
+
return { kind: "refused", reason };
|
|
345
|
+
}
|
|
346
|
+
if (ownership.kind === "unmanaged") {
|
|
347
|
+
ui.notify(
|
|
348
|
+
`The fleet session is served outside systemd: ${ownership.detail}. This install stops that server ` +
|
|
349
|
+
`first so ${DEFAULT_HERDR_UNIT} can take the session over — every pane in it restarts under the unit.`,
|
|
350
|
+
"warning",
|
|
351
|
+
);
|
|
352
|
+
transferSteps = [
|
|
353
|
+
{
|
|
354
|
+
// The fleet account's own session, so no escalation: `setup host`
|
|
355
|
+
// already runs as that account (the escalation guard proves it).
|
|
356
|
+
title: `stop the unmanaged herdr session "${session}" so ${DEFAULT_HERDR_UNIT} can own it`,
|
|
357
|
+
argv: ["herdr", "session", "stop", session],
|
|
358
|
+
unprivileged: true,
|
|
359
|
+
},
|
|
360
|
+
];
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
const outcome = await runPrivileged([...transferSteps, ...plan.steps], ui, {
|
|
302
365
|
...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
|
|
303
366
|
title: "Install and start the supervised session?",
|
|
304
367
|
answerKey: "install-host",
|
|
@@ -317,14 +380,22 @@ export async function runHostInstall(
|
|
|
317
380
|
]),
|
|
318
381
|
]),
|
|
319
382
|
"The unit runs as the account that staged it; nothing here changes that.",
|
|
320
|
-
...(
|
|
383
|
+
...(transferSteps.length === 0
|
|
384
|
+
? []
|
|
385
|
+
: [
|
|
386
|
+
`Stops the unmanaged herdr session "${session}" first — systemd cannot adopt a running process, so the ` +
|
|
387
|
+
"session has to be handed over. Every pane in it comes back under the unit.",
|
|
388
|
+
]),
|
|
389
|
+
// Retirement is named before the confirm that authorises it (#895): an
|
|
390
|
+
// operator approving this batch is approving an unmount, and a removal
|
|
391
|
+
// nobody announced is not consented to.
|
|
392
|
+
...(plan.retire === undefined
|
|
321
393
|
? []
|
|
322
394
|
: [
|
|
323
|
-
`
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
: plan.workerIdentity.pending),
|
|
395
|
+
`Retires host state an earlier release installed and this one no longer ships: ` +
|
|
396
|
+
`${[...plan.retire.units, ...plan.retire.staged].join(", ")}.`,
|
|
397
|
+
"Worker sessions run as the fleet account again, so the read-only bind that served the separate " +
|
|
398
|
+
"worker account is unmounted and its unit removed. The account itself is left alone.",
|
|
328
399
|
]),
|
|
329
400
|
"",
|
|
330
401
|
liveWorkers === 0
|
|
@@ -343,6 +414,27 @@ export async function runHostInstall(
|
|
|
343
414
|
if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
|
|
344
415
|
return { kind: "failed", reason: `${outcome.step.title} exited ${outcome.exitCode}` };
|
|
345
416
|
}
|
|
417
|
+
// Success is not "the steps exited 0" (#893). `Type=simple` returns before the
|
|
418
|
+
// session server can discover a collision and die, so the transaction proves
|
|
419
|
+
// the canonical unit is still up AND owns the session — bounded, because an
|
|
420
|
+
// auto-restart loop keeps looking momentarily alive.
|
|
421
|
+
if (plan.herdrUnit !== undefined) {
|
|
422
|
+
const settled = await proveSupervision(supervision, session);
|
|
423
|
+
if (settled.kind !== "unit") {
|
|
424
|
+
// Dispatch stays paused, deliberately and unlike a failed step: the fleet
|
|
425
|
+
// is being served by something nobody supervises, and admitting workers
|
|
426
|
+
// into that is the outcome this gate exists to prevent. The operator is
|
|
427
|
+
// told what was observed rather than a generic failure.
|
|
428
|
+
const reason =
|
|
429
|
+
`${DEFAULT_HERDR_UNIT} did not take over the fleet session: ${settled.detail}. ` +
|
|
430
|
+
`The units are installed, but supervision did not transfer, so dispatch stays paused. ` +
|
|
431
|
+
`Check \`systemctl status ${DEFAULT_HERDR_UNIT}\` and \`herdr session list\`, then re-run \`setup host\` ` +
|
|
432
|
+
`from a shell outside the session.`;
|
|
433
|
+
ui.notify(reason, "error");
|
|
434
|
+
return { kind: "failed", reason };
|
|
435
|
+
}
|
|
436
|
+
ui.notify(`${DEFAULT_HERDR_UNIT} owns the fleet session — ${settled.detail}.`, "info");
|
|
437
|
+
}
|
|
346
438
|
if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
|
|
347
439
|
ui.notify(`Installed and started ${STAGED_SERVICE_NAME}.`, "info");
|
|
348
440
|
// #541: the tick plan may have restamped agentName away from the shared
|
|
@@ -376,6 +468,84 @@ export async function runHostInstall(
|
|
|
376
468
|
return { kind: "installed", wrote };
|
|
377
469
|
}
|
|
378
470
|
|
|
471
|
+
/**
|
|
472
|
+
* Poll until the canonical unit owns the fleet session, or the bound runs out
|
|
473
|
+
* (#893). Returns the last verdict either way, so the caller reports what was
|
|
474
|
+
* actually observed rather than "timed out".
|
|
475
|
+
*
|
|
476
|
+
* `absent` and `unmanaged` are both failures here, for different reasons: the
|
|
477
|
+
* first is a unit that started and died (the collision's own signature under
|
|
478
|
+
* `Type=simple`), the second is a session still held outside it.
|
|
479
|
+
*/
|
|
480
|
+
async function proveSupervision(deps: SupervisionDeps, session: string): Promise<HerdrOwnership> {
|
|
481
|
+
const attempts = deps.attempts ?? SUPERVISION_PROBE_ATTEMPTS;
|
|
482
|
+
const intervalMs = deps.intervalMs ?? SUPERVISION_PROBE_INTERVAL_MS;
|
|
483
|
+
let verdict = herdrOwnership(deps.facts(session));
|
|
484
|
+
for (let attempt = 1; attempt < attempts && verdict.kind !== "unit"; attempt += 1) {
|
|
485
|
+
await deps.sleep(intervalMs);
|
|
486
|
+
verdict = herdrOwnership(deps.facts(session));
|
|
487
|
+
}
|
|
488
|
+
return verdict;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
/** How many reads the readiness gate takes, and how far apart. Twelve seconds
|
|
492
|
+
* total: long enough for a session server to bind its socket, short enough
|
|
493
|
+
* that a restart loop is reported rather than waited out. */
|
|
494
|
+
const SUPERVISION_PROBE_ATTEMPTS = 12;
|
|
495
|
+
const SUPERVISION_PROBE_INTERVAL_MS = 1_000;
|
|
496
|
+
|
|
497
|
+
/** The real reads: herdr's own session list, and systemd's view of the unit. */
|
|
498
|
+
function defaultSupervisionDeps(): SupervisionDeps {
|
|
499
|
+
return {
|
|
500
|
+
facts: (session) => ({
|
|
501
|
+
sessionRunning: herdrSessionRunning(session),
|
|
502
|
+
unit: herdrUnitState(),
|
|
503
|
+
}),
|
|
504
|
+
sleep: Bun.sleep,
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/** Whether herdr reports a live server for one session, or `undefined` when it
|
|
509
|
+
* could not be asked — never `false` on a failed read (#893). */
|
|
510
|
+
function herdrSessionRunning(session: string): boolean | undefined {
|
|
511
|
+
const ran = spawnSync("herdr", ["session", "list", "--json"], { encoding: "utf8", timeout: 10_000 });
|
|
512
|
+
if (ran.status !== 0 || typeof ran.stdout !== "string") return undefined;
|
|
513
|
+
try {
|
|
514
|
+
const parsed: unknown = JSON.parse(ran.stdout);
|
|
515
|
+
const sessions = parsed !== null && typeof parsed === "object" ? Reflect.get(parsed, "sessions") : undefined;
|
|
516
|
+
if (!Array.isArray(sessions)) return undefined;
|
|
517
|
+
const row = sessions.find(
|
|
518
|
+
(entry) => entry !== null && typeof entry === "object" && Reflect.get(entry, "name") === session,
|
|
519
|
+
);
|
|
520
|
+
if (row === undefined) return false;
|
|
521
|
+
return Reflect.get(row as object, "running") === true;
|
|
522
|
+
} catch {
|
|
523
|
+
return undefined;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/** The canonical unit's live state, or `undefined` when systemctl could not
|
|
528
|
+
* answer. A property systemd omits reads as unreadable rather than as zero. */
|
|
529
|
+
function herdrUnitState(): HerdrOwnershipFacts["unit"] {
|
|
530
|
+
const ran = spawnSync(
|
|
531
|
+
"systemctl",
|
|
532
|
+
["show", DEFAULT_HERDR_UNIT, "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts"],
|
|
533
|
+
{ encoding: "utf8", timeout: 10_000 },
|
|
534
|
+
);
|
|
535
|
+
if (ran.status !== 0 || typeof ran.stdout !== "string") return undefined;
|
|
536
|
+
const values = new Map<string, string>();
|
|
537
|
+
for (const line of ran.stdout.split("\n")) {
|
|
538
|
+
const eq = line.indexOf("=");
|
|
539
|
+
if (eq > 0) values.set(line.slice(0, eq), line.slice(eq + 1));
|
|
540
|
+
}
|
|
541
|
+
const activeState = values.get("ActiveState");
|
|
542
|
+
const subState = values.get("SubState");
|
|
543
|
+
const mainPid = Number(values.get("MainPID"));
|
|
544
|
+
const nRestarts = Number(values.get("NRestarts"));
|
|
545
|
+
if (activeState === undefined || subState === undefined || !Number.isFinite(mainPid)) return undefined;
|
|
546
|
+
return { activeState, subState, mainPid, nRestarts: Number.isFinite(nRestarts) ? nRestarts : 0 };
|
|
547
|
+
}
|
|
548
|
+
|
|
379
549
|
/**
|
|
380
550
|
* The pause/drain fence's default accessors, overridable per test through
|
|
381
551
|
* {@link InstallDeps.drain.deps}. Mirrors `upgrade.ts`'s {@link DrainDeps}
|
|
@@ -395,7 +565,14 @@ function hostInstallDrainDeps(ui: WizardUi, deps: InstallDeps): DrainDeps {
|
|
|
395
565
|
: { running: true, project: daemon.project, generation: `${daemon.pid}@${daemon.startedAt}` };
|
|
396
566
|
},
|
|
397
567
|
pauseState: (project) => pauseInstance(project),
|
|
398
|
-
setPaused: (v, project) =>
|
|
568
|
+
setPaused: (v, project) =>
|
|
569
|
+
// `owner` is this process (#938): the fence lives exactly as long as this
|
|
570
|
+
// transaction, so a kill leaves it behind with nothing to explain it.
|
|
571
|
+
setPaused(
|
|
572
|
+
v,
|
|
573
|
+
{ source: pauseSourceToken("setup host"), reason: "setup host, draining", owner: process.pid },
|
|
574
|
+
project,
|
|
575
|
+
),
|
|
399
576
|
sleep: Bun.sleep,
|
|
400
577
|
log: (message) => ui.notify(message, "info"),
|
|
401
578
|
};
|
package/src/setup-wizard.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
import { claimedTelegramTopics } from "./escalate.ts";
|
|
29
29
|
import { hostRamBytes, recommendedMaxWorkers, workerOvercommit } from "./host.ts";
|
|
30
30
|
import {
|
|
31
|
+
admissionAckPath,
|
|
31
32
|
daemonGeneration,
|
|
32
33
|
isPaused,
|
|
33
34
|
pausedPath,
|
|
@@ -40,8 +41,10 @@ import {
|
|
|
40
41
|
type AdmissionAckRecord,
|
|
41
42
|
type QueuePreview,
|
|
42
43
|
} from "./daemon.ts";
|
|
44
|
+
import { pauseBytesValid } from "./pause.ts";
|
|
43
45
|
import { armedMarkerPath, armTicks, fleetLayers, telegramStateDir } from "./fleet.ts";
|
|
44
46
|
import {
|
|
47
|
+
daemonControlTarget,
|
|
45
48
|
healthCheck,
|
|
46
49
|
healthServesProject,
|
|
47
50
|
livingDaemon,
|
|
@@ -2143,8 +2146,15 @@ interface DerivedPlan {
|
|
|
2143
2146
|
export async function ensureSetupArm(
|
|
2144
2147
|
projectName: string,
|
|
2145
2148
|
arm: typeof armTicks = armTicks,
|
|
2149
|
+
/**
|
|
2150
|
+
* Where the arm proof's pending heartbeat goes while it waits (#861). Setup
|
|
2151
|
+
* holds dispatch under its own fence across this call, so a silent wait here
|
|
2152
|
+
* is the whole incident: the operator sees a stalled wizard and cannot tell
|
|
2153
|
+
* a live handshake from a dead process.
|
|
2154
|
+
*/
|
|
2155
|
+
progress?: (line: string) => void,
|
|
2146
2156
|
): Promise<string> {
|
|
2147
|
-
const armed = await arm(projectName);
|
|
2157
|
+
const armed = await arm(projectName, progress === undefined ? {} : { progress });
|
|
2148
2158
|
return armed.alreadyArmed
|
|
2149
2159
|
? `existing heartbeat arm revalidated for owner ${armed.owner} at ${armed.path}`
|
|
2150
2160
|
: `heartbeat armed for owner ${armed.owner} at ${armed.path}`;
|
|
@@ -2347,7 +2357,12 @@ export interface SetupApplyDeps {
|
|
|
2347
2357
|
drain: DrainDeps;
|
|
2348
2358
|
smoke: (project: string) => Promise<SetupSmokeResult>;
|
|
2349
2359
|
restart: (o: { project?: string }) => Promise<RestartResult>;
|
|
2350
|
-
|
|
2360
|
+
/**
|
|
2361
|
+
* Prove the heartbeat channel for one project. `progress` is where the
|
|
2362
|
+
* pending proof's own heartbeat is written while it waits on the operator
|
|
2363
|
+
* (#861) — the wizard owns the surface, so the reporter comes from the call.
|
|
2364
|
+
*/
|
|
2365
|
+
arm: (project: string, progress?: (line: string) => void) => Promise<string>;
|
|
2351
2366
|
hostInstall: (
|
|
2352
2367
|
project: ProjectConfig,
|
|
2353
2368
|
caps: Caps,
|
|
@@ -2388,7 +2403,15 @@ function defaultSetupDrain(): DrainDeps {
|
|
|
2388
2403
|
projectNames: () => loadConfig().projects.map((p) => p.name),
|
|
2389
2404
|
daemonIdentity: setupDaemonIdentity,
|
|
2390
2405
|
pauseState: (project) => pauseInstance(project),
|
|
2391
|
-
setPaused: (v, project) =>
|
|
2406
|
+
setPaused: (v, project) =>
|
|
2407
|
+
// `owner` is this process (#938): the apply fence holds dispatch across
|
|
2408
|
+
// this transaction only, and the 2026-08-21 incident is exactly what an
|
|
2409
|
+
// abandoned one looks like from outside.
|
|
2410
|
+
setPaused(
|
|
2411
|
+
v,
|
|
2412
|
+
{ source: pauseSourceToken("setup"), reason: "setup apply fence", owner: process.pid },
|
|
2413
|
+
project,
|
|
2414
|
+
),
|
|
2392
2415
|
// The daemon-side admission acknowledgement: the file the running daemon
|
|
2393
2416
|
// itself writes when it observes the fence at an admission boundary, so
|
|
2394
2417
|
// the barrier never mistakes a second synchronous count for the daemon's
|
|
@@ -2458,7 +2481,7 @@ export const DEFAULT_APPLY: SetupApplyDeps = {
|
|
|
2458
2481
|
drain: defaultSetupDrain(),
|
|
2459
2482
|
smoke: runSetupSmoke,
|
|
2460
2483
|
restart: (o) => restartDaemon(o),
|
|
2461
|
-
arm: (project) => ensureSetupArm(project),
|
|
2484
|
+
arm: (project, progress) => ensureSetupArm(project, armTicks, progress),
|
|
2462
2485
|
hostInstall: (project, caps, telegramStateDir, ui) =>
|
|
2463
2486
|
runHostInstall(project, caps, telegramStateDir, ui),
|
|
2464
2487
|
graphInstall: (project, ui, options) => runGraphInstall(project, ui, options),
|
|
@@ -2487,19 +2510,6 @@ type CapturedPause =
|
|
|
2487
2510
|
| { kind: "malformed"; bytes: string; path: string }
|
|
2488
2511
|
| { kind: "unreadable"; path: string };
|
|
2489
2512
|
|
|
2490
|
-
/** Whether one sentinel's bytes parse as a valid pause instance — the exact
|
|
2491
|
-
* grammar {@link pauseInstance} reads in `daemon.ts` (a parseable ISO
|
|
2492
|
-
* timestamp line, then a `source=` provenance line). Readable bytes that
|
|
2493
|
-
* fail this are an unreadable-as-state malformed hold: valid bytes are
|
|
2494
|
-
* restored verbatim, malformed ones are refused at entry and never deleted
|
|
2495
|
-
* (#650). */
|
|
2496
|
-
function isValidPauseBytes(bytes: string): boolean {
|
|
2497
|
-
const [line1, line2] = bytes.split("\n");
|
|
2498
|
-
if (!Number.isFinite(Date.parse(line1?.trim() ?? ""))) return false;
|
|
2499
|
-
if (line2 === undefined) return false;
|
|
2500
|
-
return /^source=(\S+)(?: reason="(.*)")?$/.test(line2.trim());
|
|
2501
|
-
}
|
|
2502
|
-
|
|
2503
2513
|
/** Reads one pause sentinel, keeping absence, readable-valid bytes, malformed
|
|
2504
2514
|
* bytes and unreadability apart — never conflated: collapsing malformed to
|
|
2505
2515
|
* absence would let a refusal delete an operator hold it could not parse. */
|
|
@@ -2507,7 +2517,7 @@ function readSentinel(p: string): CapturedPause {
|
|
|
2507
2517
|
if (!existsSync(p)) return { kind: "absent" };
|
|
2508
2518
|
try {
|
|
2509
2519
|
const bytes = readFileSync(p, "utf8");
|
|
2510
|
-
return
|
|
2520
|
+
return pauseBytesValid(bytes) ? { kind: "bytes", bytes } : { kind: "malformed", bytes, path: p };
|
|
2511
2521
|
} catch {
|
|
2512
2522
|
return { kind: "unreadable", path: p };
|
|
2513
2523
|
}
|
|
@@ -2666,7 +2676,6 @@ function captureInventory(
|
|
|
2666
2676
|
};
|
|
2667
2677
|
captureIfWritten(plan.runtime.service);
|
|
2668
2678
|
captureIfWritten(plan.runtime.herdrUnit);
|
|
2669
|
-
captureIfWritten(plan.runtime.harnessMount);
|
|
2670
2679
|
captureIfWritten(plan.runtime.herdrConfig);
|
|
2671
2680
|
captureIfWritten(plan.runtime.herdrEnv);
|
|
2672
2681
|
captureIfWritten(plan.runtime.recoverUnit);
|
|
@@ -2900,23 +2909,61 @@ const FENCE_ACK_POLL_MS = 250;
|
|
|
2900
2909
|
* barrier froze, or `undefined` when it does: the acknowledgement must name
|
|
2901
2910
|
* the exact pause instance the barrier proved (not some other pause, however
|
|
2902
2911
|
* similar) and the exact daemon generation the barrier began with (not an
|
|
2903
|
-
* older or newer instance's word) (#651 review #3).
|
|
2912
|
+
* older or newer instance's word) (#651 review #3). Every problem names the
|
|
2913
|
+
* observed and expected instances, so an operator can tell "no
|
|
2914
|
+
* acknowledgement yet" from "acknowledged something else" without opening a
|
|
2915
|
+
* debugger (#865).
|
|
2904
2916
|
*/
|
|
2905
2917
|
function fenceAckProblem(ack: AdmissionAckRecord | undefined, begun: RestartBegun): string | undefined {
|
|
2906
|
-
if (ack === undefined)
|
|
2918
|
+
if (ack === undefined) {
|
|
2919
|
+
return (
|
|
2920
|
+
"the running daemon has not acknowledged the setup admission fence " +
|
|
2921
|
+
`(fence ${describePauseInstance(begun.pauseToken)}; expected daemon generation ${begun.daemon.generation}; ` +
|
|
2922
|
+
`no readable acknowledgement at ${admissionAckPath()})`
|
|
2923
|
+
);
|
|
2924
|
+
}
|
|
2907
2925
|
if (
|
|
2908
2926
|
ack.pause.source !== begun.pauseToken.source ||
|
|
2909
2927
|
ack.pause.since !== begun.pauseToken.since ||
|
|
2910
2928
|
ack.pause.reason !== begun.pauseToken.reason
|
|
2911
2929
|
) {
|
|
2912
|
-
return
|
|
2930
|
+
return (
|
|
2931
|
+
"the running daemon acknowledged a different pause than the setup admission fence " +
|
|
2932
|
+
`(acknowledged ${describePauseInstance(ack.pause)}; fence ${describePauseInstance(begun.pauseToken)})`
|
|
2933
|
+
);
|
|
2913
2934
|
}
|
|
2914
2935
|
if (ack.daemon !== begun.daemon.generation) {
|
|
2915
|
-
return
|
|
2936
|
+
return (
|
|
2937
|
+
"the running daemon's acknowledgement belongs to a different daemon generation " +
|
|
2938
|
+
`(acknowledged ${ack.daemon}; expected ${begun.daemon.generation})`
|
|
2939
|
+
);
|
|
2916
2940
|
}
|
|
2917
2941
|
return undefined;
|
|
2918
2942
|
}
|
|
2919
2943
|
|
|
2944
|
+
/** One pause instance as one readable token for refusal diagnostics (#865). */
|
|
2945
|
+
function describePauseInstance(pause: { source: string; reason?: string; since: number }): string {
|
|
2946
|
+
const reason = pause.reason === undefined ? "" : ` reason="${pause.reason}"`;
|
|
2947
|
+
return `source=${pause.source}${reason} since=${new Date(pause.since).toISOString()}`;
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2950
|
+
/**
|
|
2951
|
+
* Where a prompt-pass wake can provably land, or `undefined`: the pidfile
|
|
2952
|
+
* record's port while its process lives, else — a supervised daemon whose
|
|
2953
|
+
* runtime record went missing while the unit stayed active (#651 review #2) —
|
|
2954
|
+
* the run-control endpoint {@link daemonControlTarget} proves by answering
|
|
2955
|
+
* `/healthz`, the same consultation every run-control verb performs (#716).
|
|
2956
|
+
* A port is never guessed: when nothing provable answers, no wake is sent and
|
|
2957
|
+
* the caller's bounded wait covers the daemon's own next pass instead. The
|
|
2958
|
+
* pre-#865 barrier skipped BOTH the wake and the wait whenever the record was
|
|
2959
|
+
* unreadable, so a transient record loss refused three interviews in a row
|
|
2960
|
+
* against a healthy dispatcher that would have acknowledged within one tick.
|
|
2961
|
+
*/
|
|
2962
|
+
async function reachableWakeEndpoint(): Promise<{ port: number } | undefined> {
|
|
2963
|
+
const target = await daemonControlTarget();
|
|
2964
|
+
return target.kind === "record" || target.kind === "unit" ? { port: target.port } : undefined;
|
|
2965
|
+
}
|
|
2966
|
+
|
|
2920
2967
|
/**
|
|
2921
2968
|
* Why the acknowledged barrier no longer holds, or `undefined` when it does:
|
|
2922
2969
|
* a live worker anywhere in scope, the daemon-side admission acknowledgement
|
|
@@ -3391,39 +3438,41 @@ export async function setup(
|
|
|
3391
3438
|
// boundary and written that observation down — a second synchronous worker
|
|
3392
3439
|
// count is not an acknowledgement, because a tick already past its own
|
|
3393
3440
|
// pause gate can claim after it and before setup mutates. A running daemon
|
|
3394
|
-
// that has not yet acknowledged is woken
|
|
3395
|
-
//
|
|
3396
|
-
// acknowledged by the bounded deadline refuses before
|
|
3441
|
+
// that has not yet acknowledged is woken — through any endpoint /healthz
|
|
3442
|
+
// proves, record or record-less (#865) — to prompt an immediate pass; one
|
|
3443
|
+
// that still has not acknowledged by the bounded deadline refuses before
|
|
3444
|
+
// anything is written. The wait itself never depends on the pidfile: a
|
|
3445
|
+
// supervised daemon with a lost record still reaches its own gate within
|
|
3446
|
+
// one tick interval, and skipping the wait for it was the #865 refusal.
|
|
3397
3447
|
let requireAck = true;
|
|
3398
3448
|
if (begun.daemon.running) {
|
|
3399
3449
|
let ack = apply.drain.admissionAck?.(scope.pauseKey);
|
|
3400
|
-
let
|
|
3401
|
-
if (
|
|
3402
|
-
const
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
"
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3450
|
+
let problem = fenceAckProblem(ack, begun);
|
|
3451
|
+
if (problem !== undefined) {
|
|
3452
|
+
const wake = await reachableWakeEndpoint();
|
|
3453
|
+
ui.notify(
|
|
3454
|
+
wake !== undefined
|
|
3455
|
+
? `The running daemon has not yet acknowledged the setup admission fence — waking it on :${wake.port} to prompt a pass.`
|
|
3456
|
+
: "The running daemon has not yet acknowledged the setup admission fence — nothing provable answers /healthz to wake, so waiting for its own dispatch pass.",
|
|
3457
|
+
"info",
|
|
3458
|
+
);
|
|
3459
|
+
if (wake !== undefined) void wakeDaemon(wake.port);
|
|
3460
|
+
// `OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS` is the #399 test seam: a
|
|
3461
|
+
// regression proving this refusal burns the short test deadline
|
|
3462
|
+
// instead of the production 60s.
|
|
3463
|
+
const deadline =
|
|
3464
|
+
Date.now() + Number(process.env["OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS"] ?? FENCE_ACK_WAIT_MS);
|
|
3465
|
+
while (problem !== undefined && Date.now() < deadline) {
|
|
3466
|
+
await apply.drain.sleep(FENCE_ACK_POLL_MS);
|
|
3467
|
+
ack = apply.drain.admissionAck?.(scope.pauseKey);
|
|
3468
|
+
problem = fenceAckProblem(ack, begun);
|
|
3419
3469
|
}
|
|
3420
3470
|
}
|
|
3421
|
-
if (
|
|
3471
|
+
if (problem !== undefined) {
|
|
3422
3472
|
restorePause(priorPause, ownedFence);
|
|
3423
3473
|
ui.notify(
|
|
3424
|
-
|
|
3425
|
-
"
|
|
3426
|
-
"once it has acknowledged the fence.",
|
|
3474
|
+
`Setup stopped before writing anything: ${problem} — quiescence cannot be proven. ` +
|
|
3475
|
+
"Nothing has been changed; stop the daemon, or re-run setup once it has acknowledged the fence.",
|
|
3427
3476
|
"error",
|
|
3428
3477
|
);
|
|
3429
3478
|
return false;
|
|
@@ -3668,7 +3717,7 @@ export async function setup(
|
|
|
3668
3717
|
"info",
|
|
3669
3718
|
);
|
|
3670
3719
|
try {
|
|
3671
|
-
armLine = await apply.arm(plan.project.name);
|
|
3720
|
+
armLine = await apply.arm(plan.project.name, (line) => ui.notify(line, "info"));
|
|
3672
3721
|
} catch (err) {
|
|
3673
3722
|
ui.notify(
|
|
3674
3723
|
[
|
|
@@ -3781,6 +3830,18 @@ export async function setup(
|
|
|
3781
3830
|
`Heartbeat: ${armLine}.`,
|
|
3782
3831
|
"",
|
|
3783
3832
|
"Use the documented toy-issue drill to prove one complete worker path.",
|
|
3833
|
+
"",
|
|
3834
|
+
// The one grooming recommendation setup makes, and it is only a sentence
|
|
3835
|
+
// (#827). Deliberately inert: nothing above or below this line installs,
|
|
3836
|
+
// probes, version-checks or refuses on it, because a conductor guarantee
|
|
3837
|
+
// must never depend on prose that releases on somebody else's cadence
|
|
3838
|
+
// (#506's judgement). Printed only here, on the committed success path,
|
|
3839
|
+
// so a failed or cancelled apply can never imply setup finished.
|
|
3840
|
+
"Optional: the spec-out skills pack at github.com/mattpocock/skills (`npx skills`) adds a",
|
|
3841
|
+
"grooming workflow this fleet has no opinion about — `to-spec` / `to-tickets` for turning a",
|
|
3842
|
+
"rough idea into sized slices, and `grill-me` for the questions worth asking before work",
|
|
3843
|
+
"starts. Nothing was installed or checked just now, and conductor is fully functional",
|
|
3844
|
+
"without it: skip it freely.",
|
|
3784
3845
|
].join("\n"),
|
|
3785
3846
|
"info",
|
|
3786
3847
|
);
|