omp-conductor 0.18.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +105 -40
  2. package/REFERENCE.md +865 -30
  3. package/package.json +1 -1
  4. package/schema/config.schema.json +26 -0
  5. package/src/admission.ts +212 -26
  6. package/src/ask.ts +288 -1
  7. package/src/briefs/orchestrator.md +6 -5
  8. package/src/cli.ts +5 -1
  9. package/src/command-help.ts +9 -1
  10. package/src/command-manifest.ts +36 -3
  11. package/src/commands/arm.ts +5 -1
  12. package/src/commands/context.ts +2 -0
  13. package/src/commands/message.ts +26 -2
  14. package/src/commands/reconcile-units.ts +104 -0
  15. package/src/commands/release-composition.ts +232 -0
  16. package/src/commands/resume.ts +2 -27
  17. package/src/commands/setup.ts +101 -16
  18. package/src/commands/stats.ts +11 -30
  19. package/src/commands/tail.ts +31 -1
  20. package/src/commands/upgrade.ts +20 -3
  21. package/src/commands/verb.ts +2 -1
  22. package/src/config-schema.ts +19 -0
  23. package/src/config.ts +80 -0
  24. package/src/credential-class.ts +366 -0
  25. package/src/daemon.ts +1218 -288
  26. package/src/dashboard/app.js +504 -2
  27. package/src/dashboard/controls.ts +336 -0
  28. package/src/dashboard/index.html +30 -0
  29. package/src/dashboard/server.ts +271 -30
  30. package/src/dashboard/style.css +116 -0
  31. package/src/dashboard/transcript.ts +173 -0
  32. package/src/doctor.ts +377 -20
  33. package/src/failure-class.ts +59 -0
  34. package/src/fleet.ts +497 -15
  35. package/src/host.ts +6 -130
  36. package/src/omp.ts +29 -0
  37. package/src/orchestrator-tick.ts +343 -88
  38. package/src/pause.ts +233 -0
  39. package/src/settlement.ts +159 -2
  40. package/src/setup-answers.ts +97 -0
  41. package/src/setup-host.ts +321 -1155
  42. package/src/setup-install.ts +204 -27
  43. package/src/setup-wizard.ts +111 -50
  44. package/src/setup.ts +33 -0
  45. package/src/spend-telemetry.ts +117 -0
  46. package/src/stats.ts +35 -0
  47. package/src/status-render.ts +348 -19
  48. package/src/store.ts +1229 -55
  49. package/src/telegram-freshness.ts +269 -0
  50. package/src/to-spec.ts +27 -0
  51. package/src/types.ts +697 -4
  52. package/src/unblock.ts +22 -0
  53. package/src/unit-reconcile.ts +303 -0
  54. package/src/upgrade-verify.ts +8 -1
  55. package/src/upgrade.ts +299 -12
  56. package/src/verbs/actions.ts +124 -10
  57. package/src/verbs/protocol.ts +70 -2
  58. package/src/verbs/server.ts +447 -8
  59. package/src/wake.ts +48 -0
  60. package/src/worker.ts +403 -3
@@ -20,7 +20,8 @@ import { platform } from "node:os";
20
20
  import { join } from "node:path";
21
21
  import { loadConfig, stateDir } from "./config.ts";
22
22
  import { pauseInstance, pauseSourceToken, setPaused, statusSnapshot } from "./daemon.ts";
23
- import { fleetLayers, DEFAULT_HERDR_UNIT } from "./fleet.ts";
23
+ import { fleetLayers, DEFAULT_HERDR_SESSION, DEFAULT_HERDR_UNIT } from "./fleet.ts";
24
+ import { spawnSync } from "node:child_process";
24
25
  import { livingDaemon } from "./lifecycle.ts";
25
26
  import { dbPath, openStore } from "./store.ts";
26
27
  import {
@@ -51,8 +52,9 @@ import { runPrivileged, type PrivilegedDeps, type PrivilegedStep } from "./privi
51
52
  import {
52
53
  agentRenameVerdict,
53
54
  checkEscalation,
55
+ checkSessionRestartContext,
56
+ herdrOwnership,
54
57
  DEFAULT_AGENT_RENAME_DEPS,
55
- defaultIdentityProbes,
56
58
  defaultServiceRuntime,
57
59
  planHostRuntime,
58
60
  totalConfiguredWorkers,
@@ -62,8 +64,9 @@ import {
62
64
  RECOVER_SCRIPT_INSTALL_PATH,
63
65
  type AgentRenameDeps,
64
66
  type EscalationDeps,
67
+ type HerdrOwnership,
68
+ type HerdrOwnershipFacts,
65
69
  type ServiceRuntime,
66
- type WorkerIdentityProbes,
67
70
  } from "./setup-host.ts";
68
71
  import type { WizardUi } from "./wizard-ui.ts";
69
72
  import type { Caps, ProjectConfig, Store } from "./types.ts";
@@ -84,14 +87,6 @@ export interface InstallDeps {
84
87
  escalation?: EscalationDeps;
85
88
  /** `"linux"` gates the systemd half. Injectable so the refusal is testable. */
86
89
  platform?: () => string;
87
- /**
88
- * The worker identity plan's probes (#798). `undefined` means the real
89
- * host's read-only facts — production: every `setup host` installs and
90
- * grants the dedicated worker account. `null` disables the identity plan
91
- * entirely (the historical step list, for tests pinning it), and an object
92
- * is used as given (tests pinning a specific host state).
93
- */
94
- workerIdentityProbes?: WorkerIdentityProbes | null;
95
90
  unitDir?: string;
96
91
  /**
97
92
  * Where the recovery playbook installs. Injectable so the idempotency gate
@@ -133,6 +128,31 @@ export interface InstallDeps {
133
128
  * default is the same `herdr --session` CLI recover.sh and the tick use.
134
129
  */
135
130
  agentRename?: AgentRenameDeps;
131
+ /**
132
+ * The environment the caller's context is read from (#834): `$HERDR_ENV` says
133
+ * this shell is a pane inside the unit the install restarts. Injectable so
134
+ * the refusal is testable without a live Herdr session.
135
+ */
136
+ env?: Record<string, string | undefined>;
137
+ /**
138
+ * The live supervision facts and the one remedy for them (#893). Injectable
139
+ * so the ownership transfer and its readiness gate are testable without
140
+ * touching real systemd or a real herdr server — production reads
141
+ * `herdr session list --json` and `systemctl show`.
142
+ */
143
+ supervision?: SupervisionDeps;
144
+ }
145
+
146
+ /** The reads and the one write the #893 ownership transfer needs. */
147
+ export interface SupervisionDeps {
148
+ /** Who is serving the fleet session right now. */
149
+ facts: (session: string) => HerdrOwnershipFacts;
150
+ /** Bounded readiness polling between reads. */
151
+ sleep: (ms: number) => Promise<void>;
152
+ /** How many times the post-restart gate re-reads before giving up. */
153
+ attempts?: number;
154
+ /** Milliseconds between those reads. */
155
+ intervalMs?: number;
136
156
  }
137
157
 
138
158
  /**
@@ -199,13 +219,6 @@ export async function runHostInstall(
199
219
  // happens only after consent (#510), so the prompt's "Nothing has been run
200
220
  // yet" is true of the staged tree when it is printed, and a declined run
201
221
  // leaves every staged file byte-identical.
202
- // The worker identity plan rides the same consent (#798): production probes
203
- // the real host (account present? grants current?); a test may pass its own
204
- // probes, or `null` to plan the historical step list only.
205
- const identityProbes: WorkerIdentityProbes | null =
206
- deps.workerIdentityProbes === null
207
- ? null
208
- : deps.workerIdentityProbes ?? defaultIdentityProbes(deps.runtime ?? defaultServiceRuntime(telegramStateDir));
209
222
  const plan = planHostRuntime(
210
223
  project,
211
224
  caps,
@@ -215,7 +228,6 @@ export async function runHostInstall(
215
228
  unitDir,
216
229
  deps.recoverScriptInstallPath ?? RECOVER_SCRIPT_INSTALL_PATH,
217
230
  deps.multiProject ?? hostMultiProject(),
218
- identityProbes,
219
231
  );
220
232
  // No pane-shell key is planned (unusable login shell, unparseable config):
221
233
  // the operator hears why before the consent prompt, not after an install
@@ -257,6 +269,17 @@ export async function runHostInstall(
257
269
  return { kind: "installed", wrote: [] };
258
270
  }
259
271
 
272
+ // The caller must survive the transaction it started (#834). Placed here on
273
+ // purpose: after the plan and the no-op gate — so a host with no herdr unit,
274
+ // and a re-run with nothing to install, are never refused for a restart that
275
+ // is not going to happen — and before the drain fence and the post-consent
276
+ // staging, so a refusal has mutated nothing at all.
277
+ const context = checkSessionRestartContext("setup host", plan.steps, deps.env);
278
+ if (context.kind === "refuse") {
279
+ ui.notify(context.message, "error");
280
+ return { kind: "refused", reason: context.message };
281
+ }
282
+
260
283
  // argv, not shell: `installCommands` renders `sudo …` strings for humans to
261
284
  // read, and re-parsing those into an argv is how a path with a space becomes
262
285
  // two arguments. The plan carries the exact argv once, in
@@ -298,7 +321,47 @@ export async function runHostInstall(
298
321
  attributeRestartKills(fence.scope, project?.name ?? "", deps.drain?.drainStore);
299
322
  };
300
323
 
301
- const outcome = await runPrivileged(plan.steps, ui, {
324
+ // Who is serving the fleet session, before anything is staged or run (#893).
325
+ // A `systemctl restart` that returns 0 does not prove supervision moved: an
326
+ // unmanaged `herdr --session <name> server` holding the session makes every
327
+ // start of the unit exit 1, and `Type=simple` returns before the child finds
328
+ // out. So the collision is resolved by stopping the unmanaged session inside
329
+ // this transaction, or the transaction refuses before it mutates anything.
330
+ const supervision = deps.supervision ?? defaultSupervisionDeps();
331
+ const session = plan.herdrSession ?? DEFAULT_HERDR_SESSION;
332
+ let transferSteps: PrivilegedStep[] = [];
333
+ if (plan.herdrUnit !== undefined) {
334
+ const ownership = herdrOwnership(supervision.facts(session));
335
+ if (ownership.kind === "unknown") {
336
+ // The remedy here stops somebody's live session, so an unreadable fact is
337
+ // a refusal, never an assumption. Nothing has been staged or installed.
338
+ const reason =
339
+ `refused: ${ownership.detail}, so this install cannot tell whether ${DEFAULT_HERDR_UNIT} owns the ` +
340
+ `fleet session or an unmanaged server does. Stopping the wrong one kills a live fleet, so nothing ` +
341
+ `was staged or installed. Check \`systemctl status ${DEFAULT_HERDR_UNIT}\` and ` +
342
+ `\`herdr session list\`, then re-run.`;
343
+ ui.notify(reason, "error");
344
+ return { kind: "refused", reason };
345
+ }
346
+ if (ownership.kind === "unmanaged") {
347
+ ui.notify(
348
+ `The fleet session is served outside systemd: ${ownership.detail}. This install stops that server ` +
349
+ `first so ${DEFAULT_HERDR_UNIT} can take the session over — every pane in it restarts under the unit.`,
350
+ "warning",
351
+ );
352
+ transferSteps = [
353
+ {
354
+ // The fleet account's own session, so no escalation: `setup host`
355
+ // already runs as that account (the escalation guard proves it).
356
+ title: `stop the unmanaged herdr session "${session}" so ${DEFAULT_HERDR_UNIT} can own it`,
357
+ argv: ["herdr", "session", "stop", session],
358
+ unprivileged: true,
359
+ },
360
+ ];
361
+ }
362
+ }
363
+
364
+ const outcome = await runPrivileged([...transferSteps, ...plan.steps], ui, {
302
365
  ...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
303
366
  title: "Install and start the supervised session?",
304
367
  answerKey: "install-host",
@@ -317,14 +380,22 @@ export async function runHostInstall(
317
380
  ]),
318
381
  ]),
319
382
  "The unit runs as the account that staged it; nothing here changes that.",
320
- ...(plan.workerIdentity === undefined
383
+ ...(transferSteps.length === 0
384
+ ? []
385
+ : [
386
+ `Stops the unmanaged herdr session "${session}" first — systemd cannot adopt a running process, so the ` +
387
+ "session has to be handed over. Every pane in it comes back under the unit.",
388
+ ]),
389
+ // Retirement is named before the confirm that authorises it (#895): an
390
+ // operator approving this batch is approving an unmount, and a removal
391
+ // nobody announced is not consented to.
392
+ ...(plan.retire === undefined
321
393
  ? []
322
394
  : [
323
- `Establishes the dedicated worker identity ${plan.workerIdentity.account}: every worker session runs under it, ` +
324
- "never under the daemon's account, and it is granted search/read access to exactly the fleet paths a worker needs.",
325
- ...(plan.workerIdentity.current
326
- ? ["The worker identity is already in place nothing to change."]
327
- : plan.workerIdentity.pending),
395
+ `Retires host state an earlier release installed and this one no longer ships: ` +
396
+ `${[...plan.retire.units, ...plan.retire.staged].join(", ")}.`,
397
+ "Worker sessions run as the fleet account again, so the read-only bind that served the separate " +
398
+ "worker account is unmounted and its unit removed. The account itself is left alone.",
328
399
  ]),
329
400
  "",
330
401
  liveWorkers === 0
@@ -343,6 +414,27 @@ export async function runHostInstall(
343
414
  if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
344
415
  return { kind: "failed", reason: `${outcome.step.title} exited ${outcome.exitCode}` };
345
416
  }
417
+ // Success is not "the steps exited 0" (#893). `Type=simple` returns before the
418
+ // session server can discover a collision and die, so the transaction proves
419
+ // the canonical unit is still up AND owns the session — bounded, because an
420
+ // auto-restart loop keeps looking momentarily alive.
421
+ if (plan.herdrUnit !== undefined) {
422
+ const settled = await proveSupervision(supervision, session);
423
+ if (settled.kind !== "unit") {
424
+ // Dispatch stays paused, deliberately and unlike a failed step: the fleet
425
+ // is being served by something nobody supervises, and admitting workers
426
+ // into that is the outcome this gate exists to prevent. The operator is
427
+ // told what was observed rather than a generic failure.
428
+ const reason =
429
+ `${DEFAULT_HERDR_UNIT} did not take over the fleet session: ${settled.detail}. ` +
430
+ `The units are installed, but supervision did not transfer, so dispatch stays paused. ` +
431
+ `Check \`systemctl status ${DEFAULT_HERDR_UNIT}\` and \`herdr session list\`, then re-run \`setup host\` ` +
432
+ `from a shell outside the session.`;
433
+ ui.notify(reason, "error");
434
+ return { kind: "failed", reason };
435
+ }
436
+ ui.notify(`${DEFAULT_HERDR_UNIT} owns the fleet session — ${settled.detail}.`, "info");
437
+ }
346
438
  if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
347
439
  ui.notify(`Installed and started ${STAGED_SERVICE_NAME}.`, "info");
348
440
  // #541: the tick plan may have restamped agentName away from the shared
@@ -376,6 +468,84 @@ export async function runHostInstall(
376
468
  return { kind: "installed", wrote };
377
469
  }
378
470
 
471
+ /**
472
+ * Poll until the canonical unit owns the fleet session, or the bound runs out
473
+ * (#893). Returns the last verdict either way, so the caller reports what was
474
+ * actually observed rather than "timed out".
475
+ *
476
+ * `absent` and `unmanaged` are both failures here, for different reasons: the
477
+ * first is a unit that started and died (the collision's own signature under
478
+ * `Type=simple`), the second is a session still held outside it.
479
+ */
480
+ async function proveSupervision(deps: SupervisionDeps, session: string): Promise<HerdrOwnership> {
481
+ const attempts = deps.attempts ?? SUPERVISION_PROBE_ATTEMPTS;
482
+ const intervalMs = deps.intervalMs ?? SUPERVISION_PROBE_INTERVAL_MS;
483
+ let verdict = herdrOwnership(deps.facts(session));
484
+ for (let attempt = 1; attempt < attempts && verdict.kind !== "unit"; attempt += 1) {
485
+ await deps.sleep(intervalMs);
486
+ verdict = herdrOwnership(deps.facts(session));
487
+ }
488
+ return verdict;
489
+ }
490
+
491
+ /** How many reads the readiness gate takes, and how far apart. Twelve seconds
492
+ * total: long enough for a session server to bind its socket, short enough
493
+ * that a restart loop is reported rather than waited out. */
494
+ const SUPERVISION_PROBE_ATTEMPTS = 12;
495
+ const SUPERVISION_PROBE_INTERVAL_MS = 1_000;
496
+
497
+ /** The real reads: herdr's own session list, and systemd's view of the unit. */
498
+ function defaultSupervisionDeps(): SupervisionDeps {
499
+ return {
500
+ facts: (session) => ({
501
+ sessionRunning: herdrSessionRunning(session),
502
+ unit: herdrUnitState(),
503
+ }),
504
+ sleep: Bun.sleep,
505
+ };
506
+ }
507
+
508
+ /** Whether herdr reports a live server for one session, or `undefined` when it
509
+ * could not be asked — never `false` on a failed read (#893). */
510
+ function herdrSessionRunning(session: string): boolean | undefined {
511
+ const ran = spawnSync("herdr", ["session", "list", "--json"], { encoding: "utf8", timeout: 10_000 });
512
+ if (ran.status !== 0 || typeof ran.stdout !== "string") return undefined;
513
+ try {
514
+ const parsed: unknown = JSON.parse(ran.stdout);
515
+ const sessions = parsed !== null && typeof parsed === "object" ? Reflect.get(parsed, "sessions") : undefined;
516
+ if (!Array.isArray(sessions)) return undefined;
517
+ const row = sessions.find(
518
+ (entry) => entry !== null && typeof entry === "object" && Reflect.get(entry, "name") === session,
519
+ );
520
+ if (row === undefined) return false;
521
+ return Reflect.get(row as object, "running") === true;
522
+ } catch {
523
+ return undefined;
524
+ }
525
+ }
526
+
527
+ /** The canonical unit's live state, or `undefined` when systemctl could not
528
+ * answer. A property systemd omits reads as unreadable rather than as zero. */
529
+ function herdrUnitState(): HerdrOwnershipFacts["unit"] {
530
+ const ran = spawnSync(
531
+ "systemctl",
532
+ ["show", DEFAULT_HERDR_UNIT, "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts"],
533
+ { encoding: "utf8", timeout: 10_000 },
534
+ );
535
+ if (ran.status !== 0 || typeof ran.stdout !== "string") return undefined;
536
+ const values = new Map<string, string>();
537
+ for (const line of ran.stdout.split("\n")) {
538
+ const eq = line.indexOf("=");
539
+ if (eq > 0) values.set(line.slice(0, eq), line.slice(eq + 1));
540
+ }
541
+ const activeState = values.get("ActiveState");
542
+ const subState = values.get("SubState");
543
+ const mainPid = Number(values.get("MainPID"));
544
+ const nRestarts = Number(values.get("NRestarts"));
545
+ if (activeState === undefined || subState === undefined || !Number.isFinite(mainPid)) return undefined;
546
+ return { activeState, subState, mainPid, nRestarts: Number.isFinite(nRestarts) ? nRestarts : 0 };
547
+ }
548
+
379
549
  /**
380
550
  * The pause/drain fence's default accessors, overridable per test through
381
551
  * {@link InstallDeps.drain.deps}. Mirrors `upgrade.ts`'s {@link DrainDeps}
@@ -395,7 +565,14 @@ function hostInstallDrainDeps(ui: WizardUi, deps: InstallDeps): DrainDeps {
395
565
  : { running: true, project: daemon.project, generation: `${daemon.pid}@${daemon.startedAt}` };
396
566
  },
397
567
  pauseState: (project) => pauseInstance(project),
398
- setPaused: (v, project) => setPaused(v, { source: pauseSourceToken("setup host"), reason: "setup host, draining" }, project),
568
+ setPaused: (v, project) =>
569
+ // `owner` is this process (#938): the fence lives exactly as long as this
570
+ // transaction, so a kill leaves it behind with nothing to explain it.
571
+ setPaused(
572
+ v,
573
+ { source: pauseSourceToken("setup host"), reason: "setup host, draining", owner: process.pid },
574
+ project,
575
+ ),
399
576
  sleep: Bun.sleep,
400
577
  log: (message) => ui.notify(message, "info"),
401
578
  };
@@ -28,6 +28,7 @@ import {
28
28
  import { claimedTelegramTopics } from "./escalate.ts";
29
29
  import { hostRamBytes, recommendedMaxWorkers, workerOvercommit } from "./host.ts";
30
30
  import {
31
+ admissionAckPath,
31
32
  daemonGeneration,
32
33
  isPaused,
33
34
  pausedPath,
@@ -40,8 +41,10 @@ import {
40
41
  type AdmissionAckRecord,
41
42
  type QueuePreview,
42
43
  } from "./daemon.ts";
44
+ import { pauseBytesValid } from "./pause.ts";
43
45
  import { armedMarkerPath, armTicks, fleetLayers, telegramStateDir } from "./fleet.ts";
44
46
  import {
47
+ daemonControlTarget,
45
48
  healthCheck,
46
49
  healthServesProject,
47
50
  livingDaemon,
@@ -2143,8 +2146,15 @@ interface DerivedPlan {
2143
2146
  export async function ensureSetupArm(
2144
2147
  projectName: string,
2145
2148
  arm: typeof armTicks = armTicks,
2149
+ /**
2150
+ * Where the arm proof's pending heartbeat goes while it waits (#861). Setup
2151
+ * holds dispatch under its own fence across this call, so a silent wait here
2152
+ * is the whole incident: the operator sees a stalled wizard and cannot tell
2153
+ * a live handshake from a dead process.
2154
+ */
2155
+ progress?: (line: string) => void,
2146
2156
  ): Promise<string> {
2147
- const armed = await arm(projectName);
2157
+ const armed = await arm(projectName, progress === undefined ? {} : { progress });
2148
2158
  return armed.alreadyArmed
2149
2159
  ? `existing heartbeat arm revalidated for owner ${armed.owner} at ${armed.path}`
2150
2160
  : `heartbeat armed for owner ${armed.owner} at ${armed.path}`;
@@ -2347,7 +2357,12 @@ export interface SetupApplyDeps {
2347
2357
  drain: DrainDeps;
2348
2358
  smoke: (project: string) => Promise<SetupSmokeResult>;
2349
2359
  restart: (o: { project?: string }) => Promise<RestartResult>;
2350
- arm: (project: string) => Promise<string>;
2360
+ /**
2361
+ * Prove the heartbeat channel for one project. `progress` is where the
2362
+ * pending proof's own heartbeat is written while it waits on the operator
2363
+ * (#861) — the wizard owns the surface, so the reporter comes from the call.
2364
+ */
2365
+ arm: (project: string, progress?: (line: string) => void) => Promise<string>;
2351
2366
  hostInstall: (
2352
2367
  project: ProjectConfig,
2353
2368
  caps: Caps,
@@ -2388,7 +2403,15 @@ function defaultSetupDrain(): DrainDeps {
2388
2403
  projectNames: () => loadConfig().projects.map((p) => p.name),
2389
2404
  daemonIdentity: setupDaemonIdentity,
2390
2405
  pauseState: (project) => pauseInstance(project),
2391
- setPaused: (v, project) => setPaused(v, { source: pauseSourceToken("setup"), reason: "setup apply fence" }, project),
2406
+ setPaused: (v, project) =>
2407
+ // `owner` is this process (#938): the apply fence holds dispatch across
2408
+ // this transaction only, and the 2026-08-21 incident is exactly what an
2409
+ // abandoned one looks like from outside.
2410
+ setPaused(
2411
+ v,
2412
+ { source: pauseSourceToken("setup"), reason: "setup apply fence", owner: process.pid },
2413
+ project,
2414
+ ),
2392
2415
  // The daemon-side admission acknowledgement: the file the running daemon
2393
2416
  // itself writes when it observes the fence at an admission boundary, so
2394
2417
  // the barrier never mistakes a second synchronous count for the daemon's
@@ -2458,7 +2481,7 @@ export const DEFAULT_APPLY: SetupApplyDeps = {
2458
2481
  drain: defaultSetupDrain(),
2459
2482
  smoke: runSetupSmoke,
2460
2483
  restart: (o) => restartDaemon(o),
2461
- arm: (project) => ensureSetupArm(project),
2484
+ arm: (project, progress) => ensureSetupArm(project, armTicks, progress),
2462
2485
  hostInstall: (project, caps, telegramStateDir, ui) =>
2463
2486
  runHostInstall(project, caps, telegramStateDir, ui),
2464
2487
  graphInstall: (project, ui, options) => runGraphInstall(project, ui, options),
@@ -2487,19 +2510,6 @@ type CapturedPause =
2487
2510
  | { kind: "malformed"; bytes: string; path: string }
2488
2511
  | { kind: "unreadable"; path: string };
2489
2512
 
2490
- /** Whether one sentinel's bytes parse as a valid pause instance — the exact
2491
- * grammar {@link pauseInstance} reads in `daemon.ts` (a parseable ISO
2492
- * timestamp line, then a `source=` provenance line). Readable bytes that
2493
- * fail this are an unreadable-as-state malformed hold: valid bytes are
2494
- * restored verbatim, malformed ones are refused at entry and never deleted
2495
- * (#650). */
2496
- function isValidPauseBytes(bytes: string): boolean {
2497
- const [line1, line2] = bytes.split("\n");
2498
- if (!Number.isFinite(Date.parse(line1?.trim() ?? ""))) return false;
2499
- if (line2 === undefined) return false;
2500
- return /^source=(\S+)(?: reason="(.*)")?$/.test(line2.trim());
2501
- }
2502
-
2503
2513
  /** Reads one pause sentinel, keeping absence, readable-valid bytes, malformed
2504
2514
  * bytes and unreadability apart — never conflated: collapsing malformed to
2505
2515
  * absence would let a refusal delete an operator hold it could not parse. */
@@ -2507,7 +2517,7 @@ function readSentinel(p: string): CapturedPause {
2507
2517
  if (!existsSync(p)) return { kind: "absent" };
2508
2518
  try {
2509
2519
  const bytes = readFileSync(p, "utf8");
2510
- return isValidPauseBytes(bytes) ? { kind: "bytes", bytes } : { kind: "malformed", bytes, path: p };
2520
+ return pauseBytesValid(bytes) ? { kind: "bytes", bytes } : { kind: "malformed", bytes, path: p };
2511
2521
  } catch {
2512
2522
  return { kind: "unreadable", path: p };
2513
2523
  }
@@ -2666,7 +2676,6 @@ function captureInventory(
2666
2676
  };
2667
2677
  captureIfWritten(plan.runtime.service);
2668
2678
  captureIfWritten(plan.runtime.herdrUnit);
2669
- captureIfWritten(plan.runtime.harnessMount);
2670
2679
  captureIfWritten(plan.runtime.herdrConfig);
2671
2680
  captureIfWritten(plan.runtime.herdrEnv);
2672
2681
  captureIfWritten(plan.runtime.recoverUnit);
@@ -2900,23 +2909,61 @@ const FENCE_ACK_POLL_MS = 250;
2900
2909
  * barrier froze, or `undefined` when it does: the acknowledgement must name
2901
2910
  * the exact pause instance the barrier proved (not some other pause, however
2902
2911
  * similar) and the exact daemon generation the barrier began with (not an
2903
- * older or newer instance's word) (#651 review #3).
2912
+ * older or newer instance's word) (#651 review #3). Every problem names the
2913
+ * observed and expected instances, so an operator can tell "no
2914
+ * acknowledgement yet" from "acknowledged something else" without opening a
2915
+ * debugger (#865).
2904
2916
  */
2905
2917
  function fenceAckProblem(ack: AdmissionAckRecord | undefined, begun: RestartBegun): string | undefined {
2906
- if (ack === undefined) return "the running daemon has not acknowledged the setup admission fence";
2918
+ if (ack === undefined) {
2919
+ return (
2920
+ "the running daemon has not acknowledged the setup admission fence " +
2921
+ `(fence ${describePauseInstance(begun.pauseToken)}; expected daemon generation ${begun.daemon.generation}; ` +
2922
+ `no readable acknowledgement at ${admissionAckPath()})`
2923
+ );
2924
+ }
2907
2925
  if (
2908
2926
  ack.pause.source !== begun.pauseToken.source ||
2909
2927
  ack.pause.since !== begun.pauseToken.since ||
2910
2928
  ack.pause.reason !== begun.pauseToken.reason
2911
2929
  ) {
2912
- return "the running daemon acknowledged a different pause than the setup admission fence";
2930
+ return (
2931
+ "the running daemon acknowledged a different pause than the setup admission fence " +
2932
+ `(acknowledged ${describePauseInstance(ack.pause)}; fence ${describePauseInstance(begun.pauseToken)})`
2933
+ );
2913
2934
  }
2914
2935
  if (ack.daemon !== begun.daemon.generation) {
2915
- return "the running daemon's acknowledgement belongs to a different daemon generation";
2936
+ return (
2937
+ "the running daemon's acknowledgement belongs to a different daemon generation " +
2938
+ `(acknowledged ${ack.daemon}; expected ${begun.daemon.generation})`
2939
+ );
2916
2940
  }
2917
2941
  return undefined;
2918
2942
  }
2919
2943
 
2944
+ /** One pause instance as one readable token for refusal diagnostics (#865). */
2945
+ function describePauseInstance(pause: { source: string; reason?: string; since: number }): string {
2946
+ const reason = pause.reason === undefined ? "" : ` reason="${pause.reason}"`;
2947
+ return `source=${pause.source}${reason} since=${new Date(pause.since).toISOString()}`;
2948
+ }
2949
+
2950
+ /**
2951
+ * Where a prompt-pass wake can provably land, or `undefined`: the pidfile
2952
+ * record's port while its process lives, else — a supervised daemon whose
2953
+ * runtime record went missing while the unit stayed active (#651 review #2) —
2954
+ * the run-control endpoint {@link daemonControlTarget} proves by answering
2955
+ * `/healthz`, the same consultation every run-control verb performs (#716).
2956
+ * A port is never guessed: when nothing provable answers, no wake is sent and
2957
+ * the caller's bounded wait covers the daemon's own next pass instead. The
2958
+ * pre-#865 barrier skipped BOTH the wake and the wait whenever the record was
2959
+ * unreadable, so a transient record loss refused three interviews in a row
2960
+ * against a healthy dispatcher that would have acknowledged within one tick.
2961
+ */
2962
+ async function reachableWakeEndpoint(): Promise<{ port: number } | undefined> {
2963
+ const target = await daemonControlTarget();
2964
+ return target.kind === "record" || target.kind === "unit" ? { port: target.port } : undefined;
2965
+ }
2966
+
2920
2967
  /**
2921
2968
  * Why the acknowledged barrier no longer holds, or `undefined` when it does:
2922
2969
  * a live worker anywhere in scope, the daemon-side admission acknowledgement
@@ -3391,39 +3438,41 @@ export async function setup(
3391
3438
  // boundary and written that observation down — a second synchronous worker
3392
3439
  // count is not an acknowledgement, because a tick already past its own
3393
3440
  // pause gate can claim after it and before setup mutates. A running daemon
3394
- // that has not yet acknowledged is woken to prompt an immediate pass; one
3395
- // that cannot be reached (its record is missing) or that still has not
3396
- // acknowledged by the bounded deadline refuses before anything is written.
3441
+ // that has not yet acknowledged is woken through any endpoint /healthz
3442
+ // proves, record or record-less (#865) to prompt an immediate pass; one
3443
+ // that still has not acknowledged by the bounded deadline refuses before
3444
+ // anything is written. The wait itself never depends on the pidfile: a
3445
+ // supervised daemon with a lost record still reaches its own gate within
3446
+ // one tick interval, and skipping the wait for it was the #865 refusal.
3397
3447
  let requireAck = true;
3398
3448
  if (begun.daemon.running) {
3399
3449
  let ack = apply.drain.admissionAck?.(scope.pauseKey);
3400
- let acknowledged = fenceAckProblem(ack, begun) === undefined;
3401
- if (!acknowledged) {
3402
- const reachable = livingDaemon();
3403
- if (reachable !== undefined) {
3404
- ui.notify(
3405
- "The running daemon has not yet acknowledged the setup admission fence — waking it to prompt a pass.",
3406
- "info",
3407
- );
3408
- void wakeDaemon(reachable.port);
3409
- // `OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS` is the #399 test seam: a
3410
- // regression proving this refusal burns the short test deadline
3411
- // instead of the production 60s.
3412
- const deadline =
3413
- Date.now() + Number(process.env["OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS"] ?? FENCE_ACK_WAIT_MS);
3414
- while (!acknowledged && Date.now() < deadline) {
3415
- await apply.drain.sleep(FENCE_ACK_POLL_MS);
3416
- ack = apply.drain.admissionAck?.(scope.pauseKey);
3417
- acknowledged = fenceAckProblem(ack, begun) === undefined;
3418
- }
3450
+ let problem = fenceAckProblem(ack, begun);
3451
+ if (problem !== undefined) {
3452
+ const wake = await reachableWakeEndpoint();
3453
+ ui.notify(
3454
+ wake !== undefined
3455
+ ? `The running daemon has not yet acknowledged the setup admission fence — waking it on :${wake.port} to prompt a pass.`
3456
+ : "The running daemon has not yet acknowledged the setup admission fence — nothing provable answers /healthz to wake, so waiting for its own dispatch pass.",
3457
+ "info",
3458
+ );
3459
+ if (wake !== undefined) void wakeDaemon(wake.port);
3460
+ // `OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS` is the #399 test seam: a
3461
+ // regression proving this refusal burns the short test deadline
3462
+ // instead of the production 60s.
3463
+ const deadline =
3464
+ Date.now() + Number(process.env["OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS"] ?? FENCE_ACK_WAIT_MS);
3465
+ while (problem !== undefined && Date.now() < deadline) {
3466
+ await apply.drain.sleep(FENCE_ACK_POLL_MS);
3467
+ ack = apply.drain.admissionAck?.(scope.pauseKey);
3468
+ problem = fenceAckProblem(ack, begun);
3419
3469
  }
3420
3470
  }
3421
- if (!acknowledged) {
3471
+ if (problem !== undefined) {
3422
3472
  restorePause(priorPause, ownedFence);
3423
3473
  ui.notify(
3424
- "Setup stopped before writing anything: the running daemon has not acknowledged the setup admission " +
3425
- "fence, so quiescence cannot be proven. Nothing has been changed; stop the daemon, or re-run setup " +
3426
- "once it has acknowledged the fence.",
3474
+ `Setup stopped before writing anything: ${problem} quiescence cannot be proven. ` +
3475
+ "Nothing has been changed; stop the daemon, or re-run setup once it has acknowledged the fence.",
3427
3476
  "error",
3428
3477
  );
3429
3478
  return false;
@@ -3668,7 +3717,7 @@ export async function setup(
3668
3717
  "info",
3669
3718
  );
3670
3719
  try {
3671
- armLine = await apply.arm(plan.project.name);
3720
+ armLine = await apply.arm(plan.project.name, (line) => ui.notify(line, "info"));
3672
3721
  } catch (err) {
3673
3722
  ui.notify(
3674
3723
  [
@@ -3781,6 +3830,18 @@ export async function setup(
3781
3830
  `Heartbeat: ${armLine}.`,
3782
3831
  "",
3783
3832
  "Use the documented toy-issue drill to prove one complete worker path.",
3833
+ "",
3834
+ // The one grooming recommendation setup makes, and it is only a sentence
3835
+ // (#827). Deliberately inert: nothing above or below this line installs,
3836
+ // probes, version-checks or refuses on it, because a conductor guarantee
3837
+ // must never depend on prose that releases on somebody else's cadence
3838
+ // (#506's judgement). Printed only here, on the committed success path,
3839
+ // so a failed or cancelled apply can never imply setup finished.
3840
+ "Optional: the spec-out skills pack at github.com/mattpocock/skills (`npx skills`) adds a",
3841
+ "grooming workflow this fleet has no opinion about — `to-spec` / `to-tickets` for turning a",
3842
+ "rough idea into sized slices, and `grill-me` for the questions worth asking before work",
3843
+ "starts. Nothing was installed or checked just now, and conductor is fully functional",
3844
+ "without it: skip it freely.",
3784
3845
  ].join("\n"),
3785
3846
  "info",
3786
3847
  );