omp-conductor 0.16.2 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +38 -4
  2. package/REFERENCE.md +18 -12
  3. package/package.json +2 -1
  4. package/schema/config.schema.json +16 -0
  5. package/src/admission.ts +159 -43
  6. package/src/availability.ts +27 -1
  7. package/src/briefs/worker.md +2 -0
  8. package/src/clack-ui.ts +83 -0
  9. package/src/command-manifest.ts +16 -7
  10. package/src/commands/arm.ts +11 -3
  11. package/src/commands/decision.ts +17 -7
  12. package/src/commands/doctor.ts +18 -1
  13. package/src/commands/hold.ts +9 -7
  14. package/src/commands/ledger.ts +25 -4
  15. package/src/commands/message.ts +32 -4
  16. package/src/commands/setup.ts +61 -10
  17. package/src/commands/stats.ts +9 -5
  18. package/src/commands/status.ts +32 -5
  19. package/src/commands/tail.ts +13 -1
  20. package/src/commands/watch.ts +16 -7
  21. package/src/config-schema.ts +20 -0
  22. package/src/config.ts +37 -0
  23. package/src/daemon.ts +1240 -18
  24. package/src/doctor.ts +310 -22
  25. package/src/escalate.ts +560 -57
  26. package/src/failure-class.ts +56 -13
  27. package/src/fleet.ts +224 -47
  28. package/src/gitops.ts +103 -24
  29. package/src/lifecycle.ts +7 -2
  30. package/src/orchestrator-tick.ts +372 -157
  31. package/src/privileged.ts +3 -0
  32. package/src/release-policy.ts +177 -5
  33. package/src/setup-answers.ts +135 -0
  34. package/src/setup-host.ts +193 -4
  35. package/src/setup-install.ts +2 -0
  36. package/src/setup-probe.ts +1 -0
  37. package/src/setup-wizard.ts +1296 -101
  38. package/src/setup.ts +60 -3
  39. package/src/status-render.ts +11 -1
  40. package/src/store.ts +333 -12
  41. package/src/tracker/github.ts +562 -13
  42. package/src/types.ts +204 -2
  43. package/src/ui/progress.ts +32 -0
  44. package/src/ui/style.ts +11 -0
  45. package/src/upgrade.ts +50 -19
  46. package/src/verbs/actions.ts +66 -18
  47. package/src/verbs/protocol.ts +45 -0
  48. package/src/verbs/server.ts +212 -11
  49. package/src/wizard-ui.ts +14 -5
  50. package/src/worker.ts +26 -0
  51. package/systemd/omp-conductor-recover.sh +73 -0
  52. package/systemd/recover-unit-test.sh +61 -0
package/src/privileged.ts CHANGED
@@ -88,6 +88,8 @@ export interface RunPrivilegedOptions {
88
88
  deps?: PrivilegedDeps;
89
89
  /** Confirm title. Defaults to a generic one; callers name their verb. */
90
90
  title?: string;
91
+ /** Stable answer-file key for this batch's confirmation. */
92
+ answerKey?: string;
91
93
  /** Extra lines shown above the step list — what this batch is for. */
92
94
  preamble?: readonly string[];
93
95
  /**
@@ -201,6 +203,7 @@ export async function runPrivileged(
201
203
  const go = await ui.confirm(
202
204
  options.title ?? "Run these steps now?",
203
205
  `${steps.length} step(s), in the order shown. Anything that fails stops the rest and prints what is left.`,
206
+ { key: options.answerKey ?? "run-privileged-steps" },
204
207
  );
205
208
  // `undefined` is a dismissal rather than a "no", but for a batch that has not
206
209
  // started they mean the same thing: run nothing.
@@ -280,7 +280,7 @@ function wholePackageBunTest(segment: string): boolean {
280
280
  * the host were all plain invocations, and refusing the sanctioned parse check
281
281
  * would teach workers to skip it rather than stop the load.
282
282
  */
283
- const SHARED_HOST_SCRIPTS = [
283
+ export const SHARED_HOST_SCRIPTS: readonly string[] = [
284
284
  "herdr/test/recover-test.sh",
285
285
  "test/setup-test.sh",
286
286
  "setup.sh",
@@ -298,6 +298,147 @@ function sharedHostScriptMatch(segment: string): string | undefined {
298
298
  return undefined;
299
299
  }
300
300
 
301
+ /** The `git [-C <dir>] tag` prefix; the tail after it decides read vs write. */
302
+ const GIT_TAG_HEAD = /^git(?:\s+-[Cc]\s+\S+)*\s+tag(?=\s|$)/;
303
+
304
+ /**
305
+ * Whether a segment that opens with `git [-C <dir>] tag` is a write — tag
306
+ * creation or deletion — rather than a read. The tripwire used to match the
307
+ * `git tag` prefix alone (#696), so a worker orienting itself with
308
+ * `git tag | head -5` was refused as an attempted release and the refusal
309
+ * reached the digest as release-policy drift. Git writes tags in exactly two
310
+ * shapes, and every other form is a read:
311
+ *
312
+ * - creation needs a tagname operand: `git tag <name>` and the
313
+ * `-a`/`-s`/`-u`/`-t`/`-m`/`-F`/`-f` forms;
314
+ * - deletion needs `-d`/`--delete`;
315
+ * - bare `git tag`, `-l`/`--list` (with an optional pattern), `-n`,
316
+ * `--contains`, `--points-at`, `--sort`, `--merged`/`--no-merged`,
317
+ * `--format`, `--column` and `-v`/`--verify` all only read.
318
+ *
319
+ * A creation flag with no name is still gated: git rejects the command, but a
320
+ * create attempt is not a read, and the tripwire errs toward refusing a
321
+ * release act.
322
+ */
323
+ function gitTagIsWrite(segment: string): boolean {
324
+ const head = GIT_TAG_HEAD.exec(segment);
325
+ if (head === null) return false;
326
+ const tokens = segment.slice(head[0].length).trim().split(/\s+/).filter((token) => token.length > 0);
327
+
328
+ let deletion = false; // -d / --delete
329
+ let listRead = false; // a read-only list flag
330
+ let verifyRead = false; // -v / --verify
331
+ let createSignal = false; // -a / -s / -f / -m / -u / -t / -F
332
+ let hasOperand = false; // a positional tagname, pattern or commit
333
+
334
+ for (let i = 0; i < tokens.length; i++) {
335
+ const token = tokens[i]!;
336
+ // A `--` ends option parsing: whatever follows is a plain operand, the
337
+ // created or deleted tag's name.
338
+ if (token === "--") {
339
+ hasOperand = true;
340
+ break;
341
+ }
342
+ if (token.startsWith("--")) {
343
+ const flag = token.includes("=") ? token.slice(0, token.indexOf("=")) : token;
344
+ switch (flag) {
345
+ case "--delete":
346
+ deletion = true;
347
+ break;
348
+ case "--list":
349
+ case "--contains":
350
+ case "--no-contains":
351
+ case "--points-at":
352
+ case "--merged":
353
+ case "--no-merged":
354
+ case "--sort":
355
+ case "--format":
356
+ case "--column":
357
+ case "--no-column":
358
+ case "--color":
359
+ case "--no-color":
360
+ case "--ignore-case":
361
+ case "--no-ignore-case":
362
+ listRead = true;
363
+ break;
364
+ case "--verify":
365
+ verifyRead = true;
366
+ break;
367
+ case "--annotate":
368
+ case "--sign":
369
+ case "--force":
370
+ createSignal = true;
371
+ break;
372
+ case "--local-user":
373
+ case "--object":
374
+ case "--message":
375
+ case "--file":
376
+ // The flag's operand is part of the create parameters, not a
377
+ // tagname; `--flag value` must not be read as `--flag` plus a name.
378
+ createSignal = true;
379
+ if (!token.includes("=")) i++;
380
+ break;
381
+ default:
382
+ break; // an unknown flag: no conclusion either way
383
+ }
384
+ continue;
385
+ }
386
+ if (token.startsWith("-") && token.length > 1) {
387
+ // Combined short options (`-an5` is `-a -n 5`). An option with a
388
+ // required operand takes the rest of its token or the next token.
389
+ let consumeNext = false;
390
+ for (let j = 1; j < token.length; j++) {
391
+ const flag = token[j];
392
+ switch (flag) {
393
+ case "d":
394
+ deletion = true;
395
+ break;
396
+ case "a":
397
+ case "s":
398
+ case "f":
399
+ createSignal = true;
400
+ break;
401
+ case "l":
402
+ listRead = true;
403
+ break;
404
+ case "v":
405
+ verifyRead = true;
406
+ break;
407
+ case "n":
408
+ // `-n[<num>]` — the digit run belongs to the flag, so `-n5`
409
+ // is one token and never an operand.
410
+ listRead = true;
411
+ while (j + 1 < token.length && token[j + 1]! >= "0" && token[j + 1]! <= "9") j++;
412
+ break;
413
+ case "m":
414
+ case "u":
415
+ case "t":
416
+ case "F":
417
+ createSignal = true;
418
+ if (j + 1 < token.length) {
419
+ j = token.length;
420
+ } else {
421
+ consumeNext = true;
422
+ }
423
+ break;
424
+ default:
425
+ break; // an unknown short flag: no conclusion either way
426
+ }
427
+ }
428
+ if (consumeNext) i++;
429
+ continue;
430
+ }
431
+ // A plain operand: with a list flag it is a pattern or commit for the
432
+ // read; without one it is the tagname a creation needs.
433
+ hasOperand = true;
434
+ }
435
+
436
+ if (deletion) return true;
437
+ if (listRead || verifyRead) return false;
438
+ if (hasOperand) return true;
439
+ return createSignal;
440
+ }
441
+
301
442
  /**
302
443
  * Classify a shell command and the segment that fired. The segment is what a
303
444
  * triager reads: `bun test src/foo.test.ts && npm publish` matches on its
@@ -323,7 +464,7 @@ function releaseCommandMatch(command: string): { shape: GateShape; matched: stri
323
464
  // any future spelling under the upgrade family all need the install shape,
324
465
  // and a session without that grant is refused before the shell runs.
325
466
  if (/^omp-conductor\s+upgrade(?:\b|$)/.test(segment)) return { shape: "install", matched: segment };
326
- if (/^git(?:\s+-[Cc]\s+\S+)*\s+tag(?:\s|$)/.test(segment)) return { shape: "git-tag", matched: segment };
467
+ if (gitTagIsWrite(segment)) return { shape: "git-tag", matched: segment };
327
468
  if (
328
469
  /^git(?:\s+-[Cc]\s+\S+)*\s+push\b/.test(segment) &&
329
470
  GIT_PUSH_TAG_SHAPE.test(segment)
@@ -513,8 +654,10 @@ export function releaseShapeFromTool(
513
654
  /**
514
655
  * The refusal wording for the shared-host gate (#428). The whole-package
515
656
  * `bun test` form names the focused alternative (a guard that only denies
516
- * teaches nothing and gets worked around); the shell suites get their own
517
- * because there is no focused form of a suite script.
657
+ * teaches nothing and gets worked around); a refused shell suite names the
658
+ * script that fired and the sanctioned `bash -n` parse check (#687), because
659
+ * there is no focused form of a suite script — the alternative is to drop the
660
+ * execution segment and keep the allowed prefix, not to run something else.
518
661
  */
519
662
  function sharedHostRefusalReason(matched: string | undefined): string {
520
663
  if (matched !== undefined && /^bun\s+test\b/.test(matched)) {
@@ -524,10 +667,39 @@ function sharedHostRefusalReason(matched: string | undefined): string {
524
667
  "Run a focused `bun test <file>.test.ts` instead."
525
668
  );
526
669
  }
670
+ if (matched !== undefined) {
671
+ return (
672
+ `Blocked by sharedHostPolicy: \`${matched}\` is a shared-host shell suite and not a worker's proof ` +
673
+ "path on this shared host (it overloads the 4-core VPS that also runs Langfuse and the fleet). " +
674
+ `\`bash -n ${matched}\` is the sanctioned local check — parsing never executes the script — and CI's ` +
675
+ "`herdr plugin + installer shell suites` job owns executing it. Drop the segment that fired and keep " +
676
+ "the `bash -n` parse gate."
677
+ );
678
+ }
527
679
  return (
528
680
  "Blocked by sharedHostPolicy: this shell suite is not a worker's proof path on this shared host " +
529
681
  "(it overloads the 4-core VPS that also runs Langfuse and the fleet). " +
530
- "Run focused `bun test <file>.test.ts` unit tests instead."
682
+ "`bash -n <path>` is the sanctioned local check; CI's `herdr plugin + installer shell suites` job owns " +
683
+ "executing it."
684
+ );
685
+ }
686
+
687
+ /**
688
+ * The worker-brief notice for the shared-host gate (#687): the guarded suite
689
+ * paths and the sanctioned local check, derived from
690
+ * {@link SHARED_HOST_SCRIPTS} so the rendered brief and the refusal can never
691
+ * disagree about what is guarded. One line and non-empty whenever the list is:
692
+ * the brief's rendered line count must equal its template's, which the
693
+ * buildBrief tests pin, so the notice must never carry an internal newline.
694
+ */
695
+ export function sharedHostBriefNotice(): string {
696
+ if (SHARED_HOST_SCRIPTS.length === 0) return "";
697
+ const paths = SHARED_HOST_SCRIPTS.map((script) => `\`${script}\``).join(", ");
698
+ return (
699
+ `**Shared-host guard:** ${paths} are never executed on this shared host — \`bash -n\` on them is the ` +
700
+ "sanctioned local check (parsing never executes), and CI's `herdr plugin + installer shell suites` job " +
701
+ "owns executing them. A command that executes one is refused, and the refusal names the segment; drop " +
702
+ "that segment and keep the `bash -n` parse gate."
531
703
  );
532
704
  }
533
705
 
@@ -0,0 +1,135 @@
1
+ import { readFileSync, writeFileSync } from "node:fs";
2
+ import { z } from "zod";
3
+
4
+ import type { PromptOptions, SelectOptions, TerminalUi, WizardUi } from "./wizard-ui.ts";
5
+
6
+ const answerFileSchema = z.record(z.string(), z.union([z.string(), z.boolean()]));
7
+
8
+ export type SetupAnswerValues = Record<string, string | boolean>;
9
+
10
+ function promptName(title: string, options: PromptOptions): string {
11
+ return `answer "${options.key}" for "${title}"`;
12
+ }
13
+
14
+ function requiredAnswer(answers: SetupAnswerValues, title: string, options: PromptOptions): string | boolean {
15
+ if (!Object.hasOwn(answers, options.key)) {
16
+ throw new Error(`missing ${promptName(title, options)}; add it to the --answers file`);
17
+ }
18
+ return answers[options.key]!;
19
+ }
20
+
21
+ /** A prompt-free WizardUi backed only by validated answers. */
22
+ export function answersUi(
23
+ answers: SetupAnswerValues,
24
+ notify: WizardUi["notify"] = (message) => process.stdout.write(`${message}\n`),
25
+ ): TerminalUi {
26
+ return {
27
+ notify,
28
+ close: () => {},
29
+ async confirm(title, _message, options) {
30
+ const answer = requiredAnswer(answers, title, options);
31
+ if (typeof answer !== "boolean") {
32
+ throw new Error(`${promptName(title, options)} must be a boolean`);
33
+ }
34
+ return answer;
35
+ },
36
+ async input(title, _placeholder, options) {
37
+ const answer = requiredAnswer(answers, title, options);
38
+ if (typeof answer !== "string") {
39
+ throw new Error(`${promptName(title, options)} must be a string`);
40
+ }
41
+ return answer;
42
+ },
43
+ async select(title, choices, options) {
44
+ const answer = requiredAnswer(answers, title, options);
45
+ if (typeof answer !== "string") {
46
+ throw new Error(`${promptName(title, options)} must be a listed label`);
47
+ }
48
+ if (!choices.some((choice) => choice.label === answer)) {
49
+ throw new Error(`${promptName(title, options)} must be a listed label`);
50
+ }
51
+ return answer;
52
+ },
53
+ };
54
+ }
55
+
56
+ export function loadAnswersFile(path: string): SetupAnswerValues {
57
+ let raw: string;
58
+ try {
59
+ raw = readFileSync(path, "utf8");
60
+ } catch (err) {
61
+ throw new Error(`could not read answers file "${path}": ${err instanceof Error ? err.message : String(err)}`);
62
+ }
63
+
64
+ let decoded: unknown;
65
+ try {
66
+ decoded = JSON.parse(raw);
67
+ } catch (err) {
68
+ throw new Error(`answers file "${path}" is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
69
+ }
70
+ const parsed = answerFileSchema.safeParse(decoded);
71
+ if (!parsed.success) {
72
+ throw new Error(`answers file "${path}" must be a JSON object whose values are strings or booleans`);
73
+ }
74
+ return parsed.data;
75
+ }
76
+
77
+ export interface RecordedAnswersUi {
78
+ ui: TerminalUi;
79
+ answers: SetupAnswerValues;
80
+ }
81
+
82
+ type ClosableWizardUi = WizardUi & Partial<Pick<TerminalUi, "close">>;
83
+
84
+ /** Decorates any driver and records only answers that were actually returned. */
85
+ export function recordingAnswersUi(inner: ClosableWizardUi): RecordedAnswersUi {
86
+ const answers: SetupAnswerValues = {};
87
+ return {
88
+ answers,
89
+ ui: {
90
+ notify: (message, type) => inner.notify(message, type),
91
+ close: (message) => inner.close?.(message),
92
+ async confirm(title, message, options) {
93
+ const answer = await inner.confirm(title, message, options);
94
+ if (answer !== undefined) answers[options.key] = answer;
95
+ return answer;
96
+ },
97
+ async input(title, placeholder, options) {
98
+ const answer = await inner.input(title, placeholder, options);
99
+ if (answer !== undefined) answers[options.key] = answer;
100
+ return answer;
101
+ },
102
+ async select(title, choices, options) {
103
+ const answer = await inner.select(title, choices, options);
104
+ if (answer !== undefined) answers[options.key] = answer;
105
+ return answer;
106
+ },
107
+ },
108
+ };
109
+ }
110
+
111
+ export function saveAnswersFile(path: string, answers: SetupAnswerValues): void {
112
+ writeFileSync(path, `${JSON.stringify(answers, null, 2)}\n`, "utf8");
113
+ }
114
+
115
+ /** Turns exhausted scripted stdin into an actionable non-interactive failure. */
116
+ export function guardNonInteractiveUi(inner: TerminalUi): TerminalUi {
117
+ const exhausted = (title: string, options: PromptOptions): never => {
118
+ throw new Error(
119
+ `stdin ended before ${promptName(title, options)}; provide more piped answers or use setup --answers FILE`,
120
+ );
121
+ };
122
+ return {
123
+ notify: (message, type) => inner.notify(message, type),
124
+ close: (message) => inner.close(message),
125
+ async confirm(title, message, options) {
126
+ return (await inner.confirm(title, message, options)) ?? exhausted(title, options);
127
+ },
128
+ async input(title, placeholder, options) {
129
+ return (await inner.input(title, placeholder, options)) ?? exhausted(title, options);
130
+ },
131
+ async select(title, choices, options: SelectOptions) {
132
+ return (await inner.select(title, choices, options)) ?? exhausted(title, options);
133
+ },
134
+ };
135
+ }
package/src/setup-host.ts CHANGED
@@ -15,10 +15,13 @@ import {
15
15
  DEFAULT_PORT,
16
16
  healthCheck,
17
17
  livingDaemon,
18
+ probeUnit,
18
19
  startDaemon,
19
20
  stopDaemon,
21
+ SYSTEMD_UNIT,
20
22
  type DaemonRecord,
21
23
  type StopResult,
24
+ type UnitOwnership,
22
25
  } from "./lifecycle.ts";
23
26
  import {
24
27
  legacyArmedMarkerPath,
@@ -48,7 +51,7 @@ export const SYSTEMD_UNIT_DIR = "/etc/systemd/system";
48
51
  export const RECOVER_SERVICE_NAME = "omp-conductor-recover.service";
49
52
 
50
53
  /** The playbook file shipped in the package's `systemd/` directory. */
51
- const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh";
54
+ export const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh";
52
55
 
53
56
  /**
54
57
  * Where `setup host` installs the recovery playbook so the unit's ExecStart is
@@ -1512,8 +1515,125 @@ export function writeHostRuntime(plan: HostRuntimePlan): HostRuntimeWrite {
1512
1515
  return { wrote, warnings };
1513
1516
  }
1514
1517
 
1518
+ // ---------------------------------------------------------- rollback (#652) --
1519
+ //
1520
+ // The inverse of the writes above, for the setup apply's mutation inventory:
1521
+ // every path the apply can write is captured byte-for-byte (or as absence, or
1522
+ // as unreadable — never conflated) before the first mutation, and restored on
1523
+ // any later failure. `capturePathState` is the read half, `restorePathState`
1524
+ // the write half; the transaction that owns them lives in `setup-wizard.ts`.
1525
+
1526
+ /** One path's pre-entry state, three-valued so absence is never confused with
1527
+ * unreadability:
1528
+ * - `absent` — nothing at the path; rollback removes what the apply created.
1529
+ * - `bytes` — readable on entry (a regular file with its mode, or a symlink
1530
+ * with its target); rollback writes those exact bytes/mode or re-links it.
1531
+ * - `unreadable` — something exists but cannot be read. The transaction must
1532
+ * fail closed before changing anything: collapsing this to absence and
1533
+ * later deleting the path would remove state the apply never saw. */
1534
+ export type CapturedPathState =
1535
+ | { path: string; kind: "absent" }
1536
+ | { path: string; kind: "bytes"; bytes: string; mode: number; symlink: boolean }
1537
+ | { path: string; kind: "unreadable" };
1538
+
1539
+ /** Reads {@link CapturedPathState} for one path. Symlinks are captured as
1540
+ * their target (readlink), never followed — the brief link is a symlink by
1541
+ * contract, and a dangling one is still state worth restoring. */
1542
+ export function capturePathState(path: string): CapturedPathState {
1543
+ let st: Stats;
1544
+ try {
1545
+ st = lstatSync(path);
1546
+ } catch {
1547
+ return { path, kind: "absent" };
1548
+ }
1549
+ try {
1550
+ if (st.isSymbolicLink()) {
1551
+ return { path, kind: "bytes", bytes: readlinkSync(path, "utf8"), mode: st.mode & 0o7777, symlink: true };
1552
+ }
1553
+ return { path, kind: "bytes", bytes: readFileSync(path, "utf8"), mode: st.mode & 0o7777, symlink: false };
1554
+ } catch {
1555
+ return { path, kind: "unreadable" };
1556
+ }
1557
+ }
1558
+
1559
+ /** Options for {@link restorePathState}. */
1560
+ export interface RestorePathOptions {
1561
+ /**
1562
+ * Never remove or replace a regular file: only a symlink (or absence) is
1563
+ * touched, so an operator's file that appeared mid-transaction survives a
1564
+ * rollback that cannot clobber it — the same guard the brief-link write
1565
+ * itself applies (#652). A regular file where the entry state said
1566
+ * "absent"/"symlink" therefore reports a restoration failure: the prior
1567
+ * state is not coherent and the operator must be told.
1568
+ */
1569
+ preserveRegularFile?: boolean;
1570
+ }
1571
+
1572
+ /** Restores {@link CapturedPathState} captured before the first mutation:
1573
+ * writes the entry bytes back (with their mode), re-links a symlink's entry
1574
+ * target, or removes what the apply created when nothing was there. Returns
1575
+ * `undefined` on a complete restore, else the first failure's message — a
1576
+ * rollback that could not restore a path must be reported, never implied
1577
+ * complete. */
1578
+ export function restorePathState(state: CapturedPathState, opts: RestorePathOptions = {}): string | undefined {
1579
+ const { path } = state;
1580
+ try {
1581
+ if (state.kind === "bytes" && state.symlink) {
1582
+ // Re-link the entry target. Under `preserveRegularFile` a regular file
1583
+ // that replaced the link mid-flight is left alone — and reported — so a
1584
+ // rollback never clobbers an operator's file it did not create.
1585
+ let current: Stats | undefined;
1586
+ try {
1587
+ current = lstatSync(path);
1588
+ } catch {
1589
+ current = undefined;
1590
+ }
1591
+ if (current !== undefined && !current.isSymbolicLink() && opts.preserveRegularFile === true) {
1592
+ return `could not restore the symlink at ${path}: a regular file now occupies it`;
1593
+ }
1594
+ rmSync(path, { force: true });
1595
+ mkdirSync(dirname(path), { recursive: true });
1596
+ symlinkSync(state.bytes, path);
1597
+ return undefined;
1598
+ }
1599
+ if (state.kind === "bytes") {
1600
+ mkdirSync(dirname(path), { recursive: true });
1601
+ writeFileSync(path, state.bytes);
1602
+ // Mode is part of the pre-entry state (a staged unit 0644, the tick
1603
+ // config 0600, the recovery playbook 0755): writeFileSync preserves an
1604
+ // existing file's mode, so chmod explicitly restores the captured one.
1605
+ chmodSync(path, state.mode);
1606
+ return undefined;
1607
+ }
1608
+ // Absent at entry: remove what the apply created. A regular file that
1609
+ // appeared where the entry was absent is the operator's own unless the
1610
+ // caller says otherwise.
1611
+ let current: Stats | undefined;
1612
+ try {
1613
+ current = lstatSync(path);
1614
+ } catch {
1615
+ return undefined; // already gone — a complete restore.
1616
+ }
1617
+ if (current.isSymbolicLink() || opts.preserveRegularFile !== true) {
1618
+ rmSync(path, { force: true });
1619
+ return undefined;
1620
+ }
1621
+ return `could not restore ${path}: a regular file appeared where nothing was at entry`;
1622
+ } catch (err) {
1623
+ return `could not restore ${path}: ${err instanceof Error ? err.message : String(err)}`;
1624
+ }
1625
+ }
1626
+
1515
1627
  export interface SetupSmokeResult {
1516
1628
  mode: "temporary" | "existing";
1629
+ /**
1630
+ * Whether the smoke itself proved the daemon answers `/healthz` on
1631
+ * {@link daemon}.port. A record-less active unit has no port to probe, so
1632
+ * its "existing" smoke is unproven: it cannot count as passed until the
1633
+ * caller restarts the daemon through the lifecycle seam, whose
1634
+ * `waitForOwnedDaemon` proves MainPID + `/healthz` (#651, review #2).
1635
+ */
1636
+ healthProven: boolean;
1517
1637
  status: StatusSnapshot;
1518
1638
  daemon: DaemonRecord;
1519
1639
  }
@@ -1522,6 +1642,15 @@ export interface SetupSmokeDeps {
1522
1642
  paused(project: string): boolean;
1523
1643
  runOnce(project: string): Promise<void>;
1524
1644
  living(): DaemonRecord | undefined;
1645
+ /**
1646
+ * systemd ownership of the `omp-conductor.service` supervisor unit, probed
1647
+ * independently of the pidfile/runtime record. An active unit is a live
1648
+ * supervised daemon even when its record is missing or unreadable (#618):
1649
+ * it must never be run beside a `--once` tick, and `start()` must never be
1650
+ * aimed at it. `unknown` is preserved — never collapsed into `inactive` —
1651
+ * so a possibly-live supervised daemon is not run beside (#651, review #2).
1652
+ */
1653
+ unitOwnership(): UnitOwnership;
1525
1654
  health(port: number): Promise<{ ok: boolean; body?: string }>;
1526
1655
  start(project: string): Promise<DaemonRecord>;
1527
1656
  stop(): Promise<StopResult>;
@@ -1532,30 +1661,90 @@ const DEFAULT_SMOKE_DEPS: SetupSmokeDeps = {
1532
1661
  paused: isPaused,
1533
1662
  runOnce: async (project) => await runDaemon({ once: true, project }),
1534
1663
  living: livingDaemon,
1664
+ unitOwnership: () => probeUnit(SYSTEMD_UNIT),
1535
1665
  health: healthCheck,
1536
1666
  start: async (project) => await startDaemon({ project }),
1537
1667
  stop: stopDaemon,
1538
1668
  status: statusSnapshot,
1539
1669
  };
1540
1670
 
1671
+ /** The daemon record a smoke attributes to a live but record-less supervisor.
1672
+ * The port is unknown (the record would have named it); the "existing" mode
1673
+ * caller restarts the daemon through the lifecycle seam and recomputes the
1674
+ * real /healthz line from that restart, so this satisfies the result shape
1675
+ * without pretending to know a port. `healthProven: false` marks that this
1676
+ * is a placeholder until that restart proves the unit's MainPID answers. */
1677
+ function unrecordedDaemon(mainPid: number): DaemonRecord {
1678
+ return {
1679
+ pid: 0,
1680
+ port: 0,
1681
+ startedAt: 0,
1682
+ logFile: `<record missing — supervised daemon MainPID ${mainPid} via systemd>`,
1683
+ };
1684
+ }
1685
+
1541
1686
  export async function runSetupSmoke(
1542
1687
  project: string,
1543
1688
  deps: SetupSmokeDeps = DEFAULT_SMOKE_DEPS,
1544
1689
  ): Promise<SetupSmokeResult> {
1545
1690
  if (!deps.paused(project)) throw new Error("setup smoke requires paused dispatch");
1546
- await deps.runOnce(project);
1691
+ // Detect a live daemon BEFORE the mutating `--once` tick (#618). A paused
1692
+ // `--once` run is itself a dispatcher: it settles rows, projects labels,
1693
+ // admits and salvages workers. Running one beside a live daemon orphans and
1694
+ // salvages the runs the live process owns — exactly what the incident did to
1695
+ // three workers before failing its restart check. An already-active daemon
1696
+ // needs only its health proving (the caller restarts it through the lifecycle
1697
+ // seam so the new config takes effect) — no `--once` process ever runs beside
1698
+ // it, and no `systemctl start` is aimed at a unit that is already active.
1547
1699
  const existing = deps.living();
1548
1700
  if (existing !== undefined) {
1549
1701
  const health = await deps.health(existing.port);
1550
1702
  if (!health.ok) throw new Error(`existing daemon on :${existing.port} did not answer /healthz`);
1551
- return { mode: "existing", status: deps.status(project), daemon: existing };
1703
+ return { mode: "existing", healthProven: true, status: deps.status(project), daemon: existing };
1552
1704
  }
1553
1705
 
1706
+ // The pidfile record is absent, but that is not "no daemon": an active
1707
+ // systemd-owned unit is a live supervised dispatcher even when its record
1708
+ // went missing (a crash raced a re-record, or an unmanaged start never wrote
1709
+ // one). Without this independent probe the smoke runs a second `--once`
1710
+ // beside it, and `start()` targets the already-active unit — two dispatchers
1711
+ // owning the same state, the #618 incident shape. Treat it as an existing
1712
+ // daemon: no competing tick, no activation; the caller restarts the unit so
1713
+ // the new config takes effect (#651, review #1).
1714
+ const ownership = deps.unitOwnership();
1715
+ if (ownership.kind === "unknown") {
1716
+ // A probe failure is not the confirmed negative `inactive` claims to be:
1717
+ // collapsing `unknown` here is how a unit-owned daemon gets a second one
1718
+ // started beside it (#651, review #2). The mutating `--once` tick refuses
1719
+ // instead.
1720
+ throw new Error(
1721
+ `cannot prove whether a daemon is running: systemd ownership probe failed (${ownership.reason}) — ` +
1722
+ "refusing to run the setup smoke beside a possibly-live daemon",
1723
+ );
1724
+ }
1725
+ if (ownership.kind === "active") {
1726
+ // No record to read a port from, so no /healthz proof is possible here:
1727
+ // the smoke reports `existing` honestly as *unproven*, and the caller
1728
+ // must restart the unit through the lifecycle seam — whose
1729
+ // `waitForOwnedDaemon` proves MainPID + `/healthz` — before the smoke
1730
+ // counts as passed (#651, review #2).
1731
+ return {
1732
+ mode: "existing",
1733
+ healthProven: false,
1734
+ status: deps.status(project),
1735
+ daemon: unrecordedDaemon(ownership.pid),
1736
+ };
1737
+ }
1738
+
1739
+ // Confirmed inactive (or failed, which owns no process): a paused `--once`
1740
+ // smoke can run by itself, then a temporary daemon proves the new runtime,
1741
+ // then it is stopped again.
1742
+ await deps.runOnce(project);
1554
1743
  const daemon = await deps.start(project);
1555
1744
  try {
1556
1745
  const health = await deps.health(daemon.port);
1557
1746
  if (!health.ok) throw new Error(`temporary daemon on :${daemon.port} did not answer /healthz`);
1558
- return { mode: "temporary", status: deps.status(project), daemon };
1747
+ return { mode: "temporary", healthProven: true, status: deps.status(project), daemon };
1559
1748
  } finally {
1560
1749
  await deps.stop();
1561
1750
  }
@@ -275,6 +275,7 @@ export async function runHostInstall(
275
275
  const outcome = await runPrivileged(plan.steps, ui, {
276
276
  ...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
277
277
  title: "Install and start the supervised session?",
278
+ answerKey: "install-host",
278
279
  preamble: [
279
280
  `Installs ${plan.service.path} as ${installed}, then enables and restarts it.`,
280
281
  ...(plan.herdrUnit === undefined
@@ -547,6 +548,7 @@ export async function runGraphInstall(
547
548
  const outcome = await runPrivileged([...clones, ...install], ui, {
548
549
  ...(options.privileged === undefined ? {} : { deps: options.privileged }),
549
550
  title: unitsCurrent ? "Clone the code-graph checkouts and seed them?" : "Clone, install and enable the code-graph timer?",
551
+ answerKey: `install-code-graph.${project.name}`,
550
552
  preamble: [
551
553
  `Staged: ${staged.written.join(", ")}.`,
552
554
  ...(clones.length === 0
@@ -402,6 +402,7 @@ export async function probeProse(
402
402
  const keep = await ui.confirm(
403
403
  `Keep this ${heading}?`,
404
404
  "It becomes part of POLICY.md, which the orchestrator re-reads on every tick. You can edit that file afterwards. Declining keeps the stub.",
405
+ { key: `keep-probe.${name}` },
405
406
  );
406
407
  // A dismissed prompt is not a yes. `undefined` means the operator walked away.
407
408
  if (keep !== true) {