omp-conductor 0.16.2 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +38 -4
  2. package/REFERENCE.md +18 -12
  3. package/package.json +2 -1
  4. package/schema/config.schema.json +16 -0
  5. package/src/admission.ts +159 -43
  6. package/src/availability.ts +27 -1
  7. package/src/briefs/worker.md +2 -0
  8. package/src/clack-ui.ts +83 -0
  9. package/src/command-manifest.ts +16 -7
  10. package/src/commands/arm.ts +11 -3
  11. package/src/commands/decision.ts +17 -7
  12. package/src/commands/doctor.ts +18 -1
  13. package/src/commands/hold.ts +9 -7
  14. package/src/commands/ledger.ts +25 -4
  15. package/src/commands/message.ts +32 -4
  16. package/src/commands/setup.ts +61 -10
  17. package/src/commands/stats.ts +9 -5
  18. package/src/commands/status.ts +32 -5
  19. package/src/commands/tail.ts +13 -1
  20. package/src/commands/watch.ts +16 -7
  21. package/src/config-schema.ts +20 -0
  22. package/src/config.ts +37 -0
  23. package/src/daemon.ts +1240 -18
  24. package/src/doctor.ts +310 -22
  25. package/src/escalate.ts +560 -57
  26. package/src/failure-class.ts +56 -13
  27. package/src/fleet.ts +224 -47
  28. package/src/gitops.ts +103 -24
  29. package/src/lifecycle.ts +7 -2
  30. package/src/orchestrator-tick.ts +372 -157
  31. package/src/privileged.ts +3 -0
  32. package/src/release-policy.ts +177 -5
  33. package/src/setup-answers.ts +135 -0
  34. package/src/setup-host.ts +193 -4
  35. package/src/setup-install.ts +2 -0
  36. package/src/setup-probe.ts +1 -0
  37. package/src/setup-wizard.ts +1296 -101
  38. package/src/setup.ts +60 -3
  39. package/src/status-render.ts +11 -1
  40. package/src/store.ts +333 -12
  41. package/src/tracker/github.ts +562 -13
  42. package/src/types.ts +204 -2
  43. package/src/ui/progress.ts +32 -0
  44. package/src/ui/style.ts +11 -0
  45. package/src/upgrade.ts +50 -19
  46. package/src/verbs/actions.ts +66 -18
  47. package/src/verbs/protocol.ts +45 -0
  48. package/src/verbs/server.ts +212 -11
  49. package/src/wizard-ui.ts +14 -5
  50. package/src/worker.ts +26 -0
  51. package/systemd/omp-conductor-recover.sh +73 -0
  52. package/systemd/recover-unit-test.sh +61 -0
@@ -10,37 +10,70 @@
10
10
  * prompt's shape and default is a decision worth keeping, so none of them were
11
11
  * reworded in the move.
12
12
  */
13
- import { existsSync, readFileSync } from "node:fs";
13
+ import { randomUUID } from "node:crypto";
14
+ import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmdirSync, rmSync, writeFileSync } from "node:fs";
14
15
  import { platform } from "node:os";
15
16
  import { dirname, isAbsolute, join } from "node:path";
16
17
  import {
18
+ configBackupDir,
17
19
  configPath,
18
20
  expandHome,
19
21
  findProject,
20
22
  loadConfig,
21
23
  resolveCaps,
22
24
  saveConfig,
25
+ stateDir,
26
+ writeConfigRaw,
23
27
  } from "./config.ts";
24
28
  import { claimedTelegramTopics } from "./escalate.ts";
25
29
  import { hostRamBytes, recommendedMaxWorkers, workerOvercommit } from "./host.ts";
26
30
  import {
27
- prepareConductor,
31
+ daemonGeneration,
32
+ isPaused,
33
+ pausedPath,
34
+ pauseInstance,
35
+ pauseSourceToken,
28
36
  previewProject,
37
+ readAdmissionAck,
29
38
  setPaused,
39
+ wakeDaemon,
40
+ type AdmissionAckRecord,
30
41
  type QueuePreview,
31
42
  } from "./daemon.ts";
32
- import { armTicks, telegramStateDir } from "./fleet.ts";
33
- import { restartDaemon, type RestartResult } from "./lifecycle.ts";
43
+ import { armedMarkerPath, armTicks, fleetLayers, telegramStateDir } from "./fleet.ts";
44
+ import {
45
+ healthCheck,
46
+ healthServesProject,
47
+ livingDaemon,
48
+ probeUnit,
49
+ recordPath,
50
+ restartDaemon,
51
+ SYSTEMD_UNIT,
52
+ type RestartResult,
53
+ } from "./lifecycle.ts";
54
+ import { dbPath, liveWorkersReadOnly, openStore, vacuumInto } from "./store.ts";
55
+ import {
56
+ restartFenceProblem,
57
+ type DaemonIdentity,
58
+ type DrainDeps,
59
+ type RestartBegun,
60
+ type UpgradeScope,
61
+ } from "./upgrade.ts";
34
62
  import { defaultGraphRoot, graphRepos } from "./graph.ts";
35
63
  import {
64
+ capturePathState,
36
65
  formatHostRuntimePlan,
37
66
  planHostRuntime,
38
67
  totalConfiguredWorkers,
68
+ restorePathState,
39
69
  runSetupSmoke,
40
70
  SYSTEMD_UNIT_DIR,
41
71
  tickCwdForProject,
42
72
  writeHostRuntime,
73
+ type CapturedPathState,
43
74
  type HostRuntimePlan,
75
+ type HostRuntimeWrite,
76
+ type RestorePathOptions,
44
77
  type SetupSmokeResult,
45
78
  } from "./setup-host.ts";
46
79
  import { runGraphInstall, runHostInstall, type GraphInstallOptions, type InstallOutcome } from "./setup-install.ts";
@@ -52,6 +85,7 @@ import {
52
85
  } from "./setup-discover.ts";
53
86
  import {
54
87
  AMEND_AREAS,
88
+ ARM_PROOF_CHOICES,
55
89
  BASE_FRESHNESS_CHOICES,
56
90
  BEHIND_BASE_CHOICES,
57
91
  DRAFT_POLICY_CHOICES,
@@ -68,6 +102,7 @@ import {
68
102
  checkTokenScopes,
69
103
  createMissingLabels,
70
104
  defaultAnswers,
105
+ deleteCreatedLabels,
71
106
  detectTelegram,
72
107
  formatGates,
73
108
  orchestratorBriefPath,
@@ -86,6 +121,7 @@ import {
86
121
  type ProbedProse,
87
122
  } from "./setup.ts";
88
123
  import {
124
+ ARM_PROOFS,
89
125
  BASE_FRESHNESS,
90
126
  BEHIND_BASE_ACTIONS,
91
127
  DEFAULT_CAPS,
@@ -107,6 +143,7 @@ import {
107
143
  type ReportScopeChoice,
108
144
  type ResolvedGrants,
109
145
  } from "./types.ts";
146
+ import { withProgress } from "./ui/progress.ts";
110
147
  import type { WizardUi } from "./wizard-ui.ts";
111
148
 
112
149
  /**
@@ -132,8 +169,8 @@ const REPO_RE = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/;
132
169
  * takes it, because the harness has no pre-filled input dialog — so "Enter
133
170
  * accepts what you see" is the contract the whole wizard is built on.
134
171
  */
135
- async function ask(ui: WizardUi, title: string, fallback: string): Promise<string> {
136
- const raw = await ui.input(title, fallback.length > 0 ? fallback : undefined);
172
+ async function ask(ui: WizardUi, key: string, title: string, fallback: string): Promise<string> {
173
+ const raw = await ui.input(title, fallback.length > 0 ? fallback : undefined, { key });
137
174
  if (raw === undefined) throw new Cancelled();
138
175
  const trimmed = raw.trim();
139
176
  return trimmed.length > 0 ? trimmed : fallback;
@@ -147,8 +184,8 @@ async function ask(ui: WizardUi, title: string, fallback: string): Promise<strin
147
184
  * recorded a silent no and walked on to the next question, which is not what
148
185
  * abandoning a run means. `undefined` is the surface saying the operator left.
149
186
  */
150
- async function askYesNo(ui: WizardUi, title: string, message: string): Promise<boolean> {
151
- const answer = await ui.confirm(title, message);
187
+ async function askYesNo(ui: WizardUi, key: string, title: string, message: string): Promise<boolean> {
188
+ const answer = await ui.confirm(title, message, { key });
152
189
  if (answer === undefined) throw new Cancelled();
153
190
  return answer;
154
191
  }
@@ -160,12 +197,13 @@ async function askYesNo(ui: WizardUi, title: string, message: string): Promise<b
160
197
  */
161
198
  async function askValid(
162
199
  ui: WizardUi,
200
+ key: string,
163
201
  title: string,
164
202
  fallback: string,
165
203
  check: (value: string) => string | undefined,
166
204
  ): Promise<string> {
167
205
  for (let attempt = 0; attempt < 3; attempt++) {
168
- const value = await ask(ui, title, fallback);
206
+ const value = await ask(ui, key, title, fallback);
169
207
  const problem = check(value);
170
208
  if (problem === undefined) return value;
171
209
  ui.notify(problem, "warning");
@@ -176,8 +214,8 @@ async function askValid(
176
214
 
177
215
  /** A cap. Unparseable input keeps the current value rather than writing a NaN
178
216
  * the validator would later reject — the operator sees why, immediately. */
179
- async function askNumber(ui: WizardUi, title: string, fallback: number): Promise<number> {
180
- const raw = await ask(ui, title, String(fallback));
217
+ async function askNumber(ui: WizardUi, key: string, title: string, fallback: number): Promise<number> {
218
+ const raw = await ask(ui, key, title, String(fallback));
181
219
  const value = Number(raw);
182
220
  if (!Number.isFinite(value) || value < 0) {
183
221
  ui.notify(`"${raw}" is not a non-negative number — keeping ${fallback}.`, "warning");
@@ -193,11 +231,12 @@ async function askNumber(ui: WizardUi, title: string, fallback: number): Promise
193
231
  */
194
232
  async function askSpendCap(
195
233
  ui: WizardUi,
234
+ key: string,
196
235
  title: string,
197
236
  fallback: number | null,
198
237
  ): Promise<number | null> {
199
238
  const seed = fallback === null ? "" : String(fallback);
200
- const raw = (await ask(ui, title, seed)).trim().toLowerCase();
239
+ const raw = (await ask(ui, key, title, seed)).trim().toLowerCase();
201
240
  if (raw === "" || raw === "none" || raw === "off" || raw === "null") return null;
202
241
  const value = Number(raw);
203
242
  if (!Number.isFinite(value) || value < 0) {
@@ -232,6 +271,7 @@ async function askGates(
232
271
  // found, so it is on screen either way.
233
272
  const raw = await ask(
234
273
  ui,
274
+ `pre-push-gates.${repoName}`,
235
275
  `Pre-push gates for ${repoName} — exactly what CI runs, comma separated`,
236
276
  formatGates(seed.length > 0 ? seed : probed),
237
277
  );
@@ -263,6 +303,7 @@ async function askReportScope(ui: WizardUi, current: ReportScopeChoice): Promise
263
303
  const options = REPORT_SCOPE_CHOICES.map((c) => ({ label: c.label, description: c.description }));
264
304
  const at = REPORT_SCOPE_CHOICES.findIndex((c) => c.scope === current);
265
305
  const picked = await ui.select("What should the orchestrator report unprompted?", options, {
306
+ key: "report-scope",
266
307
  initialIndex: at === -1 ? 0 : at,
267
308
  });
268
309
  if (picked === undefined) throw new Cancelled();
@@ -292,6 +333,7 @@ async function askReportScope(ui: WizardUi, current: ReportScopeChoice): Promise
292
333
  */
293
334
  async function askLiteral<T extends string>(
294
335
  ui: WizardUi,
336
+ key: string,
295
337
  title: string,
296
338
  values: readonly T[],
297
339
  described: { readonly [K in T]: string },
@@ -301,7 +343,7 @@ async function askLiteral<T extends string>(
301
343
  const picked = await ui.select(
302
344
  title,
303
345
  values.map((v) => ({ label: v, description: described[v] })),
304
- { initialIndex: at === -1 ? 0 : at },
346
+ { key, initialIndex: at === -1 ? 0 : at },
305
347
  );
306
348
  if (picked === undefined) throw new Cancelled();
307
349
 
@@ -336,8 +378,13 @@ function parseNameList(answer: string): string[] {
336
378
 
337
379
  /** Check names, artefacts, environments: open-ended lists this package cannot
338
380
  * enumerate, so the only validation is the shape. */
339
- async function askNameList(ui: WizardUi, title: string, seed: readonly string[]): Promise<string[]> {
340
- return parseNameList(await ask(ui, title, formatNameList(seed)));
381
+ async function askNameList(
382
+ ui: WizardUi,
383
+ key: string,
384
+ title: string,
385
+ seed: readonly string[],
386
+ ): Promise<string[]> {
387
+ return parseNameList(await ask(ui, key, title, formatNameList(seed)));
341
388
  }
342
389
 
343
390
  /**
@@ -362,6 +409,7 @@ async function askReleaseRequirements(
362
409
  );
363
410
  const answered = await askValid(
364
411
  ui,
412
+ "release-requirements",
365
413
  `Release — what must have landed first (any of ${accepted}, comma separated, or "${EMPTY_LIST}")`,
366
414
  formatNameList(prior),
367
415
  (value) => {
@@ -398,11 +446,13 @@ async function askPolicyPreconditions(
398
446
  const merge = {
399
447
  requiredChecks: await askNameList(
400
448
  ui,
449
+ "merge-required-checks",
401
450
  `Merge — required checks (comma separated, "${EMPTY_LIST}" = every check the PR reports)`,
402
451
  prior.merge.requiredChecks,
403
452
  ),
404
453
  baseFreshness: await askLiteral(
405
454
  ui,
455
+ "merge-base-freshness",
406
456
  "Merge — must the PR be level with its base?",
407
457
  BASE_FRESHNESS,
408
458
  BASE_FRESHNESS_CHOICES,
@@ -410,6 +460,7 @@ async function askPolicyPreconditions(
410
460
  ),
411
461
  drafts: await askLiteral(
412
462
  ui,
463
+ "merge-drafts",
413
464
  "Merge — draft pull requests",
414
465
  DRAFT_POLICIES,
415
466
  DRAFT_POLICY_CHOICES,
@@ -417,6 +468,7 @@ async function askPolicyPreconditions(
417
468
  ),
418
469
  whenBehindBase: await askLiteral(
419
470
  ui,
471
+ "merge-behind-base",
420
472
  "Merge — a green PR that fell behind its base",
421
473
  BEHIND_BASE_ACTIONS,
422
474
  BEHIND_BASE_CHOICES,
@@ -430,16 +482,19 @@ async function askPolicyPreconditions(
430
482
  requires: await askReleaseRequirements(ui, prior.release.requires),
431
483
  requiredChecks: await askNameList(
432
484
  ui,
485
+ "release-required-checks",
433
486
  `Release — required checks (comma separated, "${EMPTY_LIST}" = every check the branch reports)`,
434
487
  prior.release.requiredChecks,
435
488
  ),
436
489
  artefacts: await askNameList(
437
490
  ui,
491
+ "release-artefacts",
438
492
  `Release — artefacts this project ships (comma separated, or "${EMPTY_LIST}")`,
439
493
  prior.release.artefacts,
440
494
  ),
441
495
  environments: await askNameList(
442
496
  ui,
497
+ "release-environments",
443
498
  `Release — environments a deploy may target (comma separated, or "${EMPTY_LIST}")`,
444
499
  prior.release.environments,
445
500
  ),
@@ -487,6 +542,7 @@ async function askAuthority(
487
542
  ): Promise<ProjectConfig["authority"]> {
488
543
  const merge = await askYesNo(
489
544
  ui,
545
+ "merge-authority",
490
546
  "Merge authority",
491
547
  "Delegate PR merging to the orchestrator session? It would land green PRs one at a time, each " +
492
548
  "re-checked against the base branch first. Default: humans merge" +
@@ -496,6 +552,7 @@ async function askAuthority(
496
552
  const unlockCount = unlock.gates.length + unlock.policy.length;
497
553
  const release = await askYesNo(
498
554
  ui,
555
+ "release-authority",
499
556
  "Release authority",
500
557
  "Delegate release cutting to the orchestrator session? It would tag, pin and publish by the " +
501
558
  "procedure you write into its brief — and its brief forbids cutting one before you have. " +
@@ -506,6 +563,7 @@ async function askAuthority(
506
563
  );
507
564
  const promotion = await askYesNo(
508
565
  ui,
566
+ "promotion-authority",
509
567
  "Promotion authority",
510
568
  "Delegate promotion to the orchestrator session? Promotion is adding the queue label to an " +
511
569
  "issue — the sign-off that lets a worker claim it, and the act that starts spend. " +
@@ -576,13 +634,14 @@ async function askJudgment(
576
634
  const picked = await ui.select(
577
635
  "Roadmap candidates found in GitHub",
578
636
  [...roadmapChoices.map((label) => ({ label })), { label: other }],
579
- { initialIndex: 0 },
637
+ { key: "roadmap-candidate", initialIndex: 0 },
580
638
  );
581
639
  if (picked === undefined) throw new Cancelled();
582
640
  roadmapSeed = picked === other ? "" : picked;
583
641
  }
584
642
  const roadmap = await ask(
585
643
  ui,
644
+ "roadmap",
586
645
  "Where does the roadmap live, and what is the current priority?",
587
646
  roadmapSeed,
588
647
  );
@@ -622,26 +681,40 @@ async function askJudgment(
622
681
  // so an operator has something to calibrate against.
623
682
  judgment.boundary = await ask(
624
683
  ui,
684
+ "release-boundary",
625
685
  "Where does the orchestrator's leg END? One sentence " +
626
686
  '(a real answer: "at the merged version pin — deploying it is operator territory")',
627
687
  prior.boundary ?? "",
628
688
  );
629
689
 
630
690
  // All five, because the brief needs each and a missing one is a hole.
631
- judgment.releaseWhat = await ask(ui, "Release — WHAT may be released, and from which branch?", prior.releaseWhat ?? "");
691
+ judgment.releaseWhat = await ask(
692
+ ui,
693
+ "release-what",
694
+ "Release — WHAT may be released, and from which branch?",
695
+ prior.releaseWhat ?? "",
696
+ );
632
697
  judgment.releaseWhen = await ask(
633
698
  ui,
699
+ "release-when",
634
700
  "Release — WHEN: batched how, after which named checks are green?",
635
701
  prior.releaseWhen ?? "",
636
702
  );
637
703
  judgment.releaseProof = await ask(
638
704
  ui,
705
+ "release-proof",
639
706
  "Release — WHAT PROOF must be held first (results actually read, not an impression)?",
640
707
  prior.releaseProof ?? "",
641
708
  );
642
- judgment.releaseAsk = await ask(ui, "Release — what must still be ASKED, every time?", prior.releaseAsk ?? "");
709
+ judgment.releaseAsk = await ask(
710
+ ui,
711
+ "release-ask",
712
+ "Release — what must still be ASKED, every time?",
713
+ prior.releaseAsk ?? "",
714
+ );
643
715
  judgment.releaseForbidden = await ask(
644
716
  ui,
717
+ "release-forbidden",
645
718
  "Release — what stays permanently FORBIDDEN?",
646
719
  prior.releaseForbidden ?? "force-push, secrets, production data",
647
720
  );
@@ -652,17 +725,19 @@ async function askJudgment(
652
725
  // `RELEASE_REQUIREMENTS`, which are mechanical preconditions, not a unit.
653
726
  judgment.worthCutting = await ask(
654
727
  ui,
728
+ "release-worth-cutting",
655
729
  "What is a release worth cutting? (a sprint, an epic's children all closed, N merged issues, urgency)",
656
730
  prior.worthCutting ?? "",
657
731
  );
658
732
 
659
- judgment.rollbackOwner = await ask(ui, "Who owns the rollback?", prior.rollbackOwner ?? "");
733
+ judgment.rollbackOwner = await ask(ui, "rollback-owner", "Who owns the rollback?", prior.rollbackOwner ?? "");
660
734
  if (namesAPerson(judgment.rollbackOwner)) {
661
735
  // Honour the consequence rather than recording a contradiction: if a person
662
736
  // rolls it back, that person owns the release, and the boundary belongs
663
737
  // before the irreversible step whatever the authority answer sounded like.
664
738
  judgment.rollbackMovesBoundary = await askYesNo(
665
739
  ui,
740
+ "rollback-moves-boundary",
666
741
  "Rollback owner is a person",
667
742
  `You named "${judgment.rollbackOwner}" as the rollback owner. That person already owns the release, ` +
668
743
  "so the honest configuration puts the orchestrator's boundary *before* the irreversible step — " +
@@ -671,6 +746,7 @@ async function askJudgment(
671
746
  if (judgment.rollbackMovesBoundary) {
672
747
  judgment.boundary = await ask(
673
748
  ui,
749
+ "release-boundary",
674
750
  "Restate the boundary, ending before the irreversible step",
675
751
  judgment.boundary ?? "",
676
752
  );
@@ -713,6 +789,7 @@ async function askReleaseGrants(
713
789
  for (const shape of RELEASE_SHAPES) {
714
790
  const open = await askYesNo(
715
791
  ui,
792
+ `release-tool-gate.${shape}`,
716
793
  `Release tool gate — ${shape}`,
717
794
  `Allow the orchestrator session to ${RELEASE_SHAPE_QUESTIONS[shape]}? Grant this only when the ` +
718
795
  "operator brief carries the procedure it must follow. A worker session is refused this " +
@@ -733,6 +810,7 @@ async function askReleaseGrants(
733
810
  async function askOrchestratorMode(ui: WizardUi, prior: OrchestratorMode): Promise<OrchestratorMode> {
734
811
  const external = await askYesNo(
735
812
  ui,
813
+ "orchestrator-mode",
736
814
  "Orchestrator session",
737
815
  "Do you already run your own orchestrator session for this project — a visible TUI session, say? " +
738
816
  "Then the daemon starts none of its own, and posts tier-1 escalations as issue comments for yours " +
@@ -762,6 +840,7 @@ async function askGraphRoot(
762
840
  ): Promise<string | undefined> {
763
841
  const wanted = await askYesNo(
764
842
  ui,
843
+ "code-graph-enabled",
765
844
  "Code-graph discovery",
766
845
  "Set up code-graph discovery for workers? Workers spend most of their turn budget finding code; " +
767
846
  'a graph answers "who calls this" in one call. Conductor keeps one disposable clone per repo, ' +
@@ -772,6 +851,7 @@ async function askGraphRoot(
772
851
 
773
852
  return await askValid(
774
853
  ui,
854
+ "code-graph-root",
775
855
  `Root for those clones — one per repo (${repoNames.join(", ")}) is created under it`,
776
856
  prior ?? defaultGraphRoot(trackerRepo),
777
857
  (v) =>
@@ -791,6 +871,7 @@ async function askOrchestratorBrief(ui: WizardUi, a: SetupAnswers): Promise<bool
791
871
  const path = orchestratorBriefPath(a);
792
872
  const wanted = await askYesNo(
793
873
  ui,
874
+ "write-operator-brief",
794
875
  `Write ${ORCHESTRATOR_BRIEF_NAME} + ${POLICY_BRIEF_NAME} under ${dirname(path)}?`,
795
876
  `Writes composed ${ORCHESTRATOR_BRIEF_NAME} (package floor, refreshed each tick) and ${POLICY_BRIEF_NAME} ` +
796
877
  `(Releases, Project context, Reporting, Amendments — yours to edit via the Learning loop). ` +
@@ -801,6 +882,7 @@ async function askOrchestratorBrief(ui: WizardUi, a: SetupAnswers): Promise<bool
801
882
 
802
883
  return await askYesNo(
803
884
  ui,
885
+ "overwrite-operator-brief",
804
886
  `Overwrite existing ${ORCHESTRATOR_BRIEF_NAME} / ${POLICY_BRIEF_NAME}?`,
805
887
  `${path} already exists. Overwriting replaces the composed brief and POLICY.md scaffold — any policy you wrote is lost.`,
806
888
  );
@@ -990,6 +1072,7 @@ async function proseFromRepos(ui: WizardUi, a: SetupAnswers): Promise<ProbedPros
990
1072
  const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
991
1073
  const trackerRepo = await askValid(
992
1074
  ui,
1075
+ "tracker-repo",
993
1076
  "Tracker repo (owner/repo) — where ready issues live",
994
1077
  a.trackerRepo,
995
1078
  (v) => (REPO_RE.test(v) ? undefined : `"${v}" is not owner/repo — e.g. acme/planning.`),
@@ -997,6 +1080,7 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
997
1080
 
998
1081
  const queueLabel = await ask(
999
1082
  ui,
1083
+ "queue-label",
1000
1084
  "Queue label — the human sign-off that makes an issue claimable",
1001
1085
  a.queueLabel,
1002
1086
  );
@@ -1007,18 +1091,20 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
1007
1091
  const stateLabels: SetupAnswers["stateLabels"] = { ...a.stateLabels };
1008
1092
  const customiseStates = await askYesNo(
1009
1093
  ui,
1094
+ "customise-state-labels",
1010
1095
  "State labels",
1011
1096
  `The conductor writes back "${stateLabels.inProgress}", "${stateLabels.blocked}" and ` +
1012
1097
  `"${stateLabels.failed}" so the tracker alone shows live state. Rename them?`,
1013
1098
  );
1014
1099
  if (customiseStates) {
1015
- stateLabels.inProgress = await ask(ui, "Label for a run in progress", stateLabels.inProgress);
1016
- stateLabels.blocked = await ask(ui, "Label for a run parked on a human", stateLabels.blocked);
1017
- stateLabels.failed = await ask(ui, "Label for a run that gave up", stateLabels.failed);
1100
+ stateLabels.inProgress = await ask(ui, "state-label-in-progress", "Label for a run in progress", stateLabels.inProgress);
1101
+ stateLabels.blocked = await ask(ui, "state-label-blocked", "Label for a run parked on a human", stateLabels.blocked);
1102
+ stateLabels.failed = await ask(ui, "state-label-failed", "Label for a run that gave up", stateLabels.failed);
1018
1103
  }
1019
1104
 
1020
1105
  const routingLabelPrefix = await ask(
1021
1106
  ui,
1107
+ "routing-label-prefix",
1022
1108
  "Routing label prefix — an issue picks its checkout with <prefix><repo>",
1023
1109
  a.routingLabelPrefix,
1024
1110
  );
@@ -1028,18 +1114,21 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
1028
1114
  const seed = a.targetRepos[i];
1029
1115
  const name = await askValid(
1030
1116
  ui,
1117
+ `routing-key.${i + 1}`,
1031
1118
  `Routing key for repo ${i + 1} — the "${routingLabelPrefix}<key>" label an issue carries`,
1032
1119
  seed?.name ?? "",
1033
1120
  (v) => (v.length > 0 ? undefined : "A routing key is required, or no issue can reach this repo."),
1034
1121
  );
1035
1122
  const cloneUrl = await askValid(
1036
1123
  ui,
1124
+ `clone-url.${name}`,
1037
1125
  `Clone URL for ${routingLabelPrefix}${name}`,
1038
1126
  seed?.cloneUrl ?? "",
1039
1127
  (v) => (v.length > 0 ? undefined : "A clone URL is required — the daemon mirrors it before every run."),
1040
1128
  );
1041
1129
  const defaultBranch = await ask(
1042
1130
  ui,
1131
+ `default-branch.${name}`,
1043
1132
  `Default branch for ${name} — worktrees are cut from it and PRs target it`,
1044
1133
  seed?.defaultBranch ?? SETUP_DEFAULTS.defaultBranch,
1045
1134
  );
@@ -1051,6 +1140,7 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
1051
1140
 
1052
1141
  const more = await askYesNo(
1053
1142
  ui,
1143
+ `another-repo.${i + 1}`,
1054
1144
  "Another repo?",
1055
1145
  `${targetRepos.map((r) => r.name).join(", ")} configured. Add another checkout this project routes to?`,
1056
1146
  );
@@ -1121,6 +1211,7 @@ const askCaps: AreaAsker = async (ui, a) => {
1121
1211
  : "";
1122
1212
  const tuneCaps = await askYesNo(
1123
1213
  ui,
1214
+ "tune-caps",
1124
1215
  "Caps",
1125
1216
  `Defaults: ${workersDefault} workers, ` +
1126
1217
  `${spendLabel}, ${DEFAULT_CAPS.workerMaxTurns} base / ` +
@@ -1143,16 +1234,19 @@ const askCaps: AreaAsker = async (ui, a) => {
1143
1234
  // not silently go unasked.
1144
1235
  caps.maxConcurrentWorkers = await askNumber(
1145
1236
  ui,
1237
+ "max-concurrent-workers",
1146
1238
  "Max concurrent workers",
1147
1239
  workersDefault,
1148
1240
  );
1149
1241
  caps.dailySpendUsd = await askSpendCap(
1150
1242
  ui,
1243
+ "daily-spend-usd",
1151
1244
  "Spend ceiling per rolling day (USD) — blank = no spend cap",
1152
1245
  caps.dailySpendUsd !== undefined ? caps.dailySpendUsd : DEFAULT_CAPS.dailySpendUsd,
1153
1246
  );
1154
1247
  const workerMaxTurns = await askNumber(
1155
1248
  ui,
1249
+ "worker-max-turns",
1156
1250
  "Turn ceiling per worker",
1157
1251
  caps.workerMaxTurns ?? DEFAULT_CAPS.workerMaxTurns,
1158
1252
  );
@@ -1164,6 +1258,7 @@ const askCaps: AreaAsker = async (ui, a) => {
1164
1258
  );
1165
1259
  const workerMaxTurnsCeiling = await askNumber(
1166
1260
  ui,
1261
+ "worker-max-turns-ceiling",
1167
1262
  "Maximum turn ceiling for one issue",
1168
1263
  turnCeilingFallback,
1169
1264
  );
@@ -1179,16 +1274,19 @@ const askCaps: AreaAsker = async (ui, a) => {
1179
1274
  }
1180
1275
  caps.workerWallClockMs = await askNumber(
1181
1276
  ui,
1277
+ "worker-wall-clock-ms",
1182
1278
  "Wall-clock ceiling per worker (ms)",
1183
1279
  caps.workerWallClockMs ?? DEFAULT_CAPS.workerWallClockMs,
1184
1280
  );
1185
1281
  caps.maxAttemptsPerIssue = await askNumber(
1186
1282
  ui,
1283
+ "max-attempts-per-issue",
1187
1284
  "Failed implementation attempts per issue before escalation",
1188
1285
  caps.maxAttemptsPerIssue ?? DEFAULT_CAPS.maxAttemptsPerIssue,
1189
1286
  );
1190
1287
  caps.maxContinuationsPerIssue = await askNumber(
1191
1288
  ui,
1289
+ "max-continuations-per-issue",
1192
1290
  "Operational continuations per issue before escalation",
1193
1291
  caps.maxContinuationsPerIssue ?? DEFAULT_CAPS.maxContinuationsPerIssue,
1194
1292
  );
@@ -1198,7 +1296,12 @@ const askCaps: AreaAsker = async (ui, a) => {
1198
1296
  /** Outside the caps block: a model is not a ceiling, and an operator who left
1199
1297
  * the caps alone may still want workers on a cheaper model. */
1200
1298
  const askWorkerModel: AreaAsker = async (ui, a) => {
1201
- const answered = await ask(ui, "Worker model pattern (blank = harness default)", a.workerModel ?? "");
1299
+ const answered = await ask(
1300
+ ui,
1301
+ "worker-model",
1302
+ "Worker model pattern (blank = harness default)",
1303
+ a.workerModel ?? "",
1304
+ );
1202
1305
  const next: SetupAnswers = { ...a };
1203
1306
  if (answered.trim().length === 0) delete next.workerModel;
1204
1307
  else next.workerModel = answered.trim();
@@ -1216,6 +1319,7 @@ const askOmpSettings: AreaAsker = async (ui, a) => {
1216
1319
  const seed = a.ompSettings === undefined ? "" : JSON.stringify(a.ompSettings);
1217
1320
  const answered = await askValid(
1218
1321
  ui,
1322
+ "omp-settings",
1219
1323
  "Omp settings overlay for workers (YAML, blank = none — omp's schema, not conductor's)",
1220
1324
  seed,
1221
1325
  (value) => {
@@ -1266,10 +1370,26 @@ const askAuthorityArea: AreaAsker = async (ui, a, _probes, discovered) => {
1266
1370
  /** What a merge and a release must satisfy. Asked straight after the grants:
1267
1371
  * who may act, then under what conditions. The release half is only asked
1268
1372
  * when the orchestrator cuts releases — with humans releasing it has no gates
1269
- * to configure, so it would be a dead prompt (#368). */
1373
+ * to configure, so it would be a dead prompt (#368). The arming proof is
1374
+ * asked after the preconditions: it gates `arm`, not merge or release, but it
1375
+ * is the same kind of declared policy, so it rides the same area (#613).
1376
+ *
1377
+ * The `claim-only` consequence is stated in the option's own description so an
1378
+ * operator chooses it rather than discovering it: `arm` is already privileged,
1379
+ * and `claim-only` means anything that can invoke it can start dispatch once
1380
+ * the live claim and poller pass.
1381
+ */
1270
1382
  const askPolicy: AreaAsker = async (ui, a) => ({
1271
1383
  ...a,
1272
1384
  policy: await askPolicyPreconditions(ui, a.policy, a.authority.release),
1385
+ armProof: await askLiteral(
1386
+ ui,
1387
+ "arm-proof",
1388
+ "Arming — how should `arm` prove a human approved dispatch?",
1389
+ ARM_PROOFS,
1390
+ ARM_PROOF_CHOICES,
1391
+ a.armProof,
1392
+ ),
1273
1393
  });
1274
1394
 
1275
1395
  /** How a stuck run reaches a human, and who triages it when it does. */
@@ -1280,12 +1400,18 @@ const askEscalation: AreaAsker = async (ui, a) => {
1280
1400
  if (telegramChatId === undefined && telegram.pairedOwnerId !== undefined) {
1281
1401
  const usePaired = await askYesNo(
1282
1402
  ui,
1403
+ "tier2-use-paired-chat",
1283
1404
  "Tier-2 escalations",
1284
1405
  `omp-telegram is paired with chat ${telegram.pairedOwnerId}. Page it when a run is stuck?`,
1285
1406
  );
1286
1407
  if (usePaired) telegramChatId = telegram.pairedOwnerId;
1287
1408
  } else {
1288
- const answered = await ask(ui, "Telegram chat id for tier-2 escalations (blank for none)", telegramChatId ?? "");
1409
+ const answered = await ask(
1410
+ ui,
1411
+ "tier2-chat-id",
1412
+ "Telegram chat id for tier-2 escalations (blank for none)",
1413
+ telegramChatId ?? "",
1414
+ );
1289
1415
  telegramChatId = answered.length > 0 ? answered : undefined;
1290
1416
  }
1291
1417
  } else {
@@ -1303,6 +1429,7 @@ const askEscalation: AreaAsker = async (ui, a) => {
1303
1429
 
1304
1430
  const fallbackToIssueComment = await askYesNo(
1305
1431
  ui,
1432
+ "escalation-fallback",
1306
1433
  "Escalation fallback",
1307
1434
  "Also comment on the issue when a run escalates? Recommended: a chat message you miss is a run nobody sees.",
1308
1435
  );
@@ -1320,14 +1447,25 @@ const askEscalation: AreaAsker = async (ui, a) => {
1320
1447
  /**
1321
1448
  * Forum topic for tier-2 pages. Claimed threads from omp-telegram's
1322
1449
  * `threads.json` are offered when readable; a missing file is silent and the
1323
- * operator can still type an id or keep flat chat (#318).
1450
+ * operator can still type an id or keep flat chat (#318). A registry that is
1451
+ * *present but unreadable* is the opposite: the bridge is in a state the
1452
+ * wizard cannot read, so the manual id would be a guess against unverifiable
1453
+ * state — surface the problem before the fallback instead of silently offering
1454
+ * either (#626).
1324
1455
  */
1325
1456
  async function askTelegramTopicId(
1326
1457
  ui: WizardUi,
1327
1458
  stateDir: string,
1328
1459
  prior: number | undefined,
1329
1460
  ): Promise<number | undefined> {
1330
- const claimed = claimedTelegramTopics(stateDir);
1461
+ const result = claimedTelegramTopics(stateDir);
1462
+ if (result.kind === "unavailable") {
1463
+ ui.notify(
1464
+ `Cannot offer claimed topics: omp-telegram's claim registry is unreadable (${result.problem}) — check the bridge state before typing a topic id, or keep flat chat`,
1465
+ "warning",
1466
+ );
1467
+ }
1468
+ const claimed = result.kind === "ok" ? result.claims : [];
1331
1469
  const manual = "Enter thread id manually";
1332
1470
  const none = "None — flat chat (0.13 behaviour)";
1333
1471
  if (claimed.length > 0) {
@@ -1346,6 +1484,7 @@ async function askTelegramTopicId(
1346
1484
  ];
1347
1485
  const priorIdx = prior === undefined ? -1 : claimed.findIndex((t) => t.threadId === prior);
1348
1486
  const picked = await ui.select("Telegram forum topic for tier-2 pages", options, {
1487
+ key: "tier2-topic",
1349
1488
  initialIndex: priorIdx >= 0 ? priorIdx : claimed.length + 1,
1350
1489
  });
1351
1490
  if (picked === undefined) throw new Cancelled();
@@ -1358,6 +1497,7 @@ async function askTelegramTopicId(
1358
1497
 
1359
1498
  const answered = await ask(
1360
1499
  ui,
1500
+ "tier2-topic-id",
1361
1501
  "Telegram forum topic id (blank for flat chat)",
1362
1502
  prior !== undefined ? String(prior) : "",
1363
1503
  );
@@ -1388,7 +1528,7 @@ const askReporting: AreaAsker = async (ui, a) => {
1388
1528
  description: "hold non-bypass interruptions outside selected local working hours",
1389
1529
  },
1390
1530
  ],
1391
- { initialIndex: a.availability === undefined ? 0 : 1 },
1531
+ { key: "operator-availability", initialIndex: a.availability === undefined ? 0 : 1 },
1392
1532
  );
1393
1533
  if (mode === undefined) throw new Cancelled();
1394
1534
 
@@ -1420,6 +1560,7 @@ const askReporting: AreaAsker = async (ui, a) => {
1420
1560
  : a.dailyDigestAt ?? (a.digestCadence === "daily" ? "model-timed" : fallback);
1421
1561
  const schedule = await askValid(
1422
1562
  ui,
1563
+ "digest-schedule",
1423
1564
  'Daily rollup time in that timezone / digest cadence ("per-tick", "model-timed", "off", or 24h HH:MM)',
1424
1565
  shown,
1425
1566
  (value) =>
@@ -1444,6 +1585,7 @@ const askReporting: AreaAsker = async (ui, a) => {
1444
1585
  a.reportingTimezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone ?? "UTC";
1445
1586
  const reportingTimezone = await askValid(
1446
1587
  ui,
1588
+ "reporting-timezone",
1447
1589
  "Operator timezone (IANA, for example Europe/London)",
1448
1590
  defaultZone,
1449
1591
  (value) => {
@@ -1457,6 +1599,7 @@ const askReporting: AreaAsker = async (ui, a) => {
1457
1599
  );
1458
1600
  const daysText = await askValid(
1459
1601
  ui,
1602
+ "working-weekdays",
1460
1603
  "Working weekdays (comma-separated: mon,tue,wed,thu,fri,sat,sun)",
1461
1604
  a.availability?.days.join(",") ?? "mon,tue,wed,thu,fri",
1462
1605
  (value) => {
@@ -1470,12 +1613,14 @@ const askReporting: AreaAsker = async (ui, a) => {
1470
1613
  const days = daysText.split(",").map((day) => day.trim().toLowerCase() as Weekday);
1471
1614
  const start = await askValid(
1472
1615
  ui,
1616
+ "availability-start",
1473
1617
  "Availability starts (24h HH:MM)",
1474
1618
  a.availability?.start ?? "09:00",
1475
1619
  (value) => (/^([01]\d|2[0-3]):[0-5]\d$/.test(value) ? undefined : "Use 24h HH:MM."),
1476
1620
  );
1477
1621
  const end = await askValid(
1478
1622
  ui,
1623
+ "availability-end",
1479
1624
  "Availability ends (24h HH:MM)",
1480
1625
  a.availability?.end ?? "17:00",
1481
1626
  (value) =>
@@ -1487,6 +1632,7 @@ const askReporting: AreaAsker = async (ui, a) => {
1487
1632
  );
1488
1633
  const bypassText = await askValid(
1489
1634
  ui,
1635
+ "quiet-hours-bypass",
1490
1636
  `Quiet-hours bypass categories (comma-separated; "none" = none; choices: ${INTERRUPT_CATEGORIES.join(",")})`,
1491
1637
  a.availability === undefined ? "fleet-stopped" : a.availability.bypass.join(",") || "none",
1492
1638
  (value) => {
@@ -1634,7 +1780,7 @@ async function chooseAmendArea(ui: WizardUi, prior: ProjectConfig, defaults: Cap
1634
1780
  description: "full interview for a new project; existing projects stay as they are",
1635
1781
  },
1636
1782
  ],
1637
- { initialIndex: 0 },
1783
+ { key: `existing-project-action.${prior.name}`, initialIndex: 0 },
1638
1784
  );
1639
1785
  if (mode === undefined) throw new Cancelled();
1640
1786
  if (mode === ADD_PROJECT) return { kind: "add-project" };
@@ -1650,7 +1796,7 @@ async function chooseAmendArea(ui: WizardUi, prior: ProjectConfig, defaults: Cap
1650
1796
  const picked = await ui.select(
1651
1797
  "Which area? Each row shows what it says now",
1652
1798
  choices.map((c) => ({ label: c.label, description: c.description })),
1653
- { initialIndex: 0 },
1799
+ { key: `amend-area.${prior.name}`, initialIndex: 0 },
1654
1800
  );
1655
1801
  if (picked === undefined) throw new Cancelled();
1656
1802
 
@@ -1693,6 +1839,7 @@ async function collectAnswers(
1693
1839
 
1694
1840
  const projectName = await askValid(
1695
1841
  ui,
1842
+ "project-name",
1696
1843
  "Project name",
1697
1844
  projectArg ?? seed.projectName,
1698
1845
  (v) => (v.length > 0 ? undefined : "A name is required — it is how `omp-conductor status --project <name>` finds this project."),
@@ -1821,7 +1968,7 @@ export async function collectSetup(
1821
1968
  description: "overwrites this project's config entry on apply; other projects untouched",
1822
1969
  },
1823
1970
  ],
1824
- { initialIndex: 0 },
1971
+ { key: `duplicate-project-action.${answers.projectName}`, initialIndex: 0 },
1825
1972
  );
1826
1973
  if (mode === undefined) throw new Cancelled();
1827
1974
  if (mode.startsWith("Amend")) {
@@ -1866,7 +2013,7 @@ export async function collectSetup(
1866
2013
  description: "overwrites this project's config entry on apply; other projects untouched",
1867
2014
  },
1868
2015
  ],
1869
- { initialIndex: 0 },
2016
+ { key: `duplicate-project-action.${answers.projectName}`, initialIndex: 0 },
1870
2017
  );
1871
2018
  if (mode === undefined) throw new Cancelled();
1872
2019
  if (mode.startsWith("Amend")) {
@@ -1884,6 +2031,7 @@ export async function collectSetup(
1884
2031
  "Walks every question again and overwrites this project's config entry on apply. " +
1885
2032
  "Other projects are left alone. Cancel and pick \"Change one area\" to amend without a full replace, " +
1886
2033
  "or \"Add another project\" to create a neighbour.",
2034
+ { key: `replace-project.${prior.name}` },
1887
2035
  );
1888
2036
  if (replace !== true) throw new Cancelled();
1889
2037
  return { answers: await collectAnswers(ui, prior, projectArg, probes) };
@@ -1942,10 +2090,39 @@ export interface SetupApplyDeps {
1942
2090
  labels: (trackerRepo: string, a: SetupAnswers) => Promise<LabelPlan[]>;
1943
2091
  preview: (project: ProjectConfig) => Promise<QueuePreview>;
1944
2092
  createLabels: (trackerRepo: string, plan: LabelPlan[]) => Promise<string[]>;
2093
+ /**
2094
+ * Compensation half of {@link createLabels}: deletes exactly the labels a
2095
+ * failed apply created, so the tracker returns to its pre-entry state.
2096
+ * Called only on rollback, where a deletion failure is a reported
2097
+ * restoration failure — the orphaned label is named, never silently left
2098
+ * behind (#652).
2099
+ */
2100
+ deleteLabels: (trackerRepo: string, created: string[]) => Promise<void>;
2101
+ /**
2102
+ * Proves a healthy runtime generation serves the COMPLETE prior configured
2103
+ * project set after a failed apply rolled the disk state back (#650). The
2104
+ * surviving entry-generation process predates the rollback: its store
2105
+ * handle points at the file the rollback replaced (writes diverge into the
2106
+ * unlinked inode until its next restart) and its in-memory config may be
2107
+ * the rejected generation — so "proving health" is never the proof. The
2108
+ * default implementation restarts through the same lifecycle seam the
2109
+ * apply's own restart leg uses and then proves /healthz names every prior
2110
+ * project. Returns false when the proof cannot be made — the caller then
2111
+ * stays fail-closed (a setup hold stays in force, with the recovery
2112
+ * command named) rather than resuming dispatch onto an unproven runtime.
2113
+ */
2114
+ proveRuntime: (priorProjects: readonly string[]) => Promise<boolean>;
2115
+ /**
2116
+ * The lifecycle/drain seam the acknowledged quiescence barrier runs on —
2117
+ * the same `DrainDeps` `upgrade`, `restart` and `setup host` use, so the
2118
+ * freeze, its token proof and its staleness checks are the seam's, never a
2119
+ * second counter-only fence (#618). Injectable so a regression can drive
2120
+ * the exact admission interleaving through real barrier code.
2121
+ */
2122
+ drain: DrainDeps;
1945
2123
  smoke: (project: string) => Promise<SetupSmokeResult>;
1946
2124
  restart: (o: { project?: string }) => Promise<RestartResult>;
1947
2125
  arm: (project: string) => Promise<string>;
1948
- resume: (project: string) => void;
1949
2126
  hostInstall: (
1950
2127
  project: ProjectConfig,
1951
2128
  caps: Caps,
@@ -1959,20 +2136,593 @@ export interface SetupApplyDeps {
1959
2136
  ) => Promise<InstallOutcome>;
1960
2137
  }
1961
2138
 
2139
+ /**
2140
+ * The default drain accessors the setup apply barrier runs on, wired exactly
2141
+ * like `setup host`'s (`setup-install.ts`) — except the pause names `setup`
2142
+ * as the actor and the barrier refuses on live workers instead of waiting for
2143
+ * a drain.
2144
+ *
2145
+ * Two deliberate divergences from the drop-in lifecycle accessors, both
2146
+ * because a setup apply owns a *host-global* transaction rather than the one
2147
+ * project a restart drains:
2148
+ *
2149
+ * - `snapshot` counts live workers through {@link liveWorkersReadOnly}, never
2150
+ * `statusSnapshot`: the observation that gates the barrier must be
2151
+ * read-only, because database preparation is itself one of the mutations
2152
+ * the barrier precedes (#651 review #2).
2153
+ * - `layers` reports the *global* sentinel's paused flag, not the project
2154
+ * bare-read `fleetLayers` derives — the barrier freezes the whole host
2155
+ * through the one global pause sentinel (`scope.pauseKey === undefined`),
2156
+ * so the fence it must re-prove is that sentinel, never a per-project one
2157
+ * (#651 review #4).
2158
+ */
2159
+ function defaultSetupDrain(): DrainDeps {
2160
+ return {
2161
+ snapshot: (project) => ({ liveWorkers: liveWorkersReadOnly(project) }),
2162
+ layers: () => ({ ...fleetLayers(), paused: isPaused() }),
2163
+ projectNames: () => loadConfig().projects.map((p) => p.name),
2164
+ daemonIdentity: setupDaemonIdentity,
2165
+ pauseState: (project) => pauseInstance(project),
2166
+ setPaused: (v, project) => setPaused(v, { source: pauseSourceToken("setup"), reason: "setup apply fence" }, project),
2167
+ // The daemon-side admission acknowledgement: the file the running daemon
2168
+ // itself writes when it observes the fence at an admission boundary, so
2169
+ // the barrier never mistakes a second synchronous count for the daemon's
2170
+ // own word that it will claim nothing (#651 review #3).
2171
+ admissionAck: () => readAdmissionAck(),
2172
+ sleep: Bun.sleep,
2173
+ log: () => {},
2174
+ };
2175
+ }
2176
+
2177
+ /**
2178
+ * The daemon identity the setup barrier acknowledges and re-proves: the known
2179
+ * pidfile record when one is live, else an ACTIVE `omp-conductor.service`
2180
+ * unit whose runtime record went missing — a supervised daemon is still a
2181
+ * running daemon without its record, and its MainPID is the generation that
2182
+ * identifies the exact instance (#651, review #2). An UNKNOWN systemd query
2183
+ * throws: "dbus blipped" is not "no daemon", and the barrier must fail closed
2184
+ * rather than acknowledge quiescence it cannot prove.
2185
+ */
2186
+ function setupDaemonIdentity(): DaemonIdentity {
2187
+ const ownership = probeUnit(SYSTEMD_UNIT);
2188
+ if (ownership.kind === "unknown") {
2189
+ throw new Error(
2190
+ `cannot prove whether the daemon is running: systemd ownership probe failed (${ownership.reason})`,
2191
+ );
2192
+ }
2193
+ const daemon = livingDaemon();
2194
+ if (daemon !== undefined) {
2195
+ // The record names the exact instance; `daemonGeneration` formats the
2196
+ // same generation the daemon's own admission acknowledgement records, so
2197
+ // the two sides of the acknowledged fence always compare like for like.
2198
+ return { running: true, project: daemon.project, generation: daemonGeneration() };
2199
+ }
2200
+ if (ownership.kind === "active") {
2201
+ // The unit owns a live MainPID but the record is absent (a crash raced a
2202
+ // re-record, an unmanaged start never wrote one): the daemon is running,
2203
+ // and the MainPID is the generation that lets the fence spot a restart.
2204
+ return { running: true, generation: `systemd:${ownership.pid}` };
2205
+ }
2206
+ return { running: false };
2207
+ }
2208
+
1962
2209
  export const DEFAULT_APPLY: SetupApplyDeps = {
1963
2210
  scopes: checkTokenScopes,
1964
2211
  labels: planLabels,
1965
2212
  preview: previewProject,
1966
2213
  createLabels: createMissingLabels,
2214
+ deleteLabels: deleteCreatedLabels,
2215
+ proveRuntime: async (priorProjects) => {
2216
+ // No short-circuit on a healthy daemon: whatever process is still
2217
+ // answering predates the rollback, so it holds a handle on the store file
2218
+ // the rollback replaced and may serve the rejected config. Restart
2219
+ // through the same fail-closed lifecycle seam the apply's own restart leg
2220
+ // uses — whose `waitForOwnedDaemon` proves MainPID + /healthz and that a
2221
+ // fresh boot loaded the restored/rolled-back state — then prove the new
2222
+ // generation serves EVERY prior configured project, not just health on
2223
+ // one (#650, PR #700 review #1).
2224
+ try {
2225
+ const restarted = await restartDaemon({});
2226
+ const health = await healthCheck(restarted.record.port);
2227
+ if (!health.ok) return false;
2228
+ return priorProjects.every((project) => healthServesProject(health.body, project));
2229
+ } catch {
2230
+ return false;
2231
+ }
2232
+ },
2233
+ drain: defaultSetupDrain(),
1967
2234
  smoke: runSetupSmoke,
1968
2235
  restart: (o) => restartDaemon(o),
1969
2236
  arm: (project) => ensureSetupArm(project),
1970
- resume: (project) => setPaused(false, undefined, project),
1971
2237
  hostInstall: (project, caps, telegramStateDir, ui) =>
1972
2238
  runHostInstall(project, caps, telegramStateDir, ui),
1973
2239
  graphInstall: (project, ui, options) => runGraphInstall(project, ui, options),
1974
2240
  };
1975
2241
 
2242
+ /** One pause sentinel's pre-entry state, four-valued so absence, readable-valid
2243
+ * bytes, malformed bytes and unreadability are never conflated.
2244
+ * - `{ kind: "absent" }` — no file on entry; rollback removes one the barrier
2245
+ * created.
2246
+ * - `{ kind: "bytes", bytes }` — readable AND valid on entry; rollback writes
2247
+ * these bytes back verbatim, so the pre-entry owner token and timestamp
2248
+ * survive a refused apply instead of being replaced by a fresh equivalent
2249
+ * pause (#618, continuation #7).
2250
+ * - `{ kind: "malformed", bytes, path }` — readable but not a valid pause
2251
+ * instance (an unparseable timestamp or provenance line). This is
2252
+ * unreadable-as-state: the barrier must fail closed BEFORE changing
2253
+ * anything and never delete it, because collapsing it to absence would
2254
+ * remove an operator hold it could not prove (#650).
2255
+ * - `{ kind: "unreadable", path }` — the file exists but cannot be read. The
2256
+ * barrier must fail closed BEFORE changing anything: collapsing this to
2257
+ * absence and later deleting the path would remove an operator hold it
2258
+ * never saw (#651 review #5). */
2259
+ type CapturedPause =
2260
+ | { kind: "absent" }
2261
+ | { kind: "bytes"; bytes: string }
2262
+ | { kind: "malformed"; bytes: string; path: string }
2263
+ | { kind: "unreadable"; path: string };
2264
+
2265
+ /** Whether one sentinel's bytes parse as a valid pause instance — the exact
2266
+ * grammar {@link pauseInstance} reads in `daemon.ts` (a parseable ISO
2267
+ * timestamp line, then a `source=` provenance line). Readable bytes that
2268
+ * fail this are an unreadable-as-state malformed hold: valid bytes are
2269
+ * restored verbatim, malformed ones are refused at entry and never deleted
2270
+ * (#650). */
2271
+ function isValidPauseBytes(bytes: string): boolean {
2272
+ const [line1, line2] = bytes.split("\n");
2273
+ if (!Number.isFinite(Date.parse(line1?.trim() ?? ""))) return false;
2274
+ if (line2 === undefined) return false;
2275
+ return /^source=(\S+)(?: reason="(.*)")?$/.test(line2.trim());
2276
+ }
2277
+
2278
+ /** Reads one pause sentinel, keeping absence, readable-valid bytes, malformed
2279
+ * bytes and unreadability apart — never conflated: collapsing malformed to
2280
+ * absence would let a refusal delete an operator hold it could not parse. */
2281
+ function readSentinel(p: string): CapturedPause {
2282
+ if (!existsSync(p)) return { kind: "absent" };
2283
+ try {
2284
+ const bytes = readFileSync(p, "utf8");
2285
+ return isValidPauseBytes(bytes) ? { kind: "bytes", bytes } : { kind: "malformed", bytes, path: p };
2286
+ } catch {
2287
+ return { kind: "unreadable", path: p };
2288
+ }
2289
+ }
2290
+
2291
+ /**
2292
+ * The pre-entry pause sentinel the setup apply itself can change. Setup
2293
+ * freezes the whole host through the one *global* sentinel — it must never
2294
+ * write a project's pause instead, which left the wrong project held when
2295
+ * adding a neighbour next to a daemon recorded for the existing project
2296
+ * (#651 review #4) — so the captured path is the global one, its scope is
2297
+ * preserved exactly (a successful apply or a refusal restores that same
2298
+ * path's pre-entry bytes, never a project-scoped sentinel), and an entry
2299
+ * that is already paused — valid, malformed or unreadable — fails closed in
2300
+ * its own way rather than being rebuilt or deleted (#650).
2301
+ */
2302
+ function capturePause(): CapturedPause {
2303
+ return readSentinel(pausedPath());
2304
+ }
2305
+
2306
+ /** Restores the sentinel {@link capturePause} captured — but only while the
2307
+ * sentinel on disk is still the fence THIS transaction froze (the bytes it
2308
+ * wrote, or the pre-entry hold it inherited). A `resume` or a newer operator
2309
+ * hold that replaced the setup freeze is a takeover the stale transaction
2310
+ * must preserve: writing the entry bytes back over it, or removing the file
2311
+ * it owns, would delete a safety hold a later actor wrote in the meantime
2312
+ * (#651 review #3). An unreadable or malformed current sentinel is also left
2313
+ * alone — overwriting what cannot be read (or parsed) is the same
2314
+ * overwrite-of-the-unseen this guard exists to prevent. Never called with a
2315
+ * malformed entry state: the barrier fails closed before mutating then; the
2316
+ * branch exists so a malformed state would still be restored byte for byte
2317
+ * rather than deleted, never treated as absence. */
2318
+ function restorePause(state: CapturedPause, owned: CapturedPause): void {
2319
+ const p = pausedPath();
2320
+ const current = readSentinel(p);
2321
+ // Only this transaction's own fence may be undone. An unreadable or
2322
+ // malformed sentinel is left alone (overwriting what cannot be read or
2323
+ // parsed is overwriting a hold the barrier never saw); a sentinel that no
2324
+ // longer matches the bytes this transaction froze was replaced by a takeover
2325
+ // — a resume or a newer hold — and that takeover stands.
2326
+ let isOwnFence = false;
2327
+ if (current.kind === "bytes" && owned.kind === "bytes") {
2328
+ isOwnFence = current.bytes === owned.bytes;
2329
+ } else if (current.kind === "absent" && owned.kind === "absent") {
2330
+ isOwnFence = true;
2331
+ }
2332
+ if (!isOwnFence) return;
2333
+ if (state.kind === "bytes" || state.kind === "malformed") {
2334
+ mkdirSync(dirname(p), { recursive: true });
2335
+ writeFileSync(p, state.bytes);
2336
+ } else {
2337
+ rmSync(p, { force: true });
2338
+ }
2339
+ }
2340
+
2341
+ // -------------------------------------------------- the mutation inventory --
2342
+ //
2343
+ // Child 2 of the setup-transaction decomposition (#652): before the first
2344
+ // mutation the apply records the COMPLETE inventory of every surface it can
2345
+ // write — config, briefs, staged units/scripts/tick, the AGENTS link, the run
2346
+ // store, the daemon record, the arm marker, the config backups and the
2347
+ // tracker labels — with byte-exact pre-entry state. Any later exception or
2348
+ // failed smoke restores every one of them (or removes what a first install
2349
+ // created), compensates the tracker exactly, and reports the first
2350
+ // restoration failure honestly instead of pretending the prior state is
2351
+ // coherent. The one surface a rollback never undoes blindly is the pause
2352
+ // sentinel: {@link restorePause} keeps child 1's ownership rule — only the
2353
+ // fence THIS transaction froze is undone, and a newer actor's hold stands.
2354
+
2355
+ /** The run store's pre-entry state: a byte-consistent snapshot when the
2356
+ * store exists, absence on a first install. */
2357
+ interface StoreCapture {
2358
+ path: string;
2359
+ kind: "absent" | "snapshot";
2360
+ snapshotPath?: string;
2361
+ }
2362
+
2363
+ /** The complete pre-entry mutation inventory of one setup apply (#652). */
2364
+ interface SetupInventory {
2365
+ /** `config.json` pre-entry state (bytes, absence, or unreadable). */
2366
+ config: CapturedPathState;
2367
+ /** The pre-entry configured project set — what the prior runtime generation
2368
+ * must be proven to serve after a rollback restores the config (#650). */
2369
+ priorProjects: string[];
2370
+ /** The brief files the apply will write: `POLICY.md` + `ORCHESTRATOR.md`. */
2371
+ briefs: CapturedPathState[];
2372
+ /** Every staged host-runtime path the apply may write (units, scripts, tick). */
2373
+ runtime: CapturedPathState[];
2374
+ /** The AGENTS.md brief link the apply will create/update, when planned. */
2375
+ briefLink?: CapturedPathState;
2376
+ /** The run store: byte-consistent snapshot when it exists, absence on a first install. */
2377
+ store: StoreCapture;
2378
+ /** The daemon pidfile record the smoke/restart legs write. */
2379
+ record: CapturedPathState;
2380
+ /** The external-heartbeat arm marker, when an external orchestrator is planned. */
2381
+ armMarker?: CapturedPathState;
2382
+ /** Directories the apply may create (workspace/tick cwds), absent at entry;
2383
+ * rollback removes them only while still empty, so operator content that
2384
+ * landed in one is never destroyed. */
2385
+ dirs: string[];
2386
+ /** Pre-entry listing of the config backup dir; backups the apply created
2387
+ * are removed on rollback so the backups return to their entry set. */
2388
+ backups: { dir: string; files: string[] };
2389
+ }
2390
+
2391
+ /** A byte-consistent pre-entry copy of the run store: VACUUM INTO reads main
2392
+ * file + WAL as one, so the snapshot holds every committed row even while a
2393
+ * paused daemon holds a writer handle. The snapshot lives under the state
2394
+ * dir for the transaction's lifetime and is removed on commit and rollback
2395
+ * alike; a partial snapshot from a throwing VACUUM is removed immediately,
2396
+ * never left as a stray temp copy in the state dir (#650, review #2). */
2397
+ function captureStore(path: string): StoreCapture {
2398
+ if (!existsSync(path)) return { path, kind: "absent" };
2399
+ const snapshotPath = join(stateDir(), `.conductor.db.setup-${process.pid}.${randomUUID()}.snapshot`);
2400
+ try {
2401
+ vacuumInto(path, snapshotPath);
2402
+ } catch (err) {
2403
+ rmSync(snapshotPath, { force: true });
2404
+ throw err;
2405
+ }
2406
+ return { path, kind: "snapshot", snapshotPath };
2407
+ }
2408
+
2409
+ /** The backup dir listing, or the empty set when the dir does not exist yet. */
2410
+ function readdirIfPresent(dir: string): string[] {
2411
+ try {
2412
+ return readdirSync(dir).sort();
2413
+ } catch {
2414
+ return [];
2415
+ }
2416
+ }
2417
+
2418
+ /**
2419
+ * The full pre-entry mutation inventory of one apply, read BEFORE the first
2420
+ * mutation. Throws when any captured path exists but cannot be read: an
2421
+ * unreadable pre-entry path is a fault, never absence — collapsing it would
2422
+ * let a rollback delete state the apply never saw, the same fail-closed rule
2423
+ * the pause sentinel capture already applies (#651 review #5).
2424
+ *
2425
+ * `priorProjectNames` is the pre-entry configured project set (from the
2426
+ * config `setup()` loaded before the interview): the set a failed apply must
2427
+ * prove the restored runtime serves again. It comes from the entry load, not
2428
+ * a post-rollback re-read, so an un-restored config cannot make the proof
2429
+ * measure the wrong set (#650).
2430
+ */
2431
+ function captureInventory(
2432
+ plan: DerivedPlan,
2433
+ answers: SetupAnswers,
2434
+ priorProjectNames: string[],
2435
+ ): SetupInventory {
2436
+ const runtime: CapturedPathState[] = [];
2437
+ const captureIfWritten = (w: { action: string; path: string } | undefined): void => {
2438
+ // Only paths the plan says will actually be written: a `keep` path is
2439
+ // untouched by the apply and needs no pre-entry state.
2440
+ if (w !== undefined && w.action !== "keep") runtime.push(capturePathState(w.path));
2441
+ };
2442
+ captureIfWritten(plan.runtime.service);
2443
+ captureIfWritten(plan.runtime.herdrUnit);
2444
+ captureIfWritten(plan.runtime.herdrConfig);
2445
+ captureIfWritten(plan.runtime.herdrEnv);
2446
+ captureIfWritten(plan.runtime.recoverUnit);
2447
+ captureIfWritten(plan.runtime.recoverScript);
2448
+ captureIfWritten(plan.runtime.tick);
2449
+
2450
+ const briefs = answers.writeOrchestratorBrief
2451
+ ? [policyPathForProject(plan.project), briefPathForProject(plan.project)].map(capturePathState)
2452
+ : [];
2453
+ const briefLink =
2454
+ plan.runtime.briefLink !== undefined &&
2455
+ (plan.runtime.briefLink.action === "create" || plan.runtime.briefLink.action === "update")
2456
+ ? capturePathState(plan.runtime.briefLink.path)
2457
+ : undefined;
2458
+ const record = capturePathState(recordPath());
2459
+ const armMarker =
2460
+ plan.project.escalation.orchestrator === "external"
2461
+ ? capturePathState(armedMarkerPath(plan.project.name))
2462
+ : undefined;
2463
+
2464
+ // Unreadable check across every captured path, before anything is written.
2465
+ const all = [
2466
+ capturePathState(configPath()),
2467
+ ...briefs,
2468
+ ...runtime,
2469
+ ...(briefLink === undefined ? [] : [briefLink]),
2470
+ record,
2471
+ ...(armMarker === undefined ? [] : [armMarker]),
2472
+ ];
2473
+ const unreadable = all.find((c) => c.kind === "unreadable");
2474
+ if (unreadable !== undefined) {
2475
+ throw new Error(
2476
+ `the path ${unreadable.path} exists but could not be read — refusing to start the apply rather than ` +
2477
+ "risk overwriting state it never saw",
2478
+ );
2479
+ }
2480
+
2481
+ // Directories the apply can create (the workspace root via the brief write,
2482
+ // the fleet cwd via the AGENTS link and the tick config). Only absent ones
2483
+ // are recorded; rollback removes them only while still empty.
2484
+ const dirs = new Set<string>();
2485
+ if (answers.writeOrchestratorBrief) dirs.add(dirname(briefPathForProject(plan.project)));
2486
+ if (briefLink !== undefined) dirs.add(dirname(briefLink.path));
2487
+ if (plan.runtime.tick !== undefined && plan.runtime.tick.action !== "keep") {
2488
+ dirs.add(dirname(plan.runtime.tick.path));
2489
+ }
2490
+
2491
+ return {
2492
+ config: all[0]!,
2493
+ priorProjects: priorProjectNames,
2494
+ briefs,
2495
+ runtime,
2496
+ ...(briefLink === undefined ? {} : { briefLink }),
2497
+ store: captureStore(dbPath()),
2498
+ record,
2499
+ ...(armMarker === undefined ? {} : { armMarker }),
2500
+ dirs: [...dirs].filter((d) => !existsSync(d)),
2501
+ backups: { dir: configBackupDir(), files: readdirIfPresent(configBackupDir()) },
2502
+ };
2503
+ }
2504
+
2505
+ /**
2506
+ * Restores every captured surface on a failed apply (#652): the config and
2507
+ * the backups the apply created, the briefs, the staged runtime files, the
2508
+ * AGENTS link, the run store, the daemon record and the arm marker; removes
2509
+ * what a first install created; and compensates tracker labels exactly.
2510
+ * Returns every restoration failure — the first included — so the caller
2511
+ * reports the rollback honestly instead of implying the prior state is
2512
+ * coherent. Never touches the pause sentinel: that restore keeps child 1's
2513
+ * ownership rule and is the caller's.
2514
+ */
2515
+ async function restoreInventory(
2516
+ inventory: SetupInventory,
2517
+ trackerRepo: string,
2518
+ createdLabels: string[],
2519
+ deleteLabels: SetupApplyDeps["deleteLabels"],
2520
+ ): Promise<string[]> {
2521
+ const failures: string[] = [];
2522
+ const restore = (state: CapturedPathState | undefined, opts?: RestorePathOptions): void => {
2523
+ if (state === undefined) return;
2524
+ const failure = restorePathState(state, opts);
2525
+ if (failure !== undefined) failures.push(failure);
2526
+ };
2527
+
2528
+ // The config first — the file everything else derives from. Restored
2529
+ // through the atomic temp-then-rename writer the upgrade rollback uses, so
2530
+ // a crash mid-restore cannot leave a truncated config; the backup that
2531
+ // writer makes of the failed config is pruned with the rest of the apply's
2532
+ // new backups below, returning the backup dir to its entry listing.
2533
+ if (inventory.config.kind === "bytes") {
2534
+ try {
2535
+ writeConfigRaw(inventory.config.bytes);
2536
+ } catch (err) {
2537
+ failures.push(
2538
+ `could not restore config.json: ${err instanceof Error ? err.message : String(err)}`,
2539
+ );
2540
+ }
2541
+ } else {
2542
+ restore(inventory.config);
2543
+ }
2544
+ const currentBackups = readdirIfPresent(inventory.backups.dir);
2545
+ for (const name of currentBackups) {
2546
+ if (inventory.backups.files.includes(name)) continue;
2547
+ try {
2548
+ rmSync(join(inventory.backups.dir, name), { force: true });
2549
+ } catch (err) {
2550
+ failures.push(
2551
+ `could not remove backup ${join(inventory.backups.dir, name)}: ${err instanceof Error ? err.message : String(err)}`,
2552
+ );
2553
+ }
2554
+ }
2555
+
2556
+ for (const brief of inventory.briefs) restore(brief);
2557
+ for (const path of inventory.runtime) restore(path);
2558
+ // The AGENTS link: only a symlink is ever removed or re-linked, so an
2559
+ // operator's regular file that appeared mid-flight survives — reported as a
2560
+ // failure, because the prior state is then not coherent.
2561
+ restore(inventory.briefLink, { preserveRegularFile: true });
2562
+
2563
+ // The run store: replace from the byte-consistent snapshot (or remove it
2564
+ // entirely on a first-install rollback).
2565
+ try {
2566
+ rmSync(inventory.store.path, { force: true });
2567
+ rmSync(`${inventory.store.path}-wal`, { force: true });
2568
+ rmSync(`${inventory.store.path}-shm`, { force: true });
2569
+ if (inventory.store.kind === "snapshot") {
2570
+ copyFileSync(inventory.store.snapshotPath!, inventory.store.path);
2571
+ }
2572
+ } catch (err) {
2573
+ failures.push(
2574
+ `could not restore the run store at ${inventory.store.path}: ${err instanceof Error ? err.message : String(err)}`,
2575
+ );
2576
+ }
2577
+
2578
+ // The daemon record: only while no daemon is live. A live process owns its
2579
+ // record — restoring the entry bytes over it would describe a daemon that
2580
+ // no longer runs, or remove the record of one that does.
2581
+ if (livingDaemon() === undefined) restore(inventory.record);
2582
+ restore(inventory.armMarker);
2583
+
2584
+ // Directories the apply created: removed only while empty (rmdirSync, never
2585
+ // a recursive rm — operator content that landed in one must survive).
2586
+ for (const dir of inventory.dirs) {
2587
+ try {
2588
+ rmdirSync(dir);
2589
+ } catch {
2590
+ // Non-empty, missing, or busy — leave it; this is not a restoration failure.
2591
+ }
2592
+ }
2593
+
2594
+ // Tracker compensation: exactly the labels THIS apply created.
2595
+ if (createdLabels.length > 0) {
2596
+ try {
2597
+ await deleteLabels(trackerRepo, createdLabels);
2598
+ } catch (err) {
2599
+ failures.push(err instanceof Error ? err.message : String(err));
2600
+ }
2601
+ }
2602
+ return failures;
2603
+ }
2604
+
2605
+ /**
2606
+ * The host-global scope a setup apply's barrier covers: every configured
2607
+ * project served by the daemon, and the global pause sentinel so one write
2608
+ * pauses the whole host. Deliberately NOT {@link resolveScope}'s
2609
+ * project-narrowing — which would pause the daemon's recorded project's
2610
+ * sentinel instead of the host-wide one when adding a neighbour next to it
2611
+ * (review #4). A first install has no config and no daemon, so the scope names
2612
+ * the bare selector and the observation proves zero without any reader.
2613
+ */
2614
+ function setupScope(drain: DrainDeps): UpgradeScope {
2615
+ let configured: readonly string[] = [];
2616
+ try {
2617
+ configured = drain.projectNames();
2618
+ } catch (err) {
2619
+ // An existing config that cannot be enumerated is a fault, never absence:
2620
+ // collapsing it to the bare selector would read "zero workers" for a fleet
2621
+ // the config still describes, then overwrite the unreadable file (#651,
2622
+ // review #2). Only a truly absent config legitimately yields the bare
2623
+ // selector (a first install), and even there the observation still fails
2624
+ // closed against the daemon identity.
2625
+ if (existsSync(configPath())) {
2626
+ throw new Error(
2627
+ `the configured projects could not be read: ${err instanceof Error ? err.message : String(err)} — ` +
2628
+ "zero workers cannot be proven while the config is unreadable",
2629
+ );
2630
+ }
2631
+ }
2632
+ // A daemon serves projects from its own memory, not from the on-disk
2633
+ // config: a project the config no longer names (or never named — a neighbour
2634
+ // this host is mid-migrating to) still owns live runs the daemon admits and
2635
+ // sweeps. If the daemon's recorded project is not among the configured
2636
+ // ones, the host-global scope must still cover it, or its workers are
2637
+ // silently skipped while setup mutates underneath them (#651 review #3). An
2638
+ // identity that cannot be proven throws here — before any freeze — so the
2639
+ // scope never reads "zero" for a fleet it cannot see.
2640
+ const daemon = drain.daemonIdentity();
2641
+ if (daemon.running && daemon.project !== undefined && !configured.includes(daemon.project)) {
2642
+ configured = [...configured, daemon.project];
2643
+ }
2644
+ return {
2645
+ selectors: configured.length === 0 ? [undefined] : configured,
2646
+ pauseKey: undefined,
2647
+ };
2648
+ }
2649
+
2650
+ /** Live worker runs across every project a host-wide transaction covers.
2651
+ * Throws — refusing — when the fleet cannot be proven quiescent: a missing
2652
+ * config does not prove zero workers while a daemon runs from its own
2653
+ * in-memory config and still owns runs (review #3), and an unreadable store
2654
+ * is a fault, never absence. */
2655
+ function liveWorkersInScope(drain: DrainDeps, scope: UpgradeScope): number {
2656
+ if (!existsSync(configPath()) && drain.daemonIdentity().running) {
2657
+ throw new Error(
2658
+ "config.json is absent while a daemon is running, so zero workers cannot be proven; " +
2659
+ "the daemon serves projects from memory",
2660
+ );
2661
+ }
2662
+ return scope.selectors.reduce((n, s) => n + drain.snapshot(s).liveWorkers, 0);
2663
+ }
2664
+
2665
+ /** How long the barrier waits — after one best-effort daemon wake — for the
2666
+ * running daemon to acknowledge the setup fence before refusing with nothing
2667
+ * written. Bounded and fail-closed: an unacknowledged fence is an unproven
2668
+ * one, and the answer is a re-run, never an unproven proceed. */
2669
+ const FENCE_ACK_WAIT_MS = 60_000;
2670
+ const FENCE_ACK_POLL_MS = 250;
2671
+
2672
+ /**
2673
+ * Why the daemon's admission acknowledgement no longer covers the fence the
2674
+ * barrier froze, or `undefined` when it does: the acknowledgement must name
2675
+ * the exact pause instance the barrier proved (not some other pause, however
2676
+ * similar) and the exact daemon generation the barrier began with (not an
2677
+ * older or newer instance's word) (#651 review #3).
2678
+ */
2679
+ function fenceAckProblem(ack: AdmissionAckRecord | undefined, begun: RestartBegun): string | undefined {
2680
+ if (ack === undefined) return "the running daemon has not acknowledged the setup admission fence";
2681
+ if (
2682
+ ack.pause.source !== begun.pauseToken.source ||
2683
+ ack.pause.since !== begun.pauseToken.since ||
2684
+ ack.pause.reason !== begun.pauseToken.reason
2685
+ ) {
2686
+ return "the running daemon acknowledged a different pause than the setup admission fence";
2687
+ }
2688
+ if (ack.daemon !== begun.daemon.generation) {
2689
+ return "the running daemon's acknowledgement belongs to a different daemon generation";
2690
+ }
2691
+ return undefined;
2692
+ }
2693
+
2694
+ /**
2695
+ * Why the acknowledged barrier no longer holds, or `undefined` when it does:
2696
+ * a live worker anywhere in scope, the daemon-side admission acknowledgement
2697
+ * no longer covering the frozen fence (when one is required), or the fence
2698
+ * its own pause/daemon-generation check (the reused lifecycle seam,
2699
+ * {@link restartFenceProblem}) says was lifted or replaced. The acknowledged
2700
+ * quiescence barrier re-runs this after every async boundary it must not
2701
+ * proceed across (#651 review #6).
2702
+ */
2703
+ function barrierDispute(
2704
+ drain: DrainDeps,
2705
+ scope: UpgradeScope,
2706
+ begun: RestartBegun,
2707
+ requireAck: boolean,
2708
+ ): string | undefined {
2709
+ try {
2710
+ const live = liveWorkersInScope(drain, scope);
2711
+ if (live > 0) return `${live} live worker(s) appeared under the setup admission freeze`;
2712
+ if (requireAck && begun.daemon.running) {
2713
+ const problem = fenceAckProblem(drain.admissionAck?.(scope.pauseKey), begun);
2714
+ if (problem !== undefined) return problem;
2715
+ }
2716
+ // The fence's own pause/daemon-generation proof (the reused lifecycle
2717
+ // seam) — wrapped with the worker count so a throwing accessor (an
2718
+ // unknown systemd probe in the identity, say) is a dispute to refuse on,
2719
+ // never an exception escaping with the freeze still written.
2720
+ return restartFenceProblem(drain, scope, begun, "setup");
2721
+ } catch (err) {
2722
+ return err instanceof Error ? err.message : String(err);
2723
+ }
2724
+ }
2725
+
1976
2726
  /**
1977
2727
  * The post-apply code-graph offer. Shared by the tail and the live-worker early
1978
2728
  * return, so neither path can silently leave the graph install unreferenced.
@@ -1990,6 +2740,7 @@ async function offerCodeGraph(
1990
2740
  "Clones each index-only checkout as you, installs and enables the reindex timer as root, then seeds one " +
1991
2741
  "indexing run so the first fetch happens while you watch. Minutes per repo. " +
1992
2742
  "`omp-conductor setup graph` does the same later; `--no-seed` skips the seeding run.",
2743
+ { key: `offer-code-graph.${project.name}` },
1993
2744
  );
1994
2745
  if (wantsGraph === true) await graphInstall(project, ui);
1995
2746
  else
@@ -2025,7 +2776,7 @@ export async function setup(
2025
2776
  }
2026
2777
  // `answers` is mutable below: the review loop re-asks one area at a time and
2027
2778
  // replaces only the fields that area owns, so everything else carries across.
2028
- const { amend, added } = collected;
2779
+ const { amend } = collected;
2029
2780
  let answers = collected.answers;
2030
2781
 
2031
2782
  const scopes = await apply.scopes();
@@ -2176,7 +2927,12 @@ export async function setup(
2176
2927
  : "",
2177
2928
  d.project.escalation.orchestrator === "external"
2178
2929
  ? "Dispatch stays paused until the existing arm marker or a new inbound Telegram proof makes the heartbeat live."
2179
- : "Dispatch resumes after the smoke succeeds.",
2930
+ : "Dispatch resumes after the smoke succeeds and the daemon is proven to serve the config it was just written.",
2931
+ // The acknowledged quiescence barrier (#618): the whole sequence refuses
2932
+ // before the first write while any configured project has a live worker
2933
+ // and until the running daemon has acknowledged the admission fence, so
2934
+ // the consent names the gate the apply runs behind.
2935
+ "Refuses before any write while any configured project has a live worker, and until the running daemon acknowledges the admission fence.",
2180
2936
  "Issues are only claimed after every setup gate succeeds.",
2181
2937
  ]
2182
2938
  .filter((s) => s.length > 0)
@@ -2202,7 +2958,7 @@ export async function setup(
2202
2958
  amend === undefined ? "Apply this setup?" : `Apply this change to ${AMEND_AREAS[amend.area].name}?`,
2203
2959
  REVIEW_CHOICES,
2204
2960
  // Bare Enter lands on Review — the one row with no effect.
2205
- { initialIndex: 2 },
2961
+ { key: "setup-review", initialIndex: 2 },
2206
2962
  );
2207
2963
  if (choice === undefined) {
2208
2964
  ui.notify("Setup cancelled — nothing was changed, and the answers were discarded.", "info");
@@ -2229,7 +2985,7 @@ export async function setup(
2229
2985
  const pickedArea = await ui.select(
2230
2986
  "Edit which area?",
2231
2987
  INTERVIEW_AREAS.map((area) => ({ label: area.label })),
2232
- { initialIndex: 0 },
2988
+ { key: "setup-edit-area", initialIndex: 0 },
2233
2989
  );
2234
2990
  if (pickedArea === undefined) {
2235
2991
  ui.notify("Editing cancelled — the answers stand as they were.", "info");
@@ -2265,87 +3021,525 @@ export async function setup(
2265
3021
  ui.notify(planBlock(plan), "info");
2266
3022
  }
2267
3023
 
2268
- // Hold first. Any later filesystem, tracker, smoke, or channel error leaves a
2269
- // partially applied setup unable to claim work. Everything below reads the
2270
- // last re-derivation: the loop above can only break here on a final "Apply",
2271
- // so this one-writer sequence runs exactly once per setup run.
2272
- prepareConductor(plan.project.name);
2273
- const created = await apply.createLabels(answers.trackerRepo, plan.labels);
2274
- saveConfig(plan.nextConfig);
2275
- const briefPath = answers.writeOrchestratorBrief ? writeOrchestratorBrief(answers, prose) : undefined;
2276
- const runtimeFiles = writeHostRuntime(plan.runtime);
2277
- for (const warning of runtimeFiles.warnings) ui.notify(warning, "warning");
2278
- const smoke = await apply.smoke(plan.project.name);
2279
- let smokeLine =
2280
- `paused daemon --once; temporary /healthz on :${smoke.daemon.port}; ` +
2281
- `stored status for ${smoke.status.project}`;
3024
+ // ------------------------------------------------------------------ #618 --
3025
+ // The acknowledged quiescence barrier. A setup apply replaces config and
3026
+ // runs a paused-daemon smoke that is itself a dispatcher (it settles rows,
3027
+ // projects labels, salvages workers), and every write below happens
3028
+ // underneath a daemon that may still admit or own workers. The fail-closed
3029
+ // envelope, reusing the same lifecycle/drain seam `upgrade`, `restart` and
3030
+ // `setup host` run on rather than a second counter-only fence: refuse before
3031
+ // the first mutation when any configured project has a live worker; freeze
3032
+ // admission host-wide — through the one global sentinel, never a project's —
3033
+ // and prove the freeze is in force (#552); then acknowledge — re-check every
3034
+ // project in scope under the proven freeze, and prove the fence still holds.
3035
+ // A pass admitted between the first zero-worker observation and that
3036
+ // acknowledgement refuses with the exact pre-entry pause restored and
3037
+ // nothing else written. After the acknowledgement, every awaited mutation/
3038
+ // lifecycle step re-proves the fence and the zero-worker scope before it
3039
+ // acts (a concurrent `resume` during the awaited label I/O is the shape that
3040
+ // used to slip through and orphan an admitted run).
3041
+ //
3042
+ // The pre-acknowledgement refusal paths are all pre-write, so "nothing has
3043
+ // been changed" is literal: no label, no config byte, no brief, no runtime
3044
+ // file, no store write, no smoke and no service mutation. The observation
3045
+ // that gates the barrier is read-only — it can never be the thing that
3046
+ // creates or upgrades the store, because database preparation is itself one
3047
+ // of the mutations the barrier precedes.
3048
+
3049
+ // The exact pre-entry host-global pause sentinel (owner token and timestamp
3050
+ // byte for byte), captured before anything mutates so a refusal — or the end
3051
+ // of a successful apply — restores it verbatim. Setup freezes the whole host
3052
+ // through the one global sentinel, so that is the only path it may change
3053
+ // (review #4); an unreadable pre-existing sentinel fails closed here, before
3054
+ // any change, rather than being read as absence and deleted (review #5).
3055
+ let scope: UpgradeScope;
3056
+ try {
3057
+ scope = setupScope(apply.drain);
3058
+ } catch (err) {
3059
+ ui.notify(
3060
+ `Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
3061
+ "Nothing has been changed; fix the config and re-run setup.",
3062
+ "error",
3063
+ );
3064
+ return false;
3065
+ }
3066
+ const priorPause = capturePause();
3067
+ if (priorPause.kind === "unreadable") {
3068
+ ui.notify(
3069
+ `Setup stopped before writing anything: the pause sentinel at ${priorPause.path} exists but ` +
3070
+ "could not be read — refusing rather than risk overwriting an operator hold. Nothing has been changed.",
3071
+ "error",
3072
+ );
3073
+ return false;
3074
+ }
3075
+ // A malformed sentinel is unreadable-as-state: readable bytes that parse as
3076
+ // no pause instance are still an operator's hold, and rebuilding them as a
3077
+ // fresh setup pause would lose the owner identity the fence preserves. Fail
3078
+ // closed before any change and never delete the file (#650).
3079
+ if (priorPause.kind === "malformed") {
3080
+ ui.notify(
3081
+ `Setup stopped before writing anything: the pause sentinel at ${priorPause.path} exists but is ` +
3082
+ "malformed (not a readable pause instance) — refusing rather than rebuild an operator hold setup " +
3083
+ "cannot prove. Nothing has been changed.",
3084
+ "error",
3085
+ );
3086
+ return false;
3087
+ }
3088
+
3089
+ // First refusal — before ANY durable mutation, the freeze included. A live
3090
+ // worker in any project the daemon serves would be orphaned, salvaged and
3091
+ // requeued by the config write and the paused-daemon smoke that follow. The
3092
+ // observation is read-only (it never creates or upgrades the store) and
3093
+ // fails closed when quiescence cannot be proven — including when the scope
3094
+ // itself cannot be enumerated (an unreadable config is a fault, never bare
3095
+ // scope, review #2) or the daemon identity cannot be proven (an unknown
3096
+ // systemd probe, review #2).
3097
+ let entryWorkers: number;
3098
+ try {
3099
+ entryWorkers = liveWorkersInScope(apply.drain, scope);
3100
+ } catch (err) {
3101
+ ui.notify(
3102
+ `Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
3103
+ "Nothing has been changed; re-run setup once the fleet is provably quiet.",
3104
+ "error",
3105
+ );
3106
+ return false;
3107
+ }
3108
+ if (entryWorkers > 0) {
3109
+ ui.notify(
3110
+ [
3111
+ `Setup stopped before writing anything: ${entryWorkers} live worker(s) run under the active daemon.`,
3112
+ "A setup apply replaces config and runs a paused-daemon smoke that would orphan, salvage and requeue those runs.",
3113
+ "Let the workers finish, then re-run setup — nothing has been changed.",
3114
+ ].join("\n"),
3115
+ "error",
3116
+ );
3117
+ return false;
3118
+ }
3119
+
3120
+ // Freeze admission host-wide, then prove the freeze is in force before
3121
+ // anything is re-checked (#552): a pause that cannot be read back as an
3122
+ // instance is a fence that cannot be acknowledged, so the barrier restores
3123
+ // the exact entry pause and refuses instead of proceeding unproven.
3124
+ const initialPaused = apply.drain.layers(scope.pauseKey).paused;
3125
+ if (!initialPaused) apply.drain.setPaused(true, scope.pauseKey);
3126
+ // The sentinel THIS transaction now owns — the bytes it just wrote, or the
3127
+ // pre-entry hold it inherited. Every later restore only touches the disk
3128
+ // while it still matches these exact bytes, so a takeover (a resume, a
3129
+ // newer hold) is preserved rather than overwritten (#651 review #3).
3130
+ const ownedFence = readSentinel(pausedPath(scope.pauseKey));
3131
+ const pauseToken = apply.drain.pauseState(scope.pauseKey);
3132
+ if (pauseToken === undefined) {
3133
+ restorePause(priorPause, ownedFence);
3134
+ ui.notify(
3135
+ "Setup stopped before writing anything: the setup freeze could not be proven and was released. " +
3136
+ "Nothing has been changed; re-run setup once dispatch is quiet.",
3137
+ "error",
3138
+ );
3139
+ return false;
3140
+ }
3141
+
3142
+ // Acknowledge the barrier: every project in scope is re-checked under the
3143
+ // proven freeze, and the fence must still hold — the pause token we proved
3144
+ // and the daemon generation we began with. A worker admitted after the
3145
+ // first zero-worker observation but before this acknowledgement (a dispatch
3146
+ // pass that passed its own pause gate before the freeze landed), a `resume`
3147
+ // that lifted the freeze, or an identity that can no longer be proven (an
3148
+ // unknown systemd probe) refuses here with the exact pre-entry pause
3149
+ // restored and nothing else written.
3150
+ let begun: RestartBegun;
3151
+ try {
3152
+ begun = { pauseToken, daemon: apply.drain.daemonIdentity() };
3153
+ } catch (err) {
3154
+ restorePause(priorPause, ownedFence);
3155
+ ui.notify(
3156
+ `Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
3157
+ "The setup freeze was released; nothing has been changed; re-run setup once the fleet is provably quiet.",
3158
+ "error",
3159
+ );
3160
+ return false;
3161
+ }
3162
+
3163
+ // The daemon-side admission acknowledgement (#651 review #3): the fence is
3164
+ // acknowledged only when the daemon ITSELF has observed it at an admission
3165
+ // boundary and written that observation down — a second synchronous worker
3166
+ // count is not an acknowledgement, because a tick already past its own
3167
+ // pause gate can claim after it and before setup mutates. A running daemon
3168
+ // that has not yet acknowledged is woken to prompt an immediate pass; one
3169
+ // that cannot be reached (its record is missing) or that still has not
3170
+ // acknowledged by the bounded deadline refuses before anything is written.
3171
+ let requireAck = true;
3172
+ if (begun.daemon.running) {
3173
+ let ack = apply.drain.admissionAck?.(scope.pauseKey);
3174
+ let acknowledged = fenceAckProblem(ack, begun) === undefined;
3175
+ if (!acknowledged) {
3176
+ const reachable = livingDaemon();
3177
+ if (reachable !== undefined) {
3178
+ ui.notify(
3179
+ "The running daemon has not yet acknowledged the setup admission fence — waking it to prompt a pass.",
3180
+ "info",
3181
+ );
3182
+ void wakeDaemon(reachable.port);
3183
+ // `OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS` is the #399 test seam: a
3184
+ // regression proving this refusal burns the short test deadline
3185
+ // instead of the production 60s.
3186
+ const deadline =
3187
+ Date.now() + Number(process.env["OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS"] ?? FENCE_ACK_WAIT_MS);
3188
+ while (!acknowledged && Date.now() < deadline) {
3189
+ await apply.drain.sleep(FENCE_ACK_POLL_MS);
3190
+ ack = apply.drain.admissionAck?.(scope.pauseKey);
3191
+ acknowledged = fenceAckProblem(ack, begun) === undefined;
3192
+ }
3193
+ }
3194
+ }
3195
+ if (!acknowledged) {
3196
+ restorePause(priorPause, ownedFence);
3197
+ ui.notify(
3198
+ "Setup stopped before writing anything: the running daemon has not acknowledged the setup admission " +
3199
+ "fence, so quiescence cannot be proven. Nothing has been changed; stop the daemon, or re-run setup " +
3200
+ "once it has acknowledged the fence.",
3201
+ "error",
3202
+ );
3203
+ return false;
3204
+ }
3205
+ }
3206
+ const ackDispute = barrierDispute(apply.drain, scope, begun, requireAck);
3207
+ if (ackDispute !== undefined) {
3208
+ restorePause(priorPause, ownedFence);
3209
+ ui.notify(
3210
+ `Setup stopped before writing anything: ${ackDispute}. ` +
3211
+ "Nothing has been changed; re-run setup once dispatch is quiet.",
3212
+ "error",
3213
+ );
3214
+ return false;
3215
+ }
3216
+
3217
+ // The re-proof every later awaited-mutation/lifecycle step stands on: the
3218
+ // acknowledged fence must still hold (pause owned, acknowledgement current,
3219
+ // daemon generation unchanged) and no worker may have appeared. A concurrent
3220
+ // `resume` while label creation is pending can reopen admission, after which
3221
+ // config/runtime/smoke would proceed with no further token or worker check
3222
+ // and orphan the admitted run (review #6) — so every one of them re-runs
3223
+ // `barrierDispute` and restores the exact entry pause on a dispute.
3224
+ const barrierHolds = (needAck: boolean): string | undefined =>
3225
+ barrierDispute(apply.drain, scope, begun, needAck);
3226
+
3227
+ // The after-acknowledgement crosscheck: a claim that lands after the
3228
+ // acknowledgement and before the first mutation (a daemon tick that passed
3229
+ // its own pause gate before the freeze and reaches its claim now) must
3230
+ // refuse here with NOTHING written — this read is the one that makes the
3231
+ // `[0, 0, 1]` interleaving a pre-write refusal, not a post-store one
3232
+ // (#651 review #3). The acknowledgement itself cannot see it: the ack is
3233
+ // the daemon's word, and this synchronous crosscheck is the last moment
3234
+ // the envelope can stay literal about "nothing has been changed".
3235
+ const preMutationReproof = barrierHolds(requireAck);
3236
+ if (preMutationReproof !== undefined) {
3237
+ restorePause(priorPause, ownedFence);
3238
+ ui.notify(
3239
+ `Setup stopped before writing anything: ${preMutationReproof}. ` +
3240
+ "Nothing has been changed; re-run setup once dispatch is quiet.",
3241
+ "error",
3242
+ );
3243
+ return false;
3244
+ }
3245
+
3246
+ // The one-writer transaction — runs exactly once per setup run, only after
3247
+ // the loop above broke on a final "Apply", and only under the acknowledged
3248
+ // barrier held above. Database preparation is the first mutation, in the
3249
+ // order the envelope promises: barrier first, then store, config,
3250
+ // brief/runtime files, smoke, service activation, tracker labels, the
3251
+ // heartbeat proof last — immediately before the commit. The labels sit
3252
+ // BEFORE the heartbeat proof so the deliberate arm-failure hold keeps the
3253
+ // tracker's planned labels while it holds the fleet (#650, PR #700 review
3254
+ // #3); every rollback path after them compensates exactly through the
3255
+ // inventory, and every failure path before them has created none (#652). A
3256
+ // throw anywhere in this leg, or a re-proof dispute after the first
3257
+ // mutation, rolls the WHOLE inventory back as one unit; the pre-write
3258
+ // refusal paths above are untouched.
3259
+ let created: string[] = [];
3260
+ let briefPath: string | undefined;
3261
+ let runtimeFiles: HostRuntimeWrite = { wrote: [], warnings: [] };
3262
+ let smoke: SetupSmokeResult | undefined;
3263
+ let smokeLine = "";
2282
3264
  let restartVia: "systemctl" | "cli" | undefined;
2283
- if (smoke.mode === "existing") {
2284
- if (smoke.status.liveWorkers > 0) {
2285
- // Runs are live: never yank the daemon. Print the reload command; the new
2286
- // project only joins the multi-tenant loop after restart (#319).
3265
+ let armLine = "embedded orchestrator — no heartbeat arm marker";
3266
+
3267
+ // The complete pre-entry mutation inventory (#652): every path the apply
3268
+ // can write config, briefs, staged units/scripts/tick, the AGENTS link,
3269
+ // the run store, the daemon record, the arm marker, the config backups —
3270
+ // with byte-exact pre-entry state, read BEFORE the first mutation so a
3271
+ // failure anywhere in the write/smoke/activation leg can restore all of
3272
+ // them. An unreadable pre-entry path refuses here: nothing has been changed.
3273
+ let inventory: SetupInventory;
3274
+ try {
3275
+ inventory = captureInventory(plan, answers, existing?.projects.map((p) => p.name) ?? []);
3276
+ } catch (err) {
3277
+ // The freeze was already written by this point, so it is released with
3278
+ // the refusal — an unreadable pre-entry path must not leave a setup-owned
3279
+ // sentinel wedging dispatch for every project (review #2).
3280
+ restorePause(priorPause, ownedFence);
3281
+ ui.notify(
3282
+ `Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
3283
+ "The setup freeze was released; nothing has been changed; re-run setup once the fleet is provably quiet.",
3284
+ "error",
3285
+ );
3286
+ return false;
3287
+ }
3288
+ // Whether a daemon was live at entry, captured before the first mutation: a
3289
+ // failed restart leg can stop that daemon without proving a replacement, so
3290
+ // only a run that entered with one must prove the runtime back before
3291
+ // dispatch may resume (#652). The capture lives INSIDE the transaction try
3292
+ // below so its refusal path is covered by the same `finally` that removes
3293
+ // the store snapshot — the byte-consistent capture the inventory just made
3294
+ // must never leak a stray temp copy in the state dir when the identity
3295
+ // cannot be proven (#650, PR #700 review #2).
3296
+ let hadLiveDaemonAtEntry = false;
3297
+ try {
3298
+ try {
3299
+ hadLiveDaemonAtEntry = apply.drain.daemonIdentity().running;
3300
+ } catch (err) {
3301
+ // The freeze was already written by this point, so it is released with
3302
+ // the refusal — an identity that cannot be proven must not leave a
3303
+ // setup-owned sentinel wedging dispatch for every project (review #2).
3304
+ restorePause(priorPause, ownedFence);
2287
3305
  ui.notify(
2288
- [
2289
- `Setup files are updated, but ${smoke.status.liveWorkers} live worker(s) still use the old daemon config.`,
2290
- "Dispatch remains paused. Let those workers finish.",
2291
- "Then run `omp-conductor restart --now`.",
2292
- plan.project.escalation.orchestrator === "external"
2293
- ? `Run \`omp-conductor arm --project ${plan.project.name}\` if ticks are disarmed, then run \`omp-conductor resume --project ${plan.project.name}\`.`
2294
- : `Then run \`omp-conductor resume --project ${plan.project.name}\`.`,
2295
- ].join("\n"),
2296
- "warning",
3306
+ `Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
3307
+ "The setup freeze was released; nothing has been changed; re-run setup once the fleet is provably quiet.",
3308
+ "error",
2297
3309
  );
2298
- // The early return must not swallow the graph offer. This host is the
2299
- // live reproduction: the shared reindex units were already installed by
2300
- // the first project, so only the clone+seed are outstanding — and nothing
2301
- // above names them. Offer regardless of live workers: cloning seeds and
2302
- // enables the timer, which does not touch the running daemon.
2303
- await offerCodeGraph(plan.project, ui, apply.graphInstall);
2304
- ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
2305
3310
  return false;
2306
3311
  }
2307
- // No live workers: still prefer an explicit reload when this run *added* a
2308
- // neighbour — auto-restart would bounce every other project's heartbeat for
2309
- // a config change they did not ask for. Amends of the only/same project keep
2310
- // the old auto-restart so a first-time install still comes up alone.
2311
- if (added === true) {
3312
+ openStore(dbPath()).close();
3313
+ saveConfig(plan.nextConfig);
3314
+ briefPath = answers.writeOrchestratorBrief ? writeOrchestratorBrief(answers, prose) : undefined;
3315
+ runtimeFiles = writeHostRuntime(plan.runtime);
3316
+ for (const warning of runtimeFiles.warnings) ui.notify(warning, "warning");
3317
+
3318
+ // Re-prove before the smoke: the paused-daemon smoke is itself a dispatcher,
3319
+ // so it must never run under a reopened fence or beside a worker admitted
3320
+ // since the acknowledgement (review #6). The store/config/brief/runtime
3321
+ // writes above are rolled back with the whole inventory (review #652).
3322
+ const smokeReproof = barrierHolds(requireAck);
3323
+ if (smokeReproof !== undefined) {
3324
+ const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
3325
+ restorePause(priorPause, ownedFence);
2312
3326
  ui.notify(
2313
- [
2314
- "Setup files are updated. The running daemon does not serve the new project until it reloads.",
2315
- "No live workers safe to reload now:",
2316
- " omp-conductor restart --now",
2317
- ].join("\n"),
2318
- "info",
3327
+ `Setup stopped before the daemon smoke: ${smokeReproof}. ` +
3328
+ (restoreFailures.length === 0
3329
+ ? "The prior state was restored."
3330
+ : `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
3331
+ "error",
2319
3332
  );
2320
- } else {
3333
+ return false;
3334
+ }
3335
+ smoke = await withProgress("Running setup smoke", "Setup smoke ran", () =>
3336
+ apply.smoke(plan.project.name),
3337
+ );
3338
+
3339
+ // Re-prove AFTER the awaited smoke too (review #2): the smoke is itself a
3340
+ // dispatcher, and a fence resumed while it ran — or a worker it admitted —
3341
+ // must stop the restart and heartbeat actions that follow. The restart,
3342
+ // if any, is the very next action, so this single re-proof covers it.
3343
+ const postSmokeReproof = barrierHolds(requireAck);
3344
+ if (postSmokeReproof !== undefined) {
3345
+ const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
3346
+ restorePause(priorPause, ownedFence);
3347
+ ui.notify(
3348
+ `Setup stopped after the daemon smoke: ${postSmokeReproof}. ` +
3349
+ (restoreFailures.length === 0
3350
+ ? "The prior state was restored."
3351
+ : `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
3352
+ "error",
3353
+ );
3354
+ return false;
3355
+ }
3356
+
3357
+ smokeLine =
3358
+ smoke.mode === "temporary"
3359
+ ? `paused daemon --once; temporary /healthz on :${smoke.daemon.port}; ` +
3360
+ `stored status for ${smoke.status.project}`
3361
+ : // An existing daemon (record or active unit) never ran a `--once` tick
3362
+ // and never staged a temporary one — the restart line below replaces
3363
+ // this one whenever the restart runs.
3364
+ `existing daemon; stored status for ${smoke.status.project}`;
3365
+ if (smoke.mode === "existing") {
3366
+ // The barrier acknowledged zero workers in every project under a proven
3367
+ // host-wide freeze before the first write, so a live worker here can only
3368
+ // mean the fence itself was bypassed. The old soft return — applied config
3369
+ // and a wedged setup pause left underneath that worker, with a "let them
3370
+ // finish" note — is the exact incident shape (#618) and is refused instead,
3371
+ // with the whole inventory rolled back (review #2, #652).
3372
+ if (smoke.status.liveWorkers > 0) {
3373
+ const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
3374
+ restorePause(priorPause, ownedFence);
3375
+ ui.notify(
3376
+ `${smoke.status.liveWorkers} live worker(s) appeared under the setup admission freeze — ` +
3377
+ "refusing to apply over a bypassed fence. " +
3378
+ (restoreFailures.length === 0
3379
+ ? "The prior state was restored."
3380
+ : `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
3381
+ "error",
3382
+ );
3383
+ return false;
3384
+ }
3385
+ // The added-project contract (#650): the newly loaded runtime must be
3386
+ // proven to serve the added project BEFORE setup reports success — and
3387
+ // more generally, an "existing" smoke means a daemon is serving the
3388
+ // config that was just replaced, so it must reload the applied config
3389
+ // through the lifecycle seam. The deferral to a manual `restart --now`
3390
+ // an operator may never run left dispatch ready on a daemon that did
3391
+ // not serve the config it was just written, so it is gone: every
3392
+ // existing-mode apply restarts, even when the smoke proved health, and
3393
+ // the seam's `waitForOwnedDaemon` proves MainPID + /healthz + that the
3394
+ // (added) project is served before the apply reports success.
2321
3395
  const restarted = await apply.restart({ project: plan.project.name });
2322
3396
  restartVia = restarted.via;
2323
3397
  smokeLine =
2324
3398
  `existing /healthz and stored status; restarted through ${restarted.via}; ` +
2325
3399
  `new /healthz on :${restarted.record.port}`;
3400
+ // The intentional restart is the generation boundary: from here the
3401
+ // fence protects the daemon we just started, not the one we began with.
3402
+ // Refresh the acknowledged world — pause re-proven, identity re-captured
3403
+ // — and drop the acknowledgement requirement, because a daemon this
3404
+ // apply itself started cannot have been mid-claim when the fence landed
3405
+ // (its first admission boundary is the tick gate, above routing); its
3406
+ // own first gate re-acknowledges on the next pass. A fence that vanished
3407
+ // while the restart ran (a concurrent resume) refuses here instead of
3408
+ // proceeding under an unproven freeze (review #3).
3409
+ const refreshedPause = apply.drain.pauseState(scope.pauseKey);
3410
+ if (refreshedPause === undefined) {
3411
+ const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
3412
+ restorePause(priorPause, ownedFence);
3413
+ ui.notify(
3414
+ "Setup stopped after the daemon restart: the setup pause was lifted while the daemon restarted, so " +
3415
+ "the acknowledged world no longer exists. " +
3416
+ (restoreFailures.length === 0
3417
+ ? "The prior state was restored."
3418
+ : `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
3419
+ "error",
3420
+ );
3421
+ return false;
3422
+ }
3423
+ begun = { pauseToken: refreshedPause, daemon: apply.drain.daemonIdentity() };
3424
+ requireAck = false;
2326
3425
  }
2327
- }
2328
3426
 
2329
- let armLine = "embedded orchestrator no heartbeat arm marker";
2330
- if (plan.project.escalation.orchestrator === "external") {
2331
- ui.notify("Setup smoke passed. Proving the external heartbeat channel…", "info");
2332
- try {
2333
- armLine = await apply.arm(plan.project.name);
2334
- } catch (err) {
3427
+ // The tracker labelsbefore the heartbeat proof, so the deliberate
3428
+ // arm-failure hold (which keeps its writes by design) leaves the tracker
3429
+ // with the planned labels instead of a config whose routing keys and
3430
+ // queue label do not exist yet (#650, PR #700 review #3). Every rollback
3431
+ // path AFTER this point compensates them exactly through the inventory;
3432
+ // every failure path before it has created none.
3433
+ created = await withProgress("Creating tracker labels", "Tracker labels ready", () =>
3434
+ apply.createLabels(answers.trackerRepo, plan.labels),
3435
+ );
3436
+
3437
+ if (plan.project.escalation.orchestrator === "external") {
2335
3438
  ui.notify(
2336
- [
2337
- "Setup files passed the paused daemon smoke, but the fleet remains held.",
2338
- err instanceof Error ? err.message : String(err),
2339
- `Start the external orchestrator in ${plan.project.workspaceRoot}, then run \`omp-conductor arm --project ${plan.project.name}\`.`,
2340
- "After the arm proof succeeds, run `omp-conductor resume`.",
2341
- ].join("\n"),
2342
- "warning",
3439
+ smoke.healthProven
3440
+ ? "Setup smoke passed. Proving the external heartbeat channel…"
3441
+ : "Setup smoke staged — the restart proved /healthz. Proving the external heartbeat channel…",
3442
+ "info",
3443
+ );
3444
+ try {
3445
+ armLine = await apply.arm(plan.project.name);
3446
+ } catch (err) {
3447
+ ui.notify(
3448
+ [
3449
+ "Setup files passed the paused daemon smoke, but the fleet remains held.",
3450
+ created.length > 0 ? `Tracker labels were created (${created.join(", ")}).` : "Planned labels already existed.",
3451
+ err instanceof Error ? err.message : String(err),
3452
+ `Start the external orchestrator in ${plan.project.workspaceRoot}, then run \`omp-conductor arm --project ${plan.project.name}\`.`,
3453
+ "After the arm proof succeeds, run `omp-conductor resume`.",
3454
+ ].join("\n"),
3455
+ "warning",
3456
+ );
3457
+ ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
3458
+ return false;
3459
+ }
3460
+ }
3461
+
3462
+ // The final barrier boundary: the awaited label and heartbeat I/O are the
3463
+ // last steps before the commit, and a fence resumed while either ran — or
3464
+ // a worker admitted — must refuse with the tracker compensated and every
3465
+ // captured path restored.
3466
+ const finalReproof = barrierHolds(requireAck);
3467
+ if (finalReproof !== undefined) {
3468
+ const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
3469
+ restorePause(priorPause, ownedFence);
3470
+ ui.notify(
3471
+ `Setup stopped before committing: ${finalReproof}. ` +
3472
+ (restoreFailures.length === 0
3473
+ ? "The prior state was restored."
3474
+ : `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
3475
+ "error",
2343
3476
  );
2344
- ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
2345
3477
  return false;
2346
3478
  }
3479
+ } catch (err) {
3480
+ // Roll the whole apply back as one transaction (#652): every captured
3481
+ // path restored byte-for-byte (or removed on a first install), the
3482
+ // tracker compensated exactly, and — because a failed restart leg may
3483
+ // have stopped the entry daemon without proving a replacement — a healthy
3484
+ // daemon proven to serve the restored config before dispatch may resume.
3485
+ // If that proof cannot be made, stay fail-closed: a setup hold stays in
3486
+ // force with the recovery command named, and the thrown error reports the
3487
+ // first restoration failure honestly rather than implying the prior state
3488
+ // is coherent.
3489
+ //
3490
+ // The labels created before a mid-way createLabels throw ride on the
3491
+ // error (createMissingLabels attaches its partial list), so compensation
3492
+ // stays exact even when the tracker step itself failed.
3493
+ const createdLabels = created.length > 0 ? created : ((err as { created?: string[] }).created ?? []);
3494
+ const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, createdLabels, apply.deleteLabels);
3495
+ let runtimeProved = !hadLiveDaemonAtEntry;
3496
+ if (hadLiveDaemonAtEntry) {
3497
+ // The proof targets the PRE-ENTRY configured set the rollback restored,
3498
+ // never the rejected generation's: the restarted runtime must be proven
3499
+ // to serve every project the prior runtime served (#650).
3500
+ runtimeProved = await apply.proveRuntime(inventory.priorProjects).catch(() => false);
3501
+ }
3502
+ // The pause ownership rule (#651 review #3): only the fence this
3503
+ // transaction froze is undone; a takeover stands.
3504
+ restorePause(priorPause, ownedFence);
3505
+ const failures = [...restoreFailures];
3506
+ if (!runtimeProved) {
3507
+ // Fail-closed: only an entry that was actually unpaused needs a hold
3508
+ // re-asserted — an entry already held keeps that exact sentinel, and a
3509
+ // takeover that replaced the fence is preserved, never overwritten.
3510
+ // The re-asserted hold is ALWAYS the host-global sentinel, never a
3511
+ // project-scoped one: a fresh `paused-<project>` written under a
3512
+ // failure that entered under a global pause is the silent fake this
3513
+ // slice exists to forbid — when the global hold later clears, the
3514
+ // hidden project pause would remain and deadlock the fleet (#650).
3515
+ if (priorPause.kind === "absent" && readSentinel(pausedPath(scope.pauseKey)).kind === "absent") {
3516
+ setPaused(true, {
3517
+ source: pauseSourceToken("setup"),
3518
+ reason: "setup apply failed and no healthy daemon could be proven",
3519
+ });
3520
+ }
3521
+ failures.push(
3522
+ `no healthy daemon could be proven after the rollback — the fleet stays held; ` +
3523
+ `run \`omp-conductor restart --project ${plan.project.name}\` to recover`,
3524
+ );
3525
+ }
3526
+ if (failures.length > 0) {
3527
+ throw new Error(
3528
+ `setup apply failed (${err instanceof Error ? err.message : String(err)}) and the prior state could not ` +
3529
+ `be fully restored: ${failures.join("; ")}`,
3530
+ );
3531
+ }
3532
+ throw err;
3533
+ } finally {
3534
+ // The byte-consistent store snapshot exists only for this transaction.
3535
+ if (inventory.store.kind === "snapshot") rmSync(inventory.store.snapshotPath!, { force: true });
2347
3536
  }
2348
- apply.resume(plan.project.name);
3537
+ // Lift the setup freeze and restore the exact pre-entry pause: an entry that
3538
+ // was already paused (an operator hold, say) keeps that sentinel byte for
3539
+ // byte, and an entry that was unpaused resumes dispatch exactly as the old
3540
+ // `resume` did. A takeover that replaced the setup fence meanwhile is
3541
+ // preserved, never overwritten (#651 review #3).
3542
+ restorePause(priorPause, ownedFence);
2349
3543
 
2350
3544
  ui.notify(
2351
3545
  [
@@ -2357,7 +3551,7 @@ export async function setup(
2357
3551
  runtimeFiles.wrote.length === 0
2358
3552
  ? "Host runtime files were already current."
2359
3553
  : `Wrote host runtime file(s): ${runtimeFiles.wrote.join(", ")}`,
2360
- `Smoke passed: ${smokeLine}.`,
3554
+ `Smoke ${smoke.healthProven ? "passed" : "staged; the restart proved /healthz"}: ${smokeLine}.`,
2361
3555
  `Heartbeat: ${armLine}.`,
2362
3556
  "",
2363
3557
  "Use the documented toy-issue drill to prove one complete worker path.",
@@ -2392,6 +3586,7 @@ export async function setup(
2392
3586
  `${plan.runtime.installedAction === "create" ? "Installs" : "Updates"} ${plan.runtime.installedPath} from ` +
2393
3587
  `${plan.runtime.service.path}, then enables and restarts it. Needs root, one step at a time, and shows every ` +
2394
3588
  "command before it runs. Skipping is fine — `omp-conductor setup host` does exactly this later.",
3589
+ { key: `install-daemon.${plan.project.name}` },
2395
3590
  );
2396
3591
  if (install === true)
2397
3592
  await apply.hostInstall(