omp-conductor 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/REFERENCE.md +18 -12
- package/package.json +2 -1
- package/schema/config.schema.json +16 -0
- package/src/admission.ts +159 -43
- package/src/availability.ts +27 -1
- package/src/briefs/worker.md +2 -0
- package/src/clack-ui.ts +83 -0
- package/src/command-manifest.ts +16 -7
- package/src/commands/arm.ts +11 -3
- package/src/commands/decision.ts +17 -7
- package/src/commands/doctor.ts +18 -1
- package/src/commands/hold.ts +9 -7
- package/src/commands/ledger.ts +25 -4
- package/src/commands/message.ts +32 -4
- package/src/commands/setup.ts +61 -10
- package/src/commands/stats.ts +9 -5
- package/src/commands/status.ts +32 -5
- package/src/commands/tail.ts +13 -1
- package/src/commands/watch.ts +16 -7
- package/src/config-schema.ts +20 -0
- package/src/config.ts +37 -0
- package/src/daemon.ts +1240 -18
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +56 -13
- package/src/fleet.ts +224 -47
- package/src/gitops.ts +103 -24
- package/src/lifecycle.ts +7 -2
- package/src/orchestrator-tick.ts +372 -157
- package/src/privileged.ts +3 -0
- package/src/release-policy.ts +177 -5
- package/src/setup-answers.ts +135 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +2 -0
- package/src/setup-probe.ts +1 -0
- package/src/setup-wizard.ts +1296 -101
- package/src/setup.ts +60 -3
- package/src/status-render.ts +11 -1
- package/src/store.ts +333 -12
- package/src/tracker/github.ts +562 -13
- package/src/types.ts +204 -2
- package/src/ui/progress.ts +32 -0
- package/src/ui/style.ts +11 -0
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +212 -11
- package/src/wizard-ui.ts +14 -5
- package/src/worker.ts +26 -0
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/setup-wizard.ts
CHANGED
|
@@ -10,37 +10,70 @@
|
|
|
10
10
|
* prompt's shape and default is a decision worth keeping, so none of them were
|
|
11
11
|
* reworded in the move.
|
|
12
12
|
*/
|
|
13
|
-
import {
|
|
13
|
+
import { randomUUID } from "node:crypto";
|
|
14
|
+
import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmdirSync, rmSync, writeFileSync } from "node:fs";
|
|
14
15
|
import { platform } from "node:os";
|
|
15
16
|
import { dirname, isAbsolute, join } from "node:path";
|
|
16
17
|
import {
|
|
18
|
+
configBackupDir,
|
|
17
19
|
configPath,
|
|
18
20
|
expandHome,
|
|
19
21
|
findProject,
|
|
20
22
|
loadConfig,
|
|
21
23
|
resolveCaps,
|
|
22
24
|
saveConfig,
|
|
25
|
+
stateDir,
|
|
26
|
+
writeConfigRaw,
|
|
23
27
|
} from "./config.ts";
|
|
24
28
|
import { claimedTelegramTopics } from "./escalate.ts";
|
|
25
29
|
import { hostRamBytes, recommendedMaxWorkers, workerOvercommit } from "./host.ts";
|
|
26
30
|
import {
|
|
27
|
-
|
|
31
|
+
daemonGeneration,
|
|
32
|
+
isPaused,
|
|
33
|
+
pausedPath,
|
|
34
|
+
pauseInstance,
|
|
35
|
+
pauseSourceToken,
|
|
28
36
|
previewProject,
|
|
37
|
+
readAdmissionAck,
|
|
29
38
|
setPaused,
|
|
39
|
+
wakeDaemon,
|
|
40
|
+
type AdmissionAckRecord,
|
|
30
41
|
type QueuePreview,
|
|
31
42
|
} from "./daemon.ts";
|
|
32
|
-
import { armTicks, telegramStateDir } from "./fleet.ts";
|
|
33
|
-
import {
|
|
43
|
+
import { armedMarkerPath, armTicks, fleetLayers, telegramStateDir } from "./fleet.ts";
|
|
44
|
+
import {
|
|
45
|
+
healthCheck,
|
|
46
|
+
healthServesProject,
|
|
47
|
+
livingDaemon,
|
|
48
|
+
probeUnit,
|
|
49
|
+
recordPath,
|
|
50
|
+
restartDaemon,
|
|
51
|
+
SYSTEMD_UNIT,
|
|
52
|
+
type RestartResult,
|
|
53
|
+
} from "./lifecycle.ts";
|
|
54
|
+
import { dbPath, liveWorkersReadOnly, openStore, vacuumInto } from "./store.ts";
|
|
55
|
+
import {
|
|
56
|
+
restartFenceProblem,
|
|
57
|
+
type DaemonIdentity,
|
|
58
|
+
type DrainDeps,
|
|
59
|
+
type RestartBegun,
|
|
60
|
+
type UpgradeScope,
|
|
61
|
+
} from "./upgrade.ts";
|
|
34
62
|
import { defaultGraphRoot, graphRepos } from "./graph.ts";
|
|
35
63
|
import {
|
|
64
|
+
capturePathState,
|
|
36
65
|
formatHostRuntimePlan,
|
|
37
66
|
planHostRuntime,
|
|
38
67
|
totalConfiguredWorkers,
|
|
68
|
+
restorePathState,
|
|
39
69
|
runSetupSmoke,
|
|
40
70
|
SYSTEMD_UNIT_DIR,
|
|
41
71
|
tickCwdForProject,
|
|
42
72
|
writeHostRuntime,
|
|
73
|
+
type CapturedPathState,
|
|
43
74
|
type HostRuntimePlan,
|
|
75
|
+
type HostRuntimeWrite,
|
|
76
|
+
type RestorePathOptions,
|
|
44
77
|
type SetupSmokeResult,
|
|
45
78
|
} from "./setup-host.ts";
|
|
46
79
|
import { runGraphInstall, runHostInstall, type GraphInstallOptions, type InstallOutcome } from "./setup-install.ts";
|
|
@@ -52,6 +85,7 @@ import {
|
|
|
52
85
|
} from "./setup-discover.ts";
|
|
53
86
|
import {
|
|
54
87
|
AMEND_AREAS,
|
|
88
|
+
ARM_PROOF_CHOICES,
|
|
55
89
|
BASE_FRESHNESS_CHOICES,
|
|
56
90
|
BEHIND_BASE_CHOICES,
|
|
57
91
|
DRAFT_POLICY_CHOICES,
|
|
@@ -68,6 +102,7 @@ import {
|
|
|
68
102
|
checkTokenScopes,
|
|
69
103
|
createMissingLabels,
|
|
70
104
|
defaultAnswers,
|
|
105
|
+
deleteCreatedLabels,
|
|
71
106
|
detectTelegram,
|
|
72
107
|
formatGates,
|
|
73
108
|
orchestratorBriefPath,
|
|
@@ -86,6 +121,7 @@ import {
|
|
|
86
121
|
type ProbedProse,
|
|
87
122
|
} from "./setup.ts";
|
|
88
123
|
import {
|
|
124
|
+
ARM_PROOFS,
|
|
89
125
|
BASE_FRESHNESS,
|
|
90
126
|
BEHIND_BASE_ACTIONS,
|
|
91
127
|
DEFAULT_CAPS,
|
|
@@ -107,6 +143,7 @@ import {
|
|
|
107
143
|
type ReportScopeChoice,
|
|
108
144
|
type ResolvedGrants,
|
|
109
145
|
} from "./types.ts";
|
|
146
|
+
import { withProgress } from "./ui/progress.ts";
|
|
110
147
|
import type { WizardUi } from "./wizard-ui.ts";
|
|
111
148
|
|
|
112
149
|
/**
|
|
@@ -132,8 +169,8 @@ const REPO_RE = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/;
|
|
|
132
169
|
* takes it, because the harness has no pre-filled input dialog — so "Enter
|
|
133
170
|
* accepts what you see" is the contract the whole wizard is built on.
|
|
134
171
|
*/
|
|
135
|
-
async function ask(ui: WizardUi, title: string, fallback: string): Promise<string> {
|
|
136
|
-
const raw = await ui.input(title, fallback.length > 0 ? fallback : undefined);
|
|
172
|
+
async function ask(ui: WizardUi, key: string, title: string, fallback: string): Promise<string> {
|
|
173
|
+
const raw = await ui.input(title, fallback.length > 0 ? fallback : undefined, { key });
|
|
137
174
|
if (raw === undefined) throw new Cancelled();
|
|
138
175
|
const trimmed = raw.trim();
|
|
139
176
|
return trimmed.length > 0 ? trimmed : fallback;
|
|
@@ -147,8 +184,8 @@ async function ask(ui: WizardUi, title: string, fallback: string): Promise<strin
|
|
|
147
184
|
* recorded a silent no and walked on to the next question, which is not what
|
|
148
185
|
* abandoning a run means. `undefined` is the surface saying the operator left.
|
|
149
186
|
*/
|
|
150
|
-
async function askYesNo(ui: WizardUi, title: string, message: string): Promise<boolean> {
|
|
151
|
-
const answer = await ui.confirm(title, message);
|
|
187
|
+
async function askYesNo(ui: WizardUi, key: string, title: string, message: string): Promise<boolean> {
|
|
188
|
+
const answer = await ui.confirm(title, message, { key });
|
|
152
189
|
if (answer === undefined) throw new Cancelled();
|
|
153
190
|
return answer;
|
|
154
191
|
}
|
|
@@ -160,12 +197,13 @@ async function askYesNo(ui: WizardUi, title: string, message: string): Promise<b
|
|
|
160
197
|
*/
|
|
161
198
|
async function askValid(
|
|
162
199
|
ui: WizardUi,
|
|
200
|
+
key: string,
|
|
163
201
|
title: string,
|
|
164
202
|
fallback: string,
|
|
165
203
|
check: (value: string) => string | undefined,
|
|
166
204
|
): Promise<string> {
|
|
167
205
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
168
|
-
const value = await ask(ui, title, fallback);
|
|
206
|
+
const value = await ask(ui, key, title, fallback);
|
|
169
207
|
const problem = check(value);
|
|
170
208
|
if (problem === undefined) return value;
|
|
171
209
|
ui.notify(problem, "warning");
|
|
@@ -176,8 +214,8 @@ async function askValid(
|
|
|
176
214
|
|
|
177
215
|
/** A cap. Unparseable input keeps the current value rather than writing a NaN
|
|
178
216
|
* the validator would later reject — the operator sees why, immediately. */
|
|
179
|
-
async function askNumber(ui: WizardUi, title: string, fallback: number): Promise<number> {
|
|
180
|
-
const raw = await ask(ui, title, String(fallback));
|
|
217
|
+
async function askNumber(ui: WizardUi, key: string, title: string, fallback: number): Promise<number> {
|
|
218
|
+
const raw = await ask(ui, key, title, String(fallback));
|
|
181
219
|
const value = Number(raw);
|
|
182
220
|
if (!Number.isFinite(value) || value < 0) {
|
|
183
221
|
ui.notify(`"${raw}" is not a non-negative number — keeping ${fallback}.`, "warning");
|
|
@@ -193,11 +231,12 @@ async function askNumber(ui: WizardUi, title: string, fallback: number): Promise
|
|
|
193
231
|
*/
|
|
194
232
|
async function askSpendCap(
|
|
195
233
|
ui: WizardUi,
|
|
234
|
+
key: string,
|
|
196
235
|
title: string,
|
|
197
236
|
fallback: number | null,
|
|
198
237
|
): Promise<number | null> {
|
|
199
238
|
const seed = fallback === null ? "" : String(fallback);
|
|
200
|
-
const raw = (await ask(ui, title, seed)).trim().toLowerCase();
|
|
239
|
+
const raw = (await ask(ui, key, title, seed)).trim().toLowerCase();
|
|
201
240
|
if (raw === "" || raw === "none" || raw === "off" || raw === "null") return null;
|
|
202
241
|
const value = Number(raw);
|
|
203
242
|
if (!Number.isFinite(value) || value < 0) {
|
|
@@ -232,6 +271,7 @@ async function askGates(
|
|
|
232
271
|
// found, so it is on screen either way.
|
|
233
272
|
const raw = await ask(
|
|
234
273
|
ui,
|
|
274
|
+
`pre-push-gates.${repoName}`,
|
|
235
275
|
`Pre-push gates for ${repoName} — exactly what CI runs, comma separated`,
|
|
236
276
|
formatGates(seed.length > 0 ? seed : probed),
|
|
237
277
|
);
|
|
@@ -263,6 +303,7 @@ async function askReportScope(ui: WizardUi, current: ReportScopeChoice): Promise
|
|
|
263
303
|
const options = REPORT_SCOPE_CHOICES.map((c) => ({ label: c.label, description: c.description }));
|
|
264
304
|
const at = REPORT_SCOPE_CHOICES.findIndex((c) => c.scope === current);
|
|
265
305
|
const picked = await ui.select("What should the orchestrator report unprompted?", options, {
|
|
306
|
+
key: "report-scope",
|
|
266
307
|
initialIndex: at === -1 ? 0 : at,
|
|
267
308
|
});
|
|
268
309
|
if (picked === undefined) throw new Cancelled();
|
|
@@ -292,6 +333,7 @@ async function askReportScope(ui: WizardUi, current: ReportScopeChoice): Promise
|
|
|
292
333
|
*/
|
|
293
334
|
async function askLiteral<T extends string>(
|
|
294
335
|
ui: WizardUi,
|
|
336
|
+
key: string,
|
|
295
337
|
title: string,
|
|
296
338
|
values: readonly T[],
|
|
297
339
|
described: { readonly [K in T]: string },
|
|
@@ -301,7 +343,7 @@ async function askLiteral<T extends string>(
|
|
|
301
343
|
const picked = await ui.select(
|
|
302
344
|
title,
|
|
303
345
|
values.map((v) => ({ label: v, description: described[v] })),
|
|
304
|
-
{ initialIndex: at === -1 ? 0 : at },
|
|
346
|
+
{ key, initialIndex: at === -1 ? 0 : at },
|
|
305
347
|
);
|
|
306
348
|
if (picked === undefined) throw new Cancelled();
|
|
307
349
|
|
|
@@ -336,8 +378,13 @@ function parseNameList(answer: string): string[] {
|
|
|
336
378
|
|
|
337
379
|
/** Check names, artefacts, environments: open-ended lists this package cannot
|
|
338
380
|
* enumerate, so the only validation is the shape. */
|
|
339
|
-
async function askNameList(
|
|
340
|
-
|
|
381
|
+
async function askNameList(
|
|
382
|
+
ui: WizardUi,
|
|
383
|
+
key: string,
|
|
384
|
+
title: string,
|
|
385
|
+
seed: readonly string[],
|
|
386
|
+
): Promise<string[]> {
|
|
387
|
+
return parseNameList(await ask(ui, key, title, formatNameList(seed)));
|
|
341
388
|
}
|
|
342
389
|
|
|
343
390
|
/**
|
|
@@ -362,6 +409,7 @@ async function askReleaseRequirements(
|
|
|
362
409
|
);
|
|
363
410
|
const answered = await askValid(
|
|
364
411
|
ui,
|
|
412
|
+
"release-requirements",
|
|
365
413
|
`Release — what must have landed first (any of ${accepted}, comma separated, or "${EMPTY_LIST}")`,
|
|
366
414
|
formatNameList(prior),
|
|
367
415
|
(value) => {
|
|
@@ -398,11 +446,13 @@ async function askPolicyPreconditions(
|
|
|
398
446
|
const merge = {
|
|
399
447
|
requiredChecks: await askNameList(
|
|
400
448
|
ui,
|
|
449
|
+
"merge-required-checks",
|
|
401
450
|
`Merge — required checks (comma separated, "${EMPTY_LIST}" = every check the PR reports)`,
|
|
402
451
|
prior.merge.requiredChecks,
|
|
403
452
|
),
|
|
404
453
|
baseFreshness: await askLiteral(
|
|
405
454
|
ui,
|
|
455
|
+
"merge-base-freshness",
|
|
406
456
|
"Merge — must the PR be level with its base?",
|
|
407
457
|
BASE_FRESHNESS,
|
|
408
458
|
BASE_FRESHNESS_CHOICES,
|
|
@@ -410,6 +460,7 @@ async function askPolicyPreconditions(
|
|
|
410
460
|
),
|
|
411
461
|
drafts: await askLiteral(
|
|
412
462
|
ui,
|
|
463
|
+
"merge-drafts",
|
|
413
464
|
"Merge — draft pull requests",
|
|
414
465
|
DRAFT_POLICIES,
|
|
415
466
|
DRAFT_POLICY_CHOICES,
|
|
@@ -417,6 +468,7 @@ async function askPolicyPreconditions(
|
|
|
417
468
|
),
|
|
418
469
|
whenBehindBase: await askLiteral(
|
|
419
470
|
ui,
|
|
471
|
+
"merge-behind-base",
|
|
420
472
|
"Merge — a green PR that fell behind its base",
|
|
421
473
|
BEHIND_BASE_ACTIONS,
|
|
422
474
|
BEHIND_BASE_CHOICES,
|
|
@@ -430,16 +482,19 @@ async function askPolicyPreconditions(
|
|
|
430
482
|
requires: await askReleaseRequirements(ui, prior.release.requires),
|
|
431
483
|
requiredChecks: await askNameList(
|
|
432
484
|
ui,
|
|
485
|
+
"release-required-checks",
|
|
433
486
|
`Release — required checks (comma separated, "${EMPTY_LIST}" = every check the branch reports)`,
|
|
434
487
|
prior.release.requiredChecks,
|
|
435
488
|
),
|
|
436
489
|
artefacts: await askNameList(
|
|
437
490
|
ui,
|
|
491
|
+
"release-artefacts",
|
|
438
492
|
`Release — artefacts this project ships (comma separated, or "${EMPTY_LIST}")`,
|
|
439
493
|
prior.release.artefacts,
|
|
440
494
|
),
|
|
441
495
|
environments: await askNameList(
|
|
442
496
|
ui,
|
|
497
|
+
"release-environments",
|
|
443
498
|
`Release — environments a deploy may target (comma separated, or "${EMPTY_LIST}")`,
|
|
444
499
|
prior.release.environments,
|
|
445
500
|
),
|
|
@@ -487,6 +542,7 @@ async function askAuthority(
|
|
|
487
542
|
): Promise<ProjectConfig["authority"]> {
|
|
488
543
|
const merge = await askYesNo(
|
|
489
544
|
ui,
|
|
545
|
+
"merge-authority",
|
|
490
546
|
"Merge authority",
|
|
491
547
|
"Delegate PR merging to the orchestrator session? It would land green PRs one at a time, each " +
|
|
492
548
|
"re-checked against the base branch first. Default: humans merge" +
|
|
@@ -496,6 +552,7 @@ async function askAuthority(
|
|
|
496
552
|
const unlockCount = unlock.gates.length + unlock.policy.length;
|
|
497
553
|
const release = await askYesNo(
|
|
498
554
|
ui,
|
|
555
|
+
"release-authority",
|
|
499
556
|
"Release authority",
|
|
500
557
|
"Delegate release cutting to the orchestrator session? It would tag, pin and publish by the " +
|
|
501
558
|
"procedure you write into its brief — and its brief forbids cutting one before you have. " +
|
|
@@ -506,6 +563,7 @@ async function askAuthority(
|
|
|
506
563
|
);
|
|
507
564
|
const promotion = await askYesNo(
|
|
508
565
|
ui,
|
|
566
|
+
"promotion-authority",
|
|
509
567
|
"Promotion authority",
|
|
510
568
|
"Delegate promotion to the orchestrator session? Promotion is adding the queue label to an " +
|
|
511
569
|
"issue — the sign-off that lets a worker claim it, and the act that starts spend. " +
|
|
@@ -576,13 +634,14 @@ async function askJudgment(
|
|
|
576
634
|
const picked = await ui.select(
|
|
577
635
|
"Roadmap candidates found in GitHub",
|
|
578
636
|
[...roadmapChoices.map((label) => ({ label })), { label: other }],
|
|
579
|
-
{ initialIndex: 0 },
|
|
637
|
+
{ key: "roadmap-candidate", initialIndex: 0 },
|
|
580
638
|
);
|
|
581
639
|
if (picked === undefined) throw new Cancelled();
|
|
582
640
|
roadmapSeed = picked === other ? "" : picked;
|
|
583
641
|
}
|
|
584
642
|
const roadmap = await ask(
|
|
585
643
|
ui,
|
|
644
|
+
"roadmap",
|
|
586
645
|
"Where does the roadmap live, and what is the current priority?",
|
|
587
646
|
roadmapSeed,
|
|
588
647
|
);
|
|
@@ -622,26 +681,40 @@ async function askJudgment(
|
|
|
622
681
|
// so an operator has something to calibrate against.
|
|
623
682
|
judgment.boundary = await ask(
|
|
624
683
|
ui,
|
|
684
|
+
"release-boundary",
|
|
625
685
|
"Where does the orchestrator's leg END? One sentence " +
|
|
626
686
|
'(a real answer: "at the merged version pin — deploying it is operator territory")',
|
|
627
687
|
prior.boundary ?? "",
|
|
628
688
|
);
|
|
629
689
|
|
|
630
690
|
// All five, because the brief needs each and a missing one is a hole.
|
|
631
|
-
judgment.releaseWhat = await ask(
|
|
691
|
+
judgment.releaseWhat = await ask(
|
|
692
|
+
ui,
|
|
693
|
+
"release-what",
|
|
694
|
+
"Release — WHAT may be released, and from which branch?",
|
|
695
|
+
prior.releaseWhat ?? "",
|
|
696
|
+
);
|
|
632
697
|
judgment.releaseWhen = await ask(
|
|
633
698
|
ui,
|
|
699
|
+
"release-when",
|
|
634
700
|
"Release — WHEN: batched how, after which named checks are green?",
|
|
635
701
|
prior.releaseWhen ?? "",
|
|
636
702
|
);
|
|
637
703
|
judgment.releaseProof = await ask(
|
|
638
704
|
ui,
|
|
705
|
+
"release-proof",
|
|
639
706
|
"Release — WHAT PROOF must be held first (results actually read, not an impression)?",
|
|
640
707
|
prior.releaseProof ?? "",
|
|
641
708
|
);
|
|
642
|
-
judgment.releaseAsk = await ask(
|
|
709
|
+
judgment.releaseAsk = await ask(
|
|
710
|
+
ui,
|
|
711
|
+
"release-ask",
|
|
712
|
+
"Release — what must still be ASKED, every time?",
|
|
713
|
+
prior.releaseAsk ?? "",
|
|
714
|
+
);
|
|
643
715
|
judgment.releaseForbidden = await ask(
|
|
644
716
|
ui,
|
|
717
|
+
"release-forbidden",
|
|
645
718
|
"Release — what stays permanently FORBIDDEN?",
|
|
646
719
|
prior.releaseForbidden ?? "force-push, secrets, production data",
|
|
647
720
|
);
|
|
@@ -652,17 +725,19 @@ async function askJudgment(
|
|
|
652
725
|
// `RELEASE_REQUIREMENTS`, which are mechanical preconditions, not a unit.
|
|
653
726
|
judgment.worthCutting = await ask(
|
|
654
727
|
ui,
|
|
728
|
+
"release-worth-cutting",
|
|
655
729
|
"What is a release worth cutting? (a sprint, an epic's children all closed, N merged issues, urgency)",
|
|
656
730
|
prior.worthCutting ?? "",
|
|
657
731
|
);
|
|
658
732
|
|
|
659
|
-
judgment.rollbackOwner = await ask(ui, "Who owns the rollback?", prior.rollbackOwner ?? "");
|
|
733
|
+
judgment.rollbackOwner = await ask(ui, "rollback-owner", "Who owns the rollback?", prior.rollbackOwner ?? "");
|
|
660
734
|
if (namesAPerson(judgment.rollbackOwner)) {
|
|
661
735
|
// Honour the consequence rather than recording a contradiction: if a person
|
|
662
736
|
// rolls it back, that person owns the release, and the boundary belongs
|
|
663
737
|
// before the irreversible step whatever the authority answer sounded like.
|
|
664
738
|
judgment.rollbackMovesBoundary = await askYesNo(
|
|
665
739
|
ui,
|
|
740
|
+
"rollback-moves-boundary",
|
|
666
741
|
"Rollback owner is a person",
|
|
667
742
|
`You named "${judgment.rollbackOwner}" as the rollback owner. That person already owns the release, ` +
|
|
668
743
|
"so the honest configuration puts the orchestrator's boundary *before* the irreversible step — " +
|
|
@@ -671,6 +746,7 @@ async function askJudgment(
|
|
|
671
746
|
if (judgment.rollbackMovesBoundary) {
|
|
672
747
|
judgment.boundary = await ask(
|
|
673
748
|
ui,
|
|
749
|
+
"release-boundary",
|
|
674
750
|
"Restate the boundary, ending before the irreversible step",
|
|
675
751
|
judgment.boundary ?? "",
|
|
676
752
|
);
|
|
@@ -713,6 +789,7 @@ async function askReleaseGrants(
|
|
|
713
789
|
for (const shape of RELEASE_SHAPES) {
|
|
714
790
|
const open = await askYesNo(
|
|
715
791
|
ui,
|
|
792
|
+
`release-tool-gate.${shape}`,
|
|
716
793
|
`Release tool gate — ${shape}`,
|
|
717
794
|
`Allow the orchestrator session to ${RELEASE_SHAPE_QUESTIONS[shape]}? Grant this only when the ` +
|
|
718
795
|
"operator brief carries the procedure it must follow. A worker session is refused this " +
|
|
@@ -733,6 +810,7 @@ async function askReleaseGrants(
|
|
|
733
810
|
async function askOrchestratorMode(ui: WizardUi, prior: OrchestratorMode): Promise<OrchestratorMode> {
|
|
734
811
|
const external = await askYesNo(
|
|
735
812
|
ui,
|
|
813
|
+
"orchestrator-mode",
|
|
736
814
|
"Orchestrator session",
|
|
737
815
|
"Do you already run your own orchestrator session for this project — a visible TUI session, say? " +
|
|
738
816
|
"Then the daemon starts none of its own, and posts tier-1 escalations as issue comments for yours " +
|
|
@@ -762,6 +840,7 @@ async function askGraphRoot(
|
|
|
762
840
|
): Promise<string | undefined> {
|
|
763
841
|
const wanted = await askYesNo(
|
|
764
842
|
ui,
|
|
843
|
+
"code-graph-enabled",
|
|
765
844
|
"Code-graph discovery",
|
|
766
845
|
"Set up code-graph discovery for workers? Workers spend most of their turn budget finding code; " +
|
|
767
846
|
'a graph answers "who calls this" in one call. Conductor keeps one disposable clone per repo, ' +
|
|
@@ -772,6 +851,7 @@ async function askGraphRoot(
|
|
|
772
851
|
|
|
773
852
|
return await askValid(
|
|
774
853
|
ui,
|
|
854
|
+
"code-graph-root",
|
|
775
855
|
`Root for those clones — one per repo (${repoNames.join(", ")}) is created under it`,
|
|
776
856
|
prior ?? defaultGraphRoot(trackerRepo),
|
|
777
857
|
(v) =>
|
|
@@ -791,6 +871,7 @@ async function askOrchestratorBrief(ui: WizardUi, a: SetupAnswers): Promise<bool
|
|
|
791
871
|
const path = orchestratorBriefPath(a);
|
|
792
872
|
const wanted = await askYesNo(
|
|
793
873
|
ui,
|
|
874
|
+
"write-operator-brief",
|
|
794
875
|
`Write ${ORCHESTRATOR_BRIEF_NAME} + ${POLICY_BRIEF_NAME} under ${dirname(path)}?`,
|
|
795
876
|
`Writes composed ${ORCHESTRATOR_BRIEF_NAME} (package floor, refreshed each tick) and ${POLICY_BRIEF_NAME} ` +
|
|
796
877
|
`(Releases, Project context, Reporting, Amendments — yours to edit via the Learning loop). ` +
|
|
@@ -801,6 +882,7 @@ async function askOrchestratorBrief(ui: WizardUi, a: SetupAnswers): Promise<bool
|
|
|
801
882
|
|
|
802
883
|
return await askYesNo(
|
|
803
884
|
ui,
|
|
885
|
+
"overwrite-operator-brief",
|
|
804
886
|
`Overwrite existing ${ORCHESTRATOR_BRIEF_NAME} / ${POLICY_BRIEF_NAME}?`,
|
|
805
887
|
`${path} already exists. Overwriting replaces the composed brief and POLICY.md scaffold — any policy you wrote is lost.`,
|
|
806
888
|
);
|
|
@@ -990,6 +1072,7 @@ async function proseFromRepos(ui: WizardUi, a: SetupAnswers): Promise<ProbedPros
|
|
|
990
1072
|
const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
|
|
991
1073
|
const trackerRepo = await askValid(
|
|
992
1074
|
ui,
|
|
1075
|
+
"tracker-repo",
|
|
993
1076
|
"Tracker repo (owner/repo) — where ready issues live",
|
|
994
1077
|
a.trackerRepo,
|
|
995
1078
|
(v) => (REPO_RE.test(v) ? undefined : `"${v}" is not owner/repo — e.g. acme/planning.`),
|
|
@@ -997,6 +1080,7 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
|
|
|
997
1080
|
|
|
998
1081
|
const queueLabel = await ask(
|
|
999
1082
|
ui,
|
|
1083
|
+
"queue-label",
|
|
1000
1084
|
"Queue label — the human sign-off that makes an issue claimable",
|
|
1001
1085
|
a.queueLabel,
|
|
1002
1086
|
);
|
|
@@ -1007,18 +1091,20 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
|
|
|
1007
1091
|
const stateLabels: SetupAnswers["stateLabels"] = { ...a.stateLabels };
|
|
1008
1092
|
const customiseStates = await askYesNo(
|
|
1009
1093
|
ui,
|
|
1094
|
+
"customise-state-labels",
|
|
1010
1095
|
"State labels",
|
|
1011
1096
|
`The conductor writes back "${stateLabels.inProgress}", "${stateLabels.blocked}" and ` +
|
|
1012
1097
|
`"${stateLabels.failed}" so the tracker alone shows live state. Rename them?`,
|
|
1013
1098
|
);
|
|
1014
1099
|
if (customiseStates) {
|
|
1015
|
-
stateLabels.inProgress = await ask(ui, "Label for a run in progress", stateLabels.inProgress);
|
|
1016
|
-
stateLabels.blocked = await ask(ui, "Label for a run parked on a human", stateLabels.blocked);
|
|
1017
|
-
stateLabels.failed = await ask(ui, "Label for a run that gave up", stateLabels.failed);
|
|
1100
|
+
stateLabels.inProgress = await ask(ui, "state-label-in-progress", "Label for a run in progress", stateLabels.inProgress);
|
|
1101
|
+
stateLabels.blocked = await ask(ui, "state-label-blocked", "Label for a run parked on a human", stateLabels.blocked);
|
|
1102
|
+
stateLabels.failed = await ask(ui, "state-label-failed", "Label for a run that gave up", stateLabels.failed);
|
|
1018
1103
|
}
|
|
1019
1104
|
|
|
1020
1105
|
const routingLabelPrefix = await ask(
|
|
1021
1106
|
ui,
|
|
1107
|
+
"routing-label-prefix",
|
|
1022
1108
|
"Routing label prefix — an issue picks its checkout with <prefix><repo>",
|
|
1023
1109
|
a.routingLabelPrefix,
|
|
1024
1110
|
);
|
|
@@ -1028,18 +1114,21 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
|
|
|
1028
1114
|
const seed = a.targetRepos[i];
|
|
1029
1115
|
const name = await askValid(
|
|
1030
1116
|
ui,
|
|
1117
|
+
`routing-key.${i + 1}`,
|
|
1031
1118
|
`Routing key for repo ${i + 1} — the "${routingLabelPrefix}<key>" label an issue carries`,
|
|
1032
1119
|
seed?.name ?? "",
|
|
1033
1120
|
(v) => (v.length > 0 ? undefined : "A routing key is required, or no issue can reach this repo."),
|
|
1034
1121
|
);
|
|
1035
1122
|
const cloneUrl = await askValid(
|
|
1036
1123
|
ui,
|
|
1124
|
+
`clone-url.${name}`,
|
|
1037
1125
|
`Clone URL for ${routingLabelPrefix}${name}`,
|
|
1038
1126
|
seed?.cloneUrl ?? "",
|
|
1039
1127
|
(v) => (v.length > 0 ? undefined : "A clone URL is required — the daemon mirrors it before every run."),
|
|
1040
1128
|
);
|
|
1041
1129
|
const defaultBranch = await ask(
|
|
1042
1130
|
ui,
|
|
1131
|
+
`default-branch.${name}`,
|
|
1043
1132
|
`Default branch for ${name} — worktrees are cut from it and PRs target it`,
|
|
1044
1133
|
seed?.defaultBranch ?? SETUP_DEFAULTS.defaultBranch,
|
|
1045
1134
|
);
|
|
@@ -1051,6 +1140,7 @@ const askTrackerAndRepos: AreaAsker = async (ui, a, probes) => {
|
|
|
1051
1140
|
|
|
1052
1141
|
const more = await askYesNo(
|
|
1053
1142
|
ui,
|
|
1143
|
+
`another-repo.${i + 1}`,
|
|
1054
1144
|
"Another repo?",
|
|
1055
1145
|
`${targetRepos.map((r) => r.name).join(", ")} configured. Add another checkout this project routes to?`,
|
|
1056
1146
|
);
|
|
@@ -1121,6 +1211,7 @@ const askCaps: AreaAsker = async (ui, a) => {
|
|
|
1121
1211
|
: "";
|
|
1122
1212
|
const tuneCaps = await askYesNo(
|
|
1123
1213
|
ui,
|
|
1214
|
+
"tune-caps",
|
|
1124
1215
|
"Caps",
|
|
1125
1216
|
`Defaults: ${workersDefault} workers, ` +
|
|
1126
1217
|
`${spendLabel}, ${DEFAULT_CAPS.workerMaxTurns} base / ` +
|
|
@@ -1143,16 +1234,19 @@ const askCaps: AreaAsker = async (ui, a) => {
|
|
|
1143
1234
|
// not silently go unasked.
|
|
1144
1235
|
caps.maxConcurrentWorkers = await askNumber(
|
|
1145
1236
|
ui,
|
|
1237
|
+
"max-concurrent-workers",
|
|
1146
1238
|
"Max concurrent workers",
|
|
1147
1239
|
workersDefault,
|
|
1148
1240
|
);
|
|
1149
1241
|
caps.dailySpendUsd = await askSpendCap(
|
|
1150
1242
|
ui,
|
|
1243
|
+
"daily-spend-usd",
|
|
1151
1244
|
"Spend ceiling per rolling day (USD) — blank = no spend cap",
|
|
1152
1245
|
caps.dailySpendUsd !== undefined ? caps.dailySpendUsd : DEFAULT_CAPS.dailySpendUsd,
|
|
1153
1246
|
);
|
|
1154
1247
|
const workerMaxTurns = await askNumber(
|
|
1155
1248
|
ui,
|
|
1249
|
+
"worker-max-turns",
|
|
1156
1250
|
"Turn ceiling per worker",
|
|
1157
1251
|
caps.workerMaxTurns ?? DEFAULT_CAPS.workerMaxTurns,
|
|
1158
1252
|
);
|
|
@@ -1164,6 +1258,7 @@ const askCaps: AreaAsker = async (ui, a) => {
|
|
|
1164
1258
|
);
|
|
1165
1259
|
const workerMaxTurnsCeiling = await askNumber(
|
|
1166
1260
|
ui,
|
|
1261
|
+
"worker-max-turns-ceiling",
|
|
1167
1262
|
"Maximum turn ceiling for one issue",
|
|
1168
1263
|
turnCeilingFallback,
|
|
1169
1264
|
);
|
|
@@ -1179,16 +1274,19 @@ const askCaps: AreaAsker = async (ui, a) => {
|
|
|
1179
1274
|
}
|
|
1180
1275
|
caps.workerWallClockMs = await askNumber(
|
|
1181
1276
|
ui,
|
|
1277
|
+
"worker-wall-clock-ms",
|
|
1182
1278
|
"Wall-clock ceiling per worker (ms)",
|
|
1183
1279
|
caps.workerWallClockMs ?? DEFAULT_CAPS.workerWallClockMs,
|
|
1184
1280
|
);
|
|
1185
1281
|
caps.maxAttemptsPerIssue = await askNumber(
|
|
1186
1282
|
ui,
|
|
1283
|
+
"max-attempts-per-issue",
|
|
1187
1284
|
"Failed implementation attempts per issue before escalation",
|
|
1188
1285
|
caps.maxAttemptsPerIssue ?? DEFAULT_CAPS.maxAttemptsPerIssue,
|
|
1189
1286
|
);
|
|
1190
1287
|
caps.maxContinuationsPerIssue = await askNumber(
|
|
1191
1288
|
ui,
|
|
1289
|
+
"max-continuations-per-issue",
|
|
1192
1290
|
"Operational continuations per issue before escalation",
|
|
1193
1291
|
caps.maxContinuationsPerIssue ?? DEFAULT_CAPS.maxContinuationsPerIssue,
|
|
1194
1292
|
);
|
|
@@ -1198,7 +1296,12 @@ const askCaps: AreaAsker = async (ui, a) => {
|
|
|
1198
1296
|
/** Outside the caps block: a model is not a ceiling, and an operator who left
|
|
1199
1297
|
* the caps alone may still want workers on a cheaper model. */
|
|
1200
1298
|
const askWorkerModel: AreaAsker = async (ui, a) => {
|
|
1201
|
-
const answered = await ask(
|
|
1299
|
+
const answered = await ask(
|
|
1300
|
+
ui,
|
|
1301
|
+
"worker-model",
|
|
1302
|
+
"Worker model pattern (blank = harness default)",
|
|
1303
|
+
a.workerModel ?? "",
|
|
1304
|
+
);
|
|
1202
1305
|
const next: SetupAnswers = { ...a };
|
|
1203
1306
|
if (answered.trim().length === 0) delete next.workerModel;
|
|
1204
1307
|
else next.workerModel = answered.trim();
|
|
@@ -1216,6 +1319,7 @@ const askOmpSettings: AreaAsker = async (ui, a) => {
|
|
|
1216
1319
|
const seed = a.ompSettings === undefined ? "" : JSON.stringify(a.ompSettings);
|
|
1217
1320
|
const answered = await askValid(
|
|
1218
1321
|
ui,
|
|
1322
|
+
"omp-settings",
|
|
1219
1323
|
"Omp settings overlay for workers (YAML, blank = none — omp's schema, not conductor's)",
|
|
1220
1324
|
seed,
|
|
1221
1325
|
(value) => {
|
|
@@ -1266,10 +1370,26 @@ const askAuthorityArea: AreaAsker = async (ui, a, _probes, discovered) => {
|
|
|
1266
1370
|
/** What a merge and a release must satisfy. Asked straight after the grants:
|
|
1267
1371
|
* who may act, then under what conditions. The release half is only asked
|
|
1268
1372
|
* when the orchestrator cuts releases — with humans releasing it has no gates
|
|
1269
|
-
* to configure, so it would be a dead prompt (#368).
|
|
1373
|
+
* to configure, so it would be a dead prompt (#368). The arming proof is
|
|
1374
|
+
* asked after the preconditions: it gates `arm`, not merge or release, but it
|
|
1375
|
+
* is the same kind of declared policy, so it rides the same area (#613).
|
|
1376
|
+
*
|
|
1377
|
+
* The `claim-only` consequence is stated in the option's own description so an
|
|
1378
|
+
* operator chooses it rather than discovering it: `arm` is already privileged,
|
|
1379
|
+
* and `claim-only` means anything that can invoke it can start dispatch once
|
|
1380
|
+
* the live claim and poller pass.
|
|
1381
|
+
*/
|
|
1270
1382
|
const askPolicy: AreaAsker = async (ui, a) => ({
|
|
1271
1383
|
...a,
|
|
1272
1384
|
policy: await askPolicyPreconditions(ui, a.policy, a.authority.release),
|
|
1385
|
+
armProof: await askLiteral(
|
|
1386
|
+
ui,
|
|
1387
|
+
"arm-proof",
|
|
1388
|
+
"Arming — how should `arm` prove a human approved dispatch?",
|
|
1389
|
+
ARM_PROOFS,
|
|
1390
|
+
ARM_PROOF_CHOICES,
|
|
1391
|
+
a.armProof,
|
|
1392
|
+
),
|
|
1273
1393
|
});
|
|
1274
1394
|
|
|
1275
1395
|
/** How a stuck run reaches a human, and who triages it when it does. */
|
|
@@ -1280,12 +1400,18 @@ const askEscalation: AreaAsker = async (ui, a) => {
|
|
|
1280
1400
|
if (telegramChatId === undefined && telegram.pairedOwnerId !== undefined) {
|
|
1281
1401
|
const usePaired = await askYesNo(
|
|
1282
1402
|
ui,
|
|
1403
|
+
"tier2-use-paired-chat",
|
|
1283
1404
|
"Tier-2 escalations",
|
|
1284
1405
|
`omp-telegram is paired with chat ${telegram.pairedOwnerId}. Page it when a run is stuck?`,
|
|
1285
1406
|
);
|
|
1286
1407
|
if (usePaired) telegramChatId = telegram.pairedOwnerId;
|
|
1287
1408
|
} else {
|
|
1288
|
-
const answered = await ask(
|
|
1409
|
+
const answered = await ask(
|
|
1410
|
+
ui,
|
|
1411
|
+
"tier2-chat-id",
|
|
1412
|
+
"Telegram chat id for tier-2 escalations (blank for none)",
|
|
1413
|
+
telegramChatId ?? "",
|
|
1414
|
+
);
|
|
1289
1415
|
telegramChatId = answered.length > 0 ? answered : undefined;
|
|
1290
1416
|
}
|
|
1291
1417
|
} else {
|
|
@@ -1303,6 +1429,7 @@ const askEscalation: AreaAsker = async (ui, a) => {
|
|
|
1303
1429
|
|
|
1304
1430
|
const fallbackToIssueComment = await askYesNo(
|
|
1305
1431
|
ui,
|
|
1432
|
+
"escalation-fallback",
|
|
1306
1433
|
"Escalation fallback",
|
|
1307
1434
|
"Also comment on the issue when a run escalates? Recommended: a chat message you miss is a run nobody sees.",
|
|
1308
1435
|
);
|
|
@@ -1320,14 +1447,25 @@ const askEscalation: AreaAsker = async (ui, a) => {
|
|
|
1320
1447
|
/**
|
|
1321
1448
|
* Forum topic for tier-2 pages. Claimed threads from omp-telegram's
|
|
1322
1449
|
* `threads.json` are offered when readable; a missing file is silent and the
|
|
1323
|
-
* operator can still type an id or keep flat chat (#318).
|
|
1450
|
+
* operator can still type an id or keep flat chat (#318). A registry that is
|
|
1451
|
+
* *present but unreadable* is the opposite: the bridge is in a state the
|
|
1452
|
+
* wizard cannot read, so the manual id would be a guess against unverifiable
|
|
1453
|
+
* state — surface the problem before the fallback instead of silently offering
|
|
1454
|
+
* either (#626).
|
|
1324
1455
|
*/
|
|
1325
1456
|
async function askTelegramTopicId(
|
|
1326
1457
|
ui: WizardUi,
|
|
1327
1458
|
stateDir: string,
|
|
1328
1459
|
prior: number | undefined,
|
|
1329
1460
|
): Promise<number | undefined> {
|
|
1330
|
-
const
|
|
1461
|
+
const result = claimedTelegramTopics(stateDir);
|
|
1462
|
+
if (result.kind === "unavailable") {
|
|
1463
|
+
ui.notify(
|
|
1464
|
+
`Cannot offer claimed topics: omp-telegram's claim registry is unreadable (${result.problem}) — check the bridge state before typing a topic id, or keep flat chat`,
|
|
1465
|
+
"warning",
|
|
1466
|
+
);
|
|
1467
|
+
}
|
|
1468
|
+
const claimed = result.kind === "ok" ? result.claims : [];
|
|
1331
1469
|
const manual = "Enter thread id manually";
|
|
1332
1470
|
const none = "None — flat chat (0.13 behaviour)";
|
|
1333
1471
|
if (claimed.length > 0) {
|
|
@@ -1346,6 +1484,7 @@ async function askTelegramTopicId(
|
|
|
1346
1484
|
];
|
|
1347
1485
|
const priorIdx = prior === undefined ? -1 : claimed.findIndex((t) => t.threadId === prior);
|
|
1348
1486
|
const picked = await ui.select("Telegram forum topic for tier-2 pages", options, {
|
|
1487
|
+
key: "tier2-topic",
|
|
1349
1488
|
initialIndex: priorIdx >= 0 ? priorIdx : claimed.length + 1,
|
|
1350
1489
|
});
|
|
1351
1490
|
if (picked === undefined) throw new Cancelled();
|
|
@@ -1358,6 +1497,7 @@ async function askTelegramTopicId(
|
|
|
1358
1497
|
|
|
1359
1498
|
const answered = await ask(
|
|
1360
1499
|
ui,
|
|
1500
|
+
"tier2-topic-id",
|
|
1361
1501
|
"Telegram forum topic id (blank for flat chat)",
|
|
1362
1502
|
prior !== undefined ? String(prior) : "",
|
|
1363
1503
|
);
|
|
@@ -1388,7 +1528,7 @@ const askReporting: AreaAsker = async (ui, a) => {
|
|
|
1388
1528
|
description: "hold non-bypass interruptions outside selected local working hours",
|
|
1389
1529
|
},
|
|
1390
1530
|
],
|
|
1391
|
-
{ initialIndex: a.availability === undefined ? 0 : 1 },
|
|
1531
|
+
{ key: "operator-availability", initialIndex: a.availability === undefined ? 0 : 1 },
|
|
1392
1532
|
);
|
|
1393
1533
|
if (mode === undefined) throw new Cancelled();
|
|
1394
1534
|
|
|
@@ -1420,6 +1560,7 @@ const askReporting: AreaAsker = async (ui, a) => {
|
|
|
1420
1560
|
: a.dailyDigestAt ?? (a.digestCadence === "daily" ? "model-timed" : fallback);
|
|
1421
1561
|
const schedule = await askValid(
|
|
1422
1562
|
ui,
|
|
1563
|
+
"digest-schedule",
|
|
1423
1564
|
'Daily rollup time in that timezone / digest cadence ("per-tick", "model-timed", "off", or 24h HH:MM)',
|
|
1424
1565
|
shown,
|
|
1425
1566
|
(value) =>
|
|
@@ -1444,6 +1585,7 @@ const askReporting: AreaAsker = async (ui, a) => {
|
|
|
1444
1585
|
a.reportingTimezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone ?? "UTC";
|
|
1445
1586
|
const reportingTimezone = await askValid(
|
|
1446
1587
|
ui,
|
|
1588
|
+
"reporting-timezone",
|
|
1447
1589
|
"Operator timezone (IANA, for example Europe/London)",
|
|
1448
1590
|
defaultZone,
|
|
1449
1591
|
(value) => {
|
|
@@ -1457,6 +1599,7 @@ const askReporting: AreaAsker = async (ui, a) => {
|
|
|
1457
1599
|
);
|
|
1458
1600
|
const daysText = await askValid(
|
|
1459
1601
|
ui,
|
|
1602
|
+
"working-weekdays",
|
|
1460
1603
|
"Working weekdays (comma-separated: mon,tue,wed,thu,fri,sat,sun)",
|
|
1461
1604
|
a.availability?.days.join(",") ?? "mon,tue,wed,thu,fri",
|
|
1462
1605
|
(value) => {
|
|
@@ -1470,12 +1613,14 @@ const askReporting: AreaAsker = async (ui, a) => {
|
|
|
1470
1613
|
const days = daysText.split(",").map((day) => day.trim().toLowerCase() as Weekday);
|
|
1471
1614
|
const start = await askValid(
|
|
1472
1615
|
ui,
|
|
1616
|
+
"availability-start",
|
|
1473
1617
|
"Availability starts (24h HH:MM)",
|
|
1474
1618
|
a.availability?.start ?? "09:00",
|
|
1475
1619
|
(value) => (/^([01]\d|2[0-3]):[0-5]\d$/.test(value) ? undefined : "Use 24h HH:MM."),
|
|
1476
1620
|
);
|
|
1477
1621
|
const end = await askValid(
|
|
1478
1622
|
ui,
|
|
1623
|
+
"availability-end",
|
|
1479
1624
|
"Availability ends (24h HH:MM)",
|
|
1480
1625
|
a.availability?.end ?? "17:00",
|
|
1481
1626
|
(value) =>
|
|
@@ -1487,6 +1632,7 @@ const askReporting: AreaAsker = async (ui, a) => {
|
|
|
1487
1632
|
);
|
|
1488
1633
|
const bypassText = await askValid(
|
|
1489
1634
|
ui,
|
|
1635
|
+
"quiet-hours-bypass",
|
|
1490
1636
|
`Quiet-hours bypass categories (comma-separated; "none" = none; choices: ${INTERRUPT_CATEGORIES.join(",")})`,
|
|
1491
1637
|
a.availability === undefined ? "fleet-stopped" : a.availability.bypass.join(",") || "none",
|
|
1492
1638
|
(value) => {
|
|
@@ -1634,7 +1780,7 @@ async function chooseAmendArea(ui: WizardUi, prior: ProjectConfig, defaults: Cap
|
|
|
1634
1780
|
description: "full interview for a new project; existing projects stay as they are",
|
|
1635
1781
|
},
|
|
1636
1782
|
],
|
|
1637
|
-
{ initialIndex: 0 },
|
|
1783
|
+
{ key: `existing-project-action.${prior.name}`, initialIndex: 0 },
|
|
1638
1784
|
);
|
|
1639
1785
|
if (mode === undefined) throw new Cancelled();
|
|
1640
1786
|
if (mode === ADD_PROJECT) return { kind: "add-project" };
|
|
@@ -1650,7 +1796,7 @@ async function chooseAmendArea(ui: WizardUi, prior: ProjectConfig, defaults: Cap
|
|
|
1650
1796
|
const picked = await ui.select(
|
|
1651
1797
|
"Which area? Each row shows what it says now",
|
|
1652
1798
|
choices.map((c) => ({ label: c.label, description: c.description })),
|
|
1653
|
-
{ initialIndex: 0 },
|
|
1799
|
+
{ key: `amend-area.${prior.name}`, initialIndex: 0 },
|
|
1654
1800
|
);
|
|
1655
1801
|
if (picked === undefined) throw new Cancelled();
|
|
1656
1802
|
|
|
@@ -1693,6 +1839,7 @@ async function collectAnswers(
|
|
|
1693
1839
|
|
|
1694
1840
|
const projectName = await askValid(
|
|
1695
1841
|
ui,
|
|
1842
|
+
"project-name",
|
|
1696
1843
|
"Project name",
|
|
1697
1844
|
projectArg ?? seed.projectName,
|
|
1698
1845
|
(v) => (v.length > 0 ? undefined : "A name is required — it is how `omp-conductor status --project <name>` finds this project."),
|
|
@@ -1821,7 +1968,7 @@ export async function collectSetup(
|
|
|
1821
1968
|
description: "overwrites this project's config entry on apply; other projects untouched",
|
|
1822
1969
|
},
|
|
1823
1970
|
],
|
|
1824
|
-
{ initialIndex: 0 },
|
|
1971
|
+
{ key: `duplicate-project-action.${answers.projectName}`, initialIndex: 0 },
|
|
1825
1972
|
);
|
|
1826
1973
|
if (mode === undefined) throw new Cancelled();
|
|
1827
1974
|
if (mode.startsWith("Amend")) {
|
|
@@ -1866,7 +2013,7 @@ export async function collectSetup(
|
|
|
1866
2013
|
description: "overwrites this project's config entry on apply; other projects untouched",
|
|
1867
2014
|
},
|
|
1868
2015
|
],
|
|
1869
|
-
{ initialIndex: 0 },
|
|
2016
|
+
{ key: `duplicate-project-action.${answers.projectName}`, initialIndex: 0 },
|
|
1870
2017
|
);
|
|
1871
2018
|
if (mode === undefined) throw new Cancelled();
|
|
1872
2019
|
if (mode.startsWith("Amend")) {
|
|
@@ -1884,6 +2031,7 @@ export async function collectSetup(
|
|
|
1884
2031
|
"Walks every question again and overwrites this project's config entry on apply. " +
|
|
1885
2032
|
"Other projects are left alone. Cancel and pick \"Change one area\" to amend without a full replace, " +
|
|
1886
2033
|
"or \"Add another project\" to create a neighbour.",
|
|
2034
|
+
{ key: `replace-project.${prior.name}` },
|
|
1887
2035
|
);
|
|
1888
2036
|
if (replace !== true) throw new Cancelled();
|
|
1889
2037
|
return { answers: await collectAnswers(ui, prior, projectArg, probes) };
|
|
@@ -1942,10 +2090,39 @@ export interface SetupApplyDeps {
|
|
|
1942
2090
|
labels: (trackerRepo: string, a: SetupAnswers) => Promise<LabelPlan[]>;
|
|
1943
2091
|
preview: (project: ProjectConfig) => Promise<QueuePreview>;
|
|
1944
2092
|
createLabels: (trackerRepo: string, plan: LabelPlan[]) => Promise<string[]>;
|
|
2093
|
+
/**
|
|
2094
|
+
* Compensation half of {@link createLabels}: deletes exactly the labels a
|
|
2095
|
+
* failed apply created, so the tracker returns to its pre-entry state.
|
|
2096
|
+
* Called only on rollback, where a deletion failure is a reported
|
|
2097
|
+
* restoration failure — the orphaned label is named, never silently left
|
|
2098
|
+
* behind (#652).
|
|
2099
|
+
*/
|
|
2100
|
+
deleteLabels: (trackerRepo: string, created: string[]) => Promise<void>;
|
|
2101
|
+
/**
|
|
2102
|
+
* Proves a healthy runtime generation serves the COMPLETE prior configured
|
|
2103
|
+
* project set after a failed apply rolled the disk state back (#650). The
|
|
2104
|
+
* surviving entry-generation process predates the rollback: its store
|
|
2105
|
+
* handle points at the file the rollback replaced (writes diverge into the
|
|
2106
|
+
* unlinked inode until its next restart) and its in-memory config may be
|
|
2107
|
+
* the rejected generation — so "proving health" is never the proof. The
|
|
2108
|
+
* default implementation restarts through the same lifecycle seam the
|
|
2109
|
+
* apply's own restart leg uses and then proves /healthz names every prior
|
|
2110
|
+
* project. Returns false when the proof cannot be made — the caller then
|
|
2111
|
+
* stays fail-closed (a setup hold stays in force, with the recovery
|
|
2112
|
+
* command named) rather than resuming dispatch onto an unproven runtime.
|
|
2113
|
+
*/
|
|
2114
|
+
proveRuntime: (priorProjects: readonly string[]) => Promise<boolean>;
|
|
2115
|
+
/**
|
|
2116
|
+
* The lifecycle/drain seam the acknowledged quiescence barrier runs on —
|
|
2117
|
+
* the same `DrainDeps` `upgrade`, `restart` and `setup host` use, so the
|
|
2118
|
+
* freeze, its token proof and its staleness checks are the seam's, never a
|
|
2119
|
+
* second counter-only fence (#618). Injectable so a regression can drive
|
|
2120
|
+
* the exact admission interleaving through real barrier code.
|
|
2121
|
+
*/
|
|
2122
|
+
drain: DrainDeps;
|
|
1945
2123
|
smoke: (project: string) => Promise<SetupSmokeResult>;
|
|
1946
2124
|
restart: (o: { project?: string }) => Promise<RestartResult>;
|
|
1947
2125
|
arm: (project: string) => Promise<string>;
|
|
1948
|
-
resume: (project: string) => void;
|
|
1949
2126
|
hostInstall: (
|
|
1950
2127
|
project: ProjectConfig,
|
|
1951
2128
|
caps: Caps,
|
|
@@ -1959,20 +2136,593 @@ export interface SetupApplyDeps {
|
|
|
1959
2136
|
) => Promise<InstallOutcome>;
|
|
1960
2137
|
}
|
|
1961
2138
|
|
|
2139
|
+
/**
|
|
2140
|
+
* The default drain accessors the setup apply barrier runs on, wired exactly
|
|
2141
|
+
* like `setup host`'s (`setup-install.ts`) — except the pause names `setup`
|
|
2142
|
+
* as the actor and the barrier refuses on live workers instead of waiting for
|
|
2143
|
+
* a drain.
|
|
2144
|
+
*
|
|
2145
|
+
* Two deliberate divergences from the drop-in lifecycle accessors, both
|
|
2146
|
+
* because a setup apply owns a *host-global* transaction rather than the one
|
|
2147
|
+
* project a restart drains:
|
|
2148
|
+
*
|
|
2149
|
+
* - `snapshot` counts live workers through {@link liveWorkersReadOnly}, never
|
|
2150
|
+
* `statusSnapshot`: the observation that gates the barrier must be
|
|
2151
|
+
* read-only, because database preparation is itself one of the mutations
|
|
2152
|
+
* the barrier precedes (#651 review #2).
|
|
2153
|
+
* - `layers` reports the *global* sentinel's paused flag, not the project
|
|
2154
|
+
* bare-read `fleetLayers` derives — the barrier freezes the whole host
|
|
2155
|
+
* through the one global pause sentinel (`scope.pauseKey === undefined`),
|
|
2156
|
+
* so the fence it must re-prove is that sentinel, never a per-project one
|
|
2157
|
+
* (#651 review #4).
|
|
2158
|
+
*/
|
|
2159
|
+
function defaultSetupDrain(): DrainDeps {
|
|
2160
|
+
return {
|
|
2161
|
+
snapshot: (project) => ({ liveWorkers: liveWorkersReadOnly(project) }),
|
|
2162
|
+
layers: () => ({ ...fleetLayers(), paused: isPaused() }),
|
|
2163
|
+
projectNames: () => loadConfig().projects.map((p) => p.name),
|
|
2164
|
+
daemonIdentity: setupDaemonIdentity,
|
|
2165
|
+
pauseState: (project) => pauseInstance(project),
|
|
2166
|
+
setPaused: (v, project) => setPaused(v, { source: pauseSourceToken("setup"), reason: "setup apply fence" }, project),
|
|
2167
|
+
// The daemon-side admission acknowledgement: the file the running daemon
|
|
2168
|
+
// itself writes when it observes the fence at an admission boundary, so
|
|
2169
|
+
// the barrier never mistakes a second synchronous count for the daemon's
|
|
2170
|
+
// own word that it will claim nothing (#651 review #3).
|
|
2171
|
+
admissionAck: () => readAdmissionAck(),
|
|
2172
|
+
sleep: Bun.sleep,
|
|
2173
|
+
log: () => {},
|
|
2174
|
+
};
|
|
2175
|
+
}
|
|
2176
|
+
|
|
2177
|
+
/**
|
|
2178
|
+
* The daemon identity the setup barrier acknowledges and re-proves: the known
|
|
2179
|
+
* pidfile record when one is live, else an ACTIVE `omp-conductor.service`
|
|
2180
|
+
* unit whose runtime record went missing — a supervised daemon is still a
|
|
2181
|
+
* running daemon without its record, and its MainPID is the generation that
|
|
2182
|
+
* identifies the exact instance (#651, review #2). An UNKNOWN systemd query
|
|
2183
|
+
* throws: "dbus blipped" is not "no daemon", and the barrier must fail closed
|
|
2184
|
+
* rather than acknowledge quiescence it cannot prove.
|
|
2185
|
+
*/
|
|
2186
|
+
function setupDaemonIdentity(): DaemonIdentity {
|
|
2187
|
+
const ownership = probeUnit(SYSTEMD_UNIT);
|
|
2188
|
+
if (ownership.kind === "unknown") {
|
|
2189
|
+
throw new Error(
|
|
2190
|
+
`cannot prove whether the daemon is running: systemd ownership probe failed (${ownership.reason})`,
|
|
2191
|
+
);
|
|
2192
|
+
}
|
|
2193
|
+
const daemon = livingDaemon();
|
|
2194
|
+
if (daemon !== undefined) {
|
|
2195
|
+
// The record names the exact instance; `daemonGeneration` formats the
|
|
2196
|
+
// same generation the daemon's own admission acknowledgement records, so
|
|
2197
|
+
// the two sides of the acknowledged fence always compare like for like.
|
|
2198
|
+
return { running: true, project: daemon.project, generation: daemonGeneration() };
|
|
2199
|
+
}
|
|
2200
|
+
if (ownership.kind === "active") {
|
|
2201
|
+
// The unit owns a live MainPID but the record is absent (a crash raced a
|
|
2202
|
+
// re-record, an unmanaged start never wrote one): the daemon is running,
|
|
2203
|
+
// and the MainPID is the generation that lets the fence spot a restart.
|
|
2204
|
+
return { running: true, generation: `systemd:${ownership.pid}` };
|
|
2205
|
+
}
|
|
2206
|
+
return { running: false };
|
|
2207
|
+
}
|
|
2208
|
+
|
|
1962
2209
|
export const DEFAULT_APPLY: SetupApplyDeps = {
|
|
1963
2210
|
scopes: checkTokenScopes,
|
|
1964
2211
|
labels: planLabels,
|
|
1965
2212
|
preview: previewProject,
|
|
1966
2213
|
createLabels: createMissingLabels,
|
|
2214
|
+
deleteLabels: deleteCreatedLabels,
|
|
2215
|
+
proveRuntime: async (priorProjects) => {
|
|
2216
|
+
// No short-circuit on a healthy daemon: whatever process is still
|
|
2217
|
+
// answering predates the rollback, so it holds a handle on the store file
|
|
2218
|
+
// the rollback replaced and may serve the rejected config. Restart
|
|
2219
|
+
// through the same fail-closed lifecycle seam the apply's own restart leg
|
|
2220
|
+
// uses — whose `waitForOwnedDaemon` proves MainPID + /healthz and that a
|
|
2221
|
+
// fresh boot loaded the restored/rolled-back state — then prove the new
|
|
2222
|
+
// generation serves EVERY prior configured project, not just health on
|
|
2223
|
+
// one (#650, PR #700 review #1).
|
|
2224
|
+
try {
|
|
2225
|
+
const restarted = await restartDaemon({});
|
|
2226
|
+
const health = await healthCheck(restarted.record.port);
|
|
2227
|
+
if (!health.ok) return false;
|
|
2228
|
+
return priorProjects.every((project) => healthServesProject(health.body, project));
|
|
2229
|
+
} catch {
|
|
2230
|
+
return false;
|
|
2231
|
+
}
|
|
2232
|
+
},
|
|
2233
|
+
drain: defaultSetupDrain(),
|
|
1967
2234
|
smoke: runSetupSmoke,
|
|
1968
2235
|
restart: (o) => restartDaemon(o),
|
|
1969
2236
|
arm: (project) => ensureSetupArm(project),
|
|
1970
|
-
resume: (project) => setPaused(false, undefined, project),
|
|
1971
2237
|
hostInstall: (project, caps, telegramStateDir, ui) =>
|
|
1972
2238
|
runHostInstall(project, caps, telegramStateDir, ui),
|
|
1973
2239
|
graphInstall: (project, ui, options) => runGraphInstall(project, ui, options),
|
|
1974
2240
|
};
|
|
1975
2241
|
|
|
2242
|
+
/** One pause sentinel's pre-entry state, four-valued so absence, readable-valid
|
|
2243
|
+
* bytes, malformed bytes and unreadability are never conflated.
|
|
2244
|
+
* - `{ kind: "absent" }` — no file on entry; rollback removes one the barrier
|
|
2245
|
+
* created.
|
|
2246
|
+
* - `{ kind: "bytes", bytes }` — readable AND valid on entry; rollback writes
|
|
2247
|
+
* these bytes back verbatim, so the pre-entry owner token and timestamp
|
|
2248
|
+
* survive a refused apply instead of being replaced by a fresh equivalent
|
|
2249
|
+
* pause (#618, continuation #7).
|
|
2250
|
+
* - `{ kind: "malformed", bytes, path }` — readable but not a valid pause
|
|
2251
|
+
* instance (an unparseable timestamp or provenance line). This is
|
|
2252
|
+
* unreadable-as-state: the barrier must fail closed BEFORE changing
|
|
2253
|
+
* anything and never delete it, because collapsing it to absence would
|
|
2254
|
+
* remove an operator hold it could not prove (#650).
|
|
2255
|
+
* - `{ kind: "unreadable", path }` — the file exists but cannot be read. The
|
|
2256
|
+
* barrier must fail closed BEFORE changing anything: collapsing this to
|
|
2257
|
+
* absence and later deleting the path would remove an operator hold it
|
|
2258
|
+
* never saw (#651 review #5). */
|
|
2259
|
+
type CapturedPause =
|
|
2260
|
+
| { kind: "absent" }
|
|
2261
|
+
| { kind: "bytes"; bytes: string }
|
|
2262
|
+
| { kind: "malformed"; bytes: string; path: string }
|
|
2263
|
+
| { kind: "unreadable"; path: string };
|
|
2264
|
+
|
|
2265
|
+
/** Whether one sentinel's bytes parse as a valid pause instance — the exact
|
|
2266
|
+
* grammar {@link pauseInstance} reads in `daemon.ts` (a parseable ISO
|
|
2267
|
+
* timestamp line, then a `source=` provenance line). Readable bytes that
|
|
2268
|
+
* fail this are an unreadable-as-state malformed hold: valid bytes are
|
|
2269
|
+
* restored verbatim, malformed ones are refused at entry and never deleted
|
|
2270
|
+
* (#650). */
|
|
2271
|
+
function isValidPauseBytes(bytes: string): boolean {
|
|
2272
|
+
const [line1, line2] = bytes.split("\n");
|
|
2273
|
+
if (!Number.isFinite(Date.parse(line1?.trim() ?? ""))) return false;
|
|
2274
|
+
if (line2 === undefined) return false;
|
|
2275
|
+
return /^source=(\S+)(?: reason="(.*)")?$/.test(line2.trim());
|
|
2276
|
+
}
|
|
2277
|
+
|
|
2278
|
+
/** Reads one pause sentinel, keeping absence, readable-valid bytes, malformed
|
|
2279
|
+
* bytes and unreadability apart — never conflated: collapsing malformed to
|
|
2280
|
+
* absence would let a refusal delete an operator hold it could not parse. */
|
|
2281
|
+
function readSentinel(p: string): CapturedPause {
|
|
2282
|
+
if (!existsSync(p)) return { kind: "absent" };
|
|
2283
|
+
try {
|
|
2284
|
+
const bytes = readFileSync(p, "utf8");
|
|
2285
|
+
return isValidPauseBytes(bytes) ? { kind: "bytes", bytes } : { kind: "malformed", bytes, path: p };
|
|
2286
|
+
} catch {
|
|
2287
|
+
return { kind: "unreadable", path: p };
|
|
2288
|
+
}
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2291
|
+
/**
|
|
2292
|
+
* The pre-entry pause sentinel the setup apply itself can change. Setup
|
|
2293
|
+
* freezes the whole host through the one *global* sentinel — it must never
|
|
2294
|
+
* write a project's pause instead, which left the wrong project held when
|
|
2295
|
+
* adding a neighbour next to a daemon recorded for the existing project
|
|
2296
|
+
* (#651 review #4) — so the captured path is the global one, its scope is
|
|
2297
|
+
* preserved exactly (a successful apply or a refusal restores that same
|
|
2298
|
+
* path's pre-entry bytes, never a project-scoped sentinel), and an entry
|
|
2299
|
+
* that is already paused — valid, malformed or unreadable — fails closed in
|
|
2300
|
+
* its own way rather than being rebuilt or deleted (#650).
|
|
2301
|
+
*/
|
|
2302
|
+
function capturePause(): CapturedPause {
|
|
2303
|
+
return readSentinel(pausedPath());
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
/** Restores the sentinel {@link capturePause} captured — but only while the
|
|
2307
|
+
* sentinel on disk is still the fence THIS transaction froze (the bytes it
|
|
2308
|
+
* wrote, or the pre-entry hold it inherited). A `resume` or a newer operator
|
|
2309
|
+
* hold that replaced the setup freeze is a takeover the stale transaction
|
|
2310
|
+
* must preserve: writing the entry bytes back over it, or removing the file
|
|
2311
|
+
* it owns, would delete a safety hold a later actor wrote in the meantime
|
|
2312
|
+
* (#651 review #3). An unreadable or malformed current sentinel is also left
|
|
2313
|
+
* alone — overwriting what cannot be read (or parsed) is the same
|
|
2314
|
+
* overwrite-of-the-unseen this guard exists to prevent. Never called with a
|
|
2315
|
+
* malformed entry state: the barrier fails closed before mutating then; the
|
|
2316
|
+
* branch exists so a malformed state would still be restored byte for byte
|
|
2317
|
+
* rather than deleted, never treated as absence. */
|
|
2318
|
+
function restorePause(state: CapturedPause, owned: CapturedPause): void {
|
|
2319
|
+
const p = pausedPath();
|
|
2320
|
+
const current = readSentinel(p);
|
|
2321
|
+
// Only this transaction's own fence may be undone. An unreadable or
|
|
2322
|
+
// malformed sentinel is left alone (overwriting what cannot be read or
|
|
2323
|
+
// parsed is overwriting a hold the barrier never saw); a sentinel that no
|
|
2324
|
+
// longer matches the bytes this transaction froze was replaced by a takeover
|
|
2325
|
+
// — a resume or a newer hold — and that takeover stands.
|
|
2326
|
+
let isOwnFence = false;
|
|
2327
|
+
if (current.kind === "bytes" && owned.kind === "bytes") {
|
|
2328
|
+
isOwnFence = current.bytes === owned.bytes;
|
|
2329
|
+
} else if (current.kind === "absent" && owned.kind === "absent") {
|
|
2330
|
+
isOwnFence = true;
|
|
2331
|
+
}
|
|
2332
|
+
if (!isOwnFence) return;
|
|
2333
|
+
if (state.kind === "bytes" || state.kind === "malformed") {
|
|
2334
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
2335
|
+
writeFileSync(p, state.bytes);
|
|
2336
|
+
} else {
|
|
2337
|
+
rmSync(p, { force: true });
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
// -------------------------------------------------- the mutation inventory --
|
|
2342
|
+
//
|
|
2343
|
+
// Child 2 of the setup-transaction decomposition (#652): before the first
|
|
2344
|
+
// mutation the apply records the COMPLETE inventory of every surface it can
|
|
2345
|
+
// write — config, briefs, staged units/scripts/tick, the AGENTS link, the run
|
|
2346
|
+
// store, the daemon record, the arm marker, the config backups and the
|
|
2347
|
+
// tracker labels — with byte-exact pre-entry state. Any later exception or
|
|
2348
|
+
// failed smoke restores every one of them (or removes what a first install
|
|
2349
|
+
// created), compensates the tracker exactly, and reports the first
|
|
2350
|
+
// restoration failure honestly instead of pretending the prior state is
|
|
2351
|
+
// coherent. The one surface a rollback never undoes blindly is the pause
|
|
2352
|
+
// sentinel: {@link restorePause} keeps child 1's ownership rule — only the
|
|
2353
|
+
// fence THIS transaction froze is undone, and a newer actor's hold stands.
|
|
2354
|
+
|
|
2355
|
+
/** The run store's pre-entry state: a byte-consistent snapshot when the
|
|
2356
|
+
* store exists, absence on a first install. */
|
|
2357
|
+
interface StoreCapture {
|
|
2358
|
+
path: string;
|
|
2359
|
+
kind: "absent" | "snapshot";
|
|
2360
|
+
snapshotPath?: string;
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
/** The complete pre-entry mutation inventory of one setup apply (#652). */
|
|
2364
|
+
interface SetupInventory {
|
|
2365
|
+
/** `config.json` pre-entry state (bytes, absence, or unreadable). */
|
|
2366
|
+
config: CapturedPathState;
|
|
2367
|
+
/** The pre-entry configured project set — what the prior runtime generation
|
|
2368
|
+
* must be proven to serve after a rollback restores the config (#650). */
|
|
2369
|
+
priorProjects: string[];
|
|
2370
|
+
/** The brief files the apply will write: `POLICY.md` + `ORCHESTRATOR.md`. */
|
|
2371
|
+
briefs: CapturedPathState[];
|
|
2372
|
+
/** Every staged host-runtime path the apply may write (units, scripts, tick). */
|
|
2373
|
+
runtime: CapturedPathState[];
|
|
2374
|
+
/** The AGENTS.md brief link the apply will create/update, when planned. */
|
|
2375
|
+
briefLink?: CapturedPathState;
|
|
2376
|
+
/** The run store: byte-consistent snapshot when it exists, absence on a first install. */
|
|
2377
|
+
store: StoreCapture;
|
|
2378
|
+
/** The daemon pidfile record the smoke/restart legs write. */
|
|
2379
|
+
record: CapturedPathState;
|
|
2380
|
+
/** The external-heartbeat arm marker, when an external orchestrator is planned. */
|
|
2381
|
+
armMarker?: CapturedPathState;
|
|
2382
|
+
/** Directories the apply may create (workspace/tick cwds), absent at entry;
|
|
2383
|
+
* rollback removes them only while still empty, so operator content that
|
|
2384
|
+
* landed in one is never destroyed. */
|
|
2385
|
+
dirs: string[];
|
|
2386
|
+
/** Pre-entry listing of the config backup dir; backups the apply created
|
|
2387
|
+
* are removed on rollback so the backups return to their entry set. */
|
|
2388
|
+
backups: { dir: string; files: string[] };
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
/** A byte-consistent pre-entry copy of the run store: VACUUM INTO reads main
|
|
2392
|
+
* file + WAL as one, so the snapshot holds every committed row even while a
|
|
2393
|
+
* paused daemon holds a writer handle. The snapshot lives under the state
|
|
2394
|
+
* dir for the transaction's lifetime and is removed on commit and rollback
|
|
2395
|
+
* alike; a partial snapshot from a throwing VACUUM is removed immediately,
|
|
2396
|
+
* never left as a stray temp copy in the state dir (#650, review #2). */
|
|
2397
|
+
function captureStore(path: string): StoreCapture {
|
|
2398
|
+
if (!existsSync(path)) return { path, kind: "absent" };
|
|
2399
|
+
const snapshotPath = join(stateDir(), `.conductor.db.setup-${process.pid}.${randomUUID()}.snapshot`);
|
|
2400
|
+
try {
|
|
2401
|
+
vacuumInto(path, snapshotPath);
|
|
2402
|
+
} catch (err) {
|
|
2403
|
+
rmSync(snapshotPath, { force: true });
|
|
2404
|
+
throw err;
|
|
2405
|
+
}
|
|
2406
|
+
return { path, kind: "snapshot", snapshotPath };
|
|
2407
|
+
}
|
|
2408
|
+
|
|
2409
|
+
/** The backup dir listing, or the empty set when the dir does not exist yet. */
|
|
2410
|
+
function readdirIfPresent(dir: string): string[] {
|
|
2411
|
+
try {
|
|
2412
|
+
return readdirSync(dir).sort();
|
|
2413
|
+
} catch {
|
|
2414
|
+
return [];
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2418
|
+
/**
|
|
2419
|
+
* The full pre-entry mutation inventory of one apply, read BEFORE the first
|
|
2420
|
+
* mutation. Throws when any captured path exists but cannot be read: an
|
|
2421
|
+
* unreadable pre-entry path is a fault, never absence — collapsing it would
|
|
2422
|
+
* let a rollback delete state the apply never saw, the same fail-closed rule
|
|
2423
|
+
* the pause sentinel capture already applies (#651 review #5).
|
|
2424
|
+
*
|
|
2425
|
+
* `priorProjectNames` is the pre-entry configured project set (from the
|
|
2426
|
+
* config `setup()` loaded before the interview): the set a failed apply must
|
|
2427
|
+
* prove the restored runtime serves again. It comes from the entry load, not
|
|
2428
|
+
* a post-rollback re-read, so an un-restored config cannot make the proof
|
|
2429
|
+
* measure the wrong set (#650).
|
|
2430
|
+
*/
|
|
2431
|
+
function captureInventory(
|
|
2432
|
+
plan: DerivedPlan,
|
|
2433
|
+
answers: SetupAnswers,
|
|
2434
|
+
priorProjectNames: string[],
|
|
2435
|
+
): SetupInventory {
|
|
2436
|
+
const runtime: CapturedPathState[] = [];
|
|
2437
|
+
const captureIfWritten = (w: { action: string; path: string } | undefined): void => {
|
|
2438
|
+
// Only paths the plan says will actually be written: a `keep` path is
|
|
2439
|
+
// untouched by the apply and needs no pre-entry state.
|
|
2440
|
+
if (w !== undefined && w.action !== "keep") runtime.push(capturePathState(w.path));
|
|
2441
|
+
};
|
|
2442
|
+
captureIfWritten(plan.runtime.service);
|
|
2443
|
+
captureIfWritten(plan.runtime.herdrUnit);
|
|
2444
|
+
captureIfWritten(plan.runtime.herdrConfig);
|
|
2445
|
+
captureIfWritten(plan.runtime.herdrEnv);
|
|
2446
|
+
captureIfWritten(plan.runtime.recoverUnit);
|
|
2447
|
+
captureIfWritten(plan.runtime.recoverScript);
|
|
2448
|
+
captureIfWritten(plan.runtime.tick);
|
|
2449
|
+
|
|
2450
|
+
const briefs = answers.writeOrchestratorBrief
|
|
2451
|
+
? [policyPathForProject(plan.project), briefPathForProject(plan.project)].map(capturePathState)
|
|
2452
|
+
: [];
|
|
2453
|
+
const briefLink =
|
|
2454
|
+
plan.runtime.briefLink !== undefined &&
|
|
2455
|
+
(plan.runtime.briefLink.action === "create" || plan.runtime.briefLink.action === "update")
|
|
2456
|
+
? capturePathState(plan.runtime.briefLink.path)
|
|
2457
|
+
: undefined;
|
|
2458
|
+
const record = capturePathState(recordPath());
|
|
2459
|
+
const armMarker =
|
|
2460
|
+
plan.project.escalation.orchestrator === "external"
|
|
2461
|
+
? capturePathState(armedMarkerPath(plan.project.name))
|
|
2462
|
+
: undefined;
|
|
2463
|
+
|
|
2464
|
+
// Unreadable check across every captured path, before anything is written.
|
|
2465
|
+
const all = [
|
|
2466
|
+
capturePathState(configPath()),
|
|
2467
|
+
...briefs,
|
|
2468
|
+
...runtime,
|
|
2469
|
+
...(briefLink === undefined ? [] : [briefLink]),
|
|
2470
|
+
record,
|
|
2471
|
+
...(armMarker === undefined ? [] : [armMarker]),
|
|
2472
|
+
];
|
|
2473
|
+
const unreadable = all.find((c) => c.kind === "unreadable");
|
|
2474
|
+
if (unreadable !== undefined) {
|
|
2475
|
+
throw new Error(
|
|
2476
|
+
`the path ${unreadable.path} exists but could not be read — refusing to start the apply rather than ` +
|
|
2477
|
+
"risk overwriting state it never saw",
|
|
2478
|
+
);
|
|
2479
|
+
}
|
|
2480
|
+
|
|
2481
|
+
// Directories the apply can create (the workspace root via the brief write,
|
|
2482
|
+
// the fleet cwd via the AGENTS link and the tick config). Only absent ones
|
|
2483
|
+
// are recorded; rollback removes them only while still empty.
|
|
2484
|
+
const dirs = new Set<string>();
|
|
2485
|
+
if (answers.writeOrchestratorBrief) dirs.add(dirname(briefPathForProject(plan.project)));
|
|
2486
|
+
if (briefLink !== undefined) dirs.add(dirname(briefLink.path));
|
|
2487
|
+
if (plan.runtime.tick !== undefined && plan.runtime.tick.action !== "keep") {
|
|
2488
|
+
dirs.add(dirname(plan.runtime.tick.path));
|
|
2489
|
+
}
|
|
2490
|
+
|
|
2491
|
+
return {
|
|
2492
|
+
config: all[0]!,
|
|
2493
|
+
priorProjects: priorProjectNames,
|
|
2494
|
+
briefs,
|
|
2495
|
+
runtime,
|
|
2496
|
+
...(briefLink === undefined ? {} : { briefLink }),
|
|
2497
|
+
store: captureStore(dbPath()),
|
|
2498
|
+
record,
|
|
2499
|
+
...(armMarker === undefined ? {} : { armMarker }),
|
|
2500
|
+
dirs: [...dirs].filter((d) => !existsSync(d)),
|
|
2501
|
+
backups: { dir: configBackupDir(), files: readdirIfPresent(configBackupDir()) },
|
|
2502
|
+
};
|
|
2503
|
+
}
|
|
2504
|
+
|
|
2505
|
+
/**
|
|
2506
|
+
* Restores every captured surface on a failed apply (#652): the config and
|
|
2507
|
+
* the backups the apply created, the briefs, the staged runtime files, the
|
|
2508
|
+
* AGENTS link, the run store, the daemon record and the arm marker; removes
|
|
2509
|
+
* what a first install created; and compensates tracker labels exactly.
|
|
2510
|
+
* Returns every restoration failure — the first included — so the caller
|
|
2511
|
+
* reports the rollback honestly instead of implying the prior state is
|
|
2512
|
+
* coherent. Never touches the pause sentinel: that restore keeps child 1's
|
|
2513
|
+
* ownership rule and is the caller's.
|
|
2514
|
+
*/
|
|
2515
|
+
async function restoreInventory(
|
|
2516
|
+
inventory: SetupInventory,
|
|
2517
|
+
trackerRepo: string,
|
|
2518
|
+
createdLabels: string[],
|
|
2519
|
+
deleteLabels: SetupApplyDeps["deleteLabels"],
|
|
2520
|
+
): Promise<string[]> {
|
|
2521
|
+
const failures: string[] = [];
|
|
2522
|
+
const restore = (state: CapturedPathState | undefined, opts?: RestorePathOptions): void => {
|
|
2523
|
+
if (state === undefined) return;
|
|
2524
|
+
const failure = restorePathState(state, opts);
|
|
2525
|
+
if (failure !== undefined) failures.push(failure);
|
|
2526
|
+
};
|
|
2527
|
+
|
|
2528
|
+
// The config first — the file everything else derives from. Restored
|
|
2529
|
+
// through the atomic temp-then-rename writer the upgrade rollback uses, so
|
|
2530
|
+
// a crash mid-restore cannot leave a truncated config; the backup that
|
|
2531
|
+
// writer makes of the failed config is pruned with the rest of the apply's
|
|
2532
|
+
// new backups below, returning the backup dir to its entry listing.
|
|
2533
|
+
if (inventory.config.kind === "bytes") {
|
|
2534
|
+
try {
|
|
2535
|
+
writeConfigRaw(inventory.config.bytes);
|
|
2536
|
+
} catch (err) {
|
|
2537
|
+
failures.push(
|
|
2538
|
+
`could not restore config.json: ${err instanceof Error ? err.message : String(err)}`,
|
|
2539
|
+
);
|
|
2540
|
+
}
|
|
2541
|
+
} else {
|
|
2542
|
+
restore(inventory.config);
|
|
2543
|
+
}
|
|
2544
|
+
const currentBackups = readdirIfPresent(inventory.backups.dir);
|
|
2545
|
+
for (const name of currentBackups) {
|
|
2546
|
+
if (inventory.backups.files.includes(name)) continue;
|
|
2547
|
+
try {
|
|
2548
|
+
rmSync(join(inventory.backups.dir, name), { force: true });
|
|
2549
|
+
} catch (err) {
|
|
2550
|
+
failures.push(
|
|
2551
|
+
`could not remove backup ${join(inventory.backups.dir, name)}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2552
|
+
);
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
|
|
2556
|
+
for (const brief of inventory.briefs) restore(brief);
|
|
2557
|
+
for (const path of inventory.runtime) restore(path);
|
|
2558
|
+
// The AGENTS link: only a symlink is ever removed or re-linked, so an
|
|
2559
|
+
// operator's regular file that appeared mid-flight survives — reported as a
|
|
2560
|
+
// failure, because the prior state is then not coherent.
|
|
2561
|
+
restore(inventory.briefLink, { preserveRegularFile: true });
|
|
2562
|
+
|
|
2563
|
+
// The run store: replace from the byte-consistent snapshot (or remove it
|
|
2564
|
+
// entirely on a first-install rollback).
|
|
2565
|
+
try {
|
|
2566
|
+
rmSync(inventory.store.path, { force: true });
|
|
2567
|
+
rmSync(`${inventory.store.path}-wal`, { force: true });
|
|
2568
|
+
rmSync(`${inventory.store.path}-shm`, { force: true });
|
|
2569
|
+
if (inventory.store.kind === "snapshot") {
|
|
2570
|
+
copyFileSync(inventory.store.snapshotPath!, inventory.store.path);
|
|
2571
|
+
}
|
|
2572
|
+
} catch (err) {
|
|
2573
|
+
failures.push(
|
|
2574
|
+
`could not restore the run store at ${inventory.store.path}: ${err instanceof Error ? err.message : String(err)}`,
|
|
2575
|
+
);
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
// The daemon record: only while no daemon is live. A live process owns its
|
|
2579
|
+
// record — restoring the entry bytes over it would describe a daemon that
|
|
2580
|
+
// no longer runs, or remove the record of one that does.
|
|
2581
|
+
if (livingDaemon() === undefined) restore(inventory.record);
|
|
2582
|
+
restore(inventory.armMarker);
|
|
2583
|
+
|
|
2584
|
+
// Directories the apply created: removed only while empty (rmdirSync, never
|
|
2585
|
+
// a recursive rm — operator content that landed in one must survive).
|
|
2586
|
+
for (const dir of inventory.dirs) {
|
|
2587
|
+
try {
|
|
2588
|
+
rmdirSync(dir);
|
|
2589
|
+
} catch {
|
|
2590
|
+
// Non-empty, missing, or busy — leave it; this is not a restoration failure.
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2593
|
+
|
|
2594
|
+
// Tracker compensation: exactly the labels THIS apply created.
|
|
2595
|
+
if (createdLabels.length > 0) {
|
|
2596
|
+
try {
|
|
2597
|
+
await deleteLabels(trackerRepo, createdLabels);
|
|
2598
|
+
} catch (err) {
|
|
2599
|
+
failures.push(err instanceof Error ? err.message : String(err));
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
return failures;
|
|
2603
|
+
}
|
|
2604
|
+
|
|
2605
|
+
/**
|
|
2606
|
+
* The host-global scope a setup apply's barrier covers: every configured
|
|
2607
|
+
* project served by the daemon, and the global pause sentinel so one write
|
|
2608
|
+
* pauses the whole host. Deliberately NOT {@link resolveScope}'s
|
|
2609
|
+
* project-narrowing — which would pause the daemon's recorded project's
|
|
2610
|
+
* sentinel instead of the host-wide one when adding a neighbour next to it
|
|
2611
|
+
* (review #4). A first install has no config and no daemon, so the scope names
|
|
2612
|
+
* the bare selector and the observation proves zero without any reader.
|
|
2613
|
+
*/
|
|
2614
|
+
function setupScope(drain: DrainDeps): UpgradeScope {
|
|
2615
|
+
let configured: readonly string[] = [];
|
|
2616
|
+
try {
|
|
2617
|
+
configured = drain.projectNames();
|
|
2618
|
+
} catch (err) {
|
|
2619
|
+
// An existing config that cannot be enumerated is a fault, never absence:
|
|
2620
|
+
// collapsing it to the bare selector would read "zero workers" for a fleet
|
|
2621
|
+
// the config still describes, then overwrite the unreadable file (#651,
|
|
2622
|
+
// review #2). Only a truly absent config legitimately yields the bare
|
|
2623
|
+
// selector (a first install), and even there the observation still fails
|
|
2624
|
+
// closed against the daemon identity.
|
|
2625
|
+
if (existsSync(configPath())) {
|
|
2626
|
+
throw new Error(
|
|
2627
|
+
`the configured projects could not be read: ${err instanceof Error ? err.message : String(err)} — ` +
|
|
2628
|
+
"zero workers cannot be proven while the config is unreadable",
|
|
2629
|
+
);
|
|
2630
|
+
}
|
|
2631
|
+
}
|
|
2632
|
+
// A daemon serves projects from its own memory, not from the on-disk
|
|
2633
|
+
// config: a project the config no longer names (or never named — a neighbour
|
|
2634
|
+
// this host is mid-migrating to) still owns live runs the daemon admits and
|
|
2635
|
+
// sweeps. If the daemon's recorded project is not among the configured
|
|
2636
|
+
// ones, the host-global scope must still cover it, or its workers are
|
|
2637
|
+
// silently skipped while setup mutates underneath them (#651 review #3). An
|
|
2638
|
+
// identity that cannot be proven throws here — before any freeze — so the
|
|
2639
|
+
// scope never reads "zero" for a fleet it cannot see.
|
|
2640
|
+
const daemon = drain.daemonIdentity();
|
|
2641
|
+
if (daemon.running && daemon.project !== undefined && !configured.includes(daemon.project)) {
|
|
2642
|
+
configured = [...configured, daemon.project];
|
|
2643
|
+
}
|
|
2644
|
+
return {
|
|
2645
|
+
selectors: configured.length === 0 ? [undefined] : configured,
|
|
2646
|
+
pauseKey: undefined,
|
|
2647
|
+
};
|
|
2648
|
+
}
|
|
2649
|
+
|
|
2650
|
+
/** Live worker runs across every project a host-wide transaction covers.
|
|
2651
|
+
* Throws — refusing — when the fleet cannot be proven quiescent: a missing
|
|
2652
|
+
* config does not prove zero workers while a daemon runs from its own
|
|
2653
|
+
* in-memory config and still owns runs (review #3), and an unreadable store
|
|
2654
|
+
* is a fault, never absence. */
|
|
2655
|
+
function liveWorkersInScope(drain: DrainDeps, scope: UpgradeScope): number {
|
|
2656
|
+
if (!existsSync(configPath()) && drain.daemonIdentity().running) {
|
|
2657
|
+
throw new Error(
|
|
2658
|
+
"config.json is absent while a daemon is running, so zero workers cannot be proven; " +
|
|
2659
|
+
"the daemon serves projects from memory",
|
|
2660
|
+
);
|
|
2661
|
+
}
|
|
2662
|
+
return scope.selectors.reduce((n, s) => n + drain.snapshot(s).liveWorkers, 0);
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2665
|
+
/** How long the barrier waits — after one best-effort daemon wake — for the
|
|
2666
|
+
* running daemon to acknowledge the setup fence before refusing with nothing
|
|
2667
|
+
* written. Bounded and fail-closed: an unacknowledged fence is an unproven
|
|
2668
|
+
* one, and the answer is a re-run, never an unproven proceed. */
|
|
2669
|
+
const FENCE_ACK_WAIT_MS = 60_000;
|
|
2670
|
+
const FENCE_ACK_POLL_MS = 250;
|
|
2671
|
+
|
|
2672
|
+
/**
|
|
2673
|
+
* Why the daemon's admission acknowledgement no longer covers the fence the
|
|
2674
|
+
* barrier froze, or `undefined` when it does: the acknowledgement must name
|
|
2675
|
+
* the exact pause instance the barrier proved (not some other pause, however
|
|
2676
|
+
* similar) and the exact daemon generation the barrier began with (not an
|
|
2677
|
+
* older or newer instance's word) (#651 review #3).
|
|
2678
|
+
*/
|
|
2679
|
+
function fenceAckProblem(ack: AdmissionAckRecord | undefined, begun: RestartBegun): string | undefined {
|
|
2680
|
+
if (ack === undefined) return "the running daemon has not acknowledged the setup admission fence";
|
|
2681
|
+
if (
|
|
2682
|
+
ack.pause.source !== begun.pauseToken.source ||
|
|
2683
|
+
ack.pause.since !== begun.pauseToken.since ||
|
|
2684
|
+
ack.pause.reason !== begun.pauseToken.reason
|
|
2685
|
+
) {
|
|
2686
|
+
return "the running daemon acknowledged a different pause than the setup admission fence";
|
|
2687
|
+
}
|
|
2688
|
+
if (ack.daemon !== begun.daemon.generation) {
|
|
2689
|
+
return "the running daemon's acknowledgement belongs to a different daemon generation";
|
|
2690
|
+
}
|
|
2691
|
+
return undefined;
|
|
2692
|
+
}
|
|
2693
|
+
|
|
2694
|
+
/**
|
|
2695
|
+
* Why the acknowledged barrier no longer holds, or `undefined` when it does:
|
|
2696
|
+
* a live worker anywhere in scope, the daemon-side admission acknowledgement
|
|
2697
|
+
* no longer covering the frozen fence (when one is required), or the fence
|
|
2698
|
+
* its own pause/daemon-generation check (the reused lifecycle seam,
|
|
2699
|
+
* {@link restartFenceProblem}) says was lifted or replaced. The acknowledged
|
|
2700
|
+
* quiescence barrier re-runs this after every async boundary it must not
|
|
2701
|
+
* proceed across (#651 review #6).
|
|
2702
|
+
*/
|
|
2703
|
+
function barrierDispute(
|
|
2704
|
+
drain: DrainDeps,
|
|
2705
|
+
scope: UpgradeScope,
|
|
2706
|
+
begun: RestartBegun,
|
|
2707
|
+
requireAck: boolean,
|
|
2708
|
+
): string | undefined {
|
|
2709
|
+
try {
|
|
2710
|
+
const live = liveWorkersInScope(drain, scope);
|
|
2711
|
+
if (live > 0) return `${live} live worker(s) appeared under the setup admission freeze`;
|
|
2712
|
+
if (requireAck && begun.daemon.running) {
|
|
2713
|
+
const problem = fenceAckProblem(drain.admissionAck?.(scope.pauseKey), begun);
|
|
2714
|
+
if (problem !== undefined) return problem;
|
|
2715
|
+
}
|
|
2716
|
+
// The fence's own pause/daemon-generation proof (the reused lifecycle
|
|
2717
|
+
// seam) — wrapped with the worker count so a throwing accessor (an
|
|
2718
|
+
// unknown systemd probe in the identity, say) is a dispute to refuse on,
|
|
2719
|
+
// never an exception escaping with the freeze still written.
|
|
2720
|
+
return restartFenceProblem(drain, scope, begun, "setup");
|
|
2721
|
+
} catch (err) {
|
|
2722
|
+
return err instanceof Error ? err.message : String(err);
|
|
2723
|
+
}
|
|
2724
|
+
}
|
|
2725
|
+
|
|
1976
2726
|
/**
|
|
1977
2727
|
* The post-apply code-graph offer. Shared by the tail and the live-worker early
|
|
1978
2728
|
* return, so neither path can silently leave the graph install unreferenced.
|
|
@@ -1990,6 +2740,7 @@ async function offerCodeGraph(
|
|
|
1990
2740
|
"Clones each index-only checkout as you, installs and enables the reindex timer as root, then seeds one " +
|
|
1991
2741
|
"indexing run so the first fetch happens while you watch. Minutes per repo. " +
|
|
1992
2742
|
"`omp-conductor setup graph` does the same later; `--no-seed` skips the seeding run.",
|
|
2743
|
+
{ key: `offer-code-graph.${project.name}` },
|
|
1993
2744
|
);
|
|
1994
2745
|
if (wantsGraph === true) await graphInstall(project, ui);
|
|
1995
2746
|
else
|
|
@@ -2025,7 +2776,7 @@ export async function setup(
|
|
|
2025
2776
|
}
|
|
2026
2777
|
// `answers` is mutable below: the review loop re-asks one area at a time and
|
|
2027
2778
|
// replaces only the fields that area owns, so everything else carries across.
|
|
2028
|
-
const { amend
|
|
2779
|
+
const { amend } = collected;
|
|
2029
2780
|
let answers = collected.answers;
|
|
2030
2781
|
|
|
2031
2782
|
const scopes = await apply.scopes();
|
|
@@ -2176,7 +2927,12 @@ export async function setup(
|
|
|
2176
2927
|
: "",
|
|
2177
2928
|
d.project.escalation.orchestrator === "external"
|
|
2178
2929
|
? "Dispatch stays paused until the existing arm marker or a new inbound Telegram proof makes the heartbeat live."
|
|
2179
|
-
: "Dispatch resumes after the smoke succeeds.",
|
|
2930
|
+
: "Dispatch resumes after the smoke succeeds and the daemon is proven to serve the config it was just written.",
|
|
2931
|
+
// The acknowledged quiescence barrier (#618): the whole sequence refuses
|
|
2932
|
+
// before the first write while any configured project has a live worker
|
|
2933
|
+
// and until the running daemon has acknowledged the admission fence, so
|
|
2934
|
+
// the consent names the gate the apply runs behind.
|
|
2935
|
+
"Refuses before any write while any configured project has a live worker, and until the running daemon acknowledges the admission fence.",
|
|
2180
2936
|
"Issues are only claimed after every setup gate succeeds.",
|
|
2181
2937
|
]
|
|
2182
2938
|
.filter((s) => s.length > 0)
|
|
@@ -2202,7 +2958,7 @@ export async function setup(
|
|
|
2202
2958
|
amend === undefined ? "Apply this setup?" : `Apply this change to ${AMEND_AREAS[amend.area].name}?`,
|
|
2203
2959
|
REVIEW_CHOICES,
|
|
2204
2960
|
// Bare Enter lands on Review — the one row with no effect.
|
|
2205
|
-
{ initialIndex: 2 },
|
|
2961
|
+
{ key: "setup-review", initialIndex: 2 },
|
|
2206
2962
|
);
|
|
2207
2963
|
if (choice === undefined) {
|
|
2208
2964
|
ui.notify("Setup cancelled — nothing was changed, and the answers were discarded.", "info");
|
|
@@ -2229,7 +2985,7 @@ export async function setup(
|
|
|
2229
2985
|
const pickedArea = await ui.select(
|
|
2230
2986
|
"Edit which area?",
|
|
2231
2987
|
INTERVIEW_AREAS.map((area) => ({ label: area.label })),
|
|
2232
|
-
{ initialIndex: 0 },
|
|
2988
|
+
{ key: "setup-edit-area", initialIndex: 0 },
|
|
2233
2989
|
);
|
|
2234
2990
|
if (pickedArea === undefined) {
|
|
2235
2991
|
ui.notify("Editing cancelled — the answers stand as they were.", "info");
|
|
@@ -2265,87 +3021,525 @@ export async function setup(
|
|
|
2265
3021
|
ui.notify(planBlock(plan), "info");
|
|
2266
3022
|
}
|
|
2267
3023
|
|
|
2268
|
-
//
|
|
2269
|
-
//
|
|
2270
|
-
//
|
|
2271
|
-
//
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
3024
|
+
// ------------------------------------------------------------------ #618 --
|
|
3025
|
+
// The acknowledged quiescence barrier. A setup apply replaces config and
|
|
3026
|
+
// runs a paused-daemon smoke that is itself a dispatcher (it settles rows,
|
|
3027
|
+
// projects labels, salvages workers), and every write below happens
|
|
3028
|
+
// underneath a daemon that may still admit or own workers. The fail-closed
|
|
3029
|
+
// envelope, reusing the same lifecycle/drain seam `upgrade`, `restart` and
|
|
3030
|
+
// `setup host` run on rather than a second counter-only fence: refuse before
|
|
3031
|
+
// the first mutation when any configured project has a live worker; freeze
|
|
3032
|
+
// admission host-wide — through the one global sentinel, never a project's —
|
|
3033
|
+
// and prove the freeze is in force (#552); then acknowledge — re-check every
|
|
3034
|
+
// project in scope under the proven freeze, and prove the fence still holds.
|
|
3035
|
+
// A pass admitted between the first zero-worker observation and that
|
|
3036
|
+
// acknowledgement refuses with the exact pre-entry pause restored and
|
|
3037
|
+
// nothing else written. After the acknowledgement, every awaited mutation/
|
|
3038
|
+
// lifecycle step re-proves the fence and the zero-worker scope before it
|
|
3039
|
+
// acts (a concurrent `resume` during the awaited label I/O is the shape that
|
|
3040
|
+
// used to slip through and orphan an admitted run).
|
|
3041
|
+
//
|
|
3042
|
+
// The pre-acknowledgement refusal paths are all pre-write, so "nothing has
|
|
3043
|
+
// been changed" is literal: no label, no config byte, no brief, no runtime
|
|
3044
|
+
// file, no store write, no smoke and no service mutation. The observation
|
|
3045
|
+
// that gates the barrier is read-only — it can never be the thing that
|
|
3046
|
+
// creates or upgrades the store, because database preparation is itself one
|
|
3047
|
+
// of the mutations the barrier precedes.
|
|
3048
|
+
|
|
3049
|
+
// The exact pre-entry host-global pause sentinel (owner token and timestamp
|
|
3050
|
+
// byte for byte), captured before anything mutates so a refusal — or the end
|
|
3051
|
+
// of a successful apply — restores it verbatim. Setup freezes the whole host
|
|
3052
|
+
// through the one global sentinel, so that is the only path it may change
|
|
3053
|
+
// (review #4); an unreadable pre-existing sentinel fails closed here, before
|
|
3054
|
+
// any change, rather than being read as absence and deleted (review #5).
|
|
3055
|
+
let scope: UpgradeScope;
|
|
3056
|
+
try {
|
|
3057
|
+
scope = setupScope(apply.drain);
|
|
3058
|
+
} catch (err) {
|
|
3059
|
+
ui.notify(
|
|
3060
|
+
`Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
|
|
3061
|
+
"Nothing has been changed; fix the config and re-run setup.",
|
|
3062
|
+
"error",
|
|
3063
|
+
);
|
|
3064
|
+
return false;
|
|
3065
|
+
}
|
|
3066
|
+
const priorPause = capturePause();
|
|
3067
|
+
if (priorPause.kind === "unreadable") {
|
|
3068
|
+
ui.notify(
|
|
3069
|
+
`Setup stopped before writing anything: the pause sentinel at ${priorPause.path} exists but ` +
|
|
3070
|
+
"could not be read — refusing rather than risk overwriting an operator hold. Nothing has been changed.",
|
|
3071
|
+
"error",
|
|
3072
|
+
);
|
|
3073
|
+
return false;
|
|
3074
|
+
}
|
|
3075
|
+
// A malformed sentinel is unreadable-as-state: readable bytes that parse as
|
|
3076
|
+
// no pause instance are still an operator's hold, and rebuilding them as a
|
|
3077
|
+
// fresh setup pause would lose the owner identity the fence preserves. Fail
|
|
3078
|
+
// closed before any change and never delete the file (#650).
|
|
3079
|
+
if (priorPause.kind === "malformed") {
|
|
3080
|
+
ui.notify(
|
|
3081
|
+
`Setup stopped before writing anything: the pause sentinel at ${priorPause.path} exists but is ` +
|
|
3082
|
+
"malformed (not a readable pause instance) — refusing rather than rebuild an operator hold setup " +
|
|
3083
|
+
"cannot prove. Nothing has been changed.",
|
|
3084
|
+
"error",
|
|
3085
|
+
);
|
|
3086
|
+
return false;
|
|
3087
|
+
}
|
|
3088
|
+
|
|
3089
|
+
// First refusal — before ANY durable mutation, the freeze included. A live
|
|
3090
|
+
// worker in any project the daemon serves would be orphaned, salvaged and
|
|
3091
|
+
// requeued by the config write and the paused-daemon smoke that follow. The
|
|
3092
|
+
// observation is read-only (it never creates or upgrades the store) and
|
|
3093
|
+
// fails closed when quiescence cannot be proven — including when the scope
|
|
3094
|
+
// itself cannot be enumerated (an unreadable config is a fault, never bare
|
|
3095
|
+
// scope, review #2) or the daemon identity cannot be proven (an unknown
|
|
3096
|
+
// systemd probe, review #2).
|
|
3097
|
+
let entryWorkers: number;
|
|
3098
|
+
try {
|
|
3099
|
+
entryWorkers = liveWorkersInScope(apply.drain, scope);
|
|
3100
|
+
} catch (err) {
|
|
3101
|
+
ui.notify(
|
|
3102
|
+
`Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
|
|
3103
|
+
"Nothing has been changed; re-run setup once the fleet is provably quiet.",
|
|
3104
|
+
"error",
|
|
3105
|
+
);
|
|
3106
|
+
return false;
|
|
3107
|
+
}
|
|
3108
|
+
if (entryWorkers > 0) {
|
|
3109
|
+
ui.notify(
|
|
3110
|
+
[
|
|
3111
|
+
`Setup stopped before writing anything: ${entryWorkers} live worker(s) run under the active daemon.`,
|
|
3112
|
+
"A setup apply replaces config and runs a paused-daemon smoke that would orphan, salvage and requeue those runs.",
|
|
3113
|
+
"Let the workers finish, then re-run setup — nothing has been changed.",
|
|
3114
|
+
].join("\n"),
|
|
3115
|
+
"error",
|
|
3116
|
+
);
|
|
3117
|
+
return false;
|
|
3118
|
+
}
|
|
3119
|
+
|
|
3120
|
+
// Freeze admission host-wide, then prove the freeze is in force before
|
|
3121
|
+
// anything is re-checked (#552): a pause that cannot be read back as an
|
|
3122
|
+
// instance is a fence that cannot be acknowledged, so the barrier restores
|
|
3123
|
+
// the exact entry pause and refuses instead of proceeding unproven.
|
|
3124
|
+
const initialPaused = apply.drain.layers(scope.pauseKey).paused;
|
|
3125
|
+
if (!initialPaused) apply.drain.setPaused(true, scope.pauseKey);
|
|
3126
|
+
// The sentinel THIS transaction now owns — the bytes it just wrote, or the
|
|
3127
|
+
// pre-entry hold it inherited. Every later restore only touches the disk
|
|
3128
|
+
// while it still matches these exact bytes, so a takeover (a resume, a
|
|
3129
|
+
// newer hold) is preserved rather than overwritten (#651 review #3).
|
|
3130
|
+
const ownedFence = readSentinel(pausedPath(scope.pauseKey));
|
|
3131
|
+
const pauseToken = apply.drain.pauseState(scope.pauseKey);
|
|
3132
|
+
if (pauseToken === undefined) {
|
|
3133
|
+
restorePause(priorPause, ownedFence);
|
|
3134
|
+
ui.notify(
|
|
3135
|
+
"Setup stopped before writing anything: the setup freeze could not be proven and was released. " +
|
|
3136
|
+
"Nothing has been changed; re-run setup once dispatch is quiet.",
|
|
3137
|
+
"error",
|
|
3138
|
+
);
|
|
3139
|
+
return false;
|
|
3140
|
+
}
|
|
3141
|
+
|
|
3142
|
+
// Acknowledge the barrier: every project in scope is re-checked under the
|
|
3143
|
+
// proven freeze, and the fence must still hold — the pause token we proved
|
|
3144
|
+
// and the daemon generation we began with. A worker admitted after the
|
|
3145
|
+
// first zero-worker observation but before this acknowledgement (a dispatch
|
|
3146
|
+
// pass that passed its own pause gate before the freeze landed), a `resume`
|
|
3147
|
+
// that lifted the freeze, or an identity that can no longer be proven (an
|
|
3148
|
+
// unknown systemd probe) refuses here with the exact pre-entry pause
|
|
3149
|
+
// restored and nothing else written.
|
|
3150
|
+
let begun: RestartBegun;
|
|
3151
|
+
try {
|
|
3152
|
+
begun = { pauseToken, daemon: apply.drain.daemonIdentity() };
|
|
3153
|
+
} catch (err) {
|
|
3154
|
+
restorePause(priorPause, ownedFence);
|
|
3155
|
+
ui.notify(
|
|
3156
|
+
`Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
|
|
3157
|
+
"The setup freeze was released; nothing has been changed; re-run setup once the fleet is provably quiet.",
|
|
3158
|
+
"error",
|
|
3159
|
+
);
|
|
3160
|
+
return false;
|
|
3161
|
+
}
|
|
3162
|
+
|
|
3163
|
+
// The daemon-side admission acknowledgement (#651 review #3): the fence is
|
|
3164
|
+
// acknowledged only when the daemon ITSELF has observed it at an admission
|
|
3165
|
+
// boundary and written that observation down — a second synchronous worker
|
|
3166
|
+
// count is not an acknowledgement, because a tick already past its own
|
|
3167
|
+
// pause gate can claim after it and before setup mutates. A running daemon
|
|
3168
|
+
// that has not yet acknowledged is woken to prompt an immediate pass; one
|
|
3169
|
+
// that cannot be reached (its record is missing) or that still has not
|
|
3170
|
+
// acknowledged by the bounded deadline refuses before anything is written.
|
|
3171
|
+
let requireAck = true;
|
|
3172
|
+
if (begun.daemon.running) {
|
|
3173
|
+
let ack = apply.drain.admissionAck?.(scope.pauseKey);
|
|
3174
|
+
let acknowledged = fenceAckProblem(ack, begun) === undefined;
|
|
3175
|
+
if (!acknowledged) {
|
|
3176
|
+
const reachable = livingDaemon();
|
|
3177
|
+
if (reachable !== undefined) {
|
|
3178
|
+
ui.notify(
|
|
3179
|
+
"The running daemon has not yet acknowledged the setup admission fence — waking it to prompt a pass.",
|
|
3180
|
+
"info",
|
|
3181
|
+
);
|
|
3182
|
+
void wakeDaemon(reachable.port);
|
|
3183
|
+
// `OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS` is the #399 test seam: a
|
|
3184
|
+
// regression proving this refusal burns the short test deadline
|
|
3185
|
+
// instead of the production 60s.
|
|
3186
|
+
const deadline =
|
|
3187
|
+
Date.now() + Number(process.env["OMP_CONDUCTOR_TEST_FENCE_ACK_WAIT_MS"] ?? FENCE_ACK_WAIT_MS);
|
|
3188
|
+
while (!acknowledged && Date.now() < deadline) {
|
|
3189
|
+
await apply.drain.sleep(FENCE_ACK_POLL_MS);
|
|
3190
|
+
ack = apply.drain.admissionAck?.(scope.pauseKey);
|
|
3191
|
+
acknowledged = fenceAckProblem(ack, begun) === undefined;
|
|
3192
|
+
}
|
|
3193
|
+
}
|
|
3194
|
+
}
|
|
3195
|
+
if (!acknowledged) {
|
|
3196
|
+
restorePause(priorPause, ownedFence);
|
|
3197
|
+
ui.notify(
|
|
3198
|
+
"Setup stopped before writing anything: the running daemon has not acknowledged the setup admission " +
|
|
3199
|
+
"fence, so quiescence cannot be proven. Nothing has been changed; stop the daemon, or re-run setup " +
|
|
3200
|
+
"once it has acknowledged the fence.",
|
|
3201
|
+
"error",
|
|
3202
|
+
);
|
|
3203
|
+
return false;
|
|
3204
|
+
}
|
|
3205
|
+
}
|
|
3206
|
+
const ackDispute = barrierDispute(apply.drain, scope, begun, requireAck);
|
|
3207
|
+
if (ackDispute !== undefined) {
|
|
3208
|
+
restorePause(priorPause, ownedFence);
|
|
3209
|
+
ui.notify(
|
|
3210
|
+
`Setup stopped before writing anything: ${ackDispute}. ` +
|
|
3211
|
+
"Nothing has been changed; re-run setup once dispatch is quiet.",
|
|
3212
|
+
"error",
|
|
3213
|
+
);
|
|
3214
|
+
return false;
|
|
3215
|
+
}
|
|
3216
|
+
|
|
3217
|
+
// The re-proof every later awaited-mutation/lifecycle step stands on: the
|
|
3218
|
+
// acknowledged fence must still hold (pause owned, acknowledgement current,
|
|
3219
|
+
// daemon generation unchanged) and no worker may have appeared. A concurrent
|
|
3220
|
+
// `resume` while label creation is pending can reopen admission, after which
|
|
3221
|
+
// config/runtime/smoke would proceed with no further token or worker check
|
|
3222
|
+
// and orphan the admitted run (review #6) — so every one of them re-runs
|
|
3223
|
+
// `barrierDispute` and restores the exact entry pause on a dispute.
|
|
3224
|
+
const barrierHolds = (needAck: boolean): string | undefined =>
|
|
3225
|
+
barrierDispute(apply.drain, scope, begun, needAck);
|
|
3226
|
+
|
|
3227
|
+
// The after-acknowledgement crosscheck: a claim that lands after the
|
|
3228
|
+
// acknowledgement and before the first mutation (a daemon tick that passed
|
|
3229
|
+
// its own pause gate before the freeze and reaches its claim now) must
|
|
3230
|
+
// refuse here with NOTHING written — this read is the one that makes the
|
|
3231
|
+
// `[0, 0, 1]` interleaving a pre-write refusal, not a post-store one
|
|
3232
|
+
// (#651 review #3). The acknowledgement itself cannot see it: the ack is
|
|
3233
|
+
// the daemon's word, and this synchronous crosscheck is the last moment
|
|
3234
|
+
// the envelope can stay literal about "nothing has been changed".
|
|
3235
|
+
const preMutationReproof = barrierHolds(requireAck);
|
|
3236
|
+
if (preMutationReproof !== undefined) {
|
|
3237
|
+
restorePause(priorPause, ownedFence);
|
|
3238
|
+
ui.notify(
|
|
3239
|
+
`Setup stopped before writing anything: ${preMutationReproof}. ` +
|
|
3240
|
+
"Nothing has been changed; re-run setup once dispatch is quiet.",
|
|
3241
|
+
"error",
|
|
3242
|
+
);
|
|
3243
|
+
return false;
|
|
3244
|
+
}
|
|
3245
|
+
|
|
3246
|
+
// The one-writer transaction — runs exactly once per setup run, only after
|
|
3247
|
+
// the loop above broke on a final "Apply", and only under the acknowledged
|
|
3248
|
+
// barrier held above. Database preparation is the first mutation, in the
|
|
3249
|
+
// order the envelope promises: barrier first, then store, config,
|
|
3250
|
+
// brief/runtime files, smoke, service activation, tracker labels, the
|
|
3251
|
+
// heartbeat proof last — immediately before the commit. The labels sit
|
|
3252
|
+
// BEFORE the heartbeat proof so the deliberate arm-failure hold keeps the
|
|
3253
|
+
// tracker's planned labels while it holds the fleet (#650, PR #700 review
|
|
3254
|
+
// #3); every rollback path after them compensates exactly through the
|
|
3255
|
+
// inventory, and every failure path before them has created none (#652). A
|
|
3256
|
+
// throw anywhere in this leg, or a re-proof dispute after the first
|
|
3257
|
+
// mutation, rolls the WHOLE inventory back as one unit; the pre-write
|
|
3258
|
+
// refusal paths above are untouched.
|
|
3259
|
+
let created: string[] = [];
|
|
3260
|
+
let briefPath: string | undefined;
|
|
3261
|
+
let runtimeFiles: HostRuntimeWrite = { wrote: [], warnings: [] };
|
|
3262
|
+
let smoke: SetupSmokeResult | undefined;
|
|
3263
|
+
let smokeLine = "";
|
|
2282
3264
|
let restartVia: "systemctl" | "cli" | undefined;
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
3265
|
+
let armLine = "embedded orchestrator — no heartbeat arm marker";
|
|
3266
|
+
|
|
3267
|
+
// The complete pre-entry mutation inventory (#652): every path the apply
|
|
3268
|
+
// can write — config, briefs, staged units/scripts/tick, the AGENTS link,
|
|
3269
|
+
// the run store, the daemon record, the arm marker, the config backups —
|
|
3270
|
+
// with byte-exact pre-entry state, read BEFORE the first mutation so a
|
|
3271
|
+
// failure anywhere in the write/smoke/activation leg can restore all of
|
|
3272
|
+
// them. An unreadable pre-entry path refuses here: nothing has been changed.
|
|
3273
|
+
let inventory: SetupInventory;
|
|
3274
|
+
try {
|
|
3275
|
+
inventory = captureInventory(plan, answers, existing?.projects.map((p) => p.name) ?? []);
|
|
3276
|
+
} catch (err) {
|
|
3277
|
+
// The freeze was already written by this point, so it is released with
|
|
3278
|
+
// the refusal — an unreadable pre-entry path must not leave a setup-owned
|
|
3279
|
+
// sentinel wedging dispatch for every project (review #2).
|
|
3280
|
+
restorePause(priorPause, ownedFence);
|
|
3281
|
+
ui.notify(
|
|
3282
|
+
`Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
|
|
3283
|
+
"The setup freeze was released; nothing has been changed; re-run setup once the fleet is provably quiet.",
|
|
3284
|
+
"error",
|
|
3285
|
+
);
|
|
3286
|
+
return false;
|
|
3287
|
+
}
|
|
3288
|
+
// Whether a daemon was live at entry, captured before the first mutation: a
|
|
3289
|
+
// failed restart leg can stop that daemon without proving a replacement, so
|
|
3290
|
+
// only a run that entered with one must prove the runtime back before
|
|
3291
|
+
// dispatch may resume (#652). The capture lives INSIDE the transaction try
|
|
3292
|
+
// below so its refusal path is covered by the same `finally` that removes
|
|
3293
|
+
// the store snapshot — the byte-consistent capture the inventory just made
|
|
3294
|
+
// must never leak a stray temp copy in the state dir when the identity
|
|
3295
|
+
// cannot be proven (#650, PR #700 review #2).
|
|
3296
|
+
let hadLiveDaemonAtEntry = false;
|
|
3297
|
+
try {
|
|
3298
|
+
try {
|
|
3299
|
+
hadLiveDaemonAtEntry = apply.drain.daemonIdentity().running;
|
|
3300
|
+
} catch (err) {
|
|
3301
|
+
// The freeze was already written by this point, so it is released with
|
|
3302
|
+
// the refusal — an identity that cannot be proven must not leave a
|
|
3303
|
+
// setup-owned sentinel wedging dispatch for every project (review #2).
|
|
3304
|
+
restorePause(priorPause, ownedFence);
|
|
2287
3305
|
ui.notify(
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
"Then run `omp-conductor restart --now`.",
|
|
2292
|
-
plan.project.escalation.orchestrator === "external"
|
|
2293
|
-
? `Run \`omp-conductor arm --project ${plan.project.name}\` if ticks are disarmed, then run \`omp-conductor resume --project ${plan.project.name}\`.`
|
|
2294
|
-
: `Then run \`omp-conductor resume --project ${plan.project.name}\`.`,
|
|
2295
|
-
].join("\n"),
|
|
2296
|
-
"warning",
|
|
3306
|
+
`Setup stopped before writing anything: ${err instanceof Error ? err.message : String(err)}. ` +
|
|
3307
|
+
"The setup freeze was released; nothing has been changed; re-run setup once the fleet is provably quiet.",
|
|
3308
|
+
"error",
|
|
2297
3309
|
);
|
|
2298
|
-
// The early return must not swallow the graph offer. This host is the
|
|
2299
|
-
// live reproduction: the shared reindex units were already installed by
|
|
2300
|
-
// the first project, so only the clone+seed are outstanding — and nothing
|
|
2301
|
-
// above names them. Offer regardless of live workers: cloning seeds and
|
|
2302
|
-
// enables the timer, which does not touch the running daemon.
|
|
2303
|
-
await offerCodeGraph(plan.project, ui, apply.graphInstall);
|
|
2304
|
-
ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
|
|
2305
3310
|
return false;
|
|
2306
3311
|
}
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
3312
|
+
openStore(dbPath()).close();
|
|
3313
|
+
saveConfig(plan.nextConfig);
|
|
3314
|
+
briefPath = answers.writeOrchestratorBrief ? writeOrchestratorBrief(answers, prose) : undefined;
|
|
3315
|
+
runtimeFiles = writeHostRuntime(plan.runtime);
|
|
3316
|
+
for (const warning of runtimeFiles.warnings) ui.notify(warning, "warning");
|
|
3317
|
+
|
|
3318
|
+
// Re-prove before the smoke: the paused-daemon smoke is itself a dispatcher,
|
|
3319
|
+
// so it must never run under a reopened fence or beside a worker admitted
|
|
3320
|
+
// since the acknowledgement (review #6). The store/config/brief/runtime
|
|
3321
|
+
// writes above are rolled back with the whole inventory (review #652).
|
|
3322
|
+
const smokeReproof = barrierHolds(requireAck);
|
|
3323
|
+
if (smokeReproof !== undefined) {
|
|
3324
|
+
const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
|
|
3325
|
+
restorePause(priorPause, ownedFence);
|
|
2312
3326
|
ui.notify(
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
"info",
|
|
3327
|
+
`Setup stopped before the daemon smoke: ${smokeReproof}. ` +
|
|
3328
|
+
(restoreFailures.length === 0
|
|
3329
|
+
? "The prior state was restored."
|
|
3330
|
+
: `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
|
|
3331
|
+
"error",
|
|
2319
3332
|
);
|
|
2320
|
-
|
|
3333
|
+
return false;
|
|
3334
|
+
}
|
|
3335
|
+
smoke = await withProgress("Running setup smoke", "Setup smoke ran", () =>
|
|
3336
|
+
apply.smoke(plan.project.name),
|
|
3337
|
+
);
|
|
3338
|
+
|
|
3339
|
+
// Re-prove AFTER the awaited smoke too (review #2): the smoke is itself a
|
|
3340
|
+
// dispatcher, and a fence resumed while it ran — or a worker it admitted —
|
|
3341
|
+
// must stop the restart and heartbeat actions that follow. The restart,
|
|
3342
|
+
// if any, is the very next action, so this single re-proof covers it.
|
|
3343
|
+
const postSmokeReproof = barrierHolds(requireAck);
|
|
3344
|
+
if (postSmokeReproof !== undefined) {
|
|
3345
|
+
const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
|
|
3346
|
+
restorePause(priorPause, ownedFence);
|
|
3347
|
+
ui.notify(
|
|
3348
|
+
`Setup stopped after the daemon smoke: ${postSmokeReproof}. ` +
|
|
3349
|
+
(restoreFailures.length === 0
|
|
3350
|
+
? "The prior state was restored."
|
|
3351
|
+
: `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
|
|
3352
|
+
"error",
|
|
3353
|
+
);
|
|
3354
|
+
return false;
|
|
3355
|
+
}
|
|
3356
|
+
|
|
3357
|
+
smokeLine =
|
|
3358
|
+
smoke.mode === "temporary"
|
|
3359
|
+
? `paused daemon --once; temporary /healthz on :${smoke.daemon.port}; ` +
|
|
3360
|
+
`stored status for ${smoke.status.project}`
|
|
3361
|
+
: // An existing daemon (record or active unit) never ran a `--once` tick
|
|
3362
|
+
// and never staged a temporary one — the restart line below replaces
|
|
3363
|
+
// this one whenever the restart runs.
|
|
3364
|
+
`existing daemon; stored status for ${smoke.status.project}`;
|
|
3365
|
+
if (smoke.mode === "existing") {
|
|
3366
|
+
// The barrier acknowledged zero workers in every project under a proven
|
|
3367
|
+
// host-wide freeze before the first write, so a live worker here can only
|
|
3368
|
+
// mean the fence itself was bypassed. The old soft return — applied config
|
|
3369
|
+
// and a wedged setup pause left underneath that worker, with a "let them
|
|
3370
|
+
// finish" note — is the exact incident shape (#618) and is refused instead,
|
|
3371
|
+
// with the whole inventory rolled back (review #2, #652).
|
|
3372
|
+
if (smoke.status.liveWorkers > 0) {
|
|
3373
|
+
const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
|
|
3374
|
+
restorePause(priorPause, ownedFence);
|
|
3375
|
+
ui.notify(
|
|
3376
|
+
`${smoke.status.liveWorkers} live worker(s) appeared under the setup admission freeze — ` +
|
|
3377
|
+
"refusing to apply over a bypassed fence. " +
|
|
3378
|
+
(restoreFailures.length === 0
|
|
3379
|
+
? "The prior state was restored."
|
|
3380
|
+
: `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
|
|
3381
|
+
"error",
|
|
3382
|
+
);
|
|
3383
|
+
return false;
|
|
3384
|
+
}
|
|
3385
|
+
// The added-project contract (#650): the newly loaded runtime must be
|
|
3386
|
+
// proven to serve the added project BEFORE setup reports success — and
|
|
3387
|
+
// more generally, an "existing" smoke means a daemon is serving the
|
|
3388
|
+
// config that was just replaced, so it must reload the applied config
|
|
3389
|
+
// through the lifecycle seam. The deferral to a manual `restart --now`
|
|
3390
|
+
// an operator may never run left dispatch ready on a daemon that did
|
|
3391
|
+
// not serve the config it was just written, so it is gone: every
|
|
3392
|
+
// existing-mode apply restarts, even when the smoke proved health, and
|
|
3393
|
+
// the seam's `waitForOwnedDaemon` proves MainPID + /healthz + that the
|
|
3394
|
+
// (added) project is served before the apply reports success.
|
|
2321
3395
|
const restarted = await apply.restart({ project: plan.project.name });
|
|
2322
3396
|
restartVia = restarted.via;
|
|
2323
3397
|
smokeLine =
|
|
2324
3398
|
`existing /healthz and stored status; restarted through ${restarted.via}; ` +
|
|
2325
3399
|
`new /healthz on :${restarted.record.port}`;
|
|
3400
|
+
// The intentional restart is the generation boundary: from here the
|
|
3401
|
+
// fence protects the daemon we just started, not the one we began with.
|
|
3402
|
+
// Refresh the acknowledged world — pause re-proven, identity re-captured
|
|
3403
|
+
// — and drop the acknowledgement requirement, because a daemon this
|
|
3404
|
+
// apply itself started cannot have been mid-claim when the fence landed
|
|
3405
|
+
// (its first admission boundary is the tick gate, above routing); its
|
|
3406
|
+
// own first gate re-acknowledges on the next pass. A fence that vanished
|
|
3407
|
+
// while the restart ran (a concurrent resume) refuses here instead of
|
|
3408
|
+
// proceeding under an unproven freeze (review #3).
|
|
3409
|
+
const refreshedPause = apply.drain.pauseState(scope.pauseKey);
|
|
3410
|
+
if (refreshedPause === undefined) {
|
|
3411
|
+
const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
|
|
3412
|
+
restorePause(priorPause, ownedFence);
|
|
3413
|
+
ui.notify(
|
|
3414
|
+
"Setup stopped after the daemon restart: the setup pause was lifted while the daemon restarted, so " +
|
|
3415
|
+
"the acknowledged world no longer exists. " +
|
|
3416
|
+
(restoreFailures.length === 0
|
|
3417
|
+
? "The prior state was restored."
|
|
3418
|
+
: `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
|
|
3419
|
+
"error",
|
|
3420
|
+
);
|
|
3421
|
+
return false;
|
|
3422
|
+
}
|
|
3423
|
+
begun = { pauseToken: refreshedPause, daemon: apply.drain.daemonIdentity() };
|
|
3424
|
+
requireAck = false;
|
|
2326
3425
|
}
|
|
2327
|
-
}
|
|
2328
3426
|
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
3427
|
+
// The tracker labels — before the heartbeat proof, so the deliberate
|
|
3428
|
+
// arm-failure hold (which keeps its writes by design) leaves the tracker
|
|
3429
|
+
// with the planned labels instead of a config whose routing keys and
|
|
3430
|
+
// queue label do not exist yet (#650, PR #700 review #3). Every rollback
|
|
3431
|
+
// path AFTER this point compensates them exactly through the inventory;
|
|
3432
|
+
// every failure path before it has created none.
|
|
3433
|
+
created = await withProgress("Creating tracker labels", "Tracker labels ready", () =>
|
|
3434
|
+
apply.createLabels(answers.trackerRepo, plan.labels),
|
|
3435
|
+
);
|
|
3436
|
+
|
|
3437
|
+
if (plan.project.escalation.orchestrator === "external") {
|
|
2335
3438
|
ui.notify(
|
|
2336
|
-
|
|
2337
|
-
"Setup
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
3439
|
+
smoke.healthProven
|
|
3440
|
+
? "Setup smoke passed. Proving the external heartbeat channel…"
|
|
3441
|
+
: "Setup smoke staged — the restart proved /healthz. Proving the external heartbeat channel…",
|
|
3442
|
+
"info",
|
|
3443
|
+
);
|
|
3444
|
+
try {
|
|
3445
|
+
armLine = await apply.arm(plan.project.name);
|
|
3446
|
+
} catch (err) {
|
|
3447
|
+
ui.notify(
|
|
3448
|
+
[
|
|
3449
|
+
"Setup files passed the paused daemon smoke, but the fleet remains held.",
|
|
3450
|
+
created.length > 0 ? `Tracker labels were created (${created.join(", ")}).` : "Planned labels already existed.",
|
|
3451
|
+
err instanceof Error ? err.message : String(err),
|
|
3452
|
+
`Start the external orchestrator in ${plan.project.workspaceRoot}, then run \`omp-conductor arm --project ${plan.project.name}\`.`,
|
|
3453
|
+
"After the arm proof succeeds, run `omp-conductor resume`.",
|
|
3454
|
+
].join("\n"),
|
|
3455
|
+
"warning",
|
|
3456
|
+
);
|
|
3457
|
+
ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
|
|
3458
|
+
return false;
|
|
3459
|
+
}
|
|
3460
|
+
}
|
|
3461
|
+
|
|
3462
|
+
// The final barrier boundary: the awaited label and heartbeat I/O are the
|
|
3463
|
+
// last steps before the commit, and a fence resumed while either ran — or
|
|
3464
|
+
// a worker admitted — must refuse with the tracker compensated and every
|
|
3465
|
+
// captured path restored.
|
|
3466
|
+
const finalReproof = barrierHolds(requireAck);
|
|
3467
|
+
if (finalReproof !== undefined) {
|
|
3468
|
+
const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, created, apply.deleteLabels);
|
|
3469
|
+
restorePause(priorPause, ownedFence);
|
|
3470
|
+
ui.notify(
|
|
3471
|
+
`Setup stopped before committing: ${finalReproof}. ` +
|
|
3472
|
+
(restoreFailures.length === 0
|
|
3473
|
+
? "The prior state was restored."
|
|
3474
|
+
: `Rollback could not fully restore the prior state: ${restoreFailures.join("; ")}`),
|
|
3475
|
+
"error",
|
|
2343
3476
|
);
|
|
2344
|
-
ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
|
|
2345
3477
|
return false;
|
|
2346
3478
|
}
|
|
3479
|
+
} catch (err) {
|
|
3480
|
+
// Roll the whole apply back as one transaction (#652): every captured
|
|
3481
|
+
// path restored byte-for-byte (or removed on a first install), the
|
|
3482
|
+
// tracker compensated exactly, and — because a failed restart leg may
|
|
3483
|
+
// have stopped the entry daemon without proving a replacement — a healthy
|
|
3484
|
+
// daemon proven to serve the restored config before dispatch may resume.
|
|
3485
|
+
// If that proof cannot be made, stay fail-closed: a setup hold stays in
|
|
3486
|
+
// force with the recovery command named, and the thrown error reports the
|
|
3487
|
+
// first restoration failure honestly rather than implying the prior state
|
|
3488
|
+
// is coherent.
|
|
3489
|
+
//
|
|
3490
|
+
// The labels created before a mid-way createLabels throw ride on the
|
|
3491
|
+
// error (createMissingLabels attaches its partial list), so compensation
|
|
3492
|
+
// stays exact even when the tracker step itself failed.
|
|
3493
|
+
const createdLabels = created.length > 0 ? created : ((err as { created?: string[] }).created ?? []);
|
|
3494
|
+
const restoreFailures = await restoreInventory(inventory, answers.trackerRepo, createdLabels, apply.deleteLabels);
|
|
3495
|
+
let runtimeProved = !hadLiveDaemonAtEntry;
|
|
3496
|
+
if (hadLiveDaemonAtEntry) {
|
|
3497
|
+
// The proof targets the PRE-ENTRY configured set the rollback restored,
|
|
3498
|
+
// never the rejected generation's: the restarted runtime must be proven
|
|
3499
|
+
// to serve every project the prior runtime served (#650).
|
|
3500
|
+
runtimeProved = await apply.proveRuntime(inventory.priorProjects).catch(() => false);
|
|
3501
|
+
}
|
|
3502
|
+
// The pause ownership rule (#651 review #3): only the fence this
|
|
3503
|
+
// transaction froze is undone; a takeover stands.
|
|
3504
|
+
restorePause(priorPause, ownedFence);
|
|
3505
|
+
const failures = [...restoreFailures];
|
|
3506
|
+
if (!runtimeProved) {
|
|
3507
|
+
// Fail-closed: only an entry that was actually unpaused needs a hold
|
|
3508
|
+
// re-asserted — an entry already held keeps that exact sentinel, and a
|
|
3509
|
+
// takeover that replaced the fence is preserved, never overwritten.
|
|
3510
|
+
// The re-asserted hold is ALWAYS the host-global sentinel, never a
|
|
3511
|
+
// project-scoped one: a fresh `paused-<project>` written under a
|
|
3512
|
+
// failure that entered under a global pause is the silent fake this
|
|
3513
|
+
// slice exists to forbid — when the global hold later clears, the
|
|
3514
|
+
// hidden project pause would remain and deadlock the fleet (#650).
|
|
3515
|
+
if (priorPause.kind === "absent" && readSentinel(pausedPath(scope.pauseKey)).kind === "absent") {
|
|
3516
|
+
setPaused(true, {
|
|
3517
|
+
source: pauseSourceToken("setup"),
|
|
3518
|
+
reason: "setup apply failed and no healthy daemon could be proven",
|
|
3519
|
+
});
|
|
3520
|
+
}
|
|
3521
|
+
failures.push(
|
|
3522
|
+
`no healthy daemon could be proven after the rollback — the fleet stays held; ` +
|
|
3523
|
+
`run \`omp-conductor restart --project ${plan.project.name}\` to recover`,
|
|
3524
|
+
);
|
|
3525
|
+
}
|
|
3526
|
+
if (failures.length > 0) {
|
|
3527
|
+
throw new Error(
|
|
3528
|
+
`setup apply failed (${err instanceof Error ? err.message : String(err)}) and the prior state could not ` +
|
|
3529
|
+
`be fully restored: ${failures.join("; ")}`,
|
|
3530
|
+
);
|
|
3531
|
+
}
|
|
3532
|
+
throw err;
|
|
3533
|
+
} finally {
|
|
3534
|
+
// The byte-consistent store snapshot exists only for this transaction.
|
|
3535
|
+
if (inventory.store.kind === "snapshot") rmSync(inventory.store.snapshotPath!, { force: true });
|
|
2347
3536
|
}
|
|
2348
|
-
|
|
3537
|
+
// Lift the setup freeze and restore the exact pre-entry pause: an entry that
|
|
3538
|
+
// was already paused (an operator hold, say) keeps that sentinel byte for
|
|
3539
|
+
// byte, and an entry that was unpaused resumes dispatch exactly as the old
|
|
3540
|
+
// `resume` did. A takeover that replaced the setup fence meanwhile is
|
|
3541
|
+
// preserved, never overwritten (#651 review #3).
|
|
3542
|
+
restorePause(priorPause, ownedFence);
|
|
2349
3543
|
|
|
2350
3544
|
ui.notify(
|
|
2351
3545
|
[
|
|
@@ -2357,7 +3551,7 @@ export async function setup(
|
|
|
2357
3551
|
runtimeFiles.wrote.length === 0
|
|
2358
3552
|
? "Host runtime files were already current."
|
|
2359
3553
|
: `Wrote host runtime file(s): ${runtimeFiles.wrote.join(", ")}`,
|
|
2360
|
-
`Smoke passed: ${smokeLine}.`,
|
|
3554
|
+
`Smoke ${smoke.healthProven ? "passed" : "staged; the restart proved /healthz"}: ${smokeLine}.`,
|
|
2361
3555
|
`Heartbeat: ${armLine}.`,
|
|
2362
3556
|
"",
|
|
2363
3557
|
"Use the documented toy-issue drill to prove one complete worker path.",
|
|
@@ -2392,6 +3586,7 @@ export async function setup(
|
|
|
2392
3586
|
`${plan.runtime.installedAction === "create" ? "Installs" : "Updates"} ${plan.runtime.installedPath} from ` +
|
|
2393
3587
|
`${plan.runtime.service.path}, then enables and restarts it. Needs root, one step at a time, and shows every ` +
|
|
2394
3588
|
"command before it runs. Skipping is fine — `omp-conductor setup host` does exactly this later.",
|
|
3589
|
+
{ key: `install-daemon.${plan.project.name}` },
|
|
2395
3590
|
);
|
|
2396
3591
|
if (install === true)
|
|
2397
3592
|
await apply.hostInstall(
|