omp-conductor 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/REFERENCE.md +18 -12
- package/package.json +2 -1
- package/schema/config.schema.json +16 -0
- package/src/admission.ts +159 -43
- package/src/availability.ts +27 -1
- package/src/briefs/worker.md +2 -0
- package/src/clack-ui.ts +83 -0
- package/src/command-manifest.ts +16 -7
- package/src/commands/arm.ts +11 -3
- package/src/commands/decision.ts +17 -7
- package/src/commands/doctor.ts +18 -1
- package/src/commands/hold.ts +9 -7
- package/src/commands/ledger.ts +25 -4
- package/src/commands/message.ts +32 -4
- package/src/commands/setup.ts +61 -10
- package/src/commands/stats.ts +9 -5
- package/src/commands/status.ts +32 -5
- package/src/commands/tail.ts +13 -1
- package/src/commands/watch.ts +16 -7
- package/src/config-schema.ts +20 -0
- package/src/config.ts +37 -0
- package/src/daemon.ts +1240 -18
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +56 -13
- package/src/fleet.ts +224 -47
- package/src/gitops.ts +103 -24
- package/src/lifecycle.ts +7 -2
- package/src/orchestrator-tick.ts +372 -157
- package/src/privileged.ts +3 -0
- package/src/release-policy.ts +177 -5
- package/src/setup-answers.ts +135 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +2 -0
- package/src/setup-probe.ts +1 -0
- package/src/setup-wizard.ts +1296 -101
- package/src/setup.ts +60 -3
- package/src/status-render.ts +11 -1
- package/src/store.ts +333 -12
- package/src/tracker/github.ts +562 -13
- package/src/types.ts +204 -2
- package/src/ui/progress.ts +32 -0
- package/src/ui/style.ts +11 -0
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +212 -11
- package/src/wizard-ui.ts +14 -5
- package/src/worker.ts +26 -0
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/privileged.ts
CHANGED
|
@@ -88,6 +88,8 @@ export interface RunPrivilegedOptions {
|
|
|
88
88
|
deps?: PrivilegedDeps;
|
|
89
89
|
/** Confirm title. Defaults to a generic one; callers name their verb. */
|
|
90
90
|
title?: string;
|
|
91
|
+
/** Stable answer-file key for this batch's confirmation. */
|
|
92
|
+
answerKey?: string;
|
|
91
93
|
/** Extra lines shown above the step list — what this batch is for. */
|
|
92
94
|
preamble?: readonly string[];
|
|
93
95
|
/**
|
|
@@ -201,6 +203,7 @@ export async function runPrivileged(
|
|
|
201
203
|
const go = await ui.confirm(
|
|
202
204
|
options.title ?? "Run these steps now?",
|
|
203
205
|
`${steps.length} step(s), in the order shown. Anything that fails stops the rest and prints what is left.`,
|
|
206
|
+
{ key: options.answerKey ?? "run-privileged-steps" },
|
|
204
207
|
);
|
|
205
208
|
// `undefined` is a dismissal rather than a "no", but for a batch that has not
|
|
206
209
|
// started they mean the same thing: run nothing.
|
package/src/release-policy.ts
CHANGED
|
@@ -280,7 +280,7 @@ function wholePackageBunTest(segment: string): boolean {
|
|
|
280
280
|
* the host were all plain invocations, and refusing the sanctioned parse check
|
|
281
281
|
* would teach workers to skip it rather than stop the load.
|
|
282
282
|
*/
|
|
283
|
-
const SHARED_HOST_SCRIPTS = [
|
|
283
|
+
export const SHARED_HOST_SCRIPTS: readonly string[] = [
|
|
284
284
|
"herdr/test/recover-test.sh",
|
|
285
285
|
"test/setup-test.sh",
|
|
286
286
|
"setup.sh",
|
|
@@ -298,6 +298,147 @@ function sharedHostScriptMatch(segment: string): string | undefined {
|
|
|
298
298
|
return undefined;
|
|
299
299
|
}
|
|
300
300
|
|
|
301
|
+
/** The `git [-C <dir>] tag` prefix; the tail after it decides read vs write. */
|
|
302
|
+
const GIT_TAG_HEAD = /^git(?:\s+-[Cc]\s+\S+)*\s+tag(?=\s|$)/;
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Whether a segment that opens with `git [-C <dir>] tag` is a write — tag
|
|
306
|
+
* creation or deletion — rather than a read. The tripwire used to match the
|
|
307
|
+
* `git tag` prefix alone (#696), so a worker orienting itself with
|
|
308
|
+
* `git tag | head -5` was refused as an attempted release and the refusal
|
|
309
|
+
* reached the digest as release-policy drift. Git writes tags in exactly two
|
|
310
|
+
* shapes, and every other form is a read:
|
|
311
|
+
*
|
|
312
|
+
* - creation needs a tagname operand: `git tag <name>` and the
|
|
313
|
+
* `-a`/`-s`/`-u`/`-t`/`-m`/`-F`/`-f` forms;
|
|
314
|
+
* - deletion needs `-d`/`--delete`;
|
|
315
|
+
* - bare `git tag`, `-l`/`--list` (with an optional pattern), `-n`,
|
|
316
|
+
* `--contains`, `--points-at`, `--sort`, `--merged`/`--no-merged`,
|
|
317
|
+
* `--format`, `--column` and `-v`/`--verify` all only read.
|
|
318
|
+
*
|
|
319
|
+
* A creation flag with no name is still gated: git rejects the command, but a
|
|
320
|
+
* create attempt is not a read, and the tripwire errs toward refusing a
|
|
321
|
+
* release act.
|
|
322
|
+
*/
|
|
323
|
+
function gitTagIsWrite(segment: string): boolean {
|
|
324
|
+
const head = GIT_TAG_HEAD.exec(segment);
|
|
325
|
+
if (head === null) return false;
|
|
326
|
+
const tokens = segment.slice(head[0].length).trim().split(/\s+/).filter((token) => token.length > 0);
|
|
327
|
+
|
|
328
|
+
let deletion = false; // -d / --delete
|
|
329
|
+
let listRead = false; // a read-only list flag
|
|
330
|
+
let verifyRead = false; // -v / --verify
|
|
331
|
+
let createSignal = false; // -a / -s / -f / -m / -u / -t / -F
|
|
332
|
+
let hasOperand = false; // a positional tagname, pattern or commit
|
|
333
|
+
|
|
334
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
335
|
+
const token = tokens[i]!;
|
|
336
|
+
// A `--` ends option parsing: whatever follows is a plain operand, the
|
|
337
|
+
// created or deleted tag's name.
|
|
338
|
+
if (token === "--") {
|
|
339
|
+
hasOperand = true;
|
|
340
|
+
break;
|
|
341
|
+
}
|
|
342
|
+
if (token.startsWith("--")) {
|
|
343
|
+
const flag = token.includes("=") ? token.slice(0, token.indexOf("=")) : token;
|
|
344
|
+
switch (flag) {
|
|
345
|
+
case "--delete":
|
|
346
|
+
deletion = true;
|
|
347
|
+
break;
|
|
348
|
+
case "--list":
|
|
349
|
+
case "--contains":
|
|
350
|
+
case "--no-contains":
|
|
351
|
+
case "--points-at":
|
|
352
|
+
case "--merged":
|
|
353
|
+
case "--no-merged":
|
|
354
|
+
case "--sort":
|
|
355
|
+
case "--format":
|
|
356
|
+
case "--column":
|
|
357
|
+
case "--no-column":
|
|
358
|
+
case "--color":
|
|
359
|
+
case "--no-color":
|
|
360
|
+
case "--ignore-case":
|
|
361
|
+
case "--no-ignore-case":
|
|
362
|
+
listRead = true;
|
|
363
|
+
break;
|
|
364
|
+
case "--verify":
|
|
365
|
+
verifyRead = true;
|
|
366
|
+
break;
|
|
367
|
+
case "--annotate":
|
|
368
|
+
case "--sign":
|
|
369
|
+
case "--force":
|
|
370
|
+
createSignal = true;
|
|
371
|
+
break;
|
|
372
|
+
case "--local-user":
|
|
373
|
+
case "--object":
|
|
374
|
+
case "--message":
|
|
375
|
+
case "--file":
|
|
376
|
+
// The flag's operand is part of the create parameters, not a
|
|
377
|
+
// tagname; `--flag value` must not be read as `--flag` plus a name.
|
|
378
|
+
createSignal = true;
|
|
379
|
+
if (!token.includes("=")) i++;
|
|
380
|
+
break;
|
|
381
|
+
default:
|
|
382
|
+
break; // an unknown flag: no conclusion either way
|
|
383
|
+
}
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
if (token.startsWith("-") && token.length > 1) {
|
|
387
|
+
// Combined short options (`-an5` is `-a -n 5`). An option with a
|
|
388
|
+
// required operand takes the rest of its token or the next token.
|
|
389
|
+
let consumeNext = false;
|
|
390
|
+
for (let j = 1; j < token.length; j++) {
|
|
391
|
+
const flag = token[j];
|
|
392
|
+
switch (flag) {
|
|
393
|
+
case "d":
|
|
394
|
+
deletion = true;
|
|
395
|
+
break;
|
|
396
|
+
case "a":
|
|
397
|
+
case "s":
|
|
398
|
+
case "f":
|
|
399
|
+
createSignal = true;
|
|
400
|
+
break;
|
|
401
|
+
case "l":
|
|
402
|
+
listRead = true;
|
|
403
|
+
break;
|
|
404
|
+
case "v":
|
|
405
|
+
verifyRead = true;
|
|
406
|
+
break;
|
|
407
|
+
case "n":
|
|
408
|
+
// `-n[<num>]` — the digit run belongs to the flag, so `-n5`
|
|
409
|
+
// is one token and never an operand.
|
|
410
|
+
listRead = true;
|
|
411
|
+
while (j + 1 < token.length && token[j + 1]! >= "0" && token[j + 1]! <= "9") j++;
|
|
412
|
+
break;
|
|
413
|
+
case "m":
|
|
414
|
+
case "u":
|
|
415
|
+
case "t":
|
|
416
|
+
case "F":
|
|
417
|
+
createSignal = true;
|
|
418
|
+
if (j + 1 < token.length) {
|
|
419
|
+
j = token.length;
|
|
420
|
+
} else {
|
|
421
|
+
consumeNext = true;
|
|
422
|
+
}
|
|
423
|
+
break;
|
|
424
|
+
default:
|
|
425
|
+
break; // an unknown short flag: no conclusion either way
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
if (consumeNext) i++;
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
// A plain operand: with a list flag it is a pattern or commit for the
|
|
432
|
+
// read; without one it is the tagname a creation needs.
|
|
433
|
+
hasOperand = true;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
if (deletion) return true;
|
|
437
|
+
if (listRead || verifyRead) return false;
|
|
438
|
+
if (hasOperand) return true;
|
|
439
|
+
return createSignal;
|
|
440
|
+
}
|
|
441
|
+
|
|
301
442
|
/**
|
|
302
443
|
* Classify a shell command and the segment that fired. The segment is what a
|
|
303
444
|
* triager reads: `bun test src/foo.test.ts && npm publish` matches on its
|
|
@@ -323,7 +464,7 @@ function releaseCommandMatch(command: string): { shape: GateShape; matched: stri
|
|
|
323
464
|
// any future spelling under the upgrade family all need the install shape,
|
|
324
465
|
// and a session without that grant is refused before the shell runs.
|
|
325
466
|
if (/^omp-conductor\s+upgrade(?:\b|$)/.test(segment)) return { shape: "install", matched: segment };
|
|
326
|
-
if (
|
|
467
|
+
if (gitTagIsWrite(segment)) return { shape: "git-tag", matched: segment };
|
|
327
468
|
if (
|
|
328
469
|
/^git(?:\s+-[Cc]\s+\S+)*\s+push\b/.test(segment) &&
|
|
329
470
|
GIT_PUSH_TAG_SHAPE.test(segment)
|
|
@@ -513,8 +654,10 @@ export function releaseShapeFromTool(
|
|
|
513
654
|
/**
|
|
514
655
|
* The refusal wording for the shared-host gate (#428). The whole-package
|
|
515
656
|
* `bun test` form names the focused alternative (a guard that only denies
|
|
516
|
-
* teaches nothing and gets worked around);
|
|
517
|
-
*
|
|
657
|
+
* teaches nothing and gets worked around); a refused shell suite names the
|
|
658
|
+
* script that fired and the sanctioned `bash -n` parse check (#687), because
|
|
659
|
+
* there is no focused form of a suite script — the alternative is to drop the
|
|
660
|
+
* execution segment and keep the allowed prefix, not to run something else.
|
|
518
661
|
*/
|
|
519
662
|
function sharedHostRefusalReason(matched: string | undefined): string {
|
|
520
663
|
if (matched !== undefined && /^bun\s+test\b/.test(matched)) {
|
|
@@ -524,10 +667,39 @@ function sharedHostRefusalReason(matched: string | undefined): string {
|
|
|
524
667
|
"Run a focused `bun test <file>.test.ts` instead."
|
|
525
668
|
);
|
|
526
669
|
}
|
|
670
|
+
if (matched !== undefined) {
|
|
671
|
+
return (
|
|
672
|
+
`Blocked by sharedHostPolicy: \`${matched}\` is a shared-host shell suite and not a worker's proof ` +
|
|
673
|
+
"path on this shared host (it overloads the 4-core VPS that also runs Langfuse and the fleet). " +
|
|
674
|
+
`\`bash -n ${matched}\` is the sanctioned local check — parsing never executes the script — and CI's ` +
|
|
675
|
+
"`herdr plugin + installer shell suites` job owns executing it. Drop the segment that fired and keep " +
|
|
676
|
+
"the `bash -n` parse gate."
|
|
677
|
+
);
|
|
678
|
+
}
|
|
527
679
|
return (
|
|
528
680
|
"Blocked by sharedHostPolicy: this shell suite is not a worker's proof path on this shared host " +
|
|
529
681
|
"(it overloads the 4-core VPS that also runs Langfuse and the fleet). " +
|
|
530
|
-
"
|
|
682
|
+
"`bash -n <path>` is the sanctioned local check; CI's `herdr plugin + installer shell suites` job owns " +
|
|
683
|
+
"executing it."
|
|
684
|
+
);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* The worker-brief notice for the shared-host gate (#687): the guarded suite
|
|
689
|
+
* paths and the sanctioned local check, derived from
|
|
690
|
+
* {@link SHARED_HOST_SCRIPTS} so the rendered brief and the refusal can never
|
|
691
|
+
* disagree about what is guarded. One line and non-empty whenever the list is:
|
|
692
|
+
* the brief's rendered line count must equal its template's, which the
|
|
693
|
+
* buildBrief tests pin, so the notice must never carry an internal newline.
|
|
694
|
+
*/
|
|
695
|
+
export function sharedHostBriefNotice(): string {
|
|
696
|
+
if (SHARED_HOST_SCRIPTS.length === 0) return "";
|
|
697
|
+
const paths = SHARED_HOST_SCRIPTS.map((script) => `\`${script}\``).join(", ");
|
|
698
|
+
return (
|
|
699
|
+
`**Shared-host guard:** ${paths} are never executed on this shared host — \`bash -n\` on them is the ` +
|
|
700
|
+
"sanctioned local check (parsing never executes), and CI's `herdr plugin + installer shell suites` job " +
|
|
701
|
+
"owns executing them. A command that executes one is refused, and the refusal names the segment; drop " +
|
|
702
|
+
"that segment and keep the `bash -n` parse gate."
|
|
531
703
|
);
|
|
532
704
|
}
|
|
533
705
|
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
import type { PromptOptions, SelectOptions, TerminalUi, WizardUi } from "./wizard-ui.ts";
|
|
5
|
+
|
|
6
|
+
const answerFileSchema = z.record(z.string(), z.union([z.string(), z.boolean()]));
|
|
7
|
+
|
|
8
|
+
export type SetupAnswerValues = Record<string, string | boolean>;
|
|
9
|
+
|
|
10
|
+
function promptName(title: string, options: PromptOptions): string {
|
|
11
|
+
return `answer "${options.key}" for "${title}"`;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function requiredAnswer(answers: SetupAnswerValues, title: string, options: PromptOptions): string | boolean {
|
|
15
|
+
if (!Object.hasOwn(answers, options.key)) {
|
|
16
|
+
throw new Error(`missing ${promptName(title, options)}; add it to the --answers file`);
|
|
17
|
+
}
|
|
18
|
+
return answers[options.key]!;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** A prompt-free WizardUi backed only by validated answers. */
|
|
22
|
+
export function answersUi(
|
|
23
|
+
answers: SetupAnswerValues,
|
|
24
|
+
notify: WizardUi["notify"] = (message) => process.stdout.write(`${message}\n`),
|
|
25
|
+
): TerminalUi {
|
|
26
|
+
return {
|
|
27
|
+
notify,
|
|
28
|
+
close: () => {},
|
|
29
|
+
async confirm(title, _message, options) {
|
|
30
|
+
const answer = requiredAnswer(answers, title, options);
|
|
31
|
+
if (typeof answer !== "boolean") {
|
|
32
|
+
throw new Error(`${promptName(title, options)} must be a boolean`);
|
|
33
|
+
}
|
|
34
|
+
return answer;
|
|
35
|
+
},
|
|
36
|
+
async input(title, _placeholder, options) {
|
|
37
|
+
const answer = requiredAnswer(answers, title, options);
|
|
38
|
+
if (typeof answer !== "string") {
|
|
39
|
+
throw new Error(`${promptName(title, options)} must be a string`);
|
|
40
|
+
}
|
|
41
|
+
return answer;
|
|
42
|
+
},
|
|
43
|
+
async select(title, choices, options) {
|
|
44
|
+
const answer = requiredAnswer(answers, title, options);
|
|
45
|
+
if (typeof answer !== "string") {
|
|
46
|
+
throw new Error(`${promptName(title, options)} must be a listed label`);
|
|
47
|
+
}
|
|
48
|
+
if (!choices.some((choice) => choice.label === answer)) {
|
|
49
|
+
throw new Error(`${promptName(title, options)} must be a listed label`);
|
|
50
|
+
}
|
|
51
|
+
return answer;
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function loadAnswersFile(path: string): SetupAnswerValues {
|
|
57
|
+
let raw: string;
|
|
58
|
+
try {
|
|
59
|
+
raw = readFileSync(path, "utf8");
|
|
60
|
+
} catch (err) {
|
|
61
|
+
throw new Error(`could not read answers file "${path}": ${err instanceof Error ? err.message : String(err)}`);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
let decoded: unknown;
|
|
65
|
+
try {
|
|
66
|
+
decoded = JSON.parse(raw);
|
|
67
|
+
} catch (err) {
|
|
68
|
+
throw new Error(`answers file "${path}" is not valid JSON: ${err instanceof Error ? err.message : String(err)}`);
|
|
69
|
+
}
|
|
70
|
+
const parsed = answerFileSchema.safeParse(decoded);
|
|
71
|
+
if (!parsed.success) {
|
|
72
|
+
throw new Error(`answers file "${path}" must be a JSON object whose values are strings or booleans`);
|
|
73
|
+
}
|
|
74
|
+
return parsed.data;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface RecordedAnswersUi {
|
|
78
|
+
ui: TerminalUi;
|
|
79
|
+
answers: SetupAnswerValues;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
type ClosableWizardUi = WizardUi & Partial<Pick<TerminalUi, "close">>;
|
|
83
|
+
|
|
84
|
+
/** Decorates any driver and records only answers that were actually returned. */
|
|
85
|
+
export function recordingAnswersUi(inner: ClosableWizardUi): RecordedAnswersUi {
|
|
86
|
+
const answers: SetupAnswerValues = {};
|
|
87
|
+
return {
|
|
88
|
+
answers,
|
|
89
|
+
ui: {
|
|
90
|
+
notify: (message, type) => inner.notify(message, type),
|
|
91
|
+
close: (message) => inner.close?.(message),
|
|
92
|
+
async confirm(title, message, options) {
|
|
93
|
+
const answer = await inner.confirm(title, message, options);
|
|
94
|
+
if (answer !== undefined) answers[options.key] = answer;
|
|
95
|
+
return answer;
|
|
96
|
+
},
|
|
97
|
+
async input(title, placeholder, options) {
|
|
98
|
+
const answer = await inner.input(title, placeholder, options);
|
|
99
|
+
if (answer !== undefined) answers[options.key] = answer;
|
|
100
|
+
return answer;
|
|
101
|
+
},
|
|
102
|
+
async select(title, choices, options) {
|
|
103
|
+
const answer = await inner.select(title, choices, options);
|
|
104
|
+
if (answer !== undefined) answers[options.key] = answer;
|
|
105
|
+
return answer;
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function saveAnswersFile(path: string, answers: SetupAnswerValues): void {
|
|
112
|
+
writeFileSync(path, `${JSON.stringify(answers, null, 2)}\n`, "utf8");
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Turns exhausted scripted stdin into an actionable non-interactive failure. */
|
|
116
|
+
export function guardNonInteractiveUi(inner: TerminalUi): TerminalUi {
|
|
117
|
+
const exhausted = (title: string, options: PromptOptions): never => {
|
|
118
|
+
throw new Error(
|
|
119
|
+
`stdin ended before ${promptName(title, options)}; provide more piped answers or use setup --answers FILE`,
|
|
120
|
+
);
|
|
121
|
+
};
|
|
122
|
+
return {
|
|
123
|
+
notify: (message, type) => inner.notify(message, type),
|
|
124
|
+
close: (message) => inner.close(message),
|
|
125
|
+
async confirm(title, message, options) {
|
|
126
|
+
return (await inner.confirm(title, message, options)) ?? exhausted(title, options);
|
|
127
|
+
},
|
|
128
|
+
async input(title, placeholder, options) {
|
|
129
|
+
return (await inner.input(title, placeholder, options)) ?? exhausted(title, options);
|
|
130
|
+
},
|
|
131
|
+
async select(title, choices, options: SelectOptions) {
|
|
132
|
+
return (await inner.select(title, choices, options)) ?? exhausted(title, options);
|
|
133
|
+
},
|
|
134
|
+
};
|
|
135
|
+
}
|
package/src/setup-host.ts
CHANGED
|
@@ -15,10 +15,13 @@ import {
|
|
|
15
15
|
DEFAULT_PORT,
|
|
16
16
|
healthCheck,
|
|
17
17
|
livingDaemon,
|
|
18
|
+
probeUnit,
|
|
18
19
|
startDaemon,
|
|
19
20
|
stopDaemon,
|
|
21
|
+
SYSTEMD_UNIT,
|
|
20
22
|
type DaemonRecord,
|
|
21
23
|
type StopResult,
|
|
24
|
+
type UnitOwnership,
|
|
22
25
|
} from "./lifecycle.ts";
|
|
23
26
|
import {
|
|
24
27
|
legacyArmedMarkerPath,
|
|
@@ -48,7 +51,7 @@ export const SYSTEMD_UNIT_DIR = "/etc/systemd/system";
|
|
|
48
51
|
export const RECOVER_SERVICE_NAME = "omp-conductor-recover.service";
|
|
49
52
|
|
|
50
53
|
/** The playbook file shipped in the package's `systemd/` directory. */
|
|
51
|
-
const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh";
|
|
54
|
+
export const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh";
|
|
52
55
|
|
|
53
56
|
/**
|
|
54
57
|
* Where `setup host` installs the recovery playbook so the unit's ExecStart is
|
|
@@ -1512,8 +1515,125 @@ export function writeHostRuntime(plan: HostRuntimePlan): HostRuntimeWrite {
|
|
|
1512
1515
|
return { wrote, warnings };
|
|
1513
1516
|
}
|
|
1514
1517
|
|
|
1518
|
+
// ---------------------------------------------------------- rollback (#652) --
|
|
1519
|
+
//
|
|
1520
|
+
// The inverse of the writes above, for the setup apply's mutation inventory:
|
|
1521
|
+
// every path the apply can write is captured byte-for-byte (or as absence, or
|
|
1522
|
+
// as unreadable — never conflated) before the first mutation, and restored on
|
|
1523
|
+
// any later failure. `capturePathState` is the read half, `restorePathState`
|
|
1524
|
+
// the write half; the transaction that owns them lives in `setup-wizard.ts`.
|
|
1525
|
+
|
|
1526
|
+
/** One path's pre-entry state, three-valued so absence is never confused with
|
|
1527
|
+
* unreadability:
|
|
1528
|
+
* - `absent` — nothing at the path; rollback removes what the apply created.
|
|
1529
|
+
* - `bytes` — readable on entry (a regular file with its mode, or a symlink
|
|
1530
|
+
* with its target); rollback writes those exact bytes/mode or re-links it.
|
|
1531
|
+
* - `unreadable` — something exists but cannot be read. The transaction must
|
|
1532
|
+
* fail closed before changing anything: collapsing this to absence and
|
|
1533
|
+
* later deleting the path would remove state the apply never saw. */
|
|
1534
|
+
export type CapturedPathState =
|
|
1535
|
+
| { path: string; kind: "absent" }
|
|
1536
|
+
| { path: string; kind: "bytes"; bytes: string; mode: number; symlink: boolean }
|
|
1537
|
+
| { path: string; kind: "unreadable" };
|
|
1538
|
+
|
|
1539
|
+
/** Reads {@link CapturedPathState} for one path. Symlinks are captured as
|
|
1540
|
+
* their target (readlink), never followed — the brief link is a symlink by
|
|
1541
|
+
* contract, and a dangling one is still state worth restoring. */
|
|
1542
|
+
export function capturePathState(path: string): CapturedPathState {
|
|
1543
|
+
let st: Stats;
|
|
1544
|
+
try {
|
|
1545
|
+
st = lstatSync(path);
|
|
1546
|
+
} catch {
|
|
1547
|
+
return { path, kind: "absent" };
|
|
1548
|
+
}
|
|
1549
|
+
try {
|
|
1550
|
+
if (st.isSymbolicLink()) {
|
|
1551
|
+
return { path, kind: "bytes", bytes: readlinkSync(path, "utf8"), mode: st.mode & 0o7777, symlink: true };
|
|
1552
|
+
}
|
|
1553
|
+
return { path, kind: "bytes", bytes: readFileSync(path, "utf8"), mode: st.mode & 0o7777, symlink: false };
|
|
1554
|
+
} catch {
|
|
1555
|
+
return { path, kind: "unreadable" };
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
/** Options for {@link restorePathState}. */
|
|
1560
|
+
export interface RestorePathOptions {
|
|
1561
|
+
/**
|
|
1562
|
+
* Never remove or replace a regular file: only a symlink (or absence) is
|
|
1563
|
+
* touched, so an operator's file that appeared mid-transaction survives a
|
|
1564
|
+
* rollback that cannot clobber it — the same guard the brief-link write
|
|
1565
|
+
* itself applies (#652). A regular file where the entry state said
|
|
1566
|
+
* "absent"/"symlink" therefore reports a restoration failure: the prior
|
|
1567
|
+
* state is not coherent and the operator must be told.
|
|
1568
|
+
*/
|
|
1569
|
+
preserveRegularFile?: boolean;
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
/** Restores {@link CapturedPathState} captured before the first mutation:
|
|
1573
|
+
* writes the entry bytes back (with their mode), re-links a symlink's entry
|
|
1574
|
+
* target, or removes what the apply created when nothing was there. Returns
|
|
1575
|
+
* `undefined` on a complete restore, else the first failure's message — a
|
|
1576
|
+
* rollback that could not restore a path must be reported, never implied
|
|
1577
|
+
* complete. */
|
|
1578
|
+
export function restorePathState(state: CapturedPathState, opts: RestorePathOptions = {}): string | undefined {
|
|
1579
|
+
const { path } = state;
|
|
1580
|
+
try {
|
|
1581
|
+
if (state.kind === "bytes" && state.symlink) {
|
|
1582
|
+
// Re-link the entry target. Under `preserveRegularFile` a regular file
|
|
1583
|
+
// that replaced the link mid-flight is left alone — and reported — so a
|
|
1584
|
+
// rollback never clobbers an operator's file it did not create.
|
|
1585
|
+
let current: Stats | undefined;
|
|
1586
|
+
try {
|
|
1587
|
+
current = lstatSync(path);
|
|
1588
|
+
} catch {
|
|
1589
|
+
current = undefined;
|
|
1590
|
+
}
|
|
1591
|
+
if (current !== undefined && !current.isSymbolicLink() && opts.preserveRegularFile === true) {
|
|
1592
|
+
return `could not restore the symlink at ${path}: a regular file now occupies it`;
|
|
1593
|
+
}
|
|
1594
|
+
rmSync(path, { force: true });
|
|
1595
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1596
|
+
symlinkSync(state.bytes, path);
|
|
1597
|
+
return undefined;
|
|
1598
|
+
}
|
|
1599
|
+
if (state.kind === "bytes") {
|
|
1600
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1601
|
+
writeFileSync(path, state.bytes);
|
|
1602
|
+
// Mode is part of the pre-entry state (a staged unit 0644, the tick
|
|
1603
|
+
// config 0600, the recovery playbook 0755): writeFileSync preserves an
|
|
1604
|
+
// existing file's mode, so chmod explicitly restores the captured one.
|
|
1605
|
+
chmodSync(path, state.mode);
|
|
1606
|
+
return undefined;
|
|
1607
|
+
}
|
|
1608
|
+
// Absent at entry: remove what the apply created. A regular file that
|
|
1609
|
+
// appeared where the entry was absent is the operator's own unless the
|
|
1610
|
+
// caller says otherwise.
|
|
1611
|
+
let current: Stats | undefined;
|
|
1612
|
+
try {
|
|
1613
|
+
current = lstatSync(path);
|
|
1614
|
+
} catch {
|
|
1615
|
+
return undefined; // already gone — a complete restore.
|
|
1616
|
+
}
|
|
1617
|
+
if (current.isSymbolicLink() || opts.preserveRegularFile !== true) {
|
|
1618
|
+
rmSync(path, { force: true });
|
|
1619
|
+
return undefined;
|
|
1620
|
+
}
|
|
1621
|
+
return `could not restore ${path}: a regular file appeared where nothing was at entry`;
|
|
1622
|
+
} catch (err) {
|
|
1623
|
+
return `could not restore ${path}: ${err instanceof Error ? err.message : String(err)}`;
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1515
1627
|
export interface SetupSmokeResult {
|
|
1516
1628
|
mode: "temporary" | "existing";
|
|
1629
|
+
/**
|
|
1630
|
+
* Whether the smoke itself proved the daemon answers `/healthz` on
|
|
1631
|
+
* {@link daemon}.port. A record-less active unit has no port to probe, so
|
|
1632
|
+
* its "existing" smoke is unproven: it cannot count as passed until the
|
|
1633
|
+
* caller restarts the daemon through the lifecycle seam, whose
|
|
1634
|
+
* `waitForOwnedDaemon` proves MainPID + `/healthz` (#651, review #2).
|
|
1635
|
+
*/
|
|
1636
|
+
healthProven: boolean;
|
|
1517
1637
|
status: StatusSnapshot;
|
|
1518
1638
|
daemon: DaemonRecord;
|
|
1519
1639
|
}
|
|
@@ -1522,6 +1642,15 @@ export interface SetupSmokeDeps {
|
|
|
1522
1642
|
paused(project: string): boolean;
|
|
1523
1643
|
runOnce(project: string): Promise<void>;
|
|
1524
1644
|
living(): DaemonRecord | undefined;
|
|
1645
|
+
/**
|
|
1646
|
+
* systemd ownership of the `omp-conductor.service` supervisor unit, probed
|
|
1647
|
+
* independently of the pidfile/runtime record. An active unit is a live
|
|
1648
|
+
* supervised daemon even when its record is missing or unreadable (#618):
|
|
1649
|
+
* it must never be run beside a `--once` tick, and `start()` must never be
|
|
1650
|
+
* aimed at it. `unknown` is preserved — never collapsed into `inactive` —
|
|
1651
|
+
* so a possibly-live supervised daemon is not run beside (#651, review #2).
|
|
1652
|
+
*/
|
|
1653
|
+
unitOwnership(): UnitOwnership;
|
|
1525
1654
|
health(port: number): Promise<{ ok: boolean; body?: string }>;
|
|
1526
1655
|
start(project: string): Promise<DaemonRecord>;
|
|
1527
1656
|
stop(): Promise<StopResult>;
|
|
@@ -1532,30 +1661,90 @@ const DEFAULT_SMOKE_DEPS: SetupSmokeDeps = {
|
|
|
1532
1661
|
paused: isPaused,
|
|
1533
1662
|
runOnce: async (project) => await runDaemon({ once: true, project }),
|
|
1534
1663
|
living: livingDaemon,
|
|
1664
|
+
unitOwnership: () => probeUnit(SYSTEMD_UNIT),
|
|
1535
1665
|
health: healthCheck,
|
|
1536
1666
|
start: async (project) => await startDaemon({ project }),
|
|
1537
1667
|
stop: stopDaemon,
|
|
1538
1668
|
status: statusSnapshot,
|
|
1539
1669
|
};
|
|
1540
1670
|
|
|
1671
|
+
/** The daemon record a smoke attributes to a live but record-less supervisor.
|
|
1672
|
+
* The port is unknown (the record would have named it); the "existing" mode
|
|
1673
|
+
* caller restarts the daemon through the lifecycle seam and recomputes the
|
|
1674
|
+
* real /healthz line from that restart, so this satisfies the result shape
|
|
1675
|
+
* without pretending to know a port. `healthProven: false` marks that this
|
|
1676
|
+
* is a placeholder until that restart proves the unit's MainPID answers. */
|
|
1677
|
+
function unrecordedDaemon(mainPid: number): DaemonRecord {
|
|
1678
|
+
return {
|
|
1679
|
+
pid: 0,
|
|
1680
|
+
port: 0,
|
|
1681
|
+
startedAt: 0,
|
|
1682
|
+
logFile: `<record missing — supervised daemon MainPID ${mainPid} via systemd>`,
|
|
1683
|
+
};
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1541
1686
|
export async function runSetupSmoke(
|
|
1542
1687
|
project: string,
|
|
1543
1688
|
deps: SetupSmokeDeps = DEFAULT_SMOKE_DEPS,
|
|
1544
1689
|
): Promise<SetupSmokeResult> {
|
|
1545
1690
|
if (!deps.paused(project)) throw new Error("setup smoke requires paused dispatch");
|
|
1546
|
-
|
|
1691
|
+
// Detect a live daemon BEFORE the mutating `--once` tick (#618). A paused
|
|
1692
|
+
// `--once` run is itself a dispatcher: it settles rows, projects labels,
|
|
1693
|
+
// admits and salvages workers. Running one beside a live daemon orphans and
|
|
1694
|
+
// salvages the runs the live process owns — exactly what the incident did to
|
|
1695
|
+
// three workers before failing its restart check. An already-active daemon
|
|
1696
|
+
// needs only its health proving (the caller restarts it through the lifecycle
|
|
1697
|
+
// seam so the new config takes effect) — no `--once` process ever runs beside
|
|
1698
|
+
// it, and no `systemctl start` is aimed at a unit that is already active.
|
|
1547
1699
|
const existing = deps.living();
|
|
1548
1700
|
if (existing !== undefined) {
|
|
1549
1701
|
const health = await deps.health(existing.port);
|
|
1550
1702
|
if (!health.ok) throw new Error(`existing daemon on :${existing.port} did not answer /healthz`);
|
|
1551
|
-
return { mode: "existing", status: deps.status(project), daemon: existing };
|
|
1703
|
+
return { mode: "existing", healthProven: true, status: deps.status(project), daemon: existing };
|
|
1552
1704
|
}
|
|
1553
1705
|
|
|
1706
|
+
// The pidfile record is absent, but that is not "no daemon": an active
|
|
1707
|
+
// systemd-owned unit is a live supervised dispatcher even when its record
|
|
1708
|
+
// went missing (a crash raced a re-record, or an unmanaged start never wrote
|
|
1709
|
+
// one). Without this independent probe the smoke runs a second `--once`
|
|
1710
|
+
// beside it, and `start()` targets the already-active unit — two dispatchers
|
|
1711
|
+
// owning the same state, the #618 incident shape. Treat it as an existing
|
|
1712
|
+
// daemon: no competing tick, no activation; the caller restarts the unit so
|
|
1713
|
+
// the new config takes effect (#651, review #1).
|
|
1714
|
+
const ownership = deps.unitOwnership();
|
|
1715
|
+
if (ownership.kind === "unknown") {
|
|
1716
|
+
// A probe failure is not the confirmed negative `inactive` claims to be:
|
|
1717
|
+
// collapsing `unknown` here is how a unit-owned daemon gets a second one
|
|
1718
|
+
// started beside it (#651, review #2). The mutating `--once` tick refuses
|
|
1719
|
+
// instead.
|
|
1720
|
+
throw new Error(
|
|
1721
|
+
`cannot prove whether a daemon is running: systemd ownership probe failed (${ownership.reason}) — ` +
|
|
1722
|
+
"refusing to run the setup smoke beside a possibly-live daemon",
|
|
1723
|
+
);
|
|
1724
|
+
}
|
|
1725
|
+
if (ownership.kind === "active") {
|
|
1726
|
+
// No record to read a port from, so no /healthz proof is possible here:
|
|
1727
|
+
// the smoke reports `existing` honestly as *unproven*, and the caller
|
|
1728
|
+
// must restart the unit through the lifecycle seam — whose
|
|
1729
|
+
// `waitForOwnedDaemon` proves MainPID + `/healthz` — before the smoke
|
|
1730
|
+
// counts as passed (#651, review #2).
|
|
1731
|
+
return {
|
|
1732
|
+
mode: "existing",
|
|
1733
|
+
healthProven: false,
|
|
1734
|
+
status: deps.status(project),
|
|
1735
|
+
daemon: unrecordedDaemon(ownership.pid),
|
|
1736
|
+
};
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1739
|
+
// Confirmed inactive (or failed, which owns no process): a paused `--once`
|
|
1740
|
+
// smoke can run by itself, then a temporary daemon proves the new runtime,
|
|
1741
|
+
// then it is stopped again.
|
|
1742
|
+
await deps.runOnce(project);
|
|
1554
1743
|
const daemon = await deps.start(project);
|
|
1555
1744
|
try {
|
|
1556
1745
|
const health = await deps.health(daemon.port);
|
|
1557
1746
|
if (!health.ok) throw new Error(`temporary daemon on :${daemon.port} did not answer /healthz`);
|
|
1558
|
-
return { mode: "temporary", status: deps.status(project), daemon };
|
|
1747
|
+
return { mode: "temporary", healthProven: true, status: deps.status(project), daemon };
|
|
1559
1748
|
} finally {
|
|
1560
1749
|
await deps.stop();
|
|
1561
1750
|
}
|
package/src/setup-install.ts
CHANGED
|
@@ -275,6 +275,7 @@ export async function runHostInstall(
|
|
|
275
275
|
const outcome = await runPrivileged(plan.steps, ui, {
|
|
276
276
|
...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
|
|
277
277
|
title: "Install and start the supervised session?",
|
|
278
|
+
answerKey: "install-host",
|
|
278
279
|
preamble: [
|
|
279
280
|
`Installs ${plan.service.path} as ${installed}, then enables and restarts it.`,
|
|
280
281
|
...(plan.herdrUnit === undefined
|
|
@@ -547,6 +548,7 @@ export async function runGraphInstall(
|
|
|
547
548
|
const outcome = await runPrivileged([...clones, ...install], ui, {
|
|
548
549
|
...(options.privileged === undefined ? {} : { deps: options.privileged }),
|
|
549
550
|
title: unitsCurrent ? "Clone the code-graph checkouts and seed them?" : "Clone, install and enable the code-graph timer?",
|
|
551
|
+
answerKey: `install-code-graph.${project.name}`,
|
|
550
552
|
preamble: [
|
|
551
553
|
`Staged: ${staged.written.join(", ")}.`,
|
|
552
554
|
...(clones.length === 0
|
package/src/setup-probe.ts
CHANGED
|
@@ -402,6 +402,7 @@ export async function probeProse(
|
|
|
402
402
|
const keep = await ui.confirm(
|
|
403
403
|
`Keep this ${heading}?`,
|
|
404
404
|
"It becomes part of POLICY.md, which the orchestrator re-reads on every tick. You can edit that file afterwards. Declining keeps the stub.",
|
|
405
|
+
{ key: `keep-probe.${name}` },
|
|
405
406
|
);
|
|
406
407
|
// A dismissed prompt is not a yes. `undefined` means the operator walked away.
|
|
407
408
|
if (keep !== true) {
|