@celilo/cli 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +16 -1
- package/package.json +4 -4
- package/src/cli/commands/hook-run.ts +5 -8
- package/src/cli/commands/ipam.ts +93 -0
- package/src/cli/commands/machine-add.ts +22 -0
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-doctor.ts +148 -5
- package/src/cli/commands/system-update.ts +2 -0
- package/src/cli/completion.ts +38 -5
- package/src/cli/index.ts +10 -1
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/schema.ts +41 -1
- package/src/hooks/artifact-retention.test.ts +136 -0
- package/src/hooks/artifact-retention.ts +159 -0
- package/src/hooks/executor.test.ts +80 -0
- package/src/hooks/executor.ts +68 -23
- package/src/hooks/test-fixtures/artifact-writing-hook.ts +25 -0
- package/src/hooks/types.ts +20 -2
- package/src/ipam/allocator.test.ts +38 -0
- package/src/ipam/allocator.ts +63 -1
- package/src/ipam/auto-allocator.ts +7 -0
- package/src/policy/module-business-baseline.ts +404 -0
- package/src/policy/no-module-business-in-core.test.ts +504 -0
- package/src/services/alerting/keys.ts +21 -1
- package/src/services/alerting/run-monitor.ts +6 -1
- package/src/services/aspect-reconcile.test.ts +460 -0
- package/src/services/aspect-runner.test.ts +1 -0
- package/src/services/aspect-runner.ts +408 -37
- package/src/services/audit/browser-pin.test.ts +167 -0
- package/src/services/audit/browser-pin.ts +185 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/types.ts +1 -0
- package/src/services/deploy-ansible-recap.test.ts +76 -0
- package/src/services/deploy-ansible.ts +56 -1
- package/src/services/health-runner.ts +15 -1
- package/src/services/module-deploy.ts +70 -16
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/system/browser-provisioning.test.ts +67 -0
- package/src/system/prereqs.test.ts +73 -0
- package/src/system/prereqs.ts +89 -12
- package/src/templates/ingress-ip.test.ts +108 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser pin-drift check (managed-browser-runtime D9).
|
|
3
|
+
*
|
|
4
|
+
* A module bundles its own `playwright-core`, and celilo provisions the
|
|
5
|
+
* browser binary. Under D2 the consumer passes an explicit
|
|
6
|
+
* `executablePath`, so a client bump can no longer move the browser out
|
|
7
|
+
* from under it — the hard failure that started that change cannot recur.
|
|
8
|
+
* What remains is softer: a client several minor versions from the binary's
|
|
9
|
+
* build drives it over a protocol the two were versioned together. Usually
|
|
10
|
+
* fine; not guaranteed.
|
|
11
|
+
*
|
|
12
|
+
* So this WARNS and never blocks (D9 / task 4.2).
|
|
13
|
+
*
|
|
14
|
+
* **It compares REVISIONS, not version strings.** The revision is what
|
|
15
|
+
* actually has to match a build, and each `playwright-core` states its own
|
|
16
|
+
* in the `browsers.json` inside the package — which is where 1193 and 1223
|
|
17
|
+
* came from. Reading it out of the bundle is what keeps this honest and
|
|
18
|
+
* means nobody maintains a version→revision table that will rot.
|
|
19
|
+
*
|
|
20
|
+
* Like `system doctor` (D4), it reads the bundle and the descriptor and
|
|
21
|
+
* never asks Playwright which executable it would use: that reports the
|
|
22
|
+
* full browser regardless of what a headless launch would open.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
26
|
+
import { dirname, join } from 'node:path';
|
|
27
|
+
import { resolveBrowser } from '@celilo/capabilities';
|
|
28
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
29
|
+
import type { DriftFinding } from './types';
|
|
30
|
+
|
|
31
|
+
/** What a module bundles, already read off disk. */
|
|
32
|
+
export interface BundledBrowserClient {
|
|
33
|
+
moduleId: string;
|
|
34
|
+
/** The bundled `playwright-core` version, e.g. `1.60.0`. */
|
|
35
|
+
clientVersion: string;
|
|
36
|
+
/** The chromium-headless-shell revision that client expects, e.g. `1223`. */
|
|
37
|
+
expectedRevision: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface BrowserPinAuditDeps {
|
|
41
|
+
/**
|
|
42
|
+
* Modules that bundle a browser client. A module with none contributes
|
|
43
|
+
* nothing — most modules never launch a browser.
|
|
44
|
+
*/
|
|
45
|
+
consumers: BundledBrowserClient[];
|
|
46
|
+
/**
|
|
47
|
+
* The provisioned browser, or `null` when this host has none.
|
|
48
|
+
*
|
|
49
|
+
* Null means silence, not a finding: installation is opt-in, so a host
|
|
50
|
+
* without a browser is the normal case and `system doctor` is already
|
|
51
|
+
* the place that reports its absence.
|
|
52
|
+
*/
|
|
53
|
+
provisioned: { revision: string; playwrightVersion: string } | null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function auditBrowserPin(deps: BrowserPinAuditDeps): Promise<DriftFinding[]> {
|
|
57
|
+
const { provisioned } = deps;
|
|
58
|
+
if (!provisioned) return [];
|
|
59
|
+
|
|
60
|
+
const findings: DriftFinding[] = [];
|
|
61
|
+
for (const consumer of deps.consumers) {
|
|
62
|
+
if (consumer.expectedRevision === provisioned.revision) continue;
|
|
63
|
+
|
|
64
|
+
// Any difference is reported. A "how far apart is too far" threshold
|
|
65
|
+
// would be a tuning knob with no evidence behind it — the revision is
|
|
66
|
+
// the thing that has to match a build, so it either does or it does not.
|
|
67
|
+
findings.push({
|
|
68
|
+
category: 'browser_pin',
|
|
69
|
+
severity: 'drift',
|
|
70
|
+
code: 'browser_client_revision_drift',
|
|
71
|
+
subject: consumer.moduleId,
|
|
72
|
+
message:
|
|
73
|
+
`${consumer.moduleId} bundles playwright-core ${consumer.clientVersion}, which expects ` +
|
|
74
|
+
`chromium revision ${consumer.expectedRevision}, but this host has revision ` +
|
|
75
|
+
`${provisioned.revision} (installed for playwright ${provisioned.playwrightVersion}).`,
|
|
76
|
+
details:
|
|
77
|
+
'Not a failure: the module passes an explicit executablePath, so it launches the ' +
|
|
78
|
+
'installed browser regardless of which revision its client expects. The risk is ' +
|
|
79
|
+
'protocol skew — a client far from its browser build drives it over a protocol the ' +
|
|
80
|
+
'two were versioned together.',
|
|
81
|
+
remediation:
|
|
82
|
+
`Align them: either republish ${consumer.moduleId} against playwright-core ` +
|
|
83
|
+
`${provisioned.playwrightVersion}, or set celilo-mgmt's playwright_version to ` +
|
|
84
|
+
`${consumer.clientVersion} and redeploy it.`,
|
|
85
|
+
actionable: false,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
return findings;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Read the browser client a module bundles, or `null` when it bundles none.
|
|
93
|
+
*
|
|
94
|
+
* Searches beside each hook script rather than walking the module tree:
|
|
95
|
+
* `scripts/node_modules` is where a module's hook runtime is bundled, and
|
|
96
|
+
* the manifest's own hook list is the authoritative statement of where its
|
|
97
|
+
* hook code lives. A module laying its hooks out differently
|
|
98
|
+
* (`celilo/scripts/…`, as the one real browser consumer does) is therefore
|
|
99
|
+
* found without hardcoding either layout.
|
|
100
|
+
*/
|
|
101
|
+
export function readBundledBrowserClient(
|
|
102
|
+
moduleId: string,
|
|
103
|
+
sourcePath: string,
|
|
104
|
+
hookScripts: string[],
|
|
105
|
+
): BundledBrowserClient | null {
|
|
106
|
+
const searched = new Set<string>([join(sourcePath, 'scripts')]);
|
|
107
|
+
for (const script of hookScripts) {
|
|
108
|
+
searched.add(dirname(join(sourcePath, script)));
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
for (const dir of searched) {
|
|
112
|
+
const pkgRoot = join(dir, 'node_modules', 'playwright-core');
|
|
113
|
+
const browsers = join(pkgRoot, 'browsers.json');
|
|
114
|
+
if (!existsSync(browsers)) continue;
|
|
115
|
+
|
|
116
|
+
const expectedRevision = revisionFor(browsers, 'chromium-headless-shell');
|
|
117
|
+
if (!expectedRevision) continue;
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
moduleId,
|
|
121
|
+
clientVersion: packageVersion(join(pkgRoot, 'package.json')) ?? 'unknown',
|
|
122
|
+
expectedRevision,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The revision a `playwright-core` states for a browser, from the
|
|
130
|
+
* `browsers.json` inside the package. Never a lookup table — the package
|
|
131
|
+
* is the only thing that knows, and a table would rot silently.
|
|
132
|
+
*/
|
|
133
|
+
function revisionFor(browsersJsonPath: string, browserName: string): string | null {
|
|
134
|
+
try {
|
|
135
|
+
const parsed = JSON.parse(readFileSync(browsersJsonPath, 'utf-8')) as {
|
|
136
|
+
browsers?: Array<{ name?: string; revision?: string | number }>;
|
|
137
|
+
};
|
|
138
|
+
const entry = parsed.browsers?.find((b) => b.name === browserName);
|
|
139
|
+
return entry?.revision === undefined ? null : String(entry.revision);
|
|
140
|
+
} catch {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function packageVersion(packageJsonPath: string): string | null {
|
|
146
|
+
try {
|
|
147
|
+
const parsed = JSON.parse(readFileSync(packageJsonPath, 'utf-8')) as { version?: string };
|
|
148
|
+
return parsed.version ?? null;
|
|
149
|
+
} catch {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Gather this check's inputs from the module store and the provisioned
|
|
156
|
+
* browser's descriptor.
|
|
157
|
+
*
|
|
158
|
+
* Lives here rather than in the CLI so the two `AuditDeps` builders stay
|
|
159
|
+
* one line each and cannot drift apart — they already each carry their own
|
|
160
|
+
* copy of every other category's wiring.
|
|
161
|
+
*/
|
|
162
|
+
export function collectBrowserPinDeps(
|
|
163
|
+
modules: Array<{ id: string; sourcePath: string; manifestData: unknown }>,
|
|
164
|
+
): BrowserPinAuditDeps {
|
|
165
|
+
const consumers: BundledBrowserClient[] = [];
|
|
166
|
+
for (const module of modules) {
|
|
167
|
+
const manifest = module.manifestData as ModuleManifest | undefined;
|
|
168
|
+
const hookScripts = Object.values(manifest?.hooks ?? {})
|
|
169
|
+
.map((hook) => (hook as { script?: string } | undefined)?.script)
|
|
170
|
+
.filter((script): script is string => typeof script === 'string');
|
|
171
|
+
const bundled = readBundledBrowserClient(module.id, module.sourcePath, hookScripts);
|
|
172
|
+
if (bundled) consumers.push(bundled);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
let provisioned: BrowserPinAuditDeps['provisioned'] = null;
|
|
176
|
+
try {
|
|
177
|
+
const browser = resolveBrowser();
|
|
178
|
+
provisioned = { revision: browser.revision, playwrightVersion: browser.playwrightVersion };
|
|
179
|
+
} catch {
|
|
180
|
+
// No browser on this host — the check stays silent. `system doctor`
|
|
181
|
+
// is where an absent browser is reported, not here.
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return { consumers, provisioned };
|
|
185
|
+
}
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
auditAbandonedOperations,
|
|
16
16
|
} from './abandoned-operations';
|
|
17
17
|
import { type BackupsAuditDeps, auditBackups } from './backups';
|
|
18
|
+
import { type BrowserPinAuditDeps, auditBrowserPin } from './browser-pin';
|
|
18
19
|
import { type CapabilityAbiAuditDeps, auditCapabilityAbi } from './capability-abi';
|
|
19
20
|
import { type CliVersionAuditDeps, auditCliVersion } from './cli-version';
|
|
20
21
|
import { type HealthAuditDeps, auditHealth } from './health';
|
|
@@ -48,6 +49,7 @@ export interface AuditDeps {
|
|
|
48
49
|
cliVersion: CliVersionAuditDeps;
|
|
49
50
|
schema: SchemaAuditDeps;
|
|
50
51
|
capabilityAbi: CapabilityAbiAuditDeps;
|
|
52
|
+
browserPin: BrowserPinAuditDeps;
|
|
51
53
|
terraformPlan: TerraformPlanAuditDeps;
|
|
52
54
|
moduleVersions: ModuleVersionsAuditDeps;
|
|
53
55
|
moduleConfigs: ModuleConfigsAuditDeps;
|
|
@@ -101,6 +103,7 @@ export async function runAudit(
|
|
|
101
103
|
wrap('cli_version', auditCliVersion(deps.cliVersion)),
|
|
102
104
|
wrap('schema', auditSchema(deps.schema)),
|
|
103
105
|
wrap('capability_abi', auditCapabilityAbi(deps.capabilityAbi)),
|
|
106
|
+
wrap('browser_pin', auditBrowserPin(deps.browserPin)),
|
|
104
107
|
wrap('terraform_plan', auditTerraformPlan(deps.terraformPlan)),
|
|
105
108
|
wrap('module_versions', auditModuleVersions(deps.moduleVersions)),
|
|
106
109
|
wrap('module_configs', auditModuleConfigs(deps.moduleConfigs)),
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parsing Ansible's PLAY RECAP, and the one classification that must not be got
|
|
3
|
+
* wrong.
|
|
4
|
+
*
|
|
5
|
+
* `--check` is how `verifyAspectCoverage` asks a HOST whether an aspect is
|
|
6
|
+
* applied, instead of consulting a stored claim that it once ran (celilo#902
|
|
7
|
+
* design D6). The trap is that check mode does not evaluate a task it cannot
|
|
8
|
+
* support — it SKIPS it — so a role built from `command:` / `shell:` tasks can
|
|
9
|
+
* finish a check run reporting `changed=0` having never been applied at all.
|
|
10
|
+
*
|
|
11
|
+
* Read as a boolean ("no changes, therefore applied") that is a confidently
|
|
12
|
+
* clean answer about an unconverged host: the same failure shape as the stored
|
|
13
|
+
* verdict this approach exists to avoid, arriving by a different route. Hence
|
|
14
|
+
* three outcomes, and hence this file.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { describe, expect, it } from 'bun:test';
|
|
18
|
+
import { parseAnsibleRecap } from './deploy-ansible';
|
|
19
|
+
|
|
20
|
+
const RECAP = `
|
|
21
|
+
PLAY RECAP *********************************************************************
|
|
22
|
+
caddy-int : ok=3 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
|
23
|
+
vpn : ok=3 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
|
|
24
|
+
legacy-box : ok=1 changed=0 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
|
|
25
|
+
dead-host : ok=0 changed=0 unreachable=1 failed=0 skipped=0 rescued=0 ignored=0
|
|
26
|
+
`;
|
|
27
|
+
|
|
28
|
+
describe('parseAnsibleRecap', () => {
|
|
29
|
+
it('reads every host line with its counters', () => {
|
|
30
|
+
const recaps = parseAnsibleRecap(RECAP);
|
|
31
|
+
expect(recaps.map((r) => r.host)).toEqual(['caddy-int', 'vpn', 'legacy-box', 'dead-host']);
|
|
32
|
+
expect(recaps[0]).toEqual({
|
|
33
|
+
host: 'caddy-int',
|
|
34
|
+
ok: 3,
|
|
35
|
+
changed: 0,
|
|
36
|
+
unreachable: 0,
|
|
37
|
+
failed: 0,
|
|
38
|
+
skipped: 0,
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('SKIPPED IS NOT UNCHANGED — the case that must never read as applied', () => {
|
|
43
|
+
// THE assertion this file exists for. `legacy-box` reports zero changes,
|
|
44
|
+
// which a two-state reading calls "applied". It is not: two of its tasks
|
|
45
|
+
// were never evaluated, so nothing about its convergence was measured.
|
|
46
|
+
// If this ever passes while a caller treats it as applied, celilo is once
|
|
47
|
+
// again confidently reporting a host it has not looked at.
|
|
48
|
+
const legacy = parseAnsibleRecap(RECAP).find((r) => r.host === 'legacy-box');
|
|
49
|
+
expect(legacy).toBeDefined();
|
|
50
|
+
expect(legacy?.changed).toBe(0);
|
|
51
|
+
expect(legacy?.skipped).toBeGreaterThan(0);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('separates a host that could not be reached from one that was measured clean', () => {
|
|
55
|
+
const recaps = parseAnsibleRecap(RECAP);
|
|
56
|
+
expect(recaps.find((r) => r.host === 'dead-host')?.unreachable).toBe(1);
|
|
57
|
+
expect(recaps.find((r) => r.host === 'caddy-int')?.unreachable).toBe(0);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('survives ANSI colour, which the progress filter leaves in', () => {
|
|
61
|
+
const coloured =
|
|
62
|
+
'\x1b[0;32mweb-01\x1b[0m : ok=5 changed=2 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0';
|
|
63
|
+
const [recap] = parseAnsibleRecap(coloured);
|
|
64
|
+
expect(recap.host).toBe('web-01');
|
|
65
|
+
expect(recap.changed).toBe(2);
|
|
66
|
+
expect(recap.skipped).toBe(1);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('returns nothing for output with no recap, rather than inventing a clean one', () => {
|
|
70
|
+
// A run that produced no recap measured nothing. Callers must not read an
|
|
71
|
+
// empty array as "every host is fine" — verifyAspectCoverage classifies a
|
|
72
|
+
// host with no recap line as unknown for exactly this reason.
|
|
73
|
+
expect(parseAnsibleRecap('fatal: could not connect\n')).toEqual([]);
|
|
74
|
+
expect(parseAnsibleRecap('')).toEqual([]);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -138,7 +138,7 @@ function parseAnsibleLine(line: string): string | null {
|
|
|
138
138
|
*/
|
|
139
139
|
export async function executeAnsible(
|
|
140
140
|
generatedPath: string,
|
|
141
|
-
options?: { noInteractive?: boolean },
|
|
141
|
+
options?: { noInteractive?: boolean; check?: boolean },
|
|
142
142
|
): Promise<AnsibleResult> {
|
|
143
143
|
const ansibleDir = join(generatedPath, 'ansible');
|
|
144
144
|
const inventoryPath = join(ansibleDir, 'inventory', 'hosts.ini');
|
|
@@ -161,6 +161,11 @@ export async function executeAnsible(
|
|
|
161
161
|
shellEscape(inventoryPath),
|
|
162
162
|
'--vault-password-file',
|
|
163
163
|
shellEscape(passwordPath),
|
|
164
|
+
// `--check` evaluates the play without changing anything, so a caller
|
|
165
|
+
// can ask "is this already applied?" of the HOST rather than of a
|
|
166
|
+
// stored record of the host (celilo#902 design D6). One argument, not a
|
|
167
|
+
// second execution path — everything else about the run is identical.
|
|
168
|
+
...(options?.check ? ['--check'] : []),
|
|
164
169
|
shellEscape(playbookPath),
|
|
165
170
|
],
|
|
166
171
|
cwd: ansibleDir,
|
|
@@ -200,3 +205,53 @@ export async function executeAnsible(
|
|
|
200
205
|
await rm(tempDir, { recursive: true, force: true });
|
|
201
206
|
}
|
|
202
207
|
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* One host's line from Ansible's `PLAY RECAP`.
|
|
211
|
+
*
|
|
212
|
+
* web-01 : ok=5 changed=2 unreachable=0 failed=0 skipped=1 …
|
|
213
|
+
*
|
|
214
|
+
* `skipped` is the field that matters and the one it is easy not to look at. A
|
|
215
|
+
* task that does NOT support check mode is not evaluated — it is skipped, and
|
|
216
|
+
* reported as skipped rather than changed. So a role built from `command:` /
|
|
217
|
+
* `shell:` tasks finishes a `--check` run with `changed=0` having never been
|
|
218
|
+
* applied to the host at all. Read as a boolean that says "applied", which is a
|
|
219
|
+
* confidently clean answer about an unconverged system. Callers must treat
|
|
220
|
+
* `skipped > 0` as NOT MEASURED rather than as applied.
|
|
221
|
+
*/
|
|
222
|
+
export interface AnsibleHostRecap {
|
|
223
|
+
host: string;
|
|
224
|
+
ok: number;
|
|
225
|
+
changed: number;
|
|
226
|
+
unreachable: number;
|
|
227
|
+
failed: number;
|
|
228
|
+
skipped: number;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const RECAP_LINE =
|
|
232
|
+
/^(\S+)\s*:\s*ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)/;
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Parse every host line out of an Ansible run's `PLAY RECAP`.
|
|
236
|
+
*
|
|
237
|
+
* Tolerant of ANSI colour and of the recap appearing anywhere in the stream,
|
|
238
|
+
* because the output here has been through a progress filter. Lines that are
|
|
239
|
+
* not recap lines are ignored rather than throwing — a run that produced no
|
|
240
|
+
* recap at all returns an empty array, which callers must not read as success.
|
|
241
|
+
*/
|
|
242
|
+
export function parseAnsibleRecap(output: string): AnsibleHostRecap[] {
|
|
243
|
+
const recaps: AnsibleHostRecap[] = [];
|
|
244
|
+
for (const raw of output.split('\n')) {
|
|
245
|
+
const match = RECAP_LINE.exec(raw.replace(ANSI_ESCAPE, '').trim());
|
|
246
|
+
if (!match) continue;
|
|
247
|
+
recaps.push({
|
|
248
|
+
host: match[1],
|
|
249
|
+
ok: Number(match[2]),
|
|
250
|
+
changed: Number(match[3]),
|
|
251
|
+
unreachable: Number(match[4]),
|
|
252
|
+
failed: Number(match[5]),
|
|
253
|
+
skipped: Number(match[6]),
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
return recaps;
|
|
257
|
+
}
|
|
@@ -31,6 +31,16 @@ export interface HealthCheckResult {
|
|
|
31
31
|
status: 'healthy' | 'degraded' | 'unhealthy' | 'no-checks' | 'error';
|
|
32
32
|
checks: HealthCheckItem[];
|
|
33
33
|
error?: string;
|
|
34
|
+
/**
|
|
35
|
+
* Files the hook wrote to its per-run artifact directory.
|
|
36
|
+
*
|
|
37
|
+
* This path is the one that runs UNATTENDED every fifteen minutes, and
|
|
38
|
+
* it used to drop `HookResult`'s artifacts on the floor entirely — the
|
|
39
|
+
* on-demand paths (`module run-hook`, deploy) surfaced them and this one
|
|
40
|
+
* did not, so the artifacts were discarded exactly when nobody was
|
|
41
|
+
* watching to re-run the check by hand.
|
|
42
|
+
*/
|
|
43
|
+
artifactPaths?: string[];
|
|
34
44
|
}
|
|
35
45
|
|
|
36
46
|
export interface HealthCheckOptions {
|
|
@@ -191,11 +201,15 @@ export async function runModuleHealthCheck(
|
|
|
191
201
|
presentation.finish(hookResult.success);
|
|
192
202
|
|
|
193
203
|
if (!hookResult.success) {
|
|
204
|
+
// Artifacts matter MOST here: the hook died, so there are no named
|
|
205
|
+
// checks to explain why, and whatever it managed to write is all the
|
|
206
|
+
// operator gets.
|
|
194
207
|
return {
|
|
195
208
|
moduleId,
|
|
196
209
|
status: 'error',
|
|
197
210
|
checks: [],
|
|
198
211
|
error: hookResult.error,
|
|
212
|
+
artifactPaths: hookResult.artifactPaths,
|
|
199
213
|
};
|
|
200
214
|
}
|
|
201
215
|
|
|
@@ -220,7 +234,7 @@ export async function runModuleHealthCheck(
|
|
|
220
234
|
db.update(modules).set({ state: nextState }).where(eq(modules.id, moduleId)).run();
|
|
221
235
|
}
|
|
222
236
|
|
|
223
|
-
return { moduleId, status, checks };
|
|
237
|
+
return { moduleId, status, checks, artifactPaths: hookResult.artifactPaths };
|
|
224
238
|
}
|
|
225
239
|
|
|
226
240
|
/**
|
|
@@ -15,6 +15,7 @@ import { capabilities, machines, modules } from '../db/schema';
|
|
|
15
15
|
import { loadCapabilityFunctions } from '../hooks/capability-loader';
|
|
16
16
|
import { invokeHook } from '../hooks/executor';
|
|
17
17
|
import { createGaugeLogger } from '../hooks/logger';
|
|
18
|
+
import { describeArtifacts } from '../hooks/types';
|
|
18
19
|
import type { HookDefinition, HookLogger, HookResult } from '../hooks/types';
|
|
19
20
|
import { type ModuleManifest, getSingularSystemSpec } from '../manifest/schema';
|
|
20
21
|
import { decryptSecret } from '../secrets/encryption';
|
|
@@ -624,10 +625,9 @@ async function deployModuleImpl(
|
|
|
624
625
|
|
|
625
626
|
if (!hookResult.success) {
|
|
626
627
|
gauge.stop(false);
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
}
|
|
628
|
+
const errorMsg =
|
|
629
|
+
(hookResult.error || 'Configuration validation failed') +
|
|
630
|
+
describeArtifacts(hookResult.artifactPaths);
|
|
631
631
|
return {
|
|
632
632
|
success: false,
|
|
633
633
|
error: errorMsg,
|
|
@@ -932,19 +932,17 @@ async function deployModuleImpl(
|
|
|
932
932
|
// provider type (machine / proxmox IPAM / DO outputs). This populates
|
|
933
933
|
// ctx.systems for on_install and is the source of truth for system.created
|
|
934
934
|
// and DNS (openspec/specs/module-systems-addressing/spec.md). API-only modules record none.
|
|
935
|
-
{
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
935
|
+
const { recordDeployedSystemForModule } = await import('./deployed-systems');
|
|
936
|
+
const recordedSystems = await recordDeployedSystemForModule(
|
|
937
|
+
moduleId,
|
|
938
|
+
manifest,
|
|
939
|
+
plan.infrastructure,
|
|
940
|
+
db,
|
|
941
|
+
);
|
|
942
|
+
if (recordedSystems.length > 0) {
|
|
943
|
+
log.success(
|
|
944
|
+
`Recorded ${recordedSystems.length} deployed system(s): ${recordedSystems.map((s) => `${s.hostname} (${s.ipv4_address})`).join(', ')}`,
|
|
942
945
|
);
|
|
943
|
-
if (recorded.length > 0) {
|
|
944
|
-
log.success(
|
|
945
|
-
`Recorded ${recorded.length} deployed system(s): ${recorded.map((s) => `${s.hostname} (${s.ipv4_address})`).join(', ')}`,
|
|
946
|
-
);
|
|
947
|
-
}
|
|
948
946
|
}
|
|
949
947
|
|
|
950
948
|
if (Object.keys(resolution.resolved).length > 0) {
|
|
@@ -1152,6 +1150,62 @@ async function deployModuleImpl(
|
|
|
1152
1150
|
phases.sshWait = true;
|
|
1153
1151
|
}
|
|
1154
1152
|
|
|
1153
|
+
// Apply the fleet's aspects to the systems THIS deploy just created, before
|
|
1154
|
+
// the module's own playbook runs against them (celilo#902, design D3).
|
|
1155
|
+
//
|
|
1156
|
+
// A base-module aspect fans out to the systems that exist when its PROVIDER
|
|
1157
|
+
// deploys and to nothing afterwards, so a host provisioned here would keep
|
|
1158
|
+
// its boot-time /etc/resolv.conf and be unable to resolve any fleet-internal
|
|
1159
|
+
// name — silently, with a green deploy on top. This is the inbound half.
|
|
1160
|
+
//
|
|
1161
|
+
// Placed here on purpose: after `waitForSSH`, so the host is proven
|
|
1162
|
+
// reachable and the aspect's Ansible does not fail UNREACHABLE on a
|
|
1163
|
+
// still-booting LXC; before `executeAnsible`, so the module's playbook and
|
|
1164
|
+
// its `on_install` hook see a correctly configured host rather than
|
|
1165
|
+
// configuring themselves on top of a broken one.
|
|
1166
|
+
//
|
|
1167
|
+
// FAILURE IS FATAL HERE, and that is DELIBERATELY the opposite of the
|
|
1168
|
+
// outbound direction below, where a failed fan-out never fails the
|
|
1169
|
+
// provider's own deploy (D4). The asymmetry is the point and is not an
|
|
1170
|
+
// oversight to be tidied up: an inbound aspect is a PREREQUISITE of the host
|
|
1171
|
+
// being configured right now, so continuing would configure a host celilo
|
|
1172
|
+
// knows is misconfigured — exactly the outcome celilo#902 produced. Nothing
|
|
1173
|
+
// is rolled back: the rows, the guest and the IPAM allocation all persist,
|
|
1174
|
+
// and re-running this deploy after fixing the cause converges on the same
|
|
1175
|
+
// system.
|
|
1176
|
+
if (recordedSystems.length > 0) {
|
|
1177
|
+
const { reconcileAspectsForSystems } = await import('./aspect-runner');
|
|
1178
|
+
const reconcile = await reconcileAspectsForSystems({
|
|
1179
|
+
systems: recordedSystems.map((sys) => ({ hostname: sys.hostname, zone: sys.zone })),
|
|
1180
|
+
db,
|
|
1181
|
+
// This module's own aspect is fanned out by `on_install` below, across
|
|
1182
|
+
// the whole fleet rather than only these hosts.
|
|
1183
|
+
excludeModuleIds: [moduleId],
|
|
1184
|
+
});
|
|
1185
|
+
if (reconcile.failures.length > 0) {
|
|
1186
|
+
// Name the PROVIDING module, not the one being deployed — otherwise
|
|
1187
|
+
// this reads as a defect in `moduleId` and the operator debugs the
|
|
1188
|
+
// wrong thing.
|
|
1189
|
+
const detail = reconcile.failures
|
|
1190
|
+
.map(
|
|
1191
|
+
(f) =>
|
|
1192
|
+
` ✗ '${f.providerModuleId}' aspect '${f.role}' on ${f.hostnames.join(', ')}: ${f.error ?? 'unknown error'}`,
|
|
1193
|
+
)
|
|
1194
|
+
.join('\n');
|
|
1195
|
+
return {
|
|
1196
|
+
success: false,
|
|
1197
|
+
phases,
|
|
1198
|
+
error: `Fleet aspects could not be applied to the system(s) this deploy created:\n${detail}\nThese are prerequisites, so '${moduleId}' was not configured on top of them. Fix the cause and re-run \`celilo module deploy ${moduleId}\` — the system is kept and will be reused.`,
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
const applied = reconcile.outcomes.filter((o) => o.ran);
|
|
1202
|
+
if (applied.length > 0) {
|
|
1203
|
+
log.success(
|
|
1204
|
+
`Applied ${applied.length} fleet aspect(s) to the new system(s): ${applied.map((o) => `${o.providerModuleId}/${o.role}`).join(', ')}`,
|
|
1205
|
+
);
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1155
1209
|
let machineId: string | undefined;
|
|
1156
1210
|
if (plan.infrastructure?.type === 'machine' && plan.infrastructure.machineId) {
|
|
1157
1211
|
// The management box deploys to ITSELF (registered in the machine pool as
|
|
@@ -72,6 +72,7 @@ const cleanAudit: AuditDeps = {
|
|
|
72
72
|
cliVersion: { installedVersion: '0.1.5', fetcher: async () => '0.1.5' },
|
|
73
73
|
schema: { journal: () => null, applied: () => [], db: fakeDb },
|
|
74
74
|
capabilityAbi: { modules: [] },
|
|
75
|
+
browserPin: { consumers: [], provisioned: null },
|
|
75
76
|
terraformPlan: { modules: [], run: async () => ({ exitCode: 0, stdout: '', stderr: '' }) },
|
|
76
77
|
moduleVersions: { installed: [], fetcher: async () => ({ latest: null }) },
|
|
77
78
|
moduleConfigs: { modules: [] },
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The browser runtime is described in two languages: an Ansible role
|
|
3
|
+
* installs it, and TypeScript reads it back. Nothing in either language
|
|
4
|
+
* can see the other, so these are the assertions that keep them agreeing.
|
|
5
|
+
*
|
|
6
|
+
* Both facts here have already drifted in other pairs of files in this
|
|
7
|
+
* repo; neither drift is visible in a diff.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, expect, test } from 'bun:test';
|
|
11
|
+
import { readFileSync } from 'node:fs';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import {
|
|
14
|
+
BROWSER_DESCRIPTOR_PATH,
|
|
15
|
+
BROWSER_EXECUTABLE_PATH,
|
|
16
|
+
BROWSER_ROOT,
|
|
17
|
+
} from '@celilo/capabilities';
|
|
18
|
+
|
|
19
|
+
const MODULE_ROOT = join(import.meta.dir, '../../../../modules/celilo-mgmt');
|
|
20
|
+
|
|
21
|
+
function read(relativePath: string): string {
|
|
22
|
+
return readFileSync(join(MODULE_ROOT, relativePath), 'utf-8');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
describe('celilo-mgmt browser provisioning', () => {
|
|
26
|
+
test('the playbook stamps the module version the manifest declares', () => {
|
|
27
|
+
// The descriptor records which celilo-mgmt wrote it. No template prefix
|
|
28
|
+
// exposes a module's own version ($self: resolves module CONFIG), so
|
|
29
|
+
// the playbook carries a literal — and this is what stops it rotting.
|
|
30
|
+
const manifestVersion = read('manifest.yml').match(/^version:\s*(\S+)$/m)?.[1];
|
|
31
|
+
const playbookVersion = read('ansible/playbook.yml.tpl').match(
|
|
32
|
+
/celilo_mgmt_version:\s*"([^"]+)"/,
|
|
33
|
+
)?.[1];
|
|
34
|
+
|
|
35
|
+
expect(manifestVersion).toBeTruthy();
|
|
36
|
+
expect(playbookVersion).toBe(manifestVersion as string);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('the role installs into the tree the resolver reads', () => {
|
|
40
|
+
const debian = read('ansible/roles/celilo-mgmt/tasks/debian.yml');
|
|
41
|
+
|
|
42
|
+
expect(BROWSER_EXECUTABLE_PATH).toBe(`${BROWSER_ROOT}/current/chrome`);
|
|
43
|
+
expect(debian).toContain(`PLAYWRIGHT_BROWSERS_PATH: ${BROWSER_ROOT}`);
|
|
44
|
+
expect(debian).toContain(`dest: ${BROWSER_EXECUTABLE_PATH}`);
|
|
45
|
+
expect(debian).toContain(`dest: ${BROWSER_DESCRIPTOR_PATH}`);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test('the install is gated, so a default host downloads nothing', () => {
|
|
49
|
+
const debian = read('ansible/roles/celilo-mgmt/tasks/debian.yml');
|
|
50
|
+
const manifest = read('manifest.yml');
|
|
51
|
+
|
|
52
|
+
expect(debian).toContain('when: celilo_install_browser | bool');
|
|
53
|
+
// The gate is worthless if the flag defaults on: the e2e topology has
|
|
54
|
+
// no real internet, and the browser comes from an external CDN.
|
|
55
|
+
expect(manifest).toMatch(/name: install_browser[\s\S]*?default: false/);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('the role provisions the headless shell, not full Chromium', () => {
|
|
59
|
+
// Reversed after measuring celilo-mgr: both builds there are headless
|
|
60
|
+
// shells and chromium.launch() has been driving one in production all
|
|
61
|
+
// along. Provisioning the full browser would change what works.
|
|
62
|
+
const debian = read('ansible/roles/celilo-mgmt/tasks/debian.yml');
|
|
63
|
+
|
|
64
|
+
expect(debian).toContain('install chromium-headless-shell');
|
|
65
|
+
expect(debian).toContain("'flavor': 'headless-shell'");
|
|
66
|
+
});
|
|
67
|
+
});
|