@celilo/cli 3.1.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_SUBSYSTEMS.md +1 -0
- package/package.json +5 -4
- package/src/cli/command-tree-parser.test.ts +18 -0
- package/src/cli/command-tree-parser.ts +12 -0
- package/src/cli/commands/api.ts +67 -14
- package/src/cli/commands/events.ts +5 -2
- package/src/cli/commands/machine-reclassify.ts +79 -0
- package/src/cli/commands/publish/workspace.ts +9 -0
- package/src/cli/commands/system-audit.ts +22 -0
- package/src/cli/commands/system-reboot.ts +126 -0
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +3 -2
- package/src/cli/index.ts +19 -0
- package/src/cli/tui/audit-state.ts +4 -0
- package/src/hooks/broker.ts +31 -0
- package/src/hooks/capability-loader.ts +87 -44
- package/src/hooks/executor.ts +20 -2
- package/src/hooks/test-fixtures/slow-capability-hook.ts +28 -0
- package/src/hooks/timeout-names-provider.test.ts +87 -0
- package/src/policy/module-business-baseline.ts +0 -6
- package/src/services/api-access.test.ts +44 -0
- package/src/services/audit/index.test.ts +7 -0
- package/src/services/audit/index.ts +15 -0
- package/src/services/audit/machine-interface-zones.test.ts +30 -0
- package/src/services/audit/machine-interface-zones.ts +43 -0
- package/src/services/audit/reboot-pending.test.ts +63 -0
- package/src/services/audit/reboot-pending.ts +85 -0
- package/src/services/audit/recurrence-gate.test.ts +21 -0
- package/src/services/audit/server-bun-pin.test.ts +44 -0
- package/src/services/audit/server-bun-pin.ts +95 -0
- package/src/services/audit/types.ts +2 -0
- package/src/services/capability-compat.test.ts +22 -0
- package/src/services/capability-compat.ts +15 -2
- package/src/services/celilo-events.test.ts +39 -0
- package/src/services/celilo-events.ts +41 -4
- package/src/services/deploy-preflight.ts +12 -3
- package/src/services/deploy-validation.test.ts +81 -2
- package/src/services/deploy-validation.ts +19 -0
- package/src/services/fleet-checks.ts +40 -10
- package/src/services/machine-interface-zones.test.ts +73 -0
- package/src/services/machine-interface-zones.ts +101 -0
- package/src/services/module-deploy.stale-provider-guard.test.ts +98 -0
- package/src/services/module-deploy.ts +48 -13
- package/src/services/provider-arrival.test.ts +62 -46
- package/src/services/provider-arrival.ts +41 -10
- package/src/services/provider-converge.test.ts +16 -0
- package/src/services/provider-converge.ts +106 -4
- package/src/services/system-reboot.test.ts +231 -0
- package/src/services/system-reboot.ts +357 -0
- package/src/services/update/orchestrator.test.ts +164 -0
- package/src/services/update/orchestrator.ts +43 -12
- package/src/services/update/types.ts +10 -1
- package/src/templates/generator.test.ts +15 -4
- package/src/templates/generator.ts +12 -4
- package/src/variables/context.ts +14 -1
- package/src/services/static-content-converge.test.ts +0 -516
- package/src/services/static-content-converge.ts +0 -390
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rebooting the management server (celilo#1378).
|
|
3
|
+
*
|
|
4
|
+
* celilo already DETECTS that celilo-mgr needs a reboot — `apt-upgrade` prints
|
|
5
|
+
* needrestart's pending-kernel block — and until now offered no way to act on
|
|
6
|
+
* it. The only path was SSH and `sudo reboot`: out of band, unrecorded, and not
|
|
7
|
+
* something an agent can reach, since agents operate celilo-mgr only through
|
|
8
|
+
* the MCP server.
|
|
9
|
+
*
|
|
10
|
+
* Rebooting this box is not `systemctl reboot`. It takes down the event
|
|
11
|
+
* dispatcher, the MCP server, the remote API and anything deploying, so the
|
|
12
|
+
* reboot is gated on nothing being in flight and on explicit consent.
|
|
13
|
+
*
|
|
14
|
+
* WHY VERIFICATION IS A SECOND CALL. The process that asks for the reboot dies
|
|
15
|
+
* in it — it runs ON the host it is rebooting. So no single invocation can both
|
|
16
|
+
* reboot and report that the host came back; a command that claimed to would be
|
|
17
|
+
* reporting a request as an outcome, which is the thing the issue asks us not
|
|
18
|
+
* to do. Instead `requestReboot` records a marker carrying the CURRENT boot id,
|
|
19
|
+
* and `verifyReturn` is what reports success: it passes only once the boot id
|
|
20
|
+
* has actually changed AND the dispatcher and CLI answer. An unverified marker
|
|
21
|
+
* is a reboot that did not come back, and the audit reports it as one.
|
|
22
|
+
*
|
|
23
|
+
* WHAT "IN FLIGHT" COVERS, and what it does not. The gate is `checkInFlight`,
|
|
24
|
+
* the module-operation lock that backup and restore already consult — a deploy,
|
|
25
|
+
* uninstall, backup or restore registers there, so all four hold the reboot off.
|
|
26
|
+
* celilo#1378 also names the e2e rig lock; that one is deliberately NOT checked,
|
|
27
|
+
* because the rig runs on a developer's machine and never on celilo-mgr, so
|
|
28
|
+
* reading it here would be a check that cannot reach its subject.
|
|
29
|
+
*
|
|
30
|
+
* Every judgement here is pure over injected facts, so the refusal and the
|
|
31
|
+
* verification are both testable against a fake host.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
35
|
+
import { release } from 'node:os';
|
|
36
|
+
import { dirname, join } from 'node:path';
|
|
37
|
+
import { getDataDir } from '../config/paths';
|
|
38
|
+
import { type InFlightConflict, checkInFlight } from './module-operations';
|
|
39
|
+
|
|
40
|
+
/** Debian writes this the moment a package needs a reboot to take effect. */
|
|
41
|
+
export const REBOOT_REQUIRED_PATH = '/var/run/reboot-required';
|
|
42
|
+
/** …and the packages that asked for it here, one per line. */
|
|
43
|
+
export const REBOOT_REQUIRED_PKGS_PATH = '/var/run/reboot-required.pkgs';
|
|
44
|
+
/** The kernel's own identity for this boot. It changes on every boot, and only then. */
|
|
45
|
+
export const BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id';
|
|
46
|
+
|
|
47
|
+
/** What the host says about whether it needs a reboot. */
|
|
48
|
+
export interface RebootPendingFacts {
|
|
49
|
+
/** `/var/run/reboot-required` is present. */
|
|
50
|
+
markerPresent: boolean;
|
|
51
|
+
/** Packages named in `/var/run/reboot-required.pkgs`. */
|
|
52
|
+
deferredPackages: string[];
|
|
53
|
+
/** `uname -r`, or null when it could not be read. */
|
|
54
|
+
runningKernel: string | null;
|
|
55
|
+
/** The newest kernel installed on disk, or null when it could not be read. */
|
|
56
|
+
installedKernel: string | null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface RebootPendingVerdict {
|
|
60
|
+
pending: boolean;
|
|
61
|
+
/** True when the host could not be asked at all — NOT the same as "no reboot needed". */
|
|
62
|
+
unmeasured: boolean;
|
|
63
|
+
/** Operator-readable reasons, in the order they were found. */
|
|
64
|
+
reasons: string[];
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Two independent tells, and either one is enough.
|
|
69
|
+
*
|
|
70
|
+
* The marker file is Debian's own flag and covers the deferred service
|
|
71
|
+
* restarts. The kernel comparison covers the case the marker misses and the
|
|
72
|
+
* case #1378 was filed from: running 6.8.0-1063-raspi with 1064 installed.
|
|
73
|
+
*/
|
|
74
|
+
export function judgeRebootPending(facts: RebootPendingFacts): RebootPendingVerdict {
|
|
75
|
+
const { markerPresent, deferredPackages, runningKernel, installedKernel } = facts;
|
|
76
|
+
|
|
77
|
+
const kernelKnown = runningKernel !== null && installedKernel !== null;
|
|
78
|
+
// Nothing was measured: no marker AND no kernel pair to compare. An absent
|
|
79
|
+
// marker alone is not evidence of a healthy host — the file lives under
|
|
80
|
+
// /var/run, which a container or a non-Debian host simply does not have.
|
|
81
|
+
if (!markerPresent && !kernelKnown) {
|
|
82
|
+
return { pending: false, unmeasured: true, reasons: [] };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const reasons: string[] = [];
|
|
86
|
+
if (kernelKnown && runningKernel !== installedKernel) {
|
|
87
|
+
reasons.push(
|
|
88
|
+
`the running kernel is ${runningKernel} but ${installedKernel} is installed — the new kernel loads on reboot and not before`,
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
if (markerPresent) {
|
|
92
|
+
reasons.push(
|
|
93
|
+
deferredPackages.length > 0
|
|
94
|
+
? `${REBOOT_REQUIRED_PATH} is present; deferred by: ${deferredPackages.join(', ')}`
|
|
95
|
+
: `${REBOOT_REQUIRED_PATH} is present`,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { pending: reasons.length > 0, unmeasured: false, reasons };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Read the host's own answer. The only impure part of the detection. */
|
|
103
|
+
export function collectRebootPendingFacts(
|
|
104
|
+
read: (path: string) => string | null = readFileOrNull,
|
|
105
|
+
runningKernel: string | null = null,
|
|
106
|
+
installedKernel: string | null = null,
|
|
107
|
+
): RebootPendingFacts {
|
|
108
|
+
return {
|
|
109
|
+
markerPresent: read(REBOOT_REQUIRED_PATH) !== null,
|
|
110
|
+
deferredPackages: (read(REBOOT_REQUIRED_PKGS_PATH) ?? '')
|
|
111
|
+
.split('\n')
|
|
112
|
+
.map((line) => line.trim())
|
|
113
|
+
.filter((line) => line.length > 0),
|
|
114
|
+
runningKernel,
|
|
115
|
+
installedKernel,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function readFileOrNull(path: string): string | null {
|
|
120
|
+
try {
|
|
121
|
+
return readFileSync(path, 'utf-8');
|
|
122
|
+
} catch {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The marker a reboot request leaves behind, so the return can be verified. */
|
|
128
|
+
export interface RebootMarker {
|
|
129
|
+
/** The boot id at the moment the reboot was requested. */
|
|
130
|
+
bootIdBefore: string;
|
|
131
|
+
/** Epoch ms. */
|
|
132
|
+
requestedAt: number;
|
|
133
|
+
/** Why the operator asked — echoed back by the audit finding. */
|
|
134
|
+
reason: string;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function rebootMarkerPath(dataDir: string = getDataDir()): string {
|
|
138
|
+
return join(dataDir, 'state', 'reboot-requested.json');
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export function readRebootMarker(path: string = rebootMarkerPath()): RebootMarker | null {
|
|
142
|
+
try {
|
|
143
|
+
const parsed = JSON.parse(readFileSync(path, 'utf-8')) as Partial<RebootMarker>;
|
|
144
|
+
if (typeof parsed.bootIdBefore !== 'string' || typeof parsed.requestedAt !== 'number') {
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
bootIdBefore: parsed.bootIdBefore,
|
|
149
|
+
requestedAt: parsed.requestedAt,
|
|
150
|
+
reason: parsed.reason ?? '',
|
|
151
|
+
};
|
|
152
|
+
} catch {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function writeRebootMarker(marker: RebootMarker, path: string = rebootMarkerPath()): void {
|
|
158
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
159
|
+
writeFileSync(path, `${JSON.stringify(marker, null, 2)}\n`, 'utf-8');
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function clearRebootMarker(path: string = rebootMarkerPath()): void {
|
|
163
|
+
rmSync(path, { force: true });
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export interface RequestRebootDeps {
|
|
167
|
+
/** In-flight module operations. Defaults to the real `checkInFlight`. */
|
|
168
|
+
conflicts?: () => InFlightConflict[];
|
|
169
|
+
/** This boot's id, or null when unreadable. */
|
|
170
|
+
bootId: () => string | null;
|
|
171
|
+
/** Fire the reboot. Only reached once every gate above has passed. */
|
|
172
|
+
issueReboot: () => { status: number | null };
|
|
173
|
+
writeMarker?: (marker: RebootMarker) => void;
|
|
174
|
+
now?: () => number;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export interface RequestRebootResult {
|
|
178
|
+
success: boolean;
|
|
179
|
+
error?: string;
|
|
180
|
+
message?: string;
|
|
181
|
+
/** The boot id recorded for `verifyReturn` to compare against. */
|
|
182
|
+
bootIdBefore?: string;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Refuse, or reboot. Consent is the CALLER's to obtain (flag or interview) —
|
|
187
|
+
* this function is the part that must not be bypassed, so it holds only the
|
|
188
|
+
* checks, never the prompt.
|
|
189
|
+
*/
|
|
190
|
+
export function requestReboot(deps: RequestRebootDeps, reason = ''): RequestRebootResult {
|
|
191
|
+
const conflicts = (deps.conflicts ?? checkInFlight)();
|
|
192
|
+
if (conflicts.length > 0) {
|
|
193
|
+
// Name what we are waiting on. "Busy, try later" sends the operator to
|
|
194
|
+
// guess, and the guess is usually a reboot anyway.
|
|
195
|
+
return {
|
|
196
|
+
success: false,
|
|
197
|
+
error: `Refusing to reboot: ${conflicts.length} module operation(s) in flight.\n${conflicts
|
|
198
|
+
.map((c) => ` • ${c.describe}`)
|
|
199
|
+
.join(
|
|
200
|
+
'\n',
|
|
201
|
+
)}\nWait for them to finish, or release them with \`celilo module operations clear\` if they are not really running.`,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const bootIdBefore = deps.bootId();
|
|
206
|
+
if (bootIdBefore === null) {
|
|
207
|
+
// Without a before-value there is nothing to compare after, so a reboot
|
|
208
|
+
// here could never be verified — and an unverifiable reboot of the
|
|
209
|
+
// control plane is exactly what this command exists to replace.
|
|
210
|
+
return {
|
|
211
|
+
success: false,
|
|
212
|
+
error: `Refusing to reboot: could not read this boot's id from ${BOOT_ID_PATH}, so the host coming back could not be verified afterwards.`,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
(deps.writeMarker ?? writeRebootMarker)({
|
|
217
|
+
bootIdBefore,
|
|
218
|
+
requestedAt: (deps.now ?? Date.now)(),
|
|
219
|
+
reason,
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
const { status } = deps.issueReboot();
|
|
223
|
+
if (status !== 0) {
|
|
224
|
+
return {
|
|
225
|
+
success: false,
|
|
226
|
+
error: `Reboot command exited ${status ?? 'on a signal'}. The host was not rebooted; the pending marker is left in place for \`celilo system reboot --verify\` to clear.`,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
return {
|
|
231
|
+
success: true,
|
|
232
|
+
bootIdBefore,
|
|
233
|
+
message:
|
|
234
|
+
'Reboot issued. This process goes down with the host, so it cannot report the return — ' +
|
|
235
|
+
'run `celilo system reboot --verify` once celilo-mgr answers again, or read `celilo system audit`, ' +
|
|
236
|
+
'which reports an unverified reboot as a finding.',
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export interface VerifyReturnDeps {
|
|
241
|
+
bootId: () => string | null;
|
|
242
|
+
readMarker?: () => RebootMarker | null;
|
|
243
|
+
clearMarker?: () => void;
|
|
244
|
+
/** Is the event dispatcher alive and emitting? */
|
|
245
|
+
heartbeatLive: () => Promise<boolean>;
|
|
246
|
+
/** Does the installed CLI answer `--version`? Returns the version, or null. */
|
|
247
|
+
cliVersion: () => string | null;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export interface VerifyReturnResult {
|
|
251
|
+
success: boolean;
|
|
252
|
+
error?: string;
|
|
253
|
+
message?: string;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* The half that is allowed to say "the host came back".
|
|
258
|
+
*
|
|
259
|
+
* All three must hold: the boot id CHANGED (so we are not reading the machine
|
|
260
|
+
* that never went down), the dispatcher is live, and the CLI answers. Any one
|
|
261
|
+
* of them alone passes on a host that is half up — a booted box whose
|
|
262
|
+
* dispatcher never started is not a successful reboot of a control plane.
|
|
263
|
+
*/
|
|
264
|
+
export async function verifyReturn(deps: VerifyReturnDeps): Promise<VerifyReturnResult> {
|
|
265
|
+
const marker = (deps.readMarker ?? (() => readRebootMarker()))();
|
|
266
|
+
if (marker === null) {
|
|
267
|
+
return {
|
|
268
|
+
success: false,
|
|
269
|
+
error: 'No reboot is pending verification — nothing recorded a reboot request.',
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const bootIdNow = deps.bootId();
|
|
274
|
+
if (bootIdNow === null) {
|
|
275
|
+
return {
|
|
276
|
+
success: false,
|
|
277
|
+
error: `Could not read ${BOOT_ID_PATH}, so the reboot could not be verified.`,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
if (bootIdNow === marker.bootIdBefore) {
|
|
281
|
+
return {
|
|
282
|
+
success: false,
|
|
283
|
+
error: `The host has NOT rebooted: the boot id is still ${bootIdNow}, the one recorded when the reboot was requested.`,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const heartbeat = await deps.heartbeatLive();
|
|
288
|
+
const version = deps.cliVersion();
|
|
289
|
+
if (!heartbeat || version === null) {
|
|
290
|
+
const missing = [
|
|
291
|
+
heartbeat ? null : 'the event dispatcher is not reporting a live heartbeat',
|
|
292
|
+
version === null ? 'the installed celilo CLI did not answer `--version`' : null,
|
|
293
|
+
].filter((m): m is string => m !== null);
|
|
294
|
+
return {
|
|
295
|
+
success: false,
|
|
296
|
+
error: `The host rebooted but did not come back fully: ${missing.join('; ')}. The reboot stays unverified.`,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
(deps.clearMarker ?? (() => clearRebootMarker()))();
|
|
301
|
+
return {
|
|
302
|
+
success: true,
|
|
303
|
+
message: `celilo-mgr rebooted and came back: dispatcher live, CLI ${version} answering.`,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* The newest kernel the host has on disk, from /boot/vmlinuz-<release>.
|
|
309
|
+
*
|
|
310
|
+
* Compared against `os.release()` — the kernel actually running — because
|
|
311
|
+
* that pair is the #1378 condition itself. Sorted numerically per dotted
|
|
312
|
+
* component so `-1064-` beats `-999-`, which a plain string sort gets wrong.
|
|
313
|
+
*/
|
|
314
|
+
export function newestInstalledKernel(bootEntries: string[], prefix = 'vmlinuz-'): string | null {
|
|
315
|
+
const releases = bootEntries
|
|
316
|
+
.filter((name) => name.startsWith(prefix))
|
|
317
|
+
.map((name) => name.slice(prefix.length))
|
|
318
|
+
.filter((release) => release.length > 0);
|
|
319
|
+
if (releases.length === 0) return null;
|
|
320
|
+
|
|
321
|
+
return releases.sort(compareKernelRelease).at(-1) ?? null;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function compareKernelRelease(a: string, b: string): number {
|
|
325
|
+
const partsOf = (s: string) => s.split(/[.\-+]/);
|
|
326
|
+
const pa = partsOf(a);
|
|
327
|
+
const pb = partsOf(b);
|
|
328
|
+
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
|
329
|
+
const na = Number(pa[i]);
|
|
330
|
+
const nb = Number(pb[i]);
|
|
331
|
+
const bothNumeric = Number.isFinite(na) && Number.isFinite(nb);
|
|
332
|
+
const cmp = bothNumeric ? na - nb : (pa[i] ?? '').localeCompare(pb[i] ?? '');
|
|
333
|
+
if (cmp !== 0) return cmp;
|
|
334
|
+
}
|
|
335
|
+
return 0;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** Everything the audit check needs, read from this host. */
|
|
339
|
+
export function collectRebootPendingDeps(
|
|
340
|
+
bootDir = '/boot',
|
|
341
|
+
markerPath: string = rebootMarkerPath(),
|
|
342
|
+
): { verdict: RebootPendingVerdict; marker: RebootMarker | null } {
|
|
343
|
+
let bootEntries: string[] = [];
|
|
344
|
+
try {
|
|
345
|
+
bootEntries = readdirSync(bootDir);
|
|
346
|
+
} catch {
|
|
347
|
+
// No /boot to read — the kernel half is simply unmeasured, which
|
|
348
|
+
// judgeRebootPending reports as such rather than as a clean host.
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return {
|
|
352
|
+
verdict: judgeRebootPending(
|
|
353
|
+
collectRebootPendingFacts(readFileOrNull, release(), newestInstalledKernel(bootEntries)),
|
|
354
|
+
),
|
|
355
|
+
marker: readRebootMarker(markerPath),
|
|
356
|
+
};
|
|
357
|
+
}
|
|
@@ -3,10 +3,12 @@ import type { DbClient } from '../../db/client';
|
|
|
3
3
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
4
4
|
import type { AuditDeps } from '../audit';
|
|
5
5
|
import { unusedPublicDnsProbe } from '../audit/public-dns';
|
|
6
|
+
import type { DriftFinding } from '../audit/types';
|
|
6
7
|
import { buildModuleGraph } from './dep-graph';
|
|
7
8
|
import {
|
|
8
9
|
type ModuleSnapshot,
|
|
9
10
|
type OrchestratorOps,
|
|
11
|
+
blockersOutOfReach,
|
|
10
12
|
consumerSkipReason,
|
|
11
13
|
runSystemUpdate,
|
|
12
14
|
} from './orchestrator';
|
|
@@ -73,6 +75,12 @@ const cleanAudit: AuditDeps = {
|
|
|
73
75
|
schema: { journal: () => null, applied: () => [], db: fakeDb },
|
|
74
76
|
capabilityAbi: { modules: [] },
|
|
75
77
|
browserPin: { consumers: [], provisioned: null },
|
|
78
|
+
serverBunPin: {
|
|
79
|
+
pinnedVersion: '1.4.2',
|
|
80
|
+
runningVersion: '1.4.2',
|
|
81
|
+
pinPath: '/etc/celilo/mise.pin.toml',
|
|
82
|
+
},
|
|
83
|
+
rebootPending: { verdict: { pending: false, unmeasured: false, reasons: [] }, marker: null },
|
|
76
84
|
terraformPlan: { modules: [], run: async () => ({ exitCode: 0, stdout: '', stderr: '' }) },
|
|
77
85
|
moduleVersions: { installed: [], fetcher: async () => ({ latest: null }) },
|
|
78
86
|
moduleConfigs: { modules: [] },
|
|
@@ -88,6 +96,7 @@ const cleanAudit: AuditDeps = {
|
|
|
88
96
|
secretsDecryptable: { results: [] },
|
|
89
97
|
servicesReachable: { results: [] },
|
|
90
98
|
machinesReachable: { results: [] },
|
|
99
|
+
machineInterfaceZones: { machines: [] },
|
|
91
100
|
diskSpace: { results: [] },
|
|
92
101
|
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
93
102
|
trustedSources: { firewalls: [], unreachableFirewalls: [] },
|
|
@@ -162,6 +171,8 @@ describe('runSystemUpdate', () => {
|
|
|
162
171
|
expect(result.audit.verdict).toBe('BLOCKED');
|
|
163
172
|
expect(result.modules).toEqual([]);
|
|
164
173
|
expect(upgradeCalls).toBe(0);
|
|
174
|
+
// Not `no-network` — the network is fine; the audit is what refused.
|
|
175
|
+
expect(result.selfUpdate).toEqual({ performed: false, reason: 'audit-blocked' });
|
|
165
176
|
});
|
|
166
177
|
|
|
167
178
|
test('happy path: clean audit + one drifting module → backup, upgrade, deploy, health, done', async () => {
|
|
@@ -471,3 +482,156 @@ describe('runSystemUpdate', () => {
|
|
|
471
482
|
expect([...seenIds]).toEqual(['shared-id']);
|
|
472
483
|
});
|
|
473
484
|
});
|
|
485
|
+
|
|
486
|
+
describe('blockersOutOfReach', () => {
|
|
487
|
+
const abiBlocker = (subject: string): DriftFinding => ({
|
|
488
|
+
category: 'capability_abi',
|
|
489
|
+
severity: 'blocked',
|
|
490
|
+
code: 'capability_abi_provider_mismatch',
|
|
491
|
+
message: `${subject} provides public_web@3.1.0 but framework runtime expects 4.0.0`,
|
|
492
|
+
subject,
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
test('an unscoped run cannot reach any blocker', () => {
|
|
496
|
+
expect(blockersOutOfReach([abiBlocker('caddy')], undefined)).toHaveLength(1);
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
test('--module X reaches a blocker about X', () => {
|
|
500
|
+
expect(blockersOutOfReach([abiBlocker('caddy')], 'caddy')).toEqual([]);
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
test('--module Y does not reach a blocker about X', () => {
|
|
504
|
+
const out = blockersOutOfReach([abiBlocker('caddy')], 'celilo-mgmt');
|
|
505
|
+
expect(out.map((f) => f.subject)).toEqual(['caddy']);
|
|
506
|
+
});
|
|
507
|
+
|
|
508
|
+
test('--module X never reaches a system-wide blocker', () => {
|
|
509
|
+
const systemBlocker: DriftFinding = {
|
|
510
|
+
category: 'schema',
|
|
511
|
+
severity: 'blocked',
|
|
512
|
+
code: 'schema_pending_migrations',
|
|
513
|
+
message: '1 pending migration',
|
|
514
|
+
subject: 'system',
|
|
515
|
+
};
|
|
516
|
+
const out = blockersOutOfReach([abiBlocker('caddy'), systemBlocker], 'caddy');
|
|
517
|
+
expect(out.map((f) => f.subject)).toEqual(['system']);
|
|
518
|
+
});
|
|
519
|
+
|
|
520
|
+
test('non-blocking findings are never returned', () => {
|
|
521
|
+
const drift: DriftFinding = {
|
|
522
|
+
category: 'module_versions',
|
|
523
|
+
severity: 'drift',
|
|
524
|
+
code: 'module_version_drift',
|
|
525
|
+
message: 'caddy 1.0.0 → 1.1.0',
|
|
526
|
+
subject: 'caddy',
|
|
527
|
+
};
|
|
528
|
+
expect(blockersOutOfReach([drift], undefined)).toEqual([]);
|
|
529
|
+
});
|
|
530
|
+
});
|
|
531
|
+
|
|
532
|
+
describe('runSystemUpdate with a module-scoped BLOCKED audit', () => {
|
|
533
|
+
// caddy is deployed and provides public_web@3.1.0 while the runtime
|
|
534
|
+
// contract is at 4.0.0 — the celilo#1381 shape. Upgrading caddy is
|
|
535
|
+
// exactly what the blocking finding asks for.
|
|
536
|
+
const abiBlockedAudit: AuditDeps = {
|
|
537
|
+
...cleanAudit,
|
|
538
|
+
capabilityAbi: {
|
|
539
|
+
modules: [
|
|
540
|
+
{
|
|
541
|
+
id: 'caddy',
|
|
542
|
+
state: 'VERIFIED',
|
|
543
|
+
manifest: makeManifest('caddy', { provides: ['public_web'] }),
|
|
544
|
+
},
|
|
545
|
+
],
|
|
546
|
+
contractVersions: { public_web: '2.0.0' },
|
|
547
|
+
},
|
|
548
|
+
};
|
|
549
|
+
|
|
550
|
+
function run(onlyModule: string | undefined, ops: OrchestratorOps) {
|
|
551
|
+
return runSystemUpdate({
|
|
552
|
+
audit: abiBlockedAudit,
|
|
553
|
+
graph: buildModuleGraph([
|
|
554
|
+
makeManifest('caddy', { provides: ['public_web'] }),
|
|
555
|
+
makeManifest('celilo-mgmt'),
|
|
556
|
+
]),
|
|
557
|
+
snapshots: new Map([
|
|
558
|
+
['caddy', makeSnapshot('caddy', { installed: '1.0.0', latest: '1.1.0' })],
|
|
559
|
+
['celilo-mgmt', makeSnapshot('celilo-mgmt', { installed: '1.0.0', latest: '1.1.0' })],
|
|
560
|
+
]),
|
|
561
|
+
ops,
|
|
562
|
+
selfUpdate: {
|
|
563
|
+
installedVersion: '0.1.5',
|
|
564
|
+
fetcher: async () => '0.1.5',
|
|
565
|
+
updater: async () => ({ ok: true, stderr: '' }),
|
|
566
|
+
},
|
|
567
|
+
progress: capturingProgress(),
|
|
568
|
+
now: () => new Date('2026-04-25T00:00:00Z'),
|
|
569
|
+
idGen: () => 'fixed-id',
|
|
570
|
+
onlyModule,
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
test('--module caddy upgrades caddy, with backup and health still run', async () => {
|
|
575
|
+
const calls: string[] = [];
|
|
576
|
+
const ops = makeOps({
|
|
577
|
+
backup: async (id) => {
|
|
578
|
+
calls.push(`backup:${id}`);
|
|
579
|
+
return { ok: true };
|
|
580
|
+
},
|
|
581
|
+
upgrade: async (id) => {
|
|
582
|
+
calls.push(`upgrade:${id}`);
|
|
583
|
+
return { ok: true };
|
|
584
|
+
},
|
|
585
|
+
deploy: async (id) => {
|
|
586
|
+
calls.push(`deploy:${id}`);
|
|
587
|
+
return { ok: true };
|
|
588
|
+
},
|
|
589
|
+
health: async (id) => {
|
|
590
|
+
calls.push(`health:${id}`);
|
|
591
|
+
return { status: 'healthy' };
|
|
592
|
+
},
|
|
593
|
+
});
|
|
594
|
+
|
|
595
|
+
const result = await run('caddy', ops);
|
|
596
|
+
|
|
597
|
+
expect(result.audit.verdict).toBe('BLOCKED');
|
|
598
|
+
expect(result.modules.map((m) => m.moduleId)).toEqual(['caddy']);
|
|
599
|
+
expect(result.modules[0]?.step).toBe('done');
|
|
600
|
+
expect(calls).toEqual(['backup:caddy', 'upgrade:caddy', 'deploy:caddy', 'health:caddy']);
|
|
601
|
+
});
|
|
602
|
+
|
|
603
|
+
test('--module celilo-mgmt refuses, naming the caddy finding, and never says no-network', async () => {
|
|
604
|
+
let upgrades = 0;
|
|
605
|
+
const ops = makeOps({
|
|
606
|
+
upgrade: async () => {
|
|
607
|
+
upgrades++;
|
|
608
|
+
return { ok: true };
|
|
609
|
+
},
|
|
610
|
+
});
|
|
611
|
+
|
|
612
|
+
const result = await run('celilo-mgmt', ops);
|
|
613
|
+
|
|
614
|
+
expect(result.ok).toBe(false);
|
|
615
|
+
expect(result.modules).toEqual([]);
|
|
616
|
+
expect(upgrades).toBe(0);
|
|
617
|
+
expect(result.selfUpdate).toEqual({ performed: false, reason: 'audit-blocked' });
|
|
618
|
+
const blocked = result.audit.findings.filter((f) => f.severity === 'blocked');
|
|
619
|
+
expect(blocked.map((f) => f.subject)).toEqual(['caddy']);
|
|
620
|
+
});
|
|
621
|
+
|
|
622
|
+
test('an unscoped run still refuses', async () => {
|
|
623
|
+
let upgrades = 0;
|
|
624
|
+
const ops = makeOps({
|
|
625
|
+
upgrade: async () => {
|
|
626
|
+
upgrades++;
|
|
627
|
+
return { ok: true };
|
|
628
|
+
},
|
|
629
|
+
});
|
|
630
|
+
|
|
631
|
+
const result = await run(undefined, ops);
|
|
632
|
+
|
|
633
|
+
expect(result.ok).toBe(false);
|
|
634
|
+
expect(result.modules).toEqual([]);
|
|
635
|
+
expect(upgrades).toBe(0);
|
|
636
|
+
});
|
|
637
|
+
});
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
import { runAudit } from '../audit';
|
|
22
22
|
import type { AuditDeps, SystemAuditReport } from '../audit';
|
|
23
|
+
import type { DriftFinding } from '../audit/types';
|
|
23
24
|
import {
|
|
24
25
|
type ModuleGraph,
|
|
25
26
|
type ModuleId,
|
|
@@ -133,6 +134,29 @@ export function consumerSkipReason(
|
|
|
133
134
|
return null; // compatible; consumer can proceed against still-installed provider
|
|
134
135
|
}
|
|
135
136
|
|
|
137
|
+
/**
|
|
138
|
+
* The blocking findings a run cannot clear by doing its own work.
|
|
139
|
+
*
|
|
140
|
+
* `system update` used to refuse on ANY blocked finding, which made the
|
|
141
|
+
* command unable to fix the condition that stopped it: when a provider is
|
|
142
|
+
* behind a capability major, upgrading that provider is the remediation,
|
|
143
|
+
* and `--module <provider>` was refused too (celilo#1381).
|
|
144
|
+
*
|
|
145
|
+
* A finding is within reach only when the run is scoped to one module
|
|
146
|
+
* (`--module`) and the finding is about that same module — that module's
|
|
147
|
+
* upgrade is what the finding asks for. Findings about `system` or about
|
|
148
|
+
* any other module are never within reach: nothing this run does touches
|
|
149
|
+
* them. An unscoped run therefore still refuses on every blocker, as before.
|
|
150
|
+
*/
|
|
151
|
+
export function blockersOutOfReach(
|
|
152
|
+
findings: DriftFinding[],
|
|
153
|
+
onlyModule: string | undefined,
|
|
154
|
+
): DriftFinding[] {
|
|
155
|
+
const blocked = findings.filter((f) => f.severity === 'blocked');
|
|
156
|
+
if (!onlyModule) return blocked;
|
|
157
|
+
return blocked.filter((f) => f.subject !== onlyModule);
|
|
158
|
+
}
|
|
159
|
+
|
|
136
160
|
/**
|
|
137
161
|
* Drive a single module through the state machine. Stops at the first
|
|
138
162
|
* failed step.
|
|
@@ -218,20 +242,27 @@ export async function runSystemUpdate(deps: RunUpdateDeps): Promise<SystemUpdate
|
|
|
218
242
|
|
|
219
243
|
deps.progress.emit({ kind: 'plan', modules: deps.snapshots.size });
|
|
220
244
|
|
|
221
|
-
// Audit first; bail if BLOCKED.
|
|
245
|
+
// Audit first; bail if BLOCKED and the selected module can't clear it.
|
|
222
246
|
const audit: SystemAuditReport = await runAudit(deps.audit);
|
|
223
247
|
if (audit.verdict === 'BLOCKED') {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
248
|
+
const stillBlocking = blockersOutOfReach(audit.findings, deps.onlyModule);
|
|
249
|
+
if (stillBlocking.length > 0) {
|
|
250
|
+
deps.progress.emit({
|
|
251
|
+
kind: 'self-update-skipped',
|
|
252
|
+
reason: `audit BLOCKED: ${stillBlocking.map((f) => `${f.subject}: ${f.message}`).join('; ')}`,
|
|
253
|
+
});
|
|
254
|
+
return {
|
|
255
|
+
version: 1,
|
|
256
|
+
updateId,
|
|
257
|
+
startedAt,
|
|
258
|
+
finishedAt: now().toISOString(),
|
|
259
|
+
audit,
|
|
260
|
+
selfUpdate: { performed: false, reason: 'audit-blocked' },
|
|
261
|
+
backupsCreated: false,
|
|
262
|
+
modules: [],
|
|
263
|
+
ok: false,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
235
266
|
}
|
|
236
267
|
|
|
237
268
|
// Self-update.
|
|
@@ -43,7 +43,16 @@ export interface ModuleUpdateState {
|
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
export type SelfUpdateResult =
|
|
46
|
-
| {
|
|
46
|
+
| {
|
|
47
|
+
performed: false;
|
|
48
|
+
/**
|
|
49
|
+
* `audit-blocked` means the run refused before the self-update was
|
|
50
|
+
* attempted. It exists because that refusal used to report
|
|
51
|
+
* `no-network` (celilo#1381), sending operators to debug a network
|
|
52
|
+
* that was fine.
|
|
53
|
+
*/
|
|
54
|
+
reason: 'already-current' | 'dev-mode' | 'no-network' | 'audit-blocked';
|
|
55
|
+
}
|
|
47
56
|
| { performed: true; from: string; to: string };
|
|
48
57
|
|
|
49
58
|
export interface SystemUpdateResult {
|