@celilo/cli 3.1.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CELILO_SUBSYSTEMS.md +1 -0
  2. package/package.json +5 -4
  3. package/src/cli/command-tree-parser.test.ts +18 -0
  4. package/src/cli/command-tree-parser.ts +12 -0
  5. package/src/cli/commands/api.ts +67 -14
  6. package/src/cli/commands/events.ts +5 -2
  7. package/src/cli/commands/machine-reclassify.ts +79 -0
  8. package/src/cli/commands/publish/workspace.ts +9 -0
  9. package/src/cli/commands/system-audit.ts +22 -0
  10. package/src/cli/commands/system-reboot.ts +126 -0
  11. package/src/cli/commands/system-update.ts +5 -0
  12. package/src/cli/completion.ts +3 -2
  13. package/src/cli/index.ts +19 -0
  14. package/src/cli/tui/audit-state.ts +4 -0
  15. package/src/hooks/broker.ts +31 -0
  16. package/src/hooks/capability-loader.ts +87 -44
  17. package/src/hooks/executor.ts +20 -2
  18. package/src/hooks/test-fixtures/slow-capability-hook.ts +28 -0
  19. package/src/hooks/timeout-names-provider.test.ts +87 -0
  20. package/src/policy/module-business-baseline.ts +0 -6
  21. package/src/services/api-access.test.ts +44 -0
  22. package/src/services/audit/index.test.ts +7 -0
  23. package/src/services/audit/index.ts +15 -0
  24. package/src/services/audit/machine-interface-zones.test.ts +30 -0
  25. package/src/services/audit/machine-interface-zones.ts +43 -0
  26. package/src/services/audit/reboot-pending.test.ts +63 -0
  27. package/src/services/audit/reboot-pending.ts +85 -0
  28. package/src/services/audit/recurrence-gate.test.ts +21 -0
  29. package/src/services/audit/server-bun-pin.test.ts +44 -0
  30. package/src/services/audit/server-bun-pin.ts +95 -0
  31. package/src/services/audit/types.ts +2 -0
  32. package/src/services/capability-compat.test.ts +22 -0
  33. package/src/services/capability-compat.ts +15 -2
  34. package/src/services/celilo-events.test.ts +39 -0
  35. package/src/services/celilo-events.ts +41 -4
  36. package/src/services/deploy-preflight.ts +12 -3
  37. package/src/services/deploy-validation.test.ts +81 -2
  38. package/src/services/deploy-validation.ts +19 -0
  39. package/src/services/fleet-checks.ts +40 -10
  40. package/src/services/machine-interface-zones.test.ts +73 -0
  41. package/src/services/machine-interface-zones.ts +101 -0
  42. package/src/services/module-deploy.stale-provider-guard.test.ts +98 -0
  43. package/src/services/module-deploy.ts +48 -13
  44. package/src/services/provider-arrival.test.ts +62 -46
  45. package/src/services/provider-arrival.ts +41 -10
  46. package/src/services/provider-converge.test.ts +16 -0
  47. package/src/services/provider-converge.ts +106 -4
  48. package/src/services/system-reboot.test.ts +231 -0
  49. package/src/services/system-reboot.ts +357 -0
  50. package/src/services/update/orchestrator.test.ts +164 -0
  51. package/src/services/update/orchestrator.ts +43 -12
  52. package/src/services/update/types.ts +10 -1
  53. package/src/templates/generator.test.ts +15 -4
  54. package/src/templates/generator.ts +12 -4
  55. package/src/variables/context.ts +14 -1
  56. package/src/services/static-content-converge.test.ts +0 -516
  57. package/src/services/static-content-converge.ts +0 -390
@@ -0,0 +1,357 @@
1
+ /**
2
+ * Rebooting the management server (celilo#1378).
3
+ *
4
+ * celilo already DETECTS that celilo-mgr needs a reboot — `apt-upgrade` prints
5
+ * needrestart's pending-kernel block — and until now offered no way to act on
6
+ * it. The only path was SSH and `sudo reboot`: out of band, unrecorded, and not
7
+ * something an agent can reach, since agents operate celilo-mgr only through
8
+ * the MCP server.
9
+ *
10
+ * Rebooting this box is not `systemctl reboot`. It takes down the event
11
+ * dispatcher, the MCP server, the remote API and anything deploying, so the
12
+ * reboot is gated on nothing being in flight and on explicit consent.
13
+ *
14
+ * WHY VERIFICATION IS A SECOND CALL. The process that asks for the reboot dies
15
+ * in it — it runs ON the host it is rebooting. So no single invocation can both
16
+ * reboot and report that the host came back; a command that claimed to would be
17
+ * reporting a request as an outcome, which is the thing the issue asks us not
18
+ * to do. Instead `requestReboot` records a marker carrying the CURRENT boot id,
19
+ * and `verifyReturn` is what reports success: it passes only once the boot id
20
+ * has actually changed AND the dispatcher and CLI answer. An unverified marker
21
+ * is a reboot that did not come back, and the audit reports it as one.
22
+ *
23
+ * WHAT "IN FLIGHT" COVERS, and what it does not. The gate is `checkInFlight`,
24
+ * the module-operation lock that backup and restore already consult — a deploy,
25
+ * uninstall, backup or restore registers there, so all four hold the reboot off.
26
+ * celilo#1378 also names the e2e rig lock; that one is deliberately NOT checked,
27
+ * because the rig runs on a developer's machine and never on celilo-mgr, so
28
+ * reading it here would be a check that cannot reach its subject.
29
+ *
30
+ * Every judgement here is pure over injected facts, so the refusal and the
31
+ * verification are both testable against a fake host.
32
+ */
33
+
34
+ import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
35
+ import { release } from 'node:os';
36
+ import { dirname, join } from 'node:path';
37
+ import { getDataDir } from '../config/paths';
38
+ import { type InFlightConflict, checkInFlight } from './module-operations';
39
+
40
+ /** Debian writes this the moment a package needs a reboot to take effect. */
41
+ export const REBOOT_REQUIRED_PATH = '/var/run/reboot-required';
42
+ /** …and the packages that asked for it here, one per line. */
43
+ export const REBOOT_REQUIRED_PKGS_PATH = '/var/run/reboot-required.pkgs';
44
+ /** The kernel's own identity for this boot. It changes on every boot, and only then. */
45
+ export const BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id';
46
+
47
+ /** What the host says about whether it needs a reboot. */
48
+ export interface RebootPendingFacts {
49
+ /** `/var/run/reboot-required` is present. */
50
+ markerPresent: boolean;
51
+ /** Packages named in `/var/run/reboot-required.pkgs`. */
52
+ deferredPackages: string[];
53
+ /** `uname -r`, or null when it could not be read. */
54
+ runningKernel: string | null;
55
+ /** The newest kernel installed on disk, or null when it could not be read. */
56
+ installedKernel: string | null;
57
+ }
58
+
59
+ export interface RebootPendingVerdict {
60
+ pending: boolean;
61
+ /** True when the host could not be asked at all — NOT the same as "no reboot needed". */
62
+ unmeasured: boolean;
63
+ /** Operator-readable reasons, in the order they were found. */
64
+ reasons: string[];
65
+ }
66
+
67
+ /**
68
+ * Two independent tells, and either one is enough.
69
+ *
70
+ * The marker file is Debian's own flag and covers the deferred service
71
+ * restarts. The kernel comparison covers the case the marker misses and the
72
+ * case #1378 was filed from: running 6.8.0-1063-raspi with 1064 installed.
73
+ */
74
+ export function judgeRebootPending(facts: RebootPendingFacts): RebootPendingVerdict {
75
+ const { markerPresent, deferredPackages, runningKernel, installedKernel } = facts;
76
+
77
+ const kernelKnown = runningKernel !== null && installedKernel !== null;
78
+ // Nothing was measured: no marker AND no kernel pair to compare. An absent
79
+ // marker alone is not evidence of a healthy host — the file lives under
80
+ // /var/run, which a container or a non-Debian host simply does not have.
81
+ if (!markerPresent && !kernelKnown) {
82
+ return { pending: false, unmeasured: true, reasons: [] };
83
+ }
84
+
85
+ const reasons: string[] = [];
86
+ if (kernelKnown && runningKernel !== installedKernel) {
87
+ reasons.push(
88
+ `the running kernel is ${runningKernel} but ${installedKernel} is installed — the new kernel loads on reboot and not before`,
89
+ );
90
+ }
91
+ if (markerPresent) {
92
+ reasons.push(
93
+ deferredPackages.length > 0
94
+ ? `${REBOOT_REQUIRED_PATH} is present; deferred by: ${deferredPackages.join(', ')}`
95
+ : `${REBOOT_REQUIRED_PATH} is present`,
96
+ );
97
+ }
98
+
99
+ return { pending: reasons.length > 0, unmeasured: false, reasons };
100
+ }
101
+
102
+ /** Read the host's own answer. The only impure part of the detection. */
103
+ export function collectRebootPendingFacts(
104
+ read: (path: string) => string | null = readFileOrNull,
105
+ runningKernel: string | null = null,
106
+ installedKernel: string | null = null,
107
+ ): RebootPendingFacts {
108
+ return {
109
+ markerPresent: read(REBOOT_REQUIRED_PATH) !== null,
110
+ deferredPackages: (read(REBOOT_REQUIRED_PKGS_PATH) ?? '')
111
+ .split('\n')
112
+ .map((line) => line.trim())
113
+ .filter((line) => line.length > 0),
114
+ runningKernel,
115
+ installedKernel,
116
+ };
117
+ }
118
+
119
+ function readFileOrNull(path: string): string | null {
120
+ try {
121
+ return readFileSync(path, 'utf-8');
122
+ } catch {
123
+ return null;
124
+ }
125
+ }
126
+
127
+ /** The marker a reboot request leaves behind, so the return can be verified. */
128
+ export interface RebootMarker {
129
+ /** The boot id at the moment the reboot was requested. */
130
+ bootIdBefore: string;
131
+ /** Epoch ms. */
132
+ requestedAt: number;
133
+ /** Why the operator asked — echoed back by the audit finding. */
134
+ reason: string;
135
+ }
136
+
137
+ export function rebootMarkerPath(dataDir: string = getDataDir()): string {
138
+ return join(dataDir, 'state', 'reboot-requested.json');
139
+ }
140
+
141
+ export function readRebootMarker(path: string = rebootMarkerPath()): RebootMarker | null {
142
+ try {
143
+ const parsed = JSON.parse(readFileSync(path, 'utf-8')) as Partial<RebootMarker>;
144
+ if (typeof parsed.bootIdBefore !== 'string' || typeof parsed.requestedAt !== 'number') {
145
+ return null;
146
+ }
147
+ return {
148
+ bootIdBefore: parsed.bootIdBefore,
149
+ requestedAt: parsed.requestedAt,
150
+ reason: parsed.reason ?? '',
151
+ };
152
+ } catch {
153
+ return null;
154
+ }
155
+ }
156
+
157
+ export function writeRebootMarker(marker: RebootMarker, path: string = rebootMarkerPath()): void {
158
+ mkdirSync(dirname(path), { recursive: true });
159
+ writeFileSync(path, `${JSON.stringify(marker, null, 2)}\n`, 'utf-8');
160
+ }
161
+
162
+ export function clearRebootMarker(path: string = rebootMarkerPath()): void {
163
+ rmSync(path, { force: true });
164
+ }
165
+
166
+ export interface RequestRebootDeps {
167
+ /** In-flight module operations. Defaults to the real `checkInFlight`. */
168
+ conflicts?: () => InFlightConflict[];
169
+ /** This boot's id, or null when unreadable. */
170
+ bootId: () => string | null;
171
+ /** Fire the reboot. Only reached once every gate above has passed. */
172
+ issueReboot: () => { status: number | null };
173
+ writeMarker?: (marker: RebootMarker) => void;
174
+ now?: () => number;
175
+ }
176
+
177
+ export interface RequestRebootResult {
178
+ success: boolean;
179
+ error?: string;
180
+ message?: string;
181
+ /** The boot id recorded for `verifyReturn` to compare against. */
182
+ bootIdBefore?: string;
183
+ }
184
+
185
+ /**
186
+ * Refuse, or reboot. Consent is the CALLER's to obtain (flag or interview) —
187
+ * this function is the part that must not be bypassed, so it holds only the
188
+ * checks, never the prompt.
189
+ */
190
+ export function requestReboot(deps: RequestRebootDeps, reason = ''): RequestRebootResult {
191
+ const conflicts = (deps.conflicts ?? checkInFlight)();
192
+ if (conflicts.length > 0) {
193
+ // Name what we are waiting on. "Busy, try later" sends the operator to
194
+ // guess, and the guess is usually a reboot anyway.
195
+ return {
196
+ success: false,
197
+ error: `Refusing to reboot: ${conflicts.length} module operation(s) in flight.\n${conflicts
198
+ .map((c) => ` • ${c.describe}`)
199
+ .join(
200
+ '\n',
201
+ )}\nWait for them to finish, or release them with \`celilo module operations clear\` if they are not really running.`,
202
+ };
203
+ }
204
+
205
+ const bootIdBefore = deps.bootId();
206
+ if (bootIdBefore === null) {
207
+ // Without a before-value there is nothing to compare after, so a reboot
208
+ // here could never be verified — and an unverifiable reboot of the
209
+ // control plane is exactly what this command exists to replace.
210
+ return {
211
+ success: false,
212
+ error: `Refusing to reboot: could not read this boot's id from ${BOOT_ID_PATH}, so the host coming back could not be verified afterwards.`,
213
+ };
214
+ }
215
+
216
+ (deps.writeMarker ?? writeRebootMarker)({
217
+ bootIdBefore,
218
+ requestedAt: (deps.now ?? Date.now)(),
219
+ reason,
220
+ });
221
+
222
+ const { status } = deps.issueReboot();
223
+ if (status !== 0) {
224
+ return {
225
+ success: false,
226
+ error: `Reboot command exited ${status ?? 'on a signal'}. The host was not rebooted; the pending marker is left in place for \`celilo system reboot --verify\` to clear.`,
227
+ };
228
+ }
229
+
230
+ return {
231
+ success: true,
232
+ bootIdBefore,
233
+ message:
234
+ 'Reboot issued. This process goes down with the host, so it cannot report the return — ' +
235
+ 'run `celilo system reboot --verify` once celilo-mgr answers again, or read `celilo system audit`, ' +
236
+ 'which reports an unverified reboot as a finding.',
237
+ };
238
+ }
239
+
240
+ export interface VerifyReturnDeps {
241
+ bootId: () => string | null;
242
+ readMarker?: () => RebootMarker | null;
243
+ clearMarker?: () => void;
244
+ /** Is the event dispatcher alive and emitting? */
245
+ heartbeatLive: () => Promise<boolean>;
246
+ /** Does the installed CLI answer `--version`? Returns the version, or null. */
247
+ cliVersion: () => string | null;
248
+ }
249
+
250
+ export interface VerifyReturnResult {
251
+ success: boolean;
252
+ error?: string;
253
+ message?: string;
254
+ }
255
+
256
+ /**
257
+ * The half that is allowed to say "the host came back".
258
+ *
259
+ * All three must hold: the boot id CHANGED (so we are not reading the machine
260
+ * that never went down), the dispatcher is live, and the CLI answers. Any one
261
+ * of them alone passes on a host that is half up — a booted box whose
262
+ * dispatcher never started is not a successful reboot of a control plane.
263
+ */
264
+ export async function verifyReturn(deps: VerifyReturnDeps): Promise<VerifyReturnResult> {
265
+ const marker = (deps.readMarker ?? (() => readRebootMarker()))();
266
+ if (marker === null) {
267
+ return {
268
+ success: false,
269
+ error: 'No reboot is pending verification — nothing recorded a reboot request.',
270
+ };
271
+ }
272
+
273
+ const bootIdNow = deps.bootId();
274
+ if (bootIdNow === null) {
275
+ return {
276
+ success: false,
277
+ error: `Could not read ${BOOT_ID_PATH}, so the reboot could not be verified.`,
278
+ };
279
+ }
280
+ if (bootIdNow === marker.bootIdBefore) {
281
+ return {
282
+ success: false,
283
+ error: `The host has NOT rebooted: the boot id is still ${bootIdNow}, the one recorded when the reboot was requested.`,
284
+ };
285
+ }
286
+
287
+ const heartbeat = await deps.heartbeatLive();
288
+ const version = deps.cliVersion();
289
+ if (!heartbeat || version === null) {
290
+ const missing = [
291
+ heartbeat ? null : 'the event dispatcher is not reporting a live heartbeat',
292
+ version === null ? 'the installed celilo CLI did not answer `--version`' : null,
293
+ ].filter((m): m is string => m !== null);
294
+ return {
295
+ success: false,
296
+ error: `The host rebooted but did not come back fully: ${missing.join('; ')}. The reboot stays unverified.`,
297
+ };
298
+ }
299
+
300
+ (deps.clearMarker ?? (() => clearRebootMarker()))();
301
+ return {
302
+ success: true,
303
+ message: `celilo-mgr rebooted and came back: dispatcher live, CLI ${version} answering.`,
304
+ };
305
+ }
306
+
307
+ /**
308
+ * The newest kernel the host has on disk, from /boot/vmlinuz-<release>.
309
+ *
310
+ * Compared against `os.release()` — the kernel actually running — because
311
+ * that pair is the #1378 condition itself. Sorted numerically per dotted
312
+ * component so `-1064-` beats `-999-`, which a plain string sort gets wrong.
313
+ */
314
+ export function newestInstalledKernel(bootEntries: string[], prefix = 'vmlinuz-'): string | null {
315
+ const releases = bootEntries
316
+ .filter((name) => name.startsWith(prefix))
317
+ .map((name) => name.slice(prefix.length))
318
+ .filter((release) => release.length > 0);
319
+ if (releases.length === 0) return null;
320
+
321
+ return releases.sort(compareKernelRelease).at(-1) ?? null;
322
+ }
323
+
324
+ function compareKernelRelease(a: string, b: string): number {
325
+ const partsOf = (s: string) => s.split(/[.\-+]/);
326
+ const pa = partsOf(a);
327
+ const pb = partsOf(b);
328
+ for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
329
+ const na = Number(pa[i]);
330
+ const nb = Number(pb[i]);
331
+ const bothNumeric = Number.isFinite(na) && Number.isFinite(nb);
332
+ const cmp = bothNumeric ? na - nb : (pa[i] ?? '').localeCompare(pb[i] ?? '');
333
+ if (cmp !== 0) return cmp;
334
+ }
335
+ return 0;
336
+ }
337
+
338
+ /** Everything the audit check needs, read from this host. */
339
+ export function collectRebootPendingDeps(
340
+ bootDir = '/boot',
341
+ markerPath: string = rebootMarkerPath(),
342
+ ): { verdict: RebootPendingVerdict; marker: RebootMarker | null } {
343
+ let bootEntries: string[] = [];
344
+ try {
345
+ bootEntries = readdirSync(bootDir);
346
+ } catch {
347
+ // No /boot to read — the kernel half is simply unmeasured, which
348
+ // judgeRebootPending reports as such rather than as a clean host.
349
+ }
350
+
351
+ return {
352
+ verdict: judgeRebootPending(
353
+ collectRebootPendingFacts(readFileOrNull, release(), newestInstalledKernel(bootEntries)),
354
+ ),
355
+ marker: readRebootMarker(markerPath),
356
+ };
357
+ }
@@ -3,10 +3,12 @@ import type { DbClient } from '../../db/client';
3
3
  import type { ModuleManifest } from '../../manifest/schema';
4
4
  import type { AuditDeps } from '../audit';
5
5
  import { unusedPublicDnsProbe } from '../audit/public-dns';
6
+ import type { DriftFinding } from '../audit/types';
6
7
  import { buildModuleGraph } from './dep-graph';
7
8
  import {
8
9
  type ModuleSnapshot,
9
10
  type OrchestratorOps,
11
+ blockersOutOfReach,
10
12
  consumerSkipReason,
11
13
  runSystemUpdate,
12
14
  } from './orchestrator';
@@ -73,6 +75,12 @@ const cleanAudit: AuditDeps = {
73
75
  schema: { journal: () => null, applied: () => [], db: fakeDb },
74
76
  capabilityAbi: { modules: [] },
75
77
  browserPin: { consumers: [], provisioned: null },
78
+ serverBunPin: {
79
+ pinnedVersion: '1.4.2',
80
+ runningVersion: '1.4.2',
81
+ pinPath: '/etc/celilo/mise.pin.toml',
82
+ },
83
+ rebootPending: { verdict: { pending: false, unmeasured: false, reasons: [] }, marker: null },
76
84
  terraformPlan: { modules: [], run: async () => ({ exitCode: 0, stdout: '', stderr: '' }) },
77
85
  moduleVersions: { installed: [], fetcher: async () => ({ latest: null }) },
78
86
  moduleConfigs: { modules: [] },
@@ -88,6 +96,7 @@ const cleanAudit: AuditDeps = {
88
96
  secretsDecryptable: { results: [] },
89
97
  servicesReachable: { results: [] },
90
98
  machinesReachable: { results: [] },
99
+ machineInterfaceZones: { machines: [] },
91
100
  diskSpace: { results: [] },
92
101
  transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
93
102
  trustedSources: { firewalls: [], unreachableFirewalls: [] },
@@ -162,6 +171,8 @@ describe('runSystemUpdate', () => {
162
171
  expect(result.audit.verdict).toBe('BLOCKED');
163
172
  expect(result.modules).toEqual([]);
164
173
  expect(upgradeCalls).toBe(0);
174
+ // Not `no-network` — the network is fine; the audit is what refused.
175
+ expect(result.selfUpdate).toEqual({ performed: false, reason: 'audit-blocked' });
165
176
  });
166
177
 
167
178
  test('happy path: clean audit + one drifting module → backup, upgrade, deploy, health, done', async () => {
@@ -471,3 +482,156 @@ describe('runSystemUpdate', () => {
471
482
  expect([...seenIds]).toEqual(['shared-id']);
472
483
  });
473
484
  });
485
+
486
+ describe('blockersOutOfReach', () => {
487
+ const abiBlocker = (subject: string): DriftFinding => ({
488
+ category: 'capability_abi',
489
+ severity: 'blocked',
490
+ code: 'capability_abi_provider_mismatch',
491
+ message: `${subject} provides public_web@3.1.0 but framework runtime expects 4.0.0`,
492
+ subject,
493
+ });
494
+
495
+ test('an unscoped run cannot reach any blocker', () => {
496
+ expect(blockersOutOfReach([abiBlocker('caddy')], undefined)).toHaveLength(1);
497
+ });
498
+
499
+ test('--module X reaches a blocker about X', () => {
500
+ expect(blockersOutOfReach([abiBlocker('caddy')], 'caddy')).toEqual([]);
501
+ });
502
+
503
+ test('--module Y does not reach a blocker about X', () => {
504
+ const out = blockersOutOfReach([abiBlocker('caddy')], 'celilo-mgmt');
505
+ expect(out.map((f) => f.subject)).toEqual(['caddy']);
506
+ });
507
+
508
+ test('--module X never reaches a system-wide blocker', () => {
509
+ const systemBlocker: DriftFinding = {
510
+ category: 'schema',
511
+ severity: 'blocked',
512
+ code: 'schema_pending_migrations',
513
+ message: '1 pending migration',
514
+ subject: 'system',
515
+ };
516
+ const out = blockersOutOfReach([abiBlocker('caddy'), systemBlocker], 'caddy');
517
+ expect(out.map((f) => f.subject)).toEqual(['system']);
518
+ });
519
+
520
+ test('non-blocking findings are never returned', () => {
521
+ const drift: DriftFinding = {
522
+ category: 'module_versions',
523
+ severity: 'drift',
524
+ code: 'module_version_drift',
525
+ message: 'caddy 1.0.0 → 1.1.0',
526
+ subject: 'caddy',
527
+ };
528
+ expect(blockersOutOfReach([drift], undefined)).toEqual([]);
529
+ });
530
+ });
531
+
532
+ describe('runSystemUpdate with a module-scoped BLOCKED audit', () => {
533
+ // caddy is deployed and provides public_web@3.1.0 while the runtime
534
+ // contract is at 4.0.0 — the celilo#1381 shape. Upgrading caddy is
535
+ // exactly what the blocking finding asks for.
536
+ const abiBlockedAudit: AuditDeps = {
537
+ ...cleanAudit,
538
+ capabilityAbi: {
539
+ modules: [
540
+ {
541
+ id: 'caddy',
542
+ state: 'VERIFIED',
543
+ manifest: makeManifest('caddy', { provides: ['public_web'] }),
544
+ },
545
+ ],
546
+ contractVersions: { public_web: '2.0.0' },
547
+ },
548
+ };
549
+
550
+ function run(onlyModule: string | undefined, ops: OrchestratorOps) {
551
+ return runSystemUpdate({
552
+ audit: abiBlockedAudit,
553
+ graph: buildModuleGraph([
554
+ makeManifest('caddy', { provides: ['public_web'] }),
555
+ makeManifest('celilo-mgmt'),
556
+ ]),
557
+ snapshots: new Map([
558
+ ['caddy', makeSnapshot('caddy', { installed: '1.0.0', latest: '1.1.0' })],
559
+ ['celilo-mgmt', makeSnapshot('celilo-mgmt', { installed: '1.0.0', latest: '1.1.0' })],
560
+ ]),
561
+ ops,
562
+ selfUpdate: {
563
+ installedVersion: '0.1.5',
564
+ fetcher: async () => '0.1.5',
565
+ updater: async () => ({ ok: true, stderr: '' }),
566
+ },
567
+ progress: capturingProgress(),
568
+ now: () => new Date('2026-04-25T00:00:00Z'),
569
+ idGen: () => 'fixed-id',
570
+ onlyModule,
571
+ });
572
+ }
573
+
574
+ test('--module caddy upgrades caddy, with backup and health still run', async () => {
575
+ const calls: string[] = [];
576
+ const ops = makeOps({
577
+ backup: async (id) => {
578
+ calls.push(`backup:${id}`);
579
+ return { ok: true };
580
+ },
581
+ upgrade: async (id) => {
582
+ calls.push(`upgrade:${id}`);
583
+ return { ok: true };
584
+ },
585
+ deploy: async (id) => {
586
+ calls.push(`deploy:${id}`);
587
+ return { ok: true };
588
+ },
589
+ health: async (id) => {
590
+ calls.push(`health:${id}`);
591
+ return { status: 'healthy' };
592
+ },
593
+ });
594
+
595
+ const result = await run('caddy', ops);
596
+
597
+ expect(result.audit.verdict).toBe('BLOCKED');
598
+ expect(result.modules.map((m) => m.moduleId)).toEqual(['caddy']);
599
+ expect(result.modules[0]?.step).toBe('done');
600
+ expect(calls).toEqual(['backup:caddy', 'upgrade:caddy', 'deploy:caddy', 'health:caddy']);
601
+ });
602
+
603
+ test('--module celilo-mgmt refuses, naming the caddy finding, and never says no-network', async () => {
604
+ let upgrades = 0;
605
+ const ops = makeOps({
606
+ upgrade: async () => {
607
+ upgrades++;
608
+ return { ok: true };
609
+ },
610
+ });
611
+
612
+ const result = await run('celilo-mgmt', ops);
613
+
614
+ expect(result.ok).toBe(false);
615
+ expect(result.modules).toEqual([]);
616
+ expect(upgrades).toBe(0);
617
+ expect(result.selfUpdate).toEqual({ performed: false, reason: 'audit-blocked' });
618
+ const blocked = result.audit.findings.filter((f) => f.severity === 'blocked');
619
+ expect(blocked.map((f) => f.subject)).toEqual(['caddy']);
620
+ });
621
+
622
+ test('an unscoped run still refuses', async () => {
623
+ let upgrades = 0;
624
+ const ops = makeOps({
625
+ upgrade: async () => {
626
+ upgrades++;
627
+ return { ok: true };
628
+ },
629
+ });
630
+
631
+ const result = await run(undefined, ops);
632
+
633
+ expect(result.ok).toBe(false);
634
+ expect(result.modules).toEqual([]);
635
+ expect(upgrades).toBe(0);
636
+ });
637
+ });
@@ -20,6 +20,7 @@
20
20
 
21
21
  import { runAudit } from '../audit';
22
22
  import type { AuditDeps, SystemAuditReport } from '../audit';
23
+ import type { DriftFinding } from '../audit/types';
23
24
  import {
24
25
  type ModuleGraph,
25
26
  type ModuleId,
@@ -133,6 +134,29 @@ export function consumerSkipReason(
133
134
  return null; // compatible; consumer can proceed against still-installed provider
134
135
  }
135
136
 
137
+ /**
138
+ * The blocking findings a run cannot clear by doing its own work.
139
+ *
140
+ * `system update` used to refuse on ANY blocked finding, which made the
141
+ * command unable to fix the condition that stopped it: when a provider is
142
+ * behind a capability major, upgrading that provider is the remediation,
143
+ * and `--module <provider>` was refused too (celilo#1381).
144
+ *
145
+ * A finding is within reach only when the run is scoped to one module
146
+ * (`--module`) and the finding is about that same module — that module's
147
+ * upgrade is what the finding asks for. Findings about `system` or about
148
+ * any other module are never within reach: nothing this run does touches
149
+ * them. An unscoped run therefore still refuses on every blocker, as before.
150
+ */
151
+ export function blockersOutOfReach(
152
+ findings: DriftFinding[],
153
+ onlyModule: string | undefined,
154
+ ): DriftFinding[] {
155
+ const blocked = findings.filter((f) => f.severity === 'blocked');
156
+ if (!onlyModule) return blocked;
157
+ return blocked.filter((f) => f.subject !== onlyModule);
158
+ }
159
+
136
160
  /**
137
161
  * Drive a single module through the state machine. Stops at the first
138
162
  * failed step.
@@ -218,20 +242,27 @@ export async function runSystemUpdate(deps: RunUpdateDeps): Promise<SystemUpdate
218
242
 
219
243
  deps.progress.emit({ kind: 'plan', modules: deps.snapshots.size });
220
244
 
221
- // Audit first; bail if BLOCKED.
245
+ // Audit first; bail if BLOCKED and the selected module can't clear it.
222
246
  const audit: SystemAuditReport = await runAudit(deps.audit);
223
247
  if (audit.verdict === 'BLOCKED') {
224
- return {
225
- version: 1,
226
- updateId,
227
- startedAt,
228
- finishedAt: now().toISOString(),
229
- audit,
230
- selfUpdate: { performed: false, reason: 'no-network' },
231
- backupsCreated: false,
232
- modules: [],
233
- ok: false,
234
- };
248
+ const stillBlocking = blockersOutOfReach(audit.findings, deps.onlyModule);
249
+ if (stillBlocking.length > 0) {
250
+ deps.progress.emit({
251
+ kind: 'self-update-skipped',
252
+ reason: `audit BLOCKED: ${stillBlocking.map((f) => `${f.subject}: ${f.message}`).join('; ')}`,
253
+ });
254
+ return {
255
+ version: 1,
256
+ updateId,
257
+ startedAt,
258
+ finishedAt: now().toISOString(),
259
+ audit,
260
+ selfUpdate: { performed: false, reason: 'audit-blocked' },
261
+ backupsCreated: false,
262
+ modules: [],
263
+ ok: false,
264
+ };
265
+ }
235
266
  }
236
267
 
237
268
  // Self-update.
@@ -43,7 +43,16 @@ export interface ModuleUpdateState {
43
43
  }
44
44
 
45
45
  export type SelfUpdateResult =
46
- | { performed: false; reason: 'already-current' | 'dev-mode' | 'no-network' }
46
+ | {
47
+ performed: false;
48
+ /**
49
+ * `audit-blocked` means the run refused before the self-update was
50
+ * attempted. It exists because that refusal used to report
51
+ * `no-network` (celilo#1381), sending operators to debug a network
52
+ * that was fine.
53
+ */
54
+ reason: 'already-current' | 'dev-mode' | 'no-network' | 'audit-blocked';
55
+ }
47
56
  | { performed: true; from: string; to: string };
48
57
 
49
58
  export interface SystemUpdateResult {