@celilo/cli 2.2.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/CELILO_CORE_MODULES.md +1 -0
  2. package/CELILO_SUBSYSTEMS.md +1 -1
  3. package/README.md +1 -1
  4. package/drizzle/0032_module_jail_policies.sql +28 -0
  5. package/drizzle/0033_build_bus_hook_runs.sql +37 -0
  6. package/drizzle/meta/_journal.json +15 -1
  7. package/package.json +3 -3
  8. package/src/__integration__/container-services-cli.integration.test.ts +1 -1
  9. package/src/api/sessions.test.ts +3 -3
  10. package/src/api-clients/proxmox.ts +10 -7
  11. package/src/cli/backup-rename.test.ts +3 -3
  12. package/src/cli/cli.test.ts +1 -1
  13. package/src/cli/commands/events.test.ts +2 -2
  14. package/src/cli/commands/firewall-interface-list.test.ts +2 -2
  15. package/src/cli/commands/machine-list.test.ts +57 -0
  16. package/src/cli/commands/machine-list.ts +35 -4
  17. package/src/cli/commands/module-health.ts +1 -0
  18. package/src/cli/commands/module-import-registry.test.ts +1 -1
  19. package/src/cli/commands/module-jail.test.ts +242 -0
  20. package/src/cli/commands/module-jail.ts +227 -0
  21. package/src/cli/commands/module-list-jail.test.ts +136 -0
  22. package/src/cli/commands/module-list.ts +30 -3
  23. package/src/cli/commands/module-publish.test.ts +6 -6
  24. package/src/cli/commands/module-update.test.ts +22 -7
  25. package/src/cli/commands/module-update.ts +4 -1
  26. package/src/cli/commands/module-upgrade.test.ts +115 -1
  27. package/src/cli/commands/module-upgrade.ts +23 -1
  28. package/src/cli/commands/module-verify.test.ts +1 -1
  29. package/src/cli/commands/publish/execute.ts +4 -1
  30. package/src/cli/commands/publish/index.ts +11 -1
  31. package/src/cli/commands/publish/module-registry.test.ts +24 -1
  32. package/src/cli/commands/publish/module-registry.ts +23 -4
  33. package/src/cli/commands/publish/plan.ts +1 -1
  34. package/src/cli/commands/publish/types.ts +7 -0
  35. package/src/cli/commands/registry-owner.test.ts +1 -1
  36. package/src/cli/commands/registry-token.test.ts +1 -1
  37. package/src/cli/commands/system-audit.ts +20 -6
  38. package/src/cli/commands/system-doctor-remediation-gate.test.ts +295 -0
  39. package/src/cli/commands/system-doctor.test.ts +76 -10
  40. package/src/cli/commands/system-doctor.ts +34 -3
  41. package/src/cli/commands/system-init-deprecation.test.ts +1 -1
  42. package/src/cli/commands/system-update.ts +5 -0
  43. package/src/cli/completion.ts +8 -2
  44. package/src/cli/flag-surface-gate.test.ts +279 -0
  45. package/src/cli/index.ts +68 -2
  46. package/src/cli/parser.test.ts +37 -1
  47. package/src/cli/restore-command.test.ts +3 -3
  48. package/src/cli/restore-migration-failure.test.ts +2 -2
  49. package/src/cli/tui/audit-state.ts +2 -0
  50. package/src/config/paths.test.ts +19 -19
  51. package/src/db/client.test.ts +46 -1
  52. package/src/db/client.ts +26 -0
  53. package/src/db/schema.ts +63 -0
  54. package/src/hooks/capability-loader-firewall.test.ts +9 -1
  55. package/src/hooks/capability-loader.ts +8 -0
  56. package/src/hooks/executor.test.ts +86 -12
  57. package/src/hooks/executor.ts +31 -1
  58. package/src/hooks/hook-jail-unreachability.test.ts +17 -18
  59. package/src/hooks/hook-trespass.test.ts +9 -2
  60. package/src/hooks/jail.test.ts +105 -17
  61. package/src/hooks/jail.ts +47 -10
  62. package/src/manifest/contracts/v1.ts +13 -0
  63. package/src/policy/module-script-scan.ts +52 -68
  64. package/src/policy/no-hand-built-ssh.test.ts +5 -1
  65. package/src/policy/no-swallowed-refusal.test.ts +14 -14
  66. package/src/registry/client.test.ts +2 -2
  67. package/src/secrets/storage.test.ts +1 -1
  68. package/src/services/alerting/keys.test.ts +4 -0
  69. package/src/services/alerting/keys.ts +12 -2
  70. package/src/services/audit/health.test.ts +97 -2
  71. package/src/services/audit/health.ts +64 -2
  72. package/src/services/audit/index.test.ts +22 -0
  73. package/src/services/audit/index.ts +8 -2
  74. package/src/services/audit/jail-exemptions.test.ts +42 -0
  75. package/src/services/audit/jail-exemptions.ts +44 -0
  76. package/src/services/audit/module-integrity.test.ts +23 -1
  77. package/src/services/audit/module-integrity.ts +7 -2
  78. package/src/services/audit/types.ts +2 -1
  79. package/src/services/backup-create.ts +9 -0
  80. package/src/services/backup-envelope-roundtrip.test.ts +1 -1
  81. package/src/services/backup-in-flight-refusal.test.ts +1 -1
  82. package/src/services/build-bus/hook-dispatch-mgmt.test.ts +48 -0
  83. package/src/services/build-bus/hook-dispatcher.ts +49 -1
  84. package/src/services/bus-ensure-flow.test.ts +1 -1
  85. package/src/services/bus-interview-park.test.ts +2 -2
  86. package/src/services/bus-interview.test.ts +2 -2
  87. package/src/services/bus-secret-flow.test.ts +1 -1
  88. package/src/services/celilo-events.test.ts +1 -1
  89. package/src/services/celilo-mgmt-hooks.test.ts +23 -5
  90. package/src/services/container-service.test.ts +1 -1
  91. package/src/services/cross-module-read.test.ts +2 -2
  92. package/src/services/deploy-preflight.ts +7 -0
  93. package/src/services/deployed-systems.ts +1 -1
  94. package/src/services/fleet-checks.test.ts +123 -4
  95. package/src/services/fleet-checks.ts +176 -3
  96. package/src/services/health-runner.test.ts +87 -2
  97. package/src/services/health-runner.ts +40 -11
  98. package/src/services/infrastructure-selector.test.ts +1 -1
  99. package/src/services/jail-exemptions.test.ts +125 -0
  100. package/src/services/jail-exemptions.ts +81 -0
  101. package/src/services/machine-pool.test.ts +1 -1
  102. package/src/services/module-deploy.ts +12 -4
  103. package/src/services/module-subscriptions.test.ts +2 -2
  104. package/src/services/network-discovery.test.ts +64 -1
  105. package/src/services/network-discovery.ts +30 -5
  106. package/src/services/responder-probe.test.ts +1 -1
  107. package/src/services/restore-from-file.test.ts +4 -4
  108. package/src/services/restore-preflight.test.ts +1 -1
  109. package/src/services/ssh-key-manager.test.ts +2 -2
  110. package/src/services/system-state-stage.test.ts +2 -2
  111. package/src/services/update/orchestrator.test.ts +2 -0
  112. package/src/test-utils/bus-responder.ts +1 -1
@@ -43,7 +43,7 @@ describe('handleModulePublish — validation', () => {
43
43
  // will skip the validation path it's trying to exercise.
44
44
  const origToken = process.env.CELILO_PUBLISH_TOKEN;
45
45
  const origDataDir = process.env.CELILO_DATA_DIR;
46
- process.env.CELILO_PUBLISH_TOKEN = undefined;
46
+ delete process.env.CELILO_PUBLISH_TOKEN;
47
47
  process.env.CELILO_DB_PATH = `/tmp/test-celilo-no-token-${Date.now()}.db`;
48
48
  process.env.CELILO_DATA_DIR = `/tmp/test-celilo-data-${Date.now()}`;
49
49
  try {
@@ -54,7 +54,7 @@ describe('handleModulePublish — validation', () => {
54
54
  if (origToken !== undefined) process.env.CELILO_PUBLISH_TOKEN = origToken;
55
55
  resetTestDbPath();
56
56
  if (origDataDir !== undefined) process.env.CELILO_DATA_DIR = origDataDir;
57
- else process.env.CELILO_DATA_DIR = undefined;
57
+ else delete process.env.CELILO_DATA_DIR;
58
58
  }
59
59
  });
60
60
 
@@ -169,7 +169,7 @@ describe('resolveToken — precedence', () => {
169
169
  expect(t).toBe('flag-token');
170
170
  } finally {
171
171
  if (origEnv !== undefined) process.env.CELILO_PUBLISH_TOKEN = origEnv;
172
- else process.env.CELILO_PUBLISH_TOKEN = undefined;
172
+ else delete process.env.CELILO_PUBLISH_TOKEN;
173
173
  resetTestDbPath();
174
174
  }
175
175
  });
@@ -183,7 +183,7 @@ describe('resolveToken — precedence', () => {
183
183
  expect(t).toBe('env-token');
184
184
  } finally {
185
185
  if (origEnv !== undefined) process.env.CELILO_PUBLISH_TOKEN = origEnv;
186
- else process.env.CELILO_PUBLISH_TOKEN = undefined;
186
+ else delete process.env.CELILO_PUBLISH_TOKEN;
187
187
  resetTestDbPath();
188
188
  }
189
189
  });
@@ -193,7 +193,7 @@ describe('resolveToken — precedence', () => {
193
193
  // DB and returns null gracefully.
194
194
  const origEnv = process.env.CELILO_PUBLISH_TOKEN;
195
195
  const origDataDir = process.env.CELILO_DATA_DIR;
196
- process.env.CELILO_PUBLISH_TOKEN = undefined;
196
+ delete process.env.CELILO_PUBLISH_TOKEN;
197
197
  process.env.CELILO_DB_PATH = `/tmp/test-celilo-resolve-empty-${Date.now()}.db`;
198
198
  process.env.CELILO_DATA_DIR = `/tmp/test-celilo-data-empty-${Date.now()}`;
199
199
  try {
@@ -203,7 +203,7 @@ describe('resolveToken — precedence', () => {
203
203
  if (origEnv !== undefined) process.env.CELILO_PUBLISH_TOKEN = origEnv;
204
204
  resetTestDbPath();
205
205
  if (origDataDir !== undefined) process.env.CELILO_DATA_DIR = origDataDir;
206
- else process.env.CELILO_DATA_DIR = undefined;
206
+ else delete process.env.CELILO_DATA_DIR;
207
207
  }
208
208
  });
209
209
  });
@@ -107,12 +107,15 @@ description: fixture
107
107
  // Seed the modules table with the "currently installed" record.
108
108
  // We deliberately put a registry-style version string in here so
109
109
  // displayVersion-less upgrades preserve the previousVersion field.
110
+ // celilo#1363: updatedAt is seeded frozen so the success assertions can
111
+ // prove the stamp actually happens whenever the version moves.
110
112
  db.insert(modules)
111
113
  .values({
112
114
  id: 'testmod',
113
115
  name: 'Test Module',
114
116
  sourcePath: installedDir,
115
117
  version: '1.0.0+5',
118
+ updatedAt: new Date('2026-04-25T16:26:31Z'),
116
119
  manifestData: {
117
120
  celilo_contract: '1.0',
118
121
  id: 'testmod',
@@ -126,7 +129,7 @@ description: fixture
126
129
  afterEach(() => {
127
130
  rmSync(tempDir, { recursive: true, force: true });
128
131
  resetTestDbPath();
129
- process.env.CELILO_ORIGINAL_CWD = undefined;
132
+ delete process.env.CELILO_ORIGINAL_CWD;
130
133
  });
131
134
 
132
135
  test('returns previousVersion + newVersion in the success outcome', async () => {
@@ -153,6 +156,18 @@ description: fixture
153
156
  expect(row.version).toBe('1.0.0+6');
154
157
  });
155
158
 
159
+ // celilo#1363: the version moved (1.0.5+5 → +6) while updatedAt stayed at
160
+ // 2026-04-25, so the audit read the module as untouched and fine. The stamp
161
+ // must happen in updateOne itself, where every version write lives.
162
+ test('writing a new version stamps updatedAt', async () => {
163
+ const result = await updateOne(srcDir, db, {}, { quiet: true, displayVersion: '1.0.0+6' });
164
+ expect(result.status).toBe('success');
165
+
166
+ const row = db.select().from(modules).all()[0];
167
+ expect(row.version).toBe('1.0.0+6');
168
+ expect(row.updatedAt?.getTime()).toBeGreaterThan(new Date('2026-04-25T16:26:31Z').getTime());
169
+ });
170
+
156
171
  test('quiet=true suppresses the per-call log lines (caller renders its own)', async () => {
157
172
  // We can't easily intercept the log helpers' output without adding test
158
173
  // hooks, so instead we exercise that the call simply succeeds and
@@ -257,7 +272,7 @@ subscriptions:
257
272
  bus.close();
258
273
  }
259
274
  } finally {
260
- process.env.EVENT_BUS_DB = undefined;
275
+ delete process.env.EVENT_BUS_DB;
261
276
  }
262
277
  });
263
278
  });
@@ -315,8 +330,8 @@ describe('registry sweep — an unanswered breaking update is not a decline', ()
315
330
  server.stop(true);
316
331
  rmSync(tempDir, { recursive: true, force: true });
317
332
  resetTestDbPath();
318
- process.env.CELILO_ORIGINAL_CWD = undefined;
319
- process.env.EVENT_BUS_DB = undefined;
333
+ delete process.env.CELILO_ORIGINAL_CWD;
334
+ delete process.env.EVENT_BUS_DB;
320
335
  });
321
336
 
322
337
  test('reports it as unanswered, never as declined, and fails the sweep', async () => {
@@ -389,7 +404,7 @@ description: fixture
389
404
  afterEach(() => {
390
405
  rmSync(tempDir, { recursive: true, force: true });
391
406
  resetTestDbPath();
392
- process.env.CELILO_ORIGINAL_CWD = undefined;
407
+ delete process.env.CELILO_ORIGINAL_CWD;
393
408
  });
394
409
 
395
410
  test('refuses by name, and writes NOTHING', async () => {
@@ -501,7 +516,7 @@ describe('updateOne — an update that fails partway leaves the install untouche
501
516
  afterEach(() => {
502
517
  rmSync(tempDir, { recursive: true, force: true });
503
518
  resetTestDbPath();
504
- process.env.CELILO_ORIGINAL_CWD = undefined;
519
+ delete process.env.CELILO_ORIGINAL_CWD;
505
520
  });
506
521
 
507
522
  test('a failed update leaves the installed tree byte-identical', async () => {
@@ -596,7 +611,7 @@ describe('updateOne — a .netapp package source', () => {
596
611
  afterEach(() => {
597
612
  rmSync(tempDir, { recursive: true, force: true });
598
613
  resetTestDbPath();
599
- process.env.CELILO_ORIGINAL_CWD = undefined;
614
+ delete process.env.CELILO_ORIGINAL_CWD;
600
615
  });
601
616
 
602
617
  // ce-h4of: the temp-dir cleanup ran BEFORE the staged swap, and
@@ -475,12 +475,15 @@ export async function updateOne(
475
475
 
476
476
  // Update manifest in database. We persist the display version (with
477
477
  // +N when known) so subsequent `module list` / `module update` calls
478
- // see the same version string the registry reported.
478
+ // see the same version string the registry reported. updatedAt moves with
479
+ // the version: a frozen timestamp beside a new version read as "nothing
480
+ // happened here" to the audit (celilo#1363).
479
481
  db.update(modules)
480
482
  .set({
481
483
  manifestData: newManifest as unknown as Record<string, unknown>,
482
484
  version: newVersion,
483
485
  name: newManifest.name,
486
+ updatedAt: new Date(),
484
487
  })
485
488
  .where(eq(modules.id, moduleId))
486
489
  .run();
@@ -1,5 +1,16 @@
1
- import { describe, expect, test } from 'bun:test';
1
+ import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
2
+ import type { Mock } from 'bun:test';
3
+ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
4
+ import { tmpdir } from 'node:os';
5
+ import { join } from 'node:path';
6
+ import { eq } from 'drizzle-orm';
7
+ import { create as tarCreate } from 'tar';
8
+ import { type DbClient, getDb } from '../../db/client';
9
+ import { type ModuleState, modules } from '../../db/schema';
2
10
  import type { ModuleManifest } from '../../manifest/schema';
11
+ import { RegistryClient } from '../../registry/client';
12
+ import * as moduleDeploy from '../../services/module-deploy';
13
+ import { resetTestDbPath } from '../../test-utils/db-path';
3
14
  import {
4
15
  type PollCandidate,
5
16
  isPollInvocation,
@@ -8,6 +19,7 @@ import {
8
19
  pickAutoUpgrade,
9
20
  pickUpgradePolicy,
10
21
  selectPollTargets,
22
+ upgradeOneModule,
11
23
  } from './module-upgrade';
12
24
 
13
25
  /** Minimal manifest fixture; only `hooks` matters for the backup gate. */
@@ -70,6 +82,7 @@ describe('selectPollTargets (ISS-0139 — opted-in + a newer registry version)',
70
82
  installed: '1.0.0',
71
83
  latest: '1.0.1',
72
84
  autoUpgrade: true,
85
+ inError: false,
73
86
  ...over,
74
87
  });
75
88
 
@@ -92,6 +105,13 @@ describe('selectPollTargets (ISS-0139 — opted-in + a newer registry version)',
92
105
  expect(selectPollTargets([base({ latest: null })])).toEqual([]);
93
106
  });
94
107
 
108
+ // celilo#1363: the poll burned five attempts redeploying celilo-website
109
+ // against a failure that never went away. A module in ERROR stays skipped
110
+ // until an attended health check clears it.
111
+ test('skips modules in ERROR state (a failed upgrade deploy)', () => {
112
+ expect(selectPollTargets([base({ inError: true })])).toEqual([]);
113
+ });
114
+
95
115
  test('picks only the eligible subset from a mixed set', () => {
96
116
  const targets = selectPollTargets([
97
117
  base({ moduleId: 'a', installed: '1.0.0', latest: '1.0.1', autoUpgrade: true }),
@@ -136,3 +156,97 @@ describe('needsPreUpgradeBackup (ISS-0168 — gate on the TARGET version manifes
136
156
  expect(needsPreUpgradeBackup('safe', manifest(undefined))).toBe(false);
137
157
  });
138
158
  });
159
+
160
+ // celilo#1363: fetchAndUpdate commits the new version to the store BEFORE the
161
+ // deploy runs, so a failed deploy left a new version recorded beside a stale
162
+ // VERIFIED state and a frozen updatedAt, and the audit read the module as
163
+ // fine. Option A: the row must say what happened — state ERROR with the
164
+ // error, and updatedAt moving whenever the version moves.
165
+ describe('upgradeOneModule — a failed deploy records the failure in the row (celilo#1363)', () => {
166
+ const FROZEN = new Date('2026-04-25T16:26:31Z');
167
+ let tempDir: string;
168
+ let installedDir: string;
169
+ let db: DbClient;
170
+ let getIndexSpy: Mock<(name: string) => Promise<unknown[]>>;
171
+ let downloadSpy: Mock<(name: string, vers: string, cksum?: string) => Promise<ArrayBuffer>>;
172
+ let deploySpy: Mock<typeof moduleDeploy.deployModule>;
173
+
174
+ beforeEach(async () => {
175
+ tempDir = mkdtempSync(join(tmpdir(), 'celilo-upgrade-'));
176
+ process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
177
+ process.env.CELILO_ORIGINAL_CWD = tempDir;
178
+
179
+ installedDir = join(tempDir, 'installed', 'testmod');
180
+ mkdirSync(installedDir, { recursive: true });
181
+ writeFileSync(
182
+ join(installedDir, 'manifest.yml'),
183
+ 'celilo_contract: "1.0"\nid: testmod\nname: Test Module\nversion: 1.0.0\ndescription: fixture\n',
184
+ );
185
+
186
+ db = getDb();
187
+ db.insert(modules)
188
+ .values({
189
+ id: 'testmod',
190
+ name: 'Test Module',
191
+ sourcePath: installedDir,
192
+ version: '1.0.0+5',
193
+ state: 'VERIFIED',
194
+ updatedAt: FROZEN,
195
+ manifestData: {
196
+ celilo_contract: '1.0',
197
+ id: 'testmod',
198
+ name: 'Test Module',
199
+ version: '1.0.0',
200
+ },
201
+ })
202
+ .run();
203
+
204
+ // Build the registry package for a patch bump (1.0.0+5 → 1.0.0+6).
205
+ // Patch/revision → fast posture, so no snapshot and no backup are needed
206
+ // on the path to the deploy.
207
+ const srcDir = join(tempDir, 'pkg-src');
208
+ mkdirSync(srcDir, { recursive: true });
209
+ writeFileSync(
210
+ join(srcDir, 'manifest.yml'),
211
+ 'celilo_contract: "1.0"\nid: testmod\nname: Test Module\nversion: 1.0.0\ndescription: fixture v6\n',
212
+ );
213
+ const pkgPath = join(tempDir, 'testmod-1.0.0.netapp');
214
+ await tarCreate({ file: pkgPath, cwd: srcDir }, ['manifest.yml']);
215
+ const pkgBytes = new Uint8Array(await Bun.file(pkgPath).arrayBuffer());
216
+
217
+ getIndexSpy = spyOn(RegistryClient.prototype, 'getIndex').mockResolvedValue([
218
+ { name: 'testmod', vers: '1.0.0+6', deps: [], cksum: 'abc', yanked: false },
219
+ ]);
220
+ downloadSpy = spyOn(RegistryClient.prototype, 'download').mockResolvedValue(pkgBytes.buffer);
221
+ deploySpy = spyOn(moduleDeploy, 'deployModule').mockResolvedValue({
222
+ success: false,
223
+ error: 'ssh: connection refused',
224
+ phases: {},
225
+ });
226
+ });
227
+
228
+ afterEach(() => {
229
+ getIndexSpy.mockRestore();
230
+ downloadSpy.mockRestore();
231
+ deploySpy.mockRestore();
232
+ rmSync(tempDir, { recursive: true, force: true });
233
+ resetTestDbPath();
234
+ delete process.env.CELILO_ORIGINAL_CWD;
235
+ });
236
+
237
+ test('a deploy failure leaves state ERROR, the new version, and a moved updatedAt', async () => {
238
+ const mod = db.select().from(modules).where(eq(modules.id, 'testmod')).get();
239
+ if (!mod) throw new Error('seed row missing');
240
+
241
+ const result = await upgradeOneModule(mod, '1.0.0+6', new RegistryClient(''), db, {});
242
+
243
+ expect(result.success).toBe(false);
244
+ const row = db.select().from(modules).where(eq(modules.id, 'testmod')).get();
245
+ if (!row) throw new Error('row missing after upgrade');
246
+ expect(row.version).toBe('1.0.0+6');
247
+ expect(row.state).toBe<ModuleState>('ERROR');
248
+ expect(row.errorMessage).toContain('1.0.0+6');
249
+ expect(row.errorMessage).toContain('ssh: connection refused');
250
+ expect(row.updatedAt?.getTime()).toBeGreaterThan(FROZEN.getTime());
251
+ });
252
+ });
@@ -127,6 +127,8 @@ export interface PollCandidate {
127
127
  /** Latest registry version, or null when the module isn't on the registry. */
128
128
  latest: string | null;
129
129
  autoUpgrade: boolean;
130
+ /** True when the module row carries state ERROR (a failed upgrade, celilo#1363). */
131
+ inError: boolean;
130
132
  }
131
133
 
132
134
  export interface PollTarget {
@@ -144,6 +146,10 @@ export function selectPollTargets(candidates: PollCandidate[]): PollTarget[] {
144
146
  const targets: PollTarget[] = [];
145
147
  for (const c of candidates) {
146
148
  if (!c.autoUpgrade || !c.latest) continue;
149
+ // A module in ERROR failed an upgrade already. Retrying it on every poll
150
+ // redeploys the same broken version indefinitely (celilo#1363), so skip it
151
+ // until an attended health check clears the state (healthy → VERIFIED).
152
+ if (c.inError) continue;
147
153
  const change = classifyVersionChange(c.installed, c.latest);
148
154
  if (change === 'up-to-date' || change === 'ahead') continue;
149
155
  targets.push({ moduleId: c.moduleId, from: c.installed, to: c.latest });
@@ -163,8 +169,10 @@ function resolveAutoUpgrade(moduleId: string): boolean {
163
169
  * Upgrade ONE module to a known target version: posture → update → backup →
164
170
  * deploy → verify. The shared core for both the single-module command and the
165
171
  * poll. `mod` is the current DB row; `targetVersion` is the registry latest.
172
+ * Exported for tests: the failure paths here are where the store diverges from
173
+ * reality (celilo#1363).
166
174
  */
167
- async function upgradeOneModule(
175
+ export async function upgradeOneModule(
168
176
  mod: typeof modules.$inferSelect,
169
177
  targetVersion: string,
170
178
  client: RegistryClient,
@@ -237,6 +245,19 @@ async function upgradeOneModule(
237
245
  // Deploy (idempotent).
238
246
  const deployed = await deployModule(moduleId, db, {});
239
247
  if (!deployed.success) {
248
+ // fetchAndUpdate already swapped the files and wrote the new version to
249
+ // the store, so the recorded version no longer matches anything verified.
250
+ // Record the failure in the state rather than leaving a stale VERIFIED
251
+ // beside the new version (celilo#1363, option A). An attended health check
252
+ // that passes moves ERROR → VERIFIED and clears the record.
253
+ db.update(modules)
254
+ .set({
255
+ state: 'ERROR',
256
+ errorMessage: `Upgrade to ${targetVersion} failed to deploy: ${deployed.error}`,
257
+ updatedAt: new Date(),
258
+ })
259
+ .where(eq(modules.id, moduleId))
260
+ .run();
240
261
  return {
241
262
  success: false,
242
263
  error: `Deploy failed for ${moduleId} (now at ${targetVersion}): ${deployed.error}`,
@@ -288,6 +309,7 @@ async function runRegistryPoll(
288
309
  installed: mod.version,
289
310
  latest: await latestRegistryVersion(client, mod.id),
290
311
  autoUpgrade: resolveAutoUpgrade(mod.id),
312
+ inError: mod.state === 'ERROR',
291
313
  });
292
314
  }
293
315
 
@@ -30,7 +30,7 @@ describe('moduleAudit (deprecation alias)', () => {
30
30
  afterEach(() => {
31
31
  rmSync(dataDir, { recursive: true, force: true });
32
32
  resetTestDbPath();
33
- if (originalDataDir === undefined) process.env.CELILO_DATA_DIR = undefined;
33
+ if (originalDataDir === undefined) delete process.env.CELILO_DATA_DIR;
34
34
  else process.env.CELILO_DATA_DIR = originalDataDir;
35
35
  });
36
36
 
@@ -63,7 +63,10 @@ export async function executePlan(input: ExecutePlanInput): Promise<PublishResul
63
63
  }
64
64
 
65
65
  if (plan.modulePublish.length > 0) {
66
- executeModulePublish(plan.modulePublish, { allowStale: plan.options.allowStale });
66
+ executeModulePublish(plan.modulePublish, {
67
+ allowStale: plan.options.allowStale,
68
+ skippedModules: plan.options.skippedModules,
69
+ });
67
70
  }
68
71
 
69
72
  return result;
@@ -20,6 +20,7 @@
20
20
  * celilo publish --release-touch # auto-fix module stale-drift
21
21
  * celilo publish --allow-stale # skip stale-version safeguards
22
22
  * celilo publish -y / --yes # auto-confirm prompts
23
+ * celilo publish --skip-module <id> # leave a module out of the registry sweep
23
24
  * celilo publish --alpha # X.Y.Z-alpha.N to @alpha
24
25
  * celilo publish --alpha --track-alpha # also force-pin alphas globally
25
26
  * celilo publish --alpha --alpha-modules # also publish module +N
@@ -120,6 +121,15 @@ export function parseOptions(argv: string[]): ParsedFlags {
120
121
  const alphaModulesFlag = argv.includes('--alpha-modules');
121
122
  const skipChangesets = argv.includes('--skip-changesets');
122
123
  const promoteArg = takeFlagValue('--promote');
124
+ const skippedModules: string[] = [];
125
+ for (let i = argv.indexOf('--skip-module'); i >= 0; i = argv.indexOf('--skip-module', i + 1)) {
126
+ const id = argv[i + 1];
127
+ if (id === undefined || id.startsWith('-')) {
128
+ console.error('✗ --skip-module requires a module id (e.g. --skip-module homebridge)');
129
+ process.exit(1);
130
+ }
131
+ skippedModules.push(id);
132
+ }
123
133
 
124
134
  if (alphaFlag && promoteArg) {
125
135
  console.error('✗ --alpha and --promote are mutually exclusive.');
@@ -152,7 +162,7 @@ export function parseOptions(argv: string[]): ParsedFlags {
152
162
  }
153
163
 
154
164
  return {
155
- options: { allowStale, autoYes: yes, mode },
165
+ options: { allowStale, autoYes: yes, mode, skippedModules },
156
166
  dryRun,
157
167
  releaseTouch,
158
168
  skipChangesets,
@@ -8,7 +8,7 @@
8
8
  */
9
9
 
10
10
  import { describe, expect, test } from 'bun:test';
11
- import { mapModuleDirsToItems } from './module-registry';
11
+ import { applyModuleSkips, mapModuleDirsToItems } from './module-registry';
12
12
 
13
13
  describe('mapModuleDirsToItems', () => {
14
14
  test('empty input → empty plan', () => {
@@ -38,3 +38,26 @@ describe('mapModuleDirsToItems', () => {
38
38
  ]);
39
39
  });
40
40
  });
41
+
42
+ describe('applyModuleSkips', () => {
43
+ const items = mapModuleDirsToItems([
44
+ '/repo/modules/caddy',
45
+ '/repo/modules/burner',
46
+ '/repo/modules/homebridge',
47
+ ]);
48
+
49
+ test('no skips leaves the plan unchanged', () => {
50
+ expect(applyModuleSkips(items, [])).toEqual(items);
51
+ });
52
+
53
+ test('drops each named module and keeps the order of the rest', () => {
54
+ expect(applyModuleSkips(items, ['burner']).map((m) => m.moduleDir)).toEqual([
55
+ '/repo/modules/caddy',
56
+ '/repo/modules/homebridge',
57
+ ]);
58
+ });
59
+
60
+ test('a name that matches no module directory throws and names it', () => {
61
+ expect(() => applyModuleSkips(items, ['burnr'])).toThrow('burnr');
62
+ });
63
+ });
@@ -11,7 +11,7 @@
11
11
 
12
12
  import { spawnSync } from 'node:child_process';
13
13
  import { existsSync } from 'node:fs';
14
- import { join, relative } from 'node:path';
14
+ import { basename, join, relative } from 'node:path';
15
15
  import { REPO_ROOT, listModuleDirs } from './helpers';
16
16
  import type { ModuleItem } from './types';
17
17
 
@@ -28,13 +28,31 @@ export function mapModuleDirsToItems(dirs: string[]): ModuleItem[] {
28
28
  return dirs.map((moduleDir) => ({ moduleDir }));
29
29
  }
30
30
 
31
- export function planModulePublish(): ModuleItem[] {
32
- return mapModuleDirsToItems(listModuleDirs());
31
+ /**
32
+ * Leave the modules named by `--skip-module` out of the sweep. A name
33
+ * that matches no module directory throws, so a typo cannot skip
34
+ * nothing and publish the module it meant to hold back.
35
+ */
36
+ export function applyModuleSkips(items: ModuleItem[], skippedModules: string[]): ModuleItem[] {
37
+ const known = new Set(items.map((m) => basename(m.moduleDir)));
38
+ const unknown = skippedModules.filter((id) => !known.has(id));
39
+ if (unknown.length > 0) {
40
+ throw new Error(`--skip-module names no module directory: ${unknown.join(', ')}`);
41
+ }
42
+ const skipped = new Set(skippedModules);
43
+ return items.filter((m) => !skipped.has(basename(m.moduleDir)));
44
+ }
45
+
46
+ export function planModulePublish(skippedModules: string[]): ModuleItem[] {
47
+ return applyModuleSkips(mapModuleDirsToItems(listModuleDirs()), skippedModules);
33
48
  }
34
49
 
35
50
  // ─── Executor ──────────────────────────────────────────────────────
36
51
 
37
- export function executeModulePublish(items: ModuleItem[], opts: { allowStale: boolean }): void {
52
+ export function executeModulePublish(
53
+ items: ModuleItem[],
54
+ opts: { allowStale: boolean; skippedModules: string[] },
55
+ ): void {
38
56
  if (items.length === 0) {
39
57
  console.log('\nNo modules with manifest.yml found — skipping module-publish phase.');
40
58
  return;
@@ -49,6 +67,7 @@ export function executeModulePublish(items: ModuleItem[], opts: { allowStale: bo
49
67
  console.log('\n──────────────────────────────────────────────');
50
68
  console.log(' Publishing modules to celilo registry');
51
69
  console.log('──────────────────────────────────────────────');
70
+ for (const id of opts.skippedModules) console.log(` modules/${id} (skipped: --skip-module)`);
52
71
  for (const m of items) console.log(` ${relative(REPO_ROOT, m.moduleDir)}`);
53
72
  console.log();
54
73
 
@@ -76,7 +76,7 @@ export function planPublish(opts: PublishOptions): PublishPlan {
76
76
  trackAlpha: opts.mode.kind === 'alpha' && opts.mode.trackAlpha,
77
77
  })
78
78
  : [],
79
- modulePublish: runPhase4 ? planModulePublish() : [],
79
+ modulePublish: runPhase4 ? planModulePublish(opts.skippedModules) : [],
80
80
  };
81
81
  }
82
82
 
@@ -61,6 +61,13 @@ export interface PublishOptions {
61
61
  allowStale: boolean;
62
62
  autoYes: boolean;
63
63
  mode: PublishMode;
64
+ /**
65
+ * Module ids to leave out of the module-registry sweep (`--skip-module
66
+ * <id>`, repeatable). For a module that cannot publish yet. The sweep
67
+ * stops at its first failed module, so one such module would otherwise
68
+ * take every module after it, and the release's .deb, down with it.
69
+ */
70
+ skippedModules: string[];
64
71
  }
65
72
 
66
73
  /**
@@ -45,7 +45,7 @@ describe('registry owner commands', () => {
45
45
  afterEach(async () => {
46
46
  fetchSpy.mockRestore();
47
47
  closeDb();
48
- process.env.CELILO_DATA_DIR = undefined;
48
+ delete process.env.CELILO_DATA_DIR;
49
49
  resetTestDbPath();
50
50
  if (existsSync(dataDir)) await rm(dataDir, { recursive: true, force: true });
51
51
  });
@@ -50,7 +50,7 @@ describe('registry token add/rm (F2 clobber fix)', () => {
50
50
 
51
51
  afterEach(async () => {
52
52
  closeDb();
53
- process.env.CELILO_DATA_DIR = undefined;
53
+ delete process.env.CELILO_DATA_DIR;
54
54
  resetTestDbPath();
55
55
  if (existsSync(dataDir)) {
56
56
  await rm(dataDir, { recursive: true, force: true });
@@ -45,7 +45,6 @@ import {
45
45
  type LatestCliVersionFetcher,
46
46
  fetchLatestCliVersion,
47
47
  } from '../../services/audit/cli-version';
48
- import type { MachineReachableResult } from '../../services/audit/machines-reachable';
49
48
  import type { ModuleVersionFetcher } from '../../services/audit/module-versions';
50
49
  import {
51
50
  loadPublicDnsEvidence,
@@ -57,8 +56,10 @@ import type { ServiceCredentialsResult } from '../../services/audit/services-cre
57
56
  import type { ServiceReachableResult } from '../../services/audit/services-reachable';
58
57
  import type { TerraformPlanRunner } from '../../services/audit/terraform-plan';
59
58
  import { getServiceCredentials, listContainerServices } from '../../services/container-service';
59
+ import { probeDiskUsage } from '../../services/disk-probe';
60
60
  import { collectFirewallReach } from '../../services/firewall-reach';
61
61
  import { runAllHealthChecks } from '../../services/health-runner';
62
+ import { collectJailExemptions } from '../../services/jail-exemptions';
62
63
  import { probeMachines } from '../../services/machine-probe';
63
64
  import { parseStoredConfigValue } from '../../services/module-config';
64
65
  import { createPublicDnsProbe, loadPublicDnsProbeSettings } from '../../services/public-dns-probe';
@@ -341,11 +342,22 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
341
342
  }),
342
343
  );
343
344
 
344
- // Machines-reachable: probe each pool machine. Shared with the monitor sweep
345
- // (services/machine-probe.ts) — this block used to be a second copy of the
346
- // same SSH logic, and both copies reported the local management box as
347
- // unreachable because celilo does not hold an SSH key for itself.
348
- const machineReachableResults: MachineReachableResult[] = await probeMachines();
345
+ // Machines-reachable and disk-space both SSH every system, so they run
346
+ // concurrently here rather than paying each other's latency. The machine
347
+ // probe is shared with the monitor sweep (services/machine-probe.ts) —
348
+ // this block used to be a second copy of the same SSH logic, and both
349
+ // copies reported the local management box as unreachable because celilo
350
+ // does not hold an SSH key for itself.
351
+ //
352
+ // The disk probe measures every system's root filesystem — the machine
353
+ // pool AND every provisioned instance, the systems celilo#1133 filled —
354
+ // in the same pre-computed measurement shape the disk_space monitor
355
+ // sweep consumes, so an audit finding and an alert page carry the same
356
+ // numbers.
357
+ const [machineReachableResults, diskUsageResults] = await Promise.all([
358
+ probeMachines(),
359
+ probeDiskUsage(db),
360
+ ]);
349
361
 
350
362
  const migrationsFolder = findMigrationsFolderSafe();
351
363
 
@@ -384,6 +396,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
384
396
  manifest: m.manifestData as ModuleManifest,
385
397
  })),
386
398
  },
399
+ jailExemptions: { exemptions: collectJailExemptions(db) },
387
400
  health: { results: healthResults },
388
401
  backups: { modules: installedBackupInfo },
389
402
  abandonedOperations: { records: loadAbandonedOperations(db) },
@@ -417,6 +430,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
417
430
  probe: createPublicDnsProbe(loadPublicDnsProbeSettings(db)),
418
431
  evidence: loadPublicDnsEvidence(db),
419
432
  },
433
+ diskSpace: { results: diskUsageResults },
420
434
  // Reads the record the poller already writes — this check never performs a
421
435
  // read of its own. One that did would drain the queue and eat the
422
436
  // acknowledgement it exists to protect (#541).