@celilo/cli 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/CELILO_CORE_MODULES.md +1 -0
  2. package/CELILO_SUBSYSTEMS.md +1 -1
  3. package/README.md +1 -1
  4. package/drizzle/0032_module_jail_policies.sql +28 -0
  5. package/drizzle/0033_build_bus_hook_runs.sql +37 -0
  6. package/drizzle/meta/_journal.json +15 -1
  7. package/package.json +3 -3
  8. package/schemas/system_config.json +5 -0
  9. package/src/__integration__/container-services-cli.integration.test.ts +1 -1
  10. package/src/ansible/inventory.test.ts +2 -1
  11. package/src/api/sessions.test.ts +5 -4
  12. package/src/api-clients/proxmox.ts +10 -7
  13. package/src/cli/backup-rename.test.ts +5 -4
  14. package/src/cli/cli.test.ts +3 -2
  15. package/src/cli/commands/console-get-chain.test.ts +2 -1
  16. package/src/cli/commands/events.test.ts +2 -2
  17. package/src/cli/commands/firewall-interface-list.test.ts +4 -3
  18. package/src/cli/commands/machine-list.test.ts +57 -0
  19. package/src/cli/commands/machine-list.ts +35 -4
  20. package/src/cli/commands/module-config.test.ts +2 -1
  21. package/src/cli/commands/module-deploy.ts +8 -2
  22. package/src/cli/commands/module-generate.test.ts +53 -0
  23. package/src/cli/commands/module-generate.ts +31 -26
  24. package/src/cli/commands/module-health.ts +1 -0
  25. package/src/cli/commands/module-import-aspect.test.ts +2 -1
  26. package/src/cli/commands/module-import-registry.test.ts +3 -2
  27. package/src/cli/commands/module-jail.test.ts +242 -0
  28. package/src/cli/commands/module-jail.ts +227 -0
  29. package/src/cli/commands/module-list-jail.test.ts +136 -0
  30. package/src/cli/commands/module-list.ts +30 -3
  31. package/src/cli/commands/module-operations.test.ts +2 -1
  32. package/src/cli/commands/module-publish.test.ts +11 -18
  33. package/src/cli/commands/module-update.test.ts +28 -12
  34. package/src/cli/commands/module-update.ts +4 -1
  35. package/src/cli/commands/module-upgrade.test.ts +130 -1
  36. package/src/cli/commands/module-upgrade.ts +77 -3
  37. package/src/cli/commands/module-verify.test.ts +3 -4
  38. package/src/cli/commands/module-verify.ts +0 -1
  39. package/src/cli/commands/notify-config.test.ts +5 -3
  40. package/src/cli/commands/publish/execute.ts +4 -1
  41. package/src/cli/commands/publish/index.ts +11 -1
  42. package/src/cli/commands/publish/module-registry.test.ts +24 -1
  43. package/src/cli/commands/publish/module-registry.ts +23 -4
  44. package/src/cli/commands/publish/plan.ts +1 -1
  45. package/src/cli/commands/publish/types.ts +7 -0
  46. package/src/cli/commands/registry-owner.test.ts +3 -2
  47. package/src/cli/commands/registry-token.test.ts +3 -2
  48. package/src/cli/commands/system-apply-config-equivalence.test.ts +5 -7
  49. package/src/cli/commands/system-audit.ts +20 -6
  50. package/src/cli/commands/system-config.test.ts +148 -0
  51. package/src/cli/commands/system-config.ts +26 -1
  52. package/src/cli/commands/system-doctor-remediation-gate.test.ts +295 -0
  53. package/src/cli/commands/system-doctor.test.ts +143 -6
  54. package/src/cli/commands/system-doctor.ts +142 -25
  55. package/src/cli/commands/system-init-deprecation.test.ts +7 -4
  56. package/src/cli/commands/system-migrate.test.ts +2 -1
  57. package/src/cli/commands/system-update.ts +5 -0
  58. package/src/cli/completion.ts +8 -2
  59. package/src/cli/flag-surface-gate.test.ts +279 -0
  60. package/src/cli/index.ts +70 -3
  61. package/src/cli/parser.test.ts +37 -1
  62. package/src/cli/restore-command.test.ts +5 -4
  63. package/src/cli/restore-migration-failure.test.ts +4 -3
  64. package/src/cli/tui/audit-state.ts +2 -0
  65. package/src/config/paths.test.ts +22 -22
  66. package/src/db/client.test.ts +46 -1
  67. package/src/db/client.ts +26 -0
  68. package/src/db/migrate.test.ts +2 -1
  69. package/src/db/migrate.ts +16 -16
  70. package/src/db/schema.ts +63 -0
  71. package/src/hooks/capability-loader-firewall.test.ts +9 -1
  72. package/src/hooks/capability-loader.test.ts +78 -0
  73. package/src/hooks/capability-loader.ts +45 -2
  74. package/src/hooks/executor.test.ts +161 -1
  75. package/src/hooks/executor.ts +87 -21
  76. package/src/hooks/hook-jail-unreachability.test.ts +17 -18
  77. package/src/hooks/hook-store.test.ts +6 -0
  78. package/src/hooks/hook-trespass.test.ts +26 -9
  79. package/src/hooks/jail.test.ts +152 -5
  80. package/src/hooks/jail.ts +93 -4
  81. package/src/manifest/contracts/v1.ts +13 -0
  82. package/src/module/packaging/audit.ts +9 -26
  83. package/src/module/packaging/build-paths.test.ts +127 -0
  84. package/src/module/packaging/build-paths.ts +175 -0
  85. package/src/module/packaging/build.test.ts +71 -1
  86. package/src/module/packaging/build.ts +60 -0
  87. package/src/module/packaging/extract.ts +1 -5
  88. package/src/policy/module-script-scan.ts +51 -107
  89. package/src/policy/no-hand-built-ssh.test.ts +5 -1
  90. package/src/policy/no-swallowed-refusal.test.ts +14 -15
  91. package/src/registry/client.test.ts +2 -2
  92. package/src/secrets/storage.test.ts +1 -1
  93. package/src/services/alerting/ack.test.ts +2 -1
  94. package/src/services/alerting/cadence-migration.test.ts +3 -2
  95. package/src/services/alerting/coverage-source.test.ts +2 -1
  96. package/src/services/alerting/deferral.test.ts +2 -1
  97. package/src/services/alerting/delivery-loop.test.ts +2 -1
  98. package/src/services/alerting/deploy-hooks.test.ts +2 -1
  99. package/src/services/alerting/inbound-poller.test.ts +2 -1
  100. package/src/services/alerting/inbound.test.ts +2 -1
  101. package/src/services/alerting/keys.test.ts +4 -0
  102. package/src/services/alerting/keys.ts +12 -2
  103. package/src/services/alerting/notification-responder.test.ts +2 -1
  104. package/src/services/alerting/run-monitor.test.ts +2 -1
  105. package/src/services/alerting/store.test.ts +2 -1
  106. package/src/services/alerting/sweep-runner.test.ts +2 -1
  107. package/src/services/alerting/tokens.test.ts +2 -1
  108. package/src/services/aspect-approvals.test.ts +2 -1
  109. package/src/services/aspect-reconcile.test.ts +4 -3
  110. package/src/services/aspect-runner.test.ts +2 -1
  111. package/src/services/audit/health.test.ts +97 -2
  112. package/src/services/audit/health.ts +64 -2
  113. package/src/services/audit/index.test.ts +22 -0
  114. package/src/services/audit/index.ts +8 -2
  115. package/src/services/audit/jail-exemptions.test.ts +42 -0
  116. package/src/services/audit/jail-exemptions.ts +44 -0
  117. package/src/services/audit/module-integrity.test.ts +23 -22
  118. package/src/services/audit/module-integrity.ts +7 -16
  119. package/src/services/audit/types.ts +2 -1
  120. package/src/services/backup-age-agreement.test.ts +2 -1
  121. package/src/services/backup-create.ts +9 -0
  122. package/src/services/backup-envelope-roundtrip.test.ts +3 -2
  123. package/src/services/backup-in-flight-refusal.test.ts +3 -2
  124. package/src/services/build-bus/hook-dispatch-mgmt.test.ts +48 -0
  125. package/src/services/build-bus/hook-dispatcher.ts +49 -1
  126. package/src/services/bus-ensure-flow.test.ts +3 -2
  127. package/src/services/bus-interview-park.test.ts +4 -3
  128. package/src/services/bus-interview.test.ts +2 -2
  129. package/src/services/bus-secret-flow.test.ts +3 -2
  130. package/src/services/capability-table-rows.test.ts +2 -1
  131. package/src/services/celilo-events.test.ts +1 -1
  132. package/src/services/celilo-mgmt-hooks.test.ts +23 -5
  133. package/src/services/consumer-cleanup.test.ts +3 -2
  134. package/src/services/container-service.test.ts +3 -2
  135. package/src/services/control-plane-bootstrap.test.ts +2 -1
  136. package/src/services/cross-module-read.test.ts +4 -3
  137. package/src/services/deploy-preflight.ts +7 -0
  138. package/src/services/deploy-validation.test.ts +3 -2
  139. package/src/services/deployed-systems.ts +1 -1
  140. package/src/services/dns-internal-records.test.ts +3 -2
  141. package/src/services/dns-provider-backfill.test.ts +2 -1
  142. package/src/services/dns-registrations.test.ts +2 -1
  143. package/src/services/ensure-interview.test.ts +3 -2
  144. package/src/services/fleet-checks.test.ts +126 -6
  145. package/src/services/fleet-checks.ts +176 -3
  146. package/src/services/fleet-key.test.ts +3 -2
  147. package/src/services/health-runner.test.ts +87 -2
  148. package/src/services/health-runner.ts +40 -11
  149. package/src/services/infrastructure-selector.test.ts +3 -2
  150. package/src/services/infrastructure-variable-resolver.test.ts +2 -1
  151. package/src/services/jail-exemptions.test.ts +125 -0
  152. package/src/services/jail-exemptions.ts +81 -0
  153. package/src/services/machine-pool.test.ts +3 -2
  154. package/src/services/module-config.test.ts +2 -1
  155. package/src/services/module-deploy.dns-repoint.test.ts +2 -1
  156. package/src/services/module-deploy.ts +48 -24
  157. package/src/services/module-operations.test.ts +2 -1
  158. package/src/services/module-subscriptions.test.ts +4 -3
  159. package/src/services/network-discovery.test.ts +64 -1
  160. package/src/services/network-discovery.ts +30 -5
  161. package/src/services/port-forwards.test.ts +2 -1
  162. package/src/services/programmatic-responder.aspect.test.ts +2 -1
  163. package/src/services/proxmox-reconcile.test.ts +2 -1
  164. package/src/services/responder-probe.test.ts +1 -1
  165. package/src/services/restore-from-file.test.ts +7 -6
  166. package/src/services/restore-preflight.test.ts +3 -2
  167. package/src/services/secret-schema-loader.test.ts +2 -1
  168. package/src/services/ssh-key-manager.test.ts +4 -3
  169. package/src/services/static-content-converge.test.ts +5 -4
  170. package/src/services/static-content-converge.ts +33 -10
  171. package/src/services/system-state-stage.test.ts +4 -3
  172. package/src/services/trusted-sources.test.ts +3 -2
  173. package/src/services/update/orchestrator.test.ts +2 -0
  174. package/src/services/zone-detector.test.ts +2 -1
  175. package/src/test-utils/bus-responder.ts +6 -4
  176. package/src/test-utils/db-path.ts +25 -0
  177. package/src/test-utils/integration.ts +7 -0
  178. package/src/test-utils/module-fixtures.ts +5 -6
  179. package/src/module/packaging/generated-plane.test.ts +0 -79
  180. package/src/module/packaging/generated-plane.ts +0 -134
@@ -15,6 +15,7 @@ import { closeDb, getDb } from '../db/client';
15
15
  import { runMigrations } from '../db/migrate';
16
16
  import { modules } from '../db/schema';
17
17
  import type { ModuleManifest } from '../manifest/schema';
18
+ import { resetTestDbPath } from '../test-utils/db-path';
18
19
  import {
19
20
  type CrossModuleIndex,
20
21
  applyCrossModuleWriteRoot,
@@ -70,8 +71,8 @@ describe('materializeCrossModuleRoot', () => {
70
71
 
71
72
  afterEach(() => {
72
73
  closeDb();
73
- process.env.CELILO_DB_PATH = undefined;
74
- process.env.CELILO_DATA_DIR = undefined;
74
+ resetTestDbPath();
75
+ delete process.env.CELILO_DATA_DIR;
75
76
  try {
76
77
  rmSync(dir, { recursive: true, force: true });
77
78
  } catch {
@@ -179,7 +180,7 @@ describe('applyCrossModuleWriteRoot', () => {
179
180
  });
180
181
 
181
182
  afterEach(() => {
182
- process.env.CELILO_DATA_DIR = undefined;
183
+ delete process.env.CELILO_DATA_DIR;
183
184
  try {
184
185
  rmSync(dir, { recursive: true, force: true });
185
186
  } catch {
@@ -27,6 +27,7 @@ import type { DbClient } from '../db/client';
27
27
  import { getDb } from '../db/client';
28
28
  import { capabilities, moduleConfigs, modules, secrets } from '../db/schema';
29
29
  import { type ModuleManifest, getSingularSystemSpec } from '../manifest/schema';
30
+ import { isPrivilegedCapability } from '../manifest/validate';
30
31
  import { buildResolutionContext } from '../variables/context';
31
32
  import { E2E_CONFLICT_FIX, runningE2eContainers } from './e2e-guard';
32
33
  import { describeCapabilityProblem, findBrokenCapabilityDerivations } from './fleet-checks';
@@ -113,6 +114,12 @@ export async function runPreflight(
113
114
  // template-generation against an ABI the provider can't fulfil.
114
115
  if (manifest.requires?.capabilities) {
115
116
  for (const cap of manifest.requires.capabilities) {
117
+ // Framework-granted privileges (e.g. cross_module_read) aren't
118
+ // provider-backed — they're gated by the allow-list at import time,
119
+ // not satisfied by deploying another module. Skip them here.
120
+ if (isPrivilegedCapability(cap.name)) {
121
+ continue;
122
+ }
116
123
  const providers = db
117
124
  .select()
118
125
  .from(capabilities)
@@ -21,6 +21,7 @@ import { join } from 'node:path';
21
21
  import { type DbClient, getDb } from '../db/client';
22
22
  import { modules, secrets } from '../db/schema';
23
23
  import type { ModuleManifest } from '../manifest/schema';
24
+ import { resetTestDbPath } from '../test-utils/db-path';
24
25
  import { findMissingSecrets } from './config-interview';
25
26
  import { findMissingBuildArtifacts, findMissingRequiredVariables } from './deploy-validation';
26
27
 
@@ -89,7 +90,7 @@ describe('findMissingRequiredVariables (deploy path)', () => {
89
90
 
90
91
  afterEach(() => {
91
92
  rmSync(tempDir, { recursive: true, force: true });
92
- process.env.CELILO_DB_PATH = undefined;
93
+ resetTestDbPath();
93
94
  });
94
95
 
95
96
  // The bug we're guarding against: the deploy used to drop type +
@@ -210,7 +211,7 @@ describe('findMissingSecrets (shared)', () => {
210
211
 
211
212
  afterEach(() => {
212
213
  rmSync(tempDir, { recursive: true, force: true });
213
- process.env.CELILO_DB_PATH = undefined;
214
+ resetTestDbPath();
214
215
  });
215
216
 
216
217
  test('accepts loose unknown[] declares (manifest read off disk)', async () => {
@@ -242,7 +242,7 @@ export function upsertDeployedSystem(
242
242
  .run();
243
243
  }
244
244
 
245
- function asZone(value: string | null | undefined): NetworkZone | null {
245
+ export function asZone(value: string | null | undefined): NetworkZone | null {
246
246
  return value && (NETWORK_ZONES as readonly string[]).includes(value)
247
247
  ? (value as NetworkZone)
248
248
  : null;
@@ -8,6 +8,7 @@ import { eq } from 'drizzle-orm';
8
8
  import type { DbClient } from '../db/client';
9
9
  import { dnsInternalRecords, modules } from '../db/schema';
10
10
  import { setupTestDatabaseAt } from '../test-utils/database';
11
+ import { resetTestDbPath } from '../test-utils/db-path';
11
12
  import {
12
13
  listDnsInternalRecords,
13
14
  listViewOverrides,
@@ -42,7 +43,7 @@ describe('dns-internal-records ledger', () => {
42
43
  });
43
44
  afterEach(() => {
44
45
  db.$client.close();
45
- process.env.CELILO_DB_PATH = undefined;
46
+ resetTestDbPath();
46
47
  try {
47
48
  rmSync(dir, { recursive: true, force: true });
48
49
  } catch {
@@ -248,7 +249,7 @@ describe('provider cascade (celilo#1010)', () => {
248
249
 
249
250
  afterEach(() => {
250
251
  db.$client.close();
251
- process.env.CELILO_DB_PATH = undefined;
252
+ resetTestDbPath();
252
253
  rmSync(dir, { recursive: true, force: true });
253
254
  });
254
255
 
@@ -6,6 +6,7 @@ import type { DnsRecordRequest, HookLogger } from '@celilo/capabilities';
6
6
  import { closeDb, getDb } from '../db/client';
7
7
  import { runMigrations } from '../db/migrate';
8
8
  import { capabilities, moduleConfigs, modules, webRoutes } from '../db/schema';
9
+ import { resetTestDbPath } from '../test-utils/db-path';
9
10
  import { backfillWebRouteDns } from './dns-provider-backfill';
10
11
 
11
12
  const silentLogger: HookLogger = {
@@ -26,7 +27,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
26
27
 
27
28
  afterEach(() => {
28
29
  closeDb();
29
- process.env.CELILO_DB_PATH = undefined;
30
+ resetTestDbPath();
30
31
  try {
31
32
  rmSync(dir, { recursive: true, force: true });
32
33
  } catch {
@@ -14,6 +14,7 @@ import type { DnsRegistrarCapability } from '@celilo/capabilities';
14
14
  import { eq } from 'drizzle-orm';
15
15
  import { type DbClient, getDb } from '../db/client';
16
16
  import { modules } from '../db/schema';
17
+ import { resetTestDbPath } from '../test-utils/db-path';
17
18
  import {
18
19
  listDnsRegistrations,
19
20
  recordDnsRegistration,
@@ -44,7 +45,7 @@ describe('dns_registrations ledger', () => {
44
45
 
45
46
  afterEach(() => {
46
47
  rmSync(tempDir, { recursive: true, force: true });
47
- process.env.CELILO_DB_PATH = undefined;
48
+ resetTestDbPath();
48
49
  });
49
50
 
50
51
  test('record + list roundtrip; a re-assert is an upsert, not a second row', () => {
@@ -8,6 +8,7 @@ import { moduleConfigs, modules, secrets } from '../db/schema';
8
8
  import type { Ensure } from '../manifest/schema';
9
9
  import { decryptSecret } from '../secrets/encryption';
10
10
  import { getOrCreateMasterKey } from '../secrets/master-key';
11
+ import { resetTestDbPath } from '../test-utils/db-path';
11
12
  import {
12
13
  findEnsureOnProvider,
13
14
  interviewForEnsureInputs,
@@ -56,7 +57,7 @@ describe('interviewForEnsureInputs', () => {
56
57
 
57
58
  afterEach(() => {
58
59
  rmSync(tempDir, { recursive: true, force: true });
59
- process.env.CELILO_DB_PATH = undefined;
60
+ resetTestDbPath();
60
61
  });
61
62
 
62
63
  test('appends to array config and sets secret object key', async () => {
@@ -185,7 +186,7 @@ describe('findEnsureOnProvider', () => {
185
186
 
186
187
  afterEach(() => {
187
188
  rmSync(tempDir, { recursive: true, force: true });
188
- process.env.CELILO_DB_PATH = undefined;
189
+ resetTestDbPath();
189
190
  });
190
191
 
191
192
  test('finds ensure by id on a registered provider', () => {
@@ -3,8 +3,10 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
3
3
  import { tmpdir } from 'node:os';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
6
+ import { eq } from 'drizzle-orm';
6
7
  import type { DbClient } from '../db/client';
7
8
  import {
9
+ buildBusHookRuns,
8
10
  capabilities as capabilitiesTable,
9
11
  dnsInternalRecords,
10
12
  moduleConfigs,
@@ -14,11 +16,13 @@ import {
14
16
  } from '../db/schema';
15
17
  import type { ModuleManifest } from '../manifest/schema';
16
18
  import { setupTestDatabaseAt } from '../test-utils/database';
19
+ import { resetTestDbPath } from '../test-utils/db-path';
17
20
  import { ensureInboundSubscriber, ensureSweepSubscriber } from './alerting/monitors';
18
21
  import { ensureBackupSweepSubscriber } from './backup-sweep';
19
22
  import { getDaemonUnitPath } from './events-daemon';
20
23
  import {
21
24
  type HostLivenessInputs,
25
+ checkBuildBusPublishing,
22
26
  checkCapabilityProviders,
23
27
  checkControlPlaneNetwork,
24
28
  checkDispatcher,
@@ -95,7 +99,7 @@ describe('checkDispatcher', () => {
95
99
  });
96
100
  afterEach(() => {
97
101
  bus.close();
98
- process.env.EVENT_BUS_DB = undefined;
102
+ delete process.env.EVENT_BUS_DB;
99
103
  try {
100
104
  rmSync(dir, { recursive: true, force: true });
101
105
  } catch {
@@ -269,8 +273,8 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
269
273
  afterEach(() => {
270
274
  bus.close();
271
275
  db.$client.close();
272
- process.env.CELILO_DB_PATH = undefined;
273
- process.env.EVENT_BUS_DB = undefined;
276
+ resetTestDbPath();
277
+ delete process.env.EVENT_BUS_DB;
274
278
  try {
275
279
  rmSync(dir, { recursive: true, force: true });
276
280
  } catch {
@@ -291,6 +295,118 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
291
295
  .run();
292
296
  }
293
297
 
298
+ describe('checkBuildBusPublishing', () => {
299
+ const now = Date.now();
300
+
301
+ /** A module whose installed manifest really declares on_upstream_publish. */
302
+ function insertHookModule(id: string): void {
303
+ const src = join(dir, `${id}-src`);
304
+ mkdirSync(src, { recursive: true });
305
+ writeFileSync(
306
+ join(src, 'manifest.yml'),
307
+ [
308
+ 'celilo_contract: "1.0"',
309
+ `id: ${id}`,
310
+ `name: ${id}`,
311
+ 'version: 1.0.0',
312
+ 'requires: { capabilities: [] }',
313
+ 'provides: { capabilities: [] }',
314
+ 'variables: { owns: [], imports: [] }',
315
+ 'hooks:',
316
+ ' on_upstream_publish:',
317
+ ' - name: self-update',
318
+ ' match: { registry: npm, tag: latest, package_pattern: "@celilo/cli" }',
319
+ ' script: ./scripts/update.sh',
320
+ ].join('\n'),
321
+ );
322
+ insertModule(id, baseManifest({ id, name: id }), 'VERIFIED');
323
+ // insertModule pins sourcePath to /src/<id>; point it at the real manifest.
324
+ db.update(modules).set({ sourcePath: src }).where(eq(modules.id, id)).run();
325
+ }
326
+
327
+ function seedRun(overrides: Partial<typeof buildBusHookRuns.$inferInsert> = {}): void {
328
+ db.insert(buildBusHookRuns)
329
+ .values({
330
+ eventId: 'evt-1',
331
+ packageName: '@celilo/cli',
332
+ packageVersion: '2.2.1',
333
+ tag: 'latest',
334
+ moduleId: 'celilo-mgmt',
335
+ hookName: 'self-update',
336
+ scriptPath: '/src/celilo-mgmt/scripts/update.sh',
337
+ exitCode: 0,
338
+ timedOut: false,
339
+ durationMs: 1200,
340
+ // Default to the freshest time, not the describe-time `now`:
341
+ // emitRaw stamps the bus event with its own Date.now(), and a
342
+ // run stamped even 1ms earlier would read as "predates the
343
+ // newest publish".
344
+ ranAt: new Date(Date.now()),
345
+ ...overrides,
346
+ })
347
+ .run();
348
+ }
349
+
350
+ it('is ok when no installed module declares on_upstream_publish', () => {
351
+ const f = checkBuildBusPublishing(bus, db, { now });
352
+ expect(f.status).toBe('ok');
353
+ expect(f.summary).toContain('not in use');
354
+ });
355
+
356
+ it('warns when a self-update module is installed but no publish has ever arrived', () => {
357
+ insertHookModule('celilo-mgmt');
358
+ const f = checkBuildBusPublishing(bus, db, { now });
359
+ expect(f.status).toBe('warn');
360
+ expect(f.summary).toContain('no build-bus publish has ever arrived');
361
+ expect(f.remediation).toContain('subscribers serve');
362
+ });
363
+
364
+ it('warns when publishes arrive but no hook has ever run', () => {
365
+ insertHookModule('celilo-mgmt');
366
+ bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
367
+ const f = checkBuildBusPublishing(bus, db, { now });
368
+ expect(f.status).toBe('warn');
369
+ expect(f.summary).toContain('no self-update hook has ever run');
370
+ });
371
+
372
+ it('warns with the stderr tail when the newest run failed', () => {
373
+ insertHookModule('celilo-mgmt');
374
+ bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
375
+ seedRun({ exitCode: 1, stderrTail: 'bun add -g failed; not promoting' });
376
+ const f = checkBuildBusPublishing(bus, db, { now });
377
+ expect(f.status).toBe('warn');
378
+ expect(f.summary).toContain('exited 1');
379
+ expect(f.detail.join('\n')).toContain('bun add -g failed');
380
+ });
381
+
382
+ it('warns when the newest run timed out', () => {
383
+ insertHookModule('celilo-mgmt');
384
+ bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
385
+ seedRun({ exitCode: null, timedOut: true, durationMs: 300000 });
386
+ const f = checkBuildBusPublishing(bus, db, { now });
387
+ expect(f.status).toBe('warn');
388
+ expect(f.summary).toContain('timed out');
389
+ });
390
+
391
+ it('warns when the newest run predates the newest publish — that publish dispatched nothing', () => {
392
+ insertHookModule('celilo-mgmt');
393
+ bus.emitRaw('build-bus.publish', { eventId: 'evt-2' });
394
+ seedRun({ ranAt: new Date(now - 60 * MINUTE) });
395
+ const f = checkBuildBusPublishing(bus, db, { now });
396
+ expect(f.status).toBe('warn');
397
+ expect(f.summary).toContain('dispatched nothing');
398
+ });
399
+
400
+ it('is ok when the newest run succeeded and is fresh', () => {
401
+ insertHookModule('celilo-mgmt');
402
+ bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
403
+ seedRun();
404
+ const f = checkBuildBusPublishing(bus, db, { now });
405
+ expect(f.status).toBe('ok');
406
+ expect(f.summary).toContain('@celilo/cli@2.2.1');
407
+ });
408
+ });
409
+
294
410
  describe('checkSubscribers', () => {
295
411
  it('fails when a deployed module declares a subscription the bus is missing', () => {
296
412
  insertModule(
@@ -448,7 +564,11 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
448
564
  .run();
449
565
  const f = checkCapabilityProviders(db);
450
566
  expect(f.status).toBe('ok');
451
- expect(f.detail.join(' ')).toContain('celilo capability chain');
567
+ // celilo#1308: `celilo capability chain` no longer exists, so the detail
568
+ // says the verification is by hand rather than naming a command that
569
+ // cannot run.
570
+ expect(f.detail.join(' ')).toContain('verify these by hand');
571
+ expect(f.detail.join(' ')).not.toContain('celilo capability chain');
452
572
  });
453
573
  });
454
574
 
@@ -978,8 +1098,8 @@ describe('runFleetChecks includes host liveness (#728)', () => {
978
1098
  afterEach(() => {
979
1099
  bus.close();
980
1100
  db.$client.close();
981
- process.env.CELILO_DB_PATH = undefined;
982
- process.env.EVENT_BUS_DB = undefined;
1101
+ resetTestDbPath();
1102
+ delete process.env.EVENT_BUS_DB;
983
1103
  try {
984
1104
  rmSync(dir, { recursive: true, force: true });
985
1105
  } catch {
@@ -17,13 +17,16 @@
17
17
  * rendering + `--fix` orchestration lives in the doctor command.
18
18
  */
19
19
 
20
+ import { readFileSync } from 'node:fs';
21
+ import { join } from 'node:path';
20
22
  import { type Bus, describeError } from '@celilo/event-bus';
21
- import { inArray } from 'drizzle-orm';
23
+ import { desc, eq, inArray } from 'drizzle-orm';
24
+ import { parse as parseYaml } from 'yaml';
22
25
  import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
23
26
  import { getModuleStoragePath } from '../config/paths';
24
27
  import { type DbClient, findMigrationsFolder } from '../db/client';
25
28
  import { getMigrationStatus } from '../db/migration-status';
26
- import { capabilities as capabilitiesTable, modules } from '../db/schema';
29
+ import { buildBusHookRuns, capabilities as capabilitiesTable, modules } from '../db/schema';
27
30
  import { findSchemaDrift } from '../db/schema-introspection';
28
31
  import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
29
32
  import type { ModuleManifest } from '../manifest/schema';
@@ -384,6 +387,172 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
384
387
  };
385
388
  }
386
389
 
390
+ /**
391
+ * The build-bus self-update chain, asserted outcome-first like every check
392
+ * here. Four facts separate the ways `on_upstream_publish` silently does
393
+ * nothing (celilo#1304: the CLI on a management box aged in place while the
394
+ * doctor's dispatcher check read green — that check covers the event-bus
395
+ * dispatcher, which no build-bus hook ever travels through):
396
+ *
397
+ * 1. no module declares the hook → nothing can self-update; not in use
398
+ * 2. `build-bus.publish` events arrive → the webhook path works end to end
399
+ * 3. hook-run rows exist for an event → the dispatcher ran matching hooks
400
+ * 4. the newest run exited 0 → the self-update itself succeeded
401
+ *
402
+ * Facts 3 and 4 read the `build_bus_hook_runs` ledger the dispatcher writes;
403
+ * before that ledger existed, a failed self-update existed only as daemon
404
+ * console output, which no surface reads.
405
+ */
406
+ export function checkBuildBusPublishing(
407
+ bus: Bus,
408
+ db: DbClient,
409
+ opts: { now?: number } = {},
410
+ ): FleetFinding {
411
+ const now = opts.now ?? Date.now();
412
+
413
+ const hookModules = db
414
+ .select({ id: modules.id })
415
+ .from(modules)
416
+ .where(inArray(modules.state, ['INSTALLED', 'VERIFIED']))
417
+ .all()
418
+ .filter((m) => hasUpstreamPublishHooks(db, m.id));
419
+
420
+ const finding = (
421
+ status: FleetFindingStatus,
422
+ summary: string,
423
+ detail: string[],
424
+ remediation: string | null,
425
+ ): FleetFinding => ({
426
+ id: 'build-bus-publishing',
427
+ title: 'build-bus webhooks arriving and self-update hooks running',
428
+ status,
429
+ summary,
430
+ detail,
431
+ remediation,
432
+ autoFixable: false,
433
+ });
434
+
435
+ if (hookModules.length === 0) {
436
+ return finding(
437
+ 'ok',
438
+ 'no installed module declares on_upstream_publish — build-bus self-update not in use here',
439
+ [],
440
+ null,
441
+ );
442
+ }
443
+
444
+ const newestPublish = bus.recentEvents({ type: 'build-bus.publish', limit: 1 })[0];
445
+ if (!newestPublish) {
446
+ return finding(
447
+ 'warn',
448
+ `${hookModules.map((m) => m.id).join(', ')} self-update can never fire: no build-bus publish has ever arrived`,
449
+ [
450
+ 'The webhook receiver (`celilo subscribers serve`) either is not running here or was never sent a verified publish — and nothing in the deploy installs it.',
451
+ ],
452
+ "start the receiver: `celilo subscribers serve --secret $CELILO_BUS_SECRET` under a supervisor unit, and register this host in the publisher's build-bus-subscribers.json with the same secret (export CELILO_BUS_SECRET in that unit)",
453
+ );
454
+ }
455
+
456
+ const runs = db
457
+ .select()
458
+ .from(buildBusHookRuns)
459
+ .orderBy(desc(buildBusHookRuns.id))
460
+ .limit(10)
461
+ .all();
462
+
463
+ const detail: string[] = [
464
+ `newest publish: ${newestPublish.emittedAt ? `${Math.round((now - newestPublish.emittedAt) / 60000)}min ago` : 'unknown age'}`,
465
+ ];
466
+
467
+ const newest = runs[0];
468
+ if (!newest) {
469
+ return finding(
470
+ 'warn',
471
+ 'publish events arrive but no self-update hook has ever run',
472
+ [
473
+ ...detail,
474
+ 'The receiver emitted to the bus, yet the in-process hook dispatcher dispatched nothing — it was started with --no-dispatch, no module matched, or the running CLI predates the run ledger.',
475
+ ],
476
+ "check how the receiver daemon is started (drop --no-dispatch) and compare the publish events against the installed modules' on_upstream_publish match rules",
477
+ );
478
+ }
479
+
480
+ const runAgeMin = newest.ranAt ? Math.round((now - newest.ranAt.getTime()) / 60000) : null;
481
+ const runLine = `newest hook run: ${newest.packageName}@${newest.packageVersion} on ${newest.moduleId}${runAgeMin != null ? `, ${runAgeMin}min ago` : ''}`;
482
+
483
+ if (newest.timedOut) {
484
+ return finding(
485
+ 'warn',
486
+ `self-update hook timed out after ${newest.durationMs}ms`,
487
+ [
488
+ ...detail,
489
+ runLine,
490
+ ` stderr tail: ${(newest.stderrTail ?? '(none)').trim().slice(0, 500)}`,
491
+ ],
492
+ 'inspect the ledger: `select * from build_bus_hook_runs order by id desc limit 10` in the celilo DB',
493
+ );
494
+ }
495
+ if (newest.exitCode !== 0) {
496
+ return finding(
497
+ 'warn',
498
+ `self-update hook exited ${newest.exitCode ?? 'without an exit code (spawn failure)'}`,
499
+ [
500
+ ...detail,
501
+ runLine,
502
+ ` stderr tail: ${(newest.stderrTail ?? '(none)').trim().slice(0, 500)}`,
503
+ ],
504
+ 'fix the failing hook script, then re-publish or re-emit the build-bus.publish event',
505
+ );
506
+ }
507
+
508
+ // A success that predates the newest publish means the newest publish was
509
+ // dropped on the dispatch side even though earlier ones ran. The ledger's
510
+ // ran_at has second resolution (unixepoch), so the event's millisecond
511
+ // stamp is truncated to the same granularity before comparing — otherwise
512
+ // a run in the same second as its event reads as older than it.
513
+ const newestPublishSec = Math.floor((newestPublish.emittedAt ?? 0) / 1000) * 1000;
514
+ if (newest.ranAt && newest.ranAt.getTime() < newestPublishSec) {
515
+ return finding(
516
+ 'warn',
517
+ 'newest publish event predates the newest hook run — that publish dispatched nothing',
518
+ [...detail, runLine],
519
+ 'check the receiver daemon logs for that event and the match rules against what the publisher sent',
520
+ );
521
+ }
522
+
523
+ return finding(
524
+ 'ok',
525
+ `last self-update hook succeeded (${newest.packageName}@${newest.packageVersion})`,
526
+ [...detail, runLine],
527
+ null,
528
+ );
529
+ }
530
+
531
+ /**
532
+ * Does module `id`'s installed manifest declare an `on_upstream_publish`
533
+ * hook? Reads the manifest from disk exactly as the hook dispatcher's own
534
+ * loader does, so the check and the dispatcher cannot disagree about what
535
+ * "in use" means. A manifest that has vanished or fails to parse reads as
536
+ * "no hooks" — the dispatcher skips it silently too (Rule 6.2: the skip is
537
+ * visible here as fact 1 rather than as a crash).
538
+ */
539
+ function hasUpstreamPublishHooks(db: DbClient, moduleId: string): boolean {
540
+ const row = db
541
+ .select({ sourcePath: modules.sourcePath })
542
+ .from(modules)
543
+ .where(eq(modules.id, moduleId))
544
+ .get();
545
+ if (!row?.sourcePath) return false;
546
+ try {
547
+ const parsed = parseYaml(readFileSync(join(row.sourcePath, 'manifest.yml'), 'utf-8')) as {
548
+ hooks?: { on_upstream_publish?: unknown[] };
549
+ };
550
+ return (parsed.hooks?.on_upstream_publish ?? []).length > 0;
551
+ } catch {
552
+ return false;
553
+ }
554
+ }
555
+
387
556
  /** A deployed module + its parsed manifest (INSTALLED/VERIFIED only). */
388
557
  interface DeployedModule {
389
558
  id: string;
@@ -668,7 +837,10 @@ export function checkCapabilityProviders(db: DbClient): FleetFinding {
668
837
  }
669
838
  if (traceNeeded.length > 0) {
670
839
  detail.push(
671
- `${traceNeeded.length} derived value(s) resolve through another capability — verify with \`celilo capability chain <module> <var>\` (ISS-0114): ${traceNeeded.join('; ')}`,
840
+ // celilo#1308: `celilo capability chain` no longer exists (the ISS-0115
841
+ // rework removed it), so prescribing it sent the operator to an
842
+ // "unknown command" error. No CLI command traces a chain; say so.
843
+ `${traceNeeded.length} derived value(s) resolve through another capability. No CLI command traces a capability chain, so verify these by hand (ISS-0114): ${traceNeeded.join('; ')}`,
672
844
  );
673
845
  }
674
846
 
@@ -1194,6 +1366,7 @@ export async function runFleetChecks(
1194
1366
  return [
1195
1367
  checkSchemaDrift(db),
1196
1368
  checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
1369
+ checkBuildBusPublishing(bus, db, { now: opts.now }),
1197
1370
  checkSubscribers(bus, db),
1198
1371
  checkCapabilityProviders(db),
1199
1372
  checkControlPlaneNetwork(db),
@@ -9,6 +9,7 @@ import { execFileSync } from 'node:child_process';
9
9
  import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
10
10
  import { tmpdir } from 'node:os';
11
11
  import { join } from 'node:path';
12
+ import { resetTestDbPath } from '../test-utils/db-path';
12
13
  import { ensureFleetKey, findFleetPrivateKey, getFleetSshDir } from './fleet-key';
13
14
 
14
15
  describe('ensureFleetKey', () => {
@@ -20,7 +21,7 @@ describe('ensureFleetKey', () => {
20
21
  });
21
22
 
22
23
  afterEach(() => {
23
- process.env.CELILO_DB_PATH = undefined;
24
+ resetTestDbPath();
24
25
  rmSync(dataDir, { recursive: true, force: true });
25
26
  });
26
27
 
@@ -61,7 +62,7 @@ describe('findFleetPrivateKey', () => {
61
62
  });
62
63
 
63
64
  afterEach(() => {
64
- process.env.CELILO_DB_PATH = undefined;
65
+ resetTestDbPath();
65
66
  process.env.HOME = realHome;
66
67
  rmSync(dataDir, { recursive: true, force: true });
67
68
  rmSync(homeDir, { recursive: true, force: true });