@celilo/cli 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/CELILO_CORE_MODULES.md +1 -0
  2. package/CELILO_SUBSYSTEMS.md +1 -1
  3. package/README.md +1 -1
  4. package/drizzle/0032_module_jail_policies.sql +28 -0
  5. package/drizzle/0033_build_bus_hook_runs.sql +37 -0
  6. package/drizzle/meta/_journal.json +15 -1
  7. package/package.json +3 -3
  8. package/schemas/system_config.json +5 -0
  9. package/src/__integration__/container-services-cli.integration.test.ts +1 -1
  10. package/src/ansible/inventory.test.ts +2 -1
  11. package/src/api/sessions.test.ts +5 -4
  12. package/src/api-clients/proxmox.ts +10 -7
  13. package/src/cli/backup-rename.test.ts +5 -4
  14. package/src/cli/cli.test.ts +3 -2
  15. package/src/cli/commands/console-get-chain.test.ts +2 -1
  16. package/src/cli/commands/events.test.ts +2 -2
  17. package/src/cli/commands/firewall-interface-list.test.ts +4 -3
  18. package/src/cli/commands/machine-list.test.ts +57 -0
  19. package/src/cli/commands/machine-list.ts +35 -4
  20. package/src/cli/commands/module-config.test.ts +2 -1
  21. package/src/cli/commands/module-deploy.ts +8 -2
  22. package/src/cli/commands/module-generate.test.ts +53 -0
  23. package/src/cli/commands/module-generate.ts +31 -26
  24. package/src/cli/commands/module-health.ts +1 -0
  25. package/src/cli/commands/module-import-aspect.test.ts +2 -1
  26. package/src/cli/commands/module-import-registry.test.ts +3 -2
  27. package/src/cli/commands/module-jail.test.ts +242 -0
  28. package/src/cli/commands/module-jail.ts +227 -0
  29. package/src/cli/commands/module-list-jail.test.ts +136 -0
  30. package/src/cli/commands/module-list.ts +30 -3
  31. package/src/cli/commands/module-operations.test.ts +2 -1
  32. package/src/cli/commands/module-publish.test.ts +11 -18
  33. package/src/cli/commands/module-update.test.ts +28 -12
  34. package/src/cli/commands/module-update.ts +4 -1
  35. package/src/cli/commands/module-upgrade.test.ts +130 -1
  36. package/src/cli/commands/module-upgrade.ts +77 -3
  37. package/src/cli/commands/module-verify.test.ts +3 -4
  38. package/src/cli/commands/module-verify.ts +0 -1
  39. package/src/cli/commands/notify-config.test.ts +5 -3
  40. package/src/cli/commands/publish/execute.ts +4 -1
  41. package/src/cli/commands/publish/index.ts +11 -1
  42. package/src/cli/commands/publish/module-registry.test.ts +24 -1
  43. package/src/cli/commands/publish/module-registry.ts +23 -4
  44. package/src/cli/commands/publish/plan.ts +1 -1
  45. package/src/cli/commands/publish/types.ts +7 -0
  46. package/src/cli/commands/registry-owner.test.ts +3 -2
  47. package/src/cli/commands/registry-token.test.ts +3 -2
  48. package/src/cli/commands/system-apply-config-equivalence.test.ts +5 -7
  49. package/src/cli/commands/system-audit.ts +20 -6
  50. package/src/cli/commands/system-config.test.ts +148 -0
  51. package/src/cli/commands/system-config.ts +26 -1
  52. package/src/cli/commands/system-doctor-remediation-gate.test.ts +295 -0
  53. package/src/cli/commands/system-doctor.test.ts +143 -6
  54. package/src/cli/commands/system-doctor.ts +142 -25
  55. package/src/cli/commands/system-init-deprecation.test.ts +7 -4
  56. package/src/cli/commands/system-migrate.test.ts +2 -1
  57. package/src/cli/commands/system-update.ts +5 -0
  58. package/src/cli/completion.ts +8 -2
  59. package/src/cli/flag-surface-gate.test.ts +279 -0
  60. package/src/cli/index.ts +70 -3
  61. package/src/cli/parser.test.ts +37 -1
  62. package/src/cli/restore-command.test.ts +5 -4
  63. package/src/cli/restore-migration-failure.test.ts +4 -3
  64. package/src/cli/tui/audit-state.ts +2 -0
  65. package/src/config/paths.test.ts +22 -22
  66. package/src/db/client.test.ts +46 -1
  67. package/src/db/client.ts +26 -0
  68. package/src/db/migrate.test.ts +2 -1
  69. package/src/db/migrate.ts +16 -16
  70. package/src/db/schema.ts +63 -0
  71. package/src/hooks/capability-loader-firewall.test.ts +9 -1
  72. package/src/hooks/capability-loader.test.ts +78 -0
  73. package/src/hooks/capability-loader.ts +45 -2
  74. package/src/hooks/executor.test.ts +161 -1
  75. package/src/hooks/executor.ts +87 -21
  76. package/src/hooks/hook-jail-unreachability.test.ts +17 -18
  77. package/src/hooks/hook-store.test.ts +6 -0
  78. package/src/hooks/hook-trespass.test.ts +26 -9
  79. package/src/hooks/jail.test.ts +152 -5
  80. package/src/hooks/jail.ts +93 -4
  81. package/src/manifest/contracts/v1.ts +13 -0
  82. package/src/module/packaging/audit.ts +9 -26
  83. package/src/module/packaging/build-paths.test.ts +127 -0
  84. package/src/module/packaging/build-paths.ts +175 -0
  85. package/src/module/packaging/build.test.ts +71 -1
  86. package/src/module/packaging/build.ts +60 -0
  87. package/src/module/packaging/extract.ts +1 -5
  88. package/src/policy/module-script-scan.ts +51 -107
  89. package/src/policy/no-hand-built-ssh.test.ts +5 -1
  90. package/src/policy/no-swallowed-refusal.test.ts +14 -15
  91. package/src/registry/client.test.ts +2 -2
  92. package/src/secrets/storage.test.ts +1 -1
  93. package/src/services/alerting/ack.test.ts +2 -1
  94. package/src/services/alerting/cadence-migration.test.ts +3 -2
  95. package/src/services/alerting/coverage-source.test.ts +2 -1
  96. package/src/services/alerting/deferral.test.ts +2 -1
  97. package/src/services/alerting/delivery-loop.test.ts +2 -1
  98. package/src/services/alerting/deploy-hooks.test.ts +2 -1
  99. package/src/services/alerting/inbound-poller.test.ts +2 -1
  100. package/src/services/alerting/inbound.test.ts +2 -1
  101. package/src/services/alerting/keys.test.ts +4 -0
  102. package/src/services/alerting/keys.ts +12 -2
  103. package/src/services/alerting/notification-responder.test.ts +2 -1
  104. package/src/services/alerting/run-monitor.test.ts +2 -1
  105. package/src/services/alerting/store.test.ts +2 -1
  106. package/src/services/alerting/sweep-runner.test.ts +2 -1
  107. package/src/services/alerting/tokens.test.ts +2 -1
  108. package/src/services/aspect-approvals.test.ts +2 -1
  109. package/src/services/aspect-reconcile.test.ts +4 -3
  110. package/src/services/aspect-runner.test.ts +2 -1
  111. package/src/services/audit/health.test.ts +97 -2
  112. package/src/services/audit/health.ts +64 -2
  113. package/src/services/audit/index.test.ts +22 -0
  114. package/src/services/audit/index.ts +8 -2
  115. package/src/services/audit/jail-exemptions.test.ts +42 -0
  116. package/src/services/audit/jail-exemptions.ts +44 -0
  117. package/src/services/audit/module-integrity.test.ts +23 -22
  118. package/src/services/audit/module-integrity.ts +7 -16
  119. package/src/services/audit/types.ts +2 -1
  120. package/src/services/backup-age-agreement.test.ts +2 -1
  121. package/src/services/backup-create.ts +9 -0
  122. package/src/services/backup-envelope-roundtrip.test.ts +3 -2
  123. package/src/services/backup-in-flight-refusal.test.ts +3 -2
  124. package/src/services/build-bus/hook-dispatch-mgmt.test.ts +48 -0
  125. package/src/services/build-bus/hook-dispatcher.ts +49 -1
  126. package/src/services/bus-ensure-flow.test.ts +3 -2
  127. package/src/services/bus-interview-park.test.ts +4 -3
  128. package/src/services/bus-interview.test.ts +2 -2
  129. package/src/services/bus-secret-flow.test.ts +3 -2
  130. package/src/services/capability-table-rows.test.ts +2 -1
  131. package/src/services/celilo-events.test.ts +1 -1
  132. package/src/services/celilo-mgmt-hooks.test.ts +23 -5
  133. package/src/services/consumer-cleanup.test.ts +3 -2
  134. package/src/services/container-service.test.ts +3 -2
  135. package/src/services/control-plane-bootstrap.test.ts +2 -1
  136. package/src/services/cross-module-read.test.ts +4 -3
  137. package/src/services/deploy-preflight.ts +7 -0
  138. package/src/services/deploy-validation.test.ts +3 -2
  139. package/src/services/deployed-systems.ts +1 -1
  140. package/src/services/dns-internal-records.test.ts +3 -2
  141. package/src/services/dns-provider-backfill.test.ts +2 -1
  142. package/src/services/dns-registrations.test.ts +2 -1
  143. package/src/services/ensure-interview.test.ts +3 -2
  144. package/src/services/fleet-checks.test.ts +126 -6
  145. package/src/services/fleet-checks.ts +176 -3
  146. package/src/services/fleet-key.test.ts +3 -2
  147. package/src/services/health-runner.test.ts +87 -2
  148. package/src/services/health-runner.ts +40 -11
  149. package/src/services/infrastructure-selector.test.ts +3 -2
  150. package/src/services/infrastructure-variable-resolver.test.ts +2 -1
  151. package/src/services/jail-exemptions.test.ts +125 -0
  152. package/src/services/jail-exemptions.ts +81 -0
  153. package/src/services/machine-pool.test.ts +3 -2
  154. package/src/services/module-config.test.ts +2 -1
  155. package/src/services/module-deploy.dns-repoint.test.ts +2 -1
  156. package/src/services/module-deploy.ts +48 -24
  157. package/src/services/module-operations.test.ts +2 -1
  158. package/src/services/module-subscriptions.test.ts +4 -3
  159. package/src/services/network-discovery.test.ts +64 -1
  160. package/src/services/network-discovery.ts +30 -5
  161. package/src/services/port-forwards.test.ts +2 -1
  162. package/src/services/programmatic-responder.aspect.test.ts +2 -1
  163. package/src/services/proxmox-reconcile.test.ts +2 -1
  164. package/src/services/responder-probe.test.ts +1 -1
  165. package/src/services/restore-from-file.test.ts +7 -6
  166. package/src/services/restore-preflight.test.ts +3 -2
  167. package/src/services/secret-schema-loader.test.ts +2 -1
  168. package/src/services/ssh-key-manager.test.ts +4 -3
  169. package/src/services/static-content-converge.test.ts +5 -4
  170. package/src/services/static-content-converge.ts +33 -10
  171. package/src/services/system-state-stage.test.ts +4 -3
  172. package/src/services/trusted-sources.test.ts +3 -2
  173. package/src/services/update/orchestrator.test.ts +2 -0
  174. package/src/services/zone-detector.test.ts +2 -1
  175. package/src/test-utils/bus-responder.ts +6 -4
  176. package/src/test-utils/db-path.ts +25 -0
  177. package/src/test-utils/integration.ts +7 -0
  178. package/src/test-utils/module-fixtures.ts +5 -6
  179. package/src/module/packaging/generated-plane.test.ts +0 -79
  180. package/src/module/packaging/generated-plane.ts +0 -134
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Tests for the `jail_exemptions` audit category (per-module-jail-policy
3
+ * task 3.3/3.4).
4
+ *
5
+ * The severity contract is the point: an operator-recorded exemption is
6
+ * `todo`, never `drift`, so `system update` never treats it as work. And the
7
+ * negative half: a fleet with no exemptions contributes no findings.
8
+ */
9
+
10
+ import { describe, expect, test } from 'bun:test';
11
+ import { auditJailExemptions } from './jail-exemptions';
12
+
13
+ const EXEMPT = { moduleId: 'caddy', policy: 'off' as const, systemPolicy: 'auto' as const };
14
+
15
+ describe('auditJailExemptions (per-module-jail-policy task 3.3)', () => {
16
+ test('no exemptions produce no findings (the negative half of task 3.4)', async () => {
17
+ expect(await auditJailExemptions({ exemptions: [] })).toEqual([]);
18
+ });
19
+
20
+ test('each exemption is one finding, severity todo, never drift', async () => {
21
+ const findings = await auditJailExemptions({
22
+ exemptions: [EXEMPT, { moduleId: 'vault', policy: 'auto', systemPolicy: 'required' }],
23
+ });
24
+ expect(findings).toHaveLength(2);
25
+ for (const f of findings) {
26
+ expect(f.category).toBe('jail_exemptions');
27
+ expect(f.severity).toBe('todo');
28
+ expect(f.severity).not.toBe('drift');
29
+ expect(f.code).toBe('hook_jail_exemption');
30
+ expect(f.actionable).toBe(false);
31
+ }
32
+ });
33
+
34
+ test('the finding names the module and both policies', async () => {
35
+ const [finding] = await auditJailExemptions({ exemptions: [EXEMPT] });
36
+ expect(finding).toBeDefined();
37
+ expect(finding?.subject).toBe('caddy');
38
+ expect(finding?.message).toContain('caddy');
39
+ expect(finding?.message).toContain("'off'");
40
+ expect(finding?.message).toContain("'auto'");
41
+ });
42
+ });
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Per-module hook jail exemption check (per-module-jail-policy task 3.3).
3
+ *
4
+ * One finding per module the operator exempted from the fleet's jail policy.
5
+ * Severity `todo`, deliberately NOT `drift`: the operator chose this, so it
6
+ * is a list of recorded exceptions to revisit, not a system that moved away
7
+ * from a desired state. `todo` never escalates the verdict past READY, which
8
+ * is what keeps `system update` from treating an exemption as work to do.
9
+ */
10
+
11
+ import type { JailExemption } from '../jail-exemptions';
12
+ import type { DriftFinding } from './types';
13
+
14
+ export interface JailExemptionsAuditDeps {
15
+ /**
16
+ * The exemptions, from the shared collector (`services/jail-exemptions.ts`).
17
+ * A fleet with none contributes nothing — that is the normal state, and
18
+ * emitting a "no exemptions" finding would be a surface reporting an
19
+ * exemption nobody set (task 3.4).
20
+ */
21
+ exemptions: JailExemption[];
22
+ }
23
+
24
+ export async function auditJailExemptions(deps: JailExemptionsAuditDeps): Promise<DriftFinding[]> {
25
+ return deps.exemptions.map((exemption) => ({
26
+ category: 'jail_exemptions' as const,
27
+ severity: 'todo' as const,
28
+ code: 'hook_jail_exemption',
29
+ subject: exemption.moduleId,
30
+ message: `${exemption.moduleId} runs under jail policy '${exemption.policy}', weaker than the system's '${exemption.systemPolicy}'`,
31
+ details: [
32
+ 'Its hooks run with the weaker posture while the rest of the fleet jails.',
33
+ 'The operator recorded this per-module policy with `celilo module jail`.',
34
+ 'Exemptions are deliberate, which is why this is todo and not drift —',
35
+ 'but an exemption nobody remembers is how a jail quietly stops covering',
36
+ 'the fleet. Revisit it: either the reason still holds, or clear it.',
37
+ '',
38
+ `To review: celilo module jail ${exemption.moduleId}`,
39
+ `To clear: celilo module jail ${exemption.moduleId} --clear`,
40
+ ].join('\n'),
41
+ remediation: `Clear it with 'celilo module jail ${exemption.moduleId} --clear' if the module now jails cleanly`,
42
+ actionable: false,
43
+ }));
44
+ }
@@ -58,7 +58,9 @@ describe('auditModuleIntegrity', () => {
58
58
  expect(findings).toHaveLength(1);
59
59
  expect(findings[0]?.code).toBe('module_integrity_stale_baseline');
60
60
  expect(findings[0]?.severity).toBe('unmeasured');
61
- expect(findings[0]?.remediation).toBe('celilo module update wireguard');
61
+ expect(findings[0]?.remediation).toBe(
62
+ 'no automatic remediation: the baseline predates version stamping',
63
+ );
62
64
  });
63
65
 
64
66
  test('file drift under a TRUSTWORTHY baseline is drift, and lists every file', () => {
@@ -82,27 +84,6 @@ describe('auditModuleIntegrity', () => {
82
84
  expect(findings[0]?.details).toContain('scripts/new.ts');
83
85
  });
84
86
 
85
- test('a stale generated project is drift, and says a deploy would ship the wrong bytes', () => {
86
- const findings = auditModuleIntegrity({
87
- results: [
88
- {
89
- ...clean('wireguard-manager'),
90
- success: false,
91
- violations: [
92
- {
93
- type: 'stale-generated',
94
- path: 'ansible/roles/vpn/files/bin',
95
- message: 'Generated project holds different bytes',
96
- },
97
- ],
98
- },
99
- ],
100
- });
101
- expect(findings.map((f) => f.code)).toEqual(['module_generated_stale']);
102
- expect(findings[0]?.severity).toBe('drift');
103
- expect(findings[0]?.remediation).toBe('celilo module generate wireguard-manager');
104
- });
105
-
106
87
  test('host-plane results carry through with the right severity', () => {
107
88
  const findings = auditModuleIntegrity({
108
89
  results: [
@@ -143,4 +124,24 @@ describe('auditModuleIntegrity', () => {
143
124
  expect(f.actionable).toBe(true);
144
125
  }
145
126
  });
127
+
128
+ test('a stale baseline says there is no command, it does not prescribe one (celilo#1308)', () => {
129
+ const findings = auditModuleIntegrity({
130
+ results: [
131
+ {
132
+ ...clean('c'),
133
+ success: false,
134
+ violations: [
135
+ { type: 'stale-baseline', path: 'x', message: 'baseline is v1, manifest v2' },
136
+ ],
137
+ },
138
+ ],
139
+ });
140
+ expect(findings).toHaveLength(1);
141
+ expect(findings[0].code).toBe('module_integrity_stale_baseline');
142
+ // `module update` takes a source path, not the module id this finding
143
+ // carries, and no other verb restamps a baseline. Name the gap.
144
+ expect(findings[0].remediation).not.toMatch(/celilo /);
145
+ expect(findings[0].actionable).toBe(false);
146
+ });
146
147
  });
@@ -52,22 +52,13 @@ export function auditModuleIntegrity(deps: ModuleIntegrityAuditDeps): DriftFindi
52
52
  severity: 'unmeasured',
53
53
  code: 'module_integrity_stale_baseline',
54
54
  message: `${result.moduleId}: ${violation.message}`,
55
- remediation: `celilo module update ${result.moduleId}`,
56
- actionable: true,
57
- subject: result.moduleId,
58
- });
59
- }
60
-
61
- const generated = result.violations.filter((v) => v.type === 'stale-generated');
62
- if (generated.length > 0) {
63
- findings.push({
64
- category: 'module_integrity',
65
- severity: 'drift',
66
- code: 'module_generated_stale',
67
- message: `${result.moduleId}: ${generated.length} generated asset(s) do not match the installed module — a deploy would ship the wrong bytes`,
68
- details: generated.map((v) => ` • ${v.path}`).join('\n'),
69
- remediation: `celilo module generate ${result.moduleId}`,
70
- actionable: true,
55
+ // No command restamps a baseline. `module update` takes a module
56
+ // source PATH, not an id (celilo#1308: prescribing it with the id
57
+ // named an argument form that does not exist), and `module upgrade`
58
+ // deploys the registry version instead. Say so rather than name a
59
+ // command that cannot run.
60
+ remediation: 'no automatic remediation: the baseline predates version stamping',
61
+ actionable: false,
71
62
  subject: result.moduleId,
72
63
  });
73
64
  }
@@ -45,7 +45,8 @@ export type DriftCategory =
45
45
  | 'trusted_sources'
46
46
  | 'interface_classification'
47
47
  | 'module_integrity'
48
- | 'detect_without_converge';
48
+ | 'detect_without_converge'
49
+ | 'jail_exemptions';
49
50
 
50
51
  /**
51
52
  * `unmeasured` is a check that could not reach its subject
@@ -18,6 +18,7 @@ import { join } from 'node:path';
18
18
  import { closeDb, getDb } from '../db/client';
19
19
  import { runMigrations } from '../db/migrate';
20
20
  import { backupStorages, backups, modules } from '../db/schema';
21
+ import { resetTestDbPath } from '../test-utils/db-path';
21
22
  import { loadBackupAuditInfo } from './audit/backup-source';
22
23
  import { findBackupEligibleModules } from './backup-create';
23
24
  import { loadBackupHistory } from './backup-metadata';
@@ -79,7 +80,7 @@ describe('backup age: the run path and the audit path read the same column', ()
79
80
 
80
81
  afterEach(() => {
81
82
  closeDb();
82
- process.env.CELILO_DB_PATH = undefined;
83
+ resetTestDbPath();
83
84
  rmSync(dir, { recursive: true, force: true });
84
85
  });
85
86
 
@@ -35,6 +35,7 @@ import {
35
35
  import type { Cadence } from './cadence';
36
36
  import { materializeCrossModuleRoot, moduleHasCrossModuleRead } from './cross-module-read';
37
37
  import { getModuleSystems } from './deployed-systems';
38
+ import { listMachines } from './machine-pool';
38
39
  import { parseStoredConfigValue } from './module-config';
39
40
  import {
40
41
  completeOperation,
@@ -367,6 +368,14 @@ export async function createModuleBackup(
367
368
 
368
369
  const staged = stageSystemState(join(tempDir, 'system-state'));
369
370
  hookInputs.system_state_root = staged.root;
371
+
372
+ // The machine inventory rides the same privilege. on_backup used to
373
+ // shell `celilo machine list --json` for this, which the hook jail kills
374
+ // (no celilo binary, celilo#1225) and which never worked before that:
375
+ // the CLI had no --json flag, so the hook's JSON.parse failed and its
376
+ // catch wrote [] into every envelope. listMachines() is the same read
377
+ // the CLI performs, offered as staged data instead of a spawn.
378
+ hookInputs.machine_pool = JSON.stringify(await listMachines());
370
379
  if (!staged.masterKeyStaged) {
371
380
  logger.warn(
372
381
  `master.key not found at ${getMasterKeyPath()} — secrets in the DB snapshot will be unreadable on restore.`,
@@ -32,6 +32,7 @@ import { closeDb, getDb } from '../db/client';
32
32
  import { runMigrations } from '../db/migrate';
33
33
  import { backups, systemConfig } from '../db/schema';
34
34
  import { getOrCreateMasterKey } from '../secrets/master-key';
35
+ import { resetTestDbPath } from '../test-utils/db-path';
35
36
  import { decryptFileToFile, encryptFileToFile } from './backup-cipher';
36
37
  import { createSystemStateBackup } from './backup-create';
37
38
  import { MANIFEST_SCHEMA_VERSION, buildManifest, parseManifest } from './backup-manifest';
@@ -64,8 +65,8 @@ describe('backup envelope round-trip', () => {
64
65
 
65
66
  afterEach(() => {
66
67
  closeDb();
67
- process.env.CELILO_DB_PATH = undefined;
68
- process.env.CELILO_MASTER_KEY_PATH = undefined;
68
+ resetTestDbPath();
69
+ delete process.env.CELILO_MASTER_KEY_PATH;
69
70
  try {
70
71
  rmSync(dir, { recursive: true, force: true });
71
72
  } catch {
@@ -18,6 +18,7 @@ import { join } from 'node:path';
18
18
  import { closeDb, getDb } from '../db/client';
19
19
  import { runMigrations } from '../db/migrate';
20
20
  import { backups, moduleOperations } from '../db/schema';
21
+ import { resetTestDbPath } from '../test-utils/db-path';
21
22
  import { createModuleBackup, createSystemStateBackup } from './backup-create';
22
23
  import { restoreModuleBackup, restoreSystemStateBackup } from './backup-restore';
23
24
  import { InFlightError } from './module-operations';
@@ -34,8 +35,8 @@ describe('backup/restore in-flight refusal', () => {
34
35
 
35
36
  afterEach(() => {
36
37
  closeDb();
37
- process.env.CELILO_DB_PATH = undefined;
38
- process.env.CELILO_MASTER_KEY_PATH = undefined;
38
+ resetTestDbPath();
39
+ delete process.env.CELILO_MASTER_KEY_PATH;
39
40
  try {
40
41
  rmSync(dir, { recursive: true, force: true });
41
42
  } catch {
@@ -155,3 +155,51 @@ exit 0
155
155
  expect(results).toEqual([]);
156
156
  });
157
157
  });
158
+
159
+ /**
160
+ * The durable outcome record (celilo#1304): every dispatched run lands in
161
+ * the `build_bus_hook_runs` ledger via the injectable recordRun. Without it
162
+ * a failed self-update existed only as daemon console output, which no
163
+ * surface reads.
164
+ */
165
+ describe('hook-dispatcher outcome record', () => {
166
+ test('a failing script records exit code, stderr tail, and the event identity', async () => {
167
+ const failDir = mkdtempSync(join(tmpdir(), 'hook-record-test-'));
168
+ const scriptPath = join(failDir, 'fail.sh');
169
+ writeFileSync(scriptPath, '#!/usr/bin/env bash\necho "boom" >&2\nexit 7\n');
170
+ chmodSync(scriptPath, 0o755);
171
+
172
+ const recorded: Array<{
173
+ event: PublishEvent;
174
+ result: { exitCode: number | null; stderr: string };
175
+ }> = [];
176
+ const dispatcher = await startHookDispatcher({
177
+ loadModules: () => [
178
+ {
179
+ moduleId: 'celilo-mgmt',
180
+ sourcePath: failDir,
181
+ hooks: [
182
+ {
183
+ name: 'self-update-celilo-cli',
184
+ match: { registry: 'npm', tag: 'latest', package_pattern: '@celilo/cli' },
185
+ script: 'fail.sh',
186
+ },
187
+ ],
188
+ },
189
+ ],
190
+ recordRun: (event, result) => recorded.push({ event, result }),
191
+ });
192
+
193
+ await dispatcher.handleEvent(
194
+ buildEvent({ package: { name: '@celilo/cli', version: '2.2.1' } }),
195
+ );
196
+ await dispatcher.stop();
197
+ rmSync(failDir, { recursive: true, force: true });
198
+
199
+ expect(recorded).toHaveLength(1);
200
+ expect(recorded[0].event.eventId).toBe('evt-1');
201
+ expect(recorded[0].event.package.name).toBe('@celilo/cli');
202
+ expect(recorded[0].result.exitCode).toBe(7);
203
+ expect(recorded[0].result.stderr).toContain('boom');
204
+ });
205
+ });
@@ -20,7 +20,7 @@ import { join } from 'node:path';
20
20
  import type { PublishEvent } from '@celilo/event-bus/build-bus';
21
21
  import { parse as parseYaml } from 'yaml';
22
22
  import { getDb } from '../../db/client';
23
- import { modules } from '../../db/schema';
23
+ import { buildBusHookRuns, modules } from '../../db/schema';
24
24
  import {
25
25
  type HookRunResult,
26
26
  type ModuleHookContext,
@@ -43,12 +43,59 @@ export interface HookDispatcherOptions {
43
43
  * modules table.
44
44
  */
45
45
  loadModules?: () => ModuleHookContext[];
46
+ /**
47
+ * Override the durable outcome record. The production default writes one
48
+ * `build_bus_hook_runs` row per run — without it a failed self-update
49
+ * exists only as daemon console output, which no surface reads
50
+ * (celilo#1304). Tests inject a capture to stay DB-free.
51
+ */
52
+ recordRun?: (event: PublishEvent, result: HookRunResult) => void;
53
+ }
54
+
55
+ /** Bound on stdout/stderr tails so one chatty hook cannot bloat the ledger. */
56
+ const TAIL_CHARS = 2000;
57
+
58
+ function tail(s: string): string {
59
+ return s.length > TAIL_CHARS ? s.slice(-TAIL_CHARS) : s;
60
+ }
61
+
62
+ /**
63
+ * The production outcome record: one `build_bus_hook_runs` row per run.
64
+ * Throwing here would 500 nothing (the webhook already succeeded) but
65
+ * would kill dispatch of later plans, so a record failure is logged and
66
+ * swallowed — the console outcome lines below remain the fallback signal.
67
+ */
68
+ function recordHookRun(event: PublishEvent, result: HookRunResult): void {
69
+ try {
70
+ getDb()
71
+ .insert(buildBusHookRuns)
72
+ .values({
73
+ eventId: event.eventId,
74
+ packageName: event.package.name,
75
+ packageVersion: event.package.version,
76
+ tag: event.tag,
77
+ moduleId: result.module,
78
+ hookName: result.hookName,
79
+ scriptPath: result.scriptPath,
80
+ exitCode: result.exitCode,
81
+ timedOut: result.timedOut,
82
+ durationMs: result.durationMs,
83
+ stdoutTail: result.stdout ? tail(result.stdout) : null,
84
+ stderrTail: result.stderr ? tail(result.stderr) : null,
85
+ })
86
+ .run();
87
+ } catch (err) {
88
+ console.warn(
89
+ `[build-bus] could not record hook run outcome: ${err instanceof Error ? err.message : String(err)}`,
90
+ );
91
+ }
46
92
  }
47
93
 
48
94
  export async function startHookDispatcher(
49
95
  opts: HookDispatcherOptions = {},
50
96
  ): Promise<HookDispatcher> {
51
97
  const loadModules = opts.loadModules ?? loadInstalledModulesWithHooks;
98
+ const recordRun = opts.recordRun ?? recordHookRun;
52
99
 
53
100
  return {
54
101
  async handleEvent(event) {
@@ -66,6 +113,7 @@ export async function startHookDispatcher(
66
113
  `[build-bus] ${event.package.name}@${event.package.version} → ${plan.module.moduleId} (${plan.hook.name ?? plan.hook.script})`,
67
114
  );
68
115
  const result = await runHookDispatch(plan);
116
+ recordRun(event, result);
69
117
  results.push(result);
70
118
  if (result.exitCode === 0) {
71
119
  console.log(` ✓ exit 0 in ${result.durationMs}ms`);
@@ -27,6 +27,7 @@ import { moduleConfigs, modules, secrets } from '../db/schema';
27
27
  import type { Ensure } from '../manifest/schema';
28
28
  import { decryptSecret } from '../secrets/encryption';
29
29
  import { getOrCreateMasterKey } from '../secrets/master-key';
30
+ import { resetTestDbPath } from '../test-utils/db-path';
30
31
  import type { EnsureRequiredPayload } from './bus-interview';
31
32
  import { interviewForEnsureInputs, writeModuleSecretKey } from './config-interview';
32
33
 
@@ -174,8 +175,8 @@ describe('bus-mediated interviewForEnsureInputs', () => {
174
175
  afterEach(() => {
175
176
  bus.close();
176
177
  rmSync(tempDir, { recursive: true, force: true });
177
- process.env.CELILO_DB_PATH = undefined;
178
- process.env.EVENT_BUS_DB = undefined;
178
+ resetTestDbPath();
179
+ delete process.env.EVENT_BUS_DB;
179
180
  });
180
181
 
181
182
  test('mixed inputs: array applied deterministic, config + secret routed through bus', async () => {
@@ -26,6 +26,7 @@ import { type Bus, type BusEvent, defineEvents, openBus } from '@celilo/event-bu
26
26
  import { handleModuleUpdate } from '../cli/commands/module-update';
27
27
  import { getDb } from '../db/client';
28
28
  import { modules } from '../db/schema';
29
+ import { resetTestDbPath } from '../test-utils/db-path';
29
30
  import { RESPONDER_PROBE_EVENT } from './responder-probe';
30
31
 
31
32
  const NO_SCHEMAS = defineEvents({});
@@ -91,9 +92,9 @@ beforeEach(() => {
91
92
  afterEach(() => {
92
93
  server.stop(true);
93
94
  rmSync(dir, { recursive: true, force: true });
94
- process.env.CELILO_DB_PATH = undefined;
95
- process.env.CELILO_ORIGINAL_CWD = undefined;
96
- process.env.EVENT_BUS_DB = undefined;
95
+ resetTestDbPath();
96
+ delete process.env.CELILO_ORIGINAL_CWD;
97
+ delete process.env.EVENT_BUS_DB;
97
98
  });
98
99
 
99
100
  test('a command parks on an unanswered interview, then resumes with an answer given later', async () => {
@@ -24,7 +24,7 @@ describe('busInterview', () => {
24
24
  process.env.EVENT_BUS_DB = dbPath;
25
25
  });
26
26
  afterEach(() => {
27
- if (origEnv === undefined) process.env.EVENT_BUS_DB = undefined;
27
+ if (origEnv === undefined) delete process.env.EVENT_BUS_DB;
28
28
  else process.env.EVENT_BUS_DB = origEnv;
29
29
  try {
30
30
  rmSync(dir, { recursive: true, force: true });
@@ -156,7 +156,7 @@ describe('multi-select payload roundtrip', () => {
156
156
  process.env.EVENT_BUS_DB = dbPath;
157
157
  });
158
158
  afterEach(() => {
159
- if (origEnv === undefined) process.env.EVENT_BUS_DB = undefined;
159
+ if (origEnv === undefined) delete process.env.EVENT_BUS_DB;
160
160
  else process.env.EVENT_BUS_DB = origEnv;
161
161
  try {
162
162
  rmSync(dir, { recursive: true, force: true });
@@ -21,6 +21,7 @@ import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
21
21
  import { type DbClient, getDb } from '../db/client';
22
22
  import { modules } from '../db/schema';
23
23
  import { getOrCreateMasterKey } from '../secrets/master-key';
24
+ import { resetTestDbPath } from '../test-utils/db-path';
24
25
  import type { SecretRequiredPayload } from './bus-interview';
25
26
  import {
26
27
  interviewForMissingSecrets,
@@ -154,8 +155,8 @@ describe('bus-mediated interviewForMissingSecrets', () => {
154
155
  afterEach(() => {
155
156
  bus.close();
156
157
  rmSync(tempDir, { recursive: true, force: true });
157
- process.env.CELILO_DB_PATH = undefined;
158
- process.env.EVENT_BUS_DB = undefined;
158
+ resetTestDbPath();
159
+ delete process.env.EVENT_BUS_DB;
159
160
  });
160
161
 
161
162
  test('user_provided: emits secret.required, responder writes, value lands in store', async () => {
@@ -20,6 +20,7 @@ import { allDeclaredTables } from '@celilo/capabilities';
20
20
  import type { DbClient } from '../db/client';
21
21
  import { dnsInternalRecords, modules, portForwards, trustedSources, webRoutes } from '../db/schema';
22
22
  import { setupTestDatabaseAt } from '../test-utils/database';
23
+ import { resetTestDbPath } from '../test-utils/db-path';
23
24
  import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
24
25
 
25
26
  const FW = '192.168.0.254';
@@ -74,7 +75,7 @@ describe('deleteClaimedRows', () => {
74
75
 
75
76
  afterEach(() => {
76
77
  db.$client.close();
77
- process.env.CELILO_DB_PATH = undefined;
78
+ resetTestDbPath();
78
79
  rmSync(dir, { recursive: true, force: true });
79
80
  });
80
81
 
@@ -29,7 +29,7 @@ describe('celilo lifecycle events', () => {
29
29
  process.env.EVENT_BUS_DB = dbPath;
30
30
  });
31
31
  afterEach(() => {
32
- process.env.EVENT_BUS_DB = undefined;
32
+ delete process.env.EVENT_BUS_DB;
33
33
  try {
34
34
  rmSync(dir, { recursive: true, force: true });
35
35
  } catch {
@@ -11,6 +11,14 @@ import type { HookContext } from '@celilo/capabilities';
11
11
  // runnable from the apps/celilo working dir.
12
12
  const HOOK_DIR = join(import.meta.dirname, '../../../../modules/celilo-mgmt/scripts');
13
13
 
14
+ // on_backup reads its machine-pool snapshot from the context input the
15
+ // framework stages (backup-create.ts, from listMachines()). These tests
16
+ // inject a JSON string so the snapshot is deterministic on every host. The
17
+ // pre-staging version of this hook shelled `celilo machine list --json` and
18
+ // these tests had to inject a runner stub to avoid depending on the installed
19
+ // CLI's age (celilo#1333); the staged input made that seam unnecessary.
20
+ const STUB_MACHINE_POOL = [{ hostname: 'stub-machine', zone: 'dmz' }];
21
+
14
22
  interface BackupHookOutput {
15
23
  artifact_count: number;
16
24
  size_bytes: number;
@@ -116,6 +124,7 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
116
124
  backup_dir: backupDir,
117
125
  cross_module_root: crossModuleRoot,
118
126
  system_state_root: systemStateRoot,
127
+ machine_pool: JSON.stringify(STUB_MACHINE_POOL),
119
128
  }),
120
129
  )) as BackupHookOutput;
121
130
 
@@ -135,7 +144,6 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
135
144
  expect(result.artifact_count).toBeGreaterThan(0);
136
145
  expect(result.size_bytes).toBeGreaterThan(0);
137
146
  });
138
-
139
147
  it('refuses when the framework did not stage celilo state', async () => {
140
148
  const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
141
149
 
@@ -148,24 +156,34 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
148
156
  ).rejects.toThrow(/system_state_root was not provided/);
149
157
  });
150
158
 
151
- it('machine-pool.json is valid JSON (array)', async () => {
159
+ it('machine-pool.json is the staged snapshot (valid JSON array)', async () => {
152
160
  const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
153
161
  await hook(
154
162
  buildContext({
155
163
  backup_dir: backupDir,
156
164
  cross_module_root: crossModuleRoot,
157
165
  system_state_root: systemStateRoot,
166
+ machine_pool: JSON.stringify(STUB_MACHINE_POOL),
158
167
  }),
159
168
  );
160
169
 
161
- const machinePool = JSON.parse(readFileSync(join(backupDir, 'machine-pool.json'), 'utf-8'));
162
- expect(Array.isArray(machinePool)).toBe(true);
170
+ // Content, not just shape. A bare Array.isArray also passes when the
171
+ // hook swallowed a snapshot failure and wrote an empty pool, which is
172
+ // exactly the failure mode this stub exists to rule out.
173
+ const machinePool: unknown = JSON.parse(
174
+ readFileSync(join(backupDir, 'machine-pool.json'), 'utf-8'),
175
+ );
176
+ expect(machinePool).toEqual(STUB_MACHINE_POOL);
163
177
  });
164
178
 
165
179
  it('proceeds (with a warning) when cross_module_root is missing', async () => {
166
180
  const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
167
181
  const result = (await hook(
168
- buildContext({ backup_dir: backupDir, system_state_root: systemStateRoot }),
182
+ buildContext({
183
+ backup_dir: backupDir,
184
+ system_state_root: systemStateRoot,
185
+ machine_pool: JSON.stringify(STUB_MACHINE_POOL),
186
+ }),
169
187
  )) as BackupHookOutput;
170
188
 
171
189
  expect(existsSync(join(backupDir, 'celilo.db'))).toBe(true);
@@ -5,6 +5,7 @@ import { join } from 'node:path';
5
5
  import type { DbClient } from '../db/client';
6
6
  import type { ModuleManifest } from '../manifest/schema';
7
7
  import { setupTestDatabaseAt } from '../test-utils/database';
8
+ import { resetTestDbPath } from '../test-utils/db-path';
8
9
  import {
9
10
  type CleanupTarget,
10
11
  type ProviderRow,
@@ -216,7 +217,7 @@ describe('runConsumerCleanup — the paused skip is reported, not silent', () =>
216
217
  });
217
218
  afterEach(() => {
218
219
  db.$client.close();
219
- process.env.CELILO_DB_PATH = undefined;
220
+ resetTestDbPath();
220
221
  rmSync(dir, { recursive: true, force: true });
221
222
  });
222
223
 
@@ -285,7 +286,7 @@ describe('runConsumerCleanup — dispatch', () => {
285
286
  });
286
287
  afterEach(() => {
287
288
  db.$client.close();
288
- process.env.CELILO_DB_PATH = undefined;
289
+ resetTestDbPath();
289
290
  rmSync(dir, { recursive: true, force: true });
290
291
  });
291
292
 
@@ -10,6 +10,7 @@ import { eq } from 'drizzle-orm';
10
10
  import { closeDb, createDbClient } from '../db/client';
11
11
  import { runMigrations } from '../db/migrate';
12
12
  import { containerServices } from '../db/schema';
13
+ import { resetTestDbPath } from '../test-utils/db-path';
13
14
  import {
14
15
  addContainerService,
15
16
  getContainerService,
@@ -53,8 +54,8 @@ describe('container-service', () => {
53
54
  }
54
55
 
55
56
  // Clear environment variables
56
- process.env.CELILO_DB_PATH = undefined;
57
- process.env.CELILO_MASTER_KEY_PATH = undefined;
57
+ resetTestDbPath();
58
+ delete process.env.CELILO_MASTER_KEY_PATH;
58
59
  });
59
60
 
60
61
  describe('addContainerService', () => {
@@ -17,6 +17,7 @@ import { eq } from 'drizzle-orm';
17
17
  import type { DbClient } from '../db/client';
18
18
  import { capabilities, modules, systemConfig } from '../db/schema';
19
19
  import { setupTestDatabaseAt } from '../test-utils/database';
20
+ import { resetTestDbPath } from '../test-utils/db-path';
20
21
  import {
21
22
  type ControlPlaneBootstrapOptions,
22
23
  bootstrapControlPlane,
@@ -58,7 +59,7 @@ describe('bootstrapControlPlane', () => {
58
59
 
59
60
  afterEach(() => {
60
61
  db.$client.close();
61
- process.env.CELILO_DB_PATH = undefined;
62
+ resetTestDbPath();
62
63
  try {
63
64
  rmSync(dir, { recursive: true, force: true });
64
65
  } catch {