@celilo/cli 2.1.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/drizzle/0031_module_config_source.sql +20 -0
  2. package/drizzle/meta/_journal.json +8 -1
  3. package/package.json +3 -3
  4. package/schemas/system_config.json +7 -1
  5. package/src/ansible/inventory.test.ts +2 -1
  6. package/src/api/sessions.test.ts +2 -1
  7. package/src/capabilities/public-web-publish.test.ts +61 -0
  8. package/src/cli/backup-rename.test.ts +2 -1
  9. package/src/cli/cli.test.ts +2 -1
  10. package/src/cli/commands/console-get-chain.test.ts +2 -1
  11. package/src/cli/commands/firewall-interface-list.test.ts +158 -8
  12. package/src/cli/commands/firewall-interface-list.ts +73 -7
  13. package/src/cli/commands/machine-add.ts +12 -55
  14. package/src/cli/commands/module-config.test.ts +22 -2
  15. package/src/cli/commands/module-deploy.ts +8 -2
  16. package/src/cli/commands/module-generate.test.ts +53 -0
  17. package/src/cli/commands/module-generate.ts +31 -26
  18. package/src/cli/commands/module-import-aspect.test.ts +2 -1
  19. package/src/cli/commands/module-import-registry.test.ts +2 -1
  20. package/src/cli/commands/module-import.ts +1 -1
  21. package/src/cli/commands/module-operations.test.ts +2 -1
  22. package/src/cli/commands/module-publish.test.ts +5 -12
  23. package/src/cli/commands/module-update.test.ts +87 -4
  24. package/src/cli/commands/module-update.ts +14 -4
  25. package/src/cli/commands/module-upgrade.test.ts +15 -0
  26. package/src/cli/commands/module-upgrade.ts +54 -2
  27. package/src/cli/commands/module-verify.test.ts +2 -3
  28. package/src/cli/commands/module-verify.ts +0 -1
  29. package/src/cli/commands/monitor.ts +2 -10
  30. package/src/cli/commands/notify-config.test.ts +5 -3
  31. package/src/cli/commands/registry-owner.test.ts +2 -1
  32. package/src/cli/commands/registry-token.test.ts +2 -1
  33. package/src/cli/commands/restore.ts +16 -6
  34. package/src/cli/commands/system-apply-config-equivalence.test.ts +5 -7
  35. package/src/cli/commands/system-config.test.ts +148 -0
  36. package/src/cli/commands/system-config.ts +26 -1
  37. package/src/cli/commands/system-doctor.test.ts +71 -0
  38. package/src/cli/commands/system-doctor.ts +110 -24
  39. package/src/cli/commands/system-init-deprecation.test.ts +6 -3
  40. package/src/cli/commands/system-migrate.test.ts +2 -1
  41. package/src/cli/generate-zsh-completion.ts +1 -1
  42. package/src/cli/index.ts +6 -4
  43. package/src/cli/restore-command.test.ts +2 -1
  44. package/src/cli/restore-migration-failure.test.ts +160 -0
  45. package/src/config/paths.test.ts +3 -3
  46. package/src/db/client.ts +5 -0
  47. package/src/db/migrate.test.ts +62 -135
  48. package/src/db/migrate.ts +17 -12
  49. package/src/db/schema.ts +10 -0
  50. package/src/hooks/broker.test.ts +106 -2
  51. package/src/hooks/broker.ts +91 -1
  52. package/src/hooks/capability-loader-firewall.test.ts +37 -0
  53. package/src/hooks/capability-loader.test.ts +78 -0
  54. package/src/hooks/capability-loader.ts +52 -3
  55. package/src/hooks/define-hook.test.ts +4 -3
  56. package/src/hooks/executor.test.ts +106 -19
  57. package/src/hooks/executor.ts +120 -13
  58. package/src/hooks/hook-jail-toolchain-reach.test.ts +3 -2
  59. package/src/hooks/hook-jail-unreachability.test.ts +4 -3
  60. package/src/hooks/hook-protocol.ts +46 -1
  61. package/src/hooks/hook-runner.ts +36 -0
  62. package/src/hooks/hook-store-proxy.test.ts +109 -0
  63. package/src/hooks/hook-store-proxy.ts +85 -0
  64. package/src/hooks/hook-store.test.ts +168 -0
  65. package/src/hooks/hook-store.ts +290 -0
  66. package/src/hooks/hook-timeout.test.ts +3 -2
  67. package/src/hooks/hook-trespass.test.ts +39 -5
  68. package/src/hooks/jail.test.ts +62 -3
  69. package/src/hooks/jail.ts +67 -8
  70. package/src/hooks/mount-set.test.ts +208 -0
  71. package/src/hooks/mount-set.ts +62 -14
  72. package/src/hooks/run-named-hook.ts +2 -0
  73. package/src/hooks/test-fixtures/jail-probe-hook.ts +1 -1
  74. package/src/hooks/test-fixtures/on-restore-staging-hook.ts +26 -0
  75. package/src/hooks/test-fixtures/store-backed.ts +47 -0
  76. package/src/hooks/test-fixtures/store-writing-hook.ts +63 -0
  77. package/src/hooks/unjailed-lint.test.ts +22 -6
  78. package/src/manifest/schema.ts +1 -0
  79. package/src/module/packaging/audit.ts +9 -26
  80. package/src/module/packaging/build-paths.test.ts +127 -0
  81. package/src/module/packaging/build-paths.ts +175 -0
  82. package/src/module/packaging/build.test.ts +71 -1
  83. package/src/module/packaging/build.ts +130 -2
  84. package/src/module/packaging/extract.ts +1 -5
  85. package/src/module/web-root.ts +17 -1
  86. package/src/policy/fixture-capability-coverage.test.ts +322 -0
  87. package/src/policy/module-script-scan.test.ts +42 -1
  88. package/src/policy/module-script-scan.ts +235 -5
  89. package/src/policy/no-hand-built-ssh.test.ts +34 -1
  90. package/src/policy/no-swallowed-refusal.test.ts +264 -0
  91. package/src/policy/no-tar-shell-out-in-services.test.ts +43 -0
  92. package/src/registry/client.test.ts +149 -0
  93. package/src/registry/client.ts +203 -11
  94. package/src/services/alerting/ack.test.ts +2 -1
  95. package/src/services/alerting/cadence-migration.test.ts +3 -2
  96. package/src/services/alerting/coverage-source.test.ts +2 -1
  97. package/src/services/alerting/deferral.test.ts +2 -1
  98. package/src/services/alerting/delivery-loop.test.ts +2 -1
  99. package/src/services/alerting/deploy-hooks.test.ts +2 -1
  100. package/src/services/alerting/format.test.ts +57 -0
  101. package/src/services/alerting/format.ts +24 -0
  102. package/src/services/alerting/inbound-poller.test.ts +2 -1
  103. package/src/services/alerting/inbound.test.ts +2 -1
  104. package/src/services/alerting/notification-responder.test.ts +2 -1
  105. package/src/services/alerting/run-monitor.test.ts +2 -1
  106. package/src/services/alerting/run-monitor.ts +2 -2
  107. package/src/services/alerting/store.test.ts +2 -1
  108. package/src/services/alerting/sweep-runner.test.ts +2 -1
  109. package/src/services/alerting/tokens.test.ts +2 -1
  110. package/src/services/aspect-approvals.test.ts +2 -1
  111. package/src/services/aspect-reconcile.test.ts +4 -3
  112. package/src/services/aspect-runner.test.ts +2 -1
  113. package/src/services/audit/module-integrity.test.ts +0 -21
  114. package/src/services/audit/module-integrity.ts +0 -14
  115. package/src/services/backup-age-agreement.test.ts +2 -1
  116. package/src/services/backup-create.ts +7 -7
  117. package/src/services/backup-envelope-roundtrip.test.ts +47 -3
  118. package/src/services/backup-in-flight-refusal.test.ts +2 -1
  119. package/src/services/backup-restore.ts +8 -4
  120. package/src/services/bus-ensure-flow.test.ts +2 -1
  121. package/src/services/bus-interview-park.test.ts +2 -1
  122. package/src/services/bus-interview.ts +37 -14
  123. package/src/services/bus-secret-flow.test.ts +2 -1
  124. package/src/services/capability-table-rows.test.ts +2 -1
  125. package/src/services/config-provenance.ts +4 -0
  126. package/src/services/consumer-cleanup.test.ts +3 -2
  127. package/src/services/container-service.test.ts +2 -1
  128. package/src/services/control-plane-bootstrap.test.ts +123 -2
  129. package/src/services/control-plane-bootstrap.ts +51 -4
  130. package/src/services/cross-module-read.test.ts +2 -1
  131. package/src/services/deploy-preflight.ts +8 -2
  132. package/src/services/deploy-validation.test.ts +25 -2
  133. package/src/services/deploy-validation.ts +8 -0
  134. package/src/services/dns-discovery.test.ts +54 -0
  135. package/src/services/dns-discovery.ts +47 -5
  136. package/src/services/dns-internal-records.test.ts +3 -2
  137. package/src/services/dns-provider-backfill.test.ts +2 -1
  138. package/src/services/dns-registrations.test.ts +2 -1
  139. package/src/services/ensure-interview.test.ts +3 -2
  140. package/src/services/fleet-checks.test.ts +3 -2
  141. package/src/services/fleet-key.test.ts +68 -3
  142. package/src/services/fleet-key.ts +54 -0
  143. package/src/services/health-runner.ts +2 -0
  144. package/src/services/infrastructure-selector.test.ts +2 -1
  145. package/src/services/infrastructure-variable-resolver.test.ts +2 -1
  146. package/src/services/machine-pool.test.ts +2 -1
  147. package/src/services/module-config.test.ts +2 -1
  148. package/src/services/module-config.ts +20 -2
  149. package/src/services/module-deploy.dns-repoint.test.ts +188 -0
  150. package/src/services/module-deploy.ts +199 -21
  151. package/src/services/module-operations.test.ts +2 -1
  152. package/src/services/module-subscriptions.test.ts +2 -1
  153. package/src/services/module-validator/git-hygiene.test.ts +122 -3
  154. package/src/services/module-validator/git-hygiene.ts +83 -14
  155. package/src/services/port-forwards.test.ts +2 -1
  156. package/src/services/programmatic-responder.aspect.test.ts +2 -1
  157. package/src/services/proxmox-reconcile.test.ts +2 -1
  158. package/src/services/restore-from-file.test.ts +23 -2
  159. package/src/services/restore-from-file.ts +21 -6
  160. package/src/services/restore-preflight.test.ts +2 -1
  161. package/src/services/secret-schema-loader.test.ts +2 -1
  162. package/src/services/ssh-key-manager.test.ts +2 -1
  163. package/src/services/static-content-converge.test.ts +144 -5
  164. package/src/services/static-content-converge.ts +87 -17
  165. package/src/services/system-config-schema-types.ts +1 -1
  166. package/src/services/system-config-validator.test.ts +36 -0
  167. package/src/services/system-config-validator.ts +11 -0
  168. package/src/services/system-state-stage.test.ts +2 -1
  169. package/src/services/trusted-sources.test.ts +33 -2
  170. package/src/services/trusted-sources.ts +47 -10
  171. package/src/services/zone-detector.test.ts +2 -1
  172. package/src/templates/generator.ts +9 -2
  173. package/src/test-utils/bus-responder.ts +5 -3
  174. package/src/test-utils/db-path.ts +25 -0
  175. package/src/test-utils/integration.ts +7 -0
  176. package/src/test-utils/module-fixtures.ts +5 -6
  177. package/src/variables/context.ts +16 -5
  178. package/src/module/packaging/generated-plane.test.ts +0 -79
  179. package/src/module/packaging/generated-plane.ts +0 -134
@@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm';
6
6
  import type { DbClient } from '../../db/client';
7
7
  import { type Monitor, alerts, monitorRuns, monitors } from '../../db/schema';
8
8
  import { setupTestDatabaseAt } from '../../test-utils/database';
9
+ import { resetTestDbPath } from '../../test-utils/db-path';
9
10
  import type { HealthCheckResult } from '../health-runner';
10
11
  import { HOOK_JAIL_CHECK, type HookJailState } from './hook-jail';
11
12
  import { moduleAlertKey, moduleCheckAlertKey } from './keys';
@@ -71,7 +72,7 @@ describe('runOneMonitor', () => {
71
72
 
72
73
  afterEach(() => {
73
74
  db.$client.close();
74
- process.env.CELILO_DB_PATH = undefined;
75
+ resetTestDbPath();
75
76
  try {
76
77
  rmSync(dir, { recursive: true, force: true });
77
78
  } catch {
@@ -47,7 +47,7 @@ export interface MonitorRunOutcomeSummary {
47
47
  monitorId: string;
48
48
  outcome: 'success' | 'error';
49
49
  errorMessage?: string;
50
- failingKeyCount: number;
50
+ failingKeys: FailingKey[];
51
51
  createdIds: string[];
52
52
  resolvedIds: string[];
53
53
  }
@@ -186,7 +186,7 @@ export async function runOneMonitor(
186
186
  monitorId: monitor.id,
187
187
  outcome: product.outcome,
188
188
  errorMessage: product.errorMessage,
189
- failingKeyCount: product.failingKeys.length,
189
+ failingKeys: product.failingKeys,
190
190
  createdIds,
191
191
  resolvedIds,
192
192
  };
@@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm';
6
6
  import type { DbClient } from '../../db/client';
7
7
  import { alerts, monitors } from '../../db/schema';
8
8
  import { setupTestDatabaseAt } from '../../test-utils/database';
9
+ import { resetTestDbPath } from '../../test-utils/db-path';
9
10
  import { moduleAlertKey, moduleCheckAlertKey } from './keys';
10
11
  import type { ReconcileAction } from './reconcile';
11
12
  import {
@@ -55,7 +56,7 @@ describe('alert store', () => {
55
56
 
56
57
  afterEach(() => {
57
58
  db.$client.close();
58
- process.env.CELILO_DB_PATH = undefined;
59
+ resetTestDbPath();
59
60
  try {
60
61
  rmSync(dir, { recursive: true, force: true });
61
62
  } catch {
@@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm';
6
6
  import type { DbClient } from '../../db/client';
7
7
  import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema';
8
8
  import { setupTestDatabaseAt } from '../../test-utils/database';
9
+ import { resetTestDbPath } from '../../test-utils/db-path';
9
10
  import type { HealthCheckResult } from '../health-runner';
10
11
  import { deleteModuleConfig, upsertModuleConfig } from '../module-config';
11
12
  import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence';
@@ -99,7 +100,7 @@ describe('runSweep', () => {
99
100
 
100
101
  afterEach(() => {
101
102
  db.$client.close();
102
- process.env.CELILO_DB_PATH = undefined;
103
+ resetTestDbPath();
103
104
  try {
104
105
  rmSync(dir, { recursive: true, force: true });
105
106
  } catch {
@@ -5,6 +5,7 @@ import { join } from 'node:path';
5
5
  import type { DbClient } from '../../db/client';
6
6
  import { modules, people, routes } from '../../db/schema';
7
7
  import { setupTestDatabaseAt } from '../../test-utils/database';
8
+ import { resetTestDbPath } from '../../test-utils/db-path';
8
9
  import {
9
10
  consumeDelivery,
10
11
  deliveriesForAlert,
@@ -75,7 +76,7 @@ describe('delivery records', () => {
75
76
 
76
77
  afterEach(() => {
77
78
  db.$client.close();
78
- process.env.CELILO_DB_PATH = undefined;
79
+ resetTestDbPath();
79
80
  try {
80
81
  rmSync(dir, { recursive: true, force: true });
81
82
  } catch {
@@ -6,6 +6,7 @@ import { closeDb, getDb } from '../db/client';
6
6
  import { runMigrations } from '../db/migrate';
7
7
  import { modules } from '../db/schema';
8
8
  import type { BaseModuleAspect } from '../manifest/schema';
9
+ import { resetTestDbPath } from '../test-utils/db-path';
9
10
  import {
10
11
  checkAspectApproval,
11
12
  computeAspectScopeHash,
@@ -44,7 +45,7 @@ describe('aspect-approvals', () => {
44
45
 
45
46
  afterEach(() => {
46
47
  closeDb();
47
- process.env.CELILO_DB_PATH = undefined;
48
+ resetTestDbPath();
48
49
  try {
49
50
  rmSync(dir, { recursive: true, force: true });
50
51
  } catch {
@@ -20,6 +20,7 @@ import { closeDb, getDb } from '../db/client';
20
20
  import { runMigrations } from '../db/migrate';
21
21
  import { type ModuleState, modules } from '../db/schema';
22
22
  import type { BaseModuleAspect } from '../manifest/schema';
23
+ import { resetTestDbPath } from '../test-utils/db-path';
23
24
  import {
24
25
  computeAspectScopeHash,
25
26
  recordAspectApproval,
@@ -159,7 +160,7 @@ describe('reconcileAspectsForSystems', () => {
159
160
 
160
161
  afterEach(() => {
161
162
  closeDb();
162
- process.env.CELILO_DB_PATH = undefined;
163
+ resetTestDbPath();
163
164
  try {
164
165
  rmSync(dir, { recursive: true, force: true });
165
166
  } catch {
@@ -310,7 +311,7 @@ describe('planAspectFanOut onlyHostnames', () => {
310
311
 
311
312
  afterEach(() => {
312
313
  closeDb();
313
- process.env.CELILO_DB_PATH = undefined;
314
+ resetTestDbPath();
314
315
  try {
315
316
  rmSync(dir, { recursive: true, force: true });
316
317
  } catch {
@@ -352,7 +353,7 @@ describe('verifyAspectCoverage', () => {
352
353
 
353
354
  afterEach(() => {
354
355
  closeDb();
355
- process.env.CELILO_DB_PATH = undefined;
356
+ resetTestDbPath();
356
357
  try {
357
358
  rmSync(dir, { recursive: true, force: true });
358
359
  } catch {
@@ -20,6 +20,7 @@ import { closeDb, getDb } from '../db/client';
20
20
  import { runMigrations } from '../db/migrate';
21
21
  import { modules } from '../db/schema';
22
22
  import type { BaseModuleAspect, ModuleManifest } from '../manifest/schema';
23
+ import { resetTestDbPath } from '../test-utils/db-path';
23
24
  import {
24
25
  computeAspectScopeHash,
25
26
  findAspectApproval,
@@ -109,7 +110,7 @@ describe('aspect-runner', () => {
109
110
 
110
111
  afterEach(() => {
111
112
  closeDb();
112
- process.env.CELILO_DB_PATH = undefined;
113
+ resetTestDbPath();
113
114
  try {
114
115
  rmSync(dir, { recursive: true, force: true });
115
116
  } catch {
@@ -82,27 +82,6 @@ describe('auditModuleIntegrity', () => {
82
82
  expect(findings[0]?.details).toContain('scripts/new.ts');
83
83
  });
84
84
 
85
- test('a stale generated project is drift, and says a deploy would ship the wrong bytes', () => {
86
- const findings = auditModuleIntegrity({
87
- results: [
88
- {
89
- ...clean('wireguard-manager'),
90
- success: false,
91
- violations: [
92
- {
93
- type: 'stale-generated',
94
- path: 'ansible/roles/vpn/files/bin',
95
- message: 'Generated project holds different bytes',
96
- },
97
- ],
98
- },
99
- ],
100
- });
101
- expect(findings.map((f) => f.code)).toEqual(['module_generated_stale']);
102
- expect(findings[0]?.severity).toBe('drift');
103
- expect(findings[0]?.remediation).toBe('celilo module generate wireguard-manager');
104
- });
105
-
106
85
  test('host-plane results carry through with the right severity', () => {
107
86
  const findings = auditModuleIntegrity({
108
87
  results: [
@@ -58,20 +58,6 @@ export function auditModuleIntegrity(deps: ModuleIntegrityAuditDeps): DriftFindi
58
58
  });
59
59
  }
60
60
 
61
- const generated = result.violations.filter((v) => v.type === 'stale-generated');
62
- if (generated.length > 0) {
63
- findings.push({
64
- category: 'module_integrity',
65
- severity: 'drift',
66
- code: 'module_generated_stale',
67
- message: `${result.moduleId}: ${generated.length} generated asset(s) do not match the installed module — a deploy would ship the wrong bytes`,
68
- details: generated.map((v) => ` • ${v.path}`).join('\n'),
69
- remediation: `celilo module generate ${result.moduleId}`,
70
- actionable: true,
71
- subject: result.moduleId,
72
- });
73
- }
74
-
75
61
  // File differences are only meaningful once the baseline is trustworthy.
76
62
  // Reporting them under a stale baseline is what made `module verify`
77
63
  // unreadable: 72 violations across two healthy modules, nearly all of them
@@ -18,6 +18,7 @@ import { join } from 'node:path';
18
18
  import { closeDb, getDb } from '../db/client';
19
19
  import { runMigrations } from '../db/migrate';
20
20
  import { backupStorages, backups, modules } from '../db/schema';
21
+ import { resetTestDbPath } from '../test-utils/db-path';
21
22
  import { loadBackupAuditInfo } from './audit/backup-source';
22
23
  import { findBackupEligibleModules } from './backup-create';
23
24
  import { loadBackupHistory } from './backup-metadata';
@@ -79,7 +80,7 @@ describe('backup age: the run path and the audit path read the same column', ()
79
80
 
80
81
  afterEach(() => {
81
82
  closeDb();
82
- process.env.CELILO_DB_PATH = undefined;
83
+ resetTestDbPath();
83
84
  rmSync(dir, { recursive: true, force: true });
84
85
  });
85
86
 
@@ -6,15 +6,16 @@
6
6
  import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
7
7
  import { join } from 'node:path';
8
8
  import { eq } from 'drizzle-orm';
9
+ import { create as tarCreate } from 'tar';
9
10
  import { getDbPath, getMasterKeyPath } from '../config/paths';
10
11
  import { getDb } from '../db/client';
11
12
  import { moduleConfigs, modules, secrets as secretsTable } from '../db/schema';
12
13
  import { invokeHook } from '../hooks/executor';
14
+ import { createHookStores } from '../hooks/hook-store';
13
15
  import { createConsoleLogger } from '../hooks/logger';
14
16
  import type { ModuleManifest } from '../manifest/schema';
15
17
  import { decryptSecret } from '../secrets/encryption';
16
18
  import { getOrCreateMasterKey } from '../secrets/master-key';
17
- import { shellEscape } from '../utils/shell';
18
19
  import { encryptFileToFile } from './backup-cipher';
19
20
  import { buildManifest } from './backup-manifest';
20
21
  import {
@@ -153,8 +154,7 @@ export async function createSystemStateBackup(
153
154
  // Tar the envelope. Operator can re-extract a backup file manually
154
155
  // for diagnostics: `age -d -p file.backup | tar -t` shows the contents.
155
156
  const tarPath = join(tempDir, 'envelope.tar');
156
- const { execSync } = await import('node:child_process');
157
- execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)} .`);
157
+ await tarCreate({ file: tarPath, cwd: envelopeDir }, ['.']);
158
158
 
159
159
  // Encrypt the tar, streamed — see backup-cipher.ts. The plaintext is
160
160
  // never held in memory, so a large fleet DB can't OOM the snapshot.
@@ -401,6 +401,7 @@ export async function createModuleBackup(
401
401
  debug: false,
402
402
  systems: getModuleSystems(moduleId, db),
403
403
  remoteAccess: remoteAccessPolicy(moduleId, db),
404
+ hookStores: () => createHookStores(db, moduleId),
404
405
  },
405
406
  );
406
407
 
@@ -434,8 +435,7 @@ export async function createModuleBackup(
434
435
 
435
436
  // Tar the envelope (manifest.json + data/).
436
437
  const tarPath = join(tempDir, 'envelope.tar');
437
- const { execSync } = await import('node:child_process');
438
- execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)} .`);
438
+ await tarCreate({ file: tarPath, cwd: envelopeDir }, ['.']);
439
439
 
440
440
  // Encrypt the tar, streamed — see backup-cipher.ts. Module artifacts run
441
441
  // to hundreds of MB (forgejo's are ~774 MB); holding one in memory is
@@ -566,6 +566,7 @@ export async function importModuleBackup(
566
566
  debug: false,
567
567
  systems: getModuleSystems(moduleId, db),
568
568
  remoteAccess: remoteAccessPolicy(moduleId, db),
569
+ hookStores: () => createHookStores(db, moduleId),
569
570
  },
570
571
  );
571
572
 
@@ -590,8 +591,7 @@ export async function importModuleBackup(
590
591
 
591
592
  // Tar the artifacts
592
593
  const tarPath = join(tempDir, 'backup.tar');
593
- const { execSync } = await import('node:child_process');
594
- execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(artifactDir)} .`);
594
+ await tarCreate({ file: tarPath, cwd: artifactDir }, ['.']);
595
595
 
596
596
  // Encrypt the tar, streamed — see backup-cipher.ts.
597
597
  const masterKey = await getOrCreateMasterKey();
@@ -16,16 +16,26 @@
16
16
 
17
17
  import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
18
18
  import { execSync } from 'node:child_process';
19
- import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
19
+ import {
20
+ copyFileSync,
21
+ existsSync,
22
+ mkdirSync,
23
+ mkdtempSync,
24
+ readFileSync,
25
+ rmSync,
26
+ writeFileSync,
27
+ } from 'node:fs';
20
28
  import { tmpdir } from 'node:os';
21
29
  import { join } from 'node:path';
30
+ import { getDbPath } from '../config/paths';
22
31
  import { closeDb, getDb } from '../db/client';
23
32
  import { runMigrations } from '../db/migrate';
24
33
  import { backups, systemConfig } from '../db/schema';
25
34
  import { getOrCreateMasterKey } from '../secrets/master-key';
35
+ import { resetTestDbPath } from '../test-utils/db-path';
26
36
  import { decryptFileToFile, encryptFileToFile } from './backup-cipher';
27
37
  import { createSystemStateBackup } from './backup-create';
28
- import { MANIFEST_SCHEMA_VERSION, parseManifest } from './backup-manifest';
38
+ import { MANIFEST_SCHEMA_VERSION, buildManifest, parseManifest } from './backup-manifest';
29
39
  import { restoreSystemStateBackup } from './backup-restore';
30
40
  import { addBackupStorage, setDefaultBackupStorage, verifyBackupStorage } from './backup-storage';
31
41
 
@@ -55,7 +65,7 @@ describe('backup envelope round-trip', () => {
55
65
 
56
66
  afterEach(() => {
57
67
  closeDb();
58
- process.env.CELILO_DB_PATH = undefined;
68
+ resetTestDbPath();
59
69
  process.env.CELILO_MASTER_KEY_PATH = undefined;
60
70
  try {
61
71
  rmSync(dir, { recursive: true, force: true });
@@ -121,6 +131,40 @@ describe('backup envelope round-trip', () => {
121
131
  expect(restoreResult.error).toBeUndefined();
122
132
  });
123
133
 
134
+ it('an archive written by SYSTEM tar (the pre-library producer) restores via the tar library', async () => {
135
+ // The tar library replaced `tar -cf/-xf` shell-outs (celilo#1235). A
136
+ // self-consistent round-trip cannot detect a layout change, because the
137
+ // new extract sites would happily read back the new layout. So this test
138
+ // builds the envelope with the SYSTEM tar — byte-for-byte the command the
139
+ // old code ran — and restores it with the current code, proving a backup
140
+ // taken BEFORE the change still restores after it.
141
+ const db = getDb();
142
+ db.insert(systemConfig).values({ key: 'pre-change-sentinel', value: 'before-backup' }).run();
143
+
144
+ // Take a real backup to get a valid row + storage location, then swap the
145
+ // artifact for one produced by the system tar the old code shelled out to.
146
+ const result = await createSystemStateBackup();
147
+ expect(result.success).toBe(true);
148
+ const artifactPath = join(storageDir, 'celilo-backups', result.storagePath as string);
149
+
150
+ const envelopeDir = join(dir, 'pre-change-envelope');
151
+ mkdirSync(envelopeDir, { recursive: true });
152
+ copyFileSync(getDbPath(), join(envelopeDir, 'celilo.db'));
153
+ writeFileSync(
154
+ join(envelopeDir, 'manifest.json'),
155
+ JSON.stringify(buildManifest({ kind: 'system' }), null, 2),
156
+ );
157
+ const tarPath = join(dir, 'pre-change.tar');
158
+ execSync(`tar -cf '${tarPath}' -C '${envelopeDir}' .`);
159
+ const masterKey = await getOrCreateMasterKey();
160
+ await encryptFileToFile(tarPath, artifactPath, masterKey);
161
+
162
+ const backupRow = db.select().from(backups).all()[0];
163
+ const restoreResult = await restoreSystemStateBackup(backupRow);
164
+ expect(restoreResult.success).toBe(true);
165
+ expect(restoreResult.error).toBeUndefined();
166
+ });
167
+
124
168
  it('system restore refuses an artifact with a wrong manifest kind', async () => {
125
169
  // Create a valid backup, then poison the manifest by re-packing.
126
170
  const result = await createSystemStateBackup();
@@ -18,6 +18,7 @@ import { join } from 'node:path';
18
18
  import { closeDb, getDb } from '../db/client';
19
19
  import { runMigrations } from '../db/migrate';
20
20
  import { backups, moduleOperations } from '../db/schema';
21
+ import { resetTestDbPath } from '../test-utils/db-path';
21
22
  import { createModuleBackup, createSystemStateBackup } from './backup-create';
22
23
  import { restoreModuleBackup, restoreSystemStateBackup } from './backup-restore';
23
24
  import { InFlightError } from './module-operations';
@@ -34,7 +35,7 @@ describe('backup/restore in-flight refusal', () => {
34
35
 
35
36
  afterEach(() => {
36
37
  closeDb();
37
- process.env.CELILO_DB_PATH = undefined;
38
+ resetTestDbPath();
38
39
  process.env.CELILO_MASTER_KEY_PATH = undefined;
39
40
  try {
40
41
  rmSync(dir, { recursive: true, force: true });
@@ -4,21 +4,21 @@
4
4
  * For system state backups, restores the Celilo database.
5
5
  */
6
6
 
7
- import { execSync } from 'node:child_process';
8
7
  import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
9
8
  import { tmpdir } from 'node:os';
10
9
  import { join } from 'node:path';
11
10
  import { eq } from 'drizzle-orm';
11
+ import { extract as tarExtract } from 'tar';
12
12
  import { getDbPath } from '../config/paths';
13
13
  import { closeDb, getDb } from '../db/client';
14
14
  import { moduleConfigs, modules, secrets as secretsTable } from '../db/schema';
15
15
  import type { Backup } from '../db/schema';
16
16
  import { invokeHook } from '../hooks/executor';
17
+ import { createHookStores } from '../hooks/hook-store';
17
18
  import { createConsoleLogger } from '../hooks/logger';
18
19
  import type { ModuleManifest } from '../manifest/schema';
19
20
  import { decryptSecret } from '../secrets/encryption';
20
21
  import { getOrCreateMasterKey } from '../secrets/master-key';
21
- import { shellEscape } from '../utils/shell';
22
22
  import { decryptFileToFile } from './backup-cipher';
23
23
  import { assertCompatibleSchema, parseManifest } from './backup-manifest';
24
24
  import { createStorageProvider } from './backup-storage';
@@ -70,7 +70,7 @@ export async function restoreSystemStateBackup(backup: Backup): Promise<RestoreR
70
70
  mkdirSync(envelopeDir, { recursive: true });
71
71
  const tarPath = join(tempDir, 'envelope.tar');
72
72
  await decryptFileToFile(encryptedPath, tarPath, masterKey);
73
- execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
73
+ await tarExtract({ file: tarPath, cwd: envelopeDir });
74
74
 
75
75
  // Read + validate manifest BEFORE touching the live DB. An
76
76
  // incompatible artifact must not get past this point.
@@ -180,6 +180,9 @@ export async function restoreModuleBackup(
180
180
  if (!backup.moduleId) {
181
181
  return { success: false, error: 'Backup has no associated module' };
182
182
  }
183
+ // Captured because the narrowing above does not survive into the hookStores
184
+ // callback below, where TypeScript widens the parameter back to string|null.
185
+ const moduleId = backup.moduleId;
183
186
 
184
187
  const db = getDb();
185
188
  const mod = db.select().from(modules).where(eq(modules.id, backup.moduleId)).get();
@@ -221,7 +224,7 @@ export async function restoreModuleBackup(
221
224
  const masterKey = await getOrCreateMasterKey();
222
225
  const tarPath = join(tempDir, 'envelope.tar');
223
226
  await decryptFileToFile(encryptedPath, tarPath, masterKey);
224
- execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
227
+ await tarExtract({ file: tarPath, cwd: envelopeDir });
225
228
 
226
229
  // Read + validate envelope manifest BEFORE invoking the hook.
227
230
  const manifestPath = join(envelopeDir, 'manifest.json');
@@ -290,6 +293,7 @@ export async function restoreModuleBackup(
290
293
  debug: false,
291
294
  systems: getModuleSystems(backup.moduleId, db),
292
295
  remoteAccess: remoteAccessPolicy(backup.moduleId, db),
296
+ hookStores: () => createHookStores(db, moduleId),
293
297
  },
294
298
  );
295
299
 
@@ -27,6 +27,7 @@ import { moduleConfigs, modules, secrets } from '../db/schema';
27
27
  import type { Ensure } from '../manifest/schema';
28
28
  import { decryptSecret } from '../secrets/encryption';
29
29
  import { getOrCreateMasterKey } from '../secrets/master-key';
30
+ import { resetTestDbPath } from '../test-utils/db-path';
30
31
  import type { EnsureRequiredPayload } from './bus-interview';
31
32
  import { interviewForEnsureInputs, writeModuleSecretKey } from './config-interview';
32
33
 
@@ -174,7 +175,7 @@ describe('bus-mediated interviewForEnsureInputs', () => {
174
175
  afterEach(() => {
175
176
  bus.close();
176
177
  rmSync(tempDir, { recursive: true, force: true });
177
- process.env.CELILO_DB_PATH = undefined;
178
+ resetTestDbPath();
178
179
  process.env.EVENT_BUS_DB = undefined;
179
180
  });
180
181
 
@@ -26,6 +26,7 @@ import { type Bus, type BusEvent, defineEvents, openBus } from '@celilo/event-bu
26
26
  import { handleModuleUpdate } from '../cli/commands/module-update';
27
27
  import { getDb } from '../db/client';
28
28
  import { modules } from '../db/schema';
29
+ import { resetTestDbPath } from '../test-utils/db-path';
29
30
  import { RESPONDER_PROBE_EVENT } from './responder-probe';
30
31
 
31
32
  const NO_SCHEMAS = defineEvents({});
@@ -91,7 +92,7 @@ beforeEach(() => {
91
92
  afterEach(() => {
92
93
  server.stop(true);
93
94
  rmSync(dir, { recursive: true, force: true });
94
- process.env.CELILO_DB_PATH = undefined;
95
+ resetTestDbPath();
95
96
  process.env.CELILO_ORIGINAL_CWD = undefined;
96
97
  process.env.EVENT_BUS_DB = undefined;
97
98
  });
@@ -16,7 +16,7 @@
16
16
 
17
17
  import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
18
18
  import { getEventBusPath } from '../config/paths';
19
- import { InterviewAbandonedError } from './interview-errors';
19
+ import { InterviewAbandonedError, InterviewUnansweredError } from './interview-errors';
20
20
  import { ensureResponderForInterview } from './responder-probe';
21
21
 
22
22
  const NO_SCHEMAS = defineEvents({});
@@ -271,17 +271,26 @@ export async function busInterview<TReply>(
271
271
  type: string,
272
272
  payload: object,
273
273
  ownerBus?: Bus,
274
+ timeoutMs = 0,
274
275
  ): Promise<TReply> {
275
276
  const ownsBus = !ownerBus;
276
277
  const bus: Bus = ownerBus ?? openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
277
278
  try {
278
279
  const replies = await bus.query(type as never, payload as never, {
279
- timeoutMs: 0, // wait forever — race semantics, no timeout
280
+ timeoutMs, // 0 = wait forever — race semantics, no timeout
280
281
  pollIntervalMs: 250,
281
282
  expect: 'first',
282
283
  });
283
284
  if (replies.length === 0) {
284
- throw new Error(`bus-interview: ${type} returned no reply`);
285
+ // Typed so a bounded caller can distinguish "nobody answered in time"
286
+ // from a decline. With the default 0 this is unreachable (the query
287
+ // waits forever), but keeping it typed costs nothing.
288
+ throw new InterviewUnansweredError(
289
+ type,
290
+ timeoutMs > 0
291
+ ? `No reply within ${timeoutMs}ms (${type})`
292
+ : `bus-interview: ${type} returned no reply`,
293
+ );
285
294
  }
286
295
  return replies[0].payload as unknown as TReply;
287
296
  } finally {
@@ -305,9 +314,10 @@ export async function busInterviewGuarded<TReply>(
305
314
  type: string,
306
315
  payload: object,
307
316
  ownerBus?: Bus,
317
+ timeoutMs = 0,
308
318
  ): Promise<TReply> {
309
319
  await ensureResponderForInterview(type);
310
- return busInterview<TReply>(type, payload, ownerBus);
320
+ return busInterview<TReply>(type, payload, ownerBus, timeoutMs);
311
321
  }
312
322
 
313
323
  /**
@@ -326,9 +336,10 @@ export async function busInterviewGuarded<TReply>(
326
336
  export async function askInterview(
327
337
  payload: InterviewRequiredPayload,
328
338
  ownerBus?: Bus,
339
+ timeoutMs = 0,
329
340
  ): Promise<unknown> {
330
341
  const type = EVENT_TYPES.interviewRequired(payload.scope, payload.key);
331
- const reply = await busInterviewGuarded<InterviewReply>(type, payload, ownerBus);
342
+ const reply = await busInterviewGuarded<InterviewReply>(type, payload, ownerBus, timeoutMs);
332
343
  if (reply.abandoned) throw new InterviewAbandonedError(type, reply.abandoned.reason);
333
344
  return reply.value;
334
345
  }
@@ -463,16 +474,28 @@ export async function askConfirm(opts: {
463
474
  message: string;
464
475
  defaultValue?: boolean;
465
476
  description?: string;
477
+ /**
478
+ * Bound the wait for a reply. 0 (the default) waits forever, which is
479
+ * right when the operator IS the responder (TTY). A headless caller whose
480
+ * responder may be attached but not staged for this question passes a
481
+ * finite bound so it skips instead of hanging until the suite budget
482
+ * kills it (module-orchestrator-primitives design.md D5).
483
+ */
484
+ timeoutMs?: number;
466
485
  }): Promise<boolean> {
467
- const value = await askInterview({
468
- scope: opts.scope,
469
- key: opts.key,
470
- kind: 'confirm',
471
- message: opts.message,
472
- required: true,
473
- defaultValue: opts.defaultValue === undefined ? undefined : String(opts.defaultValue),
474
- description: opts.description,
475
- });
486
+ const value = await askInterview(
487
+ {
488
+ scope: opts.scope,
489
+ key: opts.key,
490
+ kind: 'confirm',
491
+ message: opts.message,
492
+ required: true,
493
+ defaultValue: opts.defaultValue === undefined ? undefined : String(opts.defaultValue),
494
+ description: opts.description,
495
+ },
496
+ undefined,
497
+ opts.timeoutMs,
498
+ );
476
499
  if (typeof value !== 'boolean') {
477
500
  throw new Error(
478
501
  `askConfirm(${opts.scope}.${opts.key}): expected a boolean reply, got ${typeof value}`,
@@ -21,6 +21,7 @@ import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
21
21
  import { type DbClient, getDb } from '../db/client';
22
22
  import { modules } from '../db/schema';
23
23
  import { getOrCreateMasterKey } from '../secrets/master-key';
24
+ import { resetTestDbPath } from '../test-utils/db-path';
24
25
  import type { SecretRequiredPayload } from './bus-interview';
25
26
  import {
26
27
  interviewForMissingSecrets,
@@ -154,7 +155,7 @@ describe('bus-mediated interviewForMissingSecrets', () => {
154
155
  afterEach(() => {
155
156
  bus.close();
156
157
  rmSync(tempDir, { recursive: true, force: true });
157
- process.env.CELILO_DB_PATH = undefined;
158
+ resetTestDbPath();
158
159
  process.env.EVENT_BUS_DB = undefined;
159
160
  });
160
161
 
@@ -20,6 +20,7 @@ import { allDeclaredTables } from '@celilo/capabilities';
20
20
  import type { DbClient } from '../db/client';
21
21
  import { dnsInternalRecords, modules, portForwards, trustedSources, webRoutes } from '../db/schema';
22
22
  import { setupTestDatabaseAt } from '../test-utils/database';
23
+ import { resetTestDbPath } from '../test-utils/db-path';
23
24
  import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
24
25
 
25
26
  const FW = '192.168.0.254';
@@ -74,7 +75,7 @@ describe('deleteClaimedRows', () => {
74
75
 
75
76
  afterEach(() => {
76
77
  db.$client.close();
77
- process.env.CELILO_DB_PATH = undefined;
78
+ resetTestDbPath();
78
79
  rmSync(dir, { recursive: true, force: true });
79
80
  });
80
81
 
@@ -74,6 +74,8 @@ export function describeDerivedSource(variable: VariableDeclare): string {
74
74
  return 'from the infrastructure celilo selected for this module';
75
75
  case 'terraform':
76
76
  return 'from a Terraform output, at deploy time';
77
+ case 'hook':
78
+ return `by the module's own hooks at runtime (via context.config.set), and never by an operator`;
77
79
  default:
78
80
  return `computed by celilo (source: ${variable.source})`;
79
81
  }
@@ -98,6 +100,8 @@ export function explainNotSettable(moduleId: string, variable: VariableDeclare):
98
100
  return `${header}\n${origin}\n • node placement: set the service default for NEW deploys (celilo service reconfigure); move an existing container with 'celilo proxmox migrate'.\n • vmid / IP: auto-allocated by IPAM.`;
99
101
  case 'terraform':
100
102
  return `${header}\n${origin}\n • It is read back from Terraform outputs after the deploy creates the resource.`;
103
+ case 'hook':
104
+ return `${header}\n${origin}\n • It is written by the module's own hook via context.config.set, not by any operator path.`;
101
105
  default:
102
106
  return `${header}\n${origin}`;
103
107
  }