@celilo/cli 0.3.30 → 0.4.0-alpha.1

package/src/services/celilo-mgmt-hooks.test.ts

@@ -0,0 +1,296 @@
+/**
+ * Behavior tests for the celilo-mgmt module's on_backup / on_restore
+ * hooks (Phase 3 of v2/SYSTEM_BACKUP_TERRAFORM_STATE.md).
+ *
+ * We import the hook scripts directly and call their handlers with a
+ * synthesized HookContext, then assert on the side-effects (files
+ * written to backup_dir / restore staging dirs). This catches
+ * regressions in the hook bodies without spinning up the full
+ * deploy/backup pipeline.
+ *
+ * What this DOESN'T test: invokeHook validation, end-to-end backup
+ * envelope round-trip. Those live in backup-envelope-roundtrip.test.ts
+ * and (eventually) a fresh-box e2e.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import type { HookContext } from '@celilo/capabilities';
+
+// Hooks under test — imported via the scripts dir adjacent to the
+// modules/celilo-mgmt/manifest.yml. The dynamic import keeps the test
+// runnable from the apps/celilo working dir.
+const HOOK_DIR = join(import.meta.dirname, '../../../../modules/celilo-mgmt/scripts');
+
+interface BackupHookOutput {
+  artifact_count: number;
+  size_bytes: number;
+  schema_version: string;
+}
+interface RestoreHookOutput {
+  restored_items: number;
+}
+
+function buildLogger(): {
+  logger: HookContext['logger'];
+  messages: string[];
+} {
+  const messages: string[] = [];
+  const logger: HookContext['logger'] = {
+    info: (m: string) => messages.push(`info: ${m}`),
+    warn: (m: string) => messages.push(`warn: ${m}`),
+    error: (m: string) => messages.push(`error: ${m}`),
+    success: (m: string) => messages.push(`success: ${m}`),
+  };
+  return { logger, messages };
+}
+
+function buildContext(extras: Record<string, unknown>): HookContext {
+  const { logger } = buildLogger();
+  return {
+    logger,
+    config: {
+      db_path: '', // populated by caller via env
+      hostname: 'test-mgmt',
+      target_ip: '127.0.0.1',
+      event_bus_socket: '',
+      listen_port: 8123,
+      dns_primary: '1.1.1.1',
+      dns_fallback: '',
+      network_dmz_subnet: '',
+      network_app_subnet: '',
+      network_secure_subnet: '',
+      network_internal_subnet: '',
+      ssh_public_key: '',
+      install_docker: false,
+      install_terraform: false,
+    } as Record<string, unknown>,
+    secrets: {},
+    capabilities: {},
+    debug: false,
+    screenshotDir: '',
+    ...extras,
+  } as HookContext;
+}
+
+describe('celilo-mgmt on_backup', () => {
+  let dir: string;
+  let backupDir: string;
+  let crossModuleRoot: string;
+  let dbPath: string;
+  let keyPath: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-mgmt-backup-test-'));
+    backupDir = join(dir, 'backup-dir');
+    mkdirSync(backupDir, { recursive: true });
+    crossModuleRoot = join(dir, 'cross-module-root');
+    mkdirSync(join(crossModuleRoot, 'modules', 'caddy', 'terraform'), { recursive: true });
+    writeFileSync(
+      join(crossModuleRoot, 'modules', 'caddy', 'terraform', 'terraform.tfstate'),
+      '{"state":"caddy"}',
+    );
+    writeFileSync(
+      join(crossModuleRoot, 'index.json'),
+      JSON.stringify({
+        schemaVersion: '1.0',
+        generatedAt: '2026-05-20T00:00:00Z',
+        modules: [{ id: 'caddy', version: '0.0.1', terraformStateDir: 'modules/caddy/terraform' }],
+      }),
+    );
+
+    dbPath = join(dir, 'celilo.db');
+    writeFileSync(dbPath, 'fake-sqlite-bytes-for-test');
+    process.env.CELILO_DB_PATH = dbPath;
+
+    keyPath = join(dir, 'master.key');
+    writeFileSync(keyPath, 'fake-master-key-32-bytes-padding!');
+    process.env.CELILO_MASTER_KEY_PATH = keyPath;
+  });
+
+  afterEach(() => {
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('writes celilo.db, master.key, machine-pool.json, and cross_module_state into backup_dir', async () => {
+    const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
+    const ctx = buildContext({
+      backup_dir: backupDir,
+      cross_module_root: crossModuleRoot,
+    });
+    const result = (await hook(ctx)) as BackupHookOutput;
+
+    expect(existsSync(join(backupDir, 'celilo.db'))).toBe(true);
+    expect(existsSync(join(backupDir, 'master.key'))).toBe(true);
+    expect(existsSync(join(backupDir, 'machine-pool.json'))).toBe(true);
+    expect(
+      existsSync(
+        join(backupDir, 'cross_module_state', 'modules', 'caddy', 'terraform', 'terraform.tfstate'),
+      ),
+    ).toBe(true);
+    expect(existsSync(join(backupDir, 'cross_module_state', 'index.json'))).toBe(true);
+
+    expect(result.schema_version).toBe('1.0');
+    expect(result.artifact_count).toBeGreaterThan(0);
+    expect(result.size_bytes).toBeGreaterThan(0);
+  });
+
+  it('machine-pool.json is valid JSON (array)', async () => {
+    const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
+    await hook(buildContext({ backup_dir: backupDir, cross_module_root: crossModuleRoot }));
+
+    const machinePool = JSON.parse(readFileSync(join(backupDir, 'machine-pool.json'), 'utf-8'));
+    expect(Array.isArray(machinePool)).toBe(true);
+  });
+
+  it('proceeds (with a warning) when cross_module_root is missing', async () => {
+    const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
+    const result = (await hook(
+      buildContext({ backup_dir: backupDir }), // no cross_module_root
+    )) as BackupHookOutput;
+
+    expect(existsSync(join(backupDir, 'celilo.db'))).toBe(true);
+    expect(existsSync(join(backupDir, 'cross_module_state'))).toBe(false);
+    expect(result.schema_version).toBe('1.0');
+  });
+
+  it('proceeds (with a warning) when master.key is missing on disk', async () => {
+    rmSync(keyPath);
+    const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
+    const result = (await hook(
+      buildContext({ backup_dir: backupDir, cross_module_root: crossModuleRoot }),
+    )) as BackupHookOutput;
+
+    expect(existsSync(join(backupDir, 'master.key'))).toBe(false);
+    expect(existsSync(join(backupDir, 'celilo.db'))).toBe(true);
+    expect(result.artifact_count).toBeGreaterThan(0);
+  });
+});
+
+describe('celilo-mgmt on_restore', () => {
+  let dir: string;
+  let restoreDir: string;
+  let crossModuleWriteRoot: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-mgmt-restore-test-'));
+    restoreDir = join(dir, 'restore-dir');
+    mkdirSync(restoreDir, { recursive: true });
+    crossModuleWriteRoot = join(dir, 'cross-module-write-root');
+    mkdirSync(crossModuleWriteRoot, { recursive: true });
+  });
+
+  afterEach(() => {
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function plantArtifact(): void {
+    writeFileSync(join(restoreDir, 'celilo.db'), 'restored-db-bytes');
+    writeFileSync(join(restoreDir, 'master.key'), 'restored-master-key');
+    writeFileSync(join(restoreDir, 'machine-pool.json'), '[]');
+    mkdirSync(join(restoreDir, 'cross_module_state', 'modules', 'caddy', 'terraform'), {
+      recursive: true,
+    });
+    writeFileSync(
+      join(restoreDir, 'cross_module_state', 'modules', 'caddy', 'terraform', 'terraform.tfstate'),
+      '{"state":"caddy-restored"}',
+    );
+    mkdirSync(join(restoreDir, 'cross_module_state', 'modules', 'homebridge', 'terraform'), {
+      recursive: true,
+    });
+    writeFileSync(
+      join(
+        restoreDir,
+        'cross_module_state',
+        'modules',
+        'homebridge',
+        'terraform',
+        'terraform.tfstate',
+      ),
+      '{"state":"homebridge-restored"}',
+    );
+  }
+
+  it('moves cross-module terraform state into cross_module_write_root for the framework apply', async () => {
+    plantArtifact();
+    const { default: hook } = await import(`${HOOK_DIR}/on_restore.ts`);
+    const result = (await hook(
+      buildContext({
+        restore_dir: restoreDir,
+        schema_version: '1.0',
+        cross_module_write_root: crossModuleWriteRoot,
+      }),
+    )) as RestoreHookOutput;
+
+    expect(
+      existsSync(join(crossModuleWriteRoot, 'modules', 'caddy', 'terraform', 'terraform.tfstate')),
+    ).toBe(true);
+    expect(
+      existsSync(
+        join(crossModuleWriteRoot, 'modules', 'homebridge', 'terraform', 'terraform.tfstate'),
+      ),
+    ).toBe(true);
+
+    // 2 cross-module modules + 2 system files (db + master.key)
+    expect(result.restored_items).toBe(4);
+  });
+
+  it('stages celilo.db + master.key under restore_dir/system/ for Phase 4 wrapper', async () => {
+    plantArtifact();
+    const { default: hook } = await import(`${HOOK_DIR}/on_restore.ts`);
+    await hook(
+      buildContext({
+        restore_dir: restoreDir,
+        schema_version: '1.0',
+        cross_module_write_root: crossModuleWriteRoot,
+      }),
+    );
+
+    expect(existsSync(join(restoreDir, 'system', 'celilo.db'))).toBe(true);
+    expect(existsSync(join(restoreDir, 'system', 'master.key'))).toBe(true);
+    expect(readFileSync(join(restoreDir, 'system', 'celilo.db'), 'utf-8')).toBe(
+      'restored-db-bytes',
+    );
+  });
+
+  it('skips cross-module restore (with a warning) when cross_module_write_root is missing', async () => {
+    plantArtifact();
+    const { default: hook } = await import(`${HOOK_DIR}/on_restore.ts`);
+    const result = (await hook(
+      buildContext({ restore_dir: restoreDir, schema_version: '1.0' }), // no cross_module_write_root
+    )) as RestoreHookOutput;
+
+    // Only system files staged; cross-module skipped.
+    expect(result.restored_items).toBe(2);
+    expect(existsSync(join(restoreDir, 'system', 'celilo.db'))).toBe(true);
+  });
+
+  it('tolerates an artifact with no cross_module_state/ subdir', async () => {
+    // No plantArtifact call's cross-module branch — write only system files.
+    writeFileSync(join(restoreDir, 'celilo.db'), 'db');
+    writeFileSync(join(restoreDir, 'master.key'), 'key');
+    const { default: hook } = await import(`${HOOK_DIR}/on_restore.ts`);
+    const result = (await hook(
+      buildContext({
+        restore_dir: restoreDir,
+        schema_version: '1.0',
+        cross_module_write_root: crossModuleWriteRoot,
+      }),
+    )) as RestoreHookOutput;
+
+    expect(result.restored_items).toBe(2);
+  });
+});

package/src/services/config-interview.ts

@@ -17,6 +17,7 @@ import {
   type SecretRequiredPayload,
   busInterview,
 } from './bus-interview';
+import { parseStoredConfigValue, upsertModuleConfig } from './module-config';
 import { getSecretMetadata } from './secret-schema-loader';
 
 /**
@@ -245,20 +246,7 @@ export async function autoDeriveMachineConfig(
 
     log.success(`${variable.name} = ${derived} (auto-derived from ${machine.hostname})`);
 
-    await db
-      .insert(moduleConfigs)
-      .values({
-        moduleId,
-        key: variable.name,
-        value: derived,
-        valueJson: null,
-        createdAt: new Date(),
-        updatedAt: new Date(),
-      })
-      .onConflictDoUpdate({
-        target: [moduleConfigs.moduleId, moduleConfigs.key],
-        set: { value: derived, updatedAt: new Date() },
-      });
+    upsertModuleConfig(db, moduleId, variable.name, derived);
     configured.push(variable.name);
 
     // Handle per_selection follow-ups (e.g., zone_ip_* from zone list)
@@ -273,20 +261,7 @@ export async function autoDeriveMachineConfig(
 
         if (followUpValue !== null) {
           log.success(`${followUpKey} = ${followUpValue} (auto-derived from ${machine.hostname})`);
-          await db
-            .insert(moduleConfigs)
-            .values({
-              moduleId,
-              key: followUpKey,
-              value: followUpValue,
-              valueJson: null,
-              createdAt: new Date(),
-              updatedAt: new Date(),
-            })
-            .onConflictDoUpdate({
-              target: [moduleConfigs.moduleId, moduleConfigs.key],
-              set: { value: followUpValue, updatedAt: new Date() },
-            });
+          upsertModuleConfig(db, moduleId, followUpKey, followUpValue);
           configured.push(followUpKey);
         }
       }
@@ -328,20 +303,7 @@ export async function interviewForMissingConfig(
             const selectedValues = derived.split(',');
 
             // Store the main variable
-            await db
-              .insert(moduleConfigs)
-              .values({
-                moduleId,
-                key: variable.name,
-                value: derived,
-                valueJson: null,
-                createdAt: new Date(),
-                updatedAt: new Date(),
-              })
-              .onConflictDoUpdate({
-                target: [moduleConfigs.moduleId, moduleConfigs.key],
-                set: { value: derived, updatedAt: new Date() },
-              });
+            upsertModuleConfig(db, moduleId, variable.name, derived);
             configured.push(variable.name);
 
             // Handle per_selection follow-ups
@@ -361,20 +323,7 @@ export async function interviewForMissingConfig(
                 log.info(
                   `✓ ${followUpKey} = ${followUpValue} (from machine ${earmarkedMachine.hostname})`,
                 );
-                await db
-                  .insert(moduleConfigs)
-                  .values({
-                    moduleId,
-                    key: followUpKey,
-                    value: followUpValue,
-                    valueJson: null,
-                    createdAt: new Date(),
-                    updatedAt: new Date(),
-                  })
-                  .onConflictDoUpdate({
-                    target: [moduleConfigs.moduleId, moduleConfigs.key],
-                    set: { value: followUpValue, updatedAt: new Date() },
-                  });
+                upsertModuleConfig(db, moduleId, followUpKey, followUpValue);
                 configured.push(followUpKey);
               } else {
                 // Can't derive — bus-mediated prompt (responder
@@ -404,20 +353,7 @@ export async function interviewForMissingConfig(
           }
 
           // Simple (non-multi-select) derived variable
-          await db
-            .insert(moduleConfigs)
-            .values({
-              moduleId,
-              key: variable.name,
-              value: derived,
-              valueJson: null,
-              createdAt: new Date(),
-              updatedAt: new Date(),
-            })
-            .onConflictDoUpdate({
-              target: [moduleConfigs.moduleId, moduleConfigs.key],
-              set: { value: derived, updatedAt: new Date() },
-            });
+          upsertModuleConfig(db, moduleId, variable.name, derived);
           configured.push(variable.name);
           continue;
         }
@@ -850,14 +786,7 @@ export async function readModuleConfigKey(
     .where(and(eq(moduleConfigs.moduleId, moduleId), eq(moduleConfigs.key, key)))
     .get();
   if (!row) return undefined;
-  if (row.valueJson) {
-    try {
-      return JSON.parse(row.valueJson);
-    } catch {
-      return row.value;
-    }
-  }
-  return row.value;
+  return parseStoredConfigValue(row);
 }
 
 export async function writeModuleConfigKey(
@@ -866,23 +795,12 @@ export async function writeModuleConfigKey(
   value: unknown,
   db: DbClient,
 ): Promise<void> {
-  const valueJson = JSON.stringify(value);
-  const stringValue = typeof value === 'string' ? value : valueJson;
-  await db
-    .insert(moduleConfigs)
-    .values({
-      moduleId,
-      key,
-      value: stringValue,
-      valueJson,
-      createdAt: new Date(),
-      updatedAt: new Date(),
-    })
-    .onConflictDoUpdate({
-      target: [moduleConfigs.moduleId, moduleConfigs.key],
-      set: { value: stringValue, valueJson, updatedAt: new Date() },
-    })
-    .run();
+  upsertModuleConfig(
+    db,
+    moduleId,
+    key,
+    value as string | number | boolean | unknown[] | Record<string, unknown>,
+  );
 }
 
 export async function readModuleSecretKey(

package/src/services/cross-module-data-manager.ts

@@ -17,6 +17,7 @@ import { decryptSecret, encryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { buildResolutionContext } from '../variables/context';
 import { resolveTemplate } from '../variables/resolver';
+import { upsertModuleConfig } from './module-config';
 
 /**
  * Configuration data type
@@ -101,37 +102,7 @@ export class CrossModuleDataManager {
     key: string,
     value: string | number | boolean | unknown[] | Record<string, unknown>,
   ): Promise<void> {
-    const isComplex = typeof value === 'object' && value !== null;
-
-    const existing = this.db
-      .select()
-      .from(moduleConfigs)
-      .where(and(eq(moduleConfigs.moduleId, moduleId), eq(moduleConfigs.key, key)))
-      .get();
-
-    if (existing) {
-      // Update existing
-      this.db
-        .update(moduleConfigs)
-        .set({
-          value: isComplex ? '' : String(value),
-          valueJson: isComplex ? JSON.stringify(value) : null,
-          updatedAt: new Date(Date.now()),
-        })
-        .where(eq(moduleConfigs.id, existing.id))
-        .run();
-    } else {
-      // Insert new
-      this.db
-        .insert(moduleConfigs)
-        .values({
-          moduleId,
-          key,
-          value: isComplex ? '' : String(value),
-          valueJson: isComplex ? JSON.stringify(value) : null,
-        })
-        .run();
-    }
+    upsertModuleConfig(this.db, moduleId, key, value);
   }
 
   /**