@celilo/cli 0.3.30-alpha.0 → 0.4.0-alpha.0

package/src/services/module-config.ts

@@ -70,6 +70,75 @@ export async function parseConfigValue(
   }
 }
 
+/**
+ * Upsert a `module_configs` row. THE ONLY supported write path.
+ *
+ * Direct `db.insert(moduleConfigs).values({...})` calls are an
+ * anti-pattern — they bypass valueJson population and leave reads
+ * unable to recover the original type. Defect 1 was exactly this:
+ * primitive writes that set `value` but not `valueJson`, so reads
+ * had to guess (and got it wrong for stringly-looking-numeric
+ * values, etc.). Every write site routes through here now.
+ *
+ * Callers pass the canonical JS value (`number`, `boolean`,
+ * `string`, array, object); we JSON-stringify into `valueJson`
+ * (the typed canonical) and produce a human-readable form for
+ * `value` (used only for CLI display).
+ */
+export function upsertModuleConfig(
+  db: DbClient,
+  moduleId: string,
+  key: string,
+  value: string | number | boolean | unknown[] | Record<string, unknown> | null,
+): void {
+  if (value === null || value === undefined) {
+    throw new Error(`upsertModuleConfig(${moduleId}, ${key}): value must not be null/undefined`);
+  }
+  const valueJson = JSON.stringify(value);
+  const displayValue = isComplexValue(value) ? valueJson : String(value);
+
+  db.insert(moduleConfigs)
+    .values({ moduleId, key, value: displayValue, valueJson })
+    .onConflictDoUpdate({
+      target: [moduleConfigs.moduleId, moduleConfigs.key],
+      set: { value: displayValue, valueJson, updatedAt: new Date() },
+    })
+    .run();
+}
+
+/**
+ * Parse a stored module_configs row into its canonical typed value.
+ * Throws if valueJson is null — that's a row written before the
+ * always-populate-valueJson invariant, and we no longer support it.
+ * See schema.ts module_configs doc comment for context.
+ *
+ * Exported so other read sites (hook config loader, variables
+ * context, ansible inventory builder, etc.) can share one parsing
+ * path — a single source of truth for "DB row → typed value".
+ */
+export function parseStoredConfigValue(
+  row: typeof moduleConfigs.$inferSelect,
+): string | number | boolean | unknown[] | Record<string, unknown> {
+  if (row.valueJson === null || row.valueJson === undefined) {
+    throw new Error(
+      `module_configs row ${row.moduleId}.${row.key} has null valueJson — ` +
+        `pre-Defect-1 row, no longer supported. Re-run \`celilo module config set ${row.moduleId} ${row.key} <value>\` to populate.`,
+    );
+  }
+  try {
+    return JSON.parse(row.valueJson) as
+      | string
+      | number
+      | boolean
+      | unknown[]
+      | Record<string, unknown>;
+  } catch (error) {
+    throw new Error(
+      `Failed to parse valueJson for ${row.moduleId}.${row.key}: ${error instanceof Error ? error.message : 'Invalid JSON'}`,
+    );
+  }
+}
+
 /**
  * Get module configuration value
  * Returns parsed value (primitive or complex type)
@@ -89,41 +158,11 @@ export function getModuleConfigValue(
     return null;
   }
 
-  // Check if complex type (stored in valueJson)
-  if (config.valueJson) {
-    try {
-      const parsed = JSON.parse(config.valueJson);
-      return {
-        key: config.key,
-        value: parsed,
-        isPrimitive: false,
-      };
-    } catch (error) {
-      throw new Error(
-        `Failed to parse config value for ${moduleId}.${config.key}: ${error instanceof Error ? error.message : 'Invalid JSON'}`,
-      );
-    }
-  }
-
-  // Primitive type (stored in value)
-  // Parse the string value to correct type
-  let parsedValue: string | number | boolean = config.value;
-
-  // Boolean
-  if (config.value === 'true') parsedValue = true;
-  else if (config.value === 'false') parsedValue = false;
-  // Number
-  else {
-    const num = Number(config.value);
-    if (!Number.isNaN(num)) {
-      parsedValue = num;
-    }
-  }
-
+  const parsed = parseStoredConfigValue(config);
   return {
     key: config.key,
-    value: parsedValue,
-    isPrimitive: true,
+    value: parsed,
+    isPrimitive: !isComplexValue(parsed),
   };
 }
 
@@ -134,40 +173,11 @@ export function getAllModuleConfigValues(moduleId: string, db: DbClient = getDb(
   const configs = db.select().from(moduleConfigs).where(eq(moduleConfigs.moduleId, moduleId)).all();
 
   return configs.map((config: typeof moduleConfigs.$inferSelect) => {
-    // Check if complex type
-    if (config.valueJson) {
-      try {
-        const parsed = JSON.parse(config.valueJson);
-        return {
-          key: config.key,
-          value: parsed,
-          isPrimitive: false,
-        };
-      } catch (error) {
-        throw new Error(
-          `Failed to parse config value for ${moduleId}.${config.key}: ${error instanceof Error ? error.message : 'Invalid JSON'}`,
-        );
-      }
-    }
-
-    // Primitive type
-    let parsedValue: string | number | boolean = config.value;
-
-    // Boolean
-    if (config.value === 'true') parsedValue = true;
-    else if (config.value === 'false') parsedValue = false;
-    // Number
-    else {
-      const num = Number(config.value);
-      if (!Number.isNaN(num)) {
-        parsedValue = num;
-      }
-    }
-
+    const parsed = parseStoredConfigValue(config);
     return {
       key: config.key,
-      value: parsedValue,
-      isPrimitive: true,
+      value: parsed,
+      isPrimitive: !isComplexValue(parsed),
     };
   });
 }
@@ -228,61 +238,9 @@ export async function setModuleConfigValue(
     throw new Error(formatValidationErrors(validation.errors || []));
   }
 
-  // Determine if complex type
-  const isComplex = isComplexValue(parsedValue);
-
-  // Check if config already exists
-  const existingConfig = db
-    .select()
-    .from(moduleConfigs)
-    .where(and(eq(moduleConfigs.moduleId, moduleId), eq(moduleConfigs.key, key)))
-    .get();
-
-  if (existingConfig) {
-    // Update existing config
-    if (isComplex) {
-      // Store in valueJson column
-      db.update(moduleConfigs)
-        .set({
-          value: '', // Empty string for complex types
-          valueJson: JSON.stringify(parsedValue),
-          updatedAt: new Date(),
-        })
-        .where(eq(moduleConfigs.id, existingConfig.id))
-        .run();
-    } else {
-      // Store in value column
-      db.update(moduleConfigs)
-        .set({
-          value: String(parsedValue),
-          valueJson: null,
-          updatedAt: new Date(),
-        })
-        .where(eq(moduleConfigs.id, existingConfig.id))
-        .run();
-    }
-  } else {
-    // Insert new config
-    if (isComplex) {
-      db.insert(moduleConfigs)
-        .values({
-          moduleId,
-          key,
-          value: '', // Empty string for complex types
-          valueJson: JSON.stringify(parsedValue),
-        })
-        .run();
-    } else {
-      db.insert(moduleConfigs)
-        .values({
-          moduleId,
-          key,
-          value: String(parsedValue),
-          valueJson: null,
-        })
-        .run();
-    }
-  }
+  // Delegate to the shared upsert helper — single storage path,
+  // single place where the valueJson invariant lives.
+  upsertModuleConfig(db, moduleId, key, parsedValue);
 }
 
 /**

package/src/services/module-deploy.ts

@@ -20,6 +20,7 @@ import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { buildResolutionContext } from '../variables/context';
+import { maybeRunAspectForTrigger } from './aspect-runner';
 import {
   autoDeriveMachineConfig,
   findEnsureOnProvider,
@@ -33,6 +34,7 @@ import { planDeployment } from './deploy-planner';
 import { waitForSSH } from './deploy-ssh';
 import { executeTerraform, parseTerraformOutputs } from './deploy-terraform';
 import { validateAndPrepareDeployment } from './deploy-validation';
+import { getModuleSystems } from './deployed-systems';
 import { resolveInfrastructureVariables } from './infrastructure-variable-resolver';
 import { findMachineForModule } from './machine-pool';
 import { checkProxmoxReachable, formatProxmoxUnreachableError } from './proxmox-preflight';
@@ -263,7 +265,9 @@ export async function deployModule(
   const startedAt = Date.now();
   const { emitDeployCompleted, emitDeployFailed, emitDeployStarted, emitHealthCheckFailed } =
     await import('./celilo-events');
+  const { startOperation, completeOperation, failOperation } = await import('./module-operations');
 
+  const opId = startOperation(moduleId, 'deploy');
   emitDeployStarted({ module: moduleId, startedAt });
 
   let result: DeployResult;
@@ -271,19 +275,22 @@ export async function deployModule(
     result = await deployModuleImpl(moduleId, db, options);
   } catch (err) {
     const error = err instanceof Error ? err.message : String(err);
+    failOperation(opId, err);
     emitDeployFailed({
       module: moduleId,
       startedAt,
-      durationMs: Date.now() - startedAt,
+      durationMs: Math.max(0, Date.now() - startedAt),
       error,
     });
     throw err;
   }
 
-  const durationMs = Date.now() - startedAt;
+  const durationMs = Math.max(0, Date.now() - startedAt);
   if (result.success) {
+    completeOperation(opId);
     emitDeployCompleted({ module: moduleId, startedAt, durationMs });
   } else {
+    failOperation(opId, result.error ?? 'unknown error');
     emitDeployFailed({
       module: moduleId,
       startedAt,
@@ -607,6 +614,7 @@ async function deployModuleImpl(
             debug: options.debug,
             capabilities: capabilityFunctions,
             requiredCapabilities: manifest.requires.capabilities.map((c) => c.name),
+            systems: getModuleSystems(moduleId, db),
           },
         );
 
@@ -745,6 +753,7 @@ async function deployModuleImpl(
                 debug: options.debug,
                 capabilities: capFns,
                 requiredCapabilities: (manifest.requires?.capabilities ?? []).map((c) => c.name),
+                systems: getModuleSystems(moduleId, db),
               },
             };
           },
@@ -783,6 +792,25 @@ async function deployModuleImpl(
         }
       }
 
+      // Fan out the module's base-module aspect (if any) per the
+      // on_install trigger. Failures here don't fail the primary
+      // deploy — D4 in v2/CELILO_BASE.md: aspects are idempotent
+      // and forward-progress; a partial fleet update is expected to
+      // converge on the next fan-out. We log the result instead.
+      const aspectOutcome = await maybeRunAspectForTrigger({
+        moduleId,
+        manifest,
+        trigger: 'on_install',
+        db,
+      });
+      if (aspectOutcome.ran && !aspectOutcome.success) {
+        log.warn(
+          `Base-module aspect fan-out for '${moduleId}' failed: ${aspectOutcome.runResult?.error ?? 'unknown'}. Primary deploy succeeded; re-run \`celilo fleet redeploy ${moduleId}\` after fixing.`,
+        );
+      } else if (aspectOutcome.ran) {
+        log.success(`Base-module aspect fan-out for '${moduleId}' completed`);
+      }
+
       // Mirror the infrastructure-path success message at the end of
       // a successful deploy. Without this, config-only deploys end
       // abruptly with whatever the last hook line was — operator sees
@@ -875,6 +903,25 @@ async function deployModuleImpl(
       db,
     );
 
+    // Record the module's deployed system(s) now that the IP is known for every
+    // provider type (machine / proxmox IPAM / DO outputs). This populates
+    // ctx.systems for on_install and is the source of truth for system.created
+    // and DNS (v2/MODULE_SYSTEMS_ADDRESSING.md). API-only modules record none.
+    {
+      const { recordDeployedSystemForModule } = await import('./deployed-systems');
+      const recorded = await recordDeployedSystemForModule(
+        moduleId,
+        manifest,
+        plan.infrastructure,
+        db,
+      );
+      if (recorded.length > 0) {
+        log.success(
+          `Recorded ${recorded.length} deployed system(s): ${recorded.map((s) => `${s.hostname} (${s.ipv4_address})`).join(', ')}`,
+        );
+      }
+    }
+
     if (Object.keys(resolution.resolved).length > 0) {
       // Build complete message with all variables
       const lines = ['Infrastructure variables resolved:'];
@@ -999,6 +1046,7 @@ async function deployModuleImpl(
             debug: options.debug,
             capabilities: capFunctions,
             requiredCapabilities: providerManifest.requires.capabilities.map((c) => c.name),
+            systems: getModuleSystems(capRecord.moduleId, db),
           },
         );
 
@@ -1137,31 +1185,15 @@ async function deployModuleImpl(
           }
         }
 
-        // Persist target_ip to moduleConfigs (not just inject into the
-        // in-memory hook context). Later hooks like on_backup build
-        // their context from the DB only, so without this they'd see
-        // no target_ip and fail with "No container IP configured".
-        // Mirrors proxmox-state-recovery.ts for the machine-deployed
-        // case.
+        // Inject the machine's IP as ip.primary for hooks that still read it
+        // (firewall NAT target, etc.). The host's address for hooks now comes
+        // from ctx.systems (v2/MODULE_SYSTEMS_ADDRESSING.md), recorded into
+        // module_systems during generate — no target_ip is written here.
         if (machineId) {
           const { getMachine } = await import('./machine-pool');
           const deployMachine = await getMachine(machineId);
           if (deployMachine) {
             installConfigMap['ip.primary'] = deployMachine.ipAddress;
-            installConfigMap.target_ip = deployMachine.ipAddress;
-
-            await db
-              .insert(pcTable)
-              .values({
-                moduleId,
-                key: 'target_ip',
-                value: deployMachine.ipAddress,
-              })
-              .onConflictDoUpdate({
-                target: [pcTable.moduleId, pcTable.key],
-                set: { value: deployMachine.ipAddress },
-              })
-              .run();
           }
         }
 
@@ -1201,6 +1233,7 @@ async function deployModuleImpl(
                 debug: options.debug,
                 capabilities: capFns,
                 requiredCapabilities: manifest.requires.capabilities.map((c) => c.name),
+                systems: getModuleSystems(moduleId, db),
               },
             };
           },
@@ -1253,31 +1286,71 @@ async function deployModuleImpl(
       }
 
       // Auto-register module hostname in internal DNS (if available).
-      // Wrapped in a FuelGauge with a gauge logger so the capability calls
-      // inside (`dns_internal.registerRecord`, etc.) nest as sub-events
-      // under a single step rather than leaking to scrollback as
-      // unindented top-level lines via cli/prompts.log.instantEvent.
-      let dnsRegistered = true;
-      const dnsGauge = new FuelGauge(`Registering ${moduleId} in DNS`, {
-        skipAnimation: !process.stdout.isTTY,
-      });
-      dnsGauge.start();
+      // When a dns_internal provider deploys, backfill DNS for every
+      // already-deployed system by invoking its own on_system_event hook per
+      // host — the event path below only covers systems that deploy AFTER the
+      // provider (deliveries bind at emit time). Non-providers skip this
+      // entirely; their registration rides the system.created event below.
+      // v2/EVENT_DRIVEN_HOOK_SUBSCRIPTIONS.md.
+      const { isDnsInternalProvider, backfillProviderDns } = await import(
+        './dns-provider-backfill'
+      );
+      if (isDnsInternalProvider(moduleId, db)) {
+        // FuelGauge so the per-host hook invocations nest as sub-events under
+        // one step rather than leaking to scrollback.
+        const dnsGauge = new FuelGauge(`Backfilling internal DNS via ${moduleId}`, {
+          skipAnimation: !process.stdout.isTTY,
+        });
+        dnsGauge.start();
+        try {
+          const dnsLogger = createGaugeLogger(dnsGauge, moduleId, 'dns_backfill');
+          await backfillProviderDns(moduleId, db, dnsLogger);
+          dnsGauge.stop(true);
+        } catch (error) {
+          dnsGauge.stop(false);
+          const msg = error instanceof Error ? error.message : String(error);
+          log.warn(
+            `DNS backfill failed for '${moduleId}': ${msg}. Some internal DNS records may need manual setup.`,
+          );
+        }
+      }
+
+      // Announce each deployed system on the bus (D5/D6) — one
+      // system.created.<module> per host, so a dns_internal provider's
+      // subscription registers every one (v2/MODULE_SYSTEMS_ADDRESSING.md,
+      // v2/INTERNAL_DNS_DHCP_AND_SPLIT_HORIZON.md, v2/EVENT_DRIVEN_HOOK_SUBSCRIPTIONS.md).
       try {
-        const dnsLogger = createGaugeLogger(dnsGauge, moduleId, 'auto_register_dns');
-        const { autoRegisterDns } = await import('./dns-auto-register');
-        await autoRegisterDns(moduleId, db, dnsLogger);
-        dnsGauge.stop(true);
+        const { emitSystemCreated } = await import('./celilo-events');
+        for (const sys of getModuleSystems(moduleId, db)) {
+          emitSystemCreated({
+            module: moduleId,
+            hostname: sys.hostname,
+            targetIp: sys.ipv4_address,
+          });
+        }
       } catch (error) {
-        dnsGauge.stop(false);
+        // Best-effort: a bus hiccup must not fail an otherwise-good deploy.
         const msg = error instanceof Error ? error.message : String(error);
-        log.warn(`DNS auto-registration failed: ${msg}`);
-        dnsRegistered = false;
+        log.warn(`Failed to emit system.created for ${moduleId}: ${msg}`);
       }
 
-      if (!dnsRegistered) {
+      // Fan out the module's base-module aspect (if any) per the
+      // on_install trigger. Failures here don't fail the primary
+      // deploy — D4 in v2/CELILO_BASE.md: aspects are idempotent
+      // and forward-progress; a partial fleet update is expected to
+      // converge on the next fan-out. We log the result instead.
+      const aspectOutcome = await maybeRunAspectForTrigger({
+        moduleId,
+        manifest,
+        trigger: 'on_install',
+        db,
+      });
+      if (aspectOutcome.ran && !aspectOutcome.success) {
         log.warn(
-          `Module '${moduleId}' deployed but DNS registration failed. Internal DNS records may need manual setup.`,
+          `Base-module aspect fan-out for '${moduleId}' failed: ${aspectOutcome.runResult?.error ?? 'unknown'}. Primary deploy succeeded; re-run \`celilo fleet redeploy ${moduleId}\` after fixing.`,
         );
+      } else if (aspectOutcome.ran) {
+        log.success(`Base-module aspect fan-out for '${moduleId}' completed`);
       }
 
       log.success(`Module '${moduleId}' deployed successfully`);

package/src/services/module-operations.test.ts

@@ -0,0 +1,154 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { closeDb } from '../db/client';
+import { runMigrations } from '../db/migrate';
+import {
+  InFlightError,
+  checkInFlight,
+  completeOperation,
+  failOperation,
+  isPidAlive,
+  refuseIfInFlight,
+  startOperation,
+} from './module-operations';
+
+describe('module-operations', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-ops-test-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  describe('startOperation / completeOperation / failOperation', () => {
+    it('records an in-progress row on start, transitions to completed on complete', () => {
+      const id = startOperation('homebridge', 'deploy');
+      const inFlight = checkInFlight();
+      expect(inFlight).toHaveLength(1);
+      expect(inFlight[0].operation.id).toBe(id);
+      expect(inFlight[0].operation.status).toBe('in_progress');
+      expect(inFlight[0].operation.operation).toBe('deploy');
+      expect(inFlight[0].operation.moduleId).toBe('homebridge');
+
+      completeOperation(id);
+      expect(checkInFlight()).toHaveLength(0);
+    });
+
+    it('transitions to failed with error message on failOperation', () => {
+      const id = startOperation('caddy', 'backup');
+      failOperation(id, new Error('disk full'));
+      expect(checkInFlight()).toHaveLength(0);
+
+      // Re-querying directly to verify the failed row is recorded
+      const id2 = startOperation('caddy', 'backup');
+      const inFlight = checkInFlight();
+      expect(inFlight).toHaveLength(1);
+      expect(inFlight[0].operation.id).toBe(id2);
+    });
+
+    it('failOperation accepts non-Error values', () => {
+      const id = startOperation('caddy', 'restore');
+      failOperation(id, 'string error reason');
+      expect(checkInFlight()).toHaveLength(0);
+    });
+  });
+
+  describe('checkInFlight', () => {
+    it('returns empty when no operations are in flight', () => {
+      expect(checkInFlight()).toHaveLength(0);
+    });
+
+    it('describes conflicts with module + operation + pid', () => {
+      const id = startOperation('caddy', 'deploy');
+      const conflicts = checkInFlight();
+      expect(conflicts).toHaveLength(1);
+      expect(conflicts[0].describe).toBe(`deploy of caddy (pid ${process.pid})`);
+      completeOperation(id);
+    });
+
+    it('excludes the operation matching excludeOperationId', () => {
+      const own = startOperation('caddy', 'backup');
+      const other = startOperation('homebridge', 'deploy');
+      const conflicts = checkInFlight(own);
+      expect(conflicts).toHaveLength(1);
+      expect(conflicts[0].operation.id).toBe(other);
+      completeOperation(own);
+      completeOperation(other);
+    });
+
+    it('ignores rows whose pid is no longer alive', () => {
+      // Spawn a short-lived process, capture its pid, wait for it to exit.
+      const child = spawnSync('node', ['-e', 'process.exit(0)']);
+      const deadPid = child.pid;
+      expect(deadPid).toBeGreaterThan(0);
+      expect(isPidAlive(deadPid)).toBe(false);
+
+      // Insert a fake row with the dead pid via raw SQL (bypasses pid=process.pid in startOperation).
+      const { getDb } = require('../db/client');
+      const { moduleOperations } = require('../db/schema');
+      const db = getDb();
+      db.insert(moduleOperations)
+        .values({
+          id: 'fake-dead-row',
+          moduleId: 'orphan',
+          operation: 'deploy',
+          status: 'in_progress',
+          pid: deadPid,
+        })
+        .run();
+
+      const conflicts = checkInFlight();
+      expect(conflicts).toHaveLength(0); // dead pid filtered out
+    });
+  });
+
+  describe('refuseIfInFlight', () => {
+    it('throws InFlightError when conflicts exist', () => {
+      const id = startOperation('caddy', 'deploy');
+      expect(() => refuseIfInFlight()).toThrow(InFlightError);
+      try {
+        refuseIfInFlight();
+      } catch (err) {
+        expect(err).toBeInstanceOf(InFlightError);
+        expect((err as InFlightError).conflicts).toHaveLength(1);
+        expect((err as Error).message).toContain('deploy of caddy');
+      }
+      completeOperation(id);
+    });
+
+    it('is a no-op when no conflicts exist', () => {
+      expect(() => refuseIfInFlight()).not.toThrow();
+    });
+
+    it('respects excludeOperationId', () => {
+      const own = startOperation('caddy', 'backup');
+      expect(() => refuseIfInFlight(own)).not.toThrow();
+      completeOperation(own);
+    });
+  });
+
+  describe('isPidAlive', () => {
+    it('returns true for the current process', () => {
+      expect(isPidAlive(process.pid)).toBe(true);
+    });
+
+    it('returns false for a dead pid', () => {
+      const child = spawnSync('node', ['-e', 'process.exit(0)']);
+      expect(isPidAlive(child.pid as number)).toBe(false);
+    });
+  });
+});