@celilo/cli 0.5.0-alpha.1 → 0.5.0-alpha.11

package/src/services/module-build.test.ts

@@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm';
 import { log } from '../cli/prompts';
 import { type DbClient, createDbClient } from '../db/client';
 import { moduleBuilds, modules } from '../db/schema';
+import { skipIntegration } from '../test-utils/integration-guard';
 import { buildModuleFromSource, getModuleBuildStatus, verifyArtifactsExist } from './module-build';
 
 const TEST_DB_PATH = './test-module-build.db';
@@ -156,12 +157,14 @@ describe('Module Build Service', () => {
       // We're just testing that detection happens (message logged)
     }, 10000); // 10 second timeout
 
-    test('should record build metadata in database', async () => {
-      await mkdir(TEST_MODULE_DIR, { recursive: true });
-      await mkdir(`${TEST_MODULE_DIR}/build`, { recursive: true });
-      await writeFile(
-        `${TEST_MODULE_DIR}/build/playbook.yml`,
-        `---
+    test.skipIf(skipIntegration({ tools: ['ansible'] }))(
+      'should record build metadata in database',
+      async () => {
+        await mkdir(TEST_MODULE_DIR, { recursive: true });
+        await mkdir(`${TEST_MODULE_DIR}/build`, { recursive: true });
+        await writeFile(
+          `${TEST_MODULE_DIR}/build/playbook.yml`,
+          `---
 - name: Quick test build
   hosts: localhost
   gather_facts: false
@@ -170,43 +173,45 @@ describe('Module Build Service', () => {
       ansible.builtin.debug:
         msg: "Build test"
 `,
-      );
+        );
 
-      db.insert(modules)
-        .values({
-          id: 'record-test',
-          name: 'Record Test',
-          version: '1.0.0',
-          sourcePath: TEST_MODULE_DIR,
-          manifestData: {
+        db.insert(modules)
+          .values({
             id: 'record-test',
             name: 'Record Test',
             version: '1.0.0',
-            build: {
-              script: 'build/playbook.yml',
+            sourcePath: TEST_MODULE_DIR,
+            manifestData: {
+              id: 'record-test',
+              name: 'Record Test',
+              version: '1.0.0',
+              build: {
+                script: 'build/playbook.yml',
+              },
             },
-          },
-        })
-        .run();
-
-      const result = await buildModuleFromSource('record-test', db);
-
-      // Build should succeed (simple debug task)
-      expect(result.success).toBe(true);
-
-      // Verify build metadata was recorded
-      const buildRecord = await db
-        .select()
-        .from(moduleBuilds)
-        .where(eq(moduleBuilds.moduleId, 'record-test'))
-        .get();
-
-      expect(buildRecord).toBeDefined();
-      expect(buildRecord?.moduleId).toBe('record-test');
-      expect(buildRecord?.version).toBe('1.0.0');
-      expect(buildRecord?.status).toBe('success');
-      expect(buildRecord?.environment).toBe('system'); // No flake.nix
-    }, 10000); // 10 second timeout for ansible execution
+          })
+          .run();
+
+        const result = await buildModuleFromSource('record-test', db);
+
+        // Build should succeed (simple debug task)
+        expect(result.success).toBe(true);
+
+        // Verify build metadata was recorded
+        const buildRecord = await db
+          .select()
+          .from(moduleBuilds)
+          .where(eq(moduleBuilds.moduleId, 'record-test'))
+          .get();
+
+        expect(buildRecord).toBeDefined();
+        expect(buildRecord?.moduleId).toBe('record-test');
+        expect(buildRecord?.version).toBe('1.0.0');
+        expect(buildRecord?.status).toBe('success');
+        expect(buildRecord?.environment).toBe('system'); // No flake.nix
+      },
+      10000,
+    ); // 10 second timeout for ansible execution
 
     // Regression: a build.command that references $CELILO_MODULE_SOURCE_DIR
     // (the variable celilo-registry's manifest uses to find sibling packages

package/src/services/module-config.ts

@@ -106,6 +106,18 @@ export function upsertModuleConfig(
     .run();
 }
 
+/**
+ * Delete a module config row if present (no-op if absent). Used to drop a
+ * derived/cached key that should no longer persist — e.g. `__infra_target_node`,
+ * now resolved live from Proxmox each generate rather than cached in the DB
+ * (ISS-0090: "the DB stores intent, Proxmox reports reality").
+ */
+export function deleteModuleConfig(db: DbClient, moduleId: string, key: string): void {
+  db.delete(moduleConfigs)
+    .where(and(eq(moduleConfigs.moduleId, moduleId), eq(moduleConfigs.key, key)))
+    .run();
+}
+
 /**
  * Parse a stored module_configs row into its canonical typed value.
  * Throws if valueJson is null — that's a row written before the

package/src/services/module-deploy.ts

@@ -48,7 +48,6 @@ export interface DeployResult {
   phases: {
     validation?: boolean;
     autoGenerated?: boolean;
-    autoBuilt?: boolean;
     planning?: boolean;
     terraformInit?: boolean;
     terraformPlan?: boolean;
@@ -353,7 +352,6 @@ async function deployModuleImpl(
     const validation = await validateAndPrepareDeployment(moduleId, db);
     phases.validation = validation.success;
     phases.autoGenerated = validation.autoGenerated;
-    phases.autoBuilt = validation.autoBuilt;
     if (!validation.success) {
       return {
         success: false,
@@ -532,9 +530,7 @@ async function deployModuleImpl(
       }
     }
 
-    log.success(
-      validation.autoBuilt ? 'Templates generated and module built' : 'Templates generated',
-    );
+    log.success('Templates generated');
 
     // Run validate_config hook if defined (e.g., credential validation via Playwright)
     if (manifest.hooks?.validate_config) {
@@ -911,7 +907,12 @@ async function deployModuleImpl(
         lines.push(`  ${key} = ${value}`);
       }
       if (resolution.skipped.length > 0) {
-        lines.push(`  (skipped user-configured: ${resolution.skipped.join(', ')})`);
+        // These are infrastructure-managed vars with no value this deploy (e.g.
+        // vmid/target_node on a machine deploy). NOT honored operator overrides —
+        // such keys are rejected at `config set` (ISS-0069); don't imply otherwise.
+        lines.push(
+          `  (auto-managed, not applicable this deploy: ${resolution.skipped.join(', ')})`,
+        );
       }
       log.success(lines.join('\n'));
 

package/src/services/module-subscriptions.test.ts

@@ -3,11 +3,14 @@ import { mkdtempSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { defineEvents, openBus } from '@celilo/event-bus';
+import { closeDb, getDb } from '../db/client';
 import { ModuleSubscriptionSchema } from '../manifest/schema';
 import type { ModuleManifest } from '../manifest/schema';
+import { setupTestDatabase as migrateDbFile } from '../test-utils/setup-test-db';
 import {
   registerModuleSubscriptions,
   resolveSubscription,
+  resyncAllSubscriptions,
   unregisterModuleSubscriptions,
 } from './module-subscriptions';
 
@@ -253,3 +256,88 @@ describe('register / unregister roundtrip', () => {
     }
   });
 });
+
+describe('resyncAllSubscriptions (ISS-0088)', () => {
+  let dir: string;
+  let busPath: string;
+
+  beforeEach(async () => {
+    closeDb();
+    dir = mkdtempSync(join(tmpdir(), 'resync-test-'));
+    busPath = join(dir, 'events.db');
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    process.env.EVENT_BUS_DB = busPath;
+    // Migrate the celilo.db file getDb() will open (resyncAllSubscriptions reads it).
+    const setupDb = await migrateDbFile(join(dir, 'celilo.db'));
+    setupDb.$client.close();
+  });
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function seedModule(
+    id: string,
+    state: string,
+    subs: Array<{ name: string; pattern: string; handler: string }>,
+  ): void {
+    const manifest = JSON.stringify({
+      celilo_contract: '1.0',
+      id,
+      name: id,
+      version: '1.0.0',
+      requires: { capabilities: [] },
+      provides: { capabilities: [] },
+      variables: { owns: [], imports: [] },
+      subscriptions: subs,
+    });
+    getDb().$client.run(
+      'INSERT INTO modules (id, name, version, source_path, manifest_data, state) VALUES (?, ?, ?, ?, ?, ?)',
+      [id, id, '1.0.0', '/p', manifest, state],
+    );
+  }
+
+  function subscriberNames(): string[] {
+    const bus = openBus({ dbPath: busPath, events: defineEvents({}) });
+    try {
+      return bus.db
+        .query<{ name: string }, []>('SELECT name FROM subscribers ORDER BY name')
+        .all()
+        .map((r) => r.name);
+    } finally {
+      bus.close();
+    }
+  }
+
+  it('registers subs for DEPLOYED modules only, skipping imported/undeployed', () => {
+    seedModule('caddy', 'VERIFIED', [
+      { name: 'reconcile', pattern: 'public_web.routes_changed', handler: 'echo' },
+    ]);
+    seedModule('technitium', 'INSTALLED', [
+      { name: 'dns', pattern: 'system.created.*', handler: 'echo' },
+    ]);
+    seedModule('forgejo', 'IMPORTED', [{ name: 'x', pattern: 'y', handler: 'echo' }]);
+    seedModule('greenwave', 'VERIFIED', []); // deployed but declares no subs
+
+    const result = resyncAllSubscriptions();
+
+    expect(result.modules).toBe(2); // caddy + technitium (forgejo not deployed, greenwave has none)
+    expect(result.registered).toBe(2);
+    expect(result.failures).toEqual([]);
+    // forgejo.x absent — it was IMPORTED, not deployed.
+    expect(subscriberNames()).toEqual(['caddy.reconcile', 'technitium.dns']);
+  });
+
+  it('is idempotent — a second resync does not duplicate rows', () => {
+    seedModule('caddy', 'VERIFIED', [{ name: 'reconcile', pattern: 'a', handler: 'echo' }]);
+    resyncAllSubscriptions();
+    resyncAllSubscriptions();
+    expect(subscriberNames()).toEqual(['caddy.reconcile']);
+  });
+});

package/src/services/module-subscriptions.ts

@@ -14,8 +14,12 @@
  * colliding.
  */
 
+import { join } from 'node:path';
 import { defineEvents, openBus } from '@celilo/event-bus';
-import { getEventBusPath } from '../config/paths';
+import { inArray } from 'drizzle-orm';
+import { getEventBusPath, getModuleStoragePath } from '../config/paths';
+import { getDb } from '../db/client';
+import { modules } from '../db/schema';
 import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
 
 /**
@@ -126,6 +130,51 @@ export function unregisterModuleSubscriptions(moduleId: string): {
   }
 }
 
+/**
+ * Rebuild the event-bus `subscribers` table from every deployed module's
+ * manifest `subscriptions:`. The reactive layer (who-reacts-to-what) is
+ * registered at module DEPLOY time and lives in the bus (events.db), which a
+ * restore/migration starts EMPTY — so after a cutover, event-driven reconciles
+ * (caddy public_web, dns register, …) are dead until every module redeploys
+ * (ISS-0088). This reconstructs them from durable celilo.db state instead.
+ *
+ * Idempotent (registerModuleSubscriptions upserts by subscriber name). Per-module
+ * failures are collected, not thrown, so one bad manifest doesn't abort the rest.
+ */
+export function resyncAllSubscriptions(): {
+  modules: number;
+  registered: number;
+  failures: Array<{ moduleId: string; error: string }>;
+} {
+  const db = getDb();
+  const deployed = db
+    .select()
+    .from(modules)
+    .where(inArray(modules.state, ['INSTALLED', 'VERIFIED']))
+    .all();
+
+  let modulesWithSubs = 0;
+  let registered = 0;
+  const failures: Array<{ moduleId: string; error: string }> = [];
+
+  for (const mod of deployed) {
+    const manifest = mod.manifestData as unknown as ModuleManifest;
+    if (!manifest?.subscriptions?.length) continue;
+    // Code always lives at ${getModuleStoragePath()}/<id> by construction
+    // (import.ts) — the same path module-upgrade re-registers from (ISS-0091).
+    const modulePath = join(getModuleStoragePath(), mod.id);
+    try {
+      const result = registerModuleSubscriptions(manifest, modulePath);
+      registered += result.registered;
+      modulesWithSubs += 1;
+    } catch (err) {
+      failures.push({ moduleId: mod.id, error: err instanceof Error ? err.message : String(err) });
+    }
+  }
+
+  return { modules: modulesWithSubs, registered, failures };
+}
+
 function scopedName(moduleId: string, subName: string): string {
   return `${moduleId}.${subName}`;
 }

package/src/services/module-validator/bundled-deps.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, test } from 'bun:test';
+import { findStaleBundledDeps, refreshAndVerifyBundledDeps } from './bundled-deps';
+
+describe('findStaleBundledDeps (ISS-0104)', () => {
+  test('flags a bundled @celilo dep older than the workspace', () => {
+    const mismatches = findStaleBundledDeps(
+      { '@celilo/capabilities': '0.1.8' },
+      { '@celilo/capabilities': '0.4.1' },
+    );
+    expect(mismatches).toEqual([
+      { pkg: '@celilo/capabilities', bundled: '0.1.8', workspace: '0.4.1' },
+    ]);
+  });
+
+  test('no mismatch when bundled matches workspace', () => {
+    expect(
+      findStaleBundledDeps(
+        { '@celilo/capabilities': '0.4.1', '@celilo/event-bus': '0.1.6' },
+        { '@celilo/capabilities': '0.4.1', '@celilo/event-bus': '0.1.6' },
+      ),
+    ).toEqual([]);
+  });
+
+  test('ignores workspace packages the module does not bundle', () => {
+    // Module bundles only capabilities; cli-display is a workspace package but
+    // not in this module's closure — must not be flagged.
+    expect(
+      findStaleBundledDeps(
+        { '@celilo/capabilities': '0.4.1' },
+        { '@celilo/capabilities': '0.4.1', '@celilo/cli-display': '0.1.9' },
+      ),
+    ).toEqual([]);
+  });
+
+  test('flags each stale dep independently', () => {
+    const mismatches = findStaleBundledDeps(
+      { '@celilo/capabilities': '0.1.8', '@celilo/event-bus': '0.1.6' },
+      { '@celilo/capabilities': '0.4.1', '@celilo/event-bus': '0.1.6' },
+    );
+    expect(mismatches).toEqual([
+      { pkg: '@celilo/capabilities', bundled: '0.1.8', workspace: '0.4.1' },
+    ]);
+  });
+});
+
+describe('refreshAndVerifyBundledDeps (ISS-0104)', () => {
+  test('no-op for a module without scripts/package.json', () => {
+    let installs = 0;
+    const result = refreshAndVerifyBundledDeps('/tmp/does-not-exist-module-xyz', () => {
+      installs++;
+    });
+    expect(result).toEqual({ refreshed: false, mismatches: [] });
+    expect(installs).toBe(0);
+  });
+});

package/src/services/module-validator/bundled-deps.ts

@@ -0,0 +1,115 @@
+/**
+ * ISS-0104: a module's hooks resolve `@celilo/*` from the module's OWN
+ * `scripts/node_modules` (nearest-wins), and those are gitignored local
+ * installs that go stale. The publish bundles them as-is, so a deployed module
+ * can silently run capability code months older than what was just published
+ * (e.g. caddy shipped capabilities@0.1.8 while the workspace was at 0.4.1, so
+ * ISS-0102 never reached the rendered Caddyfile).
+ *
+ * Before packing, the publish refreshes a module's `scripts/` deps and verifies
+ * the bundled `@celilo/*` versions match the workspace — the versions being
+ * co-published. A mismatch fails the publish (unless --allow-stale).
+ */
+
+import { execSync } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+/** The @celilo workspace packages a module may bundle (by published name). */
+const CELILO_WORKSPACE_PACKAGES = ['capabilities', 'cli-display', 'event-bus'] as const;
+
+export interface DepMismatch {
+  pkg: string;
+  bundled: string;
+  workspace: string;
+}
+
+/**
+ * Pure: which bundled `@celilo/*` versions differ from the workspace. Only deps
+ * the module actually bundles AND the workspace owns are compared — a module
+ * that doesn't bundle a given package, or a package the workspace doesn't own,
+ * is ignored.
+ */
+export function findStaleBundledDeps(
+  bundled: Record<string, string>,
+  workspace: Record<string, string>,
+): DepMismatch[] {
+  const mismatches: DepMismatch[] = [];
+  for (const [pkg, wsVersion] of Object.entries(workspace)) {
+    const bundledVersion = bundled[pkg];
+    if (bundledVersion && bundledVersion !== wsVersion) {
+      mismatches.push({ pkg, bundled: bundledVersion, workspace: wsVersion });
+    }
+  }
+  return mismatches;
+}
+
+function readVersion(packageJsonPath: string): string | undefined {
+  try {
+    const parsed = JSON.parse(readFileSync(packageJsonPath, 'utf-8')) as { version?: string };
+    return parsed.version;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Find the monorepo root (the dir with `packages/capabilities`) by walking up. */
+function findWorkspaceRoot(start: string): string | undefined {
+  let dir = start;
+  for (let depth = 0; depth < 8; depth++) {
+    if (existsSync(join(dir, 'packages', 'capabilities', 'package.json'))) return dir;
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return undefined;
+}
+
+export interface RefreshResult {
+  /** Whether the module had a scripts/ package to refresh. */
+  refreshed: boolean;
+  /** Bundled @celilo/* that don't match the workspace (empty = clean). */
+  mismatches: DepMismatch[];
+}
+
+/**
+ * Refresh a module's `scripts/` deps (so the bundled hook runtime is current)
+ * and verify the bundled `@celilo/*` versions match the workspace. No-op for a
+ * module without `scripts/package.json`, or when run outside the monorepo (a
+ * standalone module has no workspace to compare against — refresh only).
+ *
+ * `run` is injected so tests can stub the `bun install` side effect.
+ */
+export function refreshAndVerifyBundledDeps(
+  moduleDir: string,
+  run: (cmd: string, cwd: string) => void = (cmd, cwd) =>
+    void execSync(cmd, { cwd, stdio: 'pipe' }),
+): RefreshResult {
+  const scriptsDir = join(moduleDir, 'scripts');
+  if (!existsSync(join(scriptsDir, 'package.json'))) {
+    return { refreshed: false, mismatches: [] };
+  }
+
+  // Reconcile node_modules with the lockfile + package.json pins so the pack
+  // bundles a CURRENT closure, not whatever the operator last installed.
+  run('bun install', scriptsDir);
+
+  const workspaceRoot = findWorkspaceRoot(moduleDir);
+  if (!workspaceRoot) {
+    return { refreshed: true, mismatches: [] };
+  }
+
+  const workspace: Record<string, string> = {};
+  for (const name of CELILO_WORKSPACE_PACKAGES) {
+    const version = readVersion(join(workspaceRoot, 'packages', name, 'package.json'));
+    if (version) workspace[`@celilo/${name}`] = version;
+  }
+
+  const bundled: Record<string, string> = {};
+  for (const pkg of Object.keys(workspace)) {
+    const version = readVersion(join(scriptsDir, 'node_modules', pkg, 'package.json'));
+    if (version) bundled[pkg] = version;
+  }
+
+  return { refreshed: true, mismatches: findStaleBundledDeps(bundled, workspace) };
+}

package/src/services/module-validator/capability-versions.test.ts

@@ -31,7 +31,7 @@ describe('validateCapabilityVersions', () => {
     expect(errors).toHaveLength(1);
     expect(errors[0]).toContain('provides[public_web]');
     expect(errors[0]).toContain('1.0.0');
-    expect(errors[0]).toContain('3.0.0');
+    expect(errors[0]).toContain('3.1.0');
   });
 
   test('error when provides[X].version is newer than runtime', () => {

package/src/services/placement-reconcile.test.ts

@@ -0,0 +1,86 @@
+import { describe, expect, test } from 'bun:test';
+import type { DeployedSystem } from '@celilo/capabilities';
+import { formatPlacementLine, resolveOnePlacement } from './placement-reconcile';
+
+function sys(
+  overrides: Partial<DeployedSystem> & { infrastructure: DeployedSystem['infrastructure'] },
+): DeployedSystem {
+  return {
+    name: 'main',
+    hostname: 'caddy',
+    ipv4_address: '10.0.10.10',
+    zone: 'dmz',
+    ...overrides,
+  } as DeployedSystem;
+}
+
+const SVC = 'svc-1';
+const cs = (vmid?: number) =>
+  sys({
+    infrastructure: {
+      type: 'container_service',
+      serviceId: SVC,
+      ...(vmid != null ? { vmid } : {}),
+    },
+  });
+
+describe('resolveOnePlacement (ISS-0060 — node reconciled from Proxmox, not a cached DB value)', () => {
+  test('machine-pool system → machine (no Proxmox node)', () => {
+    const s = sys({ infrastructure: { type: 'machine', machineId: 'm1' } });
+    expect(resolveOnePlacement(s, new Map(), new Set(), new Set())).toEqual({ kind: 'machine' });
+  });
+
+  test('non-Proxmox container service → other', () => {
+    expect(resolveOnePlacement(cs(200), new Map(), new Set(), new Set([SVC]))).toEqual({
+      kind: 'other',
+    });
+  });
+
+  test('container_service without a vmid → uncreated', () => {
+    expect(resolveOnePlacement(cs(undefined), new Map(), new Set(), new Set())).toEqual({
+      kind: 'uncreated',
+    });
+  });
+
+  test('Proxmox unreachable for the service → unreachable (never a stale fallback)', () => {
+    expect(resolveOnePlacement(cs(200), new Map(), new Set([SVC]), new Set())).toEqual({
+      kind: 'unreachable',
+    });
+  });
+
+  test('vmid present in the cluster → node (the reconciled reality)', () => {
+    expect(resolveOnePlacement(cs(200), new Map([[200, 'node2']]), new Set(), new Set())).toEqual({
+      kind: 'node',
+      node: 'node2',
+    });
+  });
+
+  test('vmid absent from the cluster → absent', () => {
+    expect(resolveOnePlacement(cs(200), new Map([[999, 'node2']]), new Set(), new Set())).toEqual({
+      kind: 'absent',
+    });
+  });
+});
+
+describe('formatPlacementLine', () => {
+  const s = cs(200);
+
+  test('node resolution shows the real node + vmid', () => {
+    expect(formatPlacementLine(s, { kind: 'node', node: 'node2' })).toBe(
+      'caddy (vmid 200) → node2 (zone dmz)',
+    );
+  });
+
+  test('unreachable is explicit — no silent stale value', () => {
+    expect(formatPlacementLine(s, { kind: 'unreachable' })).toContain('Proxmox unreachable');
+  });
+
+  test('absent flags a vmid not in the cluster', () => {
+    expect(formatPlacementLine(s, { kind: 'absent' })).toContain('not found in Proxmox');
+  });
+
+  test('machine line omits vmid/node', () => {
+    const m = sys({ hostname: 'iot', zone: 'internal', infrastructure: { type: 'machine' } });
+    expect(formatPlacementLine(m, { kind: 'machine' })).toBe('iot — machine (zone internal)');
+  });
+});