@celilo/cli 0.5.0-alpha.3 → 0.5.0-alpha.5

package/src/services/module-subscriptions.test.ts

@@ -3,11 +3,14 @@ import { mkdtempSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { defineEvents, openBus } from '@celilo/event-bus';
+import { closeDb, getDb } from '../db/client';
 import { ModuleSubscriptionSchema } from '../manifest/schema';
 import type { ModuleManifest } from '../manifest/schema';
+import { setupTestDatabase as migrateDbFile } from '../test-utils/setup-test-db';
 import {
   registerModuleSubscriptions,
   resolveSubscription,
+  resyncAllSubscriptions,
   unregisterModuleSubscriptions,
 } from './module-subscriptions';
 
@@ -253,3 +256,88 @@ describe('register / unregister roundtrip', () => {
     }
   });
 });
+
+describe('resyncAllSubscriptions (ISS-0088)', () => {
+  let dir: string;
+  let busPath: string;
+
+  beforeEach(async () => {
+    closeDb();
+    dir = mkdtempSync(join(tmpdir(), 'resync-test-'));
+    busPath = join(dir, 'events.db');
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    process.env.EVENT_BUS_DB = busPath;
+    // Migrate the celilo.db file getDb() will open (resyncAllSubscriptions reads it).
+    const setupDb = await migrateDbFile(join(dir, 'celilo.db'));
+    setupDb.$client.close();
+  });
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function seedModule(
+    id: string,
+    state: string,
+    subs: Array<{ name: string; pattern: string; handler: string }>,
+  ): void {
+    const manifest = JSON.stringify({
+      celilo_contract: '1.0',
+      id,
+      name: id,
+      version: '1.0.0',
+      requires: { capabilities: [] },
+      provides: { capabilities: [] },
+      variables: { owns: [], imports: [] },
+      subscriptions: subs,
+    });
+    getDb().$client.run(
+      'INSERT INTO modules (id, name, version, source_path, manifest_data, state) VALUES (?, ?, ?, ?, ?, ?)',
+      [id, id, '1.0.0', '/p', manifest, state],
+    );
+  }
+
+  function subscriberNames(): string[] {
+    const bus = openBus({ dbPath: busPath, events: defineEvents({}) });
+    try {
+      return bus.db
+        .query<{ name: string }, []>('SELECT name FROM subscribers ORDER BY name')
+        .all()
+        .map((r) => r.name);
+    } finally {
+      bus.close();
+    }
+  }
+
+  it('registers subs for DEPLOYED modules only, skipping imported/undeployed', () => {
+    seedModule('caddy', 'VERIFIED', [
+      { name: 'reconcile', pattern: 'public_web.routes_changed', handler: 'echo' },
+    ]);
+    seedModule('technitium', 'INSTALLED', [
+      { name: 'dns', pattern: 'system.created.*', handler: 'echo' },
+    ]);
+    seedModule('forgejo', 'IMPORTED', [{ name: 'x', pattern: 'y', handler: 'echo' }]);
+    seedModule('greenwave', 'VERIFIED', []); // deployed but declares no subs
+
+    const result = resyncAllSubscriptions();
+
+    expect(result.modules).toBe(2); // caddy + technitium (forgejo not deployed, greenwave has none)
+    expect(result.registered).toBe(2);
+    expect(result.failures).toEqual([]);
+    // forgejo.x absent — it was IMPORTED, not deployed.
+    expect(subscriberNames()).toEqual(['caddy.reconcile', 'technitium.dns']);
+  });
+
+  it('is idempotent — a second resync does not duplicate rows', () => {
+    seedModule('caddy', 'VERIFIED', [{ name: 'reconcile', pattern: 'a', handler: 'echo' }]);
+    resyncAllSubscriptions();
+    resyncAllSubscriptions();
+    expect(subscriberNames()).toEqual(['caddy.reconcile']);
+  });
+});

package/src/services/module-subscriptions.ts

@@ -14,8 +14,12 @@
  * colliding.
  */
 
+import { join } from 'node:path';
 import { defineEvents, openBus } from '@celilo/event-bus';
-import { getEventBusPath } from '../config/paths';
+import { inArray } from 'drizzle-orm';
+import { getEventBusPath, getModuleStoragePath } from '../config/paths';
+import { getDb } from '../db/client';
+import { modules } from '../db/schema';
 import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
 
 /**
@@ -126,6 +130,51 @@ export function unregisterModuleSubscriptions(moduleId: string): {
   }
 }
 
+/**
+ * Rebuild the event-bus `subscribers` table from every deployed module's
+ * manifest `subscriptions:`. The reactive layer (who-reacts-to-what) is
+ * registered at module DEPLOY time and lives in the bus (events.db), which a
+ * restore/migration starts EMPTY — so after a cutover, event-driven reconciles
+ * (caddy public_web, dns register, …) are dead until every module redeploys
+ * (ISS-0088). This reconstructs them from durable celilo.db state instead.
+ *
+ * Idempotent (registerModuleSubscriptions upserts by subscriber name). Per-module
+ * failures are collected, not thrown, so one bad manifest doesn't abort the rest.
+ */
+export function resyncAllSubscriptions(): {
+  modules: number;
+  registered: number;
+  failures: Array<{ moduleId: string; error: string }>;
+} {
+  const db = getDb();
+  const deployed = db
+    .select()
+    .from(modules)
+    .where(inArray(modules.state, ['INSTALLED', 'VERIFIED']))
+    .all();
+
+  let modulesWithSubs = 0;
+  let registered = 0;
+  const failures: Array<{ moduleId: string; error: string }> = [];
+
+  for (const mod of deployed) {
+    const manifest = mod.manifestData as unknown as ModuleManifest;
+    if (!manifest?.subscriptions?.length) continue;
+    // Code always lives at ${getModuleStoragePath()}/<id> by construction
+    // (import.ts) — the same path module-upgrade re-registers from (ISS-0091).
+    const modulePath = join(getModuleStoragePath(), mod.id);
+    try {
+      const result = registerModuleSubscriptions(manifest, modulePath);
+      registered += result.registered;
+      modulesWithSubs += 1;
+    } catch (err) {
+      failures.push({ moduleId: mod.id, error: err instanceof Error ? err.message : String(err) });
+    }
+  }
+
+  return { modules: modulesWithSubs, registered, failures };
+}
+
 function scopedName(moduleId: string, subName: string): string {
   return `${moduleId}.${subName}`;
 }

package/src/services/module-validator/bundled-deps.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, test } from 'bun:test';
+import { findStaleBundledDeps, refreshAndVerifyBundledDeps } from './bundled-deps';
+
+describe('findStaleBundledDeps (ISS-0104)', () => {
+  test('flags a bundled @celilo dep older than the workspace', () => {
+    const mismatches = findStaleBundledDeps(
+      { '@celilo/capabilities': '0.1.8' },
+      { '@celilo/capabilities': '0.4.1' },
+    );
+    expect(mismatches).toEqual([
+      { pkg: '@celilo/capabilities', bundled: '0.1.8', workspace: '0.4.1' },
+    ]);
+  });
+
+  test('no mismatch when bundled matches workspace', () => {
+    expect(
+      findStaleBundledDeps(
+        { '@celilo/capabilities': '0.4.1', '@celilo/event-bus': '0.1.6' },
+        { '@celilo/capabilities': '0.4.1', '@celilo/event-bus': '0.1.6' },
+      ),
+    ).toEqual([]);
+  });
+
+  test('ignores workspace packages the module does not bundle', () => {
+    // Module bundles only capabilities; cli-display is a workspace package but
+    // not in this module's closure — must not be flagged.
+    expect(
+      findStaleBundledDeps(
+        { '@celilo/capabilities': '0.4.1' },
+        { '@celilo/capabilities': '0.4.1', '@celilo/cli-display': '0.1.9' },
+      ),
+    ).toEqual([]);
+  });
+
+  test('flags each stale dep independently', () => {
+    const mismatches = findStaleBundledDeps(
+      { '@celilo/capabilities': '0.1.8', '@celilo/event-bus': '0.1.6' },
+      { '@celilo/capabilities': '0.4.1', '@celilo/event-bus': '0.1.6' },
+    );
+    expect(mismatches).toEqual([
+      { pkg: '@celilo/capabilities', bundled: '0.1.8', workspace: '0.4.1' },
+    ]);
+  });
+});
+
+describe('refreshAndVerifyBundledDeps (ISS-0104)', () => {
+  test('no-op for a module without scripts/package.json', () => {
+    let installs = 0;
+    const result = refreshAndVerifyBundledDeps('/tmp/does-not-exist-module-xyz', () => {
+      installs++;
+    });
+    expect(result).toEqual({ refreshed: false, mismatches: [] });
+    expect(installs).toBe(0);
+  });
+});

package/src/services/module-validator/bundled-deps.ts

@@ -0,0 +1,115 @@
+/**
+ * ISS-0104: a module's hooks resolve `@celilo/*` from the module's OWN
+ * `scripts/node_modules` (nearest-wins), and those are gitignored local
+ * installs that go stale. The publish bundles them as-is, so a deployed module
+ * can silently run capability code months older than what was just published
+ * (e.g. caddy shipped capabilities@0.1.8 while the workspace was at 0.4.1, so
+ * ISS-0102 never reached the rendered Caddyfile).
+ *
+ * Before packing, the publish refreshes a module's `scripts/` deps and verifies
+ * the bundled `@celilo/*` versions match the workspace — the versions being
+ * co-published. A mismatch fails the publish (unless --allow-stale).
+ */
+
+import { execSync } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+/** The @celilo workspace packages a module may bundle (by published name). */
+const CELILO_WORKSPACE_PACKAGES = ['capabilities', 'cli-display', 'event-bus'] as const;
+
+export interface DepMismatch {
+  pkg: string;
+  bundled: string;
+  workspace: string;
+}
+
+/**
+ * Pure: which bundled `@celilo/*` versions differ from the workspace. Only deps
+ * the module actually bundles AND the workspace owns are compared — a module
+ * that doesn't bundle a given package, or a package the workspace doesn't own,
+ * is ignored.
+ */
+export function findStaleBundledDeps(
+  bundled: Record<string, string>,
+  workspace: Record<string, string>,
+): DepMismatch[] {
+  const mismatches: DepMismatch[] = [];
+  for (const [pkg, wsVersion] of Object.entries(workspace)) {
+    const bundledVersion = bundled[pkg];
+    if (bundledVersion && bundledVersion !== wsVersion) {
+      mismatches.push({ pkg, bundled: bundledVersion, workspace: wsVersion });
+    }
+  }
+  return mismatches;
+}
+
+function readVersion(packageJsonPath: string): string | undefined {
+  try {
+    const parsed = JSON.parse(readFileSync(packageJsonPath, 'utf-8')) as { version?: string };
+    return parsed.version;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Find the monorepo root (the dir with `packages/capabilities`) by walking up. */
+function findWorkspaceRoot(start: string): string | undefined {
+  let dir = start;
+  for (let depth = 0; depth < 8; depth++) {
+    if (existsSync(join(dir, 'packages', 'capabilities', 'package.json'))) return dir;
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return undefined;
+}
+
+export interface RefreshResult {
+  /** Whether the module had a scripts/ package to refresh. */
+  refreshed: boolean;
+  /** Bundled @celilo/* that don't match the workspace (empty = clean). */
+  mismatches: DepMismatch[];
+}
+
+/**
+ * Refresh a module's `scripts/` deps (so the bundled hook runtime is current)
+ * and verify the bundled `@celilo/*` versions match the workspace. No-op for a
+ * module without `scripts/package.json`, or when run outside the monorepo (a
+ * standalone module has no workspace to compare against — refresh only).
+ *
+ * `run` is injected so tests can stub the `bun install` side effect.
+ */
+export function refreshAndVerifyBundledDeps(
+  moduleDir: string,
+  run: (cmd: string, cwd: string) => void = (cmd, cwd) =>
+    void execSync(cmd, { cwd, stdio: 'pipe' }),
+): RefreshResult {
+  const scriptsDir = join(moduleDir, 'scripts');
+  if (!existsSync(join(scriptsDir, 'package.json'))) {
+    return { refreshed: false, mismatches: [] };
+  }
+
+  // Reconcile node_modules with the lockfile + package.json pins so the pack
+  // bundles a CURRENT closure, not whatever the operator last installed.
+  run('bun install', scriptsDir);
+
+  const workspaceRoot = findWorkspaceRoot(moduleDir);
+  if (!workspaceRoot) {
+    return { refreshed: true, mismatches: [] };
+  }
+
+  const workspace: Record<string, string> = {};
+  for (const name of CELILO_WORKSPACE_PACKAGES) {
+    const version = readVersion(join(workspaceRoot, 'packages', name, 'package.json'));
+    if (version) workspace[`@celilo/${name}`] = version;
+  }
+
+  const bundled: Record<string, string> = {};
+  for (const pkg of Object.keys(workspace)) {
+    const version = readVersion(join(scriptsDir, 'node_modules', pkg, 'package.json'));
+    if (version) bundled[pkg] = version;
+  }
+
+  return { refreshed: true, mismatches: findStaleBundledDeps(bundled, workspace) };
+}

package/src/templates/generator.ts

@@ -19,6 +19,10 @@ import {
 import { getSingularSystemSpec } from '../manifest/schema';
 import type { AnsibleCollection, ModuleManifest } from '../manifest/schema';
 import { validateZoneRequirements } from '../manifest/validate';
+import {
+  describeCapabilityProblem,
+  findBrokenCapabilityDerivations,
+} from '../services/fleet-checks';
 import { selectInfrastructure } from '../services/infrastructure-selector';
 import { upsertModuleConfig } from '../services/module-config';
 import type { InfrastructureSelection } from '../types/infrastructure';
@@ -780,6 +784,23 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
     };
   }
 
+  // Policy: fail loud at the source on a broken capability chain. A
+  // required `source: capability` var whose chain doesn't resolve is
+  // silently dropped during derivation, then surfaces downstream as a
+  // cryptic `$self:<x> not found` in some template. Assert it here against
+  // the resolved capabilities map so generate names the broken link and
+  // the provider to redeploy (ISS-0115; the data-plane sibling of ISS-0088).
+  const capProblems = findBrokenCapabilityDerivations(moduleId, manifest, context.capabilities);
+  if (capProblems.length > 0) {
+    return {
+      success: false,
+      error: `Cannot generate '${moduleId}': ${capProblems.length} capability-derived variable(s) won't resolve:\n${capProblems
+        .map((p) => `  - ${describeCapabilityProblem(p)}`)
+        .join('\n')}`,
+      details: capProblems,
+    };
+  }
+
   // Execution: Store derived variables in module_configs
   // Variables with derive_from are resolved in the context but not stored in module_configs.
   // Store them so hooks and host_vars can access them.