@celilo/cli 1.2.0 → 1.3.0

package/src/ipam/auto-allocator.ts

@@ -10,6 +10,7 @@
 import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { ipAllocations } from '../db/schema';
+import { releaseIngressReservations } from './allocator';
 
 export interface IpamAllocation {
   moduleId: string;
@@ -258,6 +259,12 @@ export async function allocateForModule(
  * @returns True if allocation was removed, false if none existed
  */
 export async function deallocateForModule(moduleId: string, db: DbClient): Promise<boolean> {
+  // Ingress reservations are released FIRST and unconditionally (celilo#892).
+  // A module can hold one with no `ip_allocations` row at all — it deploys onto
+  // a machine rather than a celilo-provisioned container — so releasing it
+  // after the early return below would skip exactly the modules that leak.
+  await releaseIngressReservations(moduleId, db);
+
   // Check if allocation exists before deleting
   const existing = getAllocation(moduleId, db);
   if (!existing) {

package/src/services/aspect-reconcile.test.ts

@@ -0,0 +1,460 @@
+/**
+ * The INBOUND direction — aspects applied to systems that have just come into
+ * existence (celilo#902).
+ *
+ * The outbound direction (`maybeRunAspectForTrigger`, one provider across the
+ * whole fleet) is covered in `aspect-runner.test.ts`. Nothing there could have
+ * caught this bug: the fan-out enumerates the fleet once, at the moment the
+ * providing module deploys, and every assertion is about systems that already
+ * existed at that instant.
+ *
+ * The runner is injected throughout, so these exercise the GATING — which
+ * provider's aspect runs on which host, and why — without driving Ansible.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { closeDb, getDb } from '../db/client';
+import { runMigrations } from '../db/migrate';
+import { type ModuleState, modules } from '../db/schema';
+import type { BaseModuleAspect } from '../manifest/schema';
+import {
+  computeAspectScopeHash,
+  recordAspectApproval,
+  recordAspectConsent,
+} from './aspect-approvals';
+import {
+  type AspectRunResult,
+  planAspectFanOut,
+  reconcileAspectsForSystems,
+  verifyAspectCoverage,
+} from './aspect-runner';
+import type { AnsibleHostRecap } from './deploy-ansible';
+import { upsertDeployedSystem } from './deployed-systems';
+
+const dnsAspect: BaseModuleAspect = {
+  ansible_role: 'dns-client-config',
+  applicable_zones: ['dmz', 'app', 'secure', 'internal'],
+  // Deliberately ONLY on_install: the inbound reconcile must run anyway
+  // (design D2), and this is the manifest shape both shipping aspects carry.
+  triggers: ['on_install'],
+};
+
+/** Seed a provider module with an aspect, approved unless told otherwise. */
+function seedProvider(opts: {
+  moduleId: string;
+  aspect?: BaseModuleAspect;
+  state?: ModuleState;
+  approval?: 'approved' | 'denied' | 'none';
+}): void {
+  const db = getDb();
+  const aspect = opts.aspect ?? dnsAspect;
+  db.insert(modules)
+    .values({
+      id: opts.moduleId,
+      name: opts.moduleId,
+      version: '1.0.0',
+      state: opts.state ?? 'INSTALLED',
+      manifestData: {
+        id: opts.moduleId,
+        name: opts.moduleId,
+        version: '1.0.0',
+        celilo_contract: '1.0',
+        base_module_aspect: aspect,
+      },
+      sourcePath: `/tmp/${opts.moduleId}`,
+    })
+    .run();
+
+  const approval = opts.approval ?? 'approved';
+  if (approval === 'approved') {
+    recordAspectApproval({
+      moduleId: opts.moduleId,
+      version: '1.0.0',
+      scopeHash: computeAspectScopeHash(aspect),
+      approver: 'test',
+      db,
+    });
+  } else if (approval === 'denied') {
+    recordAspectConsent({
+      moduleId: opts.moduleId,
+      version: '1.0.0',
+      scopeHash: computeAspectScopeHash(aspect),
+      approver: 'test',
+      consented: false,
+      db,
+    });
+  }
+}
+
+/** A module with a deployed system, used as a fan-out TARGET. */
+function seedTargetSystem(opts: {
+  moduleId: string;
+  hostname: string;
+  zone: 'dmz' | 'app' | 'secure' | 'internal';
+  ip: string;
+}): void {
+  const db = getDb();
+  db.insert(modules)
+    .values({
+      id: opts.moduleId,
+      name: opts.moduleId,
+      version: '1.0.0',
+      state: 'INSTALLED',
+      manifestData: {
+        id: opts.moduleId,
+        name: opts.moduleId,
+        version: '1.0.0',
+        celilo_contract: '1.0',
+      },
+      sourcePath: `/tmp/${opts.moduleId}`,
+    })
+    .run();
+  upsertDeployedSystem(db, opts.moduleId, {
+    name: 'main',
+    hostname: opts.hostname,
+    ipv4Address: opts.ip,
+    zone: opts.zone,
+    infraType: 'container_service',
+  });
+}
+
+interface RunnerCall {
+  moduleId: string;
+  trigger: string;
+  onlyHostnames?: string[];
+}
+
+/** A stand-in runner that records what it was asked to do. */
+function fakeRunner(calls: RunnerCall[], success = true) {
+  return (async (args: {
+    moduleId: string;
+    options: { trigger: string; onlyHostnames?: string[] };
+  }): Promise<AspectRunResult> => {
+    calls.push({
+      moduleId: args.moduleId,
+      trigger: args.options.trigger,
+      onlyHostnames: args.options.onlyHostnames,
+    });
+    return {
+      success,
+      output: '',
+      error: success ? undefined : 'ansible said no',
+      plan: { targetSystems: [], skipped: [] },
+      recap: [],
+    };
+  }) as unknown as Parameters<typeof reconcileAspectsForSystems>[0]['runner'];
+}
+
+describe('reconcileAspectsForSystems', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-aspect-reconcile-test-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('applies an approved aspect to a system in a covered zone', async () => {
+    seedProvider({ moduleId: 'knot-unbound-internal' });
+    const calls: RunnerCall[] = [];
+
+    const result = await reconcileAspectsForSystems({
+      systems: [{ hostname: 'caddy-int', zone: 'dmz' }],
+      db: getDb(),
+      runner: fakeRunner(calls),
+    });
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0].moduleId).toBe('knot-unbound-internal');
+    expect(calls[0].onlyHostnames).toEqual(['caddy-int']);
+    expect(result.failures).toHaveLength(0);
+  });
+
+  it('runs an aspect that declares ONLY on_install — triggers do not gate this direction', async () => {
+    // Design D2, and the whole reason this fix does not need a manifest change.
+    // If this ever starts consulting `aspect.triggers`, every shipping aspect
+    // would need `on_new_system_in_zone` added — which changes its scope hash,
+    // invalidates the operator's approval, and raises a re-approval interview
+    // on the live fleet as a side effect of a bug fix.
+    seedProvider({ moduleId: 'knot-unbound-internal', aspect: dnsAspect });
+    expect(dnsAspect.triggers).toEqual(['on_install']);
+    const calls: RunnerCall[] = [];
+
+    await reconcileAspectsForSystems({
+      systems: [{ hostname: 'vpn', zone: 'app' }],
+      db: getDb(),
+      runner: fakeRunner(calls),
+    });
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0].trigger).toBe('on_new_system_in_zone');
+  });
+
+  it('does not apply an aspect to a system outside its applicable_zones', async () => {
+    seedProvider({
+      moduleId: 'knot-unbound-internal',
+      aspect: { ...dnsAspect, applicable_zones: ['dmz'] },
+    });
+    const calls: RunnerCall[] = [];
+
+    const result = await reconcileAspectsForSystems({
+      systems: [{ hostname: 'vpn', zone: 'app' }],
+      db: getDb(),
+      runner: fakeRunner(calls),
+    });
+
+    expect(calls).toHaveLength(0);
+    expect(result.outcomes[0].reason).toBe('no_covered_systems');
+  });
+
+  it('does not apply a DENIED aspect, and does not re-prompt', async () => {
+    seedProvider({ moduleId: 'knot-unbound-internal', approval: 'denied' });
+    const calls: RunnerCall[] = [];
+
+    const result = await reconcileAspectsForSystems({
+      systems: [{ hostname: 'caddy-int', zone: 'dmz' }],
+      db: getDb(),
+      runner: fakeRunner(calls),
+      requestConsent: async () => {
+        throw new Error('must not interview for an already-denied aspect');
+      },
+    });
+
+    expect(calls).toHaveLength(0);
+    expect(result.outcomes[0].reason).toBe('denied');
+  });
+
+  it('skips a PAUSED provider — the documented escape hatch for a wedged aspect', async () => {
+    // Design D4a. An inbound failure is fatal to the deploy, so an operator
+    // whose non-essential aspect is wedging every deploy pauses its provider,
+    // deploys, and unpauses. If this stopped working there would be no way out.
+    seedProvider({ moduleId: 'knot-unbound-internal', state: 'PAUSED' });
+    const calls: RunnerCall[] = [];
+
+    const result = await reconcileAspectsForSystems({
+      systems: [{ hostname: 'caddy-int', zone: 'dmz' }],
+      db: getDb(),
+      runner: fakeRunner(calls),
+    });
+
+    expect(calls).toHaveLength(0);
+    expect(result.outcomes[0].reason).toBe('paused');
+    expect(result.failures).toHaveLength(0);
+  });
+
+  it('skips the deploying module’s own aspect — on_install already fans it out', async () => {
+    seedProvider({ moduleId: 'knot-unbound-internal' });
+    const calls: RunnerCall[] = [];
+
+    await reconcileAspectsForSystems({
+      systems: [{ hostname: 'dns-int', zone: 'dmz' }],
+      db: getDb(),
+      excludeModuleIds: ['knot-unbound-internal'],
+      runner: fakeRunner(calls),
+    });
+
+    expect(calls).toHaveLength(0);
+  });
+
+  it('reports a failed aspect as a failure the caller can act on', async () => {
+    seedProvider({ moduleId: 'knot-unbound-internal' });
+    const calls: RunnerCall[] = [];
+
+    const result = await reconcileAspectsForSystems({
+      systems: [{ hostname: 'caddy-int', zone: 'dmz' }],
+      db: getDb(),
+      runner: fakeRunner(calls, false),
+    });
+
+    expect(result.failures).toHaveLength(1);
+    expect(result.failures[0].providerModuleId).toBe('knot-unbound-internal');
+    expect(result.failures[0].error).toBe('ansible said no');
+  });
+
+  it('does nothing when no systems were created', async () => {
+    seedProvider({ moduleId: 'knot-unbound-internal' });
+    const calls: RunnerCall[] = [];
+
+    const result = await reconcileAspectsForSystems({
+      systems: [],
+      db: getDb(),
+      runner: fakeRunner(calls),
+    });
+
+    expect(calls).toHaveLength(0);
+    expect(result.outcomes).toHaveLength(0);
+  });
+});
+
+describe('planAspectFanOut onlyHostnames', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-aspect-only-test-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('narrows the plan to the named hosts', async () => {
+    seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' });
+    seedTargetSystem({ moduleId: 'authentik', hostname: 'auth', zone: 'app', ip: '10.0.20.10' });
+
+    const all = await planAspectFanOut(dnsAspect);
+    expect(all.targetSystems.map((t) => t.hostname).sort()).toEqual(['auth', 'caddy']);
+
+    const narrowed = await planAspectFanOut(dnsAspect, { onlyHostnames: ['auth'] });
+    expect(narrowed.targetSystems.map((t) => t.hostname)).toEqual(['auth']);
+  });
+
+  it('NARROWS ONLY — a host outside applicable_zones is still not a target', async () => {
+    // Otherwise an inbound reconcile could apply an aspect to a system whose
+    // zone the operator never approved, which is the consent surface.
+    seedTargetSystem({ moduleId: 'signal', hostname: 'signal', zone: 'app', ip: '10.0.20.90' });
+
+    const dmzOnly: BaseModuleAspect = { ...dnsAspect, applicable_zones: ['dmz'] };
+    const plan = await planAspectFanOut(dmzOnly, { onlyHostnames: ['signal'] });
+
+    expect(plan.targetSystems).toHaveLength(0);
+  });
+});
+
+describe('verifyAspectCoverage', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-aspect-coverage-test-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  /** A runner that reports a fixed check-mode recap for the named host. */
+  function recapRunner(recap: AnsibleHostRecap[]) {
+    return (async (): Promise<AspectRunResult> => ({
+      success: true,
+      output: '',
+      plan: { targetSystems: [], skipped: [] },
+      recap,
+    })) as unknown as Parameters<typeof verifyAspectCoverage>[0]['runner'];
+  }
+
+  function seedFleet(): void {
+    seedProvider({ moduleId: 'knot-unbound-internal' });
+    seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' });
+  }
+
+  it('reports a host with no changes and no skips as applied', async () => {
+    seedFleet();
+    const findings = await verifyAspectCoverage({
+      db: getDb(),
+      runner: recapRunner([
+        { host: 'caddy', ok: 3, changed: 0, unreachable: 0, failed: 0, skipped: 0 },
+      ]),
+    });
+    expect(findings).toHaveLength(1);
+    expect(findings[0].state).toBe('applied');
+  });
+
+  it('reports a host the role would change as missing', async () => {
+    seedFleet();
+    const findings = await verifyAspectCoverage({
+      db: getDb(),
+      runner: recapRunner([
+        { host: 'caddy', ok: 3, changed: 1, unreachable: 0, failed: 0, skipped: 0 },
+      ]),
+    });
+    expect(findings[0].state).toBe('missing');
+    expect(findings[0].detail).toContain('celilo module deploy knot-unbound-internal');
+  });
+
+  it('reports SKIPPED tasks as unknown — never as applied', async () => {
+    // The unsafe direction, and the reason this is not a boolean. A role of
+    // command:/shell: tasks finishes a check run with changed=0 having never
+    // been applied; calling that "applied" is a confidently clean answer about
+    // an unconverged host — the same failure as the stored verdict this whole
+    // approach exists to avoid (celilo#902 design D6).
+    seedFleet();
+    const findings = await verifyAspectCoverage({
+      db: getDb(),
+      runner: recapRunner([
+        { host: 'caddy', ok: 1, changed: 0, unreachable: 0, failed: 0, skipped: 2 },
+      ]),
+    });
+    expect(findings[0].state).toBe('unknown');
+    expect(findings[0].state).not.toBe('applied');
+    expect(findings[0].detail).toContain('check mode cannot evaluate');
+  });
+
+  it('reports a host with NO recap line as unknown, not as applied', async () => {
+    // An empty recap means nothing was measured. Reading it as success is how a
+    // verification turns into the thing it was meant to replace.
+    seedFleet();
+    const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([]) });
+    expect(findings[0].state).toBe('unknown');
+    expect(findings[0].detail).toContain('no recap');
+  });
+
+  it('reports an unreachable host distinctly from a measured one', async () => {
+    seedFleet();
+    const findings = await verifyAspectCoverage({
+      db: getDb(),
+      runner: recapRunner([
+        { host: 'caddy', ok: 0, changed: 0, unreachable: 1, failed: 0, skipped: 0 },
+      ]),
+    });
+    expect(findings[0].state).toBe('unreachable');
+  });
+
+  it('reports a PAUSED provider’s entitled systems, so pausing cannot go unnoticed', async () => {
+    seedProvider({ moduleId: 'knot-unbound-internal', state: 'PAUSED' });
+    seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' });
+
+    const findings = await verifyAspectCoverage({
+      db: getDb(),
+      runner: recapRunner([]),
+    });
+    expect(findings).toHaveLength(1);
+    expect(findings[0].state).toBe('unknown');
+    expect(findings[0].detail).toContain('PAUSED');
+  });
+
+  it('does not verify an unapproved aspect, and never raises an interview', async () => {
+    seedProvider({ moduleId: 'knot-unbound-internal', approval: 'none' });
+    seedTargetSystem({ moduleId: 'caddy', hostname: 'caddy', zone: 'dmz', ip: '10.0.10.10' });
+
+    const findings = await verifyAspectCoverage({ db: getDb(), runner: recapRunner([]) });
+    expect(findings).toHaveLength(0);
+  });
+});

package/src/services/aspect-runner.test.ts

@@ -439,6 +439,7 @@ describe('aspect-runner', () => {
           output: success ? 'fake-output' : '',
           error: success ? undefined : 'fake-error',
           plan: { targetSystems: [], skipped: [] },
+          recap: [],
         };
       };
       return { fake, calls };