@celilo/cli 1.7.0 → 1.9.0

package/src/services/capability-table-rows.test.ts

@@ -0,0 +1,191 @@
+/**
+ * The ONE consumer-removal rule, driven by the declarations
+ * (openspec/changes/capability-owned-tables task 2.4).
+ *
+ * Before this, "the row dies with its consumer" had five implementations for one
+ * rule, and two of them were functions core imported BY NAME from
+ * `consumer-cleanup.ts` — the file whose whole purpose is deleting exactly that
+ * kind of per-capability special-casing.
+ *
+ * The property worth protecting is not "port forwards get deleted". It is that
+ * core clears EVERY declared table without naming any of them, so declaring a
+ * new one is sufficient to have it cleaned up.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { allDeclaredTables } from '@celilo/capabilities';
+import type { DbClient } from '../db/client';
+import { dnsInternalRecords, modules, portForwards, trustedSources, webRoutes } from '../db/schema';
+import { setupTestDatabaseAt } from '../test-utils/database';
+import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
+
+const FW = '192.168.0.254';
+
+describe('planClaimedRowDeletion', () => {
+  /**
+   * The anti-regression that matters. If someone declares a table and this plan
+   * does not grow, their rows outlive the consumer silently — which is the whole
+   * failure the declaration exists to make impossible.
+   */
+  it('covers every declared table, so declaring one is enough to have it cleared', () => {
+    const planned = new Set(planClaimedRowDeletion().map((t) => t.table));
+    const declared = allDeclaredTables().map((d) => d.declaration.table);
+
+    expect(declared.length).toBeGreaterThan(0);
+    for (const table of declared) expect(planned.has(table)).toBe(true);
+  });
+
+  it("carries each table's OWN claim column, which is spelled three ways", () => {
+    const byTable = new Map(planClaimedRowDeletion().map((t) => [t.table, t.claimColumn]));
+
+    expect(byTable.get('web_routes')).toBe('module_id');
+    expect(byTable.get('port_forwards')).toBe('registered_by');
+    expect(byTable.get('trusted_sources')).toBe('registered_by');
+    expect(byTable.get('dns_internal_records')).toBe('consumer_module_id');
+  });
+});
+
+describe('deleteClaimedRows', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'claimed-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabaseAt(dbPath);
+
+    for (const id of ['caddy', 'forgejo', 'technitium']) {
+      db.insert(modules)
+        .values({
+          id,
+          name: id,
+          version: '1.0.0',
+          state: 'VERIFIED',
+          sourcePath: `/tmp/${id}`,
+          manifestData: {},
+        })
+        .run();
+    }
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  function seed(consumer: string, hostname: string, port: number, subnet: string) {
+    db.insert(webRoutes)
+      .values({
+        slug: `${consumer}-slug`,
+        moduleId: consumer,
+        type: 'reverse_proxy',
+        path: `/${consumer}`,
+        hostname,
+      })
+      .run();
+    db.insert(portForwards)
+      .values({
+        firewallIp: FW,
+        internalIp: '10.0.20.5',
+        port,
+        protocol: 'TCP',
+        registeredBy: consumer,
+      })
+      .run();
+    db.insert(trustedSources).values({ firewallIp: FW, subnet, registeredBy: consumer }).run();
+    db.insert(dnsInternalRecords)
+      .values({
+        providerModuleId: 'technitium',
+        consumerModuleId: consumer,
+        host: hostname,
+        ip: '10.0.20.5',
+      })
+      .run();
+  }
+
+  it('clears the departing consumer from all four declared tables at once', () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+
+    const cleared = deleteClaimedRows(db, 'caddy');
+
+    expect(cleared.map((c) => c.table).sort()).toEqual([
+      'dns_internal_records',
+      'port_forwards',
+      'trusted_sources',
+      'web_routes',
+    ]);
+    expect(db.select().from(webRoutes).all()).toHaveLength(0);
+    expect(db.select().from(portForwards).all()).toHaveLength(0);
+    expect(db.select().from(trustedSources).all()).toHaveLength(0);
+    expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
+  });
+
+  /**
+   * The refcount case, the bug most likely to ship silently. Two consumers each
+   * hold their own row, and one leaving must not withdraw what the other still
+   * needs.
+   */
+  it("leaves another consumer's rows untouched", () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+    seed('forgejo', 'b.example.org', 2222, '10.2.0.0/24');
+
+    deleteClaimedRows(db, 'caddy');
+
+    expect(
+      db
+        .select()
+        .from(webRoutes)
+        .all()
+        .map((r) => r.moduleId),
+    ).toEqual(['forgejo']);
+    expect(
+      db
+        .select()
+        .from(portForwards)
+        .all()
+        .map((r) => r.registeredBy),
+    ).toEqual(['forgejo']);
+    expect(
+      db
+        .select()
+        .from(trustedSources)
+        .all()
+        .map((r) => r.registeredBy),
+    ).toEqual(['forgejo']);
+    expect(
+      db
+        .select()
+        .from(dnsInternalRecords)
+        .all()
+        .map((r) => r.consumerModuleId),
+    ).toEqual(['forgejo']);
+  });
+
+  /**
+   * `dns_internal_records` is claimed by its CONSUMER, never its provider. The
+   * declaration cannot express anything else, which is what makes celilo#1010
+   * unrepresentable — but the runtime rule has to agree, or the declaration is
+   * decoration.
+   */
+  it('does not treat the PROVIDER as the claimant of a dns_internal record', () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+
+    deleteClaimedRows(db, 'technitium');
+
+    expect(db.select().from(dnsInternalRecords).all()).toHaveLength(1);
+  });
+
+  it('is a no-op for a module that claimed nothing', () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+
+    deleteClaimedRows(db, 'forgejo');
+
+    expect(db.select().from(webRoutes).all()).toHaveLength(1);
+    expect(db.select().from(portForwards).all()).toHaveLength(1);
+  });
+});

package/src/services/capability-table-rows.ts

@@ -0,0 +1,103 @@
+/**
+ * Acting on every capability-declared table without naming any of them
+ * (openspec/changes/capability-owned-tables, design D3).
+ *
+ * "This row dies with its consumer" had five implementations for one rule: an FK
+ * cascade for `web_routes`, two functions core imported BY NAME from
+ * `consumer-cleanup.ts` for `port_forwards` and `trusted_sources`, a second
+ * table pruned on read for `dns_registrations`, a cascade on both provider and
+ * consumer for `dns_internal_records`, and a hook rewriting a JSON blob for the
+ * rest. `consumer-cleanup.ts` exists specifically to delete that kind of
+ * per-capability special-casing. It succeeded for the hook dispatch and then
+ * grew two hardcoded imports for the rows, because the rows had no generic rule.
+ *
+ * This is the generic rule. The declaration names the claim column, so core no
+ * longer has to know that three tables spell it `module_id`, `registered_by` and
+ * `consumer_module_id`.
+ */
+
+import { CAPABILITY_DECLARED_TABLES, allDeclaredTables } from '@celilo/capabilities';
+import { type Column, eq, is } from 'drizzle-orm';
+import { SQLiteTable, getTableConfig } from 'drizzle-orm/sqlite-core';
+import type { DbClient } from '../db/client';
+import * as dbSchema from '../db/schema';
+
+export interface ClaimedRowTarget {
+  capability: string;
+  /** Physical table name. */
+  table: string;
+  /** The column holding the consumer's module id. */
+  claimColumn: string;
+}
+
+/**
+ * Physical table name → the drizzle table object, for the tables celilo can
+ * actually act on. Built by walking the schema rather than by a hand-written
+ * map, so a table cannot be declared and then silently unreachable.
+ */
+function schemaTables(): Map<string, SQLiteTable> {
+  const out = new Map<string, SQLiteTable>();
+  for (const value of Object.values(dbSchema)) {
+    if (!is(value, SQLiteTable)) continue;
+    out.set(getTableConfig(value).name, value);
+  }
+  return out;
+}
+
+/**
+ * What a consumer's removal must clear. Pure — the list is a property of the
+ * declarations, so it is worth being able to assert without a database.
+ *
+ * A declaration naming a table that does not exist is skipped here rather than
+ * throwing. `policy/capability-shape-drift.test.ts` Scan A is what fails on
+ * that, loudly and at build time; a removal is the wrong moment to discover it,
+ * and refusing to clear the OTHER tables because one declaration is wrong would
+ * strand real rows.
+ */
+export function planClaimedRowDeletion(): ClaimedRowTarget[] {
+  const tables = schemaTables();
+  return allDeclaredTables()
+    .filter(({ declaration }) => tables.has(declaration.table))
+    .map(({ capability, declaration }) => ({
+      capability,
+      table: declaration.table,
+      claimColumn: declaration.claim.column,
+    }))
+    .sort((a, b) => a.table.localeCompare(b.table));
+}
+
+/**
+ * Delete every declared row this consumer claimed.
+ *
+ * Called AFTER every provider has been told the consumer is going (its D4): a
+ * provider's `on_consumer_removed` is a full converge, and converging from a set
+ * the rows had already been removed from would withdraw more than the departing
+ * consumer's share.
+ *
+ * `web_routes` and `dns_internal_records` also carry an FK cascade, so their
+ * rows would go anyway once the `modules` row is deleted. Doing it here as well
+ * is deliberate and not merely harmless: it makes the ORDER explicit rather than
+ * a consequence of when some other statement happens to run, and it is the same
+ * point in the sequence at which `port_forwards` and `trusted_sources` were
+ * already being cleared by hand.
+ */
+export function deleteClaimedRows(db: DbClient, consumer: string): ClaimedRowTarget[] {
+  const tables = schemaTables();
+  const cleared: ClaimedRowTarget[] = [];
+  for (const target of planClaimedRowDeletion()) {
+    const table = tables.get(target.table);
+    if (!table) continue;
+    const column = getTableConfig(table).columns.find((c) => c.name === target.claimColumn);
+    if (!column) continue;
+    db.delete(table)
+      .where(eq(column as Column, consumer))
+      .run();
+    cleared.push(target);
+  }
+  return cleared;
+}
+
+/** Capability names that declare at least one table — for logs and reports. */
+export function capabilitiesWithDeclaredTables(): string[] {
+  return Object.keys(CAPABILITY_DECLARED_TABLES).sort();
+}

package/src/services/consumer-cleanup.test.ts

@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import type { DbClient } from '../db/client';
 import type { ModuleManifest } from '../manifest/schema';
-import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { setupTestDatabaseAt } from '../test-utils/database';
 import {
   type CleanupTarget,
   type ProviderRow,
@@ -167,6 +167,43 @@ describe('planConsumerCleanup', () => {
   });
 });
 
+describe('planConsumerCleanup stays permissive after celilo#1072', () => {
+  it('notifies a provider the consumer has no recorded binding to', () => {
+    // celilo#1072 added `capability_bindings`, which records the provider a
+    // consumer actually CALLED. That set is precise and it is deliberately not
+    // this one. Cleanup must reach every provider that MIGHT hold minted state,
+    // including one a hook resolved, minted against, and never called again —
+    // and including one whose binding row was never written because the state
+    // was minted by an older celilo. Narrowing this to the recorded set would
+    // leave a site block in caddy or a DNAT rule in a ruleset with nothing left
+    // in the registry to reconcile it away, which is the exact failure
+    // `consumer-cleanup.ts` exists to prevent.
+    //
+    // The inputs here are manifest + provider rows and NOTHING ELSE. That is
+    // the assertion: no binding table is consulted, so no binding table can
+    // shrink the result.
+    const plan = planConsumerCleanup(
+      'tango-nexus',
+      manifest({ optional: ['external_web', 'public_web', 'source_forge', 'registry_publish'] }),
+      provides(
+        ['cpanel-host', 'external_web'],
+        ['caddy', 'public_web'],
+        ['forgejo', 'source_forge'],
+        ['celilo-registry', 'registry_publish'],
+      ),
+      deployed('cpanel-host', 'caddy', 'forgejo', 'celilo-registry'),
+    );
+
+    expect(plan.map((t) => t.providerId)).toEqual([
+      'caddy',
+      'celilo-registry',
+      'cpanel-host',
+      'forgejo',
+    ]);
+    expect(plan.every((t) => t.skip === undefined)).toBe(true);
+  });
+});
+
 describe('runConsumerCleanup — the paused skip is reported, not silent', () => {
   let dir: string;
   let db: DbClient;
@@ -175,7 +212,7 @@ describe('runConsumerCleanup — the paused skip is reported, not silent', () =>
     dir = mkdtempSync(join(tmpdir(), 'cc-'));
     const dbPath = join(dir, 'celilo.db');
     process.env.CELILO_DB_PATH = dbPath;
-    db = await setupTestDatabase(dbPath);
+    db = await setupTestDatabaseAt(dbPath);
   });
   afterEach(() => {
     db.$client.close();
@@ -244,7 +281,7 @@ describe('runConsumerCleanup — dispatch', () => {
     dir = mkdtempSync(join(tmpdir(), 'ccd-'));
     const dbPath = join(dir, 'celilo.db');
     process.env.CELILO_DB_PATH = dbPath;
-    db = await setupTestDatabase(dbPath);
+    db = await setupTestDatabaseAt(dbPath);
   });
   afterEach(() => {
     db.$client.close();

package/src/services/consumer-cleanup.ts

@@ -23,8 +23,7 @@ import { capabilities, modules } from '../db/schema';
 import { type RunNamedHookResult, runNamedHook } from '../hooks/run-named-hook';
 import type { HookLogger } from '../hooks/types';
 import type { ModuleManifest } from '../manifest/schema';
-import { deletePortForwardsForModule } from './port-forwards';
-import { deleteTrustedSourcesForModule } from './trusted-sources';
+import { deleteClaimedRows } from './capability-table-rows';
 
 /**
  * States in which a module has never resolved a capability and therefore holds
@@ -213,11 +212,18 @@ export async function runConsumerCleanup(
     );
   }
 
-  // The rows, once every provider has converged without them (D4). `web_routes`
-  // dies with the `modules` row via its FK cascade; these two carry a plain
-  // `registered_by` text column and do not.
-  deletePortForwardsForModule(db, consumer);
-  deleteTrustedSourcesForModule(db, consumer);
+  // The rows, once every provider has converged without them (D4).
+  //
+  // Driven by the capability declarations rather than by name. This used to be
+  // two hardcoded imports, `deletePortForwardsForModule` and
+  // `deleteTrustedSourcesForModule`, which is the shape of the whole problem
+  // openspec/changes/capability-owned-tables addresses: this file exists to
+  // delete per-capability special-casing, it succeeded for the hook dispatch,
+  // and then grew two capability-named calls for the rows because the rows had
+  // no generic rule. Now they have one, and core no longer has to know that the
+  // claim column is spelled `module_id` on one table, `registered_by` on two and
+  // `consumer_module_id` on a fourth.
+  deleteClaimedRows(db, consumer);
 
   return result;
 }

package/src/services/dns-internal-records.test.ts

@@ -4,9 +4,10 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import type { DnsRecordRequest } from '@celilo/capabilities';
 import type { ViewOverride } from '@celilo/capabilities';
+import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { modules } from '../db/schema';
-import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { dnsInternalRecords, modules } from '../db/schema';
+import { setupTestDatabaseAt } from '../test-utils/database';
 import {
   listDnsInternalRecords,
   listViewOverrides,
@@ -24,7 +25,7 @@ describe('dns-internal-records ledger', () => {
     dir = mkdtempSync(join(tmpdir(), 'dns-int-'));
     dbPath = join(dir, 'celilo.db');
     process.env.CELILO_DB_PATH = dbPath;
-    db = await setupTestDatabase(dbPath);
+    db = await setupTestDatabaseAt(dbPath);
     // FK targets: provider + consumer modules.
     for (const id of ['technitium', 'forgejo']) {
       db.insert(modules)
@@ -197,3 +198,73 @@ describe('dns-internal-records ledger', () => {
     });
   });
 });
+
+/**
+ * celilo#1010 — swapping the internal DNS provider must not delete the ledger.
+ *
+ * `dns_internal_records` used to cascade on `provider_module_id` as well as on
+ * the consumer, so removing `technitium` to install `knot-unbound-internal` took
+ * the fleet's entire internal DNS ledger with it, `zone_routable_ip` view
+ * overrides included — the durable desired state the resolver's split-horizon
+ * config is reconciled from. `web_routes` cascades on its consumer only, and the
+ * two tables' docblocks claimed to be siblings, so the divergence read as intent
+ * and was not.
+ *
+ * The claim on a capability-owned table is the CONSUMER
+ * (openspec/changes/capability-owned-tables D3/D8). Migration 0027 makes the
+ * table agree with the declaration, which cannot express anything else.
+ */
+describe('provider cascade (celilo#1010)', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'dns-cascade-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabaseAt(dbPath);
+    for (const id of ['technitium', 'knot-unbound-internal', 'caddy']) {
+      db.insert(modules)
+        .values({
+          id,
+          name: id,
+          version: '1.0.0',
+          state: 'VERIFIED',
+          sourcePath: `/tmp/${id}`,
+          manifestData: {},
+        })
+        .run();
+    }
+    db.insert(dnsInternalRecords)
+      .values({
+        providerModuleId: 'technitium',
+        consumerModuleId: 'caddy',
+        host: 'auth.example.org',
+        ip: '192.168.0.253',
+        zoneRoutableIp: '10.0.10.14',
+      })
+      .run();
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('keeps the ledger when the PROVIDER is removed, view overrides included', () => {
+    db.delete(modules).where(eq(modules.id, 'technitium')).run();
+
+    const left = db.select().from(dnsInternalRecords).all();
+    expect(left).toHaveLength(1);
+    // The override is the part whose loss is silent: the resolver keeps
+    // answering, just with the wrong address for in-zone clients.
+    expect(left[0]?.zoneRoutableIp).toBe('10.0.10.14');
+  });
+
+  it('still dies with its CONSUMER, which is the rule that did not change', () => {
+    db.delete(modules).where(eq(modules.id, 'caddy')).run();
+
+    expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
+  });
+});

package/src/services/fleet-checks.test.ts

@@ -13,7 +13,7 @@ import {
   systemConfig,
 } from '../db/schema';
 import type { ModuleManifest } from '../manifest/schema';
-import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { setupTestDatabaseAt } from '../test-utils/database';
 import { ensureInboundSubscriber, ensureSweepSubscriber } from './alerting/monitors';
 import { ensureBackupSweepSubscriber } from './backup-sweep';
 import { getDaemonUnitPath } from './events-daemon';
@@ -263,7 +263,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
     busPath = join(dir, 'events.db');
     process.env.CELILO_DB_PATH = dbPath;
     process.env.EVENT_BUS_DB = busPath;
-    db = await setupTestDatabase(dbPath);
+    db = await setupTestDatabaseAt(dbPath);
     bus = openBus({ dbPath: busPath, events: defineEvents({}) });
   });
   afterEach(() => {
@@ -737,7 +737,7 @@ describe('checkControlPlaneNetwork', () => {
 
   beforeEach(async () => {
     cpDir = mkdtempSync(join(tmpdir(), 'fleet-cpn-'));
-    db = await setupTestDatabase(join(cpDir, 'celilo.db'));
+    db = await setupTestDatabaseAt(join(cpDir, 'celilo.db'));
   });
 
   afterEach(() => {
@@ -971,7 +971,7 @@ describe('runFleetChecks includes host liveness (#728)', () => {
     dir = mkdtempSync(join(tmpdir(), 'fleet-liveness-'));
     process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
     process.env.EVENT_BUS_DB = join(dir, 'events.db');
-    db = await setupTestDatabase(join(dir, 'celilo.db'));
+    db = await setupTestDatabaseAt(join(dir, 'celilo.db'));
     bus = openBus({ dbPath: join(dir, 'events.db'), events: defineEvents({}) });
   });