@celilo/cli 1.7.0 → 1.8.0

package/src/services/capability-table-rows.test.ts

@@ -0,0 +1,191 @@
+/**
+ * The ONE consumer-removal rule, driven by the declarations
+ * (openspec/changes/capability-owned-tables task 2.4).
+ *
+ * Before this, "the row dies with its consumer" had five implementations for one
+ * rule, and two of them were functions core imported BY NAME from
+ * `consumer-cleanup.ts` — the file whose whole purpose is deleting exactly that
+ * kind of per-capability special-casing.
+ *
+ * The property worth protecting is not "port forwards get deleted". It is that
+ * core clears EVERY declared table without naming any of them, so declaring a
+ * new one is sufficient to have it cleaned up.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { allDeclaredTables } from '@celilo/capabilities';
+import type { DbClient } from '../db/client';
+import { dnsInternalRecords, modules, portForwards, trustedSources, webRoutes } from '../db/schema';
+import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
+
+const FW = '192.168.0.254';
+
+describe('planClaimedRowDeletion', () => {
+  /**
+   * The anti-regression that matters. If someone declares a table and this plan
+   * does not grow, their rows outlive the consumer silently — which is the whole
+   * failure the declaration exists to make impossible.
+   */
+  it('covers every declared table, so declaring one is enough to have it cleared', () => {
+    const planned = new Set(planClaimedRowDeletion().map((t) => t.table));
+    const declared = allDeclaredTables().map((d) => d.declaration.table);
+
+    expect(declared.length).toBeGreaterThan(0);
+    for (const table of declared) expect(planned.has(table)).toBe(true);
+  });
+
+  it("carries each table's OWN claim column, which is spelled three ways", () => {
+    const byTable = new Map(planClaimedRowDeletion().map((t) => [t.table, t.claimColumn]));
+
+    expect(byTable.get('web_routes')).toBe('module_id');
+    expect(byTable.get('port_forwards')).toBe('registered_by');
+    expect(byTable.get('trusted_sources')).toBe('registered_by');
+    expect(byTable.get('dns_internal_records')).toBe('consumer_module_id');
+  });
+});
+
+describe('deleteClaimedRows', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'claimed-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+
+    for (const id of ['caddy', 'forgejo', 'technitium']) {
+      db.insert(modules)
+        .values({
+          id,
+          name: id,
+          version: '1.0.0',
+          state: 'VERIFIED',
+          sourcePath: `/tmp/${id}`,
+          manifestData: {},
+        })
+        .run();
+    }
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  function seed(consumer: string, hostname: string, port: number, subnet: string) {
+    db.insert(webRoutes)
+      .values({
+        slug: `${consumer}-slug`,
+        moduleId: consumer,
+        type: 'reverse_proxy',
+        path: `/${consumer}`,
+        hostname,
+      })
+      .run();
+    db.insert(portForwards)
+      .values({
+        firewallIp: FW,
+        internalIp: '10.0.20.5',
+        port,
+        protocol: 'TCP',
+        registeredBy: consumer,
+      })
+      .run();
+    db.insert(trustedSources).values({ firewallIp: FW, subnet, registeredBy: consumer }).run();
+    db.insert(dnsInternalRecords)
+      .values({
+        providerModuleId: 'technitium',
+        consumerModuleId: consumer,
+        host: hostname,
+        ip: '10.0.20.5',
+      })
+      .run();
+  }
+
+  it('clears the departing consumer from all four declared tables at once', () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+
+    const cleared = deleteClaimedRows(db, 'caddy');
+
+    expect(cleared.map((c) => c.table).sort()).toEqual([
+      'dns_internal_records',
+      'port_forwards',
+      'trusted_sources',
+      'web_routes',
+    ]);
+    expect(db.select().from(webRoutes).all()).toHaveLength(0);
+    expect(db.select().from(portForwards).all()).toHaveLength(0);
+    expect(db.select().from(trustedSources).all()).toHaveLength(0);
+    expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
+  });
+
+  /**
+   * The refcount case, the bug most likely to ship silently. Two consumers each
+   * hold their own row, and one leaving must not withdraw what the other still
+   * needs.
+   */
+  it("leaves another consumer's rows untouched", () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+    seed('forgejo', 'b.example.org', 2222, '10.2.0.0/24');
+
+    deleteClaimedRows(db, 'caddy');
+
+    expect(
+      db
+        .select()
+        .from(webRoutes)
+        .all()
+        .map((r) => r.moduleId),
+    ).toEqual(['forgejo']);
+    expect(
+      db
+        .select()
+        .from(portForwards)
+        .all()
+        .map((r) => r.registeredBy),
+    ).toEqual(['forgejo']);
+    expect(
+      db
+        .select()
+        .from(trustedSources)
+        .all()
+        .map((r) => r.registeredBy),
+    ).toEqual(['forgejo']);
+    expect(
+      db
+        .select()
+        .from(dnsInternalRecords)
+        .all()
+        .map((r) => r.consumerModuleId),
+    ).toEqual(['forgejo']);
+  });
+
+  /**
+   * `dns_internal_records` is claimed by its CONSUMER, never its provider. The
+   * declaration cannot express anything else, which is what makes celilo#1010
+   * unrepresentable — but the runtime rule has to agree, or the declaration is
+   * decoration.
+   */
+  it('does not treat the PROVIDER as the claimant of a dns_internal record', () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+
+    deleteClaimedRows(db, 'technitium');
+
+    expect(db.select().from(dnsInternalRecords).all()).toHaveLength(1);
+  });
+
+  it('is a no-op for a module that claimed nothing', () => {
+    seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
+
+    deleteClaimedRows(db, 'forgejo');
+
+    expect(db.select().from(webRoutes).all()).toHaveLength(1);
+    expect(db.select().from(portForwards).all()).toHaveLength(1);
+  });
+});

package/src/services/capability-table-rows.ts

@@ -0,0 +1,103 @@
+/**
+ * Acting on every capability-declared table without naming any of them
+ * (openspec/changes/capability-owned-tables, design D3).
+ *
+ * "This row dies with its consumer" had five implementations for one rule: an FK
+ * cascade for `web_routes`, two functions core imported BY NAME from
+ * `consumer-cleanup.ts` for `port_forwards` and `trusted_sources`, a second
+ * table pruned on read for `dns_registrations`, a cascade on both provider and
+ * consumer for `dns_internal_records`, and a hook rewriting a JSON blob for the
+ * rest. `consumer-cleanup.ts` exists specifically to delete that kind of
+ * per-capability special-casing. It succeeded for the hook dispatch and then
+ * grew two hardcoded imports for the rows, because the rows had no generic rule.
+ *
+ * This is the generic rule. The declaration names the claim column, so core no
+ * longer has to know that three tables spell it `module_id`, `registered_by` and
+ * `consumer_module_id`.
+ */
+
+import { CAPABILITY_DECLARED_TABLES, allDeclaredTables } from '@celilo/capabilities';
+import { type Column, eq, is } from 'drizzle-orm';
+import { SQLiteTable, getTableConfig } from 'drizzle-orm/sqlite-core';
+import type { DbClient } from '../db/client';
+import * as dbSchema from '../db/schema';
+
+export interface ClaimedRowTarget {
+  capability: string;
+  /** Physical table name. */
+  table: string;
+  /** The column holding the consumer's module id. */
+  claimColumn: string;
+}
+
+/**
+ * Physical table name → the drizzle table object, for the tables celilo can
+ * actually act on. Built by walking the schema rather than by a hand-written
+ * map, so a table cannot be declared and then silently unreachable.
+ */
+function schemaTables(): Map<string, SQLiteTable> {
+  const out = new Map<string, SQLiteTable>();
+  for (const value of Object.values(dbSchema)) {
+    if (!is(value, SQLiteTable)) continue;
+    out.set(getTableConfig(value).name, value);
+  }
+  return out;
+}
+
+/**
+ * What a consumer's removal must clear. Pure — the list is a property of the
+ * declarations, so it is worth being able to assert without a database.
+ *
+ * A declaration naming a table that does not exist is skipped here rather than
+ * throwing. `policy/capability-shape-drift.test.ts` Scan A is what fails on
+ * that, loudly and at build time; a removal is the wrong moment to discover it,
+ * and refusing to clear the OTHER tables because one declaration is wrong would
+ * strand real rows.
+ */
+export function planClaimedRowDeletion(): ClaimedRowTarget[] {
+  const tables = schemaTables();
+  return allDeclaredTables()
+    .filter(({ declaration }) => tables.has(declaration.table))
+    .map(({ capability, declaration }) => ({
+      capability,
+      table: declaration.table,
+      claimColumn: declaration.claim.column,
+    }))
+    .sort((a, b) => a.table.localeCompare(b.table));
+}
+
+/**
+ * Delete every declared row this consumer claimed.
+ *
+ * Called AFTER every provider has been told the consumer is going (its D4): a
+ * provider's `on_consumer_removed` is a full converge, and converging from a set
+ * the rows had already been removed from would withdraw more than the departing
+ * consumer's share.
+ *
+ * `web_routes` and `dns_internal_records` also carry an FK cascade, so their
+ * rows would go anyway once the `modules` row is deleted. Doing it here as well
+ * is deliberate and not merely harmless: it makes the ORDER explicit rather than
+ * a consequence of when some other statement happens to run, and it is the same
+ * point in the sequence at which `port_forwards` and `trusted_sources` were
+ * already being cleared by hand.
+ */
+export function deleteClaimedRows(db: DbClient, consumer: string): ClaimedRowTarget[] {
+  const tables = schemaTables();
+  const cleared: ClaimedRowTarget[] = [];
+  for (const target of planClaimedRowDeletion()) {
+    const table = tables.get(target.table);
+    if (!table) continue;
+    const column = getTableConfig(table).columns.find((c) => c.name === target.claimColumn);
+    if (!column) continue;
+    db.delete(table)
+      .where(eq(column as Column, consumer))
+      .run();
+    cleared.push(target);
+  }
+  return cleared;
+}
+
+/** Capability names that declare at least one table — for logs and reports. */
+export function capabilitiesWithDeclaredTables(): string[] {
+  return Object.keys(CAPABILITY_DECLARED_TABLES).sort();
+}

package/src/services/consumer-cleanup.ts

@@ -23,8 +23,7 @@ import { capabilities, modules } from '../db/schema';
 import { type RunNamedHookResult, runNamedHook } from '../hooks/run-named-hook';
 import type { HookLogger } from '../hooks/types';
 import type { ModuleManifest } from '../manifest/schema';
-import { deletePortForwardsForModule } from './port-forwards';
-import { deleteTrustedSourcesForModule } from './trusted-sources';
+import { deleteClaimedRows } from './capability-table-rows';
 
 /**
  * States in which a module has never resolved a capability and therefore holds
@@ -213,11 +212,18 @@ export async function runConsumerCleanup(
     );
   }
 
-  // The rows, once every provider has converged without them (D4). `web_routes`
-  // dies with the `modules` row via its FK cascade; these two carry a plain
-  // `registered_by` text column and do not.
-  deletePortForwardsForModule(db, consumer);
-  deleteTrustedSourcesForModule(db, consumer);
+  // The rows, once every provider has converged without them (D4).
+  //
+  // Driven by the capability declarations rather than by name. This used to be
+  // two hardcoded imports, `deletePortForwardsForModule` and
+  // `deleteTrustedSourcesForModule`, which is the shape of the whole problem
+  // openspec/changes/capability-owned-tables addresses: this file exists to
+  // delete per-capability special-casing, it succeeded for the hook dispatch,
+  // and then grew two capability-named calls for the rows because the rows had
+  // no generic rule. Now they have one, and core no longer has to know that the
+  // claim column is spelled `module_id` on one table, `registered_by` on two and
+  // `consumer_module_id` on a fourth.
+  deleteClaimedRows(db, consumer);
 
   return result;
 }

package/src/services/dns-internal-records.test.ts

@@ -4,8 +4,9 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import type { DnsRecordRequest } from '@celilo/capabilities';
 import type { ViewOverride } from '@celilo/capabilities';
+import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { modules } from '../db/schema';
+import { dnsInternalRecords, modules } from '../db/schema';
 import { setupTestDatabase } from '../test-utils/setup-test-db';
 import {
   listDnsInternalRecords,
@@ -197,3 +198,73 @@ describe('dns-internal-records ledger', () => {
     });
   });
 });
+
+/**
+ * celilo#1010 — swapping the internal DNS provider must not delete the ledger.
+ *
+ * `dns_internal_records` used to cascade on `provider_module_id` as well as on
+ * the consumer, so removing `technitium` to install `knot-unbound-internal` took
+ * the fleet's entire internal DNS ledger with it, `zone_routable_ip` view
+ * overrides included — the durable desired state the resolver's split-horizon
+ * config is reconciled from. `web_routes` cascades on its consumer only, and the
+ * two tables' docblocks claimed to be siblings, so the divergence read as intent
+ * and was not.
+ *
+ * The claim on a capability-owned table is the CONSUMER
+ * (openspec/changes/capability-owned-tables D3/D8). Migration 0027 makes the
+ * table agree with the declaration, which cannot express anything else.
+ */
+describe('provider cascade (celilo#1010)', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'dns-cascade-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    for (const id of ['technitium', 'knot-unbound-internal', 'caddy']) {
+      db.insert(modules)
+        .values({
+          id,
+          name: id,
+          version: '1.0.0',
+          state: 'VERIFIED',
+          sourcePath: `/tmp/${id}`,
+          manifestData: {},
+        })
+        .run();
+    }
+    db.insert(dnsInternalRecords)
+      .values({
+        providerModuleId: 'technitium',
+        consumerModuleId: 'caddy',
+        host: 'auth.example.org',
+        ip: '192.168.0.253',
+        zoneRoutableIp: '10.0.10.14',
+      })
+      .run();
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('keeps the ledger when the PROVIDER is removed, view overrides included', () => {
+    db.delete(modules).where(eq(modules.id, 'technitium')).run();
+
+    const left = db.select().from(dnsInternalRecords).all();
+    expect(left).toHaveLength(1);
+    // The override is the part whose loss is silent: the resolver keeps
+    // answering, just with the wrong address for in-zone clients.
+    expect(left[0]?.zoneRoutableIp).toBe('10.0.10.14');
+  });
+
+  it('still dies with its CONSUMER, which is the rule that did not change', () => {
+    db.delete(modules).where(eq(modules.id, 'caddy')).run();
+
+    expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
+  });
+});

package/src/services/module-validator/capability-versions.test.ts

@@ -5,6 +5,7 @@
  */
 
 import { describe, expect, test } from 'bun:test';
+import { CAPABILITY_CONTRACT_VERSIONS } from '@celilo/capabilities';
 import { validateCapabilityVersions } from './capability-versions';
 
 describe('validateCapabilityVersions', () => {
@@ -31,7 +32,11 @@ describe('validateCapabilityVersions', () => {
     expect(errors).toHaveLength(1);
     expect(errors[0]).toContain('provides[public_web]');
     expect(errors[0]).toContain('1.0.0');
-    expect(errors[0]).toContain('3.1.0');
+    // Read from the registry, not pinned to today's number. The assertion is
+    // "the message names the RUNTIME version so the author knows what to move
+    // to"; a literal here turns a behaviour test into a version tracker that
+    // goes red on every legitimate bump, which is what happened at 3.1.0 → 3.2.0.
+    expect(errors[0]).toContain(CAPABILITY_CONTRACT_VERSIONS.public_web);
   });
 
   test('error when provides[X].version is newer than runtime', () => {

package/src/services/port-forwards.test.ts

@@ -7,7 +7,8 @@ import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { portForwards } from '../db/schema';
 import { setupTestDatabase } from '../test-utils/setup-test-db';
-import { buildPortForwardStore, deletePortForwardsForModule } from './port-forwards';
+import { deleteClaimedRows } from './capability-table-rows';
+import { buildPortForwardStore } from './port-forwards';
 
 const FW = '192.168.0.254';
 const CADDY = { internalIp: '10.0.20.5', protocol: 'TCP' as const, description: 'caddy' };
@@ -113,7 +114,10 @@ describe('port-forward store', () => {
     other.replace(FW, { ...CADDY, description: 'also 443' }, [443]);
     expect(store.list(FW)).toHaveLength(2);
 
-    deletePortForwardsForModule(db, 'caddy');
+    // Through the GENERIC declaration-driven path, which replaced
+    // `deletePortForwardsForModule`. The property is unchanged; what changed is
+    // that core no longer names this capability's table to clear it.
+    deleteClaimedRows(db, 'caddy');
 
     const left = store.list(FW);
     expect(left).toHaveLength(1);

package/src/services/port-forwards.ts

@@ -99,14 +99,3 @@ export function buildPortForwardStore(db: DbClient, registeredBy: string): PortF
     },
   };
 }
-
-/**
- * Drop every forward a departing module owns, across every firewall.
- *
- * `registered_by` is plain text rather than a FK, so unlike `web_routes` these
- * rows do NOT die with the `modules` row — core deletes them explicitly, AFTER
- * the provider has converged without them (D4).
- */
-export function deletePortForwardsForModule(db: DbClient, moduleId: string): void {
-  db.delete(portForwards).where(eq(portForwards.registeredBy, moduleId)).run();
-}

package/src/services/trusted-sources.ts

@@ -68,11 +68,6 @@ export function buildTrustedSourceStore(db: DbClient, registeredBy: string): Tru
   };
 }
 
-/** Drop every trusted source a departing module owns, across every firewall. */
-export function deleteTrustedSourcesForModule(db: DbClient, moduleId: string): void {
-  db.delete(trustedSources).where(eq(trustedSources.registeredBy, moduleId)).run();
-}
-
 /** Where a trusted subnet came from — reach into every tier must be attributable. */
 export type TrustedSubnetOrigin = 'derived-control-plane' | 'registered' | 'operator-override';
 

package/src/variables/context.ts

@@ -593,16 +593,81 @@ async function assembleResolutionContext(
         }
       }
     }
-    const dnsInternal = capabilitiesMap.dns_internal as { server?: { ip?: unknown } } | undefined;
-    // The dns_internal capability advertises the provider's address, which may
-    // carry a CIDR suffix (technitium's server.ip resolves from target_ip,
-    // e.g. "192.168.0.151/24"). A nameserver must be a bare IP — strip it.
-    const rawInternalIp =
-      typeof dnsInternal?.server?.ip === 'string' ? dnsInternal.server.ip : undefined;
-    const internalIp = rawInternalIp ? rawInternalIp.split('/')[0] : undefined;
-    const nameservers = internalIp ? [internalIp, ...publicDns] : publicDns;
-    if (nameservers.length > 0) {
-      selfConfig.lxc_nameserver = nameservers.join(' ');
+    type ResolverEndpoint = {
+      server?: { ip?: unknown; internal_ip?: unknown };
+      /** Zones the provider's base-module aspect owns ongoing DNS for (D5d). */
+      aspect?: { covered_zones?: unknown };
+    };
+    const dnsInternal = capabilitiesMap.dns_internal as ResolverEndpoint | undefined;
+    const dnsSecondary = capabilitiesMap.dns_internal_secondary as ResolverEndpoint | undefined;
+
+    // A nameserver must be a bare IP. The advertised address may carry a CIDR
+    // suffix (technitium's server.ip resolves from target_ip, e.g.
+    // "192.168.0.151/24"), so strip it.
+    const bareIp = (value: unknown): string | undefined =>
+      typeof value === 'string' && value.length > 0 ? value.split('/')[0] : undefined;
+
+    const targetZone = module?.manifestData
+      ? getSingularSystemSpec(module.manifestData as ModuleManifest)?.zone
+      : undefined;
+
+    // `internal` cannot route into the resolvers' own (dmz) subnet, so it uses
+    // the ingress addresses the firewall DNATs. Every other zone reaches them
+    // directly.
+    const endpointFor = (resolver: ResolverEndpoint | undefined): string | undefined =>
+      targetZone === 'internal'
+        ? (bareIp(resolver?.server?.internal_ip) ?? bareIp(resolver?.server?.ip))
+        : bareIp(resolver?.server?.ip);
+
+    const primaryIp = endpointFor(dnsInternal);
+    const secondaryIp = endpointFor(dnsSecondary);
+
+    // Whether the DEPLOYED provider's base-module aspect covers this system's
+    // zone, which is what decides if the public entries may go (design D5d).
+    //
+    // The predicate is aspect COVERAGE, not routing. Routing is why a zone goes
+    // uncovered — `external` and the planned `quarantine` hold systems that
+    // cannot reach a resolver inside the perimeter. Coverage is why the birth
+    // list is PERMANENT: terraform injects
+    // `lifecycle { ignore_changes = [nameserver] }`, so it cannot correct the
+    // value afterwards even in principle, and a zone no aspect covers has no
+    // owner for ongoing DNS at all. Keying on coverage therefore handles a
+    // future excluded zone with no change here, and fails safe on an accidental
+    // omission — where keeping the public entries is the lesser harm until the
+    // array is fixed.
+    // The provider DECLARES its coverage in the capability data. Core could
+    // instead find the provider's module row and read
+    // `base_module_aspect.applicable_zones` off its manifest, which would be
+    // one source of truth rather than two — but that means core naming a
+    // capability in order to find its provider, which is the pattern the
+    // module-business gate exists to stop, and its advice is exactly this: let
+    // the provider declare the behaviour. A manifest test holds the declared
+    // list and the aspect's own `applicable_zones` together so they cannot
+    // drift.
+    const aspectZones = dnsInternal?.aspect?.covered_zones;
+    const zoneIsAspectCovered =
+      targetZone !== undefined && Array.isArray(aspectZones) && aspectZones.includes(targetZone);
+
+    // Both halves are required. A pair with no aspect covering this zone is the
+    // `external` case: two addresses it cannot route to and nothing else, which
+    // does not tighten anything, it takes DNS away. A primary with no secondary
+    // is D5a: removing the fallback and shipping a secondary are ONE decision,
+    // because otherwise every resolver redeploy blanks fleet DNS.
+    const dropPublicResolvers =
+      primaryIp !== undefined && secondaryIp !== undefined && zoneIsAspectCovered;
+
+    // The uncovered branch is main's composition unchanged: the primary, then
+    // the public resolvers. The secondary is deliberately NOT added to it — in
+    // a zone that cannot route to the resolvers, a second unreachable address
+    // buys nothing but another timeout before the public entries answer.
+    const nameservers = dropPublicResolvers
+      ? [primaryIp, secondaryIp]
+      : primaryIp
+        ? [primaryIp, ...publicDns]
+        : publicDns;
+    const unique = [...new Set(nameservers)];
+    if (unique.length > 0) {
+      selfConfig.lxc_nameserver = unique.join(' ');
     }
   }