@celilo/cli 1.7.0 → 1.9.0

package/src/db/foreign-keys.test.ts

@@ -0,0 +1,101 @@
+/**
+ * Recurrence gate for celilo#1074.
+ *
+ * `db/client.ts` runs `PRAGMA foreign_keys = ON` before handing back the real
+ * database. The test helpers did not, and SQLite defaults the pragma OFF per
+ * connection, so every one of the schema's `onDelete: 'cascade'` declarations
+ * was enforced in production and inert in the suite.
+ *
+ * The direction is what made it invisible. An unenforced cascade can only ever
+ * make a test pass that should have failed, so nothing has ever gone red over
+ * it and nothing ever could. `consumer-cleanup.ts`'s whole design reasons about
+ * what the cascade removes and when, and no test could observe any of it.
+ *
+ * Two assertions, because the pragma buys two different things: rows go away
+ * when their parent does, and a row referencing an absent parent is refused.
+ * The second is what stops a structurally impossible fixture from reading as
+ * valid.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  cleanupTestDatabase,
+  setupTestDatabase,
+  setupTestDatabaseAt,
+  setupTestDatabaseFile,
+} from '../test-utils/database';
+import type { DbClient } from './client';
+
+describe('test databases enforce foreign keys', () => {
+  let db: DbClient;
+
+  beforeEach(async () => {
+    db = await setupTestDatabase();
+  });
+
+  afterEach(async () => {
+    await cleanupTestDatabase(db);
+  });
+
+  test('deleting a module takes its dependent rows with it', () => {
+    db.$client.run(
+      `INSERT INTO modules (id, name, version, source_path, manifest_data) VALUES ('caddy-consumer', 'consumer', '1.0.0', '/tmp/c', '{}')`,
+    );
+    db.$client.run(
+      `INSERT INTO web_routes (slug, module_id, type, path, hostname) VALUES ('s', 'caddy-consumer', 'reverse_proxy', '/', 'example.com')`,
+    );
+
+    db.$client.run(`DELETE FROM modules WHERE id = 'caddy-consumer'`);
+
+    const rows = db.$client
+      .query(`SELECT id FROM web_routes WHERE module_id = 'caddy-consumer'`)
+      .all();
+    expect(rows).toHaveLength(0);
+  });
+
+  test('a row referencing a module that does not exist is refused', () => {
+    expect(() =>
+      db.$client.run(
+        `INSERT INTO capabilities (module_id, capability_name, version, data) VALUES ('nonexistent', 'public_web', '1.0.0', '{}')`,
+      ),
+    ).toThrow();
+  });
+});
+
+/**
+ * Every way to get a test database, checked directly.
+ *
+ * `test-utils/database.ts` is now the only module that hands one out — the
+ * second one collided with it on two exported names and disagreed with it on
+ * this pragma, which is why the gap was invisible from any call site. A helper
+ * added here later must be added to this list; there is no longer another module
+ * for it to hide in.
+ */
+describe('every test-database helper enforces foreign keys', () => {
+  const enforced = (client: DbClient): boolean =>
+    (client.$client.query('PRAGMA foreign_keys').get() as { foreign_keys: number }).foreign_keys ===
+    1;
+
+  test('setupTestDatabase (in memory)', async () => {
+    const memory = await setupTestDatabase();
+    expect(enforced(memory)).toBe(true);
+    await cleanupTestDatabase(memory);
+  });
+
+  test('setupTestDatabaseFile (temp directory it owns)', async () => {
+    const { db: file, cleanup } = await setupTestDatabaseFile();
+    expect(enforced(file)).toBe(true);
+    await cleanup();
+  });
+
+  test('setupTestDatabaseAt (path the caller chooses)', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'celilo-fk-'));
+    const at = await setupTestDatabaseAt(join(dir, 'celilo.db'));
+    expect(enforced(at)).toBe(true);
+    at.$client.close();
+    await rm(dir, { recursive: true, force: true });
+  });
+});

package/src/db/schema.ts

@@ -153,6 +153,54 @@ export const capabilities = sqliteTable('capabilities', {
     .default(sql`(unixepoch())`),
 });
 
+/**
+ * Capability bindings — which provider a consumer actually resolved to.
+ *
+ * `capabilities` is provider-side only: it answers "who COULD provide this",
+ * never "who does this module actually use". Resolution happened at hook time
+ * and vanished into the generated project. Every consumer of that fact
+ * reconstructed the same approximation — the consumer's `requires` + `optional`
+ * crossed with `capabilities` — which names every provider a module MIGHT have
+ * bound to. For a module declaring four optional capabilities and using one,
+ * that is three false edges and no way to tell which.
+ *
+ * A row is written when a consumer's hook first CALLS a method on an injected
+ * capability, not when the loader resolves one. The loader deliberately injects
+ * every registered capability regardless of what the consumer declared
+ * (`loadCapabilityFunctions`, "not just required ones"), so resolution is a
+ * superset of the declared set and recording it would restate the
+ * approximation. The call is the binding.
+ *
+ * Unique on (consumer, capability): a redeploy re-asserts the row rather than
+ * duplicating it, and a provider swap rewrites `provider_module_id` in place.
+ * Cascaded on the consumer: the binding dies with the module that made it.
+ *
+ * This does NOT replace the permissive set. `planConsumerCleanup` must still
+ * notify every provider that might hold minted state, including ones this table
+ * has no row for.
+ *
+ * @owner celilo — cross-module bookkeeping of who resolved to whom (T8, beside `capabilities`)
+ */
+export const capabilityBindings = sqliteTable(
+  'capability_bindings',
+  {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    consumerModuleId: text('consumer_module_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    capabilityName: text('capability_name').notNull(),
+    providerModuleId: text('provider_module_id').notNull(),
+    /** Last time the consumer called into this provider. */
+    boundAt: integer('bound_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
+  },
+  (table) => ({
+    consumerCapabilityUnique: uniqueIndex('capability_bindings_consumer_capability_idx').on(
+      table.consumerModuleId,
+      table.capabilityName,
+    ),
+  }),
+);
+
 /**
  * Capability secrets table - stores encrypted secrets owned by capabilities
  * Values are encrypted with AES-256-GCM using master key
@@ -251,6 +299,100 @@ export const moduleIntegrity = sqliteTable('module_integrity', {
   updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
 });
 
+/**
+ * Lifecycle states an instance moves through, reported by the deploy worker's
+ * `list` so a parent can compare desired against observed (design D6).
+ *
+ * `failed` and `pending` must stay distinguishable: a parent that cannot tell
+ * "has not started" from "will never work" either retries a bug forever or
+ * gives up on work that was merely queued.
+ */
+export const INSTANCE_STATES = [
+  'pending',
+  'provisioning',
+  'ready',
+  'failed',
+  'destroying',
+] as const;
+
+export type InstanceState = (typeof INSTANCE_STATES)[number];
+
+/**
+ * Instances of a submodule (openspec/changes/submodules, D2 and D4).
+ *
+ * An instance IS a row in `modules`, under a celilo-derived id, which is what
+ * lets every table and reader keyed on `moduleId` keep working unchanged:
+ * `module_systems`, `module_configs`, `module_infrastructure`, health, backup,
+ * fleet status and the removal path all needed no change. `moduleId` here is
+ * both this table's key and the FK to that row, so an instance cannot exist
+ * without the module row it names.
+ *
+ * This table carries only what a module row cannot express: who owns it, which
+ * submodule it came from, and the opaque key its parent knows it by.
+ *
+ * IDENTITY IS THE TRIPLE (parent, submodule, instanceKey), not `instanceKey`
+ * alone (D2). Two parents may use the same key string, and one parent may use
+ * the same key across two of its submodules; both are legal and neither
+ * collides. `instanceKey` is opaque — celilo stores it and never interprets
+ * it, because a key derived from a display name orphans a running system the
+ * first time somebody is renamed at the identity provider.
+ * @owner celilo — instance ownership and lifecycle state
+ */
+export const moduleInstances = sqliteTable(
+  'module_instances',
+  {
+    /**
+     * The derived `modules.id` this instance runs as, e.g. `byoi-lab-sdf82c1e`.
+     * Both this table's key and the FK, so the two cannot drift apart.
+     */
+    moduleId: text('module_id')
+      .primaryKey()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    /** The module that declared the submodule and created this instance. */
+    parentId: text('parent_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    /** The submodule name the parent declared — its directory under `submodules/`. */
+    submodule: text('submodule').notNull(),
+    /** Opaque, stable id supplied by the parent. Never interpreted. */
+    instanceKey: text('instance_key').notNull(),
+    /**
+     * Non-authoritative label for operator display. NOTHING may key on it, and
+     * changing it does not change identity — that is the whole point of D2.
+     */
+    label: text('label'),
+    state: text('state').$type<InstanceState>().notNull().default('pending'),
+    /**
+     * Why a `failed` instance failed. Null unless `state = 'failed'` — the
+     * legitimate "this section is absent" case rather than a missing default.
+     */
+    failureReason: text('failure_reason'),
+    /**
+     * Whether retrying could succeed. Set with `failureReason`.
+     *
+     * A parent's reconcile loop reads this to decide whether to rebuild. A
+     * missing interview answer (D7) is deterministic and will fail identically
+     * forever, while a timed-out provision is worth another go. Without the
+     * flag a caller has to string-match `failureReason`, which is the class of
+     * bug where a reporter is trusted by its shape and quietly answers wrong.
+     */
+    retryable: integer('retryable', { mode: 'boolean' }),
+    createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
+    updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
+  },
+  (table) => ({
+    // The real identity (D2). Makes "this parent already has an instance under
+    // this key" a database fact rather than a check somebody has to remember.
+    identity: unique().on(table.parentId, table.submodule, table.instanceKey),
+    // Every reconcile call lists one parent's instances, so that must not be a
+    // table scan once a fleet carries tens of them.
+    parentIdx: index('module_instances_parent_idx').on(table.parentId),
+    // `list` filters by state, and the collapsed operator view rolls up health
+    // per parent, which reads the same way.
+    stateIdx: index('module_instances_state_idx').on(table.state),
+  }),
+);
+
 /**
  * IPAM (IP Address Management) allocations table
  * Tracks VMID and IP address assignments per module
@@ -327,6 +469,12 @@ export const moduleBuilds = sqliteTable('module_builds', {
  * something declares `network.<zone>.subnet` — normally the module that supplies
  * the network. Defining a zone here costs nothing and activates nothing.
  *
+ * - isp-transit: the private segment between a celilo firewall and the router
+ *   upstream of it. Non-placement and non-allocatable, because the upstream
+ *   router is the address authority. It exists so a deployment CAN separate the
+ *   wire to that router from the workload network, not so every deployment
+ *   must: a firewall whose egress leg IS the workload network declares
+ *   `default_route_zone: internal` and behaves exactly as it does today.
  * - internal: the semi-trusted LAN the management server sits on
  * - dmz: public-facing services
  * - app: internal application services
@@ -345,6 +493,7 @@ export const moduleBuilds = sqliteTable('module_builds', {
  *   holds a leg on and must translate for.
  */
 export const NETWORK_ZONES = [
+  'isp-transit',
   'internal',
   'dmz',
   'app',
@@ -366,8 +515,9 @@ export type NetworkZone = (typeof NETWORK_ZONES)[number];
  * Zones an IP allocation or reservation can name: every NetworkZone whose
  * addresses celilo hands out.
  *
- * Two are excluded, for the same underlying reason — somebody else is the
+ * Three are excluded, for the same underlying reason — somebody else is the
  * address authority:
+ *   - `isp-transit`, whose addresses the upstream router assigns;
  *   - `external`, whose systems are addressed by the cloud/VPS provider;
  *   - `vpn`, whose client addresses are assigned by the module terminating the
  *     tunnel. celilo allocating into that subnet would collide with the VPN
@@ -378,17 +528,21 @@ export type NetworkZone = (typeof NETWORK_ZONES)[number];
  * cast in machine-pool.ts, and the cast had already drifted (it was missing
  * `secure-mgmt`, and its comment claimed the only difference was `external`).
  */
-export type AllocatableZone = Exclude<NetworkZone, 'external' | 'control-plane-vpn'>;
+export type AllocatableZone = Exclude<
+  NetworkZone,
+  'isp-transit' | 'external' | 'control-plane-vpn'
+>;
 
 /** The zones whose addresses celilo hands out, as a runtime list. */
 export const ALLOCATABLE_ZONES: AllocatableZone[] = NETWORK_ZONES.filter(
-  (zone): zone is AllocatableZone => zone !== 'external' && zone !== 'control-plane-vpn',
+  (zone): zone is AllocatableZone =>
+    zone !== 'isp-transit' && zone !== 'external' && zone !== 'control-plane-vpn',
 );
 
 /**
  * Is this a zone celilo allocates addresses in? Use this rather than testing
- * `zone !== 'external'` by hand — that check predates `vpn` and read as "the one
- * externally-addressed zone" when there are now two.
+ * `zone !== 'external'` by hand — that check predates the other externally
+ * addressed roles and silently makes them allocatable.
  */
 export function isAllocatableZone(zone: NetworkZone): zone is AllocatableZone {
   return (ALLOCATABLE_ZONES as string[]).includes(zone);
@@ -801,16 +955,33 @@ export const publicDnsEvidence = sqliteTable('public_dns_evidence', {
  *
  * `celilo system doctor` reads this to assert service hostnames resolve to
  * the firewall natIp (LAN-reachable) and not a zone-side container IP that
- * a LAN device can't route to. Rows die with either module via FK cascade.
+ * a LAN device can't route to. Rows die with their CONSUMER via FK cascade, and
+ * NOT with their provider (celilo#1010 — see `providerModuleId`).
  * @owner capability:dns_internal — resolver configuration; migrates to the provider (T6)
  */
 export const dnsInternalRecords = sqliteTable(
   'dns_internal_records',
   {
     id: integer('id').primaryKey({ autoIncrement: true }),
-    providerModuleId: text('provider_module_id')
-      .notNull()
-      .references(() => modules.id, { onDelete: 'cascade' }),
+    /**
+     * The resolver serving this record. A PLAIN column with no foreign key, and
+     * that is the celilo#1010 correction rather than an oversight.
+     *
+     * It used to cascade, so swapping `technitium` for `knot-unbound-internal`
+     * deleted the fleet's entire internal DNS ledger, `zone_routable_ip` view
+     * overrides included. `web_routes` cascades on its consumer only and the two
+     * docblocks claimed to be siblings, so the divergence read as intent and was
+     * not. The claim on a capability-owned table is the CONSUMER
+     * (openspec/changes/capability-owned-tables D3/D8), and `dns_internal`'s
+     * declaration cannot express anything else.
+     *
+     * Attribution is still real and still enforced — it is half
+     * `dns_internal_records_provider_host_idx` — it just no longer decides when a
+     * LIVE record is forgotten. A provider leaving now leaves the ledger for the
+     * next one to reconcile from, which is what stage 1's provider-arrival
+     * backfill assumes. Migration `0027`.
+     */
+    providerModuleId: text('provider_module_id').notNull(),
     consumerModuleId: text('consumer_module_id')
       .notNull()
       .references(() => modules.id, { onDelete: 'cascade' }),
@@ -1392,3 +1563,5 @@ export type Alert = typeof alerts.$inferSelect;
 export type NewAlert = typeof alerts.$inferInsert;
 export type NotificationDelivery = typeof notificationDeliveries.$inferSelect;
 export type NewNotificationDelivery = typeof notificationDeliveries.$inferInsert;
+export type CapabilityBinding = typeof capabilityBindings.$inferSelect;
+export type NewCapabilityBinding = typeof capabilityBindings.$inferInsert;

package/src/hooks/broker.test.ts

@@ -0,0 +1,152 @@
+/**
+ * The broker: the capability surface crossing a process boundary.
+ *
+ * The claim under test is design D2's — that one generic proxy covers all
+ * twelve capabilities because every hook-facing method is already
+ * `(request: JSON) => Promise<JSON>`. So these tests fix the SHAPES a call can
+ * take (returns, throws, a structured throw the framework reads, an absent
+ * optional method, an unknown method) and say nothing about any particular
+ * capability. A per-method suite would prove the same thing thirty-seven times
+ * and go stale the moment a provider gained a method.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { execSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { MissingProviderInputError } from '@celilo/capabilities';
+import { capabilityShape } from './broker';
+import { executeHookScript } from './executor';
+import { createCapturingLogger } from './logger';
+import type { HookContext } from './types';
+
+const FIXTURES = join(__dirname, 'test-fixtures');
+
+function demoCapabilities(): Record<string, unknown> {
+  return {
+    demo: {
+      providerModuleId: 'demo-provider',
+      version: '1.0.0',
+      echo: async (request: unknown) => ({ echoed: request }),
+      returnsNothing: async () => undefined,
+      boom: async () => {
+        throw new Error('plain failure');
+      },
+      missingInput: async () => {
+        throw new MissingProviderInputError({
+          providerModuleId: 'caddy',
+          ensureId: 'hostnames',
+          value: 'foo.example.com',
+          humanContext: 'so the route resolves',
+        });
+      },
+    },
+  };
+}
+
+async function runCapabilityHook(): Promise<Record<string, unknown>> {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-broker-'));
+  try {
+    const context: HookContext = {
+      config: {},
+      secrets: {},
+      systems: [],
+      logger: createCapturingLogger().logger,
+      debug: false,
+      screenshotDir: dir,
+      capabilities: demoCapabilities(),
+    };
+    return await executeHookScript(
+      join(FIXTURES, 'capability-calling-hook.ts'),
+      context,
+      30_000,
+      30_000,
+    );
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+describe('capabilityShape', () => {
+  test('splits functions from data', () => {
+    const shape = capabilityShape(demoCapabilities());
+    expect(shape.demo.methods.sort()).toEqual(['boom', 'echo', 'missingInput', 'returnsNothing']);
+    expect(shape.demo.data).toEqual({ providerModuleId: 'demo-provider', version: '1.0.0' });
+  });
+
+  test('an unimplemented optional method is simply absent', () => {
+    // Not "present and throwing". `if (cap.registerTrustedSource)` is real
+    // code in the wireguard module and it has to keep answering correctly.
+    expect(capabilityShape(demoCapabilities()).demo.methods).not.toContain('sometimesAbsent');
+  });
+
+  test('symbol keys are dropped — they cannot cross JSON', () => {
+    const brand = Symbol('brand');
+    const shape = capabilityShape({ demo: { [brand]: 'x', ok: async () => 1 } });
+    expect(shape.demo.methods).toEqual(['ok']);
+    expect(shape.demo.data).toEqual({});
+  });
+
+  test('a value JSON cannot carry is left out of data rather than corrupted', () => {
+    const shape = capabilityShape({
+      demo: { nan: Number.NaN, inf: Number.POSITIVE_INFINITY, n: 1 },
+    });
+    expect(shape.demo.data).toEqual({ n: 1 });
+  });
+
+  test('a non-object capability entry is skipped, not crashed on', () => {
+    expect(capabilityShape({ broken: null, alsoBroken: 'string' })).toEqual({});
+  });
+});
+
+describe('capability calls across the boundary', () => {
+  test('every call shape survives the round trip', async () => {
+    const outputs = await runCapabilityHook();
+
+    expect(outputs.providerModuleId).toBe('demo-provider');
+    expect(outputs.version).toBe('1.0.0');
+    expect(outputs.optionalMethodAbsent).toBe(true);
+    expect(outputs.returned).toEqual({ echoed: { x: 1, nested: { y: [2, 3] } } });
+    expect(outputs.undefinedBecomesNull).toBeNull();
+
+    expect(outputs.plainThrow).toEqual({
+      isError: true,
+      name: 'Error',
+      message: 'plain failure',
+      hasStack: true,
+    });
+
+    // The one error the framework READS rather than displays. Lose these four
+    // fields and the cross-module ensure interview never runs — the deploy
+    // fails with a message where it should have asked a question.
+    expect(outputs.missingProviderInput).toEqual({
+      recognised: true,
+      providerModuleId: 'caddy',
+      ensureId: 'hostnames',
+      value: 'foo.example.com',
+      humanContext: 'so the route resolves',
+    });
+
+    // A method that is not in the shape is not on the proxy, so this fails on
+    // this side and never reaches the broker — the same TypeError a hook gets
+    // in-process today. The broker's own "no such method" guard is for a
+    // shape/map disagreement, which is a skew bug and not this path.
+    expect(outputs.unknownMethod).toContain('not a function');
+  }, 30_000);
+
+  test('the hook ran in a process of its own and left none behind', async () => {
+    await runCapabilityHook();
+
+    // By parent pid, not by name: the hook runner is a direct child of this
+    // process, and matching on the script name instead picks up whatever shell
+    // happens to have the filename in its own command line.
+    const orphans = execSync('ps -Ao ppid=,args=', { encoding: 'utf-8' })
+      .split('\n')
+      .filter(
+        (line) => Number.parseInt(line.trim(), 10) === process.pid && line.includes('hook-runner'),
+      );
+
+    expect(orphans).toEqual([]);
+  }, 30_000);
+});