@celilo/cli 0.5.0-alpha.4 → 0.5.0-alpha.5

package/src/db/schema-introspection.ts

@@ -0,0 +1,88 @@
+/**
+ * Schema introspection — compare the DB's actual tables/columns against the
+ * tables the running code's drizzle schema declares. The single source for
+ * "what does the code expect, and is it present?" shared by:
+ *   - the migration baseline (db/migrate.ts) — to decide which migrations are
+ *     already applied on an existing DB before running migrate() (ISS-0100), and
+ *   - the doctor's schema-drift check (services/fleet-checks.ts) — to report
+ *     missing tables/columns to the operator (ISS-0113).
+ */
+
+import type { Database } from 'bun:sqlite';
+import { is } from 'drizzle-orm';
+import { SQLiteTable, getTableConfig } from 'drizzle-orm/sqlite-core';
+import * as dbSchema from './schema';
+
+export interface SchemaDrift {
+  /** Tables the code's schema declares that the DB lacks. */
+  missingTables: string[];
+  /** `table.column` the code declares that the DB's table lacks. */
+  missingColumns: string[];
+  /** Total number of tables the code's schema declares. */
+  tableCount: number;
+}
+
+/** Every table name + column names the drizzle schema declares. */
+export function getSchemaTables(): Array<{ name: string; columns: string[] }> {
+  const out: Array<{ name: string; columns: string[] }> = [];
+  for (const value of Object.values(dbSchema)) {
+    if (!is(value, SQLiteTable)) continue;
+    const cfg = getTableConfig(value);
+    out.push({ name: cfg.name, columns: cfg.columns.map((c) => c.name) });
+  }
+  return out;
+}
+
+/** Names of all tables that physically exist in the DB. */
+export function getExistingTables(sqlite: Database): Set<string> {
+  return new Set(
+    sqlite
+      .query<{ name: string }, []>("SELECT name FROM sqlite_master WHERE type='table'")
+      .all()
+      .map((r) => r.name),
+  );
+}
+
+/** Names of all indexes that physically exist in the DB. */
+export function getExistingIndexes(sqlite: Database): Set<string> {
+  return new Set(
+    sqlite
+      .query<{ name: string }, []>("SELECT name FROM sqlite_master WHERE type='index'")
+      .all()
+      .map((r) => r.name),
+  );
+}
+
+/** Column names physically present on a table (empty if the table is absent). */
+export function getExistingColumns(sqlite: Database, table: string): Set<string> {
+  // `table` is a schema/migration identifier (never user input) — safe to inline.
+  return new Set(
+    sqlite
+      .query<{ name: string }, []>(`SELECT name FROM pragma_table_info('${table}')`)
+      .all()
+      .map((r) => r.name),
+  );
+}
+
+/**
+ * Compare the running code's drizzle schema to the DB and report what's missing.
+ * Track-agnostic: it reads actual table/column presence, so it's honest whether
+ * the DB was migrated, baselined, or hand-patched.
+ */
+export function findSchemaDrift(sqlite: Database): SchemaDrift {
+  const present = getExistingTables(sqlite);
+  const missingTables: string[] = [];
+  const missingColumns: string[] = [];
+  const tables = getSchemaTables();
+  for (const t of tables) {
+    if (!present.has(t.name)) {
+      missingTables.push(t.name);
+      continue;
+    }
+    const cols = getExistingColumns(sqlite, t.name);
+    for (const c of t.columns) {
+      if (!cols.has(c)) missingColumns.push(`${t.name}.${c}`);
+    }
+  }
+  return { missingTables, missingColumns, tableCount: tables.length };
+}

package/src/db/schema.ts

@@ -468,6 +468,44 @@ export const dnsRegistrations = sqliteTable(
   }),
 );
 
+/**
+ * Internal split-horizon DNS A-record ledger. Mirrors `dns_registrations`
+ * but for the dns_internal capability (technitium/knot): the capability
+ * loader records every `dns_internal.registerRecord({type:'A'})` here so
+ * celilo has an offline, queryable record of what hostname → IP it asked
+ * the internal resolver to serve. Without this, the only source of truth
+ * is the resolver's own DB, requiring a live probe (ISS-0094 / ISS-0111).
+ *
+ * `celilo system doctor` reads this to assert service hostnames resolve to
+ * the firewall natIp (LAN-reachable) and not a zone-side container IP that
+ * a LAN device can't route to. Rows die with either module via FK cascade.
+ */
+export const dnsInternalRecords = sqliteTable(
+  'dns_internal_records',
+  {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    providerModuleId: text('provider_module_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    consumerModuleId: text('consumer_module_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    /** The registered hostname (e.g. "git-ssh.git.celilo.computer"). */
+    host: text('host').notNull(),
+    /** The A-record value celilo asked the resolver to serve. */
+    ip: text('ip').notNull(),
+    registeredAt: integer('registered_at', { mode: 'timestamp' })
+      .notNull()
+      .default(sql`(unixepoch())`),
+  },
+  (table) => ({
+    providerHostUnique: uniqueIndex('dns_internal_records_provider_host_idx').on(
+      table.providerModuleId,
+      table.host,
+    ),
+  }),
+);
+
 /**
  * Backup storage providers - destinations for backup archives
  * Supports local filesystem and S3-compatible storage (AWS S3, MinIO, Backblaze B2, Wasabi)

package/src/hooks/capability-loader.ts

@@ -31,6 +31,7 @@ import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
 import { getModuleSystems } from '../services/deployed-systems';
+import { withDnsInternalLedger } from '../services/dns-internal-records';
 import { withDnsRegistrationLedger } from '../services/dns-registrations';
 import { loadHookConfigMap } from './load-hook-config';
 
@@ -200,19 +201,27 @@ export async function loadCapabilityFunctions(
     const providerConfig = await loadModuleConfig(capability.moduleId, db);
     const providerSecrets = await loadModuleSecrets(capability.moduleId, masterKey, db);
 
-    // dns_registrar interfaces get the registration-ledger wrapper:
-    // every successful registerHost is recorded in dns_registrations so
-    // the provider's refresh_registrations hook can re-assert it later
-    // (designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md B2). The loader is
-    // the one layer that knows both provider and consumer.
-    const withLedgerIfDnsRegistrar = (iface: unknown): unknown =>
-      capName === 'dns_registrar'
-        ? withDnsRegistrationLedger(iface as DnsRegistrarCapability, {
-            db,
-            providerModuleId: capability.moduleId,
-            consumerModuleId: consumingModuleId,
-          })
-        : iface;
+    // dns_registrar and dns_internal interfaces get a registration-ledger
+    // wrapper: every successful registerHost / registerRecord is recorded so
+    // celilo has an offline record of what it asked DNS to serve. The
+    // external ledger feeds the provider's refresh_registrations hook
+    // (DISPATCHER_DAEMON_AND_TIMER_EVENTS.md B2); the internal ledger feeds
+    // the doctor's natIp drift check (CELILO_DOCTOR_FLEET_DRIFT.md Phase 4).
+    // The loader is the one layer that knows both provider and consumer.
+    const ledgerCtx = {
+      db,
+      providerModuleId: capability.moduleId,
+      consumerModuleId: consumingModuleId,
+    };
+    const withLedger = (iface: unknown): unknown => {
+      if (capName === 'dns_registrar') {
+        return withDnsRegistrationLedger(iface as DnsRegistrarCapability, ledgerCtx);
+      }
+      if (capName === 'dns_internal') {
+        return withDnsInternalLedger(iface as DnsInternalCapability, ledgerCtx);
+      }
+      return iface;
+    };
 
     try {
       // Dynamically import the capability module. Try the default export
@@ -237,7 +246,7 @@ export async function loadCapabilityFunctions(
           systems: getModuleSystems(capability.moduleId, db),
           logger,
         });
-        result[capName] = withLedgerIfDnsRegistrar(capabilityInterface);
+        result[capName] = withLedger(capabilityInterface);
         debugLog(`${capName}: loaded via defineCapabilityFunction`);
         continue;
       }
@@ -253,7 +262,7 @@ export async function loadCapabilityFunctions(
       );
 
       if (capabilityInterface) {
-        result[capName] = withLedgerIfDnsRegistrar(
+        result[capName] = withLedger(
           wrapWithLogging(capabilityInterface as object, logger, capName),
         );
         debugLog(`${capName}: loaded via legacy factory`);

package/src/services/deploy-preflight.ts

@@ -29,6 +29,7 @@ import { capabilities, moduleConfigs, modules, secrets } from '../db/schema';
 import { type ModuleManifest, getSingularSystemSpec } from '../manifest/schema';
 import { buildResolutionContext } from '../variables/context';
 import { E2E_CONFLICT_FIX, runningE2eContainers } from './e2e-guard';
+import { describeCapabilityProblem, findBrokenCapabilityDerivations } from './fleet-checks';
 
 export interface PreflightResult {
   success: boolean;
@@ -232,6 +233,30 @@ export async function runPreflight(
         });
       }
     }
+
+    // 4b. Capability-derived variables resolve to a concrete value. The
+    // scan above only catches values that ARE present-but-templated; a
+    // required `source: capability` var whose chain is broken upstream is
+    // silently DROPPED during derivation (the hasUnresolved guard), so it's
+    // absent from selfConfig entirely and a template `$self:<x>` later dies
+    // with a cryptic "not found". Assert it here against the RESOLVED
+    // capabilities map so the operator gets the named missing link up front
+    // instead of at generate time (ISS-0095 / ISS-0115).
+    for (const problem of findBrokenCapabilityDerivations(
+      moduleId,
+      manifest,
+      context.capabilities,
+    )) {
+      errors.push({
+        category: problem.reason === 'no-provider' ? 'missing-capability' : 'unresolved-template',
+        message: describeCapabilityProblem(problem),
+        variable: problem.variable,
+        suggestion:
+          problem.reason === 'no-provider'
+            ? `Deploy a module that provides '${problem.capability}', then redeploy '${moduleId}'`
+            : `Redeploy '${problem.capability}'s provider so it populates ${problem.capability}.${problem.path}, then redeploy '${moduleId}'`,
+      });
+    }
   } catch (error) {
     // Resolution context may fail — that's informative too
     warnings.push({

package/src/services/dns-internal-records.test.ts

@@ -0,0 +1,126 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { DnsRecordRequest } from '@celilo/capabilities';
+import type { DbClient } from '../db/client';
+import { modules } from '../db/schema';
+import { setupTestDatabase } from '../test-utils/setup-test-db';
+import {
+  listDnsInternalRecords,
+  recordDnsInternalRecord,
+  removeDnsInternalRecord,
+  withDnsInternalLedger,
+} from './dns-internal-records';
+
+describe('dns-internal-records ledger', () => {
+  let dir: string;
+  let dbPath: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'dns-int-'));
+    dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    // FK targets: provider + consumer modules.
+    for (const id of ['technitium', 'forgejo']) {
+      db.insert(modules)
+        .values({
+          id,
+          name: id,
+          version: '1.0.0',
+          state: 'VERIFIED',
+          manifestData: {},
+          sourcePath: `/src/${id}`,
+        })
+        .run();
+    }
+  });
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  const ctx = () => ({ db, providerModuleId: 'technitium', consumerModuleId: 'forgejo' });
+
+  it('records, lists, and upserts by (provider, host)', () => {
+    recordDnsInternalRecord(db, { ...ctx(), host: 'git-ssh.x', ip: '10.0.20.14' });
+    let rows = listDnsInternalRecords(db);
+    expect(rows).toHaveLength(1);
+    expect(rows[0].ip).toBe('10.0.20.14');
+
+    // Same (provider, host) updates in place — the natIp fix re-registering.
+    recordDnsInternalRecord(db, { ...ctx(), host: 'git-ssh.x', ip: '192.168.0.253' });
+    rows = listDnsInternalRecords(db);
+    expect(rows).toHaveLength(1);
+    expect(rows[0].ip).toBe('192.168.0.253');
+  });
+
+  it('removes a record by (provider, host)', () => {
+    recordDnsInternalRecord(db, { ...ctx(), host: 'a.x', ip: '1.1.1.1' });
+    recordDnsInternalRecord(db, { ...ctx(), host: 'b.x', ip: '2.2.2.2' });
+    removeDnsInternalRecord(db, { providerModuleId: 'technitium', host: 'a.x' });
+    const rows = listDnsInternalRecords(db);
+    expect(rows.map((r) => r.host)).toEqual(['b.x']);
+  });
+
+  describe('withDnsInternalLedger', () => {
+    function fakeProvider() {
+      const calls: Array<['register' | 'delete', DnsRecordRequest]> = [];
+      const iface = {
+        async registerRecord(req: DnsRecordRequest) {
+          calls.push(['register', req]);
+        },
+        async deleteRecord(req: DnsRecordRequest) {
+          calls.push(['delete', req]);
+        },
+      };
+      return { iface, calls };
+    }
+
+    it('records A-record registrations and passes the call through', async () => {
+      const { iface, calls } = fakeProvider();
+      const wrapped = withDnsInternalLedger(iface, ctx());
+      await wrapped.registerRecord({ host: 'git-ssh.x', type: 'A', value: '192.168.0.253' });
+      expect(calls).toHaveLength(1); // underlying provider still called
+      const rows = listDnsInternalRecords(db);
+      expect(rows).toHaveLength(1);
+      expect(rows[0]).toMatchObject({ host: 'git-ssh.x', ip: '192.168.0.253' });
+    });
+
+    it('does NOT ledger non-A records', async () => {
+      const { iface } = fakeProvider();
+      const wrapped = withDnsInternalLedger(iface, ctx());
+      await wrapped.registerRecord({ host: 'mail.x', type: 'MX', value: 'mx.x' });
+      expect(listDnsInternalRecords(db)).toHaveLength(0);
+    });
+
+    it('removes the ledger row on A-record delete', async () => {
+      const { iface } = fakeProvider();
+      const wrapped = withDnsInternalLedger(iface, ctx());
+      await wrapped.registerRecord({ host: 'git-ssh.x', type: 'A', value: '192.168.0.253' });
+      await wrapped.deleteRecord({ host: 'git-ssh.x', type: 'A', value: '192.168.0.253' });
+      expect(listDnsInternalRecords(db)).toHaveLength(0);
+    });
+
+    it('does not write the ledger if the underlying register throws', async () => {
+      const iface = {
+        async registerRecord(): Promise<void> {
+          throw new Error('resolver down');
+        },
+        async deleteRecord(): Promise<void> {},
+      };
+      const wrapped = withDnsInternalLedger(iface, ctx());
+      await expect(
+        wrapped.registerRecord({ host: 'git-ssh.x', type: 'A', value: '1.2.3.4' }),
+      ).rejects.toThrow('resolver down');
+      expect(listDnsInternalRecords(db)).toHaveLength(0);
+    });
+  });
+});

package/src/services/dns-internal-records.ts

@@ -0,0 +1,119 @@
+/**
+ * Internal split-horizon DNS A-record ledger (designs/CELILO_DOCTOR_FLEET_DRIFT.md
+ * Phase 4, ISS-0094 / ISS-0111).
+ *
+ * The dns_internal capability (technitium/knot) registers records straight
+ * into the resolver's own DB, leaving celilo no offline record of what it
+ * asked to be served. This ledger — the internal-DNS sibling of
+ * `dns_registrations` — records every successful `registerRecord({type:'A'})`
+ * so `celilo system doctor` can assert, without a live resolver probe, that
+ * service hostnames resolve to the firewall natIp (LAN-reachable) rather than
+ * a zone-side container IP a LAN device can't route to.
+ *
+ * Row lifecycle is FK cascade — records die with their provider or consumer.
+ */
+
+import type { DnsInternalCapability, DnsRecordRequest } from '@celilo/capabilities';
+import { and, eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { dnsInternalRecords } from '../db/schema';
+
+export interface DnsInternalRecordRow {
+  host: string;
+  ip: string;
+  providerModuleId: string;
+  consumerModuleId: string;
+  registeredAt: Date;
+}
+
+export function listDnsInternalRecords(
+  db: DbClient,
+  options: { providerModuleId?: string } = {},
+): DnsInternalRecordRow[] {
+  const query = db
+    .select({
+      host: dnsInternalRecords.host,
+      ip: dnsInternalRecords.ip,
+      providerModuleId: dnsInternalRecords.providerModuleId,
+      consumerModuleId: dnsInternalRecords.consumerModuleId,
+      registeredAt: dnsInternalRecords.registeredAt,
+    })
+    .from(dnsInternalRecords);
+  return options.providerModuleId
+    ? query.where(eq(dnsInternalRecords.providerModuleId, options.providerModuleId)).all()
+    : query.all();
+}
+
+export function recordDnsInternalRecord(
+  db: DbClient,
+  record: { providerModuleId: string; consumerModuleId: string; host: string; ip: string },
+): void {
+  db.insert(dnsInternalRecords)
+    .values({
+      providerModuleId: record.providerModuleId,
+      consumerModuleId: record.consumerModuleId,
+      host: record.host,
+      ip: record.ip,
+      registeredAt: new Date(),
+    })
+    .onConflictDoUpdate({
+      target: [dnsInternalRecords.providerModuleId, dnsInternalRecords.host],
+      set: {
+        consumerModuleId: record.consumerModuleId,
+        ip: record.ip,
+        registeredAt: new Date(),
+      },
+    })
+    .run();
+}
+
+export function removeDnsInternalRecord(
+  db: DbClient,
+  record: { providerModuleId: string; host: string },
+): void {
+  db.delete(dnsInternalRecords)
+    .where(
+      and(
+        eq(dnsInternalRecords.providerModuleId, record.providerModuleId),
+        eq(dnsInternalRecords.host, record.host),
+      ),
+    )
+    .run();
+}
+
+/**
+ * Wrap a dns_internal interface so A-record register/delete calls are
+ * mirrored into the ledger. Only A records are tracked — they're the
+ * host→IP mapping the natIp drift check reasons about; CNAMEs/others pass
+ * through unrecorded. A throw from the underlying call propagates before
+ * any ledger write, so only a confirmed register/delete earns a row change.
+ */
+export function withDnsInternalLedger(
+  iface: DnsInternalCapability,
+  ctx: { db: DbClient; providerModuleId: string; consumerModuleId: string },
+): DnsInternalCapability {
+  const isA = (request: DnsRecordRequest) => request.type.toUpperCase() === 'A';
+  return {
+    ...iface,
+    async registerRecord(request: DnsRecordRequest): Promise<void> {
+      await iface.registerRecord(request);
+      if (isA(request)) {
+        recordDnsInternalRecord(ctx.db, {
+          providerModuleId: ctx.providerModuleId,
+          consumerModuleId: ctx.consumerModuleId,
+          host: request.host,
+          ip: request.value,
+        });
+      }
+    },
+    async deleteRecord(request: DnsRecordRequest): Promise<void> {
+      await iface.deleteRecord(request);
+      if (isA(request)) {
+        removeDnsInternalRecord(ctx.db, {
+          providerModuleId: ctx.providerModuleId,
+          host: request.host,
+        });
+      }
+    },
+  };
+}