@celilo/cli 0.5.0-alpha.1 → 0.5.0-alpha.11

package/src/db/schema-introspection.ts

@@ -0,0 +1,88 @@
+/**
+ * Schema introspection — compare the DB's actual tables/columns against the
+ * tables the running code's drizzle schema declares. The single source for
+ * "what does the code expect, and is it present?" shared by:
+ *   - the migration baseline (db/migrate.ts) — to decide which migrations are
+ *     already applied on an existing DB before running migrate() (ISS-0100), and
+ *   - the doctor's schema-drift check (services/fleet-checks.ts) — to report
+ *     missing tables/columns to the operator (ISS-0113).
+ */
+
+import type { Database } from 'bun:sqlite';
+import { is } from 'drizzle-orm';
+import { SQLiteTable, getTableConfig } from 'drizzle-orm/sqlite-core';
+import * as dbSchema from './schema';
+
+export interface SchemaDrift {
+  /** Tables the code's schema declares that the DB lacks. */
+  missingTables: string[];
+  /** `table.column` the code declares that the DB's table lacks. */
+  missingColumns: string[];
+  /** Total number of tables the code's schema declares. */
+  tableCount: number;
+}
+
+/** Every table name + column names the drizzle schema declares. */
+export function getSchemaTables(): Array<{ name: string; columns: string[] }> {
+  const out: Array<{ name: string; columns: string[] }> = [];
+  for (const value of Object.values(dbSchema)) {
+    if (!is(value, SQLiteTable)) continue;
+    const cfg = getTableConfig(value);
+    out.push({ name: cfg.name, columns: cfg.columns.map((c) => c.name) });
+  }
+  return out;
+}
+
+/** Names of all tables that physically exist in the DB. */
+export function getExistingTables(sqlite: Database): Set<string> {
+  return new Set(
+    sqlite
+      .query<{ name: string }, []>("SELECT name FROM sqlite_master WHERE type='table'")
+      .all()
+      .map((r) => r.name),
+  );
+}
+
+/** Names of all indexes that physically exist in the DB. */
+export function getExistingIndexes(sqlite: Database): Set<string> {
+  return new Set(
+    sqlite
+      .query<{ name: string }, []>("SELECT name FROM sqlite_master WHERE type='index'")
+      .all()
+      .map((r) => r.name),
+  );
+}
+
+/** Column names physically present on a table (empty if the table is absent). */
+export function getExistingColumns(sqlite: Database, table: string): Set<string> {
+  // `table` is a schema/migration identifier (never user input) — safe to inline.
+  return new Set(
+    sqlite
+      .query<{ name: string }, []>(`SELECT name FROM pragma_table_info('${table}')`)
+      .all()
+      .map((r) => r.name),
+  );
+}
+
+/**
+ * Compare the running code's drizzle schema to the DB and report what's missing.
+ * Track-agnostic: it reads actual table/column presence, so it's honest whether
+ * the DB was migrated, baselined, or hand-patched.
+ */
+export function findSchemaDrift(sqlite: Database): SchemaDrift {
+  const present = getExistingTables(sqlite);
+  const missingTables: string[] = [];
+  const missingColumns: string[] = [];
+  const tables = getSchemaTables();
+  for (const t of tables) {
+    if (!present.has(t.name)) {
+      missingTables.push(t.name);
+      continue;
+    }
+    const cols = getExistingColumns(sqlite, t.name);
+    for (const c of t.columns) {
+      if (!cols.has(c)) missingColumns.push(`${t.name}.${c}`);
+    }
+  }
+  return { missingTables, missingColumns, tableCount: tables.length };
+}

package/src/db/schema.ts

@@ -432,6 +432,80 @@ export const webRoutes = sqliteTable(
   }),
 );
 
+/**
+ * DNS registration ledger — one row per (provider, fqdn) the framework
+ * has successfully registered via dns_registrar.registerHost. Written
+ * framework-side by the capability loader (the only layer that knows
+ * both consumer and provider), read back to drive the provider's
+ * periodic refresh_registrations hook and the `celilo dns
+ * registrations` view. Rows die with either module via FK cascade; the
+ * remote DNS record itself stays (Namecheap DDNS has no delete API).
+ * See designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B2).
+ */
+export const dnsRegistrations = sqliteTable(
+  'dns_registrations',
+  {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    providerModuleId: text('provider_module_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    consumerModuleId: text('consumer_module_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    fqdn: text('fqdn').notNull(),
+    /** null = provider auto-detected the request's source IP (Namecheap-style). */
+    ip: text('ip'),
+    registeredAt: integer('registered_at', { mode: 'timestamp' })
+      .notNull()
+      .default(sql`(unixepoch())`),
+    refreshedAt: integer('refreshed_at', { mode: 'timestamp' }),
+  },
+  (table) => ({
+    providerFqdnUnique: uniqueIndex('dns_registrations_provider_fqdn_idx').on(
+      table.providerModuleId,
+      table.fqdn,
+    ),
+  }),
+);
+
+/**
+ * Internal split-horizon DNS A-record ledger. Mirrors `dns_registrations`
+ * but for the dns_internal capability (technitium/knot): the capability
+ * loader records every `dns_internal.registerRecord({type:'A'})` here so
+ * celilo has an offline, queryable record of what hostname → IP it asked
+ * the internal resolver to serve. Without this, the only source of truth
+ * is the resolver's own DB, requiring a live probe (ISS-0094 / ISS-0111).
+ *
+ * `celilo system doctor` reads this to assert service hostnames resolve to
+ * the firewall natIp (LAN-reachable) and not a zone-side container IP that
+ * a LAN device can't route to. Rows die with either module via FK cascade.
+ */
+export const dnsInternalRecords = sqliteTable(
+  'dns_internal_records',
+  {
+    id: integer('id').primaryKey({ autoIncrement: true }),
+    providerModuleId: text('provider_module_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    consumerModuleId: text('consumer_module_id')
+      .notNull()
+      .references(() => modules.id, { onDelete: 'cascade' }),
+    /** The registered hostname (e.g. "git-ssh.git.celilo.computer"). */
+    host: text('host').notNull(),
+    /** The A-record value celilo asked the resolver to serve. */
+    ip: text('ip').notNull(),
+    registeredAt: integer('registered_at', { mode: 'timestamp' })
+      .notNull()
+      .default(sql`(unixepoch())`),
+  },
+  (table) => ({
+    providerHostUnique: uniqueIndex('dns_internal_records_provider_host_idx').on(
+      table.providerModuleId,
+      table.host,
+    ),
+  }),
+);
+
 /**
  * Backup storage providers - destinations for backup archives
  * Supports local filesystem and S3-compatible storage (AWS S3, MinIO, Backblaze B2, Wasabi)

package/src/hooks/capability-loader-firewall.test.ts

@@ -37,7 +37,7 @@ export default defineCapabilityFunction({
   capability: 'firewall',
   handler: ({ config, secrets }) => ({
     exposeService: async (opts) => ({
-      externalIp: '71.36.99.96',
+      externalIp: '203.0.113.10',
       natIp: opts.internalIp,
     }),
     unexposeService: async () => {},
@@ -135,7 +135,7 @@ describe('Firewall Chain Building', () => {
       ports: [80],
       description: 'test',
     });
-    expect(exposed.externalIp).toBe('71.36.99.96');
+    expect(exposed.externalIp).toBe('203.0.113.10');
   });
 
   test('builds chain with two providers (iptables → greenwave)', async () => {
@@ -206,7 +206,7 @@ describe('Firewall Chain Building', () => {
     });
 
     // External IP came from greenwave (leaf)
-    expect(exposed.externalIp).toBe('71.36.99.96');
+    expect(exposed.externalIp).toBe('203.0.113.10');
     // NAT IP is iptables' NAT address
     expect(exposed.natIp).toBe('192.168.0.253');
     // iptables created local rules

package/src/hooks/capability-loader.ts

@@ -19,6 +19,7 @@ import {
 } from '@celilo/capabilities';
 import type {
   DnsInternalCapability,
+  DnsRegistrarCapability,
   HookLogger,
   RouteOps,
   RouteReadView,
@@ -30,6 +31,8 @@ import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
 import { getModuleSystems } from '../services/deployed-systems';
+import { withDnsInternalLedger } from '../services/dns-internal-records';
+import { withDnsRegistrationLedger } from '../services/dns-registrations';
 import { loadHookConfigMap } from './load-hook-config';
 
 /**
@@ -60,6 +63,10 @@ const CAPABILITY_MODULE_MAP: Record<string, { script: string; legacyFactoryName:
     script: 'scripts/idp-functions.ts',
     legacyFactoryName: 'createIdp',
   },
+  git_forge: {
+    script: 'scripts/git-forge-functions.ts',
+    legacyFactoryName: 'createForgejoGitForge',
+  },
   dhcp_server: {
     script: 'scripts/dhcp-server-functions.ts',
     legacyFactoryName: 'default',
@@ -194,6 +201,28 @@ export async function loadCapabilityFunctions(
     const providerConfig = await loadModuleConfig(capability.moduleId, db);
     const providerSecrets = await loadModuleSecrets(capability.moduleId, masterKey, db);
 
+    // dns_registrar and dns_internal interfaces get a registration-ledger
+    // wrapper: every successful registerHost / registerRecord is recorded so
+    // celilo has an offline record of what it asked DNS to serve. The
+    // external ledger feeds the provider's refresh_registrations hook
+    // (DISPATCHER_DAEMON_AND_TIMER_EVENTS.md B2); the internal ledger feeds
+    // the doctor's natIp drift check (CELILO_DOCTOR_FLEET_DRIFT.md Phase 4).
+    // The loader is the one layer that knows both provider and consumer.
+    const ledgerCtx = {
+      db,
+      providerModuleId: capability.moduleId,
+      consumerModuleId: consumingModuleId,
+    };
+    const withLedger = (iface: unknown): unknown => {
+      if (capName === 'dns_registrar') {
+        return withDnsRegistrationLedger(iface as DnsRegistrarCapability, ledgerCtx);
+      }
+      if (capName === 'dns_internal') {
+        return withDnsInternalLedger(iface as DnsInternalCapability, ledgerCtx);
+      }
+      return iface;
+    };
+
     try {
       // Dynamically import the capability module. Try the default export
       // first (the HOOK_API_V2 Phase 8 pattern), fall back to the legacy
@@ -217,7 +246,7 @@ export async function loadCapabilityFunctions(
           systems: getModuleSystems(capability.moduleId, db),
           logger,
         });
-        result[capName] = capabilityInterface;
+        result[capName] = withLedger(capabilityInterface);
         debugLog(`${capName}: loaded via defineCapabilityFunction`);
         continue;
       }
@@ -233,7 +262,9 @@ export async function loadCapabilityFunctions(
       );
 
       if (capabilityInterface) {
-        result[capName] = wrapWithLogging(capabilityInterface as object, logger, capName);
+        result[capName] = withLedger(
+          wrapWithLogging(capabilityInterface as object, logger, capName),
+        );
         debugLog(`${capName}: loaded via legacy factory`);
       }
     } catch (error) {
@@ -388,6 +419,16 @@ export async function loadCapabilityFunctions(
               `public_web reconcile for ${consumingModuleId}: ${reconcile.failed} delivery(ies) failed — caddy could not apply the route, so the hostname is not served.`,
             );
           }
+          // ISS-0087: a route WAS registered and the dispatcher IS alive, yet ZERO
+          // providers reconciled it (no subscriber consumed routes_changed). That
+          // registers a route nobody applied and would otherwise report success.
+          // `events === 0` means no routes changed (benign); `events > 0 &&
+          // succeeded === 0` means the route changed but nobody served it — a failure.
+          if (reconcile.events > 0 && reconcile.succeeded === 0) {
+            throw new Error(
+              `public_web route for ${consumingModuleId} changed (${reconcile.events} event(s)) but NO provider reconciled it — caddy has no reconcile_routes subscription on the bus, so the route is persisted yet never served (no site block, no cert). Ensure a public_web provider (caddy) is deployed and subscribed.`,
+            );
+          }
         },
       });
       debugLog(`public_web: loaded via framework implementation for ${consumingModuleId}`);

package/src/hooks/run-named-hook.ts

@@ -19,6 +19,10 @@ import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { getModuleSystems } from '../services/deployed-systems';
+import {
+  listDnsRegistrations,
+  stampDnsRegistrationsRefreshed,
+} from '../services/dns-registrations';
 import { loadCapabilityFunctions } from './capability-loader';
 import { invokeHook } from './executor';
 import { loadHookConfigMap } from './load-hook-config';
@@ -125,12 +129,26 @@ export async function runNamedHook(
     : [];
   const capabilityFunctions = await loadCapabilityFunctions(moduleId, db, logger);
 
-  return invokeHook(
+  // refresh_registrations is fed by the framework: module scripts can't
+  // read the celilo DB, so the dns_registrations ledger rows for THIS
+  // provider are injected as the contract's `registrations` input
+  // (designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md B3). Authoritative —
+  // overrides any caller-supplied value.
+  let inputs = options.inputs ?? {};
+  if (hookName === 'refresh_registrations') {
+    const registrations = listDnsRegistrations(db, { providerModuleId: moduleId }).map((r) => ({
+      fqdn: r.fqdn,
+      ip: r.ip,
+    }));
+    inputs = { ...inputs, registrations };
+  }
+
+  const result = await invokeHook(
     module.sourcePath,
     hookName,
     manifest.celilo_contract,
     hookDef,
-    options.inputs ?? {},
+    inputs,
     configMap,
     secretMap,
     logger,
@@ -141,4 +159,12 @@ export async function runNamedHook(
       systems: getModuleSystems(moduleId, db),
     },
   );
+
+  // A fully successful refresh stamps the ledger so `celilo dns
+  // registrations` shows when each provider last re-asserted.
+  if (hookName === 'refresh_registrations' && result.success) {
+    stampDnsRegistrationsRefreshed(db, moduleId);
+  }
+
+  return result;
 }

package/src/hooks/types.ts

@@ -119,7 +119,8 @@ export type HookName =
   | 'on_backup'
   | 'on_backup_analyze'
   | 'on_restore'
-  | 'on_system_event';
+  | 'on_system_event'
+  | 'refresh_registrations';
 
 /**
  * Hook manifest section - maps hook names to definitions

package/src/infrastructure/property-extractor.test.ts

@@ -78,6 +78,21 @@ describe('extractProxmoxProperties', () => {
     });
   });
 
+  test('omits vm_template when the service config has none (LXC services)', () => {
+    const properties = extractProxmoxProperties(100, '10.0.10.5', 'caddy', mockProxmoxConfig);
+    // Omitted, not empty — so a required-infrastructure `vm_template` var fails
+    // loudly rather than resolving to "".
+    expect('vm_template' in properties).toBe(false);
+  });
+
+  test('includes vm_template when the service config provides one (VM services)', () => {
+    const properties = extractProxmoxProperties(100, '10.0.10.5', 'builder', {
+      ...mockProxmoxConfig,
+      vm_template: 'ubuntu-2204-cloudinit',
+    });
+    expect(properties.vm_template).toBe('ubuntu-2204-cloudinit');
+  });
+
   test('converts vmid number to string', () => {
     const properties = extractProxmoxProperties(12345, '10.0.20.15', 'caddy', mockProxmoxConfig);
 

package/src/infrastructure/property-extractor.ts

@@ -47,6 +47,13 @@ export interface ProxmoxProviderConfig {
   default_target_node: string;
   lxc_template: string;
   storage: string;
+  /**
+   * Cloud-init VM template to clone for `requires.system.type: vm` modules — the
+   * VM analogue of `lxc_template`. Optional: only Proxmox services that host VM
+   * modules configure it. A VM module declares `vm_template` as a *required*
+   * infrastructure var, so a service missing it fails loudly at resolution.
+   */
+  vm_template?: string;
 }
 
 /**
@@ -72,6 +79,11 @@ export function extractProxmoxProperties(
     target_node: providerConfig.default_target_node,
     lxc_template: providerConfig.lxc_template,
     storage: providerConfig.storage,
+    // VM clone source — present only when the service configures it. VM modules
+    // declare `vm_template` as a required infrastructure var (resolution errors
+    // if absent); LXC modules never reference it. Omitted (not empty) when unset
+    // so the resolver's required/optional handling stays correct.
+    ...(providerConfig.vm_template ? { vm_template: providerConfig.vm_template } : {}),
   };
 }
 

package/src/manifest/contracts/v1.ts

@@ -178,6 +178,22 @@ export const V1_HOOKS: ContractHooks = {
     inputs: {},
     outputs: {},
   },
+  /**
+   * Periodic re-assertion of a dns_registrar provider's registered
+   * records (designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md B3). The
+   * framework populates `registrations` from the dns_registrations
+   * ledger (module scripts cannot read the celilo DB); the hook
+   * re-sends each {fqdn, ip} to the underlying DNS API and fails —
+   * loudly — if ANY record cannot be re-asserted. Typically driven by
+   * a `timer.tick.15m` subscription.
+   */
+  refresh_registrations: {
+    inputs: {
+      /** Array<{ fqdn: string; ip: string | null }>, framework-injected. */
+      registrations: { required: true },
+    },
+    outputs: {},
+  },
   /**
    * Build-bus upstream publish hook. The executor passes the
    * PublishEvent fields as env vars (CELILO_EVENT_PAYLOAD,

package/src/manifest/schema.ts

@@ -311,6 +311,13 @@ export const SystemResourceSchema = z.object({
   memory: z.number().int().positive().optional().describe('Recommended memory in MB'),
   disk: z.number().int().positive().optional().describe('Recommended disk size in GB'),
   storage: z.string().optional().describe('Proxmox storage backend (defaults to system config)'),
+  type: z
+    .enum(['lxc', 'vm'])
+    .default('lxc')
+    .describe(
+      'Proxmox provisioning type: lxc (default) or vm (qemu, for Docker / kernel-module workloads). ' +
+        'Modules declare this explicitly; celilo never infers it. Moot for machine-pool / external infra.',
+    ),
   zone: z
     .enum(['internal', 'dmz', 'app', 'secure', 'external'])
     .describe('Required security zone for this module'),
@@ -523,6 +530,16 @@ export const ModuleManifestSchema = z
          * [[v2/PUBLIC_WEB_PROVIDER_RECONCILE.md]].
          */
         reconcile_routes: LifecycleHookSchema.optional(),
+        /**
+         * Periodic re-assertion of a dns_registrar provider's registered
+         * records. The framework injects the provider's dns_registrations
+         * ledger rows as the `registrations` input; the hook re-sends each
+         * one to the underlying DNS API and fails loudly if any cannot be
+         * re-asserted. Providers subscribe it to a `timer.tick.*` event.
+         * Part of the dns_registrar capability contract — see
+         * designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B3).
+         */
+        refresh_registrations: LifecycleHookSchema.optional(),
         /**
          * Build-bus upstream publish hooks. Array (a module can react
          * to multiple upstream packages with different actions). See

package/src/manifest/validate.test.ts

@@ -77,6 +77,59 @@ variables:
     }
   });
 
+  test('requires.system.type defaults to lxc when omitted', () => {
+    const yaml = `
+${CONTRACT_LINE}
+id: homebridge
+name: Homebridge
+version: 1.0.0
+requires:
+  system:
+    cpu: 1
+    zone: app
+`;
+    const result = validateManifest(yaml);
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.requires.system?.type).toBe('lxc');
+    }
+  });
+
+  test('requires.system.type accepts an explicit vm', () => {
+    const yaml = `
+${CONTRACT_LINE}
+id: forgejo-runner
+name: Forgejo Runner
+version: 1.0.0
+requires:
+  system:
+    cpu: 4
+    memory: 8192
+    type: vm
+    zone: dmz
+`;
+    const result = validateManifest(yaml);
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.requires.system?.type).toBe('vm');
+    }
+  });
+
+  test('requires.system.type rejects an unknown infra type', () => {
+    const yaml = `
+${CONTRACT_LINE}
+id: homebridge
+name: Homebridge
+version: 1.0.0
+requires:
+  system:
+    type: container
+    zone: app
+`;
+    const result = validateManifest(yaml);
+    expect(result.success).toBe(false);
+  });
+
   test('should validate dns-external manifest with capability provider', () => {
     const yaml = `
 ${CONTRACT_LINE}

package/src/services/bus-interview.test.ts

@@ -42,7 +42,7 @@ describe('busInterview', () => {
     const watch = responderBus.watch('config.required.lunacycle.domain', (event) => {
       responderBus.emitRaw(
         `${event.type}.reply`,
-        { value: 'lunacycle.net' },
+        { value: 'example.net' },
         { replyFor: event.id, emittedBy: 'test-responder' },
       );
     });
@@ -58,7 +58,7 @@ describe('busInterview', () => {
       payload,
     );
 
-    expect(reply.value).toBe('lunacycle.net');
+    expect(reply.value).toBe('example.net');
 
     watch.close();
     responderBus.close();