@celilo/cli 0.5.0-alpha.3 → 0.5.0-alpha.5

package/src/services/dns-internal-records.ts

@@ -0,0 +1,119 @@
+/**
+ * Internal split-horizon DNS A-record ledger (designs/CELILO_DOCTOR_FLEET_DRIFT.md
+ * Phase 4, ISS-0094 / ISS-0111).
+ *
+ * The dns_internal capability (technitium/knot) registers records straight
+ * into the resolver's own DB, leaving celilo no offline record of what it
+ * asked to be served. This ledger — the internal-DNS sibling of
+ * `dns_registrations` — records every successful `registerRecord({type:'A'})`
+ * so `celilo system doctor` can assert, without a live resolver probe, that
+ * service hostnames resolve to the firewall natIp (LAN-reachable) rather than
+ * a zone-side container IP a LAN device can't route to.
+ *
+ * Row lifecycle is FK cascade — records die with their provider or consumer.
+ */
+
+import type { DnsInternalCapability, DnsRecordRequest } from '@celilo/capabilities';
+import { and, eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { dnsInternalRecords } from '../db/schema';
+
+export interface DnsInternalRecordRow {
+  host: string;
+  ip: string;
+  providerModuleId: string;
+  consumerModuleId: string;
+  registeredAt: Date;
+}
+
+export function listDnsInternalRecords(
+  db: DbClient,
+  options: { providerModuleId?: string } = {},
+): DnsInternalRecordRow[] {
+  const query = db
+    .select({
+      host: dnsInternalRecords.host,
+      ip: dnsInternalRecords.ip,
+      providerModuleId: dnsInternalRecords.providerModuleId,
+      consumerModuleId: dnsInternalRecords.consumerModuleId,
+      registeredAt: dnsInternalRecords.registeredAt,
+    })
+    .from(dnsInternalRecords);
+  return options.providerModuleId
+    ? query.where(eq(dnsInternalRecords.providerModuleId, options.providerModuleId)).all()
+    : query.all();
+}
+
+export function recordDnsInternalRecord(
+  db: DbClient,
+  record: { providerModuleId: string; consumerModuleId: string; host: string; ip: string },
+): void {
+  db.insert(dnsInternalRecords)
+    .values({
+      providerModuleId: record.providerModuleId,
+      consumerModuleId: record.consumerModuleId,
+      host: record.host,
+      ip: record.ip,
+      registeredAt: new Date(),
+    })
+    .onConflictDoUpdate({
+      target: [dnsInternalRecords.providerModuleId, dnsInternalRecords.host],
+      set: {
+        consumerModuleId: record.consumerModuleId,
+        ip: record.ip,
+        registeredAt: new Date(),
+      },
+    })
+    .run();
+}
+
+export function removeDnsInternalRecord(
+  db: DbClient,
+  record: { providerModuleId: string; host: string },
+): void {
+  db.delete(dnsInternalRecords)
+    .where(
+      and(
+        eq(dnsInternalRecords.providerModuleId, record.providerModuleId),
+        eq(dnsInternalRecords.host, record.host),
+      ),
+    )
+    .run();
+}
+
+/**
+ * Wrap a dns_internal interface so A-record register/delete calls are
+ * mirrored into the ledger. Only A records are tracked — they're the
+ * host→IP mapping the natIp drift check reasons about; CNAMEs/others pass
+ * through unrecorded. A throw from the underlying call propagates before
+ * any ledger write, so only a confirmed register/delete earns a row change.
+ */
+export function withDnsInternalLedger(
+  iface: DnsInternalCapability,
+  ctx: { db: DbClient; providerModuleId: string; consumerModuleId: string },
+): DnsInternalCapability {
+  const isA = (request: DnsRecordRequest) => request.type.toUpperCase() === 'A';
+  return {
+    ...iface,
+    async registerRecord(request: DnsRecordRequest): Promise<void> {
+      await iface.registerRecord(request);
+      if (isA(request)) {
+        recordDnsInternalRecord(ctx.db, {
+          providerModuleId: ctx.providerModuleId,
+          consumerModuleId: ctx.consumerModuleId,
+          host: request.host,
+          ip: request.value,
+        });
+      }
+    },
+    async deleteRecord(request: DnsRecordRequest): Promise<void> {
+      await iface.deleteRecord(request);
+      if (isA(request)) {
+        removeDnsInternalRecord(ctx.db, {
+          providerModuleId: ctx.providerModuleId,
+          host: request.host,
+        });
+      }
+    },
+  };
+}

package/src/services/fleet-checks.test.ts

@@ -0,0 +1,495 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
+import type { DbClient } from '../db/client';
+import {
+  capabilities as capabilitiesTable,
+  dnsInternalRecords,
+  moduleConfigs,
+  moduleSystems,
+  modules,
+} from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
+import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { getDaemonUnitPath } from './events-daemon';
+import {
+  checkCapabilityProviders,
+  checkDispatcher,
+  checkSchemaDrift,
+  checkServiceDns,
+  checkSubscribers,
+  describeCapabilityProblem,
+  findBrokenCapabilityDerivations,
+} from './fleet-checks';
+
+const MINUTE = 60_000;
+
+/** Seed a dispatcher heartbeat row directly (bypasses the running loop). */
+function seedHeartbeat(
+  bus: Bus,
+  opts: { startedAt: number; lastHeartbeat: number; pid?: number; version?: string },
+): void {
+  bus.db.run(
+    'INSERT INTO dispatcher_heartbeat (dispatcher_id, last_heartbeat, started_at, pid, version) VALUES (?, ?, ?, ?, ?)',
+    ['d1', opts.lastHeartbeat, opts.startedAt, opts.pid ?? 4242, opts.version ?? '0.1.0'],
+  );
+}
+
+/** Write a supervisor unit file so readInstalledUnit('user') sees it. */
+function installFakeUnit(home: string): void {
+  const path = getDaemonUnitPath('linux', home, 'user');
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, '[Unit]\nDescription=fake\n');
+}
+
+const baseManifest = (overrides: Partial<ModuleManifest> = {}): ModuleManifest => ({
+  celilo_contract: '1.0',
+  id: 'mod',
+  name: 'Mod',
+  version: '1.0.0',
+  requires: { capabilities: [] },
+  provides: { capabilities: [] },
+  variables: { owns: [], imports: [] },
+  ...overrides,
+});
+
+describe('checkDispatcher', () => {
+  let dir: string;
+  let dbPath: string;
+  let home: string;
+  let bus: Bus;
+  // Use real wall-clock: bus.health() reads Date.now() internally, so a
+  // synthetic past `now` would make every seeded heartbeat look stale.
+  let now: number;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'fleet-disp-'));
+    dbPath = join(dir, 'events.db');
+    home = join(dir, 'home');
+    now = Date.now();
+    process.env.EVENT_BUS_DB = dbPath;
+    bus = openBus({ dbPath, events: defineEvents({}) });
+  });
+  afterEach(() => {
+    bus.close();
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('fails when no dispatcher has ever heartbeated', () => {
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    expect(f.status).toBe('fail');
+    expect(f.summary).toContain('no live event dispatcher');
+    expect(f.remediation).toContain('enable --now celilo-events.service');
+  });
+
+  it('passes when running, supervised, current, and not behind on timer ticks', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    bus.subscribe({ name: 'namecheap.ddns', pattern: 'timer.tick.15m', handler: 'echo' });
+    bus.emitRaw('timer.tick.15m', { interval: '15m' });
+
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      platform: 'linux',
+      installedCodeMtimeMs: now - 2 * MINUTE, // code installed BEFORE the dispatcher started
+    });
+    expect(f.status).toBe('ok');
+  });
+
+  it('warns when the dispatcher is an orphan (no supervisor unit)', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    // no unit file written
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    expect(f.status).toBe('warn');
+    expect(f.detail.join(' ')).toContain('not under a supervisor');
+  });
+
+  it('warns when the dispatcher started before the installed code (stale)', () => {
+    seedHeartbeat(bus, { startedAt: now - 10 * MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      platform: 'linux',
+      installedCodeMtimeMs: now - 5 * MINUTE, // code newer than the running dispatcher
+    });
+    expect(f.status).toBe('warn');
+    expect(f.detail.join(' ')).toContain('stale code');
+  });
+
+  it('warns when a timer subscriber exists but no tick was ever emitted', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    bus.subscribe({ name: 'namecheap.ddns', pattern: 'timer.tick.15m', handler: 'echo' });
+    // no tick emitted
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    expect(f.status).toBe('warn');
+    expect(f.detail.join(' ')).toContain('no such tick has ever been emitted');
+  });
+
+  it('warns when the newest timer tick is older than the window', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    bus.subscribe({ name: 'namecheap.ddns', pattern: 'timer.tick.15m', handler: 'echo' });
+    bus.emitRaw('timer.tick.15m', { interval: '15m' });
+    // tick exists but is ancient relative to now (emitRaw stamps Date.now()).
+    bus.db.run("UPDATE events SET emitted_at = ? WHERE type = 'timer.tick.15m'", [
+      now - 30 * MINUTE,
+    ]);
+
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    expect(f.status).toBe('warn');
+    expect(f.detail.join(' ')).toContain('not emitting on schedule');
+  });
+});
+
+describe('checkSubscribers + checkCapabilityProviders', () => {
+  let dir: string;
+  let dbPath: string;
+  let busPath: string;
+  let db: DbClient;
+  let bus: Bus;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'fleet-subs-'));
+    dbPath = join(dir, 'celilo.db');
+    busPath = join(dir, 'events.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    process.env.EVENT_BUS_DB = busPath;
+    db = await setupTestDatabase(dbPath);
+    bus = openBus({ dbPath: busPath, events: defineEvents({}) });
+  });
+  afterEach(() => {
+    bus.close();
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.EVENT_BUS_DB = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function insertModule(id: string, manifest: ModuleManifest, state = 'VERIFIED' as const): void {
+    db.insert(modules)
+      .values({
+        id,
+        name: manifest.name,
+        version: manifest.version,
+        state,
+        manifestData: manifest as unknown as Record<string, unknown>,
+        sourcePath: `/src/${id}`,
+      })
+      .run();
+  }
+
+  describe('checkSubscribers', () => {
+    it('fails when a deployed module declares a subscription the bus is missing', () => {
+      insertModule(
+        'lunacycle',
+        baseManifest({
+          id: 'lunacycle',
+          subscriptions: [{ name: 'smoke', pattern: 'deploy.completed.$self', handler: 'echo' }],
+        }),
+      );
+      const f = checkSubscribers(bus, db);
+      expect(f.status).toBe('fail');
+      expect(f.autoFixable).toBe(true);
+      expect(f.detail.join(' ')).toContain('lunacycle.smoke');
+      expect(f.remediation).toContain('resync-subscriptions');
+    });
+
+    it('passes once the subscription is registered on the bus', () => {
+      insertModule(
+        'lunacycle',
+        baseManifest({
+          id: 'lunacycle',
+          subscriptions: [{ name: 'smoke', pattern: 'deploy.completed.$self', handler: 'echo' }],
+        }),
+      );
+      bus.subscribe({
+        name: 'lunacycle.smoke',
+        pattern: 'deploy.completed.lunacycle',
+        handler: 'echo',
+      });
+      const f = checkSubscribers(bus, db);
+      expect(f.status).toBe('ok');
+    });
+
+    it('warns about a stale subscriber with no deployed module', () => {
+      // No modules deployed, but a leftover subscriber lingers.
+      bus.subscribe({ name: 'ghost.sub', pattern: 'x', handler: 'echo' });
+      const f = checkSubscribers(bus, db);
+      expect(f.status).toBe('warn');
+      expect(f.detail.join(' ')).toContain('ghost.sub');
+    });
+  });
+
+  describe('checkCapabilityProviders', () => {
+    const consumer = (deriveFrom: string) =>
+      baseManifest({
+        id: 'forgejo',
+        name: 'Forgejo',
+        variables: {
+          owns: [
+            {
+              name: 'idp_dmz_ip',
+              type: 'string',
+              required: true,
+              source: 'capability',
+              derive_from: deriveFrom,
+            },
+          ],
+          imports: [],
+        },
+      });
+
+    it('fails when the consumed capability has no deployed provider', () => {
+      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      const f = checkCapabilityProviders(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain("no deployed module provides 'idp'");
+    });
+
+    it('fails when the provider lacks the referenced field', () => {
+      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
+      db.insert(capabilitiesTable)
+        .values({
+          moduleId: 'authentik',
+          capabilityName: 'idp',
+          version: '1.0.0',
+          data: { auth_url: 'x' },
+        })
+        .run();
+      const f = checkCapabilityProviders(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain('has no value there');
+    });
+
+    it('passes when the provider carries a concrete value', () => {
+      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
+      db.insert(capabilitiesTable)
+        .values({
+          moduleId: 'authentik',
+          capabilityName: 'idp',
+          version: '1.0.0',
+          data: { dmz_ip: '10.0.10.10' },
+        })
+        .run();
+      const f = checkCapabilityProviders(db);
+      expect(f.status).toBe('ok');
+    });
+
+    it('passes but flags a derived ref for the chain trace (ISS-0114)', () => {
+      // The workstream-B case: idp.dmz_ip is present but is itself a ref
+      // ($self:caddy_dmz_ip) — we can't verify it resolves without the walker.
+      insertModule('forgejo', consumer('$capability:idp.dmz_ip'));
+      insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
+      db.insert(capabilitiesTable)
+        .values({
+          moduleId: 'authentik',
+          capabilityName: 'idp',
+          version: '1.0.0',
+          data: { dmz_ip: '$self:caddy_dmz_ip' },
+        })
+        .run();
+      const f = checkCapabilityProviders(db);
+      expect(f.status).toBe('ok');
+      expect(f.detail.join(' ')).toContain('celilo capability chain');
+    });
+  });
+
+  describe('checkServiceDns', () => {
+    const NAT_IP = '192.168.0.253';
+
+    function seedFirewall(moduleId: string, natIp: string): void {
+      insertModule(moduleId, baseManifest({ id: moduleId, name: moduleId }));
+      db.insert(capabilitiesTable)
+        .values({ moduleId, capabilityName: 'firewall', version: '1.0.0', data: {} })
+        .run();
+      db.insert(moduleConfigs)
+        .values({ moduleId, key: 'nat_ip', value: natIp, valueJson: JSON.stringify(natIp) })
+        .run();
+    }
+
+    function seedSystem(moduleId: string, ip: string, zone: 'dmz' | 'internal'): void {
+      db.insert(moduleSystems)
+        .values({
+          moduleId,
+          name: 'main',
+          hostname: moduleId,
+          ipv4Address: ip,
+          zone,
+          infraType: 'container_service',
+        })
+        .run();
+    }
+
+    function seedRecord(provider: string, consumer: string, host: string, ip: string): void {
+      db.insert(dnsInternalRecords)
+        .values({ providerModuleId: provider, consumerModuleId: consumer, host, ip })
+        .run();
+    }
+
+    it('is ok when no internal DNS records are registered', async () => {
+      const f = await checkServiceDns(db);
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('no internal DNS records');
+    });
+
+    it('skips (ok) when records exist but no firewall natIp is configured', async () => {
+      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
+      insertModule('caddy', baseManifest({ id: 'caddy', name: 'Caddy' }));
+      seedRecord('technitium', 'caddy', 'git.celilo.computer', '10.0.10.10');
+      const f = await checkServiceDns(db);
+      expect(f.status).toBe('ok');
+      expect(f.detail.join(' ')).toContain('no firewall');
+    });
+
+    it('is ok when a service record points at the natIp', async () => {
+      seedFirewall('iptables', NAT_IP);
+      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
+      insertModule('caddy', baseManifest({ id: 'caddy', name: 'Caddy' }));
+      seedRecord('technitium', 'caddy', 'git.celilo.computer', NAT_IP);
+      const f = await checkServiceDns(db);
+      expect(f.status).toBe('ok');
+    });
+
+    it('fails when a record points at a segmented-zone container IP', async () => {
+      seedFirewall('iptables', NAT_IP);
+      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
+      insertModule('forgejo', baseManifest({ id: 'forgejo', name: 'Forgejo' }));
+      seedSystem('forgejo', '10.0.20.14', 'dmz');
+      seedRecord('technitium', 'forgejo', 'git-ssh.git.celilo.computer', '10.0.20.14');
+      const f = await checkServiceDns(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain('dmz-zone container IP');
+      expect(f.remediation).toContain('natIp');
+    });
+
+    it('is ok when a record points at an internal-zone (LAN) system IP', async () => {
+      seedFirewall('iptables', NAT_IP);
+      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
+      insertModule('homebridge', baseManifest({ id: 'homebridge', name: 'Homebridge' }));
+      seedSystem('homebridge', '192.168.0.50', 'internal');
+      seedRecord('technitium', 'homebridge', 'hb.celilo.computer', '192.168.0.50');
+      const f = await checkServiceDns(db);
+      expect(f.status).toBe('ok');
+    });
+
+    it('warns when a record points at neither the natIp nor a known system IP', async () => {
+      seedFirewall('iptables', NAT_IP);
+      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
+      insertModule('caddy', baseManifest({ id: 'caddy', name: 'Caddy' }));
+      seedRecord('technitium', 'caddy', 'stale.celilo.computer', '203.0.113.9');
+      const f = await checkServiceDns(db);
+      expect(f.status).toBe('warn');
+      expect(f.detail.join(' ')).toContain('known system IP');
+    });
+
+    it('does not crash when the ledger table is missing (defers to schema check)', async () => {
+      db.$client.run('DROP TABLE dns_internal_records');
+      const f = await checkServiceDns(db);
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('ledger not present');
+    });
+  });
+
+  describe('checkSchemaDrift', () => {
+    it('is ok when every schema table is present (fresh migrated DB)', () => {
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('schema tables present');
+    });
+
+    it('fails and names a table the running CLI expects but the DB lacks', () => {
+      db.$client.run('DROP TABLE dns_internal_records');
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain('dns_internal_records');
+      expect(f.remediation).toContain('migrations');
+    });
+  });
+});
+
+describe('findBrokenCapabilityDerivations (shared predicate)', () => {
+  const consumer = baseManifest({
+    id: 'forgejo',
+    variables: {
+      owns: [
+        {
+          name: 'idp_dmz_ip',
+          type: 'string',
+          required: true,
+          source: 'capability',
+          derive_from: '$capability:idp.dmz_ip',
+        },
+      ],
+      imports: [],
+    },
+  });
+
+  it('flags no-provider when the capability is absent from the map', () => {
+    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {});
+    expect(problems).toHaveLength(1);
+    expect(problems[0].reason).toBe('no-provider');
+    expect(describeCapabilityProblem(problems[0])).toContain("no deployed module provides 'idp'");
+  });
+
+  it('flags empty-value when the field is present but empty', () => {
+    const problems = findBrokenCapabilityDerivations('forgejo', consumer, { idp: { dmz_ip: '' } });
+    expect(problems[0].reason).toBe('empty-value');
+  });
+
+  it('flags unresolved-ref when the resolved value is still a template', () => {
+    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
+      idp: { dmz_ip: '$self:caddy_dmz_ip' },
+    });
+    expect(problems[0].reason).toBe('unresolved-ref');
+    expect(problems[0].value).toBe('$self:caddy_dmz_ip');
+  });
+
+  it('returns nothing when the field resolves to a concrete value', () => {
+    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
+      idp: { dmz_ip: '10.0.10.10' },
+    });
+    expect(problems).toHaveLength(0);
+  });
+
+  it('walks dotted paths', () => {
+    const m = baseManifest({
+      id: 'consumer',
+      variables: {
+        owns: [
+          {
+            name: 'primary',
+            type: 'string',
+            required: true,
+            source: 'capability',
+            derive_from: '$capability:dns_external.server.ip.primary',
+          },
+        ],
+        imports: [],
+      },
+    });
+    const ok = findBrokenCapabilityDerivations('consumer', m, {
+      dns_external: { server: { ip: { primary: '1.2.3.4' } } },
+    });
+    expect(ok).toHaveLength(0);
+    const broken = findBrokenCapabilityDerivations('consumer', m, {
+      dns_external: { server: { ip: {} } },
+    });
+    expect(broken[0].reason).toBe('empty-value');
+  });
+});