@celilo/cli 0.5.0-alpha.4 → 0.5.0-alpha.6

package/src/services/fleet-checks.ts

@@ -0,0 +1,663 @@
+/**
+ * Fleet runtime drift detection — the predicates behind `celilo system
+ * doctor`'s fleet section (designs/CELILO_DOCTOR_FLEET_DRIFT.md, ISS-0113).
+ *
+ * Every facet of post-migration drift that workstream B caught by hand —
+ * a dead/stale dispatcher (ISS-0086), an empty `subscribers` table
+ * (ISS-0088), a capability chain that never re-derived (ISS-0095 /
+ * idp_dmz_ip) — becomes one check here. Each check asserts the *outcome*,
+ * not a proxy for it (design D5): "a dispatcher process exists" is not the
+ * same as "it's the supervised, current one that's actually emitting timer
+ * ticks", and the difference is exactly the bug B hit.
+ *
+ * Detection is read-only and cheap (design D3). These predicates are also
+ * the building blocks the defensive wiring (workstream D) and the celilo
+ * MCP (ISS-0112) call — so they take their bus + DB handles as arguments
+ * and return structured findings rather than rendering or exiting. The
+ * rendering + `--fix` orchestration lives in the doctor command.
+ */
+
+import type { Bus } from '@celilo/event-bus';
+import { inArray } from 'drizzle-orm';
+import { getModuleStoragePath } from '../config/paths';
+import type { DbClient } from '../db/client';
+import { capabilities as capabilitiesTable, modules } from '../db/schema';
+import { findSchemaDrift } from '../db/schema-introspection';
+import { resolveFirewallNatIp } from '../hooks/capability-loader';
+import type { ModuleManifest } from '../manifest/schema';
+import { getModuleSystems } from './deployed-systems';
+import { listDnsInternalRecords } from './dns-internal-records';
+import { type SupervisorPlatform, readInstalledUnit } from './events-daemon';
+import { resolveSubscription } from './module-subscriptions';
+
+/**
+ * Zones reachable from the operator's LAN. A celilo placement zone other
+ * than `internal` is firewall-segmented — an unmanaged LAN device has no
+ * route into it, so an internal-DNS record pointing at a container IP there
+ * is unreachable (the bug). The `internal` zone IS the LAN, so a record at
+ * one of its systems' IPs is fine. (Never pin literal subnets to zones —
+ * compare by the zone role the system carries.)
+ */
+const LAN_REACHABLE_ZONE = 'internal';
+
+export type FleetFindingStatus = 'ok' | 'warn' | 'fail';
+
+/**
+ * One drift facet's verdict. `autoFixable` marks the checks `--fix` may
+ * run unattended (today: subscribers resync only) — everything else is
+ * report + a named manual remediation, never a surprise prod redeploy.
+ */
+export interface FleetFinding {
+  /** Stable id for the check (e.g. 'dispatcher'); not user-facing prose. */
+  id: string;
+  title: string;
+  status: FleetFindingStatus;
+  summary: string;
+  /** Extra context lines, rendered indented under the summary. */
+  detail: string[];
+  /** A concrete next step, or null when status is ok. */
+  remediation: string | null;
+  autoFixable: boolean;
+}
+
+/**
+ * A timer subscriber should see a fresh tick within its interval plus
+ * slack. 15m is the shortest DDNS-refresh cadence (namecheap); a tick
+ * older than this means the dispatcher isn't emitting (the workstream-B
+ * stale-orphan symptom). One window covers the common case without
+ * parsing every interval name.
+ */
+const TIMER_TICK_MAX_AGE_MS = 20 * 60 * 1000;
+
+const UNRESOLVED_REF = /\$\{?(?:self|capability|infra|infrastructure|system|secret):/;
+
+/** Worst of a set of statuses (fail > warn > ok). */
+function worst(statuses: FleetFindingStatus[]): FleetFindingStatus {
+  if (statuses.includes('fail')) return 'fail';
+  if (statuses.includes('warn')) return 'warn';
+  return 'ok';
+}
+
+/**
+ * The DB schema the running CLI expects must actually exist on this box.
+ * celilo only runs drizzle's `migrate()` on a FRESH database; an existing
+ * install gets a hand-maintained CREATE/ALTER list in db/client.ts instead
+ * (ISS-0100). When that list drifts from the shipped migrations — e.g. a new
+ * migration adds a table nobody added to the list — the running code expects
+ * a table/column the DB doesn't have, and features fail at runtime.
+ *
+ * This asserts the outcome directly (track-agnostic): every table + column in
+ * the code's drizzle schema is present in the DB. A miss means migrations
+ * haven't reached this box. It does NOT count migration rows — an existing DB
+ * patched via the hand list legitimately lags `__drizzle_migrations` while
+ * its schema is current, so presence is the honest signal.
+ */
+export function checkSchemaDrift(db: DbClient): FleetFinding {
+  const { missingTables, missingColumns, tableCount } = findSchemaDrift(db.$client);
+
+  const detail: string[] = [];
+  if (missingTables.length > 0) detail.push(`missing table(s): ${missingTables.join(', ')}`);
+  if (missingColumns.length > 0) detail.push(`missing column(s): ${missingColumns.join(', ')}`);
+  const status: FleetFindingStatus = detail.length > 0 ? 'fail' : 'ok';
+
+  return {
+    id: 'schema',
+    title: 'database schema matches the running CLI (migrations applied)',
+    status,
+    summary:
+      status === 'ok'
+        ? `all ${tableCount} schema tables present`
+        : 'database schema is behind the running CLI — migrations not applied',
+    detail,
+    remediation:
+      status === 'ok'
+        ? null
+        : 'run `celilo system migrate` to apply pending migrations on this box — see ISS-0100',
+    autoFixable: false,
+  };
+}
+
+interface HeartbeatRow {
+  dispatcher_id: string;
+  last_heartbeat: number;
+  started_at: number;
+  pid: number;
+  version: string;
+}
+
+export interface DispatcherCheckOptions {
+  now?: number;
+  /**
+   * mtime (ms) of the installed dispatcher code (`@celilo/event-bus`
+   * package.json). A dispatcher whose `started_at` predates this is
+   * running stale in-memory code — the exact workstream-B orphan. Omit
+   * to skip the staleness aspect (e.g. unit tests, or when the package
+   * can't be located).
+   */
+  installedCodeMtimeMs?: number | null;
+  /** Override for readInstalledUnit — tests point this at a temp home. */
+  home?: string;
+  platform?: SupervisorPlatform;
+}
+
+/**
+ * The dispatcher check is four-part (design D5): a dispatcher is (1)
+ * running, (2) the *supervised* one (survives reboot, not an orphan),
+ * (3) running *current* code, and (4) actually emitting timer ticks +
+ * draining deliveries. A naive "is a process up?" check reports green
+ * while broken — that's the trap this exists to avoid.
+ */
+export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): FleetFinding {
+  const now = opts.now ?? Date.now();
+  const health = bus.health();
+  const hb = bus.db
+    .query<HeartbeatRow, []>(
+      'SELECT dispatcher_id, last_heartbeat, started_at, pid, version FROM dispatcher_heartbeat ORDER BY last_heartbeat DESC LIMIT 1',
+    )
+    .get();
+
+  const detail: string[] = [];
+  const statuses: FleetFindingStatus[] = [];
+  const remediations: string[] = [];
+
+  // (1) running — a fresh heartbeat. health() already classifies a
+  // stale/absent heartbeat as no_dispatcher.
+  if (health.status === 'no_dispatcher' || !hb) {
+    statuses.push('fail');
+    detail.push(
+      'no live dispatcher — heartbeat absent or stale (events are queueing, not delivered)',
+    );
+    remediations.push(
+      'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
+    );
+    // Without a heartbeat there's nothing more to assert about it.
+    return {
+      id: 'dispatcher',
+      title: 'event dispatcher running, supervised & current',
+      status: 'fail',
+      summary: 'no live event dispatcher',
+      detail,
+      remediation: remediations.join('; '),
+      autoFixable: false,
+    };
+  }
+
+  const ageMs = health.lastHeartbeatAgeMs ?? now - hb.last_heartbeat;
+  detail.push(
+    `running (pid ${hb.pid}, heartbeat ${Math.round(ageMs / 1000)}s ago, code v${hb.version})`,
+  );
+  if (health.status === 'stuck') {
+    statuses.push('warn');
+    detail.push(
+      `${health.stuckRunningCount} delivery(ies) stuck in 'running' — possible crashed handler`,
+    );
+    remediations.push('`celilo events repair` to sweep stuck deliveries');
+  }
+
+  // (2) supervised — a unit file exists (user or system scope). A
+  // running dispatcher with NO unit is the orphan case: works now, gone
+  // after reboot.
+  const supervised =
+    readInstalledUnit({ scope: 'user', home: opts.home, platform: opts.platform }).exists ||
+    readInstalledUnit({ scope: 'system', home: opts.home, platform: opts.platform }).exists;
+  if (!supervised) {
+    statuses.push('warn');
+    detail.push('not under a supervisor unit — will not survive a reboot (orphan process)');
+    remediations.push('`celilo events install-daemon` then enable the unit so it is supervised');
+  }
+
+  // (3) current — started before the installed code was last written ⇒
+  // running stale in-memory code (delivers, but may not emit new event
+  // types like timer ticks). The workstream-B orphan, exactly.
+  if (opts.installedCodeMtimeMs != null && hb.started_at < opts.installedCodeMtimeMs) {
+    statuses.push('warn');
+    const startedAgoMin = Math.round((now - hb.started_at) / 60000);
+    detail.push(
+      `started ${startedAgoMin}min ago — before the last code update; running stale code, restart to pick it up`,
+    );
+    remediations.push('restart the dispatcher: `systemctl --user restart celilo-events.service`');
+  }
+
+  // (4) emitting + delivering. Only assert ticks if something subscribes
+  // to a timer (no subscriber ⇒ no expectation). Assert no piled-up
+  // failed deliveries either way.
+  const timerSub = bus.db
+    .query<{ pattern: string }, []>(
+      "SELECT pattern FROM subscribers WHERE pattern LIKE 'timer.tick.%' LIMIT 1",
+    )
+    .get();
+  if (timerSub) {
+    const latestTick = bus.recentEvents({ type: timerSub.pattern, limit: 1 })[0];
+    if (!latestTick) {
+      statuses.push('warn');
+      detail.push(
+        `a subscriber wants '${timerSub.pattern}' but no such tick has ever been emitted (refresh/DDNS not firing)`,
+      );
+      remediations.push('restart the dispatcher so it emits timer ticks');
+    } else if (now - latestTick.emittedAt > TIMER_TICK_MAX_AGE_MS) {
+      statuses.push('warn');
+      const ageMin = Math.round((now - latestTick.emittedAt) / 60000);
+      detail.push(
+        `last '${timerSub.pattern}' was ${ageMin}min ago — dispatcher not emitting on schedule`,
+      );
+      remediations.push('restart the dispatcher so it resumes emitting timer ticks');
+    }
+  }
+
+  const failed = bus.failedDeliveries({ limit: 50 });
+  if (failed.length > 0) {
+    statuses.push('warn');
+    const sample = failed[0]?.lastError ? ` (e.g. ${failed[0].lastError.split('\n')[0]})` : '';
+    detail.push(`${failed.length} failed/abandoned delivery(ies)${sample}`);
+    remediations.push('inspect failed deliveries and re-emit/repair as needed');
+  }
+
+  const status = worst(statuses);
+  return {
+    id: 'dispatcher',
+    title: 'event dispatcher running, supervised & current',
+    status,
+    summary:
+      status === 'ok'
+        ? 'dispatcher healthy, supervised, current, and emitting'
+        : 'dispatcher running but degraded',
+    detail,
+    remediation: remediations.length > 0 ? remediations.join('; ') : null,
+    autoFixable: false,
+  };
+}
+
+/** A deployed module + its parsed manifest (INSTALLED/VERIFIED only). */
+interface DeployedModule {
+  id: string;
+  manifest: ModuleManifest;
+}
+
+function loadDeployedModules(db: DbClient): DeployedModule[] {
+  const rows = db
+    .select()
+    .from(modules)
+    .where(inArray(modules.state, ['INSTALLED', 'VERIFIED']))
+    .all();
+  return rows.map((m) => ({ id: m.id, manifest: m.manifestData as unknown as ModuleManifest }));
+}
+
+/**
+ * The bus `subscribers` table must reflect what the deployed fleet's
+ * manifests declare. A restore/migration starts it EMPTY (ISS-0088), so
+ * every reactive subscription silently vanishes until a resync or a
+ * redeploy. Missing rows fail; stale rows (a since-removed module) warn.
+ */
+export function checkSubscribers(bus: Bus, db: DbClient): FleetFinding {
+  const deployed = loadDeployedModules(db);
+
+  // Expected: every (scoped name → pattern) the deployed manifests declare.
+  const expected = new Map<string, string>();
+  for (const mod of deployed) {
+    const subs = mod.manifest.subscriptions ?? [];
+    const modulePath = `${getModuleStoragePath()}/${mod.id}`;
+    for (const sub of subs) {
+      const resolved = resolveSubscription(sub, mod.id, modulePath);
+      expected.set(resolved.name, resolved.pattern);
+    }
+  }
+
+  const actualRows = bus.db
+    .query<{ name: string; pattern: string }, []>('SELECT name, pattern FROM subscribers')
+    .all();
+  const actual = new Map(actualRows.map((r) => [r.name, r.pattern]));
+
+  const missing: string[] = [];
+  const mismatched: string[] = [];
+  for (const [name, pattern] of expected) {
+    const have = actual.get(name);
+    if (have === undefined) missing.push(name);
+    else if (have !== pattern) mismatched.push(`${name} (manifest: ${pattern}, bus: ${have})`);
+  }
+  const stale = actualRows.map((r) => r.name).filter((name) => !expected.has(name));
+
+  const detail: string[] = [];
+  const statuses: FleetFindingStatus[] = [];
+  if (missing.length > 0) {
+    statuses.push('fail');
+    detail.push(
+      `${missing.length} subscription(s) declared by the fleet but missing from the bus: ${missing.join(', ')}`,
+    );
+  }
+  if (mismatched.length > 0) {
+    statuses.push('warn');
+    detail.push(
+      `${mismatched.length} subscription(s) with a pattern the bus disagrees on: ${mismatched.join('; ')}`,
+    );
+  }
+  if (stale.length > 0) {
+    statuses.push('warn');
+    detail.push(
+      `${stale.length} subscriber(s) on the bus with no deployed module: ${stale.join(', ')}`,
+    );
+  }
+
+  const status = worst(statuses);
+  return {
+    id: 'subscribers',
+    title: 'bus subscribers reflect the deployed fleet',
+    status,
+    summary:
+      status === 'ok'
+        ? `${expected.size} subscription(s) match the deployed fleet`
+        : 'bus subscribers drifted from the deployed fleet',
+    detail,
+    remediation: status === 'ok' ? null : '`celilo events resync-subscriptions` (safe, idempotent)',
+    autoFixable: status !== 'ok',
+  };
+}
+
+/** A `$capability:<name>.<path>` reference parsed out of a derive_from. */
+interface CapabilityRef {
+  variable: string;
+  capability: string;
+  path: string;
+}
+
+export type CapabilityDerivationReason = 'no-provider' | 'empty-value' | 'unresolved-ref';
+
+/**
+ * A broken `source: capability` derivation found on a consumer module.
+ *  - `no-provider`: nothing in the capabilities map provides `capability`.
+ *  - `empty-value`: the field exists but is null/undefined/empty.
+ *  - `unresolved-ref`: the field is itself an unresolved template ref
+ *    (e.g. authentik's `idp.dmz_ip = $self:caddy_dmz_ip`) — the chain is
+ *    broken one+ hops upstream.
+ */
+export interface CapabilityDerivationProblem {
+  consumerModule: string;
+  variable: string;
+  capability: string;
+  path: string;
+  reason: CapabilityDerivationReason;
+  /** The offending value, for `unresolved-ref`. */
+  value?: string;
+}
+
+function parseCapabilityRefs(manifest: ModuleManifest): CapabilityRef[] {
+  const refs: CapabilityRef[] = [];
+  for (const v of manifest.variables?.owns ?? []) {
+    if (v.source !== 'capability' || !v.derive_from) continue;
+    const re = /\$\{?capability:([\w-]+)\.([\w.]+)/g;
+    let m: RegExpExecArray | null = re.exec(v.derive_from);
+    while (m !== null) {
+      refs.push({ variable: v.name, capability: m[1], path: m[2] });
+      m = re.exec(v.derive_from);
+    }
+  }
+  return refs;
+}
+
+/** Walk a dotted path into a JSON object; undefined if any hop is absent. */
+function getNested(data: Record<string, unknown>, path: string): unknown {
+  let cur: unknown = data;
+  for (const seg of path.split('.')) {
+    if (cur == null || typeof cur !== 'object') return undefined;
+    cur = (cur as Record<string, unknown>)[seg];
+  }
+  return cur;
+}
+
+/**
+ * The shared capability-derivation predicate (design D4: build once, call
+ * from the detector AND the preventer). For a consumer manifest and a map
+ * of capability-name → data, return every `source: capability` derivation
+ * that won't resolve.
+ *
+ * The `capabilities` map can be either RAW (the doctor reads capability
+ * rows straight from the DB, so a still-derived field shows up as
+ * `unresolved-ref`) or RESOLVED (deploy preflight / generate pass
+ * `ResolutionContext.capabilities`, where `$self:` refs are already
+ * substituted against the provider's config — so a broken upstream link
+ * shows up as `empty-value` or `unresolved-ref`). Callers decide severity:
+ * the doctor treats `unresolved-ref` as "needs the ISS-0114 chain trace"
+ * (a note), while preflight/generate treat every reason as a hard error.
+ */
+export function findBrokenCapabilityDerivations(
+  consumerModule: string,
+  manifest: ModuleManifest,
+  capabilities: Record<string, Record<string, unknown> | undefined>,
+): CapabilityDerivationProblem[] {
+  const problems: CapabilityDerivationProblem[] = [];
+  for (const ref of parseCapabilityRefs(manifest)) {
+    const base = {
+      consumerModule,
+      variable: ref.variable,
+      capability: ref.capability,
+      path: ref.path,
+    };
+    const data = capabilities[ref.capability];
+    if (!data) {
+      problems.push({ ...base, reason: 'no-provider' });
+      continue;
+    }
+    const value = getNested(data, ref.path);
+    if (value === undefined || value === null || value === '') {
+      problems.push({ ...base, reason: 'empty-value' });
+      continue;
+    }
+    if (typeof value === 'string' && UNRESOLVED_REF.test(value)) {
+      problems.push({ ...base, reason: 'unresolved-ref', value });
+    }
+  }
+  return problems;
+}
+
+/** One-line human description of a broken derivation, shared by all callers. */
+export function describeCapabilityProblem(p: CapabilityDerivationProblem): string {
+  const head = `${p.consumerModule}.${p.variable} derives from $capability:${p.capability}.${p.path}`;
+  switch (p.reason) {
+    case 'no-provider':
+      return `${head}, but no deployed module provides '${p.capability}' — deploy/redeploy its provider first`;
+    case 'empty-value':
+      return `${head}, but the provider's '${p.capability}' data has no value there — redeploy the provider so it re-registers`;
+    case 'unresolved-ref':
+      return `${head}, which resolves to an unresolved ref (${p.value}) — its own upstream chain is broken; redeploy the provider chain (provider → consumer)`;
+  }
+}
+
+/**
+ * Every `source: capability` variable a deployed module derives must have
+ * a live provider whose capability data carries the referenced field
+ * (the forgejo `$self:idp_dmz_ip not found` class, ISS-0095/ISS-0115).
+ *
+ * Reads RAW capability data, so a present-but-still-derived field (e.g.
+ * authentik's `idp.dmz_ip = $self:caddy_dmz_ip`) can't be verified here
+ * without the backward chain-walker (ISS-0114). Rather than ship a second
+ * walker (design D2.1), those `unresolved-ref` cases are flagged as "needs
+ * the chain trace" — a note pointing at `celilo capability chain`, not a
+ * false-positive fail. (Deploy preflight + generate run the same predicate
+ * against the RESOLVED context, where the same break IS a hard error.)
+ */
+export function checkCapabilityProviders(db: DbClient): FleetFinding {
+  const deployed = loadDeployedModules(db);
+  const capRows = db
+    .select({ name: capabilitiesTable.capabilityName, data: capabilitiesTable.data })
+    .from(capabilitiesTable)
+    .all();
+  const rawMap: Record<string, Record<string, unknown>> = {};
+  for (const r of capRows) rawMap[r.name] = r.data;
+
+  const breaks: string[] = [];
+  const traceNeeded: string[] = [];
+  let refCount = 0;
+
+  for (const mod of deployed) {
+    const problems = findBrokenCapabilityDerivations(mod.id, mod.manifest, rawMap);
+    refCount += parseCapabilityRefs(mod.manifest).length;
+    for (const p of problems) {
+      if (p.reason === 'unresolved-ref') {
+        traceNeeded.push(
+          `${p.consumerModule}.${p.variable} ← ${p.capability}.${p.path} (= ${p.value})`,
+        );
+      } else {
+        breaks.push(describeCapabilityProblem(p));
+      }
+    }
+  }
+
+  const detail: string[] = [];
+  let status: FleetFindingStatus = 'ok';
+  if (breaks.length > 0) {
+    status = 'fail';
+    detail.push(...breaks);
+  }
+  if (traceNeeded.length > 0) {
+    detail.push(
+      `${traceNeeded.length} derived value(s) resolve through another capability — verify with \`celilo capability chain <module> <var>\` (ISS-0114): ${traceNeeded.join('; ')}`,
+    );
+  }
+
+  return {
+    id: 'capability-derived',
+    title: 'capability-derived config has live providers',
+    status,
+    summary:
+      status === 'ok'
+        ? `${refCount} capability-derived reference(s) have providers`
+        : 'capability-derived config is missing a provider',
+    detail,
+    remediation:
+      status === 'ok'
+        ? null
+        : 'redeploy the provider module(s) so they re-register capability data, then redeploy the consumer (provider → consumer order)',
+    autoFixable: false,
+  };
+}
+
+/**
+ * Internal split-horizon DNS records for service hostnames must resolve to
+ * the firewall natIp (the LAN-reachable DNAT ingress), not a zone-side
+ * container IP a LAN device can't route to (ISS-0094 / ISS-0111). Reads the
+ * dns_internal ledger offline — every `registerRecord({type:'A'})` the
+ * capability loader saw — and compares each to the natIp:
+ *   - == natIp                              → ok
+ *   - a segmented-zone system's container IP → fail (unroutable from the LAN)
+ *   - an `internal`-zone system's IP         → ok (that zone IS the LAN)
+ *   - anything else                          → warn (unknown / possibly stale)
+ *
+ * Skipped cleanly when no firewall advertises a natIp (a flat network has no
+ * segmented zones, so container IPs are reachable).
+ */
+export async function checkServiceDns(db: DbClient): Promise<FleetFinding> {
+  const base = {
+    id: 'service-dns',
+    title: 'service DNS points at the firewall natIp',
+    autoFixable: false,
+  } as const;
+
+  // The ledger table may be absent on a DB whose schema is behind (ISS-0100).
+  // Don't crash the whole doctor — the schema-drift check owns that signal.
+  let records: ReturnType<typeof listDnsInternalRecords>;
+  try {
+    records = listDnsInternalRecords(db);
+  } catch {
+    return {
+      ...base,
+      status: 'ok',
+      summary: 'internal-DNS ledger not present (schema behind — see the schema check)',
+      detail: [],
+      remediation: null,
+    };
+  }
+
+  if (records.length === 0) {
+    return {
+      ...base,
+      status: 'ok',
+      summary: 'no internal DNS records registered',
+      detail: [],
+      remediation: null,
+    };
+  }
+
+  const natIp = await resolveFirewallNatIp(db);
+  if (!natIp) {
+    return {
+      ...base,
+      status: 'ok',
+      summary: `${records.length} internal DNS record(s); no firewall natIp to check against`,
+      detail: ['no firewall advertises a natIp — flat network, container IPs are LAN-reachable'],
+      remediation: null,
+    };
+  }
+
+  // Map every deployed system's container IP → its zone, so a record can be
+  // recognized as pointing at a segmented-zone container (the bug) vs an
+  // internal-zone (LAN) system.
+  const ipZone = new Map<string, { moduleId: string; zone: string }>();
+  for (const mod of loadDeployedModules(db)) {
+    for (const sys of getModuleSystems(mod.id, db)) {
+      if (sys.ipv4_address) ipZone.set(sys.ipv4_address, { moduleId: mod.id, zone: sys.zone });
+    }
+  }
+
+  const atContainer: string[] = [];
+  const atOther: string[] = [];
+  for (const r of records) {
+    if (r.ip === natIp) continue;
+    const owner = ipZone.get(r.ip);
+    if (owner && owner.zone !== LAN_REACHABLE_ZONE) {
+      atContainer.push(
+        `${r.host} → ${r.ip} (${owner.moduleId}'s ${owner.zone}-zone container IP — a LAN device can't route there; should be the natIp ${natIp})`,
+      );
+    } else if (!owner) {
+      atOther.push(`${r.host} → ${r.ip} (neither the natIp ${natIp} nor a known system IP)`);
+    }
+  }
+
+  const detail: string[] = [];
+  const statuses: FleetFindingStatus[] = [];
+  if (atContainer.length > 0) {
+    statuses.push('fail');
+    detail.push(...atContainer);
+  }
+  if (atOther.length > 0) {
+    statuses.push('warn');
+    detail.push(...atOther);
+  }
+
+  const status = worst(statuses);
+  return {
+    ...base,
+    status,
+    summary:
+      status === 'ok'
+        ? `${records.length} internal DNS record(s) resolve to the natIp or a LAN-reachable system`
+        : 'internal DNS records point at zone-side IPs unreachable from the LAN',
+    detail,
+    remediation:
+      status === 'fail'
+        ? 'redeploy the provider so it registers the record at the firewall natIp (firewall.exposeService result), not the container IP'
+        : null,
+  };
+}
+
+export interface RunFleetChecksOptions {
+  now?: number;
+  installedCodeMtimeMs?: number | null;
+}
+
+/**
+ * Run every fleet check against the given handles. The caller owns
+ * gating (skip when there's no celilo DB) and rendering; this just
+ * returns the findings, in the order they're shown.
+ */
+export async function runFleetChecks(
+  bus: Bus,
+  db: DbClient,
+  opts: RunFleetChecksOptions = {},
+): Promise<FleetFinding[]> {
+  return [
+    checkSchemaDrift(db),
+    checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
+    checkSubscribers(bus, db),
+    checkCapabilityProviders(db),
+    await checkServiceDns(db),
+  ];
+}