@celilo/cli 0.12.1 → 0.13.0

package/CELILO_SUBSYSTEMS.md

@@ -26,6 +26,9 @@ see `openspec/specs/`. Companion doc: [CELILO_CORE_MODULES.md](./CELILO_CORE_MOD
 - **IPAM (IP/VMID allocation)** — `apps/celilo/src/ipam/allocator.ts` — `allocateIPFromSubnet`, `allocateVMID`, `reserveIP`/`unreserveIP`, `inferZoneFromIP`, `getAllocation`. Auto-wrapper: `apps/celilo/src/ipam/auto-allocator.ts` — `allocateForModule` / `deallocateForModule`.
 - **Infrastructure selection (container-service vs machine pool)** — `apps/celilo/src/services/machine-pool.ts` (`getMachineByHostname`, `addMachine`, `assignModuleToMachine`) and `apps/celilo/src/services/container-service.ts` (`getContainerServiceByName`, `addContainerService`, `verifyContainerService`). Provider API clients: `apps/celilo/src/api-clients/proxmox.ts`, `apps/celilo/src/api-clients/digitalocean.ts`.
 - **Zone detection / system config** — `apps/celilo/src/services/zone-detector.ts` — `detectZoneFromIp` reads `network.<zone>.subnet` from the `systemConfig` table.
+- **Zone taxonomy (canonical list)** — `apps/celilo/src/db/schema.ts` — `NETWORK_ZONES` is the single array; `NetworkZone` is DERIVED from it. Never hand-maintain a second copy: a duplicate that dropped a member made zone validation return null and silently fall back to a wrong-but-valid zone.
+- **Control-plane network (`secure-mgmt`)** — `apps/celilo/src/hooks/capability-loader.ts` — `loadControlPlaneSubnet` returns the subnet of the zone `celilo-mgmt` is deployed in. `secure-mgmt` is a placement zone AND the control-plane tier, deliberately NOT in `ZONE_TIER_ORDER` (it is not part of the `dmz → app → secure` data-plane chain; it reaches every tier by trust). The firewall's `trustedSubnets` derives from this rather than assuming celilo-mgr sits on `internal`. Reported as an actionable gap by `checkControlPlaneNetwork` in `apps/celilo/src/services/fleet-checks.ts` when the management address matches no configured subnet.
+- **A deployed system's zone** — `apps/celilo/src/services/deployed-systems.ts` — for machine-pool deploys the zone recorded is the ZONE OF THE MACHINE, not `requires.system.zone` (which is only the minimum used to *select* a host, as with sizing). Three writers must agree: `recordDeployedSystemForModule`, `backfillModuleSystems`, and `apps/celilo/src/variables/context.ts` — the last runs latest and will overwrite the others.
 - **Host discovery ("which host serves module X?")** — `apps/celilo/src/cli/commands/module-where.ts` (`celilo module where <id> [--json]`, MCP `celilo_module_where`) — reads deployed hosts from `module_systems` via `getModuleSystems`, reconciles the live Proxmox node via `reconcilePlacement`, and adds a role-based reachability hint per zone. CI/build infra (builder VM, Forgejo runners) is out of scope (not in `module_systems`).
 
 ## Capability system (cross-module data & functions)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.12.1",
+  "version": "0.13.0",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -56,7 +56,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1024.0",
-    "@celilo/capabilities": "^0.7.0",
+    "@celilo/capabilities": "^0.7.1",
     "@celilo/cli-display": "^0.1.9",
     "@celilo/core": "^0.1.0",
     "@celilo/event-bus": "^0.1.8",

package/src/capabilities/well-known.ts

@@ -12,10 +12,17 @@
  * - app: Internal services in home lab (VLAN 20, e.g., 10.0.20.0/24)
  * - secure: Auth/DB in home lab (VLAN 30, e.g., 10.0.30.0/24)
  *
+ * Control Plane:
+ * - secure-mgmt: Where celilo's own management server and management-plane modules
+ *   run. A placement zone AND the control-plane tier: it sits outside the
+ *   dmz->app->secure data-plane chain and reaches every data-plane tier by trust
+ *   instead. Modules placed here inherit that reach, so placement is a privilege
+ *   decision. celilo-mgmt may equally run in `internal`.
+ *
  * External Zone (Cloud/VPS):
  * - external: Services hosted outside home network (no VLAN, e.g., VPS on internet)
  */
-export type NetworkZone = 'internal' | 'dmz' | 'app' | 'secure' | 'external';
+export type NetworkZone = 'internal' | 'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'external';
 
 export interface WellKnownCapability {
   canonical_hostname: string;

package/src/cli/commands/ipam.ts

@@ -309,7 +309,13 @@ export async function handleIpamIpReserve(
       };
     }
 
-    if (zone !== 'dmz' && zone !== 'app' && zone !== 'secure' && zone !== 'internal') {
+    if (
+      zone !== 'dmz' &&
+      zone !== 'app' &&
+      zone !== 'secure' &&
+      zone !== 'secure-mgmt' &&
+      zone !== 'internal'
+    ) {
       return {
         success: false,
         error: `Invalid zone: ${zone}. Must be internal, dmz, app, or secure`,
@@ -376,7 +382,13 @@ export async function handleIpamIpUnreserve(
       };
     }
 
-    if (zone !== 'dmz' && zone !== 'app' && zone !== 'secure' && zone !== 'internal') {
+    if (
+      zone !== 'dmz' &&
+      zone !== 'app' &&
+      zone !== 'secure' &&
+      zone !== 'secure-mgmt' &&
+      zone !== 'internal'
+    ) {
       return {
         success: false,
         error: `Invalid zone: ${zone}. Must be internal, dmz, app, or secure`,
@@ -489,7 +501,7 @@ export async function handleIpamShow(
     lines.push('');
 
     // IP section - group by zone
-    const zones = ['internal', 'dmz', 'app', 'secure'];
+    const zones = ['internal', 'dmz', 'app', 'secure', 'secure-mgmt'];
     for (const zone of zones) {
       const zoneAllocations = allocations.filter((a) => a.zone === zone);
       const zoneReservations = ipReservations.filter((r) => r.zone === zone);

package/src/cli/commands/module-where.ts

@@ -31,6 +31,8 @@ export function reachabilityHint(zone: NetworkZone | string): string {
     case 'app':
     case 'secure':
       return `firewall-segmented (${zone}) — reach via the firewall's natIp DNAT, not routable directly from the LAN`;
+    case 'secure-mgmt':
+      return "celilo's control plane — firewall-segmented inbound like the other zones, but trusted OUTBOUND to every segmented tier";
     default:
       return `zone ${zone}`;
   }

package/src/cli/commands/service-add-digitalocean.ts

@@ -50,6 +50,7 @@ export async function handleServiceAddDigitalOcean(
           { value: 'dmz', label: 'dmz' },
           { value: 'app', label: 'app' },
           { value: 'secure', label: 'secure' },
+          { value: 'secure-mgmt', label: 'secure-mgmt' },
           { value: 'external', label: 'external' },
         ],
         required: true,

package/src/cli/commands/service-add-proxmox.ts

@@ -73,6 +73,7 @@ export async function handleServiceAddProxmox(
           { value: 'dmz', label: 'dmz' },
           { value: 'app', label: 'app' },
           { value: 'secure', label: 'secure' },
+          { value: 'secure-mgmt', label: 'secure-mgmt' },
           { value: 'external', label: 'external' },
         ],
         required: true,

package/src/cli/commands/service-config-set.ts

@@ -78,7 +78,9 @@ export async function handleServiceConfigSet(args: string[]): Promise<CommandRes
 
     if (key === 'zones') {
       // Parse as JSON array of NetworkZone
-      const ZonesSchema = z.array(z.enum(['internal', 'dmz', 'app', 'secure', 'external']));
+      const ZonesSchema = z.array(
+        z.enum(['internal', 'dmz', 'app', 'secure', 'secure-mgmt', 'external']),
+      );
       const parsed = JSON.parse(valueRaw);
       const zones = ZonesSchema.parse(parsed) as NetworkZone[];
 

package/src/db/schema.ts

@@ -197,7 +197,7 @@ export const ipAllocations = sqliteTable('ip_allocations', {
     .references(() => modules.id, { onDelete: 'cascade' }),
   vmid: integer('vmid').notNull().unique(),
   containerIp: text('container_ip').notNull().unique(), // CIDR format (e.g., "10.0.10.10/24")
-  zone: text('zone').$type<'dmz' | 'app' | 'secure' | 'internal'>().notNull(),
+  zone: text('zone').$type<'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'>().notNull(),
   allocatedAt: integer('allocated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
 });
 
@@ -210,7 +210,7 @@ export const ipReservations = sqliteTable('ip_reservations', {
   id: integer('id').primaryKey({ autoIncrement: true }),
   ipStart: text('ip_start').notNull(), // Single IP or range start
   ipEnd: text('ip_end'), // NULL for single IP, end IP for range
-  zone: text('zone').$type<'dmz' | 'app' | 'secure' | 'internal'>().notNull(),
+  zone: text('zone').$type<'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'>().notNull(),
   reason: text('reason').notNull(),
   reservedAt: integer('reserved_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
 });
@@ -253,9 +253,26 @@ export const moduleBuilds = sqliteTable('module_builds', {
  * - dmz: Public-facing services in home lab (10.0.10.0/24)
  * - app: Internal application services (10.0.20.0/24)
  * - secure: Authentication and database services (10.0.30.0/24)
+ * - secure-mgmt: celilo's own control plane (management server + management-plane
+ *   modules). Outside the data-plane tier chain; reaches every tier by trust.
  * - external: Internet-hosted services (outside home network)
  */
-export type NetworkZone = 'internal' | 'dmz' | 'app' | 'secure' | 'external';
+export const NETWORK_ZONES = [
+  'internal',
+  'dmz',
+  'app',
+  'secure',
+  'secure-mgmt',
+  'external',
+] as const;
+
+/**
+ * Derived from NETWORK_ZONES on purpose: a hand-maintained runtime copy of this
+ * list silently dropped `secure-mgmt`, so zone validation returned null for it
+ * and callers fell back to a wrong-but-valid zone. Deriving the type from the
+ * single array means a new zone cannot be added to one and missed by the other.
+ */
+export type NetworkZone = (typeof NETWORK_ZONES)[number];
 
 /**
  * Container services table

package/src/hooks/capability-loader-control-plane.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Control-plane network derivation.
+ *
+ * `trustedSubnets` is what lets celilo reach every segmented tier through a
+ * default-DROP FORWARD chain — it is the control plane. It used to be hardcoded
+ * to `network.internal.subnet`, which silently assumed the management server
+ * lives on the internal LAN. When it doesn't, celilo's own control plane is
+ * trusted by nothing and its network is absent from the resolver's split-horizon
+ * views (internal names then resolve publicly and can't be hairpinned).
+ *
+ * These tests pin the derivation: the trusted subnet follows wherever
+ * `celilo-mgmt` is actually deployed, and an install that predates
+ * celilo-mgmt-as-a-module keeps exactly today's behaviour.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import type { DbClient } from '../db/client';
+import { moduleSystems, modules, systemConfig } from '../db/schema';
+import { cleanupTestDatabase, setupTestDatabase } from '../test-utils/database';
+import { loadControlPlaneSubnet } from './capability-loader';
+
+const INTERNAL = '192.168.0.0/24';
+const DMZ = '10.0.10.0/24';
+const SECURE_MGMT = '10.0.120.0/24';
+
+describe('control-plane network derivation', () => {
+  let db: DbClient;
+
+  function setSubnet(zone: string, cidr: string) {
+    db.insert(systemConfig)
+      .values({ key: `network.${zone}.subnet`, value: cidr })
+      .run();
+  }
+
+  function deployCeliloMgmt(zone: string, ip: string) {
+    db.insert(modules)
+      .values({
+        id: 'celilo-mgmt',
+        name: 'celilo-mgmt',
+        version: '1.0.0',
+        manifestData: {},
+        sourcePath: '/tmp/celilo-mgmt',
+      })
+      .run();
+    db.insert(moduleSystems)
+      .values({
+        moduleId: 'celilo-mgmt',
+        name: 'main',
+        hostname: 'celilo-mgr',
+        ipv4Address: ip,
+        // biome-ignore lint/suspicious/noExplicitAny: zone is a NetworkZone literal
+        zone: zone as any,
+        infraType: 'machine',
+      })
+      .run();
+  }
+
+  beforeEach(async () => {
+    db = await setupTestDatabase();
+  });
+
+  afterEach(async () => {
+    await cleanupTestDatabase(db);
+  });
+
+  test('celilo-mgmt on the internal network → the internal subnet', () => {
+    setSubnet('internal', INTERNAL);
+    setSubnet('dmz', DMZ);
+    deployCeliloMgmt('internal', '192.168.0.10');
+
+    expect(loadControlPlaneSubnet(db)).toBe(INTERNAL);
+  });
+
+  test('celilo-mgmt on secure-mgmt → the secure-mgmt subnet, NOT internal', () => {
+    setSubnet('internal', INTERNAL);
+    setSubnet('secure-mgmt', SECURE_MGMT);
+    deployCeliloMgmt('secure-mgmt', '10.0.120.10');
+
+    // The production case: hardcoding internal here is exactly the bug.
+    expect(loadControlPlaneSubnet(db)).toBe(SECURE_MGMT);
+  });
+
+  test('celilo-mgmt deployed but its zone has no configured subnet → undefined', () => {
+    setSubnet('internal', INTERNAL);
+    deployCeliloMgmt('secure-mgmt', '10.0.120.10'); // no network.secure-mgmt.subnet
+
+    expect(loadControlPlaneSubnet(db)).toBeUndefined();
+  });
+
+  test('celilo-mgmt not deployed as a module → undefined (caller falls back)', () => {
+    setSubnet('internal', INTERNAL);
+
+    expect(loadControlPlaneSubnet(db)).toBeUndefined();
+  });
+
+  test('nothing configured at all → undefined, not a throw', () => {
+    expect(loadControlPlaneSubnet(db)).toBeUndefined();
+  });
+});

package/src/hooks/capability-loader.ts

@@ -661,10 +661,13 @@ const ZONE_TIER_ORDER = ['dmz', 'app', 'secure'] as const;
 interface FirewallZones {
   /** Ordered segmented tiers (dmz→app→secure adjacency) for the data-plane matrix. */
   zoneTiers: Array<{ name: string; subnet: string }>;
-  /** Trusted LAN subnets (internal) that reach every tier — celilo's control plane. */
+  /** Subnets that reach EVERY tier — celilo's control plane (see loadControlPlaneSubnet). */
   trustedSubnets: string[];
 }
 
+/** The module that IS celilo's control plane; its network is what we trust. */
+const CONTROL_PLANE_MODULE_ID = 'celilo-mgmt';
+
 function readZoneSubnet(db: DbClient, zone: string): string | undefined {
   const row = db
     .select()
@@ -674,10 +677,33 @@ function readZoneSubnet(db: DbClient, zone: string): string | undefined {
   return row?.value ?? undefined;
 }
 
+/**
+ * The subnet celilo's control plane occupies: the zone `celilo-mgmt` is actually
+ * deployed in — NOT a hardcoded zone.
+ *
+ * This used to read `network.internal.subnet` outright, on the assumption that the
+ * management server lives on the internal LAN. When it doesn't, celilo's own
+ * control plane is trusted by nothing: default-DROP then blocks the SSH it uses to
+ * run hooks/converges/Ansible on every deployed box, and the same unrecognized
+ * network is absent from the resolver's split-horizon views (so internal names
+ * resolve publicly and can't be hairpinned).
+ *
+ * `celilo-mgmt` may legitimately run in `internal` OR in `secure-mgmt`; deriving
+ * from where it landed supports both without hardcoding either.
+ */
+export function loadControlPlaneSubnet(db: DbClient): string | undefined {
+  for (const system of getModuleSystems(CONTROL_PLANE_MODULE_ID, db)) {
+    const subnet = system.zone ? readZoneSubnet(db, system.zone) : undefined;
+    if (subnet) return subnet;
+  }
+  return undefined;
+}
+
 /**
  * Read the firewall zone matrix inputs from system config (network.<zone>.subnet):
- * the segmented tiers [dmz, app, secure] and the trusted internal LAN. Zones with
- * no configured subnet are omitted — their traffic stays denied (fail-closed).
+ * the segmented tiers [dmz, app, secure] and the control-plane subnet that reaches
+ * all of them. Zones with no configured subnet are omitted — their traffic stays
+ * denied (fail-closed).
  */
 function loadFirewallZones(db: DbClient): FirewallZones {
   const zoneTiers: Array<{ name: string; subnet: string }> = [];
@@ -685,8 +711,12 @@ function loadFirewallZones(db: DbClient): FirewallZones {
     const subnet = readZoneSubnet(db, zone);
     if (subnet) zoneTiers.push({ name: zone, subnet });
   }
-  const internal = readZoneSubnet(db, 'internal');
-  return { zoneTiers, trustedSubnets: internal ? [internal] : [] };
+  // Fall back to the internal subnet when celilo-mgmt's location can't be
+  // determined (e.g. installs predating celilo-mgmt-as-a-module). Losing control-
+  // plane trust outright would brick celilo's own fleet management, so absent
+  // information preserves today's behaviour; the gap is REPORTED separately.
+  const controlPlane = loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal');
+  return { zoneTiers, trustedSubnets: controlPlane ? [controlPlane] : [] };
 }
 
 async function buildFirewallChain(

package/src/ipam/allocator.ts

@@ -16,9 +16,9 @@ import { generateIPsInSubnet, isIPInRange, isInSubnet, stripCIDR } from './subne
 type DbOrTransaction = BunSQLiteDatabase<typeof schema> | DbClient;
 
 /** Zones that support IPAM auto-allocation of VMID and container IP */
-export type IpamZone = 'dmz' | 'app' | 'secure' | 'internal';
+export type IpamZone = 'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal';
 
-const IPAM_ZONES: IpamZone[] = ['internal', 'dmz', 'app', 'secure'];
+const IPAM_ZONES: IpamZone[] = ['internal', 'dmz', 'app', 'secure', 'secure-mgmt'];
 
 /**
  * Infer which zone an IP address belongs to by checking configured zone subnets.

package/src/manifest/schema.ts

@@ -321,7 +321,7 @@ export const SystemResourceSchema = z.object({
         'Modules declare this explicitly; celilo never infers it. Moot for machine-pool / external infra.',
     ),
   zone: z
-    .enum(['internal', 'dmz', 'app', 'secure', 'external'])
+    .enum(['internal', 'dmz', 'app', 'secure', 'secure-mgmt', 'external'])
     .describe('Required security zone for this module'),
 });
 

package/src/services/aspect-runner.ts

@@ -225,12 +225,20 @@ export async function materializeAspectAnsible(args: {
   // ambient operator key — same as a normal LXC deploy (ISS-0028).
   const inventoryHosts: InventoryHost[] = [];
   for (const t of targetSystems) {
-    const keyPath = t.machineId ? await writeTemporarySshKey(t.machineId) : undefined;
+    // The management box registers itself in the machine pool as 127.0.0.1 and
+    // stores NO ssh key -- it does not need one, because Ansible reaches it with
+    // the local connection. Mirrors the module-deploy inventory path
+    // (ansible/inventory.ts). Without this the fan-out tried `ssh root@127.0.0.1`
+    // with an empty key file and the host failed UNREACHABLE with an opaque
+    // "error in libcrypto", so celilo could not configure its own resolv.conf.
+    const isLocal = t.ipAddress === '127.0.0.1';
+    const keyPath = t.machineId && !isLocal ? await writeTemporarySshKey(t.machineId) : undefined;
     inventoryHosts.push({
       hostname: t.hostname,
       ansibleHost: t.ipAddress,
       ansibleUser: t.sshUser,
       groups: ['aspect_targets', t.zone],
+      local: isLocal,
       ansibleSshPrivateKeyFile: keyPath,
     });
 

package/src/services/deployed-systems.test.ts

@@ -11,7 +11,13 @@ import {
   moduleSystems,
   modules,
 } from '../db/schema';
-import { backfillModuleSystems, getModuleSystems, upsertDeployedSystem } from './deployed-systems';
+import type { ModuleManifest } from '../manifest/schema';
+import {
+  backfillModuleSystems,
+  getModuleSystems,
+  recordDeployedSystemForModule,
+  upsertDeployedSystem,
+} from './deployed-systems';
 
 const TEST_DB_PATH = './test-deployed-systems.db';
 
@@ -209,6 +215,109 @@ describe('backfillModuleSystems', () => {
     expect(systems[0].infrastructure.vmid).toBeUndefined();
   });
 
+  // The control plane is recorded by THIS path, not the deploy path, so the two
+  // must agree on zone precedence. They didn't: the deploy path took the machine's
+  // zone while backfill still took the manifest's, so celilo-mgmt was persisted as
+  // `internal` while sitting on `secure-mgmt` — and the firewall trusted the wrong
+  // subnet, silently dropping celilo's own SSH to the fleet.
+  test('backfill records the MACHINE zone, matching the deploy path', () => {
+    db.insert(machines)
+      .values({
+        id: 'm-mgmt-bf',
+        hostname: 'celilo-mgr',
+        ipAddress: '10.0.120.100',
+        sshUser: 'root',
+        sshKeyEncrypted: JSON.stringify({ encryptedValue: '', iv: '', authTag: '' }),
+        hardware: { cpu_cores: 4, memory_mb: 8192, disk_gb: 64 },
+        zone: 'secure-mgmt',
+      })
+      .run();
+    db.insert(modules)
+      .values({
+        id: 'celilo-mgmt',
+        name: 'celilo-mgmt',
+        version: '1.0.0',
+        manifestData: { requires: { system: { zone: 'internal' } } },
+        sourcePath: '/tmp/celilo-mgmt',
+        state: 'VERIFIED',
+      })
+      .run();
+    db.insert(moduleInfrastructure)
+      .values({
+        id: 'infra-celilo-mgmt',
+        moduleId: 'celilo-mgmt',
+        infrastructureType: 'machine',
+        machineId: 'm-mgmt-bf',
+      })
+      .run();
+    db.insert(moduleConfigs)
+      .values({
+        moduleId: 'celilo-mgmt',
+        key: 'hostname',
+        value: 'celilo-mgr',
+        valueJson: '"celilo-mgr"',
+      })
+      .run();
+
+    expect(backfillModuleSystems(db)).toEqual(['celilo-mgmt']);
+    expect(getModuleSystems('celilo-mgmt', db)[0]).toMatchObject({
+      zone: 'secure-mgmt',
+      ipv4_address: '10.0.120.100',
+    });
+  });
+
+  // The control-plane case. celilo-mgmt declares `internal` (it bootstraps before
+  // any firewall exists) but in a segmented fleet the operator earmarks a box on
+  // `secure-mgmt`. What gets RECORDED must be where the system actually is, since
+  // the firewall's trusted source is derived from it — recording the manifest's
+  // zone silently trusts the wrong subnet and drops celilo's own SSH to the fleet.
+  //
+  // This fails two distinct ways before the fix: the machine's zone was never
+  // consulted, AND a hand-maintained zone list omitted `secure-mgmt`, so validating
+  // it returned null and fell back to a wrong-but-valid zone.
+  test('records the zone of the MACHINE it landed on, not the manifest minimum', async () => {
+    db.insert(machines)
+      .values({
+        id: 'm-mgmt',
+        hostname: 'celilo-mgr',
+        ipAddress: '10.0.120.100',
+        sshUser: 'root',
+        sshKeyEncrypted: JSON.stringify({ encryptedValue: '', iv: '', authTag: '' }),
+        hardware: { cpu_cores: 4, memory_mb: 8192, disk_gb: 64 },
+        zone: 'secure-mgmt',
+        earmarkedModule: 'celilo-mgmt',
+      })
+      .run();
+    db.insert(modules)
+      .values({
+        id: 'celilo-mgmt',
+        name: 'celilo-mgmt',
+        version: '1.0.0',
+        manifestData: { requires: { system: { zone: 'internal' } } },
+        sourcePath: '/tmp/celilo-mgmt',
+        state: 'VERIFIED',
+      })
+      .run();
+    db.insert(moduleConfigs)
+      .values({
+        moduleId: 'celilo-mgmt',
+        key: 'hostname',
+        value: 'celilo-mgr',
+        valueJson: '"celilo-mgr"',
+      })
+      .run();
+
+    const recorded = await recordDeployedSystemForModule(
+      'celilo-mgmt',
+      { requires: { system: { zone: 'internal' } } } as ModuleManifest,
+      { type: 'machine', machineId: 'm-mgmt' },
+      db,
+    );
+
+    expect(recorded[0]).toMatchObject({ zone: 'secure-mgmt', ipv4_address: '10.0.120.100' });
+    expect(getModuleSystems('celilo-mgmt', db)[0]).toMatchObject({ zone: 'secure-mgmt' });
+  });
+
   test('skips an API-only module (no declared systems)', () => {
     ensureProxmoxService(db);
     db.insert(modules)

package/src/services/deployed-systems.ts

@@ -2,6 +2,7 @@ import type { DeployedSystem } from '@celilo/capabilities';
 import { and, eq, inArray } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import {
+  NETWORK_ZONES,
   type NetworkZone,
   machines,
   moduleConfigs,
@@ -198,9 +199,7 @@ export function upsertDeployedSystem(
     .run();
 }
 
-const NETWORK_ZONES: readonly NetworkZone[] = ['internal', 'dmz', 'app', 'secure', 'external'];
-
-function asZone(value: string | undefined): NetworkZone | null {
+function asZone(value: string | null | undefined): NetworkZone | null {
   return value && (NETWORK_ZONES as readonly string[]).includes(value)
     ? (value as NetworkZone)
     : null;
@@ -238,19 +237,21 @@ export async function recordDeployedSystemForModule(
   // IP: the module's resolved target_ip, else ip.primary, else the assigned
   // machine's own IP (machine-pool deploys where neither was written).
   let ip = cfg('target_ip') ?? cfg('ip.primary');
-  if (!ip && infrastructure?.machineId) {
-    const machine = db
-      .select()
-      .from(machines)
-      .where(eq(machines.id, infrastructure.machineId))
-      .get();
-    ip = machine?.ipAddress;
-  }
+  const machine = infrastructure?.machineId
+    ? db.select().from(machines).where(eq(machines.id, infrastructure.machineId)).get()
+    : undefined;
+  if (!ip) ip = machine?.ipAddress;
   if (!ip) return [];
 
   // Single-system transition: take the first declared system's name + zone.
   const decl = declared[0];
-  const zone = asZone(decl.resources.zone) ?? asZone(cfg('zone'));
+  // A machine-pool deploy lands on a specific box, and THAT box's zone is where
+  // the system actually is. `requires.system.zone` is the minimum used to select
+  // a host — not a description of the result — exactly as `requires.system.memory`
+  // is a floor rather than the deployed size. Recording the manifest's zone here
+  // would report the control plane as living on `internal` while it sits on
+  // `secure-mgmt`, which is the misreading this whole change exists to remove.
+  const zone = asZone(machine?.zone) ?? asZone(decl.resources.zone) ?? asZone(cfg('zone'));
   if (!zone) {
     throw new Error(
       `Cannot record deployed system for '${moduleId}': no resolvable network zone (checked requires.systems[].resources.zone and config.zone).`,
@@ -324,13 +325,19 @@ export function backfillModuleSystems(db: DbClient): string[] {
     // IP (machine-pool deploy where neither was written). Mirrors
     // recordDeployedSystemForModule. CIDR is stripped by upsertDeployedSystem.
     let ip = cfg('target_ip') ?? cfg('ip.primary');
-    if (!ip && infra.machineId) {
-      ip = db.select().from(machines).where(eq(machines.id, infra.machineId)).get()?.ipAddress;
-    }
+    const machine = infra.machineId
+      ? db.select().from(machines).where(eq(machines.id, infra.machineId)).get()
+      : undefined;
+    if (!ip) ip = machine?.ipAddress;
     if (!ip) continue;
 
     const decl = declared[0];
-    const zone = asZone(decl.resources.zone) ?? asZone(cfg('zone'));
+    // Same precedence as recordDeployedSystemForModule: the machine a deploy
+    // landed on is where the system IS; `requires.system.zone` is only the
+    // minimum used to pick a host. These two paths must agree — the control
+    // plane is recorded HERE, not by the deploy path, so a divergence between
+    // them is invisible until the firewall trusts the wrong subnet.
+    const zone = asZone(machine?.zone) ?? asZone(decl.resources.zone) ?? asZone(cfg('zone'));
     // Can't address a system without a zone; skip rather than abort the whole
     // backfill (a re-deploy will record it properly).
     if (!zone) continue;

package/src/services/fleet-checks.test.ts

@@ -10,12 +10,14 @@ import {
   moduleConfigs,
   moduleSystems,
   modules,
+  systemConfig,
 } from '../db/schema';
 import type { ModuleManifest } from '../manifest/schema';
 import { setupTestDatabase } from '../test-utils/setup-test-db';
 import { getDaemonUnitPath } from './events-daemon';
 import {
   checkCapabilityProviders,
+  checkControlPlaneNetwork,
   checkDispatcher,
   checkSchemaDrift,
   checkServiceDns,
@@ -508,3 +510,96 @@ describe('findBrokenCapabilityDerivations (shared predicate)', () => {
     expect(broken[0].reason).toBe('empty-value');
   });
 });
+
+describe('checkControlPlaneNetwork', () => {
+  let db: DbClient;
+  let cpDir: string;
+
+  function setSubnet(zone: string, cidr: string) {
+    db.insert(systemConfig)
+      .values({ key: `network.${zone}.subnet`, value: cidr })
+      .run();
+  }
+
+  function deployCeliloMgmt(zone: string, ip: string) {
+    db.insert(modules)
+      .values({
+        id: 'celilo-mgmt',
+        name: 'celilo-mgmt',
+        version: '1.0.0',
+        manifestData: {},
+        sourcePath: '/tmp/celilo-mgmt',
+      })
+      .run();
+    db.insert(moduleSystems)
+      .values({
+        moduleId: 'celilo-mgmt',
+        name: 'main',
+        hostname: 'celilo-mgr',
+        ipv4Address: ip,
+        zone: zone as 'internal' | 'secure-mgmt',
+        infraType: 'machine',
+      })
+      .run();
+  }
+
+  beforeEach(async () => {
+    cpDir = mkdtempSync(join(tmpdir(), 'fleet-cpn-'));
+    db = await setupTestDatabase(join(cpDir, 'celilo.db'));
+  });
+
+  afterEach(() => {
+    try {
+      rmSync(cpDir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('is ok when the control-plane network is a configured zone', () => {
+    setSubnet('secure-mgmt', '10.0.120.0/24');
+    deployCeliloMgmt('secure-mgmt', '10.0.120.10');
+
+    const finding = checkControlPlaneNetwork(db);
+    expect(finding.status).toBe('ok');
+    expect(finding.summary).toContain('10.0.120.0/24');
+    expect(finding.remediation).toBeNull();
+  });
+
+  it('warns — naming BOTH consequences — when the network is unrecognized', () => {
+    // The production case: celilo-mgr on a network with no configured subnet.
+    deployCeliloMgmt('secure-mgmt', '10.0.120.10');
+
+    const finding = checkControlPlaneNetwork(db);
+    expect(finding.status).toBe('warn');
+    const detail = finding.detail.join(' ');
+    // Silence is the bug; the report must name what actually breaks.
+    expect(detail).toContain('firewall');
+    expect(detail).toContain('resolver');
+    expect(detail).toContain('hairpin');
+    // ...and where it is, so the operator can act.
+    expect(detail).toContain('10.0.120.10');
+  });
+
+  it('remediation names the concrete config key to set', () => {
+    deployCeliloMgmt('secure-mgmt', '10.0.120.10');
+
+    const finding = checkControlPlaneNetwork(db);
+    expect(finding.remediation).toContain('network.secure-mgmt.subnet');
+  });
+
+  it('warns when celilo-mgmt is not deployed at all', () => {
+    const finding = checkControlPlaneNetwork(db);
+    expect(finding.status).toBe('warn');
+    expect(finding.detail.join(' ')).toContain('no deployed systems');
+  });
+
+  it('is ok for the common case: control plane on the internal LAN', () => {
+    setSubnet('internal', '192.168.0.0/24');
+    deployCeliloMgmt('internal', '192.168.0.10');
+
+    const finding = checkControlPlaneNetwork(db);
+    expect(finding.status).toBe('ok');
+    expect(finding.summary).toContain('192.168.0.0/24');
+  });
+});

package/src/services/fleet-checks.ts

@@ -23,8 +23,11 @@ import { getModuleStoragePath } from '../config/paths';
 import type { DbClient } from '../db/client';
 import { capabilities as capabilitiesTable, modules } from '../db/schema';
 import { findSchemaDrift } from '../db/schema-introspection';
-import { resolveFirewallNatIp } from '../hooks/capability-loader';
+import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
 import type { ModuleManifest } from '../manifest/schema';
+
+/** The module that IS celilo's control plane. */
+const CONTROL_PLANE_MODULE = 'celilo-mgmt';
 import { getModuleSystems } from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import { type SupervisorPlatform, readInstalledUnit } from './events-daemon';
@@ -663,6 +666,68 @@ export interface RunFleetChecksOptions {
  * gating (skip when there's no celilo DB) and rendering; this just
  * returns the findings, in the order they're shown.
  */
+/**
+ * celilo's control plane must sit on a network celilo RECOGNIZES. When it doesn't,
+ * two things break silently and in different subsystems:
+ *
+ *  1. the firewall's trusted sources don't cover it, so a default-DROP FORWARD
+ *     chain blocks celilo's own SSH to every deployed box;
+ *  2. the internal resolver has no split-horizon view for it, so it answers
+ *     NOERROR with ZERO records — indistinguishable from "no such name". The name
+ *     then falls through to public DNS and the box tries to hairpin off the WAN IP.
+ *
+ * Neither surfaces where the cause is. In production this presented as
+ * `apt-get update` timing out against the celilo-hosted apt repo — three layers
+ * from the actual misconfiguration, and it took a multi-step investigation to
+ * locate precisely because every layer degraded politely. Hence this check.
+ */
+export function checkControlPlaneNetwork(db: DbClient): FleetFinding {
+  const systems = getModuleSystems(CONTROL_PLANE_MODULE, db);
+  const subnet = loadControlPlaneSubnet(db);
+
+  if (subnet) {
+    return {
+      id: 'control-plane-network',
+      title: "celilo's own network is recognized (firewall trust + internal DNS)",
+      status: 'ok',
+      summary: `control plane on ${subnet}`,
+      detail: [],
+      remediation: null,
+      autoFixable: false,
+    };
+  }
+
+  const notDeployed = systems.length === 0;
+  const zones = [...new Set(systems.map((sys) => sys.zone).filter(Boolean))];
+  const addresses = systems.map((sys) => sys.ipv4_address).filter(Boolean);
+
+  const detail = notDeployed
+    ? [`no deployed systems found for '${CONTROL_PLANE_MODULE}'`]
+    : [
+        `'${CONTROL_PLANE_MODULE}' is in zone(s): ${zones.join(', ') || '(unset)'}`,
+        `address(es): ${addresses.join(', ') || '(unknown)'}`,
+        'no network.<zone>.subnet is configured for that zone',
+      ];
+  detail.push(
+    'consequence 1: firewall trusted sources fall back to network.internal.subnet — celilo may be unable to reach segmented zones',
+    'consequence 2: the internal resolver has no view for this network — internal names resolve publicly and cannot be hairpinned',
+  );
+
+  return {
+    id: 'control-plane-network',
+    title: "celilo's own network is recognized (firewall trust + internal DNS)",
+    status: 'warn',
+    summary: notDeployed
+      ? "cannot determine celilo's control-plane network — falling back to the internal subnet"
+      : "celilo's control-plane network is not a configured zone",
+    detail,
+    remediation: notDeployed
+      ? `deploy '${CONTROL_PLANE_MODULE}' so celilo knows where its control plane runs, or set network.internal.subnet if it lives on the internal LAN`
+      : `run \`celilo system config set network.${zones[0] ?? '<zone>'}.subnet <cidr>\` for the control plane's network, then reconcile the firewall and the internal resolver`,
+    autoFixable: false,
+  };
+}
+
 export async function runFleetChecks(
   bus: Bus,
   db: DbClient,
@@ -673,6 +738,7 @@ export async function runFleetChecks(
     checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
     checkSubscribers(bus, db),
     checkCapabilityProviders(db),
+    checkControlPlaneNetwork(db),
     await checkServiceDns(db),
   ];
 }

package/src/services/infrastructure-selector.ts

@@ -130,9 +130,16 @@ export async function selectInfrastructure(module: Module): Promise<Infrastructu
   const zone = requirements.zone;
   const moduleId = module.id;
 
-  // 0. Check for earmarked machines first (highest priority)
-  const earmarkedMachines = await listMachines({ zone });
-  const earmarked = earmarkedMachines.find((m) => m.earmarkedModule === moduleId);
+  // 0. Check for earmarked machines first (highest priority).
+  // Deliberately NOT zone-filtered: an earmark is the operator stating outright
+  // which box a module goes on, so it outranks the manifest's zone requirement
+  // (which exists to *pick* a host when nobody said). Filtering by zone made an
+  // earmarked machine silently invisible and produced a "no infrastructure in
+  // zone X" error that never mentioned the box the operator had named. The
+  // control plane is the case that forced this: celilo-mgmt declares `internal`
+  // (it bootstraps before any firewall exists) but legitimately lives on
+  // `secure-mgmt` in a segmented fleet. Role validation below still applies.
+  const earmarked = (await listMachines()).find((m) => m.earmarkedModule === moduleId);
   if (earmarked) {
     validateMachineRoleForModule(earmarked, module);
     return {

package/src/services/ssh-key-manager.test.ts

@@ -16,6 +16,51 @@ import {
   writeTemporarySshKey,
 } from './ssh-key-manager';
 
+describe('a machine with no stored SSH key fails loudly, not silently', () => {
+  // Regression: the management box registers itself in the machine pool as
+  // 127.0.0.1 with NO ssh key -- it does not need one, because Ansible reaches
+  // it over the local connection. writeTemporarySshKey happily wrote a 0-byte
+  // file, and the failure only surfaced much later inside Ansible as
+  //   Load key "/tmp/celilo-ansible-keys/machine-<uuid>.key": error in libcrypto
+  //   root@127.0.0.1: Permission denied (publickey)
+  // naming neither the machine nor the missing key. The host went UNREACHABLE
+  // and celilo could not configure its own resolv.conf.
+  let testDir2: string;
+
+  beforeEach(async () => {
+    testDir2 = mkdtempSync(join(tmpdir(), 'celilo-nokey-'));
+    process.env.CELILO_DB_PATH = join(testDir2, 'test.db');
+    process.env.CELILO_DATA_DIR = join(testDir2, 'data');
+    await runMigrations(join(testDir2, 'test.db'));
+    const masterKeyPath = join(testDir2, 'data', 'master.key');
+    process.env.CELILO_MASTER_KEY_PATH = masterKeyPath;
+    const fs = await import('node:fs/promises');
+    await fs.mkdir(join(testDir2, 'data'), { recursive: true });
+    await fs.writeFile(masterKeyPath, 'a'.repeat(64), 'utf8');
+  });
+
+  afterEach(() => {
+    closeDb();
+    rmSync(testDir2, { recursive: true, force: true });
+  });
+
+  it('throws instead of writing a 0-byte key file', async () => {
+    const machine = await addMachine({
+      hostname: 'celilo-mgr',
+      zone: 'internal',
+      ipAddress: '127.0.0.1',
+      sshUser: 'root',
+      sshKey: '',
+      hardware: { cpu_cores: 4, memory_mb: 4096, disk_gb: 100 },
+      role: 'host',
+      interfaces: [],
+      assignedModuleIds: [],
+    });
+
+    await expect(writeTemporarySshKey(machine.id)).rejects.toThrow(/has no SSH key stored/);
+  });
+});
+
 describe('ssh-key-manager', () => {
   let testDbPath: string;
   let testDir: string;

package/src/services/ssh-key-manager.ts

@@ -47,6 +47,18 @@ export async function writeTemporarySshKey(machineId: string): Promise<string> {
   // Get decrypted SSH key from database
   const keyContent = await getMachineSshKey(machineId);
 
+  // Refuse to write an empty key. A machine with no stored key (e.g. the
+  // management box, which registers itself as 127.0.0.1 and is reached by
+  // Ansible's LOCAL connection) previously produced a 0-byte file here; ssh
+  // then failed far downstream with "Load key ...: error in libcrypto" and the
+  // host went UNREACHABLE, naming neither the machine nor the missing key.
+  // Callers that legitimately have no key must not ask for one.
+  if (keyContent.trim() === '') {
+    throw new Error(
+      `Machine ${machineId} has no SSH key stored, so no key file can be written. If this is the local management box, use an Ansible local connection (InventoryHost.local) instead of pinning a key.`,
+    );
+  }
+
   // Write to temporary file with restrictive permissions
   const keyPath = getTempKeyPath(machineId);
   writeFileSync(keyPath, keyContent, { mode: 0o600 });

package/src/services/zone-detector.ts

@@ -73,7 +73,7 @@ async function getZoneSubnet(zone: NetworkZone): Promise<string | null> {
  */
 export async function detectZoneFromIp(ip: string): Promise<NetworkZone> {
   // Check each zone's subnet
-  const zones: NetworkZone[] = ['internal', 'dmz', 'app', 'secure'];
+  const zones: NetworkZone[] = ['internal', 'dmz', 'app', 'secure', 'secure-mgmt'];
 
   for (const zone of zones) {
     const subnet = await getZoneSubnet(zone);

package/src/variables/context.ts

@@ -368,7 +368,14 @@ export async function buildResolutionContext(
           name: decl.name,
           hostname,
           ipv4Address: machineRow.ipAddress,
-          zone,
+          // The machine's own zone, matching recordDeployedSystemForModule and
+          // backfillModuleSystems. This is the THIRD place that decides a
+          // deployed system's zone, and it ran last — so while the other two
+          // took the machine's zone, this one kept overwriting it with the
+          // manifest's, and the control plane stayed recorded as `internal`.
+          // `requires.system.zone` is the minimum used to SELECT a host; the
+          // machine we selected is where the system actually is.
+          zone: machineRow.zone ?? zone,
           infraType: 'machine',
           machineId: infraSelection.machineId,
         });