@celilo/cli 0.24.1 → 0.25.1

package/src/manifest/network-requirement-schema.test.ts

@@ -0,0 +1,141 @@
+/**
+ * `requires.networks` — a module NAMES the networks it depends on and never
+ * carries their values
+ * (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md).
+ *
+ * The rejection below is the schema-level half of "a module has no path to
+ * create a network definition". The other half is the write path
+ * ([[cli/commands/system-apply-config.ts]]).
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { NetworkRequirementSchema, getRequiredNetworkNames } from './schema';
+import type { ModuleManifest } from './schema';
+
+describe('NetworkRequirementSchema', () => {
+  test('a requirement is just a name', () => {
+    expect(NetworkRequirementSchema.safeParse({ name: 'control-plane-vpn' }).success).toBe(true);
+  });
+
+  test('names are kebab-case, like every other user-facing identifier', () => {
+    expect(NetworkRequirementSchema.safeParse({ name: 'control_plane_vpn' }).success).toBe(false);
+    expect(NetworkRequirementSchema.safeParse({ name: 'DMZ' }).success).toBe(false);
+  });
+
+  test('a requirement carrying a subnet is rejected, and the message says why', () => {
+    const result = NetworkRequirementSchema.safeParse({
+      name: 'control-plane-vpn',
+      subnet: '10.255.255.0/24',
+    });
+
+    expect(result.success).toBe(false);
+    if (result.success) return;
+    const message = result.error.errors.map((e) => e.message).join(' ');
+    expect(message).toContain('celilo owns');
+    expect(message).toContain('$system:network.<name>.subnet');
+  });
+
+  test('a gateway or vlan on the requirement is rejected too', () => {
+    expect(NetworkRequirementSchema.safeParse({ name: 'dmz', gateway: '10.0.10.1' }).success).toBe(
+      false,
+    );
+    expect(NetworkRequirementSchema.safeParse({ name: 'dmz', vlan: 10 }).success).toBe(false);
+  });
+
+  /**
+   * `from:` is the second form, for a module whose required set is decided per
+   * install rather than at authoring time — a firewall requires the networks it
+   * has legs on, and which legs it has is a property of the box it lands on.
+   * It still names networks and still carries no values.
+   */
+  test('a requirement may name a config array instead of a literal', () => {
+    expect(NetworkRequirementSchema.safeParse({ from: '$self:zones' }).success).toBe(true);
+  });
+
+  test("from must reference this module's own config, not another source", () => {
+    // `$machine:` / `$capability:` would make some other thing the author of the
+    // required set, which is the authority question this change exists to settle.
+    expect(NetworkRequirementSchema.safeParse({ from: '$machine:zones' }).success).toBe(false);
+    expect(NetworkRequirementSchema.safeParse({ from: '$capability:firewall.zones' }).success).toBe(
+      false,
+    );
+    expect(NetworkRequirementSchema.safeParse({ from: 'zones' }).success).toBe(false);
+  });
+
+  test('exactly one of name or from — neither both nor neither', () => {
+    expect(NetworkRequirementSchema.safeParse({ name: 'dmz', from: '$self:zones' }).success).toBe(
+      false,
+    );
+    expect(NetworkRequirementSchema.safeParse({}).success).toBe(false);
+  });
+});
+
+describe('getRequiredNetworkNames', () => {
+  function manifestWith(
+    networks: Array<{ name?: string; from?: string }> | undefined,
+  ): ModuleManifest {
+    return {
+      requires: { capabilities: [], networks },
+    } as unknown as ModuleManifest;
+  }
+
+  test('is empty for a module that requires no network', () => {
+    expect(getRequiredNetworkNames(manifestWith(undefined))).toEqual([]);
+  });
+
+  test('keeps declaration order and drops duplicates', () => {
+    expect(
+      getRequiredNetworkNames(manifestWith([{ name: 'dmz' }, { name: 'app' }, { name: 'dmz' }])),
+    ).toEqual(['dmz', 'app']);
+  });
+
+  test("resolves a from: requirement against the module's own config", () => {
+    expect(
+      getRequiredNetworkNames(manifestWith([{ from: '$self:zones' }]), {
+        zones: ['dmz', 'app', 'secure'],
+      }),
+    ).toEqual(['dmz', 'app', 'secure']);
+  });
+
+  test('an array stored as JSON resolves too', () => {
+    // Array config arrives as either shape depending on how it was written, and
+    // a requirement that silently resolved to nothing would leave a firewall leg
+    // undeclared — which is how an interface ends up classified alien.
+    expect(
+      getRequiredNetworkNames(manifestWith([{ from: '$self:zones' }]), {
+        zones: '["dmz","secure-mgmt"]',
+      }),
+    ).toEqual(['dmz', 'secure-mgmt']);
+  });
+
+  /**
+   * `external` is the RESIDUAL: an interface is external because it is publicly
+   * routable and matched no declared zone, never because it is contained in a
+   * subnet. Giving it one would reintroduce the overload that
+   * `readDeclaredNetworks` and `deployFirewall` both already exclude.
+   */
+  test('external is dropped, however it is named', () => {
+    expect(
+      getRequiredNetworkNames(manifestWith([{ from: '$self:zones' }]), {
+        zones: ['dmz', 'external'],
+      }),
+    ).toEqual(['dmz']);
+    expect(getRequiredNetworkNames(manifestWith([{ name: 'external' }]))).toEqual([]);
+  });
+
+  test('an unanswered config array requires nothing rather than throwing', () => {
+    // The firewall's zones are answered by the config interview, which runs
+    // before this. An empty result here means the interview has not happened
+    // yet, not that the module needs no networks.
+    expect(getRequiredNetworkNames(manifestWith([{ from: '$self:zones' }]), {})).toEqual([]);
+  });
+
+  test('literal and dynamic requirements combine', () => {
+    expect(
+      getRequiredNetworkNames(
+        manifestWith([{ name: 'control-plane-vpn' }, { from: '$self:zones' }]),
+        { zones: ['dmz'] },
+      ),
+    ).toEqual(['control-plane-vpn', 'dmz']);
+  });
+});

package/src/manifest/schema.ts

@@ -354,6 +354,68 @@ export const SystemResourceSchema = z.object({
   zone: z.enum(NETWORK_ZONES).describe('Required security zone for this module'),
 });
 
+/**
+ * A network a module depends on, declared under `requires.networks`
+ * (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md).
+ *
+ * It names the network and stops there. celilo owns the network namespace: the
+ * range belongs to the operator's topology, not to whichever module happens to
+ * arrive first and need it. A module that carried the value here would be
+ * defining the network by another route, which is the authority this change
+ * reverses — so `.strict()` is load-bearing rather than tidiness, and its
+ * message says why.
+ *
+ * When the named network has no `network.<name>.subnet` in system config, the
+ * deploy asks for one BEFORE anything is generated or any hook runs
+ * ([[services/network-ensure.ts]]).
+ *
+ * ── Two forms, because some requirements are not knowable at authoring time ──
+ *
+ * `name:` is a literal — `wireguard` always needs `control-plane-vpn`, and says
+ * so once. `from:` names a config array whose VALUES are the network names, for
+ * a module whose set is decided per install: a firewall requires the networks it
+ * has legs on, and which legs it has is a property of the box it lands on.
+ *
+ * `from:` is not a loophole in "a requirement carries no value". It still names
+ * networks and still carries none — it just names them indirectly. What it fixes
+ * is the split that `firewall-interface-classification` §3 already diagnosed from
+ * the harness side: the legs a firewall has and the networks it declares were two
+ * hand-maintained lists, and every one of the ~20 call sites that had to keep
+ * them in step got it wrong. A leg whose network has no declared subnet
+ * classifies `alien`, so under-declaring is not cosmetic — it is how an interface
+ * gets isolated. One list makes that structurally impossible.
+ */
+export const NetworkRequirementSchema = z
+  .object({
+    name: z
+      .string()
+      .regex(/^[a-z0-9]+(-[a-z0-9]+)*$/, 'network name must be kebab-case')
+      .describe('Name of a network this module needs defined before it deploys')
+      .optional(),
+    from: z
+      .string()
+      .regex(
+        /^\$self:[a-zA-Z_][a-zA-Z0-9_]*$/,
+        'from must reference one of this module\'s own config values, e.g. "$self:zones"',
+      )
+      .describe('A config array whose values are network names, e.g. "$self:zones"')
+      .optional(),
+  })
+  .strict(
+    'A network requirement NAMES networks; it never carries their values. celilo owns ' +
+      'the network namespace — the range is supplied before the deploy runs — so a ' +
+      'requirement takes `name:` (a literal) or `from:` (a $self: config array of names) ' +
+      'and nothing else. Read the range with `source: system` / ' +
+      '`derive_from: "$system:network.<name>.subnet"`.',
+  )
+  .refine((requirement) => Boolean(requirement.name) !== Boolean(requirement.from), {
+    message:
+      'A network requirement declares exactly one of `name:` (a literal network) or ' +
+      '`from:` (a $self: config array whose values are network names)',
+  });
+
+export type NetworkRequirement = z.infer<typeof NetworkRequirementSchema>;
+
 /**
  * One system a module deploys (openspec/specs/module-systems-addressing/spec.md). A module
  * declares 0..N of these under `requires.systems`. `name` is the stable
@@ -505,6 +567,11 @@ export const ModuleManifestSchema = z
          * `systems` gets one host per entry, each addressable as `$infra:<name>`.
          */
         systems: z.array(SystemDeclarationSchema).optional(),
+        /**
+         * Networks that must be DEFINED in celilo's own config before this
+         * module deploys. Names only — see `NetworkRequirementSchema`.
+         */
+        networks: z.array(NetworkRequirementSchema).optional(),
         /**
          * Singular form — sugar for a single unnamed system. Normalized to one
          * `systems` entry (name `main`) by `getDeclaredSystems`. Optional because
@@ -857,3 +924,60 @@ export function getDeclaredSystems(manifest: ModuleManifest): SystemDeclaration[
 export function getSingularSystemSpec(manifest: ModuleManifest): SystemResource | undefined {
   return manifest.requires?.system;
 }
+
+/**
+ * The network names a module requires
+ * (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md).
+ * Deduplicated and in declaration order. Empty for the modules that depend on no
+ * network, which is most of them.
+ *
+ * `from:` entries are resolved against `config` — the module's own values, as
+ * answered by the config interview. A module whose required set is decided per
+ * install (a firewall requires the networks it has legs on) therefore needs its
+ * config populated before this is meaningful, which is why the ensure runs after
+ * the config interview and before generation.
+ *
+ * `external` is dropped wherever it appears. It is the RESIDUAL — an interface is
+ * `external` because it is publicly routable and matched no declared zone, never
+ * because it is contained in a subnet — so giving it one would reintroduce the
+ * overload that `readDeclaredNetworks` and `deployFirewall` both already exclude.
+ */
+export function getRequiredNetworkNames(
+  manifest: ModuleManifest,
+  config: Record<string, unknown> = {},
+): string[] {
+  const names: string[] = [];
+  for (const requirement of manifest.requires?.networks ?? []) {
+    if (requirement.name) {
+      names.push(requirement.name);
+      continue;
+    }
+    if (!requirement.from) continue;
+    const key = requirement.from.slice('$self:'.length);
+    names.push(...asNetworkNameArray(config[key]));
+  }
+  return [...new Set(names)].filter((name) => name !== 'external');
+}
+
+/**
+ * Narrow a `from:`-referenced config value to network names.
+ *
+ * Tolerates the JSON-string form as well as a real array: an array config value
+ * arrives as either depending on how it was written, and a requirement that
+ * silently resolved to nothing would leave a leg undeclared — which is the
+ * failure this whole mechanism exists to prevent.
+ */
+function asNetworkNameArray(raw: unknown): string[] {
+  const value =
+    typeof raw === 'string' && raw.trim().startsWith('[')
+      ? (() => {
+          try {
+            return JSON.parse(raw) as unknown;
+          } catch {
+            return undefined;
+          }
+        })()
+      : raw;
+  if (!Array.isArray(value)) return [];
+  return value.filter((entry): entry is string => typeof entry === 'string' && entry.length > 0);
+}

package/src/services/module-deploy.ts

@@ -38,6 +38,7 @@ import { getModuleSystems } from './deployed-systems';
 import { E2E_CONFLICT_MESSAGE, runningE2eContainers } from './e2e-guard';
 import { resolveInfrastructureVariables } from './infrastructure-variable-resolver';
 import { findMachineForModule } from './machine-pool';
+import { ensureRequiredNetworks } from './network-ensure';
 import { checkProxmoxReachable, formatProxmoxUnreachableError } from './proxmox-preflight';
 import { republishStaticWebConsumers } from './public-web-republish';
 import { LOCAL_MACHINE_IP, deleteTemporarySshKey, writeTemporarySshKey } from './ssh-key-manager';
@@ -367,6 +368,32 @@ async function deployModuleImpl(
       return { success: false, error: E2E_CONFLICT_MESSAGE, phases };
     }
 
+    // Every network this module REQUIRES must be defined before ANYTHING else
+    // (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md).
+    //
+    // First, not merely "before the hooks". `validateAndPrepareDeployment`
+    // generates templates itself when no variable is missing, and generation is
+    // where `$system:` derivations resolve and get persisted. A module reading
+    // `network.<name>.subnet` would derive it BEFORE the network existed, get
+    // nothing, and — because generation does not run twice — never get it. The
+    // hook would then fail on an unset value the deploy had just been told to
+    // supply. Ensuring the network up front is what makes "the value is
+    // available before any hook runs" true rather than nearly true.
+    const declaringModule = await db.select().from(modules).where(eq(modules.id, moduleId)).get();
+    if (declaringModule?.manifestData) {
+      const networksEnsured = await ensureRequiredNetworks(
+        moduleId,
+        declaringModule.manifestData as ModuleManifest,
+        db,
+      );
+      if (!networksEnsured.success) {
+        return { success: false, error: networksEnsured.error, phases };
+      }
+      for (const line of networksEnsured.applied) {
+        log.success(`network defined: ${line}`);
+      }
+    }
+
     const validation = await validateAndPrepareDeployment(moduleId, db);
     phases.validation = validation.success;
     phases.autoGenerated = validation.autoGenerated;
@@ -517,9 +544,10 @@ async function deployModuleImpl(
     // The bus-mediated interview has fired and any responders have
     // answered; the operator can inspect the encrypted store and
     // module config to confirm what landed without spinning up
-    // terraform / ansible / actual hooks. Cross-module `ensure`
-    // events fire later from hook execution, so they're NOT
-    // exercised here — that requires a real run.
+    // terraform / ansible / actual hooks. The required-network
+    // interview above HAS run by this point; the cross-module
+    // `ensure` events fire later from hook execution, so those are
+    // NOT exercised here — that requires a real run.
     if (options.stopAfterInterview) {
       log.info('--stop-after-interview: exiting before infrastructure phase.');
       return {

package/src/services/network-discovery.test.ts

@@ -0,0 +1,198 @@
+/**
+ * celilo discovering the network of the box it is installed on.
+ *
+ * These tests moved here with the code, from
+ * `modules/celilo-mgmt/scripts/discovery.test.ts`. They were always claims about
+ * CELILO — that it reflects whatever network it finds and bakes in no range of
+ * its own — being made in a module's test file because that is where the code
+ * happened to live
+ * (openspec/changes/networks-are-declared-not-written/specs/network-declaration/spec.md).
+ *
+ * The headline test feeds a RANDOM RFC1918 /24 and asserts celilo uses exactly
+ * that: the product-level guarantee that no specific address range is baked in.
+ */
+
+import { beforeEach, describe, expect, test } from 'bun:test';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { systemConfig } from '../db/schema';
+import { setupTestDatabase } from '../test-utils/database';
+import {
+  discoverAndRecordNetwork,
+  discoveredNetworkKeys,
+  parseInternalNetwork,
+} from './network-discovery';
+
+/** Build an `ip route` output for a given /24 base, gateway, and host IP. */
+function ipRouteFor(base: string, gateway: string, hostIp: string, dev = 'eth0'): string {
+  return [
+    `default via ${gateway} dev ${dev} proto dhcp src ${hostIp} metric 100`,
+    `${base}/24 dev ${dev} proto kernel scope link src ${hostIp} metric 100`,
+    '',
+  ].join('\n');
+}
+
+/** Pick a random RFC1918 /24 base + a host IP + gateway within it. */
+function randomRfc1918Slash24(): { base: string; gateway: string; hostIp: string } {
+  const r = Math.random();
+  let a: number;
+  let b: number;
+  if (r < 0.34) {
+    a = 10;
+    b = Math.floor(Math.random() * 256);
+  } else if (r < 0.67) {
+    a = 172;
+    b = 16 + Math.floor(Math.random() * 16); // 172.16–172.31
+  } else {
+    a = 192;
+    b = 168;
+  }
+  const c = Math.floor(Math.random() * 256);
+  return {
+    base: `${a}.${b}.${c}.0`,
+    gateway: `${a}.${b}.${c}.1`,
+    hostIp: `${a}.${b}.${c}.${10 + Math.floor(Math.random() * 200)}`,
+  };
+}
+
+describe('parseInternalNetwork', () => {
+  test('uses whatever /24 the host is actually on — no hard-coded range', () => {
+    for (let i = 0; i < 50; i++) {
+      const { base, gateway, hostIp } = randomRfc1918Slash24();
+      const result = parseInternalNetwork(ipRouteFor(base, gateway, hostIp));
+      expect(result).toEqual({ subnet: `${base}/24`, gateway });
+      // Guard against any latent assumption of the old default.
+      if (base !== '192.168.0.0') {
+        expect(result?.subnet).not.toBe('192.168.0.0/24');
+      }
+    }
+  });
+
+  test('parses a typical dhcp default route + connected subnet', () => {
+    const out = ipRouteFor('10.37.42.0', '10.37.42.1', '10.37.42.50');
+    expect(parseInternalNetwork(out)).toEqual({ subnet: '10.37.42.0/24', gateway: '10.37.42.1' });
+  });
+
+  test('honors a non-/24 prefix length', () => {
+    const out = [
+      'default via 172.20.0.1 dev ens3',
+      '172.20.0.0/16 dev ens3 proto kernel scope link src 172.20.5.9',
+    ].join('\n');
+    expect(parseInternalNetwork(out)).toEqual({ subnet: '172.20.0.0/16', gateway: '172.20.0.1' });
+  });
+
+  test('returns null when there is no default route', () => {
+    expect(parseInternalNetwork('10.0.0.0/24 dev eth0 proto kernel scope link src 10.0.0.5')).toBe(
+      null,
+    );
+  });
+
+  test('returns null when the connected subnet route is missing', () => {
+    expect(parseInternalNetwork('default via 10.0.0.1 dev eth0')).toBe(null);
+  });
+});
+
+describe('discoveredNetworkKeys (issue #300 redeploy idempotency)', () => {
+  const internal = { subnet: '192.168.1.0/24', gateway: '192.168.1.1' };
+
+  test('writes discovered subnet+gateway on first install (unset)', () => {
+    expect(discoveredNetworkKeys(internal, false)).toEqual({
+      'network.internal.subnet': '192.168.1.0/24',
+      'network.internal.gateway': '192.168.1.1',
+    });
+  });
+
+  test('preserves an already-set subnet on redeploy (no override)', () => {
+    // The regression: a redeploy on a box off the internal network discovers the
+    // WRONG subnet — it must not clobber the operator's value.
+    expect(discoveredNetworkKeys(internal, true)).toEqual({});
+  });
+
+  test('box IS on the configured internal network → nothing to record', () => {
+    expect(discoveredNetworkKeys(internal, true, { internalSubnet: '192.168.1.0/24' })).toEqual({});
+  });
+
+  test('box is OFF the internal network → records it as secure-mgmt, not discarded', () => {
+    // #300 discarded this value to protect `internal`. Correct, but it left celilo
+    // blind to its own network: untrusted by the firewall, and absent from the
+    // resolver's split-horizon views (NOERROR/0-records → public DNS → hairpin).
+    const mgmtBox = { subnet: '10.0.120.0/24', gateway: '10.0.120.1' };
+    expect(discoveredNetworkKeys(mgmtBox, true, { internalSubnet: '192.168.0.0/24' })).toEqual({
+      'network.secure-mgmt.subnet': '10.0.120.0/24',
+      'network.secure-mgmt.gateway': '10.0.120.1',
+    });
+  });
+
+  test('never clobbers an already-set secure-mgmt subnet', () => {
+    const mgmtBox = { subnet: '10.0.120.0/24', gateway: '10.0.120.1' };
+    expect(
+      discoveredNetworkKeys(mgmtBox, true, {
+        internalSubnet: '192.168.0.0/24',
+        secureMgmtAlreadySet: true,
+      }),
+    ).toEqual({});
+  });
+
+  test('internal set but its value unreadable → stays conservative, writes nothing', () => {
+    // Without knowing the configured subnet we cannot tell whether this box is on
+    // it; guessing wrong would mislabel the LAN. Preserve #300's behaviour.
+    expect(discoveredNetworkKeys(internal, true, {})).toEqual({});
+  });
+
+  test('writes nothing when discovery failed and nothing is set', () => {
+    expect(discoveredNetworkKeys(null, false)).toEqual({});
+  });
+});
+
+describe('discoverAndRecordNetwork', () => {
+  let db: DbClient;
+
+  beforeEach(async () => {
+    db = await setupTestDatabase();
+  });
+
+  function read(key: string): string | undefined {
+    return db.select().from(systemConfig).where(eq(systemConfig.key, key)).get()?.value;
+  }
+
+  test('records the discovered network on a box that has none', () => {
+    const result = discoverAndRecordNetwork(db, () =>
+      ipRouteFor('10.31.7.0', '10.31.7.1', '10.31.7.20'),
+    );
+
+    expect(result.applied).toHaveLength(2);
+    expect(read('network.internal.subnet')).toBe('10.31.7.0/24');
+    expect(read('network.internal.gateway')).toBe('10.31.7.1');
+  });
+
+  test('a redeploy on the same box changes nothing and says so', () => {
+    const routes = () => ipRouteFor('10.31.7.0', '10.31.7.1', '10.31.7.20');
+    discoverAndRecordNetwork(db, routes);
+    const second = discoverAndRecordNetwork(db, routes);
+
+    expect(second.applied).toEqual([]);
+    expect(second.skipped).toContain('already accounts for');
+    expect(read('network.internal.subnet')).toBe('10.31.7.0/24');
+  });
+
+  test('a box off the internal LAN records its own network as secure-mgmt', () => {
+    db.insert(systemConfig)
+      .values({ key: 'network.internal.subnet', value: '192.168.0.0/24' })
+      .run();
+
+    discoverAndRecordNetwork(db, () => ipRouteFor('10.0.120.0', '10.0.120.1', '10.0.120.10'));
+
+    // The operator's internal value is untouched, and celilo is no longer blind
+    // to the network its own control plane occupies.
+    expect(read('network.internal.subnet')).toBe('192.168.0.0/24');
+    expect(read('network.secure-mgmt.subnet')).toBe('10.0.120.0/24');
+  });
+
+  test('an unreadable routing table is reported, not guessed at', () => {
+    const result = discoverAndRecordNetwork(db, () => null);
+
+    expect(result.applied).toEqual([]);
+    expect(result.skipped).toContain('ip route');
+    expect(read('network.internal.subnet')).toBeUndefined();
+  });
+});