@celilo/cli 2.0.0 → 2.2.0

package/src/services/static-content-converge.test.ts

@@ -1,12 +1,21 @@
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { moduleConfigs, modules, webRoutes } from '../db/schema';
 import { setupTestDatabaseAt } from '../test-utils/database';
 import {
+  ansibleTreeMentions,
   buildStaticContentPlan,
   planStaticContent,
   resolveStaticContentRetention,
@@ -246,6 +255,82 @@ describe('staticContentVarsYaml — deterministic desired state', () => {
   });
 });
 
+describe('the state web overlay (celilo#1265)', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'static-converge-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabaseAt(dbPath);
+    writeSite('lunacycle', '<html>luna</html>');
+    db.insert(modules)
+      .values({
+        id: 'lunacycle',
+        name: 'Lunacycle',
+        version: '1.0.0',
+        manifestData: {},
+        sourcePath: join(dir, 'modules', 'lunacycle'),
+      })
+      .onConflictDoNothing()
+      .run();
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function writeSite(moduleId: string, html: string): string {
+    const webRoot = join(dir, 'modules', moduleId, 'site', 'dist');
+    mkdirSync(webRoot, { recursive: true });
+    writeFileSync(join(webRoot, 'index.html'), html, 'utf-8');
+    return webRoot;
+  }
+
+  function insertRoute(route: { slug: string; moduleId: string; hostname: string }) {
+    db.insert(webRoutes)
+      .values({
+        slug: route.slug,
+        moduleId: route.moduleId,
+        type: 'static',
+        path: '/',
+        hostname: route.hostname,
+        websocket: false,
+        contentHash: HASH1,
+      })
+      .run();
+  }
+
+  it('a release with a state overlay carries its directory; one without does not', () => {
+    insertRoute({ slug: 'lunacycle', moduleId: 'lunacycle', hostname: 'lunacycle.net' });
+
+    // No overlay in state: the release names no overlay_dir, and the converge
+    // role has nothing second to copy.
+    const bare = planStaticContent(db);
+    expect(bare).toHaveLength(1);
+    expect(bare[0]?.overlayDir).toBeUndefined();
+    expect(staticContentVarsYaml({ retention: 1, releases: bare })).not.toContain('overlay_dir');
+
+    // The hook wrote generated files into `<module>/state/site`: the converge
+    // must copy them over the release, or a rebuilt host loses the file.
+    const overlay = join(dir, 'modules', 'lunacycle', 'state', 'site');
+    mkdirSync(overlay, { recursive: true });
+    writeFileSync(join(overlay, 'ca.crt'), '---CERT---', 'utf-8');
+
+    const overlaid = planStaticContent(db);
+    expect(overlaid[0]?.overlayDir).toBe(overlay);
+    const yaml = staticContentVarsYaml({ retention: 1, releases: overlaid });
+    expect(yaml).toContain('overlay_dir:');
+  });
+});
+
 describe('writeStaticContentVars — lands in the generated inventory', () => {
   let dir: string;
   let db: DbClient;
@@ -336,3 +421,56 @@ describe('writeStaticContentVars — lands in the generated inventory', () => {
     expect(plan.releases).toHaveLength(1);
   });
 });
+
+describe('the stale-project guard reaches the whole generated tree (celilo#1248)', () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'static-guard-reach-'));
+  });
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('sees a converge that lives in a role task file, not in playbook.yml', () => {
+    // Mirror DERIVED from the real module, never hand-written: a hand-built
+    // tree would give confident reach data about a world that does not exist,
+    // which is the same bug one level up.
+    const real = resolve(import.meta.dir, '../../../..', 'modules/caddy/ansible');
+    cpSync(real, join(dir, 'ansible'), { recursive: true });
+
+    // The premise this guards. caddy's playbook is a thin hosts/vars/roles
+    // stanza; if it ever DOES name static_releases, this test stops proving
+    // anything and should be re-derived rather than deleted.
+    const playbook = readFileSync(join(dir, 'ansible/playbook.yml.tpl'), 'utf-8');
+    expect(playbook).not.toContain('static_releases');
+
+    expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(true);
+  });
+
+  it('still refuses a project that mentions the var nowhere', () => {
+    // A pre-converge generated project. Hand-built deliberately, and it has to
+    // be: no deploy on today's code can produce a tree without the role tasks,
+    // so this legacy shape is the fixture and not a shortcut.
+    mkdirSync(join(dir, 'ansible/roles/caddy/tasks'), { recursive: true });
+    writeFileSync(
+      join(dir, 'ansible/playbook.yml'),
+      '---\n- hosts: caddy\n  roles:\n    - caddy\n',
+    );
+    writeFileSync(
+      join(dir, 'ansible/roles/caddy/tasks/main.yml'),
+      '---\n- name: install\n  debug: {}\n',
+    );
+
+    expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(false);
+  });
+
+  it('ignores non-text files rather than reading assets as utf-8', () => {
+    mkdirSync(join(dir, 'ansible/files'), { recursive: true });
+    writeFileSync(join(dir, 'ansible/files/logo.png'), Buffer.from([0x89, 0x50, 0x4e, 0x47]));
+    writeFileSync(join(dir, 'ansible/site.yml'), 'vars:\n  static_releases: []\n');
+
+    expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(true);
+    expect(ansibleTreeMentions(join(dir, 'ansible'), 'PNG')).toBe(false);
+  });
+});

package/src/services/static-content-converge.ts

@@ -23,14 +23,14 @@
  * `module verify` (celilo#1018, deliberately left to that thread).
  */
 
-import { existsSync, readFileSync } from 'node:fs';
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
 import { mkdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { and, eq, isNotNull } from 'drizzle-orm';
 import { stringify as stringifyYaml } from 'yaml';
 import type { DbClient } from '../db/client';
 import { moduleConfigs, modules, webRoutes } from '../db/schema';
-import { resolveModuleWebRoot } from '../module/web-root';
+import { resolveModuleStateWebRoot, resolveModuleWebRoot } from '../module/web-root';
 import { parseStoredConfigValue } from './module-config';
 
 /** One content-hashed release the provider host must serve. */
@@ -43,6 +43,13 @@ export interface StaticRelease {
   hostnames: string[];
   /** Absolute celilo-mgr path core derived: `<module source_path>/site/dist`. */
   sourceDir: string;
+  /**
+   * Absolute path to `<module source_path>/state/site` (celilo#1265), present
+   * ONLY when that directory exists. The role copies it over the release
+   * after the web root, so a hook-generated file wins over a built one. A
+   * module with no generated site content has no overlay and no row.
+   */
+  overlayDir?: string;
 }
 
 export interface StaticContentPlan {
@@ -172,11 +179,17 @@ export function planStaticContent(db: DbClient): StaticRelease[] {
       );
     }
 
+    // Present only when the module's hook actually wrote an overlay —
+    // otherwise the vars file would name a directory that does not exist
+    // and the role's copy task would have nothing to copy from.
+    const stateWebRoot = resolveModuleStateWebRoot(moduleId, db);
+    const overlayDir = stateWebRoot && existsSync(stateWebRoot) ? stateWebRoot : undefined;
     releases.push({
       slug,
       contentHash,
       hostnames: [...new Set(group.map((r) => r.hostname))].sort(),
       sourceDir,
+      ...(overlayDir ? { overlayDir } : {}),
     });
   }
 
@@ -208,6 +221,9 @@ export function staticContentVarsYaml(plan: StaticContentPlan): string {
       content_hash: r.contentHash,
       hostnames: r.hostnames,
       source_dir: r.sourceDir,
+      // Absent when there is no overlay, so the role's `when:` guard reads a
+      // clean variable — not an empty string a copy task would choke on.
+      ...(r.overlayDir ? { overlay_dir: r.overlayDir } : {}),
     })),
   };
   const header =
@@ -249,6 +265,40 @@ export async function writeStaticContentVars(
  * never happen, so the playbook is checked for the converge's own variable
  * first and a stale project fails loudly with the redeploy that fixes it.
  */
+/**
+ * Whether any text file in a generated Ansible tree mentions `needle`.
+ *
+ * Scans the TREE rather than `playbook.yml`, because a provider's converge
+ * belongs in one of its role's task files and a generated `playbook.yml` is a
+ * thin hosts/vars/roles stanza that names no variable at all. caddy is the
+ * worked example: `static_releases` appears in
+ * `roles/caddy/tasks/static-content.yml`, `.../main.yml.tpl` and
+ * `.../converge-release.yml`, and zero times in its playbook.
+ *
+ * @psbanka - 2026-09: this used to read `ansible/playbook.yml` alone, which
+ * could not reach the string it was looking for. The guard below therefore
+ * refused EVERY provider that had the converge support, and its remedy told the
+ * operator to redeploy — which regenerates the same thin playbook and cannot
+ * help. Guard and role tasks landed in the same commit (1fd3595e), so it had
+ * never once passed, and it held three e2e suites red where nobody was looking.
+ * celilo#1248.
+ */
+export function ansibleTreeMentions(dir: string, needle: string): boolean {
+  if (!existsSync(dir)) return false;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (ansibleTreeMentions(full, needle)) return true;
+      continue;
+    }
+    // Text only. A generated tree can carry static assets, and reading those
+    // as utf-8 to look for a variable name is waste at best.
+    if (!/\.(ya?ml|j2|tpl|cfg|ini|conf)$/.test(entry.name)) continue;
+    if (readFileSync(full, 'utf-8').includes(needle)) return true;
+  }
+  return false;
+}
+
 export async function convergeStaticContent(
   db: DbClient,
   providerModuleId: string,
@@ -271,12 +321,9 @@ export async function convergeStaticContent(
 
   // A generated project that predates the converge role tasks would read the
   // new vars file and converge nothing. `static_releases` appearing nowhere in
-  // the playbook is the tell. Generic string check — core names no module.
-  const playbookPath = join(generatedPath, 'ansible', 'playbook.yml');
-  if (
-    existsSync(playbookPath) &&
-    !readFileSync(playbookPath, 'utf-8').includes('static_releases')
-  ) {
+  // the project is the tell. Generic string check — core names no module.
+  const ansiblePath = join(generatedPath, 'ansible');
+  if (existsSync(ansiblePath) && !ansibleTreeMentions(ansiblePath, 'static_releases')) {
     return {
       success: false,
       error: `The generated project for '${providerModuleId}' predates the static-content converge (its playbook never reads static_releases). Redeploy the provider (\`celilo module deploy ${providerModuleId}\`) to regenerate it, then retry the publish.`,

package/src/services/system-config-schema-types.ts

@@ -14,7 +14,7 @@ export interface SystemConfigProperty {
   maximum?: number;
   default?: string | number | boolean;
   description?: string;
-  format?: 'email';
+  format?: 'email' | 'cidr-list';
 }
 
 /**

package/src/services/system-config-validator.test.ts

@@ -119,6 +119,42 @@ describe('validateValue', () => {
       expect(result.valid).toBe(false);
     });
   });
+
+  describe('cidr-list format validation', () => {
+    const property: SystemConfigProperty = {
+      type: 'string',
+      format: 'cidr-list',
+      description: 'Trusted source subnets',
+    };
+
+    it('accepts both stored forms: comma-separated CIDRs and a JSON array', () => {
+      expect(validateValue('firewall.trusted_subnets', '10.0.0.0/24', property).valid).toBe(true);
+      expect(
+        validateValue('firewall.trusted_subnets', '10.0.0.0/24, 10.1.0.0/24', property).valid,
+      ).toBe(true);
+      expect(validateValue('firewall.trusted_subnets', '["10.226.120.0/24"]', property).valid).toBe(
+        true,
+      );
+    });
+
+    it('rejects a token that is not a CIDR, naming it, at write time', () => {
+      const result = validateValue(
+        'firewall.trusted_subnets',
+        '["10.226.120.0/24", oops]',
+        property,
+      );
+      expect(result.valid).toBe(false);
+      if (result.valid) return;
+      expect(result.error).toContain('looks like JSON but does not parse');
+    });
+
+    it('rejects a JSON-looking value outright rather than mangling it into a subnet name', () => {
+      const result = validateValue('firewall.trusted_subnets', 'nonsense', property);
+      expect(result.valid).toBe(false);
+      if (result.valid) return;
+      expect(result.error).toContain('nonsense');
+    });
+  });
 });
 
 describe('validateKey', () => {

package/src/services/system-config-validator.ts

@@ -5,6 +5,7 @@
  */
 
 import type { SystemConfigProperty, SystemConfigSchema } from './system-config-schema-types';
+import { parseOperatorTrustedSubnets } from './trusted-sources';
 
 /**
  * Validation result (discriminated union)
@@ -55,6 +56,16 @@ export function validateValue(
     }
   }
 
+  // CIDR lists validate through the key's own parser, so the write-time check
+  // and the converge-time read can never disagree about what is valid.
+  if (property.format === 'cidr-list') {
+    try {
+      parseOperatorTrustedSubnets(value);
+    } catch (e) {
+      return { valid: false, error: e instanceof Error ? e.message : String(e) };
+    }
+  }
+
   return { valid: true };
 }
 

package/src/services/trusted-sources.test.ts

@@ -256,5 +256,35 @@ describe('parsing the operator override', () => {
   it('an unset key contributes nothing', () => {
     expect(parseOperatorTrustedSubnets(undefined)).toEqual([]);
     expect(parseOperatorTrustedSubnets('')).toEqual([]);
+    expect(parseOperatorTrustedSubnets('   ')).toEqual([]);
+  });
+
+  it('parses the JSON-array form every sibling config key uses', () => {
+    expect(parseOperatorTrustedSubnets('["10.226.120.0/24"]')).toEqual(['10.226.120.0/24']);
+    expect(parseOperatorTrustedSubnets('[ "10.1.0.0/24" , "10.2.0.0/24" ]')).toEqual([
+      '10.1.0.0/24',
+      '10.2.0.0/24',
+    ]);
+  });
+
+  it('refuses a JSON-looking value that does not parse to an array of strings', () => {
+    // The exact e2e failure (ce-yaqa): a JSON array read as ONE subnet named
+    // ["10.226.120.0/24"], rendered verbatim into rules.v4, and
+    // iptables-restore failed the converge with "Bad argument `]'".
+    expect(() => parseOperatorTrustedSubnets('{"a":1}')).toThrow(/JSON array of CIDR strings/);
+    expect(() => parseOperatorTrustedSubnets('["10.0.0.0/24", 5]')).toThrow(
+      /JSON array of CIDR strings/,
+    );
+    expect(() => parseOperatorTrustedSubnets('[10.0.0.0/24')).toThrow(
+      /looks like JSON but does not parse/,
+    );
+  });
+
+  it('refuses a token that is not an IPv4 CIDR, naming it', () => {
+    expect(() => parseOperatorTrustedSubnets('10.0.0.1')).toThrow(/10\.0\.0\.1/);
+    expect(() => parseOperatorTrustedSubnets('10.0.0.1')).toThrow(/not an IPv4 CIDR/);
+    expect(() => parseOperatorTrustedSubnets('10.0.0.0/24, hostname.local')).toThrow(
+      /hostname\.local/,
+    );
   });
 });

package/src/services/trusted-sources.ts

@@ -14,6 +14,7 @@ import type {
   TrustedSource,
   TrustedSourceStore,
 } from '@celilo/capabilities';
+import { isCidr } from '@celilo/capabilities';
 import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { trustedSources } from '../db/schema';
@@ -80,20 +81,56 @@ export interface TrustedSubnetEntry {
 }
 
 /**
- * The operator's explicit trusted sources, as a comma-separated CIDR list in
- * system config. ADDITIVE rather than replacing the derived control-plane
- * subnet: dropping control-plane trust would block the SSH celilo uses to run
- * every hook and converge, so an operator cannot strand celilo's own management
- * by naming a subnet here.
+ * The operator's explicit trusted sources, in system config. Accepts the two
+ * forms a config value takes in this area: a comma-separated CIDR list and a
+ * JSON array of CIDRs (the form every sibling key uses, e.g. ddns_passwords).
+ *
+ * Every token must be a CIDR. The parser is the single choke point between the
+ * operator's value and the ruleset renderer, which emits each subnet verbatim
+ * into rules.v4 — a token that is not a CIDR used to ride through as a literal
+ * subnet name and iptables-restore failed the whole converge with
+ * "Bad argument `]'". Failing here names the token instead.
+ *
+ * ADDITIVE rather than replacing the derived control-plane subnet: dropping
+ * control-plane trust would block the SSH celilo uses to run every hook and
+ * converge, so an operator cannot strand celilo's own management by naming a
+ * subnet here.
  */
 export const TRUSTED_SUBNETS_CONFIG_KEY = 'firewall.trusted_subnets';
 
 export function parseOperatorTrustedSubnets(raw: string | undefined): string[] {
-  if (!raw) return [];
-  return raw
-    .split(',')
-    .map((s) => s.trim())
-    .filter((s) => s.length > 0);
+  if (!raw || raw.trim().length === 0) return [];
+
+  let tokens: string[];
+  const trimmed = raw.trim();
+  if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      throw new Error(
+        `${TRUSTED_SUBNETS_CONFIG_KEY} looks like JSON but does not parse: ${raw}. Expected a JSON array of CIDRs, e.g. ["10.0.0.0/24"], or a comma-separated CIDR list.`,
+      );
+    }
+    if (!Array.isArray(parsed) || parsed.some((t) => typeof t !== 'string')) {
+      throw new Error(
+        `${TRUSTED_SUBNETS_CONFIG_KEY} must be a JSON array of CIDR strings, e.g. ["10.0.0.0/24"], or a comma-separated CIDR list.`,
+      );
+    }
+    tokens = parsed;
+  } else {
+    tokens = trimmed.split(',');
+  }
+
+  const subnets = tokens.map((s) => s.trim()).filter((s) => s.length > 0);
+  for (const subnet of subnets) {
+    if (!isCidr(subnet)) {
+      throw new Error(
+        `${TRUSTED_SUBNETS_CONFIG_KEY} contains "${subnet}", which is not an IPv4 CIDR. Expected e.g. 10.0.0.0/24 (comma-separated) or ["10.0.0.0/24"] (JSON array).`,
+      );
+    }
+  }
+  return subnets;
 }
 
 /**

package/src/templates/generator.ts

@@ -817,9 +817,16 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
     const configMap = new Map(configs.map((c) => [c.key, c.value || c.valueJson]));
 
     // Check for missing required variables
-    // Skip infrastructure and terraform variables - they're auto-populated during/after generation
+    // Skip infrastructure and terraform variables - they're auto-populated during/after generation.
+    // Skip hook-owned variables too: they are discovered by the module's own
+    // hooks at runtime (hook-owned-state D2), so a hook that has not run yet
+    // is not a configuration error.
     const missingVars = requiredVars.filter(
-      (v) => v.source !== 'infrastructure' && v.source !== 'terraform' && !configMap.has(v.name),
+      (v) =>
+        v.source !== 'infrastructure' &&
+        v.source !== 'terraform' &&
+        v.source !== 'hook' &&
+        !configMap.has(v.name),
     );
 
     if (missingVars.length > 0) {

package/src/variables/context.ts

@@ -577,19 +577,30 @@ async function assembleResolutionContext(
   // LXC boots with a working resolver — before Ansible's apt update, and
   // independent of which Proxmox node it lands on (the nodes' DNS defaults
   // diverge): the internal dns_internal resolver first (it recurses, so it
-  // also answers external names), then the public resolvers as fallback.
+  // also answers external names), then the configured fleet resolvers as
+  // fallback.
   // Bootstrap (no dns_internal provider registered yet — e.g. the DNS module's
   // own LXC) → public only, so apt still works. Injected as
   // $self:lxc_nameserver, mirroring how inventory.* are auto-derived; the
   // proxmox_lxc terraform template drops it straight into `nameserver`.
   {
-    const publicDns: string[] = [];
+    // Named for what the keys hold, not for what a reader hopes they hold:
+    // `dns.primary` / `dns.fallback` mean "the resolvers systems should use",
+    // and an operator pointing the fleet at its own resolver (a deployed
+    // dns_internal provider's address) is established, deliberate practice
+    // (celilo#1239). These entries may therefore be private. The guard against
+    // a resolver celilo installed ending up here lives where the keys are
+    // WRITTEN from host state — discovery refuses a deployed provider's
+    // address (dns-discovery.ts) — not in a privateness check here, which
+    // would break that practice while fixing nothing: in an uncovered zone
+    // dropping the entries takes DNS away rather than tightening anything.
+    const configuredDns: string[] = [];
     for (const key of ['dns.primary', 'dns.fallback']) {
       const raw = systemConfigMap[key];
       if (raw) {
         for (const part of raw.split(',')) {
           const ip = part.trim();
-          if (ip) publicDns.push(ip);
+          if (ip) configuredDns.push(ip);
         }
       }
     }
@@ -663,8 +674,8 @@ async function assembleResolutionContext(
     const nameservers = dropPublicResolvers
       ? [primaryIp, secondaryIp]
       : primaryIp
-        ? [primaryIp, ...publicDns]
-        : publicDns;
+        ? [primaryIp, ...configuredDns]
+        : configuredDns;
     const unique = [...new Set(nameservers)];
     if (unique.length > 0) {
       selfConfig.lxc_nameserver = unique.join(' ');