@celilo/cli 0.8.2 → 0.9.0

package/src/templates/generator.ts

@@ -27,7 +27,11 @@ import {
   findBrokenCapabilityDerivations,
 } from '../services/fleet-checks';
 import { selectInfrastructure } from '../services/infrastructure-selector';
-import { deleteModuleConfig, upsertModuleConfig } from '../services/module-config';
+import {
+  deleteModuleConfig,
+  getModuleConfigValue,
+  upsertModuleConfig,
+} from '../services/module-config';
 import type { InfrastructureSelection } from '../types/infrastructure';
 import { convertSecretsToJinja } from '../variables/ansible-resolver';
 import { buildResolutionContext } from '../variables/context';
@@ -730,6 +734,50 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
     }
   }
 
+  // Dedicated DNS-ingress IP (ISS-0156). A dns_internal provider now deploys into
+  // a PROTECTED zone (dmz) so it can see protected-zone query sources for
+  // split-horizon views (v2/INTERNAL_DNS_ZONE_VIEWS.md). `internal` devices have
+  // no route into the 10-net, so they reach the resolver through a firewall DNAT
+  // on a dedicated `internal`-subnet address. A module opts in by declaring a
+  // `dns_ingress_ip` infrastructure variable; we allocate a free IP from the
+  // `internal` subnet via IPAM and RESERVE it (so it's never re-handed-out),
+  // idempotently (reuse the stored value on re-generate). The resolver's
+  // on_install passes it to firewall.exposeService({ ingressIp }).
+  const wantsDnsIngress = manifest.variables?.owns?.some(
+    (v) => v.name === 'dns_ingress_ip' && v.source === 'infrastructure',
+  );
+  if (wantsDnsIngress) {
+    const existing = getModuleConfigValue(moduleId, 'dns_ingress_ip', db)?.value;
+    if (typeof existing === 'string' && existing.length > 0) {
+      log.success(`Using existing DNS-ingress IP ${existing} for ${moduleId}`);
+    } else {
+      const subnetRow = db.$client
+        .prepare('SELECT value FROM system_config WHERE key = ?')
+        .get('network.internal.subnet') as { value: string } | undefined;
+      if (!subnetRow?.value) {
+        return {
+          success: false,
+          error:
+            'network.internal.subnet is not configured — required to allocate the ' +
+            'dns_internal DNS-ingress IP (ISS-0156). Ensure the internal network is set up first.',
+        };
+      }
+      const { allocateIPFromSubnet, reserveIP } = await import('../ipam/allocator');
+      const { stripCIDR } = await import('../ipam/subnet-parser');
+      try {
+        const ip = stripCIDR(await allocateIPFromSubnet(subnetRow.value, 'internal', db));
+        await reserveIP(ip, 'internal', `dns-ingress:${moduleId}`, null, db);
+        upsertModuleConfig(db, moduleId, 'dns_ingress_ip', ip);
+        log.success(`Allocated DNS-ingress IP ${ip} (internal subnet) for ${moduleId}`);
+      } catch (error) {
+        return {
+          success: false,
+          error: `DNS-ingress IP allocation failed: ${error instanceof Error ? error.message : String(error)}`,
+        };
+      }
+    }
+  }
+
   // Infrastructure Properties Resolution (Proxmox provider config)
   // For Proxmox services, extract provider config and store as temporary values
   // This happens during generation so templates can access target_node, lxc_template, etc.

package/src/variables/context.ts

@@ -8,6 +8,7 @@ import {
   machines,
   moduleConfigs,
   moduleInfrastructure,
+  moduleSystems,
   modules,
   secrets,
   systemConfig,
@@ -264,20 +265,48 @@ export async function buildResolutionContext(
     const systemResources = getSingularSystemSpec(manifest);
 
     if (systemResources) {
-      // Map manifest fields to module variable names and apply defaults
+      // The DEPLOYED size is the SYSTEM's canonical state (ISS-0150), seeded from
+      // requires.system at first provision and thereafter owned by
+      // `celilo proxmox … resize`. So sizing flows: module_systems → these config
+      // vars → `$self:{cores,memory,disk}` in the instance Terraform.
+      //
+      // Precedence: the recorded system size WINS and overwrites the cached
+      // config (a resize must propagate on the next generate); only when this
+      // module has no recorded system size yet (the very first provision, before
+      // recordDeployedSystemForModule runs below) do we fall back to
+      // requires.system — and seed-when-unset, matching the prior behavior so the
+      // first-deploy / golden output is unchanged. `requires.system` stays the
+      // minimum floor, never the canonical size. (CLAUDE.md / ISS-0150.)
+      const sizedRow = db
+        .select({
+          cpu: moduleSystems.cpu,
+          memory: moduleSystems.memory,
+          disk: moduleSystems.disk,
+        })
+        .from(moduleSystems)
+        .where(eq(moduleSystems.moduleId, moduleId))
+        .all()
+        .find((r) => r.cpu != null || r.memory != null || r.disk != null);
+
       const resourceMappings: Array<{
         manifestKey: keyof typeof systemResources;
         configKey: string;
+        systemValue: number | null | undefined;
       }> = [
-        { manifestKey: 'cpu', configKey: 'cores' }, // manifest.requires.system.cpu → cores variable
-        { manifestKey: 'memory', configKey: 'memory' },
-        { manifestKey: 'disk', configKey: 'disk' },
-        { manifestKey: 'storage', configKey: 'storage' },
+        { manifestKey: 'cpu', configKey: 'cores', systemValue: sizedRow?.cpu }, // requires.system.cpu → cores
+        { manifestKey: 'memory', configKey: 'memory', systemValue: sizedRow?.memory },
+        { manifestKey: 'disk', configKey: 'disk', systemValue: sizedRow?.disk },
+        { manifestKey: 'storage', configKey: 'storage', systemValue: undefined }, // pool name, not sizing
       ];
 
-      for (const { manifestKey, configKey } of resourceMappings) {
+      for (const { manifestKey, configKey, systemValue } of resourceMappings) {
+        if (systemValue != null) {
+          // Canonical system size — always wins so a resize propagates.
+          upsertModuleConfig(db, moduleId, configKey, systemValue);
+          selfConfig[configKey] = String(systemValue);
+          continue;
+        }
         const value = systemResources[manifestKey];
-
         // Manifest fields are typed (cpu: number, storage: string, etc.).
         // Pass them through unstringified so valueJson preserves the
         // shape — see comment in the variable-defaults block above.

package/CLI_USAGE.md

@@ -1,433 +0,0 @@
-# Celilo CLI Usage Guide
-
-Common workflows and command patterns for the Celilo CLI.
-
-## Quick Reference
-
-```bash
-# Import and setup
-celilo module import <path>
-celilo system config set <key> <value>
-celilo module config set <module> <key> <value>
-celilo secret set <module> <name> <value>
-celilo module generate <module>
-
-# Query and inspect
-celilo module list
-celilo module config get <module>
-celilo system config get [key]
-celilo system vault-password
-
-# Manage
-celilo module remove <module>
-celilo help
-```
-
-## Common Workflows
-
-### 1. Setting Up a New Module
-
-```bash
-# Step 1: Import the module
-./celilo module import modules/homebridge
-# Output: Successfully imported module: homebridge
-#         Files copied to: /tmp/celilo/modules/homebridge
-
-# Step 2: Configure system-wide settings (if not already set)
-./celilo system config set dns.primary 192.168.0.1
-./celilo system config set dns.fallback "8.8.8.8 1.1.1.1"
-./celilo system config set routing.internal_gateway 192.168.0.254
-
-# Step 3: Configure module-specific settings
-./celilo module config set homebridge vmid 2110
-./celilo module config set homebridge hostname iot
-./celilo module config set homebridge container_ip "192.168.0.110/24"
-./celilo module config set homebridge gateway 192.168.0.254
-./celilo module config set homebridge vlan 192
-./celilo module config set homebridge cores 2
-./celilo module config set homebridge memory 2048
-./celilo module config set homebridge storage datacenter
-./celilo module config set homebridge rootfs_size 20G
-
-# Step 4: Set secrets
-./celilo secret set homebridge api_key "your_api_key_here"
-./celilo secret set homebridge db_password "your_password_here"
-
-# Step 5: Generate infrastructure code
-./celilo module generate homebridge
-# Output: Successfully generated 8 files:
-#           - terraform/main.tf
-#           - terraform/variables.tf
-#           - ...
-
-# Step 6: Review generated files
-ls -la /tmp/celilo/modules/homebridge/generated/
-cd /tmp/celilo/modules/homebridge/generated/
-```
-
-### 2. Inspecting Configuration
-
-```bash
-# List all modules
-./celilo module list
-
-# Get all config for a module
-./celilo module config get homebridge
-
-# Get specific config value
-./celilo module config get homebridge hostname
-
-# Get all system config
-./celilo system config get
-
-# Get specific system config
-./celilo system config get dns.primary
-```
-
-### 3. Working with Secrets
-
-```bash
-# Set a secret
-./celilo secret set homebridge api_key "secret_value"
-
-# View encrypted secrets file
-MODULE=homebridge
-SECRETS_FILE=/tmp/celilo/modules/$MODULE/generated/ansible/inventory/secrets.yml
-
-# Check it's encrypted
-head -1 $SECRETS_FILE
-# Should show: $ANSIBLE_VAULT;1.1;AES256
-
-# Decrypt and view
-ansible-vault view $SECRETS_FILE \
-  --vault-password-file=<(./celilo system vault-password)
-
-# Edit encrypted secrets directly (if needed)
-ansible-vault edit $SECRETS_FILE \
-  --vault-password-file=<(./celilo system vault-password)
-```
-
-### 4. Regenerating After Changes
-
-```bash
-# Change configuration
-./celilo module config set homebridge cores 4
-./celilo module config set homebridge memory 4096
-
-# Regenerate
-./celilo module generate homebridge
-
-# Generated files will reflect new configuration
-```
-
-### 5. Removing a Module
-
-```bash
-# Remove module (cascade deletes all config and secrets)
-./celilo module remove homebridge
-
-# Verify removal
-./celilo module list
-# Should no longer show homebridge
-
-# Manually clean up generated files if needed
-rm -rf /tmp/celilo/modules/homebridge/
-```
-
-### 6. Using Generated Infrastructure
-
-```bash
-# Terraform workflow
-cd /tmp/celilo/modules/homebridge/generated/terraform
-terraform init
-terraform plan
-terraform apply
-
-# Ansible workflow
-cd /tmp/celilo/modules/homebridge/generated/ansible
-ansible-playbook playbook.yml \
-  -i inventory/ \
-  --vault-password-file=<(celilo system vault-password)
-```
-
-## Advanced Usage
-
-### Running from Different Directories
-
-```bash
-# From anywhere (using wrapper script)
-/path/to/celilo/celilo module list
-
-# From backend directory (using bun)
-cd /path/to/celilo/backend
-bun run src/cli/index.ts module list
-
-# From backend directory (using npm script)
-cd /path/to/celilo/backend
-bun run dev -- module list
-```
-
-### Custom Database Location
-
-```bash
-# Set environment variable
-export CELILO_DB_PATH=/custom/path/celilo.db
-
-# Run migrations
-bun run db:migrate
-
-# Use CLI (will use custom database)
-./celilo module list
-```
-
-### Custom Output Directory
-
-```bash
-# Generate to specific location
-./celilo module generate homebridge --output /custom/output/path
-```
-
-### Scripting with Celilo
-
-```bash
-#!/bin/bash
-# Example: setup-homebridge.sh
-
-set -e  # Exit on error
-
-MODULE=homebridge
-
-# Import module
-./celilo module import "modules/$MODULE"
-
-# Configure system
-./celilo system config set dns.primary 192.168.0.1
-./celilo system config set routing.internal_gateway 192.168.0.254
-
-# Configure module (read from config file or environment)
-./celilo module config set $MODULE vmid "${VMID}"
-./celilo module config set $MODULE hostname "${HOSTNAME}"
-./celilo module config set $MODULE container_ip "${CONTAINER_IP}"
-
-# Set secrets (from secure source)
-./celilo secret set $MODULE api_key "${API_KEY}"
-
-# Generate
-./celilo module generate $MODULE
-
-echo "Setup complete! Generated files at:"
-echo "  /tmp/celilo/modules/$MODULE/generated/"
-```
-
-### Debugging
-
-```bash
-# Check if module was imported correctly
-./celilo module list
-
-# Verify configuration
-./celilo module config get homebridge
-
-# Check database directly
-sqlite3 celilo.db "SELECT * FROM modules;"
-sqlite3 celilo.db "SELECT * FROM module_configs WHERE module_id='homebridge';"
-
-# View encrypted secrets (verify they're not plaintext)
-sqlite3 celilo.db "SELECT name, substr(encrypted_value, 1, 20) || '...' AS encrypted FROM secrets WHERE module_id='homebridge';"
-
-# Check generated files exist
-ls -la /tmp/celilo/modules/homebridge/generated/
-
-# Validate generated Terraform
-cd /tmp/celilo/modules/homebridge/generated/terraform
-terraform fmt -check
-terraform validate
-```
-
-## Common Patterns
-
-### Bulk Configuration
-
-```bash
-# Set multiple values in sequence
-for key in vmid hostname container_ip gateway vlan cores memory storage; do
-  ./celilo module config set homebridge $key "${!key}"
-done
-```
-
-### Configuration from File
-
-```bash
-# Read config from YAML/JSON and apply
-# Example config.yml:
-#   homebridge:
-#     vmid: 2110
-#     hostname: iot
-#     ...
-
-# Using yq or jq
-while IFS='=' read -r key value; do
-  ./celilo module config set homebridge "$key" "$value"
-done < <(yq eval '.homebridge | to_entries | .[] | .key + "=" + (.value | tostring)' config.yml)
-```
-
-### Export Configuration
-
-```bash
-# Export all config for a module
-./celilo module config get homebridge > homebridge-config.txt
-
-# Export system config
-./celilo system config get > system-config.txt
-```
-
-### Validate Before Generate
-
-```bash
-# Check all required config is set
-REQUIRED_KEYS="vmid hostname container_ip gateway vlan"
-
-for key in $REQUIRED_KEYS; do
-  if ! ./celilo module config get homebridge $key &>/dev/null; then
-    echo "ERROR: Missing required config: $key"
-    exit 1
-  fi
-done
-
-# All required config present, generate
-./celilo module generate homebridge
-```
-
-## Tips & Tricks
-
-### 1. Use Absolute Paths for Module Import
-
-```bash
-# Relative path (preferred - works from celilo root)
-./celilo module import ../modules/homebridge
-
-# Absolute path (use your actual path)
-./celilo module import /path/to/celilo/modules/homebridge
-```
-
-The wrapper script handles relative path conversion automatically for `module import`.
-
-### 2. Quote Values with Spaces or Special Characters
-
-```bash
-# Correct
-./celilo module config set homebridge bridge_name "Home Bridge"
-./celilo system config set dns.fallback "8.8.8.8 1.1.1.1"
-
-# Wrong (will only set first word)
-./celilo module config set homebridge bridge_name Home Bridge
-```
-
-### 3. Store Vault Password Securely
-
-```bash
-# DO NOT store vault password in plain files
-# BAD: echo "password" > vault-pass.txt
-
-# GOOD: Use celilo CLI with process substitution
-ansible-vault view secrets.yml \
-  --vault-password-file=<(./celilo system vault-password)
-
-# GOOD: Export to environment (for scripts)
-export ANSIBLE_VAULT_PASSWORD=$(./celilo system vault-password)
-```
-
-### 4. Check Help for Any Command
-
-```bash
-./celilo help                    # General help
-./celilo module --help           # Module command help
-./celilo module config --help    # Config subcommand help (shows same general help)
-```
-
-### 5. Validate Manifests Before Import
-
-```bash
-# Module imports will validate automatically, but you can check manually
-cd backend
-bun run src/cli/index.ts module import ../modules/homebridge
-# Look for validation error messages
-```
-
-## Troubleshooting
-
-### "Module not found" After Import
-
-**Check import succeeded**:
-```bash
-./celilo module list
-```
-
-**Verify files copied**:
-```bash
-ls -la /tmp/celilo/modules/homebridge/
-```
-
-**Check database**:
-```bash
-sqlite3 celilo.db "SELECT id, name, state FROM modules;"
-```
-
-### "Variable not found" During Generate
-
-**Check configuration is set**:
-```bash
-./celilo module config get homebridge
-```
-
-**Check system config**:
-```bash
-./celilo system config get
-```
-
-**Identify missing variables**:
-```bash
-# Error message will show which variables are missing:
-# Error: Failed to resolve variables in templates:
-# terraform/main.tf:
-#   $self:cores: Self variable 'cores' not found in module configuration
-```
-
-Set the missing config and regenerate.
-
-### "Failed to generate Ansible secrets"
-
-**Check Ansible is installed**:
-```bash
-ansible-vault --version
-```
-
-**If Ansible is missing**:
-```bash
-# macOS
-brew install ansible
-
-# Ubuntu/Debian
-sudo apt-get install ansible
-```
-
-### Generated Files Not Where Expected
-
-**Default location**: `/tmp/celilo/modules/<module-id>/generated/`
-
-**Custom location**:
-```bash
-./celilo module generate homebridge --output /custom/path
-```
-
-**Check generation output**:
-```bash
-./celilo module generate homebridge
-# Output shows: "Output: /tmp/celilo/modules/homebridge/generated"
-```
-
-## Next Steps
-
-- Read [INTEGRATION_TESTS.md](backend/INTEGRATION_TESTS.md) for testing workflows
-- Read [backend/README.md](backend/README.md) for architecture details
-- Check [SETUP.md](SETUP.md) for installation and setup guides