@celilo/cli 0.3.30-alpha.0 → 0.4.0-alpha.0

package/src/services/proxmox-reconcile.ts

@@ -0,0 +1,156 @@
+/**
+ * Proxmox reconciliation (v2/CELILO_BASE.md D5).
+ *
+ * When a base-module aspect declares `proxmox_reconcile.tfvars` and
+ * its fan-out plan touches Proxmox-provisioned LXCs, the running
+ * config (Ansible writes /etc/resolv.conf) is only half the story:
+ * the LXC's persisted Proxmox terraform config also needs the new
+ * value, otherwise a re-provision would revert to a stale resolver
+ * and break the fleet.
+ *
+ * SC5 SCOPE (this file):
+ *   - Plan: pure function that takes the aspect + fan-out targets
+ *     + resolution context, returns one action per affected LXC.
+ *   - Execute: surfaces a clear WARN per action, documenting which
+ *     tfvar would be updated on which owning-module's terraform
+ *     state.
+ *
+ * DEFERRED to a follow-up (Phase 1c / Phase 2):
+ *   - Actually running terraform apply per LXC owning-module. The
+ *     value needs to flow through celilo's persistent state
+ *     (system_config / module_config) AND be picked up by the
+ *     owning module's terraform template on next regular deploy,
+ *     not just injected as a one-off -var. Doing that right
+ *     requires (a) a celilo system-config key with a clear
+ *     semantic, (b) updates to every Proxmox-using module's
+ *     terraform template to read from it, and (c) reconciliation
+ *     against existing tfvars files so the next deploy doesn't
+ *     overwrite.
+ *
+ *     That's meaningful design work and the simulator's Phase 1
+ *     e2e doesn't have a real Proxmox to validate against. The
+ *     responsible move is to ship the SHAPE (manifest schema +
+ *     planning + observable warn-on-skip), defer the persistence
+ *     layer, and pick it up when production rollout actually
+ *     touches Proxmox LXCs.
+ */
+
+import { log } from '../cli/prompts';
+import type { getDb } from '../db/client';
+import type { BaseModuleAspect } from '../manifest/schema';
+import { resolveAspectTemplateRecord } from './aspect-template-resolver';
+import { type ContainerSystem, getContainerSystemsByZone } from './machine-pool';
+
+type DbClient = ReturnType<typeof getDb>;
+
+export interface ProxmoxReconcileAction {
+  /** The owning module of the affected LXC — its terraform state holds the var. */
+  moduleId: string;
+  /** The container_service this LXC was provisioned through. */
+  serviceId: string;
+  /** Resolved tfvar updates, mapping tfvar name → final string value. */
+  tfvarUpdates: Record<string, string>;
+  /** Source LXC for logging / observability. */
+  containerSystem: ContainerSystem;
+}
+
+export interface ProxmoxReconcilePlan {
+  actions: ProxmoxReconcileAction[];
+  /** Containers that matched the zones but were skipped (non-Proxmox provider). */
+  skipped: Array<{ system: ContainerSystem; reason: string }>;
+}
+
+/**
+ * Planning phase: figure out which Proxmox LXCs in the aspect's
+ * zone scope would need a terraform-config update. Pure aside from
+ * the DB reads (container_service rows + value resolution).
+ *
+ * The `providerModuleId` is the module whose aspect we're acting
+ * on (e.g., knot-unbound-internal). Its config + capability data
+ * is what tfvar value templates resolve against — the aspect knows
+ * its own service's IP, the resolution context just makes that
+ * available.
+ */
+export async function planProxmoxReconcile(args: {
+  aspect: BaseModuleAspect;
+  providerModuleId: string;
+  db: DbClient;
+}): Promise<ProxmoxReconcilePlan> {
+  const { aspect, providerModuleId, db } = args;
+  if (!aspect.proxmox_reconcile) {
+    return { actions: [], skipped: [] };
+  }
+
+  const systems = await getContainerSystemsByZone(aspect.applicable_zones);
+
+  const actions: ProxmoxReconcileAction[] = [];
+  const skipped: ProxmoxReconcilePlan['skipped'] = [];
+
+  for (const sys of systems) {
+    if (sys.providerName !== 'proxmox') {
+      // DO droplets / future providers don't ride the proxmox_lxc
+      // terraform path. The aspect's Ansible run still hit them;
+      // they just don't get a tfvar-overlay update from this
+      // mechanism.
+      skipped.push({ system: sys, reason: `non-proxmox provider (${sys.providerName})` });
+      continue;
+    }
+
+    // Resolve every tfvar template against the provider module's
+    // context. Any resolution failure surfaces as a thrown error
+    // up to the caller — aspect authors are expected to declare
+    // resolvable templates.
+    const tfvarUpdates = await resolveAspectTemplateRecord(
+      aspect.proxmox_reconcile.tfvars,
+      providerModuleId,
+      db,
+      'proxmox_reconcile.tfvars',
+    );
+
+    actions.push({
+      moduleId: sys.moduleId,
+      serviceId: sys.serviceId,
+      tfvarUpdates,
+      containerSystem: sys,
+    });
+  }
+
+  return { actions, skipped };
+}
+
+/**
+ * Execution phase. **Currently surfaces operator warnings instead
+ * of running terraform** — see file header for why.
+ *
+ * The warning is structured so operators see exactly which
+ * Proxmox LXC has drifted from its desired tfvar state and can
+ * manually reconcile via:
+ *
+ *   celilo module deploy <owning-module>
+ *
+ * (which re-runs that module's terraform with whatever the
+ * current celilo state says the tfvars should be).
+ *
+ * When the persistence layer lands, this function gains a real
+ * terraform-apply path; the planning function above stays
+ * unchanged.
+ */
+export function executeProxmoxReconcile(plan: ProxmoxReconcilePlan): void {
+  if (plan.actions.length === 0) {
+    return;
+  }
+  log.warn(
+    `Proxmox reconciliation pending (${plan.actions.length} LXC${plan.actions.length === 1 ? '' : 's'} affected). The aspect's Ansible run updated the running config on each, but the persisted Proxmox terraform config has NOT been changed. Re-provisioning would revert. Affected:`,
+  );
+  for (const action of plan.actions) {
+    const tfvarSummary = Object.entries(action.tfvarUpdates)
+      .map(([k, v]) => `${k}=${v}`)
+      .join(', ');
+    log.warn(
+      `  - LXC owned by module '${action.moduleId}' (service ${action.serviceId}, vmid ${(action.containerSystem.containerMetadata as { vmid?: string | number } | null)?.vmid ?? '?'}): would set ${tfvarSummary}`,
+    );
+  }
+  log.warn(
+    'To persist these values, re-deploy the owning modules after the underlying celilo state catches up. Tracked as Phase 1c in v2/CELILO_BASE.md.',
+  );
+}

package/src/services/proxmox-state-recovery.ts

@@ -11,6 +11,7 @@ import { and, eq } from 'drizzle-orm';
 import { log } from '../cli/prompts';
 import type { DbClient } from '../db/client';
 import { moduleConfigs } from '../db/schema';
+import { upsertModuleConfig } from './module-config';
 
 /**
  * Terraform state file structure (subset we care about)
@@ -107,33 +108,11 @@ export async function ensureProxmoxConfigFromState(
   log.warn('  Recovering vmid and target_ip from Terraform state...');
 
   if (!existingVmid) {
-    await db
-      .insert(moduleConfigs)
-      .values({
-        moduleId,
-        key: 'vmid',
-        value: vmid.toString(),
-      })
-      .onConflictDoUpdate({
-        target: [moduleConfigs.moduleId, moduleConfigs.key],
-        set: { value: vmid.toString() },
-      })
-      .run();
+    upsertModuleConfig(db, moduleId, 'vmid', vmid);
   }
 
   if (!existingContainerIp) {
-    await db
-      .insert(moduleConfigs)
-      .values({
-        moduleId,
-        key: 'target_ip',
-        value: containerIp,
-      })
-      .onConflictDoUpdate({
-        target: [moduleConfigs.moduleId, moduleConfigs.key],
-        set: { value: containerIp },
-      })
-      .run();
+    upsertModuleConfig(db, moduleId, 'target_ip', containerIp);
   }
 
   log.success(`  Recovered: vmid=${vmid}, target_ip=${containerIp}`);

package/src/services/restore-from-file.test.ts

@@ -0,0 +1,177 @@
+/**
+ * Tests for the file-direct restore service (Phase 4).
+ *
+ * Focused on the apply-staged-system-files step + restoreFromArtifactFile's
+ * error-handling for malformed / mismatched / missing artifacts.
+ *
+ * Full round-trip (create artifact via celilo-mgmt backup, then restore
+ * via this service) is an e2e concern, not exercised at the unit level.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { closeDb } from '../db/client';
+import { runMigrations } from '../db/migrate';
+import { applyStagedSystemFiles, restoreFromArtifactFile } from './restore-from-file';
+
+describe('applyStagedSystemFiles', () => {
+  let dir: string;
+  let stagingDir: string;
+  let livePath: string;
+  let keyPath: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-staged-apply-test-'));
+    stagingDir = join(dir, 'system');
+    mkdirSync(stagingDir, { recursive: true });
+    livePath = join(dir, 'celilo.db');
+    keyPath = join(dir, 'master.key');
+    process.env.CELILO_DB_PATH = livePath;
+    process.env.CELILO_MASTER_KEY_PATH = keyPath;
+  });
+
+  afterEach(() => {
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('copies celilo.db + master.key into their live paths', () => {
+    writeFileSync(join(stagingDir, 'celilo.db'), 'restored-db');
+    writeFileSync(join(stagingDir, 'master.key'), 'restored-key');
+
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.dbApplied).toBe(true);
+    expect(result.keyApplied).toBe(true);
+    expect(readFileSync(livePath, 'utf-8')).toBe('restored-db');
+    expect(readFileSync(keyPath, 'utf-8')).toBe('restored-key');
+  });
+
+  it('overwrites pre-existing files at the live paths', () => {
+    writeFileSync(livePath, 'old-db');
+    writeFileSync(keyPath, 'old-key');
+    writeFileSync(join(stagingDir, 'celilo.db'), 'new-db');
+    writeFileSync(join(stagingDir, 'master.key'), 'new-key');
+
+    applyStagedSystemFiles(stagingDir);
+    expect(readFileSync(livePath, 'utf-8')).toBe('new-db');
+    expect(readFileSync(keyPath, 'utf-8')).toBe('new-key');
+  });
+
+  it('cleans up stale WAL/SHM siblings of the restored DB', () => {
+    writeFileSync(`${livePath}-wal`, 'stale-wal');
+    writeFileSync(`${livePath}-shm`, 'stale-shm');
+    writeFileSync(join(stagingDir, 'celilo.db'), 'fresh-db');
+
+    applyStagedSystemFiles(stagingDir);
+    expect(existsSync(`${livePath}-wal`)).toBe(false);
+    expect(existsSync(`${livePath}-shm`)).toBe(false);
+  });
+
+  it('handles partial staging (only master.key, no DB)', () => {
+    writeFileSync(join(stagingDir, 'master.key'), 'just-the-key');
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.dbApplied).toBe(false);
+    expect(result.keyApplied).toBe(true);
+  });
+
+  it('handles partial staging (only DB, no master.key)', () => {
+    writeFileSync(join(stagingDir, 'celilo.db'), 'just-the-db');
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.dbApplied).toBe(true);
+    expect(result.keyApplied).toBe(false);
+    expect(result.sshApplied).toBe(false);
+  });
+
+  it('restores the fleet SSH keypair next to the DB with strict perms', () => {
+    const stagedSsh = join(stagingDir, 'ssh');
+    mkdirSync(stagedSsh, { recursive: true });
+    writeFileSync(join(stagedSsh, 'id_ed25519'), 'PRIVATE-KEY-BYTES');
+    writeFileSync(join(stagedSsh, 'id_ed25519.pub'), 'ssh-ed25519 AAAA... celilo-fleet');
+
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.sshApplied).toBe(true);
+
+    // Lands at dirname(getDbPath())/.ssh — which is `dir` here (CELILO_DB_PATH=dir/celilo.db).
+    const liveSshDir = join(dir, '.ssh');
+    expect(readFileSync(join(liveSshDir, 'id_ed25519'), 'utf-8')).toBe('PRIVATE-KEY-BYTES');
+    expect(readFileSync(join(liveSshDir, 'id_ed25519.pub'), 'utf-8')).toBe(
+      'ssh-ed25519 AAAA... celilo-fleet',
+    );
+    // Private key 0600, public 0644, dir 0700.
+    expect(statSync(join(liveSshDir, 'id_ed25519')).mode & 0o777).toBe(0o600);
+    expect(statSync(join(liveSshDir, 'id_ed25519.pub')).mode & 0o777).toBe(0o644);
+    expect(statSync(liveSshDir).mode & 0o777).toBe(0o700);
+  });
+
+  it('is a no-op when staging dir does not exist', () => {
+    rmSync(stagingDir, { recursive: true });
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.dbApplied).toBe(false);
+    expect(result.keyApplied).toBe(false);
+  });
+});
+
+describe('restoreFromArtifactFile error paths', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-restore-from-file-test-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    process.env.CELILO_DATA_DIR = dir;
+    process.env.CELILO_MASTER_KEY_PATH = join(dir, 'master.key');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.CELILO_DATA_DIR = undefined;
+    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('returns an error for a non-existent file', async () => {
+    const result = await restoreFromArtifactFile('/tmp/definitely-does-not-exist.backup');
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('not found');
+  });
+
+  it('returns an error for a file that is not valid JSON', async () => {
+    const bogus = join(dir, 'bogus.backup');
+    writeFileSync(bogus, 'this is not JSON');
+    const result = await restoreFromArtifactFile(bogus);
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('JSON');
+  });
+
+  it('returns an error for valid JSON that fails decryption', async () => {
+    // Valid JSON shape that decryptSecret will fail on.
+    const bogus = join(dir, 'malformed.backup');
+    writeFileSync(
+      bogus,
+      JSON.stringify({ encryptedValue: 'not-real', iv: 'not-real', authTag: 'not-real' }),
+    );
+    const result = await restoreFromArtifactFile(bogus);
+    expect(result.success).toBe(false);
+    expect((result.error ?? '').toLowerCase()).toMatch(/decrypt|json|artifact/);
+  });
+});