@celilo/cli 0.1.5 → 0.1.6

package/src/cli/commands/system-audit.ts

@@ -0,0 +1,571 @@
+/**
+ * System audit command — reports drift across the system.
+ *
+ * Usage:
+ *   celilo system audit            # human-readable table
+ *   celilo system audit --json     # machine-readable for scripts
+ *
+ * Read-only. No mutations. Wires the per-category audit checks
+ * (`apps/celilo/src/services/audit/*`) to the live DB / registry /
+ * health-runner / terraform binary, then formats the resulting
+ * `SystemAuditReport` for stdout.
+ *
+ * The actual audit logic lives in services/audit/. This command is a
+ * thin adapter — per Rule 10.5, it's a "thin adapter": parse args,
+ * compose deps, call `runAudit`, format output, return.
+ */
+
+import { execFile } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+import { eq, isNotNull } from 'drizzle-orm';
+import { testDigitalOceanConnection } from '../../api-clients/digitalocean';
+import { testProxmoxConnection } from '../../api-clients/proxmox';
+import { getDb } from '../../db/client';
+import {
+  backups,
+  capabilitySecrets,
+  moduleConfigs as moduleConfigsTbl,
+  modules,
+  secrets,
+  systemSecrets,
+} from '../../db/schema';
+import type { ModuleManifest } from '../../manifest/schema';
+import { RegistryClient } from '../../registry/client';
+import { decryptSecret } from '../../secrets/encryption';
+import { getOrCreateMasterKey } from '../../secrets/master-key';
+import { runAudit } from '../../services/audit';
+import type { DriftFinding, SystemAuditReport } from '../../services/audit';
+import {
+  type LatestCliVersionFetcher,
+  fetchLatestCliVersion,
+} from '../../services/audit/cli-version';
+import type { MachineReachableResult } from '../../services/audit/machines-reachable';
+import type { ModuleVersionFetcher } from '../../services/audit/module-versions';
+import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
+import type { SecretCheckResult } from '../../services/audit/secrets-decryptable';
+import type { ServiceCredentialsResult } from '../../services/audit/services-credentials';
+import type { ServiceReachableResult } from '../../services/audit/services-reachable';
+import type { TerraformPlanRunner } from '../../services/audit/terraform-plan';
+import { getServiceCredentials, listContainerServices } from '../../services/container-service';
+import { runAllHealthChecks } from '../../services/health-runner';
+import { listMachines } from '../../services/machine-pool';
+import { buildTerraformEnvForModule } from '../../services/terraform-env';
+import { hasFlag } from '../parser';
+import type { CommandResult } from '../types';
+
+const execFileAsync = promisify(execFile);
+
+/** Read the running CLI version from the bundled package.json. */
+function readInstalledCliVersion(): string {
+  const here = dirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    join(here, '..', '..', '..', 'package.json'),
+    join(process.cwd(), 'package.json'),
+  ];
+  for (const path of candidates) {
+    if (existsSync(path)) {
+      try {
+        const pkg = JSON.parse(readFileSync(path, 'utf-8')) as { version?: string };
+        if (pkg.version) return pkg.version;
+      } catch {
+        // try the next candidate
+      }
+    }
+  }
+  return '0.0.0'; // unknown — caller treats as "ahead of latest"
+}
+
+function makeRegistryFetcher(client: RegistryClient): ModuleVersionFetcher {
+  return async (moduleId) => {
+    const entries = await client.getIndex(moduleId);
+    if (entries.length === 0) return { latest: null };
+    const latest = client.latestVersion(entries);
+    if (!latest) return { latest: null };
+    return { latest: latest.vers, intermediateCount: entries.length };
+  };
+}
+
+const realTerraformPlan: TerraformPlanRunner = async (terraformDir, envVars) => {
+  try {
+    const result = await execFileAsync(
+      'terraform',
+      ['plan', '-detailed-exitcode', '-no-color', '-input=false'],
+      {
+        cwd: terraformDir,
+        timeout: 120_000,
+        // Inherit current env, then layer module-specific TF_VAR_*
+        // credentials so terraform sees the same world `module deploy`
+        // would. Without this, modules with provider creds (Proxmox,
+        // DigitalOcean) fail immediately with "no value for required
+        // variable" — a false positive, not real drift.
+        env: { ...process.env, ...envVars },
+      },
+    );
+    return { exitCode: 0, stdout: result.stdout, stderr: result.stderr };
+  } catch (err) {
+    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string };
+    return {
+      // detailed-exitcode: 0 = no diff, 1 = error, 2 = diff
+      exitCode: e.code ?? 1,
+      stdout: e.stdout ?? '',
+      stderr: e.stderr ?? e.message ?? 'terraform plan failed',
+    };
+  }
+};
+
+function findMigrationsFolderSafe(): string | null {
+  // Mirror db/client.ts:findMigrationsFolder, but return null instead
+  // of throwing so the audit gracefully reports "schema check skipped"
+  // rather than crashing.
+  try {
+    const here = dirname(fileURLToPath(import.meta.url));
+    const candidates = [
+      join(here, '..', '..', '..', 'drizzle'),
+      join(process.cwd(), 'drizzle'),
+      join(process.cwd(), 'apps', 'celilo', 'drizzle'),
+    ];
+    for (const c of candidates) {
+      if (existsSync(join(c, 'meta', '_journal.json'))) return c;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Build the dependency object for `runAudit` from the live system.
+ *
+ * `onProgress` (when supplied) suppresses stdout-bound progress
+ * indicators (FuelGauge) inside the audit subroutines and routes
+ * progress messages to the caller instead — used by the TUI so
+ * audit progress appears inside the alt-screen render.
+ */
+async function buildAuditDeps(onProgress?: (msg: string) => void) {
+  const db = getDb();
+
+  const installed = db.select().from(modules).all();
+  const deployedModules = installed.filter((m) => ['INSTALLED', 'VERIFIED'].includes(m.state));
+
+  const registryClient = new RegistryClient();
+
+  // Most recent successful backup per module.
+  // The `backups` table is added via inline ALTER statements in db/client.ts
+  // for upgraded databases; a freshly-initialized DB created from the
+  // drizzle journal alone may not have it yet. Treat absence as "no
+  // backups recorded" rather than crashing the audit.
+  const latestBackupByModule = new Map<string, number>();
+  try {
+    const successfulBackups = db
+      .select()
+      .from(backups)
+      .where(eq(backups.status, 'completed'))
+      .all();
+    for (const b of successfulBackups) {
+      if (!b.moduleId || !b.completedAt) continue;
+      const ts = b.completedAt.getTime();
+      const prev = latestBackupByModule.get(b.moduleId);
+      if (prev === undefined || ts > prev) latestBackupByModule.set(b.moduleId, ts);
+    }
+  } catch {
+    // backups table missing — leave map empty
+  }
+
+  // Build per-module config map for module-configs check.
+  const allConfigs = db
+    .select()
+    .from(moduleConfigsTbl)
+    .where(isNotNull(moduleConfigsTbl.moduleId))
+    .all();
+  const configsByModule = new Map<string, Record<string, unknown>>();
+  for (const c of allConfigs) {
+    const map = configsByModule.get(c.moduleId) ?? {};
+    map[c.key] = c.valueJson ? JSON.parse(c.valueJson) : c.value;
+    configsByModule.set(c.moduleId, map);
+  }
+
+  const installedConfigs = deployedModules.map((m) => ({
+    id: m.id,
+    manifest: m.manifestData as ModuleManifest,
+    configs: configsByModule.get(m.id) ?? {},
+  }));
+
+  const installedBackupInfo = deployedModules.map((m) => ({
+    id: m.id,
+    manifest: m.manifestData as ModuleManifest,
+    lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
+  }));
+
+  // Compose per-module TF_VAR_* env vars in parallel — each call hits
+  // the secret store / DB so they're not free, but they're all
+  // independent. Empty record for machine-bound modules.
+  const terraformEnvByModule = new Map<string, Record<string, string>>();
+  await Promise.all(
+    deployedModules.map(async (m) => {
+      try {
+        terraformEnvByModule.set(m.id, await buildTerraformEnvForModule(m.id, db));
+      } catch {
+        // Credential lookup failed — leave empty; terraform plan will
+        // surface the genuine "missing var" error and the user can
+        // fix the underlying credential.
+        terraformEnvByModule.set(m.id, {});
+      }
+    }),
+  );
+
+  const terraformModules = deployedModules.map((m) => ({
+    id: m.id,
+    terraformDir: existsSync(join(m.sourcePath, 'generated', 'terraform'))
+      ? join(m.sourcePath, 'generated', 'terraform')
+      : null,
+    envVars: terraformEnvByModule.get(m.id) ?? {},
+  }));
+
+  const healthResults = await runAllHealthChecks(db, { onProgress });
+
+  // Services-credentials: try decrypting each container service's
+  // credential envelope. Failures (missing master key, wrong
+  // provider shape, corrupt envelope) become BLOCKED audit findings.
+  const allServices = await listContainerServices();
+  const serviceCredResults: ServiceCredentialsResult[] = await Promise.all(
+    allServices.map(async (s) => {
+      try {
+        await getServiceCredentials(s.id);
+        return {
+          serviceId: s.serviceId,
+          name: s.name,
+          providerName: s.providerName,
+          error: null,
+        };
+      } catch (err) {
+        return {
+          serviceId: s.serviceId,
+          name: s.name,
+          providerName: s.providerName,
+          error: err instanceof Error ? err.message : String(err),
+        };
+      }
+    }),
+  );
+
+  // Secrets-decryptable: try decrypting every encrypted secret in
+  // the DB (modules / system / capabilities). One bad master key
+  // makes every entry fail; the audit collapses that case into a
+  // single "master key mismatch" finding.
+  const secretResults: SecretCheckResult[] = [];
+  try {
+    const masterKey = await getOrCreateMasterKey();
+    const tryDecrypt = (env: { encryptedValue: string; iv: string; authTag: string }):
+      | string
+      | null => {
+      try {
+        decryptSecret(env, masterKey);
+        return null;
+      } catch (err) {
+        return err instanceof Error ? err.message : String(err);
+      }
+    };
+    for (const s of db.select().from(secrets).all()) {
+      secretResults.push({
+        scope: 'module',
+        subject: s.moduleId,
+        name: s.name,
+        error: tryDecrypt(s),
+      });
+    }
+    for (const s of db.select().from(systemSecrets).all()) {
+      secretResults.push({
+        scope: 'system',
+        subject: 'system',
+        name: s.key,
+        error: tryDecrypt(s),
+      });
+    }
+    for (const s of db.select().from(capabilitySecrets).all()) {
+      // Capability secrets can be metadata-only (encrypted fields
+      // null) — skip those, they have nothing to decrypt.
+      if (!s.encryptedValue || !s.iv || !s.authTag) continue;
+      secretResults.push({
+        scope: 'capability',
+        subject: `capability:${s.capabilityId}`,
+        name: s.name,
+        error: tryDecrypt({
+          encryptedValue: s.encryptedValue,
+          iv: s.iv,
+          authTag: s.authTag,
+        }),
+      });
+    }
+  } catch (err) {
+    // Master key unavailable — surface as a single system-level finding.
+    secretResults.push({
+      scope: 'system',
+      subject: 'system',
+      name: '<master-key>',
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+
+  // Services-reachable: ping each container service's API in parallel
+  // (Proxmox /version, DigitalOcean /v2/account). Reuses the same
+  // probes the `service add` wizard runs. Services with bad
+  // credentials get reachable=true here so we don't double-report —
+  // the credentials audit already flagged them.
+  const serviceReachableResults: ServiceReachableResult[] = await Promise.all(
+    allServices.map(async (s): Promise<ServiceReachableResult> => {
+      try {
+        const creds = await getServiceCredentials(s.id);
+        let probe: { success: boolean; message?: string };
+        if (s.providerName === 'proxmox' && 'api_url' in creds) {
+          probe = await testProxmoxConnection(creds);
+        } else if (s.providerName === 'digitalocean' && 'api_token' in creds) {
+          probe = await testDigitalOceanConnection(creds);
+        } else {
+          return {
+            serviceId: s.serviceId,
+            name: s.name,
+            providerName: s.providerName,
+            reachable: true,
+          };
+        }
+        return {
+          serviceId: s.serviceId,
+          name: s.name,
+          providerName: s.providerName,
+          reachable: probe.success,
+          message: probe.success ? undefined : (probe.message ?? 'unreachable'),
+        };
+      } catch {
+        return {
+          serviceId: s.serviceId,
+          name: s.name,
+          providerName: s.providerName,
+          reachable: true,
+        };
+      }
+    }),
+  );
+
+  // Machines-reachable: SSH probe each pool machine in parallel.
+  // BatchMode=yes prevents password prompt hangs; ConnectTimeout=5
+  // bounds wait on unresponsive hosts. The audit's collapse logic
+  // turns "all unreachable" into a single host/network finding.
+  const allMachines = await listMachines();
+  const machineReachableResults: MachineReachableResult[] = await Promise.all(
+    allMachines.map(async (m): Promise<MachineReachableResult> => {
+      try {
+        await execFileAsync(
+          'ssh',
+          [
+            '-o',
+            'BatchMode=yes',
+            '-o',
+            'ConnectTimeout=5',
+            '-o',
+            'StrictHostKeyChecking=no',
+            '-o',
+            'UserKnownHostsFile=/dev/null',
+            `${m.sshUser}@${m.ipAddress}`,
+            'true',
+          ],
+          { timeout: 8000 },
+        );
+        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+      } catch (err) {
+        const e = err as { stderr?: string; message?: string };
+        return {
+          id: m.id,
+          hostname: m.hostname,
+          ipAddress: m.ipAddress,
+          reachable: false,
+          message: (e.stderr || e.message || 'unknown error').slice(0, 200),
+        };
+      }
+    }),
+  );
+
+  const migrationsFolder = findMigrationsFolderSafe();
+
+  return {
+    cliVersion: {
+      installedVersion: readInstalledCliVersion(),
+      fetcher: fetchLatestCliVersion satisfies LatestCliVersionFetcher,
+    },
+    schema: {
+      journal: migrationsFolder ? makeJournalReader(migrationsFolder) : () => null,
+      applied: readAppliedMigrations,
+      db,
+    },
+    capabilityAbi: {
+      modules: deployedModules.map((m) => ({
+        id: m.id,
+        manifest: m.manifestData as ModuleManifest,
+      })),
+    },
+    terraformPlan: {
+      modules: terraformModules,
+      run: realTerraformPlan,
+    },
+    moduleVersions: {
+      installed: deployedModules.map((m) => ({ id: m.id, version: m.version })),
+      fetcher: makeRegistryFetcher(registryClient),
+    },
+    moduleConfigs: { modules: installedConfigs },
+    health: { results: healthResults },
+    backups: { modules: installedBackupInfo },
+    undeployedModules: {
+      modules: installed.map((m) => ({ id: m.id, state: m.state })),
+    },
+    unconfiguredModules: {
+      modules: installed.map((m) => ({
+        id: m.id,
+        state: m.state,
+        configCount: configsByModule.get(m.id)
+          ? Object.keys(configsByModule.get(m.id) ?? {}).length
+          : 0,
+      })),
+    },
+    servicesCredentials: { results: serviceCredResults },
+    secretsDecryptable: { results: secretResults },
+    servicesReachable: { results: serviceReachableResults },
+    machinesReachable: { results: machineReachableResults },
+  };
+}
+
+const VERDICT_ICON: Record<string, string> = {
+  READY: '●',
+  DRIFT: '⚠',
+  BLOCKED: '✗',
+};
+
+const SEVERITY_ICON: Record<string, string> = {
+  drift: '⚠',
+  blocked: '✗',
+};
+
+/** Format a `SystemAuditReport` for a human reader. */
+export function formatReport(report: SystemAuditReport): string {
+  const lines: string[] = [];
+  lines.push('');
+  lines.push(
+    `${VERDICT_ICON[report.verdict] ?? '?'} ${report.verdict} (${report.findings.length} finding${report.findings.length === 1 ? '' : 's'})`,
+  );
+  lines.push(`  generated at ${report.generatedAt}`);
+
+  if (report.findings.length === 0) {
+    lines.push('');
+    lines.push('  System is up to date. No drift detected.');
+    return lines.join('\n');
+  }
+
+  // Group findings by severity, then by category.
+  const byCategory = new Map<string, DriftFinding[]>();
+  for (const f of report.findings) {
+    const key = `${f.severity}/${f.category}`;
+    const arr = byCategory.get(key) ?? [];
+    arr.push(f);
+    byCategory.set(key, arr);
+  }
+
+  // Sort: blocked first, then by category name.
+  const keys = [...byCategory.keys()].sort((a, b) => {
+    const aBlocked = a.startsWith('blocked/');
+    const bBlocked = b.startsWith('blocked/');
+    if (aBlocked !== bBlocked) return aBlocked ? -1 : 1;
+    return a.localeCompare(b);
+  });
+
+  for (const key of keys) {
+    const findings = byCategory.get(key) ?? [];
+    const [severity, category] = key.split('/');
+    lines.push('');
+    lines.push(
+      `  ${SEVERITY_ICON[severity]} ${severity.toUpperCase()} • ${category} (${findings.length})`,
+    );
+    for (const f of findings) {
+      lines.push(`    - ${f.message}`);
+      if (f.details) {
+        for (const detail of f.details.split('\n')) {
+          lines.push(`      ${detail}`);
+        }
+      }
+      if (f.remediation) {
+        lines.push(`      → ${f.remediation}`);
+      }
+    }
+  }
+
+  return lines.join('\n');
+}
+
+export async function handleSystemAudit(
+  _args: string[],
+  flags: Record<string, string | boolean>,
+): Promise<CommandResult> {
+  const json = hasFlag(flags, 'json');
+  const explicitTui = hasFlag(flags, 'tui');
+  const noTui = hasFlag(flags, 'no-tui');
+
+  // Default to the TUI when stdout is an interactive terminal AND
+  // the user hasn't asked for a non-interactive output mode. JSON
+  // and --no-tui both opt out; piped/redirected stdout falls back
+  // to the static text report so the audit stays scriptable.
+  const tui = !json && !noTui && (explicitTui || Boolean(process.stdout.isTTY));
+
+  if (tui) {
+    // Defer the audit to inside the TUI so progress messages render
+    // in the alt-screen rather than leaking to the user's scrollback.
+    const themeRaw = String(flags.theme ?? '').toLowerCase();
+    const theme = themeRaw === 'light' ? 'light' : themeRaw === 'dark' ? 'dark' : undefined;
+    const { renderAuditTui } = await import('../tui/audit-tui');
+    try {
+      await renderAuditTui(
+        async (onProgress, onCategory) => {
+          // Bracket the two long phases so the user sees something
+          // before health checks begin emitting per-module messages,
+          // and so the spinner caption keeps changing during the
+          // parallel-audit phase (which doesn't emit its own).
+          onProgress('Reading system state…');
+          const deps = await buildAuditDeps(onProgress);
+          onProgress('Analyzing drift across categories…');
+          return runAudit(deps, onCategory);
+        },
+        { theme },
+      );
+      return { success: true, message: '' };
+    } catch (err) {
+      return {
+        success: false,
+        error: err instanceof Error ? err.message : String(err),
+      };
+    }
+  }
+
+  let deps: Awaited<ReturnType<typeof buildAuditDeps>>;
+  try {
+    deps = await buildAuditDeps();
+  } catch (err) {
+    return {
+      success: false,
+      error: `audit failed to gather state: ${err instanceof Error ? err.message : String(err)}`,
+    };
+  }
+
+  const report = await runAudit(deps);
+
+  if (json) {
+    return {
+      success: true,
+      message: JSON.stringify(report, null, 2),
+      rawOutput: true,
+    };
+  }
+
+  return {
+    success: true,
+    message: formatReport(report),
+  };
+}