@celilo/cli 0.1.5 → 0.1.7

package/src/services/audit/index.ts

@@ -0,0 +1,118 @@
+/**
+ * System audit aggregator.
+ *
+ * Calls each drift-category check in parallel and assembles a
+ * `SystemAuditReport`. Each category check is dependency-injected so
+ * the aggregator can be tested in isolation and the CLI command can
+ * wire the real DB / registry / health-runner / terraform binary.
+ *
+ * No mutations are performed by `runAudit`. The single source of
+ * truth for what audit can detect is this file's `AuditDeps` shape.
+ */
+
+import { type BackupsAuditDeps, auditBackups } from './backups';
+import { type CapabilityAbiAuditDeps, auditCapabilityAbi } from './capability-abi';
+import { type CliVersionAuditDeps, auditCliVersion } from './cli-version';
+import { type HealthAuditDeps, auditHealth } from './health';
+import { type MachinesReachableAuditDeps, auditMachinesReachable } from './machines-reachable';
+import { type ModuleConfigsAuditDeps, auditModuleConfigs } from './module-configs';
+import { type ModuleVersionsAuditDeps, auditModuleVersions } from './module-versions';
+import { type SchemaAuditDeps, auditSchema } from './schema';
+import { type SecretsDecryptableAuditDeps, auditSecretsDecryptable } from './secrets-decryptable';
+import {
+  type ServicesCredentialsAuditDeps,
+  auditServicesCredentials,
+} from './services-credentials';
+import { type ServicesReachableAuditDeps, auditServicesReachable } from './services-reachable';
+import { type TerraformPlanAuditDeps, auditTerraformPlan } from './terraform-plan';
+import {
+  type DriftCategory,
+  type DriftFinding,
+  type SystemAuditReport,
+  computeVerdict,
+} from './types';
+import {
+  type UnconfiguredModulesAuditDeps,
+  auditUnconfiguredModules,
+} from './unconfigured-modules';
+import { type UndeployedModulesAuditDeps, auditUndeployedModules } from './undeployed-modules';
+
+export interface AuditDeps {
+  cliVersion: CliVersionAuditDeps;
+  schema: SchemaAuditDeps;
+  capabilityAbi: CapabilityAbiAuditDeps;
+  terraformPlan: TerraformPlanAuditDeps;
+  moduleVersions: ModuleVersionsAuditDeps;
+  moduleConfigs: ModuleConfigsAuditDeps;
+  health: HealthAuditDeps;
+  backups: BackupsAuditDeps;
+  undeployedModules: UndeployedModulesAuditDeps;
+  unconfiguredModules: UnconfiguredModulesAuditDeps;
+  servicesCredentials: ServicesCredentialsAuditDeps;
+  secretsDecryptable: SecretsDecryptableAuditDeps;
+  servicesReachable: ServicesReachableAuditDeps;
+  machinesReachable: MachinesReachableAuditDeps;
+  /** Defaults to `Date.now()`-based ISO string. */
+  now?: () => Date;
+}
+
+/**
+ * Per-category lifecycle event emitted by `runAudit`. Lets a UI
+ * (the audit TUI) display per-category fuel-gauges instead of one
+ * opaque spinner. `start` fires when the category's promise enters
+ * the parallel pool; `end` fires when its promise resolves.
+ */
+export interface AuditCategoryEvent {
+  category: DriftCategory;
+  phase: 'start' | 'end';
+  /** Findings array — present on `phase: 'end'` only. */
+  findings?: DriftFinding[];
+}
+
+export async function runAudit(
+  deps: AuditDeps,
+  onProgress?: (event: AuditCategoryEvent) => void,
+): Promise<SystemAuditReport> {
+  // Wrap each category's promise so the caller sees per-category
+  // start/end events. `start` fires synchronously when this function
+  // schedules the category; in practice all 14 fire roughly at once
+  // since Promise.all spawns them in parallel.
+  const wrap = (category: DriftCategory, p: Promise<DriftFinding[]>): Promise<DriftFinding[]> => {
+    onProgress?.({ category, phase: 'start' });
+    return p.then((findings) => {
+      onProgress?.({ category, phase: 'end', findings });
+      return findings;
+    });
+  };
+
+  // Run all categories in parallel; each returns its own array.
+  const groups = await Promise.all([
+    wrap('cli_version', auditCliVersion(deps.cliVersion)),
+    wrap('schema', auditSchema(deps.schema)),
+    wrap('capability_abi', auditCapabilityAbi(deps.capabilityAbi)),
+    wrap('terraform_plan', auditTerraformPlan(deps.terraformPlan)),
+    wrap('module_versions', auditModuleVersions(deps.moduleVersions)),
+    wrap('module_configs', auditModuleConfigs(deps.moduleConfigs)),
+    wrap('health', auditHealth(deps.health)),
+    wrap('backups', auditBackups(deps.backups)),
+    wrap('undeployed_modules', auditUndeployedModules(deps.undeployedModules)),
+    wrap('unconfigured_modules', auditUnconfiguredModules(deps.unconfiguredModules)),
+    wrap('services_credentials', auditServicesCredentials(deps.servicesCredentials)),
+    wrap('secrets_decryptable', auditSecretsDecryptable(deps.secretsDecryptable)),
+    wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
+    wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
+  ]);
+
+  const findings: DriftFinding[] = groups.flat();
+  const now = (deps.now ?? (() => new Date()))();
+
+  return {
+    version: 1,
+    verdict: computeVerdict(findings),
+    generatedAt: now.toISOString(),
+    findings,
+  };
+}
+
+// Re-export types for callers.
+export type { SystemAuditReport, DriftFinding, AuditVerdict, DriftCategory } from './types';

package/src/services/audit/machines-reachable.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, test } from 'bun:test';
+import { auditMachinesReachable } from './machines-reachable';
+
+describe('auditMachinesReachable', () => {
+  test('no findings when every machine is reachable', async () => {
+    const result = await auditMachinesReachable({
+      results: [
+        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        { id: 'm2', hostname: 'dns-ext', ipAddress: '203.0.113.5', reachable: true },
+      ],
+    });
+    expect(result).toEqual([]);
+  });
+
+  test('per-machine drift finding for one unreachable host', async () => {
+    const result = await auditMachinesReachable({
+      results: [
+        { id: 'm1', hostname: 'iot', ipAddress: '10.0.0.10', reachable: true },
+        {
+          id: 'm2',
+          hostname: 'dns-ext',
+          ipAddress: '203.0.113.5',
+          reachable: false,
+          message: 'Connection timed out',
+        },
+      ],
+    });
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({
+      category: 'machines_reachable',
+      severity: 'drift',
+      code: 'machine_unreachable',
+      subject: 'm2',
+      actionable: false,
+    });
+    expect(result[0].message).toContain('dns-ext');
+    expect(result[0].details).toContain('Connection timed out');
+    expect(result[0].remediation).toContain('celilo machine remove dns-ext');
+  });
+
+  test('collapses to "all_machines_unreachable" when every machine fails', async () => {
+    const result = await auditMachinesReachable({
+      results: [
+        {
+          id: 'm1',
+          hostname: 'iot',
+          ipAddress: '10.0.0.10',
+          reachable: false,
+          message: 'host down',
+        },
+        {
+          id: 'm2',
+          hostname: 'dns-ext',
+          ipAddress: '203.0.113.5',
+          reachable: false,
+          message: 'host down',
+        },
+      ],
+    });
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({
+      category: 'machines_reachable',
+      severity: 'drift',
+      code: 'all_machines_unreachable',
+      subject: 'system',
+      actionable: false,
+    });
+    expect(result[0].message).toContain('All 2 machines unreachable');
+  });
+
+  test('does NOT collapse when only one machine is in the pool', async () => {
+    // A single machine failing is per-machine, not a system-wide signal.
+    const result = await auditMachinesReachable({
+      results: [
+        {
+          id: 'm1',
+          hostname: 'iot',
+          ipAddress: '10.0.0.10',
+          reachable: false,
+          message: 'down',
+        },
+      ],
+    });
+    expect(result).toHaveLength(1);
+    expect(result[0].code).toBe('machine_unreachable');
+  });
+});

package/src/services/audit/machines-reachable.ts

@@ -0,0 +1,87 @@
+/**
+ * Machines-reachable check.
+ *
+ * For each machine in the pool, attempts a quick SSH probe (5s
+ * timeout, BatchMode=yes so it never hangs on a password prompt).
+ * Catches:
+ *
+ * - VPS terminated or paused.
+ * - Raspberry Pi powered off / unplugged.
+ * - SSH key rotated; old key no longer works.
+ * - Network partition (machine on a zone the celilo host can't
+ *   reach right now).
+ *
+ * Severity is `drift` — these are operational, not architectural.
+ * The audit consumes pre-computed probe results so it stays unit-
+ * testable without a live SSH client.
+ */
+
+import type { DriftFinding } from './types';
+
+export interface MachineReachableResult {
+  /** Stable machine ID (UUID in the DB). */
+  id: string;
+  /** Hostname for the finding message. */
+  hostname: string;
+  ipAddress: string;
+  /** True if SSH probe succeeded. */
+  reachable: boolean;
+  /** Short description of failure when `!reachable`. */
+  message?: string;
+}
+
+export interface MachinesReachableAuditDeps {
+  results: MachineReachableResult[];
+}
+
+export async function auditMachinesReachable(
+  deps: MachinesReachableAuditDeps,
+): Promise<DriftFinding[]> {
+  const findings: DriftFinding[] = [];
+
+  // Collapse "all machines unreachable" → one finding pointing at the
+  // network/SSH-key/celilo-host root cause, not N identical lines.
+  const failed = deps.results.filter((r) => !r.reachable);
+  if (failed.length > 1 && failed.length === deps.results.length) {
+    findings.push({
+      category: 'machines_reachable',
+      severity: 'drift',
+      code: 'all_machines_unreachable',
+      message: `All ${failed.length} machines unreachable from this host`,
+      details:
+        'Every machine in the pool failed the SSH probe. Likely a\n' +
+        'network or SSH-key issue on the celilo host, not per-\n' +
+        'machine failure.',
+      remediation: [
+        'Check from this host:',
+        '  - Network: can you ping the machines directly?',
+        '  - SSH key: ls ~/.ssh/  (the key celilo uses to deploy)',
+        '  - DNS: do machine hostnames resolve?',
+      ].join('\n'),
+      actionable: false,
+      subject: 'system',
+    });
+    return findings;
+  }
+
+  for (const r of failed) {
+    findings.push({
+      category: 'machines_reachable',
+      severity: 'drift',
+      code: 'machine_unreachable',
+      message: `${r.hostname} (${r.ipAddress}): SSH unreachable`,
+      details: r.message,
+      remediation: [
+        'Verify the machine is powered on and reachable on the',
+        'network. If the SSH key was rotated, re-add the machine:',
+        `  celilo machine remove ${r.hostname}`,
+        '  celilo machine add  # (re-run the wizard)',
+      ].join('\n'),
+      // Multi-step / interactive; not a one-shot.
+      actionable: false,
+      subject: r.id,
+    });
+  }
+
+  return findings;
+}

package/src/services/audit/module-configs.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, test } from 'bun:test';
+import type { ModuleManifest, VariableDeclare } from '../../manifest/schema';
+import { type InstalledModuleConfig, auditModuleConfigs } from './module-configs';
+
+function makeVariable(overrides: Partial<VariableDeclare>): VariableDeclare {
+  return {
+    name: 'foo',
+    type: 'string',
+    required: false,
+    source: 'user',
+    ...overrides,
+  } as VariableDeclare;
+}
+
+function makeModule(
+  id: string,
+  variables: VariableDeclare[],
+  configs: Record<string, unknown> = {},
+): InstalledModuleConfig {
+  const manifest = {
+    id,
+    name: id,
+    version: '1.0.0',
+    celilo_contract: '1.0',
+    variables: { owns: variables, imports: [] },
+  } as unknown as ModuleManifest;
+  return { id, manifest, configs };
+}
+
+describe('auditModuleConfigs', () => {
+  test('no findings when every required variable has a value', async () => {
+    const result = await auditModuleConfigs({
+      modules: [
+        makeModule('caddy', [makeVariable({ name: 'hostname', required: true })], {
+          hostname: 'www',
+        }),
+      ],
+    });
+    expect(result).toEqual([]);
+  });
+
+  test('blocked finding when required variable has no value and no default', async () => {
+    const result = await auditModuleConfigs({
+      modules: [makeModule('caddy', [makeVariable({ name: 'acme_email', required: true })], {})],
+    });
+
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({
+      severity: 'blocked',
+      code: 'module_config_required_unset',
+      subject: 'caddy',
+    });
+    expect(result[0].message).toContain('acme_email');
+  });
+
+  test('no finding when required variable has a default (default applies)', async () => {
+    const result = await auditModuleConfigs({
+      modules: [
+        makeModule(
+          'caddy',
+          [makeVariable({ name: 'hostname', required: true, default: 'www' })],
+          {},
+        ),
+      ],
+    });
+    expect(result).toEqual([]);
+  });
+
+  test('no finding when optional variable is unset (no default needed)', async () => {
+    const result = await auditModuleConfigs({
+      modules: [makeModule('caddy', [makeVariable({ name: 'acme_ca', required: false })], {})],
+    });
+    expect(result).toEqual([]);
+  });
+
+  test('no finding when optional variable with default is unset', async () => {
+    const result = await auditModuleConfigs({
+      modules: [
+        makeModule(
+          'greenwave',
+          [makeVariable({ name: 'port_forwards', required: false, default: [] })],
+          {},
+        ),
+      ],
+    });
+    expect(result).toEqual([]);
+  });
+
+  test('skips non-user-sourced variables (infrastructure/capability/system)', async () => {
+    const result = await auditModuleConfigs({
+      modules: [
+        makeModule(
+          'caddy',
+          [
+            makeVariable({ name: 'target_node', required: true, source: 'infrastructure' }),
+            makeVariable({ name: 'primary_domain', required: true, source: 'capability' }),
+            makeVariable({ name: 'dns_primary', required: true, source: 'system' }),
+          ],
+          {},
+        ),
+      ],
+    });
+    expect(result).toEqual([]);
+  });
+
+  test('treats empty string as unset', async () => {
+    const result = await auditModuleConfigs({
+      modules: [
+        makeModule('caddy', [makeVariable({ name: 'acme_email', required: true })], {
+          acme_email: '',
+        }),
+      ],
+    });
+    expect(result).toHaveLength(1);
+    expect(result[0].severity).toBe('blocked');
+  });
+
+  test('reports across multiple modules (only required-without-default)', async () => {
+    const result = await auditModuleConfigs({
+      modules: [
+        makeModule('caddy', [makeVariable({ name: 'a', required: true })], {}),
+        // iptables.b has a default, so even though it's required and unset,
+        // the default resolves the value — no finding.
+        makeModule('iptables', [makeVariable({ name: 'b', required: true, default: 'x' })], {}),
+      ],
+    });
+
+    expect(result).toHaveLength(1);
+    expect(result[0].subject).toBe('caddy');
+  });
+});

package/src/services/audit/module-configs.ts

@@ -0,0 +1,80 @@
+/**
+ * Module config drift check.
+ *
+ * For each installed module, walks `manifest.variables.owns` looking
+ * for variables that *must* have a value but don't. The check
+ * ignores `source != 'user'` variables — those are auto-derived from
+ * infrastructure / capability data and aren't supposed to be set by
+ * the user.
+ *
+ * Severity rules:
+ * - `required: true` AND no current value AND no `default:` → BLOCKED
+ *   (the user has to set it; `system update` would otherwise fail at
+ *   deploy time anyway).
+ * - All other unset cases → no finding. An optional variable with a
+ *   default is by definition fine when unset (the default applies);
+ *   a required variable with a default is also fine (the default
+ *   resolves the value). Either case as drift would just be noise.
+ */
+
+import type { ModuleManifest, VariableDeclare } from '../../manifest/schema';
+import type { DriftFinding } from './types';
+
+export interface InstalledModuleConfig {
+  id: string;
+  manifest: ModuleManifest;
+  /** Map of config key → current value (string for primitives, parsed object for complex). */
+  configs: Record<string, unknown>;
+}
+
+export interface ModuleConfigsAuditDeps {
+  modules: InstalledModuleConfig[];
+}
+
+function isUnset(value: unknown): boolean {
+  return value === undefined || value === null || value === '';
+}
+
+function ownVariableFindings(
+  moduleId: string,
+  variable: VariableDeclare,
+  currentValue: unknown,
+): DriftFinding[] {
+  // Only audit user-sourced variables. Infrastructure/capability/system/
+  // terraform variables are auto-derived elsewhere; complaining about
+  // them being unset is noise.
+  if (variable.source !== 'user') return [];
+  if (!isUnset(currentValue)) return [];
+
+  const hasDefault = variable.default !== undefined && variable.default !== null;
+
+  if (variable.required && !hasDefault) {
+    return [
+      {
+        category: 'module_configs',
+        severity: 'blocked',
+        code: 'module_config_required_unset',
+        message: `${moduleId}: required config "${variable.name}" is not set`,
+        details: variable.description,
+        remediation: `celilo module config set ${moduleId} ${variable.name} <value>`,
+        actionable: true,
+        subject: moduleId,
+      },
+    ];
+  }
+
+  return [];
+}
+
+export async function auditModuleConfigs(deps: ModuleConfigsAuditDeps): Promise<DriftFinding[]> {
+  const findings: DriftFinding[] = [];
+
+  for (const m of deps.modules) {
+    const owned = m.manifest.variables?.owns ?? [];
+    for (const variable of owned) {
+      findings.push(...ownVariableFindings(m.id, variable, m.configs[variable.name]));
+    }
+  }
+
+  return findings;
+}

package/src/services/audit/module-versions.test.ts

@@ -0,0 +1,99 @@
+import { describe, expect, test } from 'bun:test';
+import { type ModuleVersionFetcher, auditModuleVersions } from './module-versions';
+
+const installed = [
+  { id: 'caddy', version: '1.2.0' },
+  { id: 'iptables', version: '1.0.0' },
+  { id: 'lunacycle', version: '1.0.0' },
+];
+
+describe('auditModuleVersions', () => {
+  test('no findings when every installed module is at registry latest', async () => {
+    const fetcher: ModuleVersionFetcher = async (id) => ({
+      latest: installed.find((m) => m.id === id)?.version ?? '0.0.0',
+    });
+    const result = await auditModuleVersions({ installed, fetcher });
+    expect(result).toEqual([]);
+  });
+
+  test('drift finding per outdated module', async () => {
+    const fetcher: ModuleVersionFetcher = async (id) => {
+      if (id === 'caddy') return { latest: '1.3.0' };
+      if (id === 'iptables') return { latest: '1.0.0' };
+      if (id === 'lunacycle') return { latest: '1.1.0' };
+      throw new Error('unexpected id');
+    };
+
+    const result = await auditModuleVersions({ installed, fetcher });
+
+    expect(result).toHaveLength(2);
+    expect(result.map((f) => f.subject).sort()).toEqual(['caddy', 'lunacycle']);
+    expect(result.find((f) => f.subject === 'caddy')?.message).toContain('1.2.0 → 1.3.0');
+  });
+
+  test('appends "intervening releases" when intermediateCount > 1', async () => {
+    const fetcher: ModuleVersionFetcher = async () => ({
+      latest: '1.5.0',
+      intermediateCount: 3,
+    });
+    const result = await auditModuleVersions({
+      installed: [{ id: 'caddy', version: '1.2.0' }],
+      fetcher,
+    });
+    expect(result[0].message).toContain('3 intervening releases');
+  });
+
+  test('renders release.json messages as details when supplied', async () => {
+    const fetcher: ModuleVersionFetcher = async () => ({
+      latest: '1.5.0',
+      intermediateCount: 3,
+      intermediateMessages: [
+        { version: '1.5.0', message: 'Add HTTP/3 support' },
+        { version: '1.4.0', message: 'Fix DNS race' },
+      ],
+    });
+    const result = await auditModuleVersions({
+      installed: [{ id: 'caddy', version: '1.2.0' }],
+      fetcher,
+    });
+
+    expect(result[0].details).toContain('Add HTTP/3 support');
+    expect(result[0].details).toContain('1.5.0');
+    expect(result[0].details).toContain('Fix DNS race');
+    expect(result[0].details).toContain('1.4.0');
+  });
+
+  test('module not in registry produces explanatory finding', async () => {
+    const fetcher: ModuleVersionFetcher = async () => ({ latest: null });
+    const result = await auditModuleVersions({
+      installed: [{ id: 'private-module', version: '1.0.0' }],
+      fetcher,
+    });
+    expect(result).toHaveLength(1);
+    expect(result[0].code).toBe('module_not_in_registry');
+  });
+
+  test('per-module fetcher error becomes a drift finding (does not abort)', async () => {
+    const fetcher: ModuleVersionFetcher = async (id) => {
+      if (id === 'iptables') throw new Error('socket hang up');
+      return { latest: installed.find((m) => m.id === id)?.version ?? '0.0.0' };
+    };
+    const result = await auditModuleVersions({ installed, fetcher });
+
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({
+      code: 'module_version_lookup_failed',
+      subject: 'iptables',
+    });
+    expect(result[0].message).toContain('socket hang up');
+  });
+
+  test('local module ahead of registry produces no finding', async () => {
+    const fetcher: ModuleVersionFetcher = async () => ({ latest: '1.0.0' });
+    const result = await auditModuleVersions({
+      installed: [{ id: 'caddy', version: '1.2.0' }],
+      fetcher,
+    });
+    expect(result).toEqual([]);
+  });
+});