@celilo/cli 0.1.0

package/src/services/deploy-terraform.ts

@@ -0,0 +1,445 @@
+import { join } from 'node:path';
+import { log } from '../cli/prompts';
+import { executeBuildWithProgress } from './build-stream';
+import { validateTerraformPlanSafety } from './terraform-safety';
+
+export interface TerraformResult {
+  success: boolean;
+  output: string;
+  error?: string;
+  exitCode?: number;
+}
+
+/**
+ * Execute Terraform workflow with safety validation and streaming progress
+ * Execution function - performs infrastructure provisioning
+ *
+ * @param generatedPath - Path to generated module artifacts
+ * @param phases - Phase tracking object to update during execution
+ * @param extraEnvVars - Optional environment variables (e.g., TF_VAR_* for credentials)
+ * @returns Terraform execution result
+ */
+export async function executeTerraform(
+  generatedPath: string,
+  phases?: {
+    terraformInit?: boolean;
+    terraformPlan?: boolean;
+    terraformApply?: boolean;
+  },
+  extraEnvVars?: Record<string, string>,
+  options?: { noInteractive?: boolean },
+): Promise<TerraformResult> {
+  const terraformDir = join(generatedPath, 'terraform');
+
+  // Build environment variables for Terraform
+  const terraformEnv: Record<string, string> = {
+    TF_IN_AUTOMATION: '1', // Disable interactive prompts
+    ...extraEnvVars, // Merge in any additional env vars (e.g., TF_VAR_* credentials)
+  };
+
+  const noInteractive = options?.noInteractive;
+
+  // 1. Always run terraform init -upgrade
+  // This ensures provider versions are updated when templates change
+  // It's idempotent and safe to run on every deployment
+  const initResult = await executeTerraformCommand(
+    'init',
+    ['-upgrade'],
+    terraformDir,
+    'Initializing Terraform',
+    terraformEnv,
+    noInteractive,
+  );
+  if (!initResult.success) {
+    if (phases) phases.terraformInit = false;
+    const errorMessage = parseTerraformError(initResult.output);
+    return { ...initResult, error: errorMessage };
+  }
+
+  if (phases) phases.terraformInit = true;
+
+  // 2. Run terraform plan with streaming progress
+  let planResult = await executeBuildWithProgress({
+    command: 'terraform',
+    args: ['plan', '-detailed-exitcode', '-no-color'],
+    cwd: terraformDir,
+    title: 'Planning infrastructure',
+    env: terraformEnv,
+    noInteractive,
+  });
+
+  // Auto-recover from stale state lock (e.g., interrupted previous deploy)
+  if (!planResult.success && planResult.output.includes('Error acquiring the state lock')) {
+    const unlocked = await autoForceUnlock(
+      terraformDir,
+      planResult.output,
+      terraformEnv,
+      noInteractive,
+    );
+    if (unlocked) {
+      log.info('Retrying plan after lock recovery...');
+      planResult = await executeBuildWithProgress({
+        command: 'terraform',
+        args: ['plan', '-detailed-exitcode', '-no-color'],
+        cwd: terraformDir,
+        title: 'Planning infrastructure',
+        env: terraformEnv,
+        noInteractive,
+      });
+    }
+  }
+
+  // Terraform plan exit codes:
+  // 0 = no changes
+  // 1 = error
+  // 2 = changes present (success!)
+  const planExitCode = planResult.exitCode;
+
+  if (planExitCode !== 0 && planExitCode !== 2) {
+    if (phases) phases.terraformPlan = false;
+
+    // Parse common errors into clear messages
+    const errorMessage = parseTerraformError(planResult.output);
+
+    return {
+      success: false,
+      output: planResult.output,
+      error: errorMessage,
+      exitCode: planExitCode,
+    };
+  }
+
+  if (phases) phases.terraformPlan = true;
+
+  // Check exit code: 0 = no changes, 2 = changes present
+  if (planExitCode === 0) {
+    log.message('  No changes needed');
+    if (phases) phases.terraformApply = true;
+    return { success: true, output: planResult.output, exitCode: 0 };
+  }
+
+  // Exit code 2 = changes present, validate safety
+  const safetyCheck = validateTerraformPlanSafety(planResult.output);
+  if (!safetyCheck.safe) {
+    if (phases) phases.terraformApply = false;
+    return {
+      success: false,
+      output: planResult.output,
+      error: safetyCheck.error,
+      exitCode: 2,
+    };
+  }
+
+  // 3. Run terraform apply with streaming progress
+  let applyResult = await executeTerraformCommand(
+    'apply',
+    ['-auto-approve'],
+    terraformDir,
+    'Applying infrastructure',
+    terraformEnv,
+    noInteractive,
+  );
+
+  // Auto-recover from stale state lock during apply
+  if (!applyResult.success && applyResult.output.includes('Error acquiring the state lock')) {
+    const unlocked = await autoForceUnlock(
+      terraformDir,
+      applyResult.output,
+      terraformEnv,
+      noInteractive,
+    );
+    if (unlocked) {
+      log.info('Retrying apply after lock recovery...');
+      applyResult = await executeTerraformCommand(
+        'apply',
+        ['-auto-approve'],
+        terraformDir,
+        'Applying infrastructure',
+        terraformEnv,
+        noInteractive,
+      );
+    }
+  }
+
+  if (!applyResult.success) {
+    // Check if failure is due to "already exists" - this happens when Terraform
+    // loses connection during creation but the resource was actually created
+    if (applyResult.output.includes('already exists')) {
+      log.warn('Resource already exists - attempting automatic recovery...');
+
+      // Try to import existing resource into state
+      const importResult = await attemptAutoImport(terraformDir, applyResult.output, terraformEnv);
+
+      if (importResult.success) {
+        log.success('Resource imported into Terraform state');
+        log.message('  Retrying deployment...');
+
+        // Retry apply after successful import
+        const retryResult = await executeTerraformCommand(
+          'apply',
+          ['-auto-approve'],
+          terraformDir,
+          'Applying infrastructure',
+          terraformEnv,
+          noInteractive,
+        );
+
+        if (!retryResult.success) {
+          if (phases) phases.terraformApply = false;
+          return retryResult;
+        }
+
+        if (phases) phases.terraformApply = true;
+        log.success('Infrastructure deployed (recovered from state drift)');
+        return retryResult;
+      }
+      log.error('Automatic recovery failed - manual import required');
+      if (phases) phases.terraformApply = false;
+      return {
+        ...applyResult,
+        error: `${applyResult.error}\n\nAutomatic recovery failed. Manual fix:\ncd "${terraformDir}"\nterraform import ${importResult.suggestion || '<resource>'}`,
+      };
+    }
+
+    if (phases) phases.terraformApply = false;
+    return applyResult;
+  }
+
+  if (phases) phases.terraformApply = true;
+  log.success('Infrastructure deployed');
+  return applyResult;
+}
+
+/**
+ * Attempt to automatically import a resource that already exists
+ * Parses Terraform error output to determine resource type and ID, then imports it
+ *
+ * @param terraformDir - Terraform working directory
+ * @param errorOutput - Terraform error output containing "already exists"
+ * @param terraformEnv - Environment variables including provider credentials
+ * @returns Import result with success status and suggestion
+ */
+async function attemptAutoImport(
+  terraformDir: string,
+  errorOutput: string,
+  terraformEnv: Record<string, string>,
+): Promise<{ success: boolean; suggestion?: string }> {
+  // Parse error message to extract resource info
+  // Example: "CT 200 already exists on node 'node2'"
+  // Example: "with proxmox_lxc.homebridge"
+
+  const resourceMatch = errorOutput.match(/with\s+([\w_]+\.\w+)/);
+  if (!resourceMatch) {
+    return { success: false, suggestion: '<resource> <id>' };
+  }
+
+  const resourceName = resourceMatch[1]; // e.g., "proxmox_lxc.homebridge"
+
+  // Try to extract resource ID from error message
+  let resourceId: string | null = null;
+
+  // LXC container: "CT 200 already exists on node 'node2'" -> "node2/lxc/200"
+  const lxcMatch = errorOutput.match(/CT (\d+) already exists on node '(\w+)'/);
+  if (lxcMatch) {
+    const vmid = lxcMatch[1];
+    const node = lxcMatch[2];
+    resourceId = `${node}/lxc/${vmid}`;
+  }
+
+  // VM: "VM 100 already exists" -> need to determine format
+  // Add more patterns as needed for other resource types
+
+  if (!resourceId) {
+    return {
+      success: false,
+      suggestion: `${resourceName} <id>`,
+    };
+  }
+
+  // Attempt the import
+  log.info(`  Importing ${resourceName} as ${resourceId}...`);
+
+  const result = await executeBuildWithProgress({
+    command: 'terraform',
+    args: ['import', resourceName, resourceId],
+    cwd: terraformDir,
+    title: 'Importing existing resource',
+    env: {
+      ...terraformEnv,
+      TF_IN_AUTOMATION: '1',
+    },
+  });
+
+  return {
+    success: result.success,
+    suggestion: `${resourceName} ${resourceId}`,
+  };
+}
+
+/**
+ * Execute a single Terraform command with streaming output
+ * Execution function - runs terraform command with fuel-gauge progress
+ *
+ * @param command - Terraform subcommand (init, apply, etc.)
+ * @param args - Additional arguments
+ * @param cwd - Working directory
+ * @param title - Progress indicator title
+ * @param env - Environment variables
+ * @returns Execution result
+ */
+/**
+ * Auto-recover from a stale Terraform state lock.
+ * Parses the lock ID from the error output and runs force-unlock.
+ */
+async function autoForceUnlock(
+  terraformDir: string,
+  errorOutput: string,
+  terraformEnv: Record<string, string>,
+  noInteractive?: boolean,
+): Promise<boolean> {
+  const lockIdMatch = errorOutput.match(/ID:\s+([0-9a-f-]+)/);
+  if (!lockIdMatch) {
+    log.warn('State lock detected but could not parse lock ID');
+    return false;
+  }
+
+  const lockId = lockIdMatch[1];
+  log.warn(`Stale state lock detected (${lockId}) -- auto-recovering...`);
+
+  const result = await executeBuildWithProgress({
+    command: 'terraform',
+    args: ['force-unlock', '-force', lockId],
+    cwd: terraformDir,
+    title: 'Removing stale state lock',
+    env: { ...terraformEnv, TF_IN_AUTOMATION: '1' },
+    noInteractive,
+  });
+
+  if (result.success) {
+    log.success('State lock removed');
+    return true;
+  }
+
+  log.error(`Failed to remove state lock: ${result.error || result.output}`);
+  return false;
+}
+
+/**
+ * Parse Terraform error output into a clear, actionable message
+ */
+function parseTerraformError(output: string): string {
+  // Proxmox unreachable
+  if (output.includes('dial tcp') && output.includes('connect: operation timed out')) {
+    const ipMatch = output.match(/dial tcp ([^:]+:\d+)/);
+    const target = ipMatch ? ipMatch[1] : 'unknown';
+    return `Proxmox server unreachable at ${target}\n\nCheck:\n  - Is the Proxmox server running?\n  - Can this machine reach ${target}?\n  - Is a firewall blocking the connection?`;
+  }
+
+  // Proxmox auth error
+  if (output.includes('401') && output.includes('proxmox')) {
+    return 'Proxmox authentication failed\n\nCheck:\n  - API token ID and secret in service config\n  - Token permissions in Proxmox';
+  }
+
+  // Digital Ocean auth error
+  if (output.includes('401') && output.includes('digitalocean')) {
+    return 'Digital Ocean authentication failed\n\nCheck:\n  - API token in service config\n  - Token has write permissions';
+  }
+
+  // Connection refused
+  if (output.includes('connection refused')) {
+    const ipMatch = output.match(/dial tcp ([^:]+:\d+)/);
+    const target = ipMatch ? ipMatch[1] : 'unknown';
+    return `Connection refused to ${target}\n\nThe server is not accepting connections on that port.`;
+  }
+
+  // DNS resolution failure
+  if (output.includes('no such host') || output.includes('could not resolve')) {
+    return 'DNS resolution failed for the infrastructure provider\n\nCheck your network connection and DNS settings.';
+  }
+
+  // Proxmox permission denied (e.g., API token can't set certain LXC feature flags)
+  if (output.includes('Permission check failed') && output.includes('proxmox')) {
+    const reasonMatch = output.match(/Permission check failed \(([^)]+)\)/);
+    const reason = reasonMatch ? reasonMatch[1] : 'insufficient permissions';
+    return `Proxmox permission denied: ${reason}\n\nYour API token lacks the required privileges.\n\nCheck:\n  - Token permissions in Proxmox (Datacenter → Permissions → API Tokens)\n  - Some operations (e.g., LXC feature flags like keyctl) require root@pam\n  - Consider removing unsupported features from the module's Terraform template`;
+  }
+
+  // Terraform registry unreachable (provider download failure during init)
+  if (
+    output.includes('registry.terraform.io') &&
+    (output.includes('request canceled') ||
+      output.includes('Timeout exceeded') ||
+      output.includes('could not connect'))
+  ) {
+    return 'Cannot reach Terraform registry (registry.terraform.io)\n\nTerraform needs internet access to download provider plugins.\n\nCheck:\n  - Network connectivity from the machine running Celilo\n  - DNS resolution: nslookup registry.terraform.io\n  - Firewall rules allowing outbound HTTPS (port 443)';
+  }
+
+  // Generic: extract the Error: line
+  const errorLineMatch = output.match(/Error: (.+?)(?:\n|$)/);
+  if (errorLineMatch) {
+    return errorLineMatch[1].trim();
+  }
+
+  return `Terraform failed:\n${output.substring(0, 300)}`;
+}
+
+async function executeTerraformCommand(
+  command: string,
+  args: string[],
+  cwd: string,
+  title: string,
+  env: Record<string, string>,
+  noInteractive?: boolean,
+): Promise<TerraformResult> {
+  // Execute with streaming progress
+  const result = await executeBuildWithProgress({
+    command: 'terraform',
+    args: [command, ...args],
+    cwd,
+    title,
+    env,
+    noInteractive,
+  });
+
+  return {
+    success: result.success,
+    output: result.output,
+    error: result.error,
+    exitCode: result.exitCode,
+  };
+}
+
+/**
+ * Parse Terraform outputs after successful apply
+ * Execution function - queries Terraform state for outputs
+ *
+ * @param terraformDir - Terraform working directory
+ * @returns Parsed Terraform outputs or null if none exist
+ */
+export async function parseTerraformOutputs(
+  terraformDir: string,
+): Promise<Record<string, unknown> | null> {
+  const result = await executeBuildWithProgress({
+    command: 'terraform',
+    args: ['output', '-json'],
+    cwd: terraformDir,
+    title: 'Reading Terraform outputs',
+    env: { TF_IN_AUTOMATION: '1' },
+  });
+
+  if (!result.success) {
+    // No outputs defined is not an error - return null
+    if (result.output.includes('no outputs')) {
+      return null;
+    }
+    throw new Error(`Failed to read Terraform outputs: ${result.error}`);
+  }
+
+  try {
+    return JSON.parse(result.output) as Record<string, unknown>;
+  } catch (error) {
+    throw new Error(
+      `Failed to parse Terraform outputs JSON: ${error instanceof Error ? error.message : 'Unknown error'}`,
+    );
+  }
+}