@rulebricks/cli 2.1.3 → 2.1.5

package/dist/lib/terraform.d.ts

@@ -24,14 +24,19 @@ export declare function terraformPlan(deploymentName: string): Promise<void>;
  */
 export declare function terraformApply(deploymentName: string): Promise<void>;
 /**
- * Cleans up orphaned cloud resources that may linger after a failed deploy or
- * incomplete destroy. Best-effort: failures are logged but never thrown.
+ * Lightweight pre-deploy cleanup for the CloudWatch log group that the EKS module
+ * no longer manages (create_cloudwatch_log_group = false). Safe to call before
+ * terraform apply since it targets a resource outside terraform's control.
  */
 export declare function cleanupOrphanedResources(provider: CloudProvider, clusterName: string, region: string): Promise<void>;
 /**
- * Destroys Terraform infrastructure with retry logic.
- * Runs init first to ensure .terraform folder exists (handles partial deployments).
- * After all attempts, sweeps orphaned cloud resources that Terraform doesn't manage.
+ * Destroys Terraform infrastructure, then sweeps remaining cloud resources.
+ *
+ * Flow:
+ *   1. terraform destroy (single attempt)
+ *   2. Cloud-native cleanup ALWAYS runs (terraform can report success while
+ *      resources still exist)
+ *   3. If terraform reported failure, try once more now that blockers are gone
  */
 export declare function terraformDestroy(deploymentName: string, cloudContext?: {
     provider: CloudProvider;

package/dist/lib/terraform.js

@@ -169,14 +169,12 @@ export async function terraformApply(deploymentName) {
     }
 }
 /**
- * Cleans up orphaned cloud resources that may linger after a failed deploy or
- * incomplete destroy. Best-effort: failures are logged but never thrown.
+ * Lightweight pre-deploy cleanup for the CloudWatch log group that the EKS module
+ * no longer manages (create_cloudwatch_log_group = false). Safe to call before
+ * terraform apply since it targets a resource outside terraform's control.
  */
 export async function cleanupOrphanedResources(provider, clusterName, region) {
     if (provider === 'aws') {
-        // The EKS module (or AWS itself) creates /aws/eks/<cluster>/cluster.
-        // Since we disabled Terraform management of this log group, we must
-        // delete it ourselves to ensure a clean slate.
         const logGroupName = `/aws/eks/${clusterName}/cluster`;
         try {
             await execa('aws', [
@@ -189,14 +187,368 @@ export async function cleanupOrphanedResources(provider, clusterName, region) {
             // Log group may not exist — that's fine
         }
     }
-    // GCP and Azure don't have an equivalent orphan problem today
 }
-const DESTROY_MAX_ATTEMPTS = 3;
-const DESTROY_RETRY_DELAY_MS = 5_000;
+// ============================================================================
+// Post-destroy cloud-native cleanup (AWS)
+//
+// Handles every uniquely-named resource that terraform tends to leave behind
+// after a failed destroy or partial apply. Runs unconditionally after every
+// terraform destroy since terraform can report success while resources linger.
+// Every step is best-effort: failures are silently swallowed.
+// ============================================================================
+async function deleteAwsEksNodeGroups(clusterName, region) {
+    let nodeGroups;
+    try {
+        const { stdout } = await execa('aws', [
+            'eks', 'list-nodegroups',
+            '--cluster-name', clusterName,
+            '--region', region,
+            '--output', 'json',
+        ]);
+        const parsed = JSON.parse(stdout);
+        nodeGroups = parsed.nodegroups ?? [];
+    }
+    catch {
+        return; // Cluster may not exist
+    }
+    for (const ng of nodeGroups) {
+        try {
+            await execa('aws', [
+                'eks', 'delete-nodegroup',
+                '--cluster-name', clusterName,
+                '--nodegroup-name', ng,
+                '--region', region,
+            ]);
+        }
+        catch { /* already gone */ }
+    }
+    // Wait for all node groups to finish deleting
+    for (const ng of nodeGroups) {
+        try {
+            await execa('aws', [
+                'eks', 'wait', 'nodegroup-deleted',
+                '--cluster-name', clusterName,
+                '--nodegroup-name', ng,
+                '--region', region,
+            ]);
+        }
+        catch { /* timeout or already gone */ }
+    }
+}
+async function deleteAwsEksCluster(clusterName, region) {
+    try {
+        await execa('aws', [
+            'eks', 'delete-cluster',
+            '--name', clusterName,
+            '--region', region,
+        ]);
+    }
+    catch {
+        return; // Cluster may not exist
+    }
+    try {
+        await execa('aws', [
+            'eks', 'wait', 'cluster-deleted',
+            '--name', clusterName,
+            '--region', region,
+        ]);
+    }
+    catch { /* timeout or already gone */ }
+}
+async function deleteAwsCloudWatchLogGroup(clusterName, region) {
+    try {
+        await execa('aws', [
+            'logs', 'delete-log-group',
+            '--log-group-name', `/aws/eks/${clusterName}/cluster`,
+            '--region', region,
+        ]);
+    }
+    catch { /* may not exist */ }
+}
+/**
+ * Captures the OIDC issuer URL from an EKS cluster before it's deleted.
+ * The URL uses a random cluster ID (not the cluster name), so we must
+ * grab it while the cluster still exists to identify the OIDC provider later.
+ */
+async function getEksOidcIssuer(clusterName, region) {
+    try {
+        const { stdout } = await execa('aws', [
+            'eks', 'describe-cluster',
+            '--name', clusterName,
+            '--region', region,
+            '--query', 'cluster.identity.oidc.issuer',
+            '--output', 'text',
+        ]);
+        const url = stdout.trim();
+        return url && url !== 'None' ? url : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+async function deleteAwsOidcProvider(oidcIssuerUrl) {
+    if (!oidcIssuerUrl)
+        return;
+    // Strip the https:// prefix to match how IAM stores the URL
+    const issuerHost = oidcIssuerUrl.replace('https://', '');
+    let providerArns;
+    try {
+        const { stdout } = await execa('aws', [
+            'iam', 'list-open-id-connect-providers',
+            '--output', 'json',
+        ]);
+        const parsed = JSON.parse(stdout);
+        providerArns = (parsed.OpenIDConnectProviderList ?? []).map((p) => p.Arn);
+    }
+    catch {
+        return;
+    }
+    for (const arn of providerArns) {
+        try {
+            const { stdout } = await execa('aws', [
+                'iam', 'get-open-id-connect-provider',
+                '--open-id-connect-provider-arn', arn,
+                '--output', 'json',
+            ]);
+            const parsed = JSON.parse(stdout);
+            if (parsed.Url && issuerHost.includes(parsed.Url)) {
+                await execa('aws', [
+                    'iam', 'delete-open-id-connect-provider',
+                    '--open-id-connect-provider-arn', arn,
+                ]);
+            }
+        }
+        catch { /* skip */ }
+    }
+}
+async function releaseAwsElasticIps(clusterName, region) {
+    try {
+        const { stdout } = await execa('aws', [
+            'ec2', 'describe-addresses',
+            '--filters', `Name=tag:Name,Values=*${clusterName}*`,
+            '--region', region,
+            '--query', 'Addresses[?AssociationId==null].AllocationId',
+            '--output', 'json',
+        ]);
+        const allocationIds = JSON.parse(stdout);
+        for (const id of allocationIds) {
+            try {
+                await execa('aws', [
+                    'ec2', 'release-address',
+                    '--allocation-id', id,
+                    '--region', region,
+                ]);
+            }
+            catch { /* may already be released */ }
+        }
+    }
+    catch { /* skip */ }
+}
+async function deleteAwsIamRole(roleName) {
+    // Detach all managed policies
+    try {
+        const { stdout } = await execa('aws', [
+            'iam', 'list-attached-role-policies',
+            '--role-name', roleName,
+            '--output', 'json',
+        ]);
+        const parsed = JSON.parse(stdout);
+        for (const policy of parsed.AttachedPolicies ?? []) {
+            try {
+                await execa('aws', [
+                    'iam', 'detach-role-policy',
+                    '--role-name', roleName,
+                    '--policy-arn', policy.PolicyArn,
+                ]);
+            }
+            catch { /* skip */ }
+        }
+    }
+    catch { /* role may not exist */ }
+    // Delete inline policies
+    try {
+        const { stdout } = await execa('aws', [
+            'iam', 'list-role-policies',
+            '--role-name', roleName,
+            '--output', 'json',
+        ]);
+        const parsed = JSON.parse(stdout);
+        for (const policyName of parsed.PolicyNames ?? []) {
+            try {
+                await execa('aws', [
+                    'iam', 'delete-role-policy',
+                    '--role-name', roleName,
+                    '--policy-name', policyName,
+                ]);
+            }
+            catch { /* skip */ }
+        }
+    }
+    catch { /* role may not exist */ }
+    // Delete the role itself
+    try {
+        await execa('aws', ['iam', 'delete-role', '--role-name', roleName]);
+    }
+    catch { /* may not exist */ }
+}
+async function deleteAwsKmsAlias(clusterName, region) {
+    const aliasName = `alias/eks/${clusterName}`;
+    let keyId;
+    // Find the KMS key behind the alias so we can schedule it for deletion
+    try {
+        const { stdout } = await execa('aws', [
+            'kms', 'list-aliases',
+            '--query', `Aliases[?AliasName=='${aliasName}'].TargetKeyId | [0]`,
+            '--output', 'text',
+            '--region', region,
+        ]);
+        const id = stdout.trim();
+        if (id && id !== 'None') {
+            keyId = id;
+        }
+    }
+    catch { /* skip */ }
+    // Delete the alias (unique name constraint -- blocks re-deploy if left behind)
+    try {
+        await execa('aws', [
+            'kms', 'delete-alias',
+            '--alias-name', aliasName,
+            '--region', region,
+        ]);
+    }
+    catch { /* may not exist */ }
+    // Schedule the underlying key for deletion (7-day mandatory minimum)
+    if (keyId) {
+        try {
+            await execa('aws', [
+                'kms', 'schedule-key-deletion',
+                '--key-id', keyId,
+                '--pending-window-in-days', '7',
+                '--region', region,
+            ]);
+        }
+        catch { /* key may already be pending deletion or not exist */ }
+    }
+}
+/**
+ * Finds KMS keys by the description the EKS module uses, and schedules them for
+ * deletion. Catches keys that survive after their alias is already deleted.
+ */
+async function scheduleAwsOrphanedKmsKeys(clusterName, region) {
+    try {
+        const { stdout } = await execa('aws', [
+            'kms', 'list-keys',
+            '--region', region,
+            '--query', 'Keys[].KeyId',
+            '--output', 'json',
+        ]);
+        const keyIds = JSON.parse(stdout);
+        for (const keyId of keyIds) {
+            try {
+                const { stdout: meta } = await execa('aws', [
+                    'kms', 'describe-key',
+                    '--key-id', keyId,
+                    '--region', region,
+                    '--query', 'KeyMetadata.{State:KeyState,Desc:Description,Manager:KeyManager}',
+                    '--output', 'json',
+                ]);
+                const info = JSON.parse(meta);
+                if (info.Manager === 'CUSTOMER' &&
+                    info.State === 'Enabled' &&
+                    info.Desc.includes(clusterName)) {
+                    await execa('aws', [
+                        'kms', 'schedule-key-deletion',
+                        '--key-id', keyId,
+                        '--pending-window-in-days', '7',
+                        '--region', region,
+                    ]);
+                }
+            }
+            catch { /* skip individual key */ }
+        }
+    }
+    catch { /* skip */ }
+}
+async function deleteAwsLaunchTemplates(clusterName, region) {
+    try {
+        const { stdout } = await execa('aws', [
+            'ec2', 'describe-launch-templates',
+            '--filters', `Name=tag:Environment,Values=rulebricks`,
+            '--region', region,
+            '--query', 'LaunchTemplates[].LaunchTemplateId',
+            '--output', 'json',
+        ]);
+        const ids = JSON.parse(stdout);
+        for (const id of ids) {
+            try {
+                await execa('aws', [
+                    'ec2', 'delete-launch-template',
+                    '--launch-template-id', id,
+                    '--region', region,
+                ]);
+            }
+            catch { /* may not exist or in use */ }
+        }
+    }
+    catch { /* skip */ }
+}
+async function deleteAwsIamPolicy(policyName) {
+    try {
+        const { stdout } = await execa('aws', [
+            'iam', 'list-policies',
+            '--query', `Policies[?PolicyName=='${policyName}']`,
+            '--output', 'json',
+        ]);
+        const policies = JSON.parse(stdout);
+        for (const policy of policies) {
+            try {
+                await execa('aws', ['iam', 'delete-policy', '--policy-arn', policy.Arn]);
+            }
+            catch { /* may have attachments or not exist */ }
+        }
+    }
+    catch { /* skip */ }
+}
+/**
+ * Comprehensive post-destroy cleanup of AWS resources that terraform leaves
+ * behind. Handles the full dependency chain in the correct order.
+ * Entirely best-effort: every step silently swallows errors.
+ */
+async function cleanupAwsResources(clusterName, region) {
+    // Capture the OIDC issuer URL BEFORE deleting the cluster -- the URL uses a
+    // random cluster ID (not the cluster name) so we can't find it after deletion.
+    const oidcIssuerUrl = await getEksOidcIssuer(clusterName, region);
+    // 1. EKS node groups (must be deleted before cluster)
+    await deleteAwsEksNodeGroups(clusterName, region);
+    // 2. EKS cluster
+    await deleteAwsEksCluster(clusterName, region);
+    // 3. CloudWatch log group (now safe -- cluster is gone, won't be recreated)
+    await deleteAwsCloudWatchLogGroup(clusterName, region);
+    // 4. OIDC provider (matched by issuer URL captured above)
+    await deleteAwsOidcProvider(oidcIssuerUrl);
+    // 5. IAM roles created by terraform modules
+    await deleteAwsIamRole(`${clusterName}-ebs-csi`);
+    await deleteAwsIamRole(`${clusterName}-external-dns`);
+    await deleteAwsIamRole(`${clusterName}-vector`);
+    // 6. Customer-managed IAM policies
+    await deleteAwsIamPolicy(`${clusterName}-vector-s3`);
+    // 7. KMS key + alias (created by EKS module for envelope encryption)
+    await deleteAwsKmsAlias(clusterName, region);
+    // 8. KMS keys that lost their alias but are still Enabled (matched by description)
+    await scheduleAwsOrphanedKmsKeys(clusterName, region);
+    // 9. Launch templates (created by EKS managed node groups)
+    await deleteAwsLaunchTemplates(clusterName, region);
+    // 10. Elastic IPs (created by VPC module for NAT gateways, cost money if leaked)
+    await releaseAwsElasticIps(clusterName, region);
+}
 /**
- * Destroys Terraform infrastructure with retry logic.
- * Runs init first to ensure .terraform folder exists (handles partial deployments).
- * After all attempts, sweeps orphaned cloud resources that Terraform doesn't manage.
+ * Destroys Terraform infrastructure, then sweeps remaining cloud resources.
+ *
+ * Flow:
+ *   1. terraform destroy (single attempt)
+ *   2. Cloud-native cleanup ALWAYS runs (terraform can report success while
+ *      resources still exist)
+ *   3. If terraform reported failure, try once more now that blockers are gone
  */
 export async function terraformDestroy(deploymentName, cloudContext) {
     const workDir = getTerraformDir(deploymentName);
@@ -211,36 +563,41 @@ export async function terraformDestroy(deploymentName, cloudContext) {
         if (execaInitError.stdout || execaInitError.stderr) {
             await saveLogFile(workDir, 'destroy-init', execaInitError.stdout || '', execaInitError.stderr || '');
         }
-        // Don't throw — continue to try destroy anyway
     }
-    let lastError;
-    for (let attempt = 1; attempt <= DESTROY_MAX_ATTEMPTS; attempt++) {
+    // First terraform destroy attempt
+    let firstAttemptFailed = false;
+    try {
+        await execa('terraform', ['destroy', '-auto-approve'], {
+            cwd: workDir
+        });
+    }
+    catch (error) {
+        firstAttemptFailed = true;
+        const execaError = error;
+        if (execaError.stdout || execaError.stderr) {
+            await saveLogFile(workDir, 'destroy', execaError.stdout || '', execaError.stderr || '');
+        }
+    }
+    // ALWAYS run cloud-native cleanup -- terraform can't be trusted to report
+    // accurately whether all resources were actually destroyed
+    if (cloudContext?.provider === 'aws') {
+        await cleanupAwsResources(cloudContext.clusterName, cloudContext.region);
+    }
+    // If terraform failed, try once more now that cloud-native cleanup removed blockers
+    if (firstAttemptFailed) {
         try {
             await execa('terraform', ['destroy', '-auto-approve'], {
                 cwd: workDir
             });
-            lastError = undefined;
-            break;
         }
         catch (error) {
             const execaError = error;
             if (execaError.stdout || execaError.stderr) {
-                await saveLogFile(workDir, `destroy-attempt-${attempt}`, execaError.stdout || '', execaError.stderr || '');
-            }
-            lastError = new Error(`Terraform destroy failed (attempt ${attempt}/${DESTROY_MAX_ATTEMPTS}):\n` +
-                `${getErrorMessage(error, 'Unknown error')}\n\nLogs saved to: ${workDir}`);
-            if (attempt < DESTROY_MAX_ATTEMPTS) {
-                await new Promise((r) => setTimeout(r, DESTROY_RETRY_DELAY_MS));
+                await saveLogFile(workDir, 'destroy-final', execaError.stdout || '', execaError.stderr || '');
             }
+            throw new Error(`Terraform destroy failed:\n${getErrorMessage(error, 'Unknown error')}\n\nLogs saved to: ${workDir}`);
         }
     }
-    // Best-effort cleanup of orphaned cloud resources regardless of destroy outcome
-    if (cloudContext) {
-        await cleanupOrphanedResources(cloudContext.provider, cloudContext.clusterName, cloudContext.region);
-    }
-    if (lastError) {
-        throw lastError;
-    }
 }
 /**
  * Gets Terraform outputs

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulebricks/cli",
-  "version": "2.1.3",
+  "version": "2.1.5",
   "description": "CLI for deploying and managing private Rulebricks instances",
   "type": "module",
   "bin": {