@fishawack/lab-env 5.7.0-beta.1 → 5.7.0-beta.3

package/commands/create/services/aws/index.js

@@ -1,5 +1,3 @@
-const md5 = require("apache-md5");
-const fs = require("fs-extra");
 const generator = require("generate-password");
 const { S3Client, ListBucketsCommand } = require("@aws-sdk/client-s3");
 const {
@@ -10,9 +8,17 @@ const {
     SecretsManagerClient,
     DescribeSecretCommand,
 } = require("@aws-sdk/client-secrets-manager");
+const {
+    OpenSearchClient,
+    DescribeDomainCommand,
+} = require("@aws-sdk/client-opensearch");
+const {
+    ElastiCacheClient,
+    DescribeReplicationGroupsCommand,
+} = require("@aws-sdk/client-elasticache");
 
-const { nameSafe, Spinner } = require("../../libs/utilities.js");
-const { eb } = require("../../libs/vars.js");
+const { nameSafe, Spinner, pollAll } = require("../../libs/utilities.js");
+const { eb, secretGroups } = require("../../libs/vars.js");
 
 module.exports.s3 = require("./s3.js");
 module.exports.cloudfront = require("./cloudfront.js");
@@ -20,7 +26,11 @@ module.exports.iam = require("./iam.js");
 module.exports.elasticbeanstalk = require("./elasticbeanstalk.js");
 module.exports.ec2 = require("./ec2.js");
 module.exports.rds = require("./rds.js");
+module.exports.opensearch = require("./opensearch.js");
+module.exports.elasticache = require("./elasticache.js");
 module.exports.secretsmanager = require("./secretsmanager.js");
+module.exports.acm = require("./acm.js");
+module.exports.ses = require("./ses.js");
 
 module.exports.slug = (repo, client, branch, service = "s3") =>
     nameSafe(`${branch}-${repo}-${client}`, service);
@@ -29,13 +39,27 @@ module.exports.fullstackPreflight = async (
     name,
     repo,
     branch,
-    { storage = false, database = false } = {},
+    {
+        storage = false,
+        database = false,
+        opensearch = false,
+        cache = false,
+    } = {},
 ) => {
-    const conflicts = [];
     const existing = {
+        environment: false,
         storage: false,
         database: false,
-        secrets: { app: null, storage: null, database: null },
+        opensearch: false,
+        cache: false,
+        secrets: {
+            app: null,
+            storage: null,
+            database: null,
+            opensearch: null,
+            cache: null,
+            mail: null,
+        },
     };
 
     const spinner = new Spinner("Running pre-flight checks", true);
@@ -50,9 +74,7 @@ module.exports.fullstackPreflight = async (
                 (e) => e.Status !== "Terminated",
             );
             if (active.length) {
-                conflicts.push(
-                    `Elastic Beanstalk environment "${name}" already exists (status: ${active[0].Status})`,
-                );
+                existing.environment = active[0];
             }
         } catch {
             /* does not exist */
@@ -105,8 +127,56 @@ module.exports.fullstackPreflight = async (
             }
         }
 
+        // Check OpenSearch
+        if (opensearch) {
+            const osSlug = module.exports.slug(
+                repo,
+                process.env.AWS_PROFILE,
+                branch,
+                "os",
+            );
+
+            spinner.ora.text = "Checking for existing OpenSearch domain";
+            try {
+                const { DomainStatus } = await new OpenSearchClient({}).send(
+                    new DescribeDomainCommand({ DomainName: osSlug }),
+                );
+                if (DomainStatus && !DomainStatus.Deleted) {
+                    existing.opensearch = true;
+                }
+            } catch {
+                /* does not exist */
+            }
+        }
+
+        // Check ElastiCache
+        if (cache) {
+            const cacheSlug = module.exports.slug(
+                repo,
+                process.env.AWS_PROFILE,
+                branch,
+                "cache",
+            );
+
+            spinner.ora.text = "Checking for existing ElastiCache cluster";
+            try {
+                const { ReplicationGroups } = await new ElastiCacheClient(
+                    {},
+                ).send(
+                    new DescribeReplicationGroupsCommand({
+                        ReplicationGroupId: cacheSlug,
+                    }),
+                );
+                const group = ReplicationGroups[0];
+                if (group && group.Status !== "deleting") {
+                    existing.cache = true;
+                }
+            } catch {
+                /* does not exist */
+            }
+        }
+
         // Check for existing secrets
-        const secretGroups = ["app", "storage", "database"];
         for (const group of secretGroups) {
             spinner.ora.text = `Checking for existing ${group} secret`;
             try {
@@ -125,7 +195,7 @@ module.exports.fullstackPreflight = async (
         throw e;
     }
 
-    return { conflicts, existing };
+    return { existing };
 };
 
 module.exports.clients = [
@@ -272,140 +342,659 @@ module.exports.staticTerminate = async (name, repo, branch, id) => {
     }
 };
 
-module.exports.fullstack = async (
-    name,
-    tags = [],
-    credentials = [],
-    repo,
-    branch,
-    framework,
-    availability,
-    database = false,
-    storage = false,
-    existingSecrets = { app: null, storage: null, database: null },
-    snapshot = null,
-) => {
-    const { platform, language } = eb.configurations[framework];
+// Private provisioner functions for parallel execution
 
-    let s3Slug;
+const provisionStorage = async (
+    aws,
+    s3Slug,
+    tags,
+    isLaravel,
+    existingResources,
+) => {
     let s3Data = {};
 
-    if (storage) {
-        s3Slug = module.exports.slug(
-            repo,
-            process.env.AWS_PROFILE,
-            branch,
-            "s3",
-        );
+    if (!existingResources.storage) {
+        if (isLaravel) {
+            await aws.s3.createS3Bucket(s3Slug, tags, {
+                BlockPublicAcls: true,
+                IgnorePublicAcls: true,
+                BlockPublicPolicy: false,
+                RestrictPublicBuckets: false,
+            });
 
-        await module.exports.iam.createIAMUser(s3Slug, tags);
-        await module.exports.iam.attachIAMPolicy(
-            s3Slug,
-            "arn:aws:iam::aws:policy/AmazonS3FullAccess",
-        );
-        const {
-            AccessKey: { AccessKeyId },
-            AccessKey: { SecretAccessKey },
-        } = await module.exports.iam.createAccessKey(s3Slug);
+            await aws.s3.setPublicReadPolicy(s3Slug, "public/*");
+        } else {
+            await aws.iam.createIAMUser(s3Slug, tags);
+            await aws.iam.attachIAMPolicy(
+                s3Slug,
+                "arn:aws:iam::aws:policy/AmazonS3FullAccess",
+            );
+            const {
+                AccessKey: { AccessKeyId },
+                AccessKey: { SecretAccessKey },
+            } = await aws.iam.createAccessKey(s3Slug);
 
-        await module.exports.s3.createS3Bucket(
-            s3Slug,
-            tags,
-            false,
-            "BucketOwnerPreferred",
-        );
+            await aws.s3.createS3Bucket(
+                s3Slug,
+                tags,
+                false,
+                "BucketOwnerPreferred",
+            );
 
-        s3Data = { s3Slug, AccessKeyId, SecretAccessKey };
+            s3Data = { s3Slug, AccessKeyId, SecretAccessKey };
+        }
     }
 
-    await module.exports.iam.ensureEBInstanceProfileExists();
-    await module.exports.iam.ensureEBManagedUpdateProfileExists();
+    if (!s3Data.s3Slug) {
+        s3Data = { s3Slug };
+    }
 
-    // Create decoupled RDS instance if database is required
-    let rdsData = {};
+    return s3Data;
+};
 
-    if (database) {
-        const rdsSlug = module.exports.slug(
-            repo,
-            process.env.AWS_PROFILE,
-            branch,
-            "rds",
-        );
+const provisionRDS = async (
+    aws,
+    name,
+    repo,
+    branch,
+    tags,
+    vpcData,
+    existingResources,
+    existingSecrets,
+    snapshot,
+) => {
+    const rdsSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "rds");
+    const rdsClient = new RDSClient({});
 
+    if (!existingResources.database) {
         const dbPassword = generator.generate({
             length: 10,
             numbers: true,
         });
 
-        await module.exports.rds.ensureSSLParameterGroup();
+        await aws.rds.ensureSSLParameterGroup();
 
-        const rdsSecurityGroupId =
-            await module.exports.ec2.createRDSSecurityGroup(rdsSlug, tags);
+        const rdsSecurityGroupId = await aws.ec2.createRDSSecurityGroup(
+            rdsSlug,
+            tags,
+        );
+
+        const dbSubnetGroupName = await aws.rds.ensureDBSubnetGroup(
+            vpcData.privateSubnetIds,
+        );
 
         if (snapshot) {
-            await module.exports.rds.restoreRDSInstanceFromSnapshot(
+            await aws.rds.restoreRDSInstanceFromSnapshot(
                 rdsSlug,
                 snapshot,
                 rdsSecurityGroupId,
                 tags,
+                dbSubnetGroupName,
             );
 
-            await module.exports.rds.waitForRDSInstance(rdsSlug);
-
-            await module.exports.rds.modifyRDSPassword(rdsSlug, dbPassword);
+            await aws.rds.waitForRDSInstance(rdsSlug);
+            await aws.rds.modifyRDSPassword(rdsSlug, dbPassword);
         } else {
-            await module.exports.rds.createRDSInstance(
+            await aws.rds.createRDSInstance(
                 rdsSlug,
                 tags,
                 dbPassword,
                 rdsSecurityGroupId,
+                dbSubnetGroupName,
             );
         }
 
-        const rdsInstance =
-            await module.exports.rds.waitForRDSInstance(rdsSlug);
+        return {
+            rdsData: { RDS_PASSWORD: dbPassword },
+            rdsSlug,
+            poll: {
+                name: "RDS",
+                cb: async () =>
+                    (
+                        await rdsClient.send(
+                            new DescribeDBInstancesCommand({
+                                DBInstanceIdentifier: rdsSlug,
+                            }),
+                        )
+                    ).DBInstances[0],
+                shouldContinue: ({ DBInstanceStatus }) =>
+                    DBInstanceStatus !== "available",
+            },
+        };
+    } else if (existingResources.database && !existingSecrets.database) {
+        const dbPassword = generator.generate({
+            length: 10,
+            numbers: true,
+        });
+
+        await aws.rds.modifyRDSPassword(rdsSlug, dbPassword);
+
+        return {
+            rdsData: { RDS_PASSWORD: dbPassword },
+            rdsSlug,
+            poll: {
+                name: "RDS",
+                cb: async () =>
+                    (
+                        await rdsClient.send(
+                            new DescribeDBInstancesCommand({
+                                DBInstanceIdentifier: rdsSlug,
+                            }),
+                        )
+                    ).DBInstances[0],
+                shouldContinue: ({ DBInstanceStatus }) =>
+                    DBInstanceStatus !== "available",
+            },
+        };
+    } else {
+        const secretValue = await aws.secretsmanager.getSecretValue(
+            `${name}-database`,
+        );
+        return {
+            rdsData: {
+                RDS_HOSTNAME: secretValue.RDS_HOSTNAME,
+                RDS_PORT: secretValue.RDS_PORT,
+                RDS_PASSWORD: secretValue.RDS_PASSWORD,
+            },
+            rdsSlug,
+        };
+    }
+};
+
+const provisionOpenSearch = async (
+    aws,
+    osSlug,
+    tags,
+    vpcData,
+    instanceProfileName,
+    existingResources,
+) => {
+    const osClient = new OpenSearchClient({});
+
+    if (!existingResources.opensearch) {
+        await aws.iam.ensureOpenSearchServiceRole();
+
+        const roleArn = instanceProfileName
+            ? (await aws.iam.getRole(instanceProfileName)).Role.Arn
+            : undefined;
+
+        const osSecurityGroupId = await aws.ec2.createOpenSearchSecurityGroup(
+            osSlug,
+            tags,
+        );
+
+        await aws.opensearch.createOpenSearchDomain(
+            osSlug,
+            osSecurityGroupId,
+            vpcData.privateSubnetIds,
+            roleArn,
+            tags,
+        );
+
+        return {
+            osData: {},
+            poll: {
+                name: "OpenSearch",
+                cb: async () => {
+                    const { DomainStatus } = await osClient.send(
+                        new DescribeDomainCommand({ DomainName: osSlug }),
+                    );
+                    return DomainStatus;
+                },
+                shouldContinue: (status) =>
+                    status.Processing ||
+                    (!status.Endpoint && !status.Endpoints),
+            },
+        };
+    } else {
+        const osDomain = await aws.opensearch.describeOpenSearchDomain(osSlug);
+
+        const osEndpoint =
+            osDomain.Endpoint || (osDomain.Endpoints && osDomain.Endpoints.vpc);
+
+        return { osData: { SEARCH_HOST: `https://${osEndpoint}` } };
+    }
+};
+
+const provisionCache = async (
+    aws,
+    name,
+    cacheSlug,
+    tags,
+    vpcData,
+    existingResources,
+    existingSecrets,
+) => {
+    const cacheClient = new ElastiCacheClient({});
+
+    if (!existingResources.cache) {
+        const cachePassword = generator.generate({
+            length: 16,
+            numbers: true,
+            symbols: false,
+            strict: true,
+        });
+
+        const cacheSubnetGroupName =
+            await aws.elasticache.ensureCacheSubnetGroup(
+                vpcData.privateSubnetIds,
+            );
+
+        const cacheSecurityGroupId = await aws.ec2.createCacheSecurityGroup(
+            cacheSlug,
+            tags,
+        );
+
+        await aws.elasticache.createCacheCluster(
+            cacheSlug,
+            cacheSecurityGroupId,
+            cacheSubnetGroupName,
+            cachePassword,
+            tags,
+        );
+
+        return {
+            cacheData: { REDIS_PASSWORD: cachePassword },
+            poll: {
+                name: "ElastiCache",
+                cb: async () => {
+                    const { ReplicationGroups } = await cacheClient.send(
+                        new DescribeReplicationGroupsCommand({
+                            ReplicationGroupId: cacheSlug,
+                        }),
+                    );
+                    return ReplicationGroups[0];
+                },
+                shouldContinue: (group) => group.Status !== "available",
+            },
+        };
+    } else if (existingResources.cache && !existingSecrets.cache) {
+        const cluster = await aws.elasticache.describeCacheCluster(cacheSlug);
+
+        const cacheEndpoint = cluster.NodeGroups[0].PrimaryEndpoint;
 
-        rdsData = {
-            RDS_HOSTNAME: rdsInstance.Endpoint.Address,
-            RDS_PORT: String(rdsInstance.Endpoint.Port),
-            RDS_PASSWORD: dbPassword,
+        return {
+            cacheData: {
+                REDIS_HOST: cacheEndpoint.Address,
+                REDIS_PORT: String(cacheEndpoint.Port),
+                REDIS_PASSWORD: "",
+            },
+        };
+    } else {
+        const secretValue = await aws.secretsmanager.getSecretValue(
+            `${name}-cache`,
+        );
+        return {
+            cacheData: {
+                REDIS_HOST: secretValue.REDIS_HOST,
+                REDIS_PORT: secretValue.REDIS_PORT,
+                REDIS_PASSWORD: secretValue.REDIS_PASSWORD,
+            },
         };
     }
+};
+
+const provisionMail = async (aws, name, tags, mailData) => {
+    const {
+        IAMClient,
+        CreateUserCommand,
+        GetUserCommand,
+        CreateAccessKeyCommand,
+        ListAccessKeysCommand,
+    } = require("@aws-sdk/client-iam");
+    const { fromIni } = require("@aws-sdk/credential-providers");
+
+    const sesProfile = "fishawack";
+    const sesRegion = "us-east-1";
+    const groupName = "AWSSESSendingGroupDoNotRename";
+
+    const iamClient = new IAMClient({
+        region: sesRegion,
+        credentials: fromIni({ profile: sesProfile }),
+    });
+
+    const userName = `${name}-ses`;
 
+    // Create user in the fishawack account
     try {
-        await module.exports.elasticbeanstalk.createElasticBeanstalkApplication(
-            repo,
+        await Spinner.prototype.simple(
+            `Creating SES IAM user ${userName}`,
+            () => {
+                return iamClient.send(
+                    new CreateUserCommand({
+                        UserName: userName,
+                        Tags: [
+                            { Key: "client", Value: process.env.AWS_PROFILE },
+                        ].concat(
+                            tags.map((t) => ({
+                                Key: t.Key,
+                                Value: String(t.Value),
+                            })),
+                        ),
+                    }),
+                );
+            },
+        );
+    } catch {
+        await Spinner.prototype.simple(
+            `Retrieving existing SES IAM user ${userName}`,
+            () => {
+                return iamClient.send(
+                    new GetUserCommand({ UserName: userName }),
+                );
+            },
         );
+    }
+
+    // Add user to SES sending group
+    await aws.iam.addUserToGroup(userName, groupName, iamClient);
+
+    // Create access key
+    const existingKeys = await iamClient.send(
+        new ListAccessKeysCommand({ UserName: userName }),
+    );
+
+    if (existingKeys.AccessKeyMetadata.length) {
+        // Keys already exist — we can't retrieve the secret again so return what we have in secrets
+        return {
+            MAIL_USERNAME: existingKeys.AccessKeyMetadata[0].AccessKeyId,
+            MAIL_PASSWORD: "",
+            MAIL_FROM_ADDRESS: mailData.fromAddress,
+            MAIL_FROM_NAME: mailData.fromName,
+        };
+    }
+
+    const keyRes = await Spinner.prototype.simple(
+        `Creating access key for SES user`,
+        () => {
+            return iamClient.send(
+                new CreateAccessKeyCommand({ UserName: userName }),
+            );
+        },
+    );
+
+    // Derive SMTP credentials
+    const smtpPassword = aws.ses.deriveSmtpPassword(
+        keyRes.AccessKey.SecretAccessKey,
+        sesRegion,
+    );
+
+    return {
+        MAIL_USERNAME: keyRes.AccessKey.AccessKeyId,
+        MAIL_PASSWORD: smtpPassword,
+        MAIL_FROM_ADDRESS: mailData.fromAddress,
+        MAIL_FROM_NAME: mailData.fromName,
+    };
+};
+
+const authorizeIngress = async (aws, name, slug, service, authorizer) => {
+    try {
+        const sg = await aws.ec2.findSecurityGroupByName(`${slug}-${service}`);
+        const ebSg = await aws.ec2.findSecurityGroupByTag([
+            {
+                Name: "tag:elasticbeanstalk:environment-name",
+                Values: [name],
+            },
+            {
+                Name: "tag:aws:cloudformation:logical-id",
+                Values: ["AWSEBSecurityGroup"],
+            },
+        ]);
+        if (sg && ebSg) {
+            await authorizer(sg.GroupId, ebSg.GroupId);
+        }
     } catch {
         /* empty */
     }
+};
+
+module.exports.fullstack = async (
+    name,
+    tags = [],
+    repo,
+    branch,
+    framework,
+    database = false,
+    storage = false,
+    opensearch = false,
+    cache = false,
+    mail = false,
+    existingSecrets = {
+        app: null,
+        storage: null,
+        database: null,
+        opensearch: null,
+        cache: null,
+        mail: null,
+    },
+    snapshot = null,
+    elbScheme = "public",
+    existingEnvironment = false,
+    existingResources = {
+        storage: false,
+        database: false,
+        opensearch: false,
+        cache: false,
+    },
+    appName,
+    appDomain,
+    certArn,
+    instanceType = "t3.micro",
+    mailData = {},
+) => {
+    const { platform, language } = eb.configurations[framework];
+    const isLaravel = framework === "laravel";
+    const aws = module.exports;
+
+    // Compute slugs
+    const s3Slug =
+        storage || existingSecrets.storage
+            ? aws.slug(repo, process.env.AWS_PROFILE, branch, "s3")
+            : undefined;
+
+    const osSlug =
+        opensearch || existingSecrets.opensearch
+            ? aws.slug(repo, process.env.AWS_PROFILE, branch, "os")
+            : undefined;
+
+    const cacheSlug =
+        cache || existingSecrets.cache
+            ? aws.slug(repo, process.env.AWS_PROFILE, branch, "cache")
+            : undefined;
+
+    const opensearchDomainArn = osSlug
+        ? `arn:aws:es:${process.env.AWS_REGION}:*:domain/${osSlug}`
+        : undefined;
+
+    // Phase 1: IAM prerequisites (needed before OpenSearch and EB env)
+    let instanceProfileName;
+
+    if (isLaravel) {
+        instanceProfileName = await aws.iam.ensureEnvironmentInstanceProfile(
+            name,
+            s3Slug,
+            opensearchDomainArn,
+        );
+    } else {
+        await aws.iam.ensureEBInstanceProfileExists();
+    }
 
-    let hasKeyPair = false;
+    // Phase 2: VPC + EB app in parallel
+    const [vpcData] = await Promise.all([
+        aws.ec2.ensurePrivateSubnets(),
+        aws.iam.ensureEBManagedUpdateProfileExists(),
+    ]);
 
     try {
-        await module.exports.ec2.getKeyPair("id_rsa_prev");
-        hasKeyPair = true;
+        await aws.elasticbeanstalk.createElasticBeanstalkApplication(
+            repo,
+            tags,
+        );
     } catch {
         /* empty */
     }
 
-    // LetsEncrypt CN records are limited to 64 chars so use only the hash from the generated name
+    // Phase 3: Provision services in parallel
+    const servicePromises = [];
+
+    if (storage) {
+        servicePromises.push(
+            provisionStorage(aws, s3Slug, tags, isLaravel, existingResources),
+        );
+    } else {
+        servicePromises.push(Promise.resolve(s3Slug ? { s3Slug } : {}));
+    }
+
+    if (database) {
+        servicePromises.push(
+            provisionRDS(
+                aws,
+                name,
+                repo,
+                branch,
+                tags,
+                vpcData,
+                existingResources,
+                existingSecrets,
+                snapshot,
+            ),
+        );
+    } else {
+        servicePromises.push(Promise.resolve({ rdsData: {}, rdsSlug: null }));
+    }
+
+    if (opensearch) {
+        servicePromises.push(
+            provisionOpenSearch(
+                aws,
+                osSlug,
+                tags,
+                vpcData,
+                instanceProfileName,
+                existingResources,
+            ),
+        );
+    } else {
+        servicePromises.push(Promise.resolve({ osData: {} }));
+    }
+
+    if (cache) {
+        servicePromises.push(
+            provisionCache(
+                aws,
+                name,
+                cacheSlug,
+                tags,
+                vpcData,
+                existingResources,
+                existingSecrets,
+            ),
+        );
+    } else {
+        servicePromises.push(Promise.resolve({ cacheData: {} }));
+    }
+
+    const [s3Data, rdsResult, osResult, cacheResult] =
+        await Promise.all(servicePromises);
+
+    // Phase 3b: Wait for all pending services with single spinner
+    const pendingPolls = [rdsResult, osResult, cacheResult]
+        .filter((r) => r && r.poll)
+        .map((r) => r.poll);
+
+    const pollResults = pendingPolls.length ? await pollAll(pendingPolls) : [];
+
+    // Extract endpoint data from poll results
+    let rdsData = rdsResult.rdsData || {};
+
+    if (rdsResult.poll) {
+        const rdsInstance = pollResults[pendingPolls.indexOf(rdsResult.poll)];
+        rdsData.RDS_HOSTNAME = rdsInstance.Endpoint.Address;
+        rdsData.RDS_PORT = String(rdsInstance.Endpoint.Port);
+    }
+
+    let osData = osResult.osData || {};
+
+    if (osResult.poll) {
+        const osDomain = pollResults[pendingPolls.indexOf(osResult.poll)];
+        const osEndpoint =
+            osDomain.Endpoint || (osDomain.Endpoints && osDomain.Endpoints.vpc);
+        osData.SEARCH_HOST = `https://${osEndpoint}`;
+    }
+
+    let cacheData = cacheResult.cacheData || {};
+
+    if (cacheResult.poll) {
+        const cluster = pollResults[pendingPolls.indexOf(cacheResult.poll)];
+        const cacheEndpoint = cluster.NodeGroups[0].PrimaryEndpoint;
+        cacheData.REDIS_HOST = cacheEndpoint.Address;
+        cacheData.REDIS_PORT = String(cacheEndpoint.Port);
+    }
+
+    // Phase 4: Build secrets and EB environment (sequential)
     const CNAMEPrefix = `fw-auto-${name.split("-")[name.split("-").length - 1]}`;
     process.env.DOMAIN_LINK = `${CNAMEPrefix}.${process.env.AWS_REGION}.elasticbeanstalk.com`;
 
-    // Build secret payloads and create/reuse secrets
+    // Provision mail credentials if needed
+    let mailCreds = {};
+    if (mail && !existingSecrets.mail) {
+        mailCreds = await provisionMail(aws, name, tags, mailData);
+    }
+
     const payloads = eb.buildSecretPayloads(
-        { framework, availability, platform, language, database, storage },
-        { ...s3Data, ...rdsData },
+        {
+            framework,
+            platform,
+            language,
+            database,
+            storage,
+            opensearch,
+            cache,
+            mail: mail && !existingSecrets.mail,
+        },
+        {
+            ...s3Data,
+            ...rdsData,
+            ...osData,
+            ...cacheData,
+            ...mailCreds,
+            APP_NAME: appName,
+            APP_DOMAIN: appDomain,
+            SESSION_DOMAIN: appDomain
+                ? appDomain.split(".").length > 2
+                    ? appDomain.split(".").slice(1).join(".")
+                    : appDomain
+                : undefined,
+        },
     );
 
     const secretArns = {};
 
-    for (const group of ["app", "storage", "database"]) {
+    for (const group of secretGroups) {
         if (existingSecrets[group]) {
             secretArns[`${group}SecretArn`] = existingSecrets[group];
+
+            if (Object.keys(payloads[group]).length) {
+                const currentValue = await aws.secretsmanager.getSecretValue(
+                    `${name}-${group}`,
+                );
+                const newKeys = Object.keys(payloads[group]).filter(
+                    (k) => !(k in currentValue),
+                );
+
+                if (newKeys.length) {
+                    const merged = { ...currentValue };
+                    for (const k of newKeys) {
+                        merged[k] = payloads[group][k];
+                    }
+                    await aws.secretsmanager.updateSecretValue(
+                        `${name}-${group}`,
+                        merged,
+                    );
+                }
+            }
         } else if (Object.keys(payloads[group]).length) {
-            const res = await module.exports.secretsmanager.createSecret(
+            const res = await aws.secretsmanager.createSecret(
                 `${name}-${group}`,
                 payloads[group],
                 tags,
@@ -414,20 +1003,18 @@ module.exports.fullstack = async (
         }
     }
 
-    // Auto-generate environmentsecrets entries from secret keys + ARNs
     const secretSettings = [];
 
-    for (const group of ["app", "storage", "database"]) {
+    for (const group of secretGroups) {
         const arn = secretArns[`${group}SecretArn`];
         if (!arn) continue;
 
         let keys;
 
         if (existingSecrets[group]) {
-            const secretValue =
-                await module.exports.secretsmanager.getSecretValue(
-                    `${name}-${group}`,
-                );
+            const secretValue = await aws.secretsmanager.getSecretValue(
+                `${name}-${group}`,
+            );
             keys = Object.keys(secretValue);
         } else {
             keys = Object.keys(payloads[group]);
@@ -443,115 +1030,470 @@ module.exports.fullstack = async (
         }
     }
 
-    const OptionSettings = eb.merge(
-        "environments",
-        {
-            framework,
-            availability,
-            platform,
-            language,
-            database,
-            storage,
-        },
-        { ...s3Data, ...rdsData },
-    );
+    // Phase 5: Create or update EB environment
+    let environment;
 
-    OptionSettings.push(...secretSettings);
+    if (existingEnvironment) {
+        const SECRETS_NAMESPACE =
+            "aws:elasticbeanstalk:application:environmentsecrets";
+        const currentSettings =
+            await aws.elasticbeanstalk.describeConfigurationSettings(
+                repo,
+                name,
+            );
+        const currentSecrets = currentSettings.filter(
+            (s) => s.Namespace === SECRETS_NAMESPACE,
+        );
+
+        const currentKeys = new Set(currentSecrets.map((s) => s.OptionName));
+        const expectedKeys = new Set(secretSettings.map((s) => s.OptionName));
 
-    if (hasKeyPair) {
-        OptionSettings.push({
-            OptionName: "EC2KeyName",
-            Value: "id_rsa_prev",
-            Namespace: "aws:autoscaling:launchconfiguration",
+        const toAdd = secretSettings.filter(
+            (s) => !currentKeys.has(s.OptionName),
+        );
+        const toUpdate = secretSettings.filter((s) => {
+            const current = currentSecrets.find(
+                (c) => c.OptionName === s.OptionName,
+            );
+            return current && current.Value !== s.Value;
         });
-    }
+        const toRemove = currentSecrets
+            .filter((s) => !expectedKeys.has(s.OptionName))
+            .map((s) => ({
+                OptionName: s.OptionName,
+                Namespace: SECRETS_NAMESPACE,
+            }));
+
+        const optionSettings = [...toAdd, ...toUpdate];
+
+        if (optionSettings.length || toRemove.length) {
+            await aws.elasticbeanstalk.updateEnvironmentSettings(
+                name,
+                optionSettings,
+                toRemove,
+            );
+        }
 
-    const environment =
-        await module.exports.elasticbeanstalk.createElasticBeanstalkEnvironment(
-            name,
-            { framework, availability, platform, language },
-            repo,
-            OptionSettings,
-            CNAMEPrefix,
-            tags,
+        environment = existingEnvironment;
+    } else {
+        const OptionSettings = eb.merge(
+            "environments",
+            {
+                framework,
+                platform,
+                language,
+                database,
+                storage,
+                opensearch,
+                vpcId: vpcData.vpcId,
+                privateSubnetIds: vpcData.privateSubnetIds.join(","),
+                publicSubnetIds: vpcData.publicSubnetIds.join(","),
+                elbScheme,
+                instanceProfileName,
+                instanceType,
+            },
+            {
+                ...s3Data,
+                ...rdsData,
+                ...osData,
+                ...cacheData,
+                APP_NAME: appName,
+                APP_DOMAIN: appDomain,
+                SESSION_DOMAIN: appDomain
+                    ? appDomain.split(".").length > 2
+                        ? appDomain.split(".").slice(1).join(".")
+                        : appDomain
+                    : undefined,
+            },
         );
 
-    // Add RDS security group ingress rule from EB security group
-    const rdsSlugForIngress = module.exports.slug(
+        if (certArn) {
+            OptionSettings.push(
+                {
+                    OptionName: "ListenerEnabled",
+                    Value: "true",
+                    Namespace: "aws:elbv2:listener:443",
+                },
+                {
+                    OptionName: "Protocol",
+                    Value: "HTTPS",
+                    Namespace: "aws:elbv2:listener:443",
+                },
+                {
+                    OptionName: "SSLCertificateArns",
+                    Value: certArn,
+                    Namespace: "aws:elbv2:listener:443",
+                },
+                {
+                    OptionName: "SSLPolicy",
+                    Value: "ELBSecurityPolicy-FS-1-2-Res-2020-10",
+                    Namespace: "aws:elbv2:listener:443",
+                },
+                {
+                    OptionName: "DefaultProcess",
+                    Value: "default",
+                    Namespace: "aws:elbv2:listener:443",
+                },
+            );
+        }
+
+        OptionSettings.push(...secretSettings);
+
+        environment =
+            await aws.elasticbeanstalk.createElasticBeanstalkEnvironment(
+                name,
+                { framework, platform, language },
+                repo,
+                OptionSettings,
+                CNAMEPrefix,
+                tags,
+            );
+    }
+
+    // Phase 6: SG ingress rules in parallel
+    const rdsSlugForIngress = aws.slug(
         repo,
         process.env.AWS_PROFILE,
         branch,
         "rds",
     );
 
-    try {
-        const rdsSg = await module.exports.ec2.findSecurityGroupByName(
-            `${rdsSlugForIngress}-rds`,
+    const ingressPromises = [
+        authorizeIngress(
+            aws,
+            name,
+            rdsSlugForIngress,
+            "rds",
+            aws.ec2.authorizeRDSIngress,
+        ),
+    ];
+
+    if (osSlug) {
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                osSlug,
+                "os",
+                aws.ec2.authorizeOpenSearchIngress,
+            ),
         );
-        const ebSg = await module.exports.ec2.findSecurityGroupByTag([
-            {
-                Name: "tag:elasticbeanstalk:environment-name",
-                Values: [name],
-            },
-            {
-                Name: "tag:aws:cloudformation:logical-id",
-                Values: ["AWSEBSecurityGroup"],
-            },
-        ]);
-        if (rdsSg && ebSg) {
-            await module.exports.ec2.authorizeRDSIngress(
-                rdsSg.GroupId,
-                ebSg.GroupId,
-            );
-        }
-    } catch {
-        /* empty */
     }
 
+    if (cacheSlug) {
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                cacheSlug,
+                "cache",
+                aws.ec2.authorizeCacheIngress,
+            ),
+        );
+    }
+
+    await Promise.all(ingressPromises);
+
+    // Phase 7: Copy configs and return
     const configurations = eb.merge("configurations", {
         framework,
-        availability,
         platform,
         language,
     });
 
-    if (credentials.length) {
-        configurations.push(
-            `.platform/${platform}/conf.d/elasticbeanstalk/${platform === "httpd" ? `z-${framework}-` : ""}htpasswd.conf`,
-        );
-        fs.outputFileSync(
-            `.platform/${platform}/conf.d/elasticbeanstalk/.htpasswd-${branch}`,
-            credentials
-                .map((d) => `${d.username}:${md5(d.password)}`)
-                .join("\n"),
+    await aws.elasticbeanstalk.copyElasticBeanstalkConfigs(
+        configurations,
+        branch,
+    );
+
+    let config = {
+        url: `https://${environment.CNAME}`,
+        environment: environment.EnvironmentName,
+        paths: [`_IaC/${branch}/*`],
+        copy: [
+            {
+                src: `_IaC/${branch}/.elasticbeanstalk/config.yml`,
+                dest: `.elasticbeanstalk/config.yml`,
+            },
+        ],
+    };
+
+    return config;
+};
+
+module.exports.fullstackClone = async (
+    name,
+    tags,
+    repo,
+    branch,
+    applicationName,
+    sourceEnvironmentName,
+    sourceVersionLabel,
+    sourceBranch,
+) => {
+    const aws = module.exports;
+
+    // Phase 1: Read source configuration
+    const sourceSettings =
+        await aws.elasticbeanstalk.describeConfigurationSettings(
+            applicationName,
+            sourceEnvironmentName,
         );
+
+    // Phase 2: Filter settings — keep relevant namespaces, exclude immutable/auto-managed
+    const excludedOptions = new Set([
+        "aws:elasticbeanstalk:environment::EnvironmentName",
+    ]);
+
+    const includedNamespaces = new Set([
+        "aws:elasticbeanstalk:application:environment",
+        "aws:elasticbeanstalk:application:environmentsecrets",
+        "aws:autoscaling:launchconfiguration",
+        "aws:autoscaling:asg",
+        "aws:ec2:vpc",
+        "aws:elbv2:listener:443",
+        "aws:elbv2:listener:default",
+        "aws:elasticbeanstalk:environment",
+        "aws:elasticbeanstalk:environment:process:default",
+        "aws:elasticbeanstalk:healthreporting:system",
+        "aws:elasticbeanstalk:managedactions",
+        "aws:elasticbeanstalk:managedactions:platformupdate",
+        "aws:elasticbeanstalk:cloudwatch:logs",
+        "aws:elasticbeanstalk:cloudwatch:logs:health",
+    ]);
+
+    const OptionSettings = sourceSettings
+        .filter((s) => {
+            if (!includedNamespaces.has(s.Namespace)) return false;
+            const key = `${s.Namespace}::${s.OptionName}`;
+            if (excludedOptions.has(key)) return false;
+            // Exclude empty values
+            if (s.Value === undefined || s.Value === null || s.Value === "")
+                return false;
+            return true;
+        })
+        .map((s) => ({
+            Namespace: s.Namespace,
+            OptionName: s.OptionName,
+            Value: s.Value,
+        }));
+
+    // Detect which services exist from source secrets
+    const sourceSecrets = sourceSettings.filter(
+        (s) =>
+            s.Namespace ===
+            "aws:elasticbeanstalk:application:environmentsecrets",
+    );
+
+    const hasOpenSearch = sourceSecrets.some(
+        (s) => s.OptionName === "SEARCH_HOST",
+    );
+
+    // Set SECRET_MIRROR_ENV so setvars.sh reads secrets from the source env
+    OptionSettings.push({
+        Namespace: "aws:elasticbeanstalk:application:environment",
+        OptionName: "SECRET_MIRROR_ENV",
+        Value: sourceEnvironmentName,
+    });
+
+    // Override HealthCheckGracePeriod for initial deploy window
+    const healthGraceIdx = OptionSettings.findIndex(
+        (s) =>
+            s.Namespace ===
+                "aws:elasticbeanstalk:environment:process:default" &&
+            s.OptionName === "HealthCheckGracePeriod",
+    );
+    if (healthGraceIdx !== -1) {
+        OptionSettings[healthGraceIdx].Value = "600";
     }
 
-    await module.exports.elasticbeanstalk.copyElasticBeanstalkConfigs(
-        configurations,
+    // Phase 4: Create the environment
+    const CNAMEPrefix = `fw-auto-${name.split("-")[name.split("-").length - 1]}`;
+
+    const environment =
+        await aws.elasticbeanstalk.createElasticBeanstalkEnvironment(
+            name,
+            { language: detectLanguageFromSettings(sourceSettings) },
+            applicationName,
+            OptionSettings,
+            CNAMEPrefix,
+            tags,
+            sourceVersionLabel,
+        );
+
+    // Phase 5: Authorize SG ingress on source service security groups
+    const rdsSlug = aws.slug(
+        repo,
+        process.env.AWS_PROFILE,
+        sourceBranch,
+        "rds",
+    );
+    const cacheSlug = aws.slug(
+        repo,
+        process.env.AWS_PROFILE,
+        sourceBranch,
+        "cache",
     );
 
-    if (credentials.length) {
-        configurations.push(
-            `.platform/${platform}/conf.d/elasticbeanstalk/.htpasswd-${branch}`,
+    const ingressPromises = [
+        authorizeIngress(
+            aws,
+            name,
+            rdsSlug,
+            "rds",
+            aws.ec2.authorizeRDSIngress,
+        ),
+    ];
+
+    if (hasOpenSearch) {
+        const sourceOsSlug = aws.slug(
+            repo,
+            process.env.AWS_PROFILE,
+            sourceBranch,
+            "os",
+        );
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                sourceOsSlug,
+                "os",
+                aws.ec2.authorizeOpenSearchIngress,
+            ),
         );
     }
 
-    let config = {
+    if (sourceSecrets.some((s) => s.OptionName === "REDIS_HOST")) {
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                cacheSlug,
+                "cache",
+                aws.ec2.authorizeCacheIngress,
+            ),
+        );
+    }
+
+    await Promise.all(ingressPromises);
+
+    // Phase 6: Copy configs
+    const framework = detectFrameworkFromSettings(sourceSettings);
+    const { platform, language } = eb.configurations[framework];
+    const configurations = eb.merge("configurations", {
+        framework,
+        platform,
+        language,
+    });
+
+    await aws.elasticbeanstalk.copyElasticBeanstalkConfigs(
+        configurations,
+        branch,
+    );
+
+    return {
         url: `https://${environment.CNAME}`,
         environment: environment.EnvironmentName,
-        paths: configurations.map((d) => {
-            const obj = { src: d, dest: d };
-            // Fix for core mistakenly mapping .htpasswd to directory
-            if (d.includes(`.htpasswd-${branch}`)) {
-                obj.file = true;
-                obj.dest = d.split(`-${branch}`)[0];
-            }
-            return obj;
-        }),
+        paths: [`_IaC/${branch}/*`],
+        copy: [
+            {
+                src: `_IaC/${branch}/.elasticbeanstalk/config.yml`,
+                dest: `.elasticbeanstalk/config.yml`,
+            },
+        ],
     };
+};
 
-    return config;
+const detectLanguageFromSettings = (settings) => {
+    const envVars = settings.filter(
+        (s) =>
+            s.Namespace === "aws:elasticbeanstalk:application:environment" ||
+            s.Namespace ===
+                "aws:elasticbeanstalk:application:environmentsecrets",
+    );
+    const keys = envVars.map((s) => s.OptionName);
+    if (keys.includes("DJANGO_SETTINGS_MODULE")) return "python";
+    if (keys.includes("NODE_ENV")) return "node";
+    return "php";
+};
+
+const detectFrameworkFromSettings = (settings) => {
+    const envVars = settings.filter(
+        (s) =>
+            s.Namespace === "aws:elasticbeanstalk:application:environment" ||
+            s.Namespace ===
+                "aws:elasticbeanstalk:application:environmentsecrets",
+    );
+    const keys = envVars.map((s) => s.OptionName);
+    if (keys.includes("DJANGO_SETTINGS_MODULE")) return "python";
+    if (keys.includes("ADONIS_ACE_CWD")) return "adonis";
+    if (keys.includes("APP_KEY")) return "laravel";
+    return "laravel";
+};
+
+module.exports.fullstackCloneTerminate = async (
+    name,
+    repo,
+    branch,
+    sourceBranch,
+) => {
+    const aws = module.exports;
+
+    const rdsSlug = aws.slug(
+        repo,
+        process.env.AWS_PROFILE,
+        sourceBranch,
+        "rds",
+    );
+    const osSlug = aws.slug(repo, process.env.AWS_PROFILE, sourceBranch, "os");
+    const cacheSlug = aws.slug(
+        repo,
+        process.env.AWS_PROFILE,
+        sourceBranch,
+        "cache",
+    );
+
+    // Phase 1: Revoke SG ingress rules
+    await Promise.all([
+        authorizeIngress(aws, name, rdsSlug, "rds", aws.ec2.revokeRDSIngress),
+        authorizeIngress(
+            aws,
+            name,
+            osSlug,
+            "os",
+            aws.ec2.revokeOpenSearchIngress,
+        ),
+        authorizeIngress(
+            aws,
+            name,
+            cacheSlug,
+            "cache",
+            aws.ec2.revokeCacheIngress,
+        ),
+    ]);
+
+    // Phase 2: Terminate EB environment
+    try {
+        await aws.elasticbeanstalk.removeElasticBeanstalkEnvironment(name);
+    } catch {
+        /* empty */
+    }
+
+    // Phase 3: Terminate EB environment + try removing app if empty
+    await Promise.all([
+        (async () => {
+            try {
+                const envs = await aws.elasticbeanstalk.listEnvironments(repo);
+                if (!envs.length) {
+                    await aws.elasticbeanstalk.removeElasticBeanstalkApplication(
+                        repo,
+                    );
+                }
+            } catch {
+                /* empty */
+            }
+        })(),
+    ]);
 };
 
 module.exports.fullstackTerminate = async (
@@ -561,128 +1503,324 @@ module.exports.fullstackTerminate = async (
     {
         deleteStorage = false,
         deleteDatabase = false,
+        deleteOpenSearch = false,
+        deleteCache = false,
+        deleteMail = false,
         deleteAppSecret = false,
     } = {},
 ) => {
-    if (deleteStorage) {
-        const s3Slug = module.exports.slug(
-            repo,
-            process.env.AWS_PROFILE,
-            branch,
-            "s3",
-        );
+    const aws = module.exports;
 
-        try {
-            await module.exports.iam.removeIAMUser(s3Slug);
-        } catch {
-            /* empty */
-        }
+    const rdsSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "rds");
+    const osSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "os");
+    const cacheSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "cache");
 
-        try {
-            await module.exports.s3.emptyS3Bucket(s3Slug);
-        } catch {
-            /* empty */
-        }
+    // Phase 1: Storage cleanup + revoke all SG ingress rules in parallel
+    const phase1 = [];
 
-        try {
-            await module.exports.s3.removeS3Bucket(s3Slug);
-        } catch {
-            /* empty */
-        }
+    if (deleteStorage) {
+        const s3Slug = aws.slug(repo, process.env.AWS_PROFILE, branch, "s3");
+
+        phase1.push(
+            (async () => {
+                try {
+                    await aws.iam.removeIAMUser(s3Slug);
+                } catch {
+                    /* empty */
+                }
+                try {
+                    await aws.s3.emptyS3Bucket(s3Slug);
+                } catch {
+                    /* empty */
+                }
+                try {
+                    await aws.s3.removeS3Bucket(s3Slug);
+                } catch {
+                    /* empty */
+                }
+            })(),
+        );
     }
 
-    // Remove RDS inbound rule referencing EB security group before terminating EB
-    const rdsSlug = module.exports.slug(
-        repo,
-        process.env.AWS_PROFILE,
-        branch,
-        "rds",
+    phase1.push(
+        authorizeIngress(aws, name, rdsSlug, "rds", aws.ec2.revokeRDSIngress),
+        authorizeIngress(
+            aws,
+            name,
+            osSlug,
+            "os",
+            aws.ec2.revokeOpenSearchIngress,
+        ),
+        authorizeIngress(
+            aws,
+            name,
+            cacheSlug,
+            "cache",
+            aws.ec2.revokeCacheIngress,
+        ),
     );
 
+    await Promise.all(phase1);
+
+    // Phase 2: Terminate EB environment (must complete before SG deletion)
     try {
-        const rdsSg = await module.exports.ec2.findSecurityGroupByName(
-            `${rdsSlug}-rds`,
-        );
-        const ebSg = await module.exports.ec2.findSecurityGroupByTag([
-            {
-                Name: "tag:elasticbeanstalk:environment-name",
-                Values: [name],
-            },
-            {
-                Name: "tag:aws:cloudformation:logical-id",
-                Values: ["AWSEBSecurityGroup"],
-            },
-        ]);
-        if (rdsSg && ebSg) {
-            await module.exports.ec2.revokeRDSIngress(
-                rdsSg.GroupId,
-                ebSg.GroupId,
-            );
-        }
+        await aws.elasticbeanstalk.removeElasticBeanstalkEnvironment(name);
     } catch {
         /* empty */
     }
 
-    try {
-        await module.exports.elasticbeanstalk.removeElasticBeanstalkEnvironment(
-            name,
+    // Phase 3: EB app + instance profile + initiate service deletions in parallel
+    const phase3 = [
+        (async () => {
+            try {
+                await aws.elasticbeanstalk.removeElasticBeanstalkApplication(
+                    repo,
+                );
+            } catch {
+                /* empty */
+            }
+        })(),
+        (async () => {
+            try {
+                await aws.iam.removeEnvironmentInstanceProfile(name);
+            } catch {
+                /* empty */
+            }
+        })(),
+    ];
+
+    if (deleteDatabase) {
+        phase3.push(aws.rds.deleteRDSInstance(rdsSlug).catch(() => {}));
+    }
+
+    if (deleteOpenSearch) {
+        phase3.push(
+            aws.opensearch.deleteOpenSearchDomain(osSlug).catch(() => {}),
         );
-    } catch {
-        /* empty */
     }
 
-    try {
-        await module.exports.elasticbeanstalk.removeElasticBeanstalkApplication(
-            repo,
+    if (deleteCache) {
+        phase3.push(
+            aws.elasticache.deleteCacheCluster(cacheSlug).catch(() => {}),
         );
-    } catch {
-        /* empty */
     }
 
+    await Promise.all(phase3);
+
+    // Phase 4: Wait for service deletions with round-robin polling
+    const deletionPolls = [];
+
     if (deleteDatabase) {
-        try {
-            await module.exports.rds.deleteRDSInstance(rdsSlug);
-            await module.exports.rds.waitForRDSDeletion(rdsSlug);
-        } catch {
-            /* empty */
-        }
+        const rdsClient = new RDSClient({});
+
+        deletionPolls.push({
+            name: "RDS",
+            cb: async () => {
+                try {
+                    const res = await rdsClient.send(
+                        new DescribeDBInstancesCommand({
+                            DBInstanceIdentifier: rdsSlug,
+                        }),
+                    );
+                    return res.DBInstances[0];
+                } catch (e) {
+                    if (e.name === "DBInstanceNotFoundFault") {
+                        return { DBInstanceStatus: "deleted" };
+                    }
+                    throw e;
+                }
+            },
+            shouldContinue: ({ DBInstanceStatus }) =>
+                DBInstanceStatus !== "deleted",
+        });
+    }
 
-        try {
-            const sg = await module.exports.ec2.findSecurityGroupByName(
-                `${rdsSlug}-rds`,
-            );
-            if (sg) {
-                await module.exports.ec2.deleteSecurityGroup(sg.GroupId);
-            }
-        } catch {
-            /* empty */
-        }
+    if (deleteOpenSearch) {
+        const osClient = new OpenSearchClient({});
+
+        deletionPolls.push({
+            name: "OpenSearch",
+            cb: async () => {
+                try {
+                    const { DomainStatus } = await osClient.send(
+                        new DescribeDomainCommand({ DomainName: osSlug }),
+                    );
+                    return DomainStatus;
+                } catch (e) {
+                    if (e.name === "ResourceNotFoundException") {
+                        return { deleted: true };
+                    }
+                    throw e;
+                }
+            },
+            shouldContinue: (status) => !status.deleted,
+        });
     }
 
-    // Delete secrets
-    if (deleteStorage) {
-        try {
-            await module.exports.secretsmanager.deleteSecret(`${name}-storage`);
-        } catch {
-            /* empty */
-        }
+    if (deleteCache) {
+        const cacheClient = new ElastiCacheClient({});
+
+        deletionPolls.push({
+            name: "ElastiCache",
+            cb: async () => {
+                try {
+                    const { ReplicationGroups } = await cacheClient.send(
+                        new DescribeReplicationGroupsCommand({
+                            ReplicationGroupId: cacheSlug,
+                        }),
+                    );
+                    return ReplicationGroups[0];
+                } catch (e) {
+                    if (e.name === "ReplicationGroupNotFoundFault") {
+                        return { deleted: true };
+                    }
+                    throw e;
+                }
+            },
+            shouldContinue: (group) => !group.deleted,
+        });
+    }
+
+    if (deletionPolls.length) {
+        await pollAll(deletionPolls, "Waiting for services to terminate");
     }
 
+    // Phase 5: Delete security groups after services are gone
+    const sgDeletions = [];
+
     if (deleteDatabase) {
-        try {
-            await module.exports.secretsmanager.deleteSecret(
-                `${name}-database`,
-            );
-        } catch {
-            /* empty */
-        }
+        sgDeletions.push(
+            (async () => {
+                try {
+                    const sg = await aws.ec2.findSecurityGroupByName(
+                        `${rdsSlug}-rds`,
+                    );
+                    if (sg) {
+                        await aws.ec2.deleteSecurityGroup(sg.GroupId);
+                    }
+                } catch {
+                    /* empty */
+                }
+            })(),
+        );
     }
 
+    if (deleteOpenSearch) {
+        sgDeletions.push(
+            (async () => {
+                try {
+                    const sg = await aws.ec2.findSecurityGroupByName(
+                        `${osSlug}-os`,
+                    );
+                    if (sg) {
+                        await aws.ec2.deleteSecurityGroup(sg.GroupId);
+                    }
+                } catch {
+                    /* empty */
+                }
+            })(),
+        );
+    }
+
+    if (deleteCache) {
+        sgDeletions.push(
+            (async () => {
+                try {
+                    const sg = await aws.ec2.findSecurityGroupByName(
+                        `${cacheSlug}-cache`,
+                    );
+                    if (sg) {
+                        await aws.ec2.deleteSecurityGroup(sg.GroupId);
+                    }
+                } catch {
+                    /* empty */
+                }
+            })(),
+        );
+    }
+
+    if (sgDeletions.length) {
+        await Promise.all(sgDeletions);
+    }
+
+    // Phase 6: Delete secrets in parallel
+    const secretDeletions = [];
+
+    if (deleteStorage) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-storage`).catch(() => {}),
+        );
+    }
+    if (deleteDatabase) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-database`).catch(() => {}),
+        );
+    }
+    if (deleteOpenSearch) {
+        secretDeletions.push(
+            aws.secretsmanager
+                .deleteSecret(`${name}-opensearch`)
+                .catch(() => {}),
+        );
+    }
+    if (deleteCache) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-cache`).catch(() => {}),
+        );
+    }
     if (deleteAppSecret) {
-        try {
-            await module.exports.secretsmanager.deleteSecret(`${name}-app`);
-        } catch {
-            /* empty */
-        }
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-app`).catch(() => {}),
+        );
+    }
+    if (deleteMail) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-mail`).catch(() => {}),
+        );
+        secretDeletions.push(
+            (async () => {
+                const {
+                    IAMClient,
+                    DeleteAccessKeyCommand,
+                    ListAccessKeysCommand,
+                    RemoveUserFromGroupCommand,
+                    DeleteUserCommand,
+                } = require("@aws-sdk/client-iam");
+                const { fromIni } = require("@aws-sdk/credential-providers");
+
+                const iamClient = new IAMClient({
+                    region: "us-east-1",
+                    credentials: fromIni({ profile: "fishawack" }),
+                });
+                const userName = `${name}-ses`;
+
+                try {
+                    const keys = await iamClient.send(
+                        new ListAccessKeysCommand({ UserName: userName }),
+                    );
+                    for (const key of keys.AccessKeyMetadata || []) {
+                        await iamClient.send(
+                            new DeleteAccessKeyCommand({
+                                UserName: userName,
+                                AccessKeyId: key.AccessKeyId,
+                            }),
+                        );
+                    }
+                    await iamClient.send(
+                        new RemoveUserFromGroupCommand({
+                            UserName: userName,
+                            GroupName: "AWSSESSendingGroupDoNotRename",
+                        }),
+                    );
+                    await iamClient.send(
+                        new DeleteUserCommand({ UserName: userName }),
+                    );
+                } catch {
+                    /* user may not exist */
+                }
+            })(),
+        );
     }
+
+    await Promise.all(secretDeletions);
 };