@fishawack/lab-env 5.7.0-beta.2 → 5.7.0-beta.3

package/commands/create/services/aws/index.js

@@ -1,5 +1,3 @@
-const md5 = require("apache-md5");
-const fs = require("fs-extra");
 const generator = require("generate-password");
 const { S3Client, ListBucketsCommand } = require("@aws-sdk/client-s3");
 const {
@@ -14,8 +12,12 @@ const {
     OpenSearchClient,
     DescribeDomainCommand,
 } = require("@aws-sdk/client-opensearch");
+const {
+    ElastiCacheClient,
+    DescribeReplicationGroupsCommand,
+} = require("@aws-sdk/client-elasticache");
 
-const { nameSafe, Spinner } = require("../../libs/utilities.js");
+const { nameSafe, Spinner, pollAll } = require("../../libs/utilities.js");
 const { eb, secretGroups } = require("../../libs/vars.js");
 
 module.exports.s3 = require("./s3.js");
@@ -25,8 +27,10 @@ module.exports.elasticbeanstalk = require("./elasticbeanstalk.js");
 module.exports.ec2 = require("./ec2.js");
 module.exports.rds = require("./rds.js");
 module.exports.opensearch = require("./opensearch.js");
+module.exports.elasticache = require("./elasticache.js");
 module.exports.secretsmanager = require("./secretsmanager.js");
 module.exports.acm = require("./acm.js");
+module.exports.ses = require("./ses.js");
 
 module.exports.slug = (repo, client, branch, service = "s3") =>
     nameSafe(`${branch}-${repo}-${client}`, service);
@@ -35,14 +39,27 @@ module.exports.fullstackPreflight = async (
     name,
     repo,
     branch,
-    { storage = false, database = false, opensearch = false } = {},
+    {
+        storage = false,
+        database = false,
+        opensearch = false,
+        cache = false,
+    } = {},
 ) => {
     const existing = {
         environment: false,
         storage: false,
         database: false,
         opensearch: false,
-        secrets: { app: null, storage: null, database: null, opensearch: null },
+        cache: false,
+        secrets: {
+            app: null,
+            storage: null,
+            database: null,
+            opensearch: null,
+            cache: null,
+            mail: null,
+        },
     };
 
     const spinner = new Spinner("Running pre-flight checks", true);
@@ -132,6 +149,33 @@ module.exports.fullstackPreflight = async (
             }
         }
 
+        // Check ElastiCache
+        if (cache) {
+            const cacheSlug = module.exports.slug(
+                repo,
+                process.env.AWS_PROFILE,
+                branch,
+                "cache",
+            );
+
+            spinner.ora.text = "Checking for existing ElastiCache cluster";
+            try {
+                const { ReplicationGroups } = await new ElastiCacheClient(
+                    {},
+                ).send(
+                    new DescribeReplicationGroupsCommand({
+                        ReplicationGroupId: cacheSlug,
+                    }),
+                );
+                const group = ReplicationGroups[0];
+                if (group && group.Status !== "deleting") {
+                    existing.cache = true;
+                }
+            } catch {
+                /* does not exist */
+            }
+        }
+
         // Check for existing secrets
         for (const group of secretGroups) {
             spinner.ora.text = `Checking for existing ${group} secret`;
@@ -298,75 +342,39 @@ module.exports.staticTerminate = async (name, repo, branch, id) => {
     }
 };
 
-module.exports.fullstack = async (
-    name,
-    tags = [],
-    credentials = [],
-    repo,
-    branch,
-    framework,
-    availability,
-    database = false,
-    storage = false,
-    opensearch = false,
-    existingSecrets = {
-        app: null,
-        storage: null,
-        database: null,
-        opensearch: null,
-    },
-    snapshot = null,
-    elbScheme = "public",
-    existingEnvironment = false,
-    existingResources = {
-        storage: false,
-        database: false,
-        opensearch: false,
-    },
-    appName,
-    appDomain,
-    certArn,
-) => {
-    const { platform, language } = eb.configurations[framework];
-    const isLaravel = framework === "laravel";
-
-    // Always compute s3Slug if storage is requested or pre-exists
-    // (needed for instance profile S3 access even when bucket creation is skipped)
-    const s3Slug =
-        storage || existingSecrets.storage
-            ? module.exports.slug(repo, process.env.AWS_PROFILE, branch, "s3")
-            : undefined;
-
-    // Always compute osSlug if opensearch is requested or pre-exists
-    const osSlug =
-        opensearch || existingSecrets.opensearch
-            ? module.exports.slug(repo, process.env.AWS_PROFILE, branch, "os")
-            : undefined;
+// Private provisioner functions for parallel execution
 
+const provisionStorage = async (
+    aws,
+    s3Slug,
+    tags,
+    isLaravel,
+    existingResources,
+) => {
     let s3Data = {};
 
-    if (storage && !existingResources.storage) {
+    if (!existingResources.storage) {
         if (isLaravel) {
-            await module.exports.s3.createS3Bucket(s3Slug, tags, {
+            await aws.s3.createS3Bucket(s3Slug, tags, {
                 BlockPublicAcls: true,
                 IgnorePublicAcls: true,
                 BlockPublicPolicy: false,
                 RestrictPublicBuckets: false,
             });
 
-            await module.exports.s3.setPublicReadPolicy(s3Slug, "public/*");
+            await aws.s3.setPublicReadPolicy(s3Slug, "public/*");
         } else {
-            await module.exports.iam.createIAMUser(s3Slug, tags);
-            await module.exports.iam.attachIAMPolicy(
+            await aws.iam.createIAMUser(s3Slug, tags);
+            await aws.iam.attachIAMPolicy(
                 s3Slug,
                 "arn:aws:iam::aws:policy/AmazonS3FullAccess",
             );
             const {
                 AccessKey: { AccessKeyId },
                 AccessKey: { SecretAccessKey },
-            } = await module.exports.iam.createAccessKey(s3Slug);
+            } = await aws.iam.createAccessKey(s3Slug);
 
-            await module.exports.s3.createS3Bucket(
+            await aws.s3.createS3Bucket(
                 s3Slug,
                 tags,
                 false,
@@ -377,58 +385,46 @@ module.exports.fullstack = async (
         }
     }
 
-    if (storage && !s3Data.s3Slug) {
+    if (!s3Data.s3Slug) {
         s3Data = { s3Slug };
     }
 
-    let instanceProfileName;
-    let opensearchDomainArn;
-
-    if (osSlug) {
-        opensearchDomainArn = `arn:aws:es:${process.env.AWS_REGION}:*:domain/${osSlug}`;
-    }
-
-    if (isLaravel) {
-        instanceProfileName =
-            await module.exports.iam.ensureEnvironmentInstanceProfile(
-                name,
-                s3Slug,
-                opensearchDomainArn,
-            );
-    } else {
-        await module.exports.iam.ensureEBInstanceProfileExists();
-    }
-
-    await module.exports.iam.ensureEBManagedUpdateProfileExists();
-    const vpcData = await module.exports.ec2.ensurePrivateSubnets();
-
-    // Create decoupled RDS instance if database is required
-    let rdsData = {};
+    return s3Data;
+};
 
-    if (database && !existingResources.database) {
-        const rdsSlug = module.exports.slug(
-            repo,
-            process.env.AWS_PROFILE,
-            branch,
-            "rds",
-        );
+const provisionRDS = async (
+    aws,
+    name,
+    repo,
+    branch,
+    tags,
+    vpcData,
+    existingResources,
+    existingSecrets,
+    snapshot,
+) => {
+    const rdsSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "rds");
+    const rdsClient = new RDSClient({});
 
+    if (!existingResources.database) {
         const dbPassword = generator.generate({
             length: 10,
             numbers: true,
         });
 
-        await module.exports.rds.ensureSSLParameterGroup();
+        await aws.rds.ensureSSLParameterGroup();
 
-        const rdsSecurityGroupId =
-            await module.exports.ec2.createRDSSecurityGroup(rdsSlug, tags);
+        const rdsSecurityGroupId = await aws.ec2.createRDSSecurityGroup(
+            rdsSlug,
+            tags,
+        );
 
-        const dbSubnetGroupName = await module.exports.rds.ensureDBSubnetGroup(
+        const dbSubnetGroupName = await aws.rds.ensureDBSubnetGroup(
             vpcData.privateSubnetIds,
         );
 
         if (snapshot) {
-            await module.exports.rds.restoreRDSInstanceFromSnapshot(
+            await aws.rds.restoreRDSInstanceFromSnapshot(
                 rdsSlug,
                 snapshot,
                 rdsSecurityGroupId,
@@ -436,11 +432,10 @@ module.exports.fullstack = async (
                 dbSubnetGroupName,
             );
 
-            await module.exports.rds.waitForRDSInstance(rdsSlug);
-
-            await module.exports.rds.modifyRDSPassword(rdsSlug, dbPassword);
+            await aws.rds.waitForRDSInstance(rdsSlug);
+            await aws.rds.modifyRDSPassword(rdsSlug, dbPassword);
         } else {
-            await module.exports.rds.createRDSInstance(
+            await aws.rds.createRDSInstance(
                 rdsSlug,
                 tags,
                 dbPassword,
@@ -449,194 +444,577 @@ module.exports.fullstack = async (
             );
         }
 
-        const rdsInstance =
-            await module.exports.rds.waitForRDSInstance(rdsSlug);
-
-        rdsData = {
-            RDS_HOSTNAME: rdsInstance.Endpoint.Address,
-            RDS_PORT: String(rdsInstance.Endpoint.Port),
-            RDS_PASSWORD: dbPassword,
+        return {
+            rdsData: { RDS_PASSWORD: dbPassword },
+            rdsSlug,
+            poll: {
+                name: "RDS",
+                cb: async () =>
+                    (
+                        await rdsClient.send(
+                            new DescribeDBInstancesCommand({
+                                DBInstanceIdentifier: rdsSlug,
+                            }),
+                        )
+                    ).DBInstances[0],
+                shouldContinue: ({ DBInstanceStatus }) =>
+                    DBInstanceStatus !== "available",
+            },
         };
-    } else if (
-        database &&
-        existingResources.database &&
-        !existingSecrets.database
-    ) {
-        // Recovery: RDS exists but secret was lost — reset password and populate data
-        const rdsSlug = module.exports.slug(
-            repo,
-            process.env.AWS_PROFILE,
-            branch,
-            "rds",
-        );
-
+    } else if (existingResources.database && !existingSecrets.database) {
         const dbPassword = generator.generate({
             length: 10,
             numbers: true,
         });
 
-        await module.exports.rds.modifyRDSPassword(rdsSlug, dbPassword);
-        const rdsInstance =
-            await module.exports.rds.waitForRDSInstance(rdsSlug);
-
-        rdsData = {
-            RDS_HOSTNAME: rdsInstance.Endpoint.Address,
-            RDS_PORT: String(rdsInstance.Endpoint.Port),
-            RDS_PASSWORD: dbPassword,
+        await aws.rds.modifyRDSPassword(rdsSlug, dbPassword);
+
+        return {
+            rdsData: { RDS_PASSWORD: dbPassword },
+            rdsSlug,
+            poll: {
+                name: "RDS",
+                cb: async () =>
+                    (
+                        await rdsClient.send(
+                            new DescribeDBInstancesCommand({
+                                DBInstanceIdentifier: rdsSlug,
+                            }),
+                        )
+                    ).DBInstances[0],
+                shouldContinue: ({ DBInstanceStatus }) =>
+                    DBInstanceStatus !== "available",
+            },
         };
-    } else if (
-        database &&
-        existingResources.database &&
-        existingSecrets.database
-    ) {
-        // Existing RDS + secret — read secret values for template interpolation
-        const secretValue = await module.exports.secretsmanager.getSecretValue(
+    } else {
+        const secretValue = await aws.secretsmanager.getSecretValue(
             `${name}-database`,
         );
-        rdsData = {
-            RDS_HOSTNAME: secretValue.RDS_HOSTNAME,
-            RDS_PORT: secretValue.RDS_PORT,
-            RDS_PASSWORD: secretValue.RDS_PASSWORD,
+        return {
+            rdsData: {
+                RDS_HOSTNAME: secretValue.RDS_HOSTNAME,
+                RDS_PORT: secretValue.RDS_PORT,
+                RDS_PASSWORD: secretValue.RDS_PASSWORD,
+            },
+            rdsSlug,
         };
     }
+};
 
-    // Create decoupled OpenSearch domain if requested
-    let osData = {};
+const provisionOpenSearch = async (
+    aws,
+    osSlug,
+    tags,
+    vpcData,
+    instanceProfileName,
+    existingResources,
+) => {
+    const osClient = new OpenSearchClient({});
 
-    if (opensearch && !existingResources.opensearch) {
-        await module.exports.iam.ensureOpenSearchServiceRole();
+    if (!existingResources.opensearch) {
+        await aws.iam.ensureOpenSearchServiceRole();
 
         const roleArn = instanceProfileName
-            ? (await module.exports.iam.getRole(instanceProfileName)).Role.Arn
+            ? (await aws.iam.getRole(instanceProfileName)).Role.Arn
             : undefined;
 
-        const osSecurityGroupId =
-            await module.exports.ec2.createOpenSearchSecurityGroup(
-                osSlug,
-                tags,
-            );
+        const osSecurityGroupId = await aws.ec2.createOpenSearchSecurityGroup(
+            osSlug,
+            tags,
+        );
 
-        await module.exports.opensearch.createOpenSearchDomain(
+        await aws.opensearch.createOpenSearchDomain(
             osSlug,
             osSecurityGroupId,
             vpcData.privateSubnetIds,
             roleArn,
+            tags,
         );
 
-        const osDomain =
-            await module.exports.opensearch.waitForOpenSearchDomain(osSlug);
+        return {
+            osData: {},
+            poll: {
+                name: "OpenSearch",
+                cb: async () => {
+                    const { DomainStatus } = await osClient.send(
+                        new DescribeDomainCommand({ DomainName: osSlug }),
+                    );
+                    return DomainStatus;
+                },
+                shouldContinue: (status) =>
+                    status.Processing ||
+                    (!status.Endpoint && !status.Endpoints),
+            },
+        };
+    } else {
+        const osDomain = await aws.opensearch.describeOpenSearchDomain(osSlug);
 
         const osEndpoint =
             osDomain.Endpoint || (osDomain.Endpoints && osDomain.Endpoints.vpc);
 
-        osData = {
-            SEARCH_HOST: `https://${osEndpoint}`,
+        return { osData: { SEARCH_HOST: `https://${osEndpoint}` } };
+    }
+};
+
+const provisionCache = async (
+    aws,
+    name,
+    cacheSlug,
+    tags,
+    vpcData,
+    existingResources,
+    existingSecrets,
+) => {
+    const cacheClient = new ElastiCacheClient({});
+
+    if (!existingResources.cache) {
+        const cachePassword = generator.generate({
+            length: 16,
+            numbers: true,
+            symbols: false,
+            strict: true,
+        });
+
+        const cacheSubnetGroupName =
+            await aws.elasticache.ensureCacheSubnetGroup(
+                vpcData.privateSubnetIds,
+            );
+
+        const cacheSecurityGroupId = await aws.ec2.createCacheSecurityGroup(
+            cacheSlug,
+            tags,
+        );
+
+        await aws.elasticache.createCacheCluster(
+            cacheSlug,
+            cacheSecurityGroupId,
+            cacheSubnetGroupName,
+            cachePassword,
+            tags,
+        );
+
+        return {
+            cacheData: { REDIS_PASSWORD: cachePassword },
+            poll: {
+                name: "ElastiCache",
+                cb: async () => {
+                    const { ReplicationGroups } = await cacheClient.send(
+                        new DescribeReplicationGroupsCommand({
+                            ReplicationGroupId: cacheSlug,
+                        }),
+                    );
+                    return ReplicationGroups[0];
+                },
+                shouldContinue: (group) => group.Status !== "available",
+            },
         };
-    } else if (opensearch && existingResources.opensearch) {
-        // Existing domain — describe to populate osData for secret payloads
-        const osDomain =
-            await module.exports.opensearch.describeOpenSearchDomain(osSlug);
+    } else if (existingResources.cache && !existingSecrets.cache) {
+        const cluster = await aws.elasticache.describeCacheCluster(cacheSlug);
 
-        const osEndpoint =
-            osDomain.Endpoint || (osDomain.Endpoints && osDomain.Endpoints.vpc);
+        const cacheEndpoint = cluster.NodeGroups[0].PrimaryEndpoint;
 
-        osData = {
-            SEARCH_HOST: `https://${osEndpoint}`,
+        return {
+            cacheData: {
+                REDIS_HOST: cacheEndpoint.Address,
+                REDIS_PORT: String(cacheEndpoint.Port),
+                REDIS_PASSWORD: "",
+            },
+        };
+    } else {
+        const secretValue = await aws.secretsmanager.getSecretValue(
+            `${name}-cache`,
+        );
+        return {
+            cacheData: {
+                REDIS_HOST: secretValue.REDIS_HOST,
+                REDIS_PORT: secretValue.REDIS_PORT,
+                REDIS_PASSWORD: secretValue.REDIS_PASSWORD,
+            },
         };
     }
+};
+
+const provisionMail = async (aws, name, tags, mailData) => {
+    const {
+        IAMClient,
+        CreateUserCommand,
+        GetUserCommand,
+        CreateAccessKeyCommand,
+        ListAccessKeysCommand,
+    } = require("@aws-sdk/client-iam");
+    const { fromIni } = require("@aws-sdk/credential-providers");
+
+    const sesProfile = "fishawack";
+    const sesRegion = "us-east-1";
+    const groupName = "AWSSESSendingGroupDoNotRename";
+
+    const iamClient = new IAMClient({
+        region: sesRegion,
+        credentials: fromIni({ profile: sesProfile }),
+    });
+
+    const userName = `${name}-ses`;
 
+    // Create user in the fishawack account
     try {
-        await module.exports.elasticbeanstalk.createElasticBeanstalkApplication(
-            repo,
+        await Spinner.prototype.simple(
+            `Creating SES IAM user ${userName}`,
+            () => {
+                return iamClient.send(
+                    new CreateUserCommand({
+                        UserName: userName,
+                        Tags: [
+                            { Key: "client", Value: process.env.AWS_PROFILE },
+                        ].concat(
+                            tags.map((t) => ({
+                                Key: t.Key,
+                                Value: String(t.Value),
+                            })),
+                        ),
+                    }),
+                );
+            },
         );
     } catch {
-        /* empty */
+        await Spinner.prototype.simple(
+            `Retrieving existing SES IAM user ${userName}`,
+            () => {
+                return iamClient.send(
+                    new GetUserCommand({ UserName: userName }),
+                );
+            },
+        );
     }
 
-    let hasKeyPair = false;
+    // Add user to SES sending group
+    await aws.iam.addUserToGroup(userName, groupName, iamClient);
 
-    try {
-        await module.exports.ec2.getKeyPair("id_rsa_prev");
-        hasKeyPair = true;
-    } catch {
-        /* empty */
-    }
+    // Create access key
+    const existingKeys = await iamClient.send(
+        new ListAccessKeysCommand({ UserName: userName }),
+    );
 
-    // LetsEncrypt CN records are limited to 64 chars so use only the hash from the generated name
-    const CNAMEPrefix = `fw-auto-${name.split("-")[name.split("-").length - 1]}`;
-    process.env.DOMAIN_LINK = `${CNAMEPrefix}.${process.env.AWS_REGION}.elasticbeanstalk.com`;
+    if (existingKeys.AccessKeyMetadata.length) {
+        // Keys already exist — we can't retrieve the secret again so return what we have in secrets
+        return {
+            MAIL_USERNAME: existingKeys.AccessKeyMetadata[0].AccessKeyId,
+            MAIL_PASSWORD: "",
+            MAIL_FROM_ADDRESS: mailData.fromAddress,
+            MAIL_FROM_NAME: mailData.fromName,
+        };
+    }
 
-    // Build secret payloads and create/reuse secrets
-    const payloads = eb.buildSecretPayloads(
-        {
-            framework,
-            availability,
-            platform,
-            language,
-            database,
-            storage,
-            opensearch,
-        },
-        {
-            ...s3Data,
-            ...rdsData,
-            ...osData,
-            APP_NAME: appName,
-            APP_DOMAIN: appDomain,
+    const keyRes = await Spinner.prototype.simple(
+        `Creating access key for SES user`,
+        () => {
+            return iamClient.send(
+                new CreateAccessKeyCommand({ UserName: userName }),
+            );
         },
     );
 
-    const secretArns = {};
-
-    for (const group of secretGroups) {
-        if (existingSecrets[group]) {
-            secretArns[`${group}SecretArn`] = existingSecrets[group];
+    // Derive SMTP credentials
+    const smtpPassword = aws.ses.deriveSmtpPassword(
+        keyRes.AccessKey.SecretAccessKey,
+        sesRegion,
+    );
 
-            // Merge new keys into existing secret if payload has additions
-            if (Object.keys(payloads[group]).length) {
-                const currentValue =
-                    await module.exports.secretsmanager.getSecretValue(
-                        `${name}-${group}`,
-                    );
-                const newKeys = Object.keys(payloads[group]).filter(
-                    (k) => !(k in currentValue),
-                );
+    return {
+        MAIL_USERNAME: keyRes.AccessKey.AccessKeyId,
+        MAIL_PASSWORD: smtpPassword,
+        MAIL_FROM_ADDRESS: mailData.fromAddress,
+        MAIL_FROM_NAME: mailData.fromName,
+    };
+};
 
-                if (newKeys.length) {
-                    const merged = { ...currentValue };
-                    for (const k of newKeys) {
-                        merged[k] = payloads[group][k];
-                    }
-                    await module.exports.secretsmanager.updateSecretValue(
-                        `${name}-${group}`,
-                        merged,
-                    );
-                }
-            }
-        } else if (Object.keys(payloads[group]).length) {
-            const res = await module.exports.secretsmanager.createSecret(
-                `${name}-${group}`,
-                payloads[group],
-                tags,
-            );
-            secretArns[`${group}SecretArn`] = res.ARN;
+const authorizeIngress = async (aws, name, slug, service, authorizer) => {
+    try {
+        const sg = await aws.ec2.findSecurityGroupByName(`${slug}-${service}`);
+        const ebSg = await aws.ec2.findSecurityGroupByTag([
+            {
+                Name: "tag:elasticbeanstalk:environment-name",
+                Values: [name],
+            },
+            {
+                Name: "tag:aws:cloudformation:logical-id",
+                Values: ["AWSEBSecurityGroup"],
+            },
+        ]);
+        if (sg && ebSg) {
+            await authorizer(sg.GroupId, ebSg.GroupId);
         }
+    } catch {
+        /* empty */
     }
+};
 
-    // Auto-generate environmentsecrets entries from secret keys + ARNs
-    const secretSettings = [];
+module.exports.fullstack = async (
+    name,
+    tags = [],
+    repo,
+    branch,
+    framework,
+    database = false,
+    storage = false,
+    opensearch = false,
+    cache = false,
+    mail = false,
+    existingSecrets = {
+        app: null,
+        storage: null,
+        database: null,
+        opensearch: null,
+        cache: null,
+        mail: null,
+    },
+    snapshot = null,
+    elbScheme = "public",
+    existingEnvironment = false,
+    existingResources = {
+        storage: false,
+        database: false,
+        opensearch: false,
+        cache: false,
+    },
+    appName,
+    appDomain,
+    certArn,
+    instanceType = "t3.micro",
+    mailData = {},
+) => {
+    const { platform, language } = eb.configurations[framework];
+    const isLaravel = framework === "laravel";
+    const aws = module.exports;
 
-    for (const group of secretGroups) {
-        const arn = secretArns[`${group}SecretArn`];
-        if (!arn) continue;
+    // Compute slugs
+    const s3Slug =
+        storage || existingSecrets.storage
+            ? aws.slug(repo, process.env.AWS_PROFILE, branch, "s3")
+            : undefined;
 
-        let keys;
+    const osSlug =
+        opensearch || existingSecrets.opensearch
+            ? aws.slug(repo, process.env.AWS_PROFILE, branch, "os")
+            : undefined;
 
-        if (existingSecrets[group]) {
-            const secretValue =
-                await module.exports.secretsmanager.getSecretValue(
+    const cacheSlug =
+        cache || existingSecrets.cache
+            ? aws.slug(repo, process.env.AWS_PROFILE, branch, "cache")
+            : undefined;
+
+    const opensearchDomainArn = osSlug
+        ? `arn:aws:es:${process.env.AWS_REGION}:*:domain/${osSlug}`
+        : undefined;
+
+    // Phase 1: IAM prerequisites (needed before OpenSearch and EB env)
+    let instanceProfileName;
+
+    if (isLaravel) {
+        instanceProfileName = await aws.iam.ensureEnvironmentInstanceProfile(
+            name,
+            s3Slug,
+            opensearchDomainArn,
+        );
+    } else {
+        await aws.iam.ensureEBInstanceProfileExists();
+    }
+
+    // Phase 2: VPC + EB app in parallel
+    const [vpcData] = await Promise.all([
+        aws.ec2.ensurePrivateSubnets(),
+        aws.iam.ensureEBManagedUpdateProfileExists(),
+    ]);
+
+    try {
+        await aws.elasticbeanstalk.createElasticBeanstalkApplication(
+            repo,
+            tags,
+        );
+    } catch {
+        /* empty */
+    }
+
+    // Phase 3: Provision services in parallel
+    const servicePromises = [];
+
+    if (storage) {
+        servicePromises.push(
+            provisionStorage(aws, s3Slug, tags, isLaravel, existingResources),
+        );
+    } else {
+        servicePromises.push(Promise.resolve(s3Slug ? { s3Slug } : {}));
+    }
+
+    if (database) {
+        servicePromises.push(
+            provisionRDS(
+                aws,
+                name,
+                repo,
+                branch,
+                tags,
+                vpcData,
+                existingResources,
+                existingSecrets,
+                snapshot,
+            ),
+        );
+    } else {
+        servicePromises.push(Promise.resolve({ rdsData: {}, rdsSlug: null }));
+    }
+
+    if (opensearch) {
+        servicePromises.push(
+            provisionOpenSearch(
+                aws,
+                osSlug,
+                tags,
+                vpcData,
+                instanceProfileName,
+                existingResources,
+            ),
+        );
+    } else {
+        servicePromises.push(Promise.resolve({ osData: {} }));
+    }
+
+    if (cache) {
+        servicePromises.push(
+            provisionCache(
+                aws,
+                name,
+                cacheSlug,
+                tags,
+                vpcData,
+                existingResources,
+                existingSecrets,
+            ),
+        );
+    } else {
+        servicePromises.push(Promise.resolve({ cacheData: {} }));
+    }
+
+    const [s3Data, rdsResult, osResult, cacheResult] =
+        await Promise.all(servicePromises);
+
+    // Phase 3b: Wait for all pending services with single spinner
+    const pendingPolls = [rdsResult, osResult, cacheResult]
+        .filter((r) => r && r.poll)
+        .map((r) => r.poll);
+
+    const pollResults = pendingPolls.length ? await pollAll(pendingPolls) : [];
+
+    // Extract endpoint data from poll results
+    let rdsData = rdsResult.rdsData || {};
+
+    if (rdsResult.poll) {
+        const rdsInstance = pollResults[pendingPolls.indexOf(rdsResult.poll)];
+        rdsData.RDS_HOSTNAME = rdsInstance.Endpoint.Address;
+        rdsData.RDS_PORT = String(rdsInstance.Endpoint.Port);
+    }
+
+    let osData = osResult.osData || {};
+
+    if (osResult.poll) {
+        const osDomain = pollResults[pendingPolls.indexOf(osResult.poll)];
+        const osEndpoint =
+            osDomain.Endpoint || (osDomain.Endpoints && osDomain.Endpoints.vpc);
+        osData.SEARCH_HOST = `https://${osEndpoint}`;
+    }
+
+    let cacheData = cacheResult.cacheData || {};
+
+    if (cacheResult.poll) {
+        const cluster = pollResults[pendingPolls.indexOf(cacheResult.poll)];
+        const cacheEndpoint = cluster.NodeGroups[0].PrimaryEndpoint;
+        cacheData.REDIS_HOST = cacheEndpoint.Address;
+        cacheData.REDIS_PORT = String(cacheEndpoint.Port);
+    }
+
+    // Phase 4: Build secrets and EB environment (sequential)
+    const CNAMEPrefix = `fw-auto-${name.split("-")[name.split("-").length - 1]}`;
+    process.env.DOMAIN_LINK = `${CNAMEPrefix}.${process.env.AWS_REGION}.elasticbeanstalk.com`;
+
+    // Provision mail credentials if needed
+    let mailCreds = {};
+    if (mail && !existingSecrets.mail) {
+        mailCreds = await provisionMail(aws, name, tags, mailData);
+    }
+
+    const payloads = eb.buildSecretPayloads(
+        {
+            framework,
+            platform,
+            language,
+            database,
+            storage,
+            opensearch,
+            cache,
+            mail: mail && !existingSecrets.mail,
+        },
+        {
+            ...s3Data,
+            ...rdsData,
+            ...osData,
+            ...cacheData,
+            ...mailCreds,
+            APP_NAME: appName,
+            APP_DOMAIN: appDomain,
+            SESSION_DOMAIN: appDomain
+                ? appDomain.split(".").length > 2
+                    ? appDomain.split(".").slice(1).join(".")
+                    : appDomain
+                : undefined,
+        },
+    );
+
+    const secretArns = {};
+
+    for (const group of secretGroups) {
+        if (existingSecrets[group]) {
+            secretArns[`${group}SecretArn`] = existingSecrets[group];
+
+            if (Object.keys(payloads[group]).length) {
+                const currentValue = await aws.secretsmanager.getSecretValue(
                     `${name}-${group}`,
                 );
+                const newKeys = Object.keys(payloads[group]).filter(
+                    (k) => !(k in currentValue),
+                );
+
+                if (newKeys.length) {
+                    const merged = { ...currentValue };
+                    for (const k of newKeys) {
+                        merged[k] = payloads[group][k];
+                    }
+                    await aws.secretsmanager.updateSecretValue(
+                        `${name}-${group}`,
+                        merged,
+                    );
+                }
+            }
+        } else if (Object.keys(payloads[group]).length) {
+            const res = await aws.secretsmanager.createSecret(
+                `${name}-${group}`,
+                payloads[group],
+                tags,
+            );
+            secretArns[`${group}SecretArn`] = res.ARN;
+        }
+    }
+
+    const secretSettings = [];
+
+    for (const group of secretGroups) {
+        const arn = secretArns[`${group}SecretArn`];
+        if (!arn) continue;
+
+        let keys;
+
+        if (existingSecrets[group]) {
+            const secretValue = await aws.secretsmanager.getSecretValue(
+                `${name}-${group}`,
+            );
             keys = Object.keys(secretValue);
         } else {
             keys = Object.keys(payloads[group]);
@@ -652,14 +1030,14 @@ module.exports.fullstack = async (
         }
     }
 
+    // Phase 5: Create or update EB environment
     let environment;
 
     if (existingEnvironment) {
-        // Re-provision: sync secrets to existing EB environment
         const SECRETS_NAMESPACE =
             "aws:elasticbeanstalk:application:environmentsecrets";
         const currentSettings =
-            await module.exports.elasticbeanstalk.describeConfigurationSettings(
+            await aws.elasticbeanstalk.describeConfigurationSettings(
                 repo,
                 name,
             );
@@ -689,7 +1067,7 @@ module.exports.fullstack = async (
         const optionSettings = [...toAdd, ...toUpdate];
 
         if (optionSettings.length || toRemove.length) {
-            await module.exports.elasticbeanstalk.updateEnvironmentSettings(
+            await aws.elasticbeanstalk.updateEnvironmentSettings(
                 name,
                 optionSettings,
                 toRemove,
@@ -702,7 +1080,6 @@ module.exports.fullstack = async (
             "environments",
             {
                 framework,
-                availability,
                 platform,
                 language,
                 database,
@@ -713,17 +1090,23 @@ module.exports.fullstack = async (
                 publicSubnetIds: vpcData.publicSubnetIds.join(","),
                 elbScheme,
                 instanceProfileName,
+                instanceType,
             },
             {
                 ...s3Data,
                 ...rdsData,
                 ...osData,
+                ...cacheData,
                 APP_NAME: appName,
                 APP_DOMAIN: appDomain,
+                SESSION_DOMAIN: appDomain
+                    ? appDomain.split(".").length > 2
+                        ? appDomain.split(".").slice(1).join(".")
+                        : appDomain
+                    : undefined,
             },
         );
 
-        // HTTPS listener via ACM cert
         if (certArn) {
             OptionSettings.push(
                 {
@@ -756,18 +1139,10 @@ module.exports.fullstack = async (
 
         OptionSettings.push(...secretSettings);
 
-        if (hasKeyPair) {
-            OptionSettings.push({
-                OptionName: "EC2KeyName",
-                Value: "id_rsa_prev",
-                Namespace: "aws:autoscaling:launchconfiguration",
-            });
-        }
-
         environment =
-            await module.exports.elasticbeanstalk.createElasticBeanstalkEnvironment(
+            await aws.elasticbeanstalk.createElasticBeanstalkEnvironment(
                 name,
-                { framework, availability, platform, language },
+                { framework, platform, language },
                 repo,
                 OptionSettings,
                 CNAMEPrefix,
@@ -775,309 +1150,677 @@ module.exports.fullstack = async (
             );
     }
 
-    // Add RDS security group ingress rule from EB security group
-    const rdsSlugForIngress = module.exports.slug(
+    // Phase 6: SG ingress rules in parallel
+    const rdsSlugForIngress = aws.slug(
         repo,
         process.env.AWS_PROFILE,
         branch,
         "rds",
     );
 
-    try {
-        const rdsSg = await module.exports.ec2.findSecurityGroupByName(
-            `${rdsSlugForIngress}-rds`,
+    const ingressPromises = [
+        authorizeIngress(
+            aws,
+            name,
+            rdsSlugForIngress,
+            "rds",
+            aws.ec2.authorizeRDSIngress,
+        ),
+    ];
+
+    if (osSlug) {
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                osSlug,
+                "os",
+                aws.ec2.authorizeOpenSearchIngress,
+            ),
         );
-        const ebSg = await module.exports.ec2.findSecurityGroupByTag([
-            {
-                Name: "tag:elasticbeanstalk:environment-name",
-                Values: [name],
-            },
-            {
-                Name: "tag:aws:cloudformation:logical-id",
-                Values: ["AWSEBSecurityGroup"],
-            },
-        ]);
-        if (rdsSg && ebSg) {
-            await module.exports.ec2.authorizeRDSIngress(
-                rdsSg.GroupId,
-                ebSg.GroupId,
-            );
-        }
-    } catch {
-        /* empty */
     }
 
-    // Add OpenSearch security group ingress rule from EB security group
-    if (osSlug) {
-        try {
-            const osSg = await module.exports.ec2.findSecurityGroupByName(
-                `${osSlug}-os`,
-            );
-            const ebSgForOs = await module.exports.ec2.findSecurityGroupByTag([
-                {
-                    Name: "tag:elasticbeanstalk:environment-name",
-                    Values: [name],
-                },
-                {
-                    Name: "tag:aws:cloudformation:logical-id",
-                    Values: ["AWSEBSecurityGroup"],
-                },
-            ]);
-            if (osSg && ebSgForOs) {
-                await module.exports.ec2.authorizeOpenSearchIngress(
-                    osSg.GroupId,
-                    ebSgForOs.GroupId,
-                );
-            }
-        } catch {
-            /* empty */
-        }
+    if (cacheSlug) {
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                cacheSlug,
+                "cache",
+                aws.ec2.authorizeCacheIngress,
+            ),
+        );
     }
 
+    await Promise.all(ingressPromises);
+
+    // Phase 7: Copy configs and return
     const configurations = eb.merge("configurations", {
         framework,
-        availability,
         platform,
         language,
     });
 
-    if (credentials.length) {
-        configurations.push(
-            `.platform/${platform}/conf.d/elasticbeanstalk/${platform === "httpd" ? `z-${framework}-` : ""}htpasswd.conf`,
-        );
-        fs.outputFileSync(
-            `.platform/${platform}/conf.d/elasticbeanstalk/.htpasswd-${branch}`,
-            credentials
-                .map((d) => `${d.username}:${md5(d.password)}`)
-                .join("\n"),
-        );
-    }
-
-    await module.exports.elasticbeanstalk.copyElasticBeanstalkConfigs(
+    await aws.elasticbeanstalk.copyElasticBeanstalkConfigs(
         configurations,
+        branch,
     );
 
-    if (credentials.length) {
-        configurations.push(
-            `.platform/${platform}/conf.d/elasticbeanstalk/.htpasswd-${branch}`,
-        );
-    }
-
     let config = {
         url: `https://${environment.CNAME}`,
         environment: environment.EnvironmentName,
-        paths: configurations.map((d) => {
-            const obj = { src: d, dest: d };
-            // Fix for core mistakenly mapping .htpasswd to directory
-            if (d.includes(`.htpasswd-${branch}`)) {
-                obj.file = true;
-                obj.dest = d.split(`-${branch}`)[0];
-            }
-            return obj;
-        }),
+        paths: [`_IaC/${branch}/*`],
+        copy: [
+            {
+                src: `_IaC/${branch}/.elasticbeanstalk/config.yml`,
+                dest: `.elasticbeanstalk/config.yml`,
+            },
+        ],
     };
 
     return config;
 };
 
-module.exports.fullstackTerminate = async (
+module.exports.fullstackClone = async (
     name,
+    tags,
     repo,
     branch,
-    {
-        deleteStorage = false,
-        deleteDatabase = false,
-        deleteOpenSearch = false,
-        deleteAppSecret = false,
-    } = {},
+    applicationName,
+    sourceEnvironmentName,
+    sourceVersionLabel,
+    sourceBranch,
 ) => {
-    if (deleteStorage) {
-        const s3Slug = module.exports.slug(
-            repo,
-            process.env.AWS_PROFILE,
-            branch,
-            "s3",
+    const aws = module.exports;
+
+    // Phase 1: Read source configuration
+    const sourceSettings =
+        await aws.elasticbeanstalk.describeConfigurationSettings(
+            applicationName,
+            sourceEnvironmentName,
         );
 
-        try {
-            await module.exports.iam.removeIAMUser(s3Slug);
-        } catch {
-            /* empty */
-        }
+    // Phase 2: Filter settings — keep relevant namespaces, exclude immutable/auto-managed
+    const excludedOptions = new Set([
+        "aws:elasticbeanstalk:environment::EnvironmentName",
+    ]);
+
+    const includedNamespaces = new Set([
+        "aws:elasticbeanstalk:application:environment",
+        "aws:elasticbeanstalk:application:environmentsecrets",
+        "aws:autoscaling:launchconfiguration",
+        "aws:autoscaling:asg",
+        "aws:ec2:vpc",
+        "aws:elbv2:listener:443",
+        "aws:elbv2:listener:default",
+        "aws:elasticbeanstalk:environment",
+        "aws:elasticbeanstalk:environment:process:default",
+        "aws:elasticbeanstalk:healthreporting:system",
+        "aws:elasticbeanstalk:managedactions",
+        "aws:elasticbeanstalk:managedactions:platformupdate",
+        "aws:elasticbeanstalk:cloudwatch:logs",
+        "aws:elasticbeanstalk:cloudwatch:logs:health",
+    ]);
+
+    const OptionSettings = sourceSettings
+        .filter((s) => {
+            if (!includedNamespaces.has(s.Namespace)) return false;
+            const key = `${s.Namespace}::${s.OptionName}`;
+            if (excludedOptions.has(key)) return false;
+            // Exclude empty values
+            if (s.Value === undefined || s.Value === null || s.Value === "")
+                return false;
+            return true;
+        })
+        .map((s) => ({
+            Namespace: s.Namespace,
+            OptionName: s.OptionName,
+            Value: s.Value,
+        }));
+
+    // Detect which services exist from source secrets
+    const sourceSecrets = sourceSettings.filter(
+        (s) =>
+            s.Namespace ===
+            "aws:elasticbeanstalk:application:environmentsecrets",
+    );
 
-        try {
-            await module.exports.s3.emptyS3Bucket(s3Slug);
-        } catch {
-            /* empty */
-        }
+    const hasOpenSearch = sourceSecrets.some(
+        (s) => s.OptionName === "SEARCH_HOST",
+    );
 
-        try {
-            await module.exports.s3.removeS3Bucket(s3Slug);
-        } catch {
-            /* empty */
-        }
+    // Set SECRET_MIRROR_ENV so setvars.sh reads secrets from the source env
+    OptionSettings.push({
+        Namespace: "aws:elasticbeanstalk:application:environment",
+        OptionName: "SECRET_MIRROR_ENV",
+        Value: sourceEnvironmentName,
+    });
+
+    // Override HealthCheckGracePeriod for initial deploy window
+    const healthGraceIdx = OptionSettings.findIndex(
+        (s) =>
+            s.Namespace ===
+                "aws:elasticbeanstalk:environment:process:default" &&
+            s.OptionName === "HealthCheckGracePeriod",
+    );
+    if (healthGraceIdx !== -1) {
+        OptionSettings[healthGraceIdx].Value = "600";
     }
 
-    // Remove RDS inbound rule referencing EB security group before terminating EB
-    const rdsSlug = module.exports.slug(
+    // Phase 4: Create the environment
+    const CNAMEPrefix = `fw-auto-${name.split("-")[name.split("-").length - 1]}`;
+
+    const environment =
+        await aws.elasticbeanstalk.createElasticBeanstalkEnvironment(
+            name,
+            { language: detectLanguageFromSettings(sourceSettings) },
+            applicationName,
+            OptionSettings,
+            CNAMEPrefix,
+            tags,
+            sourceVersionLabel,
+        );
+
+    // Phase 5: Authorize SG ingress on source service security groups
+    const rdsSlug = aws.slug(
         repo,
         process.env.AWS_PROFILE,
-        branch,
+        sourceBranch,
         "rds",
     );
+    const cacheSlug = aws.slug(
+        repo,
+        process.env.AWS_PROFILE,
+        sourceBranch,
+        "cache",
+    );
 
-    try {
-        const rdsSg = await module.exports.ec2.findSecurityGroupByName(
-            `${rdsSlug}-rds`,
+    const ingressPromises = [
+        authorizeIngress(
+            aws,
+            name,
+            rdsSlug,
+            "rds",
+            aws.ec2.authorizeRDSIngress,
+        ),
+    ];
+
+    if (hasOpenSearch) {
+        const sourceOsSlug = aws.slug(
+            repo,
+            process.env.AWS_PROFILE,
+            sourceBranch,
+            "os",
+        );
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                sourceOsSlug,
+                "os",
+                aws.ec2.authorizeOpenSearchIngress,
+            ),
         );
-        const ebSg = await module.exports.ec2.findSecurityGroupByTag([
-            {
-                Name: "tag:elasticbeanstalk:environment-name",
-                Values: [name],
-            },
-            {
-                Name: "tag:aws:cloudformation:logical-id",
-                Values: ["AWSEBSecurityGroup"],
-            },
-        ]);
-        if (rdsSg && ebSg) {
-            await module.exports.ec2.revokeRDSIngress(
-                rdsSg.GroupId,
-                ebSg.GroupId,
-            );
-        }
-    } catch {
-        /* empty */
     }
 
-    // Remove OpenSearch inbound rule referencing EB security group before terminating EB
-    const osSlug = module.exports.slug(
-        repo,
-        process.env.AWS_PROFILE,
+    if (sourceSecrets.some((s) => s.OptionName === "REDIS_HOST")) {
+        ingressPromises.push(
+            authorizeIngress(
+                aws,
+                name,
+                cacheSlug,
+                "cache",
+                aws.ec2.authorizeCacheIngress,
+            ),
+        );
+    }
+
+    await Promise.all(ingressPromises);
+
+    // Phase 6: Copy configs
+    const framework = detectFrameworkFromSettings(sourceSettings);
+    const { platform, language } = eb.configurations[framework];
+    const configurations = eb.merge("configurations", {
+        framework,
+        platform,
+        language,
+    });
+
+    await aws.elasticbeanstalk.copyElasticBeanstalkConfigs(
+        configurations,
         branch,
-        "os",
     );
 
-    try {
-        const osSg = await module.exports.ec2.findSecurityGroupByName(
-            `${osSlug}-os`,
-        );
-        const ebSgForOs = await module.exports.ec2.findSecurityGroupByTag([
+    return {
+        url: `https://${environment.CNAME}`,
+        environment: environment.EnvironmentName,
+        paths: [`_IaC/${branch}/*`],
+        copy: [
             {
-                Name: "tag:elasticbeanstalk:environment-name",
-                Values: [name],
+                src: `_IaC/${branch}/.elasticbeanstalk/config.yml`,
+                dest: `.elasticbeanstalk/config.yml`,
             },
-            {
-                Name: "tag:aws:cloudformation:logical-id",
-                Values: ["AWSEBSecurityGroup"],
-            },
-        ]);
-        if (osSg && ebSgForOs) {
-            await module.exports.ec2.revokeOpenSearchIngress(
-                osSg.GroupId,
-                ebSgForOs.GroupId,
-            );
-        }
-    } catch {
-        /* empty */
-    }
+        ],
+    };
+};
 
-    try {
-        await module.exports.elasticbeanstalk.removeElasticBeanstalkEnvironment(
+const detectLanguageFromSettings = (settings) => {
+    const envVars = settings.filter(
+        (s) =>
+            s.Namespace === "aws:elasticbeanstalk:application:environment" ||
+            s.Namespace ===
+                "aws:elasticbeanstalk:application:environmentsecrets",
+    );
+    const keys = envVars.map((s) => s.OptionName);
+    if (keys.includes("DJANGO_SETTINGS_MODULE")) return "python";
+    if (keys.includes("NODE_ENV")) return "node";
+    return "php";
+};
+
+const detectFrameworkFromSettings = (settings) => {
+    const envVars = settings.filter(
+        (s) =>
+            s.Namespace === "aws:elasticbeanstalk:application:environment" ||
+            s.Namespace ===
+                "aws:elasticbeanstalk:application:environmentsecrets",
+    );
+    const keys = envVars.map((s) => s.OptionName);
+    if (keys.includes("DJANGO_SETTINGS_MODULE")) return "python";
+    if (keys.includes("ADONIS_ACE_CWD")) return "adonis";
+    if (keys.includes("APP_KEY")) return "laravel";
+    return "laravel";
+};
+
+module.exports.fullstackCloneTerminate = async (
+    name,
+    repo,
+    branch,
+    sourceBranch,
+) => {
+    const aws = module.exports;
+
+    const rdsSlug = aws.slug(
+        repo,
+        process.env.AWS_PROFILE,
+        sourceBranch,
+        "rds",
+    );
+    const osSlug = aws.slug(repo, process.env.AWS_PROFILE, sourceBranch, "os");
+    const cacheSlug = aws.slug(
+        repo,
+        process.env.AWS_PROFILE,
+        sourceBranch,
+        "cache",
+    );
+
+    // Phase 1: Revoke SG ingress rules
+    await Promise.all([
+        authorizeIngress(aws, name, rdsSlug, "rds", aws.ec2.revokeRDSIngress),
+        authorizeIngress(
+            aws,
             name,
-        );
+            osSlug,
+            "os",
+            aws.ec2.revokeOpenSearchIngress,
+        ),
+        authorizeIngress(
+            aws,
+            name,
+            cacheSlug,
+            "cache",
+            aws.ec2.revokeCacheIngress,
+        ),
+    ]);
+
+    // Phase 2: Terminate EB environment
+    try {
+        await aws.elasticbeanstalk.removeElasticBeanstalkEnvironment(name);
     } catch {
         /* empty */
     }
 
-    try {
-        await module.exports.elasticbeanstalk.removeElasticBeanstalkApplication(
-            repo,
+    // Phase 3: Terminate EB environment + try removing app if empty
+    await Promise.all([
+        (async () => {
+            try {
+                const envs = await aws.elasticbeanstalk.listEnvironments(repo);
+                if (!envs.length) {
+                    await aws.elasticbeanstalk.removeElasticBeanstalkApplication(
+                        repo,
+                    );
+                }
+            } catch {
+                /* empty */
+            }
+        })(),
+    ]);
+};
+
+module.exports.fullstackTerminate = async (
+    name,
+    repo,
+    branch,
+    {
+        deleteStorage = false,
+        deleteDatabase = false,
+        deleteOpenSearch = false,
+        deleteCache = false,
+        deleteMail = false,
+        deleteAppSecret = false,
+    } = {},
+) => {
+    const aws = module.exports;
+
+    const rdsSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "rds");
+    const osSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "os");
+    const cacheSlug = aws.slug(repo, process.env.AWS_PROFILE, branch, "cache");
+
+    // Phase 1: Storage cleanup + revoke all SG ingress rules in parallel
+    const phase1 = [];
+
+    if (deleteStorage) {
+        const s3Slug = aws.slug(repo, process.env.AWS_PROFILE, branch, "s3");
+
+        phase1.push(
+            (async () => {
+                try {
+                    await aws.iam.removeIAMUser(s3Slug);
+                } catch {
+                    /* empty */
+                }
+                try {
+                    await aws.s3.emptyS3Bucket(s3Slug);
+                } catch {
+                    /* empty */
+                }
+                try {
+                    await aws.s3.removeS3Bucket(s3Slug);
+                } catch {
+                    /* empty */
+                }
+            })(),
         );
-    } catch {
-        /* empty */
     }
 
+    phase1.push(
+        authorizeIngress(aws, name, rdsSlug, "rds", aws.ec2.revokeRDSIngress),
+        authorizeIngress(
+            aws,
+            name,
+            osSlug,
+            "os",
+            aws.ec2.revokeOpenSearchIngress,
+        ),
+        authorizeIngress(
+            aws,
+            name,
+            cacheSlug,
+            "cache",
+            aws.ec2.revokeCacheIngress,
+        ),
+    );
+
+    await Promise.all(phase1);
+
+    // Phase 2: Terminate EB environment (must complete before SG deletion)
     try {
-        await module.exports.iam.removeEnvironmentInstanceProfile(name);
+        await aws.elasticbeanstalk.removeElasticBeanstalkEnvironment(name);
     } catch {
         /* empty */
     }
 
+    // Phase 3: EB app + instance profile + initiate service deletions in parallel
+    const phase3 = [
+        (async () => {
+            try {
+                await aws.elasticbeanstalk.removeElasticBeanstalkApplication(
+                    repo,
+                );
+            } catch {
+                /* empty */
+            }
+        })(),
+        (async () => {
+            try {
+                await aws.iam.removeEnvironmentInstanceProfile(name);
+            } catch {
+                /* empty */
+            }
+        })(),
+    ];
+
     if (deleteDatabase) {
-        try {
-            await module.exports.rds.deleteRDSInstance(rdsSlug);
-            await module.exports.rds.waitForRDSDeletion(rdsSlug);
-        } catch {
-            /* empty */
-        }
+        phase3.push(aws.rds.deleteRDSInstance(rdsSlug).catch(() => {}));
+    }
 
-        try {
-            const sg = await module.exports.ec2.findSecurityGroupByName(
-                `${rdsSlug}-rds`,
-            );
-            if (sg) {
-                await module.exports.ec2.deleteSecurityGroup(sg.GroupId);
-            }
-        } catch {
-            /* empty */
-        }
+    if (deleteOpenSearch) {
+        phase3.push(
+            aws.opensearch.deleteOpenSearchDomain(osSlug).catch(() => {}),
+        );
+    }
+
+    if (deleteCache) {
+        phase3.push(
+            aws.elasticache.deleteCacheCluster(cacheSlug).catch(() => {}),
+        );
+    }
+
+    await Promise.all(phase3);
+
+    // Phase 4: Wait for service deletions with round-robin polling
+    const deletionPolls = [];
+
+    if (deleteDatabase) {
+        const rdsClient = new RDSClient({});
+
+        deletionPolls.push({
+            name: "RDS",
+            cb: async () => {
+                try {
+                    const res = await rdsClient.send(
+                        new DescribeDBInstancesCommand({
+                            DBInstanceIdentifier: rdsSlug,
+                        }),
+                    );
+                    return res.DBInstances[0];
+                } catch (e) {
+                    if (e.name === "DBInstanceNotFoundFault") {
+                        return { DBInstanceStatus: "deleted" };
+                    }
+                    throw e;
+                }
+            },
+            shouldContinue: ({ DBInstanceStatus }) =>
+                DBInstanceStatus !== "deleted",
+        });
     }
 
     if (deleteOpenSearch) {
-        try {
-            await module.exports.opensearch.deleteOpenSearchDomain(osSlug);
-            await module.exports.opensearch.waitForOpenSearchDeletion(osSlug);
-        } catch {
-            /* empty */
-        }
+        const osClient = new OpenSearchClient({});
+
+        deletionPolls.push({
+            name: "OpenSearch",
+            cb: async () => {
+                try {
+                    const { DomainStatus } = await osClient.send(
+                        new DescribeDomainCommand({ DomainName: osSlug }),
+                    );
+                    return DomainStatus;
+                } catch (e) {
+                    if (e.name === "ResourceNotFoundException") {
+                        return { deleted: true };
+                    }
+                    throw e;
+                }
+            },
+            shouldContinue: (status) => !status.deleted,
+        });
+    }
 
-        try {
-            const sg = await module.exports.ec2.findSecurityGroupByName(
-                `${osSlug}-os`,
-            );
-            if (sg) {
-                await module.exports.ec2.deleteSecurityGroup(sg.GroupId);
-            }
-        } catch {
-            /* empty */
-        }
+    if (deleteCache) {
+        const cacheClient = new ElastiCacheClient({});
+
+        deletionPolls.push({
+            name: "ElastiCache",
+            cb: async () => {
+                try {
+                    const { ReplicationGroups } = await cacheClient.send(
+                        new DescribeReplicationGroupsCommand({
+                            ReplicationGroupId: cacheSlug,
+                        }),
+                    );
+                    return ReplicationGroups[0];
+                } catch (e) {
+                    if (e.name === "ReplicationGroupNotFoundFault") {
+                        return { deleted: true };
+                    }
+                    throw e;
+                }
+            },
+            shouldContinue: (group) => !group.deleted,
+        });
     }
 
-    // Delete secrets
-    if (deleteStorage) {
-        try {
-            await module.exports.secretsmanager.deleteSecret(`${name}-storage`);
-        } catch {
-            /* empty */
-        }
+    if (deletionPolls.length) {
+        await pollAll(deletionPolls, "Waiting for services to terminate");
     }
 
+    // Phase 5: Delete security groups after services are gone
+    const sgDeletions = [];
+
     if (deleteDatabase) {
-        try {
-            await module.exports.secretsmanager.deleteSecret(
-                `${name}-database`,
-            );
-        } catch {
-            /* empty */
-        }
+        sgDeletions.push(
+            (async () => {
+                try {
+                    const sg = await aws.ec2.findSecurityGroupByName(
+                        `${rdsSlug}-rds`,
+                    );
+                    if (sg) {
+                        await aws.ec2.deleteSecurityGroup(sg.GroupId);
+                    }
+                } catch {
+                    /* empty */
+                }
+            })(),
+        );
     }
 
     if (deleteOpenSearch) {
-        try {
-            await module.exports.secretsmanager.deleteSecret(
-                `${name}-opensearch`,
-            );
-        } catch {
-            /* empty */
-        }
+        sgDeletions.push(
+            (async () => {
+                try {
+                    const sg = await aws.ec2.findSecurityGroupByName(
+                        `${osSlug}-os`,
+                    );
+                    if (sg) {
+                        await aws.ec2.deleteSecurityGroup(sg.GroupId);
+                    }
+                } catch {
+                    /* empty */
+                }
+            })(),
+        );
+    }
+
+    if (deleteCache) {
+        sgDeletions.push(
+            (async () => {
+                try {
+                    const sg = await aws.ec2.findSecurityGroupByName(
+                        `${cacheSlug}-cache`,
+                    );
+                    if (sg) {
+                        await aws.ec2.deleteSecurityGroup(sg.GroupId);
+                    }
+                } catch {
+                    /* empty */
+                }
+            })(),
+        );
     }
 
+    if (sgDeletions.length) {
+        await Promise.all(sgDeletions);
+    }
+
+    // Phase 6: Delete secrets in parallel
+    const secretDeletions = [];
+
+    if (deleteStorage) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-storage`).catch(() => {}),
+        );
+    }
+    if (deleteDatabase) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-database`).catch(() => {}),
+        );
+    }
+    if (deleteOpenSearch) {
+        secretDeletions.push(
+            aws.secretsmanager
+                .deleteSecret(`${name}-opensearch`)
+                .catch(() => {}),
+        );
+    }
+    if (deleteCache) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-cache`).catch(() => {}),
+        );
+    }
     if (deleteAppSecret) {
-        try {
-            await module.exports.secretsmanager.deleteSecret(`${name}-app`);
-        } catch {
-            /* empty */
-        }
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-app`).catch(() => {}),
+        );
     }
+    if (deleteMail) {
+        secretDeletions.push(
+            aws.secretsmanager.deleteSecret(`${name}-mail`).catch(() => {}),
+        );
+        secretDeletions.push(
+            (async () => {
+                const {
+                    IAMClient,
+                    DeleteAccessKeyCommand,
+                    ListAccessKeysCommand,
+                    RemoveUserFromGroupCommand,
+                    DeleteUserCommand,
+                } = require("@aws-sdk/client-iam");
+                const { fromIni } = require("@aws-sdk/credential-providers");
+
+                const iamClient = new IAMClient({
+                    region: "us-east-1",
+                    credentials: fromIni({ profile: "fishawack" }),
+                });
+                const userName = `${name}-ses`;
+
+                try {
+                    const keys = await iamClient.send(
+                        new ListAccessKeysCommand({ UserName: userName }),
+                    );
+                    for (const key of keys.AccessKeyMetadata || []) {
+                        await iamClient.send(
+                            new DeleteAccessKeyCommand({
+                                UserName: userName,
+                                AccessKeyId: key.AccessKeyId,
+                            }),
+                        );
+                    }
+                    await iamClient.send(
+                        new RemoveUserFromGroupCommand({
+                            UserName: userName,
+                            GroupName: "AWSSESSendingGroupDoNotRename",
+                        }),
+                    );
+                    await iamClient.send(
+                        new DeleteUserCommand({ UserName: userName }),
+                    );
+                } catch {
+                    /* user may not exist */
+                }
+            })(),
+        );
+    }
+
+    await Promise.all(secretDeletions);
 };