@fishawack/lab-env 5.7.0-beta.1 → 5.7.0-beta.3

package/commands/create/services/aws/ec2.js

@@ -6,16 +6,29 @@ const {
     RevokeSecurityGroupIngressCommand,
     DeleteSecurityGroupCommand,
     DescribeSecurityGroupsCommand,
+    DescribeVpcsCommand,
+    DescribeSubnetsCommand,
+    CreateSubnetCommand,
+    CreateRouteTableCommand,
+    CreateRouteCommand,
+    AssociateRouteTableCommand,
+    CreateNatGatewayCommand,
+    DescribeNatGatewaysCommand,
+    AllocateAddressCommand,
+    DescribeAvailabilityZonesCommand,
+    DescribeRouteTablesCommand,
+    DescribeInternetGatewaysCommand,
 } = require("@aws-sdk/client-ec2");
-const { Spinner } = require("../../libs/utilities");
+const { Spinner, poll } = require("../../libs/utilities");
+const { awsClientConfig } = require("../../libs/vars");
 
-module.exports.getKeyPair = async (KeyName) => {
-    const client = new EC2Client({});
+const getClient = () => new EC2Client(awsClientConfig);
 
+module.exports.getKeyPair = async (KeyName) => {
     let res = await Spinner.prototype.simple(
         `Retrieving the KeyPair ${KeyName}`,
         () => {
-            return client.send(
+            return getClient().send(
                 new DescribeKeyPairsCommand({ KeyNames: [KeyName] }),
             );
         },
@@ -25,50 +38,55 @@ module.exports.getKeyPair = async (KeyName) => {
 };
 
 module.exports.createRDSSecurityGroup = async (name, tags = []) => {
-    const client = new EC2Client({});
-
     const groupName = `${name}-rds`;
 
-    const sg = await Spinner.prototype.simple(
-        `Creating security group ${groupName}`,
-        () => {
-            return client.send(
-                new CreateSecurityGroupCommand({
-                    GroupName: groupName,
-                    Description: `RDS access for ${name}`,
-                    TagSpecifications: [
-                        {
-                            ResourceType: "security-group",
-                            Tags: [
-                                {
-                                    Key: "Name",
-                                    Value: groupName,
-                                },
-                                {
-                                    Key: "client",
-                                    Value: process.env.AWS_PROFILE,
-                                },
-                            ].concat(tags),
-                        },
-                    ],
-                }),
-            );
-        },
-    );
+    try {
+        const sg = await Spinner.prototype.simple(
+            `Creating security group ${groupName}`,
+            () => {
+                return getClient().send(
+                    new CreateSecurityGroupCommand({
+                        GroupName: groupName,
+                        Description: `RDS access for ${name}`,
+                        TagSpecifications: [
+                            {
+                                ResourceType: "security-group",
+                                Tags: [
+                                    {
+                                        Key: "Name",
+                                        Value: groupName,
+                                    },
+                                    {
+                                        Key: "client",
+                                        Value: process.env.AWS_PROFILE,
+                                    },
+                                ].concat(tags),
+                            },
+                        ],
+                    }),
+                );
+            },
+        );
 
-    return sg.GroupId;
+        return sg.GroupId;
+    } catch (e) {
+        if (e.Code === "InvalidGroup.Duplicate") {
+            const existing =
+                await module.exports.findSecurityGroupByName(groupName);
+            return existing.GroupId;
+        }
+        throw e;
+    }
 };
 
 module.exports.authorizeRDSIngress = async (
     rdsSecurityGroupId,
     sourceSecurityGroupId,
 ) => {
-    const client = new EC2Client({});
-
     await Spinner.prototype.simple(
         `Adding MySQL inbound rule from EB security group`,
         () => {
-            return client.send(
+            return getClient().send(
                 new AuthorizeSecurityGroupIngressCommand({
                     GroupId: rdsSecurityGroupId,
                     IpPermissions: [
@@ -95,12 +113,10 @@ module.exports.revokeRDSIngress = async (
     rdsSecurityGroupId,
     sourceSecurityGroupId,
 ) => {
-    const client = new EC2Client({});
-
     await Spinner.prototype.simple(
         `Removing MySQL inbound rule from RDS security group`,
         () => {
-            return client.send(
+            return getClient().send(
                 new RevokeSecurityGroupIngressCommand({
                     GroupId: rdsSecurityGroupId,
                     IpPermissions: [
@@ -121,13 +137,211 @@ module.exports.revokeRDSIngress = async (
     );
 };
 
-module.exports.findSecurityGroupByTag = async (filters = []) => {
-    const client = new EC2Client({});
+module.exports.createOpenSearchSecurityGroup = async (name, tags = []) => {
+    const groupName = `${name}-os`;
+
+    try {
+        const sg = await Spinner.prototype.simple(
+            `Creating security group ${groupName}`,
+            () => {
+                return getClient().send(
+                    new CreateSecurityGroupCommand({
+                        GroupName: groupName,
+                        Description: `OpenSearch access for ${name}`,
+                        TagSpecifications: [
+                            {
+                                ResourceType: "security-group",
+                                Tags: [
+                                    {
+                                        Key: "Name",
+                                        Value: groupName,
+                                    },
+                                    {
+                                        Key: "client",
+                                        Value: process.env.AWS_PROFILE,
+                                    },
+                                ].concat(tags),
+                            },
+                        ],
+                    }),
+                );
+            },
+        );
+
+        return sg.GroupId;
+    } catch (e) {
+        if (e.Code === "InvalidGroup.Duplicate") {
+            const existing =
+                await module.exports.findSecurityGroupByName(groupName);
+            return existing.GroupId;
+        }
+        throw e;
+    }
+};
+
+module.exports.authorizeOpenSearchIngress = async (
+    osSecurityGroupId,
+    sourceSecurityGroupId,
+) => {
+    await Spinner.prototype.simple(
+        `Adding HTTPS inbound rule from EB security group`,
+        () => {
+            return getClient().send(
+                new AuthorizeSecurityGroupIngressCommand({
+                    GroupId: osSecurityGroupId,
+                    IpPermissions: [
+                        {
+                            IpProtocol: "tcp",
+                            FromPort: 443,
+                            ToPort: 443,
+                            UserIdGroupPairs: [
+                                {
+                                    GroupId: sourceSecurityGroupId,
+                                    Description:
+                                        "HTTPS access for EB instances",
+                                },
+                            ],
+                        },
+                    ],
+                }),
+            );
+        },
+    );
+};
+
+module.exports.revokeOpenSearchIngress = async (
+    osSecurityGroupId,
+    sourceSecurityGroupId,
+) => {
+    await Spinner.prototype.simple(
+        `Removing HTTPS inbound rule from OpenSearch security group`,
+        () => {
+            return getClient().send(
+                new RevokeSecurityGroupIngressCommand({
+                    GroupId: osSecurityGroupId,
+                    IpPermissions: [
+                        {
+                            IpProtocol: "tcp",
+                            FromPort: 443,
+                            ToPort: 443,
+                            UserIdGroupPairs: [
+                                {
+                                    GroupId: sourceSecurityGroupId,
+                                },
+                            ],
+                        },
+                    ],
+                }),
+            );
+        },
+    );
+};
+
+module.exports.createCacheSecurityGroup = async (name, tags = []) => {
+    const groupName = `${name}-cache`;
+
+    try {
+        const sg = await Spinner.prototype.simple(
+            `Creating security group ${groupName}`,
+            () => {
+                return getClient().send(
+                    new CreateSecurityGroupCommand({
+                        GroupName: groupName,
+                        Description: `ElastiCache access for ${name}`,
+                        TagSpecifications: [
+                            {
+                                ResourceType: "security-group",
+                                Tags: [
+                                    {
+                                        Key: "Name",
+                                        Value: groupName,
+                                    },
+                                    {
+                                        Key: "client",
+                                        Value: process.env.AWS_PROFILE,
+                                    },
+                                ].concat(tags),
+                            },
+                        ],
+                    }),
+                );
+            },
+        );
+
+        return sg.GroupId;
+    } catch (e) {
+        if (e.Code === "InvalidGroup.Duplicate") {
+            const existing =
+                await module.exports.findSecurityGroupByName(groupName);
+            return existing.GroupId;
+        }
+        throw e;
+    }
+};
+
+module.exports.authorizeCacheIngress = async (
+    cacheSecurityGroupId,
+    sourceSecurityGroupId,
+) => {
+    await Spinner.prototype.simple(
+        `Adding Redis inbound rule from EB security group`,
+        () => {
+            return getClient().send(
+                new AuthorizeSecurityGroupIngressCommand({
+                    GroupId: cacheSecurityGroupId,
+                    IpPermissions: [
+                        {
+                            IpProtocol: "tcp",
+                            FromPort: 6380,
+                            ToPort: 6380,
+                            UserIdGroupPairs: [
+                                {
+                                    GroupId: sourceSecurityGroupId,
+                                    Description:
+                                        "Redis/Valkey access for EB instances",
+                                },
+                            ],
+                        },
+                    ],
+                }),
+            );
+        },
+    );
+};
+
+module.exports.revokeCacheIngress = async (
+    cacheSecurityGroupId,
+    sourceSecurityGroupId,
+) => {
+    await Spinner.prototype.simple(
+        `Removing Redis inbound rule from cache security group`,
+        () => {
+            return getClient().send(
+                new RevokeSecurityGroupIngressCommand({
+                    GroupId: cacheSecurityGroupId,
+                    IpPermissions: [
+                        {
+                            IpProtocol: "tcp",
+                            FromPort: 6380,
+                            ToPort: 6380,
+                            UserIdGroupPairs: [
+                                {
+                                    GroupId: sourceSecurityGroupId,
+                                },
+                            ],
+                        },
+                    ],
+                }),
+            );
+        },
+    );
+};
 
+module.exports.findSecurityGroupByTag = async (filters = []) => {
     const res = await Spinner.prototype.simple(
         `Looking up security group by filters`,
         () => {
-            return client.send(
+            return getClient().send(
                 new DescribeSecurityGroupsCommand({
                     Filters: filters,
                 }),
@@ -139,12 +353,10 @@ module.exports.findSecurityGroupByTag = async (filters = []) => {
 };
 
 module.exports.findSecurityGroupByName = async (name) => {
-    const client = new EC2Client({});
-
     const res = await Spinner.prototype.simple(
         `Looking up security group ${name}`,
         () => {
-            return client.send(
+            return getClient().send(
                 new DescribeSecurityGroupsCommand({
                     Filters: [{ Name: "group-name", Values: [name] }],
                 }),
@@ -156,11 +368,426 @@ module.exports.findSecurityGroupByName = async (name) => {
 };
 
 module.exports.deleteSecurityGroup = async (groupId) => {
-    const client = new EC2Client({});
-
     await Spinner.prototype.simple(`Deleting security group ${groupId}`, () => {
-        return client.send(
+        return getClient().send(
             new DeleteSecurityGroupCommand({ GroupId: groupId }),
         );
     });
 };
+
+const discoverPublicSubnets = async (vpcId, excludeIds, azs) => {
+    const { Subnets: allSubnets } = await getClient().send(
+        new DescribeSubnetsCommand({
+            Filters: [{ Name: "vpc-id", Values: [vpcId] }],
+        }),
+    );
+
+    const { RouteTables } = await getClient().send(
+        new DescribeRouteTablesCommand({
+            Filters: [{ Name: "vpc-id", Values: [vpcId] }],
+        }),
+    );
+
+    const publicByAz = {};
+    for (const subnet of allSubnets) {
+        if (excludeIds.includes(subnet.SubnetId)) continue;
+        if (!azs.includes(subnet.AvailabilityZone)) continue;
+
+        let rt = RouteTables.find((r) =>
+            r.Associations.some((a) => a.SubnetId === subnet.SubnetId),
+        );
+        if (!rt) {
+            rt = RouteTables.find((r) => r.Associations.some((a) => a.Main));
+        }
+        if (!rt) continue;
+
+        const hasIgwRoute = rt.Routes.some(
+            (route) =>
+                route.GatewayId &&
+                route.GatewayId.startsWith("igw-") &&
+                route.DestinationCidrBlock === "0.0.0.0/0",
+        );
+
+        if (hasIgwRoute && !publicByAz[subnet.AvailabilityZone]) {
+            publicByAz[subnet.AvailabilityZone] = subnet.SubnetId;
+        }
+    }
+
+    return azs.map((az) => publicByAz[az]).filter(Boolean);
+};
+
+module.exports.ensurePrivateSubnets = async () => {
+    // Get default VPC
+    const { Vpcs } = await getClient().send(
+        new DescribeVpcsCommand({
+            Filters: [{ Name: "isDefault", Values: ["true"] }],
+        }),
+    );
+
+    if (!Vpcs.length) {
+        throw new Error("No default VPC found in this region");
+    }
+
+    const vpc = Vpcs[0];
+    const vpcId = vpc.VpcId;
+
+    // Preflight: check all private networking resources
+    const { Subnets: existingSubnets } = await getClient().send(
+        new DescribeSubnetsCommand({
+            Filters: [
+                { Name: "vpc-id", Values: [vpcId] },
+                { Name: "tag:private", Values: ["true"] },
+            ],
+        }),
+    );
+
+    const { NatGateways: allNats } = await getClient().send(
+        new DescribeNatGatewaysCommand({
+            Filter: [
+                { Name: "vpc-id", Values: [vpcId] },
+                { Name: "tag:private", Values: ["true"] },
+            ],
+        }),
+    );
+    const existingNats = allNats.filter(
+        (n) => n.State !== "deleted" && n.State !== "deleting",
+    );
+
+    const { RouteTables: existingRtbs } = await getClient().send(
+        new DescribeRouteTablesCommand({
+            Filters: [
+                { Name: "vpc-id", Values: [vpcId] },
+                { Name: "tag:private", Values: ["true"] },
+            ],
+        }),
+    );
+
+    // Check for inconsistencies (exactly 1 of any resource type)
+    const counts = {
+        "private subnets": existingSubnets.length,
+        "NAT gateways": existingNats.length,
+        "route tables": existingRtbs.length,
+    };
+
+    const inconsistent = Object.entries(counts).filter(
+        ([, count]) => count === 1,
+    );
+
+    if (inconsistent.length) {
+        const detail = inconsistent
+            .map(([name, count]) => `${count} ${name}`)
+            .join(", ");
+        throw new Error(
+            `Inconsistent private networking state: ${detail}. Expected 0 or 2 of each resource tagged private=true. Please resolve manually.`,
+        );
+    }
+
+    // Determine what needs creation
+    const needsSubnets = existingSubnets.length < 2;
+    const needsNats = existingNats.length < 2;
+    const needsRtbs = existingRtbs.length < 2;
+
+    // All resources exist — nothing to do
+    if (!needsSubnets && !needsNats && !needsRtbs) {
+        const publicSubnetIds = await discoverPublicSubnets(
+            vpcId,
+            existingSubnets.map((s) => s.SubnetId),
+            existingSubnets.slice(0, 2).map((s) => s.AvailabilityZone),
+        );
+        return {
+            privateSubnetIds: existingSubnets.map((s) => s.SubnetId),
+            publicSubnetIds,
+            vpcId,
+        };
+    }
+
+    const spinner = new Spinner("Setting up private subnets", true);
+
+    try {
+        // Discover AZs — use existing subnets' AZs if available, otherwise first 2
+        let azs;
+        if (!needsSubnets) {
+            azs = existingSubnets.slice(0, 2).map((s) => s.AvailabilityZone);
+        } else {
+            spinner.ora.text = "Discovering availability zones";
+            const { AvailabilityZones } = await getClient().send(
+                new DescribeAvailabilityZonesCommand({
+                    Filters: [{ Name: "state", Values: ["available"] }],
+                }),
+            );
+            azs = AvailabilityZones.slice(0, 2).map((az) => az.ZoneName);
+        }
+
+        if (azs.length < 2) {
+            throw new Error("Need at least 2 availability zones");
+        }
+
+        // Step 1: Subnets
+        let privateSubnetIds;
+        if (needsSubnets) {
+            spinner.ora.text = "Discovering available CIDR blocks";
+            const { Subnets: allSubnets } = await getClient().send(
+                new DescribeSubnetsCommand({
+                    Filters: [{ Name: "vpc-id", Values: [vpcId] }],
+                }),
+            );
+
+            const usedBlocks = allSubnets.map((s) => {
+                const [ip, prefix] = s.CidrBlock.split("/");
+                const octets = ip.split(".").map(Number);
+                const start =
+                    (octets[0] << 24) |
+                    (octets[1] << 16) |
+                    (octets[2] << 8) |
+                    octets[3];
+                const size = 1 << (32 - Number(prefix));
+                return { start, end: start + size };
+            });
+
+            const vpcOctets = vpc.CidrBlock.split("/")[0]
+                .split(".")
+                .map(Number);
+            const vpcStart =
+                (vpcOctets[0] << 24) |
+                (vpcOctets[1] << 16) |
+                (vpcOctets[2] << 8) |
+                vpcOctets[3];
+            const vpcSize = 1 << (32 - Number(vpc.CidrBlock.split("/")[1]));
+            const vpcEnd = vpcStart + vpcSize;
+            const blockSize = 4096; // /20
+
+            const availableBlocks = [];
+            for (
+                let candidate = vpcStart;
+                candidate + blockSize <= vpcEnd && availableBlocks.length < 2;
+                candidate += blockSize
+            ) {
+                const candidateEnd = candidate + blockSize;
+                const overlaps = usedBlocks.some(
+                    (b) => candidate < b.end && candidateEnd > b.start,
+                );
+                if (!overlaps) {
+                    const o1 = (candidate >>> 24) & 0xff;
+                    const o2 = (candidate >>> 16) & 0xff;
+                    const o3 = (candidate >>> 8) & 0xff;
+                    const o4 = candidate & 0xff;
+                    availableBlocks.push(`${o1}.${o2}.${o3}.${o4}/20`);
+                }
+            }
+
+            if (availableBlocks.length < 2) {
+                throw new Error(
+                    "Could not find 2 available /20 CIDR blocks in the default VPC",
+                );
+            }
+
+            privateSubnetIds = [];
+            for (let i = 0; i < 2; i++) {
+                spinner.ora.text = `Creating private subnet in ${azs[i]}`;
+                const subnet = await getClient().send(
+                    new CreateSubnetCommand({
+                        VpcId: vpcId,
+                        CidrBlock: availableBlocks[i],
+                        AvailabilityZone: azs[i],
+                        TagSpecifications: [
+                            {
+                                ResourceType: "subnet",
+                                Tags: [
+                                    {
+                                        Key: "Name",
+                                        Value: `fw-auto-private-subnet-${azs[i]}`,
+                                    },
+                                    { Key: "private", Value: "true" },
+                                ],
+                            },
+                        ],
+                    }),
+                );
+                privateSubnetIds.push(subnet.Subnet.SubnetId);
+            }
+        } else {
+            privateSubnetIds = existingSubnets.map((s) => s.SubnetId);
+        }
+
+        // Step 2: NAT gateways (+ Elastic IPs)
+        let natGatewayIds;
+        if (needsNats) {
+            // Find public subnets per AZ
+            spinner.ora.text = "Identifying public subnets";
+            const { InternetGateways } = await getClient().send(
+                new DescribeInternetGatewaysCommand({
+                    Filters: [{ Name: "attachment.vpc-id", Values: [vpcId] }],
+                }),
+            );
+
+            if (!InternetGateways.length) {
+                throw new Error("No Internet Gateway found on default VPC");
+            }
+
+            const { Subnets: vpcSubnets } = await getClient().send(
+                new DescribeSubnetsCommand({
+                    Filters: [{ Name: "vpc-id", Values: [vpcId] }],
+                }),
+            );
+
+            const { RouteTables } = await getClient().send(
+                new DescribeRouteTablesCommand({
+                    Filters: [{ Name: "vpc-id", Values: [vpcId] }],
+                }),
+            );
+
+            const publicSubnetsByAz = {};
+            for (const subnet of vpcSubnets) {
+                if (privateSubnetIds.includes(subnet.SubnetId)) continue;
+
+                let rt = RouteTables.find((r) =>
+                    r.Associations.some((a) => a.SubnetId === subnet.SubnetId),
+                );
+                if (!rt) {
+                    rt = RouteTables.find((r) =>
+                        r.Associations.some((a) => a.Main),
+                    );
+                }
+                if (!rt) continue;
+
+                const hasIgwRoute = rt.Routes.some(
+                    (route) =>
+                        route.GatewayId &&
+                        route.GatewayId.startsWith("igw-") &&
+                        route.DestinationCidrBlock === "0.0.0.0/0",
+                );
+
+                if (
+                    hasIgwRoute &&
+                    !publicSubnetsByAz[subnet.AvailabilityZone]
+                ) {
+                    publicSubnetsByAz[subnet.AvailabilityZone] =
+                        subnet.SubnetId;
+                }
+            }
+
+            for (const az of azs) {
+                if (!publicSubnetsByAz[az]) {
+                    throw new Error(
+                        `No public subnet found in ${az} to host NAT gateway`,
+                    );
+                }
+            }
+
+            natGatewayIds = [];
+            for (let i = 0; i < 2; i++) {
+                spinner.ora.text = `Allocating Elastic IP for NAT gateway in ${azs[i]}`;
+                const eip = await getClient().send(
+                    new AllocateAddressCommand({
+                        Domain: "vpc",
+                        TagSpecifications: [
+                            {
+                                ResourceType: "elastic-ip",
+                                Tags: [
+                                    {
+                                        Key: "Name",
+                                        Value: `fw-auto-nat-eip-${azs[i]}`,
+                                    },
+                                    { Key: "private", Value: "true" },
+                                ],
+                            },
+                        ],
+                    }),
+                );
+
+                spinner.ora.text = `Creating NAT gateway in ${azs[i]}`;
+                const nat = await getClient().send(
+                    new CreateNatGatewayCommand({
+                        SubnetId: publicSubnetsByAz[azs[i]],
+                        AllocationId: eip.AllocationId,
+                        TagSpecifications: [
+                            {
+                                ResourceType: "natgateway",
+                                Tags: [
+                                    {
+                                        Key: "Name",
+                                        Value: `fw-auto-nat-gateway-${azs[i]}`,
+                                    },
+                                    { Key: "private", Value: "true" },
+                                ],
+                            },
+                        ],
+                    }),
+                );
+
+                natGatewayIds.push(nat.NatGateway.NatGatewayId);
+            }
+
+            for (let i = 0; i < 2; i++) {
+                spinner.ora.text = `Waiting for NAT gateway in ${azs[i]} to become available`;
+                await poll(
+                    async () => {
+                        const res = await getClient().send(
+                            new DescribeNatGatewaysCommand({
+                                NatGatewayIds: [natGatewayIds[i]],
+                            }),
+                        );
+                        return res.NatGateways[0];
+                    },
+                    (nat) => nat.State !== "available",
+                );
+            }
+        } else {
+            natGatewayIds = existingNats.map((n) => n.NatGatewayId);
+        }
+
+        // Step 3: Route tables
+        if (needsRtbs) {
+            for (let i = 0; i < 2; i++) {
+                spinner.ora.text = `Creating route table for ${azs[i]}`;
+                const rt = await getClient().send(
+                    new CreateRouteTableCommand({
+                        VpcId: vpcId,
+                        TagSpecifications: [
+                            {
+                                ResourceType: "route-table",
+                                Tags: [
+                                    {
+                                        Key: "Name",
+                                        Value: `fw-auto-private-rtb-${azs[i]}`,
+                                    },
+                                    { Key: "private", Value: "true" },
+                                ],
+                            },
+                        ],
+                    }),
+                );
+
+                spinner.ora.text = `Adding NAT gateway route for ${azs[i]}`;
+                await getClient().send(
+                    new CreateRouteCommand({
+                        RouteTableId: rt.RouteTable.RouteTableId,
+                        DestinationCidrBlock: "0.0.0.0/0",
+                        NatGatewayId: natGatewayIds[i],
+                    }),
+                );
+
+                spinner.ora.text = `Associating route table with private subnet in ${azs[i]}`;
+                await getClient().send(
+                    new AssociateRouteTableCommand({
+                        RouteTableId: rt.RouteTable.RouteTableId,
+                        SubnetId: privateSubnetIds[i],
+                    }),
+                );
+            }
+        }
+
+        spinner.ora.succeed("Private subnets configured");
+
+        const publicSubnetIds = await discoverPublicSubnets(
+            vpcId,
+            privateSubnetIds,
+            azs,
+        );
+
+        return { privateSubnetIds, publicSubnetIds, vpcId };
+    } catch (e) {
+        spinner.ora.fail("Failed to set up private subnets");
+        throw e;
+    }
+};