@infoxchange/make-it-so 2.19.2 → 2.20.0

package/README.md

@@ -421,6 +421,88 @@ new IxWebsiteRedirect(scope, "WebsiteRedirect", {
 
 </details>
 
+<details>
+<summary><strong>IxTroubleshootingBastion</strong> - Deploys a temporary EC2 host for debugging things only reachable from inside the VPC.</summary>
+
+Sometimes you need to poke at something that only exists inside the VPC — an RDS instance, an Elasticache cluster, an internal API. This deploys a small host in the standard IX VPC that you can SSH into and use as a jump box.
+
+It is built on [CDK's BastionHostLinux](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.BastionHostLinux.html) and accepts all the same props, with these differences:
+
+- It always attaches to the standard IX VPC and its private subnets (via `IxVpcDetails`), so there is no `vpc` prop.
+- It is set up for SSH rather than SSM Session Manager. EC2 generates a key pair and CloudFormation stores the private key in Parameter Store, so no key material passes through the CloudFormation template, an env var, or your shell history.
+- It defaults to the smallest instance type available, `t4g.nano`.
+- It tags itself with `CreatorResource=IxTroubleshootingBastion` so you can find it without knowing the stack or instance name.
+
+```typescript
+import { IxTroubleshootingBastion } from "@infoxchange/make-it-so/cdk-constructs";
+
+// Deploy it behind an env var so it isn't part of a normal deploy
+if (process.env.TROUBLESHOOTING_BASTION) {
+  new IxTroubleshootingBastion(stack, "TroubleshootingBastion", {
+    packages: ["postgresql16"],
+  });
+}
+```
+
+> [!WARNING]
+> This is a debugging tool, not something to leave running. It should be reasonably secure as it's private IP means it's
+> only connectable over the VPN and it can only be connected to with the SSH key. But still better not to tempt fate so
+> remove it once you are done.
+
+#### Connecting:
+
+On a CI deploy the instructions below are emitted as a workflow annotation with the values already filled in. Otherwise,
+find the host and download it's key pair:
+
+```shell
+ix aws auth -A app:$APP_NAME/$APP_ENV -- aws ec2 describe-instances \
+  --filters Name=tag:CreatorResource,Values=IxTroubleshootingBastion \
+            Name=instance-state-name,Values=running \
+  --query "Reservations[].Instances[].[PrivateIpAddress,KeyName]" --output text
+
+ix aws auth -A app:$APP_NAME/$APP_ENV -- aws ec2 describe-key-pairs --key-names <key name> \
+  --query "KeyPairs[].KeyPairId" --output text
+
+ix aws auth -A app:$APP_NAME/$APP_ENV -- aws ssm get-parameter --name /ec2/keypair/<key pair id> --with-decryption \
+  --query Parameter.Value --output text > /tmp/troubleshooting-bastion.pem
+
+chmod 400 /tmp/troubleshooting-bastion.pem # SSH will not accept the key otherwise
+```
+
+If you are connected to the VPN you should be able to SSH to the bastion host with:
+
+```shell
+ssh -i /tmp/troubleshooting-bastion.pem ec2-user@<private ip>
+```
+
+To reach a service through the bastion rather than from it, use a local port forward. For example to get at Postgres on RDS from your machine:
+
+```shell
+ssh -N -i /tmp/troubleshooting-bastion.pem \
+  -L 5433:<rds endpoint>:5432 \
+  ec2-user@<private ip>
+```
+
+Note that whatever you are connecting to needs a security group that accepts traffic from the bastion, either by allowing the private subnet CIDRs or by having the bastion's `securityGroup` added to it.
+
+#### Options:
+
+| Prop                       | Type     | Description                                                                                                                                                      |
+| -------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| allowSshFrom               | string[] | (optional) CIDRs allowed to reach the host on port 22. Defaults to `["0.0.0.0/0"]`                                                                               |
+| packages                   | string[] | (optional) Extra packages to `dnf install` at boot, eg a database client to poke at RDS with                                                                     |
+| [...BastionHostLinuxProps] |          | Any props accepted by [BastionHostLinux](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.BastionHostLinux.html#construct-props) other than `vpc` |
+
+#### Properties:
+
+| Properties    | Type             | Description                                                                            |
+| ------------- | ---------------- | -------------------------------------------------------------------------------------- |
+| bastion       | BastionHostLinux | The underlying AWS CDK BastionHostLinux instance                                       |
+| keyPair       | KeyPair          | The generated EC2 key pair. Its `keyPairId` gives the Parameter Store path for the key |
+| securityGroup | ISecurityGroup   | The host's security group, to add to the security groups of whatever it needs to reach |
+
+</details>
+
 <details>
 <summary><strong>IxVpcDetails</strong> - Fetches the standard VPC and subnets that exist in all IX workload aws accounts.</summary>
 

package/dist/cdk-constructs/IxTroubleshootingBastion.d.ts

@@ -0,0 +1,50 @@
+import { BastionHostLinux, ISecurityGroup, KeyPair } from "aws-cdk-lib/aws-ec2";
+import { Construct } from "constructs";
+type ConstructScope = ConstructorParameters<typeof Construct>[0];
+type ConstructId = ConstructorParameters<typeof Construct>[1];
+type BastionHostProps = ConstructorParameters<typeof BastionHostLinux>[2];
+type Props = Omit<BastionHostProps, "vpc"> & {
+    /**
+     * CIDRs allowed to reach the bastion on port 22. Defaults to anywhere, which is only reasonable because the bastion
+     * has no public IP — the subnet is private, so reaching it still means coming in over the VPN.
+     */
+    allowSshFrom?: string[];
+    /** Extra packages to `dnf install` at boot, eg a database client to poke at RDS with. */
+    packages?: string[];
+};
+/**
+ * A short-lived EC2 host for poking at things that are only reachable from inside the VPC (an RDS instance, a redis
+ * cluster, an internal API).
+ *
+ * Unlike a plain BastionHostLinux this is set up for SSH rather than SSM Session Manager. EC2 generates the key pair
+ * and CloudFormation stores the private key in Parameter Store, so no key material passes through the template or an
+ * env var. On a CI deploy the instructions for connecting are emitted as a workflow annotation. Otherwise the host
+ * tags itself so you can find it, and its key, without knowing the stack or instance name:
+ *
+ *     aws ec2 describe-instances \
+ *       --filters Name=tag:CreatorResource,Values=IxTroubleshootingBastion \
+ *                 Name=instance-state-name,Values=running \
+ *       --query "Reservations[].Instances[].[PrivateIpAddress,KeyName]" --output text
+ *     aws ec2 describe-key-pairs --key-names <key name> --query "KeyPairs[].KeyPairId" --output text
+ *     aws ssm get-parameter --name /ec2/keypair/<key pair id> --with-decryption \
+ *       --query Parameter.Value --output text > /tmp/troubleshooting-bastion.pem
+ *     chmod 400 /tmp/troubleshooting-bastion.pem
+ *     ssh -i /tmp/troubleshooting-bastion.pem ec2-user@<private ip>
+ *
+ * CloudFormation deletes that parameter along with the key pair, so nothing is left behind — but it needs
+ * `ssm:PutParameter` and `ssm:DeleteParameter` on the deploying role to create and remove the stack. The delete check
+ * runs against a fabricated parameter name, so a missing permission shows up as an AccessDeniedException naming a
+ * parameter that doesn't exist.
+ *
+ * This is a debugging tool, not something to leave running: it is a host in the private subnets accepting SSH from
+ * anywhere by default, so remove it once you are done.
+ */
+export declare class IxTroubleshootingBastion extends Construct {
+    bastion: BastionHostLinux;
+    keyPair: KeyPair;
+    securityGroup: ISecurityGroup;
+    constructor(scope: ConstructScope, id: ConstructId, { allowSshFrom, securityGroup, packages, ...bastionProps }?: Props);
+    private installPackages;
+}
+export {};
+//# sourceMappingURL=IxTroubleshootingBastion.d.ts.map

package/dist/cdk-constructs/IxTroubleshootingBastion.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IxTroubleshootingBastion.d.ts","sourceRoot":"","sources":["../../src/cdk-constructs/IxTroubleshootingBastion.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,gBAAgB,EAKhB,cAAc,EACd,OAAO,EAOR,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAKvC,KAAK,cAAc,GAAG,qBAAqB,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AACjE,KAAK,WAAW,GAAG,qBAAqB,CAAC,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,KAAK,gBAAgB,GAAG,qBAAqB,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;AAE1E,KAAK,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,GAAG;IAC3C;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,yFAAyF;IACzF,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,qBAAa,wBAAyB,SAAQ,SAAS;IAC9C,OAAO,EAAE,gBAAgB,CAAC;IAE1B,OAAO,EAAE,OAAO,CAAC;IAEjB,aAAa,EAAE,cAAc,CAAC;gBAGnC,KAAK,EAAE,cAAc,EACrB,EAAE,EAAE,WAAW,EACf,EACE,YAA4B,EAC5B,aAAa,EACb,QAAa,EACb,GAAG,YAAY,EAChB,GAAE,KAAU;IAwFf,OAAO,CAAC,eAAe;CAkBxB"}

package/dist/cdk-constructs/IxTroubleshootingBastion.js

@@ -0,0 +1,118 @@
+import { CfnOutput, Tags } from "aws-cdk-lib";
+import { AmazonLinuxCpuType, BastionHostLinux, InstanceArchitecture, InstanceClass, InstanceSize, InstanceType, KeyPair, KeyPairType, MachineImage, Peer, Port, SecurityGroup, SubnetType, } from "aws-cdk-lib/aws-ec2";
+import { Construct } from "constructs";
+import { IxVpcDetails } from "./IxVpcDetails.js";
+import deployConfig from "../deployConfig.js";
+import { deindent } from "../lib/utils/strings.js";
+/**
+ * A short-lived EC2 host for poking at things that are only reachable from inside the VPC (an RDS instance, a redis
+ * cluster, an internal API).
+ *
+ * Unlike a plain BastionHostLinux this is set up for SSH rather than SSM Session Manager. EC2 generates the key pair
+ * and CloudFormation stores the private key in Parameter Store, so no key material passes through the template or an
+ * env var. On a CI deploy the instructions for connecting are emitted as a workflow annotation. Otherwise the host
+ * tags itself so you can find it, and its key, without knowing the stack or instance name:
+ *
+ *     aws ec2 describe-instances \
+ *       --filters Name=tag:CreatorResource,Values=IxTroubleshootingBastion \
+ *                 Name=instance-state-name,Values=running \
+ *       --query "Reservations[].Instances[].[PrivateIpAddress,KeyName]" --output text
+ *     aws ec2 describe-key-pairs --key-names <key name> --query "KeyPairs[].KeyPairId" --output text
+ *     aws ssm get-parameter --name /ec2/keypair/<key pair id> --with-decryption \
+ *       --query Parameter.Value --output text > /tmp/troubleshooting-bastion.pem
+ *     chmod 400 /tmp/troubleshooting-bastion.pem
+ *     ssh -i /tmp/troubleshooting-bastion.pem ec2-user@<private ip>
+ *
+ * CloudFormation deletes that parameter along with the key pair, so nothing is left behind — but it needs
+ * `ssm:PutParameter` and `ssm:DeleteParameter` on the deploying role to create and remove the stack. The delete check
+ * runs against a fabricated parameter name, so a missing permission shows up as an AccessDeniedException naming a
+ * parameter that doesn't exist.
+ *
+ * This is a debugging tool, not something to leave running: it is a host in the private subnets accepting SSH from
+ * anywhere by default, so remove it once you are done.
+ */
+export class IxTroubleshootingBastion extends Construct {
+    bastion;
+    keyPair;
+    securityGroup;
+    constructor(scope, id, { allowSshFrom = ["0.0.0.0/0"], securityGroup, packages = [], ...bastionProps } = {}) {
+        super(scope, id);
+        const { vpc } = new IxVpcDetails(this, `${id}-IxVpcDetails`);
+        this.securityGroup =
+            securityGroup ??
+                new SecurityGroup(this, `${id}-SecurityGroup`, {
+                    vpc,
+                    allowAllOutbound: true,
+                    description: `Security group for ${deployConfig.appName} ${deployConfig.environment} troubleshooting bastion`,
+                });
+        for (const cidr of allowSshFrom) {
+            this.securityGroup.addIngressRule(Peer.ipv4(cidr), Port.tcp(22), `Allow SSH to the troubleshooting bastion from ${cidr}`);
+        }
+        // Omitting publicKeyMaterial is what makes this a generated key pair, which is what gets the private key written
+        // to Parameter Store.
+        this.keyPair = new KeyPair(this, `${id}-KeyPair`, {
+            type: KeyPairType.ED25519,
+        });
+        // The smallest instance type available, and the cheapest of the 512MiB ones.
+        const instanceType = bastionProps.instanceType ??
+            InstanceType.of(InstanceClass.T4G, InstanceSize.NANO);
+        this.bastion = new BastionHostLinux(this, `${id}-Bastion`, {
+            instanceName: `${deployConfig.appName}-${deployConfig.environment}-${id}`,
+            // IxVpcDetails registers its subnets as isolated, so the BastionHostLinux default of PRIVATE_WITH_EGRESS finds
+            // no subnets to place the instance in.
+            subnetSelection: { subnetType: SubnetType.PRIVATE_ISOLATED },
+            instanceType,
+            // Pinned rather than left to the `bastionHostUseAmazonLinux2023ByDefault` feature flag so the OS the packages
+            // below are installed with is predictable.
+            machineImage: MachineImage.latestAmazonLinux2023({
+                cpuType: instanceType.architecture === InstanceArchitecture.ARM_64
+                    ? AmazonLinuxCpuType.ARM_64
+                    : AmazonLinuxCpuType.X86_64,
+            }),
+            ...bastionProps,
+            vpc,
+            securityGroup: this.securityGroup,
+        });
+        // BastionHostLinux takes no keyPair/keyName prop of its own (aws-cdk#18565), so this goes on the L1 directly.
+        // cloud-init picks the key up from instance metadata and writes it to the default user's authorized_keys, which
+        // is why there's no key handling in the user data below.
+        this.bastion.instance.instance.keyName = this.keyPair.keyPairName;
+        // Lets the host be found with `describe-instances --filters Name=tag:CreatorResource,...` without having to know
+        // the stack or instance name. See the connection instructions above.
+        Tags.of(this.bastion).add("CreatorResource", "IxTroubleshootingBastion");
+        this.installPackages(packages);
+        const tmpCertPath = "/tmp/troubleshooting-bastion.pem";
+        const message = deindent(`
+      A troubleshooting bastion host has been deployed. MAKE SURE TO REMOVE IT AFTER USE.
+
+      To fetch the private key used to connect to the bastion host run:
+        ix aws auth -A app:${deployConfig.appName}/${deployConfig.environment} -- aws ssm get-parameter --name /ec2/keypair/${this.keyPair.keyPairId} --with-decryption --query Parameter.Value --output text > ${tmpCertPath}
+        chmod 400 ${tmpCertPath}
+
+      To connect to the bastion host ensure you're connected to the VPN then run:
+        ssh -i ${tmpCertPath} ec2-user@${this.bastion.instancePrivateIp}
+    `);
+        let outputValue;
+        if (process.env.CI) {
+            outputValue = `The following note is available as an annotation in the CI workflow run:\n::warning title=Troubleshooting Bastion::${message.replaceAll("\n", "%0A")}`;
+        }
+        else {
+            outputValue = message;
+        }
+        // We can't just log this with console.log() since the CDK tokens aren't resolvable at that point.
+        new CfnOutput(this, "TroubleshootingBastionNote", {
+            value: outputValue,
+        });
+    }
+    installPackages(packages) {
+        if (!packages.length) {
+            return;
+        }
+        const { userData } = this.bastion.instance;
+        // The subnet has no direct egress, so reaching the package repos goes via the proxy.
+        if (deployConfig.vpcHttpProxy) {
+            userData.addCommands(`export http_proxy=${deployConfig.vpcHttpProxy}`, `export https_proxy=${deployConfig.vpcHttpProxy}`, `export no_proxy=169.254.169.254,localhost,127.0.0.1`);
+        }
+        userData.addCommands(`dnf install -y ${packages.join(" ")}`);
+    }
+}

package/dist/cdk-constructs/index.d.ts

@@ -8,5 +8,6 @@ export * from "./IxElasticache.js";
 export * from "./IxApi.js";
 export * from "./IxQuicksightWorkspace.js";
 export * from "./IxCloudWatchAlarm.js";
+export * from "./IxTroubleshootingBastion.js";
 export * from "./SiteOidcAuth/index.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/cdk-constructs/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cdk-constructs/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cdk-constructs/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,wBAAwB,CAAC;AACvC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC"}

package/dist/cdk-constructs/index.js

@@ -8,4 +8,5 @@ export * from "./IxElasticache.js";
 export * from "./IxApi.js";
 export * from "./IxQuicksightWorkspace.js";
 export * from "./IxCloudWatchAlarm.js";
+export * from "./IxTroubleshootingBastion.js";
 export * from "./SiteOidcAuth/index.js";

package/dist/deployConfig.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deployConfig.d.ts","sourceRoot":"","sources":["../src/deployConfig.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyFA,wBAA0C;AAG1C,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAAmC,CAAC"}
+{"version":3,"file":"deployConfig.d.ts","sourceRoot":"","sources":["../src/deployConfig.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqGA,wBAA0C;AAG1C,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAAmC,CAAC"}

package/dist/deployConfig.js

@@ -26,12 +26,14 @@ const ixDeployConfigSchema = z
     environment: z.enum(["dev", "test", "uat", "prod"]),
     workloadGroup: z.enum(["ds", "srs"]),
     primaryAwsRegion: z.literal("ap-southeast-2"),
-    siteDomains: z
-        .string()
-        .transform((val) => val.split(",").map((domain) => domain.trim())),
-    siteDomainAliases: z
-        .string()
-        .transform((val) => val.split(",").map((domain) => domain.trim())),
+    siteDomains: z.string().transform((val) => val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean)),
+    siteDomainAliases: z.string().transform((val) => val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean)),
     isInternalApp: z.coerce.boolean(),
     deploymentType: z.enum(["docker", "serverless"]),
     sourceCommitRef: z.string().min(1),
@@ -52,12 +54,14 @@ const nonIxDeployConfigSchema = z
     environment: z.string(),
     workloadGroup: z.string(),
     primaryAwsRegion: z.string(),
-    siteDomains: z
-        .string()
-        .transform((val) => val.split(",").map((domain) => domain.trim())),
-    siteDomainAliases: z
-        .string()
-        .transform((val) => val.split(",").map((domain) => domain.trim())),
+    siteDomains: z.string().transform((val) => val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean)),
+    siteDomainAliases: z.string().transform((val) => val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean)),
     isInternalApp: z
         .string()
         .transform((val) => (val ? val.toLowerCase() === "true" : undefined)),

package/dist/lib/utils/strings.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Strips the shared leading whitespace off every line of a template literal, so a multi-line string can be indented
+ * to match the code it sits in without that indentation ending up in the output. Blank lines are emptied rather than
+ * kept as whitespace, and leading/trailing blank lines are dropped.
+ */
+export declare function deindent(str: string): string;
+//# sourceMappingURL=strings.d.ts.map

package/dist/lib/utils/strings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"strings.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/strings.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAa5C"}

package/dist/lib/utils/strings.js

@@ -0,0 +1,18 @@
+/**
+ * Strips the shared leading whitespace off every line of a template literal, so a multi-line string can be indented
+ * to match the code it sits in without that indentation ending up in the output. Blank lines are emptied rather than
+ * kept as whitespace, and leading/trailing blank lines are dropped.
+ */
+export function deindent(str) {
+    const lines = str.split("\n");
+    const minIndent = lines
+        .filter((line) => line.trim().length > 0)
+        .reduce((min, line) => {
+        const indent = line.match(/^\s*/)?.[0].length ?? 0;
+        return Math.min(min, indent);
+    }, Infinity);
+    return lines
+        .map((line) => (line.trim().length === 0 ? "" : line.slice(minIndent)))
+        .join("\n")
+        .trim();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@infoxchange/make-it-so",
-  "version": "2.19.2",
+  "version": "2.20.0",
   "description": "Makes deploying services to IX infra easy",
   "repository": "github:infoxchange/make-it-so",
   "publishConfig": {

package/src/cdk-constructs/IxTroubleshootingBastion.ts

@@ -0,0 +1,185 @@
+import { CfnOutput, Tags } from "aws-cdk-lib";
+import {
+  AmazonLinuxCpuType,
+  BastionHostLinux,
+  InstanceArchitecture,
+  InstanceClass,
+  InstanceSize,
+  InstanceType,
+  ISecurityGroup,
+  KeyPair,
+  KeyPairType,
+  MachineImage,
+  Peer,
+  Port,
+  SecurityGroup,
+  SubnetType,
+} from "aws-cdk-lib/aws-ec2";
+import { Construct } from "constructs";
+import { IxVpcDetails } from "./IxVpcDetails.js";
+import deployConfig from "../deployConfig.js";
+import { deindent } from "../lib/utils/strings.js";
+
+type ConstructScope = ConstructorParameters<typeof Construct>[0];
+type ConstructId = ConstructorParameters<typeof Construct>[1];
+type BastionHostProps = ConstructorParameters<typeof BastionHostLinux>[2];
+
+type Props = Omit<BastionHostProps, "vpc"> & {
+  /**
+   * CIDRs allowed to reach the bastion on port 22. Defaults to anywhere, which is only reasonable because the bastion
+   * has no public IP — the subnet is private, so reaching it still means coming in over the VPN.
+   */
+  allowSshFrom?: string[];
+  /** Extra packages to `dnf install` at boot, eg a database client to poke at RDS with. */
+  packages?: string[];
+};
+
+/**
+ * A short-lived EC2 host for poking at things that are only reachable from inside the VPC (an RDS instance, a redis
+ * cluster, an internal API).
+ *
+ * Unlike a plain BastionHostLinux this is set up for SSH rather than SSM Session Manager. EC2 generates the key pair
+ * and CloudFormation stores the private key in Parameter Store, so no key material passes through the template or an
+ * env var. On a CI deploy the instructions for connecting are emitted as a workflow annotation. Otherwise the host
+ * tags itself so you can find it, and its key, without knowing the stack or instance name:
+ *
+ *     aws ec2 describe-instances \
+ *       --filters Name=tag:CreatorResource,Values=IxTroubleshootingBastion \
+ *                 Name=instance-state-name,Values=running \
+ *       --query "Reservations[].Instances[].[PrivateIpAddress,KeyName]" --output text
+ *     aws ec2 describe-key-pairs --key-names <key name> --query "KeyPairs[].KeyPairId" --output text
+ *     aws ssm get-parameter --name /ec2/keypair/<key pair id> --with-decryption \
+ *       --query Parameter.Value --output text > /tmp/troubleshooting-bastion.pem
+ *     chmod 400 /tmp/troubleshooting-bastion.pem
+ *     ssh -i /tmp/troubleshooting-bastion.pem ec2-user@<private ip>
+ *
+ * CloudFormation deletes that parameter along with the key pair, so nothing is left behind — but it needs
+ * `ssm:PutParameter` and `ssm:DeleteParameter` on the deploying role to create and remove the stack. The delete check
+ * runs against a fabricated parameter name, so a missing permission shows up as an AccessDeniedException naming a
+ * parameter that doesn't exist.
+ *
+ * This is a debugging tool, not something to leave running: it is a host in the private subnets accepting SSH from
+ * anywhere by default, so remove it once you are done.
+ */
+export class IxTroubleshootingBastion extends Construct {
+  public bastion: BastionHostLinux;
+
+  public keyPair: KeyPair;
+
+  public securityGroup: ISecurityGroup;
+
+  constructor(
+    scope: ConstructScope,
+    id: ConstructId,
+    {
+      allowSshFrom = ["0.0.0.0/0"],
+      securityGroup,
+      packages = [],
+      ...bastionProps
+    }: Props = {},
+  ) {
+    super(scope, id);
+
+    const { vpc } = new IxVpcDetails(this, `${id}-IxVpcDetails`);
+
+    this.securityGroup =
+      securityGroup ??
+      new SecurityGroup(this, `${id}-SecurityGroup`, {
+        vpc,
+        allowAllOutbound: true,
+        description: `Security group for ${deployConfig.appName} ${deployConfig.environment} troubleshooting bastion`,
+      });
+
+    for (const cidr of allowSshFrom) {
+      this.securityGroup.addIngressRule(
+        Peer.ipv4(cidr),
+        Port.tcp(22),
+        `Allow SSH to the troubleshooting bastion from ${cidr}`,
+      );
+    }
+
+    // Omitting publicKeyMaterial is what makes this a generated key pair, which is what gets the private key written
+    // to Parameter Store.
+    this.keyPair = new KeyPair(this, `${id}-KeyPair`, {
+      type: KeyPairType.ED25519,
+    });
+
+    // The smallest instance type available, and the cheapest of the 512MiB ones.
+    const instanceType =
+      bastionProps.instanceType ??
+      InstanceType.of(InstanceClass.T4G, InstanceSize.NANO);
+
+    this.bastion = new BastionHostLinux(this, `${id}-Bastion`, {
+      instanceName: `${deployConfig.appName}-${deployConfig.environment}-${id}`,
+      // IxVpcDetails registers its subnets as isolated, so the BastionHostLinux default of PRIVATE_WITH_EGRESS finds
+      // no subnets to place the instance in.
+      subnetSelection: { subnetType: SubnetType.PRIVATE_ISOLATED },
+      instanceType,
+      // Pinned rather than left to the `bastionHostUseAmazonLinux2023ByDefault` feature flag so the OS the packages
+      // below are installed with is predictable.
+      machineImage: MachineImage.latestAmazonLinux2023({
+        cpuType:
+          instanceType.architecture === InstanceArchitecture.ARM_64
+            ? AmazonLinuxCpuType.ARM_64
+            : AmazonLinuxCpuType.X86_64,
+      }),
+      ...bastionProps,
+      vpc,
+      securityGroup: this.securityGroup,
+    });
+
+    // BastionHostLinux takes no keyPair/keyName prop of its own (aws-cdk#18565), so this goes on the L1 directly.
+    // cloud-init picks the key up from instance metadata and writes it to the default user's authorized_keys, which
+    // is why there's no key handling in the user data below.
+    this.bastion.instance.instance.keyName = this.keyPair.keyPairName;
+
+    // Lets the host be found with `describe-instances --filters Name=tag:CreatorResource,...` without having to know
+    // the stack or instance name. See the connection instructions above.
+    Tags.of(this.bastion).add("CreatorResource", "IxTroubleshootingBastion");
+
+    this.installPackages(packages);
+
+    const tmpCertPath = "/tmp/troubleshooting-bastion.pem";
+    const message = deindent(`
+      A troubleshooting bastion host has been deployed. MAKE SURE TO REMOVE IT AFTER USE.
+
+      To fetch the private key used to connect to the bastion host run:
+        ix aws auth -A app:${deployConfig.appName}/${deployConfig.environment} -- aws ssm get-parameter --name /ec2/keypair/${this.keyPair.keyPairId} --with-decryption --query Parameter.Value --output text > ${tmpCertPath}
+        chmod 400 ${tmpCertPath}
+
+      To connect to the bastion host ensure you're connected to the VPN then run:
+        ssh -i ${tmpCertPath} ec2-user@${this.bastion.instancePrivateIp}
+    `);
+
+    let outputValue;
+    if (process.env.CI) {
+      outputValue = `The following note is available as an annotation in the CI workflow run:\n::warning title=Troubleshooting Bastion::${message.replaceAll("\n", "%0A")}`;
+    } else {
+      outputValue = message;
+    }
+
+    // We can't just log this with console.log() since the CDK tokens aren't resolvable at that point.
+    new CfnOutput(this, "TroubleshootingBastionNote", {
+      value: outputValue,
+    });
+  }
+
+  private installPackages(packages: string[]) {
+    if (!packages.length) {
+      return;
+    }
+
+    const { userData } = this.bastion.instance;
+
+    // The subnet has no direct egress, so reaching the package repos goes via the proxy.
+    if (deployConfig.vpcHttpProxy) {
+      userData.addCommands(
+        `export http_proxy=${deployConfig.vpcHttpProxy}`,
+        `export https_proxy=${deployConfig.vpcHttpProxy}`,
+        `export no_proxy=169.254.169.254,localhost,127.0.0.1`,
+      );
+    }
+
+    userData.addCommands(`dnf install -y ${packages.join(" ")}`);
+  }
+}

package/src/cdk-constructs/index.ts

@@ -8,4 +8,5 @@ export * from "./IxElasticache.js";
 export * from "./IxApi.js";
 export * from "./IxQuicksightWorkspace.js";
 export * from "./IxCloudWatchAlarm.js";
+export * from "./IxTroubleshootingBastion.js";
 export * from "./SiteOidcAuth/index.js";

package/src/deployConfig.ts

@@ -29,12 +29,18 @@ const ixDeployConfigSchema = z
     environment: z.enum(["dev", "test", "uat", "prod"]),
     workloadGroup: z.enum(["ds", "srs"]),
     primaryAwsRegion: z.literal("ap-southeast-2"),
-    siteDomains: z
-      .string()
-      .transform((val) => val.split(",").map((domain) => domain.trim())),
-    siteDomainAliases: z
-      .string()
-      .transform((val) => val.split(",").map((domain) => domain.trim())),
+    siteDomains: z.string().transform((val) =>
+      val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean),
+    ),
+    siteDomainAliases: z.string().transform((val) =>
+      val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean),
+    ),
     isInternalApp: z.coerce.boolean(),
     deploymentType: z.enum(["docker", "serverless"]),
     sourceCommitRef: z.string().min(1),
@@ -56,12 +62,18 @@ const nonIxDeployConfigSchema = z
     environment: z.string(),
     workloadGroup: z.string(),
     primaryAwsRegion: z.string(),
-    siteDomains: z
-      .string()
-      .transform((val) => val.split(",").map((domain) => domain.trim())),
-    siteDomainAliases: z
-      .string()
-      .transform((val) => val.split(",").map((domain) => domain.trim())),
+    siteDomains: z.string().transform((val) =>
+      val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean),
+    ),
+    siteDomainAliases: z.string().transform((val) =>
+      val
+        .split(",")
+        .map((domain) => domain.trim())
+        .filter(Boolean),
+    ),
     isInternalApp: z
       .string()
       .transform((val) => (val ? val.toLowerCase() === "true" : undefined)),

package/src/lib/utils/strings.ts

@@ -0,0 +1,19 @@
+/**
+ * Strips the shared leading whitespace off every line of a template literal, so a multi-line string can be indented
+ * to match the code it sits in without that indentation ending up in the output. Blank lines are emptied rather than
+ * kept as whitespace, and leading/trailing blank lines are dropped.
+ */
+export function deindent(str: string): string {
+  const lines = str.split("\n");
+  const minIndent = lines
+    .filter((line) => line.trim().length > 0)
+    .reduce((min, line) => {
+      const indent = line.match(/^\s*/)?.[0].length ?? 0;
+      return Math.min(min, indent);
+    }, Infinity);
+
+  return lines
+    .map((line) => (line.trim().length === 0 ? "" : line.slice(minIndent)))
+    .join("\n")
+    .trim();
+}