adhoc-deploy 1.1.0

package/.nvmrc

@@ -0,0 +1 @@
+20.18.0

package/Dockerfile

@@ -0,0 +1,18 @@
+FROM node:20.18.0
+
+WORKDIR /usr/src/app
+
+COPY package.json /usr/src/app/
+
+RUN npm install
+
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install
+
+RUN apt-get update && \
+    apt-get -y install zip
+
+COPY . /usr/src/app/
+
+CMD [ "node", "index.mjs" ]

package/README.md

@@ -0,0 +1,14 @@
+# adhoc-deploy
+
+AWS Batch job container for deploying QA/Demo environments.
+
+## Deploy
+
+You must be logged in using the wp-role profile and have Docker running.
+
+1. `npm run deploy`
+2. `npm run deploy:verify`
+
+## Bootstrap
+
+Run `npm run bootstrap` after modifying job definitions in `aws-config/` to register updated definitions with AWS Batch.

package/aws-config/bootstrap.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Register updated Batch job definitions with disable_discord_notifications parameter
+
+aws batch register-job-definition --profile wp-role --cli-input-json file://qa-api-deploy.json
+aws batch register-job-definition --profile wp-role --cli-input-json file://qa-webapp-deploy.json
+aws batch register-job-definition --profile wp-role --cli-input-json file://demo-api-deploy.json
+aws batch register-job-definition --profile wp-role --cli-input-json file://demo-webapp-deploy.json
+
+echo "✅ All job definitions updated"

package/aws-config/demo-api-deploy.json

@@ -0,0 +1,88 @@
+{
+    "jobDefinitionName": "demo-api-deploy",
+    "type": "container",
+    "parameters": {
+        "branch": "master"
+    },
+    "retryStrategy": {
+        "attempts": 1,
+        "evaluateOnExit": []
+    },
+    "containerProperties": {
+        "image": "147724020626.dkr.ecr.us-east-1.amazonaws.com/deployments",
+        "command": [
+            "node",
+            "index.mjs",
+            "--branch",
+            "Ref::branch",
+            "--disable_discord_notifications",
+            "Ref::disable_discord_notifications"
+        ],
+        "jobRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "executionRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "volumes": [],
+        "environment": [
+            {
+                "name": "AWS_REGION",
+                "value": "us-east-1"
+            },
+            {
+                "name": "REPO_KEY_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/configuration/wp-code-deploy-api"
+            },
+            {
+                "name": "ARTIFACT_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/builds/demo/api/source.zip"
+            },
+            {
+                "name": "REPO_URL",
+                "value": "git@github.com:WholesalePayroll/service-layer.git"
+            },
+            {
+                "name": "VPC_ID",
+                "value": "vpc-097160e523f14ec6f"
+            },
+            {
+                "name": "LAUNCH_TEMPLATE",
+                "value": "lt-0f04c819ccc51da3d,lt-0e546a64ec43647b6"
+            },
+            {
+                "name": "SOURCE_PACKAGE",
+                "value": "api.zip"
+            },
+            {
+                "name": "GIT_SSH_COMMAND",
+                "value": "ssh -i ~/.ssh/wp-code-deploy -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+            }
+        ],
+        "mountPoints": [],
+        "ulimits": [],
+        "resourceRequirements": [
+            {
+                "value": "1",
+                "type": "VCPU"
+            },
+            {
+                "value": "1024",
+                "type": "MEMORY"
+            }
+        ],
+        "logConfiguration": {
+            "logDriver": "awslogs",
+            "options": {},
+            "secretOptions": []
+        },
+        "secrets": []
+    },
+    "timeout": {
+        "attemptDurationSeconds": 600
+    },
+    "tags": {
+        "Role": "api",
+        "Environment": "demo"
+    },
+    "propagateTags": false,
+    "platformCapabilities": [
+        "EC2"
+    ]
+}

package/aws-config/demo-webapp-deploy.json

@@ -0,0 +1,88 @@
+{
+    "jobDefinitionName": "demo-webapp-deploy",
+    "type": "container",
+    "parameters": {
+        "branch": "master"
+    },
+    "retryStrategy": {
+        "attempts": 1,
+        "evaluateOnExit": []
+    },
+    "containerProperties": {
+        "image": "147724020626.dkr.ecr.us-east-1.amazonaws.com/deployments",
+        "command": [
+            "node",
+            "index.mjs",
+            "--branch",
+            "Ref::branch",
+            "--disable_discord_notifications",
+            "Ref::disable_discord_notifications"
+        ],
+        "jobRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "executionRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "volumes": [],
+        "environment": [
+            {
+                "name": "AWS_REGION",
+                "value": "us-east-1"
+            },
+            {
+                "name": "REPO_KEY_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/configuration/wp-code-deploy"
+            },
+            {
+                "name": "ARTIFACT_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/builds/demo/web/source.zip"
+            },
+            {
+                "name": "REPO_URL",
+                "value": "git@github.com:WholesalePayroll/web-application.git"
+            },
+            {
+                "name": "VPC_ID",
+                "value": "vpc-097160e523f14ec6f"
+            },
+            {
+                "name": "LAUNCH_TEMPLATE",
+                "value": "lt-05f023dfa7b4e668e"
+            },
+            {
+                "name": "SOURCE_PACKAGE",
+                "value": "webapp.zip"
+            },
+            {
+                "name": "GIT_SSH_COMMAND",
+                "value": "ssh -i ~/.ssh/wp-code-deploy -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+            }
+        ],
+        "mountPoints": [],
+        "ulimits": [],
+        "resourceRequirements": [
+            {
+                "value": "1",
+                "type": "VCPU"
+            },
+            {
+                "value": "1024",
+                "type": "MEMORY"
+            }
+        ],
+        "logConfiguration": {
+            "logDriver": "awslogs",
+            "options": {},
+            "secretOptions": []
+        },
+        "secrets": []
+    },
+    "timeout": {
+        "attemptDurationSeconds": 600
+    },
+    "tags": {
+        "Role": "webapp",
+        "Environment": "demo"
+    },
+    "propagateTags": false,
+    "platformCapabilities": [
+        "EC2"
+    ]
+}

package/aws-config/qa-api-deploy.json

@@ -0,0 +1,88 @@
+{
+    "jobDefinitionName": "qa-api-deploy",
+    "type": "container",
+    "parameters": {
+        "branch": "master"
+    },
+    "retryStrategy": {
+        "attempts": 1,
+        "evaluateOnExit": []
+    },
+    "containerProperties": {
+        "image": "147724020626.dkr.ecr.us-east-1.amazonaws.com/deployments",
+        "command": [
+            "node",
+            "index.mjs",
+            "--branch",
+            "Ref::branch",
+            "--disable_discord_notifications",
+            "Ref::disable_discord_notifications"
+        ],
+        "jobRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "executionRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "volumes": [],
+        "environment": [
+            {
+                "name": "AWS_REGION",
+                "value": "us-east-1"
+            },
+            {
+                "name": "REPO_KEY_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/configuration/wp-code-deploy-api"
+            },
+            {
+                "name": "ARTIFACT_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/builds/qa/api/source.zip"
+            },
+            {
+                "name": "REPO_URL",
+                "value": "git@github.com:WholesalePayroll/service-layer.git"
+            },
+            {
+                "name": "VPC_ID",
+                "value": "vpc-097160e523f14ec6f"
+            },
+            {
+                "name": "LAUNCH_TEMPLATE",
+                "value": "lt-0d81fd1772f186129,lt-02cd3cda3bdad1e9f"
+            },
+            {
+                "name": "SOURCE_PACKAGE",
+                "value": "api.zip"
+            },
+            {
+                "name": "GIT_SSH_COMMAND",
+                "value": "ssh -i ~/.ssh/wp-code-deploy -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+            }
+        ],
+        "mountPoints": [],
+        "ulimits": [],
+        "resourceRequirements": [
+            {
+                "value": "1",
+                "type": "VCPU"
+            },
+            {
+                "value": "1024",
+                "type": "MEMORY"
+            }
+        ],
+        "logConfiguration": {
+            "logDriver": "awslogs",
+            "options": {},
+            "secretOptions": []
+        },
+        "secrets": []
+    },
+    "timeout": {
+        "attemptDurationSeconds": 600
+    },
+    "tags": {
+        "Role": "api",
+        "Environment": "qa"
+    },
+    "propagateTags": false,
+    "platformCapabilities": [
+        "EC2"
+    ]
+}

package/aws-config/qa-webapp-deploy.json

@@ -0,0 +1,88 @@
+{
+    "jobDefinitionName": "qa-webapp-deploy",
+    "type": "container",
+    "parameters": {
+        "branch": "master"
+    },
+    "retryStrategy": {
+        "attempts": 1,
+        "evaluateOnExit": []
+    },
+    "containerProperties": {
+        "image": "147724020626.dkr.ecr.us-east-1.amazonaws.com/deployments",
+        "command": [
+            "node",
+            "index.mjs",
+            "--branch",
+            "Ref::branch",
+            "--disable_discord_notifications",
+            "Ref::disable_discord_notifications"
+        ],
+        "jobRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "executionRoleArn": "arn:aws:iam::147724020626:role/any-deployer-batch-task",
+        "volumes": [],
+        "environment": [
+            {
+                "name": "AWS_REGION",
+                "value": "us-east-1"
+            },
+            {
+                "name": "REPO_KEY_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/configuration/wp-code-deploy"
+            },
+            {
+                "name": "ARTIFACT_URL",
+                "value": "s3://wholesalepayroll-com-batch-deployments/builds/qa/web/source.zip"
+            },
+            {
+                "name": "REPO_URL",
+                "value": "git@github.com:WholesalePayroll/web-application.git"
+            },
+            {
+                "name": "VPC_ID",
+                "value": "vpc-097160e523f14ec6f"
+            },
+            {
+                "name": "LAUNCH_TEMPLATE",
+                "value": "lt-03169692c25c944ba"
+            },
+            {
+                "name": "SOURCE_PACKAGE",
+                "value": "webapp.zip"
+            },
+            {
+                "name": "GIT_SSH_COMMAND",
+                "value": "ssh -i ~/.ssh/wp-code-deploy -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+            }
+        ],
+        "mountPoints": [],
+        "ulimits": [],
+        "resourceRequirements": [
+            {
+                "value": "1",
+                "type": "VCPU"
+            },
+            {
+                "value": "1024",
+                "type": "MEMORY"
+            }
+        ],
+        "logConfiguration": {
+            "logDriver": "awslogs",
+            "options": {},
+            "secretOptions": []
+        },
+        "secrets": []
+    },
+    "timeout": {
+        "attemptDurationSeconds": 600
+    },
+    "tags": {
+        "Role": "webapp",
+        "Environment": "qa"
+    },
+    "propagateTags": false,
+    "platformCapabilities": [
+        "EC2"
+    ]
+}

package/config.mjs

@@ -0,0 +1,41 @@
+class Config {
+    constructor(args) {
+        this._artifactUrl = process.env.ARTIFACT_URL;
+        this._branch = args.branch;
+        this._disableDiscordNotifications = args.disable_discord_notifications;
+        this._launchTemplateId = process.env.LAUNCH_TEMPLATE.split(",");
+        this._repoSshKey = process.env.REPO_KEY_URL;
+        this._repoUrl = process.env.REPO_URL;
+        this._vpcId = process.env.VPC_ID;
+    }
+
+    get artifactUrl() {
+        return this._artifactUrl;
+    }
+
+    get branch() {
+        return this._branch;
+    }
+
+    get disableDiscordNotifications() {
+        return this._disableDiscordNotifications;
+    }
+
+    get launchTemplateIds() {
+        return this._launchTemplateId;
+    }
+
+    get repoSshKey() {
+        return this._repoSshKey;
+    }
+
+    get repoUrl() {
+        return this._repoUrl;
+    }
+
+    get vpcId() {
+        return this._vpcId;
+    }
+}
+
+export default Config;

package/index.mjs

@@ -0,0 +1,271 @@
+import {
+    EC2Client,
+    DescribeInstancesCommand,
+    DescribeSubnetsCommand,
+    RunInstancesCommand
+} from "@aws-sdk/client-ec2";
+
+import {
+    ExecuteStatementCommand,
+    RDSDataClient
+} from "@aws-sdk/client-rds-data";
+
+import {
+    SNSClient,
+    PublishCommand
+} from "@aws-sdk/client-sns";
+
+const argDefinition = [
+    { name: "branch", alias: "b", type: String },
+    { name: "disable_discord_notifications", type: String }
+];
+
+import commandLineArgs from "command-line-args";
+const commandOptions = commandLineArgs(argDefinition);
+
+console.dir(commandOptions);
+
+import Config from "./config.mjs";
+const config = new Config(commandOptions);
+
+import os from "os";
+import path from "path";
+import fs from "fs";
+import child_process from "child_process";
+
+const ec2Client = new EC2Client();
+const rdsClient = new RDSDataClient();
+const snsClient = new SNSClient();
+
+async function checkExistingInstance() {
+  // Fortunately AWS tags instances created from a launch template.
+  // So we just need to see if an existing instance can be found
+  for (const launchTemplateId of config.launchTemplateIds) {
+    const describeInstancesCommand = new DescribeInstancesCommand({
+        Filters: [
+            { Name: "tag:aws:ec2launchtemplate:id", Values: [launchTemplateId] },
+            { Name: "instance-state-name", Values: ["running"]}
+        ]
+    });
+
+    const response = await ec2Client.send(describeInstancesCommand);
+    console.log(JSON.stringify(response));
+
+    const reservation = (response.Reservations && response.Reservations[0]);
+    if (reservation && reservation.Instances && reservation.Instances.length > 0) {
+      return true;
+    }
+  }
+  return false;
+}
+
+async function createDeploymentArtifact() {
+    const sshPath = path.join(os.homedir(),".ssh");
+    const keyPath =  path.format({
+        dir: sshPath,
+        name: "wp-code-deploy"
+    });
+    console.log(`Writing private deployment key to ${keyPath}`);
+    fs.mkdirSync(sshPath);
+
+    downloadFileFromS3(config.repoSshKey, keyPath);
+    fs.chmodSync(keyPath, fs.constants.S_IRUSR | fs.constants.S_IWUSR);
+
+    console.log(`Fetching git repo ${config.repoUrl}...`);
+    const cloneArgs = ["clone", config.repoUrl, "code-staging"];
+    const gitResult = child_process.spawnSync(
+        "git",
+        cloneArgs,
+        { cwd: os.tmpdir() }
+    );
+    reportSpawnError(gitResult);
+
+    const codeWorkingDir = path.join(os.tmpdir(), "code-staging");
+
+    // now switch to the desired branch
+    console.log("changing branches...");
+    const gitBranchResult = child_process.spawnSync(
+        "git",
+        ["checkout", config.branch],
+        { cwd: codeWorkingDir }
+    );
+    reportSpawnError(gitBranchResult);
+
+    // Now run the zip builder
+    console.log("building zip....");
+    const zipBuildResult = child_process.spawnSync(
+        "npm",
+        ["run", "deploy-aws"],
+        { cwd: codeWorkingDir }
+    );
+    reportSpawnError(zipBuildResult);
+
+    // And finally upload the results to s3
+    const uploadResult = child_process.spawnSync(
+        "aws",
+        ["s3","cp", process.env.SOURCE_PACKAGE, config.artifactUrl],
+        { cwd: codeWorkingDir }
+    );
+    reportSpawnError(uploadResult);
+
+    // Publish to topic
+    const unused1 = await publishProgress({
+        type: "batch:job",
+        step: "source:build",
+        status: "success"
+    });
+    return true;
+}
+
+async function createInstance() {
+  const instancePromises = [];
+
+  for (const launchTemplateId of config.launchTemplateIds) {
+    // create the instance using the launch template specified
+    const commandInput = {};
+    commandInput.LaunchTemplate = {
+        LaunchTemplateId: launchTemplateId,
+        Version: "$Default"
+    };
+    commandInput.MaxCount = 1;
+    commandInput.MinCount = 1;
+
+    const subnet = await getRandomSubnetForVPC();
+    commandInput.SubnetId = subnet.SubnetId;
+
+    console.dir(`CommandInput => ${commandInput}`);
+    const runCommand = new RunInstancesCommand(commandInput);
+    const response = await ec2Client.send(runCommand);
+    const instance = response && response.Instances && response.Instances[0];
+    if (!instance) {
+        throw new Error("Unable to create new instance");
+    }
+    console.log(`instance started => ${instance.InstanceId}`);
+
+    const describeCommand = new DescribeInstancesCommand({
+        InstanceIds: [instance.InstanceId]
+    });
+    instancePromises.push(
+      new Promise((resolve, reject) => {
+        const interval = setInterval(
+            async () => {
+                console.log("polling for state....")
+                const describeResponse = await ec2Client.send(describeCommand);
+                const reservation = describeResponse.Reservations && describeResponse.Reservations[0];
+                const foundInstance = reservation.Instances && reservation.Instances[0];
+                console.log(`received ${foundInstance}`);
+                if (foundInstance && foundInstance.State.Name == "running") {
+                    const unused2 = await publishProgress({
+                        type: "batch:job",
+                        step: "instance:create",
+                        status: "success"
+                    });
+                    resolve(foundInstance);
+                }
+            },
+            30000
+        );
+      })
+    );
+  }
+  return Promise.all(instancePromises);
+}
+
+function downloadFileFromS3(s3Uri, outputPath) {
+    const downloadResult = child_process.spawnSync(
+        "aws",
+        ["s3","cp", s3Uri, outputPath]
+    );
+    reportSpawnError(downloadResult);
+}
+
+async function getRandomSubnetForVPC() {
+    console.log(`Fetching subnets for => ${config.vpcId}`);
+
+    const command = new DescribeSubnetsCommand({
+        Filters: [
+            { Name: "vpc-id", Values: [config.vpcId] },
+            { Name: "tag-key", Values: ["IncludeDeployments"] }
+        ]
+    });
+    const response = await ec2Client.send(command);
+    const subnets = response.Subnets;
+    subnets.forEach((subnet) => { console.log(subnet.SubnetId); });
+    const subnetIndex = Math.floor(Math.random() * subnets.length)
+    console.log(`Found subnet => ${subnets[subnetIndex].SubnetId}`);
+    return subnets[subnetIndex];
+}
+
+async function main() {
+    console.log("Checking for existing instance...");
+    const instanceCheck = await checkExistingInstance();
+    if (instanceCheck) {
+        const unused = await publishProgress({
+            type: "batch:job",
+            step: "instance:check",
+            status: "failed"
+        });
+        process.exit(-1);
+    }
+
+    await wakeupDatabaseCluster();
+    await createInstance();
+    await createDeploymentArtifact();
+}
+
+async function publishProgress(message) {
+    // Skip Discord notifications if disabled (GitHub Actions deployments)
+    // Parameter is optional - Discord Lambda doesn't pass it, only GitHub Actions does
+    if (config.disableDiscordNotifications === "true") {
+        console.log("Discord notifications disabled, skipping SNS publish:", message);
+        return;
+    }
+
+    const publishCommand = new PublishCommand({
+        Message: JSON.stringify(message),
+        TopicArn: "arn:aws:sns:us-east-1:147724020626:deployment-progress"
+    });
+    return snsClient.send(publishCommand);
+}
+
+function reportSpawnError(spawnResult) {
+    console.log(`spawn output: ${spawnResult.stdout.toString()}`);
+    console.log(`spawn error: ${spawnResult.stderr.toString()}`);
+    if (spawnResult.error) {
+        throw new Error(zipBuildResult.error)
+    }
+    if (spawnResult.status) {
+        throw new Error(zipBuildResult.status)
+    }
+}
+
+async function wakeupDatabaseCluster() {
+    // RJTODO: move some of these strings out of code
+    const command = new ExecuteStatementCommand({
+        continueAfterTimeout: false,
+        database: "payroll",
+        includeResultMetadata: true,
+        resourceArn: "arn:aws:rds:us-east-1:147724020626:cluster:staging-2022-02-12",
+        secretArn: "arn:aws:secretsmanager:us-east-1:147724020626:secret:staging/rds/connection-RuggqK",
+        sql: "select count(id) from default_config"
+    });
+    try {
+        const response = await rdsClient.send(command);
+    } catch (error) {
+        console.log("error while booting rds");
+        console.log(error);
+    }
+}
+
+try {
+    await main();
+    process.exit(0);
+} catch (err) {
+    const unused = await publishProgress({
+        type: "batch:job",
+        step: "unknown",
+        status: "failed"
+    });
+    console.error(err);
+    process.exit(1);
+}

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "adhoc-deploy",
+  "version": "1.1.0",
+  "scripts": {
+    "bootstrap": "cd aws-config && ./bootstrap.sh",
+    "build": "npm run clean && npx tsc",
+    "clean": "rimraf dist",
+    "ecr:build": "docker build --platform linux/amd64 -t deployments .",
+    "ecr:login": "aws ecr get-login-password --region us-east-1 --profile wp-role | docker login --username AWS --password-stdin 147724020626.dkr.ecr.us-east-1.amazonaws.com",
+    "ecr:publish": "docker tag deployments:latest 147724020626.dkr.ecr.us-east-1.amazonaws.com/deployments:latest && docker push 147724020626.dkr.ecr.us-east-1.amazonaws.com/deployments:latest",
+    "deploy": "npm run ecr:build && npm run ecr:login && npm run ecr:publish",
+    "deploy:verify": "aws ecr describe-images --repository-name deployments --image-ids imageTag=latest --profile wp-role",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "@aws-sdk/client-ec2": "^3.49.0",
+    "@aws-sdk/client-rds-data": "^3.72.0",
+    "@aws-sdk/client-s3": "^3.50.0",
+    "@aws-sdk/client-sns": "^3.51.0",
+    "@aws-sdk/client-ssm": "^3.50.0",
+    "command-line-args": "^5.2.1",
+    "dotenv": "^16.0.0"
+  }
+}