@fishawack/lab-env 5.7.0-beta.2 → 5.7.0-beta.4

package/commands/create/cmds/rekey.js

@@ -0,0 +1,218 @@
+const _ = require("../../../globals.js");
+const inquirer = require("inquirer");
+const aws = require("../services/aws/index.js");
+const outputCredentials = require("../libs/output-credentials.js");
+const { misc } = require("../libs/vars.js");
+const {
+    setAWSClientDefaults,
+} = require("../../../commands/create/services/aws/misc.js");
+const parallelRunner = require("../libs/parallel-runner.js");
+const resolveOperator = require("../libs/resolve-operator.js");
+
+module.exports = [
+    "rekey",
+    _.config.preset === "devops"
+        ? "Rotates AWS access keys and credentials"
+        : false,
+    () => {},
+    async () => {
+        let operator = await resolveOperator();
+
+        let users = [];
+        let clients = [];
+        let sendEmail = false;
+
+        let answer = await inquirer.prompt([
+            {
+                type: "confirm",
+                name: "check",
+                message: `Rotate keys for all users`,
+                default: "Y",
+            },
+        ]);
+
+        if (answer.check) {
+            users = _.config.users.map((d) => d.username);
+        } else {
+            answer = await inquirer.prompt([
+                {
+                    type: "checkbox",
+                    name: "users",
+                    message: "Select users",
+                    choices: _.config.users.map((d) => d.username),
+                },
+            ]);
+
+            users = answer.users;
+        }
+
+        answer = await inquirer.prompt([
+            {
+                type: "confirm",
+                name: "check",
+                message: `Rotate keys for all clients`,
+                default: "Y",
+            },
+        ]);
+
+        if (answer.check) {
+            clients = aws.clients;
+        } else {
+            answer = await inquirer.prompt([
+                {
+                    type: "checkbox",
+                    name: "clients",
+                    message: "Select clients",
+                    choices: aws.clients,
+                },
+            ]);
+
+            clients = answer.clients;
+        }
+
+        answer = await inquirer.prompt([
+            {
+                type: "confirm",
+                name: "check",
+                message: `Would you like to email out the credentials after creation (requires office365 credentials in ~/targets/misc.json)?`,
+                default: "Y",
+            },
+        ]);
+
+        sendEmail = answer.check;
+
+        // Check for SES email credentials if user wants to send emails
+        if (sendEmail && !misc.nodemailer?.AWSSES?.username) {
+            answer = await inquirer.prompt([
+                {
+                    type: "confirm",
+                    name: "check",
+                    message:
+                        "No email credentials found, create them via AWS SES?",
+                    default: "Y",
+                },
+            ]);
+
+            if (answer.check) {
+                try {
+                    const creds = await aws.ses.ensureOperatorSESCredentials(
+                        operator.username,
+                    );
+
+                    if (creds) {
+                        aws.ses.updateMiscNodemailer(creds);
+                        console.log(
+                            "\n  \u2713 SES email credentials created and saved to misc.json\n",
+                        );
+                    } else {
+                        console.log(
+                            "\n  \u26a0 SES IAM user exists but keys were already created. Rotate with `fw rekey` to generate new credentials.\n",
+                        );
+                    }
+                } catch (err) {
+                    console.log(
+                        `\n  \u26a0 Failed to create SES credentials: ${err.message}\n`,
+                    );
+                }
+            }
+        }
+
+        // Separate non-operator users and operator
+        // Operator MUST be processed last per account — rotating the
+        // operator's key invalidates the credentials used for all API calls
+        let nonOperatorUsers = users.filter((u) => u !== operator.username);
+        let includeOperator = users.includes(operator.username);
+
+        let credentials = {};
+
+        // Process non-operator users in parallel
+        if (nonOperatorUsers.length) {
+            const batches = clients.map((client) => ({
+                users: nonOperatorUsers,
+                client,
+                fn: async (user) => {
+                    setAWSClientDefaults(client);
+
+                    await aws.iam.removeIAMUser(`fw-automation-${user}`);
+
+                    return aws.iam.createFWIAMUser(
+                        `fw-automation-${user}`,
+                        _.config.users.find((d) => d.username === user)
+                            .permissions,
+                    );
+                },
+            }));
+
+            const resultsMap = await parallelRunner(batches, {
+                concurrency: 10,
+                verb: "Rotate credentials",
+            });
+
+            for (const [user, entries] of resultsMap) {
+                credentials[user] = {};
+
+                for (const { client, res } of entries) {
+                    credentials[user][client] = {
+                        key:
+                            (res.AccessKey && res.AccessKey.AccessKeyId) ||
+                            res.AccessKeyMetadata[0].AccessKeyId,
+                        secret:
+                            (res.AccessKey && res.AccessKey.SecretAccessKey) ||
+                            "** secret **",
+                    };
+                }
+            }
+        }
+
+        // Process operator sequentially — must be last per account
+        if (includeOperator) {
+            const operatorBatches = clients.map((client) => ({
+                users: [operator.username],
+                client,
+                fn: async (user) => {
+                    setAWSClientDefaults(client);
+
+                    return aws.iam.rotateFWIAMUser(
+                        `fw-automation-${user}`,
+                        operator.permissions,
+                    );
+                },
+            }));
+
+            const operatorResults = await parallelRunner(operatorBatches, {
+                concurrency: 1,
+                verb: "Rotate operator credentials",
+            });
+
+            for (const [user, entries] of operatorResults) {
+                credentials[user] = {};
+
+                for (const { client, res } of entries) {
+                    credentials[user][client] = {
+                        key: res.AccessKey.AccessKeyId,
+                        secret: res.AccessKey.SecretAccessKey,
+                    };
+                }
+            }
+        }
+
+        // Rotate SES email credentials (after operator keys so fishawack creds are still valid)
+        if (misc.nodemailer?.AWSSES?.username) {
+            try {
+                const creds = await aws.ses.rotateOperatorSESCredentials(
+                    operator.username,
+                );
+                aws.ses.updateMiscNodemailer(creds);
+                console.log(
+                    "\n  \u2713 SES email credentials rotated and saved to misc.json\n",
+                );
+            } catch (err) {
+                console.log(
+                    `\n  \u26a0 Failed to rotate SES credentials: ${err.message}\n`,
+                );
+            }
+        }
+
+        await outputCredentials(credentials, sendEmail);
+    },
+];

package/commands/create/libs/output-credentials.js

@@ -0,0 +1,59 @@
+const _ = require("../../../globals.js");
+const utilities = require("../libs/utilities");
+const email = require("../services/email.js");
+const { readFileSync } = require("fs");
+const htmlEmail = readFileSync(`${__dirname}/../templates/credentials.html`, {
+    encoding: "utf8",
+});
+
+module.exports = async function outputCredentials(credentials, sendEmail) {
+    let output = "";
+
+    for (var user in credentials) {
+        output += utilities.colorize(`\n${user}\n`, "title");
+
+        let outputUser = "";
+        for (var client in credentials[user]) {
+            outputUser += `\n[${client}]\n`;
+            outputUser += `aws_access_key_id = ${credentials[user][client].key}\n`;
+            outputUser += `aws_secret_access_key = ${credentials[user][client].secret}\n`;
+        }
+        output += outputUser;
+
+        if (sendEmail) {
+            await email.send(
+                _.config.users.find((d) => d.username === user).email,
+                "New AWS Keys",
+                `${htmlEmail}<pre><code>${outputUser.replace(/\n/g, "<br>")}</code></pre>`,
+            );
+        }
+    }
+
+    console.log(output);
+
+    // Output base64-encoded credentials for the CI runner user
+    let runner =
+        _.config.users.find((d) => d.runner) ||
+        _.config.users.find((d) => d.username === "aws-runner");
+
+    if (runner && credentials[runner.username]) {
+        let runnerCreds = "";
+
+        for (var c in credentials[runner.username]) {
+            runnerCreds += `[${c}]\n`;
+            runnerCreds += `aws_access_key_id = ${credentials[runner.username][c].key}\n`;
+            runnerCreds += `aws_secret_access_key = ${credentials[runner.username][c].secret}\n\n`;
+        }
+
+        let base64 = Buffer.from(runnerCreds.trim()).toString("base64");
+
+        console.log(
+            utilities.colorize("\n CI Runner Base64 Credentials\n", "title"),
+        );
+        console.log(
+            " Use this base64 string as the CI variable for pipeline AWS authentication.\n",
+        );
+        console.log(base64);
+        console.log("");
+    }
+};

package/commands/create/libs/parallel-runner.js

@@ -0,0 +1,204 @@
+const logUpdate = require("log-update");
+const chalk = require("chalk");
+const { Spinner } = require("./utilities");
+
+const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+
+function formatElapsed(ms) {
+    const secs = Math.floor(ms / 1000);
+    if (secs < 60) return `${secs}s`;
+    const mins = Math.floor(secs / 60);
+    return `${mins}m${secs % 60}s`;
+}
+
+/**
+ * Runs batches of tasks sequentially (one batch per client/account) with
+ * parallel execution within each batch. Users are processed in groups of
+ * `concurrency` — each group completes all batches before the next starts.
+ *
+ * @param {Array<{ users: string[], client: string, fn: (user: string) => Promise<any> }>} batches
+ * @param {{ concurrency?: number, verb?: string }} options
+ * @returns {Promise<Map<string, Array<{ client: string, res: any }>>>} results keyed by user
+ */
+module.exports = async function parallelRunner(
+    batches,
+    { concurrency = 5, verb = "Processing" } = {},
+) {
+    const allUsers = [...new Set(batches.flatMap((b) => b.users))];
+    const totalBatches = batches.length;
+    const results = new Map(allUsers.map((u) => [u, []]));
+    const startTime = Date.now();
+    let frame = 0;
+    let throttleCount = 0;
+
+    // Silence Spinner.prototype.simple — track throttling via slow calls
+    const originalSimple = Spinner.prototype.simple;
+    Spinner.prototype.simple = async (_message, action) => {
+        const t0 = Date.now();
+        const res = await action();
+        // If a single API call took > 3s it likely hit a retry/backoff
+        if (Date.now() - t0 > 3000) {
+            throttleCount++;
+        }
+        return res;
+    };
+
+    // Chunk users into groups of `concurrency`
+    const groups = [];
+    for (let i = 0; i < allUsers.length; i += concurrency) {
+        groups.push(allUsers.slice(i, i + concurrency));
+    }
+
+    // Track state for users (populated as groups start)
+    const groupStates = new Map();
+    const errors = [];
+
+    // Render loop — header + one line per user
+    const render = setInterval(() => {
+        frame = (frame + 1) % SPINNER_FRAMES.length;
+        const elapsed = formatElapsed(Date.now() - startTime);
+
+        // Header line with timing and throttle info
+        let header = chalk.dim(`  ${elapsed} elapsed`);
+        if (throttleCount > 0) {
+            header += chalk.yellow(
+                ` │ ⚠ ${throttleCount} throttled call${throttleCount > 1 ? "s" : ""} (retrying automatically)`,
+            );
+        }
+
+        const lines = [header];
+
+        for (const user of allUsers) {
+            const state = groupStates.get(user);
+
+            if (!state) {
+                lines.push(
+                    chalk.dim(`  - ${verb}: ${user} (0/${totalBatches})`),
+                );
+                continue;
+            }
+
+            if (state.status === "active") {
+                const taskElapsed = Date.now() - state.taskStart;
+                let suffix = "";
+                if (taskElapsed > 5000) {
+                    suffix = chalk.yellow(` ${formatElapsed(taskElapsed)}`);
+                }
+                lines.push(
+                    chalk.cyan(
+                        `  ${SPINNER_FRAMES[frame]} ${verb}: ${user} - ${state.client} (${state.batchesDone + 1}/${totalBatches})`,
+                    ) + suffix,
+                );
+            } else if (state.status === "error") {
+                lines.push(
+                    chalk.red(
+                        `  ✖ ${verb}: ${user} - ${state.client} (${state.batchesDone}/${totalBatches}) ${state.errorMsg}`,
+                    ),
+                );
+            } else if (state.batchesDone === totalBatches) {
+                lines.push(
+                    chalk.green(
+                        `  ✓ ${verb}: ${user} (${totalBatches}/${totalBatches})`,
+                    ),
+                );
+            } else {
+                lines.push(
+                    chalk.dim(
+                        `  - ${verb}: ${user} (${state.batchesDone}/${totalBatches})`,
+                    ),
+                );
+            }
+        }
+
+        logUpdate(lines.join("\n"));
+    }, 80);
+
+    // Process each group to full completion before starting the next
+    for (const group of groups) {
+        for (const u of group) {
+            groupStates.set(u, {
+                client: "",
+                status: "pending",
+                batchesDone: 0,
+                taskStart: 0,
+                hasError: false,
+                errorMsg: "",
+            });
+        }
+
+        for (const batch of batches) {
+            const groupUsers = group.filter((u) => batch.users.includes(u));
+
+            await Promise.all(
+                groupUsers.map(async (user) => {
+                    const state = groupStates.get(user);
+                    state.client = batch.client;
+                    state.status = "active";
+                    state.taskStart = Date.now();
+
+                    try {
+                        const res = await batch.fn(user);
+                        results.get(user).push({ client: batch.client, res });
+                        state.status = "done";
+                    } catch (err) {
+                        state.status = "error";
+                        state.hasError = true;
+                        state.errorMsg = err.message || String(err);
+                        errors.push({ user, client: batch.client, error: err });
+                    } finally {
+                        state.batchesDone++;
+                    }
+                }),
+            );
+        }
+    }
+
+    clearInterval(render);
+
+    // Final render
+    const elapsed = formatElapsed(Date.now() - startTime);
+    let header = chalk.dim(`  ${elapsed} elapsed`);
+    if (throttleCount > 0) {
+        header += chalk.yellow(
+            ` │ ⚠ ${throttleCount} throttled call${throttleCount > 1 ? "s" : ""}`,
+        );
+    }
+
+    const lines = [header];
+    for (const user of allUsers) {
+        const state = groupStates.get(user);
+        if (state.hasError) {
+            lines.push(
+                chalk.red(
+                    `  ✖ ${verb}: ${user} (${state.batchesDone}/${totalBatches}) ${state.errorMsg}`,
+                ),
+            );
+        } else {
+            lines.push(
+                chalk.green(
+                    `  ✓ ${verb}: ${user} (${totalBatches}/${totalBatches})`,
+                ),
+            );
+        }
+    }
+    logUpdate(lines.join("\n"));
+    logUpdate.done();
+
+    // Restore spinner
+    Spinner.prototype.simple = originalSimple;
+
+    if (errors.length) {
+        const summary = errors
+            .map(
+                (e) => `${e.user} (${e.client}): ${e.error.message || e.error}`,
+            )
+            .join("\n");
+        const err = new Error(
+            `${errors.length} task${errors.length > 1 ? "s" : ""} failed:\n${summary}`,
+        );
+        err.errors = errors;
+        throw err;
+    }
+
+    return results;
+};

package/commands/create/libs/resolve-operator.js

@@ -0,0 +1,59 @@
+const _ = require("../../../globals.js");
+const inquirer = require("inquirer");
+const os = require("os");
+const { writeFileSync } = require("fs");
+
+module.exports = async function resolveOperator() {
+    let operators = _.config.users.filter((d) => d.operator);
+
+    if (operators.length === 1) {
+        return operators[0];
+    }
+
+    let candidates =
+        operators.length > 1
+            ? operators
+            : _.config.users.filter(
+                  (d) =>
+                      d.permissions && d.permissions.includes("manage-users"),
+              );
+
+    if (!candidates.length) {
+        throw new Error(
+            "No users with manage-users permission found. A user with IAMFullAccess is required to rotate keys.",
+        );
+    }
+
+    let message =
+        operators.length > 1
+            ? "Multiple operators found. Which user are you?"
+            : "No operator configured. Which user are you?";
+
+    let answer = await inquirer.prompt([
+        {
+            type: "list",
+            name: "operator",
+            message,
+            choices: candidates.map((d) => d.username),
+        },
+    ]);
+
+    _.config.users.forEach((d) => {
+        delete d.operator;
+    });
+
+    let selected = _.config.users.find((d) => d.username === answer.operator);
+    selected.operator = true;
+
+    writeFileSync(
+        `${os.homedir()}/.lab-env`,
+        JSON.stringify(_.config, null, 4),
+        { encoding: "utf8" },
+    );
+
+    console.log(
+        `\nOperator set to "${selected.username}" and saved to ~/.lab-env\n`,
+    );
+
+    return selected;
+};

package/commands/create/libs/utilities.js

@@ -28,6 +28,7 @@ const colorize = (module.exports.colorize = (str, type) => {
 });
 
 let latestSpinner = null;
+let spinnerQueue = Promise.resolve();
 
 module.exports.Spinner = class Spinner {
     constructor(startMessage, simple = false) {
@@ -67,31 +68,42 @@ module.exports.Spinner = class Spinner {
     }
 
     simple(message, action) {
+        const ready = spinnerQueue;
+        let release;
+        spinnerQueue = new Promise((r) => {
+            release = r;
+        });
+
         // eslint-disable-next-line no-async-promise-executor
         return new Promise(async (resolve, reject) => {
+            await ready;
+
             let instance = new this.constructor(message, true);
             let res;
 
             try {
-                res = await action();
+                res = await action(instance);
 
                 instance.ora.succeed();
             } catch (e) {
                 instance.ora.fail();
+                release();
                 reject(e);
+                return;
             }
 
+            release();
             resolve(res);
         });
     }
 
-    ping() {
-        latestSpinner.ora.color =
-            latestSpinner.ora.color === "cyan" ? "magenta" : "cyan";
+    ping(spinner) {
+        const target = spinner || latestSpinner;
+        target.ora.color = target.ora.color === "cyan" ? "magenta" : "cyan";
         let message = " - check: ";
-        let split = latestSpinner.ora.text.split(message);
+        let split = target.ora.text.split(message);
         let iteration = (split[1] && +split[1].split(":")[0]) || 0;
-        latestSpinner.ora.text = `${split[0]}${message}${iteration + 1}`;
+        target.ora.text = `${split[0]}${message}${iteration + 1}`;
     }
 };
 
@@ -124,6 +136,8 @@ module.exports.nameSafe = (name, service = "s3") => {
         maxLength = 40;
     } else if (service === "os") {
         maxLength = 28;
+    } else if (service === "cache") {
+        maxLength = 40;
     }
 
     safe = safe.substring(0, maxLength - suffix.length - prefix.length);
@@ -146,13 +160,13 @@ module.exports.copyToClipboard = (text) => {
     }
 };
 
-module.exports.poll = async (cb, resolve, reject) => {
+module.exports.poll = async (cb, resolve, reject, spinner) => {
     let res;
 
     do {
         await new Promise((resolve) => setTimeout(() => resolve(), 5000));
 
-        await module.exports.Spinner.prototype.ping();
+        await module.exports.Spinner.prototype.ping(spinner);
 
         res = await cb();
 
@@ -163,3 +177,52 @@ module.exports.poll = async (cb, resolve, reject) => {
 
     return res;
 };
+
+module.exports.pollAll = async (
+    services,
+    message = "Waiting for services to provision",
+) => {
+    const instance = new module.exports.Spinner(
+        `${message} - checking (${services[0].name})`,
+        true,
+    );
+
+    const results = new Array(services.length).fill(null);
+    const pending = services.map((s, i) => ({ ...s, index: i, checks: 0 }));
+    let cursor = 0;
+
+    while (pending.length > 0) {
+        const service = pending[cursor % pending.length];
+        service.checks++;
+
+        instance.ora.text = `${message} - checking (${service.name}) - check: ${service.checks}`;
+
+        await new Promise((r) => setTimeout(r, 5000));
+
+        instance.ora.color = instance.ora.color === "cyan" ? "magenta" : "cyan";
+
+        const res = await service.cb();
+
+        if (service.reject && service.reject(res)) {
+            instance.ora.fail();
+            throw new Error(
+                `Polling received reject response for ${service.name}`,
+            );
+        }
+
+        if (!service.shouldContinue(res)) {
+            results[service.index] = res;
+            pending.splice(cursor % pending.length, 1);
+
+            if (pending.length > 0) {
+                cursor = cursor % pending.length;
+            }
+        } else {
+            cursor++;
+        }
+    }
+
+    instance.ora.succeed();
+
+    return results;
+};