@fishawack/lab-env 5.7.0-beta.4 → 5.7.0-beta.6

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 ## Changelog
 
+### 5.7.0-beta.6 (2026-07-03)
+
+#### Features
+
+* **devops:** misc.json setup, SES rotation recovery, runner ordering ([3c6ec70](https://bitbucket.org/fishawackdigital/lab-env/commits/3c6ec70df5e54ad4683a9a5b4e8e3accb681121d))
+
+#### Bug Fixes
+
+* add timeout for aws key calls themselves ([389444c](https://bitbucket.org/fishawackdigital/lab-env/commits/389444cf7c0c61273b8dc7edf2d0d6695a1047a7))
+* concurrency set back to 5 for more stable key commands ([a5e7382](https://bitbucket.org/fishawackdigital/lab-env/commits/a5e7382b9731ad2a5a60110a54e27ebe4af45391))
+
+### 5.7.0-beta.5 (2026-07-03)
+
+#### Bug Fixes
+
+* allow rekey outside of projects ([94b511b](https://bitbucket.org/fishawackdigital/lab-env/commits/94b511b629fbf80b842a9ab5db9f3671f1f603f0))
+* retry on rate limits and aws errors on key commands ([0244221](https://bitbucket.org/fishawackdigital/lab-env/commits/0244221b8a263256f015380280024c862213b885))
+* safe check for misc bitbucket vars ([cee035c](https://bitbucket.org/fishawackdigital/lab-env/commits/cee035c25f927d1b5f55c00f10fa35f08733e4bf))
+
 ### 5.7.0-beta.4 (2026-07-03)
 
 #### Bug Fixes

package/commands/create/cmds/dekey.js

@@ -75,7 +75,7 @@ module.exports = [
         }));
 
         await parallelRunner(batches, {
-            concurrency: 10,
+            concurrency: 5,
             verb: "Remove credentials",
         });
     },

package/commands/create/cmds/diagnose.js

@@ -142,6 +142,16 @@ module.exports = [
                 }
             }
 
+            if (preset === "devops") {
+                while (!(await test.targets())) {
+                    await guide.targets(argv);
+                }
+
+                while (!(await test.misc())) {
+                    await guide.misc(argv);
+                }
+            }
+
             // Update diagnosis version in ~/.lab-env config file
             let spinner = new Spinner(
                 `Writing options to ${path.join(os.homedir(), "", ".lab-env")}`,

package/commands/create/cmds/key.js

@@ -95,18 +95,32 @@ module.exports = [
 
             if (answer.check) {
                 try {
-                    const creds = await aws.ses.ensureOperatorSESCredentials(
+                    let creds = await aws.ses.ensureOperatorSESCredentials(
                         operator.username,
                     );
 
+                    if (!creds) {
+                        answer = await inquirer.prompt([
+                            {
+                                type: "confirm",
+                                name: "check",
+                                message:
+                                    "SES IAM user exists but secret is not stored locally. Rotate credentials to recapture?",
+                                default: "Y",
+                            },
+                        ]);
+
+                        if (answer.check) {
+                            creds = await aws.ses.rotateOperatorSESCredentials(
+                                operator.username,
+                            );
+                        }
+                    }
+
                     if (creds) {
                         aws.ses.updateMiscNodemailer(creds);
                         console.log(
-                            "\n  ✓ SES email credentials created and saved to misc.json\n",
-                        );
-                    } else {
-                        console.log(
-                            "\n  ⚠ SES IAM user exists but keys were already created. Rotate with `fw rekey` to generate new credentials.\n",
+                            "\n  ✓ SES email credentials saved to misc.json\n",
                         );
                     }
                 } catch (err) {
@@ -133,7 +147,7 @@ module.exports = [
         }));
 
         const resultsMap = await parallelRunner(batches, {
-            concurrency: 10,
+            concurrency: 5,
             verb: "Create credentials",
         });
 

package/commands/create/cmds/rekey.js

@@ -95,18 +95,32 @@ module.exports = [
 
             if (answer.check) {
                 try {
-                    const creds = await aws.ses.ensureOperatorSESCredentials(
+                    let creds = await aws.ses.ensureOperatorSESCredentials(
                         operator.username,
                     );
 
+                    if (!creds) {
+                        answer = await inquirer.prompt([
+                            {
+                                type: "confirm",
+                                name: "check",
+                                message:
+                                    "SES IAM user exists but secret is not stored locally. Rotate credentials to recapture?",
+                                default: "Y",
+                            },
+                        ]);
+
+                        if (answer.check) {
+                            creds = await aws.ses.rotateOperatorSESCredentials(
+                                operator.username,
+                            );
+                        }
+                    }
+
                     if (creds) {
                         aws.ses.updateMiscNodemailer(creds);
                         console.log(
-                            "\n  \u2713 SES email credentials created and saved to misc.json\n",
-                        );
-                    } else {
-                        console.log(
-                            "\n  \u26a0 SES IAM user exists but keys were already created. Rotate with `fw rekey` to generate new credentials.\n",
+                            "\n  \u2713 SES email credentials saved to misc.json\n",
                         );
                     }
                 } catch (err) {
@@ -117,10 +131,19 @@ module.exports = [
             }
         }
 
-        // Separate non-operator users and operator
-        // Operator MUST be processed last per account — rotating the
-        // operator's key invalidates the credentials used for all API calls
-        let nonOperatorUsers = users.filter((u) => u !== operator.username);
+        // Separate users into: regular, runners, operator
+        // Processing order: regular users → runners → operator (last)
+        let runners = users.filter((u) => {
+            let config = _.config.users.find((d) => d.username === u);
+            return (
+                u !== operator.username &&
+                config &&
+                (config.runner || u === "aws-runner")
+            );
+        });
+        let nonOperatorUsers = users.filter(
+            (u) => u !== operator.username && !runners.includes(u),
+        );
         let includeOperator = users.includes(operator.username);
 
         let credentials = {};
@@ -144,7 +167,7 @@ module.exports = [
             }));
 
             const resultsMap = await parallelRunner(batches, {
-                concurrency: 10,
+                concurrency: 5,
                 verb: "Rotate credentials",
             });
 
@@ -164,6 +187,45 @@ module.exports = [
             }
         }
 
+        // Process runner users as a group (after regular, before operator)
+        if (runners.length) {
+            const runnerBatches = clients.map((client) => ({
+                users: runners,
+                client,
+                fn: async (user) => {
+                    setAWSClientDefaults(client);
+
+                    await aws.iam.removeIAMUser(`fw-automation-${user}`);
+
+                    return aws.iam.createFWIAMUser(
+                        `fw-automation-${user}`,
+                        _.config.users.find((d) => d.username === user)
+                            .permissions,
+                    );
+                },
+            }));
+
+            const runnerResults = await parallelRunner(runnerBatches, {
+                concurrency: 5,
+                verb: "Rotate runner credentials",
+            });
+
+            for (const [user, entries] of runnerResults) {
+                credentials[user] = {};
+
+                for (const { client, res } of entries) {
+                    credentials[user][client] = {
+                        key:
+                            (res.AccessKey && res.AccessKey.AccessKeyId) ||
+                            res.AccessKeyMetadata[0].AccessKeyId,
+                        secret:
+                            (res.AccessKey && res.AccessKey.SecretAccessKey) ||
+                            "** secret **",
+                    };
+                }
+            }
+        }
+
         // Process operator sequentially — must be last per account
         if (includeOperator) {
             const operatorBatches = clients.map((client) => ({

package/commands/create/libs/parallel-runner.js

@@ -3,6 +3,34 @@ const chalk = require("chalk");
 const { Spinner } = require("./utilities");
 
 const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+const MAX_RETRIES = 15;
+const INITIAL_DELAY = 2000;
+const MAX_DELAY = 60000;
+const TASK_TIMEOUT = 120000; // 2 min hard timeout per attempt — prevents hung TCP connections
+
+const RETRYABLE_ERRORS = [
+    "Throttling",
+    "ThrottlingException",
+    "TooManyRequestsException",
+    "RequestLimitExceeded",
+    "ServiceUnavailable",
+    "InternalFailure",
+    "RequestTimeout",
+    "RequestTimeoutException",
+];
+
+function isRetryable(err) {
+    if (
+        RETRYABLE_ERRORS.includes(err.name) ||
+        RETRYABLE_ERRORS.includes(err.Code)
+    ) {
+        return true;
+    }
+
+    const statusCode = err.$metadata?.httpStatusCode;
+
+    return statusCode === 429 || statusCode === 503 || statusCode === 500;
+}
 
 function formatElapsed(ms) {
     const secs = Math.floor(ms / 1000);
@@ -78,7 +106,20 @@ module.exports = async function parallelRunner(
                 continue;
             }
 
-            if (state.status === "active") {
+            if (state.status === "retrying") {
+                const remaining = Math.max(
+                    0,
+                    Math.ceil(
+                        (state.retryDelay - (Date.now() - state.retryStart)) /
+                            1000,
+                    ),
+                );
+                lines.push(
+                    chalk.yellow(
+                        `  ↻ ${verb}: ${user} - ${state.client} (${state.batchesDone + 1}/${totalBatches}) retry ${state.retryAttempt}/${MAX_RETRIES} ${remaining}s`,
+                    ),
+                );
+            } else if (state.status === "active") {
                 const taskElapsed = Date.now() - state.taskStart;
                 let suffix = "";
                 if (taskElapsed > 5000) {
@@ -130,24 +171,75 @@ module.exports = async function parallelRunner(
             const groupUsers = group.filter((u) => batch.users.includes(u));
 
             await Promise.all(
-                groupUsers.map(async (user) => {
+                groupUsers.map(async (user, userIndex) => {
                     const state = groupStates.get(user);
                     state.client = batch.client;
-                    state.status = "active";
                     state.taskStart = Date.now();
 
-                    try {
-                        const res = await batch.fn(user);
-                        results.get(user).push({ client: batch.client, res });
-                        state.status = "done";
-                    } catch (err) {
-                        state.status = "error";
-                        state.hasError = true;
-                        state.errorMsg = err.message || String(err);
-                        errors.push({ user, client: batch.client, error: err });
-                    } finally {
-                        state.batchesDone++;
+                    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+                        state.status = "active";
+                        state.taskStart = Date.now();
+
+                        try {
+                            let timeoutHandle;
+                            const res = await Promise.race([
+                                batch.fn(user),
+                                new Promise((_, reject) => {
+                                    timeoutHandle = setTimeout(
+                                        () =>
+                                            reject(
+                                                Object.assign(
+                                                    new Error(
+                                                        "Request timed out after 2 minutes — will retry",
+                                                    ),
+                                                    { name: "RequestTimeout" },
+                                                ),
+                                            ),
+                                        TASK_TIMEOUT,
+                                    );
+                                }),
+                            ]).finally(() => clearTimeout(timeoutHandle));
+                            results
+                                .get(user)
+                                .push({ client: batch.client, res });
+                            state.status = "done";
+                            state.retryAttempt = 0;
+                            break;
+                        } catch (err) {
+                            if (!isRetryable(err) || attempt === MAX_RETRIES) {
+                                state.status = "error";
+                                state.hasError = true;
+                                state.errorMsg = err.message || String(err);
+                                errors.push({
+                                    user,
+                                    client: batch.client,
+                                    error: err,
+                                });
+                                break;
+                            }
+
+                            let delay = Math.min(
+                                INITIAL_DELAY * Math.pow(2, attempt),
+                                MAX_DELAY,
+                            );
+                            // Stagger retries per user to avoid thundering herd
+                            delay +=
+                                Math.floor(Math.random() * delay * 0.3) +
+                                userIndex * 1000;
+
+                            state.status = "retrying";
+                            state.retryAttempt = attempt + 1;
+                            state.retryDelay = delay;
+                            state.retryStart = Date.now();
+                            throttleCount++;
+
+                            await new Promise((resolve) =>
+                                setTimeout(resolve, delay),
+                            );
+                        }
                     }
+
+                    state.batchesDone++;
                 }),
             );
         }

package/commands/create/libs/vars.js

@@ -68,17 +68,17 @@ const bitbucketApi = (module.exports.bitbucketApi =
     "https://api.bitbucket.org/2.0/repositories");
 
 module.exports.apis = {
-    bbWorkspaceAPI: `${bitbucketApi}/${misc.bitbucket.workspace}`,
+    bbWorkspaceAPI: `${bitbucketApi}/${misc?.bitbucket?.workspace}`,
 };
 
 // URLs
 module.exports.urls = {
-    bitbucketSSH: `git@bitbucket.org:${misc.bitbucket.workspace}`,
+    bitbucketSSH: `git@bitbucket.org:${misc?.bitbucket?.workspace}`,
 };
 
 // Request Headers
 module.exports.headers = {
-    bbHeaders: encode(misc.bitbucket.username, misc.bitbucket.token),
+    bbHeaders: encode(misc?.bitbucket?.username, misc?.bitbucket?.token),
 };
 
 // Required global node modules

package/commands/create/services/aws/ses.js

@@ -176,7 +176,15 @@ const MISC_PATH = `${os.homedir()}/targets/misc.json`;
 module.exports.updateMiscNodemailer = (creds) => {
     const { misc } = require("../../libs/vars");
 
-    const file = JSON.parse(readFileSync(MISC_PATH, { encoding: "utf8" }));
+    let file = {};
+
+    try {
+        file = JSON.parse(readFileSync(MISC_PATH, { encoding: "utf8" }));
+    } catch {
+        const { mkdirSync } = require("fs");
+        mkdirSync(require("path").dirname(MISC_PATH), { recursive: true });
+        file = { bitbucket: {}, gitlab: {} };
+    }
 
     file.nodemailer = {
         driver: "AWSSES",

package/globals.js

@@ -279,6 +279,7 @@ if (composer && composer.require && composer.require["laravel/framework"]) {
             args[0] !== "new" &&
             args[0] !== "key" &&
             args[0] !== "dekey" &&
+            args[0] !== "rekey" &&
             args[0] !== "lint" &&
             args[0] !== "workspace" &&
             args[0] !== "prune"

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@fishawack/lab-env",
-    "version": "5.7.0-beta.4",
+    "version": "5.7.0-beta.6",
     "description": "Docker manager for FW",
     "main": "cli.js",
     "scripts": {