hereya-cli 0.103.0 → 0.105.0

package/dist/lib/executor-start/job-serialization.js

@@ -0,0 +1,134 @@
+/**
+ * Per-(target, workspace) job serialization for the remote executor.
+ *
+ * Production incident: a user resubmitted a deploy that looked stuck, and the
+ * executor ran BOTH identical deploys of the same project+workspace
+ * concurrently. They raced each other over the same cloned repo dirs, local
+ * hereya state and tofu/CDK state, and both wedged. Jobs that mutate the same
+ * logical target in the same workspace must run serially on one executor;
+ * unrelated jobs (e.g. the normal boot burst of provisions for different
+ * packages) must keep running concurrently.
+ */
+/**
+ * Escape '%' (the escape character) and ':' (the key segment separator) so
+ * two different (target, workspace) pairs can never collide on the same key
+ * (e.g. project `a:b` in workspace `c` vs project `a` in workspace `b:c`).
+ */
+function escapeSegment(value) {
+    return value.replaceAll('%', '%25').replaceAll(':', '%3A');
+}
+function buildKey(kind, target, workspace) {
+    return `${kind}:${escapeSegment(target)}:${escapeSegment(workspace)}`;
+}
+function nonEmptyString(value) {
+    if (typeof value !== 'string')
+        return undefined;
+    const trimmed = value.trim();
+    return trimmed === '' ? undefined : trimmed;
+}
+/**
+ * Derives a stable key identifying "same logical target in the same
+ * workspace", or null when the job must NOT be serialized (read-only jobs,
+ * unknown job types, malformed payloads — those run immediately, no lock).
+ * Never throws.
+ *
+ * Key derivation per job type (the kind prefix prevents cross-kind
+ * collisions, e.g. a project named like an app):
+ *   - deploy / undeploy:        `project:<payload.project>:<payload.workspace>`
+ *   - init:                     `project:<payload.projectName>:<payload.workspace>`
+ *   - app-deploy / app-destroy: `app:<payload.appName>:<payload.workspace>`
+ *   - provision / destroy:      `pkg:<payload.package>:<payload.workspace>`
+ *   - resolve-env:              null (read-only, never serialized)
+ */
+export function deriveJobSerializationKey(job) {
+    try {
+        const { payload } = job;
+        if (!payload || typeof payload !== 'object')
+            return null;
+        const workspace = nonEmptyString(payload.workspace);
+        if (!workspace)
+            return null;
+        switch (job.type) {
+            case 'app-deploy':
+            case 'app-destroy': {
+                const appName = nonEmptyString(payload.appName);
+                return appName ? buildKey('app', appName, workspace) : null;
+            }
+            case 'deploy':
+            case 'undeploy': {
+                // The deploy payload carries the project NAME (the repo URL lives in
+                // the project metadata fetched later), so the name is the identity.
+                const project = nonEmptyString(payload.project);
+                return project ? buildKey('project', project, workspace) : null;
+            }
+            case 'destroy':
+            case 'provision': {
+                const pkg = nonEmptyString(payload.package);
+                return pkg ? buildKey('pkg', pkg, workspace) : null;
+            }
+            case 'init': {
+                // Shares the `project:` key space with deploy/undeploy on purpose:
+                // an init and a deploy of the same project must not interleave.
+                const projectName = nonEmptyString(payload.projectName);
+                return projectName ? buildKey('project', projectName, workspace) : null;
+            }
+            default: {
+                // resolve-env (read-only) and any unknown/future job type.
+                return null;
+            }
+        }
+    }
+    catch {
+        // Serialization is best-effort protection; a weird payload must never
+        // prevent the job from running at all.
+        return null;
+    }
+}
+/**
+ * Minimal keyed mutex: chains async functions per key so runs with the same
+ * key execute strictly serially (FIFO), while runs with different keys — or a
+ * null key — start immediately and proceed concurrently.
+ *
+ * - A rejecting fn never blocks its successors: each fn is chained via
+ *   `.then(fn, fn)` onto a rejection-swallowed tail.
+ * - The map entry for a key is removed once its chain drains, so the map
+ *   cannot grow unboundedly over executor uptime.
+ */
+export class KeyedMutex {
+    chains = new Map();
+    /** Number of keys with in-flight or queued runs (introspection/test aid). */
+    get size() {
+        return this.chains.size;
+    }
+    /**
+     * True when at least one run for this key is in flight or queued — i.e. a
+     * new run() with this key would have to wait. A null key is never busy.
+     */
+    isBusy(key) {
+        return key !== null && this.chains.has(key);
+    }
+    async run(key, fn) {
+        if (key === null) {
+            return fn();
+        }
+        const existing = this.chains.get(key);
+        const entry = existing ?? { pending: 0, tail: Promise.resolve() };
+        if (!existing) {
+            this.chains.set(key, entry);
+        }
+        entry.pending += 1;
+        const chained = entry.tail.then(fn, fn);
+        // The stored tail must never reject (a failed predecessor must not block
+        // or poison successors); once this run settles, drop the map entry if no
+        // other run is in flight or queued for the key.
+        entry.tail = chained
+            .then(() => { }, () => { })
+            .finally(() => {
+            entry.pending -= 1;
+            if (entry.pending === 0 && this.chains.get(key) === entry) {
+                this.chains.delete(key);
+            }
+        });
+        return chained;
+    }
+}

package/dist/lib/executor-start/job-timeout.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Per-job execution timeout in minutes. Overridable via
+ * HEREYA_EXECUTOR_JOB_TIMEOUT_MIN (positive number, in minutes).
+ * A wedged child process (e.g. a tool blocking on a prompt) must not hold
+ * its executor slot forever while the heartbeat keeps the job "alive".
+ */
+export declare function executorJobTimeoutMin(): number;

package/dist/lib/executor-start/job-timeout.js

@@ -0,0 +1,17 @@
+const DEFAULT_EXECUTOR_JOB_TIMEOUT_MIN = 60;
+/**
+ * Per-job execution timeout in minutes. Overridable via
+ * HEREYA_EXECUTOR_JOB_TIMEOUT_MIN (positive number, in minutes).
+ * A wedged child process (e.g. a tool blocking on a prompt) must not hold
+ * its executor slot forever while the heartbeat keeps the job "alive".
+ */
+export function executorJobTimeoutMin() {
+    const fromEnv = process.env.HEREYA_EXECUTOR_JOB_TIMEOUT_MIN;
+    if (fromEnv !== undefined) {
+        const parsed = Number(fromEnv);
+        if (Number.isFinite(parsed) && parsed > 0) {
+            return parsed;
+        }
+    }
+    return DEFAULT_EXECUTOR_JOB_TIMEOUT_MIN;
+}

package/dist/lib/remote-job-utils.js

@@ -1,3 +1,4 @@
+import { REQUEST_TIMED_OUT } from '../backend/cloud/cloud-backend/executor-jobs.js';
 const DEFAULT_TIMEOUT_MIN = 90;
 const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000;
 /**
@@ -11,6 +12,12 @@ const REWAKE_INTERVAL_MS = 60_000;
  * a chance to land before we ask for another one.
  */
 const REWAKE_INITIAL_DELAY_MS = 60_000;
+/**
+ * A status fetch aborted by its timeout (see executor-jobs.ts) is transient —
+ * retry a bounded number of consecutive times with a short backoff.
+ */
+const MAX_CONSECUTIVE_POLL_TIMEOUTS = 3;
+const POLL_TIMEOUT_RETRY_DELAY_MS = 2000;
 /**
  * Default poll timeout, in ms. Overridable via HEREYA_JOB_TIMEOUT_MIN env var
  * (positive integer, in minutes). Falls back to DEFAULT_TIMEOUT_MIN.
@@ -49,6 +56,33 @@ async function maybeRewake(args) {
     }
     return now;
 }
+/**
+ * Forwards new job logs to the logger, or emits a periodic "still waiting"
+ * heartbeat when the job has been quiet. Returns the updated cursors.
+ */
+function forwardJobProgress(args) {
+    const { job, jobId, logger, startTime, timeout } = args;
+    let { lastLogLength, lastUserHeartbeat } = args;
+    const sawNewLog = Boolean(job.logs && job.logs.length > lastLogLength);
+    if (sawNewLog) {
+        const newLogs = job.logs.slice(lastLogLength);
+        if (newLogs.trim()) {
+            logger?.info?.(newLogs);
+        }
+        lastLogLength = job.logs.length;
+        lastUserHeartbeat = Date.now();
+    }
+    else {
+        const now = Date.now();
+        if (now - lastUserHeartbeat >= HEARTBEAT_INTERVAL_MS) {
+            const elapsedMin = Math.round((now - startTime) / 60_000);
+            const totalMin = Math.round(timeout / 60_000);
+            logger?.info?.(`still waiting on job ${jobId} (${elapsedMin}m elapsed, ${totalMin}m timeout). Set HEREYA_JOB_TIMEOUT_MIN to extend.`);
+            lastUserHeartbeat = now;
+        }
+    }
+    return { lastLogLength, lastUserHeartbeat };
+}
 export async function pollExecutorJob(input) {
     const { cloudBackend, jobId, logger, workspace } = input;
     const timeout = input.timeoutMs ?? defaultTimeoutMs();
@@ -58,6 +92,7 @@ export async function pollExecutorJob(input) {
     let lastStatus = 'pending';
     let lastUserHeartbeat = startTime;
     let lastRewakeAt = startTime;
+    let consecutiveTimeouts = 0;
     while (Date.now() - startTime < timeout) {
         // eslint-disable-next-line no-await-in-loop
         const statusResult = await cloudBackend.getExecutorJobStatus({
@@ -67,28 +102,29 @@ export async function pollExecutorJob(input) {
             workspace,
         });
         if (!statusResult.success) {
+            if (statusResult.reason === REQUEST_TIMED_OUT && consecutiveTimeouts < MAX_CONSECUTIVE_POLL_TIMEOUTS) {
+                consecutiveTimeouts += 1;
+                logger?.debug?.(`Job status poll timed out (attempt ${consecutiveTimeouts}/${MAX_CONSECUTIVE_POLL_TIMEOUTS}); retrying...`);
+                // eslint-disable-next-line no-await-in-loop
+                await new Promise((resolve) => {
+                    setTimeout(resolve, POLL_TIMEOUT_RETRY_DELAY_MS);
+                });
+                continue;
+            }
             return { reason: `Failed to get job status: ${statusResult.reason}`, success: false };
         }
+        consecutiveTimeouts = 0;
         const { job } = statusResult;
         lastStatus = job.status;
-        const sawNewLog = Boolean(job.logs && job.logs.length > lastLogLength);
-        if (sawNewLog) {
-            const newLogs = job.logs.slice(lastLogLength);
-            if (newLogs.trim()) {
-                logger?.info?.(newLogs);
-            }
-            lastLogLength = job.logs.length;
-            lastUserHeartbeat = Date.now();
-        }
-        else {
-            const now = Date.now();
-            if (now - lastUserHeartbeat >= HEARTBEAT_INTERVAL_MS) {
-                const elapsedMin = Math.round((now - startTime) / 60_000);
-                const totalMin = Math.round(timeout / 60_000);
-                logger?.info?.(`still waiting on job ${jobId} (${elapsedMin}m elapsed, ${totalMin}m timeout). Set HEREYA_JOB_TIMEOUT_MIN to extend.`);
-                lastUserHeartbeat = now;
-            }
-        }
+        ({ lastLogLength, lastUserHeartbeat } = forwardJobProgress({
+            job,
+            jobId,
+            lastLogLength,
+            lastUserHeartbeat,
+            logger,
+            startTime,
+            timeout,
+        }));
         // eslint-disable-next-line no-await-in-loop
         lastRewakeAt = await maybeRewake({
             cloudBackend,

package/dist/lib/shell.js

@@ -34,8 +34,10 @@ export async function runShell(cmd, args, options = {}) {
             cwd: options.directory ?? process.cwd(),
             env: { ...process.env, ...options.env },
             shell: true,
-            // If not explicitly set, use 'inherit' when debugging, or 'pipe' otherwise
-            stdio: options.stdio ?? 'pipe',
+            // Default: stdin ignored (closed), stdout/stderr piped. An open stdin
+            // pipe nothing writes to lets prompting tools (tofu, git, npm) block
+            // forever; with 'ignore' they read EOF and fail fast instead.
+            stdio: options.stdio ?? ['ignore', 'pipe', 'pipe'],
         };
         logger.debug(`Executing: ${cmd} ${args.join(' ')}\n`);
         const child = spawn(cmd, args, spawnOptions);

package/oclif.manifest.json

@@ -2402,6 +2402,61 @@
         "index.js"
       ]
     },
+    "flow:remove": {
+      "aliases": [],
+      "args": {
+        "package": {
+          "description": "Remove a previously added package.",
+          "name": "package",
+          "required": true
+        }
+      },
+      "description": "Remove a package from the project in a git branch-based workspace",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> cloudy/docker_postgres",
+        "<%= config.bin %> <%= command.id %> cloudy/docker_postgres --profile staging"
+      ],
+      "flags": {
+        "chdir": {
+          "description": "\n        Directory where the command will be executed.\n        If not specified, it defaults to the current working directory.\n        Alternatively, you can define the project root by setting the HEREYA_PROJECT_ROOT_DIR environment variable.\n      ",
+          "name": "chdir",
+          "required": false,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "debug": {
+          "description": "enable debug mode",
+          "name": "debug",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "profile": {
+          "description": "profile to use for the workspace (will be appended to workspace name)",
+          "name": "profile",
+          "required": false,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "flow:remove",
+      "pluginAlias": "hereya-cli",
+      "pluginName": "hereya-cli",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": true,
+      "relativePath": [
+        "dist",
+        "commands",
+        "flow",
+        "remove",
+        "index.js"
+      ]
+    },
     "flow:run": {
       "aliases": [],
       "args": {
@@ -2992,61 +3047,6 @@
         "index.js"
       ]
     },
-    "flow:remove": {
-      "aliases": [],
-      "args": {
-        "package": {
-          "description": "Remove a previously added package.",
-          "name": "package",
-          "required": true
-        }
-      },
-      "description": "Remove a package from the project in a git branch-based workspace",
-      "examples": [
-        "<%= config.bin %> <%= command.id %> cloudy/docker_postgres",
-        "<%= config.bin %> <%= command.id %> cloudy/docker_postgres --profile staging"
-      ],
-      "flags": {
-        "chdir": {
-          "description": "\n        Directory where the command will be executed.\n        If not specified, it defaults to the current working directory.\n        Alternatively, you can define the project root by setting the HEREYA_PROJECT_ROOT_DIR environment variable.\n      ",
-          "name": "chdir",
-          "required": false,
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "debug": {
-          "description": "enable debug mode",
-          "name": "debug",
-          "allowNo": false,
-          "type": "boolean"
-        },
-        "profile": {
-          "description": "profile to use for the workspace (will be appended to workspace name)",
-          "name": "profile",
-          "required": false,
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        }
-      },
-      "hasDynamicHelp": false,
-      "hiddenAliases": [],
-      "id": "flow:remove",
-      "pluginAlias": "hereya-cli",
-      "pluginName": "hereya-cli",
-      "pluginType": "core",
-      "strict": true,
-      "enableJsonFlag": false,
-      "isESM": true,
-      "relativePath": [
-        "dist",
-        "commands",
-        "flow",
-        "remove",
-        "index.js"
-      ]
-    },
     "devenv:project:init": {
       "aliases": [],
       "args": {
@@ -3314,6 +3314,51 @@
         "index.js"
       ]
     },
+    "workspace:env:unset": {
+      "aliases": [],
+      "args": {},
+      "description": "unset an env var for a workspace",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> -w my-workspace -n myVar"
+      ],
+      "flags": {
+        "name": {
+          "char": "n",
+          "description": "name of the env var to unset",
+          "name": "name",
+          "required": true,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "workspace": {
+          "char": "w",
+          "description": "name of the workspace to unset an env var for",
+          "name": "workspace",
+          "required": true,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "workspace:env:unset",
+      "pluginAlias": "hereya-cli",
+      "pluginName": "hereya-cli",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": true,
+      "relativePath": [
+        "dist",
+        "commands",
+        "workspace",
+        "env",
+        "unset",
+        "index.js"
+      ]
+    },
     "workspace:executor:install": {
       "aliases": [],
       "args": {},
@@ -3470,52 +3515,7 @@
         "uninstall",
         "index.js"
       ]
-    },
-    "workspace:env:unset": {
-      "aliases": [],
-      "args": {},
-      "description": "unset an env var for a workspace",
-      "examples": [
-        "<%= config.bin %> <%= command.id %> -w my-workspace -n myVar"
-      ],
-      "flags": {
-        "name": {
-          "char": "n",
-          "description": "name of the env var to unset",
-          "name": "name",
-          "required": true,
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        },
-        "workspace": {
-          "char": "w",
-          "description": "name of the workspace to unset an env var for",
-          "name": "workspace",
-          "required": true,
-          "hasDynamicHelp": false,
-          "multiple": false,
-          "type": "option"
-        }
-      },
-      "hasDynamicHelp": false,
-      "hiddenAliases": [],
-      "id": "workspace:env:unset",
-      "pluginAlias": "hereya-cli",
-      "pluginName": "hereya-cli",
-      "pluginType": "core",
-      "strict": true,
-      "enableJsonFlag": false,
-      "isESM": true,
-      "relativePath": [
-        "dist",
-        "commands",
-        "workspace",
-        "env",
-        "unset",
-        "index.js"
-      ]
     }
   },
-  "version": "0.103.0"
+  "version": "0.105.0"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hereya-cli",
   "description": "Infrastructure as Package",
-  "version": "0.103.0",
+  "version": "0.105.0",
   "author": "Hereya Developers",
   "bin": {
     "hereya": "./bin/run.cjs"