hereya-cli 0.103.0 → 0.104.0

package/dist/backend/cloud/cloud-backend/executor-jobs.js

@@ -1,4 +1,22 @@
 import { safeResponseJson } from './utils.js';
+/**
+ * Failure reason reported when an executor HTTP call is aborted by its
+ * timeout signal. Callers (e.g. remote-job-utils) match on this exact string
+ * to treat the failure as transient/retryable.
+ */
+export const REQUEST_TIMED_OUT = 'Request timed out';
+/**
+ * True when the error is an AbortSignal timeout/abort (possibly wrapped by
+ * undici as `TypeError: fetch failed` with the abort error as `cause`).
+ */
+function isAbortLikeError(error) {
+    if (!(error instanceof Error))
+        return false;
+    if (error.name === 'AbortError' || error.name === 'TimeoutError')
+        return true;
+    const { cause } = error;
+    return cause instanceof Error && (cause.name === 'AbortError' || cause.name === 'TimeoutError');
+}
 export async function cancelExecutorJob(config, input) {
     const response = await fetch(`${config.url}/api/workspaces/${encodeURIComponent(input.workspace)}/jobs/${encodeURIComponent(input.jobId)}/cancel`, {
         body: JSON.stringify({ force: input.force === true }),
@@ -38,10 +56,23 @@ export async function getExecutorJobStatus(config, input) {
     if (input.lastStatus) {
         url.searchParams.set('lastStatus', input.lastStatus);
     }
-    const response = await fetch(url.toString(), {
-        headers: { 'Authorization': `Bearer ${config.accessToken}` },
-        method: 'GET',
-    });
+    let response;
+    try {
+        // Server-side long-poll holds the request for at most ~30s; 45s means a
+        // healthy long-poll never trips this, but a dead connection cannot hang
+        // the caller's wait loop forever.
+        response = await fetch(url.toString(), {
+            headers: { 'Authorization': `Bearer ${config.accessToken}` },
+            method: 'GET',
+            signal: AbortSignal.timeout(45_000),
+        });
+    }
+    catch (error) {
+        if (isAbortLikeError(error)) {
+            return { reason: REQUEST_TIMED_OUT, success: false };
+        }
+        throw error;
+    }
     if (!response.ok) {
         if (response.status === 401) {
             return { reason: 'Unauthorized', success: false };
@@ -77,10 +108,23 @@ export async function pollExecutorJobs(config, input) {
     if (input.executorId) {
         url.searchParams.set('executorId', input.executorId);
     }
-    const response = await fetch(url.toString(), {
-        headers: { 'Authorization': `Bearer ${config.accessToken}` },
-        method: 'GET',
-    });
+    let response;
+    try {
+        // Server-side long-poll holds the request for at most ~30s; 45s means a
+        // healthy long-poll never trips this, but a dead connection cannot kill
+        // the executor's poll loop silently.
+        response = await fetch(url.toString(), {
+            headers: { 'Authorization': `Bearer ${config.accessToken}` },
+            method: 'GET',
+            signal: AbortSignal.timeout(45_000),
+        });
+    }
+    catch (error) {
+        if (isAbortLikeError(error)) {
+            return { reason: REQUEST_TIMED_OUT, success: false };
+        }
+        throw error;
+    }
     if (!response.ok) {
         if (response.status === 401) {
             return { reason: 'Unauthorized', success: false, unauthorized: true };
@@ -166,11 +210,21 @@ export async function updateExecutorJob(config, input) {
     if (input.result) {
         formData.append('result', JSON.stringify(input.result));
     }
-    const response = await fetch(`${config.url}/api/executor/jobs/${encodeURIComponent(input.jobId)}`, {
-        body: formData,
-        headers: { 'Authorization': `Bearer ${config.accessToken}` },
-        method: 'PATCH',
-    });
+    let response;
+    try {
+        response = await fetch(`${config.url}/api/executor/jobs/${encodeURIComponent(input.jobId)}`, {
+            body: formData,
+            headers: { 'Authorization': `Bearer ${config.accessToken}` },
+            method: 'PATCH',
+            signal: AbortSignal.timeout(30_000),
+        });
+    }
+    catch (error) {
+        if (isAbortLikeError(error)) {
+            return { reason: REQUEST_TIMED_OUT, success: false };
+        }
+        throw error;
+    }
     if (!response.ok) {
         if (response.status === 401) {
             return { reason: 'Unauthorized', success: false, unauthorized: true };

package/dist/commands/executor/start/index.js

@@ -7,6 +7,7 @@ import { LocalExecutor } from '../../../executor/local.js';
 import { createAuthenticatedBackend } from '../../../lib/executor-start/auth.js';
 import { executeAppJob } from '../../../lib/executor-start/execute-app-job.js';
 import { executeInitJob } from '../../../lib/executor-start/execute-init-job.js';
+import { createExecutorHooks } from '../../../lib/executor-start/hooks.js';
 import { dispatchJob } from '../../../lib/executor-start/job-dispatch.js';
 export { redactCredentials } from '../../../lib/executor-start/format.js';
 export default class ExecutorStart extends Command {
@@ -70,7 +71,15 @@ Set the HEREYA_CLOUD_URL environment variable to target a specific hereya cloud
         process.on('SIGINT', shutdown);
         process.on('SIGTERM', shutdown);
         const activeJobs = new Set();
-        const hooks = { log: (m) => this.log(m), warn: (m) => this.warn(m) };
+        // Ephemeral mode = an idle-timeout is configured (the EC2/ASG executor).
+        // There, a job that hits the execution timeout triggers a delayed
+        // non-zero exit so systemd replaces the wedged instance; in non-ephemeral
+        // mode the hook only logs a warning. See createExecutorHooks.
+        const hooks = createExecutorHooks({
+            ephemeral: idleTimeout !== undefined,
+            log: (m) => this.log(m),
+            warn: (m) => this.warn(m),
+        });
         // Idle clock advances on every successful poll OR while jobs are running.
         // Generic transient errors (5xx, network) deliberately do NOT advance it
         // — a flaky cloud must not keep the EC2 alive forever. Persistent 401s

package/dist/iac/terraform.js

@@ -5,7 +5,7 @@ import path from 'node:path';
 import { pipeline } from 'node:stream/promises';
 import { Extract as decompress } from 'unzip-stream';
 import { mapObject, paramToEnvString } from '../lib/object-utils.js';
-import { runShell } from '../lib/shell.js';
+import { shellUtils } from '../lib/shell.js';
 export class Terraform {
     async apply(input) {
         if (input.infraConfig?.localStatePath) {
@@ -41,7 +41,7 @@ export class Terraform {
         }
         try {
             const terraform = await this.getTerraformBinary(input.logger);
-            await runShell(terraform, ['init'], {
+            await shellUtils.runShell(terraform, ['init', '-input=false'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -56,7 +56,10 @@ export class Terraform {
                 },
                 logger: input.logger,
             });
-            await runShell(terraform, ['apply', '-auto-approve'], {
+            // -input=false: never prompt (a prompt would hang the executor forever).
+            // -lock-timeout=5m: wait bounded time for a concurrently-held state lock
+            // instead of failing instantly.
+            await shellUtils.runShell(terraform, ['apply', '-auto-approve', '-input=false', '-lock-timeout=5m'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -119,7 +122,7 @@ export class Terraform {
             await fs.promises.writeFile(backendFile, backendConfig);
         }
         try {
-            await runShell(terraform, ['init'], {
+            await shellUtils.runShell(terraform, ['init', '-input=false'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -135,7 +138,9 @@ export class Terraform {
                 logger: input.logger,
             });
             const env = await this.getEnv(input.pkgPath);
-            await runShell(terraform, ['destroy', '-auto-approve'], {
+            // See apply(): -input=false prevents interactive prompts from hanging
+            // the executor; -lock-timeout=5m bounds waiting on a held state lock.
+            await shellUtils.runShell(terraform, ['destroy', '-auto-approve', '-input=false', '-lock-timeout=5m'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -244,9 +249,8 @@ export class Terraform {
     }
     async getEnv(pkgPath, logger) {
         const terraform = await this.getTerraformBinary(logger);
-        const resourceOut = await runShell(terraform, ['output', '--json'], {
+        const resourceOut = await shellUtils.runShell(terraform, ['output', '--json'], {
             directory: pkgPath,
-            stdio: 'pipe',
         });
         let outStr = resourceOut.stdout.toString().trim();
         const start = outStr.indexOf('{');

package/dist/lib/executor-start/hooks.d.ts

@@ -0,0 +1,27 @@
+import { JobDispatchHooks } from './job-dispatch.js';
+/**
+ * Delay before the process exits after a job timeout on an ephemeral
+ * executor — long enough for the job-failed PATCH (and final warn logs)
+ * to flush to hereya cloud.
+ */
+export declare const JOB_TIMEOUT_EXIT_DELAY_MS = 5000;
+/**
+ * Builds the dispatch hooks for the executor start loop, including the
+ * per-job-timeout policy:
+ *
+ * - Non-ephemeral executors (no idle-timeout configured) just log a warning
+ *   and keep running — an operator owns the process lifecycle.
+ * - Ephemeral executors (idle-timeout configured) initiate a graceful exit
+ *   with a non-zero code after a short delay. On the ephemeral executor,
+ *   systemd's OnFailure/ExecStopPost drain then replaces the instance, which
+ *   un-wedges everything: other in-flight jobs on the box lose their
+ *   heartbeat, go stale, and the server-side reaper resets them for retry —
+ *   that is the intended behavior.
+ */
+export declare function createExecutorHooks(opts: {
+    ephemeral: boolean;
+    exit?: (code: number) => void;
+    exitDelayMs?: number;
+    log: (msg: string) => void;
+    warn: (msg: string) => void;
+}): JobDispatchHooks;

package/dist/lib/executor-start/hooks.js

@@ -0,0 +1,42 @@
+/**
+ * Delay before the process exits after a job timeout on an ephemeral
+ * executor — long enough for the job-failed PATCH (and final warn logs)
+ * to flush to hereya cloud.
+ */
+export const JOB_TIMEOUT_EXIT_DELAY_MS = 5000;
+/**
+ * Builds the dispatch hooks for the executor start loop, including the
+ * per-job-timeout policy:
+ *
+ * - Non-ephemeral executors (no idle-timeout configured) just log a warning
+ *   and keep running — an operator owns the process lifecycle.
+ * - Ephemeral executors (idle-timeout configured) initiate a graceful exit
+ *   with a non-zero code after a short delay. On the ephemeral executor,
+ *   systemd's OnFailure/ExecStopPost drain then replaces the instance, which
+ *   un-wedges everything: other in-flight jobs on the box lose their
+ *   heartbeat, go stale, and the server-side reaper resets them for retry —
+ *   that is the intended behavior.
+ */
+export function createExecutorHooks(opts) {
+    const { ephemeral, log, warn } = opts;
+    const exitDelayMs = opts.exitDelayMs ?? JOB_TIMEOUT_EXIT_DELAY_MS;
+    const exit = opts.exit ??
+        ((code) => {
+            // eslint-disable-next-line n/no-process-exit, unicorn/no-process-exit
+            process.exit(code);
+        });
+    return {
+        log,
+        onJobTimeout(jobId) {
+            if (!ephemeral) {
+                warn(`Job ${jobId} hit the execution timeout. Executor keeps running (non-ephemeral mode); check the host for wedged child processes.`);
+                return;
+            }
+            warn(`Job ${jobId} hit the execution timeout. Ephemeral executor exits with a non-zero code in ${Math.round(exitDelayMs / 1000)}s so the instance gets replaced.`);
+            const timer = setTimeout(() => exit(1), exitDelayMs);
+            // Don't let the exit timer keep an otherwise-finished process alive.
+            timer.unref?.();
+        },
+        warn,
+    };
+}

package/dist/lib/executor-start/job-dispatch.d.ts

@@ -2,11 +2,19 @@ import { CloudBackend } from '../../backend/cloud/cloud-backend.js';
 import { LocalExecutor } from '../../executor/local.js';
 export interface JobDispatchHooks {
     log(msg: string): void;
+    /**
+     * Fired after a job exceeds the execution timeout (job already PATCHed as
+     * failed). Lets the caller decide process-level policy, e.g. exit for
+     * instance replacement on an ephemeral executor.
+     */
+    onJobTimeout?(jobId: string): void;
     warn(msg: string): void;
 }
 /**
  * Dispatches a single job to the right handler, with periodic log flushing
- * and heartbeating to the cloud. Errors are caught and reported as failed jobs.
+ * and heartbeating to the cloud. Errors are caught and reported as failed
+ * jobs. Execution is raced against a per-job timeout so a hung child can
+ * never hold its slot indefinitely.
  */
 export declare function dispatchJob(job: {
     id: string;

package/dist/lib/executor-start/job-dispatch.js

@@ -1,6 +1,23 @@
 import { executeAppJob } from './execute-app-job.js';
 import { executeDeployJob } from './execute-deploy-job.js';
 import { executeInitJob } from './execute-init-job.js';
+const DEFAULT_EXECUTOR_JOB_TIMEOUT_MIN = 60;
+/**
+ * Per-job execution timeout in minutes. Overridable via
+ * HEREYA_EXECUTOR_JOB_TIMEOUT_MIN (positive number, in minutes).
+ * A wedged child process (e.g. a tool blocking on a prompt) must not hold
+ * its executor slot forever while the heartbeat keeps the job "alive".
+ */
+function executorJobTimeoutMin() {
+    const fromEnv = process.env.HEREYA_EXECUTOR_JOB_TIMEOUT_MIN;
+    if (fromEnv !== undefined) {
+        const parsed = Number(fromEnv);
+        if (Number.isFinite(parsed) && parsed > 0) {
+            return parsed;
+        }
+    }
+    return DEFAULT_EXECUTOR_JOB_TIMEOUT_MIN;
+}
 /**
  * Builds a verbose error description for server-side job logs. If the thrown
  * error is a `ShellCommandError` (or anything carrying `.stderr`/`.stdout`),
@@ -27,7 +44,9 @@ function formatJobError(error) {
 }
 /**
  * Dispatches a single job to the right handler, with periodic log flushing
- * and heartbeating to the cloud. Errors are caught and reported as failed jobs.
+ * and heartbeating to the cloud. Errors are caught and reported as failed
+ * jobs. Execution is raced against a per-job timeout so a hung child can
+ * never hold its slot indefinitely.
  */
 export async function dispatchJob(job, executor, cloudBackend, hooks) {
     let logBuffer = '';
@@ -62,7 +81,14 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
             logBuffer += msg + '\n';
         },
     };
+    const timeoutMin = executorJobTimeoutMin();
+    let timedOut = false;
+    let timeoutTimer;
     const finalize = async (result, label) => {
+        // The timeout path already reported this job as failed; a late completion
+        // from the still-running handler must not overwrite that terminal status.
+        if (timedOut)
+            return;
         clearInterval(logInterval);
         clearInterval(heartbeatInterval);
         await cloudBackend.updateExecutorJob({
@@ -73,7 +99,7 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
         });
         hooks.log(`Job ${job.id} ${result.success ? 'completed' : 'failed'}${label ? ` (${label})` : ''}`);
     };
-    try {
+    const execute = async () => {
         if (job.type === 'resolve-env') {
             const resolved = await executor.resolveEnvValues(job.payload);
             await finalize({ env: resolved, success: true }, 'resolve-env');
@@ -98,6 +124,41 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
             ? await executor.provision({ ...job.payload, logger })
             : await executor.destroy({ ...job.payload, logger });
         await finalize(result, '');
+    };
+    try {
+        const timeoutPromise = new Promise((resolve) => {
+            timeoutTimer = setTimeout(() => resolve('timeout'), timeoutMin * 60 * 1000);
+            // Belt-and-braces: a pending job timer must never keep the process alive.
+            timeoutTimer.unref?.();
+        });
+        const executePromise = execute().then(() => 'done');
+        // Keep the chain explicitly handled even after the timeout wins the race:
+        // a late rejection from the wedged handler (hung tool killed by OOM,
+        // network drop, ...) must never surface as an unhandledRejection and
+        // crash a non-ephemeral executor. The job row is already failed at that
+        // point, so just log it. When execute() rejects BEFORE the timeout, the
+        // race still rejects into the catch block below and the timedOut guard
+        // keeps this extra warn silent.
+        executePromise.catch((error) => {
+            if (timedOut) {
+                hooks.warn(`Job ${job.id} late failure after timeout: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        });
+        const outcome = await Promise.race([executePromise, timeoutPromise]);
+        if (outcome === 'timeout') {
+            timedOut = true;
+            clearInterval(logInterval);
+            clearInterval(heartbeatInterval);
+            const reason = `Job execution timed out on the executor after ${timeoutMin} minutes (HEREYA_EXECUTOR_JOB_TIMEOUT_MIN to adjust)`;
+            await cloudBackend.updateExecutorJob({
+                jobId: job.id,
+                logs: logBuffer + `\nError: ${reason}\n`,
+                result: { reason, success: false },
+                status: 'failed',
+            });
+            hooks.warn(`Job ${job.id} failed: ${reason}`);
+            hooks.onJobTimeout?.(job.id);
+        }
     }
     catch (error) {
         clearInterval(logInterval);
@@ -111,4 +172,8 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
         });
         hooks.warn(`Job ${job.id} failed: ${reason}`);
     }
+    finally {
+        if (timeoutTimer)
+            clearTimeout(timeoutTimer);
+    }
 }

package/dist/lib/remote-job-utils.js

@@ -1,3 +1,4 @@
+import { REQUEST_TIMED_OUT } from '../backend/cloud/cloud-backend/executor-jobs.js';
 const DEFAULT_TIMEOUT_MIN = 90;
 const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000;
 /**
@@ -11,6 +12,12 @@ const REWAKE_INTERVAL_MS = 60_000;
  * a chance to land before we ask for another one.
  */
 const REWAKE_INITIAL_DELAY_MS = 60_000;
+/**
+ * A status fetch aborted by its timeout (see executor-jobs.ts) is transient —
+ * retry a bounded number of consecutive times with a short backoff.
+ */
+const MAX_CONSECUTIVE_POLL_TIMEOUTS = 3;
+const POLL_TIMEOUT_RETRY_DELAY_MS = 2000;
 /**
  * Default poll timeout, in ms. Overridable via HEREYA_JOB_TIMEOUT_MIN env var
  * (positive integer, in minutes). Falls back to DEFAULT_TIMEOUT_MIN.
@@ -49,6 +56,33 @@ async function maybeRewake(args) {
     }
     return now;
 }
+/**
+ * Forwards new job logs to the logger, or emits a periodic "still waiting"
+ * heartbeat when the job has been quiet. Returns the updated cursors.
+ */
+function forwardJobProgress(args) {
+    const { job, jobId, logger, startTime, timeout } = args;
+    let { lastLogLength, lastUserHeartbeat } = args;
+    const sawNewLog = Boolean(job.logs && job.logs.length > lastLogLength);
+    if (sawNewLog) {
+        const newLogs = job.logs.slice(lastLogLength);
+        if (newLogs.trim()) {
+            logger?.info?.(newLogs);
+        }
+        lastLogLength = job.logs.length;
+        lastUserHeartbeat = Date.now();
+    }
+    else {
+        const now = Date.now();
+        if (now - lastUserHeartbeat >= HEARTBEAT_INTERVAL_MS) {
+            const elapsedMin = Math.round((now - startTime) / 60_000);
+            const totalMin = Math.round(timeout / 60_000);
+            logger?.info?.(`still waiting on job ${jobId} (${elapsedMin}m elapsed, ${totalMin}m timeout). Set HEREYA_JOB_TIMEOUT_MIN to extend.`);
+            lastUserHeartbeat = now;
+        }
+    }
+    return { lastLogLength, lastUserHeartbeat };
+}
 export async function pollExecutorJob(input) {
     const { cloudBackend, jobId, logger, workspace } = input;
     const timeout = input.timeoutMs ?? defaultTimeoutMs();
@@ -58,6 +92,7 @@ export async function pollExecutorJob(input) {
     let lastStatus = 'pending';
     let lastUserHeartbeat = startTime;
     let lastRewakeAt = startTime;
+    let consecutiveTimeouts = 0;
     while (Date.now() - startTime < timeout) {
         // eslint-disable-next-line no-await-in-loop
         const statusResult = await cloudBackend.getExecutorJobStatus({
@@ -67,28 +102,29 @@ export async function pollExecutorJob(input) {
             workspace,
         });
         if (!statusResult.success) {
+            if (statusResult.reason === REQUEST_TIMED_OUT && consecutiveTimeouts < MAX_CONSECUTIVE_POLL_TIMEOUTS) {
+                consecutiveTimeouts += 1;
+                logger?.debug?.(`Job status poll timed out (attempt ${consecutiveTimeouts}/${MAX_CONSECUTIVE_POLL_TIMEOUTS}); retrying...`);
+                // eslint-disable-next-line no-await-in-loop
+                await new Promise((resolve) => {
+                    setTimeout(resolve, POLL_TIMEOUT_RETRY_DELAY_MS);
+                });
+                continue;
+            }
             return { reason: `Failed to get job status: ${statusResult.reason}`, success: false };
         }
+        consecutiveTimeouts = 0;
         const { job } = statusResult;
         lastStatus = job.status;
-        const sawNewLog = Boolean(job.logs && job.logs.length > lastLogLength);
-        if (sawNewLog) {
-            const newLogs = job.logs.slice(lastLogLength);
-            if (newLogs.trim()) {
-                logger?.info?.(newLogs);
-            }
-            lastLogLength = job.logs.length;
-            lastUserHeartbeat = Date.now();
-        }
-        else {
-            const now = Date.now();
-            if (now - lastUserHeartbeat >= HEARTBEAT_INTERVAL_MS) {
-                const elapsedMin = Math.round((now - startTime) / 60_000);
-                const totalMin = Math.round(timeout / 60_000);
-                logger?.info?.(`still waiting on job ${jobId} (${elapsedMin}m elapsed, ${totalMin}m timeout). Set HEREYA_JOB_TIMEOUT_MIN to extend.`);
-                lastUserHeartbeat = now;
-            }
-        }
+        ({ lastLogLength, lastUserHeartbeat } = forwardJobProgress({
+            job,
+            jobId,
+            lastLogLength,
+            lastUserHeartbeat,
+            logger,
+            startTime,
+            timeout,
+        }));
         // eslint-disable-next-line no-await-in-loop
         lastRewakeAt = await maybeRewake({
             cloudBackend,

package/dist/lib/shell.js

@@ -34,8 +34,10 @@ export async function runShell(cmd, args, options = {}) {
             cwd: options.directory ?? process.cwd(),
             env: { ...process.env, ...options.env },
             shell: true,
-            // If not explicitly set, use 'inherit' when debugging, or 'pipe' otherwise
-            stdio: options.stdio ?? 'pipe',
+            // Default: stdin ignored (closed), stdout/stderr piped. An open stdin
+            // pipe nothing writes to lets prompting tools (tofu, git, npm) block
+            // forever; with 'ignore' they read EOF and fail fast instead.
+            stdio: options.stdio ?? ['ignore', 'pipe', 'pipe'],
         };
         logger.debug(`Executing: ${cmd} ${args.join(' ')}\n`);
         const child = spawn(cmd, args, spawnOptions);