hereya-cli 0.103.0 → 0.105.0

package/dist/backend/cloud/cloud-backend/executor-jobs.js

@@ -1,4 +1,22 @@
 import { safeResponseJson } from './utils.js';
+/**
+ * Failure reason reported when an executor HTTP call is aborted by its
+ * timeout signal. Callers (e.g. remote-job-utils) match on this exact string
+ * to treat the failure as transient/retryable.
+ */
+export const REQUEST_TIMED_OUT = 'Request timed out';
+/**
+ * True when the error is an AbortSignal timeout/abort (possibly wrapped by
+ * undici as `TypeError: fetch failed` with the abort error as `cause`).
+ */
+function isAbortLikeError(error) {
+    if (!(error instanceof Error))
+        return false;
+    if (error.name === 'AbortError' || error.name === 'TimeoutError')
+        return true;
+    const { cause } = error;
+    return cause instanceof Error && (cause.name === 'AbortError' || cause.name === 'TimeoutError');
+}
 export async function cancelExecutorJob(config, input) {
     const response = await fetch(`${config.url}/api/workspaces/${encodeURIComponent(input.workspace)}/jobs/${encodeURIComponent(input.jobId)}/cancel`, {
         body: JSON.stringify({ force: input.force === true }),
@@ -38,10 +56,23 @@ export async function getExecutorJobStatus(config, input) {
     if (input.lastStatus) {
         url.searchParams.set('lastStatus', input.lastStatus);
     }
-    const response = await fetch(url.toString(), {
-        headers: { 'Authorization': `Bearer ${config.accessToken}` },
-        method: 'GET',
-    });
+    let response;
+    try {
+        // Server-side long-poll holds the request for at most ~30s; 45s means a
+        // healthy long-poll never trips this, but a dead connection cannot hang
+        // the caller's wait loop forever.
+        response = await fetch(url.toString(), {
+            headers: { 'Authorization': `Bearer ${config.accessToken}` },
+            method: 'GET',
+            signal: AbortSignal.timeout(45_000),
+        });
+    }
+    catch (error) {
+        if (isAbortLikeError(error)) {
+            return { reason: REQUEST_TIMED_OUT, success: false };
+        }
+        throw error;
+    }
     if (!response.ok) {
         if (response.status === 401) {
             return { reason: 'Unauthorized', success: false };
@@ -77,10 +108,23 @@ export async function pollExecutorJobs(config, input) {
     if (input.executorId) {
         url.searchParams.set('executorId', input.executorId);
     }
-    const response = await fetch(url.toString(), {
-        headers: { 'Authorization': `Bearer ${config.accessToken}` },
-        method: 'GET',
-    });
+    let response;
+    try {
+        // Server-side long-poll holds the request for at most ~30s; 45s means a
+        // healthy long-poll never trips this, but a dead connection cannot kill
+        // the executor's poll loop silently.
+        response = await fetch(url.toString(), {
+            headers: { 'Authorization': `Bearer ${config.accessToken}` },
+            method: 'GET',
+            signal: AbortSignal.timeout(45_000),
+        });
+    }
+    catch (error) {
+        if (isAbortLikeError(error)) {
+            return { reason: REQUEST_TIMED_OUT, success: false };
+        }
+        throw error;
+    }
     if (!response.ok) {
         if (response.status === 401) {
             return { reason: 'Unauthorized', success: false, unauthorized: true };
@@ -166,11 +210,21 @@ export async function updateExecutorJob(config, input) {
     if (input.result) {
         formData.append('result', JSON.stringify(input.result));
     }
-    const response = await fetch(`${config.url}/api/executor/jobs/${encodeURIComponent(input.jobId)}`, {
-        body: formData,
-        headers: { 'Authorization': `Bearer ${config.accessToken}` },
-        method: 'PATCH',
-    });
+    let response;
+    try {
+        response = await fetch(`${config.url}/api/executor/jobs/${encodeURIComponent(input.jobId)}`, {
+            body: formData,
+            headers: { 'Authorization': `Bearer ${config.accessToken}` },
+            method: 'PATCH',
+            signal: AbortSignal.timeout(30_000),
+        });
+    }
+    catch (error) {
+        if (isAbortLikeError(error)) {
+            return { reason: REQUEST_TIMED_OUT, success: false };
+        }
+        throw error;
+    }
     if (!response.ok) {
         if (response.status === 401) {
             return { reason: 'Unauthorized', success: false, unauthorized: true };

package/dist/commands/executor/start/index.js

@@ -7,7 +7,9 @@ import { LocalExecutor } from '../../../executor/local.js';
 import { createAuthenticatedBackend } from '../../../lib/executor-start/auth.js';
 import { executeAppJob } from '../../../lib/executor-start/execute-app-job.js';
 import { executeInitJob } from '../../../lib/executor-start/execute-init-job.js';
+import { createExecutorHooks } from '../../../lib/executor-start/hooks.js';
 import { dispatchJob } from '../../../lib/executor-start/job-dispatch.js';
+import { KeyedMutex } from '../../../lib/executor-start/job-serialization.js';
 export { redactCredentials } from '../../../lib/executor-start/format.js';
 export default class ExecutorStart extends Command {
     static description = `Start the remote executor process (polls for jobs from hereya cloud).
@@ -70,7 +72,18 @@ Set the HEREYA_CLOUD_URL environment variable to target a specific hereya cloud
         process.on('SIGINT', shutdown);
         process.on('SIGTERM', shutdown);
         const activeJobs = new Set();
-        const hooks = { log: (m) => this.log(m), warn: (m) => this.warn(m) };
+        // One shared mutex per executor process: jobs targeting the same
+        // (target, workspace) run serially, unrelated jobs stay concurrent.
+        const serialization = new KeyedMutex();
+        // Ephemeral mode = an idle-timeout is configured (the EC2/ASG executor).
+        // There, a job that hits the execution timeout triggers a delayed
+        // non-zero exit so systemd replaces the wedged instance; in non-ephemeral
+        // mode the hook only logs a warning. See createExecutorHooks.
+        const hooks = createExecutorHooks({
+            ephemeral: idleTimeout !== undefined,
+            log: (m) => this.log(m),
+            warn: (m) => this.warn(m),
+        });
         // Idle clock advances on every successful poll OR while jobs are running.
         // Generic transient errors (5xx, network) deliberately do NOT advance it
         // — a flaky cloud must not keep the EC2 alive forever. Persistent 401s
@@ -98,6 +111,7 @@ Set the HEREYA_CLOUD_URL environment variable to target a specific hereya cloud
                     lastJobAt,
                     lastSuccessfulPollAt,
                     maxConsecutive401s: MAX_CONSECUTIVE_401S,
+                    serialization,
                     workspace: flags.workspace,
                 });
                 consecutive401s = step.consecutive401s;
@@ -156,7 +170,7 @@ Set the HEREYA_CLOUD_URL environment variable to target a specific hereya cloud
         return { exit: false };
     }
     async runPollIteration(args) {
-        const { activeJobs, cloudBackend, cloudUrl, concurrency, executor, executorId, hereyaToken, hooks, idleTimeout, maxConsecutive401s, workspace, } = args;
+        const { activeJobs, cloudBackend, cloudUrl, concurrency, executor, executorId, hereyaToken, hooks, idleTimeout, maxConsecutive401s, serialization, workspace, } = args;
         let { consecutive401s, lastJobAt, lastSuccessfulPollAt } = args;
         if (activeJobs.size > 0) {
             lastJobAt = Date.now();
@@ -192,7 +206,7 @@ Set the HEREYA_CLOUD_URL environment variable to target a specific hereya cloud
             const { job } = pollResult;
             this.log(`Received job ${job.id} (${job.type}) [${activeJobs.size + 1}/${concurrency} slots used]`);
             lastJobAt = Date.now();
-            const jobPromise = dispatchJob(job, executor, cloudBackend, hooks).finally(() => {
+            const jobPromise = dispatchJob(job, executor, cloudBackend, hooks, serialization).finally(() => {
                 activeJobs.delete(jobPromise);
             });
             activeJobs.add(jobPromise);

package/dist/iac/terraform.js

@@ -5,7 +5,7 @@ import path from 'node:path';
 import { pipeline } from 'node:stream/promises';
 import { Extract as decompress } from 'unzip-stream';
 import { mapObject, paramToEnvString } from '../lib/object-utils.js';
-import { runShell } from '../lib/shell.js';
+import { shellUtils } from '../lib/shell.js';
 export class Terraform {
     async apply(input) {
         if (input.infraConfig?.localStatePath) {
@@ -41,7 +41,7 @@ export class Terraform {
         }
         try {
             const terraform = await this.getTerraformBinary(input.logger);
-            await runShell(terraform, ['init'], {
+            await shellUtils.runShell(terraform, ['init', '-input=false'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -56,7 +56,10 @@ export class Terraform {
                 },
                 logger: input.logger,
             });
-            await runShell(terraform, ['apply', '-auto-approve'], {
+            // -input=false: never prompt (a prompt would hang the executor forever).
+            // -lock-timeout=5m: wait bounded time for a concurrently-held state lock
+            // instead of failing instantly.
+            await shellUtils.runShell(terraform, ['apply', '-auto-approve', '-input=false', '-lock-timeout=5m'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -119,7 +122,7 @@ export class Terraform {
             await fs.promises.writeFile(backendFile, backendConfig);
         }
         try {
-            await runShell(terraform, ['init'], {
+            await shellUtils.runShell(terraform, ['init', '-input=false'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -135,7 +138,9 @@ export class Terraform {
                 logger: input.logger,
             });
             const env = await this.getEnv(input.pkgPath);
-            await runShell(terraform, ['destroy', '-auto-approve'], {
+            // See apply(): -input=false prevents interactive prompts from hanging
+            // the executor; -lock-timeout=5m bounds waiting on a held state lock.
+            await shellUtils.runShell(terraform, ['destroy', '-auto-approve', '-input=false', '-lock-timeout=5m'], {
                 directory: input.pkgPath,
                 env: {
                     ...mapObject(input.env ?? {}, (key, value) => [
@@ -244,9 +249,8 @@ export class Terraform {
     }
     async getEnv(pkgPath, logger) {
         const terraform = await this.getTerraformBinary(logger);
-        const resourceOut = await runShell(terraform, ['output', '--json'], {
+        const resourceOut = await shellUtils.runShell(terraform, ['output', '--json'], {
             directory: pkgPath,
-            stdio: 'pipe',
         });
         let outStr = resourceOut.stdout.toString().trim();
         const start = outStr.indexOf('{');

package/dist/lib/executor-start/format.d.ts

@@ -4,6 +4,15 @@
  * the same sanitization.
  */
 export declare function sanitizeWorkspaceForFilename(workspace: string): string;
+/**
+ * Builds a verbose error description for server-side job logs. If the thrown
+ * error is a `ShellCommandError` (or anything carrying `.stderr`/`.stdout`),
+ * include the full buffers so operators can see the complete failure context.
+ */
+export declare function formatJobError(error: unknown): {
+    fullErr: string;
+    reason: string;
+};
 /**
  * Strip credentials from log strings before flushing them to the cloud.
  * Removes:

package/dist/lib/executor-start/format.js

@@ -6,6 +6,30 @@
 export function sanitizeWorkspaceForFilename(workspace) {
     return workspace.replaceAll('/', '-');
 }
+/**
+ * Builds a verbose error description for server-side job logs. If the thrown
+ * error is a `ShellCommandError` (or anything carrying `.stderr`/`.stdout`),
+ * include the full buffers so operators can see the complete failure context.
+ */
+export function formatJobError(error) {
+    const reason = error && typeof error === 'object' && 'message' in error && typeof error.message === 'string'
+        ? error.message
+        : String(error);
+    const stderrBuffer = error && typeof error === 'object' && typeof error.stderr === 'string'
+        ? error.stderr
+        : '';
+    const stdoutBuffer = error && typeof error === 'object' && typeof error.stdout === 'string'
+        ? error.stdout
+        : '';
+    let extra = '';
+    if (stderrBuffer.trim()) {
+        extra = `\n--- stderr ---\n${stderrBuffer}`;
+    }
+    else if (stdoutBuffer.trim()) {
+        extra = `\n--- stdout ---\n${stdoutBuffer}`;
+    }
+    return { fullErr: reason + extra, reason };
+}
 /**
  * Strip credentials from log strings before flushing them to the cloud.
  * Removes:

package/dist/lib/executor-start/hooks.d.ts

@@ -0,0 +1,27 @@
+import { JobDispatchHooks } from './job-dispatch.js';
+/**
+ * Delay before the process exits after a job timeout on an ephemeral
+ * executor — long enough for the job-failed PATCH (and final warn logs)
+ * to flush to hereya cloud.
+ */
+export declare const JOB_TIMEOUT_EXIT_DELAY_MS = 5000;
+/**
+ * Builds the dispatch hooks for the executor start loop, including the
+ * per-job-timeout policy:
+ *
+ * - Non-ephemeral executors (no idle-timeout configured) just log a warning
+ *   and keep running — an operator owns the process lifecycle.
+ * - Ephemeral executors (idle-timeout configured) initiate a graceful exit
+ *   with a non-zero code after a short delay. On the ephemeral executor,
+ *   systemd's OnFailure/ExecStopPost drain then replaces the instance, which
+ *   un-wedges everything: other in-flight jobs on the box lose their
+ *   heartbeat, go stale, and the server-side reaper resets them for retry —
+ *   that is the intended behavior.
+ */
+export declare function createExecutorHooks(opts: {
+    ephemeral: boolean;
+    exit?: (code: number) => void;
+    exitDelayMs?: number;
+    log: (msg: string) => void;
+    warn: (msg: string) => void;
+}): JobDispatchHooks;

package/dist/lib/executor-start/hooks.js

@@ -0,0 +1,42 @@
+/**
+ * Delay before the process exits after a job timeout on an ephemeral
+ * executor — long enough for the job-failed PATCH (and final warn logs)
+ * to flush to hereya cloud.
+ */
+export const JOB_TIMEOUT_EXIT_DELAY_MS = 5000;
+/**
+ * Builds the dispatch hooks for the executor start loop, including the
+ * per-job-timeout policy:
+ *
+ * - Non-ephemeral executors (no idle-timeout configured) just log a warning
+ *   and keep running — an operator owns the process lifecycle.
+ * - Ephemeral executors (idle-timeout configured) initiate a graceful exit
+ *   with a non-zero code after a short delay. On the ephemeral executor,
+ *   systemd's OnFailure/ExecStopPost drain then replaces the instance, which
+ *   un-wedges everything: other in-flight jobs on the box lose their
+ *   heartbeat, go stale, and the server-side reaper resets them for retry —
+ *   that is the intended behavior.
+ */
+export function createExecutorHooks(opts) {
+    const { ephemeral, log, warn } = opts;
+    const exitDelayMs = opts.exitDelayMs ?? JOB_TIMEOUT_EXIT_DELAY_MS;
+    const exit = opts.exit ??
+        ((code) => {
+            // eslint-disable-next-line n/no-process-exit, unicorn/no-process-exit
+            process.exit(code);
+        });
+    return {
+        log,
+        onJobTimeout(jobId) {
+            if (!ephemeral) {
+                warn(`Job ${jobId} hit the execution timeout. Executor keeps running (non-ephemeral mode); check the host for wedged child processes.`);
+                return;
+            }
+            warn(`Job ${jobId} hit the execution timeout. Ephemeral executor exits with a non-zero code in ${Math.round(exitDelayMs / 1000)}s so the instance gets replaced.`);
+            const timer = setTimeout(() => exit(1), exitDelayMs);
+            // Don't let the exit timer keep an otherwise-finished process alive.
+            timer.unref?.();
+        },
+        warn,
+    };
+}

package/dist/lib/executor-start/job-dispatch.d.ts

@@ -1,15 +1,28 @@
 import { CloudBackend } from '../../backend/cloud/cloud-backend.js';
 import { LocalExecutor } from '../../executor/local.js';
+import { KeyedMutex } from './job-serialization.js';
 export interface JobDispatchHooks {
     log(msg: string): void;
+    /**
+     * Fired after a job exceeds the execution timeout (job already PATCHed as
+     * failed). Lets the caller decide process-level policy, e.g. exit for
+     * instance replacement on an ephemeral executor.
+     */
+    onJobTimeout?(jobId: string): void;
     warn(msg: string): void;
 }
 /**
  * Dispatches a single job to the right handler, with periodic log flushing
- * and heartbeating to the cloud. Errors are caught and reported as failed jobs.
+ * and heartbeating to the cloud. Errors are caught and reported as failed
+ * jobs. Execution is raced against a per-job timeout so a hung child can
+ * never hold its slot indefinitely.
+ *
+ * `serialization` must be the executor-process-wide KeyedMutex: jobs that
+ * mutate the same logical (target, workspace) run serially on it, everything
+ * else stays concurrent. See job-serialization.ts.
  */
 export declare function dispatchJob(job: {
     id: string;
     payload: any;
     type: string;
-}, executor: LocalExecutor, cloudBackend: CloudBackend, hooks: JobDispatchHooks): Promise<void>;
+}, executor: LocalExecutor, cloudBackend: CloudBackend, hooks: JobDispatchHooks, serialization: KeyedMutex): Promise<void>;

package/dist/lib/executor-start/job-dispatch.js

@@ -1,35 +1,21 @@
 import { executeAppJob } from './execute-app-job.js';
 import { executeDeployJob } from './execute-deploy-job.js';
 import { executeInitJob } from './execute-init-job.js';
-/**
- * Builds a verbose error description for server-side job logs. If the thrown
- * error is a `ShellCommandError` (or anything carrying `.stderr`/`.stdout`),
- * include the full buffers so operators can see the complete failure context.
- */
-function formatJobError(error) {
-    const reason = error && typeof error === 'object' && 'message' in error && typeof error.message === 'string'
-        ? error.message
-        : String(error);
-    const stderrBuffer = error && typeof error === 'object' && typeof error.stderr === 'string'
-        ? error.stderr
-        : '';
-    const stdoutBuffer = error && typeof error === 'object' && typeof error.stdout === 'string'
-        ? error.stdout
-        : '';
-    let extra = '';
-    if (stderrBuffer.trim()) {
-        extra = `\n--- stderr ---\n${stderrBuffer}`;
-    }
-    else if (stdoutBuffer.trim()) {
-        extra = `\n--- stdout ---\n${stdoutBuffer}`;
-    }
-    return { fullErr: reason + extra, reason };
-}
+import { formatJobError } from './format.js';
+import { deriveJobSerializationKey } from './job-serialization.js';
+import { executorJobTimeoutMin } from './job-timeout.js';
 /**
  * Dispatches a single job to the right handler, with periodic log flushing
- * and heartbeating to the cloud. Errors are caught and reported as failed jobs.
+ * and heartbeating to the cloud. Errors are caught and reported as failed
+ * jobs. Execution is raced against a per-job timeout so a hung child can
+ * never hold its slot indefinitely.
+ *
+ * `serialization` must be the executor-process-wide KeyedMutex: jobs that
+ * mutate the same logical (target, workspace) run serially on it, everything
+ * else stays concurrent. See job-serialization.ts.
  */
-export async function dispatchJob(job, executor, cloudBackend, hooks) {
+// eslint-disable-next-line max-params -- explicit DI of the process-wide mutex beats hidden module state
+export async function dispatchJob(job, executor, cloudBackend, hooks, serialization) {
     let logBuffer = '';
     const logInterval = setInterval(async () => {
         if (logBuffer) {
@@ -62,7 +48,14 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
             logBuffer += msg + '\n';
         },
     };
+    const timeoutMin = executorJobTimeoutMin();
+    let timedOut = false;
+    let timeoutTimer;
     const finalize = async (result, label) => {
+        // The timeout path already reported this job as failed; a late completion
+        // from the still-running handler must not overwrite that terminal status.
+        if (timedOut)
+            return;
         clearInterval(logInterval);
         clearInterval(heartbeatInterval);
         await cloudBackend.updateExecutorJob({
@@ -73,7 +66,7 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
         });
         hooks.log(`Job ${job.id} ${result.success ? 'completed' : 'failed'}${label ? ` (${label})` : ''}`);
     };
-    try {
+    const execute = async () => {
         if (job.type === 'resolve-env') {
             const resolved = await executor.resolveEnvValues(job.payload);
             await finalize({ env: resolved, success: true }, 'resolve-env');
@@ -98,6 +91,63 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
             ? await executor.provision({ ...job.payload, logger })
             : await executor.destroy({ ...job.payload, logger });
         await finalize(result, '');
+    };
+    // Jobs mutating the same logical (target, workspace) — e.g. a resubmitted
+    // duplicate of a deploy that looked stuck — must run serially: concurrent
+    // runs race over the same cloned repo dirs, local hereya state and tofu/CDK
+    // state. The wait happens INSIDE the job lifecycle, AFTER the log/heartbeat
+    // intervals started above, so a queued job keeps heartbeating — otherwise
+    // the server-side stale reaper would reset it and this same executor would
+    // claim it again. The per-job execution timeout below INTENTIONALLY covers
+    // queue-wait + execution: a job stuck behind a wedged predecessor is failed
+    // and retried rather than waiting forever.
+    const serializationKey = deriveJobSerializationKey(job);
+    const executeSerialized = async () => {
+        if (serialization.isBusy(serializationKey)) {
+            logger.info(`Waiting for an in-flight job on the same target (${serializationKey}) to finish before starting...`);
+        }
+        await serialization.run(serializationKey, async () => {
+            // A job that timed out while queued must not ghost-execute after its
+            // predecessor finally drains — its row is already failed.
+            if (timedOut)
+                return;
+            await execute();
+        });
+    };
+    try {
+        const timeoutPromise = new Promise((resolve) => {
+            timeoutTimer = setTimeout(() => resolve('timeout'), timeoutMin * 60 * 1000);
+            // Belt-and-braces: a pending job timer must never keep the process alive.
+            timeoutTimer.unref?.();
+        });
+        const executePromise = executeSerialized().then(() => 'done');
+        // Keep the chain explicitly handled even after the timeout wins the race:
+        // a late rejection from the wedged handler (hung tool killed by OOM,
+        // network drop, ...) must never surface as an unhandledRejection and
+        // crash a non-ephemeral executor. The job row is already failed at that
+        // point, so just log it. When execute() rejects BEFORE the timeout, the
+        // race still rejects into the catch block below and the timedOut guard
+        // keeps this extra warn silent.
+        executePromise.catch((error) => {
+            if (timedOut) {
+                hooks.warn(`Job ${job.id} late failure after timeout: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        });
+        const outcome = await Promise.race([executePromise, timeoutPromise]);
+        if (outcome === 'timeout') {
+            timedOut = true;
+            clearInterval(logInterval);
+            clearInterval(heartbeatInterval);
+            const reason = `Job execution timed out on the executor after ${timeoutMin} minutes (HEREYA_EXECUTOR_JOB_TIMEOUT_MIN to adjust)`;
+            await cloudBackend.updateExecutorJob({
+                jobId: job.id,
+                logs: logBuffer + `\nError: ${reason}\n`,
+                result: { reason, success: false },
+                status: 'failed',
+            });
+            hooks.warn(`Job ${job.id} failed: ${reason}`);
+            hooks.onJobTimeout?.(job.id);
+        }
     }
     catch (error) {
         clearInterval(logInterval);
@@ -111,4 +161,8 @@ export async function dispatchJob(job, executor, cloudBackend, hooks) {
         });
         hooks.warn(`Job ${job.id} failed: ${reason}`);
     }
+    finally {
+        if (timeoutTimer)
+            clearTimeout(timeoutTimer);
+    }
 }

package/dist/lib/executor-start/job-serialization.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Per-(target, workspace) job serialization for the remote executor.
+ *
+ * Production incident: a user resubmitted a deploy that looked stuck, and the
+ * executor ran BOTH identical deploys of the same project+workspace
+ * concurrently. They raced each other over the same cloned repo dirs, local
+ * hereya state and tofu/CDK state, and both wedged. Jobs that mutate the same
+ * logical target in the same workspace must run serially on one executor;
+ * unrelated jobs (e.g. the normal boot burst of provisions for different
+ * packages) must keep running concurrently.
+ */
+/**
+ * Derives a stable key identifying "same logical target in the same
+ * workspace", or null when the job must NOT be serialized (read-only jobs,
+ * unknown job types, malformed payloads — those run immediately, no lock).
+ * Never throws.
+ *
+ * Key derivation per job type (the kind prefix prevents cross-kind
+ * collisions, e.g. a project named like an app):
+ *   - deploy / undeploy:        `project:<payload.project>:<payload.workspace>`
+ *   - init:                     `project:<payload.projectName>:<payload.workspace>`
+ *   - app-deploy / app-destroy: `app:<payload.appName>:<payload.workspace>`
+ *   - provision / destroy:      `pkg:<payload.package>:<payload.workspace>`
+ *   - resolve-env:              null (read-only, never serialized)
+ */
+export declare function deriveJobSerializationKey(job: {
+    id: string;
+    payload: any;
+    type: string;
+}): null | string;
+/**
+ * Minimal keyed mutex: chains async functions per key so runs with the same
+ * key execute strictly serially (FIFO), while runs with different keys — or a
+ * null key — start immediately and proceed concurrently.
+ *
+ * - A rejecting fn never blocks its successors: each fn is chained via
+ *   `.then(fn, fn)` onto a rejection-swallowed tail.
+ * - The map entry for a key is removed once its chain drains, so the map
+ *   cannot grow unboundedly over executor uptime.
+ */
+export declare class KeyedMutex {
+    private readonly chains;
+    /** Number of keys with in-flight or queued runs (introspection/test aid). */
+    get size(): number;
+    /**
+     * True when at least one run for this key is in flight or queued — i.e. a
+     * new run() with this key would have to wait. A null key is never busy.
+     */
+    isBusy(key: null | string): boolean;
+    run<T>(key: null | string, fn: () => Promise<T>): Promise<T>;
+}