@crewx/workflow 0.3.22-rc.12 → 0.3.22-rc.121

package/dist/src/run-manager.js

@@ -33,13 +33,14 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RunManager = exports.SKILL_TASK_DEFAULT_TIMEOUT = void 0;
+exports.RunManager = exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = exports.WORKFLOW_RUN_INTERRUPTED_ERROR = exports.SKILL_TASK_DEFAULT_TIMEOUT = exports.FileLock = void 0;
 exports.parseSetArgs = parseSetArgs;
 exports.readStdin = readStdin;
 exports.interpolateTemplate = interpolateTemplate;
 exports.safeEvaluate = safeEvaluate;
 exports.safeEvaluateBranchCondition = safeEvaluateBranchCondition;
 exports.validateSafeBranchCondition = validateSafeBranchCondition;
+exports.isRetryableFailure = isRetryableFailure;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const yaml = __importStar(require("js-yaml"));
@@ -48,11 +49,15 @@ const sdk_1 = require("@crewx/sdk");
 const file_lock_1 = require("./file-lock");
 const output_format_1 = require("./utils/output-format");
 const shell_security_1 = require("./utils/shell-security");
+var file_lock_2 = require("./file-lock");
+Object.defineProperty(exports, "FileLock", { enumerable: true, get: function () { return file_lock_2.FileLock; } });
 exports.SKILL_TASK_DEFAULT_TIMEOUT = 600000;
+exports.WORKFLOW_RUN_INTERRUPTED_ERROR = 'Workflow run interrupted: runner process exited before completion';
+exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = 10 * 60000;
 const KNOWN_AUTO_NODE_TYPES = new Set([
-    'agent_task', 'skill_task', 'shell_task', 'branch', 'expression', 'join', 'end',
+    'agent_task', 'skill_task', 'shell_task', 'branch', 'expression', 'join', 'end', 'error',
 ]);
-const SELF_ADVANCING_NODE_TYPES = new Set(['branch', 'expression', 'end']);
+const SELF_ADVANCING_NODE_TYPES = new Set(['branch', 'expression', 'end', 'error']);
 function killProcessGroup(child) {
     try {
         if (process.platform === 'win32') {
@@ -81,7 +86,7 @@ async function runProcess(command, argv, opts) {
             child = cp.spawn(command, argv, {
                 cwd: opts.cwd,
                 env: opts.env,
-                stdio: ['ignore', 'pipe', 'pipe'],
+                stdio: [opts.stdin !== undefined ? 'pipe' : 'ignore', 'pipe', 'pipe'],
                 shell: opts.shell ?? false,
                 detached: process.platform !== 'win32',
                 windowsHide: opts.windowsHide ?? true,
@@ -109,6 +114,9 @@ async function runProcess(command, argv, opts) {
             timer.unref();
         child.stdout?.on('data', (d) => { stdout += d.toString(); });
         child.stderr?.on('data', (d) => { stderr += d.toString(); });
+        if (opts.stdin !== undefined) {
+            child.stdin?.end(opts.stdin);
+        }
         child.on('error', (err) => finish({ status: null, stdout, stderr, timedOut: false, error: err }));
         child.on('close', (code) => finish({ status: code, stdout, stderr, timedOut: false }));
     });
@@ -521,12 +529,62 @@ function safeEvaluateBranchCondition(expr, state) {
 function validateSafeBranchCondition(expr) {
     safeEvaluateBranchCondition(expr, {});
 }
+function readRunEngineState(run) {
+    const raw = run.state['__engine'];
+    if (!raw || typeof raw !== 'object')
+        return { visitCounts: {}, iterations: 0 };
+    const record = raw;
+    const rawCounts = record['visitCounts'];
+    const visitCounts = {};
+    if (rawCounts && typeof rawCounts === 'object') {
+        for (const [key, value] of Object.entries(rawCounts)) {
+            if (typeof value === 'number' && Number.isFinite(value))
+                visitCounts[key] = value;
+        }
+    }
+    const iterations = typeof record['iterations'] === 'number' && Number.isFinite(record['iterations'])
+        ? record['iterations']
+        : 0;
+    return { visitCounts, iterations };
+}
+const inFlightAutoRuns = new Set();
+function isProcessAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    if (pid === process.pid)
+        return true;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+}
+function isTerminalStatus(status) {
+    return status === 'completed' || status === 'failed' || status === 'cancelled';
+}
+function isRetryableFailure(run, spec) {
+    if (run.status !== 'failed')
+        return false;
+    if (!run.current_node)
+        return false;
+    const node = spec.nodes?.[run.current_node];
+    if (!node)
+        return false;
+    return node.type !== 'error';
+}
 class RunManager {
     constructor(runsDir = '.crewx/workflow-runs', projectRoot) {
         this.runsDir = runsDir;
         const base = projectRoot || process.env.CREWX_WORKSPACE || process.cwd();
         this.resolvedDir = path.resolve(base, this.runsDir);
         this.projectRoot = path.resolve(base);
+        const hasConfig = fs.existsSync(path.join(this.projectRoot, 'crewx.yaml'))
+            || fs.existsSync(path.join(this.projectRoot, 'crewx.yml'));
+        if (!hasConfig) {
+            console.warn(`[RunManager] projectRoot "${this.projectRoot}" has no crewx.yaml/crewx.yml — agent_task nodes may fail to resolve agents.`);
+        }
     }
     ensureRunsDir() {
         if (!fs.existsSync(this.resolvedDir)) {
@@ -538,8 +596,7 @@ class RunManager {
             throw new Error(`Invalid execution ID: "${execId}"`);
         }
     }
-    loadRun(execId) {
-        this.validateExecId(execId);
+    readRun(execId) {
         const filePath = path.join(this.resolvedDir, `${execId}.json`);
         if (!fs.existsSync(filePath))
             return null;
@@ -550,6 +607,78 @@ class RunManager {
             throw new Error(`Failed to parse execution file ${filePath}: ${e.message}`);
         }
     }
+    isLegacyRetryStale(run) {
+        if (run.status !== undefined
+            || run.auto_run
+            || run.manual_resume
+            || !run.current_node) {
+            return false;
+        }
+        const lastAudit = run.audit?.at(-1);
+        if (lastAudit?.type !== 'retry')
+            return false;
+        const updatedAt = Date.parse(run.updated_at);
+        if (!Number.isFinite(updatedAt) || Date.now() - updatedAt < exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS) {
+            return false;
+        }
+        const doc = this.loadWorkflowYaml(run.workflow_file);
+        const spec = doc?.workflows[run.workflow_id];
+        const node = spec?.nodes?.[run.current_node];
+        return Boolean(node && KNOWN_AUTO_NODE_TYPES.has(node.type));
+    }
+    hasStaleRunner(run) {
+        if (run.auto_run)
+            return !isProcessAlive(run.auto_run.pid);
+        return this.isLegacyRetryStale(run);
+    }
+    recoverStaleRunner(execId) {
+        const filePath = path.join(this.resolvedDir, `${execId}.json`);
+        const lock = new file_lock_1.FileLock(filePath);
+        lock.acquire();
+        try {
+            const run = this.readRun(execId);
+            if (!run || !this.hasStaleRunner(run))
+                return run;
+            if (isTerminalStatus(run.status)) {
+                delete run.auto_run;
+                delete run.manual_resume;
+                this.saveRun(run);
+                return run;
+            }
+            const lease = run.auto_run;
+            const nodeId = lease?.node_id || run.current_node;
+            const now = new Date().toISOString();
+            run.status = 'failed';
+            run.error = exports.WORKFLOW_RUN_INTERRUPTED_ERROR;
+            delete run.auto_run;
+            delete run.manual_resume;
+            run.audit = run.audit ?? [];
+            run.audit.push({
+                exec_id: run.id,
+                node_id: nodeId,
+                type: 'interrupted',
+                summary: exports.WORKFLOW_RUN_INTERRUPTED_ERROR,
+                started_at: now,
+                ended_at: now,
+                duration_ms: 0,
+                exit_code: null,
+                error: exports.WORKFLOW_RUN_INTERRUPTED_ERROR,
+                trigger: 'auto',
+            });
+            this.saveRun(run);
+            return run;
+        }
+        finally {
+            lock.release();
+        }
+    }
+    loadRun(execId) {
+        this.validateExecId(execId);
+        const run = this.readRun(execId);
+        if (!run || !this.hasStaleRunner(run))
+            return run;
+        return this.recoverStaleRunner(execId);
+    }
     saveRun(run) {
         this.ensureRunsDir();
         run.updated_at = new Date().toISOString();
@@ -563,7 +692,7 @@ class RunManager {
         const lock = new file_lock_1.FileLock(filePath);
         lock.acquire();
         try {
-            const run = this.loadRun(execId);
+            const run = this.readRun(execId);
             if (!run)
                 throw new Error(`Execution not found: ${execId}`);
             updater(run);
@@ -581,7 +710,7 @@ class RunManager {
         const lock = new file_lock_1.FileLock(filePath);
         await lock.acquireAsync();
         try {
-            const run = this.loadRun(execId);
+            const run = this.readRun(execId);
             if (!run)
                 throw new Error(`Execution not found: ${execId}`);
             updater(run);
@@ -601,6 +730,12 @@ class RunManager {
             return null;
         return doc;
     }
+    getNodeType(workflowFile, workflowId, nodeId) {
+        const doc = this.loadWorkflowYaml(workflowFile);
+        const spec = doc?.workflows[workflowId];
+        const node = spec?.nodes?.[nodeId];
+        return typeof node?.type === 'string' ? node.type : null;
+    }
     start(yamlPath, workflowId, overrides) {
         const doc = this.loadWorkflowYaml(yamlPath);
         if (!doc) {
@@ -684,6 +819,7 @@ class RunManager {
         const nodesToComplete = [];
         let currentNode = nodeId;
         let status;
+        let failureMessage;
         let nodeTaskId;
         const trigger = opts?.trigger ?? 'manual';
         const startedAt = new Date();
@@ -711,19 +847,39 @@ class RunManager {
                     if (!nodeSpec.agent) {
                         throw new Error(`Node "${nodeId}" is agent_task but has no agent defined`);
                     }
+                    const agentCtx = { ...snapshot.state, state: snapshot.state };
+                    const resolvedAgent = interpolateTemplate(nodeSpec.agent, agentCtx).trim();
+                    if (!resolvedAgent) {
+                        throw new Error(`Node "${nodeId}": assignee not resolved (agent field "${nodeSpec.agent}" interpolated to empty)`);
+                    }
+                    const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, agentCtx) : undefined, this.projectRoot);
+                    if (!cwdCheck.ok)
+                        throw new Error(`Node "${nodeId}": ${cwdCheck.error}`);
+                    const nodeCwd = cwdCheck.resolved;
+                    const rootCheck = (0, shell_security_1.normalizeCwd)(undefined, this.projectRoot);
+                    if (!rootCheck.ok)
+                        throw new Error(`Node "${nodeId}": ${rootCheck.error}`);
+                    const workspaceRoot = rootCheck.resolved;
                     const input = nodeSpec.input
-                        ? interpolateTemplate(nodeSpec.input, { ...snapshot.state, state: snapshot.state })
+                        ? interpolateTemplate(nodeSpec.input, agentCtx)
                         : '';
+                    const preamble = process.env.CREWX_WORKFLOW_PREAMBLE === 'off'
+                        ? ''
+                        : (0, output_format_1.buildWorkflowPreamble)({
+                            workflowId: snapshot.workflow_id,
+                            nodeId,
+                            execId: snapshot.id,
+                        });
                     const finalInput = nodeSpec.output_format === 'json'
-                        ? input + (0, output_format_1.buildJsonOutputSuffix)(nodeSpec.output_schema, nodeSpec.output_strict)
-                        : input;
+                        ? preamble + input + (0, output_format_1.buildJsonOutputSuffix)(nodeSpec.output_schema, nodeSpec.output_strict)
+                        : preamble + input;
                     const mode = nodeSpec.mode === 'query' ? 'q' : 'x';
-                    const prompt = `@${nodeSpec.agent} ${finalInput}`;
+                    const prompt = `@${resolvedAgent} ${finalInput}`;
                     if (opts?.dryRun) {
-                        console.log(`[dry-run] Would execute: crewx ${mode} "@${nodeSpec.agent} ..."`);
+                        console.log(`[dry-run] Would execute: crewx ${mode} "@${resolvedAgent} ..."`);
                         break;
                     }
-                    console.log(`Executing: @${nodeSpec.agent} (${mode} mode)`);
+                    console.log(`Executing: @${resolvedAgent} (${mode} mode)`);
                     const crewxCli = process.env.CREWX_CLI || 'npx crewx';
                     const cliParts = crewxCli.split(/\s+/);
                     const bin = cliParts[0];
@@ -748,9 +904,10 @@ class RunManager {
                     });
                     const invocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, wfArgs);
                     const result = await runProcess(invocation.command, invocation.argv, {
-                        cwd: process.cwd(),
+                        cwd: nodeCwd,
                         env: {
                             ...process.env,
+                            CREWX_WORKSPACE: workspaceRoot,
                             CREWX_WORKFLOW_EXEC_ID: snapshot.id,
                             CREWX_WORKFLOW_NODE_ID: nodeId,
                             CREWX_WORKFLOW_ID: snapshot.workflow_id,
@@ -775,6 +932,10 @@ class RunManager {
                         throw new Error(`Agent execution failed (exit ${result.status}): ${stderr}`);
                     }
                     const output = result.stdout.trim();
+                    const signal = (0, output_format_1.detectWorkflowSignal)(output);
+                    if (signal) {
+                        throw new Error(`Node "${nodeId}" agent signaled failure: ${signal.reason ?? '(no reason given)'}`);
+                    }
                     if (nodeSpec.output) {
                         if (nodeSpec.output_format === 'json') {
                             const retryMax = nodeSpec.output_retry ?? 1;
@@ -787,7 +948,7 @@ class RunManager {
                                     retryInput = finalInput +
                                         `\n\n## ⚠️ PREVIOUS OUTPUT FAILED JSON.parse: ${lastError.message}\n` +
                                         `Return STRICT JSON ONLY. First char "{", last char "}". No prose, no fences.`;
-                                    const retryPrompt = `@${nodeSpec.agent} ${retryInput}`;
+                                    const retryPrompt = `@${resolvedAgent} ${retryInput}`;
                                     const retryArgs = [...baseArgs, mode, retryPrompt];
                                     if (process.env.CREWX_WORKFLOW_THREAD === 'on') {
                                         retryArgs.push(`--thread=workflow:${snapshot.id}`);
@@ -795,9 +956,10 @@ class RunManager {
                                     retryArgs.push('--metadata', JSON.stringify(wfMetadata));
                                     const retryInvocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, retryArgs);
                                     const retryResult = await runProcess(retryInvocation.command, retryInvocation.argv, {
-                                        cwd: process.cwd(),
+                                        cwd: nodeCwd,
                                         env: {
                                             ...process.env,
+                                            CREWX_WORKSPACE: workspaceRoot,
                                             CREWX_WORKFLOW_EXEC_ID: snapshot.id,
                                             CREWX_WORKFLOW_NODE_ID: nodeId,
                                             CREWX_WORKFLOW_ID: snapshot.workflow_id,
@@ -817,6 +979,10 @@ class RunManager {
                                         throw new Error(`Agent retry failed (exit ${retryResult.status}): ${retryResult.stderr.trim()}`);
                                     }
                                     attemptOutput = retryResult.stdout.trim();
+                                    const retrySignal = (0, output_format_1.detectWorkflowSignal)(attemptOutput);
+                                    if (retrySignal) {
+                                        throw new Error(`Node "${nodeId}" agent signaled failure: ${retrySignal.reason ?? '(no reason given)'}`);
+                                    }
                                 }
                                 try {
                                     parsed = JSON.parse((0, output_format_1.extractJson)(attemptOutput));
@@ -851,9 +1017,16 @@ class RunManager {
                     }
                     const ctx = { ...snapshot.state, state: snapshot.state };
                     const skillArgs = (nodeSpec.args ?? []).map((a) => interpolateTemplate(String(a), ctx));
+                    const skillStdin = nodeSpec.stdin !== undefined
+                        ? interpolateTemplate(String(nodeSpec.stdin), ctx)
+                        : undefined;
                     const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, ctx) : undefined, this.projectRoot);
                     if (!cwdCheck.ok)
                         throw new Error(`Node "${nodeId}": ${cwdCheck.error}`);
+                    const skillRootCheck = (0, shell_security_1.normalizeCwd)(undefined, this.projectRoot);
+                    if (!skillRootCheck.ok)
+                        throw new Error(`Node "${nodeId}": ${skillRootCheck.error}`);
+                    const skillWorkspaceRoot = skillRootCheck.resolved;
                     const timeout = nodeSpec.timeout ?? exports.SKILL_TASK_DEFAULT_TIMEOUT;
                     if (opts?.dryRun) {
                         console.log(`[dry-run] Would execute: crewx skill ${nodeSpec.skill} ${skillArgs.join(' ')}`.trim());
@@ -875,6 +1048,7 @@ class RunManager {
                         cwd: cwdCheck.resolved,
                         env: {
                             ...process.env,
+                            CREWX_WORKSPACE: skillWorkspaceRoot,
                             CREWX_WORKFLOW_EXEC_ID: snapshot.id,
                             CREWX_WORKFLOW_NODE_ID: nodeId,
                             CREWX_WORKFLOW_ID: snapshot.workflow_id,
@@ -882,6 +1056,7 @@ class RunManager {
                         timeout,
                         shell: invocation.shell ?? false,
                         windowsHide: invocation.windowsHide,
+                        stdin: skillStdin,
                     });
                     nodeStdout = result.stdout;
                     nodeStderr = result.stderr;
@@ -932,11 +1107,15 @@ class RunManager {
                     childEnv.CREWX_WORKFLOW_EXEC_ID = snapshot.id;
                     childEnv.CREWX_WORKFLOW_NODE_ID = nodeId;
                     childEnv.CREWX_WORKFLOW_ID = snapshot.workflow_id;
+                    const shellStdin = nodeSpec.stdin !== undefined
+                        ? interpolateTemplate(String(nodeSpec.stdin), ctx)
+                        : undefined;
                     const result = await runProcess(command[0], command.slice(1), {
                         cwd: cwdCheck.resolved,
                         env: childEnv,
                         timeout,
                         shell: false,
+                        stdin: shellStdin,
                     });
                     nodeStdout = result.stdout;
                     nodeStderr = result.stderr;
@@ -986,6 +1165,15 @@ class RunManager {
                     console.log(`End "${nodeId}": workflow execution completed`);
                     break;
                 }
+                case 'error': {
+                    status = 'failed';
+                    const ctx = { ...snapshot.state, state: snapshot.state };
+                    failureMessage = nodeSpec.message
+                        ? interpolateTemplate(nodeSpec.message, ctx)
+                        : `Workflow failed at error node "${nodeId}"`;
+                    console.log(`Error "${nodeId}": workflow execution failed — ${failureMessage}`);
+                    break;
+                }
                 case 'expression': {
                     if (!nodeSpec.set || typeof nodeSpec.set !== 'object') {
                         throw new Error(`Expression node "${nodeId}" requires "set" field`);
@@ -1009,6 +1197,14 @@ class RunManager {
                 default:
                     console.log(`Unknown node type "${nodeSpec.type}" for node "${nodeId}"`);
             }
+            if (nodeSpec.fail_when &&
+                !opts?.dryRun &&
+                (nodeSpec.type === 'agent_task' || nodeSpec.type === 'skill_task' || nodeSpec.type === 'shell_task')) {
+                const judged = safeEvaluateBranchCondition(String(nodeSpec.fail_when), { ...snapshot.state, ...stateUpdates });
+                if (judged === true || judged === 'true') {
+                    throw new Error(`Node "${nodeId}" judged failed by fail_when: ${nodeSpec.fail_when}`);
+                }
+            }
         }
         catch (e) {
             const errMessage = e.message;
@@ -1020,6 +1216,8 @@ class RunManager {
             await this.atomicUpdateAsync(execId, (run) => {
                 if (run.status === 'cancelled')
                     return;
+                Object.assign(run.state, stateUpdates);
+                delete run.manual_resume;
                 run.current_node = nodeId;
                 run.status = 'failed';
                 run.error = errMessage;
@@ -1033,6 +1231,7 @@ class RunManager {
             if (run.status === 'cancelled')
                 return;
             Object.assign(run.state, stateUpdates);
+            delete run.manual_resume;
             for (const node of nodesToComplete) {
                 if (!run.completed_nodes.includes(node)) {
                     run.completed_nodes.push(node);
@@ -1042,8 +1241,18 @@ class RunManager {
                 run.completed_nodes.push(prevNode);
             }
             run.current_node = currentNode;
-            if (status)
+            if (status) {
                 run.status = status;
+                if (status !== 'failed') {
+                    delete run.error;
+                }
+            }
+            else if (run.status === 'failed') {
+                run.status = 'running';
+                delete run.error;
+            }
+            if (failureMessage)
+                run.error = failureMessage;
             if (nodeTaskId) {
                 run.tasks = run.tasks ?? {};
                 run.tasks[nodeId] = nodeTaskId;
@@ -1052,92 +1261,155 @@ class RunManager {
             run.audit.push(successAudit);
         });
     }
-    async runAuto(execId) {
-        const initial = this.loadRun(execId);
-        if (!initial)
-            throw new Error(`Execution not found: ${execId}`);
-        const doc = this.loadWorkflowYaml(initial.workflow_file);
-        if (!doc)
-            throw new Error(`Failed to load workflow file: ${initial.workflow_file}`);
-        const spec = doc.workflows[initial.workflow_id];
-        if (!spec) {
-            throw new Error(`Workflow "${initial.workflow_id}" not found in ${initial.workflow_file}`);
-        }
-        const executed = [];
-        if (initial.status === 'completed' || initial.status === 'failed' || initial.status === 'cancelled') {
-            return {
-                run: initial,
-                outcome: initial.status === 'completed' ? 'completed' : initial.status === 'cancelled' ? 'cancelled' : 'failed',
-                reason: `run already ${initial.status}`,
-                executed,
+    async claimAutoRunLease(execId, nodeId, preferredToken) {
+        const updated = await this.atomicUpdateAsync(execId, (run) => {
+            const existing = run.auto_run;
+            const ownedByThisProcess = existing?.pid === process.pid && typeof existing.token === 'string';
+            if (existing && !ownedByThisProcess && isProcessAlive(existing.pid)) {
+                throw new Error(`Workflow run "${execId}" is already owned by another runner process`);
+            }
+            const token = preferredToken ?? (ownedByThisProcess ? existing.token : (0, sdk_1.generateId)('wfr'));
+            run.auto_run = {
+                pid: process.pid,
+                token,
+                node_id: nodeId,
+                started_at: ownedByThisProcess && existing.token === token
+                    ? existing.started_at
+                    : new Date().toISOString(),
             };
+            delete run.manual_resume;
+        });
+        return updated.auto_run.token;
+    }
+    async releaseAutoRunLease(execId, token) {
+        await this.atomicUpdateAsync(execId, (run) => {
+            if (run.auto_run?.token === token)
+                delete run.auto_run;
+        });
+    }
+    async runAuto(execId) {
+        const leaseKey = `${this.resolvedDir}::${execId}`;
+        if (inFlightAutoRuns.has(leaseKey)) {
+            const run = this.loadRun(execId);
+            if (!run)
+                throw new Error(`Execution not found: ${execId}`);
+            console.log(`runAuto "${execId}": already in-flight — skipping duplicate auto-run (single-flight guard)`);
+            return { run, outcome: 'paused', reason: 'execution already in-flight (single-flight guard)', executed: [] };
         }
-        const firstNode = Object.keys(spec.nodes ?? {})[0];
-        const maxIterations = (typeof spec.max_iterations === 'number' && spec.max_iterations > 0)
-            ? spec.max_iterations
-            : 30;
-        const visitCounts = {};
-        let iterations = 0;
-        let run = initial;
-        while (true) {
-            run = this.loadRun(execId);
-            if (run.status === 'completed')
-                return { run, outcome: 'completed', executed };
-            if (run.status === 'failed')
-                return { run, outcome: 'failed', reason: run.error, executed };
-            if (run.status === 'cancelled')
-                return { run, outcome: 'cancelled', reason: run.error, executed };
-            const nodeId = run.current_node || firstNode;
-            if (!nodeId)
-                return { run, outcome: 'failed', reason: 'workflow has no nodes', executed };
-            const node = spec.nodes[nodeId];
-            if (!node) {
-                run = await this.atomicUpdateAsync(execId, (r) => {
-                    r.current_node = nodeId;
-                    r.status = 'failed';
-                    r.error = `node "${nodeId}" not found in workflow`;
-                });
-                return { run, outcome: 'failed', reason: run.error, executed };
-            }
-            if (node.type === 'approval') {
-                return { run, outcome: 'paused', reason: `approval node "${nodeId}" requires manual decision`, executed };
+        inFlightAutoRuns.add(leaseKey);
+        let runnerToken;
+        try {
+            const initial = this.loadRun(execId);
+            if (!initial)
+                throw new Error(`Execution not found: ${execId}`);
+            if (initial.auto_run?.pid === process.pid) {
+                runnerToken = initial.auto_run.token;
             }
-            if (node.type === 'parallel') {
-                return { run, outcome: 'paused', reason: `parallel auto-run not implemented (node "${nodeId}")`, executed };
+            const doc = this.loadWorkflowYaml(initial.workflow_file);
+            if (!doc)
+                throw new Error(`Failed to load workflow file: ${initial.workflow_file}`);
+            const spec = doc.workflows[initial.workflow_id];
+            if (!spec) {
+                throw new Error(`Workflow "${initial.workflow_id}" not found in ${initial.workflow_file}`);
             }
-            if (!KNOWN_AUTO_NODE_TYPES.has(node.type)) {
-                return { run, outcome: 'paused', reason: `unknown node type "${node.type}" (node "${nodeId}")`, executed };
+            const executed = [];
+            if (initial.status === 'completed' || initial.status === 'failed' || initial.status === 'cancelled') {
+                return {
+                    run: initial,
+                    outcome: initial.status === 'completed' ? 'completed' : initial.status === 'cancelled' ? 'cancelled' : 'failed',
+                    reason: `run already ${initial.status}`,
+                    executed,
+                };
             }
-            visitCounts[nodeId] = (visitCounts[nodeId] ?? 0) + 1;
-            iterations++;
-            if (visitCounts[nodeId] > maxIterations || iterations > maxIterations) {
+            const firstNode = Object.keys(spec.nodes ?? {})[0];
+            const maxIterations = (typeof spec.max_iterations === 'number' && spec.max_iterations > 0)
+                ? spec.max_iterations
+                : 30;
+            let run = initial;
+            while (true) {
+                run = this.loadRun(execId);
+                if (run.status === 'completed')
+                    return { run, outcome: 'completed', executed };
+                if (run.status === 'failed')
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                if (run.status === 'cancelled')
+                    return { run, outcome: 'cancelled', reason: run.error, executed };
+                const nodeId = run.current_node || firstNode;
+                if (!nodeId)
+                    return { run, outcome: 'failed', reason: 'workflow has no nodes', executed };
+                const node = spec.nodes[nodeId];
+                if (!node) {
+                    run = await this.atomicUpdateAsync(execId, (r) => {
+                        r.current_node = nodeId;
+                        r.status = 'failed';
+                        r.error = `node "${nodeId}" not found in workflow`;
+                    });
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
+                if (node.type === 'approval') {
+                    if (run.current_node !== nodeId) {
+                        run = await this.atomicUpdateAsync(execId, (r) => { r.current_node = nodeId; });
+                    }
+                    return { run, outcome: 'paused', reason: `approval node "${nodeId}" requires manual decision`, executed };
+                }
+                if (node.type === 'parallel') {
+                    if (run.current_node !== nodeId) {
+                        run = await this.atomicUpdateAsync(execId, (r) => { r.current_node = nodeId; });
+                    }
+                    return { run, outcome: 'paused', reason: `parallel auto-run not implemented (node "${nodeId}")`, executed };
+                }
+                if (!KNOWN_AUTO_NODE_TYPES.has(node.type)) {
+                    if (run.current_node !== nodeId) {
+                        run = await this.atomicUpdateAsync(execId, (r) => { r.current_node = nodeId; });
+                    }
+                    return { run, outcome: 'paused', reason: `unknown node type "${node.type}" (node "${nodeId}")`, executed };
+                }
+                runnerToken = await this.claimAutoRunLease(execId, nodeId, runnerToken);
                 run = await this.atomicUpdateAsync(execId, (r) => {
-                    r.status = 'failed';
-                    r.error = `auto runner exceeded max_iterations (${maxIterations}); possible loop at "${nodeId}"`;
+                    const engine = readRunEngineState(r);
+                    engine.visitCounts[nodeId] = (engine.visitCounts[nodeId] ?? 0) + 1;
+                    engine.iterations += 1;
+                    r.state['__engine'] = engine;
+                    if (engine.visitCounts[nodeId] > maxIterations || engine.iterations > maxIterations) {
+                        r.status = 'failed';
+                        r.error = `auto runner exceeded max_iterations (${maxIterations}); possible loop at "${nodeId}"`;
+                    }
                 });
-                return { run, outcome: 'failed', reason: run.error, executed };
-            }
-            try {
-                run = await this.executeNode(execId, nodeId, { trigger: 'auto' });
-                executed.push(nodeId);
-            }
-            catch (e) {
-                run = this.loadRun(execId);
-                return { run, outcome: 'failed', reason: e.message, executed };
+                if (run.status === 'failed') {
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
+                try {
+                    run = await this.executeNode(execId, nodeId, { trigger: 'auto' });
+                    executed.push(nodeId);
+                }
+                catch (e) {
+                    run = this.loadRun(execId);
+                    return { run, outcome: 'failed', reason: e.message, executed };
+                }
+                if (run.status === 'completed')
+                    return { run, outcome: 'completed', executed };
+                if (run.status === 'failed')
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                if (run.status === 'cancelled')
+                    return { run, outcome: 'cancelled', reason: run.error, executed };
+                if (!SELF_ADVANCING_NODE_TYPES.has(node.type)) {
+                    const next = node.next;
+                    if (!next) {
+                        return { run, outcome: 'paused', reason: `node "${nodeId}" has no "next"; stopping`, executed };
+                    }
+                    run = this.moveNode(execId, next);
+                }
             }
-            if (run.status === 'completed')
-                return { run, outcome: 'completed', executed };
-            if (run.status === 'failed')
-                return { run, outcome: 'failed', reason: run.error, executed };
-            if (run.status === 'cancelled')
-                return { run, outcome: 'cancelled', reason: run.error, executed };
-            if (!SELF_ADVANCING_NODE_TYPES.has(node.type)) {
-                const next = node.next;
-                if (!next) {
-                    return { run, outcome: 'paused', reason: `node "${nodeId}" has no "next"; stopping`, executed };
+        }
+        finally {
+            if (runnerToken) {
+                try {
+                    await this.releaseAutoRunLease(execId, runnerToken);
+                }
+                catch {
                 }
-                run = this.moveNode(execId, next);
             }
+            inFlightAutoRuns.delete(leaseKey);
         }
     }
     cancelRun(execId, reason = 'Cancelled by user') {
@@ -1148,6 +1420,8 @@ class RunManager {
             const now = new Date();
             run.status = 'cancelled';
             run.error = reason;
+            delete run.auto_run;
+            delete run.manual_resume;
             run.audit = run.audit ?? [];
             run.audit.push({
                 exec_id: run.id,
@@ -1168,6 +1442,57 @@ class RunManager {
             run.state = JSON.parse(JSON.stringify(run.initial_state));
             run.current_node = '';
             run.completed_nodes = [];
+            delete run.auto_run;
+            delete run.manual_resume;
+        });
+    }
+    retry(execId, options = {}) {
+        const snapshot = this.loadRun(execId);
+        if (!snapshot)
+            throw new Error(`Execution not found: ${execId}`);
+        const doc = this.loadWorkflowYaml(snapshot.workflow_file);
+        if (!doc)
+            throw new Error(`Failed to load workflow file: ${snapshot.workflow_file}`);
+        const spec = doc.workflows[snapshot.workflow_id];
+        if (!spec)
+            throw new Error(`Workflow "${snapshot.workflow_id}" not found`);
+        if (snapshot.status !== 'failed') {
+            throw new Error(`Run "${execId}" is not failed (status=${snapshot.status ?? 'running'}); only failed runs can be retried`);
+        }
+        if (!isRetryableFailure(snapshot, spec)) {
+            throw new Error(`Run "${execId}" failed at a non-retryable node "${snapshot.current_node}" (error node); use reset to restart`);
+        }
+        return this.atomicUpdate(execId, (run) => {
+            const failedNode = run.current_node;
+            delete run.status;
+            delete run.error;
+            delete run.manual_resume;
+            run.retry_count = (run.retry_count ?? 0) + 1;
+            run.audit = run.audit ?? [];
+            const now = new Date().toISOString();
+            if (options.autoRun === false) {
+                delete run.auto_run;
+                run.manual_resume = { node_id: failedNode, requested_at: now };
+            }
+            else {
+                run.auto_run = {
+                    pid: process.pid,
+                    token: (0, sdk_1.generateId)('wfr'),
+                    node_id: failedNode,
+                    started_at: now,
+                };
+            }
+            run.audit.push({
+                exec_id: run.id,
+                node_id: failedNode,
+                type: 'retry',
+                summary: `manual retry #${run.retry_count} of node "${failedNode}"`,
+                started_at: now,
+                ended_at: now,
+                duration_ms: 0,
+                exit_code: null,
+                trigger: 'manual',
+            });
         });
     }
     listRuns(opts) {
@@ -1176,7 +1501,10 @@ class RunManager {
         const runs = [];
         for (const file of files) {
             try {
-                const run = JSON.parse(fs.readFileSync(path.join(this.resolvedDir, file), 'utf-8'));
+                const execId = file.slice(0, -'.json'.length);
+                const run = this.loadRun(execId);
+                if (!run)
+                    continue;
                 if (opts?.workflowId && run.workflow_id !== opts.workflowId)
                     continue;
                 runs.push(run);