@crewx/workflow 0.3.22-rc.128 → 0.3.22-rc.129

package/dist/src/run-manager.js

@@ -40,24 +40,46 @@ exports.interpolateTemplate = interpolateTemplate;
 exports.safeEvaluate = safeEvaluate;
 exports.safeEvaluateBranchCondition = safeEvaluateBranchCondition;
 exports.validateSafeBranchCondition = validateSafeBranchCondition;
+exports.normalizeRunStatus = normalizeRunStatus;
 exports.isRetryableFailure = isRetryableFailure;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const yaml = __importStar(require("js-yaml"));
 const cp = __importStar(require("child_process"));
 const sdk_1 = require("@crewx/sdk");
+const failure_1 = require("./failure");
 const file_lock_1 = require("./file-lock");
 const output_format_1 = require("./utils/output-format");
 const shell_security_1 = require("./utils/shell-security");
+const retry_policy_1 = require("./retry-policy");
 var file_lock_2 = require("./file-lock");
 Object.defineProperty(exports, "FileLock", { enumerable: true, get: function () { return file_lock_2.FileLock; } });
 exports.SKILL_TASK_DEFAULT_TIMEOUT = 600000;
 exports.WORKFLOW_RUN_INTERRUPTED_ERROR = 'Workflow run interrupted: runner process exited before completion';
 exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = 10 * 60000;
+const RETRY_WAIT_SLICE_MS = 1000;
+function sleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
 const KNOWN_AUTO_NODE_TYPES = new Set([
     'agent_task', 'skill_task', 'shell_task', 'branch', 'expression', 'join', 'end', 'error',
 ]);
 const SELF_ADVANCING_NODE_TYPES = new Set(['branch', 'expression', 'end', 'error']);
+function resolveCrewxChildInvocation(env, cwd) {
+    const resolution = (0, sdk_1.resolveCrewxExecutable)({ env, cwd });
+    if (!resolution.ok) {
+        throw (0, failure_1.createWorkflowFailureError)('CLI_UNRESOLVABLE', (0, sdk_1.formatCrewxExecutableFailure)(resolution));
+    }
+    return {
+        argv: resolution.argv,
+        env: {
+            ...env,
+            CREWX_CLI_ARGV: JSON.stringify(resolution.argv),
+        },
+    };
+}
 function killProcessGroup(child) {
     try {
         if (process.platform === 'win32') {
@@ -93,12 +115,23 @@ async function runProcess(command, argv, opts) {
             });
         }
         catch (err) {
-            resolve({ status: null, stdout: '', stderr: '', timedOut: false, error: err });
+            resolve({
+                status: null,
+                stdout: '',
+                stderr: '',
+                timedOut: false,
+                signal: null,
+                errorOrigin: 'spawn',
+                error: err,
+            });
             return;
         }
         let stdout = '';
         let stderr = '';
         let settled = false;
+        let spawned = false;
+        let postSpawnError;
+        let stdinError;
         const finish = (result) => {
             if (settled)
                 return;
@@ -108,38 +141,120 @@ async function runProcess(command, argv, opts) {
         };
         const timer = setTimeout(() => {
             killProcessGroup(child);
-            finish({ status: null, stdout, stderr, timedOut: true });
+            finish({
+                status: null,
+                stdout,
+                stderr,
+                timedOut: true,
+                signal: null,
+                errorOrigin: stdinError ? 'stdin' : postSpawnError ? 'post_spawn' : null,
+                ...(stdinError || postSpawnError ? { error: stdinError ?? postSpawnError } : {}),
+            });
         }, opts.timeout);
         if (typeof timer.unref === 'function')
             timer.unref();
         child.stdout?.on('data', (d) => { stdout += d.toString(); });
         child.stderr?.on('data', (d) => { stderr += d.toString(); });
+        child.once('spawn', () => { spawned = true; });
+        child.on('error', (err) => {
+            if (!spawned) {
+                finish({
+                    status: null,
+                    stdout,
+                    stderr,
+                    timedOut: false,
+                    signal: null,
+                    errorOrigin: 'spawn',
+                    error: err,
+                });
+                return;
+            }
+            postSpawnError = postSpawnError ?? err;
+        });
         if (opts.stdin !== undefined) {
-            child.stdin?.end(opts.stdin);
+            child.stdin?.on('error', (err) => {
+                stdinError = stdinError ?? err;
+            });
+            try {
+                child.stdin?.end(opts.stdin);
+            }
+            catch (err) {
+                stdinError = stdinError ?? err;
+            }
         }
-        child.on('error', (err) => finish({ status: null, stdout, stderr, timedOut: false, error: err }));
-        child.on('close', (code) => finish({ status: code, stdout, stderr, timedOut: false }));
+        child.on('close', (code, signal) => finish({
+            status: code,
+            stdout,
+            stderr,
+            timedOut: false,
+            signal,
+            errorOrigin: stdinError ? 'stdin' : postSpawnError ? 'post_spawn' : null,
+            ...(stdinError || postSpawnError ? { error: stdinError ?? postSpawnError } : {}),
+        }));
     });
 }
-function maskArg(arg) {
-    return arg.replace(/^(--?[\w-]*(?:token|secret|password|passwd|key|auth)[\w-]*)=(.+)$/i, '$1=***');
+const AUDIT_IDENTIFIER_RE = /^[A-Za-z0-9_.-]{1,128}$/;
+function safeAuditIdentifier(value) {
+    if (value === undefined)
+        return undefined;
+    const match = AUDIT_IDENTIFIER_RE.exec(value);
+    return match?.[0] === value ? value : undefined;
 }
 function summarizeNode(node) {
     switch (node.type) {
-        case 'agent_task':
-            return `@${node.agent ?? '?'} (${node.mode === 'query' ? 'q' : 'x'})`;
-        case 'skill_task':
-            return `skill ${node.skill ?? '?'} ${(node.args ?? []).map(maskArg).join(' ')}`.trim();
+        case 'agent_task': {
+            const agent = safeAuditIdentifier(node.agent);
+            return agent
+                ? `@${agent} (${node.mode === 'query' ? 'q' : 'x'})`
+                : `agent_task (${node.mode === 'query' ? 'q' : 'x'})`;
+        }
+        case 'skill_task': {
+            const skill = safeAuditIdentifier(node.skill);
+            return skill ? `skill ${skill}` : 'skill_task';
+        }
         case 'shell_task':
-            return (node.command ?? []).map(maskArg).join(' ');
+            return 'shell_task';
         case 'branch':
-            return `branch: ${node.condition ?? ''}`;
+            return 'branch';
         case 'expression':
-            return `set ${Object.keys(node.set ?? {}).join(', ')}`;
+            return 'expression';
         default:
-            return node.type;
+            return 'workflow node';
     }
 }
+function getErrorMessage(error) {
+    if (error instanceof Error)
+        return error.message;
+    return typeof error === 'string' ? error : 'Node execution failed';
+}
+function toProcessFailureMetadata(result) {
+    return {
+        status: result.status,
+        timedOut: result.timedOut,
+        signal: result.signal,
+        errorOrigin: result.errorOrigin,
+    };
+}
+function appendFailureAudit(run, failure, type, trigger, now = new Date().toISOString()) {
+    run.audit = run.audit ?? [];
+    run.audit.push({
+        exec_id: run.id,
+        node_id: failure.failedNode,
+        type,
+        summary: failure.causeMessage,
+        started_at: now,
+        ended_at: now,
+        duration_ms: 0,
+        exit_code: failure.exitCode,
+        error: failure.causeMessage,
+        trigger,
+    });
+}
+function assignRunFailure(run, failure, rawError) {
+    run.status = 'failed';
+    run.error = (0, failure_1.sanitizeLocalRunError)(rawError, failure.causeMessage);
+    run.last_failure = failure;
+}
 async function storeNodeOutput(nodeSpec, nodeId, stdout, stateUpdates) {
     if (!nodeSpec.output)
         return;
@@ -148,12 +263,12 @@ async function storeNodeOutput(nodeSpec, nodeId, stdout, stateUpdates) {
         let parsed;
         try {
             parsed = JSON.parse((0, output_format_1.extractJson)(trimmed));
+            if (nodeSpec.output_schema) {
+                await (0, output_format_1.validateJsonSchema)(parsed, nodeSpec.output_schema);
+            }
         }
         catch (e) {
-            throw new Error(`Node "${nodeId}" output_format=json parse failed: ${e.message}`);
-        }
-        if (nodeSpec.output_schema) {
-            await (0, output_format_1.validateJsonSchema)(parsed, nodeSpec.output_schema);
+            throw (0, failure_1.createWorkflowFailureError)('OUTPUT_INVALID', `Node "${nodeId}" output_format=json parse or validation failed: ${getErrorMessage(e)}`);
         }
         stateUpdates[nodeSpec.output] = parsed;
     }
@@ -564,6 +679,34 @@ function isProcessAlive(pid) {
 function isTerminalStatus(status) {
     return status === 'completed' || status === 'failed' || status === 'cancelled';
 }
+function isRecentLegacyRetry(run, nowMs) {
+    if (run.status !== undefined || run.auto_run)
+        return false;
+    const lastAudit = run.audit?.at(-1);
+    if (lastAudit?.type !== 'retry')
+        return false;
+    const updatedAt = Date.parse(run.updated_at);
+    return Number.isFinite(updatedAt)
+        && nowMs - updatedAt < exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS;
+}
+function normalizeRunStatus(run, nowMs = Date.now()) {
+    if (run.status !== undefined)
+        return run;
+    let status;
+    if (run.current_node === '') {
+        status = 'pending';
+    }
+    else if (run.auto_run) {
+        status = 'running';
+    }
+    else if (isRecentLegacyRetry(run, nowMs)) {
+        status = 'retrying';
+    }
+    else {
+        status = 'pending';
+    }
+    return { ...run, status };
+}
 function isRetryableFailure(run, spec) {
     if (run.status !== 'failed')
         return false;
@@ -575,8 +718,9 @@ function isRetryableFailure(run, spec) {
     return node.type !== 'error';
 }
 class RunManager {
-    constructor(runsDir = '.crewx/workflow-runs', projectRoot) {
+    constructor(runsDir = '.crewx/workflow-runs', projectRoot, deps) {
         this.runsDir = runsDir;
+        this.sleep = deps?.sleep ?? sleep;
         const base = projectRoot || process.env.CREWX_WORKSPACE || process.cwd();
         this.resolvedDir = path.resolve(base, this.runsDir);
         this.projectRoot = path.resolve(base);
@@ -596,7 +740,7 @@ class RunManager {
             throw new Error(`Invalid execution ID: "${execId}"`);
         }
     }
-    readRun(execId) {
+    readRunRaw(execId) {
         const filePath = path.join(this.resolvedDir, `${execId}.json`);
         if (!fs.existsSync(filePath))
             return null;
@@ -607,16 +751,18 @@ class RunManager {
             throw new Error(`Failed to parse execution file ${filePath}: ${e.message}`);
         }
     }
-    isLegacyRetryStale(run) {
+    readRun(execId) {
+        const run = this.readRunRaw(execId);
+        return run ? normalizeRunStatus(run) : null;
+    }
+    isLegacyStatuslessStale(run) {
         if (run.status !== undefined
             || run.auto_run
             || run.manual_resume
-            || !run.current_node) {
+            || !run.current_node
+            || !run.audit?.length) {
             return false;
         }
-        const lastAudit = run.audit?.at(-1);
-        if (lastAudit?.type !== 'retry')
-            return false;
         const updatedAt = Date.parse(run.updated_at);
         if (!Number.isFinite(updatedAt) || Date.now() - updatedAt < exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS) {
             return false;
@@ -629,16 +775,16 @@ class RunManager {
     hasStaleRunner(run) {
         if (run.auto_run)
             return !isProcessAlive(run.auto_run.pid);
-        return this.isLegacyRetryStale(run);
+        return this.isLegacyStatuslessStale(run);
     }
     recoverStaleRunner(execId) {
         const filePath = path.join(this.resolvedDir, `${execId}.json`);
         const lock = new file_lock_1.FileLock(filePath);
         lock.acquire();
         try {
-            const run = this.readRun(execId);
+            const run = this.readRunRaw(execId);
             if (!run || !this.hasStaleRunner(run))
-                return run;
+                return run ? normalizeRunStatus(run) : null;
             if (isTerminalStatus(run.status)) {
                 delete run.auto_run;
                 delete run.manual_resume;
@@ -648,23 +794,18 @@ class RunManager {
             const lease = run.auto_run;
             const nodeId = lease?.node_id || run.current_node;
             const now = new Date().toISOString();
-            run.status = 'failed';
-            run.error = exports.WORKFLOW_RUN_INTERRUPTED_ERROR;
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt: (0, failure_1.readNodeAttempt)(run, nodeId),
+                causeCode: 'INTERRUPTED',
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            });
+            assignRunFailure(run, failure, exports.WORKFLOW_RUN_INTERRUPTED_ERROR);
             delete run.auto_run;
             delete run.manual_resume;
-            run.audit = run.audit ?? [];
-            run.audit.push({
-                exec_id: run.id,
-                node_id: nodeId,
-                type: 'interrupted',
-                summary: exports.WORKFLOW_RUN_INTERRUPTED_ERROR,
-                started_at: now,
-                ended_at: now,
-                duration_ms: 0,
-                exit_code: null,
-                error: exports.WORKFLOW_RUN_INTERRUPTED_ERROR,
-                trigger: 'auto',
-            });
+            appendFailureAudit(run, failure, 'interrupted', 'auto', now);
             this.saveRun(run);
             return run;
         }
@@ -674,9 +815,9 @@ class RunManager {
     }
     loadRun(execId) {
         this.validateExecId(execId);
-        const run = this.readRun(execId);
+        const run = this.readRunRaw(execId);
         if (!run || !this.hasStaleRunner(run))
-            return run;
+            return run ? normalizeRunStatus(run) : null;
         return this.recoverStaleRunner(execId);
     }
     saveRun(run) {
@@ -721,6 +862,32 @@ class RunManager {
             lock.release();
         }
     }
+    async reserveNodeAttempt(execId, nodeId) {
+        let reservedAttempt = 1;
+        await this.atomicUpdateAsync(execId, (run) => {
+            if (run.status === 'cancelled') {
+                reservedAttempt = (0, failure_1.readNodeAttempt)(run, nodeId);
+                return;
+            }
+            const storedAttempt = run.node_attempts?.[nodeId];
+            const previousAttempt = typeof storedAttempt === 'number'
+                && Number.isInteger(storedAttempt)
+                && storedAttempt > 0
+                ? storedAttempt
+                : 0;
+            reservedAttempt = previousAttempt + 1;
+            run.node_attempts = run.node_attempts ?? {};
+            run.node_attempts[nodeId] = reservedAttempt;
+        });
+        return reservedAttempt;
+    }
+    clearNodeAttempt(run, nodeId) {
+        if (!run.node_attempts)
+            return;
+        delete run.node_attempts[nodeId];
+        if (Object.keys(run.node_attempts).length === 0)
+            delete run.node_attempts;
+    }
     loadWorkflowYaml(yamlPath) {
         if (!fs.existsSync(yamlPath))
             return null;
@@ -773,6 +940,7 @@ class RunManager {
             completed_nodes: [],
             state,
             initial_state: { ...initialState, ...sets },
+            status: 'pending',
         };
         this.saveRun(run);
         return run;
@@ -804,15 +972,15 @@ class RunManager {
         }
         const doc = this.loadWorkflowYaml(snapshot.workflow_file);
         if (!doc) {
-            throw new Error(`Failed to load workflow file: ${snapshot.workflow_file}`);
+            throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Failed to load workflow file: ${snapshot.workflow_file}`);
         }
         const spec = doc.workflows[snapshot.workflow_id];
         if (!spec) {
-            throw new Error(`Workflow "${snapshot.workflow_id}" not found in ${snapshot.workflow_file}`);
+            throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Workflow "${snapshot.workflow_id}" not found in ${snapshot.workflow_file}`);
         }
         const nodeSpec = spec.nodes?.[nodeId];
         if (!nodeSpec) {
-            throw new Error(`Node "${nodeId}" not found in workflow "${snapshot.workflow_id}"`);
+            throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" not found in workflow "${snapshot.workflow_id}"`);
         }
         const prevNode = snapshot.current_node;
         const stateUpdates = {};
@@ -820,12 +988,20 @@ class RunManager {
         let currentNode = nodeId;
         let status;
         let failureMessage;
+        let failureCode;
         let nodeTaskId;
         const trigger = opts?.trigger ?? 'manual';
         const startedAt = new Date();
-        let nodeStdout = '';
-        let nodeStderr = '';
         let nodeExitCode = null;
+        let nodeSignal = null;
+        let nodeErrorOrigin = null;
+        let processFailure;
+        const recordProcessResult = (result) => {
+            nodeExitCode = result.status;
+            nodeSignal = result.signal;
+            nodeErrorOrigin = result.errorOrigin;
+            processFailure = toProcessFailureMetadata(result);
+        };
         const buildAudit = (error) => {
             const endedAt = new Date();
             return {
@@ -841,24 +1017,27 @@ class RunManager {
                 trigger,
             };
         };
+        const attempt = await this.reserveNodeAttempt(execId, nodeId);
         try {
             switch (nodeSpec.type) {
                 case 'agent_task': {
                     if (!nodeSpec.agent) {
-                        throw new Error(`Node "${nodeId}" is agent_task but has no agent defined`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" is agent_task but has no agent defined`);
                     }
                     const agentCtx = { ...snapshot.state, state: snapshot.state };
                     const resolvedAgent = interpolateTemplate(nodeSpec.agent, agentCtx).trim();
                     if (!resolvedAgent) {
-                        throw new Error(`Node "${nodeId}": assignee not resolved (agent field "${nodeSpec.agent}" interpolated to empty)`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": assignee not resolved (agent field "${nodeSpec.agent}" interpolated to empty)`);
                     }
                     const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, agentCtx) : undefined, this.projectRoot);
-                    if (!cwdCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${cwdCheck.error}`);
+                    if (!cwdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cwdCheck.error}`);
+                    }
                     const nodeCwd = cwdCheck.resolved;
                     const rootCheck = (0, shell_security_1.normalizeCwd)(undefined, this.projectRoot);
-                    if (!rootCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${rootCheck.error}`);
+                    if (!rootCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${rootCheck.error}`);
+                    }
                     const workspaceRoot = rootCheck.resolved;
                     const input = nodeSpec.input
                         ? interpolateTemplate(nodeSpec.input, agentCtx)
@@ -880,12 +1059,15 @@ class RunManager {
                         break;
                     }
                     console.log(`Executing: @${resolvedAgent} (${mode} mode)`);
-                    const crewxCli = process.env.CREWX_CLI || 'npx crewx';
-                    const cliParts = crewxCli.split(/\s+/);
-                    const bin = cliParts[0];
-                    const baseArgs = cliParts.slice(1);
                     const nodeTimeout = Number(process.env[mode === 'q' ? 'CREWX_TIMEOUT_QUERY' : 'CREWX_TIMEOUT_EXECUTE']) || 8 * 3600000;
-                    const wfArgs = [...baseArgs, mode, prompt];
+                    const childInvocation = resolveCrewxChildInvocation({
+                        ...process.env,
+                        CREWX_WORKSPACE: workspaceRoot,
+                        CREWX_WORKFLOW_EXEC_ID: snapshot.id,
+                        CREWX_WORKFLOW_NODE_ID: nodeId,
+                        CREWX_WORKFLOW_ID: snapshot.workflow_id,
+                    }, nodeCwd);
+                    const wfArgs = [...childInvocation.argv.slice(1), mode, prompt];
                     if (process.env.CREWX_WORKFLOW_THREAD === 'on') {
                         wfArgs.push(`--thread=workflow:${snapshot.id}`);
                     }
@@ -898,33 +1080,27 @@ class RunManager {
                     };
                     wfArgs.push('--metadata', JSON.stringify(wfMetadata));
                     const program = (0, sdk_1.resolveWindowsSpawnProgram)({
-                        command: bin,
+                        command: childInvocation.argv[0],
+                        env: childInvocation.env,
+                        cwd: nodeCwd,
                         allowShellFallback: true,
                         packageName: 'crewx',
                     });
                     const invocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, wfArgs);
                     const result = await runProcess(invocation.command, invocation.argv, {
                         cwd: nodeCwd,
-                        env: {
-                            ...process.env,
-                            CREWX_WORKSPACE: workspaceRoot,
-                            CREWX_WORKFLOW_EXEC_ID: snapshot.id,
-                            CREWX_WORKFLOW_NODE_ID: nodeId,
-                            CREWX_WORKFLOW_ID: snapshot.workflow_id,
-                        },
+                        env: childInvocation.env,
                         timeout: nodeTimeout,
                         shell: invocation.shell ?? false,
                         windowsHide: invocation.windowsHide,
                     });
-                    nodeStdout = result.stdout;
-                    nodeStderr = result.stderr;
-                    nodeExitCode = result.status;
-                    if (result.error) {
-                        throw result.error;
-                    }
+                    recordProcessResult(result);
                     if (result.timedOut) {
                         throw new Error(`Agent execution timed out after ${nodeTimeout}ms`);
                     }
+                    if (result.error) {
+                        throw result.error;
+                    }
                     const taskIdMatch = result.stderr.match(/crewx kill (tsk_\w+)/);
                     nodeTaskId = taskIdMatch?.[1];
                     if (result.status !== 0) {
@@ -949,7 +1125,7 @@ class RunManager {
                                         `\n\n## ⚠️ PREVIOUS OUTPUT FAILED JSON.parse: ${lastError.message}\n` +
                                         `Return STRICT JSON ONLY. First char "{", last char "}". No prose, no fences.`;
                                     const retryPrompt = `@${resolvedAgent} ${retryInput}`;
-                                    const retryArgs = [...baseArgs, mode, retryPrompt];
+                                    const retryArgs = [...childInvocation.argv.slice(1), mode, retryPrompt];
                                     if (process.env.CREWX_WORKFLOW_THREAD === 'on') {
                                         retryArgs.push(`--thread=workflow:${snapshot.id}`);
                                     }
@@ -957,24 +1133,16 @@ class RunManager {
                                     const retryInvocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, retryArgs);
                                     const retryResult = await runProcess(retryInvocation.command, retryInvocation.argv, {
                                         cwd: nodeCwd,
-                                        env: {
-                                            ...process.env,
-                                            CREWX_WORKSPACE: workspaceRoot,
-                                            CREWX_WORKFLOW_EXEC_ID: snapshot.id,
-                                            CREWX_WORKFLOW_NODE_ID: nodeId,
-                                            CREWX_WORKFLOW_ID: snapshot.workflow_id,
-                                        },
+                                        env: childInvocation.env,
                                         timeout: nodeTimeout,
                                         shell: retryInvocation.shell ?? false,
                                         windowsHide: retryInvocation.windowsHide,
                                     });
-                                    nodeStdout = retryResult.stdout;
-                                    nodeStderr = retryResult.stderr;
-                                    nodeExitCode = retryResult.status;
-                                    if (retryResult.error)
-                                        throw retryResult.error;
+                                    recordProcessResult(retryResult);
                                     if (retryResult.timedOut)
                                         throw new Error(`Agent retry timed out after ${nodeTimeout}ms`);
+                                    if (retryResult.error)
+                                        throw retryResult.error;
                                     if (retryResult.status !== 0) {
                                         throw new Error(`Agent retry failed (exit ${retryResult.status}): ${retryResult.stderr.trim()}`);
                                     }
@@ -996,7 +1164,7 @@ class RunManager {
                                 }
                             }
                             if (parsed === null) {
-                                throw new Error(`Node "${nodeId}" output_format=json parse failed after ${retryMax + 1} attempt(s): ${lastError.message}`);
+                                throw (0, failure_1.createWorkflowFailureError)('OUTPUT_INVALID', `Node "${nodeId}" output_format=json parse failed after ${retryMax + 1} attempt(s): ${lastError.message}`);
                             }
                             stateUpdates[nodeSpec.output] = parsed;
                             console.log(`Result stored in state.${nodeSpec.output} (JSON object)`);
@@ -1010,10 +1178,10 @@ class RunManager {
                 }
                 case 'skill_task': {
                     if (!nodeSpec.skill) {
-                        throw new Error(`Node "${nodeId}" is skill_task but has no skill defined`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" is skill_task but has no skill defined`);
                     }
                     if (!shell_security_1.SKILL_NAME_RE.test(nodeSpec.skill)) {
-                        throw new Error(`Node "${nodeId}" has invalid skill name "${nodeSpec.skill}" (must match ^[a-z0-9][a-z0-9-]*$)`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" has invalid skill name "${nodeSpec.skill}" (must match ^[a-z0-9][a-z0-9-]*$)`);
                     }
                     const ctx = { ...snapshot.state, state: snapshot.state };
                     const skillArgs = (nodeSpec.args ?? []).map((a) => interpolateTemplate(String(a), ctx));
@@ -1021,11 +1189,13 @@ class RunManager {
                         ? interpolateTemplate(String(nodeSpec.stdin), ctx)
                         : undefined;
                     const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, ctx) : undefined, this.projectRoot);
-                    if (!cwdCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${cwdCheck.error}`);
+                    if (!cwdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cwdCheck.error}`);
+                    }
                     const skillRootCheck = (0, shell_security_1.normalizeCwd)(undefined, this.projectRoot);
-                    if (!skillRootCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${skillRootCheck.error}`);
+                    if (!skillRootCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${skillRootCheck.error}`);
+                    }
                     const skillWorkspaceRoot = skillRootCheck.resolved;
                     const timeout = nodeSpec.timeout ?? exports.SKILL_TASK_DEFAULT_TIMEOUT;
                     if (opts?.dryRun) {
@@ -1033,38 +1203,35 @@ class RunManager {
                         break;
                     }
                     console.log(`Executing skill: ${nodeSpec.skill} ${skillArgs.join(' ')}`.trim());
-                    const crewxCli = process.env.CREWX_CLI || 'npx crewx';
-                    const cliParts = crewxCli.split(/\s+/);
-                    const bin = cliParts[0];
-                    const baseArgs = cliParts.slice(1);
-                    const skillArgv = [...baseArgs, 'skill', nodeSpec.skill, ...skillArgs];
+                    const childInvocation = resolveCrewxChildInvocation({
+                        ...process.env,
+                        CREWX_WORKSPACE: skillWorkspaceRoot,
+                        CREWX_WORKFLOW_EXEC_ID: snapshot.id,
+                        CREWX_WORKFLOW_NODE_ID: nodeId,
+                        CREWX_WORKFLOW_ID: snapshot.workflow_id,
+                    }, cwdCheck.resolved);
+                    const skillArgv = [...childInvocation.argv.slice(1), 'skill', nodeSpec.skill, ...skillArgs];
                     const program = (0, sdk_1.resolveWindowsSpawnProgram)({
-                        command: bin,
+                        command: childInvocation.argv[0],
+                        env: childInvocation.env,
+                        cwd: cwdCheck.resolved,
                         allowShellFallback: true,
                         packageName: 'crewx',
                     });
                     const invocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, skillArgv);
                     const result = await runProcess(invocation.command, invocation.argv, {
                         cwd: cwdCheck.resolved,
-                        env: {
-                            ...process.env,
-                            CREWX_WORKSPACE: skillWorkspaceRoot,
-                            CREWX_WORKFLOW_EXEC_ID: snapshot.id,
-                            CREWX_WORKFLOW_NODE_ID: nodeId,
-                            CREWX_WORKFLOW_ID: snapshot.workflow_id,
-                        },
+                        env: childInvocation.env,
                         timeout,
                         shell: invocation.shell ?? false,
                         windowsHide: invocation.windowsHide,
                         stdin: skillStdin,
                     });
-                    nodeStdout = result.stdout;
-                    nodeStderr = result.stderr;
-                    nodeExitCode = result.status;
-                    if (result.error)
-                        throw result.error;
+                    recordProcessResult(result);
                     if (result.timedOut)
                         throw new Error(`skill_task "${nodeId}" timed out after ${timeout}ms`);
+                    if (result.error)
+                        throw result.error;
                     if (result.status !== 0) {
                         throw new Error(`skill_task "${nodeId}" failed (exit ${result.status}): ${result.stderr.trim()}`);
                     }
@@ -1078,25 +1245,29 @@ class RunManager {
                     const shellEnvEnabled = process.env.CREWX_WORKFLOW_SHELL === '1';
                     const shellMetaAllowed = spec.metadata?.shell_task_allowed === true;
                     if (!shellEnvEnabled || !shellMetaAllowed) {
-                        throw new Error('shell_task is disabled. Set CREWX_WORKFLOW_SHELL=1 and metadata.shell_task_allowed=true to enable.');
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', 'shell_task is disabled. Set CREWX_WORKFLOW_SHELL=1 and metadata.shell_task_allowed=true to enable.');
                     }
                     const cmdCheck = (0, shell_security_1.validateShellCommand)(nodeSpec.command);
-                    if (!cmdCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${cmdCheck.error}`);
+                    if (!cmdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cmdCheck.error}`);
+                    }
                     const envCheck = (0, shell_security_1.validateEnvKeys)(nodeSpec.env);
-                    if (!envCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${envCheck.error}`);
+                    if (!envCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${envCheck.error}`);
+                    }
                     const ctx = { ...snapshot.state, state: snapshot.state };
                     const command = nodeSpec.command.map((c) => interpolateTemplate(c, ctx));
                     const postCheck = (0, shell_security_1.validateShellCommand)(command);
-                    if (!postCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${postCheck.error}`);
+                    if (!postCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${postCheck.error}`);
+                    }
                     const interpolatedEnv = nodeSpec.env
                         ? Object.fromEntries(Object.entries(nodeSpec.env).map(([k, v]) => [k, interpolateTemplate(String(v), ctx)]))
                         : undefined;
                     const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, ctx) : undefined, this.projectRoot);
-                    if (!cwdCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${cwdCheck.error}`);
+                    if (!cwdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cwdCheck.error}`);
+                    }
                     const timeout = nodeSpec.timeout ?? exports.SKILL_TASK_DEFAULT_TIMEOUT;
                     if (opts?.dryRun) {
                         console.log(`[dry-run] Would execute: ${command.join(' ')}`);
@@ -1117,13 +1288,11 @@ class RunManager {
                         shell: false,
                         stdin: shellStdin,
                     });
-                    nodeStdout = result.stdout;
-                    nodeStderr = result.stderr;
-                    nodeExitCode = result.status;
-                    if (result.error)
-                        throw result.error;
+                    recordProcessResult(result);
                     if (result.timedOut)
                         throw new Error(`shell_task "${nodeId}" timed out after ${timeout}ms`);
+                    if (result.error)
+                        throw result.error;
                     if (result.status !== 0) {
                         throw new Error(`shell_task "${nodeId}" failed (exit ${result.status}): ${result.stderr.trim()}`);
                     }
@@ -1135,13 +1304,13 @@ class RunManager {
                 }
                 case 'branch': {
                     if (!nodeSpec.condition) {
-                        throw new Error(`Branch node "${nodeId}" has no condition`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Branch node "${nodeId}" has no condition`);
                     }
                     const condResult = String(safeEvaluateBranchCondition(String(nodeSpec.condition), snapshot.state));
                     const branches = nodeSpec.branches;
                     const targetNode = branches?.[condResult] ?? nodeSpec.default;
                     if (!targetNode) {
-                        throw new Error(`Branch "${nodeId}": no matching branch for "${condResult}" and no default`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Branch "${nodeId}": no matching branch for "${condResult}" and no default`);
                     }
                     console.log(`Branch "${nodeId}": condition="${nodeSpec.condition}" → ${condResult} → ${targetNode}`);
                     nodesToComplete.push(nodeId);
@@ -1167,6 +1336,7 @@ class RunManager {
                 }
                 case 'error': {
                     status = 'failed';
+                    failureCode = 'UNKNOWN';
                     const ctx = { ...snapshot.state, state: snapshot.state };
                     failureMessage = nodeSpec.message
                         ? interpolateTemplate(nodeSpec.message, ctx)
@@ -1176,7 +1346,7 @@ class RunManager {
                 }
                 case 'expression': {
                     if (!nodeSpec.set || typeof nodeSpec.set !== 'object') {
-                        throw new Error(`Expression node "${nodeId}" requires "set" field`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Expression node "${nodeId}" requires "set" field`);
                     }
                     for (const [key, expr] of Object.entries(nodeSpec.set)) {
                         const result = safeEvaluate(String(expr), snapshot.state);
@@ -1207,26 +1377,46 @@ class RunManager {
             }
         }
         catch (e) {
-            const errMessage = e.message;
-            const partial = [
-                nodeStdout.trim() ? `stdout: ${nodeStdout.trim().slice(-500)}` : '',
-                nodeStderr.trim() ? `stderr: ${nodeStderr.trim().slice(-500)}` : '',
-            ].filter(Boolean).join(' | ');
-            const auditEntry = buildAudit(partial ? `${errMessage} (${partial})` : errMessage);
+            const errMessage = getErrorMessage(e);
+            const causeCode = (0, failure_1.classifyWorkflowFailure)({
+                errorCode: (0, failure_1.getWorkflowFailureCode)(e),
+                process: processFailure,
+            });
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt,
+                causeCode,
+                exitCode: causeCode === 'NODE_EXIT_NONZERO' && typeof nodeExitCode === 'number'
+                    ? nodeExitCode
+                    : null,
+                signal: causeCode === 'INTERRUPTED' ? nodeSignal : null,
+                errorOrigin: nodeErrorOrigin,
+                timedOut: processFailure?.timedOut ?? false,
+            });
+            const auditEntry = buildAudit(failure.causeMessage);
             await this.atomicUpdateAsync(execId, (run) => {
                 if (run.status === 'cancelled')
                     return;
                 Object.assign(run.state, stateUpdates);
                 delete run.manual_resume;
                 run.current_node = nodeId;
-                run.status = 'failed';
-                run.error = errMessage;
+                assignRunFailure(run, failure, errMessage);
                 run.audit = run.audit ?? [];
                 run.audit.push(auditEntry);
             });
             throw e;
         }
-        const successAudit = buildAudit();
+        const successFailure = failureCode
+            ? (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt,
+                causeCode: failureCode,
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            })
+            : null;
+        const successAudit = buildAudit(successFailure?.causeMessage);
         return this.atomicUpdateAsync(execId, (run) => {
             if (run.status === 'cancelled')
                 return;
@@ -1241,18 +1431,20 @@ class RunManager {
                 run.completed_nodes.push(prevNode);
             }
             run.current_node = currentNode;
-            if (status) {
-                run.status = status;
-                if (status !== 'failed') {
-                    delete run.error;
-                }
+            if (successFailure) {
+                assignRunFailure(run, successFailure, failureMessage ?? successFailure.causeMessage);
             }
-            else if (run.status === 'failed') {
-                run.status = 'running';
+            else {
                 delete run.error;
+                delete run.last_failure;
+                this.clearNodeAttempt(run, nodeId);
+                if (status) {
+                    run.status = status;
+                }
+                else if (run.status !== 'completed') {
+                    run.status = 'running';
+                }
             }
-            if (failureMessage)
-                run.error = failureMessage;
             if (nodeTaskId) {
                 run.tasks = run.tasks ?? {};
                 run.tasks[nodeId] = nodeTaskId;
@@ -1262,13 +1454,19 @@ class RunManager {
         });
     }
     async claimAutoRunLease(execId, nodeId, preferredToken) {
+        let claimed = false;
         const updated = await this.atomicUpdateAsync(execId, (run) => {
+            if (isTerminalStatus(run.status))
+                return;
             const existing = run.auto_run;
             const ownedByThisProcess = existing?.pid === process.pid && typeof existing.token === 'string';
             if (existing && !ownedByThisProcess && isProcessAlive(existing.pid)) {
                 throw new Error(`Workflow run "${execId}" is already owned by another runner process`);
             }
             const token = preferredToken ?? (ownedByThisProcess ? existing.token : (0, sdk_1.generateId)('wfr'));
+            if (!isTerminalStatus(run.status)) {
+                run.status = 'running';
+            }
             run.auto_run = {
                 pid: process.pid,
                 token,
@@ -1278,8 +1476,9 @@ class RunManager {
                     : new Date().toISOString(),
             };
             delete run.manual_resume;
+            claimed = true;
         });
-        return updated.auto_run.token;
+        return claimed ? updated.auto_run?.token ?? null : null;
     }
     async releaseAutoRunLease(execId, token) {
         await this.atomicUpdateAsync(execId, (run) => {
@@ -1287,6 +1486,145 @@ class RunManager {
                 delete run.auto_run;
         });
     }
+    async failAutoConfiguration(execId, nodeId, reason) {
+        return this.atomicUpdateAsync(execId, (run) => {
+            if (run.status === 'cancelled')
+                return;
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt: (0, failure_1.readNodeAttempt)(run, nodeId),
+                causeCode: 'CONFIG_INVALID',
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            });
+            assignRunFailure(run, failure, reason);
+            appendFailureAudit(run, failure, 'config', 'auto');
+        });
+    }
+    async reserveAutoRetry(execId, nodeId, runnerToken, policy, delayMs) {
+        let reserved = false;
+        const run = await this.atomicUpdateAsync(execId, (draft) => {
+            if (draft.status === 'cancelled')
+                return;
+            if (draft.status !== 'failed')
+                return;
+            if (draft.auto_run?.token !== runnerToken)
+                return;
+            const decision = (0, retry_policy_1.decideRetry)({ policy, run: draft, nodeId });
+            if (!decision.retry || decision.attempt === undefined)
+                return;
+            const now = new Date().toISOString();
+            draft.status = 'retrying';
+            delete draft.error;
+            delete draft.manual_resume;
+            draft.audit = draft.audit ?? [];
+            draft.audit.push({
+                exec_id: draft.id,
+                node_id: nodeId,
+                type: 'retry',
+                summary: `auto retry ${decision.attempt + 1}/${policy.max + 1} in ${delayMs}ms`,
+                started_at: now,
+                ended_at: now,
+                duration_ms: 0,
+                exit_code: null,
+                trigger: 'auto',
+            });
+            reserved = true;
+        });
+        return { run, reserved };
+    }
+    async waitForRetry(execId, runnerToken, delayMs) {
+        const inspect = () => {
+            const run = this.loadRun(execId);
+            if (!run)
+                throw new Error(`Execution not found: ${execId}`);
+            if (run.status === 'cancelled') {
+                return { run, outcome: 'cancelled', reason: 'run cancelled during retry delay' };
+            }
+            if (run.auto_run?.token !== runnerToken) {
+                return { run, outcome: 'paused', reason: 'auto-run lease lost during retry delay' };
+            }
+            if (run.status !== 'retrying') {
+                return { run, outcome: 'paused', reason: 'run state changed during retry delay' };
+            }
+            return null;
+        };
+        let remaining = Math.max(0, delayMs);
+        while (remaining > 0) {
+            const slice = Math.min(remaining, RETRY_WAIT_SLICE_MS);
+            await this.sleep(slice);
+            remaining -= slice;
+            const stopped = inspect();
+            if (stopped)
+                return stopped;
+        }
+        const stopped = inspect();
+        if (stopped)
+            return stopped;
+        const run = this.loadRun(execId);
+        if (!run)
+            throw new Error(`Execution not found: ${execId}`);
+        return { run, outcome: 'ready' };
+    }
+    async runNodeWithRetry(execId, nodeId, runnerToken, policy) {
+        let token = runnerToken;
+        while (true) {
+            try {
+                const run = await this.executeNode(execId, nodeId, { trigger: 'auto' });
+                return { run, runnerToken: token, outcome: 'success' };
+            }
+            catch (caught) {
+                const run = this.loadRun(execId);
+                if (!run)
+                    throw new Error(`Execution not found: ${execId}`);
+                if (run.status === 'cancelled') {
+                    return { run, runnerToken: token, outcome: 'cancelled', reason: run.error };
+                }
+                if (run.status !== 'failed' || !policy) {
+                    return { run, runnerToken: token, outcome: run.status === 'failed' ? 'failed' : 'paused', reason: getErrorMessage(caught) };
+                }
+                const decision = (0, retry_policy_1.decideRetry)({ policy, run, nodeId });
+                if (!decision.retry || decision.attempt === undefined) {
+                    return { run, runnerToken: token, outcome: 'failed', reason: getErrorMessage(caught) };
+                }
+                const delayMs = (0, retry_policy_1.computeRetryDelayMs)(policy, decision.attempt);
+                const reservation = await this.reserveAutoRetry(execId, nodeId, token, policy, delayMs);
+                if (!reservation.reserved) {
+                    const latest = reservation.run;
+                    if (latest.status === 'cancelled') {
+                        return { run: latest, runnerToken: token, outcome: 'cancelled', reason: latest.error };
+                    }
+                    return {
+                        run: latest,
+                        runnerToken: token,
+                        outcome: latest.status === 'failed' ? 'failed' : 'paused',
+                        reason: getErrorMessage(caught),
+                    };
+                }
+                const waited = await this.waitForRetry(execId, token, delayMs);
+                if (waited.outcome !== 'ready') {
+                    return { run: waited.run, runnerToken: token, outcome: waited.outcome, reason: waited.reason };
+                }
+                const nextToken = await this.claimAutoRunLease(execId, nodeId, token);
+                if (!nextToken) {
+                    const latest = this.loadRun(execId);
+                    if (!latest)
+                        throw new Error(`Execution not found: ${execId}`);
+                    if (latest.status === 'cancelled') {
+                        return { run: latest, runnerToken: token, outcome: 'cancelled', reason: latest.error };
+                    }
+                    return {
+                        run: latest,
+                        runnerToken: token,
+                        outcome: 'paused',
+                        reason: 'run stopped before retry lease could be reclaimed',
+                    };
+                }
+                token = nextToken;
+            }
+        }
+    }
     async runAuto(execId) {
         const leaseKey = `${this.resolvedDir}::${execId}`;
         if (inFlightAutoRuns.has(leaseKey)) {
@@ -1306,11 +1644,12 @@ class RunManager {
                 runnerToken = initial.auto_run.token;
             }
             const doc = this.loadWorkflowYaml(initial.workflow_file);
-            if (!doc)
-                throw new Error(`Failed to load workflow file: ${initial.workflow_file}`);
+            if (!doc) {
+                throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Failed to load workflow file: ${initial.workflow_file}`);
+            }
             const spec = doc.workflows[initial.workflow_id];
             if (!spec) {
-                throw new Error(`Workflow "${initial.workflow_id}" not found in ${initial.workflow_file}`);
+                throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Workflow "${initial.workflow_id}" not found in ${initial.workflow_file}`);
             }
             const executed = [];
             if (initial.status === 'completed' || initial.status === 'failed' || initial.status === 'cancelled') {
@@ -1335,17 +1674,46 @@ class RunManager {
                 if (run.status === 'cancelled')
                     return { run, outcome: 'cancelled', reason: run.error, executed };
                 const nodeId = run.current_node || firstNode;
-                if (!nodeId)
-                    return { run, outcome: 'failed', reason: 'workflow has no nodes', executed };
+                if (!nodeId) {
+                    run = await this.atomicUpdateAsync(execId, (r) => {
+                        const failure = (0, failure_1.buildNodeFailure)({
+                            failedNode: r.current_node || '',
+                            attempt: (0, failure_1.readNodeAttempt)(r, r.current_node || ''),
+                            causeCode: 'CONFIG_INVALID',
+                            exitCode: null,
+                            signal: null,
+                            errorOrigin: null,
+                        });
+                        assignRunFailure(r, failure, 'workflow has no nodes');
+                        appendFailureAudit(r, failure, 'config', 'auto');
+                    });
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
                 const node = spec.nodes[nodeId];
                 if (!node) {
                     run = await this.atomicUpdateAsync(execId, (r) => {
                         r.current_node = nodeId;
-                        r.status = 'failed';
-                        r.error = `node "${nodeId}" not found in workflow`;
+                        const failure = (0, failure_1.buildNodeFailure)({
+                            failedNode: nodeId,
+                            attempt: (0, failure_1.readNodeAttempt)(r, nodeId),
+                            causeCode: 'CONFIG_INVALID',
+                            exitCode: null,
+                            signal: null,
+                            errorOrigin: null,
+                        });
+                        assignRunFailure(r, failure, `node "${nodeId}" not found in workflow`);
+                        appendFailureAudit(r, failure, 'config', 'auto');
                     });
                     return { run, outcome: 'failed', reason: run.error, executed };
                 }
+                const parsedRetryPolicy = (0, retry_policy_1.parseRetryPolicy)(node);
+                if (!parsedRetryPolicy.ok) {
+                    run = await this.failAutoConfiguration(execId, nodeId, parsedRetryPolicy.reason);
+                    if (run.status === 'cancelled') {
+                        return { run, outcome: 'cancelled', reason: run.error, executed };
+                    }
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
                 if (node.type === 'approval') {
                     if (run.current_node !== nodeId) {
                         run = await this.atomicUpdateAsync(execId, (r) => { r.current_node = nodeId; });
@@ -1364,28 +1732,45 @@ class RunManager {
                     }
                     return { run, outcome: 'paused', reason: `unknown node type "${node.type}" (node "${nodeId}")`, executed };
                 }
-                runnerToken = await this.claimAutoRunLease(execId, nodeId, runnerToken);
+                const claimedToken = await this.claimAutoRunLease(execId, nodeId, runnerToken);
+                if (!claimedToken) {
+                    run = this.loadRun(execId);
+                    return {
+                        run,
+                        outcome: run.status === 'cancelled' ? 'cancelled' : 'paused',
+                        reason: run.status === 'cancelled' ? run.error : 'run stopped before auto-run lease could be claimed',
+                        executed,
+                    };
+                }
+                runnerToken = claimedToken;
                 run = await this.atomicUpdateAsync(execId, (r) => {
                     const engine = readRunEngineState(r);
                     engine.visitCounts[nodeId] = (engine.visitCounts[nodeId] ?? 0) + 1;
                     engine.iterations += 1;
                     r.state['__engine'] = engine;
                     if (engine.visitCounts[nodeId] > maxIterations || engine.iterations > maxIterations) {
-                        r.status = 'failed';
-                        r.error = `auto runner exceeded max_iterations (${maxIterations}); possible loop at "${nodeId}"`;
+                        const failure = (0, failure_1.buildNodeFailure)({
+                            failedNode: nodeId,
+                            attempt: (0, failure_1.readNodeAttempt)(r, nodeId),
+                            causeCode: 'CONFIG_INVALID',
+                            exitCode: null,
+                            signal: null,
+                            errorOrigin: null,
+                        });
+                        assignRunFailure(r, failure, `auto runner exceeded max_iterations (${maxIterations}); possible loop at "${nodeId}"`);
+                        appendFailureAudit(r, failure, 'config', 'auto');
                     }
                 });
                 if (run.status === 'failed') {
                     return { run, outcome: 'failed', reason: run.error, executed };
                 }
-                try {
-                    run = await this.executeNode(execId, nodeId, { trigger: 'auto' });
-                    executed.push(nodeId);
-                }
-                catch (e) {
-                    run = this.loadRun(execId);
-                    return { run, outcome: 'failed', reason: e.message, executed };
+                const nodeResult = await this.runNodeWithRetry(execId, nodeId, runnerToken, parsedRetryPolicy.policy);
+                run = nodeResult.run;
+                runnerToken = nodeResult.runnerToken;
+                if (nodeResult.outcome !== 'success') {
+                    return { run, outcome: nodeResult.outcome, reason: nodeResult.reason, executed };
                 }
+                executed.push(nodeId);
                 if (run.status === 'completed')
                     return { run, outcome: 'completed', executed };
                 if (run.status === 'failed')
@@ -1418,23 +1803,19 @@ class RunManager {
                 return;
             }
             const now = new Date();
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: run.current_node || '',
+                attempt: (0, failure_1.readNodeAttempt)(run, run.current_node || ''),
+                causeCode: 'CANCELLED',
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            });
+            assignRunFailure(run, failure, reason);
             run.status = 'cancelled';
-            run.error = reason;
             delete run.auto_run;
             delete run.manual_resume;
-            run.audit = run.audit ?? [];
-            run.audit.push({
-                exec_id: run.id,
-                node_id: run.current_node || '',
-                type: 'cancel',
-                summary: reason,
-                started_at: now.toISOString(),
-                ended_at: now.toISOString(),
-                duration_ms: 0,
-                exit_code: null,
-                error: reason,
-                trigger: 'manual',
-            });
+            appendFailureAudit(run, failure, 'cancel', 'manual', now.toISOString());
         });
     }
     reset(execId) {
@@ -1442,6 +1823,10 @@ class RunManager {
             run.state = JSON.parse(JSON.stringify(run.initial_state));
             run.current_node = '';
             run.completed_nodes = [];
+            run.status = 'pending';
+            delete run.error;
+            delete run.last_failure;
+            delete run.node_attempts;
             delete run.auto_run;
             delete run.manual_resume;
         });
@@ -1463,8 +1848,20 @@ class RunManager {
             throw new Error(`Run "${execId}" failed at a non-retryable node "${snapshot.current_node}" (error node); use reset to restart`);
         }
         return this.atomicUpdate(execId, (run) => {
+            if (run.status !== 'failed') {
+                throw new Error(`Run "${execId}" is not failed (status=${run.status ?? 'pending'}); only failed runs can be retried`);
+            }
             const failedNode = run.current_node;
-            delete run.status;
+            if (failedNode && run.last_failure) {
+                const lastAttempt = run.last_failure.attempt;
+                const storedAttempt = run.node_attempts?.[failedNode];
+                if (typeof lastAttempt === 'number' && Number.isInteger(lastAttempt) && lastAttempt > 0
+                    && (typeof storedAttempt !== 'number' || storedAttempt < lastAttempt)) {
+                    run.node_attempts = run.node_attempts ?? {};
+                    run.node_attempts[failedNode] = lastAttempt;
+                }
+            }
+            run.status = 'retrying';
             delete run.error;
             delete run.manual_resume;
             run.retry_count = (run.retry_count ?? 0) + 1;
@@ -1486,7 +1883,7 @@ class RunManager {
                 exec_id: run.id,
                 node_id: failedNode,
                 type: 'retry',
-                summary: `manual retry #${run.retry_count} of node "${failedNode}"`,
+                summary: `manual retry #${run.retry_count}`,
                 started_at: now,
                 ended_at: now,
                 duration_ms: 0,