@crewx/workflow 0.3.22-rc.19 → 0.3.22-rc.191

package/dist/src/run-manager.js

@@ -33,105 +33,310 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RunManager = exports.SKILL_TASK_DEFAULT_TIMEOUT = void 0;
+exports.RunManager = exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = exports.WORKFLOW_RUN_INTERRUPTED_ERROR = exports.WORKFLOW_CANCEL_GRACE_MS = exports.SKILL_TASK_DEFAULT_TIMEOUT = exports.FileLock = void 0;
 exports.parseSetArgs = parseSetArgs;
 exports.readStdin = readStdin;
 exports.interpolateTemplate = interpolateTemplate;
 exports.safeEvaluate = safeEvaluate;
 exports.safeEvaluateBranchCondition = safeEvaluateBranchCondition;
 exports.validateSafeBranchCondition = validateSafeBranchCondition;
+exports.normalizeRunStatus = normalizeRunStatus;
+exports.isRetryableFailure = isRetryableFailure;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const yaml = __importStar(require("js-yaml"));
 const cp = __importStar(require("child_process"));
 const sdk_1 = require("@crewx/sdk");
+const failure_1 = require("./failure");
 const file_lock_1 = require("./file-lock");
 const output_format_1 = require("./utils/output-format");
 const shell_security_1 = require("./utils/shell-security");
+const retry_policy_1 = require("./retry-policy");
+var file_lock_2 = require("./file-lock");
+Object.defineProperty(exports, "FileLock", { enumerable: true, get: function () { return file_lock_2.FileLock; } });
 exports.SKILL_TASK_DEFAULT_TIMEOUT = 600000;
+exports.WORKFLOW_CANCEL_GRACE_MS = 2000;
+const WORKFLOW_CANCEL_POLL_INTERVAL_MS = 100;
+exports.WORKFLOW_RUN_INTERRUPTED_ERROR = 'Workflow run interrupted: runner process exited before completion';
+exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = 10 * 60000;
+const RETRY_WAIT_SLICE_MS = 1000;
+function sleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
 const KNOWN_AUTO_NODE_TYPES = new Set([
     'agent_task', 'skill_task', 'shell_task', 'branch', 'expression', 'join', 'end', 'error',
 ]);
 const SELF_ADVANCING_NODE_TYPES = new Set(['branch', 'expression', 'end', 'error']);
-function killProcessGroup(child) {
+function resolveCrewxChildInvocation(env, cwd) {
+    const resolution = (0, sdk_1.resolveCrewxExecutable)({ env, cwd });
+    if (!resolution.ok) {
+        throw (0, failure_1.createWorkflowFailureError)('CLI_UNRESOLVABLE', (0, sdk_1.formatCrewxExecutableFailure)(resolution));
+    }
+    return (0, sdk_1.createCrewxChildInvocation)(resolution, { baseEnv: env });
+}
+async function killProcessGroup(child, signal = 'SIGKILL') {
+    if (!child.pid)
+        return;
     try {
-        if (process.platform === 'win32') {
-            if (child.pid) {
-                cp.spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' });
-            }
-        }
-        else if (child.pid) {
+        const result = await (0, sdk_1.terminateProcessTree)(child.pid, { signal });
+        if (result.status === 'failed' || result.status === 'not_found') {
             try {
-                process.kill(-child.pid, 'SIGKILL');
-            }
-            catch {
-                try {
-                    child.kill('SIGKILL');
-                }
-                catch { }
+                child.kill(signal);
             }
+            catch { }
+        }
+    }
+    catch {
+        try {
+            child.kill(signal);
         }
+        catch { }
     }
-    catch { }
 }
 async function runProcess(command, argv, opts) {
     return new Promise((resolve) => {
+        const isCancelled = () => {
+            try {
+                return opts.cancellation?.isCancelled() === true;
+            }
+            catch {
+                return false;
+            }
+        };
+        if (isCancelled()) {
+            resolve({
+                status: null,
+                stdout: '',
+                stderr: '',
+                timedOut: false,
+                cancelled: true,
+                signal: null,
+                errorOrigin: null,
+            });
+            return;
+        }
         let child;
         try {
             child = cp.spawn(command, argv, {
                 cwd: opts.cwd,
                 env: opts.env,
-                stdio: ['ignore', 'pipe', 'pipe'],
+                stdio: [opts.stdin !== undefined ? 'pipe' : 'ignore', 'pipe', 'pipe'],
                 shell: opts.shell ?? false,
                 detached: process.platform !== 'win32',
                 windowsHide: opts.windowsHide ?? true,
             });
         }
         catch (err) {
-            resolve({ status: null, stdout: '', stderr: '', timedOut: false, error: err });
+            resolve({
+                status: null,
+                stdout: '',
+                stderr: '',
+                timedOut: false,
+                signal: null,
+                errorOrigin: 'spawn',
+                error: err,
+            });
             return;
         }
         let stdout = '';
         let stderr = '';
         let settled = false;
+        let finishing = false;
+        let spawned = false;
+        let cancellationRequested = false;
+        let postSpawnError;
+        let stdinError;
+        let timeoutTimer;
+        let cancellationEscalationTimer;
+        let cancellationPollTimer;
+        let activeController;
+        const outDecoder = new sdk_1.ConsoleDecoder();
+        const errDecoder = new sdk_1.ConsoleDecoder();
+        let stdoutDecodeChain = Promise.resolve();
+        let stderrDecodeChain = Promise.resolve();
+        const cleanup = () => {
+            if (timeoutTimer)
+                clearTimeout(timeoutTimer);
+            if (cancellationEscalationTimer)
+                clearTimeout(cancellationEscalationTimer);
+            if (cancellationPollTimer)
+                clearInterval(cancellationPollTimer);
+            if (opts.cancellation
+                && activeController
+                && activeProcessControllers.get(opts.cancellation.key) === activeController) {
+                activeProcessControllers.delete(opts.cancellation.key);
+            }
+        };
         const finish = (result) => {
             if (settled)
                 return;
             settled = true;
-            clearTimeout(timer);
-            resolve(result);
+            cleanup();
+            resolve(cancellationRequested ? { ...result, cancelled: true, timedOut: false } : result);
         };
-        const timer = setTimeout(() => {
-            killProcessGroup(child);
-            finish({ status: null, stdout, stderr, timedOut: true });
+        const requestCancellation = () => {
+            if (settled || cancellationRequested)
+                return;
+            cancellationRequested = true;
+            void killProcessGroup(child, 'SIGTERM');
+            cancellationEscalationTimer = setTimeout(() => {
+                if (!settled)
+                    void killProcessGroup(child, 'SIGKILL');
+            }, exports.WORKFLOW_CANCEL_GRACE_MS);
+            if (typeof cancellationEscalationTimer.unref === 'function') {
+                cancellationEscalationTimer.unref();
+            }
+        };
+        if (opts.cancellation) {
+            activeController = { cancel: requestCancellation };
+            activeProcessControllers.set(opts.cancellation.key, activeController);
+            cancellationPollTimer = setInterval(() => {
+                if (isCancelled())
+                    requestCancellation();
+            }, WORKFLOW_CANCEL_POLL_INTERVAL_MS);
+            if (typeof cancellationPollTimer.unref === 'function')
+                cancellationPollTimer.unref();
+        }
+        timeoutTimer = setTimeout(() => {
+            void killProcessGroup(child);
+            finishAfterDecode(() => ({
+                status: null,
+                stdout,
+                stderr,
+                timedOut: true,
+                signal: null,
+                errorOrigin: stdinError ? 'stdin' : postSpawnError ? 'post_spawn' : null,
+                ...(stdinError || postSpawnError ? { error: stdinError ?? postSpawnError } : {}),
+            }));
         }, opts.timeout);
-        if (typeof timer.unref === 'function')
-            timer.unref();
-        child.stdout?.on('data', (d) => { stdout += d.toString(); });
-        child.stderr?.on('data', (d) => { stderr += d.toString(); });
-        child.on('error', (err) => finish({ status: null, stdout, stderr, timedOut: false, error: err }));
-        child.on('close', (code) => finish({ status: code, stdout, stderr, timedOut: false }));
+        if (typeof timeoutTimer.unref === 'function')
+            timeoutTimer.unref();
+        child.stdout?.on('data', (d) => {
+            stdoutDecodeChain = stdoutDecodeChain.then(async () => {
+                stdout += await outDecoder.write(d);
+            });
+        });
+        child.stderr?.on('data', (d) => {
+            stderrDecodeChain = stderrDecodeChain.then(async () => {
+                stderr += await errDecoder.write(d);
+            });
+        });
+        child.once('spawn', () => { spawned = true; });
+        child.on('error', (err) => {
+            if (!spawned) {
+                finish({
+                    status: null,
+                    stdout,
+                    stderr,
+                    timedOut: false,
+                    signal: null,
+                    errorOrigin: 'spawn',
+                    error: err,
+                });
+                return;
+            }
+            postSpawnError = postSpawnError ?? err;
+        });
+        if (opts.stdin !== undefined) {
+            child.stdin?.on('error', (err) => {
+                stdinError = stdinError ?? err;
+            });
+            try {
+                child.stdin?.end(opts.stdin);
+            }
+            catch (err) {
+                stdinError = stdinError ?? err;
+            }
+        }
+        const finishAfterDecode = (createResult) => {
+            if (settled || finishing)
+                return;
+            finishing = true;
+            void (async () => {
+                await Promise.all([stdoutDecodeChain, stderrDecodeChain]);
+                const [tailOut, tailErr] = await Promise.all([outDecoder.end(), errDecoder.end()]);
+                stdout += tailOut;
+                stderr += tailErr;
+                finish(createResult());
+            })().catch(() => finish(createResult()));
+        };
+        child.on('close', (code, signal) => finishAfterDecode(() => ({
+            status: code,
+            stdout,
+            stderr,
+            timedOut: false,
+            signal,
+            errorOrigin: stdinError ? 'stdin' : postSpawnError ? 'post_spawn' : null,
+            ...(stdinError || postSpawnError ? { error: stdinError ?? postSpawnError } : {}),
+        })));
+        if (isCancelled())
+            requestCancellation();
     });
 }
-function maskArg(arg) {
-    return arg.replace(/^(--?[\w-]*(?:token|secret|password|passwd|key|auth)[\w-]*)=(.+)$/i, '$1=***');
+const AUDIT_IDENTIFIER_RE = /^[A-Za-z0-9_.-]{1,128}$/;
+function safeAuditIdentifier(value) {
+    if (value === undefined)
+        return undefined;
+    const match = AUDIT_IDENTIFIER_RE.exec(value);
+    return match?.[0] === value ? value : undefined;
 }
 function summarizeNode(node) {
     switch (node.type) {
-        case 'agent_task':
-            return `@${node.agent ?? '?'} (${node.mode === 'query' ? 'q' : 'x'})`;
-        case 'skill_task':
-            return `skill ${node.skill ?? '?'} ${(node.args ?? []).map(maskArg).join(' ')}`.trim();
+        case 'agent_task': {
+            const agent = safeAuditIdentifier(node.agent);
+            return agent
+                ? `@${agent} (${node.mode === 'query' ? 'q' : 'x'})`
+                : `agent_task (${node.mode === 'query' ? 'q' : 'x'})`;
+        }
+        case 'skill_task': {
+            const skill = safeAuditIdentifier(node.skill);
+            return skill ? `skill ${skill}` : 'skill_task';
+        }
         case 'shell_task':
-            return (node.command ?? []).map(maskArg).join(' ');
+            return 'shell_task';
         case 'branch':
-            return `branch: ${node.condition ?? ''}`;
+            return 'branch';
         case 'expression':
-            return `set ${Object.keys(node.set ?? {}).join(', ')}`;
+            return 'expression';
         default:
-            return node.type;
+            return 'workflow node';
     }
 }
+function getErrorMessage(error) {
+    if (error instanceof Error)
+        return error.message;
+    return typeof error === 'string' ? error : 'Node execution failed';
+}
+function toProcessFailureMetadata(result) {
+    return {
+        status: result.status,
+        timedOut: result.timedOut,
+        signal: result.signal,
+        errorOrigin: result.errorOrigin,
+    };
+}
+function appendFailureAudit(run, failure, type, trigger, now = new Date().toISOString()) {
+    run.audit = run.audit ?? [];
+    run.audit.push({
+        exec_id: run.id,
+        node_id: failure.failedNode,
+        type,
+        summary: failure.causeMessage,
+        started_at: now,
+        ended_at: now,
+        duration_ms: 0,
+        exit_code: failure.exitCode,
+        error: failure.causeMessage,
+        trigger,
+    });
+}
+function assignRunFailure(run, failure, rawError) {
+    run.status = 'failed';
+    run.error = (0, failure_1.sanitizeLocalRunError)(rawError, failure.causeMessage);
+    run.last_failure = failure;
+}
 async function storeNodeOutput(nodeSpec, nodeId, stdout, stateUpdates) {
     if (!nodeSpec.output)
         return;
@@ -140,12 +345,12 @@ async function storeNodeOutput(nodeSpec, nodeId, stdout, stateUpdates) {
         let parsed;
         try {
             parsed = JSON.parse((0, output_format_1.extractJson)(trimmed));
+            if (nodeSpec.output_schema) {
+                await (0, output_format_1.validateJsonSchema)(parsed, nodeSpec.output_schema);
+            }
         }
         catch (e) {
-            throw new Error(`Node "${nodeId}" output_format=json parse failed: ${e.message}`);
-        }
-        if (nodeSpec.output_schema) {
-            await (0, output_format_1.validateJsonSchema)(parsed, nodeSpec.output_schema);
+            throw (0, failure_1.createWorkflowFailureError)('OUTPUT_INVALID', `Node "${nodeId}" output_format=json parse or validation failed: ${getErrorMessage(e)}`);
         }
         stateUpdates[nodeSpec.output] = parsed;
     }
@@ -521,12 +726,99 @@ function safeEvaluateBranchCondition(expr, state) {
 function validateSafeBranchCondition(expr) {
     safeEvaluateBranchCondition(expr, {});
 }
+function readRunEngineState(run) {
+    const raw = run.state['__engine'];
+    if (!raw || typeof raw !== 'object')
+        return { visitCounts: {}, iterations: 0 };
+    const record = raw;
+    const rawCounts = record['visitCounts'];
+    const visitCounts = {};
+    if (rawCounts && typeof rawCounts === 'object') {
+        for (const [key, value] of Object.entries(rawCounts)) {
+            if (typeof value === 'number' && Number.isFinite(value))
+                visitCounts[key] = value;
+        }
+    }
+    const iterations = typeof record['iterations'] === 'number' && Number.isFinite(record['iterations'])
+        ? record['iterations']
+        : 0;
+    return { visitCounts, iterations };
+}
+const inFlightAutoRuns = new Set();
+const activeProcessControllers = new Map();
+function processControllerKey(resolvedDir, execId) {
+    return `${resolvedDir}::${execId}`;
+}
+function requestActiveProcessCancellation(resolvedDir, execId) {
+    activeProcessControllers.get(processControllerKey(resolvedDir, execId))?.cancel();
+}
+function isProcessAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    if (pid === process.pid)
+        return true;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+}
+function isTerminalStatus(status) {
+    return status === 'completed' || status === 'failed' || status === 'cancelled';
+}
+function isRecentLegacyRetry(run, nowMs) {
+    if (run.status !== undefined || run.auto_run)
+        return false;
+    const lastAudit = run.audit?.at(-1);
+    if (lastAudit?.type !== 'retry')
+        return false;
+    const updatedAt = Date.parse(run.updated_at);
+    return Number.isFinite(updatedAt)
+        && nowMs - updatedAt < exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS;
+}
+function normalizeRunStatus(run, nowMs = Date.now()) {
+    if (run.status !== undefined)
+        return run;
+    let status;
+    if (run.current_node === '') {
+        status = 'pending';
+    }
+    else if (run.auto_run) {
+        status = 'running';
+    }
+    else if (isRecentLegacyRetry(run, nowMs)) {
+        status = 'retrying';
+    }
+    else {
+        status = 'pending';
+    }
+    return { ...run, status };
+}
+function isRetryableFailure(run, spec) {
+    if (run.status !== 'failed')
+        return false;
+    if (!run.current_node)
+        return false;
+    const node = spec.nodes?.[run.current_node];
+    if (!node)
+        return false;
+    return node.type !== 'error';
+}
 class RunManager {
-    constructor(runsDir = '.crewx/workflow-runs', projectRoot) {
+    constructor(runsDir = '.crewx/workflow-runs', projectRoot, deps) {
         this.runsDir = runsDir;
+        this.sleep = deps?.sleep ?? sleep;
+        this.onRunPersisted = deps?.onRunPersisted;
         const base = projectRoot || process.env.CREWX_WORKSPACE || process.cwd();
         this.resolvedDir = path.resolve(base, this.runsDir);
         this.projectRoot = path.resolve(base);
+        const hasConfig = fs.existsSync(path.join(this.projectRoot, 'crewx.yaml'))
+            || fs.existsSync(path.join(this.projectRoot, 'crewx.yml'));
+        if (!hasConfig) {
+            console.warn(`[RunManager] projectRoot "${this.projectRoot}" has no crewx.yaml/crewx.yml — agent_task nodes may fail to resolve agents.`);
+        }
     }
     ensureRunsDir() {
         if (!fs.existsSync(this.resolvedDir)) {
@@ -538,8 +830,21 @@ class RunManager {
             throw new Error(`Invalid execution ID: "${execId}"`);
         }
     }
-    loadRun(execId) {
-        this.validateExecId(execId);
+    isRunCancelled(execId) {
+        try {
+            return this.readRunRaw(execId)?.status === 'cancelled';
+        }
+        catch {
+            return false;
+        }
+    }
+    processCancellation(execId) {
+        return {
+            key: processControllerKey(this.resolvedDir, execId),
+            isCancelled: () => this.isRunCancelled(execId),
+        };
+    }
+    readRunRaw(execId) {
         const filePath = path.join(this.resolvedDir, `${execId}.json`);
         if (!fs.existsSync(filePath))
             return null;
@@ -550,11 +855,145 @@ class RunManager {
             throw new Error(`Failed to parse execution file ${filePath}: ${e.message}`);
         }
     }
+    readRun(execId) {
+        this.validateExecId(execId);
+        const run = this.readRunRaw(execId);
+        return run ? normalizeRunStatus(run) : null;
+    }
+    isLegacyStatuslessStale(run) {
+        if (run.status !== undefined
+            || run.auto_run
+            || run.manual_resume
+            || !run.current_node
+            || !run.audit?.length) {
+            return false;
+        }
+        const updatedAt = Date.parse(run.updated_at);
+        if (!Number.isFinite(updatedAt) || Date.now() - updatedAt < exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS) {
+            return false;
+        }
+        const doc = this.loadWorkflowYaml(run.workflow_file);
+        const spec = doc?.workflows[run.workflow_id];
+        const node = spec?.nodes?.[run.current_node];
+        return Boolean(node && KNOWN_AUTO_NODE_TYPES.has(node.type));
+    }
+    withRunLock(execId, callback) {
+        this.validateExecId(execId);
+        const filePath = path.join(this.resolvedDir, `${execId}.json`);
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`Execution not found: ${execId}`);
+        }
+        const lock = new file_lock_1.FileLock(filePath);
+        lock.acquire();
+        try {
+            const run = this.readRunRaw(execId);
+            if (!run)
+                throw new Error(`Execution not found: ${execId}`);
+            return callback(normalizeRunStatus(run));
+        }
+        finally {
+            lock.release();
+        }
+    }
+    hasStaleRunner(run) {
+        if (run.auto_run)
+            return !isProcessAlive(run.auto_run.pid);
+        return this.isLegacyStatuslessStale(run);
+    }
+    recoverStaleRunner(execId) {
+        const filePath = path.join(this.resolvedDir, `${execId}.json`);
+        const lock = new file_lock_1.FileLock(filePath);
+        lock.acquire();
+        try {
+            const run = this.readRunRaw(execId);
+            if (!run || !this.hasStaleRunner(run))
+                return run ? normalizeRunStatus(run) : null;
+            if (isTerminalStatus(run.status)) {
+                delete run.auto_run;
+                delete run.manual_resume;
+                this.saveRun(run);
+                return run;
+            }
+            const lease = run.auto_run;
+            const nodeId = lease?.node_id || run.current_node;
+            const now = new Date().toISOString();
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt: (0, failure_1.readNodeAttempt)(run, nodeId),
+                causeCode: 'INTERRUPTED',
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            });
+            assignRunFailure(run, failure, exports.WORKFLOW_RUN_INTERRUPTED_ERROR);
+            delete run.auto_run;
+            delete run.manual_resume;
+            appendFailureAudit(run, failure, 'interrupted', 'auto', now);
+            this.saveRun(run);
+            return run;
+        }
+        finally {
+            lock.release();
+        }
+    }
+    loadRun(execId) {
+        this.validateExecId(execId);
+        const run = this.readRunRaw(execId);
+        if (!run || !this.hasStaleRunner(run))
+            return run ? normalizeRunStatus(run) : null;
+        return this.recoverStaleRunner(execId);
+    }
     saveRun(run) {
         this.ensureRunsDir();
         run.updated_at = new Date().toISOString();
         const filePath = path.join(this.resolvedDir, `${run.id}.json`);
-        fs.writeFileSync(filePath, JSON.stringify(run, null, 2) + '\n', 'utf-8');
+        let existingMode;
+        try {
+            existingMode = fs.statSync(filePath).mode & 0o7777;
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+        }
+        const tempPath = path.join(this.resolvedDir, `.${run.id}.tmp-${process.pid}-${(0, sdk_1.generateId)('wfr')}`);
+        const serialized = JSON.stringify(run, null, 2) + '\n';
+        let tempFd;
+        let tempCreated = false;
+        try {
+            tempFd = fs.openSync(tempPath, 'wx');
+            tempCreated = true;
+            fs.writeFileSync(tempFd, serialized, 'utf-8');
+            if (existingMode !== undefined)
+                fs.fchmodSync(tempFd, existingMode);
+            fs.closeSync(tempFd);
+            tempFd = undefined;
+            fs.renameSync(tempPath, filePath);
+            tempCreated = false;
+        }
+        finally {
+            if (tempFd !== undefined) {
+                try {
+                    fs.closeSync(tempFd);
+                }
+                catch {
+                }
+            }
+            if (tempCreated) {
+                try {
+                    fs.unlinkSync(tempPath);
+                }
+                catch {
+                }
+            }
+        }
+        if (!this.onRunPersisted)
+            return;
+        try {
+            this.onRunPersisted(run);
+        }
+        catch (error) {
+            console.warn(`[RunManager] onRunPersisted failed for ${run.id}: ${getErrorMessage(error)}`);
+        }
     }
     atomicUpdate(execId, updater) {
         this.validateExecId(execId);
@@ -563,7 +1002,7 @@ class RunManager {
         const lock = new file_lock_1.FileLock(filePath);
         lock.acquire();
         try {
-            const run = this.loadRun(execId);
+            const run = this.readRun(execId);
             if (!run)
                 throw new Error(`Execution not found: ${execId}`);
             updater(run);
@@ -581,7 +1020,7 @@ class RunManager {
         const lock = new file_lock_1.FileLock(filePath);
         await lock.acquireAsync();
         try {
-            const run = this.loadRun(execId);
+            const run = this.readRun(execId);
             if (!run)
                 throw new Error(`Execution not found: ${execId}`);
             updater(run);
@@ -592,6 +1031,32 @@ class RunManager {
             lock.release();
         }
     }
+    async reserveNodeAttempt(execId, nodeId) {
+        let reservedAttempt = 1;
+        await this.atomicUpdateAsync(execId, (run) => {
+            if (run.status === 'cancelled') {
+                reservedAttempt = (0, failure_1.readNodeAttempt)(run, nodeId);
+                return;
+            }
+            const storedAttempt = run.node_attempts?.[nodeId];
+            const previousAttempt = typeof storedAttempt === 'number'
+                && Number.isInteger(storedAttempt)
+                && storedAttempt > 0
+                ? storedAttempt
+                : 0;
+            reservedAttempt = previousAttempt + 1;
+            run.node_attempts = run.node_attempts ?? {};
+            run.node_attempts[nodeId] = reservedAttempt;
+        });
+        return reservedAttempt;
+    }
+    clearNodeAttempt(run, nodeId) {
+        if (!run.node_attempts)
+            return;
+        delete run.node_attempts[nodeId];
+        if (Object.keys(run.node_attempts).length === 0)
+            delete run.node_attempts;
+    }
     loadWorkflowYaml(yamlPath) {
         if (!fs.existsSync(yamlPath))
             return null;
@@ -601,6 +1066,12 @@ class RunManager {
             return null;
         return doc;
     }
+    getNodeType(workflowFile, workflowId, nodeId) {
+        const doc = this.loadWorkflowYaml(workflowFile);
+        const spec = doc?.workflows[workflowId];
+        const node = spec?.nodes?.[nodeId];
+        return typeof node?.type === 'string' ? node.type : null;
+    }
     start(yamlPath, workflowId, overrides) {
         const doc = this.loadWorkflowYaml(yamlPath);
         if (!doc) {
@@ -638,6 +1109,7 @@ class RunManager {
             completed_nodes: [],
             state,
             initial_state: { ...initialState, ...sets },
+            status: 'pending',
         };
         this.saveRun(run);
         return run;
@@ -656,6 +1128,8 @@ class RunManager {
     }
     moveNode(execId, nodeId) {
         return this.atomicUpdate(execId, (run) => {
+            if (run.status === 'cancelled')
+                return;
             if (run.current_node && !run.completed_nodes.includes(run.current_node)) {
                 run.completed_nodes.push(run.current_node);
             }
@@ -667,17 +1141,20 @@ class RunManager {
         if (!snapshot) {
             throw new Error(`Execution not found: ${execId}`);
         }
+        if (snapshot.status === 'cancelled') {
+            throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `Workflow run "${execId}" is already cancelled`);
+        }
         const doc = this.loadWorkflowYaml(snapshot.workflow_file);
         if (!doc) {
-            throw new Error(`Failed to load workflow file: ${snapshot.workflow_file}`);
+            throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Failed to load workflow file: ${snapshot.workflow_file}`);
         }
         const spec = doc.workflows[snapshot.workflow_id];
         if (!spec) {
-            throw new Error(`Workflow "${snapshot.workflow_id}" not found in ${snapshot.workflow_file}`);
+            throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Workflow "${snapshot.workflow_id}" not found in ${snapshot.workflow_file}`);
         }
         const nodeSpec = spec.nodes?.[nodeId];
         if (!nodeSpec) {
-            throw new Error(`Node "${nodeId}" not found in workflow "${snapshot.workflow_id}"`);
+            throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" not found in workflow "${snapshot.workflow_id}"`);
         }
         const prevNode = snapshot.current_node;
         const stateUpdates = {};
@@ -685,12 +1162,30 @@ class RunManager {
         let currentNode = nodeId;
         let status;
         let failureMessage;
+        let failureCode;
         let nodeTaskId;
         const trigger = opts?.trigger ?? 'manual';
         const startedAt = new Date();
-        let nodeStdout = '';
-        let nodeStderr = '';
         let nodeExitCode = null;
+        let nodeSignal = null;
+        let nodeErrorOrigin = null;
+        let processFailure;
+        let failWhenReason;
+        const recordProcessResult = (result) => {
+            nodeExitCode = result.status;
+            nodeSignal = result.signal;
+            nodeErrorOrigin = result.errorOrigin;
+            processFailure = toProcessFailureMetadata(result);
+        };
+        const reserveAgentTaskId = async () => {
+            const taskId = (0, sdk_1.generateId)('tsk');
+            await this.atomicUpdateAsync(execId, (run) => {
+                run.tasks = run.tasks ?? {};
+                run.tasks[nodeId] = taskId;
+            });
+            nodeTaskId = taskId;
+            return taskId;
+        };
         const buildAudit = (error) => {
             const endedAt = new Date();
             return {
@@ -706,31 +1201,57 @@ class RunManager {
                 trigger,
             };
         };
+        const attempt = await this.reserveNodeAttempt(execId, nodeId);
         try {
             switch (nodeSpec.type) {
                 case 'agent_task': {
                     if (!nodeSpec.agent) {
-                        throw new Error(`Node "${nodeId}" is agent_task but has no agent defined`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" is agent_task but has no agent defined`);
+                    }
+                    const agentCtx = { ...snapshot.state, state: snapshot.state };
+                    const resolvedAgent = interpolateTemplate(nodeSpec.agent, agentCtx).trim();
+                    if (!resolvedAgent) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": assignee not resolved (agent field "${nodeSpec.agent}" interpolated to empty)`);
+                    }
+                    const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, agentCtx) : undefined, this.projectRoot);
+                    if (!cwdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cwdCheck.error}`);
                     }
+                    const nodeCwd = cwdCheck.resolved;
+                    const rootCheck = (0, shell_security_1.normalizeCwd)(undefined, this.projectRoot);
+                    if (!rootCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${rootCheck.error}`);
+                    }
+                    const workspaceRoot = rootCheck.resolved;
                     const input = nodeSpec.input
-                        ? interpolateTemplate(nodeSpec.input, { ...snapshot.state, state: snapshot.state })
+                        ? interpolateTemplate(nodeSpec.input, agentCtx)
                         : '';
+                    const preamble = process.env.CREWX_WORKFLOW_PREAMBLE === 'off'
+                        ? ''
+                        : (0, output_format_1.buildWorkflowPreamble)({
+                            workflowId: snapshot.workflow_id,
+                            nodeId,
+                            execId: snapshot.id,
+                        });
                     const finalInput = nodeSpec.output_format === 'json'
-                        ? input + (0, output_format_1.buildJsonOutputSuffix)(nodeSpec.output_schema, nodeSpec.output_strict)
-                        : input;
+                        ? preamble + input + (0, output_format_1.buildJsonOutputSuffix)(nodeSpec.output_schema, nodeSpec.output_strict)
+                        : preamble + input;
                     const mode = nodeSpec.mode === 'query' ? 'q' : 'x';
-                    const prompt = `@${nodeSpec.agent} ${finalInput}`;
+                    const prompt = `@${resolvedAgent} ${finalInput}`;
                     if (opts?.dryRun) {
-                        console.log(`[dry-run] Would execute: crewx ${mode} "@${nodeSpec.agent} ..."`);
+                        console.log(`[dry-run] Would execute: crewx ${mode} "@${resolvedAgent} ..."`);
                         break;
                     }
-                    console.log(`Executing: @${nodeSpec.agent} (${mode} mode)`);
-                    const crewxCli = process.env.CREWX_CLI || 'npx crewx';
-                    const cliParts = crewxCli.split(/\s+/);
-                    const bin = cliParts[0];
-                    const baseArgs = cliParts.slice(1);
+                    console.log(`Executing: @${resolvedAgent} (${mode} mode)`);
                     const nodeTimeout = Number(process.env[mode === 'q' ? 'CREWX_TIMEOUT_QUERY' : 'CREWX_TIMEOUT_EXECUTE']) || 8 * 3600000;
-                    const wfArgs = [...baseArgs, mode, prompt];
+                    const childInvocation = resolveCrewxChildInvocation({
+                        ...process.env,
+                        CREWX_WORKSPACE: workspaceRoot,
+                        CREWX_WORKFLOW_EXEC_ID: snapshot.id,
+                        CREWX_WORKFLOW_NODE_ID: nodeId,
+                        CREWX_WORKFLOW_ID: snapshot.workflow_id,
+                    }, workspaceRoot);
+                    const wfArgs = [...childInvocation.args, mode, prompt];
                     if (process.env.CREWX_WORKFLOW_THREAD === 'on') {
                         wfArgs.push(`--thread=workflow:${snapshot.id}`);
                     }
@@ -743,39 +1264,47 @@ class RunManager {
                     };
                     wfArgs.push('--metadata', JSON.stringify(wfMetadata));
                     const program = (0, sdk_1.resolveWindowsSpawnProgram)({
-                        command: bin,
+                        command: childInvocation.command,
+                        env: childInvocation.env,
+                        cwd: nodeCwd,
                         allowShellFallback: true,
                         packageName: 'crewx',
                     });
                     const invocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, wfArgs);
+                    const taskId = await reserveAgentTaskId();
                     const result = await runProcess(invocation.command, invocation.argv, {
-                        cwd: process.cwd(),
-                        env: {
-                            ...process.env,
-                            CREWX_WORKFLOW_EXEC_ID: snapshot.id,
-                            CREWX_WORKFLOW_NODE_ID: nodeId,
-                            CREWX_WORKFLOW_ID: snapshot.workflow_id,
-                        },
+                        cwd: nodeCwd,
+                        env: (0, sdk_1.withoutCrewxRootContext)({
+                            ...childInvocation.env,
+                            CREWX_TRACE_ID: taskId,
+                        }),
                         timeout: nodeTimeout,
                         shell: invocation.shell ?? false,
                         windowsHide: invocation.windowsHide,
+                        cancellation: this.processCancellation(execId),
                     });
-                    nodeStdout = result.stdout;
-                    nodeStderr = result.stderr;
-                    nodeExitCode = result.status;
-                    if (result.error) {
-                        throw result.error;
+                    recordProcessResult(result);
+                    if (result.cancelled) {
+                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `Agent execution cancelled for node "${nodeId}"`);
                     }
                     if (result.timedOut) {
                         throw new Error(`Agent execution timed out after ${nodeTimeout}ms`);
                     }
+                    if (result.error) {
+                        throw result.error;
+                    }
                     const taskIdMatch = result.stderr.match(/crewx kill (tsk_\w+)/);
-                    nodeTaskId = taskIdMatch?.[1];
+                    if (taskIdMatch?.[1])
+                        nodeTaskId = taskIdMatch[1];
                     if (result.status !== 0) {
                         const stderr = result.stderr.trim();
                         throw new Error(`Agent execution failed (exit ${result.status}): ${stderr}`);
                     }
                     const output = result.stdout.trim();
+                    const signal = (0, output_format_1.detectWorkflowSignal)(output);
+                    if (signal) {
+                        throw new Error(`Node "${nodeId}" agent signaled failure: ${signal.reason ?? '(no reason given)'}`);
+                    }
                     if (nodeSpec.output) {
                         if (nodeSpec.output_format === 'json') {
                             const retryMax = nodeSpec.output_retry ?? 1;
@@ -788,36 +1317,41 @@ class RunManager {
                                     retryInput = finalInput +
                                         `\n\n## ⚠️ PREVIOUS OUTPUT FAILED JSON.parse: ${lastError.message}\n` +
                                         `Return STRICT JSON ONLY. First char "{", last char "}". No prose, no fences.`;
-                                    const retryPrompt = `@${nodeSpec.agent} ${retryInput}`;
-                                    const retryArgs = [...baseArgs, mode, retryPrompt];
+                                    const retryPrompt = `@${resolvedAgent} ${retryInput}`;
+                                    const retryArgs = [...childInvocation.args, mode, retryPrompt];
                                     if (process.env.CREWX_WORKFLOW_THREAD === 'on') {
                                         retryArgs.push(`--thread=workflow:${snapshot.id}`);
                                     }
                                     retryArgs.push('--metadata', JSON.stringify(wfMetadata));
                                     const retryInvocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, retryArgs);
+                                    const retryTaskId = await reserveAgentTaskId();
                                     const retryResult = await runProcess(retryInvocation.command, retryInvocation.argv, {
-                                        cwd: process.cwd(),
-                                        env: {
-                                            ...process.env,
-                                            CREWX_WORKFLOW_EXEC_ID: snapshot.id,
-                                            CREWX_WORKFLOW_NODE_ID: nodeId,
-                                            CREWX_WORKFLOW_ID: snapshot.workflow_id,
-                                        },
+                                        cwd: nodeCwd,
+                                        env: (0, sdk_1.withoutCrewxRootContext)({
+                                            ...childInvocation.env,
+                                            CREWX_TRACE_ID: retryTaskId,
+                                        }),
                                         timeout: nodeTimeout,
                                         shell: retryInvocation.shell ?? false,
                                         windowsHide: retryInvocation.windowsHide,
+                                        cancellation: this.processCancellation(execId),
                                     });
-                                    nodeStdout = retryResult.stdout;
-                                    nodeStderr = retryResult.stderr;
-                                    nodeExitCode = retryResult.status;
-                                    if (retryResult.error)
-                                        throw retryResult.error;
+                                    recordProcessResult(retryResult);
+                                    if (retryResult.cancelled) {
+                                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `Agent retry cancelled for node "${nodeId}"`);
+                                    }
                                     if (retryResult.timedOut)
                                         throw new Error(`Agent retry timed out after ${nodeTimeout}ms`);
+                                    if (retryResult.error)
+                                        throw retryResult.error;
                                     if (retryResult.status !== 0) {
                                         throw new Error(`Agent retry failed (exit ${retryResult.status}): ${retryResult.stderr.trim()}`);
                                     }
                                     attemptOutput = retryResult.stdout.trim();
+                                    const retrySignal = (0, output_format_1.detectWorkflowSignal)(attemptOutput);
+                                    if (retrySignal) {
+                                        throw new Error(`Node "${nodeId}" agent signaled failure: ${retrySignal.reason ?? '(no reason given)'}`);
+                                    }
                                 }
                                 try {
                                     parsed = JSON.parse((0, output_format_1.extractJson)(attemptOutput));
@@ -831,7 +1365,7 @@ class RunManager {
                                 }
                             }
                             if (parsed === null) {
-                                throw new Error(`Node "${nodeId}" output_format=json parse failed after ${retryMax + 1} attempt(s): ${lastError.message}`);
+                                throw (0, failure_1.createWorkflowFailureError)('OUTPUT_INVALID', `Node "${nodeId}" output_format=json parse failed after ${retryMax + 1} attempt(s): ${lastError.message}`);
                             }
                             stateUpdates[nodeSpec.output] = parsed;
                             console.log(`Result stored in state.${nodeSpec.output} (JSON object)`);
@@ -845,52 +1379,64 @@ class RunManager {
                 }
                 case 'skill_task': {
                     if (!nodeSpec.skill) {
-                        throw new Error(`Node "${nodeId}" is skill_task but has no skill defined`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" is skill_task but has no skill defined`);
                     }
                     if (!shell_security_1.SKILL_NAME_RE.test(nodeSpec.skill)) {
-                        throw new Error(`Node "${nodeId}" has invalid skill name "${nodeSpec.skill}" (must match ^[a-z0-9][a-z0-9-]*$)`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}" has invalid skill name "${nodeSpec.skill}" (must match ^[a-z0-9][a-z0-9-]*$)`);
                     }
                     const ctx = { ...snapshot.state, state: snapshot.state };
                     const skillArgs = (nodeSpec.args ?? []).map((a) => interpolateTemplate(String(a), ctx));
+                    const skillStdin = nodeSpec.stdin !== undefined
+                        ? interpolateTemplate(String(nodeSpec.stdin), ctx)
+                        : undefined;
                     const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, ctx) : undefined, this.projectRoot);
-                    if (!cwdCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${cwdCheck.error}`);
+                    if (!cwdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cwdCheck.error}`);
+                    }
+                    const skillRootCheck = (0, shell_security_1.normalizeCwd)(undefined, this.projectRoot);
+                    if (!skillRootCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${skillRootCheck.error}`);
+                    }
+                    const skillWorkspaceRoot = skillRootCheck.resolved;
                     const timeout = nodeSpec.timeout ?? exports.SKILL_TASK_DEFAULT_TIMEOUT;
                     if (opts?.dryRun) {
                         console.log(`[dry-run] Would execute: crewx skill ${nodeSpec.skill} ${skillArgs.join(' ')}`.trim());
                         break;
                     }
                     console.log(`Executing skill: ${nodeSpec.skill} ${skillArgs.join(' ')}`.trim());
-                    const crewxCli = process.env.CREWX_CLI || 'npx crewx';
-                    const cliParts = crewxCli.split(/\s+/);
-                    const bin = cliParts[0];
-                    const baseArgs = cliParts.slice(1);
-                    const skillArgv = [...baseArgs, 'skill', nodeSpec.skill, ...skillArgs];
+                    const childInvocation = resolveCrewxChildInvocation({
+                        ...process.env,
+                        CREWX_WORKSPACE: skillWorkspaceRoot,
+                        CREWX_WORKFLOW_EXEC_ID: snapshot.id,
+                        CREWX_WORKFLOW_NODE_ID: nodeId,
+                        CREWX_WORKFLOW_ID: snapshot.workflow_id,
+                    }, skillWorkspaceRoot);
+                    const skillArgv = [...childInvocation.args, 'skill', nodeSpec.skill, ...skillArgs];
                     const program = (0, sdk_1.resolveWindowsSpawnProgram)({
-                        command: bin,
+                        command: childInvocation.command,
+                        env: childInvocation.env,
+                        cwd: cwdCheck.resolved,
                         allowShellFallback: true,
                         packageName: 'crewx',
                     });
                     const invocation = (0, sdk_1.materializeWindowsSpawnProgram)(program, skillArgv);
                     const result = await runProcess(invocation.command, invocation.argv, {
                         cwd: cwdCheck.resolved,
-                        env: {
-                            ...process.env,
-                            CREWX_WORKFLOW_EXEC_ID: snapshot.id,
-                            CREWX_WORKFLOW_NODE_ID: nodeId,
-                            CREWX_WORKFLOW_ID: snapshot.workflow_id,
-                        },
+                        env: childInvocation.env,
                         timeout,
                         shell: invocation.shell ?? false,
                         windowsHide: invocation.windowsHide,
+                        stdin: skillStdin,
+                        cancellation: this.processCancellation(execId),
                     });
-                    nodeStdout = result.stdout;
-                    nodeStderr = result.stderr;
-                    nodeExitCode = result.status;
-                    if (result.error)
-                        throw result.error;
+                    recordProcessResult(result);
+                    if (result.cancelled) {
+                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `skill_task "${nodeId}" was cancelled`);
+                    }
                     if (result.timedOut)
                         throw new Error(`skill_task "${nodeId}" timed out after ${timeout}ms`);
+                    if (result.error)
+                        throw result.error;
                     if (result.status !== 0) {
                         throw new Error(`skill_task "${nodeId}" failed (exit ${result.status}): ${result.stderr.trim()}`);
                     }
@@ -904,25 +1450,29 @@ class RunManager {
                     const shellEnvEnabled = process.env.CREWX_WORKFLOW_SHELL === '1';
                     const shellMetaAllowed = spec.metadata?.shell_task_allowed === true;
                     if (!shellEnvEnabled || !shellMetaAllowed) {
-                        throw new Error('shell_task is disabled. Set CREWX_WORKFLOW_SHELL=1 and metadata.shell_task_allowed=true to enable.');
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', 'shell_task is disabled. Set CREWX_WORKFLOW_SHELL=1 and metadata.shell_task_allowed=true to enable.');
                     }
                     const cmdCheck = (0, shell_security_1.validateShellCommand)(nodeSpec.command);
-                    if (!cmdCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${cmdCheck.error}`);
+                    if (!cmdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cmdCheck.error}`);
+                    }
                     const envCheck = (0, shell_security_1.validateEnvKeys)(nodeSpec.env);
-                    if (!envCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${envCheck.error}`);
+                    if (!envCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${envCheck.error}`);
+                    }
                     const ctx = { ...snapshot.state, state: snapshot.state };
                     const command = nodeSpec.command.map((c) => interpolateTemplate(c, ctx));
                     const postCheck = (0, shell_security_1.validateShellCommand)(command);
-                    if (!postCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${postCheck.error}`);
+                    if (!postCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${postCheck.error}`);
+                    }
                     const interpolatedEnv = nodeSpec.env
                         ? Object.fromEntries(Object.entries(nodeSpec.env).map(([k, v]) => [k, interpolateTemplate(String(v), ctx)]))
                         : undefined;
                     const cwdCheck = (0, shell_security_1.normalizeCwd)(nodeSpec.cwd ? interpolateTemplate(nodeSpec.cwd, ctx) : undefined, this.projectRoot);
-                    if (!cwdCheck.ok)
-                        throw new Error(`Node "${nodeId}": ${cwdCheck.error}`);
+                    if (!cwdCheck.ok) {
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Node "${nodeId}": ${cwdCheck.error}`);
+                    }
                     const timeout = nodeSpec.timeout ?? exports.SKILL_TASK_DEFAULT_TIMEOUT;
                     if (opts?.dryRun) {
                         console.log(`[dry-run] Would execute: ${command.join(' ')}`);
@@ -933,19 +1483,25 @@ class RunManager {
                     childEnv.CREWX_WORKFLOW_EXEC_ID = snapshot.id;
                     childEnv.CREWX_WORKFLOW_NODE_ID = nodeId;
                     childEnv.CREWX_WORKFLOW_ID = snapshot.workflow_id;
+                    const shellStdin = nodeSpec.stdin !== undefined
+                        ? interpolateTemplate(String(nodeSpec.stdin), ctx)
+                        : undefined;
                     const result = await runProcess(command[0], command.slice(1), {
                         cwd: cwdCheck.resolved,
                         env: childEnv,
                         timeout,
                         shell: false,
+                        stdin: shellStdin,
+                        cancellation: this.processCancellation(execId),
                     });
-                    nodeStdout = result.stdout;
-                    nodeStderr = result.stderr;
-                    nodeExitCode = result.status;
-                    if (result.error)
-                        throw result.error;
+                    recordProcessResult(result);
+                    if (result.cancelled) {
+                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `shell_task "${nodeId}" was cancelled`);
+                    }
                     if (result.timedOut)
                         throw new Error(`shell_task "${nodeId}" timed out after ${timeout}ms`);
+                    if (result.error)
+                        throw result.error;
                     if (result.status !== 0) {
                         throw new Error(`shell_task "${nodeId}" failed (exit ${result.status}): ${result.stderr.trim()}`);
                     }
@@ -957,13 +1513,13 @@ class RunManager {
                 }
                 case 'branch': {
                     if (!nodeSpec.condition) {
-                        throw new Error(`Branch node "${nodeId}" has no condition`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Branch node "${nodeId}" has no condition`);
                     }
                     const condResult = String(safeEvaluateBranchCondition(String(nodeSpec.condition), snapshot.state));
                     const branches = nodeSpec.branches;
                     const targetNode = branches?.[condResult] ?? nodeSpec.default;
                     if (!targetNode) {
-                        throw new Error(`Branch "${nodeId}": no matching branch for "${condResult}" and no default`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Branch "${nodeId}": no matching branch for "${condResult}" and no default`);
                     }
                     console.log(`Branch "${nodeId}": condition="${nodeSpec.condition}" → ${condResult} → ${targetNode}`);
                     nodesToComplete.push(nodeId);
@@ -989,6 +1545,7 @@ class RunManager {
                 }
                 case 'error': {
                     status = 'failed';
+                    failureCode = 'UNKNOWN';
                     const ctx = { ...snapshot.state, state: snapshot.state };
                     failureMessage = nodeSpec.message
                         ? interpolateTemplate(nodeSpec.message, ctx)
@@ -998,7 +1555,7 @@ class RunManager {
                 }
                 case 'expression': {
                     if (!nodeSpec.set || typeof nodeSpec.set !== 'object') {
-                        throw new Error(`Expression node "${nodeId}" requires "set" field`);
+                        throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Expression node "${nodeId}" requires "set" field`);
                     }
                     for (const [key, expr] of Object.entries(nodeSpec.set)) {
                         const result = safeEvaluate(String(expr), snapshot.state);
@@ -1019,30 +1576,73 @@ class RunManager {
                 default:
                     console.log(`Unknown node type "${nodeSpec.type}" for node "${nodeId}"`);
             }
+            if (nodeSpec.fail_when &&
+                !opts?.dryRun &&
+                (nodeSpec.type === 'agent_task' || nodeSpec.type === 'skill_task' || nodeSpec.type === 'shell_task')) {
+                const judged = safeEvaluateBranchCondition(String(nodeSpec.fail_when), { ...snapshot.state, ...stateUpdates });
+                if (judged === true || judged === 'true') {
+                    const output = nodeSpec.output ? stateUpdates[nodeSpec.output] : undefined;
+                    if (output && typeof output === 'object' && !Array.isArray(output)) {
+                        const reason = output.reason;
+                        if (typeof reason === 'string') {
+                            const redactedReason = (0, sdk_1.redactTechnicalMessage)(reason);
+                            const firstLine = redactedReason
+                                ?.split(/\r\n|\r|\n/u)
+                                .find((line) => line.trim());
+                            if (firstLine)
+                                failWhenReason = firstLine.trim().slice(0, 180);
+                        }
+                    }
+                    throw new Error(`Node "${nodeId}" judged failed by fail_when: ${nodeSpec.fail_when}`);
+                }
+            }
         }
         catch (e) {
-            const errMessage = e.message;
-            const partial = [
-                nodeStdout.trim() ? `stdout: ${nodeStdout.trim().slice(-500)}` : '',
-                nodeStderr.trim() ? `stderr: ${nodeStderr.trim().slice(-500)}` : '',
-            ].filter(Boolean).join(' | ');
-            const auditEntry = buildAudit(partial ? `${errMessage} (${partial})` : errMessage);
+            const errMessage = getErrorMessage(e);
+            const causeCode = (0, failure_1.classifyWorkflowFailure)({
+                errorCode: (0, failure_1.getWorkflowFailureCode)(e),
+                process: processFailure,
+            });
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt,
+                causeCode,
+                exitCode: causeCode === 'NODE_EXIT_NONZERO' && typeof nodeExitCode === 'number'
+                    ? nodeExitCode
+                    : null,
+                signal: causeCode === 'INTERRUPTED' ? nodeSignal : null,
+                errorOrigin: nodeErrorOrigin,
+                timedOut: processFailure?.timedOut ?? false,
+            });
+            const auditEntry = buildAudit(failWhenReason ?? failure.causeMessage);
             await this.atomicUpdateAsync(execId, (run) => {
                 if (run.status === 'cancelled')
                     return;
+                Object.assign(run.state, stateUpdates);
+                delete run.manual_resume;
                 run.current_node = nodeId;
-                run.status = 'failed';
-                run.error = errMessage;
+                assignRunFailure(run, failure, failWhenReason ?? errMessage);
                 run.audit = run.audit ?? [];
                 run.audit.push(auditEntry);
             });
             throw e;
         }
-        const successAudit = buildAudit();
+        const successFailure = failureCode
+            ? (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt,
+                causeCode: failureCode,
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            })
+            : null;
+        const successAudit = buildAudit(successFailure?.causeMessage);
         return this.atomicUpdateAsync(execId, (run) => {
             if (run.status === 'cancelled')
                 return;
             Object.assign(run.state, stateUpdates);
+            delete run.manual_resume;
             for (const node of nodesToComplete) {
                 if (!run.completed_nodes.includes(node)) {
                     run.completed_nodes.push(node);
@@ -1052,10 +1652,20 @@ class RunManager {
                 run.completed_nodes.push(prevNode);
             }
             run.current_node = currentNode;
-            if (status)
-                run.status = status;
-            if (failureMessage)
-                run.error = failureMessage;
+            if (successFailure) {
+                assignRunFailure(run, successFailure, failureMessage ?? successFailure.causeMessage);
+            }
+            else {
+                delete run.error;
+                delete run.last_failure;
+                this.clearNodeAttempt(run, nodeId);
+                if (status) {
+                    run.status = status;
+                }
+                else if (run.status !== 'completed') {
+                    run.status = 'running';
+                }
+            }
             if (nodeTaskId) {
                 run.tasks = run.tasks ?? {};
                 run.tasks[nodeId] = nodeTaskId;
@@ -1064,123 +1674,471 @@ class RunManager {
             run.audit.push(successAudit);
         });
     }
-    async runAuto(execId) {
-        const initial = this.loadRun(execId);
-        if (!initial)
-            throw new Error(`Execution not found: ${execId}`);
-        const doc = this.loadWorkflowYaml(initial.workflow_file);
-        if (!doc)
-            throw new Error(`Failed to load workflow file: ${initial.workflow_file}`);
-        const spec = doc.workflows[initial.workflow_id];
-        if (!spec) {
-            throw new Error(`Workflow "${initial.workflow_id}" not found in ${initial.workflow_file}`);
-        }
-        const executed = [];
-        if (initial.status === 'completed' || initial.status === 'failed' || initial.status === 'cancelled') {
-            return {
-                run: initial,
-                outcome: initial.status === 'completed' ? 'completed' : initial.status === 'cancelled' ? 'cancelled' : 'failed',
-                reason: `run already ${initial.status}`,
-                executed,
+    async claimAutoRunLease(execId, nodeId, preferredToken) {
+        let claimed = false;
+        const updated = await this.atomicUpdateAsync(execId, (run) => {
+            if (isTerminalStatus(run.status))
+                return;
+            const existing = run.auto_run;
+            const ownedByThisProcess = existing?.pid === process.pid && typeof existing.token === 'string';
+            if (existing && !ownedByThisProcess && isProcessAlive(existing.pid)) {
+                throw new Error(`Workflow run "${execId}" is already owned by another runner process`);
+            }
+            const token = preferredToken ?? (ownedByThisProcess ? existing.token : (0, sdk_1.generateId)('wfr'));
+            if (!isTerminalStatus(run.status)) {
+                run.status = 'running';
+            }
+            if (!run.current_node)
+                run.current_node = nodeId;
+            run.auto_run = {
+                pid: process.pid,
+                token,
+                node_id: nodeId,
+                started_at: ownedByThisProcess && existing.token === token
+                    ? existing.started_at
+                    : new Date().toISOString(),
             };
+            delete run.manual_resume;
+            claimed = true;
+        });
+        return claimed ? updated.auto_run?.token ?? null : null;
+    }
+    async releaseAutoRunLease(execId, token) {
+        await this.atomicUpdateAsync(execId, (run) => {
+            if (run.auto_run?.token === token)
+                delete run.auto_run;
+        });
+    }
+    async failAutoConfiguration(execId, nodeId, reason) {
+        return this.atomicUpdateAsync(execId, (run) => {
+            if (run.status === 'cancelled')
+                return;
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: nodeId,
+                attempt: (0, failure_1.readNodeAttempt)(run, nodeId),
+                causeCode: 'CONFIG_INVALID',
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            });
+            assignRunFailure(run, failure, reason);
+            appendFailureAudit(run, failure, 'config', 'auto');
+        });
+    }
+    async reserveAutoRetry(execId, nodeId, runnerToken, policy, delayMs) {
+        let reserved = false;
+        const run = await this.atomicUpdateAsync(execId, (draft) => {
+            if (draft.status === 'cancelled')
+                return;
+            if (draft.status !== 'failed')
+                return;
+            if (draft.auto_run?.token !== runnerToken)
+                return;
+            const decision = (0, retry_policy_1.decideRetry)({ policy, run: draft, nodeId });
+            if (!decision.retry || decision.attempt === undefined)
+                return;
+            const now = new Date().toISOString();
+            draft.status = 'retrying';
+            delete draft.error;
+            delete draft.manual_resume;
+            draft.audit = draft.audit ?? [];
+            draft.audit.push({
+                exec_id: draft.id,
+                node_id: nodeId,
+                type: 'retry',
+                summary: `auto retry ${decision.attempt + 1}/${policy.max + 1} in ${delayMs}ms`,
+                started_at: now,
+                ended_at: now,
+                duration_ms: 0,
+                exit_code: null,
+                trigger: 'auto',
+            });
+            reserved = true;
+        });
+        return { run, reserved };
+    }
+    async waitForRetry(execId, runnerToken, delayMs) {
+        const inspect = () => {
+            const run = this.loadRun(execId);
+            if (!run)
+                throw new Error(`Execution not found: ${execId}`);
+            if (run.status === 'cancelled') {
+                return { run, outcome: 'cancelled', reason: 'run cancelled during retry delay' };
+            }
+            if (run.auto_run?.token !== runnerToken) {
+                return { run, outcome: 'paused', reason: 'auto-run lease lost during retry delay' };
+            }
+            if (run.status !== 'retrying') {
+                return { run, outcome: 'paused', reason: 'run state changed during retry delay' };
+            }
+            return null;
+        };
+        let remaining = Math.max(0, delayMs);
+        while (remaining > 0) {
+            const slice = Math.min(remaining, RETRY_WAIT_SLICE_MS);
+            await this.sleep(slice);
+            remaining -= slice;
+            const stopped = inspect();
+            if (stopped)
+                return stopped;
         }
-        const firstNode = Object.keys(spec.nodes ?? {})[0];
-        const maxIterations = (typeof spec.max_iterations === 'number' && spec.max_iterations > 0)
-            ? spec.max_iterations
-            : 30;
-        const visitCounts = {};
-        let iterations = 0;
-        let run = initial;
+        const stopped = inspect();
+        if (stopped)
+            return stopped;
+        const run = this.loadRun(execId);
+        if (!run)
+            throw new Error(`Execution not found: ${execId}`);
+        return { run, outcome: 'ready' };
+    }
+    async runNodeWithRetry(execId, nodeId, runnerToken, policy) {
+        let token = runnerToken;
         while (true) {
-            run = this.loadRun(execId);
-            if (run.status === 'completed')
-                return { run, outcome: 'completed', executed };
-            if (run.status === 'failed')
-                return { run, outcome: 'failed', reason: run.error, executed };
-            if (run.status === 'cancelled')
-                return { run, outcome: 'cancelled', reason: run.error, executed };
-            const nodeId = run.current_node || firstNode;
-            if (!nodeId)
-                return { run, outcome: 'failed', reason: 'workflow has no nodes', executed };
-            const node = spec.nodes[nodeId];
-            if (!node) {
-                run = await this.atomicUpdateAsync(execId, (r) => {
-                    r.current_node = nodeId;
-                    r.status = 'failed';
-                    r.error = `node "${nodeId}" not found in workflow`;
-                });
-                return { run, outcome: 'failed', reason: run.error, executed };
+            try {
+                const run = await this.executeNode(execId, nodeId, { trigger: 'auto' });
+                if (run.status === 'cancelled') {
+                    return { run, runnerToken: token, outcome: 'cancelled', reason: run.error };
+                }
+                return { run, runnerToken: token, outcome: 'success' };
+            }
+            catch (caught) {
+                const run = this.loadRun(execId);
+                if (!run)
+                    throw new Error(`Execution not found: ${execId}`);
+                if (run.status === 'cancelled') {
+                    return { run, runnerToken: token, outcome: 'cancelled', reason: run.error };
+                }
+                if (run.status !== 'failed' || !policy) {
+                    return { run, runnerToken: token, outcome: run.status === 'failed' ? 'failed' : 'paused', reason: getErrorMessage(caught) };
+                }
+                const decision = (0, retry_policy_1.decideRetry)({ policy, run, nodeId });
+                if (!decision.retry || decision.attempt === undefined) {
+                    return { run, runnerToken: token, outcome: 'failed', reason: getErrorMessage(caught) };
+                }
+                const delayMs = (0, retry_policy_1.computeRetryDelayMs)(policy, decision.attempt);
+                const reservation = await this.reserveAutoRetry(execId, nodeId, token, policy, delayMs);
+                if (!reservation.reserved) {
+                    const latest = reservation.run;
+                    if (latest.status === 'cancelled') {
+                        return { run: latest, runnerToken: token, outcome: 'cancelled', reason: latest.error };
+                    }
+                    return {
+                        run: latest,
+                        runnerToken: token,
+                        outcome: latest.status === 'failed' ? 'failed' : 'paused',
+                        reason: getErrorMessage(caught),
+                    };
+                }
+                const waited = await this.waitForRetry(execId, token, delayMs);
+                if (waited.outcome !== 'ready') {
+                    return { run: waited.run, runnerToken: token, outcome: waited.outcome, reason: waited.reason };
+                }
+                const nextToken = await this.claimAutoRunLease(execId, nodeId, token);
+                if (!nextToken) {
+                    const latest = this.loadRun(execId);
+                    if (!latest)
+                        throw new Error(`Execution not found: ${execId}`);
+                    if (latest.status === 'cancelled') {
+                        return { run: latest, runnerToken: token, outcome: 'cancelled', reason: latest.error };
+                    }
+                    return {
+                        run: latest,
+                        runnerToken: token,
+                        outcome: 'paused',
+                        reason: 'run stopped before retry lease could be reclaimed',
+                    };
+                }
+                token = nextToken;
+            }
+        }
+    }
+    async runAuto(execId) {
+        const leaseKey = `${this.resolvedDir}::${execId}`;
+        if (inFlightAutoRuns.has(leaseKey)) {
+            const run = this.loadRun(execId);
+            if (!run)
+                throw new Error(`Execution not found: ${execId}`);
+            console.log(`runAuto "${execId}": already in-flight — skipping duplicate auto-run (single-flight guard)`);
+            return { run, outcome: 'paused', reason: 'execution already in-flight (single-flight guard)', executed: [] };
+        }
+        inFlightAutoRuns.add(leaseKey);
+        let runnerToken;
+        try {
+            const initial = this.loadRun(execId);
+            if (!initial)
+                throw new Error(`Execution not found: ${execId}`);
+            if (initial.auto_run?.pid === process.pid) {
+                runnerToken = initial.auto_run.token;
             }
-            if (node.type === 'approval') {
-                return { run, outcome: 'paused', reason: `approval node "${nodeId}" requires manual decision`, executed };
+            const doc = this.loadWorkflowYaml(initial.workflow_file);
+            if (!doc) {
+                throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Failed to load workflow file: ${initial.workflow_file}`);
             }
-            if (node.type === 'parallel') {
-                return { run, outcome: 'paused', reason: `parallel auto-run not implemented (node "${nodeId}")`, executed };
+            const spec = doc.workflows[initial.workflow_id];
+            if (!spec) {
+                throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Workflow "${initial.workflow_id}" not found in ${initial.workflow_file}`);
             }
-            if (!KNOWN_AUTO_NODE_TYPES.has(node.type)) {
-                return { run, outcome: 'paused', reason: `unknown node type "${node.type}" (node "${nodeId}")`, executed };
+            const executed = [];
+            if (initial.status === 'completed' || initial.status === 'failed' || initial.status === 'cancelled') {
+                return {
+                    run: initial,
+                    outcome: initial.status === 'completed' ? 'completed' : initial.status === 'cancelled' ? 'cancelled' : 'failed',
+                    reason: `run already ${initial.status}`,
+                    executed,
+                };
             }
-            visitCounts[nodeId] = (visitCounts[nodeId] ?? 0) + 1;
-            iterations++;
-            if (visitCounts[nodeId] > maxIterations || iterations > maxIterations) {
+            const firstNode = Object.keys(spec.nodes ?? {})[0];
+            const maxIterations = (typeof spec.max_iterations === 'number' && spec.max_iterations > 0)
+                ? spec.max_iterations
+                : 30;
+            let run = initial;
+            while (true) {
+                run = this.loadRun(execId);
+                if (run.status === 'completed')
+                    return { run, outcome: 'completed', executed };
+                if (run.status === 'failed')
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                if (run.status === 'cancelled')
+                    return { run, outcome: 'cancelled', reason: run.error, executed };
+                const nodeId = run.current_node || firstNode;
+                if (!nodeId) {
+                    run = await this.atomicUpdateAsync(execId, (r) => {
+                        const failure = (0, failure_1.buildNodeFailure)({
+                            failedNode: r.current_node || '',
+                            attempt: (0, failure_1.readNodeAttempt)(r, r.current_node || ''),
+                            causeCode: 'CONFIG_INVALID',
+                            exitCode: null,
+                            signal: null,
+                            errorOrigin: null,
+                        });
+                        assignRunFailure(r, failure, 'workflow has no nodes');
+                        appendFailureAudit(r, failure, 'config', 'auto');
+                    });
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
+                const node = spec.nodes[nodeId];
+                if (!node) {
+                    run = await this.atomicUpdateAsync(execId, (r) => {
+                        r.current_node = nodeId;
+                        const failure = (0, failure_1.buildNodeFailure)({
+                            failedNode: nodeId,
+                            attempt: (0, failure_1.readNodeAttempt)(r, nodeId),
+                            causeCode: 'CONFIG_INVALID',
+                            exitCode: null,
+                            signal: null,
+                            errorOrigin: null,
+                        });
+                        assignRunFailure(r, failure, `node "${nodeId}" not found in workflow`);
+                        appendFailureAudit(r, failure, 'config', 'auto');
+                    });
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
+                const parsedRetryPolicy = (0, retry_policy_1.parseRetryPolicy)(node);
+                if (!parsedRetryPolicy.ok) {
+                    run = await this.failAutoConfiguration(execId, nodeId, parsedRetryPolicy.reason);
+                    if (run.status === 'cancelled') {
+                        return { run, outcome: 'cancelled', reason: run.error, executed };
+                    }
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
+                if (node.type === 'approval') {
+                    if (run.current_node !== nodeId) {
+                        run = await this.atomicUpdateAsync(execId, (r) => { r.current_node = nodeId; });
+                    }
+                    return { run, outcome: 'paused', reason: `approval node "${nodeId}" requires manual decision`, executed };
+                }
+                if (node.type === 'parallel') {
+                    if (run.current_node !== nodeId) {
+                        run = await this.atomicUpdateAsync(execId, (r) => { r.current_node = nodeId; });
+                    }
+                    return { run, outcome: 'paused', reason: `parallel auto-run not implemented (node "${nodeId}")`, executed };
+                }
+                if (!KNOWN_AUTO_NODE_TYPES.has(node.type)) {
+                    if (run.current_node !== nodeId) {
+                        run = await this.atomicUpdateAsync(execId, (r) => { r.current_node = nodeId; });
+                    }
+                    return { run, outcome: 'paused', reason: `unknown node type "${node.type}" (node "${nodeId}")`, executed };
+                }
+                const claimedToken = await this.claimAutoRunLease(execId, nodeId, runnerToken);
+                if (!claimedToken) {
+                    run = this.loadRun(execId);
+                    return {
+                        run,
+                        outcome: run.status === 'cancelled' ? 'cancelled' : 'paused',
+                        reason: run.status === 'cancelled' ? run.error : 'run stopped before auto-run lease could be claimed',
+                        executed,
+                    };
+                }
+                runnerToken = claimedToken;
                 run = await this.atomicUpdateAsync(execId, (r) => {
-                    r.status = 'failed';
-                    r.error = `auto runner exceeded max_iterations (${maxIterations}); possible loop at "${nodeId}"`;
+                    if (r.status === 'cancelled')
+                        return;
+                    const engine = readRunEngineState(r);
+                    engine.visitCounts[nodeId] = (engine.visitCounts[nodeId] ?? 0) + 1;
+                    engine.iterations += 1;
+                    r.state['__engine'] = engine;
+                    if (engine.visitCounts[nodeId] > maxIterations || engine.iterations > maxIterations) {
+                        const failure = (0, failure_1.buildNodeFailure)({
+                            failedNode: nodeId,
+                            attempt: (0, failure_1.readNodeAttempt)(r, nodeId),
+                            causeCode: 'CONFIG_INVALID',
+                            exitCode: null,
+                            signal: null,
+                            errorOrigin: null,
+                        });
+                        assignRunFailure(r, failure, `auto runner exceeded max_iterations (${maxIterations}); possible loop at "${nodeId}"`);
+                        appendFailureAudit(r, failure, 'config', 'auto');
+                    }
                 });
-                return { run, outcome: 'failed', reason: run.error, executed };
-            }
-            try {
-                run = await this.executeNode(execId, nodeId, { trigger: 'auto' });
+                if (run.status === 'cancelled') {
+                    return { run, outcome: 'cancelled', reason: run.error, executed };
+                }
+                if (run.status === 'failed') {
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                }
+                const nodeResult = await this.runNodeWithRetry(execId, nodeId, runnerToken, parsedRetryPolicy.policy);
+                run = nodeResult.run;
+                runnerToken = nodeResult.runnerToken;
+                if (nodeResult.outcome !== 'success') {
+                    return { run, outcome: nodeResult.outcome, reason: nodeResult.reason, executed };
+                }
                 executed.push(nodeId);
+                if (run.status === 'completed')
+                    return { run, outcome: 'completed', executed };
+                if (run.status === 'failed')
+                    return { run, outcome: 'failed', reason: run.error, executed };
+                if (run.status === 'cancelled')
+                    return { run, outcome: 'cancelled', reason: run.error, executed };
+                if (!SELF_ADVANCING_NODE_TYPES.has(node.type)) {
+                    const next = node.next;
+                    if (!next) {
+                        return { run, outcome: 'paused', reason: `node "${nodeId}" has no "next"; stopping`, executed };
+                    }
+                    run = this.moveNode(execId, next);
+                }
             }
-            catch (e) {
-                run = this.loadRun(execId);
-                return { run, outcome: 'failed', reason: e.message, executed };
-            }
-            if (run.status === 'completed')
-                return { run, outcome: 'completed', executed };
-            if (run.status === 'failed')
-                return { run, outcome: 'failed', reason: run.error, executed };
-            if (run.status === 'cancelled')
-                return { run, outcome: 'cancelled', reason: run.error, executed };
-            if (!SELF_ADVANCING_NODE_TYPES.has(node.type)) {
-                const next = node.next;
-                if (!next) {
-                    return { run, outcome: 'paused', reason: `node "${nodeId}" has no "next"; stopping`, executed };
+        }
+        finally {
+            if (runnerToken) {
+                try {
+                    await this.releaseAutoRunLease(execId, runnerToken);
+                }
+                catch {
                 }
-                run = this.moveNode(execId, next);
             }
+            inFlightAutoRuns.delete(leaseKey);
         }
     }
     cancelRun(execId, reason = 'Cancelled by user') {
-        return this.atomicUpdate(execId, (run) => {
+        const cancelled = this.atomicUpdate(execId, (run) => {
             if (run.status === 'completed' || run.status === 'failed' || run.status === 'cancelled') {
                 return;
             }
             const now = new Date();
+            const activeNodeId = run.auto_run?.node_id || run.current_node || '';
+            const failure = (0, failure_1.buildNodeFailure)({
+                failedNode: activeNodeId,
+                attempt: (0, failure_1.readNodeAttempt)(run, activeNodeId),
+                causeCode: 'CANCELLED',
+                exitCode: null,
+                signal: null,
+                errorOrigin: null,
+            });
+            assignRunFailure(run, failure, reason);
             run.status = 'cancelled';
-            run.error = reason;
+            delete run.auto_run;
+            delete run.manual_resume;
+            appendFailureAudit(run, failure, 'cancel', 'manual', now.toISOString());
+        });
+        if (cancelled.status === 'cancelled') {
+            requestActiveProcessCancellation(this.resolvedDir, execId);
+        }
+        return cancelled;
+    }
+    reset(execId) {
+        return this.atomicUpdate(execId, (run) => {
+            run.state = JSON.parse(JSON.stringify(run.initial_state));
+            run.current_node = '';
+            run.completed_nodes = [];
+            run.status = 'pending';
+            delete run.error;
+            delete run.last_failure;
+            delete run.node_attempts;
+            delete run.auto_run;
+            delete run.manual_resume;
+        });
+    }
+    retry(execId, options = {}) {
+        const snapshot = this.loadRun(execId);
+        if (!snapshot)
+            throw new Error(`Execution not found: ${execId}`);
+        const doc = this.loadWorkflowYaml(snapshot.workflow_file);
+        if (!doc)
+            throw new Error(`Failed to load workflow file: ${snapshot.workflow_file}`);
+        const spec = doc.workflows[snapshot.workflow_id];
+        if (!spec)
+            throw new Error(`Workflow "${snapshot.workflow_id}" not found`);
+        if (snapshot.status !== 'failed') {
+            throw new Error(`Run "${execId}" is not failed (status=${snapshot.status ?? 'running'}); only failed runs can be retried`);
+        }
+        if (!isRetryableFailure(snapshot, spec)) {
+            throw new Error(`Run "${execId}" failed at a non-retryable node "${snapshot.current_node}" (error node); use reset to restart`);
+        }
+        return this.atomicUpdate(execId, (run) => {
+            if (run.status !== 'failed') {
+                throw new Error(`Run "${execId}" is not failed (status=${run.status ?? 'pending'}); only failed runs can be retried`);
+            }
+            const failedNode = run.current_node;
+            if (failedNode && run.last_failure) {
+                const lastAttempt = run.last_failure.attempt;
+                const storedAttempt = run.node_attempts?.[failedNode];
+                if (typeof lastAttempt === 'number' && Number.isInteger(lastAttempt) && lastAttempt > 0
+                    && (typeof storedAttempt !== 'number' || storedAttempt < lastAttempt)) {
+                    run.node_attempts = run.node_attempts ?? {};
+                    run.node_attempts[failedNode] = lastAttempt;
+                }
+            }
+            run.status = 'retrying';
+            delete run.error;
+            delete run.manual_resume;
+            run.retry_count = (run.retry_count ?? 0) + 1;
             run.audit = run.audit ?? [];
+            const now = new Date().toISOString();
+            if (options.autoRun === false) {
+                delete run.auto_run;
+                run.manual_resume = { node_id: failedNode, requested_at: now };
+            }
+            else {
+                run.auto_run = {
+                    pid: process.pid,
+                    token: (0, sdk_1.generateId)('wfr'),
+                    node_id: failedNode,
+                    started_at: now,
+                };
+            }
             run.audit.push({
                 exec_id: run.id,
-                node_id: run.current_node || '',
-                type: 'cancel',
-                summary: reason,
-                started_at: now.toISOString(),
-                ended_at: now.toISOString(),
+                node_id: failedNode,
+                type: 'retry',
+                summary: `manual retry #${run.retry_count}`,
+                started_at: now,
+                ended_at: now,
                 duration_ms: 0,
                 exit_code: null,
-                error: reason,
                 trigger: 'manual',
             });
         });
     }
-    reset(execId) {
-        return this.atomicUpdate(execId, (run) => {
-            run.state = JSON.parse(JSON.stringify(run.initial_state));
-            run.current_node = '';
-            run.completed_nodes = [];
-        });
+    listRunIds() {
+        try {
+            return fs.readdirSync(this.resolvedDir, { withFileTypes: true })
+                .filter((entry) => entry.isFile() && !entry.name.startsWith('.') && entry.name.endsWith('.json'))
+                .map((entry) => entry.name.slice(0, -'.json'.length));
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return [];
+            throw error;
+        }
     }
     listRuns(opts) {
         this.ensureRunsDir();
@@ -1188,7 +2146,10 @@ class RunManager {
         const runs = [];
         for (const file of files) {
             try {
-                const run = JSON.parse(fs.readFileSync(path.join(this.resolvedDir, file), 'utf-8'));
+                const execId = file.slice(0, -'.json'.length);
+                const run = this.loadRun(execId);
+                if (!run)
+                    continue;
                 if (opts?.workflowId && run.workflow_id !== opts.workflowId)
                     continue;
                 runs.push(run);