@crewx/workflow 0.3.22-rc.136 → 0.3.22-rc.138

package/dist/src/run-manager.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RunManager = exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = exports.WORKFLOW_RUN_INTERRUPTED_ERROR = exports.SKILL_TASK_DEFAULT_TIMEOUT = exports.FileLock = void 0;
+exports.RunManager = exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = exports.WORKFLOW_RUN_INTERRUPTED_ERROR = exports.WORKFLOW_CANCEL_GRACE_MS = exports.SKILL_TASK_DEFAULT_TIMEOUT = exports.FileLock = void 0;
 exports.parseSetArgs = parseSetArgs;
 exports.readStdin = readStdin;
 exports.interpolateTemplate = interpolateTemplate;
@@ -55,6 +55,8 @@ const retry_policy_1 = require("./retry-policy");
 var file_lock_2 = require("./file-lock");
 Object.defineProperty(exports, "FileLock", { enumerable: true, get: function () { return file_lock_2.FileLock; } });
 exports.SKILL_TASK_DEFAULT_TIMEOUT = 600000;
+exports.WORKFLOW_CANCEL_GRACE_MS = 2000;
+const WORKFLOW_CANCEL_POLL_INTERVAL_MS = 100;
 exports.WORKFLOW_RUN_INTERRUPTED_ERROR = 'Workflow run interrupted: runner process exited before completion';
 exports.WORKFLOW_RUN_LEGACY_RECOVERY_GRACE_MS = 10 * 60000;
 const RETRY_WAIT_SLICE_MS = 1000;
@@ -80,20 +82,23 @@ function resolveCrewxChildInvocation(env, cwd) {
         },
     };
 }
-function killProcessGroup(child) {
+function killProcessGroup(child, signal = 'SIGKILL') {
     try {
         if (process.platform === 'win32') {
             if (child.pid) {
-                cp.spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' });
+                const args = ['/pid', String(child.pid), '/T'];
+                if (signal === 'SIGKILL')
+                    args.push('/F');
+                cp.spawn('taskkill', args, { stdio: 'ignore' });
             }
         }
         else if (child.pid) {
             try {
-                process.kill(-child.pid, 'SIGKILL');
+                process.kill(-child.pid, signal);
             }
             catch {
                 try {
-                    child.kill('SIGKILL');
+                    child.kill(signal);
                 }
                 catch { }
             }
@@ -103,6 +108,26 @@ function killProcessGroup(child) {
 }
 async function runProcess(command, argv, opts) {
     return new Promise((resolve) => {
+        const isCancelled = () => {
+            try {
+                return opts.cancellation?.isCancelled() === true;
+            }
+            catch {
+                return false;
+            }
+        };
+        if (isCancelled()) {
+            resolve({
+                status: null,
+                stdout: '',
+                stderr: '',
+                timedOut: false,
+                cancelled: true,
+                signal: null,
+                errorOrigin: null,
+            });
+            return;
+        }
         let child;
         try {
             child = cp.spawn(command, argv, {
@@ -129,19 +154,65 @@ async function runProcess(command, argv, opts) {
         let stdout = '';
         let stderr = '';
         let settled = false;
+        let finishing = false;
         let spawned = false;
+        let cancellationRequested = false;
         let postSpawnError;
         let stdinError;
+        let timeoutTimer;
+        let cancellationEscalationTimer;
+        let cancellationPollTimer;
+        let activeController;
+        const outDecoder = new sdk_1.ConsoleDecoder();
+        const errDecoder = new sdk_1.ConsoleDecoder();
+        let stdoutDecodeChain = Promise.resolve();
+        let stderrDecodeChain = Promise.resolve();
+        const cleanup = () => {
+            if (timeoutTimer)
+                clearTimeout(timeoutTimer);
+            if (cancellationEscalationTimer)
+                clearTimeout(cancellationEscalationTimer);
+            if (cancellationPollTimer)
+                clearInterval(cancellationPollTimer);
+            if (opts.cancellation
+                && activeController
+                && activeProcessControllers.get(opts.cancellation.key) === activeController) {
+                activeProcessControllers.delete(opts.cancellation.key);
+            }
+        };
         const finish = (result) => {
             if (settled)
                 return;
             settled = true;
-            clearTimeout(timer);
-            resolve(result);
+            cleanup();
+            resolve(cancellationRequested ? { ...result, cancelled: true, timedOut: false } : result);
         };
-        const timer = setTimeout(() => {
+        const requestCancellation = () => {
+            if (settled || cancellationRequested)
+                return;
+            cancellationRequested = true;
+            killProcessGroup(child, 'SIGTERM');
+            cancellationEscalationTimer = setTimeout(() => {
+                if (!settled)
+                    killProcessGroup(child, 'SIGKILL');
+            }, exports.WORKFLOW_CANCEL_GRACE_MS);
+            if (typeof cancellationEscalationTimer.unref === 'function') {
+                cancellationEscalationTimer.unref();
+            }
+        };
+        if (opts.cancellation) {
+            activeController = { cancel: requestCancellation };
+            activeProcessControllers.set(opts.cancellation.key, activeController);
+            cancellationPollTimer = setInterval(() => {
+                if (isCancelled())
+                    requestCancellation();
+            }, WORKFLOW_CANCEL_POLL_INTERVAL_MS);
+            if (typeof cancellationPollTimer.unref === 'function')
+                cancellationPollTimer.unref();
+        }
+        timeoutTimer = setTimeout(() => {
             killProcessGroup(child);
-            finish({
+            finishAfterDecode(() => ({
                 status: null,
                 stdout,
                 stderr,
@@ -149,12 +220,20 @@ async function runProcess(command, argv, opts) {
                 signal: null,
                 errorOrigin: stdinError ? 'stdin' : postSpawnError ? 'post_spawn' : null,
                 ...(stdinError || postSpawnError ? { error: stdinError ?? postSpawnError } : {}),
-            });
+            }));
         }, opts.timeout);
-        if (typeof timer.unref === 'function')
-            timer.unref();
-        child.stdout?.on('data', (d) => { stdout += d.toString(); });
-        child.stderr?.on('data', (d) => { stderr += d.toString(); });
+        if (typeof timeoutTimer.unref === 'function')
+            timeoutTimer.unref();
+        child.stdout?.on('data', (d) => {
+            stdoutDecodeChain = stdoutDecodeChain.then(async () => {
+                stdout += await outDecoder.write(d);
+            });
+        });
+        child.stderr?.on('data', (d) => {
+            stderrDecodeChain = stderrDecodeChain.then(async () => {
+                stderr += await errDecoder.write(d);
+            });
+        });
         child.once('spawn', () => { spawned = true; });
         child.on('error', (err) => {
             if (!spawned) {
@@ -182,7 +261,19 @@ async function runProcess(command, argv, opts) {
                 stdinError = stdinError ?? err;
             }
         }
-        child.on('close', (code, signal) => finish({
+        const finishAfterDecode = (createResult) => {
+            if (settled || finishing)
+                return;
+            finishing = true;
+            void (async () => {
+                await Promise.all([stdoutDecodeChain, stderrDecodeChain]);
+                const [tailOut, tailErr] = await Promise.all([outDecoder.end(), errDecoder.end()]);
+                stdout += tailOut;
+                stderr += tailErr;
+                finish(createResult());
+            })().catch(() => finish(createResult()));
+        };
+        child.on('close', (code, signal) => finishAfterDecode(() => ({
             status: code,
             stdout,
             stderr,
@@ -190,7 +281,9 @@ async function runProcess(command, argv, opts) {
             signal,
             errorOrigin: stdinError ? 'stdin' : postSpawnError ? 'post_spawn' : null,
             ...(stdinError || postSpawnError ? { error: stdinError ?? postSpawnError } : {}),
-        }));
+        })));
+        if (isCancelled())
+            requestCancellation();
     });
 }
 const AUDIT_IDENTIFIER_RE = /^[A-Za-z0-9_.-]{1,128}$/;
@@ -663,6 +756,13 @@ function readRunEngineState(run) {
     return { visitCounts, iterations };
 }
 const inFlightAutoRuns = new Set();
+const activeProcessControllers = new Map();
+function processControllerKey(resolvedDir, execId) {
+    return `${resolvedDir}::${execId}`;
+}
+function requestActiveProcessCancellation(resolvedDir, execId) {
+    activeProcessControllers.get(processControllerKey(resolvedDir, execId))?.cancel();
+}
 function isProcessAlive(pid) {
     if (!Number.isInteger(pid) || pid <= 0)
         return false;
@@ -721,6 +821,7 @@ class RunManager {
     constructor(runsDir = '.crewx/workflow-runs', projectRoot, deps) {
         this.runsDir = runsDir;
         this.sleep = deps?.sleep ?? sleep;
+        this.onRunPersisted = deps?.onRunPersisted;
         const base = projectRoot || process.env.CREWX_WORKSPACE || process.cwd();
         this.resolvedDir = path.resolve(base, this.runsDir);
         this.projectRoot = path.resolve(base);
@@ -740,6 +841,20 @@ class RunManager {
             throw new Error(`Invalid execution ID: "${execId}"`);
         }
     }
+    isRunCancelled(execId) {
+        try {
+            return this.readRunRaw(execId)?.status === 'cancelled';
+        }
+        catch {
+            return false;
+        }
+    }
+    processCancellation(execId) {
+        return {
+            key: processControllerKey(this.resolvedDir, execId),
+            isCancelled: () => this.isRunCancelled(execId),
+        };
+    }
     readRunRaw(execId) {
         const filePath = path.join(this.resolvedDir, `${execId}.json`);
         if (!fs.existsSync(filePath))
@@ -752,6 +867,7 @@ class RunManager {
         }
     }
     readRun(execId) {
+        this.validateExecId(execId);
         const run = this.readRunRaw(execId);
         return run ? normalizeRunStatus(run) : null;
     }
@@ -772,6 +888,24 @@ class RunManager {
         const node = spec?.nodes?.[run.current_node];
         return Boolean(node && KNOWN_AUTO_NODE_TYPES.has(node.type));
     }
+    withRunLock(execId, callback) {
+        this.validateExecId(execId);
+        const filePath = path.join(this.resolvedDir, `${execId}.json`);
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`Execution not found: ${execId}`);
+        }
+        const lock = new file_lock_1.FileLock(filePath);
+        lock.acquire();
+        try {
+            const run = this.readRunRaw(execId);
+            if (!run)
+                throw new Error(`Execution not found: ${execId}`);
+            return callback(normalizeRunStatus(run));
+        }
+        finally {
+            lock.release();
+        }
+    }
     hasStaleRunner(run) {
         if (run.auto_run)
             return !isProcessAlive(run.auto_run.pid);
@@ -824,7 +958,53 @@ class RunManager {
         this.ensureRunsDir();
         run.updated_at = new Date().toISOString();
         const filePath = path.join(this.resolvedDir, `${run.id}.json`);
-        fs.writeFileSync(filePath, JSON.stringify(run, null, 2) + '\n', 'utf-8');
+        let existingMode;
+        try {
+            existingMode = fs.statSync(filePath).mode & 0o7777;
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+        }
+        const tempPath = path.join(this.resolvedDir, `.${run.id}.tmp-${process.pid}-${(0, sdk_1.generateId)('wfr')}`);
+        const serialized = JSON.stringify(run, null, 2) + '\n';
+        let tempFd;
+        let tempCreated = false;
+        try {
+            tempFd = fs.openSync(tempPath, 'wx');
+            tempCreated = true;
+            fs.writeFileSync(tempFd, serialized, 'utf-8');
+            if (existingMode !== undefined)
+                fs.fchmodSync(tempFd, existingMode);
+            fs.closeSync(tempFd);
+            tempFd = undefined;
+            fs.renameSync(tempPath, filePath);
+            tempCreated = false;
+        }
+        finally {
+            if (tempFd !== undefined) {
+                try {
+                    fs.closeSync(tempFd);
+                }
+                catch {
+                }
+            }
+            if (tempCreated) {
+                try {
+                    fs.unlinkSync(tempPath);
+                }
+                catch {
+                }
+            }
+        }
+        if (!this.onRunPersisted)
+            return;
+        try {
+            this.onRunPersisted(run);
+        }
+        catch (error) {
+            console.warn(`[RunManager] onRunPersisted failed for ${run.id}: ${getErrorMessage(error)}`);
+        }
     }
     atomicUpdate(execId, updater) {
         this.validateExecId(execId);
@@ -959,6 +1139,8 @@ class RunManager {
     }
     moveNode(execId, nodeId) {
         return this.atomicUpdate(execId, (run) => {
+            if (run.status === 'cancelled')
+                return;
             if (run.current_node && !run.completed_nodes.includes(run.current_node)) {
                 run.completed_nodes.push(run.current_node);
             }
@@ -970,6 +1152,9 @@ class RunManager {
         if (!snapshot) {
             throw new Error(`Execution not found: ${execId}`);
         }
+        if (snapshot.status === 'cancelled') {
+            throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `Workflow run "${execId}" is already cancelled`);
+        }
         const doc = this.loadWorkflowYaml(snapshot.workflow_file);
         if (!doc) {
             throw (0, failure_1.createWorkflowFailureError)('CONFIG_INVALID', `Failed to load workflow file: ${snapshot.workflow_file}`);
@@ -1108,8 +1293,12 @@ class RunManager {
                         timeout: nodeTimeout,
                         shell: invocation.shell ?? false,
                         windowsHide: invocation.windowsHide,
+                        cancellation: this.processCancellation(execId),
                     });
                     recordProcessResult(result);
+                    if (result.cancelled) {
+                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `Agent execution cancelled for node "${nodeId}"`);
+                    }
                     if (result.timedOut) {
                         throw new Error(`Agent execution timed out after ${nodeTimeout}ms`);
                     }
@@ -1159,8 +1348,12 @@ class RunManager {
                                         timeout: nodeTimeout,
                                         shell: retryInvocation.shell ?? false,
                                         windowsHide: retryInvocation.windowsHide,
+                                        cancellation: this.processCancellation(execId),
                                     });
                                     recordProcessResult(retryResult);
+                                    if (retryResult.cancelled) {
+                                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `Agent retry cancelled for node "${nodeId}"`);
+                                    }
                                     if (retryResult.timedOut)
                                         throw new Error(`Agent retry timed out after ${nodeTimeout}ms`);
                                     if (retryResult.error)
@@ -1248,8 +1441,12 @@ class RunManager {
                         shell: invocation.shell ?? false,
                         windowsHide: invocation.windowsHide,
                         stdin: skillStdin,
+                        cancellation: this.processCancellation(execId),
                     });
                     recordProcessResult(result);
+                    if (result.cancelled) {
+                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `skill_task "${nodeId}" was cancelled`);
+                    }
                     if (result.timedOut)
                         throw new Error(`skill_task "${nodeId}" timed out after ${timeout}ms`);
                     if (result.error)
@@ -1309,8 +1506,12 @@ class RunManager {
                         timeout,
                         shell: false,
                         stdin: shellStdin,
+                        cancellation: this.processCancellation(execId),
                     });
                     recordProcessResult(result);
+                    if (result.cancelled) {
+                        throw (0, failure_1.createWorkflowFailureError)('CANCELLED', `shell_task "${nodeId}" was cancelled`);
+                    }
                     if (result.timedOut)
                         throw new Error(`shell_task "${nodeId}" timed out after ${timeout}ms`);
                     if (result.error)
@@ -1489,6 +1690,8 @@ class RunManager {
             if (!isTerminalStatus(run.status)) {
                 run.status = 'running';
             }
+            if (!run.current_node)
+                run.current_node = nodeId;
             run.auto_run = {
                 pid: process.pid,
                 token,
@@ -1594,6 +1797,9 @@ class RunManager {
         while (true) {
             try {
                 const run = await this.executeNode(execId, nodeId, { trigger: 'auto' });
+                if (run.status === 'cancelled') {
+                    return { run, runnerToken: token, outcome: 'cancelled', reason: run.error };
+                }
                 return { run, runnerToken: token, outcome: 'success' };
             }
             catch (caught) {
@@ -1766,6 +1972,8 @@ class RunManager {
                 }
                 runnerToken = claimedToken;
                 run = await this.atomicUpdateAsync(execId, (r) => {
+                    if (r.status === 'cancelled')
+                        return;
                     const engine = readRunEngineState(r);
                     engine.visitCounts[nodeId] = (engine.visitCounts[nodeId] ?? 0) + 1;
                     engine.iterations += 1;
@@ -1783,6 +1991,9 @@ class RunManager {
                         appendFailureAudit(r, failure, 'config', 'auto');
                     }
                 });
+                if (run.status === 'cancelled') {
+                    return { run, outcome: 'cancelled', reason: run.error, executed };
+                }
                 if (run.status === 'failed') {
                     return { run, outcome: 'failed', reason: run.error, executed };
                 }
@@ -1820,14 +2031,15 @@ class RunManager {
         }
     }
     cancelRun(execId, reason = 'Cancelled by user') {
-        return this.atomicUpdate(execId, (run) => {
+        const cancelled = this.atomicUpdate(execId, (run) => {
             if (run.status === 'completed' || run.status === 'failed' || run.status === 'cancelled') {
                 return;
             }
             const now = new Date();
+            const activeNodeId = run.auto_run?.node_id || run.current_node || '';
             const failure = (0, failure_1.buildNodeFailure)({
-                failedNode: run.current_node || '',
-                attempt: (0, failure_1.readNodeAttempt)(run, run.current_node || ''),
+                failedNode: activeNodeId,
+                attempt: (0, failure_1.readNodeAttempt)(run, activeNodeId),
                 causeCode: 'CANCELLED',
                 exitCode: null,
                 signal: null,
@@ -1839,6 +2051,10 @@ class RunManager {
             delete run.manual_resume;
             appendFailureAudit(run, failure, 'cancel', 'manual', now.toISOString());
         });
+        if (cancelled.status === 'cancelled') {
+            requestActiveProcessCancellation(this.resolvedDir, execId);
+        }
+        return cancelled;
     }
     reset(execId) {
         return this.atomicUpdate(execId, (run) => {