@xioflow/kernel 0.2.0 → 0.3.0

package/dist/supervisor/supervisor.js

@@ -1,7 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
-import { UnsupportedCapabilityError, OperationNotActiveError, DuplicateOperationError, } from '../types.js';
+import { UnsupportedCapabilityError, OperationNotActiveError, OperationIdConflictError, RecoveryRequiredError, } from '../types.js';
 /**
  * 规范化计算完整输入的 SHA-256 指纹 (ARCHITECTURE §2 / P0-12)
  * 覆盖 execPath, args, cwd, 键排序的 envWhiteList, inheritEnv, sha256(stdin), 排序后的 requiredResources, timeoutMs, resourceBudget.
@@ -99,439 +99,543 @@ export class ProcessSupervisor {
                 throw new UnsupportedCapabilityError('cpuLimit', this.driver.name, 'hard');
             }
         }
-        // N8 守卫（契约 #40）：显式拒绝已存在或在飞的 opId，在触碰任何内存状态或 DB 租约之前同步拦截
+        const inputFingerprint = options.inputFingerprint || computeInputFingerprint(options);
+        // 0. 验证调用 Run 状态（N3 契约）：终态 Run 禁止登记或重放操作
+        const callingRun = this.domain.getStore().getRun(options.runId);
+        if (!callingRun) {
+            throw new Error(`Run "${options.runId}" is not registered in domain "${this.domain.domainId}". ` +
+                'Register the task and run first (store.saveTask() + store.saveRun()), then execute operations for that run.');
+        }
+        if (callingRun.status === 'succeeded' ||
+            callingRun.status === 'failed' ||
+            callingRun.status === 'cancelled' ||
+            callingRun.status === 'indeterminate') {
+            throw new Error(`Cannot register operation "${options.opId}" for Run "${options.runId}" because the Run is already finalized with status "${callingRun.status}".`);
+        }
+        // 1. 在飞操作重放判定（ARCHITECTURE §3.7 / 契约 #46）
         if (this.activeOperations.has(options.opId)) {
-            const existing = this.activeOperations.get(options.opId);
-            throw new DuplicateOperationError(options.opId, existing.phase, existing.runId);
+            const inFlight = this.activeOperations.get(options.opId);
+            if (inFlight.inputFingerprint !== inputFingerprint) {
+                throw new OperationIdConflictError(options.opId, inFlight.inputFingerprint, inputFingerprint, inFlight.phase, inFlight.runId);
+            }
+            this.domain.getStore().recordReplay(this.domain.domainId, options.opId, 'joined', options.runId);
+            if (options.onStreamChunk) {
+                inFlight.streamSubscribers.add(options.onStreamChunk);
+            }
+            if (options.abortSignal) {
+                const signal = options.abortSignal;
+                if (signal.aborted) {
+                    if (options.onStreamChunk) {
+                        inFlight.streamSubscribers.delete(options.onStreamChunk);
+                    }
+                    throw signal.reason || new Error('Operation wait aborted');
+                }
+                return await new Promise((resolve, reject) => {
+                    const onAbort = () => {
+                        if (options.onStreamChunk) {
+                            inFlight.streamSubscribers.delete(options.onStreamChunk);
+                        }
+                        reject(signal.reason || new Error('Operation wait aborted'));
+                    };
+                    signal.addEventListener('abort', onAbort, { once: true });
+                    inFlight.resultPromise.then((res) => {
+                        signal.removeEventListener('abort', onAbort);
+                        if (options.onStreamChunk) {
+                            inFlight.streamSubscribers.delete(options.onStreamChunk);
+                        }
+                        resolve({ ...res, replayed: true });
+                    }, (err) => {
+                        signal.removeEventListener('abort', onAbort);
+                        if (options.onStreamChunk) {
+                            inFlight.streamSubscribers.delete(options.onStreamChunk);
+                        }
+                        reject(err);
+                    });
+                });
+            }
+            const res = await inFlight.resultPromise;
+            if (options.onStreamChunk) {
+                inFlight.streamSubscribers.delete(options.onStreamChunk);
+            }
+            return {
+                ...res,
+                replayed: true,
+            };
         }
+        // 2. 已持久化操作重放判定（ARCHITECTURE §3.7 / 契约 #45, #47, #48, #49）
         const existingRecorded = this.domain.getStore().getOperation(options.opId);
         if (existingRecorded) {
-            throw new DuplicateOperationError(options.opId, existingRecorded.status, existingRecorded.runId);
+            if (existingRecorded.inputFingerprint !== inputFingerprint) {
+                throw new OperationIdConflictError(options.opId, existingRecorded.inputFingerprint, inputFingerprint, existingRecorded.status, existingRecorded.runId);
+            }
+            // 未终结状态且不在内存中 → 必须先执行恢复（崩溃残留现场）
+            if (existingRecorded.status !== 'done') {
+                throw new RecoveryRequiredError(options.opId, existingRecorded.status, existingRecorded.runId);
+            }
+            // indeterminate 状态：绝不重跑，原样返回，保留租约
+            if (existingRecorded.result?.status === 'indeterminate') {
+                this.domain.getStore().recordReplay(this.domain.domainId, options.opId, 'indeterminate', options.runId);
+                return {
+                    ...existingRecorded.result,
+                    replayed: true,
+                    runId: existingRecorded.runId,
+                };
+            }
+            // 已结清终态 (succeeded / failed / cancelled)：返回已持久化结果
+            this.domain.getStore().recordReplay(this.domain.domainId, options.opId, 'recorded', options.runId);
+            return {
+                ...existingRecorded.result,
+                replayed: true,
+                runId: existingRecorded.runId,
+            };
         }
         const op = {
             id: options.opId,
             runId: options.runId,
             kind: 'process',
             name: options.name,
-            inputFingerprint: options.inputFingerprint || computeInputFingerprint(options),
+            inputFingerprint,
             requiredResources: options.requiredResources,
             timeoutMs: options.timeoutMs,
             resourceBudget: options.resourceBudget,
             status: 'pending',
         };
+        const streamSubscribers = new Set();
+        if (options.onStreamChunk) {
+            streamSubscribers.add(options.onStreamChunk);
+        }
         const opState = {
             opId: options.opId,
             runId: options.runId,
             phase: 'waiting_resources',
             startTime,
+            inputFingerprint,
+            streamSubscribers,
             cancelRequested: false,
             timedOut: false,
         };
         this.activeOperations.set(options.opId, opState);
-        let timeoutTimer = null;
-        let samplingInterval = null;
-        try {
-            // 1. [资源分配与排队等待]
-            await this.domain.allocateResourcesWithWait(options.opId, options.requiredResources || [], options.waitTimeoutMs ?? 0, options.resourceBudget, () => opState.cancelRequested === true);
-            if (opState.cancelRequested) {
-                return this.handlePreSpawnCancellation(options, opState, startTime);
-            }
-            // 2. [启动协议步骤 1] 写入 SQLite (status: intent_registered)
-            try {
-                this.domain.getStore().registerOperationIntent(op, this.domain.domainId);
-                opState.phase = 'intent_registered';
-            }
-            catch (err) {
-                this.domain.internalReleaseResources(options.opId, options.requiredResources);
-                throw err;
-            }
-            if (opState.cancelRequested) {
-                return this.handlePreSpawnCancellation(options, opState, startTime);
-            }
-            let handle;
-            opState.phase = 'spawning';
-            try {
-                // 3. [启动协议步骤 2] 请求平台驱动启动
-                handle = await this.driver.spawn(options.command);
-                opState.handle = handle;
-                opState.command = options.command;
+        if (options.abortSignal) {
+            if (options.abortSignal.aborted) {
+                opState.cancelRequested = true;
             }
-            catch (spawnError) {
-                opState.phase = 'done';
-                opState.stopResolve?.({ stopped: 'confirmed_stopped', scope: 'direct_child', errorDetails: spawnError?.message });
-                opState.stopResolve = undefined;
-                // 启动失败（如可执行文件不存在），不产生假运行状态，直接失败收尾并释放资源
-                const failResult = {
-                    kind: 'process',
-                    status: 'failed',
-                    exitCode: 127,
-                    signal: null,
-                    stdout: '',
-                    stderr: spawnError?.message || String(spawnError),
-                    spawnFailure: spawnError?.message || String(spawnError),
-                    isTruncated: false,
-                    identityVerification: 'not_original_process',
-                    durationMs: Date.now() - startTime,
-                    completedAt: new Date().toISOString(),
+            else {
+                const onAbort = () => {
+                    this.cancelOperation(options.opId).catch(() => { });
                 };
-                return this.finalizeOperation(options.opId, failResult, options.requiredResources, true);
+                options.abortSignal.addEventListener('abort', onAbort, { once: true });
             }
-            // 4. [启动协议步骤 3] 登记执行身份，状态推进为 active
+        }
+        const executionPromise = (async () => {
+            let timeoutTimer = null;
+            let samplingInterval = null;
+            let stdoutDrainer;
+            let stderrDrainer;
             try {
-                this.domain.getStore().updateOperationStatus(options.opId, 'active', handle.identity);
-                opState.phase = 'active';
-                handle.releaseGate?.();
-            }
-            catch (statusError) {
-                handle.destroyGate?.();
-                // N4: spawn 成功但登记 active 失败，防止产生孤儿进程
-                opState.phase = 'stopping';
-                const stopRes = await this.driver.terminate(handle.identity, 2000);
-                // 有界等待退出与关闭流，防止管道悬挂
-                await Promise.race([
-                    (handle.onRootExit ?? handle.onExit).catch(() => { }),
-                    new Promise((resolve) => setTimeout(resolve, 500)),
-                ]);
+                // 1. [资源分配与排队等待]
+                await this.domain.allocateResourcesWithWait(options.opId, options.requiredResources || [], options.waitTimeoutMs ?? 0, options.resourceBudget, () => opState.cancelRequested === true);
+                if (opState.cancelRequested) {
+                    return this.handlePreSpawnCancellation(options, opState, startTime);
+                }
+                // 2. [启动协议步骤 1] 写入 SQLite (status: intent_registered)
                 try {
-                    handle.stdout.destroy?.();
-                    handle.stderr.destroy?.();
+                    this.domain.getStore().registerOperationIntent(op, this.domain.domainId);
+                    opState.phase = 'intent_registered';
                 }
-                catch { }
-                opState.phase = 'done';
-                if (opState.stopResolve) {
-                    const resolveFn = opState.stopResolve;
-                    opState.stopResolve = undefined;
-                    resolveFn(stopRes);
+                catch (err) {
+                    this.domain.internalReleaseResources(options.opId, options.requiredResources);
+                    throw err;
                 }
-                if (stopRes.stopped === 'confirmed_stopped') {
+                if (opState.cancelRequested) {
+                    return this.handlePreSpawnCancellation(options, opState, startTime);
+                }
+                let handle;
+                opState.phase = 'spawning';
+                try {
+                    // 3. [启动协议步骤 2] 请求平台驱动启动
+                    handle = await this.driver.spawn(options.command);
+                    opState.handle = handle;
+                    opState.command = options.command;
+                }
+                catch (spawnError) {
+                    opState.phase = 'done';
+                    opState.stopResolve?.({ stopped: 'confirmed_stopped', scope: 'direct_child', errorDetails: spawnError?.message });
+                    opState.stopResolve = undefined;
+                    // 启动失败（如可执行文件不存在），不产生假运行状态，直接失败收尾并释放资源
                     const failResult = {
                         kind: 'process',
                         status: 'failed',
-                        exitCode: null,
+                        exitCode: 127,
                         signal: null,
                         stdout: '',
-                        stderr: `Failed to register active status: ${statusError?.message || String(statusError)}`,
-                        spawnFailure: statusError?.message || String(statusError),
+                        stderr: spawnError?.message || String(spawnError),
+                        spawnFailure: spawnError?.message || String(spawnError),
                         isTruncated: false,
                         identityVerification: 'not_original_process',
                         durationMs: Date.now() - startTime,
                         completedAt: new Date().toISOString(),
                     };
-                    this.finalizeOperation(options.opId, failResult, options.requiredResources, true);
-                }
-                else {
-                    const indetResult = {
-                        kind: 'indeterminate',
-                        status: 'indeterminate',
-                        reason: `Failed to register active status: ${statusError?.message || String(statusError)}, and process ${handle.identity.pid} could not be confirmed stopped: ${stopRes.errorDetails || 'residual processes still alive'}`,
-                        recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
-                        durationMs: Date.now() - startTime,
-                        completedAt: new Date().toISOString(),
-                    };
-                    this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
+                    return this.finalizeOperation(options.opId, failResult, options.requiredResources, true);
                 }
-                throw statusError;
-            }
-            // 准备 artifacts 溢出转储目录
-            const artifactsDir = options.artifactsDir || path.join(this.domain.domainPath, 'artifacts');
-            let artifactsDirError = undefined;
-            if (!fs.existsSync(artifactsDir)) {
-                try {
-                    fs.mkdirSync(artifactsDir, { recursive: true });
-                }
-                catch (dirErr) {
-                    artifactsDirError = dirErr.message || String(dirErr);
-                }
-            }
-            const stdoutSpillPath = path.join(artifactsDir, `${options.opId}-stdout.log`);
-            const stderrSpillPath = path.join(artifactsDir, `${options.opId}-stderr.log`);
-            // 5. [正常监督、有界排空与流式转储 (Spill to Artifacts)]
-            let totalOutputBytes = 0;
-            const onChunk = (bytes) => {
-                totalOutputBytes += bytes;
-                if (options.resourceBudget?.enforcement === 'soft' &&
-                    options.resourceBudget.maxOutputBytes &&
-                    totalOutputBytes > options.resourceBudget.maxOutputBytes &&
-                    !opState.terminationReason) {
-                    opState.terminationReason = 'output_exceeded';
-                    this.handleStopPipeline(options.opId, 'output_exceeded', 1000).catch(() => { });
-                }
-            };
-            let streamCallbackError;
-            const forwardChunk = (stream) => (chunk) => {
-                if (!options.onStreamChunk)
-                    return;
+                // 4. [启动协议步骤 3] 登记执行身份，状态推进为 active
                 try {
-                    options.onStreamChunk(stream, chunk);
-                }
-                catch (err) {
-                    streamCallbackError ??= err instanceof Error ? err.message : String(err);
-                }
-            };
-            stdoutDrainer = this.setupStreamDrainer(handle.stdout, maxBytes, stdoutSpillPath, onChunk, forwardChunk('stdout'), artifactsDirError);
-            stderrDrainer = this.setupStreamDrainer(handle.stderr, maxBytes, stderrSpillPath, onChunk, forwardChunk('stderr'), artifactsDirError);
-            // 检查在 spawn 途中是否已被请求取消：若已请求取消，立即触发停止流水线向底层发送终止信号
-            if (opState.cancelRequested) {
-                this.handleStopPipeline(options.opId, 'user_cancelled', opState.cancelGraceMs ?? 2000).catch(() => { });
-            }
-            const timeoutPromise = options.timeoutMs && options.timeoutMs > 0
-                ? new Promise((resolve) => {
-                    timeoutTimer = setTimeout(() => resolve('timeout'), options.timeoutMs);
-                })
-                : new Promise(() => { });
-            // Soft / Observe 模式资源采样治理
-            let peakMemoryBytes = 0;
-            let peakCpuTimeMs = 0;
-            if (this.driver.sampleMetrics &&
-                options.resourceBudget &&
-                (options.resourceBudget.enforcement === 'soft' || options.resourceBudget.enforcement === 'observe')) {
-                samplingInterval = setInterval(async () => {
+                    this.domain.getStore().updateOperationStatus(options.opId, 'active', handle.identity);
+                    opState.phase = 'active';
+                    handle.releaseGate?.();
+                }
+                catch (statusError) {
+                    handle.destroyGate?.();
+                    // N4: spawn 成功但登记 active 失败，防止产生孤儿进程
+                    opState.phase = 'stopping';
+                    const stopRes = await this.driver.terminate(handle.identity, 2000);
+                    // 有界等待退出与关闭流，防止管道悬挂
+                    await Promise.race([
+                        (handle.onRootExit ?? handle.onExit).catch(() => { }),
+                        new Promise((resolve) => setTimeout(resolve, 500)),
+                    ]);
                     try {
-                        const metrics = await this.driver.sampleMetrics(handle.identity);
-                        peakMemoryBytes = Math.max(peakMemoryBytes, metrics.rssBytes);
-                        peakCpuTimeMs = Math.max(peakCpuTimeMs, metrics.cpuTimeMs);
-                        if (options.resourceBudget?.enforcement === 'soft') {
-                            if (options.resourceBudget.maxMemoryBytes && metrics.rssBytes > options.resourceBudget.maxMemoryBytes) {
-                                opState.terminationReason = 'memory_exceeded';
-                                if (samplingInterval)
-                                    clearInterval(samplingInterval);
-                                await this.handleStopPipeline(options.opId, 'memory_exceeded', 1000);
-                            }
-                            else if (options.resourceBudget.maxCpuTimeMs && metrics.cpuTimeMs > options.resourceBudget.maxCpuTimeMs) {
-                                opState.terminationReason = 'cpu_exceeded';
-                                if (samplingInterval)
-                                    clearInterval(samplingInterval);
-                                await this.handleStopPipeline(options.opId, 'cpu_exceeded', 1000);
-                            }
-                            else if (options.resourceBudget.maxPids && metrics.pidsCount > options.resourceBudget.maxPids) {
-                                opState.terminationReason = 'pids_exceeded';
-                                if (samplingInterval)
-                                    clearInterval(samplingInterval);
-                                await this.handleStopPipeline(options.opId, 'pids_exceeded', 1000);
-                            }
-                        }
+                        handle.stdout.destroy?.();
+                        handle.stderr.destroy?.();
                     }
                     catch { }
-                }, 100);
-            }
-            let notifyStop;
-            const stopTriggerPromise = new Promise((resolve) => {
-                notifyStop = resolve;
-            });
-            const previousResolve = opState.stopResolve;
-            opState.stopResolve = (res) => {
-                previousResolve?.(res);
-                notifyStop?.(res);
-            };
-            // 等待根进程退出、超时触发、或外部停止完成
-            const rootExitPromise = handle.onRootExit ?? handle.onExit;
-            const stopCandidate = opState.stopPromise ? opState.stopPromise : stopTriggerPromise;
-            const exitOrTimeout = await Promise.race([
-                rootExitPromise.then((res) => ({ type: 'exit', res, stopRes: null })),
-                timeoutPromise.then((type) => ({ type, res: null, stopRes: null })),
-                stopCandidate.then((stopRes) => ({ type: 'stop', res: null, stopRes })),
-            ]);
-            if (timeoutTimer)
-                clearTimeout(timeoutTimer);
-            if (samplingInterval)
-                clearInterval(samplingInterval);
-            let exitResult;
-            if (exitOrTimeout.type === 'stop') {
-                const stopRes = exitOrTimeout.stopRes;
-                if (stopRes.stopped !== 'confirmed_stopped') {
-                    if (handle.rawProcess && typeof handle.rawProcess.kill === 'function') {
-                        try {
-                            handle.rawProcess.kill('SIGKILL');
-                        }
-                        catch { }
+                    opState.phase = 'done';
+                    if (opState.stopResolve) {
+                        const resolveFn = opState.stopResolve;
+                        opState.stopResolve = undefined;
+                        resolveFn(stopRes);
                     }
+                    if (stopRes.stopped === 'confirmed_stopped') {
+                        const failResult = {
+                            kind: 'process',
+                            status: 'failed',
+                            exitCode: null,
+                            signal: null,
+                            stdout: '',
+                            stderr: `Failed to register active status: ${statusError?.message || String(statusError)}`,
+                            spawnFailure: statusError?.message || String(statusError),
+                            isTruncated: false,
+                            identityVerification: 'not_original_process',
+                            durationMs: Date.now() - startTime,
+                            completedAt: new Date().toISOString(),
+                        };
+                        this.finalizeOperation(options.opId, failResult, options.requiredResources, true);
+                    }
+                    else {
+                        const indetResult = {
+                            kind: 'indeterminate',
+                            status: 'indeterminate',
+                            reason: `Failed to register active status: ${statusError?.message || String(statusError)}, and process ${handle.identity.pid} could not be confirmed stopped: ${stopRes.errorDetails || 'residual processes still alive'}`,
+                            recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
+                            durationMs: Date.now() - startTime,
+                            completedAt: new Date().toISOString(),
+                        };
+                        this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
+                    }
+                    throw statusError;
+                }
+                // 准备 artifacts 溢出转储目录
+                const artifactsDir = options.artifactsDir || path.join(this.domain.domainPath, 'artifacts');
+                let artifactsDirError = undefined;
+                if (!fs.existsSync(artifactsDir)) {
                     try {
-                        handle.stdout.destroy?.();
-                        handle.stderr.destroy?.();
+                        fs.mkdirSync(artifactsDir, { recursive: true });
                     }
-                    catch { }
-                    const stored = this.domain.getStore().getOperation(options.opId);
-                    if (stored && stored.status === 'done') {
-                        return (stored.result ?? null);
+                    catch (dirErr) {
+                        artifactsDirError = dirErr.message || String(dirErr);
                     }
-                    const indetResult = {
-                        kind: 'indeterminate',
-                        status: 'indeterminate',
-                        reason: `Process ${handle.identity.pid} was cancelled or stopped but could not be confirmed: ${stopRes.errorDetails || 'residual processes still alive'}`,
-                        recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
-                        durationMs: Date.now() - startTime,
-                        completedAt: new Date().toISOString(),
-                    };
-                    return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
                 }
-                const rootExitWaitMs = 1000;
-                const rootExitOrTimeout = await Promise.race([
-                    rootExitPromise.then((res) => ({ exited: true, res })),
-                    new Promise((resolve) => setTimeout(() => resolve({ exited: false }), rootExitWaitMs)),
-                ]);
-                exitResult = rootExitOrTimeout.exited ? rootExitOrTimeout.res : { exitCode: null, signal: 'SIGKILL' };
-            }
-            else if (exitOrTimeout.type === 'timeout') {
-                opState.timedOut = true;
-                opState.terminationReason = 'timed_out';
-                const stopRes = await this.handleStopPipeline(options.opId, 'timed_out', 1500);
-                // 超时路径：根进程已由停止流水线终止，等待根进程退出事实（避免被持有管道的逃逸后代挂死）
-                // 关键防护（问题 4）：给等待根进程退出增加有界上限（1000ms），
-                // 若驱动停止未确认（stopped !== 'confirmed_stopped'）或根进程超出上限仍未退出，转入 indeterminate
-                const rootExitWaitMs = 1000;
-                const rootExitOrTimeout = await Promise.race([
-                    (handle.onRootExit ?? handle.onExit).then((res) => ({ exited: true, res })),
-                    new Promise((resolve) => setTimeout(() => resolve({ exited: false }), rootExitWaitMs)),
+                const stdoutSpillPath = path.join(artifactsDir, `${options.opId}-stdout.log`);
+                const stderrSpillPath = path.join(artifactsDir, `${options.opId}-stderr.log`);
+                // 5. [正常监督、有界排空与流式转储 (Spill to Artifacts)]
+                let totalOutputBytes = 0;
+                const onChunk = (bytes) => {
+                    totalOutputBytes += bytes;
+                    if (options.resourceBudget?.enforcement === 'soft' &&
+                        options.resourceBudget.maxOutputBytes &&
+                        totalOutputBytes > options.resourceBudget.maxOutputBytes &&
+                        !opState.terminationReason) {
+                        opState.terminationReason = 'output_exceeded';
+                        this.handleStopPipeline(options.opId, 'output_exceeded', 1000).catch(() => { });
+                    }
+                };
+                let streamCallbackError;
+                const forwardChunk = (stream) => (chunk) => {
+                    for (const subscriber of opState.streamSubscribers) {
+                        try {
+                            subscriber(stream, chunk);
+                        }
+                        catch (err) {
+                            streamCallbackError ??= err instanceof Error ? err.message : String(err);
+                        }
+                    }
+                };
+                stdoutDrainer = this.setupStreamDrainer(handle.stdout, maxBytes, stdoutSpillPath, onChunk, forwardChunk('stdout'), artifactsDirError);
+                stderrDrainer = this.setupStreamDrainer(handle.stderr, maxBytes, stderrSpillPath, onChunk, forwardChunk('stderr'), artifactsDirError);
+                // 检查在 spawn 途中是否已被请求取消：若已请求取消，立即触发停止流水线向底层发送终止信号
+                if (opState.cancelRequested) {
+                    this.handleStopPipeline(options.opId, 'user_cancelled', opState.cancelGraceMs ?? 2000).catch(() => { });
+                }
+                const timeoutPromise = options.timeoutMs && options.timeoutMs > 0
+                    ? new Promise((resolve) => {
+                        timeoutTimer = setTimeout(() => resolve('timeout'), options.timeoutMs);
+                    })
+                    : new Promise(() => { });
+                // Soft / Observe 模式资源采样治理
+                let peakMemoryBytes = 0;
+                let peakCpuTimeMs = 0;
+                if (this.driver.sampleMetrics &&
+                    options.resourceBudget &&
+                    (options.resourceBudget.enforcement === 'soft' || options.resourceBudget.enforcement === 'observe')) {
+                    samplingInterval = setInterval(async () => {
+                        try {
+                            const metrics = await this.driver.sampleMetrics(handle.identity);
+                            peakMemoryBytes = Math.max(peakMemoryBytes, metrics.rssBytes);
+                            peakCpuTimeMs = Math.max(peakCpuTimeMs, metrics.cpuTimeMs);
+                            if (options.resourceBudget?.enforcement === 'soft') {
+                                if (options.resourceBudget.maxMemoryBytes && metrics.rssBytes > options.resourceBudget.maxMemoryBytes) {
+                                    opState.terminationReason = 'memory_exceeded';
+                                    if (samplingInterval)
+                                        clearInterval(samplingInterval);
+                                    await this.handleStopPipeline(options.opId, 'memory_exceeded', 1000);
+                                }
+                                else if (options.resourceBudget.maxCpuTimeMs && metrics.cpuTimeMs > options.resourceBudget.maxCpuTimeMs) {
+                                    opState.terminationReason = 'cpu_exceeded';
+                                    if (samplingInterval)
+                                        clearInterval(samplingInterval);
+                                    await this.handleStopPipeline(options.opId, 'cpu_exceeded', 1000);
+                                }
+                                else if (options.resourceBudget.maxPids && metrics.pidsCount > options.resourceBudget.maxPids) {
+                                    opState.terminationReason = 'pids_exceeded';
+                                    if (samplingInterval)
+                                        clearInterval(samplingInterval);
+                                    await this.handleStopPipeline(options.opId, 'pids_exceeded', 1000);
+                                }
+                            }
+                        }
+                        catch { }
+                    }, 100);
+                }
+                let notifyStop;
+                const stopTriggerPromise = new Promise((resolve) => {
+                    notifyStop = resolve;
+                });
+                const previousResolve = opState.stopResolve;
+                opState.stopResolve = (res) => {
+                    previousResolve?.(res);
+                    notifyStop?.(res);
+                };
+                // 等待根进程退出、超时触发、或外部停止完成
+                const rootExitPromise = handle.onRootExit ?? handle.onExit;
+                const stopCandidate = opState.stopPromise ? opState.stopPromise : stopTriggerPromise;
+                const exitOrTimeout = await Promise.race([
+                    rootExitPromise.then((res) => ({ type: 'exit', res, stopRes: null })),
+                    timeoutPromise.then((type) => ({ type, res: null, stopRes: null })),
+                    stopCandidate.then((stopRes) => ({ type: 'stop', res: null, stopRes })),
                 ]);
-                if (stopRes.stopped !== 'confirmed_stopped' || !rootExitOrTimeout.exited) {
-                    const stored = this.domain.getStore().getOperation(options.opId);
-                    if (stored && stored.status === 'done' && stored.result?.status === 'indeterminate') {
-                        return stored.result;
+                if (timeoutTimer)
+                    clearTimeout(timeoutTimer);
+                if (samplingInterval)
+                    clearInterval(samplingInterval);
+                let exitResult;
+                if (exitOrTimeout.type === 'stop') {
+                    const stopRes = exitOrTimeout.stopRes;
+                    if (stopRes.stopped !== 'confirmed_stopped') {
+                        if (handle.rawProcess && typeof handle.rawProcess.kill === 'function') {
+                            try {
+                                handle.rawProcess.kill('SIGKILL');
+                            }
+                            catch { }
+                        }
+                        try {
+                            handle.stdout.destroy?.();
+                            handle.stderr.destroy?.();
+                        }
+                        catch { }
+                        const stored = this.domain.getStore().getOperation(options.opId);
+                        if (stored && stored.status === 'done') {
+                            return (stored.result ?? null);
+                        }
+                        const indetResult = {
+                            kind: 'indeterminate',
+                            status: 'indeterminate',
+                            reason: `Process ${handle.identity.pid} was cancelled or stopped but could not be confirmed: ${stopRes.errorDetails || 'residual processes still alive'}`,
+                            recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
+                            durationMs: Date.now() - startTime,
+                            completedAt: new Date().toISOString(),
+                        };
+                        return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
                     }
-                    const indetResult = {
-                        kind: 'indeterminate',
-                        status: 'indeterminate',
-                        reason: `Process ${handle.identity.pid} timed out and root process could not be confirmed exited within ${rootExitWaitMs}ms bound: ${stopRes.errorDetails || 'root process still alive'}`,
-                        recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
-                        durationMs: Date.now() - startTime,
-                        completedAt: new Date().toISOString(),
-                    };
-                    return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
+                    const rootExitWaitMs = 1000;
+                    const rootExitOrTimeout = await Promise.race([
+                        rootExitPromise.then((res) => ({ exited: true, res })),
+                        new Promise((resolve) => setTimeout(() => resolve({ exited: false }), rootExitWaitMs)),
+                    ]);
+                    exitResult = rootExitOrTimeout.exited ? rootExitOrTimeout.res : { exitCode: null, signal: 'SIGKILL' };
+                }
+                else if (exitOrTimeout.type === 'timeout') {
+                    opState.timedOut = true;
+                    opState.terminationReason = 'timed_out';
+                    const stopRes = await this.handleStopPipeline(options.opId, 'timed_out', 1500);
+                    // 超时路径：根进程已由停止流水线终止，等待根进程退出事实（避免被持有管道的逃逸后代挂死）
+                    // 关键防护（问题 4）：给等待根进程退出增加有界上限（1000ms），
+                    // 若驱动停止未确认（stopped !== 'confirmed_stopped'）或根进程超出上限仍未退出，转入 indeterminate
+                    const rootExitWaitMs = 1000;
+                    const rootExitOrTimeout = await Promise.race([
+                        (handle.onRootExit ?? handle.onExit).then((res) => ({ exited: true, res })),
+                        new Promise((resolve) => setTimeout(() => resolve({ exited: false }), rootExitWaitMs)),
+                    ]);
+                    if (stopRes.stopped !== 'confirmed_stopped' || !rootExitOrTimeout.exited) {
+                        const stored = this.domain.getStore().getOperation(options.opId);
+                        if (stored && stored.status === 'done' && stored.result?.status === 'indeterminate') {
+                            return stored.result;
+                        }
+                        const indetResult = {
+                            kind: 'indeterminate',
+                            status: 'indeterminate',
+                            reason: `Process ${handle.identity.pid} timed out and root process could not be confirmed exited within ${rootExitWaitMs}ms bound: ${stopRes.errorDetails || 'root process still alive'}`,
+                            recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
+                            durationMs: Date.now() - startTime,
+                            completedAt: new Date().toISOString(),
+                        };
+                        return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
+                    }
+                    exitResult = rootExitOrTimeout.res;
                 }
-                exitResult = rootExitOrTimeout.res;
-            }
-            else {
-                exitResult = exitOrTimeout.res;
-            }
-            // 有界等待流排空
-            const drainPromise = Promise.all([stdoutDrainer.finishPromise, stderrDrainer.finishPromise]);
-            const drainTimeoutPromise = new Promise((resolve) => setTimeout(resolve, drainTimeoutMs));
-            let drained = await Promise.race([
-                drainPromise.then(() => true),
-                drainTimeoutPromise.then(() => false),
-            ]);
-            // 排空超时说明根进程已退出但仍有后代持有管道：回收整个进程组，
-            // 否则操作会一直挂到超时，且 root 的真实退出事实会被超时掩盖。
-            let residualProcessesReaped = false;
-            if (!drained && exitOrTimeout.type === 'exit') {
-                const reapResult = await this.driver.terminate(handle.identity, 1500);
-                if (reapResult.stopped !== 'confirmed_stopped') {
-                    const indetResult = {
-                        kind: 'indeterminate',
-                        status: 'indeterminate',
-                        reason: `Process ${handle.identity.pid} exited but residual descendants could not be confirmed stopped: ${reapResult.errorDetails}`,
-                        recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
-                        durationMs: Date.now() - startTime,
-                        completedAt: new Date().toISOString(),
-                    };
-                    return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
+                else {
+                    exitResult = exitOrTimeout.res;
                 }
-                residualProcessesReaped = true;
-                // 被杀死的后代释放管道后，给排空最后一次机会
-                drained = await Promise.race([
+                // 有界等待流排空
+                const drainPromise = Promise.all([stdoutDrainer.finishPromise, stderrDrainer.finishPromise]);
+                const drainTimeoutPromise = new Promise((resolve) => setTimeout(resolve, drainTimeoutMs));
+                let drained = await Promise.race([
                     drainPromise.then(() => true),
-                    new Promise((resolve) => setTimeout(() => resolve(false), 250)),
+                    drainTimeoutPromise.then(() => false),
                 ]);
-            }
-            if (!drained) {
-                // 排空超时路径强制调用 fsyncSync + closeSync 结清刷盘
-                stdoutDrainer.forceFinalize();
-                stderrDrainer.forceFinalize();
-            }
-            const stdoutData = stdoutDrainer.getResult();
-            const stderrData = stderrDrainer.getResult();
-            const isTruncated = stdoutData.isTruncated || stderrData.isTruncated;
-            const spillError = stdoutData.spillError || stderrData.spillError || artifactsDirError;
-            const outputRef = (!spillError && (stdoutData.outputRef || stderrData.outputRef)) || undefined;
-            const outputHash = (!spillError && (stdoutData.outputHash || stderrData.outputHash)) || undefined;
-            // 校验进程身份
-            const idVerify = await this.driver.verifyIdentity(handle.identity);
-            const isResourceOrTimeoutStopped = opState.timedOut ||
-                opState.terminationReason === 'memory_exceeded' ||
-                opState.terminationReason === 'cpu_exceeded' ||
-                opState.terminationReason === 'pids_exceeded' ||
-                opState.terminationReason === 'output_exceeded';
-            const isSucceeded = exitResult.exitCode === 0 &&
-                !isResourceOrTimeoutStopped &&
-                !opState.cancelRequested;
-            const finalStatus = isResourceOrTimeoutStopped
-                ? 'failed'
-                : opState.cancelRequested
-                    ? 'cancelled'
-                    : isSucceeded
-                        ? 'succeeded'
-                        : 'failed';
-            const terminationReason = opState.terminationReason || (finalStatus === 'succeeded' ? 'completed' : undefined);
-            const result = {
-                kind: 'process',
-                status: finalStatus,
-                exitCode: exitResult.exitCode,
-                signal: exitResult.signal,
-                stdout: stdoutData.content,
-                stderr: stderrData.content,
-                isTruncated,
-                stdoutTruncated: stdoutData.isTruncated,
-                stderrTruncated: stderrData.isTruncated,
-                stdoutRef: !stdoutData.spillError ? stdoutData.outputRef : undefined,
-                stderrRef: !stderrData.spillError ? stderrData.outputRef : undefined,
-                stdoutBytes: stdoutData.bytesSeen,
-                stderrBytes: stderrData.bytesSeen,
-                stdoutHash: !stdoutData.spillError ? stdoutData.outputHash : undefined,
-                stderrHash: !stderrData.spillError ? stderrData.outputHash : undefined,
-                ...(residualProcessesReaped ? { residualProcessesReaped: true } : {}),
-                ...(streamCallbackError ? { streamCallbackError } : {}),
-                ...(spillError ? { spillError } : {}),
-                ...(stdoutData.spillError ? { stdoutSpillError: stdoutData.spillError } : {}),
-                ...(stderrData.spillError ? { stderrSpillError: stderrData.spillError } : {}),
-                outputRef,
-                outputHash,
-                terminationReason,
-                peakMemoryBytes,
-                cpuTimeMs: peakCpuTimeMs,
-                identityVerification: idVerify,
-                durationMs: Date.now() - startTime,
-                completedAt: new Date().toISOString(),
-            };
-            if (opState.stopPromise) {
-                const stopRes = await opState.stopPromise;
-                if (stopRes.stopped !== 'confirmed_stopped') {
-                    const stored = this.domain.getStore().getOperation(options.opId);
-                    if (stored && stored.status === 'done') {
-                        return (stored.result ?? null);
+                // 排空超时说明根进程已退出但仍有后代持有管道：回收整个进程组，
+                // 否则操作会一直挂到超时，且 root 的真实退出事实会被超时掩盖。
+                let residualProcessesReaped = false;
+                if (!drained && exitOrTimeout.type === 'exit') {
+                    const reapResult = await this.driver.terminate(handle.identity, 1500);
+                    if (reapResult.stopped !== 'confirmed_stopped') {
+                        const indetResult = {
+                            kind: 'indeterminate',
+                            status: 'indeterminate',
+                            reason: `Process ${handle.identity.pid} exited but residual descendants could not be confirmed stopped: ${reapResult.errorDetails}`,
+                            recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
+                            durationMs: Date.now() - startTime,
+                            completedAt: new Date().toISOString(),
+                        };
+                        return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
+                    }
+                    residualProcessesReaped = true;
+                    // 被杀死的后代释放管道后，给排空最后一次机会
+                    drained = await Promise.race([
+                        drainPromise.then(() => true),
+                        new Promise((resolve) => setTimeout(() => resolve(false), 250)),
+                    ]);
+                }
+                if (!drained) {
+                    // 排空超时路径强制调用 fsyncSync + closeSync 结清刷盘
+                    stdoutDrainer.forceFinalize();
+                    stderrDrainer.forceFinalize();
+                }
+                const stdoutData = stdoutDrainer.getResult();
+                const stderrData = stderrDrainer.getResult();
+                const isTruncated = stdoutData.isTruncated || stderrData.isTruncated;
+                const spillError = stdoutData.spillError || stderrData.spillError || artifactsDirError;
+                const outputRef = (!spillError && (stdoutData.outputRef || stderrData.outputRef)) || undefined;
+                const outputHash = (!spillError && (stdoutData.outputHash || stderrData.outputHash)) || undefined;
+                // 校验进程身份
+                const idVerify = await this.driver.verifyIdentity(handle.identity);
+                const isResourceOrTimeoutStopped = opState.timedOut ||
+                    opState.terminationReason === 'memory_exceeded' ||
+                    opState.terminationReason === 'cpu_exceeded' ||
+                    opState.terminationReason === 'pids_exceeded' ||
+                    opState.terminationReason === 'output_exceeded';
+                const isSucceeded = exitResult.exitCode === 0 &&
+                    !isResourceOrTimeoutStopped &&
+                    !opState.cancelRequested;
+                const finalStatus = isResourceOrTimeoutStopped
+                    ? 'failed'
+                    : opState.cancelRequested
+                        ? 'cancelled'
+                        : isSucceeded
+                            ? 'succeeded'
+                            : 'failed';
+                const terminationReason = opState.terminationReason || (finalStatus === 'succeeded' ? 'completed' : undefined);
+                const result = {
+                    kind: 'process',
+                    status: finalStatus,
+                    exitCode: exitResult.exitCode,
+                    signal: exitResult.signal,
+                    stdout: stdoutData.content,
+                    stderr: stderrData.content,
+                    isTruncated,
+                    stdoutTruncated: stdoutData.isTruncated,
+                    stderrTruncated: stderrData.isTruncated,
+                    stdoutRef: !stdoutData.spillError ? stdoutData.outputRef : undefined,
+                    stderrRef: !stderrData.spillError ? stderrData.outputRef : undefined,
+                    stdoutBytes: stdoutData.bytesSeen,
+                    stderrBytes: stderrData.bytesSeen,
+                    stdoutHash: !stdoutData.spillError ? stdoutData.outputHash : undefined,
+                    stderrHash: !stderrData.spillError ? stderrData.outputHash : undefined,
+                    ...(residualProcessesReaped ? { residualProcessesReaped: true } : {}),
+                    ...(streamCallbackError ? { streamCallbackError } : {}),
+                    ...(spillError ? { spillError } : {}),
+                    ...(stdoutData.spillError ? { stdoutSpillError: stdoutData.spillError } : {}),
+                    ...(stderrData.spillError ? { stderrSpillError: stderrData.spillError } : {}),
+                    outputRef,
+                    outputHash,
+                    terminationReason,
+                    peakMemoryBytes,
+                    cpuTimeMs: peakCpuTimeMs,
+                    identityVerification: idVerify,
+                    durationMs: Date.now() - startTime,
+                    completedAt: new Date().toISOString(),
+                };
+                if (opState.stopPromise) {
+                    const stopRes = await opState.stopPromise;
+                    if (stopRes.stopped !== 'confirmed_stopped') {
+                        const stored = this.domain.getStore().getOperation(options.opId);
+                        if (stored && stored.status === 'done') {
+                            return (stored.result ?? null);
+                        }
+                        const indetResult = {
+                            kind: 'indeterminate',
+                            status: 'indeterminate',
+                            reason: `Process ${handle.identity.pid} was cancelled or stopped but could not be confirmed: ${stopRes.errorDetails || 'residual processes still alive'}`,
+                            recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
+                            durationMs: Date.now() - startTime,
+                            completedAt: new Date().toISOString(),
+                        };
+                        return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
                     }
-                    const indetResult = {
-                        kind: 'indeterminate',
-                        status: 'indeterminate',
-                        reason: `Process ${handle.identity.pid} was cancelled or stopped but could not be confirmed: ${stopRes.errorDetails || 'residual processes still alive'}`,
-                        recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
-                        durationMs: Date.now() - startTime,
-                        completedAt: new Date().toISOString(),
-                    };
-                    return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
                 }
+                // 6. [事务提交结果与释放资源 - Single Writer]
+                // 统一由 finalizeOperation 写入事实与处理资源
+                return this.finalizeOperation(options.opId, result, options.requiredResources, true, artifactsDir);
+            }
+            finally {
+                stdoutDrainer?.forceFinalize();
+                stderrDrainer?.forceFinalize();
+                if (timeoutTimer)
+                    clearTimeout(timeoutTimer);
+                if (samplingInterval)
+                    clearInterval(samplingInterval);
+                if (opState.stopResolve) {
+                    const resolveFn = opState.stopResolve;
+                    opState.stopResolve = undefined;
+                    resolveFn({
+                        stopped: 'confirmed_stopped',
+                        scope: 'direct_child',
+                        errorDetails: opState.phase === 'done' ? undefined : 'Operation terminated before process activation',
+                    });
+                }
+                this.activeOperations.delete(options.opId);
             }
-            // 6. [事务提交结果与释放资源 - Single Writer]
-            // 统一由 finalizeOperation 写入事实与处理资源
-            return this.finalizeOperation(options.opId, result, options.requiredResources, true, artifactsDir);
-        }
-        finally {
-            stdoutDrainer?.forceFinalize();
-            stderrDrainer?.forceFinalize();
-            if (timeoutTimer)
-                clearTimeout(timeoutTimer);
-            if (samplingInterval)
-                clearInterval(samplingInterval);
-            if (opState.stopResolve) {
-                const resolveFn = opState.stopResolve;
-                opState.stopResolve = undefined;
-                resolveFn({
-                    stopped: 'confirmed_stopped',
-                    scope: 'direct_child',
-                    errorDetails: opState.phase === 'done' ? undefined : 'Operation terminated before process activation',
-                });
-            }
-            this.activeOperations.delete(options.opId);
-        }
+        })();
+        opState.resultPromise = executionPromise;
+        return await executionPromise;
     }
     /**
      * 停止确认流水线 (Stopping Pipeline)
@@ -662,6 +766,9 @@ export class ProcessSupervisor {
         if (existing && existing.status === 'done') {
             throw new Error(`Cannot finalize operation "${opId}": operation is already finalized with status "${existing.status}"`);
         }
+        if (existing) {
+            result.runId = existing.runId;
+        }
         if (result.kind === 'process') {
             const procRes = result;
             const targetDir = artifactsDir || path.join(this.domain.domainPath, 'artifacts');