@xioflow/kernel 0.1.5 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +50 -5
  2. package/dist/domain.d.ts +60 -6
  3. package/dist/domain.d.ts.map +1 -1
  4. package/dist/domain.js +405 -24
  5. package/dist/domain.js.map +1 -1
  6. package/dist/driver/node-driver.d.ts +10 -24
  7. package/dist/driver/node-driver.d.ts.map +1 -1
  8. package/dist/driver/node-driver.js +201 -280
  9. package/dist/driver/node-driver.js.map +1 -1
  10. package/dist/driver/process-facts.d.ts +25 -0
  11. package/dist/driver/process-facts.d.ts.map +1 -0
  12. package/dist/driver/process-facts.js +164 -0
  13. package/dist/driver/process-facts.js.map +1 -0
  14. package/dist/driver/sampler.d.ts +51 -0
  15. package/dist/driver/sampler.d.ts.map +1 -0
  16. package/dist/driver/sampler.js +224 -0
  17. package/dist/driver/sampler.js.map +1 -0
  18. package/dist/driver/types.d.ts +21 -1
  19. package/dist/driver/types.d.ts.map +1 -1
  20. package/dist/index.d.ts +1 -0
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +1 -0
  23. package/dist/index.js.map +1 -1
  24. package/dist/quick-run.d.ts +27 -0
  25. package/dist/quick-run.d.ts.map +1 -0
  26. package/dist/quick-run.js +82 -0
  27. package/dist/quick-run.js.map +1 -0
  28. package/dist/recovery/engine.d.ts.map +1 -1
  29. package/dist/recovery/engine.js +103 -19
  30. package/dist/recovery/engine.js.map +1 -1
  31. package/dist/store/sqlite.d.ts +15 -0
  32. package/dist/store/sqlite.d.ts.map +1 -1
  33. package/dist/store/sqlite.js +210 -12
  34. package/dist/store/sqlite.js.map +1 -1
  35. package/dist/supervisor/supervisor.d.ts +16 -0
  36. package/dist/supervisor/supervisor.d.ts.map +1 -1
  37. package/dist/supervisor/supervisor.js +820 -308
  38. package/dist/supervisor/supervisor.js.map +1 -1
  39. package/dist/testing/contract-suite.d.ts.map +1 -1
  40. package/dist/testing/contract-suite.js +479 -11
  41. package/dist/testing/contract-suite.js.map +1 -1
  42. package/dist/types.d.ts +54 -0
  43. package/dist/types.d.ts.map +1 -1
  44. package/dist/types.js +59 -0
  45. package/dist/types.js.map +1 -1
  46. package/package.json +2 -2
@@ -1,7 +1,68 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import crypto from 'node:crypto';
4
- import { UnsupportedCapabilityError, } from '../types.js';
4
+ import { UnsupportedCapabilityError, OperationNotActiveError, OperationIdConflictError, RecoveryRequiredError, } from '../types.js';
5
+ /**
6
+ * 规范化计算完整输入的 SHA-256 指纹 (ARCHITECTURE §2 / P0-12)
7
+ * 覆盖 execPath, args, cwd, 键排序的 envWhiteList, inheritEnv, sha256(stdin), 排序后的 requiredResources, timeoutMs, resourceBudget.
8
+ */
9
+ export function computeInputFingerprint(options) {
10
+ const cmd = options.command;
11
+ let stdinHash = null;
12
+ if (cmd.stdin !== undefined && cmd.stdin !== null) {
13
+ stdinHash = crypto.createHash('sha256').update(cmd.stdin).digest('hex');
14
+ }
15
+ let envSorted = null;
16
+ if (cmd.envWhiteList) {
17
+ const keys = Object.keys(cmd.envWhiteList).sort();
18
+ envSorted = keys.map((k) => [k, cmd.envWhiteList[k]]);
19
+ }
20
+ const canonical = {
21
+ args: cmd.args || [],
22
+ cwd: cmd.cwd || '',
23
+ envWhiteList: envSorted,
24
+ execPath: cmd.execPath || '',
25
+ inheritEnv: cmd.inheritEnv ?? null,
26
+ requiredResources: [...(options.requiredResources || [])].sort(),
27
+ resourceBudget: options.resourceBudget
28
+ ? {
29
+ enforcement: options.resourceBudget.enforcement ?? null,
30
+ maxCpuTimeMs: options.resourceBudget.maxCpuTimeMs ?? null,
31
+ maxMemoryBytes: options.resourceBudget.maxMemoryBytes ?? null,
32
+ maxOutputBytes: options.resourceBudget.maxOutputBytes ?? null,
33
+ maxPids: options.resourceBudget.maxPids ?? null,
34
+ }
35
+ : null,
36
+ stdinHash,
37
+ timeoutMs: options.timeoutMs ?? null,
38
+ };
39
+ const canonicalJson = JSON.stringify(canonical);
40
+ return crypto.createHash('sha256').update(canonicalJson).digest('hex');
41
+ }
42
+ function trimToValidUtf8(buf) {
43
+ const len = buf.length;
44
+ for (let i = 1; i <= Math.min(3, len); i++) {
45
+ const b = buf[len - i];
46
+ if ((b & 0xc0) === 0xc0) {
47
+ const needed = (b & 0xe0) === 0xc0 ? 2 : (b & 0xf0) === 0xe0 ? 3 : (b & 0xf8) === 0xf0 ? 4 : 1;
48
+ if (i < needed) {
49
+ return Buffer.from(buf.subarray(0, len - i));
50
+ }
51
+ break;
52
+ }
53
+ else if ((b & 0x80) === 0) {
54
+ break;
55
+ }
56
+ }
57
+ return buf;
58
+ }
59
+ function trimStartToValidUtf8(buf) {
60
+ let start = 0;
61
+ while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
62
+ start++;
63
+ }
64
+ return Buffer.from(buf.subarray(start));
65
+ }
5
66
  export class ProcessSupervisor {
6
67
  domain;
7
68
  driver;
@@ -9,6 +70,7 @@ export class ProcessSupervisor {
9
70
  constructor(domain, driver) {
10
71
  this.domain = domain;
11
72
  this.driver = driver;
73
+ this.domain.setDriver?.(driver);
12
74
  }
13
75
  getDomain() {
14
76
  return this.domain;
@@ -23,6 +85,8 @@ export class ProcessSupervisor {
23
85
  const startTime = Date.now();
24
86
  const maxBytes = options.maxOutputBytes ?? 10 * 1024 * 1024; // 默认 10MB
25
87
  const drainTimeoutMs = options.drainTimeoutMs ?? 2000; // 默认 2000ms
88
+ let stdoutDrainer;
89
+ let stderrDrainer;
26
90
  // 0. [准入期强校验 Pre-admission Capability Check]
27
91
  if (options.resourceBudget?.enforcement === 'hard') {
28
92
  if (options.resourceBudget.maxMemoryBytes && !this.driver.capabilities.memoryHardLimit) {
@@ -35,328 +99,590 @@ export class ProcessSupervisor {
35
99
  throw new UnsupportedCapabilityError('cpuLimit', this.driver.name, 'hard');
36
100
  }
37
101
  }
102
+ const inputFingerprint = options.inputFingerprint || computeInputFingerprint(options);
103
+ // 0. 验证调用 Run 状态(N3 契约):终态 Run 禁止登记或重放操作
104
+ const callingRun = this.domain.getStore().getRun(options.runId);
105
+ if (!callingRun) {
106
+ throw new Error(`Run "${options.runId}" is not registered in domain "${this.domain.domainId}". ` +
107
+ 'Register the task and run first (store.saveTask() + store.saveRun()), then execute operations for that run.');
108
+ }
109
+ if (callingRun.status === 'succeeded' ||
110
+ callingRun.status === 'failed' ||
111
+ callingRun.status === 'cancelled' ||
112
+ callingRun.status === 'indeterminate') {
113
+ throw new Error(`Cannot register operation "${options.opId}" for Run "${options.runId}" because the Run is already finalized with status "${callingRun.status}".`);
114
+ }
115
+ // 1. 在飞操作重放判定(ARCHITECTURE §3.7 / 契约 #46)
116
+ if (this.activeOperations.has(options.opId)) {
117
+ const inFlight = this.activeOperations.get(options.opId);
118
+ if (inFlight.inputFingerprint !== inputFingerprint) {
119
+ throw new OperationIdConflictError(options.opId, inFlight.inputFingerprint, inputFingerprint, inFlight.phase, inFlight.runId);
120
+ }
121
+ this.domain.getStore().recordReplay(this.domain.domainId, options.opId, 'joined', options.runId);
122
+ if (options.onStreamChunk) {
123
+ inFlight.streamSubscribers.add(options.onStreamChunk);
124
+ }
125
+ if (options.abortSignal) {
126
+ const signal = options.abortSignal;
127
+ if (signal.aborted) {
128
+ if (options.onStreamChunk) {
129
+ inFlight.streamSubscribers.delete(options.onStreamChunk);
130
+ }
131
+ throw signal.reason || new Error('Operation wait aborted');
132
+ }
133
+ return await new Promise((resolve, reject) => {
134
+ const onAbort = () => {
135
+ if (options.onStreamChunk) {
136
+ inFlight.streamSubscribers.delete(options.onStreamChunk);
137
+ }
138
+ reject(signal.reason || new Error('Operation wait aborted'));
139
+ };
140
+ signal.addEventListener('abort', onAbort, { once: true });
141
+ inFlight.resultPromise.then((res) => {
142
+ signal.removeEventListener('abort', onAbort);
143
+ if (options.onStreamChunk) {
144
+ inFlight.streamSubscribers.delete(options.onStreamChunk);
145
+ }
146
+ resolve({ ...res, replayed: true });
147
+ }, (err) => {
148
+ signal.removeEventListener('abort', onAbort);
149
+ if (options.onStreamChunk) {
150
+ inFlight.streamSubscribers.delete(options.onStreamChunk);
151
+ }
152
+ reject(err);
153
+ });
154
+ });
155
+ }
156
+ const res = await inFlight.resultPromise;
157
+ if (options.onStreamChunk) {
158
+ inFlight.streamSubscribers.delete(options.onStreamChunk);
159
+ }
160
+ return {
161
+ ...res,
162
+ replayed: true,
163
+ };
164
+ }
165
+ // 2. 已持久化操作重放判定(ARCHITECTURE §3.7 / 契约 #45, #47, #48, #49)
166
+ const existingRecorded = this.domain.getStore().getOperation(options.opId);
167
+ if (existingRecorded) {
168
+ if (existingRecorded.inputFingerprint !== inputFingerprint) {
169
+ throw new OperationIdConflictError(options.opId, existingRecorded.inputFingerprint, inputFingerprint, existingRecorded.status, existingRecorded.runId);
170
+ }
171
+ // 未终结状态且不在内存中 → 必须先执行恢复(崩溃残留现场)
172
+ if (existingRecorded.status !== 'done') {
173
+ throw new RecoveryRequiredError(options.opId, existingRecorded.status, existingRecorded.runId);
174
+ }
175
+ // indeterminate 状态:绝不重跑,原样返回,保留租约
176
+ if (existingRecorded.result?.status === 'indeterminate') {
177
+ this.domain.getStore().recordReplay(this.domain.domainId, options.opId, 'indeterminate', options.runId);
178
+ return {
179
+ ...existingRecorded.result,
180
+ replayed: true,
181
+ runId: existingRecorded.runId,
182
+ };
183
+ }
184
+ // 已结清终态 (succeeded / failed / cancelled):返回已持久化结果
185
+ this.domain.getStore().recordReplay(this.domain.domainId, options.opId, 'recorded', options.runId);
186
+ return {
187
+ ...existingRecorded.result,
188
+ replayed: true,
189
+ runId: existingRecorded.runId,
190
+ };
191
+ }
38
192
  const op = {
39
193
  id: options.opId,
40
194
  runId: options.runId,
41
195
  kind: 'process',
42
196
  name: options.name,
43
- inputFingerprint: options.inputFingerprint || `${options.command.execPath}:${options.command.args.join(' ')}`,
197
+ inputFingerprint,
44
198
  requiredResources: options.requiredResources,
45
199
  timeoutMs: options.timeoutMs,
46
200
  resourceBudget: options.resourceBudget,
47
201
  status: 'pending',
48
202
  };
49
- // 1. [资源分配与排队等待]
50
- await this.domain.allocateResourcesWithWait(options.opId, options.requiredResources, options.waitTimeoutMs ?? 0, options.resourceBudget);
51
- // 2. [启动协议步骤 1] 写入 SQLite (status: intent_registered)
52
- try {
53
- this.domain.getStore().registerOperationIntent(op, this.domain.domainId);
54
- }
55
- catch (err) {
56
- this.domain.releaseResources(options.opId, options.requiredResources);
57
- throw err;
203
+ const streamSubscribers = new Set();
204
+ if (options.onStreamChunk) {
205
+ streamSubscribers.add(options.onStreamChunk);
58
206
  }
59
- let handle;
60
- try {
61
- // 3. [启动协议步骤 2] 请求平台驱动启动
62
- handle = await this.driver.spawn(options.command);
63
- }
64
- catch (spawnError) {
65
- // 启动失败(如可执行文件不存在),不产生假运行状态,直接失败收尾并释放资源
66
- const failResult = {
67
- kind: 'process',
68
- status: 'failed',
69
- exitCode: 127,
70
- signal: null,
71
- stdout: '',
72
- stderr: spawnError?.message || String(spawnError),
73
- spawnFailure: spawnError?.message || String(spawnError),
74
- isTruncated: false,
75
- identityVerification: 'not_original_process',
76
- durationMs: Date.now() - startTime,
77
- completedAt: new Date().toISOString(),
78
- };
79
- this.domain.getStore().recordOperationResult(options.opId, failResult, true);
80
- this.domain.releaseResources(options.opId, options.requiredResources);
81
- return failResult;
82
- }
83
- // 4. [启动协议步骤 3] 登记执行身份,状态推进为 active
84
- this.domain.getStore().updateOperationStatus(options.opId, 'active', handle.identity);
85
207
  const opState = {
86
- handle,
87
- command: options.command,
208
+ opId: options.opId,
209
+ runId: options.runId,
210
+ phase: 'waiting_resources',
211
+ startTime,
212
+ inputFingerprint,
213
+ streamSubscribers,
88
214
  cancelRequested: false,
89
215
  timedOut: false,
90
- terminationReason: undefined,
91
- stopPromise: undefined,
92
216
  };
93
217
  this.activeOperations.set(options.opId, opState);
94
- // 准备 artifacts 溢出转储目录
95
- const artifactsDir = options.artifactsDir || path.join(this.domain.domainPath, 'artifacts');
96
- if (!fs.existsSync(artifactsDir)) {
97
- try {
98
- fs.mkdirSync(artifactsDir, { recursive: true });
218
+ if (options.abortSignal) {
219
+ if (options.abortSignal.aborted) {
220
+ opState.cancelRequested = true;
99
221
  }
100
- catch { }
101
- }
102
- const stdoutSpillPath = path.join(artifactsDir, `${options.opId}-stdout.log`);
103
- const stderrSpillPath = path.join(artifactsDir, `${options.opId}-stderr.log`);
104
- // 5. [正常监督、有界排空与流式转储 (Spill to Artifacts)]
105
- let totalOutputBytes = 0;
106
- const onChunk = (bytes) => {
107
- totalOutputBytes += bytes;
108
- if (options.resourceBudget?.enforcement === 'soft' &&
109
- options.resourceBudget.maxOutputBytes &&
110
- totalOutputBytes > options.resourceBudget.maxOutputBytes &&
111
- !opState.terminationReason) {
112
- opState.terminationReason = 'output_exceeded';
113
- this.handleStopPipeline(options.opId, 'output_exceeded', 1000).catch(() => { });
222
+ else {
223
+ const onAbort = () => {
224
+ this.cancelOperation(options.opId).catch(() => { });
225
+ };
226
+ options.abortSignal.addEventListener('abort', onAbort, { once: true });
114
227
  }
115
- };
116
- let streamCallbackError;
117
- const forwardChunk = (stream) => (chunk) => {
118
- if (!options.onStreamChunk)
119
- return;
228
+ }
229
+ const executionPromise = (async () => {
230
+ let timeoutTimer = null;
231
+ let samplingInterval = null;
232
+ let stdoutDrainer;
233
+ let stderrDrainer;
120
234
  try {
121
- options.onStreamChunk(stream, chunk);
122
- }
123
- catch (err) {
124
- streamCallbackError ??= err instanceof Error ? err.message : String(err);
125
- }
126
- };
127
- const stdoutDrainer = this.setupStreamDrainer(handle.stdout, maxBytes, stdoutSpillPath, onChunk, forwardChunk('stdout'));
128
- const stderrDrainer = this.setupStreamDrainer(handle.stderr, maxBytes, stderrSpillPath, onChunk, forwardChunk('stderr'));
129
- let timeoutTimer = null;
130
- const timeoutPromise = options.timeoutMs && options.timeoutMs > 0
131
- ? new Promise((resolve) => {
132
- timeoutTimer = setTimeout(() => resolve('timeout'), options.timeoutMs);
133
- })
134
- : new Promise(() => { });
135
- // Soft / Observe 模式资源采样治理
136
- let peakMemoryBytes = 0;
137
- let peakCpuTimeMs = 0;
138
- let samplingInterval = null;
139
- if (this.driver.sampleMetrics &&
140
- options.resourceBudget &&
141
- (options.resourceBudget.enforcement === 'soft' || options.resourceBudget.enforcement === 'observe')) {
142
- samplingInterval = setInterval(async () => {
235
+ // 1. [资源分配与排队等待]
236
+ await this.domain.allocateResourcesWithWait(options.opId, options.requiredResources || [], options.waitTimeoutMs ?? 0, options.resourceBudget, () => opState.cancelRequested === true);
237
+ if (opState.cancelRequested) {
238
+ return this.handlePreSpawnCancellation(options, opState, startTime);
239
+ }
240
+ // 2. [启动协议步骤 1] 写入 SQLite (status: intent_registered)
241
+ try {
242
+ this.domain.getStore().registerOperationIntent(op, this.domain.domainId);
243
+ opState.phase = 'intent_registered';
244
+ }
245
+ catch (err) {
246
+ this.domain.internalReleaseResources(options.opId, options.requiredResources);
247
+ throw err;
248
+ }
249
+ if (opState.cancelRequested) {
250
+ return this.handlePreSpawnCancellation(options, opState, startTime);
251
+ }
252
+ let handle;
253
+ opState.phase = 'spawning';
143
254
  try {
144
- const metrics = await this.driver.sampleMetrics(handle.identity);
145
- peakMemoryBytes = Math.max(peakMemoryBytes, metrics.rssBytes);
146
- peakCpuTimeMs = Math.max(peakCpuTimeMs, metrics.cpuTimeMs);
147
- if (options.resourceBudget?.enforcement === 'soft') {
148
- if (options.resourceBudget.maxMemoryBytes && metrics.rssBytes > options.resourceBudget.maxMemoryBytes) {
149
- opState.terminationReason = 'memory_exceeded';
150
- if (samplingInterval)
151
- clearInterval(samplingInterval);
152
- await this.handleStopPipeline(options.opId, 'memory_exceeded', 1000);
255
+ // 3. [启动协议步骤 2] 请求平台驱动启动
256
+ handle = await this.driver.spawn(options.command);
257
+ opState.handle = handle;
258
+ opState.command = options.command;
259
+ }
260
+ catch (spawnError) {
261
+ opState.phase = 'done';
262
+ opState.stopResolve?.({ stopped: 'confirmed_stopped', scope: 'direct_child', errorDetails: spawnError?.message });
263
+ opState.stopResolve = undefined;
264
+ // 启动失败(如可执行文件不存在),不产生假运行状态,直接失败收尾并释放资源
265
+ const failResult = {
266
+ kind: 'process',
267
+ status: 'failed',
268
+ exitCode: 127,
269
+ signal: null,
270
+ stdout: '',
271
+ stderr: spawnError?.message || String(spawnError),
272
+ spawnFailure: spawnError?.message || String(spawnError),
273
+ isTruncated: false,
274
+ identityVerification: 'not_original_process',
275
+ durationMs: Date.now() - startTime,
276
+ completedAt: new Date().toISOString(),
277
+ };
278
+ return this.finalizeOperation(options.opId, failResult, options.requiredResources, true);
279
+ }
280
+ // 4. [启动协议步骤 3] 登记执行身份,状态推进为 active
281
+ try {
282
+ this.domain.getStore().updateOperationStatus(options.opId, 'active', handle.identity);
283
+ opState.phase = 'active';
284
+ handle.releaseGate?.();
285
+ }
286
+ catch (statusError) {
287
+ handle.destroyGate?.();
288
+ // N4: spawn 成功但登记 active 失败,防止产生孤儿进程
289
+ opState.phase = 'stopping';
290
+ const stopRes = await this.driver.terminate(handle.identity, 2000);
291
+ // 有界等待退出与关闭流,防止管道悬挂
292
+ await Promise.race([
293
+ (handle.onRootExit ?? handle.onExit).catch(() => { }),
294
+ new Promise((resolve) => setTimeout(resolve, 500)),
295
+ ]);
296
+ try {
297
+ handle.stdout.destroy?.();
298
+ handle.stderr.destroy?.();
299
+ }
300
+ catch { }
301
+ opState.phase = 'done';
302
+ if (opState.stopResolve) {
303
+ const resolveFn = opState.stopResolve;
304
+ opState.stopResolve = undefined;
305
+ resolveFn(stopRes);
306
+ }
307
+ if (stopRes.stopped === 'confirmed_stopped') {
308
+ const failResult = {
309
+ kind: 'process',
310
+ status: 'failed',
311
+ exitCode: null,
312
+ signal: null,
313
+ stdout: '',
314
+ stderr: `Failed to register active status: ${statusError?.message || String(statusError)}`,
315
+ spawnFailure: statusError?.message || String(statusError),
316
+ isTruncated: false,
317
+ identityVerification: 'not_original_process',
318
+ durationMs: Date.now() - startTime,
319
+ completedAt: new Date().toISOString(),
320
+ };
321
+ this.finalizeOperation(options.opId, failResult, options.requiredResources, true);
322
+ }
323
+ else {
324
+ const indetResult = {
325
+ kind: 'indeterminate',
326
+ status: 'indeterminate',
327
+ reason: `Failed to register active status: ${statusError?.message || String(statusError)}, and process ${handle.identity.pid} could not be confirmed stopped: ${stopRes.errorDetails || 'residual processes still alive'}`,
328
+ recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
329
+ durationMs: Date.now() - startTime,
330
+ completedAt: new Date().toISOString(),
331
+ };
332
+ this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
333
+ }
334
+ throw statusError;
335
+ }
336
+ // 准备 artifacts 溢出转储目录
337
+ const artifactsDir = options.artifactsDir || path.join(this.domain.domainPath, 'artifacts');
338
+ let artifactsDirError = undefined;
339
+ if (!fs.existsSync(artifactsDir)) {
340
+ try {
341
+ fs.mkdirSync(artifactsDir, { recursive: true });
342
+ }
343
+ catch (dirErr) {
344
+ artifactsDirError = dirErr.message || String(dirErr);
345
+ }
346
+ }
347
+ const stdoutSpillPath = path.join(artifactsDir, `${options.opId}-stdout.log`);
348
+ const stderrSpillPath = path.join(artifactsDir, `${options.opId}-stderr.log`);
349
+ // 5. [正常监督、有界排空与流式转储 (Spill to Artifacts)]
350
+ let totalOutputBytes = 0;
351
+ const onChunk = (bytes) => {
352
+ totalOutputBytes += bytes;
353
+ if (options.resourceBudget?.enforcement === 'soft' &&
354
+ options.resourceBudget.maxOutputBytes &&
355
+ totalOutputBytes > options.resourceBudget.maxOutputBytes &&
356
+ !opState.terminationReason) {
357
+ opState.terminationReason = 'output_exceeded';
358
+ this.handleStopPipeline(options.opId, 'output_exceeded', 1000).catch(() => { });
359
+ }
360
+ };
361
+ let streamCallbackError;
362
+ const forwardChunk = (stream) => (chunk) => {
363
+ for (const subscriber of opState.streamSubscribers) {
364
+ try {
365
+ subscriber(stream, chunk);
153
366
  }
154
- else if (options.resourceBudget.maxCpuTimeMs && metrics.cpuTimeMs > options.resourceBudget.maxCpuTimeMs) {
155
- opState.terminationReason = 'cpu_exceeded';
156
- if (samplingInterval)
157
- clearInterval(samplingInterval);
158
- await this.handleStopPipeline(options.opId, 'cpu_exceeded', 1000);
367
+ catch (err) {
368
+ streamCallbackError ??= err instanceof Error ? err.message : String(err);
159
369
  }
160
- else if (options.resourceBudget.maxPids && metrics.pidsCount > options.resourceBudget.maxPids) {
161
- opState.terminationReason = 'pids_exceeded';
162
- if (samplingInterval)
163
- clearInterval(samplingInterval);
164
- await this.handleStopPipeline(options.opId, 'pids_exceeded', 1000);
370
+ }
371
+ };
372
+ stdoutDrainer = this.setupStreamDrainer(handle.stdout, maxBytes, stdoutSpillPath, onChunk, forwardChunk('stdout'), artifactsDirError);
373
+ stderrDrainer = this.setupStreamDrainer(handle.stderr, maxBytes, stderrSpillPath, onChunk, forwardChunk('stderr'), artifactsDirError);
374
+ // 检查在 spawn 途中是否已被请求取消:若已请求取消,立即触发停止流水线向底层发送终止信号
375
+ if (opState.cancelRequested) {
376
+ this.handleStopPipeline(options.opId, 'user_cancelled', opState.cancelGraceMs ?? 2000).catch(() => { });
377
+ }
378
+ const timeoutPromise = options.timeoutMs && options.timeoutMs > 0
379
+ ? new Promise((resolve) => {
380
+ timeoutTimer = setTimeout(() => resolve('timeout'), options.timeoutMs);
381
+ })
382
+ : new Promise(() => { });
383
+ // Soft / Observe 模式资源采样治理
384
+ let peakMemoryBytes = 0;
385
+ let peakCpuTimeMs = 0;
386
+ if (this.driver.sampleMetrics &&
387
+ options.resourceBudget &&
388
+ (options.resourceBudget.enforcement === 'soft' || options.resourceBudget.enforcement === 'observe')) {
389
+ samplingInterval = setInterval(async () => {
390
+ try {
391
+ const metrics = await this.driver.sampleMetrics(handle.identity);
392
+ peakMemoryBytes = Math.max(peakMemoryBytes, metrics.rssBytes);
393
+ peakCpuTimeMs = Math.max(peakCpuTimeMs, metrics.cpuTimeMs);
394
+ if (options.resourceBudget?.enforcement === 'soft') {
395
+ if (options.resourceBudget.maxMemoryBytes && metrics.rssBytes > options.resourceBudget.maxMemoryBytes) {
396
+ opState.terminationReason = 'memory_exceeded';
397
+ if (samplingInterval)
398
+ clearInterval(samplingInterval);
399
+ await this.handleStopPipeline(options.opId, 'memory_exceeded', 1000);
400
+ }
401
+ else if (options.resourceBudget.maxCpuTimeMs && metrics.cpuTimeMs > options.resourceBudget.maxCpuTimeMs) {
402
+ opState.terminationReason = 'cpu_exceeded';
403
+ if (samplingInterval)
404
+ clearInterval(samplingInterval);
405
+ await this.handleStopPipeline(options.opId, 'cpu_exceeded', 1000);
406
+ }
407
+ else if (options.resourceBudget.maxPids && metrics.pidsCount > options.resourceBudget.maxPids) {
408
+ opState.terminationReason = 'pids_exceeded';
409
+ if (samplingInterval)
410
+ clearInterval(samplingInterval);
411
+ await this.handleStopPipeline(options.opId, 'pids_exceeded', 1000);
412
+ }
413
+ }
414
+ }
415
+ catch { }
416
+ }, 100);
417
+ }
418
+ let notifyStop;
419
+ const stopTriggerPromise = new Promise((resolve) => {
420
+ notifyStop = resolve;
421
+ });
422
+ const previousResolve = opState.stopResolve;
423
+ opState.stopResolve = (res) => {
424
+ previousResolve?.(res);
425
+ notifyStop?.(res);
426
+ };
427
+ // 等待根进程退出、超时触发、或外部停止完成
428
+ const rootExitPromise = handle.onRootExit ?? handle.onExit;
429
+ const stopCandidate = opState.stopPromise ? opState.stopPromise : stopTriggerPromise;
430
+ const exitOrTimeout = await Promise.race([
431
+ rootExitPromise.then((res) => ({ type: 'exit', res, stopRes: null })),
432
+ timeoutPromise.then((type) => ({ type, res: null, stopRes: null })),
433
+ stopCandidate.then((stopRes) => ({ type: 'stop', res: null, stopRes })),
434
+ ]);
435
+ if (timeoutTimer)
436
+ clearTimeout(timeoutTimer);
437
+ if (samplingInterval)
438
+ clearInterval(samplingInterval);
439
+ let exitResult;
440
+ if (exitOrTimeout.type === 'stop') {
441
+ const stopRes = exitOrTimeout.stopRes;
442
+ if (stopRes.stopped !== 'confirmed_stopped') {
443
+ if (handle.rawProcess && typeof handle.rawProcess.kill === 'function') {
444
+ try {
445
+ handle.rawProcess.kill('SIGKILL');
446
+ }
447
+ catch { }
165
448
  }
449
+ try {
450
+ handle.stdout.destroy?.();
451
+ handle.stderr.destroy?.();
452
+ }
453
+ catch { }
454
+ const stored = this.domain.getStore().getOperation(options.opId);
455
+ if (stored && stored.status === 'done') {
456
+ return (stored.result ?? null);
457
+ }
458
+ const indetResult = {
459
+ kind: 'indeterminate',
460
+ status: 'indeterminate',
461
+ reason: `Process ${handle.identity.pid} was cancelled or stopped but could not be confirmed: ${stopRes.errorDetails || 'residual processes still alive'}`,
462
+ recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
463
+ durationMs: Date.now() - startTime,
464
+ completedAt: new Date().toISOString(),
465
+ };
466
+ return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
166
467
  }
468
+ const rootExitWaitMs = 1000;
469
+ const rootExitOrTimeout = await Promise.race([
470
+ rootExitPromise.then((res) => ({ exited: true, res })),
471
+ new Promise((resolve) => setTimeout(() => resolve({ exited: false }), rootExitWaitMs)),
472
+ ]);
473
+ exitResult = rootExitOrTimeout.exited ? rootExitOrTimeout.res : { exitCode: null, signal: 'SIGKILL' };
167
474
  }
168
- catch { }
169
- }, 100);
170
- }
171
- try {
172
- // 等待根进程退出或超时触发;有 onRootExit 时不等待持有管道的后代
173
- const rootExitPromise = handle.onRootExit ?? handle.onExit;
174
- const exitOrTimeout = await Promise.race([
175
- rootExitPromise.then((res) => ({ type: 'exit', res })),
176
- timeoutPromise.then((type) => ({ type, res: null })),
177
- ]);
178
- if (timeoutTimer)
179
- clearTimeout(timeoutTimer);
180
- if (samplingInterval)
181
- clearInterval(samplingInterval);
182
- let exitResult;
183
- if (exitOrTimeout.type === 'timeout') {
184
- opState.timedOut = true;
185
- opState.terminationReason = 'timed_out';
186
- await this.handleStopPipeline(options.opId, 'timed_out', 1500);
187
- exitResult = await handle.onExit;
188
- }
189
- else {
190
- exitResult = exitOrTimeout.res;
191
- }
192
- // 有界等待流排空
193
- const drainPromise = Promise.all([stdoutDrainer.finishPromise, stderrDrainer.finishPromise]);
194
- const drainTimeoutPromise = new Promise((resolve) => setTimeout(resolve, drainTimeoutMs));
195
- let drained = await Promise.race([
196
- drainPromise.then(() => true),
197
- drainTimeoutPromise.then(() => false),
198
- ]);
199
- // 排空超时说明根进程已退出但仍有后代持有管道:回收整个进程组,
200
- // 否则操作会一直挂到超时,且 root 的真实退出事实会被超时掩盖。
201
- let residualProcessesReaped = false;
202
- if (!drained && exitOrTimeout.type === 'exit') {
203
- const reapResult = await this.driver.terminate(handle.identity, 1500);
204
- if (!reapResult.stopped) {
205
- const indetResult = {
206
- kind: 'indeterminate',
207
- status: 'indeterminate',
208
- reason: `Process ${handle.identity.pid} exited but residual descendants could not be confirmed stopped: ${reapResult.errorDetails}`,
209
- recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
210
- durationMs: Date.now() - startTime,
211
- completedAt: new Date().toISOString(),
212
- };
213
- this.domain.getStore().recordOperationResult(options.opId, indetResult, false);
214
- this.domain.getStore().updateRunStatus(options.runId, 'indeterminate', 'crash_detected');
215
- return indetResult;
475
+ else if (exitOrTimeout.type === 'timeout') {
476
+ opState.timedOut = true;
477
+ opState.terminationReason = 'timed_out';
478
+ const stopRes = await this.handleStopPipeline(options.opId, 'timed_out', 1500);
479
+ // 超时路径:根进程已由停止流水线终止,等待根进程退出事实(避免被持有管道的逃逸后代挂死)
480
+ // 关键防护(问题 4):给等待根进程退出增加有界上限(1000ms),
481
+ // 若驱动停止未确认(stopped !== 'confirmed_stopped')或根进程超出上限仍未退出,转入 indeterminate
482
+ const rootExitWaitMs = 1000;
483
+ const rootExitOrTimeout = await Promise.race([
484
+ (handle.onRootExit ?? handle.onExit).then((res) => ({ exited: true, res })),
485
+ new Promise((resolve) => setTimeout(() => resolve({ exited: false }), rootExitWaitMs)),
486
+ ]);
487
+ if (stopRes.stopped !== 'confirmed_stopped' || !rootExitOrTimeout.exited) {
488
+ const stored = this.domain.getStore().getOperation(options.opId);
489
+ if (stored && stored.status === 'done' && stored.result?.status === 'indeterminate') {
490
+ return stored.result;
491
+ }
492
+ const indetResult = {
493
+ kind: 'indeterminate',
494
+ status: 'indeterminate',
495
+ reason: `Process ${handle.identity.pid} timed out and root process could not be confirmed exited within ${rootExitWaitMs}ms bound: ${stopRes.errorDetails || 'root process still alive'}`,
496
+ recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
497
+ durationMs: Date.now() - startTime,
498
+ completedAt: new Date().toISOString(),
499
+ };
500
+ return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
501
+ }
502
+ exitResult = rootExitOrTimeout.res;
503
+ }
504
+ else {
505
+ exitResult = exitOrTimeout.res;
216
506
  }
217
- residualProcessesReaped = true;
218
- // 被杀死的后代释放管道后,给排空最后一次机会
219
- drained = await Promise.race([
507
+ // 有界等待流排空
508
+ const drainPromise = Promise.all([stdoutDrainer.finishPromise, stderrDrainer.finishPromise]);
509
+ const drainTimeoutPromise = new Promise((resolve) => setTimeout(resolve, drainTimeoutMs));
510
+ let drained = await Promise.race([
220
511
  drainPromise.then(() => true),
221
- new Promise((resolve) => setTimeout(() => resolve(false), 250)),
512
+ drainTimeoutPromise.then(() => false),
222
513
  ]);
514
+ // 排空超时说明根进程已退出但仍有后代持有管道:回收整个进程组,
515
+ // 否则操作会一直挂到超时,且 root 的真实退出事实会被超时掩盖。
516
+ let residualProcessesReaped = false;
517
+ if (!drained && exitOrTimeout.type === 'exit') {
518
+ const reapResult = await this.driver.terminate(handle.identity, 1500);
519
+ if (reapResult.stopped !== 'confirmed_stopped') {
520
+ const indetResult = {
521
+ kind: 'indeterminate',
522
+ status: 'indeterminate',
523
+ reason: `Process ${handle.identity.pid} exited but residual descendants could not be confirmed stopped: ${reapResult.errorDetails}`,
524
+ recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
525
+ durationMs: Date.now() - startTime,
526
+ completedAt: new Date().toISOString(),
527
+ };
528
+ return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
529
+ }
530
+ residualProcessesReaped = true;
531
+ // 被杀死的后代释放管道后,给排空最后一次机会
532
+ drained = await Promise.race([
533
+ drainPromise.then(() => true),
534
+ new Promise((resolve) => setTimeout(() => resolve(false), 250)),
535
+ ]);
536
+ }
537
+ if (!drained) {
538
+ // 排空超时路径强制调用 fsyncSync + closeSync 结清刷盘
539
+ stdoutDrainer.forceFinalize();
540
+ stderrDrainer.forceFinalize();
541
+ }
542
+ const stdoutData = stdoutDrainer.getResult();
543
+ const stderrData = stderrDrainer.getResult();
544
+ const isTruncated = stdoutData.isTruncated || stderrData.isTruncated;
545
+ const spillError = stdoutData.spillError || stderrData.spillError || artifactsDirError;
546
+ const outputRef = (!spillError && (stdoutData.outputRef || stderrData.outputRef)) || undefined;
547
+ const outputHash = (!spillError && (stdoutData.outputHash || stderrData.outputHash)) || undefined;
548
+ // 校验进程身份
549
+ const idVerify = await this.driver.verifyIdentity(handle.identity);
550
+ const isResourceOrTimeoutStopped = opState.timedOut ||
551
+ opState.terminationReason === 'memory_exceeded' ||
552
+ opState.terminationReason === 'cpu_exceeded' ||
553
+ opState.terminationReason === 'pids_exceeded' ||
554
+ opState.terminationReason === 'output_exceeded';
555
+ const isSucceeded = exitResult.exitCode === 0 &&
556
+ !isResourceOrTimeoutStopped &&
557
+ !opState.cancelRequested;
558
+ const finalStatus = isResourceOrTimeoutStopped
559
+ ? 'failed'
560
+ : opState.cancelRequested
561
+ ? 'cancelled'
562
+ : isSucceeded
563
+ ? 'succeeded'
564
+ : 'failed';
565
+ const terminationReason = opState.terminationReason || (finalStatus === 'succeeded' ? 'completed' : undefined);
566
+ const result = {
567
+ kind: 'process',
568
+ status: finalStatus,
569
+ exitCode: exitResult.exitCode,
570
+ signal: exitResult.signal,
571
+ stdout: stdoutData.content,
572
+ stderr: stderrData.content,
573
+ isTruncated,
574
+ stdoutTruncated: stdoutData.isTruncated,
575
+ stderrTruncated: stderrData.isTruncated,
576
+ stdoutRef: !stdoutData.spillError ? stdoutData.outputRef : undefined,
577
+ stderrRef: !stderrData.spillError ? stderrData.outputRef : undefined,
578
+ stdoutBytes: stdoutData.bytesSeen,
579
+ stderrBytes: stderrData.bytesSeen,
580
+ stdoutHash: !stdoutData.spillError ? stdoutData.outputHash : undefined,
581
+ stderrHash: !stderrData.spillError ? stderrData.outputHash : undefined,
582
+ ...(residualProcessesReaped ? { residualProcessesReaped: true } : {}),
583
+ ...(streamCallbackError ? { streamCallbackError } : {}),
584
+ ...(spillError ? { spillError } : {}),
585
+ ...(stdoutData.spillError ? { stdoutSpillError: stdoutData.spillError } : {}),
586
+ ...(stderrData.spillError ? { stderrSpillError: stderrData.spillError } : {}),
587
+ outputRef,
588
+ outputHash,
589
+ terminationReason,
590
+ peakMemoryBytes,
591
+ cpuTimeMs: peakCpuTimeMs,
592
+ identityVerification: idVerify,
593
+ durationMs: Date.now() - startTime,
594
+ completedAt: new Date().toISOString(),
595
+ };
596
+ if (opState.stopPromise) {
597
+ const stopRes = await opState.stopPromise;
598
+ if (stopRes.stopped !== 'confirmed_stopped') {
599
+ const stored = this.domain.getStore().getOperation(options.opId);
600
+ if (stored && stored.status === 'done') {
601
+ return (stored.result ?? null);
602
+ }
603
+ const indetResult = {
604
+ kind: 'indeterminate',
605
+ status: 'indeterminate',
606
+ reason: `Process ${handle.identity.pid} was cancelled or stopped but could not be confirmed: ${stopRes.errorDetails || 'residual processes still alive'}`,
607
+ recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
608
+ durationMs: Date.now() - startTime,
609
+ completedAt: new Date().toISOString(),
610
+ };
611
+ return this.finalizeOperation(options.opId, indetResult, options.requiredResources, false);
612
+ }
613
+ }
614
+ // 6. [事务提交结果与释放资源 - Single Writer]
615
+ // 统一由 finalizeOperation 写入事实与处理资源
616
+ return this.finalizeOperation(options.opId, result, options.requiredResources, true, artifactsDir);
223
617
  }
224
- if (!drained) {
225
- // 排空超时路径强制调用 fsyncSync + closeSync 结清刷盘
226
- stdoutDrainer.forceFinalize();
227
- stderrDrainer.forceFinalize();
228
- }
229
- const stdoutData = stdoutDrainer.getResult();
230
- const stderrData = stderrDrainer.getResult();
231
- const isTruncated = stdoutData.isTruncated || stderrData.isTruncated;
232
- const outputRef = stdoutData.outputRef || stderrData.outputRef;
233
- const outputHash = stdoutData.outputHash || stderrData.outputHash;
234
- // 校验进程身份
235
- const idVerify = await this.driver.verifyIdentity(handle.identity);
236
- const isResourceOrTimeoutStopped = opState.timedOut ||
237
- opState.terminationReason === 'memory_exceeded' ||
238
- opState.terminationReason === 'cpu_exceeded' ||
239
- opState.terminationReason === 'pids_exceeded' ||
240
- opState.terminationReason === 'output_exceeded';
241
- const isSucceeded = exitResult.exitCode === 0 &&
242
- !isResourceOrTimeoutStopped &&
243
- !opState.cancelRequested;
244
- const finalStatus = isResourceOrTimeoutStopped
245
- ? 'failed'
246
- : opState.cancelRequested
247
- ? 'cancelled'
248
- : isSucceeded
249
- ? 'succeeded'
250
- : 'failed';
251
- const terminationReason = opState.terminationReason || (finalStatus === 'succeeded' ? 'completed' : undefined);
252
- const result = {
253
- kind: 'process',
254
- status: finalStatus,
255
- exitCode: exitResult.exitCode,
256
- signal: exitResult.signal,
257
- stdout: stdoutData.content,
258
- stderr: stderrData.content,
259
- isTruncated,
260
- stdoutTruncated: stdoutData.isTruncated,
261
- stderrTruncated: stderrData.isTruncated,
262
- stdoutRef: stdoutData.outputRef,
263
- stderrRef: stderrData.outputRef,
264
- stdoutBytes: stdoutData.bytesSeen,
265
- stderrBytes: stderrData.bytesSeen,
266
- stdoutHash: stdoutData.outputHash,
267
- stderrHash: stderrData.outputHash,
268
- ...(residualProcessesReaped ? { residualProcessesReaped: true } : {}),
269
- ...(streamCallbackError ? { streamCallbackError } : {}),
270
- outputRef,
271
- outputHash,
272
- terminationReason,
273
- peakMemoryBytes,
274
- cpuTimeMs: peakCpuTimeMs,
275
- identityVerification: idVerify,
276
- durationMs: Date.now() - startTime,
277
- completedAt: new Date().toISOString(),
278
- };
279
- if (opState.stopPromise) {
280
- await opState.stopPromise;
281
- }
282
- const existingOp = this.domain.getStore().getOperation(options.opId);
283
- if (existingOp && existingOp.status === 'done') {
284
- const storedResult = existingOp.result;
285
- if (storedResult?.status === 'indeterminate') {
286
- // 驱动无法确认停止:保留隔离事实与资源锁,绝不覆盖
287
- return existingOp.result;
288
- }
289
- // 内部停止流水线(超时 / 资源超限)会先写一份粗糙终态;这里用本次采集到的
290
- // 真实退出码、输出与耗时覆盖它,避免把已捕获的输出证据丢掉。
291
- this.domain.getStore().recordOperationResult(options.opId, result, true);
292
- return result;
293
- }
294
- // 6. [事务提交结果与释放资源]
295
- this.domain.getStore().recordOperationResult(options.opId, result, true);
296
- this.domain.releaseResources(options.opId, options.requiredResources);
297
- if (terminationReason && terminationReason !== 'completed') {
298
- this.domain.getStore().updateRunStatus(options.runId, finalStatus === 'cancelled' ? 'cancelled' : 'failed', terminationReason);
618
+ finally {
619
+ stdoutDrainer?.forceFinalize();
620
+ stderrDrainer?.forceFinalize();
621
+ if (timeoutTimer)
622
+ clearTimeout(timeoutTimer);
623
+ if (samplingInterval)
624
+ clearInterval(samplingInterval);
625
+ if (opState.stopResolve) {
626
+ const resolveFn = opState.stopResolve;
627
+ opState.stopResolve = undefined;
628
+ resolveFn({
629
+ stopped: 'confirmed_stopped',
630
+ scope: 'direct_child',
631
+ errorDetails: opState.phase === 'done' ? undefined : 'Operation terminated before process activation',
632
+ });
633
+ }
634
+ this.activeOperations.delete(options.opId);
299
635
  }
300
- return result;
301
- }
302
- finally {
303
- if (timeoutTimer)
304
- clearTimeout(timeoutTimer);
305
- if (samplingInterval)
306
- clearInterval(samplingInterval);
307
- this.activeOperations.delete(options.opId);
308
- }
636
+ })();
637
+ opState.resultPromise = executionPromise;
638
+ return await executionPromise;
309
639
  }
310
640
  /**
311
641
  * 停止确认流水线 (Stopping Pipeline)
312
642
  */
313
643
  async cancelOperation(opId, graceMs = 2000) {
314
- return this.handleStopPipeline(opId, 'user_cancelled', graceMs);
644
+ const active = this.ensureActiveOperation(opId);
645
+ active.cancelRequested = true;
646
+ if (active.cancelGraceMs === undefined) {
647
+ active.cancelGraceMs = graceMs;
648
+ }
649
+ active.terminationReason = 'user_cancelled';
650
+ // 若操作仍在等待队列中排队,立刻唤醒
651
+ this.domain.cancelWait(opId);
652
+ if (active.phase === 'active' || active.phase === 'stopping') {
653
+ return this.handleStopPipeline(opId, 'user_cancelled', graceMs);
654
+ }
655
+ // 操作仍在等资源、intent_registered 或 spawn 途中,返回已有的或新建的 stopPromise
656
+ if (!active.stopPromise) {
657
+ active.stopPromise = new Promise((resolve) => {
658
+ active.stopResolve = resolve;
659
+ });
660
+ }
661
+ return active.stopPromise;
315
662
  }
316
663
  async handleStopPipeline(opId, reason, graceMs) {
317
- const active = this.activeOperations.get(opId);
318
- if (!active) {
319
- return { stopped: true, scope: 'direct_child' };
320
- }
321
- if (active.stopPromise) {
664
+ const active = this.ensureActiveOperation(opId);
665
+ if (active.phase === 'stopping' && active.stopPromise) {
322
666
  return active.stopPromise;
323
667
  }
324
- active.stopPromise = (async () => {
325
- if (this.domain.isClosed()) {
326
- return { stopped: true, scope: 'direct_child' };
327
- }
328
- if (reason === 'user_cancelled') {
329
- active.cancelRequested = true;
668
+ active.phase = 'stopping';
669
+ if (reason === 'user_cancelled') {
670
+ active.cancelRequested = true;
671
+ }
672
+ active.terminationReason = reason;
673
+ const existingResolve = active.stopResolve;
674
+ const runPipeline = (async () => {
675
+ if (!active.handle) {
676
+ const res = { stopped: 'confirmed_stopped', scope: 'direct_child' };
677
+ existingResolve?.(res);
678
+ return res;
330
679
  }
331
- active.terminationReason = reason;
332
680
  // 1. 进入 stopping 中间态(保持排他资源锁定,阻断新操作)
333
681
  this.domain.getStore().updateOperationStatus(opId, 'stopping');
334
682
  // 2. 调用驱动停止流水线 (SIGINT -> grace -> SIGTERM -> SIGKILL)
335
683
  const stopResult = await this.driver.terminate(active.handle.identity, graceMs);
336
- if (stopResult.stopped) {
337
- // 3a. 驱动确认停止 -> 记录终态并安全释放资源
338
- const op = this.domain.getStore().getOperation(opId);
339
- if (op && op.status !== 'done') {
340
- const cancelResult = {
341
- kind: 'process',
342
- status: reason === 'user_cancelled' ? 'cancelled' : 'failed',
343
- exitCode: null,
344
- signal: 'SIGKILL',
345
- stdout: '',
346
- stderr: `Operation stopped via stopping pipeline (${reason})`,
347
- isTruncated: false,
348
- terminationReason: reason,
349
- identityVerification: 'not_original_process',
350
- durationMs: 0,
351
- completedAt: new Date().toISOString(),
352
- };
353
- this.domain.getStore().recordOperationResult(opId, cancelResult, true);
354
- this.domain.releaseResources(opId, op.requiredResources);
355
- this.domain.getStore().updateRunStatus(op.runId, reason === 'user_cancelled' ? 'cancelled' : 'failed', reason);
356
- }
357
- }
358
- else {
359
- // 3b. 驱动无法确认完全停止 -> 转入 indeterminate,绝对保留隔离屏障与锁!
684
+ // 3. 驱动无法确认停止 -> 立即转入 indeterminate,绝对保留隔离屏障与锁!
685
+ if (stopResult.stopped !== 'confirmed_stopped') {
360
686
  const op = this.domain.getStore().getOperation(opId);
361
687
  if (op && op.status !== 'done') {
362
688
  const indetResult = {
@@ -364,32 +690,148 @@ export class ProcessSupervisor {
364
690
  status: 'indeterminate',
365
691
  reason: `Process ${active.handle.identity.pid} could not be confirmed stopped: ${stopResult.errorDetails}`,
366
692
  recoveryGuidance: 'Residual PID detected. Inspect system processes manually before releasing resources.',
367
- durationMs: 0,
693
+ durationMs: Date.now() - active.startTime,
368
694
  completedAt: new Date().toISOString(),
369
695
  };
370
- // 传递 releaseResources = false,绝对不释放锁
371
- this.domain.getStore().recordOperationResult(opId, indetResult, false);
372
- this.domain.getStore().updateRunStatus(op.runId, 'indeterminate', 'crash_detected');
696
+ this.finalizeOperation(opId, indetResult, undefined, false);
373
697
  }
374
698
  }
699
+ existingResolve?.(stopResult);
375
700
  return stopResult;
376
701
  })();
377
- return active.stopPromise;
702
+ active.stopPromise = runPipeline;
703
+ return runPipeline;
704
+ }
705
+ ensureActiveOperation(opId) {
706
+ if (this.domain.isClosed()) {
707
+ throw new OperationNotActiveError(opId, 'domain_closed');
708
+ }
709
+ const active = this.activeOperations.get(opId);
710
+ if (!active || active.phase === 'done') {
711
+ const stored = this.domain.getStore().getOperation(opId);
712
+ if (!stored) {
713
+ throw new OperationNotActiveError(opId, 'not_found');
714
+ }
715
+ if (stored.status === 'done') {
716
+ throw new OperationNotActiveError(opId, 'already_completed');
717
+ }
718
+ throw new OperationNotActiveError(opId, 'not_found', `Operation "${opId}" is not active (current status: ${stored.status}).`);
719
+ }
720
+ return active;
721
+ }
722
+ handlePreSpawnCancellation(options, opState, startTime) {
723
+ opState.phase = 'done';
724
+ const stopRes = { stopped: 'confirmed_stopped', scope: 'direct_child' };
725
+ opState.stopResolve?.(stopRes);
726
+ opState.stopResolve = undefined;
727
+ const cancelResult = {
728
+ kind: 'process',
729
+ status: 'cancelled',
730
+ exitCode: null,
731
+ signal: null,
732
+ stdout: '',
733
+ stderr: '',
734
+ isTruncated: false,
735
+ terminationReason: 'user_cancelled',
736
+ evidence: 'unobserved',
737
+ identityVerification: 'not_original_process',
738
+ durationMs: Date.now() - startTime,
739
+ completedAt: new Date().toISOString(),
740
+ };
741
+ const stored = this.domain.getStore().getOperation(options.opId);
742
+ if (!stored) {
743
+ // 此时操作是在 waiting_resources 阶段就被取消,尚未获取资源租约,亦未正常注册意图。
744
+ // 为保证 store 中有该操作记录以便写入 cancelled 结果,直接记录预意图取消操作,
745
+ // 绝不向 resource_leases 写入虚假租约,亦不产生虚假的 OPERATION_INTENT_REGISTERED 事件。
746
+ const op = {
747
+ id: options.opId,
748
+ runId: options.runId,
749
+ kind: 'process',
750
+ name: options.name,
751
+ inputFingerprint: options.inputFingerprint || computeInputFingerprint(options),
752
+ requiredResources: options.requiredResources || [],
753
+ status: 'done',
754
+ };
755
+ this.domain.getStore().recordPreIntentCancelledOperation(op, this.domain.domainId, cancelResult);
756
+ this.domain.internalReleaseResources(options.opId, options.requiredResources);
757
+ return cancelResult;
758
+ }
759
+ return this.finalizeOperation(options.opId, cancelResult, options.requiredResources, true);
378
760
  }
379
- setupStreamDrainer(stream, maxBytes, spillFilePath, onChunk, onData) {
380
- const chunks = [];
381
- let currentBytes = 0;
761
+ pruneArtifacts(filter) {
762
+ return this.domain.pruneArtifacts(filter);
763
+ }
764
+ finalizeOperation(opId, result, requiredResources, releaseResourceLock = true, artifactsDir) {
765
+ const existing = this.domain.getStore().getOperation(opId);
766
+ if (existing && existing.status === 'done') {
767
+ throw new Error(`Cannot finalize operation "${opId}": operation is already finalized with status "${existing.status}"`);
768
+ }
769
+ if (existing) {
770
+ result.runId = existing.runId;
771
+ }
772
+ if (result.kind === 'process') {
773
+ const procRes = result;
774
+ const targetDir = artifactsDir || path.join(this.domain.domainPath, 'artifacts');
775
+ const stdoutSpillPath = path.join(targetDir, `${opId}-stdout.log`);
776
+ const stderrSpillPath = path.join(targetDir, `${opId}-stderr.log`);
777
+ if (!procRes.stdoutRef && fs.existsSync(stdoutSpillPath)) {
778
+ try {
779
+ fs.unlinkSync(stdoutSpillPath);
780
+ }
781
+ catch (unlinkErr) {
782
+ procRes.spillError = procRes.spillError || unlinkErr.message || String(unlinkErr);
783
+ }
784
+ }
785
+ if (!procRes.stderrRef && fs.existsSync(stderrSpillPath)) {
786
+ try {
787
+ fs.unlinkSync(stderrSpillPath);
788
+ }
789
+ catch (unlinkErr) {
790
+ procRes.spillError = procRes.spillError || unlinkErr.message || String(unlinkErr);
791
+ }
792
+ }
793
+ }
794
+ if (result.status === 'indeterminate') {
795
+ this.domain.getStore().recordOperationResult(opId, result, false);
796
+ const op = this.domain.getStore().getOperation(opId);
797
+ if (op) {
798
+ const termReason = result.terminationReason;
799
+ this.domain.getStore().updateRunStatus(op.runId, 'indeterminate', termReason);
800
+ }
801
+ }
802
+ else {
803
+ this.domain.getStore().recordOperationResult(opId, result, true);
804
+ if (releaseResourceLock) {
805
+ this.domain.internalReleaseResources(opId, requiredResources);
806
+ }
807
+ }
808
+ return result;
809
+ }
810
+ setupStreamDrainer(stream, maxBytes, spillFilePath, onChunk, onData, spillInitError) {
811
+ const overhead = maxBytes >= 80 ? Math.min(64, Math.floor(maxBytes / 4)) : 0;
812
+ const effectiveMax = maxBytes - overhead;
813
+ const headMaxBytes = overhead > 0 ? Math.floor(effectiveMax * 0.75) : maxBytes;
814
+ const tailMaxBytes = overhead > 0 ? Math.max(0, effectiveMax - headMaxBytes) : 0;
815
+ const headChunks = [];
816
+ let headBytes = 0;
382
817
  let bytesSeen = 0;
383
818
  let isTruncated = false;
384
819
  let spillFd = null;
820
+ let spillError = spillInitError;
385
821
  const hash = crypto.createHash('sha256');
386
822
  let isFinalized = false;
387
823
  let outputHash = undefined;
388
- if (spillFilePath) {
824
+ const tailRing = tailMaxBytes > 0 ? Buffer.alloc(tailMaxBytes) : null;
825
+ let tailHead = 0;
826
+ let tailCount = 0;
827
+ if (spillFilePath && !spillError) {
389
828
  try {
390
829
  spillFd = fs.openSync(spillFilePath, 'w');
391
830
  }
392
- catch { }
831
+ catch (openErr) {
832
+ spillError = openErr.message || String(openErr);
833
+ spillFd = null;
834
+ }
393
835
  }
394
836
  const doFinalize = () => {
395
837
  if (isFinalized)
@@ -400,13 +842,20 @@ export class ProcessSupervisor {
400
842
  fs.fsyncSync(spillFd);
401
843
  fs.closeSync(spillFd);
402
844
  }
403
- catch { }
845
+ catch (syncErr) {
846
+ spillError = spillError || syncErr.message || String(syncErr);
847
+ }
404
848
  spillFd = null;
405
849
  }
406
- try {
407
- outputHash = hash.digest('hex');
850
+ if (!spillError) {
851
+ try {
852
+ outputHash = hash.digest('hex');
853
+ }
854
+ catch { }
855
+ }
856
+ else {
857
+ outputHash = undefined;
408
858
  }
409
- catch { }
410
859
  };
411
860
  const finishPromise = new Promise((resolve) => {
412
861
  const finalize = () => {
@@ -414,6 +863,10 @@ export class ProcessSupervisor {
414
863
  resolve();
415
864
  };
416
865
  stream.on('data', (chunk) => {
866
+ if (isFinalized) {
867
+ // finalize 之后,流继续读走数据避免堵塞或 SIGPIPE,但跳过 hash、落盘和转发
868
+ return;
869
+ }
417
870
  const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
418
871
  bytesSeen += buf.length;
419
872
  if (onChunk) {
@@ -422,32 +875,66 @@ export class ProcessSupervisor {
422
875
  }
423
876
  catch { }
424
877
  }
425
- hash.update(buf);
426
- // 持续落盘转储全部流
878
+ // 持续落盘转储全部流与写盘同链哈希 (P0-9)
427
879
  if (spillFd !== null) {
428
880
  try {
429
881
  fs.writeSync(spillFd, buf);
882
+ hash.update(buf);
883
+ }
884
+ catch (writeErr) {
885
+ spillError = spillError || writeErr.message || String(writeErr);
886
+ try {
887
+ fs.closeSync(spillFd);
888
+ }
889
+ catch { }
890
+ spillFd = null;
891
+ }
892
+ }
893
+ else if (!spillFilePath && !spillError) {
894
+ try {
895
+ hash.update(buf);
430
896
  }
431
897
  catch { }
432
898
  }
433
899
  if (onData) {
434
900
  onData(buf);
435
901
  }
436
- // 内存有界保留
437
- if (currentBytes + buf.length <= maxBytes) {
438
- chunks.push(buf);
439
- currentBytes += buf.length;
440
- }
441
- else {
442
- if (!isTruncated) {
443
- const remaining = maxBytes - currentBytes;
902
+ // Head 内存有界保留
903
+ if (headBytes < headMaxBytes) {
904
+ if (headBytes + buf.length <= headMaxBytes) {
905
+ headChunks.push(buf);
906
+ headBytes += buf.length;
907
+ }
908
+ else {
909
+ const remaining = headMaxBytes - headBytes;
444
910
  if (remaining > 0) {
445
- chunks.push(buf.subarray(0, remaining));
446
- currentBytes += remaining;
911
+ headChunks.push(buf.subarray(0, remaining));
912
+ headBytes += remaining;
447
913
  }
448
914
  isTruncated = true;
449
915
  }
450
916
  }
917
+ else {
918
+ isTruncated = true;
919
+ }
920
+ if (bytesSeen > maxBytes) {
921
+ isTruncated = true;
922
+ }
923
+ // Tail 环形缓冲保留
924
+ if (tailRing !== null && tailMaxBytes > 0) {
925
+ if (buf.length >= tailMaxBytes) {
926
+ buf.copy(tailRing, 0, buf.length - tailMaxBytes);
927
+ tailHead = 0;
928
+ tailCount = tailMaxBytes;
929
+ }
930
+ else {
931
+ for (let i = 0; i < buf.length; i++) {
932
+ tailRing[tailHead] = buf[i];
933
+ tailHead = (tailHead + 1) % tailMaxBytes;
934
+ }
935
+ tailCount = Math.min(tailMaxBytes, tailCount + buf.length);
936
+ }
937
+ }
451
938
  });
452
939
  stream.on('end', finalize);
453
940
  stream.on('close', finalize);
@@ -456,12 +943,37 @@ export class ProcessSupervisor {
456
943
  return {
457
944
  getResult: () => {
458
945
  doFinalize();
946
+ let content;
947
+ if (bytesSeen <= maxBytes) {
948
+ content = Buffer.concat(headChunks).toString('utf8');
949
+ }
950
+ else {
951
+ const rawHead = trimToValidUtf8(Buffer.concat(headChunks));
952
+ let cleanTail = Buffer.alloc(0);
953
+ if (tailRing !== null && tailCount > 0) {
954
+ const rawTail = Buffer.alloc(tailCount);
955
+ if (tailCount < tailMaxBytes) {
956
+ tailRing.copy(rawTail, 0, 0, tailCount);
957
+ }
958
+ else {
959
+ const part1 = tailRing.subarray(tailHead, tailMaxBytes);
960
+ const part2 = tailRing.subarray(0, tailHead);
961
+ part1.copy(rawTail, 0);
962
+ part2.copy(rawTail, part1.length);
963
+ }
964
+ cleanTail = trimStartToValidUtf8(rawTail);
965
+ }
966
+ const truncatedBytes = Math.max(0, bytesSeen - rawHead.length - cleanTail.length);
967
+ const marker = `\n[... truncated ${truncatedBytes} bytes ...]\n`;
968
+ content = rawHead.toString('utf8') + marker + cleanTail.toString('utf8');
969
+ }
459
970
  return {
460
- content: Buffer.concat(chunks).toString('utf8'),
971
+ content,
461
972
  isTruncated,
462
- outputRef: isTruncated && spillFilePath ? spillFilePath : undefined,
463
- outputHash,
973
+ outputRef: isTruncated && spillFilePath && !spillError ? spillFilePath : undefined,
974
+ outputHash: !spillError ? outputHash : undefined,
464
975
  bytesSeen,
976
+ spillError,
465
977
  };
466
978
  },
467
979
  finishPromise,