@actiondock/core 2.2.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/app/app.d.ts +2 -0
- package/dist/app/app.js +6 -0
- package/dist/app/types.d.ts +12 -0
- package/dist/errors.d.ts +38 -0
- package/dist/errors.js +44 -0
- package/dist/execution/service.d.ts +2 -0
- package/dist/execution/service.js +13 -2
- package/dist/execution/types.d.ts +17 -0
- package/dist/host/host.js +46 -5
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/ipc/host.d.ts +5 -0
- package/dist/ipc/host.js +43 -1
- package/dist/ipc/target.d.ts +18 -0
- package/dist/ipc/target.js +74 -6
- package/dist/ipc/types.d.ts +10 -1
- package/dist/platform/default.d.ts +7 -1
- package/dist/platform/default.js +40 -2
- package/dist/process/context-process.d.ts +83 -0
- package/dist/process/context-process.js +168 -0
- package/dist/process/cursor.d.ts +52 -0
- package/dist/process/cursor.js +144 -0
- package/dist/process/driver.d.ts +214 -0
- package/dist/process/driver.js +197 -0
- package/dist/process/index.d.ts +6 -0
- package/dist/process/index.js +6 -0
- package/dist/process/metadata-store.d.ts +205 -0
- package/dist/process/metadata-store.js +462 -0
- package/dist/process/output-log.d.ts +136 -0
- package/dist/process/output-log.js +331 -0
- package/dist/process/process-manager.d.ts +279 -0
- package/dist/process/process-manager.js +1879 -0
- package/dist/profile/client.js +63 -21
- package/dist/project/init.js +2 -2
- package/dist/registry/registry.js +12 -3
- package/dist/runtime/context.d.ts +9 -6
- package/dist/runtime/context.js +31 -8
- package/dist/runtime/process.d.ts +19 -2
- package/dist/runtime/process.js +103 -131
- package/dist/runtime/runner.d.ts +45 -24
- package/dist/runtime/runner.js +106 -25
- package/dist/target/remote.js +7 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,1879 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { decodeBytes, encodeBytes, } from "@actiondock/sdk";
|
|
3
|
+
import { ACCESS_DENIED, CONTROL_BUSY, CONTROL_EXPIRED, CONTROL_REVOKED, INPUT_CLOSED, INPUT_OUTCOME_UNKNOWN, INPUT_VALIDATION_FAILED, NOT_FOUND, PROCESS_CANCELLED, PROCESS_LOST, PROCESS_QUARANTINED, PROCESS_SPAWN_ERROR, PROCESS_TIMEOUT, QUEUE_FULL, QUOTA_EXCEEDED, REQUEST_CONFLICT, SERVER_ERROR, UNSUPPORTED_CAPABILITY, INVALID_CURSOR, OUTPUT_UNAVAILABLE, ProcessError, } from "../errors.js";
|
|
4
|
+
import { parseCursor, compareCursorPos } from "./cursor.js";
|
|
5
|
+
import { ContextProcessAPI } from "./context-process.js";
|
|
6
|
+
import { MemoryProcessMetadataStore, } from "./metadata-store.js";
|
|
7
|
+
import { ProcessOutputLog } from "./output-log.js";
|
|
8
|
+
/**
|
|
9
|
+
* 格式化生成作用域唯一隔离字符串。
|
|
10
|
+
*/
|
|
11
|
+
export function formatProcessScope(owner) {
|
|
12
|
+
return `${owner.tenantId}:${owner.principalId}:${owner.packageInstanceId}:${owner.generationId}`;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* 校验调用方所有者凭据与目标记录是否完全匹配。
|
|
16
|
+
*/
|
|
17
|
+
function checkOwnerAuthorized(owner, target) {
|
|
18
|
+
if (!owner ||
|
|
19
|
+
!owner.tenantId ||
|
|
20
|
+
!owner.principalId ||
|
|
21
|
+
!owner.packageInstanceId ||
|
|
22
|
+
!owner.generationId) {
|
|
23
|
+
throw new ProcessError(ACCESS_DENIED, "Missing required owner identity fields");
|
|
24
|
+
}
|
|
25
|
+
if (owner.tenantId !== target.tenantId ||
|
|
26
|
+
owner.principalId !== target.principalId ||
|
|
27
|
+
owner.packageInstanceId !== target.packageInstanceId ||
|
|
28
|
+
owner.generationId !== target.generationId) {
|
|
29
|
+
throw new ProcessError(ACCESS_DENIED, "Access denied: owner identity mismatch");
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* 计算任意 JSON 负载对象的 SHA-256 哈希值。
|
|
34
|
+
*/
|
|
35
|
+
function hashRequestPayload(payload) {
|
|
36
|
+
return createHash("sha256").update(JSON.stringify(payload)).digest("hex");
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* 将持久化记录转换为 SDK 标准 ProcessInfo 快照。
|
|
40
|
+
*/
|
|
41
|
+
function toSdkProcessInfo(record) {
|
|
42
|
+
const ctrl = (record.control ?? record.controlState ?? "free");
|
|
43
|
+
const exitDetails = record.exitCode !== undefined || record.exitSignal !== undefined
|
|
44
|
+
? { code: record.exitCode ?? null, signal: record.exitSignal ?? null }
|
|
45
|
+
: undefined;
|
|
46
|
+
return {
|
|
47
|
+
id: record.processId,
|
|
48
|
+
hostEpoch: record.hostEpoch,
|
|
49
|
+
state: record.state,
|
|
50
|
+
control: ctrl,
|
|
51
|
+
io: (record.ioConfig ?? { mode: "pipe" }),
|
|
52
|
+
capabilities: (record.capabilities ?? {
|
|
53
|
+
pty: false,
|
|
54
|
+
resize: false,
|
|
55
|
+
inputEOF: false,
|
|
56
|
+
interruptForeground: false,
|
|
57
|
+
terminationScope: "process",
|
|
58
|
+
}),
|
|
59
|
+
createdAt: record.createdAt ?? new Date().toISOString(),
|
|
60
|
+
exit: exitDetails,
|
|
61
|
+
endReason: record.endReason,
|
|
62
|
+
outputClosed: Boolean(record.outputClosed),
|
|
63
|
+
outputEndReason: record.outputEndReason,
|
|
64
|
+
effectiveLimits: (record.effectiveLimits ?? {
|
|
65
|
+
idleMs: 60000,
|
|
66
|
+
lifetimeMs: 3600000,
|
|
67
|
+
outputBufferBytes: 4 * 1024 * 1024,
|
|
68
|
+
}),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* 将内部模型转换为持久化记录。
|
|
73
|
+
*/
|
|
74
|
+
function toStoredProcessRecord(owner, info, startRequestId) {
|
|
75
|
+
return {
|
|
76
|
+
processId: info.id,
|
|
77
|
+
tenantId: owner.tenantId,
|
|
78
|
+
principalId: owner.principalId,
|
|
79
|
+
packageInstanceId: owner.packageInstanceId,
|
|
80
|
+
generationId: owner.generationId,
|
|
81
|
+
hostEpoch: info.hostEpoch,
|
|
82
|
+
state: info.state,
|
|
83
|
+
controlState: info.control,
|
|
84
|
+
control: info.control,
|
|
85
|
+
ioConfig: info.io,
|
|
86
|
+
capabilities: info.capabilities,
|
|
87
|
+
createdAt: info.createdAt,
|
|
88
|
+
exitCode: info.exit ? info.exit.code : null,
|
|
89
|
+
exitSignal: info.exit ? info.exit.signal : null,
|
|
90
|
+
endReason: info.endReason ?? null,
|
|
91
|
+
outputClosed: info.outputClosed,
|
|
92
|
+
outputEndReason: info.outputEndReason ?? null,
|
|
93
|
+
inputClosed: false,
|
|
94
|
+
effectiveLimits: info.effectiveLimits,
|
|
95
|
+
startRequestId,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* 格式化同步幂等预占键。
|
|
100
|
+
*/
|
|
101
|
+
function formatReservationKey(key, operation) {
|
|
102
|
+
return `${key.hostEpoch}:${key.scope}:${key.processId ?? ""}:${operation}:${key.requestId}`;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* 校验正整数毫秒时长参数,非法时抛出参数错误。
|
|
106
|
+
*/
|
|
107
|
+
function checkPositiveDurationMs(value, field) {
|
|
108
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
109
|
+
throw new ProcessError(INPUT_VALIDATION_FAILED, `Invalid ${field}: must be a positive integer (milliseconds)`, {
|
|
110
|
+
[field]: value,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* 受管进程核心管理器。
|
|
116
|
+
*
|
|
117
|
+
* 职责范畴:
|
|
118
|
+
* - 宿主身份与配额检查(每作用域活跃进程、每宿主进程、输出缓冲预算、等待者上限、输入队列容量)。
|
|
119
|
+
* - 严格归属所有者鉴权校验(基于 tenantId, principalId, packageInstanceId, generationId)。
|
|
120
|
+
* - 控制权状态机推进(free -> held -> quarantined -> closed)。
|
|
121
|
+
* - 输入队列去重与串行调度执行。
|
|
122
|
+
* - 资源生命周期定时器管理(idleTimeout, maxLifetime, drainDeadline)。
|
|
123
|
+
*/
|
|
124
|
+
export class ProcessManager {
|
|
125
|
+
hostEpoch;
|
|
126
|
+
driver;
|
|
127
|
+
metadataStore;
|
|
128
|
+
quotas;
|
|
129
|
+
defaultLimits;
|
|
130
|
+
drainDeadlineMs;
|
|
131
|
+
terminalLogRetentionMs;
|
|
132
|
+
logger;
|
|
133
|
+
processes = new Map();
|
|
134
|
+
/** 已驱逐进程的输出日志缓存:附带时间戳支撑 TTL 过期回收与 LRU 淘汰 */
|
|
135
|
+
evictedOutputLogs = new Map();
|
|
136
|
+
/** 已淘汰输出日志的墓碑缓存:防止因日志淘汰静默丢失输出数据,支撑缺口明确告知与不可用异常 */
|
|
137
|
+
evictedOutputTombstones = new Map();
|
|
138
|
+
/** 同步幂等预占表:以复合键在异步落盘窗口内锁定并发重复请求 */
|
|
139
|
+
requestReservations = new Map();
|
|
140
|
+
initPromise;
|
|
141
|
+
isShutdown = false;
|
|
142
|
+
/** 内部诊断日志:未注入 logger 时保留最近的持久化与驱动错误,避免静默吞没异常 */
|
|
143
|
+
diagnostics = [];
|
|
144
|
+
constructor(options) {
|
|
145
|
+
this.hostEpoch = options.hostEpoch ?? `epoch-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
146
|
+
this.driver = options.driver;
|
|
147
|
+
this.metadataStore = options.metadataStore ?? new MemoryProcessMetadataStore();
|
|
148
|
+
this.quotas = {
|
|
149
|
+
maxActiveProcessesPerScope: options.quotas?.maxActiveProcessesPerScope ?? 8,
|
|
150
|
+
maxActiveProcessesPerHost: options.quotas?.maxActiveProcessesPerHost ?? 64,
|
|
151
|
+
maxOutputBufferBytesPerProcess: options.quotas?.maxOutputBufferBytesPerProcess ?? 4 * 1024 * 1024,
|
|
152
|
+
maxOutputBufferBytesPerHost: options.quotas?.maxOutputBufferBytesPerHost ?? 128 * 1024 * 1024,
|
|
153
|
+
maxPendingQueueBytesPerProcess: options.quotas?.maxPendingQueueBytesPerProcess ?? 1 * 1024 * 1024,
|
|
154
|
+
maxPendingQueueBytesPerHost: options.quotas?.maxPendingQueueBytesPerHost ?? 16 * 1024 * 1024,
|
|
155
|
+
maxWaitersPerProcess: options.quotas?.maxWaitersPerProcess ?? 8,
|
|
156
|
+
maxWaitersPerHost: options.quotas?.maxWaitersPerHost ?? 256,
|
|
157
|
+
};
|
|
158
|
+
this.defaultLimits = {
|
|
159
|
+
idleMs: options.defaultLimits?.idleMs ?? 60000,
|
|
160
|
+
lifetimeMs: options.defaultLimits?.lifetimeMs ?? 3600000,
|
|
161
|
+
outputBufferBytes: options.defaultLimits?.outputBufferBytes ?? 4 * 1024 * 1024,
|
|
162
|
+
};
|
|
163
|
+
this.drainDeadlineMs = options.drainDeadlineMs ?? 5000;
|
|
164
|
+
this.terminalLogRetentionMs = options.terminalLogRetentionMs ?? 10 * 60 * 1000;
|
|
165
|
+
this.logger = options.logger;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* 初始化宿主环境,原子性收敛旧宿主遗留的非终态进程。
|
|
169
|
+
* 幂等:重复调用返回同一份缓存 Promise。
|
|
170
|
+
*/
|
|
171
|
+
async initialize() {
|
|
172
|
+
if (!this.initPromise) {
|
|
173
|
+
this.initPromise = this.metadataStore
|
|
174
|
+
.initializeHost(this.hostEpoch)
|
|
175
|
+
.catch((err) => {
|
|
176
|
+
this.initPromise = undefined;
|
|
177
|
+
this.recordDiagnostic("initializeHost failed", err);
|
|
178
|
+
throw err instanceof ProcessError
|
|
179
|
+
? err
|
|
180
|
+
: new ProcessError(SERVER_ERROR, `Failed to initialize process host: ${err instanceof Error ? err.message : String(err)}`);
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
return this.initPromise;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* 惰性收敛入口:所有公共操作前调用,确保崩溃恢复无需平台显式接线。
|
|
187
|
+
*/
|
|
188
|
+
ensureInitialized() {
|
|
189
|
+
return this.initialize();
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* 记录内部诊断信息:优先写入注入的 logger,缺失时保留在内存环形缓冲区。
|
|
193
|
+
*/
|
|
194
|
+
recordDiagnostic(message, err) {
|
|
195
|
+
const detail = err instanceof Error ? `${err.name}: ${err.message}` : err !== undefined ? String(err) : "";
|
|
196
|
+
const line = detail ? `${message} (${detail})` : message;
|
|
197
|
+
this.diagnostics.push(`${new Date().toISOString()} ${line}`);
|
|
198
|
+
if (this.diagnostics.length > 100) {
|
|
199
|
+
this.diagnostics.shift();
|
|
200
|
+
}
|
|
201
|
+
this.logger?.warn("[ProcessManager] " + line);
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* 获取最近的内部诊断日志快照(最近条目在前)。
|
|
205
|
+
*/
|
|
206
|
+
get recentDiagnostics() {
|
|
207
|
+
return [...this.diagnostics].reverse();
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* 异步落盘进程状态:捕获并记录持久化失败,杜绝静默吞没异常。
|
|
211
|
+
*/
|
|
212
|
+
async persistState(processId, patch) {
|
|
213
|
+
try {
|
|
214
|
+
await this.metadataStore.updateProcessState(processId, patch);
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
this.recordDiagnostic(`Failed to persist state for process '${processId}'`, err);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* 异步落盘请求凭据:捕获并记录持久化失败,杜绝静默吞没异常。
|
|
222
|
+
*/
|
|
223
|
+
async persistReceipt(key, receipt, payloadHash) {
|
|
224
|
+
try {
|
|
225
|
+
await this.metadataStore.recordRequest(key, receipt, payloadHash);
|
|
226
|
+
}
|
|
227
|
+
catch (err) {
|
|
228
|
+
this.recordDiagnostic(`Failed to persist receipt for request '${key.requestId}'`, err);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* 同步预占幂等请求:跨 await 的检查与落盘窗口内锁定同复合键并发调用。
|
|
233
|
+
* 返回 undefined 表示预占成功,调用方继续异步路径并在完成时调用 commit/cancel。
|
|
234
|
+
* 返回 RequestReservation 表示已有同请求进行中,调用方可等待其结果。
|
|
235
|
+
* 若已有请求负载或操作类型不同,立即抛出 REQUEST_CONFLICT 杜绝混用结果。
|
|
236
|
+
*/
|
|
237
|
+
reserveRequest(key, payloadHash, operation) {
|
|
238
|
+
const reservationKey = formatReservationKey(key, operation);
|
|
239
|
+
const existing = this.requestReservations.get(reservationKey);
|
|
240
|
+
if (existing) {
|
|
241
|
+
if (existing.payloadHash !== payloadHash || existing.operation !== operation) {
|
|
242
|
+
throw new ProcessError(REQUEST_CONFLICT, "Request conflict: identical requestId with different payload", { requestId: key.requestId });
|
|
243
|
+
}
|
|
244
|
+
return existing;
|
|
245
|
+
}
|
|
246
|
+
let resolveFn = () => { };
|
|
247
|
+
let rejectFn = () => { };
|
|
248
|
+
const promise = new Promise((res, rej) => {
|
|
249
|
+
resolveFn = res;
|
|
250
|
+
rejectFn = rej;
|
|
251
|
+
});
|
|
252
|
+
// 预占 promise 可能永远无人等待:预先挂接空捕获,避免拒绝时触发 unhandledRejection
|
|
253
|
+
promise.catch(() => { });
|
|
254
|
+
const reservation = {
|
|
255
|
+
payloadHash,
|
|
256
|
+
operation,
|
|
257
|
+
promise,
|
|
258
|
+
resolve: resolveFn,
|
|
259
|
+
reject: rejectFn,
|
|
260
|
+
};
|
|
261
|
+
this.requestReservations.set(reservationKey, reservation);
|
|
262
|
+
return undefined;
|
|
263
|
+
}
|
|
264
|
+
commitReservation(key, operation, value) {
|
|
265
|
+
const reservationKey = formatReservationKey(key, operation);
|
|
266
|
+
const reservation = this.requestReservations.get(reservationKey);
|
|
267
|
+
if (reservation) {
|
|
268
|
+
this.requestReservations.delete(reservationKey);
|
|
269
|
+
reservation.resolve(value);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
rejectReservation(key, operation, err) {
|
|
273
|
+
const reservationKey = formatReservationKey(key, operation);
|
|
274
|
+
const reservation = this.requestReservations.get(reservationKey);
|
|
275
|
+
if (reservation) {
|
|
276
|
+
this.requestReservations.delete(reservationKey);
|
|
277
|
+
reservation.reject(err);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* 停止全部受管进程、清理全部定时器并置为关闭态,供宿主优雅退出使用。
|
|
282
|
+
*/
|
|
283
|
+
async shutdown() {
|
|
284
|
+
if (this.isShutdown) {
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
this.isShutdown = true;
|
|
288
|
+
const targets = Array.from(this.processes.values()).filter((proc) => proc.info.state !== "exited" &&
|
|
289
|
+
proc.info.state !== "failed" &&
|
|
290
|
+
proc.info.state !== "lost");
|
|
291
|
+
for (const proc of targets) {
|
|
292
|
+
proc.info.endReason = proc.info.endReason ?? "requested";
|
|
293
|
+
try {
|
|
294
|
+
await this.stop(proc.owner, proc.info.id, {
|
|
295
|
+
requestId: `shutdown-${proc.info.id}`,
|
|
296
|
+
graceMs: 1000,
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
catch (err) {
|
|
300
|
+
this.recordDiagnostic(`Failed to stop process '${proc.info.id}' during shutdown`, err);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
// 防御性清理残余定时器(stop 内部已清理,此处兜底)
|
|
304
|
+
for (const proc of this.processes.values()) {
|
|
305
|
+
if (proc.ttlTimer) {
|
|
306
|
+
clearTimeout(proc.ttlTimer);
|
|
307
|
+
proc.ttlTimer = undefined;
|
|
308
|
+
}
|
|
309
|
+
if (proc.idleTimer) {
|
|
310
|
+
clearTimeout(proc.idleTimer);
|
|
311
|
+
proc.idleTimer = undefined;
|
|
312
|
+
}
|
|
313
|
+
if (proc.lifetimeTimer) {
|
|
314
|
+
clearTimeout(proc.lifetimeTimer);
|
|
315
|
+
proc.lifetimeTimer = undefined;
|
|
316
|
+
}
|
|
317
|
+
if (proc.drainTimer) {
|
|
318
|
+
clearTimeout(proc.drainTimer);
|
|
319
|
+
proc.drainTimer = undefined;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
for (const [reservationKey, res] of Array.from(this.requestReservations.entries())) {
|
|
323
|
+
this.requestReservations.delete(reservationKey);
|
|
324
|
+
res.reject(new ProcessError(PROCESS_CANCELLED, "Process manager has been shut down"));
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
assertNotShutdown() {
|
|
328
|
+
if (this.isShutdown) {
|
|
329
|
+
throw new ProcessError(PROCESS_CANCELLED, "Process manager has been shut down");
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* 为指定所有者与运行上下文创建 ProcessAPI 代理适配器。
|
|
334
|
+
*/
|
|
335
|
+
forOwner(owner, runId, signal) {
|
|
336
|
+
return new ContextProcessAPI(this, owner, runId ?? randomUUID(), signal, Boolean(runId));
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* 启动新的受管进程资源。
|
|
340
|
+
*/
|
|
341
|
+
async start(owner, input, call) {
|
|
342
|
+
checkOwnerAuthorized(owner, owner);
|
|
343
|
+
await this.ensureInitialized();
|
|
344
|
+
this.assertNotShutdown();
|
|
345
|
+
const scope = formatProcessScope(owner);
|
|
346
|
+
const key = {
|
|
347
|
+
hostEpoch: this.hostEpoch,
|
|
348
|
+
scope,
|
|
349
|
+
requestId: input.requestId,
|
|
350
|
+
};
|
|
351
|
+
const payloadHash = hashRequestPayload({ spec: input.spec, limits: input.limits });
|
|
352
|
+
// 同步预占幂等请求,跨 await 窗口内拦截并发同 requestId 双重执行
|
|
353
|
+
const inFlight = this.reserveRequest(key, payloadHash, "start");
|
|
354
|
+
if (inFlight) {
|
|
355
|
+
return (await inFlight.promise);
|
|
356
|
+
}
|
|
357
|
+
try {
|
|
358
|
+
// 请求去重校验
|
|
359
|
+
const existing = await this.metadataStore.getRequest(key);
|
|
360
|
+
if (existing) {
|
|
361
|
+
if (existing.payloadHash !== payloadHash) {
|
|
362
|
+
throw new ProcessError(REQUEST_CONFLICT, "Request conflict: identical requestId with different payload", {
|
|
363
|
+
requestId: input.requestId,
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
const cached = existing.receipt;
|
|
367
|
+
this.commitReservation(key, "start", cached);
|
|
368
|
+
return cached;
|
|
369
|
+
}
|
|
370
|
+
// 宿主与作用域配额检查
|
|
371
|
+
this.checkSpawnQuotas(scope, input.limits?.outputBufferBytes);
|
|
372
|
+
// 驱动能力校验
|
|
373
|
+
const caps = this.driver.getCapabilities();
|
|
374
|
+
if (input.spec.io.mode === "pty" && !caps.pty) {
|
|
375
|
+
throw new ProcessError(UNSUPPORTED_CAPABILITY, "Driver does not support PTY mode");
|
|
376
|
+
}
|
|
377
|
+
const processId = `proc-${randomUUID()}`;
|
|
378
|
+
const effectiveLimits = {
|
|
379
|
+
idleMs: input.limits?.idleMs ?? this.defaultLimits.idleMs,
|
|
380
|
+
lifetimeMs: input.limits?.lifetimeMs ?? this.defaultLimits.lifetimeMs,
|
|
381
|
+
outputBufferBytes: input.limits?.outputBufferBytes ?? this.defaultLimits.outputBufferBytes,
|
|
382
|
+
};
|
|
383
|
+
const outputLog = new ProcessOutputLog(this.hostEpoch, processId, {
|
|
384
|
+
maxBufferBytes: effectiveLimits.outputBufferBytes,
|
|
385
|
+
maxWaiters: this.quotas.maxWaitersPerProcess,
|
|
386
|
+
});
|
|
387
|
+
const info = {
|
|
388
|
+
id: processId,
|
|
389
|
+
hostEpoch: this.hostEpoch,
|
|
390
|
+
state: "starting",
|
|
391
|
+
control: "free",
|
|
392
|
+
io: input.spec.io,
|
|
393
|
+
capabilities: caps,
|
|
394
|
+
createdAt: new Date().toISOString(),
|
|
395
|
+
outputClosed: false,
|
|
396
|
+
effectiveLimits,
|
|
397
|
+
};
|
|
398
|
+
const procRecord = {
|
|
399
|
+
info,
|
|
400
|
+
owner: { ...owner },
|
|
401
|
+
scope,
|
|
402
|
+
outputLog,
|
|
403
|
+
controlEpoch: 0,
|
|
404
|
+
cancelEpoch: 0,
|
|
405
|
+
inputQueue: [],
|
|
406
|
+
pendingInputBytes: 0,
|
|
407
|
+
acquireWaiters: [],
|
|
408
|
+
inputClosed: false,
|
|
409
|
+
isDispatching: false,
|
|
410
|
+
effectiveLimits,
|
|
411
|
+
};
|
|
412
|
+
this.processes.set(processId, procRecord);
|
|
413
|
+
await this.metadataStore.saveProcess(toStoredProcessRecord(owner, info, input.requestId));
|
|
414
|
+
// 调用驱动派生进程
|
|
415
|
+
let abortListener;
|
|
416
|
+
try {
|
|
417
|
+
const callbacks = {
|
|
418
|
+
onOutput: (stream, data) => {
|
|
419
|
+
procRecord.outputLog.append(stream, data);
|
|
420
|
+
},
|
|
421
|
+
onExit: (exit) => {
|
|
422
|
+
this.handleProcessExit(procRecord, exit);
|
|
423
|
+
},
|
|
424
|
+
onOutputClosed: (reason) => {
|
|
425
|
+
this.handleOutputClosed(procRecord, reason);
|
|
426
|
+
},
|
|
427
|
+
onError: (err) => {
|
|
428
|
+
this.handleProcessError(procRecord, err);
|
|
429
|
+
},
|
|
430
|
+
};
|
|
431
|
+
if (call?.signal?.aborted) {
|
|
432
|
+
throw call.signal.reason instanceof ProcessError
|
|
433
|
+
? call.signal.reason
|
|
434
|
+
: new ProcessError(PROCESS_CANCELLED, "Process start was cancelled");
|
|
435
|
+
}
|
|
436
|
+
const spawnPromise = this.driver.spawn(processId, input.spec, callbacks);
|
|
437
|
+
if (call?.signal) {
|
|
438
|
+
const abortPromise = new Promise((_, reject) => {
|
|
439
|
+
abortListener = () => {
|
|
440
|
+
reject(call.signal?.reason instanceof ProcessError
|
|
441
|
+
? call.signal.reason
|
|
442
|
+
: new ProcessError(PROCESS_CANCELLED, "Process start was cancelled"));
|
|
443
|
+
};
|
|
444
|
+
call.signal?.addEventListener("abort", abortListener, { once: true });
|
|
445
|
+
});
|
|
446
|
+
try {
|
|
447
|
+
procRecord.handle = await Promise.race([spawnPromise, abortPromise]);
|
|
448
|
+
}
|
|
449
|
+
catch (raceErr) {
|
|
450
|
+
void spawnPromise
|
|
451
|
+
.then((h) => {
|
|
452
|
+
if (h && typeof h.terminate === "function") {
|
|
453
|
+
void h.terminate(0);
|
|
454
|
+
}
|
|
455
|
+
})
|
|
456
|
+
.catch(() => { });
|
|
457
|
+
throw raceErr;
|
|
458
|
+
}
|
|
459
|
+
finally {
|
|
460
|
+
if (abortListener) {
|
|
461
|
+
call.signal.removeEventListener("abort", abortListener);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
else {
|
|
466
|
+
procRecord.handle = await spawnPromise;
|
|
467
|
+
}
|
|
468
|
+
if (call?.signal?.aborted) {
|
|
469
|
+
if (procRecord.handle && typeof procRecord.handle.terminate === "function") {
|
|
470
|
+
await procRecord.handle.terminate(0);
|
|
471
|
+
}
|
|
472
|
+
throw call.signal.reason instanceof ProcessError
|
|
473
|
+
? call.signal.reason
|
|
474
|
+
: new ProcessError(PROCESS_CANCELLED, "Process start was cancelled");
|
|
475
|
+
}
|
|
476
|
+
// 仅在进程仍处于 starting 初始阶段时转为 running 并挂接定时器;
|
|
477
|
+
// 若驱动在 spawn 完成前已触发 exited / failed / stopping,切勿回写为 running 且严禁启动空闲与寿命定时器
|
|
478
|
+
if (procRecord.info.state === "starting") {
|
|
479
|
+
procRecord.info.state = "running";
|
|
480
|
+
await this.metadataStore.updateProcessState(processId, { state: "running" });
|
|
481
|
+
this.startIdleTimer(procRecord);
|
|
482
|
+
this.startLifetimeTimer(procRecord);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
catch (err) {
|
|
486
|
+
if ((err instanceof ProcessError && err.code === PROCESS_CANCELLED) || call?.signal?.aborted) {
|
|
487
|
+
if (procRecord.info.state !== "exited" && procRecord.info.state !== "failed") {
|
|
488
|
+
procRecord.info.state = "exited";
|
|
489
|
+
procRecord.info.endReason = "requested";
|
|
490
|
+
procRecord.info.control = "closed";
|
|
491
|
+
procRecord.outputLog.closeOutput("natural");
|
|
492
|
+
await this.metadataStore.updateProcessState(processId, {
|
|
493
|
+
state: "exited",
|
|
494
|
+
endReason: "requested",
|
|
495
|
+
control: "closed",
|
|
496
|
+
controlState: "closed",
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
throw err instanceof ProcessError
|
|
500
|
+
? err
|
|
501
|
+
: new ProcessError(PROCESS_CANCELLED, "Process start was cancelled");
|
|
502
|
+
}
|
|
503
|
+
if (procRecord.info.state !== "exited" && procRecord.info.state !== "failed") {
|
|
504
|
+
procRecord.info.state = "failed";
|
|
505
|
+
procRecord.info.endReason = "spawn-failure";
|
|
506
|
+
procRecord.info.control = "closed";
|
|
507
|
+
procRecord.outputLog.closeOutput("natural");
|
|
508
|
+
await this.metadataStore.updateProcessState(processId, {
|
|
509
|
+
state: "failed",
|
|
510
|
+
endReason: "spawn-failure",
|
|
511
|
+
control: "closed",
|
|
512
|
+
controlState: "closed",
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
throw new ProcessError(PROCESS_SPAWN_ERROR, `Failed to spawn process: ${err?.message || String(err)}`);
|
|
516
|
+
}
|
|
517
|
+
const startResult = {
|
|
518
|
+
process: { ...procRecord.info },
|
|
519
|
+
initialCursor: outputLog.earliestCursor,
|
|
520
|
+
};
|
|
521
|
+
await this.metadataStore.recordRequest(key, startResult, payloadHash);
|
|
522
|
+
this.commitReservation(key, "start", startResult);
|
|
523
|
+
return startResult;
|
|
524
|
+
}
|
|
525
|
+
finally {
|
|
526
|
+
// 成功路径已在 commit 中移除预占;此处仅对异常路径释放并唤醒等待方
|
|
527
|
+
this.rejectReservation(key, "start", new ProcessError(REQUEST_CONFLICT, "Concurrent start request did not produce a result", { requestId: input.requestId }));
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* 查看指定受管进程资源的当前最新状态。
|
|
532
|
+
*/
|
|
533
|
+
async inspect(owner, id, call) {
|
|
534
|
+
await this.ensureInitialized();
|
|
535
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
536
|
+
return { ...proc.info };
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* 分页列出指定归属所有者可见的受管进程列表。
|
|
540
|
+
*/
|
|
541
|
+
async list(owner, input, call) {
|
|
542
|
+
checkOwnerAuthorized(owner, owner);
|
|
543
|
+
await this.ensureInitialized();
|
|
544
|
+
const res = await this.metadataStore.listProcesses(owner, input.pageToken, input.limit);
|
|
545
|
+
const processes = [];
|
|
546
|
+
for (const item of res.processes) {
|
|
547
|
+
const active = this.processes.get(item.processId);
|
|
548
|
+
if (active) {
|
|
549
|
+
processes.push({ ...active.info });
|
|
550
|
+
}
|
|
551
|
+
else {
|
|
552
|
+
processes.push(toSdkProcessInfo(item));
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return {
|
|
556
|
+
processes,
|
|
557
|
+
nextPageToken: res.nextPageToken,
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
/**
|
|
561
|
+
* 申请指定受管进程的独占控制令牌。
|
|
562
|
+
*/
|
|
563
|
+
async acquire(owner, id, input, call, runId) {
|
|
564
|
+
checkPositiveDurationMs(input.waitMs, "waitMs");
|
|
565
|
+
checkPositiveDurationMs(input.ttlMs, "ttlMs");
|
|
566
|
+
await this.ensureInitialized();
|
|
567
|
+
this.assertNotShutdown();
|
|
568
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
569
|
+
if (proc.info.state === "exited" ||
|
|
570
|
+
proc.info.state === "failed" ||
|
|
571
|
+
proc.info.state === "lost") {
|
|
572
|
+
throw new ProcessError(CONTROL_REVOKED, "Process has terminated");
|
|
573
|
+
}
|
|
574
|
+
if (proc.info.control === "quarantined") {
|
|
575
|
+
throw new ProcessError(PROCESS_QUARANTINED, "Process is in quarantined state");
|
|
576
|
+
}
|
|
577
|
+
if (proc.info.control === "closed") {
|
|
578
|
+
throw new ProcessError(CONTROL_REVOKED, "Process control is closed");
|
|
579
|
+
}
|
|
580
|
+
const key = {
|
|
581
|
+
hostEpoch: this.hostEpoch,
|
|
582
|
+
scope: proc.scope,
|
|
583
|
+
processId: id,
|
|
584
|
+
requestId: input.requestId,
|
|
585
|
+
};
|
|
586
|
+
const payloadHash = hashRequestPayload({ waitMs: input.waitMs, ttlMs: input.ttlMs });
|
|
587
|
+
// 同步预占幂等请求,跨 await 窗口内拦截并发同 requestId 双重授权
|
|
588
|
+
const inFlight = this.reserveRequest(key, payloadHash, "acquire");
|
|
589
|
+
if (inFlight) {
|
|
590
|
+
return (await inFlight.promise);
|
|
591
|
+
}
|
|
592
|
+
try {
|
|
593
|
+
// 幂等去重检查
|
|
594
|
+
const existing = await this.metadataStore.getRequest(key);
|
|
595
|
+
if (existing) {
|
|
596
|
+
if (existing.payloadHash !== payloadHash) {
|
|
597
|
+
throw new ProcessError(REQUEST_CONFLICT, "Request conflict: identical requestId with different payload", {
|
|
598
|
+
requestId: input.requestId,
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
const cached = existing.receipt;
|
|
602
|
+
this.commitReservation(key, "acquire", cached);
|
|
603
|
+
return cached;
|
|
604
|
+
}
|
|
605
|
+
// 若控制权空闲且无等待者,直接授予
|
|
606
|
+
if (proc.info.control === "free" && proc.acquireWaiters.length === 0) {
|
|
607
|
+
const grant = this.grantControl(proc, input.requestId, input.ttlMs, runId);
|
|
608
|
+
await this.metadataStore.recordRequest(key, grant, payloadHash);
|
|
609
|
+
this.commitReservation(key, "acquire", grant);
|
|
610
|
+
return grant;
|
|
611
|
+
}
|
|
612
|
+
// 控制权已被持有,检查等待队列配额并进入 FIFO 排队
|
|
613
|
+
if (proc.acquireWaiters.length >= this.quotas.maxWaitersPerProcess) {
|
|
614
|
+
throw new ProcessError(QUOTA_EXCEEDED, "Process acquire waiter quota exceeded", {
|
|
615
|
+
limit: this.quotas.maxWaitersPerProcess,
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
const totalHostWaiters = this.countHostAcquireWaiters();
|
|
619
|
+
if (totalHostWaiters >= this.quotas.maxWaitersPerHost) {
|
|
620
|
+
throw new ProcessError(QUOTA_EXCEEDED, "Host acquire waiter quota exceeded", {
|
|
621
|
+
limit: this.quotas.maxWaitersPerHost,
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
// 排队路径:由 wakeNextAcquireWaiter 在授予时落盘凭据并结算预占
|
|
625
|
+
const grant = await new Promise((resolve, reject) => {
|
|
626
|
+
let timer;
|
|
627
|
+
const waiter = {
|
|
628
|
+
requestId: input.requestId,
|
|
629
|
+
waitMs: input.waitMs,
|
|
630
|
+
ttlMs: input.ttlMs,
|
|
631
|
+
runId,
|
|
632
|
+
resolve: (granted) => {
|
|
633
|
+
cleanup();
|
|
634
|
+
resolve(granted);
|
|
635
|
+
},
|
|
636
|
+
reject: (err) => {
|
|
637
|
+
cleanup();
|
|
638
|
+
reject(err);
|
|
639
|
+
},
|
|
640
|
+
};
|
|
641
|
+
const cleanup = () => {
|
|
642
|
+
if (timer)
|
|
643
|
+
clearTimeout(timer);
|
|
644
|
+
const idx = proc.acquireWaiters.indexOf(waiter);
|
|
645
|
+
if (idx !== -1) {
|
|
646
|
+
proc.acquireWaiters.splice(idx, 1);
|
|
647
|
+
}
|
|
648
|
+
if (call?.signal && waiter.onAbort) {
|
|
649
|
+
call.signal.removeEventListener("abort", waiter.onAbort);
|
|
650
|
+
}
|
|
651
|
+
};
|
|
652
|
+
if (call?.signal) {
|
|
653
|
+
if (call.signal.aborted) {
|
|
654
|
+
reject(call.signal.reason ?? new ProcessError(PROCESS_CANCELLED, "Acquire cancelled"));
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
waiter.onAbort = () => {
|
|
658
|
+
cleanup();
|
|
659
|
+
reject(call.signal?.reason ?? new ProcessError(PROCESS_CANCELLED, "Acquire cancelled"));
|
|
660
|
+
};
|
|
661
|
+
call.signal.addEventListener("abort", waiter.onAbort, { once: true });
|
|
662
|
+
}
|
|
663
|
+
timer = setTimeout(() => {
|
|
664
|
+
cleanup();
|
|
665
|
+
reject(new ProcessError(CONTROL_BUSY, "Timed out waiting for process control", {
|
|
666
|
+
waitMs: input.waitMs,
|
|
667
|
+
}));
|
|
668
|
+
}, input.waitMs);
|
|
669
|
+
if (typeof timer?.unref === "function") {
|
|
670
|
+
timer.unref();
|
|
671
|
+
}
|
|
672
|
+
proc.acquireWaiters.push(waiter);
|
|
673
|
+
});
|
|
674
|
+
this.commitReservation(key, "acquire", grant);
|
|
675
|
+
return grant;
|
|
676
|
+
}
|
|
677
|
+
finally {
|
|
678
|
+
// 成功路径已在 commit 中移除预占;此处仅对异常路径释放并唤醒等待方
|
|
679
|
+
this.rejectReservation(key, "acquire", new ProcessError(REQUEST_CONFLICT, "Concurrent acquire request did not produce a result", { requestId: input.requestId }));
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
/**
|
|
683
|
+
* 延长当前有效控制令牌的存活时间。
|
|
684
|
+
*/
|
|
685
|
+
async renew(owner, id, token, ttlMs, call) {
|
|
686
|
+
checkPositiveDurationMs(ttlMs, "ttlMs");
|
|
687
|
+
await this.ensureInitialized();
|
|
688
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
689
|
+
if (proc.info.control === "quarantined") {
|
|
690
|
+
throw new ProcessError(PROCESS_QUARANTINED, "Process is in quarantined state");
|
|
691
|
+
}
|
|
692
|
+
if (proc.info.control !== "held" || !proc.currentGrant) {
|
|
693
|
+
throw new ProcessError(CONTROL_REVOKED, "Process control is not currently held");
|
|
694
|
+
}
|
|
695
|
+
if (proc.currentGrant.token !== token) {
|
|
696
|
+
throw new ProcessError(ACCESS_DENIED, "Invalid control token");
|
|
697
|
+
}
|
|
698
|
+
if (new Date(proc.currentGrant.expiresAt).getTime() <= Date.now()) {
|
|
699
|
+
throw new ProcessError(CONTROL_EXPIRED, "Control token has expired");
|
|
700
|
+
}
|
|
701
|
+
if (proc.ttlTimer) {
|
|
702
|
+
clearTimeout(proc.ttlTimer);
|
|
703
|
+
}
|
|
704
|
+
const expiresAt = new Date(Date.now() + ttlMs).toISOString();
|
|
705
|
+
proc.currentGrant.ttlMs = ttlMs;
|
|
706
|
+
proc.currentGrant.expiresAt = expiresAt;
|
|
707
|
+
proc.ttlTimer = setTimeout(() => {
|
|
708
|
+
this.handleGrantTtlExpired(proc);
|
|
709
|
+
}, ttlMs);
|
|
710
|
+
if (typeof proc.ttlTimer.unref === "function") {
|
|
711
|
+
proc.ttlTimer.unref();
|
|
712
|
+
}
|
|
713
|
+
// 延长控制权刷新 idleTimeout
|
|
714
|
+
this.refreshIdleTimer(proc);
|
|
715
|
+
return {
|
|
716
|
+
token: proc.currentGrant.token,
|
|
717
|
+
expiresAt,
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
/**
|
|
721
|
+
* 显式释放控制令牌,允许后续控制者申请。
|
|
722
|
+
*/
|
|
723
|
+
async release(owner, id, token, call) {
|
|
724
|
+
await this.ensureInitialized();
|
|
725
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
726
|
+
if (proc.info.control === "quarantined") {
|
|
727
|
+
throw new ProcessError(PROCESS_QUARANTINED, "Process is in quarantined state");
|
|
728
|
+
}
|
|
729
|
+
if (proc.info.control !== "held" || !proc.currentGrant) {
|
|
730
|
+
throw new ProcessError(CONTROL_REVOKED, "Process control is not currently held");
|
|
731
|
+
}
|
|
732
|
+
if (proc.currentGrant.token !== token) {
|
|
733
|
+
throw new ProcessError(ACCESS_DENIED, "Invalid control token");
|
|
734
|
+
}
|
|
735
|
+
// 当队列中存在待 dispatch 的操作时抛出 CONTROL_BUSY 拒绝 release
|
|
736
|
+
if (proc.inputQueue.length > 0) {
|
|
737
|
+
throw new ProcessError(CONTROL_BUSY, "Cannot release control while operations are pending in queue");
|
|
738
|
+
}
|
|
739
|
+
if (proc.ttlTimer) {
|
|
740
|
+
clearTimeout(proc.ttlTimer);
|
|
741
|
+
proc.ttlTimer = undefined;
|
|
742
|
+
}
|
|
743
|
+
proc.currentGrant = undefined;
|
|
744
|
+
proc.info.control = "free";
|
|
745
|
+
await this.persistState(id, {
|
|
746
|
+
control: "free",
|
|
747
|
+
controlState: "free",
|
|
748
|
+
});
|
|
749
|
+
// 正常 release 唤醒下一个等待者
|
|
750
|
+
this.wakeNextAcquireWaiter(proc);
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* 向受管进程输入流写入原始字节数据。
|
|
754
|
+
*/
|
|
755
|
+
async write(owner, id, input, call) {
|
|
756
|
+
await this.ensureInitialized();
|
|
757
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
758
|
+
const key = {
|
|
759
|
+
hostEpoch: this.hostEpoch,
|
|
760
|
+
scope: proc.scope,
|
|
761
|
+
processId: id,
|
|
762
|
+
requestId: input.requestId,
|
|
763
|
+
};
|
|
764
|
+
const payloadHash = hashRequestPayload({ token: input.token, data: input.data });
|
|
765
|
+
// 同步预占幂等请求,跨 await 窗口内拦截并发同 requestId 双重入队
|
|
766
|
+
const inFlight = this.reserveRequest(key, payloadHash, "write");
|
|
767
|
+
if (inFlight) {
|
|
768
|
+
return (await inFlight.promise);
|
|
769
|
+
}
|
|
770
|
+
try {
|
|
771
|
+
// 幂等去重检查
|
|
772
|
+
const existing = await this.metadataStore.getRequest(key);
|
|
773
|
+
if (existing) {
|
|
774
|
+
if (existing.payloadHash !== payloadHash) {
|
|
775
|
+
throw new ProcessError(REQUEST_CONFLICT, "Request conflict: identical requestId with different payload", {
|
|
776
|
+
requestId: input.requestId,
|
|
777
|
+
});
|
|
778
|
+
}
|
|
779
|
+
const cached = existing.receipt;
|
|
780
|
+
this.commitReservation(key, "write", cached);
|
|
781
|
+
return cached;
|
|
782
|
+
}
|
|
783
|
+
// 入队前校验授权与控制状态
|
|
784
|
+
this.checkOperationAuth(proc, input.token);
|
|
785
|
+
if (proc.inputClosed) {
|
|
786
|
+
throw new ProcessError(INPUT_CLOSED, "Input stream is closed");
|
|
787
|
+
}
|
|
788
|
+
const rawBytes = decodeBytes(input.data);
|
|
789
|
+
const dataSize = rawBytes.byteLength;
|
|
790
|
+
// 待写入队列容量检查:在任何 await 前同步原子校验并预占配额
|
|
791
|
+
if (proc.pendingInputBytes + dataSize > this.quotas.maxPendingQueueBytesPerProcess) {
|
|
792
|
+
throw new ProcessError(QUEUE_FULL, "Process pending input queue limit exceeded", {
|
|
793
|
+
limit: this.quotas.maxPendingQueueBytesPerProcess,
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
const totalHostPending = this.countHostPendingInputBytes();
|
|
797
|
+
if (totalHostPending + dataSize > this.quotas.maxPendingQueueBytesPerHost) {
|
|
798
|
+
throw new ProcessError(QUEUE_FULL, "Host pending input queue limit exceeded", {
|
|
799
|
+
limit: this.quotas.maxPendingQueueBytesPerHost,
|
|
800
|
+
});
|
|
801
|
+
}
|
|
802
|
+
// 同步占位预扣队列配额,杜绝并发 await 窗口穿透
|
|
803
|
+
proc.pendingInputBytes += dataSize;
|
|
804
|
+
let pendingBytesCommitted = false;
|
|
805
|
+
try {
|
|
806
|
+
const receipt = {
|
|
807
|
+
requestId: input.requestId,
|
|
808
|
+
state: "queued",
|
|
809
|
+
};
|
|
810
|
+
await this.metadataStore.recordRequest(key, receipt, payloadHash);
|
|
811
|
+
proc.inputQueue.push({
|
|
812
|
+
type: "write",
|
|
813
|
+
requestId: input.requestId,
|
|
814
|
+
token: input.token,
|
|
815
|
+
bytes: rawBytes,
|
|
816
|
+
receipt,
|
|
817
|
+
key,
|
|
818
|
+
payloadHash,
|
|
819
|
+
cancelEpoch: proc.cancelEpoch,
|
|
820
|
+
});
|
|
821
|
+
pendingBytesCommitted = true;
|
|
822
|
+
// 异步调度推进
|
|
823
|
+
queueMicrotask(() => void this.dispatchNext(proc));
|
|
824
|
+
const queued = { ...receipt };
|
|
825
|
+
this.commitReservation(key, "write", queued);
|
|
826
|
+
return queued;
|
|
827
|
+
}
|
|
828
|
+
finally {
|
|
829
|
+
if (!pendingBytesCommitted) {
|
|
830
|
+
proc.pendingInputBytes = Math.max(0, proc.pendingInputBytes - dataSize);
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
finally {
|
|
835
|
+
// 成功路径已在 commit 中移除预占;此处仅对异常路径释放并唤醒等待方
|
|
836
|
+
this.rejectReservation(key, "write", new ProcessError(REQUEST_CONFLICT, "Concurrent write request did not produce a result", { requestId: input.requestId }));
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* 向受管进程发送结构化控制指令。
|
|
841
|
+
*/
|
|
842
|
+
async control(owner, id, input, call) {
|
|
843
|
+
await this.ensureInitialized();
|
|
844
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
845
|
+
const key = {
|
|
846
|
+
hostEpoch: this.hostEpoch,
|
|
847
|
+
scope: proc.scope,
|
|
848
|
+
processId: id,
|
|
849
|
+
requestId: input.requestId,
|
|
850
|
+
};
|
|
851
|
+
const payloadHash = hashRequestPayload({ token: input.token, action: input.action });
|
|
852
|
+
// 同步预占幂等请求,跨 await 窗口内拦截并发同 requestId 双重入队
|
|
853
|
+
const inFlight = this.reserveRequest(key, payloadHash, "control");
|
|
854
|
+
if (inFlight) {
|
|
855
|
+
return (await inFlight.promise);
|
|
856
|
+
}
|
|
857
|
+
try {
|
|
858
|
+
// 幂等去重检查
|
|
859
|
+
const existing = await this.metadataStore.getRequest(key);
|
|
860
|
+
if (existing) {
|
|
861
|
+
if (existing.payloadHash !== payloadHash) {
|
|
862
|
+
throw new ProcessError(REQUEST_CONFLICT, "Request conflict: identical requestId with different payload", {
|
|
863
|
+
requestId: input.requestId,
|
|
864
|
+
});
|
|
865
|
+
}
|
|
866
|
+
const cached = existing.receipt;
|
|
867
|
+
this.commitReservation(key, "control", cached);
|
|
868
|
+
return cached;
|
|
869
|
+
}
|
|
870
|
+
// 入队前校验授权与控制状态
|
|
871
|
+
this.checkOperationAuth(proc, input.token);
|
|
872
|
+
// 校验能力支持
|
|
873
|
+
if (input.action.type === "input-eof" && !proc.info.capabilities.inputEOF) {
|
|
874
|
+
throw new ProcessError(UNSUPPORTED_CAPABILITY, "Driver does not support input-eof action");
|
|
875
|
+
}
|
|
876
|
+
if (input.action.type === "interrupt-foreground" &&
|
|
877
|
+
!proc.info.capabilities.interruptForeground) {
|
|
878
|
+
throw new ProcessError(UNSUPPORTED_CAPABILITY, "Driver does not support interrupt-foreground action");
|
|
879
|
+
}
|
|
880
|
+
if (input.action.type === "resize" &&
|
|
881
|
+
(!proc.info.capabilities.resize || proc.info.io.mode !== "pty")) {
|
|
882
|
+
throw new ProcessError(UNSUPPORTED_CAPABILITY, "Driver does not support resize or IO mode is not pty");
|
|
883
|
+
}
|
|
884
|
+
const receipt = {
|
|
885
|
+
requestId: input.requestId,
|
|
886
|
+
state: "queued",
|
|
887
|
+
};
|
|
888
|
+
await this.metadataStore.recordRequest(key, receipt, payloadHash);
|
|
889
|
+
proc.inputQueue.push({
|
|
890
|
+
type: "control",
|
|
891
|
+
requestId: input.requestId,
|
|
892
|
+
token: input.token,
|
|
893
|
+
action: input.action,
|
|
894
|
+
receipt,
|
|
895
|
+
key,
|
|
896
|
+
payloadHash,
|
|
897
|
+
cancelEpoch: proc.cancelEpoch,
|
|
898
|
+
});
|
|
899
|
+
queueMicrotask(() => void this.dispatchNext(proc));
|
|
900
|
+
const queued = { ...receipt };
|
|
901
|
+
this.commitReservation(key, "control", queued);
|
|
902
|
+
return queued;
|
|
903
|
+
}
|
|
904
|
+
finally {
|
|
905
|
+
// 成功路径已在 commit 中移除预占;此处仅对异常路径释放并唤醒等待方
|
|
906
|
+
this.rejectReservation(key, "control", new ProcessError(REQUEST_CONFLICT, "Concurrent control request did not produce a result", { requestId: input.requestId }));
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
910
|
+
* 查询指定请求标识的操作执行收据。
|
|
911
|
+
*/
|
|
912
|
+
async operation(owner, id, requestId, call) {
|
|
913
|
+
await this.ensureInitialized();
|
|
914
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
915
|
+
const key = {
|
|
916
|
+
hostEpoch: this.hostEpoch,
|
|
917
|
+
scope: proc.scope,
|
|
918
|
+
processId: id,
|
|
919
|
+
requestId,
|
|
920
|
+
};
|
|
921
|
+
const record = await this.metadataStore.getRequest(key);
|
|
922
|
+
if (!record) {
|
|
923
|
+
throw new ProcessError(NOT_FOUND, `Operation receipt not found for request ID: ${requestId}`);
|
|
924
|
+
}
|
|
925
|
+
return record.receipt;
|
|
926
|
+
}
|
|
927
|
+
/**
|
|
928
|
+
* 按游标读取受管进程输出流。
|
|
929
|
+
*/
|
|
930
|
+
async read(owner, id, input, call) {
|
|
931
|
+
await this.ensureInitialized();
|
|
932
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
933
|
+
if (proc.outputUnavailable) {
|
|
934
|
+
const requestedPos = parseCursor(input.cursor, proc.info.hostEpoch, id);
|
|
935
|
+
const tombstone = proc.outputTombstone ?? this.evictedOutputTombstones.get(id);
|
|
936
|
+
if (tombstone) {
|
|
937
|
+
const tailPos = parseCursor(tombstone.tailCursor, proc.info.hostEpoch, id);
|
|
938
|
+
const cmp = compareCursorPos(requestedPos, tailPos);
|
|
939
|
+
if (cmp > 0) {
|
|
940
|
+
throw new ProcessError(INVALID_CURSOR, "Requested cursor is beyond the end of the output log", {
|
|
941
|
+
cursor: input.cursor,
|
|
942
|
+
tailCursor: tombstone.tailCursor,
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
if (cmp === 0) {
|
|
946
|
+
return {
|
|
947
|
+
chunks: [],
|
|
948
|
+
nextCursor: tombstone.tailCursor,
|
|
949
|
+
earliestCursor: tombstone.tailCursor,
|
|
950
|
+
tailCursor: tombstone.tailCursor,
|
|
951
|
+
truncated: false,
|
|
952
|
+
eof: true,
|
|
953
|
+
process: { ...proc.info },
|
|
954
|
+
};
|
|
955
|
+
}
|
|
956
|
+
const gapMode = input.onGap ?? "error";
|
|
957
|
+
if (gapMode === "error") {
|
|
958
|
+
throw new ProcessError(OUTPUT_UNAVAILABLE, `Process output for '${id}' is unavailable because it has been evicted from memory`, { processId: id });
|
|
959
|
+
}
|
|
960
|
+
return {
|
|
961
|
+
chunks: [],
|
|
962
|
+
nextCursor: tombstone.tailCursor,
|
|
963
|
+
earliestCursor: tombstone.tailCursor,
|
|
964
|
+
tailCursor: tombstone.tailCursor,
|
|
965
|
+
truncated: true,
|
|
966
|
+
gap: { fromCursor: input.cursor, toCursor: tombstone.tailCursor },
|
|
967
|
+
eof: true,
|
|
968
|
+
process: { ...proc.info },
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
throw new ProcessError(OUTPUT_UNAVAILABLE, `Process output for '${id}' is unavailable because it has been evicted from memory`, { processId: id });
|
|
972
|
+
}
|
|
973
|
+
const logResult = await proc.outputLog.waitForData(input.cursor, input.waitMs, call?.signal, {
|
|
974
|
+
maxBytes: input.maxBytes,
|
|
975
|
+
onGap: input.onGap,
|
|
976
|
+
});
|
|
977
|
+
const chunks = logResult.chunks.map((chunk) => ({
|
|
978
|
+
stream: chunk.stream,
|
|
979
|
+
data: encodeBytes(chunk.data),
|
|
980
|
+
}));
|
|
981
|
+
return {
|
|
982
|
+
chunks,
|
|
983
|
+
nextCursor: logResult.nextCursor,
|
|
984
|
+
earliestCursor: logResult.earliestCursor,
|
|
985
|
+
tailCursor: logResult.tailCursor,
|
|
986
|
+
truncated: logResult.truncated,
|
|
987
|
+
gap: logResult.gap,
|
|
988
|
+
eof: logResult.eof,
|
|
989
|
+
process: { ...proc.info },
|
|
990
|
+
};
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* 独立鉴权紧急终止通道,强行回收资源。
|
|
994
|
+
*/
|
|
995
|
+
async stop(owner, id, input, call) {
|
|
996
|
+
await this.ensureInitialized();
|
|
997
|
+
const proc = await this.getOrLoadProcess(owner, id);
|
|
998
|
+
// 幂等返回已终止状态
|
|
999
|
+
if (proc.info.state === "exited" ||
|
|
1000
|
+
proc.info.state === "failed" ||
|
|
1001
|
+
proc.info.state === "lost") {
|
|
1002
|
+
return { ...proc.info };
|
|
1003
|
+
}
|
|
1004
|
+
// 撤销控制权与定时器
|
|
1005
|
+
if (proc.ttlTimer) {
|
|
1006
|
+
clearTimeout(proc.ttlTimer);
|
|
1007
|
+
proc.ttlTimer = undefined;
|
|
1008
|
+
}
|
|
1009
|
+
proc.currentGrant = undefined;
|
|
1010
|
+
if (proc.idleTimer) {
|
|
1011
|
+
clearTimeout(proc.idleTimer);
|
|
1012
|
+
proc.idleTimer = undefined;
|
|
1013
|
+
}
|
|
1014
|
+
if (proc.lifetimeTimer) {
|
|
1015
|
+
clearTimeout(proc.lifetimeTimer);
|
|
1016
|
+
proc.lifetimeTimer = undefined;
|
|
1017
|
+
}
|
|
1018
|
+
if (proc.drainTimer) {
|
|
1019
|
+
clearTimeout(proc.drainTimer);
|
|
1020
|
+
proc.drainTimer = undefined;
|
|
1021
|
+
}
|
|
1022
|
+
// 递增取消纪元:接管输入队列,使挂起中的 dispatch 放弃过期结算
|
|
1023
|
+
proc.cancelEpoch = (proc.cancelEpoch ?? 0) + 1;
|
|
1024
|
+
// 取消待 dispatch 输入
|
|
1025
|
+
for (const op of proc.inputQueue) {
|
|
1026
|
+
op.receipt.state = "failed";
|
|
1027
|
+
op.receipt.errorCode = PROCESS_CANCELLED;
|
|
1028
|
+
await this.persistReceipt(op.key, op.receipt, op.payloadHash);
|
|
1029
|
+
}
|
|
1030
|
+
proc.inputQueue = [];
|
|
1031
|
+
proc.pendingInputBytes = 0;
|
|
1032
|
+
// 拒绝排队等待者
|
|
1033
|
+
while (proc.acquireWaiters.length > 0) {
|
|
1034
|
+
const waiter = proc.acquireWaiters.shift();
|
|
1035
|
+
waiter.reject(new ProcessError(CONTROL_REVOKED, "Process was stopped"));
|
|
1036
|
+
}
|
|
1037
|
+
proc.info.endReason = proc.info.endReason ?? "requested";
|
|
1038
|
+
proc.info.state = "stopping";
|
|
1039
|
+
proc.info.control = "closed";
|
|
1040
|
+
await this.persistState(id, {
|
|
1041
|
+
state: proc.info.state,
|
|
1042
|
+
control: "closed",
|
|
1043
|
+
controlState: "closed",
|
|
1044
|
+
endReason: proc.info.endReason,
|
|
1045
|
+
});
|
|
1046
|
+
// 调用底层驱动终止;真实退出状态与输出流关闭由底层观察者事件收敛
|
|
1047
|
+
try {
|
|
1048
|
+
if (proc.handle) {
|
|
1049
|
+
await proc.handle.terminate(input.graceMs);
|
|
1050
|
+
}
|
|
1051
|
+
else if (this.driver.terminate) {
|
|
1052
|
+
await this.driver.terminate(id, input.graceMs);
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
catch (err) {
|
|
1056
|
+
this.recordDiagnostic(`Failed to terminate process '${id}' during stop`, err);
|
|
1057
|
+
}
|
|
1058
|
+
return { ...proc.info };
|
|
1059
|
+
}
|
|
1060
|
+
/**
|
|
1061
|
+
* 将进程置入隔离状态,撤销有效控制权并取消待 dispatch 的输入队列。
|
|
1062
|
+
*/
|
|
1063
|
+
async quarantineProcess(owner, processId, token, reason) {
|
|
1064
|
+
const proc = this.processes.get(processId);
|
|
1065
|
+
if (!proc) {
|
|
1066
|
+
return;
|
|
1067
|
+
}
|
|
1068
|
+
checkOwnerAuthorized(owner, proc.owner);
|
|
1069
|
+
if (token && proc.currentGrant && proc.currentGrant.token !== token) {
|
|
1070
|
+
return;
|
|
1071
|
+
}
|
|
1072
|
+
if (proc.info.control === "quarantined" || proc.info.control === "closed") {
|
|
1073
|
+
return;
|
|
1074
|
+
}
|
|
1075
|
+
proc.info.control = "quarantined";
|
|
1076
|
+
if (proc.ttlTimer) {
|
|
1077
|
+
clearTimeout(proc.ttlTimer);
|
|
1078
|
+
proc.ttlTimer = undefined;
|
|
1079
|
+
}
|
|
1080
|
+
proc.currentGrant = undefined;
|
|
1081
|
+
// 递增取消纪元:接管输入队列,使挂起中的 dispatch 放弃过期结算
|
|
1082
|
+
proc.cancelEpoch = (proc.cancelEpoch ?? 0) + 1;
|
|
1083
|
+
// 取消队列中所有待 dispatch 操作
|
|
1084
|
+
for (const op of proc.inputQueue) {
|
|
1085
|
+
op.receipt.state = "failed";
|
|
1086
|
+
op.receipt.errorCode = CONTROL_REVOKED;
|
|
1087
|
+
await this.persistReceipt(op.key, op.receipt, op.payloadHash);
|
|
1088
|
+
}
|
|
1089
|
+
proc.inputQueue = [];
|
|
1090
|
+
proc.pendingInputBytes = 0;
|
|
1091
|
+
// 拒绝排队等待者
|
|
1092
|
+
while (proc.acquireWaiters.length > 0) {
|
|
1093
|
+
const waiter = proc.acquireWaiters.shift();
|
|
1094
|
+
waiter.reject(new ProcessError(PROCESS_QUARANTINED, reason || "Process entered quarantined state"));
|
|
1095
|
+
}
|
|
1096
|
+
await this.persistState(processId, {
|
|
1097
|
+
control: "quarantined",
|
|
1098
|
+
controlState: "quarantined",
|
|
1099
|
+
});
|
|
1100
|
+
}
|
|
1101
|
+
/**
|
|
1102
|
+
* 一次性运行外部命令,收集有限输出并支持协作式取消与超时回收。
|
|
1103
|
+
*/
|
|
1104
|
+
async run(owner, input, call) {
|
|
1105
|
+
checkOwnerAuthorized(owner, owner);
|
|
1106
|
+
await this.ensureInitialized();
|
|
1107
|
+
this.assertNotShutdown();
|
|
1108
|
+
const ioMode = input.spec.io?.mode ?? "pipe";
|
|
1109
|
+
if (ioMode !== "pipe") {
|
|
1110
|
+
throw new ProcessError(UNSUPPORTED_CAPABILITY, "Run requires IO mode to be 'pipe'");
|
|
1111
|
+
}
|
|
1112
|
+
const effectiveSpec = {
|
|
1113
|
+
...input.spec,
|
|
1114
|
+
io: {
|
|
1115
|
+
...input.spec.io,
|
|
1116
|
+
mode: "pipe",
|
|
1117
|
+
},
|
|
1118
|
+
};
|
|
1119
|
+
const runProcessId = `run-${randomUUID()}`;
|
|
1120
|
+
const chunks = [];
|
|
1121
|
+
let totalBytes = 0;
|
|
1122
|
+
let truncated = false;
|
|
1123
|
+
let timedOut = false;
|
|
1124
|
+
let cancelled = false;
|
|
1125
|
+
let resolveExit;
|
|
1126
|
+
let rejectError;
|
|
1127
|
+
const exitPromise = new Promise((resolve, reject) => {
|
|
1128
|
+
resolveExit = resolve;
|
|
1129
|
+
rejectError = reject;
|
|
1130
|
+
});
|
|
1131
|
+
let driverHandle;
|
|
1132
|
+
const callbacks = {
|
|
1133
|
+
onOutput: (stream, data) => {
|
|
1134
|
+
if (truncated)
|
|
1135
|
+
return;
|
|
1136
|
+
if (stream !== "stdout" && stream !== "stderr")
|
|
1137
|
+
return;
|
|
1138
|
+
const remaining = input.maxOutputBytes - totalBytes;
|
|
1139
|
+
if (remaining <= 0) {
|
|
1140
|
+
truncated = true;
|
|
1141
|
+
if (driverHandle) {
|
|
1142
|
+
void driverHandle.terminate(1000);
|
|
1143
|
+
}
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
if (data.byteLength > remaining) {
|
|
1147
|
+
const slice = data.subarray(0, remaining);
|
|
1148
|
+
chunks.push({
|
|
1149
|
+
stream,
|
|
1150
|
+
data: encodeBytes(slice),
|
|
1151
|
+
});
|
|
1152
|
+
totalBytes += slice.byteLength;
|
|
1153
|
+
truncated = true;
|
|
1154
|
+
if (driverHandle) {
|
|
1155
|
+
void driverHandle.terminate(1000);
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
else {
|
|
1159
|
+
chunks.push({
|
|
1160
|
+
stream,
|
|
1161
|
+
data: encodeBytes(data),
|
|
1162
|
+
});
|
|
1163
|
+
totalBytes += data.byteLength;
|
|
1164
|
+
}
|
|
1165
|
+
},
|
|
1166
|
+
onExit: (exit) => {
|
|
1167
|
+
resolveExit(exit);
|
|
1168
|
+
},
|
|
1169
|
+
onError: (err) => {
|
|
1170
|
+
rejectError(err);
|
|
1171
|
+
},
|
|
1172
|
+
};
|
|
1173
|
+
// 对齐 start 防御顺序:派生前检查取消信号,已中止直接抛出取消错误不再派生进程
|
|
1174
|
+
if (call?.signal?.aborted) {
|
|
1175
|
+
throw call.signal.reason instanceof ProcessError
|
|
1176
|
+
? call.signal.reason
|
|
1177
|
+
: new ProcessError(PROCESS_CANCELLED, "Process run was cancelled");
|
|
1178
|
+
}
|
|
1179
|
+
driverHandle = await this.driver.spawn(runProcessId, effectiveSpec, callbacks);
|
|
1180
|
+
let timeoutTimer;
|
|
1181
|
+
if (input.timeoutMs > 0) {
|
|
1182
|
+
timeoutTimer = setTimeout(() => {
|
|
1183
|
+
timedOut = true;
|
|
1184
|
+
if (driverHandle) {
|
|
1185
|
+
void driverHandle.terminate(1000);
|
|
1186
|
+
}
|
|
1187
|
+
}, input.timeoutMs);
|
|
1188
|
+
if (typeof timeoutTimer?.unref === "function") {
|
|
1189
|
+
timeoutTimer.unref();
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
let onAbort;
|
|
1193
|
+
if (call?.signal) {
|
|
1194
|
+
if (call.signal.aborted) {
|
|
1195
|
+
if (timeoutTimer)
|
|
1196
|
+
clearTimeout(timeoutTimer);
|
|
1197
|
+
if (driverHandle)
|
|
1198
|
+
void driverHandle.terminate(1000);
|
|
1199
|
+
throw call.signal.reason ?? new ProcessError(PROCESS_CANCELLED, "Process run was cancelled");
|
|
1200
|
+
}
|
|
1201
|
+
onAbort = () => {
|
|
1202
|
+
cancelled = true;
|
|
1203
|
+
if (timeoutTimer)
|
|
1204
|
+
clearTimeout(timeoutTimer);
|
|
1205
|
+
if (driverHandle)
|
|
1206
|
+
void driverHandle.terminate(1000);
|
|
1207
|
+
};
|
|
1208
|
+
call.signal.addEventListener("abort", onAbort, { once: true });
|
|
1209
|
+
}
|
|
1210
|
+
try {
|
|
1211
|
+
const exit = await exitPromise;
|
|
1212
|
+
if (timeoutTimer)
|
|
1213
|
+
clearTimeout(timeoutTimer);
|
|
1214
|
+
if (call?.signal && onAbort) {
|
|
1215
|
+
call.signal.removeEventListener("abort", onAbort);
|
|
1216
|
+
}
|
|
1217
|
+
if (timedOut) {
|
|
1218
|
+
throw new ProcessError(PROCESS_TIMEOUT, `Process exceeded timeout of ${input.timeoutMs}ms`);
|
|
1219
|
+
}
|
|
1220
|
+
if (cancelled || call?.signal?.aborted) {
|
|
1221
|
+
throw call?.signal?.reason ?? new ProcessError(PROCESS_CANCELLED, "Process run was cancelled");
|
|
1222
|
+
}
|
|
1223
|
+
return {
|
|
1224
|
+
exit,
|
|
1225
|
+
chunks,
|
|
1226
|
+
truncated,
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
catch (err) {
|
|
1230
|
+
if (timeoutTimer)
|
|
1231
|
+
clearTimeout(timeoutTimer);
|
|
1232
|
+
if (call?.signal && onAbort) {
|
|
1233
|
+
call.signal.removeEventListener("abort", onAbort);
|
|
1234
|
+
}
|
|
1235
|
+
if (timedOut) {
|
|
1236
|
+
throw new ProcessError(PROCESS_TIMEOUT, `Process exceeded timeout of ${input.timeoutMs}ms`);
|
|
1237
|
+
}
|
|
1238
|
+
if (cancelled || call?.signal?.aborted) {
|
|
1239
|
+
throw call?.signal?.reason ?? new ProcessError(PROCESS_CANCELLED, "Process run was cancelled");
|
|
1240
|
+
}
|
|
1241
|
+
throw err;
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
/**
|
|
1245
|
+
* 授予指定受管进程控制令牌。
|
|
1246
|
+
*/
|
|
1247
|
+
grantControl(proc, requestId, ttlMs, runId) {
|
|
1248
|
+
proc.controlEpoch += 1;
|
|
1249
|
+
const token = `tok_${proc.info.id}_${proc.controlEpoch}_${randomUUID().replace(/-/g, "")}`;
|
|
1250
|
+
const expiresAt = new Date(Date.now() + ttlMs).toISOString();
|
|
1251
|
+
proc.info.control = "held";
|
|
1252
|
+
proc.currentGrant = {
|
|
1253
|
+
token,
|
|
1254
|
+
epoch: proc.controlEpoch,
|
|
1255
|
+
ttlMs,
|
|
1256
|
+
expiresAt,
|
|
1257
|
+
runId,
|
|
1258
|
+
requestId,
|
|
1259
|
+
};
|
|
1260
|
+
proc.ttlTimer = setTimeout(() => {
|
|
1261
|
+
this.handleGrantTtlExpired(proc);
|
|
1262
|
+
}, ttlMs);
|
|
1263
|
+
if (typeof proc.ttlTimer.unref === "function") {
|
|
1264
|
+
proc.ttlTimer.unref();
|
|
1265
|
+
}
|
|
1266
|
+
void this.persistState(proc.info.id, {
|
|
1267
|
+
control: "held",
|
|
1268
|
+
controlState: "held",
|
|
1269
|
+
});
|
|
1270
|
+
return {
|
|
1271
|
+
token,
|
|
1272
|
+
expiresAt,
|
|
1273
|
+
};
|
|
1274
|
+
}
|
|
1275
|
+
/**
|
|
1276
|
+
* 控制权到期未释放时自动转为隔离状态。
|
|
1277
|
+
*/
|
|
1278
|
+
handleGrantTtlExpired(proc) {
|
|
1279
|
+
if (proc.info.control !== "held" || !proc.currentGrant) {
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1282
|
+
void this.quarantineProcess(proc.owner, proc.info.id, proc.currentGrant.token, "Control token TTL expired");
|
|
1283
|
+
}
|
|
1284
|
+
/**
|
|
1285
|
+
* 唤醒排队等待控制权的下一个调用者。
|
|
1286
|
+
*/
|
|
1287
|
+
wakeNextAcquireWaiter(proc) {
|
|
1288
|
+
if (proc.acquireWaiters.length === 0) {
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
const next = proc.acquireWaiters.shift();
|
|
1292
|
+
const grant = this.grantControl(proc, next.requestId, next.ttlMs, next.runId);
|
|
1293
|
+
const key = {
|
|
1294
|
+
hostEpoch: this.hostEpoch,
|
|
1295
|
+
scope: proc.scope,
|
|
1296
|
+
processId: proc.info.id,
|
|
1297
|
+
requestId: next.requestId,
|
|
1298
|
+
};
|
|
1299
|
+
const payloadHash = hashRequestPayload({ waitMs: next.waitMs, ttlMs: next.ttlMs });
|
|
1300
|
+
void this.persistReceipt(key, grant, payloadHash);
|
|
1301
|
+
next.resolve(grant);
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* 串行调度执行输入队列操作。
|
|
1305
|
+
*/
|
|
1306
|
+
async dispatchNext(proc) {
|
|
1307
|
+
if (proc.isDispatching || proc.inputQueue.length === 0) {
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
1310
|
+
proc.isDispatching = true;
|
|
1311
|
+
try {
|
|
1312
|
+
while (proc.inputQueue.length > 0) {
|
|
1313
|
+
const op = proc.inputQueue[0];
|
|
1314
|
+
// dispatch 前校验 token、epoch 与授权
|
|
1315
|
+
if (proc.info.control !== "held" ||
|
|
1316
|
+
!proc.currentGrant ||
|
|
1317
|
+
proc.currentGrant.token !== op.token ||
|
|
1318
|
+
new Date(proc.currentGrant.expiresAt).getTime() <= Date.now() ||
|
|
1319
|
+
op.cancelEpoch !== proc.cancelEpoch) {
|
|
1320
|
+
if (op.cancelEpoch === proc.cancelEpoch) {
|
|
1321
|
+
op.receipt.state = "failed";
|
|
1322
|
+
op.receipt.errorCode =
|
|
1323
|
+
proc.info.control === "quarantined" ? PROCESS_QUARANTINED : CONTROL_REVOKED;
|
|
1324
|
+
await this.persistReceipt(op.key, op.receipt, op.payloadHash);
|
|
1325
|
+
}
|
|
1326
|
+
proc.inputQueue.shift();
|
|
1327
|
+
if (op.bytes && op.cancelEpoch === proc.cancelEpoch) {
|
|
1328
|
+
proc.pendingInputBytes -= op.bytes.byteLength;
|
|
1329
|
+
}
|
|
1330
|
+
continue;
|
|
1331
|
+
}
|
|
1332
|
+
op.receipt.state = "dispatching";
|
|
1333
|
+
await this.persistReceipt(op.key, op.receipt, op.payloadHash);
|
|
1334
|
+
if (op.type === "write" && op.bytes) {
|
|
1335
|
+
try {
|
|
1336
|
+
if (!proc.handle) {
|
|
1337
|
+
throw new Error("Missing driver handle");
|
|
1338
|
+
}
|
|
1339
|
+
await proc.handle.write(op.bytes);
|
|
1340
|
+
// 写入返回后校验取消纪元与队首位置:已被 stop 或 quarantine 接管则放弃过期结算
|
|
1341
|
+
if (op.cancelEpoch !== proc.cancelEpoch || proc.inputQueue[0] !== op) {
|
|
1342
|
+
continue;
|
|
1343
|
+
}
|
|
1344
|
+
op.receipt.state = "completed";
|
|
1345
|
+
op.receipt.acceptedBytes = op.bytes.byteLength;
|
|
1346
|
+
this.refreshIdleTimer(proc);
|
|
1347
|
+
}
|
|
1348
|
+
catch (err) {
|
|
1349
|
+
// 写入返回异常时同样校验取消纪元:被接管则放弃过期结算
|
|
1350
|
+
if (op.cancelEpoch !== proc.cancelEpoch || proc.inputQueue[0] !== op) {
|
|
1351
|
+
continue;
|
|
1352
|
+
}
|
|
1353
|
+
// 若写入出现不确定失败,结果标记为 unknown,进程转入 quarantined
|
|
1354
|
+
op.receipt.state = "unknown";
|
|
1355
|
+
op.receipt.errorCode = INPUT_OUTCOME_UNKNOWN;
|
|
1356
|
+
await this.persistReceipt(op.key, op.receipt, op.payloadHash);
|
|
1357
|
+
proc.inputQueue.shift();
|
|
1358
|
+
proc.pendingInputBytes -= op.bytes.byteLength;
|
|
1359
|
+
await this.quarantineProcess(proc.owner, proc.info.id, op.token, "Write failed with uncertain outcome");
|
|
1360
|
+
break;
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
else if (op.type === "control" && op.action) {
|
|
1364
|
+
try {
|
|
1365
|
+
if (!proc.handle) {
|
|
1366
|
+
throw new Error("Missing driver handle");
|
|
1367
|
+
}
|
|
1368
|
+
if (op.action.type === "input-eof") {
|
|
1369
|
+
if (proc.handle.sendInputEOF) {
|
|
1370
|
+
await proc.handle.sendInputEOF();
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
else if (op.action.type === "interrupt-foreground") {
|
|
1374
|
+
if (proc.handle.interruptForeground) {
|
|
1375
|
+
await proc.handle.interruptForeground();
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
else if (op.action.type === "resize") {
|
|
1379
|
+
if (proc.handle.resize) {
|
|
1380
|
+
await proc.handle.resize(op.action.cols, op.action.rows);
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
// 控制指令返回后校验取消纪元与队首位置:已被接管则放弃过期结算
|
|
1384
|
+
if (op.cancelEpoch !== proc.cancelEpoch || proc.inputQueue[0] !== op) {
|
|
1385
|
+
continue;
|
|
1386
|
+
}
|
|
1387
|
+
if (op.action.type === "input-eof") {
|
|
1388
|
+
proc.inputClosed = true;
|
|
1389
|
+
await this.persistState(proc.info.id, { inputClosed: true });
|
|
1390
|
+
}
|
|
1391
|
+
op.receipt.state = "completed";
|
|
1392
|
+
this.refreshIdleTimer(proc);
|
|
1393
|
+
}
|
|
1394
|
+
catch (err) {
|
|
1395
|
+
if (op.cancelEpoch !== proc.cancelEpoch || proc.inputQueue[0] !== op) {
|
|
1396
|
+
continue;
|
|
1397
|
+
}
|
|
1398
|
+
op.receipt.state = "failed";
|
|
1399
|
+
op.receipt.errorCode = err instanceof ProcessError ? err.code : "CONTROL_FAILED";
|
|
1400
|
+
if (err instanceof Error && err.message) {
|
|
1401
|
+
op.receipt.errorMessage = err.message;
|
|
1402
|
+
}
|
|
1403
|
+
this.recordDiagnostic(`Control action '${op.action.type}' failed for request '${op.requestId}'`, err);
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
// 结算前再次校验:若结算窗口内被 stop 或 quarantine 接管则放弃改写
|
|
1407
|
+
if (op.cancelEpoch !== proc.cancelEpoch || proc.inputQueue[0] !== op) {
|
|
1408
|
+
continue;
|
|
1409
|
+
}
|
|
1410
|
+
await this.persistReceipt(op.key, op.receipt, op.payloadHash);
|
|
1411
|
+
proc.inputQueue.shift();
|
|
1412
|
+
if (op.bytes) {
|
|
1413
|
+
proc.pendingInputBytes -= op.bytes.byteLength;
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
finally {
|
|
1418
|
+
proc.isDispatching = false;
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
* 操作提交前校验持有者令牌与授权。
|
|
1423
|
+
*/
|
|
1424
|
+
checkOperationAuth(proc, token) {
|
|
1425
|
+
if (proc.info.control === "quarantined") {
|
|
1426
|
+
throw new ProcessError(PROCESS_QUARANTINED, "Process is in quarantined state");
|
|
1427
|
+
}
|
|
1428
|
+
if (proc.info.control !== "held" || !proc.currentGrant) {
|
|
1429
|
+
throw new ProcessError(ACCESS_DENIED, "Process control is not currently held");
|
|
1430
|
+
}
|
|
1431
|
+
if (proc.currentGrant.token !== token) {
|
|
1432
|
+
throw new ProcessError(ACCESS_DENIED, "Invalid control token");
|
|
1433
|
+
}
|
|
1434
|
+
if (new Date(proc.currentGrant.expiresAt).getTime() <= Date.now()) {
|
|
1435
|
+
throw new ProcessError(CONTROL_EXPIRED, "Control token has expired");
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
/**
|
|
1439
|
+
* 启动空闲超时定时器。
|
|
1440
|
+
*/
|
|
1441
|
+
startIdleTimer(proc) {
|
|
1442
|
+
if (proc.effectiveLimits.idleMs <= 0) {
|
|
1443
|
+
return;
|
|
1444
|
+
}
|
|
1445
|
+
if (proc.idleTimer) {
|
|
1446
|
+
clearTimeout(proc.idleTimer);
|
|
1447
|
+
}
|
|
1448
|
+
proc.idleTimer = setTimeout(() => {
|
|
1449
|
+
this.handleIdleTimeout(proc);
|
|
1450
|
+
}, proc.effectiveLimits.idleMs);
|
|
1451
|
+
if (typeof proc.idleTimer.unref === "function") {
|
|
1452
|
+
proc.idleTimer.unref();
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
/**
|
|
1456
|
+
* 刷新空闲超时定时器(仅在成功输入与续租时刷新,输出不刷新)。
|
|
1457
|
+
*/
|
|
1458
|
+
refreshIdleTimer(proc) {
|
|
1459
|
+
if (proc.info.state === "exited" ||
|
|
1460
|
+
proc.info.state === "failed" ||
|
|
1461
|
+
proc.info.state === "lost") {
|
|
1462
|
+
return;
|
|
1463
|
+
}
|
|
1464
|
+
this.startIdleTimer(proc);
|
|
1465
|
+
}
|
|
1466
|
+
/**
|
|
1467
|
+
* 启动硬性存活上限定时器。
|
|
1468
|
+
*/
|
|
1469
|
+
startLifetimeTimer(proc) {
|
|
1470
|
+
if (proc.effectiveLimits.lifetimeMs <= 0) {
|
|
1471
|
+
return;
|
|
1472
|
+
}
|
|
1473
|
+
proc.lifetimeTimer = setTimeout(() => {
|
|
1474
|
+
this.handleLifetimeTimeout(proc);
|
|
1475
|
+
}, proc.effectiveLimits.lifetimeMs);
|
|
1476
|
+
if (typeof proc.lifetimeTimer.unref === "function") {
|
|
1477
|
+
proc.lifetimeTimer.unref();
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
/**
|
|
1481
|
+
* 处理空闲超时到期。
|
|
1482
|
+
*/
|
|
1483
|
+
handleIdleTimeout(proc) {
|
|
1484
|
+
if (proc.info.state === "exited" ||
|
|
1485
|
+
proc.info.state === "failed" ||
|
|
1486
|
+
proc.info.state === "lost") {
|
|
1487
|
+
return;
|
|
1488
|
+
}
|
|
1489
|
+
proc.info.endReason = "idle";
|
|
1490
|
+
this.stop(proc.owner, proc.info.id, {
|
|
1491
|
+
requestId: `idle-${randomUUID()}`,
|
|
1492
|
+
graceMs: 1000,
|
|
1493
|
+
}).catch((err) => {
|
|
1494
|
+
this.recordDiagnostic(`Idle timeout stop failed for process '${proc.info.id}'`, err);
|
|
1495
|
+
});
|
|
1496
|
+
}
|
|
1497
|
+
/**
|
|
1498
|
+
* 处理存活上限到期。
|
|
1499
|
+
*/
|
|
1500
|
+
handleLifetimeTimeout(proc) {
|
|
1501
|
+
if (proc.info.state === "exited" ||
|
|
1502
|
+
proc.info.state === "failed" ||
|
|
1503
|
+
proc.info.state === "lost") {
|
|
1504
|
+
return;
|
|
1505
|
+
}
|
|
1506
|
+
proc.info.endReason = "lifetime";
|
|
1507
|
+
this.stop(proc.owner, proc.info.id, {
|
|
1508
|
+
requestId: `lifetime-${randomUUID()}`,
|
|
1509
|
+
graceMs: 1000,
|
|
1510
|
+
}).catch((err) => {
|
|
1511
|
+
this.recordDiagnostic(`Lifetime timeout stop failed for process '${proc.info.id}'`, err);
|
|
1512
|
+
});
|
|
1513
|
+
}
|
|
1514
|
+
/**
|
|
1515
|
+
* 处理驱动通知的自然退出或信号退出。
|
|
1516
|
+
*/
|
|
1517
|
+
handleProcessExit(proc, exit) {
|
|
1518
|
+
if (proc.info.state === "exited" ||
|
|
1519
|
+
proc.info.state === "failed" ||
|
|
1520
|
+
proc.info.state === "lost") {
|
|
1521
|
+
return;
|
|
1522
|
+
}
|
|
1523
|
+
if (proc.idleTimer)
|
|
1524
|
+
clearTimeout(proc.idleTimer);
|
|
1525
|
+
if (proc.lifetimeTimer)
|
|
1526
|
+
clearTimeout(proc.lifetimeTimer);
|
|
1527
|
+
if (proc.ttlTimer)
|
|
1528
|
+
clearTimeout(proc.ttlTimer);
|
|
1529
|
+
proc.info.state = "exited";
|
|
1530
|
+
proc.info.control = "closed";
|
|
1531
|
+
proc.info.exit = { code: exit.code, signal: exit.signal };
|
|
1532
|
+
if (!proc.info.endReason) {
|
|
1533
|
+
proc.info.endReason = "natural";
|
|
1534
|
+
}
|
|
1535
|
+
proc.currentGrant = undefined;
|
|
1536
|
+
// 拒绝排队等待者
|
|
1537
|
+
while (proc.acquireWaiters.length > 0) {
|
|
1538
|
+
const waiter = proc.acquireWaiters.shift();
|
|
1539
|
+
waiter.reject(new ProcessError(CONTROL_REVOKED, "Process has exited"));
|
|
1540
|
+
}
|
|
1541
|
+
void this.persistState(proc.info.id, {
|
|
1542
|
+
state: "exited",
|
|
1543
|
+
control: "closed",
|
|
1544
|
+
controlState: "closed",
|
|
1545
|
+
endReason: proc.info.endReason,
|
|
1546
|
+
exitCode: exit.code,
|
|
1547
|
+
exitSignal: exit.signal,
|
|
1548
|
+
});
|
|
1549
|
+
// 启动 drain 宽限期
|
|
1550
|
+
proc.drainTimer = setTimeout(() => {
|
|
1551
|
+
if (!proc.outputLog.outputClosed) {
|
|
1552
|
+
proc.outputLog.closeOutput("drain-timeout");
|
|
1553
|
+
proc.info.outputClosed = true;
|
|
1554
|
+
proc.info.outputEndReason = "drain-timeout";
|
|
1555
|
+
void this.persistState(proc.info.id, {
|
|
1556
|
+
outputClosed: true,
|
|
1557
|
+
outputEndReason: "drain-timeout",
|
|
1558
|
+
});
|
|
1559
|
+
this.maybeEvictProcess(proc);
|
|
1560
|
+
}
|
|
1561
|
+
}, this.drainDeadlineMs);
|
|
1562
|
+
if (typeof proc.drainTimer.unref === "function") {
|
|
1563
|
+
proc.drainTimer.unref();
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
/**
|
|
1567
|
+
* 处理驱动通知的输出流彻底关闭事件。
|
|
1568
|
+
*/
|
|
1569
|
+
handleOutputClosed(proc, reason = "natural") {
|
|
1570
|
+
if (proc.drainTimer) {
|
|
1571
|
+
clearTimeout(proc.drainTimer);
|
|
1572
|
+
proc.drainTimer = undefined;
|
|
1573
|
+
}
|
|
1574
|
+
if (!proc.outputLog.outputClosed) {
|
|
1575
|
+
proc.outputLog.closeOutput(reason);
|
|
1576
|
+
proc.info.outputClosed = true;
|
|
1577
|
+
proc.info.outputEndReason = reason;
|
|
1578
|
+
void this.persistState(proc.info.id, {
|
|
1579
|
+
outputClosed: true,
|
|
1580
|
+
outputEndReason: reason,
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1583
|
+
this.maybeEvictProcess(proc);
|
|
1584
|
+
}
|
|
1585
|
+
/**
|
|
1586
|
+
* 处理驱动通知的底层故障。
|
|
1587
|
+
*/
|
|
1588
|
+
handleProcessError(proc, err) {
|
|
1589
|
+
if (proc.info.state === "exited" ||
|
|
1590
|
+
proc.info.state === "failed" ||
|
|
1591
|
+
proc.info.state === "lost") {
|
|
1592
|
+
return;
|
|
1593
|
+
}
|
|
1594
|
+
if (proc.idleTimer)
|
|
1595
|
+
clearTimeout(proc.idleTimer);
|
|
1596
|
+
if (proc.lifetimeTimer)
|
|
1597
|
+
clearTimeout(proc.lifetimeTimer);
|
|
1598
|
+
if (proc.ttlTimer)
|
|
1599
|
+
clearTimeout(proc.ttlTimer);
|
|
1600
|
+
proc.info.state = "failed";
|
|
1601
|
+
proc.info.control = "closed";
|
|
1602
|
+
proc.info.endReason = "natural";
|
|
1603
|
+
proc.outputLog.closeOutput("natural");
|
|
1604
|
+
proc.info.outputClosed = true;
|
|
1605
|
+
while (proc.acquireWaiters.length > 0) {
|
|
1606
|
+
const waiter = proc.acquireWaiters.shift();
|
|
1607
|
+
waiter.reject(new ProcessError(CONTROL_REVOKED, `Process error: ${err.message}`));
|
|
1608
|
+
}
|
|
1609
|
+
void this.persistState(proc.info.id, {
|
|
1610
|
+
state: "failed",
|
|
1611
|
+
control: "closed",
|
|
1612
|
+
controlState: "closed",
|
|
1613
|
+
endReason: "natural",
|
|
1614
|
+
outputClosed: true,
|
|
1615
|
+
outputEndReason: "natural",
|
|
1616
|
+
});
|
|
1617
|
+
this.maybeEvictProcess(proc);
|
|
1618
|
+
}
|
|
1619
|
+
/**
|
|
1620
|
+
* 加载或读取受管进程记录并校验所有者鉴权。
|
|
1621
|
+
*/
|
|
1622
|
+
async getOrLoadProcess(owner, processId) {
|
|
1623
|
+
const memoryProc = this.processes.get(processId);
|
|
1624
|
+
if (memoryProc) {
|
|
1625
|
+
checkOwnerAuthorized(owner, memoryProc.owner);
|
|
1626
|
+
return memoryProc;
|
|
1627
|
+
}
|
|
1628
|
+
const record = await this.metadataStore.getProcess(processId);
|
|
1629
|
+
if (!record) {
|
|
1630
|
+
throw new ProcessError(NOT_FOUND, `Process '${processId}' not found`, { processId });
|
|
1631
|
+
}
|
|
1632
|
+
checkOwnerAuthorized(owner, record);
|
|
1633
|
+
// 旧纪元非终态记录:宿主已丢失,拒绝后续操作并标记 lost
|
|
1634
|
+
if (record.hostEpoch !== this.hostEpoch &&
|
|
1635
|
+
record.state !== "exited" &&
|
|
1636
|
+
record.state !== "failed" &&
|
|
1637
|
+
record.state !== "lost") {
|
|
1638
|
+
await this.metadataStore.updateProcessState(processId, {
|
|
1639
|
+
state: "lost",
|
|
1640
|
+
controlState: "closed",
|
|
1641
|
+
control: "closed",
|
|
1642
|
+
endReason: "host-lost",
|
|
1643
|
+
outputClosed: true,
|
|
1644
|
+
outputEndReason: "host-lost",
|
|
1645
|
+
});
|
|
1646
|
+
throw new ProcessError(PROCESS_LOST, `Process '${processId}' belonged to a previous host epoch '${record.hostEpoch}' and has been marked lost; current epoch is '${this.hostEpoch}'`, { processId, hostEpoch: record.hostEpoch, currentHostEpoch: this.hostEpoch });
|
|
1647
|
+
}
|
|
1648
|
+
const info = toSdkProcessInfo(record);
|
|
1649
|
+
const retainedLog = this.getRetainedOutputLog(processId);
|
|
1650
|
+
const tombstone = this.evictedOutputTombstones.get(processId);
|
|
1651
|
+
const isTerminalLoaded = info.state === "exited" || info.state === "failed" || info.state === "lost";
|
|
1652
|
+
const isOutputUnavailable = !retainedLog && (Boolean(tombstone) || (isTerminalLoaded && Boolean(record.outputClosed)));
|
|
1653
|
+
const outputLog = retainedLog ??
|
|
1654
|
+
new ProcessOutputLog(record.hostEpoch, processId, {
|
|
1655
|
+
maxBufferBytes: info.effectiveLimits.outputBufferBytes,
|
|
1656
|
+
maxWaiters: this.quotas.maxWaitersPerProcess,
|
|
1657
|
+
});
|
|
1658
|
+
if (record.outputClosed && !outputLog.outputClosed) {
|
|
1659
|
+
outputLog.closeOutput(record.outputEndReason ?? "natural");
|
|
1660
|
+
}
|
|
1661
|
+
const loaded = {
|
|
1662
|
+
info,
|
|
1663
|
+
owner: {
|
|
1664
|
+
tenantId: record.tenantId,
|
|
1665
|
+
principalId: record.principalId,
|
|
1666
|
+
packageInstanceId: record.packageInstanceId,
|
|
1667
|
+
generationId: record.generationId,
|
|
1668
|
+
},
|
|
1669
|
+
scope: formatProcessScope(owner),
|
|
1670
|
+
outputLog,
|
|
1671
|
+
outputUnavailable: isOutputUnavailable,
|
|
1672
|
+
outputTombstone: tombstone,
|
|
1673
|
+
controlEpoch: 0,
|
|
1674
|
+
cancelEpoch: 0,
|
|
1675
|
+
inputQueue: [],
|
|
1676
|
+
pendingInputBytes: 0,
|
|
1677
|
+
acquireWaiters: [],
|
|
1678
|
+
inputClosed: Boolean(record.inputClosed),
|
|
1679
|
+
isDispatching: false,
|
|
1680
|
+
effectiveLimits: info.effectiveLimits,
|
|
1681
|
+
};
|
|
1682
|
+
// 终态且输出已关闭的记录不再回填内存表,保持终态驱逐语义
|
|
1683
|
+
if (isTerminalLoaded && info.outputClosed) {
|
|
1684
|
+
return loaded;
|
|
1685
|
+
}
|
|
1686
|
+
this.processes.set(processId, loaded);
|
|
1687
|
+
return loaded;
|
|
1688
|
+
}
|
|
1689
|
+
/**
|
|
1690
|
+
* 记录已淘汰输出日志的墓碑信息(保留最近 2048 条,防止内存无限积压)。
|
|
1691
|
+
*/
|
|
1692
|
+
recordTombstone(processId, tombstone) {
|
|
1693
|
+
if (this.evictedOutputTombstones.size >= 2048) {
|
|
1694
|
+
const oldestKey = this.evictedOutputTombstones.keys().next().value;
|
|
1695
|
+
if (oldestKey) {
|
|
1696
|
+
this.evictedOutputTombstones.delete(oldestKey);
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
this.evictedOutputTombstones.set(processId, tombstone);
|
|
1700
|
+
}
|
|
1701
|
+
/**
|
|
1702
|
+
* 清理已过期的终态输出日志条目。
|
|
1703
|
+
*/
|
|
1704
|
+
cleanExpiredTerminalLogs() {
|
|
1705
|
+
const now = Date.now();
|
|
1706
|
+
for (const [id, entry] of this.evictedOutputLogs.entries()) {
|
|
1707
|
+
if (now - entry.evictedAt > this.terminalLogRetentionMs) {
|
|
1708
|
+
this.recordTombstone(id, {
|
|
1709
|
+
tailCursor: entry.log.tailCursor,
|
|
1710
|
+
earliestCursor: entry.log.earliestCursor,
|
|
1711
|
+
evictedAt: now,
|
|
1712
|
+
});
|
|
1713
|
+
this.evictedOutputLogs.delete(id);
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
/**
|
|
1718
|
+
* 统计终态保留日志当前在内存中实际占用的输出缓冲字节总数。
|
|
1719
|
+
*/
|
|
1720
|
+
countRetainedOutputBufferBytes() {
|
|
1721
|
+
this.cleanExpiredTerminalLogs();
|
|
1722
|
+
let bytes = 0;
|
|
1723
|
+
for (const entry of this.evictedOutputLogs.values()) {
|
|
1724
|
+
bytes += entry.log.currentBytes;
|
|
1725
|
+
}
|
|
1726
|
+
return bytes;
|
|
1727
|
+
}
|
|
1728
|
+
/**
|
|
1729
|
+
* 将终态输出日志存入保留缓存,并根据宿主配额执行 LRU 与 TTL 淘汰。
|
|
1730
|
+
*/
|
|
1731
|
+
retainTerminalOutputLog(processId, log) {
|
|
1732
|
+
this.cleanExpiredTerminalLogs();
|
|
1733
|
+
// 若新加入条目会导致总输出配额超限或数量超限,按 LRU 顺序淘汰最旧条目
|
|
1734
|
+
while ((this.countRetainedOutputBufferBytes() + log.currentBytes > this.quotas.maxOutputBufferBytesPerHost ||
|
|
1735
|
+
this.evictedOutputLogs.size >= 512) &&
|
|
1736
|
+
this.evictedOutputLogs.size > 0) {
|
|
1737
|
+
const oldestKey = this.evictedOutputLogs.keys().next().value;
|
|
1738
|
+
if (!oldestKey)
|
|
1739
|
+
break;
|
|
1740
|
+
const oldestEntry = this.evictedOutputLogs.get(oldestKey);
|
|
1741
|
+
if (oldestEntry) {
|
|
1742
|
+
this.recordTombstone(oldestKey, {
|
|
1743
|
+
tailCursor: oldestEntry.log.tailCursor,
|
|
1744
|
+
earliestCursor: oldestEntry.log.earliestCursor,
|
|
1745
|
+
evictedAt: Date.now(),
|
|
1746
|
+
});
|
|
1747
|
+
}
|
|
1748
|
+
this.evictedOutputLogs.delete(oldestKey);
|
|
1749
|
+
}
|
|
1750
|
+
this.evictedOutputLogs.set(processId, {
|
|
1751
|
+
log,
|
|
1752
|
+
evictedAt: Date.now(),
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
/**
|
|
1756
|
+
* 获取终态保留日志(附带过期清理与 LRU 触达更新)。
|
|
1757
|
+
*/
|
|
1758
|
+
getRetainedOutputLog(processId) {
|
|
1759
|
+
this.cleanExpiredTerminalLogs();
|
|
1760
|
+
const entry = this.evictedOutputLogs.get(processId);
|
|
1761
|
+
if (!entry)
|
|
1762
|
+
return undefined;
|
|
1763
|
+
// 触达刷新 LRU 顺序
|
|
1764
|
+
this.evictedOutputLogs.delete(processId);
|
|
1765
|
+
this.evictedOutputLogs.set(processId, entry);
|
|
1766
|
+
return entry.log;
|
|
1767
|
+
}
|
|
1768
|
+
/**
|
|
1769
|
+
* 终态驱逐与驱动释放:进程到达终态且输出已关闭后移除内存记录,
|
|
1770
|
+
* 并防御性释放底层驱动句柄(失败仅记录诊断不中断)。
|
|
1771
|
+
*/
|
|
1772
|
+
maybeEvictProcess(proc) {
|
|
1773
|
+
const isTerminal = proc.info.state === "exited" || proc.info.state === "failed" || proc.info.state === "lost";
|
|
1774
|
+
if (!isTerminal || !proc.info.outputClosed) {
|
|
1775
|
+
return;
|
|
1776
|
+
}
|
|
1777
|
+
// 仍存在排队等待者或未结算输入时暂缓驱逐
|
|
1778
|
+
if (proc.acquireWaiters.length > 0 || proc.inputQueue.length > 0) {
|
|
1779
|
+
return;
|
|
1780
|
+
}
|
|
1781
|
+
this.processes.delete(proc.info.id);
|
|
1782
|
+
// 保留输出日志支撑后续游标读取(内存输出无法从持久层恢复)
|
|
1783
|
+
this.retainTerminalOutputLog(proc.info.id, proc.outputLog);
|
|
1784
|
+
if (proc.handle && typeof this.driver.dispose === "function") {
|
|
1785
|
+
try {
|
|
1786
|
+
const disposed = this.driver.dispose(proc.handle);
|
|
1787
|
+
if (disposed && typeof disposed.catch === "function") {
|
|
1788
|
+
disposed.catch((err) => {
|
|
1789
|
+
this.recordDiagnostic(`Driver dispose failed for process '${proc.info.id}'`, err);
|
|
1790
|
+
});
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
catch (err) {
|
|
1794
|
+
this.recordDiagnostic(`Driver dispose failed for process '${proc.info.id}'`, err);
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
/**
|
|
1799
|
+
* 启动受管进程时的配额容量校验。
|
|
1800
|
+
*/
|
|
1801
|
+
checkSpawnQuotas(scope, requestedBufferBytes) {
|
|
1802
|
+
let scopeActive = 0;
|
|
1803
|
+
let hostActive = 0;
|
|
1804
|
+
let totalBuffer = 0;
|
|
1805
|
+
for (const p of this.processes.values()) {
|
|
1806
|
+
if (p.info.state === "starting" ||
|
|
1807
|
+
p.info.state === "running" ||
|
|
1808
|
+
p.info.state === "stopping") {
|
|
1809
|
+
hostActive += 1;
|
|
1810
|
+
totalBuffer += p.effectiveLimits.outputBufferBytes;
|
|
1811
|
+
if (p.scope === scope) {
|
|
1812
|
+
scopeActive += 1;
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
if (scopeActive >= this.quotas.maxActiveProcessesPerScope) {
|
|
1817
|
+
throw new ProcessError(QUOTA_EXCEEDED, "Scope active process quota exceeded", {
|
|
1818
|
+
scope,
|
|
1819
|
+
limit: this.quotas.maxActiveProcessesPerScope,
|
|
1820
|
+
});
|
|
1821
|
+
}
|
|
1822
|
+
if (hostActive >= this.quotas.maxActiveProcessesPerHost) {
|
|
1823
|
+
throw new ProcessError(QUOTA_EXCEEDED, "Host active process quota exceeded", {
|
|
1824
|
+
limit: this.quotas.maxActiveProcessesPerHost,
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
const perProcBuf = requestedBufferBytes ?? this.defaultLimits.outputBufferBytes;
|
|
1828
|
+
if (perProcBuf > this.quotas.maxOutputBufferBytesPerProcess) {
|
|
1829
|
+
throw new ProcessError(QUOTA_EXCEEDED, "Output buffer per process quota exceeded", {
|
|
1830
|
+
requested: perProcBuf,
|
|
1831
|
+
limit: this.quotas.maxOutputBufferBytesPerProcess,
|
|
1832
|
+
});
|
|
1833
|
+
}
|
|
1834
|
+
// 终态保留输出日志实际占用字节数纳入宿主预算
|
|
1835
|
+
totalBuffer += this.countRetainedOutputBufferBytes();
|
|
1836
|
+
// 若新进程申请的缓冲使总预算超限,优先淘汰已有的终态保留日志(LRU 策略)
|
|
1837
|
+
while (totalBuffer + perProcBuf > this.quotas.maxOutputBufferBytesPerHost &&
|
|
1838
|
+
this.evictedOutputLogs.size > 0) {
|
|
1839
|
+
const oldestKey = this.evictedOutputLogs.keys().next().value;
|
|
1840
|
+
if (!oldestKey)
|
|
1841
|
+
break;
|
|
1842
|
+
const oldest = this.evictedOutputLogs.get(oldestKey);
|
|
1843
|
+
if (oldest) {
|
|
1844
|
+
this.recordTombstone(oldestKey, {
|
|
1845
|
+
tailCursor: oldest.log.tailCursor,
|
|
1846
|
+
earliestCursor: oldest.log.earliestCursor,
|
|
1847
|
+
evictedAt: Date.now(),
|
|
1848
|
+
});
|
|
1849
|
+
totalBuffer -= oldest.log.currentBytes;
|
|
1850
|
+
}
|
|
1851
|
+
this.evictedOutputLogs.delete(oldestKey);
|
|
1852
|
+
}
|
|
1853
|
+
if (totalBuffer + perProcBuf > this.quotas.maxOutputBufferBytesPerHost) {
|
|
1854
|
+
throw new ProcessError(QUOTA_EXCEEDED, "Host output buffer quota exceeded", {
|
|
1855
|
+
limit: this.quotas.maxOutputBufferBytesPerHost,
|
|
1856
|
+
});
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
/**
|
|
1860
|
+
* 统计当前宿主所有活跃进程累计排队等待控制权的调用者总数。
|
|
1861
|
+
*/
|
|
1862
|
+
countHostAcquireWaiters() {
|
|
1863
|
+
let total = 0;
|
|
1864
|
+
for (const p of this.processes.values()) {
|
|
1865
|
+
total += p.acquireWaiters.length;
|
|
1866
|
+
}
|
|
1867
|
+
return total;
|
|
1868
|
+
}
|
|
1869
|
+
/**
|
|
1870
|
+
* 统计当前宿主所有活跃进程累计输入队列待写入字节总数。
|
|
1871
|
+
*/
|
|
1872
|
+
countHostPendingInputBytes() {
|
|
1873
|
+
let total = 0;
|
|
1874
|
+
for (const p of this.processes.values()) {
|
|
1875
|
+
total += p.pendingInputBytes;
|
|
1876
|
+
}
|
|
1877
|
+
return total;
|
|
1878
|
+
}
|
|
1879
|
+
}
|