@zhushanwen/subagent-core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -2
- package/agents/analyst.md +60 -0
- package/agents/coder.md +69 -0
- package/agents/debugger.md +66 -0
- package/agents/doc-reviewer.md +49 -0
- package/agents/explorer.md +63 -0
- package/agents/general-purpose.md +32 -0
- package/agents/orchestrator.md +61 -0
- package/agents/planner.md +53 -0
- package/agents/researcher.md +67 -0
- package/agents/reviewer.md +73 -0
- package/dist/chunk-4KN4TTG7.js +240 -0
- package/dist/chunk-APZY4IME.js +27 -0
- package/dist/chunk-X7SZ5HLQ.js +66 -0
- package/dist/execution/engine/engines/zcode/constants.cjs +65 -0
- package/dist/execution/engine/engines/zcode/constants.d.cts +93 -1
- package/dist/execution/engine/engines/zcode/constants.d.ts +93 -1
- package/dist/execution/engine/engines/zcode/constants.js +43 -1
- package/dist/execution/engine/engines/zcode/reader.d.cts +1 -1
- package/dist/execution/engine/engines/zcode/reader.d.ts +1 -1
- package/dist/execution/engine/engines/zcode/reader.js +4 -234
- package/dist/execution/engine/paths.js +7 -19
- package/dist/index.cjs +14709 -100
- package/dist/index.d.cts +4053 -200
- package/dist/index.d.ts +4053 -200
- package/dist/index.js +14358 -102
- package/dist/{types-BxyAidGf.d.cts → types-DpUO16pj.d.cts} +464 -1
- package/dist/{types-BxyAidGf.d.ts → types-DpUO16pj.d.ts} +464 -1
- package/package.json +6 -3
- package/src/__tests__/review-fix-loop-script.test.ts +90 -2
- package/src/__tests__/review-fix-loop-utils.test.ts +21 -0
- package/src/__tests__/smoke.test.ts +9 -2
- package/src/execution/__tests__/agent-profile.test.ts +232 -0
- package/src/execution/__tests__/agents-assembly.test.ts +219 -0
- package/src/execution/__tests__/chat-engine-routing.test.ts +71 -0
- package/src/execution/__tests__/chatmode-first-round-closure-spawn.test.ts +1 -0
- package/src/execution/__tests__/create-concurrency-pool.test.ts +194 -0
- package/src/execution/__tests__/delivery-methods.test.ts +2 -0
- package/src/execution/__tests__/descendant-sweep.test.ts +269 -0
- package/src/execution/__tests__/dialog-queue.test.ts +199 -1
- package/src/execution/__tests__/epipe-fallback.test.ts +2 -0
- package/src/execution/__tests__/execution-runtime-face.test.ts +272 -0
- package/src/execution/__tests__/finalize-record.test.ts +106 -0
- package/src/execution/__tests__/gc-timer.test.ts +2 -0
- package/src/execution/__tests__/get-record-for-action-restart.test.ts +2 -0
- package/src/execution/__tests__/get-state-handshake.test.ts +127 -0
- package/src/execution/__tests__/keep-alive-no-progress.test.ts +378 -0
- package/src/execution/__tests__/kill-all-escalation.test.ts +196 -0
- package/src/execution/__tests__/lifecycle-manager-idle-timer-identity.test.ts +117 -0
- package/src/execution/__tests__/lifecycle-manager.test.ts +39 -0
- package/src/execution/__tests__/manifest-store-tmp-recovery.test.ts +99 -0
- package/src/execution/__tests__/max-turns-to-watchdog-ms.test.ts +113 -0
- package/src/execution/__tests__/notify-ledger.test.ts +294 -0
- package/src/execution/__tests__/record-store-orphan-revive.test.ts +159 -0
- package/src/execution/__tests__/record-store.test.ts +83 -0
- package/src/execution/__tests__/run-and-finalize-chatmode.test.ts +4 -0
- package/src/execution/__tests__/run-spawn-chatmode-settled.test.ts +1 -0
- package/src/execution/__tests__/run-spawn-edges.test.ts +407 -17
- package/src/execution/__tests__/run-spawn-integration.test.ts +1 -1
- package/src/execution/__tests__/run-spawn-stdout-callback-throw.test.ts +12 -4
- package/src/execution/__tests__/service-kill-escalation.test.ts +89 -0
- package/src/execution/__tests__/session-pending.test.ts +228 -0
- package/src/execution/__tests__/session-runner-branch-cache-lru.test.ts +110 -0
- package/src/execution/__tests__/session-runner-close-prune.test.ts +181 -0
- package/src/execution/__tests__/session-runner-epipe.test.ts +1 -0
- package/src/execution/__tests__/session-runner-heartbeat-idle-fallback.test.ts +226 -0
- package/src/execution/__tests__/settled-watchdog.test.ts +289 -0
- package/src/execution/__tests__/spawned-children.test.ts +153 -10
- package/src/execution/__tests__/subagent-actions-core.test.ts +997 -0
- package/src/execution/__tests__/subagent-service-message-close.test.ts +71 -10
- package/src/execution/__tests__/subagent-service-multiproc-guard.test.ts +174 -0
- package/src/execution/__tests__/subagent-service-notify-gate.test.ts +266 -0
- package/src/execution/__tests__/subagent-service-parent-guard.test.ts +3 -0
- package/src/execution/__tests__/subagent-service-recovery-bounds.test.ts +316 -0
- package/src/execution/__tests__/timeout-integration.test.ts +7 -7
- package/src/execution/__tests__/ui-request-handler-factory.test.ts +95 -1
- package/src/execution/__tests__/ui-request-queue.test.ts +25 -0
- package/src/execution/__tests__/worktree-git-ops.test.ts +571 -0
- package/src/execution/__tests__/worktree-reconcile-aging.test.ts +204 -0
- package/src/execution/agent-registry.ts +225 -1
- package/src/execution/agents-assembly.ts +88 -0
- package/src/execution/concurrency-pool.ts +71 -9
- package/src/execution/dialog-queue.ts +101 -3
- package/src/execution/engine/__tests__/conformance/contract.abort.test.ts +49 -2
- package/src/execution/engine/__tests__/conformance/contract.agent-events.test.ts +87 -1
- package/src/execution/engine/__tests__/conformance/engine-conformance.live.test.ts +26 -0
- package/src/execution/engine/__tests__/conformance/golden-replay.zcode.test.ts +83 -1
- package/src/execution/engine/__tests__/conformance/zcode-appserver-harness.ts +131 -0
- package/src/execution/engine/__tests__/registry.test.ts +90 -1
- package/src/execution/engine/engine-discovery.ts +12 -16
- package/src/execution/engine/engines/pi/task-spec-mapper.ts +3 -3
- package/src/execution/engine/engines/zcode/__tests__/__fixtures__/fake-appserver.mjs +274 -0
- package/src/execution/engine/engines/zcode/__tests__/__fixtures__/zcode-golden-appserver.json +36 -0
- package/src/execution/engine/engines/zcode/__tests__/connection.test.ts +472 -0
- package/src/execution/engine/engines/zcode/__tests__/preparer-appserver.test.ts +387 -0
- package/src/execution/engine/engines/zcode/__tests__/session-channel.test.ts +780 -0
- package/src/execution/engine/engines/zcode/__tests__/zcode-engine-appserver.test.ts +815 -0
- package/src/execution/engine/engines/zcode/__tests__/zcode-engine-degrade.test.ts +462 -0
- package/src/execution/engine/engines/zcode/__tests__/zcode-engine.live.test.ts +119 -1
- package/src/execution/engine/engines/zcode/__tests__/zcode-engine.test.ts +136 -6
- package/src/execution/engine/engines/zcode/appserver-home.ts +442 -0
- package/src/execution/engine/engines/zcode/appserver-probe.ts +141 -0
- package/src/execution/engine/engines/zcode/connection.ts +585 -0
- package/src/execution/engine/engines/zcode/constants.ts +129 -0
- package/src/execution/engine/engines/zcode/golden-sample.ts +54 -8
- package/src/execution/engine/engines/zcode/preparer.ts +25 -22
- package/src/execution/engine/engines/zcode/session-channel.ts +656 -0
- package/src/execution/engine/engines/zcode/zcode-engine.ts +909 -41
- package/src/execution/engine/host-task-spec.ts +3 -3
- package/src/execution/engine/port.ts +41 -1
- package/src/execution/engine/registry.ts +53 -0
- package/src/execution/finalize-record.ts +63 -5
- package/src/execution/get-state-handshake.ts +85 -10
- package/src/execution/lifecycle-manager.ts +27 -3
- package/src/execution/manifest-store.ts +53 -62
- package/src/execution/notifier.ts +10 -10
- package/src/execution/notify-ledger.ts +117 -16
- package/src/execution/record-entry.ts +8 -2
- package/src/execution/record-store.ts +29 -15
- package/src/execution/session-pending.ts +213 -62
- package/src/execution/session-runner.ts +974 -128
- package/src/execution/sessions-index.ts +10 -55
- package/src/execution/settled-watchdog.ts +99 -0
- package/src/execution/subagent-actions-core.ts +686 -0
- package/src/execution/subagent-service.ts +396 -49
- package/src/execution/ui-request-handler-factory.ts +27 -4
- package/src/execution/worktree-git-ops.ts +397 -0
- package/src/execution/worktree-manager.ts +72 -10
- package/src/execution/worktree-registry.ts +6 -12
- package/src/index.ts +413 -6
- package/src/orchestration/__tests__/__fixtures__/worker-template.snapshot.txt +42 -3
- package/src/orchestration/__tests__/agent-call-catch-fallback.test.ts +2 -4
- package/src/orchestration/__tests__/args-meta.test.ts +358 -0
- package/src/orchestration/__tests__/error-recovery-handlers.test.ts +1 -8
- package/src/orchestration/__tests__/error-recovery-rebuild-failure.test.ts +312 -0
- package/src/orchestration/__tests__/error-recovery-terminal-hardening.test.ts +382 -0
- package/src/orchestration/__tests__/file-run-store-prune.test.ts +168 -0
- package/src/orchestration/__tests__/file-run-store-throttle.test.ts +168 -0
- package/src/orchestration/__tests__/file-run-store.test.ts +389 -0
- package/src/orchestration/__tests__/helpers/flush-microtasks.ts +13 -0
- package/src/orchestration/__tests__/launcher-nested-workflow.test.ts +34 -0
- package/src/orchestration/__tests__/lifecycle-abort-broadcast-signal.test.ts +357 -0
- package/src/orchestration/__tests__/lifecycle-recover-crashed.test.ts +289 -0
- package/src/orchestration/__tests__/lifecycle.test.ts +77 -0
- package/src/orchestration/__tests__/run-snapshot.test.ts +353 -0
- package/src/orchestration/__tests__/script-generate.test.ts +314 -0
- package/src/orchestration/__tests__/worker-pending-timeout-abort.test.ts +275 -0
- package/src/orchestration/__tests__/worker-script-builder-runtime.test.ts +78 -1
- package/src/orchestration/__tests__/workflow-files.test.ts +186 -0
- package/src/orchestration/__tests__/workflow-run-summary.test.ts +119 -0
- package/src/orchestration/__tests__/workflow-script-registry-impl.test.ts +124 -0
- package/src/orchestration/agent-opts-resolver.ts +7 -7
- package/src/orchestration/args-meta.ts +198 -0
- package/src/orchestration/error-recovery.ts +416 -146
- package/src/orchestration/execute-agent-call.ts +9 -9
- package/src/orchestration/file-run-store.ts +327 -0
- package/src/orchestration/launcher.ts +35 -26
- package/src/orchestration/lifecycle.ts +355 -79
- package/src/orchestration/models/agent-call.ts +7 -7
- package/src/orchestration/models/budget.ts +5 -5
- package/src/orchestration/models/run-runtime.ts +13 -13
- package/src/orchestration/models/trace.ts +8 -8
- package/src/orchestration/models/workflow-run.ts +27 -27
- package/src/orchestration/models/workflow-script.ts +4 -4
- package/src/orchestration/run-snapshot.ts +266 -0
- package/src/orchestration/script-generate.ts +154 -0
- package/src/orchestration/script-lint.ts +33 -33
- package/src/orchestration/worker-handle.ts +10 -10
- package/src/orchestration/worker-host.ts +10 -10
- package/src/orchestration/worker-script-builder.ts +401 -346
- package/src/orchestration/workflow-files.ts +37 -11
- package/src/orchestration/workflow-run-summary.ts +69 -0
- package/src/orchestration/workflow-script-registry-impl.ts +31 -9
- package/src/shared/__tests__/agent-ref.test.ts +209 -2
- package/src/shared/__tests__/atomic-write.test.ts +267 -0
- package/src/shared/__tests__/bounded-serialize.test.ts +236 -0
- package/src/shared/__tests__/injection-render.test.ts +518 -0
- package/src/shared/__tests__/resource-discovery-host-roots.test.ts +474 -0
- package/src/shared/__tests__/resource-discovery.test.ts +3 -2
- package/src/shared/agent-ref.ts +143 -2
- package/src/shared/atomic-write.ts +320 -0
- package/src/shared/bounded-serialize.ts +154 -0
- package/src/shared/injection-render.ts +279 -0
- package/src/shared/meta-parser.ts +41 -1
- package/src/shared/resource-discovery.ts +104 -37
- package/src/shared/resource-meta.ts +14 -0
- package/src/shared/xml-injection.ts +9 -9
- package/workflows/README.md +9 -9
- package/workflows/chain.js +4 -2
- package/workflows/map-reduce.js +5 -3
- package/workflows/parallel.js +5 -3
- package/workflows/review-fix-loop-utils.cjs +14 -3
- package/workflows/review-fix-loop.js +18 -7
- package/workflows/scatter-gather.js +4 -2
- package/dist/chunk-3VOERJPJ.js +0 -22
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Worker 失败处理 free functions(D-12)。
|
|
5
5
|
*
|
|
6
6
|
* 4 个导出函数(domain-models.md §失败处理矩阵):
|
|
7
|
-
* - handleWorkerMessage(run, raw, deps, handlers) — 路由 agent_call/return/error
|
|
7
|
+
* - handleWorkerMessage(run, raw, deps, handlers) — 路由 agent_call/return/error/log
|
|
8
8
|
* - handleWorkerError(run, err, deps, handlers) — worker uncaught error
|
|
9
9
|
* - handleWorkerExit(run, code, handle, deps, handlers) — worker exit
|
|
10
10
|
* - handleScriptError(run, msg, deps, handlers) — type:"error" from worker
|
|
@@ -13,6 +13,11 @@
|
|
|
13
13
|
* - worker error/exit(非零)→ 3 次重试 + 指数退避 1s/2s/4s;超限 failed
|
|
14
14
|
* - script error → 3 次重试 + 指数退避;超限 failed
|
|
15
15
|
* - 重试前 rebuildRuntime(G3-001:整个 RunRuntime 重建:worker+controller)
|
|
16
|
+
* - [OR-2] 重建动作本身失败(workerHost.start 抛错)回灌本矩阵:计入
|
|
17
|
+
* workerErrorCount,未超限再走退避+重建,超限收敛 done,failed(见
|
|
18
|
+
* scheduleRebuild / handleRebuildStartFailure)——恢复机制不得在它自己的
|
|
19
|
+
* 恢复路径上开口(旧实现裸调 rebuildRuntime → run 永久 running +
|
|
20
|
+
* rejection 经 void 变 unhandledRejection)
|
|
16
21
|
*
|
|
17
22
|
* 关键不变式:
|
|
18
23
|
* - 重试前必须 rebuildRuntime(worker+controller 整体重建,避免孤儿资源)。
|
|
@@ -66,6 +71,22 @@ const MAX_ERROR_LOGS = 500;
|
|
|
66
71
|
/** malformed agent-call 日志中 opts JSON 的预览截断长度(字符)。 */
|
|
67
72
|
const MALFORMED_MSG_LOG_PREVIEW_CHARS = 200;
|
|
68
73
|
|
|
74
|
+
/**
|
|
75
|
+
* [OR-8] run 到达 done 终态时残留 in-flight call 的收口文案。
|
|
76
|
+
*
|
|
77
|
+
* trace/call 状态枚举封闭(无 "cancelled" 态),以 failed + 本固定文案表达
|
|
78
|
+
* 「run 终态前被收口」——GUI/快照侧不再出现 done run 含 running 节点的不一致。
|
|
79
|
+
*/
|
|
80
|
+
const IN_FLIGHT_CALL_CANCELLED_MSG =
|
|
81
|
+
"Cancelled: run reached terminal state while this call was in flight";
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* [P-SD/S-D] 测试钩子 env(设计 §7.3 P-SD):设为正整数 N 时 rebuildRuntime
|
|
85
|
+
* 第 N 次及以后的每次调用抛错,供 S-D「worker 崩溃后重建失败」验收注入。
|
|
86
|
+
* 安全约束:仅显式设置时激活 + 激活即 warn 留痕(见 resolveRebuildFailureInjectionThreshold)。
|
|
87
|
+
*/
|
|
88
|
+
const REBUILD_FAILURE_INJECT_ENV = "XYZ_SUBAGENT_TEST_INJECT_REBUILD_FAILURE";
|
|
89
|
+
|
|
69
90
|
/**
|
|
70
91
|
* [F1] worker 交付前退出(无终态消息)的归因文案。
|
|
71
92
|
*
|
|
@@ -114,7 +135,14 @@ interface WorkflowCallMsg {
|
|
|
114
135
|
args: Record<string, unknown>;
|
|
115
136
|
}
|
|
116
137
|
|
|
117
|
-
|
|
138
|
+
/** 脚本 log() 全局发出的独立诊断消息(协议见 worker-script-builder 头注释,OR-6)。 */
|
|
139
|
+
interface LogMsg {
|
|
140
|
+
type: "log";
|
|
141
|
+
phase?: string;
|
|
142
|
+
message: string;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
type WorkerMsg = AgentCallMsg | WorkflowCallMsg | ReturnMsg | ErrorMsg | LogMsg;
|
|
118
146
|
|
|
119
147
|
// ── 内部 helper ──────────────────────────────────────────────
|
|
120
148
|
|
|
@@ -178,8 +206,105 @@ function delay(ms: number): Promise<void> {
|
|
|
178
206
|
});
|
|
179
207
|
}
|
|
180
208
|
|
|
209
|
+
/**
|
|
210
|
+
* [OR-4][B-4] pending:unregister emit 围栏(M12 同款 try 围栏)。
|
|
211
|
+
*
|
|
212
|
+
* eventBus 是通知总线(pending-notifications 注销信号灯)——listener 同步抛错不得
|
|
213
|
+
* 经 worker-host 绑定处的 `void handlers.onXxx(...)` 变 unhandledRejection 崩宿主。
|
|
214
|
+
* catch 后 error 留痕,调用方继续。lifecycle.terminateRunningRuns 的 emit(无
|
|
215
|
+
* onRunDone 的终态路径)也消费本函数——围栏语义两处单源。
|
|
216
|
+
*/
|
|
217
|
+
export function emitPendingUnregister(run: WorkflowRun, deps: LifecycleDeps, context: string): void {
|
|
218
|
+
try {
|
|
219
|
+
deps.log?.("debug", "workflow:error-recovery", "emit pending:unregister", { runId: run.runId, reason: run.state.reason });
|
|
220
|
+
deps.eventBus?.emit("pending:unregister", { id: run.runId, reason: run.state.reason ?? "completed" });
|
|
221
|
+
deps.log?.("debug", "workflow:error-recovery", "emit pending:unregister done", { runId: run.runId });
|
|
222
|
+
} catch (err) {
|
|
223
|
+
const m = err instanceof Error ? err.message : String(err);
|
|
224
|
+
logger.error(`[workflow] pending:unregister emit failed (${context}): ${m}`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* [OR-4] 终态收尾副作用围栏:emit pending:unregister + onRunDone,各自独立 M12 同款
|
|
230
|
+
* try 围栏。
|
|
231
|
+
*
|
|
232
|
+
* 这两个是真实副作用(通知注销 + Interface 层完成回调,后者内部含 evictDoneRunsBeyondCap
|
|
233
|
+
* 内存淘汰)——任一同步抛错不得经 worker-host 绑定处的 `void handlers.onXxx(...)`
|
|
234
|
+
* 变 unhandledRejection 崩宿主,各自 catch 后 error 留痕,调用方继续。
|
|
235
|
+
*
|
|
236
|
+
* [B-4] 拆分为两个独立围栏(旧实现同一 try:emit 抛错会跳过 onRunDone)——通知总线
|
|
237
|
+
* 故障不得吞掉 Interface 层完成回调(runAndWait 轮询依赖 onRunDone 语义收口,被跳过
|
|
238
|
+
* 即悬挂)。本函数自身不重试不上抛。
|
|
239
|
+
*
|
|
240
|
+
* 旧实现四处裸调(handleReturn / handleWorkerError / handleScriptError /
|
|
241
|
+
* handleWorkerExit 的终态路径)与 budget 分支的内联围栏不对称——统一收敛到本函数。
|
|
242
|
+
*/
|
|
243
|
+
export function emitTerminalSideEffects(run: WorkflowRun, deps: LifecycleDeps, context: string): void {
|
|
244
|
+
emitPendingUnregister(run, deps, context);
|
|
245
|
+
try {
|
|
246
|
+
deps.onRunDone?.(run);
|
|
247
|
+
} catch (err) {
|
|
248
|
+
const m = err instanceof Error ? err.message : String(err);
|
|
249
|
+
logger.error(`[workflow] onRunDone failed (${context}): ${m}`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
181
253
|
// ── rebuildRuntime(G3-001 整重建) ─────────────────────────
|
|
182
254
|
|
|
255
|
+
/** [P-SD] rebuildRuntime 进程级调用计数(注入阈值「第 N 次」的判定基准)。 */
|
|
256
|
+
let rebuildRuntimeInvocationCount = 0;
|
|
257
|
+
/** [P-SD] 钩子激活/非法值 warn 是否已发(多轮 rebuild 只留痕一次,防刷屏)。 */
|
|
258
|
+
let rebuildFailureHookWarned = false;
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* [P-SD/S-D] 读取重建失败注入阈值:env XYZ_SUBAGENT_TEST_INJECT_REBUILD_FAILURE=<N>
|
|
262
|
+
* 使 rebuildRuntime 第 N 次及以后的每次调用抛错(配合脚本内 process.exit 制造
|
|
263
|
+
* 「worker 崩溃后重建失败」的 S-D 验收场景)。
|
|
264
|
+
*
|
|
265
|
+
* 安全约束(设计 §7.3 P-SD,对齐 T7① 可见性原则):
|
|
266
|
+
* - 仅显式设置时激活;未设置/空串 = 钩子完全不激活(零行为差);
|
|
267
|
+
* - 首次读取到该 env(无论合法非法)即 logger.warn 留痕一次,杜绝静默生效;
|
|
268
|
+
* - 非法值(非正整数)不激活且 warn 指明原值,杜绝「以为注入了、实际没有」的
|
|
269
|
+
* 静默失效(LC-7 同族教训)。
|
|
270
|
+
*
|
|
271
|
+
* 语义取「第 N 次及以后每次」而非「仅第 N 次」:S-D 验收要求 run 收敛 done,failed——
|
|
272
|
+
* 仅注入一次会在重试预算(MAX_WORKER_RETRIES)耗尽前放行后续重建,run 可能正常完成,
|
|
273
|
+
* 验收不可证伪。连续注入让重试矩阵确定性走完:耗尽后经 handleRebuildStartFailure
|
|
274
|
+
* 收敛 done,failed。
|
|
275
|
+
*
|
|
276
|
+
* @returns 注入阈值(调用序数 >= 阈值的 rebuild 抛错);undefined = 未激活
|
|
277
|
+
*/
|
|
278
|
+
function resolveRebuildFailureInjectionThreshold(): number | undefined {
|
|
279
|
+
const raw = process.env[REBUILD_FAILURE_INJECT_ENV];
|
|
280
|
+
if (raw === undefined || raw === "") return undefined;
|
|
281
|
+
const parsed = Number(raw);
|
|
282
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
283
|
+
if (!rebuildFailureHookWarned) {
|
|
284
|
+
rebuildFailureHookWarned = true;
|
|
285
|
+
logger.warn(
|
|
286
|
+
`[workflow] ${REBUILD_FAILURE_INJECT_ENV}="${raw}" is not a positive integer — ` +
|
|
287
|
+
"test hook INACTIVE, no rebuild failure will be injected",
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
return undefined;
|
|
291
|
+
}
|
|
292
|
+
if (!rebuildFailureHookWarned) {
|
|
293
|
+
rebuildFailureHookWarned = true;
|
|
294
|
+
logger.warn(
|
|
295
|
+
`[workflow] ${REBUILD_FAILURE_INJECT_ENV}=${raw} ACTIVE — rebuildRuntime invocations ` +
|
|
296
|
+
`#${parsed} and later will throw (S-D test hook; NEVER set in production)`,
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
return parsed;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** 测试辅助:重置注入计数与 warn 状态(仅 __tests__ 导入,生产勿用)。 */
|
|
303
|
+
export function resetRebuildFailureInjectionForTest(): void {
|
|
304
|
+
rebuildRuntimeInvocationCount = 0;
|
|
305
|
+
rebuildFailureHookWarned = false;
|
|
306
|
+
}
|
|
307
|
+
|
|
183
308
|
/**
|
|
184
309
|
* 移除 run 中未真正完成的在飞 call(status !== "done")及其 trace 节点。
|
|
185
310
|
*
|
|
@@ -203,6 +328,50 @@ function discardInFlightCalls(run: WorkflowRun): number[] {
|
|
|
203
328
|
return inFlight.sort((a, b) => a - b);
|
|
204
329
|
}
|
|
205
330
|
|
|
331
|
+
/**
|
|
332
|
+
* [OR-8] run 到达 done 终态时,把 calls Map 残留的 in-flight call(status !== "done")
|
|
333
|
+
* 收口为取消终态(不删除条目——保留调用痕迹,快照/GUI 不再出现 done run 含
|
|
334
|
+
* running 节点的不一致)。
|
|
335
|
+
*
|
|
336
|
+
* 场景:脚本 fire-and-forget agent()(不 await)后 return;或 worker 死亡/abort 时
|
|
337
|
+
* 已 dispatch 未完成的 call。旧实现 trace 节点永久 "running" 并原样落盘
|
|
338
|
+
* (run-snapshot 序列化不做状态修正)。
|
|
339
|
+
*
|
|
340
|
+
* 收口语义(trace/call 状态枚举封闭,无 "cancelled" 态):
|
|
341
|
+
* - AgentCall 补齐 pending→running→done 状态机(markDone 要求 running 前置),
|
|
342
|
+
* result 以 IN_FLIGHT_CALL_CANCELLED_MSG 承载取消原因;
|
|
343
|
+
* - trace 节点置 failed + 固定取消文案 + completedAt(「failed + Cancelled 文案」
|
|
344
|
+
* 即取消的既有表达形态,不新增状态枚举);
|
|
345
|
+
* - node.live 清除(终态 run 无 TUI 轮询,防 ExecutionRecord 滞留)。
|
|
346
|
+
*
|
|
347
|
+
* 调用点约定:每个 transition("done") 成功后、store.save 之前——先收口再落盘,
|
|
348
|
+
* 内存态与持久化快照在同一时点收敛(「run-snapshot 落盘前」的实现形态)。
|
|
349
|
+
* 返回被收口的 callId 数组(升序)供调用方记日志。
|
|
350
|
+
*/
|
|
351
|
+
export function closeOutInFlightCalls(run: WorkflowRun): number[] {
|
|
352
|
+
const inFlight: number[] = [];
|
|
353
|
+
for (const [callId, call] of run.state.calls) {
|
|
354
|
+
if (call.status !== "done") inFlight.push(callId);
|
|
355
|
+
}
|
|
356
|
+
const completedAt = new Date().toISOString();
|
|
357
|
+
for (const callId of inFlight) {
|
|
358
|
+
const call = run.state.calls.get(callId);
|
|
359
|
+
if (!call) continue; // 防御:迭代后被删(正常路径不可达)
|
|
360
|
+
if (call.status === "pending") call.markRunning();
|
|
361
|
+
if (call.status === "running") {
|
|
362
|
+
call.markDone({ content: "", error: IN_FLIGHT_CALL_CANCELLED_MSG });
|
|
363
|
+
}
|
|
364
|
+
call.traceNode.live = undefined;
|
|
365
|
+
run.state.trace.update(callId, {
|
|
366
|
+
status: "failed",
|
|
367
|
+
result: { content: "", error: IN_FLIGHT_CALL_CANCELLED_MSG },
|
|
368
|
+
error: IN_FLIGHT_CALL_CANCELLED_MSG,
|
|
369
|
+
completedAt,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
return inFlight.sort((a, b) => a - b);
|
|
373
|
+
}
|
|
374
|
+
|
|
206
375
|
/**
|
|
207
376
|
* 计算 run 的剩余时间预算(ms)[race-F3]。
|
|
208
377
|
*
|
|
@@ -244,12 +413,11 @@ async function finalizeTimeBudgetExhausted(run: WorkflowRun, deps: LifecycleDeps
|
|
|
244
413
|
void te;
|
|
245
414
|
}
|
|
246
415
|
if (!transitioned) return;
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
deps.onRunDone?.(run);
|
|
416
|
+
// [OR-8] 终态收口残留 in-flight call(先收口再落盘)
|
|
417
|
+
closeOutInFlightCalls(run);
|
|
418
|
+
await saveRunBestEffort(run, deps, "finalizeTimeBudgetExhausted (done,time_limited)");
|
|
419
|
+
// [OR-4] M12 同款围栏(终态副作用不经 void 变 unhandledRejection)
|
|
420
|
+
emitTerminalSideEffects(run, deps, "finalizeTimeBudgetExhausted (done,time_limited)");
|
|
253
421
|
}
|
|
254
422
|
|
|
255
423
|
/**
|
|
@@ -275,26 +443,36 @@ export function rebuildRuntime(
|
|
|
275
443
|
deps: LifecycleDeps,
|
|
276
444
|
handlers: WorkerHandlers,
|
|
277
445
|
): void {
|
|
278
|
-
|
|
279
|
-
|
|
446
|
+
// [P-SD] 测试钩子注入点:先于任何副作用(模拟 workerHost.start 抛错)。抛错由
|
|
447
|
+
// scheduleRebuild 的 catch 接住回灌重试矩阵([OR-2]),不再裸抛。
|
|
448
|
+
rebuildRuntimeInvocationCount += 1;
|
|
449
|
+
const injectThreshold = resolveRebuildFailureInjectionThreshold();
|
|
450
|
+
if (injectThreshold !== undefined && rebuildRuntimeInvocationCount >= injectThreshold) {
|
|
451
|
+
throw new Error(
|
|
452
|
+
`[S-D test hook] injected rebuildRuntime failure ` +
|
|
453
|
+
`(invocation #${rebuildRuntimeInvocationCount}, ${REBUILD_FAILURE_INJECT_ENV}>=${injectThreshold})`,
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
// OB3(可观察性):rebuild 关键节点 debug 日志——此前函数体 0 处 deps.log,
|
|
457
|
+
// 崩溃自愈只能靠行为证据诊断(L1 入口 / L2 重排 / L3 discard / L4 完成)。
|
|
280
458
|
deps.log?.("debug", "workflow:error-recovery", "runtime rebuild start", {
|
|
281
459
|
runId: run.runId,
|
|
282
460
|
budgetTimeMs: run.spec.budgetTimeMs,
|
|
283
461
|
});
|
|
284
462
|
const controller = new AbortController();
|
|
285
463
|
const worker = deps.workerHost.start(run.spec, run.spec.args, handlers);
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
464
|
+
// D-12 regression fix (round-2 #2):重新调度 run 级墙钟预算计时器。
|
|
465
|
+
// replaceRuntime 释放旧 runtime 时 clearTimeout 了旧计时器(run-runtime.release),
|
|
466
|
+
// 新 runtime 必须重排,否则带 budgetTimeMs 的 run 命中一次 worker/script 错误重试后
|
|
467
|
+
// 时间预算静默失效(直到 rebuildRuntime 才重排——本函数即唯一重排点)。
|
|
468
|
+
// deps.scheduleTimeBudget 由 Interface 层注入;未注入时(旧测试)跳过重排(兼容,
|
|
469
|
+
// 不影响无时间预算的 run)。
|
|
470
|
+
// 重排分支改为 if——语义与原三元一致(同一条件调 scheduleTimeBudget),仅为在
|
|
471
|
+
// 分支内记 L2 日志,控制流/异常语义零变化。
|
|
472
|
+
// [race-F3] 重排值改为剩余墙钟(remainingTimeBudgetMs)而非满额——重试不重置预算;
|
|
473
|
+
// L2 日志 payload 同步报实际重排值(排障时与 setTimeout 对得上)。remaining > 0
|
|
474
|
+
// 由调用方 scheduleRebuild 保证(耗尽在那里转 time_limited,不进本函数);本处
|
|
475
|
+
// remaining <= 0 时不挂 timer(防御直调,宁可不挂也不能挂出 0ms 立即触发)。
|
|
298
476
|
let timeBudgetTimer: ReturnType<typeof setTimeout> | undefined;
|
|
299
477
|
const remainingBudgetMs = remainingTimeBudgetMs(run);
|
|
300
478
|
if (remainingBudgetMs !== undefined && remainingBudgetMs > 0 && deps.scheduleTimeBudget) {
|
|
@@ -305,21 +483,21 @@ export function rebuildRuntime(
|
|
|
305
483
|
});
|
|
306
484
|
}
|
|
307
485
|
run.replaceRuntime(new RunRuntime(worker, controller, timeBudgetTimer));
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
486
|
+
// 清除被旧 runtime abort 的在飞 call——必须在 replaceRuntime 之后同步执行(无
|
|
487
|
+
// await 间隔):replaceRuntime 同步 abort 旧 controller + terminate 旧 worker,
|
|
488
|
+
// 在飞 executeAgentCall 的 finalize 发生在 `await runner.run` resolve 后的
|
|
489
|
+
// microtask,此刻在飞 call 仍为 "running"/"pending"(status !== "done")可精确
|
|
490
|
+
// 清理;genuinely-done 的 call 保留(重跑 replay)。放 delay 退避之前会误删退避
|
|
491
|
+
// 期间自然完成的真结果(重跑重复耗 token);放任何 await 之后,假失败已 finalize
|
|
492
|
+
// 为 "done" 挡不住——重跑 replay 会把 abort 错误当真结果回放,静默污染输出。
|
|
493
|
+
// 注意:discard 只清 Map/trace 条目,旧 executeAgentCall 的 promise 链仍会醒来
|
|
494
|
+
// finalize。markDone 在孤儿实例上无害,但后续投递并非 no-op——postAgentResult
|
|
495
|
+
// 会投给 run.runtime(已是新 worker)的同 callId pending,劫持重跑调用(实测
|
|
496
|
+
// S7-second 竞态:旧失败结果被 worker 侧 resolve 为空串 → 脚本假成功);
|
|
497
|
+
// finalizeCall 的 trace.update 在重跑已 append 同 stepIndex 新节点时命中新节点
|
|
498
|
+
// (瞬时污染,由重跑完成时的 update 覆盖)。该投递由 dispatchAgentCall 的
|
|
499
|
+
// 孤儿守卫(isOrphanedCall)拦截,trace.update 的瞬时污染由 executeAgentCall
|
|
500
|
+
// 的 isOrphaned 谓词(OB2)拦截,此处不重复设防。
|
|
323
501
|
const discardedCallIds = discardInFlightCalls(run);
|
|
324
502
|
deps.log?.("debug", "workflow:error-recovery", "in-flight calls discarded", {
|
|
325
503
|
runId: run.runId,
|
|
@@ -339,6 +517,11 @@ export function rebuildRuntime(
|
|
|
339
517
|
* agent_call → 派发 executeAgentCall(异步,不 await——立即返回让 worker 继续发消息)
|
|
340
518
|
* return → transition done,completed(脚本正常返回)
|
|
341
519
|
* error → handleScriptError(脚本主动抛错)
|
|
520
|
+
* log → 计入 run.state.errorLogs + debug 留痕([OR-6/T7④] 主线程半边——worker 侧
|
|
521
|
+
* log() 双通路的独立消息面消费点;u-m0a 已接 workerLogs 随 return/error 带回,
|
|
522
|
+
* 本 case 消费独立 {type:"log"} 消息,与 worker-script-builder 协议注释对齐)
|
|
523
|
+
* default → warn 留痕后丢弃([OR-6/T7④] 协议漂移防线——协议文档与实现漂移零
|
|
524
|
+
* 可观测的反面;未知类型静默丢弃会让协议单方面演化不可发现)
|
|
342
525
|
*
|
|
343
526
|
* 终态(done)下的 stale 消息丢弃(P0-1)。
|
|
344
527
|
*/
|
|
@@ -348,10 +531,10 @@ export async function handleWorkerMessage(
|
|
|
348
531
|
deps: LifecycleDeps,
|
|
349
532
|
handlers: WorkerHandlers,
|
|
350
533
|
): Promise<void> {
|
|
351
|
-
|
|
534
|
+
// 终态(done)丢弃 stale 消息(P0-1)
|
|
352
535
|
if (isTerminal(run)) return;
|
|
353
536
|
|
|
354
|
-
|
|
537
|
+
// M7: 形状校验——防畸形 IPC 消息(worker 崩溃/发非对象)导致下游 TypeError
|
|
355
538
|
if (typeof raw !== "object" || raw === null) return;
|
|
356
539
|
const msg = raw as WorkerMsg;
|
|
357
540
|
switch (msg.type) {
|
|
@@ -369,7 +552,7 @@ export async function handleWorkerMessage(
|
|
|
369
552
|
await handleReturn(run, msg, deps);
|
|
370
553
|
return;
|
|
371
554
|
case "error":
|
|
372
|
-
|
|
555
|
+
// M1: 传 handlers(rebuildRuntime 需要)
|
|
373
556
|
// [F1] 同 return——error 也是终态消息,标记本代际已交付(同上防误判)。
|
|
374
557
|
if (run.runtime) run.runtime.receivedTerminalMessage = true;
|
|
375
558
|
await handleScriptError(
|
|
@@ -380,7 +563,44 @@ export async function handleWorkerMessage(
|
|
|
380
563
|
handlers,
|
|
381
564
|
);
|
|
382
565
|
return;
|
|
566
|
+
case "log":
|
|
567
|
+
handleWorkerLog(run, msg, deps);
|
|
568
|
+
return;
|
|
569
|
+
default:
|
|
570
|
+
// [OR-6/T7④] 协议漂移防线:未知消息类型 warn 留痕后丢弃。畸形消息(M7 形状
|
|
571
|
+
// 校验之上、已知类型之外的 type)此前静默穿过 switch——协议注释新增消息类型
|
|
572
|
+
// 而主线程未接线时,这里提供可观测信号(而非零痕迹丢弃)。
|
|
573
|
+
logger.warn(
|
|
574
|
+
`[workflow] unknown worker message type dropped (runId=${run.runId}): ` +
|
|
575
|
+
`${JSON.stringify((msg as { type?: unknown }).type)}`,
|
|
576
|
+
);
|
|
577
|
+
deps.log?.("warn", "workflow:error-recovery", "unknown worker message type", {
|
|
578
|
+
runId: run.runId,
|
|
579
|
+
type: (msg as { type?: unknown }).type,
|
|
580
|
+
});
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* 消费 worker 的独立 log 消息([OR-6/T7④] 主线程半边)。
|
|
587
|
+
*
|
|
588
|
+
* 计入 run.state.errorLogs(与 workerLogs 通路的 L9 追加/上限语义一致——该容器
|
|
589
|
+
* 本就承载全级别 worker 日志,"log" 级条目已在其中)+ deps.log debug 留痕
|
|
590
|
+
* (含 phase,协议字段不落 WorkerLogEntry 但排查时可见)。
|
|
591
|
+
* 终态守卫(isTerminal)已由 handleWorkerMessage 前置——此处只管写入。
|
|
592
|
+
*/
|
|
593
|
+
function handleWorkerLog(run: WorkflowRun, msg: LogMsg, deps: LifecycleDeps): void {
|
|
594
|
+
const message = typeof msg.message === "string" ? msg.message : String(msg.message);
|
|
595
|
+
run.state.errorLogs.push({ level: "log", message });
|
|
596
|
+
if (run.state.errorLogs.length > MAX_ERROR_LOGS) {
|
|
597
|
+
run.state.errorLogs = run.state.errorLogs.slice(-MAX_ERROR_LOGS);
|
|
383
598
|
}
|
|
599
|
+
deps.log?.("debug", "workflow:error-recovery", "worker log", {
|
|
600
|
+
runId: run.runId,
|
|
601
|
+
phase: msg.phase,
|
|
602
|
+
message,
|
|
603
|
+
});
|
|
384
604
|
}
|
|
385
605
|
|
|
386
606
|
/**
|
|
@@ -418,7 +638,7 @@ function dispatchAgentCall(
|
|
|
418
638
|
return;
|
|
419
639
|
}
|
|
420
640
|
|
|
421
|
-
|
|
641
|
+
// 已缓存的调用直接 replay(跨 rebuild——崩溃重建后重跑脚本,已完成调用按 callId 命中缓存)
|
|
422
642
|
const cached = run.state.calls.get(msg.callId);
|
|
423
643
|
if (cached && cached.status === "done") {
|
|
424
644
|
postAgentResult(run, msg.callId, cached.result!, true);
|
|
@@ -453,8 +673,8 @@ function dispatchAgentCall(
|
|
|
453
673
|
};
|
|
454
674
|
run.state.trace.append(node);
|
|
455
675
|
|
|
456
|
-
|
|
457
|
-
|
|
676
|
+
// 构建 AgentCall(opts 形状对齐 AgentCallOpts;schema: unknown → Record)
|
|
677
|
+
// 跨进程 IPC 边界的 schema 为 unknown,窄化前加 typeof guard 兜底。
|
|
458
678
|
const rawSchema = msg.opts.schema;
|
|
459
679
|
const opts: AgentCallOpts = {
|
|
460
680
|
...msg.opts,
|
|
@@ -464,11 +684,11 @@ function dispatchAgentCall(
|
|
|
464
684
|
: undefined,
|
|
465
685
|
};
|
|
466
686
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
687
|
+
// BL-1:解析 skill/schema → skillPath / schemaEnv / appendSystemPrompt。
|
|
688
|
+
// M2 修正后 resolveAgentOpts 单参数,只处理 schema SO 指令(内容直传)+ skill。
|
|
689
|
+
// agent ref 处理(systemPrompt/model/thinkingLevel)交 resolveIdentity(经
|
|
690
|
+
// getAgentConfig + resolveModel 完整覆盖),消除双重注入与 model 层级混乱。
|
|
691
|
+
// 解析失败(skill 未找到)走 error 路径,不发 slot、不 spawn。
|
|
472
692
|
const resolved = resolveAgentOpts(opts);
|
|
473
693
|
if (resolved.error) {
|
|
474
694
|
const call = new AgentCall(msg.callId, opts, node);
|
|
@@ -493,14 +713,14 @@ function dispatchAgentCall(
|
|
|
493
713
|
const call = new AgentCall(msg.callId, resolved.opts, node);
|
|
494
714
|
run.state.calls.set(msg.callId, call);
|
|
495
715
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
716
|
+
// C-3:agent call 执行入口。
|
|
717
|
+
// (原经 gate.withSlot 包装,并发门闩 gate 已删——no-op 抽象,实际并发由
|
|
718
|
+
// SubagentService ConcurrencyPool 管理;仅保留其 pre-abort 检查语义,见下方
|
|
719
|
+
// dispatchCall 内 signal.aborted 分支。)executeAgentCall 管 retry/budget/stale-context;
|
|
720
|
+
// runner(runner.run)管 spawn pi 子进程。
|
|
721
|
+
// assignRuntime/replaceRuntime 保证 status==="running" ⟺ runtime defined,
|
|
722
|
+
// 故 run.runtime 在此必存在(dispatchAgentCall 仅从 handleWorkerMessage 调用,
|
|
723
|
+
// 后者已守 terminal(isTerminal)早期 return)。fallback new AbortController 已移除。
|
|
504
724
|
const runtime = run.runtime!;
|
|
505
725
|
const signal = runtime.controller.signal;
|
|
506
726
|
// D-005: onEvent 签名升级——executeAndAwait 直接出 AgentEvent(强类型,
|
|
@@ -516,8 +736,8 @@ function dispatchAgentCall(
|
|
|
516
736
|
const stream = deps.streamSink
|
|
517
737
|
? new SubagentStream(`${run.runId}-${msg.callId}`, deps.streamSink)
|
|
518
738
|
: undefined;
|
|
519
|
-
|
|
520
|
-
|
|
739
|
+
// 原 gate.withSlot(fn, signal) 语义内联:pre-aborted 时 reject AbortError(
|
|
740
|
+
// 下方 .catch 依赖此约定不记错),否则直接执行——并发调度归 ConcurrencyPool。
|
|
521
741
|
const dispatchCall = async (): Promise<void> => {
|
|
522
742
|
if (signal.aborted) {
|
|
523
743
|
const abortErr = new Error("Operation aborted before start");
|
|
@@ -525,9 +745,9 @@ function dispatchAgentCall(
|
|
|
525
745
|
throw abortErr;
|
|
526
746
|
}
|
|
527
747
|
try {
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
748
|
+
// OB2(S7 残留):isOrphaned 谓词注入——旧代际 finalize 在 trace.update 前被
|
|
749
|
+
// 拦截(判定语义与下方 .then/.catch 守卫同一 isOrphanedCall,详见
|
|
750
|
+
// execute-agent-call.ts finalizeCall 文档注释)。
|
|
531
751
|
await executeAgentCall(call, deps.runner, run.state.budget, signal, run.state.trace, onEvent, stream, () => isOrphanedCall(run, msg.callId, call));
|
|
532
752
|
} finally {
|
|
533
753
|
stream?.dispose();
|
|
@@ -535,35 +755,35 @@ function dispatchAgentCall(
|
|
|
535
755
|
};
|
|
536
756
|
void dispatchCall()
|
|
537
757
|
.then(() => {
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
758
|
+
// 清除 live record:终态已由 executeAgentCall → finalizeCall 写入 node.result,
|
|
759
|
+
// live 不再需要(且含可变状态,不保留)。无论 stale 与否都清,避免内存泄漏。
|
|
760
|
+
// M4: 必须在 stale guard 之前清,否则跨 rebuild 的迟到 completion 会累积 live record。
|
|
541
761
|
node.live = undefined;
|
|
542
|
-
|
|
762
|
+
// run 终止(终态)后到达的 stale completion 不写 state
|
|
543
763
|
if (run.state.status !== "running") return;
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
764
|
+
// 孤儿 call 守卫(S7-second 竞态):rebuild 的 discardInFlightCalls 已移除本
|
|
765
|
+
// call、或重跑 dispatch 已用新实例替换同 callId 条目时,本 completion 属于旧
|
|
766
|
+
// runtime 代际。postAgentResult 的投递目标是 run.runtime(已是新 worker),
|
|
767
|
+
// 迟到结果会劫持新 worker 内重跑 agent() 的 pending Promise——跳过投递 /
|
|
768
|
+
// budget 同步 / 持久化,仅留日志。executeAgentCall 内 finalizeCall 的
|
|
769
|
+
// trace.update 若已命中重跑新节点(瞬时污染),由重跑完成时的 update 覆盖。
|
|
550
770
|
if (isOrphanedCall(run, msg.callId, call)) {
|
|
551
771
|
deps.log?.("debug", "workflow:error-recovery", "orphan agent call completion dropped", { runId: run.runId, callId: msg.callId });
|
|
552
772
|
return;
|
|
553
773
|
}
|
|
554
774
|
if (call.result) postAgentResult(run, msg.callId, call.result, false);
|
|
555
|
-
|
|
556
|
-
|
|
775
|
+
// D-12 regression fix (round-2 #1):executeAgentCall 内 consume/incrementCallCount
|
|
776
|
+
// 后同步 worker $BUDGET(否则 $BUDGET.spent()/remaining() 恒为 0)
|
|
557
777
|
postBudgetUpdate(run);
|
|
558
778
|
deps.store.save(run).catch((e: unknown) => {
|
|
559
779
|
const m = e instanceof Error ? e.message : String(e);
|
|
560
780
|
logger.error(`[workflow] store.save failed (agent call ${msg.callId}): ${m}`);
|
|
561
781
|
});
|
|
562
782
|
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
783
|
+
// C-2:budget 超限 → 终止整个 run(避免继续 spawn 烧预算)
|
|
784
|
+
// 内联 terminate(不调 lifecycle.abortRun 避免 engine 内循环依赖):
|
|
785
|
+
// 若 run 仍非终态,transition done,budget_limited + 持久化。
|
|
786
|
+
// 上方 status !== "running" 已保证此处非 done(且 transition 内含 done no-op 守卫)。
|
|
567
787
|
if (run.state.budget.isExceeded()) {
|
|
568
788
|
run.state.error = run.state.error ?? "Budget exceeded";
|
|
569
789
|
deps.log?.("debug", "workflow:error-recovery", "budget exceeded, transition done", { runId: run.runId });
|
|
@@ -577,55 +797,50 @@ function dispatchAgentCall(
|
|
|
577
797
|
void te;
|
|
578
798
|
}
|
|
579
799
|
if (transitioned) {
|
|
800
|
+
// [OR-8] 终态收口残留 in-flight call(先收口再落盘)
|
|
801
|
+
closeOutInFlightCalls(run);
|
|
580
802
|
deps.store.save(run).catch((e: unknown) => {
|
|
581
803
|
const m = e instanceof Error ? e.message : String(e);
|
|
582
804
|
logger.error(`[workflow] store.save failed (budget done): ${m}`);
|
|
583
805
|
});
|
|
584
806
|
deps.log?.("debug", "workflow:error-recovery", "run saved after budget done", { runId: run.runId, reason: run.state.reason });
|
|
585
807
|
// M12: onRunDone/emit 单独 try——这些是真实副作用,错误不应被静默吞掉
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
deps.eventBus?.emit("pending:unregister", { id: run.runId, reason: run.state.reason ?? "completed" });
|
|
589
|
-
deps.log?.("debug", "workflow:error-recovery", "emit pending:unregister done", { runId: run.runId });
|
|
590
|
-
deps.onRunDone?.(run);
|
|
591
|
-
} catch (err) {
|
|
592
|
-
const m = err instanceof Error ? err.message : String(err);
|
|
593
|
-
logger.error(`[workflow] onRunDone/emit failed (budget done): ${m}`);
|
|
594
|
-
}
|
|
808
|
+
//([OR-4] 围栏收敛到共享 helper,语义与原内联围栏一致)
|
|
809
|
+
emitTerminalSideEffects(run, deps, "budget done");
|
|
595
810
|
}
|
|
596
811
|
}
|
|
597
812
|
})
|
|
598
813
|
.catch((err: unknown) => {
|
|
599
|
-
|
|
814
|
+
// pre-abort 检查(原 gate.withSlot 语义)在 dispatchCall 入口 reject AbortError——预期,不记错。
|
|
600
815
|
if (err instanceof Error && err.name === "AbortError") return;
|
|
601
816
|
const message = err instanceof Error ? err.message : String(err);
|
|
602
817
|
logger.error(`[workflow] agent call ${msg.callId} failed: ${message}`);
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
818
|
+
// 兜底回发:executeAgentCall 抛非 Abort 异常时(如 runner undefined 的 TypeError、
|
|
819
|
+
// dispatchCall 内部 bug)原 catch 仅 console.error,worker 内对 callId 的 pending
|
|
820
|
+
// Promise 永不 resolve → agent() 永久 await → worker 脚本挂死。构造 failed AgentResult
|
|
821
|
+
//(与 resolveAgentOpts 失败路径 L262-275 一致的模式)postAgentResult 回 worker,
|
|
822
|
+
// 让 pending Promise resolve(结果为 error),脚本可继续或失败退出。
|
|
823
|
+
// 孤儿 call 守卫(与 .then 对称,S7-second 竞态):rebuild 后本 call 已被 discard
|
|
824
|
+
// 移除/替换——markDone 虽在孤儿实例上无害,但 trace.update 会污染重跑新建的同
|
|
825
|
+
// stepIndex 节点、postAgentResult 会劫持新 worker 的同 callId pending。孤儿时只
|
|
826
|
+
// 留日志,全部跳过。node.live 无条件先清(旧节点已脱离 trace,防御性统一)。
|
|
612
827
|
node.live = undefined;
|
|
613
828
|
if (isOrphanedCall(run, msg.callId, call)) {
|
|
614
829
|
deps.log?.("debug", "workflow:error-recovery", "orphan agent call failure dropped", { runId: run.runId, callId: msg.callId });
|
|
615
830
|
return;
|
|
616
831
|
}
|
|
617
832
|
const errorResult: AgentResult = { content: "", error: message };
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
833
|
+
// call 已 done(executeAgentCall 内 finalizeCall 已 markDone)时跳过,避免重复 markDone。
|
|
834
|
+
// status 理论上必为 running(executeAgentCall L130 markRunning 先于 reject),pending
|
|
835
|
+
// 分支为防御性保护。非 running/done 意外态:跳过 markDone(markDone 要求 running)。
|
|
621
836
|
if (call.status !== "done") {
|
|
622
837
|
if (call.status === "pending") call.markRunning();
|
|
623
838
|
call.markDone(errorResult);
|
|
624
839
|
}
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
840
|
+
// state 一致性三件套(与 resolveAgentOpts 失败 L268-276 / .then L319-325 对等):
|
|
841
|
+
// trace 标 failed + 清 live record(防泄漏)+ 持久化(catch 恰是最需留证的场景)。
|
|
842
|
+
// stale 终态(run 已 done)时 run.runtime 为 undefined,postAgentResult 用
|
|
843
|
+
// optional chaining 跳过 worker 回发;trace/state 写入仍执行(无害,终态快照已存)。
|
|
629
844
|
run.state.trace.update(msg.callId, {
|
|
630
845
|
status: "failed",
|
|
631
846
|
result: errorResult,
|
|
@@ -795,7 +1010,7 @@ async function handleReturn(
|
|
|
795
1010
|
deps: LifecycleDeps,
|
|
796
1011
|
): Promise<void> {
|
|
797
1012
|
deps.log?.("debug", "workflow:error-recovery", "handleReturn", { runId: run.runId, status: run.state.status });
|
|
798
|
-
|
|
1013
|
+
// 捕获 worker 诊断日志(P2-2)
|
|
799
1014
|
// L9: 追加而非覆盖——保留重试历史的诊断日志(各 worker 实例的 console 输出)
|
|
800
1015
|
if (msg.workerLogs && msg.workerLogs.length > 0) {
|
|
801
1016
|
run.state.errorLogs.push(...msg.workerLogs);
|
|
@@ -805,14 +1020,15 @@ async function handleReturn(
|
|
|
805
1020
|
}
|
|
806
1021
|
run.state.scriptResult = msg.result;
|
|
807
1022
|
run.transition("done", "completed");
|
|
1023
|
+
// [OR-8] 脚本 return 时可能仍有 fire-and-forget 的 in-flight call(不 await)——
|
|
1024
|
+
// 终态收口为 cancelled,先收口再落盘(快照不再含 running 节点)
|
|
1025
|
+
closeOutInFlightCalls(run);
|
|
808
1026
|
// [SW-DATA-3] save 失败不阻断终态推进(原 await 裸抛 → unhandledRejection + 幽灵注销)
|
|
809
1027
|
await saveRunBestEffort(run, deps, "handleReturn (done,completed)");
|
|
810
1028
|
deps.log?.("debug", "workflow:error-recovery", "run saved after return", { runId: run.runId, reason: run.state.reason });
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
deps.log?.("debug", "workflow:error-recovery", "emit pending:unregister done", { runId: run.runId });
|
|
815
|
-
deps.onRunDone?.(run);
|
|
1029
|
+
// C-4: run 到达 done 终态 → 注销 pending-notification + 通知 Interface 层
|
|
1030
|
+
// [OR-4] M12 同款围栏(此前裸调——listener 同步抛错经 void 变 unhandledRejection)
|
|
1031
|
+
emitTerminalSideEffects(run, deps, "handleReturn (done,completed)");
|
|
816
1032
|
}
|
|
817
1033
|
|
|
818
1034
|
// ── handleWorkerError ────────────────────────────────────────
|
|
@@ -835,18 +1051,18 @@ export async function handleWorkerError(
|
|
|
835
1051
|
deps: LifecycleDeps,
|
|
836
1052
|
handlers: WorkerHandlers,
|
|
837
1053
|
): Promise<void> {
|
|
838
|
-
|
|
839
|
-
|
|
1054
|
+
// 与 handleWorkerMessage 对称——终态(done)丢弃 stale error。
|
|
1055
|
+
// 否则终态后到达的 worker error 仍会 workerErrorCount++(污染跨 runtime 计数)。
|
|
840
1056
|
if (isTerminal(run)) return;
|
|
841
1057
|
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
1058
|
+
// [R4-F1] 同代际幂等守卫:worker 崩溃时 error + exit(1) 双事件各派发一次
|
|
1059
|
+
// handleWorkerError(onError 先到,exit 非 0 经 handleWorkerExit 委托二次到达)——
|
|
1060
|
+
// 旧实现单次崩溃 workerErrorCount +2、两个 scheduleRebuild 并行交错(双 rebuild
|
|
1061
|
+
// 各自 new Worker,旧 handle 的 terminate/exit 事件与新 handle 的生命周期互相踩踏)。
|
|
1062
|
+
// 复用 R4 的 receivedTerminalMessage 代际标志(RunRuntime 字段,rebuild 自然重置):
|
|
1063
|
+
// 进入处理前置 true 标记「本代际已有 error/terminal 处理」,第二个事件(无论
|
|
1064
|
+
// onError 直达还是 exit(1) 委托)命中标志直接跳过。新代际的 handleWorkerError
|
|
1065
|
+
// 不受影响(新 RunRuntime 的标志为 false)。
|
|
850
1066
|
if (run.runtime?.receivedTerminalMessage) return;
|
|
851
1067
|
if (run.runtime) run.runtime.receivedTerminalMessage = true;
|
|
852
1068
|
|
|
@@ -858,18 +1074,18 @@ export async function handleWorkerError(
|
|
|
858
1074
|
return;
|
|
859
1075
|
}
|
|
860
1076
|
|
|
861
|
-
|
|
1077
|
+
// 超限 → failed
|
|
862
1078
|
run.state.error = err.message;
|
|
863
1079
|
deps.log?.("debug", "workflow:error-recovery", "handleWorkerError retries exceeded, transition done", { runId: run.runId, count });
|
|
864
1080
|
run.transition("done", "failed");
|
|
1081
|
+
// [OR-8] 终态收口残留 in-flight call(先收口再落盘)
|
|
1082
|
+
closeOutInFlightCalls(run);
|
|
865
1083
|
// [SW-DATA-3] save 失败不阻断终态推进
|
|
866
1084
|
await saveRunBestEffort(run, deps, "handleWorkerError (done,failed)");
|
|
867
1085
|
deps.log?.("debug", "workflow:error-recovery", "run saved after worker error", { runId: run.runId, reason: run.state.reason });
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
deps.log?.("debug", "workflow:error-recovery", "emit pending:unregister done", { runId: run.runId });
|
|
872
|
-
deps.onRunDone?.(run);
|
|
1086
|
+
// C-4: run 到达 done 终态 → 注销 pending-notification + 通知 Interface 层
|
|
1087
|
+
// [OR-4] M12 同款围栏
|
|
1088
|
+
emitTerminalSideEffects(run, deps, "handleWorkerError (done,failed)");
|
|
873
1089
|
}
|
|
874
1090
|
|
|
875
1091
|
// ── handleWorkerExit ─────────────────────────────────────────
|
|
@@ -896,7 +1112,7 @@ export async function handleWorkerExit(
|
|
|
896
1112
|
deps: LifecycleDeps,
|
|
897
1113
|
handlers: WorkerHandlers,
|
|
898
1114
|
): Promise<void> {
|
|
899
|
-
|
|
1115
|
+
// G-025: stale exit 事件丢弃(handle 已不是当前 runtime 的 worker)
|
|
900
1116
|
if (!handle.isCurrent) return;
|
|
901
1117
|
if (isTerminal(run)) return;
|
|
902
1118
|
|
|
@@ -910,17 +1126,17 @@ export async function handleWorkerExit(
|
|
|
910
1126
|
deps.log?.("debug", "workflow:error-recovery", "worker exited without terminal message, transition done", { runId: run.runId });
|
|
911
1127
|
run.state.error = WORKER_EXITED_WITHOUT_RESULT_MSG;
|
|
912
1128
|
run.transition("done", "failed");
|
|
1129
|
+
// [OR-8] 终态收口残留 in-flight call(先收口再落盘)
|
|
1130
|
+
closeOutInFlightCalls(run);
|
|
913
1131
|
await saveRunBestEffort(run, deps, "handleWorkerExit (done,failed, no terminal message)");
|
|
914
1132
|
deps.log?.("debug", "workflow:error-recovery", "run saved after exit without result", { runId: run.runId, reason: run.state.reason });
|
|
915
1133
|
// C-4: run 到达 done 终态 → 注销 pending-notification + 通知 Interface 层
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
deps.log?.("debug", "workflow:error-recovery", "emit pending:unregister done", { runId: run.runId });
|
|
919
|
-
deps.onRunDone?.(run);
|
|
1134
|
+
// [OR-4] M12 同款围栏
|
|
1135
|
+
emitTerminalSideEffects(run, deps, "handleWorkerExit (done,failed, no terminal message)");
|
|
920
1136
|
return;
|
|
921
1137
|
}
|
|
922
1138
|
|
|
923
|
-
|
|
1139
|
+
// 非零 exit → 委托 handleWorkerError(C.3: onExit 传 handle 用于竞态防护)
|
|
924
1140
|
await handleWorkerError(
|
|
925
1141
|
run,
|
|
926
1142
|
new Error(`Worker exited with code ${code}`),
|
|
@@ -947,10 +1163,10 @@ export async function handleScriptError(
|
|
|
947
1163
|
deps: LifecycleDeps,
|
|
948
1164
|
handlers: WorkerHandlers,
|
|
949
1165
|
): Promise<void> {
|
|
950
|
-
|
|
1166
|
+
// 与 handleWorkerMessage/handleWorkerError 对称——终态守卫前置。
|
|
951
1167
|
if (isTerminal(run)) return;
|
|
952
1168
|
|
|
953
|
-
|
|
1169
|
+
// P2-2: 捕获 worker 诊断日志
|
|
954
1170
|
// L9: 追加而非覆盖
|
|
955
1171
|
if (workerLogs.length > 0) {
|
|
956
1172
|
run.state.errorLogs.push(...workerLogs);
|
|
@@ -967,18 +1183,18 @@ export async function handleScriptError(
|
|
|
967
1183
|
return;
|
|
968
1184
|
}
|
|
969
1185
|
|
|
970
|
-
|
|
1186
|
+
// 超限 → failed
|
|
971
1187
|
run.state.error = `Workflow failed after ${MAX_WORKER_RETRIES} retries: ${errorMsg}`;
|
|
972
1188
|
deps.log?.("debug", "workflow:error-recovery", "handleScriptError retries exceeded, transition done", { runId: run.runId, count });
|
|
973
1189
|
run.transition("done", "failed");
|
|
1190
|
+
// [OR-8] 终态收口残留 in-flight call(先收口再落盘)
|
|
1191
|
+
closeOutInFlightCalls(run);
|
|
974
1192
|
// [SW-DATA-3] save 失败不阻断终态推进
|
|
975
1193
|
await saveRunBestEffort(run, deps, "handleScriptError (done,failed)");
|
|
976
1194
|
deps.log?.("debug", "workflow:error-recovery", "run saved after script error", { runId: run.runId, reason: run.state.reason });
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
deps.log?.("debug", "workflow:error-recovery", "emit pending:unregister done", { runId: run.runId });
|
|
981
|
-
deps.onRunDone?.(run);
|
|
1195
|
+
// C-4: run 到达 done 终态 → 注销 pending-notification + 通知 Interface 层
|
|
1196
|
+
// [OR-4] M12 同款围栏
|
|
1197
|
+
emitTerminalSideEffects(run, deps, "handleScriptError (done,failed)");
|
|
982
1198
|
}
|
|
983
1199
|
|
|
984
1200
|
// ── scheduleRebuild(退避 + 重建) ──────────────────────────
|
|
@@ -988,31 +1204,85 @@ export async function handleScriptError(
|
|
|
988
1204
|
*
|
|
989
1205
|
* 退避期间 run 可能被 abort(转终态 done)——rebuildRuntime 前重检状态,终态时
|
|
990
1206
|
* 跳过重建(避免给已终止的 run 启新 worker)。
|
|
1207
|
+
*
|
|
1208
|
+
* [OR-2] rebuildRuntime 抛错(workerHost.start 失败:线程/内存耗尽、eval 编译失败等)
|
|
1209
|
+
* 不再裸抛——裸抛会沿 handleWorkerError/handleScriptError 的 await 链冒泡到
|
|
1210
|
+
* worker-host 的 `void handlers.onXxx(...)` 变 unhandledRejection,且 run 卡 running
|
|
1211
|
+
* (旧 worker 已死、新 worker 未建,再无任何事件可达)。本函数 catch 后回灌重试矩阵
|
|
1212
|
+
* (handleRebuildStartFailure):计入 workerErrorCount,未超限再退避重建,超限收敛
|
|
1213
|
+
* done,failed——恢复机制在它自己的恢复路径上不再开口。
|
|
991
1214
|
*/
|
|
992
1215
|
async function scheduleRebuild(
|
|
993
1216
|
run: WorkflowRun,
|
|
994
1217
|
deps: LifecycleDeps,
|
|
995
1218
|
handlers: WorkerHandlers,
|
|
996
1219
|
): Promise<void> {
|
|
997
|
-
|
|
1220
|
+
// 用当前重试计数算退避(workerErrorCount 或 scriptErrorCount 已递增)
|
|
998
1221
|
const retryIndex = Math.max(
|
|
999
1222
|
run.meta.workerErrorCount ?? 0,
|
|
1000
1223
|
run.meta.scriptErrorCount ?? 0,
|
|
1001
1224
|
);
|
|
1002
1225
|
await delay(backoffDelay(retryIndex));
|
|
1003
1226
|
|
|
1004
|
-
|
|
1227
|
+
// 退避期间状态可能变化——重检
|
|
1005
1228
|
if (isTerminal(run)) return;
|
|
1006
1229
|
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1230
|
+
// [race-F3] 时间预算折算后已耗尽 → 不再 rebuild 重试,直接 time_limited 终态。
|
|
1231
|
+
// 必须在退避 delay 之后、rebuildRuntime 之前检查:检查前移会在「退避期间耗尽」的
|
|
1232
|
+
// 窗口漏判(rebuild 挂不出 timer,run 预算静默失效);检查点与 rebuildRuntime 的
|
|
1233
|
+
// 计时器挂载之间无 await,remaining > 0 判定不会失效。
|
|
1011
1234
|
const remainingMs = remainingTimeBudgetMs(run);
|
|
1012
1235
|
if (remainingMs !== undefined && remainingMs <= 0) {
|
|
1013
1236
|
await finalizeTimeBudgetExhausted(run, deps);
|
|
1014
1237
|
return;
|
|
1015
1238
|
}
|
|
1016
1239
|
|
|
1017
|
-
|
|
1240
|
+
try {
|
|
1241
|
+
rebuildRuntime(run, deps, handlers);
|
|
1242
|
+
} catch (err) {
|
|
1243
|
+
await handleRebuildStartFailure(run, err, deps, handlers);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
/**
|
|
1248
|
+
* [OR-2] rebuildRuntime 抛错回灌重试矩阵。
|
|
1249
|
+
*
|
|
1250
|
+
* 重建动作本身失败按 worker 家族计数(重建的就是 worker)——计入 run.meta.workerErrorCount
|
|
1251
|
+
* (跨 runtime 存活的重试计数载体),与既有 handleWorkerError 共用同一上限
|
|
1252
|
+
* MAX_WORKER_RETRIES 与退避序列:
|
|
1253
|
+
* - count <= MAX → 递归 scheduleRebuild(天然复用退避 / isTerminal 重检 / 预算折算守卫;
|
|
1254
|
+
* 每轮计数 +1,递归深度有界 ≤ MAX_WORKER_RETRIES);
|
|
1255
|
+
* - count > MAX → 收敛 done,failed(transition + 收口 in-flight + 持久化 + 围栏副作用),
|
|
1256
|
+
* run 不再卡 running。
|
|
1257
|
+
*
|
|
1258
|
+
* 终态路径顺序与其余 handle* 对齐:transition → closeOut → save → 围栏 emit/onRunDone。
|
|
1259
|
+
*/
|
|
1260
|
+
async function handleRebuildStartFailure(
|
|
1261
|
+
run: WorkflowRun,
|
|
1262
|
+
err: unknown,
|
|
1263
|
+
deps: LifecycleDeps,
|
|
1264
|
+
handlers: WorkerHandlers,
|
|
1265
|
+
): Promise<void> {
|
|
1266
|
+
if (isTerminal(run)) return;
|
|
1267
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1268
|
+
const count = (run.meta.workerErrorCount ?? 0) + 1;
|
|
1269
|
+
run.meta.workerErrorCount = count;
|
|
1270
|
+
logger.error(
|
|
1271
|
+
`[workflow] rebuildRuntime failed (runId=${run.runId}, attempt ${count}/${MAX_WORKER_RETRIES}): ${message}`,
|
|
1272
|
+
);
|
|
1273
|
+
|
|
1274
|
+
if (count <= MAX_WORKER_RETRIES) {
|
|
1275
|
+
await scheduleRebuild(run, deps, handlers);
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
// 耗尽 → 收敛 done,failed(不卡 running)
|
|
1280
|
+
run.state.error = `Runtime rebuild failed after ${MAX_WORKER_RETRIES} retries: ${message}`;
|
|
1281
|
+
deps.log?.("debug", "workflow:error-recovery", "rebuild retries exhausted, transition done", { runId: run.runId, count });
|
|
1282
|
+
run.transition("done", "failed");
|
|
1283
|
+
// [OR-8] 终态收口残留 in-flight call(先收口再落盘)
|
|
1284
|
+
closeOutInFlightCalls(run);
|
|
1285
|
+
await saveRunBestEffort(run, deps, "handleRebuildStartFailure (done,failed)");
|
|
1286
|
+
// [OR-4] M12 同款围栏
|
|
1287
|
+
emitTerminalSideEffects(run, deps, "handleRebuildStartFailure (done,failed)");
|
|
1018
1288
|
}
|