@octopi-agent/engine 0.86.38 → 0.97.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/dist/config-admin/agent-presets.d.ts +70 -0
  2. package/dist/config-admin/agent-presets.js +243 -0
  3. package/dist/config-admin/discover.d.ts +30 -0
  4. package/dist/config-admin/discover.js +48 -0
  5. package/dist/config-admin/env.d.ts +22 -0
  6. package/dist/config-admin/env.js +41 -0
  7. package/dist/config-admin/index.d.ts +20 -0
  8. package/dist/config-admin/index.js +15 -0
  9. package/dist/config-admin/model-presets.d.ts +75 -0
  10. package/dist/config-admin/model-presets.js +249 -0
  11. package/dist/config-admin/name-guard.d.ts +14 -0
  12. package/dist/config-admin/name-guard.js +24 -0
  13. package/dist/config-admin/patch.d.ts +56 -0
  14. package/dist/config-admin/patch.js +201 -0
  15. package/dist/config-admin/port.d.ts +68 -0
  16. package/dist/config-admin/port.js +21 -0
  17. package/dist/config-admin/redact.d.ts +39 -0
  18. package/dist/config-admin/redact.js +42 -0
  19. package/dist/config-schema/agent.d.ts +43 -1
  20. package/dist/config-schema/agent.js +12 -5
  21. package/dist/config-schema/context.d.ts +7 -2
  22. package/dist/config-schema/context.js +8 -0
  23. package/dist/config-schema/engine.d.ts +110 -8
  24. package/dist/config-schema/governance.d.ts +43 -1
  25. package/dist/config-schema/governance.js +160 -2
  26. package/dist/config-schema/models.d.ts +7 -4
  27. package/dist/config-schema/models.js +4 -1
  28. package/dist/config-schema/observability.d.ts +9 -0
  29. package/dist/config-schema/observability.js +29 -0
  30. package/dist/config-schema/run.d.ts +3 -0
  31. package/dist/config-schema/run.js +14 -0
  32. package/dist/config-schema/shared.d.ts +3 -0
  33. package/dist/config-schema/shared.js +12 -0
  34. package/dist/config-schema/substrate.d.ts +1 -1
  35. package/dist/config.d.ts +60 -13
  36. package/dist/config.js +81 -78
  37. package/dist/harness/activation/dispatcher.d.ts +8 -2
  38. package/dist/harness/activation/dispatcher.js +33 -5
  39. package/dist/harness/activation/inbox.d.ts +87 -0
  40. package/dist/harness/activation/inbox.js +169 -0
  41. package/dist/harness/activation/runtime.d.ts +9 -8
  42. package/dist/harness/activation/runtime.js +100 -8
  43. package/dist/harness/activation/types.d.ts +19 -0
  44. package/dist/harness/activation/types.js +2 -0
  45. package/dist/harness/agent/builder.d.ts +18 -3
  46. package/dist/harness/agent/builder.js +42 -17
  47. package/dist/harness/agent/config-bridge.d.ts +5 -0
  48. package/dist/harness/agent/config-bridge.js +7 -1
  49. package/dist/harness/agent/config-integrity.d.ts +72 -0
  50. package/dist/harness/agent/config-integrity.js +146 -0
  51. package/dist/harness/agent/tool-purpose.d.ts +9 -1
  52. package/dist/harness/agent/tool-purpose.js +11 -2
  53. package/dist/harness/agent/wire-agent-builder.js +11 -3
  54. package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.d.ts +11 -1
  55. package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.js +11 -1
  56. package/dist/harness/collaboration/autonomous-subsystem/loader.d.ts +13 -0
  57. package/dist/harness/collaboration/autonomous-subsystem/loader.js +16 -1
  58. package/dist/harness/collaboration/autonomous-subsystem/runtime.d.ts +4 -0
  59. package/dist/harness/collaboration/autonomous-subsystem/runtime.js +3 -1
  60. package/dist/harness/collaboration/autonomous-subsystem/think/executor.js +3 -1
  61. package/dist/harness/collaboration/delegation/child-run.js +21 -5
  62. package/dist/harness/collaboration/delegation/notifier.js +4 -3
  63. package/dist/harness/collaboration/delegation/service.js +2 -3
  64. package/dist/harness/collaboration/delegation/wiring.js +6 -5
  65. package/dist/harness/collaboration/orchestration/reflector/llm-reflector.d.ts +1 -1
  66. package/dist/harness/collaboration/orchestration/reflector/llm-reflector.js +1 -1
  67. package/dist/harness/context/capabilities/compact/engine.d.ts +30 -0
  68. package/dist/harness/context/capabilities/compact/engine.js +198 -46
  69. package/dist/harness/context/capabilities/compact/handover.d.ts +78 -0
  70. package/dist/harness/context/capabilities/compact/handover.js +107 -0
  71. package/dist/harness/context/capabilities/compact/index.d.ts +3 -1
  72. package/dist/harness/context/capabilities/compact/index.js +2 -1
  73. package/dist/harness/context/capabilities/compact/types.d.ts +30 -1
  74. package/dist/harness/context/capabilities/summary/executor.js +2 -1
  75. package/dist/harness/context/capabilities/summary/policies.js +8 -5
  76. package/dist/harness/context/constitution/default-agents.md +11 -10
  77. package/dist/harness/context/default-context-engine.d.ts +34 -4
  78. package/dist/harness/context/default-context-engine.js +198 -53
  79. package/dist/harness/context/hybrid-compressor.js +6 -0
  80. package/dist/harness/context/layer-types.d.ts +36 -0
  81. package/dist/harness/context/layer-types.js +38 -0
  82. package/dist/harness/context/llm-summarizer.d.ts +2 -8
  83. package/dist/harness/context/llm-summarizer.js +69 -91
  84. package/dist/harness/context/message-selector.js +4 -3
  85. package/dist/harness/context/root-instructions.js +1 -1
  86. package/dist/harness/context/summarize.d.ts +2 -0
  87. package/dist/harness/context/summarize.js +7 -2
  88. package/dist/harness/context/system-prompt-assembler.js +10 -4
  89. package/dist/harness/context/tool-result-aging.js +2 -2
  90. package/dist/harness/context/truncate-compressor.js +7 -0
  91. package/dist/harness/context/turn-envelope.d.ts +4 -3
  92. package/dist/harness/context/turn-envelope.js +4 -3
  93. package/dist/harness/context/types.d.ts +33 -1
  94. package/dist/harness/extension/execution-environment/dangerous-commands.d.ts +28 -0
  95. package/dist/harness/extension/execution-environment/dangerous-commands.js +129 -0
  96. package/dist/harness/extension/execution-environment/effect-policy.d.ts +133 -4
  97. package/dist/harness/extension/execution-environment/effect-policy.js +263 -14
  98. package/dist/harness/extension/execution-environment/index.d.ts +4 -2
  99. package/dist/harness/extension/execution-environment/index.js +2 -1
  100. package/dist/harness/extension/execution-environment/isolation.js +5 -1
  101. package/dist/harness/extension/execution-environment/kill-switch.d.ts +34 -0
  102. package/dist/harness/extension/execution-environment/kill-switch.js +46 -0
  103. package/dist/harness/extension/execution-environment/ops-audit.d.ts +200 -0
  104. package/dist/harness/extension/execution-environment/ops-audit.js +338 -0
  105. package/dist/harness/extension/execution-environment/privileged-exec.d.ts +158 -0
  106. package/dist/harness/extension/execution-environment/privileged-exec.js +418 -0
  107. package/dist/harness/extension/execution-environment/process-info.d.ts +180 -0
  108. package/dist/harness/extension/execution-environment/process-info.js +466 -0
  109. package/dist/harness/extension/execution-environment/process-isolation.d.ts +7 -0
  110. package/dist/harness/extension/execution-environment/process-isolation.js +9 -0
  111. package/dist/harness/extension/execution-environment/sandbox.d.ts +109 -0
  112. package/dist/harness/extension/execution-environment/sandbox.js +264 -29
  113. package/dist/harness/extension/execution-environment/tool-capabilities.d.ts +1 -1
  114. package/dist/harness/extension/execution-environment/tool-capabilities.js +6 -0
  115. package/dist/harness/extension/execution-environment/types.d.ts +19 -0
  116. package/dist/harness/extension/plugin-ecosystem/channel-types.d.ts +5 -0
  117. package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.d.ts +6 -0
  118. package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.js +50 -0
  119. package/dist/harness/extension/plugin-ecosystem/tools/ask-user.d.ts +4 -5
  120. package/dist/harness/extension/plugin-ecosystem/tools/ask-user.js +5 -15
  121. package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +3 -0
  122. package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +53 -2
  123. package/dist/harness/extension/plugin-ecosystem/tools/file-search.js +10 -17
  124. package/dist/harness/extension/plugin-ecosystem/tools/git-runner.d.ts +31 -0
  125. package/dist/harness/extension/plugin-ecosystem/tools/git-runner.js +104 -0
  126. package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.d.ts +111 -0
  127. package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.js +197 -0
  128. package/dist/harness/extension/plugin-ecosystem/tools/git-tools.d.ts +8 -8
  129. package/dist/harness/extension/plugin-ecosystem/tools/git-tools.js +9 -100
  130. package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +6 -0
  131. package/dist/harness/extension/plugin-ecosystem/tools/index.js +4 -0
  132. package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.d.ts +43 -0
  133. package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.js +350 -0
  134. package/dist/harness/extension/plugin-ecosystem/tools/process-tools.d.ts +30 -0
  135. package/dist/harness/extension/plugin-ecosystem/tools/process-tools.js +137 -0
  136. package/dist/harness/governance/human-in-the-loop/decision-cache.d.ts +54 -4
  137. package/dist/harness/governance/human-in-the-loop/decision-cache.js +42 -16
  138. package/dist/harness/governance/security/temp-path.d.ts +9 -0
  139. package/dist/harness/governance/security/temp-path.js +13 -2
  140. package/dist/harness/governance/session-acl/seed-roles.js +14 -0
  141. package/dist/harness/index.d.ts +4 -0
  142. package/dist/harness/index.js +3 -0
  143. package/dist/harness/knowledge/file-identity.d.ts +8 -0
  144. package/dist/harness/knowledge/file-identity.js +26 -1
  145. package/dist/harness/knowledge/fts.js +1 -2
  146. package/dist/harness/knowledge/grounding.js +5 -5
  147. package/dist/harness/knowledge/index-store.d.ts +31 -4
  148. package/dist/harness/knowledge/index-store.js +180 -48
  149. package/dist/harness/knowledge/ingest.d.ts +115 -115
  150. package/dist/harness/knowledge/ingest.js +150 -150
  151. package/dist/harness/knowledge/knowledge-serve-child.js +1 -1
  152. package/dist/harness/knowledge/purge.js +4 -1
  153. package/dist/harness/memory/backfill-coverage.js +3 -1
  154. package/dist/harness/memory/sqlite/keyword-search.d.ts +1 -0
  155. package/dist/harness/memory/sqlite/keyword-search.js +4 -3
  156. package/dist/harness/observability/observer/hub.js +38 -1
  157. package/dist/harness/observability/observer/run-snapshot.js +1 -0
  158. package/dist/harness/observability/observer/types.d.ts +27 -0
  159. package/dist/harness/run/agent/agent.d.ts +7 -0
  160. package/dist/harness/run/agent/agent.js +23 -1
  161. package/dist/harness/run/reliability/fallback-provider.d.ts +117 -5
  162. package/dist/harness/run/reliability/fallback-provider.js +262 -22
  163. package/dist/harness/run/reliability/harness-events.d.ts +24 -1
  164. package/dist/harness/run/reliability/model-error-policy.d.ts +32 -0
  165. package/dist/harness/run/reliability/model-error-policy.js +42 -0
  166. package/dist/harness/run/reliability/provider-wrapper.js +29 -2
  167. package/dist/harness/run/reliability/refusal-guard.d.ts +20 -0
  168. package/dist/harness/run/reliability/refusal-guard.js +102 -0
  169. package/dist/harness/run/reliability/run-agent.js +231 -33
  170. package/dist/harness/run/run-guard/default-run-guard.js +1 -1
  171. package/dist/harness/run/run-scope.d.ts +31 -0
  172. package/dist/harness/run/runner.d.ts +132 -2
  173. package/dist/harness/run/runner.js +679 -60
  174. package/dist/harness/session/attachments/llm-intent.js +2 -1
  175. package/dist/harness/session/history/port.js +15 -5
  176. package/dist/harness/session/in-memory-store.d.ts +5 -0
  177. package/dist/harness/session/in-memory-store.js +33 -2
  178. package/dist/harness/session/tasks/render.d.ts +4 -1
  179. package/dist/harness/session/tasks/render.js +10 -7
  180. package/dist/harness/session/title.js +25 -9
  181. package/dist/harness/shared/events/agent-event-map.d.ts +92 -0
  182. package/dist/harness/shared/events/agent-event-map.js +7 -0
  183. package/dist/index.d.ts +6 -4
  184. package/dist/index.js +4 -3
  185. package/dist/integration/agent-runtime/channel-message-source.js +19 -1
  186. package/dist/integration/index.d.ts +2 -0
  187. package/dist/integration/index.js +1 -0
  188. package/dist/integration/observability/loop-watchdog.d.ts +56 -0
  189. package/dist/integration/observability/loop-watchdog.js +177 -0
  190. package/dist/integration/observability/metrics.d.ts +3 -0
  191. package/dist/integration/observability/metrics.js +18 -7
  192. package/dist/integration/observability/run-telemetry.d.ts +8 -0
  193. package/dist/integration/observability/run-telemetry.js +31 -8
  194. package/dist/integration/observability/trace-collector.js +26 -4
  195. package/dist/integration/observability/trace-events.d.ts +1 -0
  196. package/dist/integration/observability/trace-events.js +2 -0
  197. package/dist/integration/observability/trace-logger.js +27 -17
  198. package/dist/integration/providers/anthropic.d.ts +23 -0
  199. package/dist/integration/providers/anthropic.js +97 -12
  200. package/dist/integration/providers/fetch-timeout.d.ts +42 -0
  201. package/dist/integration/providers/fetch-timeout.js +51 -0
  202. package/dist/integration/providers/openai.d.ts +10 -0
  203. package/dist/integration/providers/openai.js +58 -11
  204. package/dist/integration/providers/provider-diagnostics.d.ts +80 -0
  205. package/dist/integration/providers/provider-diagnostics.js +89 -0
  206. package/dist/integration/providers/timeout-coherence.d.ts +52 -0
  207. package/dist/integration/providers/timeout-coherence.js +68 -0
  208. package/dist/integration/providers/wire-dump.d.ts +62 -0
  209. package/dist/integration/providers/wire-dump.js +111 -2
  210. package/dist/integration/storage/jsonl.d.ts +19 -1
  211. package/dist/integration/storage/jsonl.js +100 -4
  212. package/dist/paths.d.ts +19 -0
  213. package/dist/paths.js +38 -1
  214. package/dist/subsystems/cognition-steward/conceptualize/handler.js +1 -1
  215. package/dist/subsystems/memory-steward/backfill/handler.js +1 -1
  216. package/dist/subsystems/safety-guard/config.yaml +3 -1
  217. package/dist/subsystems/wisdom-steward/formation/handler.js +1 -1
  218. package/package.json +3 -2
@@ -0,0 +1,177 @@
1
+ /**
2
+ * 事件循环停滞看门狗(宿主自检器官)
3
+ *
4
+ * 背景(2026-10-28 事故):网关事件循环被同步阻塞整体冻结约 4.5 小时——两个 run 的
5
+ * turn 同时停摆、HTTP accept 停止、SIGTERM handler 无法运行,最终被 `serve restart`
6
+ * 的 SIGKILL 升级收尾。事后缺一份阻塞现场的栈采样,确切阻塞点永久失考。
7
+ *
8
+ * 机制:独立 worker 线程(不随主循环冻结)定期 ping 主线程;主线程 message handler
9
+ * 回 pong 以证明事件循环存活。主循环冻结 → pong 静默超阈值 → worker 判定停滞:
10
+ * 写 JSONL 报告 +(macOS)自动 `sample <pid>` 抓主线程栈。检测与取证全部发生在
11
+ * worker 侧,不依赖已被冻结的事件循环。
12
+ *
13
+ * 第二道闸:进程 RSS 越过 `footprintThresholdMb` 即同样落报告 + 采栈。停滞检测
14
+ * 只能证明「已经僵死」,内存失控则可能在冻死前给出一段可干预的窗口——2026-10-30
15
+ * 网关膨胀到 30.5G 的进程在采样完成后几分钟就被系统收走。
16
+ *
17
+ * 设计约束:
18
+ * - 观测 fail-open:看门狗自身的任何失败不得影响宿主(写盘/采样失败静默吞掉)。
19
+ * - worker unref:不阻止进程退出;宿主应显式 `stop()` 收尾。
20
+ * - 复发 runbook:报告文件给出停滞起点(lastPongAt),`.sample-*.txt` 给出阻塞栈;
21
+ * 若自动采样失败,在杀进程前手动执行 `sample <pid> 5 -f /tmp/octopi-hang.txt`。
22
+ *
23
+ * @module integration/observability/loop-watchdog
24
+ */
25
+ import { Worker } from 'node:worker_threads';
26
+ import { join } from 'node:path';
27
+ import { getOctopiHome } from '../../paths.js';
28
+ /**
29
+ * worker 侧脚本(eval 载入;此处为字符串以规避 dist/源码双态下的模块解析差异)
30
+ *
31
+ * 职责:心跳 ping、停滞判定、报告落盘、栈采样。不得引用 worker_threads 之外的
32
+ * 外部模块形态(eval worker 无 ESM import,只能 require)。
33
+ */
34
+ const WORKER_SRC = `
35
+ const { parentPort, workerData } = require('node:worker_threads');
36
+ const { appendFileSync, mkdirSync } = require('node:fs');
37
+ const { dirname } = require('node:path');
38
+ const { spawnSync } = require('node:child_process');
39
+
40
+ let lastPongAt = Date.now();
41
+ let stalledSince = undefined;
42
+ let footprintLatched = false;
43
+
44
+ const writeReport = (record) => {
45
+ try {
46
+ mkdirSync(dirname(workerData.reportFile), { recursive: true });
47
+ appendFileSync(workerData.reportFile, JSON.stringify(record) + '\\n');
48
+ } catch {
49
+ // 观测 fail-open:报告写盘失败不得影响 worker 自身的监测
50
+ }
51
+ };
52
+
53
+ const captureStack = (tag) => {
54
+ if (!workerData.captureSample || process.platform !== 'darwin') return;
55
+ const sampleFile = workerData.sampleFile.replace('{{ts}}', tag + '-' + Date.now());
56
+ try {
57
+ const r = spawnSync('sample', [String(process.pid), '3', '-f', sampleFile], {
58
+ timeout: 15000,
59
+ stdio: 'ignore',
60
+ });
61
+ // spawnSync 失败不抛(ENOENT 等走返回值):必须按返回值如实留痕,
62
+ // 否则「宣称采样成功」会把运维引向不存在的文件
63
+ if (r.error == null && r.status === 0) {
64
+ console.error('[LoopWatchdog] stack sample written: ' + sampleFile);
65
+ } else {
66
+ console.error('[LoopWatchdog] stack sample FAILED (' + (r.error ? r.error.message : 'exit ' + r.status) + '); run manually: sample ' + process.pid + ' 5 -f /tmp/octopi-hang.txt');
67
+ }
68
+ } catch (err) {
69
+ console.error('[LoopWatchdog] stack sample FAILED: ' + String(err) + '; run manually: sample ' + process.pid + ' 5 -f /tmp/octopi-hang.txt');
70
+ }
71
+ };
72
+
73
+ parentPort.on('message', (msg) => {
74
+ if (msg && msg.type === 'pong') {
75
+ lastPongAt = Date.now();
76
+ if (stalledSince !== undefined) {
77
+ writeReport({
78
+ event: 'recovered',
79
+ detectedAt: new Date().toISOString(),
80
+ stalledForMs: Date.now() - stalledSince,
81
+ });
82
+ console.error('[LoopWatchdog] event loop recovered after ' + (Date.now() - stalledSince) + 'ms stall');
83
+ stalledSince = undefined;
84
+ }
85
+ }
86
+ });
87
+
88
+ setInterval(() => {
89
+ parentPort.postMessage({ type: 'ping' });
90
+ const silentForMs = Date.now() - lastPongAt;
91
+ if (stalledSince === undefined && silentForMs >= workerData.stallThresholdMs) {
92
+ stalledSince = lastPongAt;
93
+ writeReport({
94
+ event: 'stall',
95
+ detectedAt: new Date().toISOString(),
96
+ silentForMs,
97
+ lastPongAt,
98
+ pid: process.pid,
99
+ });
100
+ console.error('[LoopWatchdog] event loop stalled: no heartbeat for ' + silentForMs + 'ms (pid ' + process.pid + ')');
101
+ captureStack('stall');
102
+ }
103
+
104
+ // 内存足迹阈值:在撑爆前留出干预窗口。
105
+ // 2026-10-30 事故里 footprint 涨到 30.5G 时主线程已被原生 h2 扩容循环吃满,
106
+ // 等到心跳消失再采样只是事后取证——进程几分钟后就被系统收走,无从止损。
107
+ // 滞回(跌回 80% 以下才复位)避免在阈值附近反复采样。
108
+ const thresholdMb = workerData.footprintThresholdMb;
109
+ if (thresholdMb > 0) {
110
+ const rssMb = Math.round(process.memoryUsage().rss / 1048576);
111
+ if (!footprintLatched && rssMb >= thresholdMb) {
112
+ footprintLatched = true;
113
+ writeReport({
114
+ event: 'footprint',
115
+ detectedAt: new Date().toISOString(),
116
+ rssMb,
117
+ thresholdMb,
118
+ pid: process.pid,
119
+ });
120
+ console.error('[LoopWatchdog] footprint ' + rssMb + 'MB >= threshold ' + thresholdMb + 'MB (pid ' + process.pid + ')');
121
+ captureStack('footprint');
122
+ } else if (footprintLatched && rssMb < thresholdMb * 0.8) {
123
+ footprintLatched = false;
124
+ writeReport({
125
+ event: 'footprint_recovered',
126
+ detectedAt: new Date().toISOString(),
127
+ rssMb,
128
+ thresholdMb,
129
+ });
130
+ console.error('[LoopWatchdog] footprint back under threshold: ' + rssMb + 'MB');
131
+ }
132
+ }
133
+ }, workerData.intervalMs);
134
+ `;
135
+ /**
136
+ * 启动事件循环停滞看门狗
137
+ *
138
+ * @param options - 心跳间隔 / 停滞阈值 / 报告路径 / 是否自动栈采样
139
+ * @returns 看门狗句柄(`stop()` 回收 worker)
140
+ */
141
+ export function startLoopWatchdog(options = {}) {
142
+ if (options.enabled === false) {
143
+ return { stop: () => { } };
144
+ }
145
+ const reportFile = options.reportFile ?? join(getOctopiHome(), 'logs', 'loop-watchdog.jsonl');
146
+ const worker = new Worker(WORKER_SRC, {
147
+ eval: true,
148
+ workerData: {
149
+ intervalMs: options.intervalMs ?? 5_000,
150
+ stallThresholdMs: options.stallThresholdMs ?? 30_000,
151
+ reportFile,
152
+ captureSample: options.captureSample ?? true,
153
+ sampleFile: `${reportFile}.sample-{{ts}}.txt`,
154
+ footprintThresholdMb: options.footprintThresholdMb ?? 4096,
155
+ },
156
+ });
157
+ // 心跳应答(主循环冻结时此 handler 不再被调度——正是停滞信号)
158
+ worker.on('message', (msg) => {
159
+ if (msg?.type === 'ping') {
160
+ worker.postMessage({ type: 'pong' });
161
+ }
162
+ });
163
+ worker.on('error', (err) => {
164
+ // 观测 fail-open:worker 自身崩溃不得影响宿主;但必须留痕——否则自检能力
165
+ // 整个进程生命周期无声缺席(eval worker 启动失败/平台差异)
166
+ console.warn(`[LoopWatchdog] worker error, stall detection disabled: ${String(err)}`);
167
+ });
168
+ // 不阻止进程退出
169
+ worker.unref();
170
+ return {
171
+ stop: () => {
172
+ worker.terminate().catch(() => {
173
+ // 关停竞态(worker 已退出)无害
174
+ });
175
+ },
176
+ };
177
+ }
@@ -50,6 +50,8 @@ export interface MetricsSnapshot {
50
50
  sessionsActive: number;
51
51
  /** 总 turn 数 */
52
52
  turnsTotal: number;
53
+ /** 以失败告终的 turn 数(turn.end 带 error;含重试中失败尝试,失败不再是指标盲区) */
54
+ failedTurns: number;
53
55
  /** 空回复数 */
54
56
  emptyResponses: number;
55
57
  /** 重试次数 */
@@ -84,6 +86,7 @@ export declare class MetricsAggregator {
84
86
  private _toolErrors;
85
87
  private _sessionsActive;
86
88
  private _turnsTotal;
89
+ private _failedTurns;
87
90
  private _emptyResponses;
88
91
  private _retriesTotal;
89
92
  private _snapshots;
@@ -63,6 +63,7 @@ export class MetricsAggregator {
63
63
  // Session 指标
64
64
  _sessionsActive = new Set();
65
65
  _turnsTotal = 0;
66
+ _failedTurns = 0;
66
67
  _emptyResponses = 0;
67
68
  _retriesTotal = 0;
68
69
  // 快照历史
@@ -123,14 +124,22 @@ export class MetricsAggregator {
123
124
  break;
124
125
  }
125
126
  // ── Turn ──
126
- case 'turn.start':
127
- this._turnsTotal++;
128
- break;
129
- case 'turn.end':
130
- if (!event.data?.content || event.data.content.length === 0) {
131
- this._emptyResponses++;
127
+ // 只计完整结束的 turn:重试中失败尝试的 turn_end(error) 不是用户语义上的轮次,
128
+ // 否则 turnsTotal 会把失败尝试计成 turn、与 llmCalls 倒挂;失败调用的空输出
129
+ // 也不是「空响应」。
130
+ case 'turn.end': {
131
+ const failed = Boolean(event.data?.error);
132
+ if (!failed) {
133
+ this._turnsTotal++;
134
+ if (!event.data?.content || event.data.content.length === 0) {
135
+ this._emptyResponses++;
136
+ }
137
+ }
138
+ else {
139
+ this._failedTurns++;
132
140
  }
133
141
  break;
142
+ }
134
143
  // ── 重试 ──
135
144
  case 'retry':
136
145
  this._retriesTotal++;
@@ -162,6 +171,7 @@ export class MetricsAggregator {
162
171
  toolErrors: Object.fromEntries(this._toolErrors),
163
172
  sessionsActive: this._sessionsActive.size,
164
173
  turnsTotal: this._turnsTotal,
174
+ failedTurns: this._failedTurns,
165
175
  emptyResponses: this._emptyResponses,
166
176
  retriesTotal: this._retriesTotal,
167
177
  estimatedCostUsd: Math.round(estimatedCostUsd * 10000) / 10000,
@@ -188,6 +198,7 @@ export class MetricsAggregator {
188
198
  this._toolErrors.clear();
189
199
  this._sessionsActive.clear();
190
200
  this._turnsTotal = 0;
201
+ this._failedTurns = 0;
191
202
  this._emptyResponses = 0;
192
203
  this._retriesTotal = 0;
193
204
  this._snapshots = [];
@@ -211,7 +222,7 @@ export function formatMetricsSnapshot(snap) {
211
222
  lines.push(`${'─'.repeat(50)}`);
212
223
  lines.push(`Uptime: ${(snap.uptimeMs / 1000).toFixed(0)}s`);
213
224
  lines.push(`Sessions: ${snap.sessionsActive} active`);
214
- lines.push(`Turns: ${snap.turnsTotal}`);
225
+ lines.push(`Turns: ${snap.turnsTotal} (${snap.failedTurns} failed)`);
215
226
  lines.push(`Empty replies: ${snap.emptyResponses}`);
216
227
  lines.push(`Retries: ${snap.retriesTotal}`);
217
228
  lines.push('');
@@ -18,6 +18,14 @@ export interface CreateRunTelemetryOptions {
18
18
  /** 预创建实例(测试注入) */
19
19
  collector?: TraceCollector;
20
20
  metricsInstance?: MetricsAggregator;
21
+ /**
22
+ * 归因上下文解析(组合根注入,如 RunScope ALS)。
23
+ * LoopObserver 路径的 model.call.* 行靠它带 sessionId/agentId。
24
+ */
25
+ resolveContext?: () => {
26
+ sessionId?: string;
27
+ agentId?: string;
28
+ };
21
29
  }
22
30
  /**
23
31
  * 创建一套 RunTelemetry
@@ -45,31 +45,54 @@ export function createRunTelemetry(options = {}) {
45
45
  /**
46
46
  * 只补 LLM 调用生命周期(Runner 事件流无 model.call.*)。
47
47
  * 工具事件由 onEvent 接收 Runner 的 tool.exec.*,避免双计。
48
+ *
49
+ * 计时按 callId 隔离(同 agent 并发 run 共用本 observer,单变量会互相覆盖)。
48
50
  */
49
51
  createLoopObserver(traceOpts) {
50
52
  const c = ensureCollector(traceOpts);
51
- let llmStartTs = 0;
53
+ /** callId → 计时状态(onLLMEnd 清理,无泄漏) */
54
+ const calls = new Map();
55
+ const ctx = () => options.resolveContext?.() ?? {};
56
+ /** 无 callId 的旧调用方回退键(顺序调用沿用旧单变量语义;并发场景请传 callId) */
57
+ const keyOf = (callId) => callId ?? 'default';
52
58
  return {
53
59
  onLLMStart(params) {
54
- llmStartTs = Date.now();
60
+ const startTs = Date.now();
61
+ calls.set(keyOf(params.callId), { startTs });
55
62
  c.record({
56
63
  type: 'model.call.start',
57
- timestamp: llmStartTs,
58
- data: { model: params.model },
59
- }, {});
64
+ timestamp: startTs,
65
+ data: {
66
+ model: params.model,
67
+ callId: params.callId,
68
+ reqChars: params.reqChars,
69
+ toolChars: params.toolChars,
70
+ },
71
+ }, ctx());
72
+ },
73
+ onLLMFirstToken(params) {
74
+ const call = calls.get(keyOf(params.callId));
75
+ if (call)
76
+ call.ttftMs = params.ttftMs;
60
77
  },
61
78
  onLLMEnd(params) {
62
- const durationMs = llmStartTs ? Date.now() - llmStartTs : 0;
63
- llmStartTs = 0;
79
+ const call = calls.get(keyOf(params.callId));
80
+ calls.delete(keyOf(params.callId));
81
+ // 无 start 记录(异常调用序)时 duration 缺省,不伪造 0
82
+ const durationMs = call ? Date.now() - call.startTs : undefined;
64
83
  c.record({
65
84
  type: 'model.call.end',
66
85
  timestamp: Date.now(),
67
86
  data: {
68
87
  model: params.model,
88
+ callId: params.callId,
69
89
  usage: params.usage,
70
90
  durationMs,
91
+ ttftMs: params.ttftMs ?? call?.ttftMs,
92
+ outChars: params.outChars,
93
+ finishReason: params.finishReason,
71
94
  },
72
- }, {});
95
+ }, ctx());
73
96
  },
74
97
  };
75
98
  },
@@ -46,13 +46,20 @@ export class TraceCollector {
46
46
  const turnId = `turn_${++this.turnCount}`;
47
47
  this.currentTurnId = turnId;
48
48
  this.logger.info(TRACE_EVENTS.TURN_START, { turnId }, ctx);
49
+ let sawTurnEnd = false;
49
50
  for await (const event of events) {
51
+ if (event.type === 'turn.end')
52
+ sawTurnEnd = true;
50
53
  // 记录到 trace
51
54
  this.recordEvent(event, ctx, turnId);
52
55
  // 原样传递
53
56
  yield event;
54
57
  }
55
- this.logger.info(TRACE_EVENTS.TURN_END, { turnId }, ctx);
58
+ // 收口行仅在事件流自身无 turn.end 时补——recordEvent 已把 turn.end 映射为 TURN_END,
59
+ // 再补一行即双计(turn 统计翻倍)
60
+ if (!sawTurnEnd) {
61
+ this.logger.info(TRACE_EVENTS.TURN_END, { turnId }, ctx);
62
+ }
56
63
  }
57
64
  /**
58
65
  * 获取 MetricsAggregator 实例
@@ -92,29 +99,44 @@ export class TraceCollector {
92
99
  case 'turn.start':
93
100
  this.logger.info(TRACE_EVENTS.TURN_START, { turnId }, base);
94
101
  break;
95
- // Runner 词表:turn.end 携带 usage,映射为 model.call.end
102
+ // Runner 词表:turn.end 携带 usage。F3-c 起不再冒充 model.call.end
103
+ //(LoopObserver 的 model.call.end 才带 durationMs/ttft;两行各归各位,p50/p95 取数不双计。
104
+ // F3-d 起 turn.end 事件还带 llmTiming 供 UI timeline 展示,本白名单仍不落盘——trace 指标只认 model.call.*)
96
105
  case 'turn.end':
97
- this.logger.info(TRACE_EVENTS.MODEL_CALL_END, {
106
+ this.logger.info(TRACE_EVENTS.TURN_END, {
98
107
  contentLength: event.data?.content?.length ?? 0,
99
108
  usage: event.data?.usage,
100
109
  turnId,
101
110
  }, base);
102
- this.logger.info(TRACE_EVENTS.TURN_END, { turnId }, base);
103
111
  break;
104
112
  case 'model.call.start':
105
113
  this.logger.info(TRACE_EVENTS.MODEL_CALL_START, {
106
114
  model: event.data?.model,
115
+ callId: event.data?.callId,
116
+ reqChars: event.data?.reqChars,
117
+ toolChars: event.data?.toolChars,
107
118
  turnId,
108
119
  }, base);
109
120
  break;
110
121
  case 'model.call.end':
111
122
  this.logger.info(TRACE_EVENTS.MODEL_CALL_END, {
123
+ model: event.data?.model,
124
+ callId: event.data?.callId,
125
+ // durationMs/ttftMs/reqChars 是 F3 回归指标口径(long-session-latency §7):白名单漏掉即断档
126
+ durationMs: event.data?.durationMs,
127
+ ttftMs: event.data?.ttftMs,
128
+ outChars: event.data?.outChars,
129
+ finishReason: event.data?.finishReason,
112
130
  contentLength: event.data?.content?.length ?? 0,
113
131
  toolCallCount: event.data?.toolCallCount ?? 0,
114
132
  usage: event.data?.usage,
115
133
  turnId,
116
134
  }, base);
117
135
  break;
136
+ // Runner 收口计时(wait/build/run_end;`[timing]` 探针转正,F3)
137
+ case 'run.timing':
138
+ this.logger.info(TRACE_EVENTS.RUN_TIMING, event.data, base);
139
+ break;
118
140
  case 'model.call.error':
119
141
  this.logger.error(TRACE_EVENTS.MODEL_CALL_ERROR, {
120
142
  error: event.data?.error,
@@ -47,6 +47,7 @@ export declare const TRACE_EVENTS: {
47
47
  readonly TOOL_EXEC_END: "tool.exec.end";
48
48
  readonly SESSION_LOAD: "session.load";
49
49
  readonly SESSION_SAVE: "session.save";
50
+ readonly RUN_TIMING: "run.timing";
50
51
  readonly MODEL_RETRY: "model.retry";
51
52
  readonly MODEL_FALLBACK: "model.fallback";
52
53
  readonly EMPTY_RESPONSE: "empty.response";
@@ -42,6 +42,8 @@ export const TRACE_EVENTS = {
42
42
  TOOL_EXEC_END: 'tool.exec.end',
43
43
  SESSION_LOAD: 'session.load',
44
44
  SESSION_SAVE: 'session.save',
45
+ // Runner 收口计时(wait/build/run_end;F3 `[timing]` 探针转正)
46
+ RUN_TIMING: 'run.timing',
45
47
  // Level 2: WARN
46
48
  MODEL_RETRY: 'model.retry',
47
49
  MODEL_FALLBACK: 'model.fallback',
@@ -22,6 +22,8 @@ const DEFAULT_CONFIG = {
22
22
  filePrefix: 'trace',
23
23
  sync: false,
24
24
  };
25
+ /** 进程内 trace 文件序号(同毫秒多实例防撞) */
26
+ let fileSeq = 0;
25
27
  /**
26
28
  * TraceLogger
27
29
  */
@@ -35,26 +37,34 @@ export class TraceLogger {
35
37
  this.config = { ...DEFAULT_CONFIG, ...config };
36
38
  // 初始化导出器
37
39
  this.exporters = config?.exporters ?? [];
38
- // 初始化文件输出(兼容旧的 outputDir 模式)
40
+ // 初始化文件输出(兼容旧的 outputDir 模式)。失败 fail-open:观测绝不影响业务构建
39
41
  if (this.config.outputDir) {
40
42
  const dir = this.config.outputDir;
41
- if (!existsSync(dir)) {
42
- mkdirSync(dir, { recursive: true });
43
+ try {
44
+ if (!existsSync(dir)) {
45
+ mkdirSync(dir, { recursive: true });
46
+ }
47
+ // 文件名唯一化:毫秒时间戳 + pid + 进程内序号(同秒多 agent / 多实例不共写一个文件)
48
+ const ts = new Date().toISOString().replace(/[:.]/g, '-');
49
+ const prefix = this.config.filePrefix ?? 'trace';
50
+ this.filePath = join(dir, `${prefix}-${ts}-${process.pid}-${++fileSeq}.jsonl`);
51
+ // 写入 trace 头部
52
+ this.writeToFile({
53
+ ts: this.startTime,
54
+ level: TraceLevel.INFO,
55
+ type: 'trace.start',
56
+ data: {
57
+ level: TRACE_LEVEL_NAMES[this.config.level],
58
+ outputDir: this.config.outputDir,
59
+ filePath: this.filePath,
60
+ },
61
+ });
62
+ }
63
+ catch (err) {
64
+ // 目录不可用(非法路径/只读):禁用文件通道并告警一次;writeToFile 的空 catch 是写期瞬时失败兜底
65
+ this.filePath = undefined;
66
+ console.warn(`[observability] trace file output disabled (${dir}): ${err instanceof Error ? err.message : String(err)}`);
43
67
  }
44
- const ts = new Date().toISOString().replace(/[:.]/g, '-').substring(0, 19);
45
- const prefix = this.config.filePrefix ?? 'trace';
46
- this.filePath = join(dir, `${prefix}-${ts}.jsonl`);
47
- // 写入 trace 头部
48
- this.writeToFile({
49
- ts: this.startTime,
50
- level: TraceLevel.INFO,
51
- type: 'trace.start',
52
- data: {
53
- level: TRACE_LEVEL_NAMES[this.config.level],
54
- outputDir: this.config.outputDir,
55
- filePath: this.filePath,
56
- },
57
- });
58
68
  }
59
69
  }
60
70
  /**
@@ -49,6 +49,10 @@ export declare class AnthropicProvider implements ModelProvider {
49
49
  private baseUrl;
50
50
  private version;
51
51
  private timeoutMs;
52
+ /** 与 timeoutMs 对齐的 fetch dispatcher(transport 不得在配置值之下提前掐断) */
53
+ private dispatcher;
54
+ /** HTTP transport(可注入替换供测试;默认 undici fetch,错误形态与内置 fetch 一致) */
55
+ private fetch;
52
56
  readonly defaultModel: string;
53
57
  constructor(config: AnthropicProviderConfig);
54
58
  /**
@@ -72,6 +76,12 @@ export declare class AnthropicProvider implements ModelProvider {
72
76
  * 检查 provider 是否可用
73
77
  */
74
78
  isAvailable(): Promise<boolean>;
79
+ /**
80
+ * 释放 transport 资源(keep-alive dispatcher 的连接池)
81
+ *
82
+ * 热更替换(gateway.registerProvider)或宿主显式收口时调用;重复调用安全。
83
+ */
84
+ dispose(): Promise<void>;
75
85
  /**
76
86
  * 构造实际发送的请求体(wire dump 观测用)
77
87
  *
@@ -86,6 +96,19 @@ export declare class AnthropicProvider implements ModelProvider {
86
96
  * OpenAI 格式 → Anthropic 格式
87
97
  */
88
98
  private toAnthropicRequest;
99
+ /**
100
+ * 将消息列表转换为 Anthropic 消息列表(含 tool_result 聚合)
101
+ *
102
+ * Anthropic 协议要求:一个 assistant 回合发出的全部 `tool_use`,其 `tool_result`
103
+ * 必须位于**紧随其后的同一条 user 消息**内。引擎里并行工具调用的结果是一条 tool
104
+ * 消息携带多个 `toolResults`,而历史中相邻的 tool 消息也可能出现;逐条转换会产出
105
+ * 多条连续 user 消息,导致前面的 tool_use 无配对结果而被拒(火山等严格端点只回
106
+ * 通用 InvalidParameter,不指出 param)。
107
+ *
108
+ * @param messages - 非 system 消息列表
109
+ * @returns Anthropic 消息列表(连续 tool 消息已聚合为一条 user 消息)
110
+ */
111
+ private toAnthropicMessages;
89
112
  /**
90
113
  * 将 OpenAI 消息格式转换为 Anthropic 消息格式
91
114
  */