@octopi-agent/engine 0.86.38 → 0.97.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/dist/config-admin/agent-presets.d.ts +70 -0
  2. package/dist/config-admin/agent-presets.js +243 -0
  3. package/dist/config-admin/discover.d.ts +30 -0
  4. package/dist/config-admin/discover.js +48 -0
  5. package/dist/config-admin/env.d.ts +22 -0
  6. package/dist/config-admin/env.js +41 -0
  7. package/dist/config-admin/index.d.ts +20 -0
  8. package/dist/config-admin/index.js +15 -0
  9. package/dist/config-admin/model-presets.d.ts +75 -0
  10. package/dist/config-admin/model-presets.js +249 -0
  11. package/dist/config-admin/name-guard.d.ts +14 -0
  12. package/dist/config-admin/name-guard.js +24 -0
  13. package/dist/config-admin/patch.d.ts +56 -0
  14. package/dist/config-admin/patch.js +201 -0
  15. package/dist/config-admin/port.d.ts +68 -0
  16. package/dist/config-admin/port.js +21 -0
  17. package/dist/config-admin/redact.d.ts +39 -0
  18. package/dist/config-admin/redact.js +42 -0
  19. package/dist/config-schema/agent.d.ts +43 -1
  20. package/dist/config-schema/agent.js +12 -5
  21. package/dist/config-schema/context.d.ts +7 -2
  22. package/dist/config-schema/context.js +8 -0
  23. package/dist/config-schema/engine.d.ts +110 -8
  24. package/dist/config-schema/governance.d.ts +43 -1
  25. package/dist/config-schema/governance.js +160 -2
  26. package/dist/config-schema/models.d.ts +7 -4
  27. package/dist/config-schema/models.js +4 -1
  28. package/dist/config-schema/observability.d.ts +9 -0
  29. package/dist/config-schema/observability.js +29 -0
  30. package/dist/config-schema/run.d.ts +3 -0
  31. package/dist/config-schema/run.js +14 -0
  32. package/dist/config-schema/shared.d.ts +3 -0
  33. package/dist/config-schema/shared.js +12 -0
  34. package/dist/config-schema/substrate.d.ts +1 -1
  35. package/dist/config.d.ts +60 -13
  36. package/dist/config.js +81 -78
  37. package/dist/harness/activation/dispatcher.d.ts +8 -2
  38. package/dist/harness/activation/dispatcher.js +33 -5
  39. package/dist/harness/activation/inbox.d.ts +87 -0
  40. package/dist/harness/activation/inbox.js +169 -0
  41. package/dist/harness/activation/runtime.d.ts +9 -8
  42. package/dist/harness/activation/runtime.js +100 -8
  43. package/dist/harness/activation/types.d.ts +19 -0
  44. package/dist/harness/activation/types.js +2 -0
  45. package/dist/harness/agent/builder.d.ts +18 -3
  46. package/dist/harness/agent/builder.js +42 -17
  47. package/dist/harness/agent/config-bridge.d.ts +5 -0
  48. package/dist/harness/agent/config-bridge.js +7 -1
  49. package/dist/harness/agent/config-integrity.d.ts +72 -0
  50. package/dist/harness/agent/config-integrity.js +146 -0
  51. package/dist/harness/agent/tool-purpose.d.ts +9 -1
  52. package/dist/harness/agent/tool-purpose.js +11 -2
  53. package/dist/harness/agent/wire-agent-builder.js +11 -3
  54. package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.d.ts +11 -1
  55. package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.js +11 -1
  56. package/dist/harness/collaboration/autonomous-subsystem/loader.d.ts +13 -0
  57. package/dist/harness/collaboration/autonomous-subsystem/loader.js +16 -1
  58. package/dist/harness/collaboration/autonomous-subsystem/runtime.d.ts +4 -0
  59. package/dist/harness/collaboration/autonomous-subsystem/runtime.js +3 -1
  60. package/dist/harness/collaboration/autonomous-subsystem/think/executor.js +3 -1
  61. package/dist/harness/collaboration/delegation/child-run.js +21 -5
  62. package/dist/harness/collaboration/delegation/notifier.js +4 -3
  63. package/dist/harness/collaboration/delegation/service.js +2 -3
  64. package/dist/harness/collaboration/delegation/wiring.js +6 -5
  65. package/dist/harness/collaboration/orchestration/reflector/llm-reflector.d.ts +1 -1
  66. package/dist/harness/collaboration/orchestration/reflector/llm-reflector.js +1 -1
  67. package/dist/harness/context/capabilities/compact/engine.d.ts +30 -0
  68. package/dist/harness/context/capabilities/compact/engine.js +198 -46
  69. package/dist/harness/context/capabilities/compact/handover.d.ts +78 -0
  70. package/dist/harness/context/capabilities/compact/handover.js +107 -0
  71. package/dist/harness/context/capabilities/compact/index.d.ts +3 -1
  72. package/dist/harness/context/capabilities/compact/index.js +2 -1
  73. package/dist/harness/context/capabilities/compact/types.d.ts +30 -1
  74. package/dist/harness/context/capabilities/summary/executor.js +2 -1
  75. package/dist/harness/context/capabilities/summary/policies.js +8 -5
  76. package/dist/harness/context/constitution/default-agents.md +11 -10
  77. package/dist/harness/context/default-context-engine.d.ts +34 -4
  78. package/dist/harness/context/default-context-engine.js +198 -53
  79. package/dist/harness/context/hybrid-compressor.js +6 -0
  80. package/dist/harness/context/layer-types.d.ts +36 -0
  81. package/dist/harness/context/layer-types.js +38 -0
  82. package/dist/harness/context/llm-summarizer.d.ts +2 -8
  83. package/dist/harness/context/llm-summarizer.js +69 -91
  84. package/dist/harness/context/message-selector.js +4 -3
  85. package/dist/harness/context/root-instructions.js +1 -1
  86. package/dist/harness/context/summarize.d.ts +2 -0
  87. package/dist/harness/context/summarize.js +7 -2
  88. package/dist/harness/context/system-prompt-assembler.js +10 -4
  89. package/dist/harness/context/tool-result-aging.js +2 -2
  90. package/dist/harness/context/truncate-compressor.js +7 -0
  91. package/dist/harness/context/turn-envelope.d.ts +4 -3
  92. package/dist/harness/context/turn-envelope.js +4 -3
  93. package/dist/harness/context/types.d.ts +33 -1
  94. package/dist/harness/extension/execution-environment/dangerous-commands.d.ts +28 -0
  95. package/dist/harness/extension/execution-environment/dangerous-commands.js +129 -0
  96. package/dist/harness/extension/execution-environment/effect-policy.d.ts +133 -4
  97. package/dist/harness/extension/execution-environment/effect-policy.js +263 -14
  98. package/dist/harness/extension/execution-environment/index.d.ts +4 -2
  99. package/dist/harness/extension/execution-environment/index.js +2 -1
  100. package/dist/harness/extension/execution-environment/isolation.js +5 -1
  101. package/dist/harness/extension/execution-environment/kill-switch.d.ts +34 -0
  102. package/dist/harness/extension/execution-environment/kill-switch.js +46 -0
  103. package/dist/harness/extension/execution-environment/ops-audit.d.ts +200 -0
  104. package/dist/harness/extension/execution-environment/ops-audit.js +338 -0
  105. package/dist/harness/extension/execution-environment/privileged-exec.d.ts +158 -0
  106. package/dist/harness/extension/execution-environment/privileged-exec.js +418 -0
  107. package/dist/harness/extension/execution-environment/process-info.d.ts +180 -0
  108. package/dist/harness/extension/execution-environment/process-info.js +466 -0
  109. package/dist/harness/extension/execution-environment/process-isolation.d.ts +7 -0
  110. package/dist/harness/extension/execution-environment/process-isolation.js +9 -0
  111. package/dist/harness/extension/execution-environment/sandbox.d.ts +109 -0
  112. package/dist/harness/extension/execution-environment/sandbox.js +264 -29
  113. package/dist/harness/extension/execution-environment/tool-capabilities.d.ts +1 -1
  114. package/dist/harness/extension/execution-environment/tool-capabilities.js +6 -0
  115. package/dist/harness/extension/execution-environment/types.d.ts +19 -0
  116. package/dist/harness/extension/plugin-ecosystem/channel-types.d.ts +5 -0
  117. package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.d.ts +6 -0
  118. package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.js +50 -0
  119. package/dist/harness/extension/plugin-ecosystem/tools/ask-user.d.ts +4 -5
  120. package/dist/harness/extension/plugin-ecosystem/tools/ask-user.js +5 -15
  121. package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +3 -0
  122. package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +53 -2
  123. package/dist/harness/extension/plugin-ecosystem/tools/file-search.js +10 -17
  124. package/dist/harness/extension/plugin-ecosystem/tools/git-runner.d.ts +31 -0
  125. package/dist/harness/extension/plugin-ecosystem/tools/git-runner.js +104 -0
  126. package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.d.ts +111 -0
  127. package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.js +197 -0
  128. package/dist/harness/extension/plugin-ecosystem/tools/git-tools.d.ts +8 -8
  129. package/dist/harness/extension/plugin-ecosystem/tools/git-tools.js +9 -100
  130. package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +6 -0
  131. package/dist/harness/extension/plugin-ecosystem/tools/index.js +4 -0
  132. package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.d.ts +43 -0
  133. package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.js +350 -0
  134. package/dist/harness/extension/plugin-ecosystem/tools/process-tools.d.ts +30 -0
  135. package/dist/harness/extension/plugin-ecosystem/tools/process-tools.js +137 -0
  136. package/dist/harness/governance/human-in-the-loop/decision-cache.d.ts +54 -4
  137. package/dist/harness/governance/human-in-the-loop/decision-cache.js +42 -16
  138. package/dist/harness/governance/security/temp-path.d.ts +9 -0
  139. package/dist/harness/governance/security/temp-path.js +13 -2
  140. package/dist/harness/governance/session-acl/seed-roles.js +14 -0
  141. package/dist/harness/index.d.ts +4 -0
  142. package/dist/harness/index.js +3 -0
  143. package/dist/harness/knowledge/file-identity.d.ts +8 -0
  144. package/dist/harness/knowledge/file-identity.js +26 -1
  145. package/dist/harness/knowledge/fts.js +1 -2
  146. package/dist/harness/knowledge/grounding.js +5 -5
  147. package/dist/harness/knowledge/index-store.d.ts +31 -4
  148. package/dist/harness/knowledge/index-store.js +180 -48
  149. package/dist/harness/knowledge/ingest.d.ts +115 -115
  150. package/dist/harness/knowledge/ingest.js +150 -150
  151. package/dist/harness/knowledge/knowledge-serve-child.js +1 -1
  152. package/dist/harness/knowledge/purge.js +4 -1
  153. package/dist/harness/memory/backfill-coverage.js +3 -1
  154. package/dist/harness/memory/sqlite/keyword-search.d.ts +1 -0
  155. package/dist/harness/memory/sqlite/keyword-search.js +4 -3
  156. package/dist/harness/observability/observer/hub.js +38 -1
  157. package/dist/harness/observability/observer/run-snapshot.js +1 -0
  158. package/dist/harness/observability/observer/types.d.ts +27 -0
  159. package/dist/harness/run/agent/agent.d.ts +7 -0
  160. package/dist/harness/run/agent/agent.js +23 -1
  161. package/dist/harness/run/reliability/fallback-provider.d.ts +117 -5
  162. package/dist/harness/run/reliability/fallback-provider.js +262 -22
  163. package/dist/harness/run/reliability/harness-events.d.ts +24 -1
  164. package/dist/harness/run/reliability/model-error-policy.d.ts +32 -0
  165. package/dist/harness/run/reliability/model-error-policy.js +42 -0
  166. package/dist/harness/run/reliability/provider-wrapper.js +29 -2
  167. package/dist/harness/run/reliability/refusal-guard.d.ts +20 -0
  168. package/dist/harness/run/reliability/refusal-guard.js +102 -0
  169. package/dist/harness/run/reliability/run-agent.js +231 -33
  170. package/dist/harness/run/run-guard/default-run-guard.js +1 -1
  171. package/dist/harness/run/run-scope.d.ts +31 -0
  172. package/dist/harness/run/runner.d.ts +132 -2
  173. package/dist/harness/run/runner.js +679 -60
  174. package/dist/harness/session/attachments/llm-intent.js +2 -1
  175. package/dist/harness/session/history/port.js +15 -5
  176. package/dist/harness/session/in-memory-store.d.ts +5 -0
  177. package/dist/harness/session/in-memory-store.js +33 -2
  178. package/dist/harness/session/tasks/render.d.ts +4 -1
  179. package/dist/harness/session/tasks/render.js +10 -7
  180. package/dist/harness/session/title.js +25 -9
  181. package/dist/harness/shared/events/agent-event-map.d.ts +92 -0
  182. package/dist/harness/shared/events/agent-event-map.js +7 -0
  183. package/dist/index.d.ts +6 -4
  184. package/dist/index.js +4 -3
  185. package/dist/integration/agent-runtime/channel-message-source.js +19 -1
  186. package/dist/integration/index.d.ts +2 -0
  187. package/dist/integration/index.js +1 -0
  188. package/dist/integration/observability/loop-watchdog.d.ts +56 -0
  189. package/dist/integration/observability/loop-watchdog.js +177 -0
  190. package/dist/integration/observability/metrics.d.ts +3 -0
  191. package/dist/integration/observability/metrics.js +18 -7
  192. package/dist/integration/observability/run-telemetry.d.ts +8 -0
  193. package/dist/integration/observability/run-telemetry.js +31 -8
  194. package/dist/integration/observability/trace-collector.js +26 -4
  195. package/dist/integration/observability/trace-events.d.ts +1 -0
  196. package/dist/integration/observability/trace-events.js +2 -0
  197. package/dist/integration/observability/trace-logger.js +27 -17
  198. package/dist/integration/providers/anthropic.d.ts +23 -0
  199. package/dist/integration/providers/anthropic.js +97 -12
  200. package/dist/integration/providers/fetch-timeout.d.ts +42 -0
  201. package/dist/integration/providers/fetch-timeout.js +51 -0
  202. package/dist/integration/providers/openai.d.ts +10 -0
  203. package/dist/integration/providers/openai.js +58 -11
  204. package/dist/integration/providers/provider-diagnostics.d.ts +80 -0
  205. package/dist/integration/providers/provider-diagnostics.js +89 -0
  206. package/dist/integration/providers/timeout-coherence.d.ts +52 -0
  207. package/dist/integration/providers/timeout-coherence.js +68 -0
  208. package/dist/integration/providers/wire-dump.d.ts +62 -0
  209. package/dist/integration/providers/wire-dump.js +111 -2
  210. package/dist/integration/storage/jsonl.d.ts +19 -1
  211. package/dist/integration/storage/jsonl.js +100 -4
  212. package/dist/paths.d.ts +19 -0
  213. package/dist/paths.js +38 -1
  214. package/dist/subsystems/cognition-steward/conceptualize/handler.js +1 -1
  215. package/dist/subsystems/memory-steward/backfill/handler.js +1 -1
  216. package/dist/subsystems/safety-guard/config.yaml +3 -1
  217. package/dist/subsystems/wisdom-steward/formation/handler.js +1 -1
  218. package/package.json +3 -2
@@ -1,26 +1,81 @@
1
1
  /**
2
2
  * FallbackProvider — 模型回退提供器
3
3
  *
4
- * 包装多个 ModelProvider,按顺序尝试。
5
- * 主 provider 失败时自动切换到下一个回退 provider。
4
+ * 包装多个 ModelProvider,按顺序尝试。主 provider 失败时自动切换到下一个回退 provider。
6
5
  *
7
6
  * 使用场景:
8
7
  * - 主模型不可用(circuit breaker open、限流、超时)
9
8
  * - 主模型返回错误
10
9
  * - 跨 provider 灾备(如 OpenAI -> Anthropic)
10
+ *
11
+ * ## 单节点首块预算(2026-10-27 修复)
12
+ *
13
+ * 引擎 watchdog(core/loop/call-model.ts)把**整条链**当作一次调用计时,只认内容 chunk。
14
+ * 于是主 provider 若「连上了但长期不产首块」(上游排队 / 长 prefill,期间只发 SSE ping),
15
+ * 会独占整个 watchdog 窗口,备选模型一次机会都拿不到——实测一次 52 分钟停滞即由此而来。
16
+ *
17
+ * 因此链内每个节点各有自己的「等首个内容 chunk」预算(`attemptTtftTimeoutMs`):
18
+ * - 仅在**还有下一个候选**时生效(最后一棒没有备选可切,交给外层 watchdog 兜底)
19
+ * - 超时即 abort 该次尝试(子 AbortController,不波及父 signal),切下一节点
20
+ * - 一旦首个 chunk 产出,本次调用完全委托该节点,预算不再起作用
21
+ * (流中途的静默由外层 idle watchdog 负责,见 stream() 注释)
22
+ *
23
+ * ## 失败节点降级
24
+ *
25
+ * 超时/失败过的节点进入降级窗口(`degradeTtlMs`,缺省等于首块预算),
26
+ * 后续调用把它排到最后。这使「重试」不再是原地重撞同一节点——
27
+ * 这也是条件化重试的落地方式:重试时条件(节点顺序)确实变了。
28
+ */
29
+ import { classifyError } from '@octopi-agent/core/loop/error-classifier.js';
30
+ /**
31
+ * 单节点等待首个内容 chunk 的默认预算(毫秒)
32
+ *
33
+ * 依据:同一时段实测备选 provider 处理同体积请求 1–16s;
34
+ * 主 provider 出现 271s / 383s 级首块延迟。90s 足以放过正常抖动,
35
+ * 又不至于让一次用户提问白白等满 watchdog 窗口。
11
36
  */
37
+ export const DEFAULT_FALLBACK_ATTEMPT_TTFT_MS = 90_000;
38
+ /**
39
+ * 单节点非流式(chat)尝试的默认预算(毫秒)
40
+ *
41
+ * chat 没有「首块」概念,预算覆盖的是**整段响应**;复用 90s 的 TTFT 值会把
42
+ * 合法的长响应(推理型模型、长输出)误判为节点故障并降级。
43
+ * 取 240s:低于 undici `bodyTimeout` 的 300s,从而能在被 undici 以
44
+ * 含糊的 `terminated` 掐断之前把控制权交回引擎、切换下一节点。
45
+ */
46
+ export const DEFAULT_FALLBACK_CHAT_ATTEMPT_MS = 240_000;
47
+ /** 单次尝试的首块预算被耗尽的标记(用于区分「自家 abort」与父级 abort) */
48
+ const ATTEMPT_BUDGET_EXHAUSTED = Symbol('fallback.attemptBudgetExhausted');
12
49
  /**
13
50
  * 回退感知的 ModelProvider
14
51
  *
15
52
  * 尝试主 provider,失败后按 fallbackModels 顺序尝试备选。
16
- * 每次调用独立决策,不持久化状态。
53
+ * 每个节点有独立首块预算,失败节点在降级窗口内被排到最后。
17
54
  */
18
55
  export class FallbackProvider {
19
56
  name;
20
57
  defaultModel;
21
58
  chain;
22
- constructor(mainProvider, mainModel, fallbackModels, providerMap) {
59
+ attemptTtftTimeoutMs;
60
+ attemptChatTimeoutMs;
61
+ degradeTtlMs;
62
+ now;
63
+ /** 节点下标 → 降级截止时间戳 */
64
+ degradedUntil = new Map();
65
+ constructor(mainProvider, mainModel, fallbackModels, providerMap, options = {}) {
23
66
  this.name = mainProvider.name;
67
+ this.attemptTtftTimeoutMs =
68
+ options.attemptTtftTimeoutMs && options.attemptTtftTimeoutMs > 0
69
+ ? options.attemptTtftTimeoutMs
70
+ : DEFAULT_FALLBACK_ATTEMPT_TTFT_MS;
71
+ this.attemptChatTimeoutMs =
72
+ options.attemptChatTimeoutMs && options.attemptChatTimeoutMs > 0
73
+ ? options.attemptChatTimeoutMs
74
+ : DEFAULT_FALLBACK_CHAT_ATTEMPT_MS;
75
+ this.degradeTtlMs = options.degradeTtlMs && options.degradeTtlMs > 0
76
+ ? options.degradeTtlMs
77
+ : this.attemptTtftTimeoutMs;
78
+ this.now = options.now ?? Date.now;
24
79
  this.chain = [{ provider: mainProvider, model: mainModel, defaultModel: mainProvider.defaultModel }];
25
80
  for (const fb of fallbackModels) {
26
81
  const fbProvider = providerMap.get(fb.provider);
@@ -32,55 +87,211 @@ export class FallbackProvider {
32
87
  }
33
88
  this.defaultModel = mainModel;
34
89
  }
90
+ /**
91
+ * 本次调用的尝试顺序:未降级节点保持原相对顺序在前,降级节点在后。
92
+ *
93
+ * @returns 带原下标与「是否仍有下一候选」的尝试序列
94
+ */
95
+ attemptOrder() {
96
+ const nowTs = this.now();
97
+ const active = [];
98
+ const degraded = [];
99
+ for (let i = 0; i < this.chain.length; i++) {
100
+ const item = { entry: this.chain[i], index: i };
101
+ const until = this.degradedUntil.get(i);
102
+ if (until !== undefined && until > nowTs)
103
+ degraded.push(item);
104
+ else {
105
+ // 过期条目顺手清理,避免 Map 长期增长
106
+ if (until !== undefined)
107
+ this.degradedUntil.delete(i);
108
+ active.push(item);
109
+ }
110
+ }
111
+ return [...active, ...degraded];
112
+ }
113
+ /** 标记节点失败(进入降级窗口) */
114
+ markDegraded(index) {
115
+ this.degradedUntil.set(index, this.now() + this.degradeTtlMs);
116
+ }
117
+ /** 标记节点恢复(成功一次即解除降级) */
118
+ markHealthy(index) {
119
+ this.degradedUntil.delete(index);
120
+ }
121
+ /**
122
+ * 仅在「节点健康类」错误上降级
123
+ *
124
+ * 401/403(鉴权)、400(上下文超长/请求非法)、billing 等是**请求级**问题,
125
+ * 换节点不会变好;把它们算作节点故障会让主节点被无谓地排后,
126
+ * 并在下一次调用里静默换到别的模型(计费/缓存口径随之错位)。
127
+ *
128
+ * @param index - 节点下标
129
+ * @param err - 该节点的错误
130
+ */
131
+ degradeIfNodeUnhealthy(index, err) {
132
+ const { reason } = classifyError(err);
133
+ if (reason === 'timeout' || reason === 'network' || reason === 'server' || reason === 'rate_limit') {
134
+ this.markDegraded(index);
135
+ }
136
+ }
137
+ /**
138
+ * 由备选节点服务时留下一行可检索的提示
139
+ *
140
+ * 换模型对调用方不可见(响应里的 model 由各 provider 自填,用量/计费口径仍在主模型名下),
141
+ * 若不显式记一笔,「为什么这次的回答风格/质量变了」将无从追查。
142
+ *
143
+ * @param entry - 实际服务的节点
144
+ * @param pos - 其在本次尝试顺序中的位置(0 = 主节点)
145
+ */
146
+ warnIfServedByFallback(entry, pos) {
147
+ if (pos === 0)
148
+ return;
149
+ console.warn(`[FallbackProvider] served by fallback node ${pos}: ${entry.provider.name}/${entry.model} ` +
150
+ `(primary ${this.name}/${this.defaultModel} unavailable or degraded)`);
151
+ }
152
+ /**
153
+ * 在预算内等待一个 promise;超时则 reject 预算耗尽标记
154
+ *
155
+ * @param work - 待等待的 promise
156
+ * @param budgetMs - 预算;undefined 表示不限(最后一个候选)
157
+ * @param onTimeout - 预算耗尽时的清理动作(abort 子 signal)
158
+ * @returns work 的结果
159
+ */
160
+ async withBudget(work, budgetMs, onTimeout) {
161
+ if (budgetMs === undefined)
162
+ return work;
163
+ let timer;
164
+ const budget = new Promise((_resolve, reject) => {
165
+ timer = setTimeout(() => {
166
+ onTimeout();
167
+ reject(ATTEMPT_BUDGET_EXHAUSTED);
168
+ }, budgetMs);
169
+ });
170
+ try {
171
+ return await Promise.race([work, budget]);
172
+ }
173
+ finally {
174
+ if (timer)
175
+ clearTimeout(timer);
176
+ }
177
+ }
35
178
  async chat(request) {
179
+ const order = this.attemptOrder();
36
180
  let lastError;
37
- for (const entry of this.chain) {
181
+ for (let pos = 0; pos < order.length; pos++) {
182
+ const { entry, index } = order[pos];
183
+ const hasNext = pos < order.length - 1;
184
+ // 子 controller:预算耗尽时只取消本次尝试,不波及父级 signal
185
+ const attempt = new AbortController();
186
+ const onParentAbort = () => attempt.abort();
187
+ if (request.signal?.aborted)
188
+ attempt.abort();
189
+ else
190
+ request.signal?.addEventListener('abort', onParentAbort, { once: true });
38
191
  try {
39
- return await entry.provider.chat({
40
- ...request,
41
- model: entry.model,
42
- });
192
+ const response = await this.withBudget(entry.provider.chat({ ...request, model: entry.model, signal: attempt.signal }),
193
+ // chat 是非流式:预算覆盖整段响应,与 stream 的「首块」语义不同(见 DEFAULT_FALLBACK_CHAT_ATTEMPT_MS)
194
+ hasNext ? this.attemptChatTimeoutMs : undefined, () => attempt.abort());
195
+ this.markHealthy(index);
196
+ this.warnIfServedByFallback(entry, pos);
197
+ return response;
43
198
  }
44
199
  catch (err) {
200
+ if (request.signal?.aborted)
201
+ throw err;
202
+ if (err === ATTEMPT_BUDGET_EXHAUSTED) {
203
+ this.markDegraded(index);
204
+ // 消息显式带 "timeout":classifyError 依文本归类时才能落到 timeout
205
+ //(否则落 unknown → 直接中止,丢掉了「换节点后再试一次」的机会)
206
+ lastError = new Error(`FallbackProvider: ${entry.provider.name}/${entry.model} response timeout after ${this.attemptChatTimeoutMs}ms (no response received)`);
207
+ attachAttemptBudgetExhausted(lastError, this.attemptChatTimeoutMs);
208
+ console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} attempt budget exhausted (${this.attemptChatTimeoutMs}ms), trying next`);
209
+ continue;
210
+ }
45
211
  lastError = err;
212
+ this.degradeIfNodeUnhealthy(index, err);
46
213
  const msg = err instanceof Error ? err.message : String(err);
47
214
  console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} failed: ${msg}`);
48
215
  }
216
+ finally {
217
+ request.signal?.removeEventListener('abort', onParentAbort);
218
+ }
49
219
  }
50
220
  throw lastError ?? new Error('All fallback providers exhausted');
51
221
  }
52
222
  /**
53
223
  * 流式生成回复
54
224
  *
55
- * 尝试主 provider 的 stream(),失败则按 fallbackModels 顺序尝试备选。
225
+ * 每个节点在独立首块预算内等待首个 chunk;超时切下一节点。
56
226
  *
57
227
  * 注意:一旦某个 provider 成功产出第一个 chunk,后续就完全委托给该 provider。
58
228
  * 如果该 provider 在流中途抛异常,错误会直接传递给调用方,不会触发回退。
59
229
  * 这是流式生成的固有限制——已产出的内容无法撤回。
230
+ * 流中途的静默由外层 idle watchdog 负责(它会 abort 父 signal)。
60
231
  */
61
232
  async *stream(request) {
233
+ const order = this.attemptOrder();
62
234
  let lastError;
63
- for (const entry of this.chain) {
235
+ for (let pos = 0; pos < order.length; pos++) {
236
+ const { entry, index } = order[pos];
237
+ const hasNext = pos < order.length - 1;
238
+ // 子 controller:预算耗尽时取消本次尝试的 HTTP,不波及父级 signal
239
+ const attempt = new AbortController();
240
+ const onParentAbort = () => attempt.abort();
241
+ if (request.signal?.aborted)
242
+ attempt.abort();
243
+ else
244
+ request.signal?.addEventListener('abort', onParentAbort, { once: true });
245
+ let gen;
246
+ let first;
64
247
  // try/catch 仅覆盖初始化阶段(创建 stream + 获取首个 chunk)。
65
248
  // 一旦首个 chunk 成功产出,后续 yield* 在 try/catch 之外执行,
66
249
  // 确保流中途错误直接向上传播,不会被吞掉后静默回退。
67
- let gen;
68
- let first;
69
250
  try {
70
- gen = entry.provider.stream({ ...request, model: entry.model });
71
- first = await gen.next();
251
+ gen = entry.provider.stream({ ...request, model: entry.model, signal: attempt.signal });
252
+ first = await this.withBudget(gen.next(), hasNext ? this.attemptTtftTimeoutMs : undefined, () => attempt.abort());
72
253
  }
73
254
  catch (err) {
74
- lastError = err;
75
- const msg = err instanceof Error ? err.message : String(err);
76
- console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} stream failed: ${msg}`);
255
+ // 放弃被预算掐断的尝试:abort 底层 HTTP 并回收 generator
256
+ void gen?.return?.(undefined)?.catch(() => undefined);
257
+ request.signal?.removeEventListener('abort', onParentAbort);
258
+ // 父级中止:不是本节点的问题,直接上抛(不降级、不切下一个)
259
+ if (request.signal?.aborted)
260
+ throw err;
261
+ if (err === ATTEMPT_BUDGET_EXHAUSTED) {
262
+ this.markDegraded(index);
263
+ lastError = new Error(`FallbackProvider: ${entry.provider.name}/${entry.model} first-chunk timeout after ${this.attemptTtftTimeoutMs}ms (no chunk received)`);
264
+ attachAttemptBudgetExhausted(lastError, this.attemptTtftTimeoutMs);
265
+ console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} attempt budget exhausted (${this.attemptTtftTimeoutMs}ms), trying next`);
266
+ }
267
+ else {
268
+ lastError = err;
269
+ this.degradeIfNodeUnhealthy(index, err);
270
+ const msg = err instanceof Error ? err.message : String(err);
271
+ console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} stream failed: ${msg}`);
272
+ }
77
273
  continue;
78
274
  }
79
- if (first.done)
275
+ if (first.done) {
276
+ // 空流:本轮不算成功,交给下一节点(与旧行为一致)
277
+ request.signal?.removeEventListener('abort', onParentAbort);
278
+ this.markDegraded(index);
80
279
  continue;
81
- // 首个 chunk 成功,完全委托给当前 provider(无 try/catch 包裹)
82
- yield first.value;
83
- yield* gen;
280
+ }
281
+ this.markHealthy(index);
282
+ this.warnIfServedByFallback(entry, pos);
283
+ // 首个 chunk 成功,完全委托给当前 provider(无 try/catch 包裹)。
284
+ // finally 里必须 abort:消费者 break / 提前 return 时 gen.return() 打断不了
285
+ // 挂在 reader.read() 上的 await,只有 abort 子 signal 才能真正关掉底层 HTTP。
286
+ // 对已完成/正常的流 abort 是无害的(provider 侧 signal 已不再被等待)。
287
+ try {
288
+ yield first.value;
289
+ yield* gen;
290
+ }
291
+ finally {
292
+ attempt.abort();
293
+ request.signal?.removeEventListener('abort', onParentAbort);
294
+ }
84
295
  return;
85
296
  }
86
297
  throw lastError ?? new Error('All fallback providers exhausted');
@@ -116,3 +327,32 @@ export class FallbackProvider {
116
327
  return infos;
117
328
  }
118
329
  }
330
+ /**
331
+ * 「单节点预算耗尽」标记(观测用,与 STREAM_DIAG 同属 Symbol 通道,不影响错误分类)
332
+ *
333
+ * 定义在 harness(产生方);`integration/providers/wire-dump` 作为外层读取它落盘。
334
+ */
335
+ export const ATTEMPT_BUDGET_EXHAUSTED_DIAG = Symbol.for('@octopi-agent/engine.provider.attemptBudgetExhausted');
336
+ /**
337
+ * 在错误对象上标记「单节点预算耗尽」
338
+ *
339
+ * @param err - 待标记错误
340
+ * @param budgetMs - 被耗尽的预算
341
+ */
342
+ export function attachAttemptBudgetExhausted(err, budgetMs) {
343
+ if (typeof err !== 'object' || err === null)
344
+ return;
345
+ err[ATTEMPT_BUDGET_EXHAUSTED_DIAG] = budgetMs;
346
+ }
347
+ /**
348
+ * 读取「单节点预算耗尽」标记
349
+ *
350
+ * @param err - 任意错误
351
+ * @returns 被耗尽的预算毫秒数;未标记时 undefined
352
+ */
353
+ export function readAttemptBudgetExhausted(err) {
354
+ if (typeof err !== 'object' || err === null)
355
+ return undefined;
356
+ const v = err[ATTEMPT_BUDGET_EXHAUSTED_DIAG];
357
+ return typeof v === 'number' ? v : undefined;
358
+ }
@@ -107,7 +107,30 @@ export type SecurityBlockedEvent = {
107
107
  }>;
108
108
  };
109
109
  };
110
+ /**
111
+ * 模型调用即将重试(用户可见)
112
+ *
113
+ * 背景:一次 52 分钟的会话停滞期间前端零反馈(发送看门狗 10s 内即被首个事件清掉),
114
+ * 用户只看到 runStatus 停在 waiting。该事件给重试窗口提供可观测锚点:
115
+ * 每次重试必须能被看见,且带「第几次 / 还剩几次 / 为什么 / 等多久」。
116
+ */
117
+ export type ModelRetryEvent = {
118
+ type: 'model_retry';
119
+ timestamp: number;
120
+ data: {
121
+ /** 本次是第几次重试(从 1 起) */
122
+ attempt: number;
123
+ /** 该错误原因允许的最大重试次数 */
124
+ maxAttempts: number;
125
+ /** 分类后的错误原因(timeout / network / server / rate_limit …) */
126
+ reason: string;
127
+ /** 重试前等待时长 */
128
+ delayMs: number;
129
+ /** 错误摘要(截断,供 UI 直接展示) */
130
+ error: string;
131
+ };
132
+ };
110
133
  /** Harness 层扩展事件 */
111
- export type HarnessLoopExtension = BudgetExceededEvent | BudgetControlEvent | RunGuardRecoveredEvent | RunGuardStoppedEvent | RunGuardMetricsEvent | SecurityBlockedEvent;
134
+ export type HarnessLoopExtension = BudgetExceededEvent | BudgetControlEvent | RunGuardRecoveredEvent | RunGuardStoppedEvent | RunGuardMetricsEvent | SecurityBlockedEvent | ModelRetryEvent;
112
135
  /** reliability 包装后的完整事件流 */
113
136
  export type HarnessLoopEvent = AgentLoopEvent | HarnessLoopExtension;
@@ -0,0 +1,32 @@
1
+ /**
2
+ * 模型调用错误的重试上限(单一事实源)
3
+ *
4
+ * 为什么集中在一处:生产链路由 `AgentBuilder` 注入 `DefaultErrorStrategy`,
5
+ * `runAgentWithReliability` 的兜底分支只在未注入时生效。两处各写一套数字,
6
+ * 很容易出现「改了一处、线上还是旧策略」的漂移。
7
+ *
8
+ * ## 为什么 timeout 保持 3 次(2026-10-27 修正)
9
+ *
10
+ * 曾经把它降到 1 次,理由是「一次 timeout 已耗掉整个 watchdog 窗口,再赌 3 轮就是
11
+ * 52 分钟黑屏」。那是把**症状**当成了**病因**:52 分钟来自「单次尝试耗时过长」
12
+ * 叠加「全程零反馈」,不是来自重试次数。砍次数只是让最坏时长变小,代价是
13
+ * 削弱了上游抖动时的自愈能力——这正是「让系统看起来好了」而不是修好了。
14
+ *
15
+ * 真正的修复在别处:
16
+ * - 单次尝试有界:回退链单节点预算(`fallbackAttemptTtftMs`)使有备选时 90s 即切换
17
+ * - 重试过程可见:`engine.model_retry` 事件 + WebUI 角标,用户知道在第几次、还要等多久
18
+ * - 重试过程可中止:退避等待响应 abort,用户不必等满整个序列
19
+ *
20
+ * 三者俱备后,3 次重试是安全的:用户全程看得到、随时能停。
21
+ *
22
+ * @module harness/run/reliability/model-error-policy
23
+ */
24
+ /** 各错误原因的最大重试次数(不含首次尝试) */
25
+ export declare const MODEL_ERROR_MAX_ATTEMPTS: Readonly<Record<string, number>>;
26
+ /**
27
+ * 查询某错误原因允许的重试次数
28
+ *
29
+ * @param reason - `classifyError` 给出的错误原因
30
+ * @returns 最大重试次数;不可重试的原因返回 0
31
+ */
32
+ export declare function maxAttemptsFor(reason: string): number;
@@ -0,0 +1,42 @@
1
+ /**
2
+ * 模型调用错误的重试上限(单一事实源)
3
+ *
4
+ * 为什么集中在一处:生产链路由 `AgentBuilder` 注入 `DefaultErrorStrategy`,
5
+ * `runAgentWithReliability` 的兜底分支只在未注入时生效。两处各写一套数字,
6
+ * 很容易出现「改了一处、线上还是旧策略」的漂移。
7
+ *
8
+ * ## 为什么 timeout 保持 3 次(2026-10-27 修正)
9
+ *
10
+ * 曾经把它降到 1 次,理由是「一次 timeout 已耗掉整个 watchdog 窗口,再赌 3 轮就是
11
+ * 52 分钟黑屏」。那是把**症状**当成了**病因**:52 分钟来自「单次尝试耗时过长」
12
+ * 叠加「全程零反馈」,不是来自重试次数。砍次数只是让最坏时长变小,代价是
13
+ * 削弱了上游抖动时的自愈能力——这正是「让系统看起来好了」而不是修好了。
14
+ *
15
+ * 真正的修复在别处:
16
+ * - 单次尝试有界:回退链单节点预算(`fallbackAttemptTtftMs`)使有备选时 90s 即切换
17
+ * - 重试过程可见:`engine.model_retry` 事件 + WebUI 角标,用户知道在第几次、还要等多久
18
+ * - 重试过程可中止:退避等待响应 abort,用户不必等满整个序列
19
+ *
20
+ * 三者俱备后,3 次重试是安全的:用户全程看得到、随时能停。
21
+ *
22
+ * @module harness/run/reliability/model-error-policy
23
+ */
24
+ /** 各错误原因的最大重试次数(不含首次尝试) */
25
+ export const MODEL_ERROR_MAX_ATTEMPTS = {
26
+ rate_limit: 3,
27
+ timeout: 3,
28
+ network: 3,
29
+ server: 2,
30
+ // 内容过滤拒答多为阈值型误杀(同请求重试常过);给 2 次原样重试,
31
+ // 仍被拦则如实收口(不教模型换说法绕过审核)
32
+ content_filter: 2,
33
+ };
34
+ /**
35
+ * 查询某错误原因允许的重试次数
36
+ *
37
+ * @param reason - `classifyError` 给出的错误原因
38
+ * @returns 最大重试次数;不可重试的原因返回 0
39
+ */
40
+ export function maxAttemptsFor(reason) {
41
+ return MODEL_ERROR_MAX_ATTEMPTS[reason] ?? 0;
42
+ }
@@ -11,6 +11,28 @@
11
11
  *
12
12
  * @module
13
13
  */
14
+ /**
15
+ * 判定错误是否属于「取消/中止」而非「节点故障」
16
+ *
17
+ * 两类来源:(1) 调用方主动 abort(用户停止、run 取消);
18
+ * (2) 回退链的单节点预算耗尽 —— 它 abort 的是子 signal,节点本身可能只是慢。
19
+ * 二者都不应累计熔断失败数,否则健康节点会被连续 5 次「慢」直接熔断。
20
+ *
21
+ * @param err - provider 抛出的错误
22
+ * @param signal - 该次请求的 signal
23
+ * @returns 是否为取消类
24
+ */
25
+ function isCancellationLike(err, signal) {
26
+ if (signal?.aborted)
27
+ return true;
28
+ if (typeof err !== 'object' || err === null)
29
+ return false;
30
+ const name = err.name;
31
+ if (name === 'AbortError')
32
+ return true;
33
+ const message = err.message;
34
+ return typeof message === 'string' && /abort/i.test(message);
35
+ }
14
36
  /**
15
37
  * 用熔断器包装 ModelProvider
16
38
  *
@@ -35,7 +57,10 @@ export function wrapProviderWithCircuitBreaker(provider, cb) {
35
57
  return result;
36
58
  }
37
59
  catch (err) {
38
- cb.recordFailure();
60
+ // abort / 预算类错误不计入熔断:节点只是「慢」或「被取消」,
61
+ // 把它当故障会误伤健康节点(连续 5 次即熔断,随后只剩 unknown 错误)。
62
+ if (!isCancellationLike(err, request.signal))
63
+ cb.recordFailure();
39
64
  throw err;
40
65
  }
41
66
  };
@@ -50,7 +75,9 @@ export function wrapProviderWithCircuitBreaker(provider, cb) {
50
75
  cb.recordSuccess();
51
76
  }
52
77
  catch (err) {
53
- cb.recordFailure();
78
+ // 同 chat:取消/预算类错误不是节点故障
79
+ if (!isCancellationLike(err, request.signal))
80
+ cb.recordFailure();
54
81
  throw err;
55
82
  }
56
83
  };
@@ -0,0 +1,20 @@
1
+ /**
2
+ * 内容过滤拒答守卫 — ModelProvider 装饰器
3
+ *
4
+ * 把「正文=罐头拒答」的 200 内联返回转成 ContentFilterError(content_filter 模型错误),
5
+ * 复用既有 onError/重试链路:阈值型误杀原样重试常过;重试耗尽走既有诚实收口
6
+ * (stopReason=llm_error + 「模型调用失败…可继续对话重试」),拒答语永远不会落成
7
+ * 助手回答。
8
+ *
9
+ * 流式路径按前缀扣发罐头正文:正常回答仅在「累计正文仍是拒答签名前缀」的短暂窗口内
10
+ * 缓冲,一旦脱离签名前缀立即冲刷透传;reasoning / tool_call 块不经缓冲(有工具调用的
11
+ * 响应不是拒答,遇 tool_call 即转全透传)。
12
+ */
13
+ import type { ModelProvider } from '@octopi-agent/core/interfaces/model-provider.js';
14
+ /**
15
+ * 为 ModelProvider 套上内容过滤拒答守卫。
16
+ *
17
+ * @param model - 被包装的 provider(可与 FallbackProvider 等装饰器叠放)
18
+ * @returns 守卫后的 provider(接口同构,方法逐个显式委托——展开抄不到原型方法)
19
+ */
20
+ export declare function guardRefusalProvider(model: ModelProvider): ModelProvider;
@@ -0,0 +1,102 @@
1
+ /**
2
+ * 内容过滤拒答守卫 — ModelProvider 装饰器
3
+ *
4
+ * 把「正文=罐头拒答」的 200 内联返回转成 ContentFilterError(content_filter 模型错误),
5
+ * 复用既有 onError/重试链路:阈值型误杀原样重试常过;重试耗尽走既有诚实收口
6
+ * (stopReason=llm_error + 「模型调用失败…可继续对话重试」),拒答语永远不会落成
7
+ * 助手回答。
8
+ *
9
+ * 流式路径按前缀扣发罐头正文:正常回答仅在「累计正文仍是拒答签名前缀」的短暂窗口内
10
+ * 缓冲,一旦脱离签名前缀立即冲刷透传;reasoning / tool_call 块不经缓冲(有工具调用的
11
+ * 响应不是拒答,遇 tool_call 即转全透传)。
12
+ */
13
+ import { ContentFilterError, isCannedRefusalText, isCannedRefusalPrefix, } from '@octopi-agent/core/loop/canned-refusal.js';
14
+ /**
15
+ * 响应级拒答判定:正文恰为罐头拒答且无工具调用时抛 ContentFilterError。
16
+ *
17
+ * @param response - 同步调用响应
18
+ * @throws {ContentFilterError} 正文为罐头拒答时
19
+ */
20
+ function assertNotCannedRefusal(response) {
21
+ if (response.toolCalls?.length)
22
+ return; // 有工具调用的响应不是拒答
23
+ if (response.content && isCannedRefusalText(response.content)) {
24
+ throw new ContentFilterError(response.content);
25
+ }
26
+ }
27
+ /**
28
+ * 流式守卫:内容块按前缀扣发,落定为罐头拒答时抛 ContentFilterError。
29
+ *
30
+ * @param model - 被包装的 provider
31
+ * @param request - LLM 请求
32
+ * @returns 受守卫的流(正常内容照常流出,仅拒答前缀窗口内缓冲)
33
+ * @throws {ContentFilterError} 流的最终正文为罐头拒答时
34
+ */
35
+ async function* guardedStream(model, request) {
36
+ let contentBuf = '';
37
+ let sawToolCall = false;
38
+ for await (const chunk of model.stream(request)) {
39
+ if (chunk.type === 'tool_call') {
40
+ sawToolCall = true;
41
+ if (contentBuf) {
42
+ yield { type: 'content', content: contentBuf };
43
+ contentBuf = '';
44
+ }
45
+ yield chunk;
46
+ continue;
47
+ }
48
+ if (chunk.type === 'content' && !sawToolCall) {
49
+ contentBuf += chunk.content ?? '';
50
+ if (isCannedRefusalPrefix(contentBuf))
51
+ continue; // 疑似罐头:继续扣发
52
+ yield { type: 'content', content: contentBuf };
53
+ contentBuf = '';
54
+ continue;
55
+ }
56
+ if (chunk.type === 'done') {
57
+ if (!sawToolCall && contentBuf && isCannedRefusalText(contentBuf)) {
58
+ throw new ContentFilterError(contentBuf);
59
+ }
60
+ if (contentBuf) {
61
+ yield { type: 'content', content: contentBuf };
62
+ contentBuf = '';
63
+ }
64
+ yield chunk;
65
+ continue;
66
+ }
67
+ yield chunk;
68
+ }
69
+ // 流异常结束(无 done):非罐头的残留正文不丢输出
70
+ if (contentBuf && !isCannedRefusalText(contentBuf)) {
71
+ yield { type: 'content', content: contentBuf };
72
+ }
73
+ }
74
+ /**
75
+ * 为 ModelProvider 套上内容过滤拒答守卫。
76
+ *
77
+ * @param model - 被包装的 provider(可与 FallbackProvider 等装饰器叠放)
78
+ * @returns 守卫后的 provider(接口同构,方法逐个显式委托——展开抄不到原型方法)
79
+ */
80
+ export function guardRefusalProvider(model) {
81
+ return {
82
+ name: model.name,
83
+ defaultModel: model.defaultModel,
84
+ ...(model.models ? { models: model.models } : {}),
85
+ getModelInfo: (modelName) => model.getModelInfo(modelName),
86
+ getModelInfos: () => model.getModelInfos(),
87
+ isAvailable: () => model.isAvailable(),
88
+ ...(model.dispose
89
+ ? {
90
+ dispose: () => {
91
+ model.dispose?.();
92
+ },
93
+ }
94
+ : {}),
95
+ chat: async (request) => {
96
+ const response = await model.chat(request);
97
+ assertNotCannedRefusal(response);
98
+ return response;
99
+ },
100
+ stream: (request) => guardedStream(model, request),
101
+ };
102
+ }