@octopi-agent/engine 0.90.11 → 0.97.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-admin/agent-presets.d.ts +70 -0
- package/dist/config-admin/agent-presets.js +243 -0
- package/dist/config-admin/discover.d.ts +30 -0
- package/dist/config-admin/discover.js +48 -0
- package/dist/config-admin/env.d.ts +22 -0
- package/dist/config-admin/env.js +41 -0
- package/dist/config-admin/index.d.ts +20 -0
- package/dist/config-admin/index.js +15 -0
- package/dist/config-admin/model-presets.d.ts +75 -0
- package/dist/config-admin/model-presets.js +249 -0
- package/dist/config-admin/name-guard.d.ts +14 -0
- package/dist/config-admin/name-guard.js +24 -0
- package/dist/config-admin/patch.d.ts +56 -0
- package/dist/config-admin/patch.js +201 -0
- package/dist/config-admin/port.d.ts +68 -0
- package/dist/config-admin/port.js +21 -0
- package/dist/config-admin/redact.d.ts +39 -0
- package/dist/config-admin/redact.js +42 -0
- package/dist/config-schema/agent.d.ts +43 -1
- package/dist/config-schema/context.d.ts +6 -1
- package/dist/config-schema/context.js +8 -0
- package/dist/config-schema/engine.d.ts +106 -4
- package/dist/config-schema/governance.d.ts +43 -1
- package/dist/config-schema/governance.js +157 -0
- package/dist/config-schema/models.d.ts +5 -2
- package/dist/config-schema/models.js +4 -1
- package/dist/config-schema/observability.d.ts +9 -0
- package/dist/config-schema/observability.js +29 -0
- package/dist/config-schema/run.d.ts +3 -0
- package/dist/config-schema/run.js +14 -0
- package/dist/config.d.ts +29 -4
- package/dist/config.js +15 -4
- package/dist/harness/activation/dispatcher.d.ts +8 -2
- package/dist/harness/activation/dispatcher.js +33 -5
- package/dist/harness/activation/inbox.d.ts +87 -0
- package/dist/harness/activation/inbox.js +169 -0
- package/dist/harness/activation/runtime.d.ts +9 -8
- package/dist/harness/activation/runtime.js +100 -8
- package/dist/harness/activation/types.d.ts +19 -0
- package/dist/harness/activation/types.js +2 -0
- package/dist/harness/agent/builder.d.ts +15 -1
- package/dist/harness/agent/builder.js +31 -10
- package/dist/harness/agent/config-bridge.d.ts +5 -0
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/agent/config-integrity.d.ts +72 -0
- package/dist/harness/agent/config-integrity.js +146 -0
- package/dist/harness/agent/wire-agent-builder.js +2 -1
- package/dist/harness/collaboration/autonomous-subsystem/think/executor.js +3 -1
- package/dist/harness/collaboration/delegation/child-run.js +21 -5
- package/dist/harness/collaboration/delegation/notifier.js +4 -3
- package/dist/harness/collaboration/delegation/service.js +2 -3
- package/dist/harness/collaboration/delegation/wiring.js +6 -5
- package/dist/harness/collaboration/orchestration/reflector/llm-reflector.d.ts +1 -1
- package/dist/harness/collaboration/orchestration/reflector/llm-reflector.js +1 -1
- package/dist/harness/context/capabilities/compact/engine.d.ts +30 -0
- package/dist/harness/context/capabilities/compact/engine.js +198 -46
- package/dist/harness/context/capabilities/compact/handover.d.ts +78 -0
- package/dist/harness/context/capabilities/compact/handover.js +107 -0
- package/dist/harness/context/capabilities/compact/index.d.ts +3 -1
- package/dist/harness/context/capabilities/compact/index.js +2 -1
- package/dist/harness/context/capabilities/compact/types.d.ts +30 -1
- package/dist/harness/context/capabilities/summary/executor.js +2 -1
- package/dist/harness/context/capabilities/summary/policies.js +8 -5
- package/dist/harness/context/constitution/default-agents.md +11 -10
- package/dist/harness/context/default-context-engine.d.ts +34 -4
- package/dist/harness/context/default-context-engine.js +194 -49
- package/dist/harness/context/hybrid-compressor.js +6 -0
- package/dist/harness/context/llm-summarizer.d.ts +2 -8
- package/dist/harness/context/llm-summarizer.js +69 -91
- package/dist/harness/context/message-selector.js +4 -3
- package/dist/harness/context/summarize.d.ts +2 -0
- package/dist/harness/context/summarize.js +7 -2
- package/dist/harness/context/truncate-compressor.js +7 -0
- package/dist/harness/context/turn-envelope.d.ts +3 -2
- package/dist/harness/context/turn-envelope.js +3 -2
- package/dist/harness/context/types.d.ts +33 -1
- package/dist/harness/extension/execution-environment/dangerous-commands.d.ts +28 -0
- package/dist/harness/extension/execution-environment/dangerous-commands.js +129 -0
- package/dist/harness/extension/execution-environment/effect-policy.d.ts +127 -8
- package/dist/harness/extension/execution-environment/effect-policy.js +244 -19
- package/dist/harness/extension/execution-environment/index.d.ts +4 -2
- package/dist/harness/extension/execution-environment/index.js +2 -1
- package/dist/harness/extension/execution-environment/kill-switch.d.ts +34 -0
- package/dist/harness/extension/execution-environment/kill-switch.js +46 -0
- package/dist/harness/extension/execution-environment/ops-audit.d.ts +200 -0
- package/dist/harness/extension/execution-environment/ops-audit.js +338 -0
- package/dist/harness/extension/execution-environment/privileged-exec.d.ts +158 -0
- package/dist/harness/extension/execution-environment/privileged-exec.js +418 -0
- package/dist/harness/extension/execution-environment/process-info.d.ts +180 -0
- package/dist/harness/extension/execution-environment/process-info.js +466 -0
- package/dist/harness/extension/execution-environment/process-isolation.d.ts +7 -0
- package/dist/harness/extension/execution-environment/process-isolation.js +9 -0
- package/dist/harness/extension/execution-environment/sandbox.d.ts +109 -0
- package/dist/harness/extension/execution-environment/sandbox.js +228 -21
- package/dist/harness/extension/execution-environment/tool-capabilities.d.ts +1 -1
- package/dist/harness/extension/execution-environment/tool-capabilities.js +6 -0
- package/dist/harness/extension/execution-environment/types.d.ts +19 -0
- package/dist/harness/extension/plugin-ecosystem/channel-types.d.ts +5 -0
- package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.js +33 -0
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.d.ts +4 -5
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.js +5 -15
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +3 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +53 -2
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.d.ts +43 -0
- package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.js +350 -0
- package/dist/harness/extension/plugin-ecosystem/tools/process-tools.d.ts +30 -0
- package/dist/harness/extension/plugin-ecosystem/tools/process-tools.js +137 -0
- package/dist/harness/governance/human-in-the-loop/decision-cache.d.ts +54 -4
- package/dist/harness/governance/human-in-the-loop/decision-cache.js +42 -16
- package/dist/harness/governance/security/temp-path.d.ts +9 -0
- package/dist/harness/governance/security/temp-path.js +13 -2
- package/dist/harness/governance/session-acl/seed-roles.js +14 -0
- package/dist/harness/index.d.ts +2 -0
- package/dist/harness/index.js +2 -0
- package/dist/harness/knowledge/fts.js +1 -2
- package/dist/harness/knowledge/index-store.js +59 -9
- package/dist/harness/memory/sqlite/keyword-search.d.ts +1 -0
- package/dist/harness/memory/sqlite/keyword-search.js +4 -3
- package/dist/harness/observability/observer/hub.js +8 -0
- package/dist/harness/observability/observer/run-snapshot.js +1 -0
- package/dist/harness/observability/observer/types.d.ts +4 -0
- package/dist/harness/run/agent/agent.d.ts +7 -0
- package/dist/harness/run/agent/agent.js +23 -1
- package/dist/harness/run/reliability/fallback-provider.d.ts +117 -5
- package/dist/harness/run/reliability/fallback-provider.js +262 -22
- package/dist/harness/run/reliability/harness-events.d.ts +24 -1
- package/dist/harness/run/reliability/model-error-policy.d.ts +32 -0
- package/dist/harness/run/reliability/model-error-policy.js +42 -0
- package/dist/harness/run/reliability/provider-wrapper.js +29 -2
- package/dist/harness/run/reliability/refusal-guard.d.ts +20 -0
- package/dist/harness/run/reliability/refusal-guard.js +102 -0
- package/dist/harness/run/reliability/run-agent.js +231 -33
- package/dist/harness/run/run-guard/default-run-guard.js +1 -1
- package/dist/harness/run/run-scope.d.ts +31 -0
- package/dist/harness/run/runner.d.ts +116 -2
- package/dist/harness/run/runner.js +547 -41
- package/dist/harness/session/attachments/llm-intent.js +2 -1
- package/dist/harness/session/tasks/render.d.ts +4 -1
- package/dist/harness/session/tasks/render.js +10 -7
- package/dist/harness/shared/events/agent-event-map.d.ts +66 -0
- package/dist/harness/shared/events/agent-event-map.js +5 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2 -2
- package/dist/integration/agent-runtime/channel-message-source.js +19 -1
- package/dist/integration/index.d.ts +2 -0
- package/dist/integration/index.js +1 -0
- package/dist/integration/observability/loop-watchdog.d.ts +56 -0
- package/dist/integration/observability/loop-watchdog.js +177 -0
- package/dist/integration/observability/metrics.d.ts +3 -0
- package/dist/integration/observability/metrics.js +18 -7
- package/dist/integration/providers/anthropic.d.ts +23 -0
- package/dist/integration/providers/anthropic.js +97 -12
- package/dist/integration/providers/fetch-timeout.d.ts +42 -0
- package/dist/integration/providers/fetch-timeout.js +51 -0
- package/dist/integration/providers/openai.d.ts +10 -0
- package/dist/integration/providers/openai.js +58 -11
- package/dist/integration/providers/provider-diagnostics.d.ts +80 -0
- package/dist/integration/providers/provider-diagnostics.js +89 -0
- package/dist/integration/providers/timeout-coherence.d.ts +52 -0
- package/dist/integration/providers/timeout-coherence.js +68 -0
- package/dist/integration/providers/wire-dump.d.ts +62 -0
- package/dist/integration/providers/wire-dump.js +111 -2
- package/dist/integration/storage/jsonl.js +4 -1
- package/dist/subsystems/cognition-steward/conceptualize/handler.js +1 -1
- package/dist/subsystems/memory-steward/backfill/handler.js +1 -1
- package/dist/subsystems/wisdom-steward/formation/handler.js +1 -1
- package/package.json +3 -2
|
@@ -35,6 +35,7 @@ export interface AgentOptions {
|
|
|
35
35
|
shouldStopAfterTurn?: AgentLoopConfig['shouldStopAfterTurn'];
|
|
36
36
|
onTurnComplete?: AgentLoopConfig['onTurnComplete'];
|
|
37
37
|
prepareNextTurn?: AgentLoopConfig['prepareNextTurn'];
|
|
38
|
+
onToolUpdate?: AgentLoopConfig['onToolUpdate'];
|
|
38
39
|
onError?: AgentLoopConfig['onError'];
|
|
39
40
|
getSteeringMessages?: AgentLoopConfig['getSteeringMessages'];
|
|
40
41
|
getFollowUpMessages?: AgentLoopConfig['getFollowUpMessages'];
|
|
@@ -159,6 +160,12 @@ export declare class Agent {
|
|
|
159
160
|
context?: AgentContext;
|
|
160
161
|
/** 本 Run 身份(sessionId/agentId/systemPrompt/toolRuntime) */
|
|
161
162
|
runScope?: import('../run-scope.js').RunScope;
|
|
163
|
+
/** 本 Run 的 steering/followUp 接续;与实例回调组合,不改共享配置。 */
|
|
164
|
+
consumeMessages?: () => Promise<Message[]>;
|
|
165
|
+
/** 只读运行期提示(如委派状态):只进 steering 接缝,绝不经 followUp 续命。 */
|
|
166
|
+
steeringExtras?: () => Promise<Message[]>;
|
|
167
|
+
/** 本 Run 的工具部分输出通道(tool.exec.update 展示事件的发射端) */
|
|
168
|
+
onToolUpdate?: AgentLoopConfig['onToolUpdate'];
|
|
162
169
|
/** @deprecated 旧路径:仅绑定 provider */
|
|
163
170
|
model?: ModelProvider;
|
|
164
171
|
/** @deprecated 旧路径:仅窗口 */
|
|
@@ -41,6 +41,7 @@ export class Agent {
|
|
|
41
41
|
transformContext: options.transformContext,
|
|
42
42
|
beforeToolCall: options.beforeToolCall,
|
|
43
43
|
afterToolCall: options.afterToolCall,
|
|
44
|
+
onToolUpdate: options.onToolUpdate,
|
|
44
45
|
shouldStopAfterTurn: options.shouldStopAfterTurn,
|
|
45
46
|
onTurnComplete: options.onTurnComplete,
|
|
46
47
|
prepareNextTurn: options.prepareNextTurn,
|
|
@@ -202,7 +203,28 @@ export class Agent {
|
|
|
202
203
|
}
|
|
203
204
|
const resolved = options?.resolvedModel;
|
|
204
205
|
const loopModel = resolved?.provider ?? options?.model;
|
|
205
|
-
const config =
|
|
206
|
+
const config = {
|
|
207
|
+
...this._config,
|
|
208
|
+
...(loopModel ? { model: loopModel } : {}),
|
|
209
|
+
...(options?.onToolUpdate ? { onToolUpdate: options.onToolUpdate } : {}),
|
|
210
|
+
...(options?.consumeMessages || options?.steeringExtras ? (() => {
|
|
211
|
+
const consume = options.consumeMessages;
|
|
212
|
+
const extras = options.steeringExtras;
|
|
213
|
+
return {
|
|
214
|
+
getSteeringMessages: async () => [
|
|
215
|
+
...(consume ? await consume() : []),
|
|
216
|
+
...(extras ? await extras() : []),
|
|
217
|
+
...(this._config.getSteeringMessages ? await this._config.getSteeringMessages() : []),
|
|
218
|
+
],
|
|
219
|
+
getFollowUpMessages: async () => [
|
|
220
|
+
// followUp 是「续命」接缝:只喂真实输入;只读提示(ephemeral)
|
|
221
|
+
// 绝不能出现在这里——否则 run 会被状态快照无限续命(评审 F1)
|
|
222
|
+
...((consume ? await consume() : [])).filter((m) => m.ephemeral !== true),
|
|
223
|
+
...(this._config.getFollowUpMessages ? await this._config.getFollowUpMessages() : []),
|
|
224
|
+
],
|
|
225
|
+
};
|
|
226
|
+
})() : {}),
|
|
227
|
+
};
|
|
206
228
|
const context = options?.context ?? this._context;
|
|
207
229
|
const runScope = {
|
|
208
230
|
sessionId: options?.runScope?.sessionId ?? this._contextSessionId,
|
|
@@ -1,40 +1,152 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FallbackProvider — 模型回退提供器
|
|
3
3
|
*
|
|
4
|
-
* 包装多个 ModelProvider
|
|
5
|
-
* 主 provider 失败时自动切换到下一个回退 provider。
|
|
4
|
+
* 包装多个 ModelProvider,按顺序尝试。主 provider 失败时自动切换到下一个回退 provider。
|
|
6
5
|
*
|
|
7
6
|
* 使用场景:
|
|
8
7
|
* - 主模型不可用(circuit breaker open、限流、超时)
|
|
9
8
|
* - 主模型返回错误
|
|
10
9
|
* - 跨 provider 灾备(如 OpenAI -> Anthropic)
|
|
10
|
+
*
|
|
11
|
+
* ## 单节点首块预算(2026-10-27 修复)
|
|
12
|
+
*
|
|
13
|
+
* 引擎 watchdog(core/loop/call-model.ts)把**整条链**当作一次调用计时,只认内容 chunk。
|
|
14
|
+
* 于是主 provider 若「连上了但长期不产首块」(上游排队 / 长 prefill,期间只发 SSE ping),
|
|
15
|
+
* 会独占整个 watchdog 窗口,备选模型一次机会都拿不到——实测一次 52 分钟停滞即由此而来。
|
|
16
|
+
*
|
|
17
|
+
* 因此链内每个节点各有自己的「等首个内容 chunk」预算(`attemptTtftTimeoutMs`):
|
|
18
|
+
* - 仅在**还有下一个候选**时生效(最后一棒没有备选可切,交给外层 watchdog 兜底)
|
|
19
|
+
* - 超时即 abort 该次尝试(子 AbortController,不波及父 signal),切下一节点
|
|
20
|
+
* - 一旦首个 chunk 产出,本次调用完全委托该节点,预算不再起作用
|
|
21
|
+
* (流中途的静默由外层 idle watchdog 负责,见 stream() 注释)
|
|
22
|
+
*
|
|
23
|
+
* ## 失败节点降级
|
|
24
|
+
*
|
|
25
|
+
* 超时/失败过的节点进入降级窗口(`degradeTtlMs`,缺省等于首块预算),
|
|
26
|
+
* 后续调用把它排到最后。这使「重试」不再是原地重撞同一节点——
|
|
27
|
+
* 这也是条件化重试的落地方式:重试时条件(节点顺序)确实变了。
|
|
11
28
|
*/
|
|
12
29
|
import type { ModelProvider, LLMRequest, LLMResponse, LLMStreamChunk } from '@octopi-agent/core/interfaces/model-provider.js';
|
|
13
30
|
import type { ModelInfo } from '@octopi-agent/core/types/agent-definition.js';
|
|
14
31
|
import type { ModelConfig } from '../../shared/types/agent-definition.js';
|
|
32
|
+
/**
|
|
33
|
+
* 单节点等待首个内容 chunk 的默认预算(毫秒)
|
|
34
|
+
*
|
|
35
|
+
* 依据:同一时段实测备选 provider 处理同体积请求 1–16s;
|
|
36
|
+
* 主 provider 出现 271s / 383s 级首块延迟。90s 足以放过正常抖动,
|
|
37
|
+
* 又不至于让一次用户提问白白等满 watchdog 窗口。
|
|
38
|
+
*/
|
|
39
|
+
export declare const DEFAULT_FALLBACK_ATTEMPT_TTFT_MS = 90000;
|
|
40
|
+
/**
|
|
41
|
+
* 单节点非流式(chat)尝试的默认预算(毫秒)
|
|
42
|
+
*
|
|
43
|
+
* chat 没有「首块」概念,预算覆盖的是**整段响应**;复用 90s 的 TTFT 值会把
|
|
44
|
+
* 合法的长响应(推理型模型、长输出)误判为节点故障并降级。
|
|
45
|
+
* 取 240s:低于 undici `bodyTimeout` 的 300s,从而能在被 undici 以
|
|
46
|
+
* 含糊的 `terminated` 掐断之前把控制权交回引擎、切换下一节点。
|
|
47
|
+
*/
|
|
48
|
+
export declare const DEFAULT_FALLBACK_CHAT_ATTEMPT_MS = 240000;
|
|
49
|
+
/** 构造选项 */
|
|
50
|
+
export interface FallbackProviderOptions {
|
|
51
|
+
/** 单节点等待首个内容 chunk 的上限(仅 stream);<=0 或未提供时用 DEFAULT_FALLBACK_ATTEMPT_TTFT_MS */
|
|
52
|
+
attemptTtftTimeoutMs?: number;
|
|
53
|
+
/** 单节点非流式整响应的上限(仅 chat);<=0 或未提供时用 DEFAULT_FALLBACK_CHAT_ATTEMPT_MS */
|
|
54
|
+
attemptChatTimeoutMs?: number;
|
|
55
|
+
/** 失败节点降级窗口;缺省等于 attemptTtftTimeoutMs */
|
|
56
|
+
degradeTtlMs?: number;
|
|
57
|
+
/** 时钟注入(测试用) */
|
|
58
|
+
now?: () => number;
|
|
59
|
+
}
|
|
15
60
|
/**
|
|
16
61
|
* 回退感知的 ModelProvider
|
|
17
62
|
*
|
|
18
63
|
* 尝试主 provider,失败后按 fallbackModels 顺序尝试备选。
|
|
19
|
-
*
|
|
64
|
+
* 每个节点有独立首块预算,失败节点在降级窗口内被排到最后。
|
|
20
65
|
*/
|
|
21
66
|
export declare class FallbackProvider implements ModelProvider {
|
|
22
67
|
readonly name: string;
|
|
23
68
|
readonly defaultModel?: string;
|
|
24
69
|
private chain;
|
|
25
|
-
|
|
70
|
+
private readonly attemptTtftTimeoutMs;
|
|
71
|
+
private readonly attemptChatTimeoutMs;
|
|
72
|
+
private readonly degradeTtlMs;
|
|
73
|
+
private readonly now;
|
|
74
|
+
/** 节点下标 → 降级截止时间戳 */
|
|
75
|
+
private readonly degradedUntil;
|
|
76
|
+
constructor(mainProvider: ModelProvider, mainModel: string, fallbackModels: ModelConfig[], providerMap: Map<string, ModelProvider>, options?: FallbackProviderOptions);
|
|
77
|
+
/**
|
|
78
|
+
* 本次调用的尝试顺序:未降级节点保持原相对顺序在前,降级节点在后。
|
|
79
|
+
*
|
|
80
|
+
* @returns 带原下标与「是否仍有下一候选」的尝试序列
|
|
81
|
+
*/
|
|
82
|
+
private attemptOrder;
|
|
83
|
+
/** 标记节点失败(进入降级窗口) */
|
|
84
|
+
private markDegraded;
|
|
85
|
+
/** 标记节点恢复(成功一次即解除降级) */
|
|
86
|
+
private markHealthy;
|
|
87
|
+
/**
|
|
88
|
+
* 仅在「节点健康类」错误上降级
|
|
89
|
+
*
|
|
90
|
+
* 401/403(鉴权)、400(上下文超长/请求非法)、billing 等是**请求级**问题,
|
|
91
|
+
* 换节点不会变好;把它们算作节点故障会让主节点被无谓地排后,
|
|
92
|
+
* 并在下一次调用里静默换到别的模型(计费/缓存口径随之错位)。
|
|
93
|
+
*
|
|
94
|
+
* @param index - 节点下标
|
|
95
|
+
* @param err - 该节点的错误
|
|
96
|
+
*/
|
|
97
|
+
private degradeIfNodeUnhealthy;
|
|
98
|
+
/**
|
|
99
|
+
* 由备选节点服务时留下一行可检索的提示
|
|
100
|
+
*
|
|
101
|
+
* 换模型对调用方不可见(响应里的 model 由各 provider 自填,用量/计费口径仍在主模型名下),
|
|
102
|
+
* 若不显式记一笔,「为什么这次的回答风格/质量变了」将无从追查。
|
|
103
|
+
*
|
|
104
|
+
* @param entry - 实际服务的节点
|
|
105
|
+
* @param pos - 其在本次尝试顺序中的位置(0 = 主节点)
|
|
106
|
+
*/
|
|
107
|
+
private warnIfServedByFallback;
|
|
108
|
+
/**
|
|
109
|
+
* 在预算内等待一个 promise;超时则 reject 预算耗尽标记
|
|
110
|
+
*
|
|
111
|
+
* @param work - 待等待的 promise
|
|
112
|
+
* @param budgetMs - 预算;undefined 表示不限(最后一个候选)
|
|
113
|
+
* @param onTimeout - 预算耗尽时的清理动作(abort 子 signal)
|
|
114
|
+
* @returns work 的结果
|
|
115
|
+
*/
|
|
116
|
+
private withBudget;
|
|
26
117
|
chat(request: LLMRequest): Promise<LLMResponse>;
|
|
27
118
|
/**
|
|
28
119
|
* 流式生成回复
|
|
29
120
|
*
|
|
30
|
-
*
|
|
121
|
+
* 每个节点在独立首块预算内等待首个 chunk;超时切下一节点。
|
|
31
122
|
*
|
|
32
123
|
* 注意:一旦某个 provider 成功产出第一个 chunk,后续就完全委托给该 provider。
|
|
33
124
|
* 如果该 provider 在流中途抛异常,错误会直接传递给调用方,不会触发回退。
|
|
34
125
|
* 这是流式生成的固有限制——已产出的内容无法撤回。
|
|
126
|
+
* 流中途的静默由外层 idle watchdog 负责(它会 abort 父 signal)。
|
|
35
127
|
*/
|
|
36
128
|
stream(request: LLMRequest): AsyncGenerator<LLMStreamChunk>;
|
|
37
129
|
isAvailable(): Promise<boolean>;
|
|
38
130
|
getModelInfo(modelName: string): ModelInfo | null;
|
|
39
131
|
getModelInfos(): ModelInfo[];
|
|
40
132
|
}
|
|
133
|
+
/**
|
|
134
|
+
* 「单节点预算耗尽」标记(观测用,与 STREAM_DIAG 同属 Symbol 通道,不影响错误分类)
|
|
135
|
+
*
|
|
136
|
+
* 定义在 harness(产生方);`integration/providers/wire-dump` 作为外层读取它落盘。
|
|
137
|
+
*/
|
|
138
|
+
export declare const ATTEMPT_BUDGET_EXHAUSTED_DIAG: unique symbol;
|
|
139
|
+
/**
|
|
140
|
+
* 在错误对象上标记「单节点预算耗尽」
|
|
141
|
+
*
|
|
142
|
+
* @param err - 待标记错误
|
|
143
|
+
* @param budgetMs - 被耗尽的预算
|
|
144
|
+
*/
|
|
145
|
+
export declare function attachAttemptBudgetExhausted(err: unknown, budgetMs: number): void;
|
|
146
|
+
/**
|
|
147
|
+
* 读取「单节点预算耗尽」标记
|
|
148
|
+
*
|
|
149
|
+
* @param err - 任意错误
|
|
150
|
+
* @returns 被耗尽的预算毫秒数;未标记时 undefined
|
|
151
|
+
*/
|
|
152
|
+
export declare function readAttemptBudgetExhausted(err: unknown): number | undefined;
|
|
@@ -1,26 +1,81 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FallbackProvider — 模型回退提供器
|
|
3
3
|
*
|
|
4
|
-
* 包装多个 ModelProvider
|
|
5
|
-
* 主 provider 失败时自动切换到下一个回退 provider。
|
|
4
|
+
* 包装多个 ModelProvider,按顺序尝试。主 provider 失败时自动切换到下一个回退 provider。
|
|
6
5
|
*
|
|
7
6
|
* 使用场景:
|
|
8
7
|
* - 主模型不可用(circuit breaker open、限流、超时)
|
|
9
8
|
* - 主模型返回错误
|
|
10
9
|
* - 跨 provider 灾备(如 OpenAI -> Anthropic)
|
|
10
|
+
*
|
|
11
|
+
* ## 单节点首块预算(2026-10-27 修复)
|
|
12
|
+
*
|
|
13
|
+
* 引擎 watchdog(core/loop/call-model.ts)把**整条链**当作一次调用计时,只认内容 chunk。
|
|
14
|
+
* 于是主 provider 若「连上了但长期不产首块」(上游排队 / 长 prefill,期间只发 SSE ping),
|
|
15
|
+
* 会独占整个 watchdog 窗口,备选模型一次机会都拿不到——实测一次 52 分钟停滞即由此而来。
|
|
16
|
+
*
|
|
17
|
+
* 因此链内每个节点各有自己的「等首个内容 chunk」预算(`attemptTtftTimeoutMs`):
|
|
18
|
+
* - 仅在**还有下一个候选**时生效(最后一棒没有备选可切,交给外层 watchdog 兜底)
|
|
19
|
+
* - 超时即 abort 该次尝试(子 AbortController,不波及父 signal),切下一节点
|
|
20
|
+
* - 一旦首个 chunk 产出,本次调用完全委托该节点,预算不再起作用
|
|
21
|
+
* (流中途的静默由外层 idle watchdog 负责,见 stream() 注释)
|
|
22
|
+
*
|
|
23
|
+
* ## 失败节点降级
|
|
24
|
+
*
|
|
25
|
+
* 超时/失败过的节点进入降级窗口(`degradeTtlMs`,缺省等于首块预算),
|
|
26
|
+
* 后续调用把它排到最后。这使「重试」不再是原地重撞同一节点——
|
|
27
|
+
* 这也是条件化重试的落地方式:重试时条件(节点顺序)确实变了。
|
|
28
|
+
*/
|
|
29
|
+
import { classifyError } from '@octopi-agent/core/loop/error-classifier.js';
|
|
30
|
+
/**
|
|
31
|
+
* 单节点等待首个内容 chunk 的默认预算(毫秒)
|
|
32
|
+
*
|
|
33
|
+
* 依据:同一时段实测备选 provider 处理同体积请求 1–16s;
|
|
34
|
+
* 主 provider 出现 271s / 383s 级首块延迟。90s 足以放过正常抖动,
|
|
35
|
+
* 又不至于让一次用户提问白白等满 watchdog 窗口。
|
|
11
36
|
*/
|
|
37
|
+
export const DEFAULT_FALLBACK_ATTEMPT_TTFT_MS = 90_000;
|
|
38
|
+
/**
|
|
39
|
+
* 单节点非流式(chat)尝试的默认预算(毫秒)
|
|
40
|
+
*
|
|
41
|
+
* chat 没有「首块」概念,预算覆盖的是**整段响应**;复用 90s 的 TTFT 值会把
|
|
42
|
+
* 合法的长响应(推理型模型、长输出)误判为节点故障并降级。
|
|
43
|
+
* 取 240s:低于 undici `bodyTimeout` 的 300s,从而能在被 undici 以
|
|
44
|
+
* 含糊的 `terminated` 掐断之前把控制权交回引擎、切换下一节点。
|
|
45
|
+
*/
|
|
46
|
+
export const DEFAULT_FALLBACK_CHAT_ATTEMPT_MS = 240_000;
|
|
47
|
+
/** 单次尝试的首块预算被耗尽的标记(用于区分「自家 abort」与父级 abort) */
|
|
48
|
+
const ATTEMPT_BUDGET_EXHAUSTED = Symbol('fallback.attemptBudgetExhausted');
|
|
12
49
|
/**
|
|
13
50
|
* 回退感知的 ModelProvider
|
|
14
51
|
*
|
|
15
52
|
* 尝试主 provider,失败后按 fallbackModels 顺序尝试备选。
|
|
16
|
-
*
|
|
53
|
+
* 每个节点有独立首块预算,失败节点在降级窗口内被排到最后。
|
|
17
54
|
*/
|
|
18
55
|
export class FallbackProvider {
|
|
19
56
|
name;
|
|
20
57
|
defaultModel;
|
|
21
58
|
chain;
|
|
22
|
-
|
|
59
|
+
attemptTtftTimeoutMs;
|
|
60
|
+
attemptChatTimeoutMs;
|
|
61
|
+
degradeTtlMs;
|
|
62
|
+
now;
|
|
63
|
+
/** 节点下标 → 降级截止时间戳 */
|
|
64
|
+
degradedUntil = new Map();
|
|
65
|
+
constructor(mainProvider, mainModel, fallbackModels, providerMap, options = {}) {
|
|
23
66
|
this.name = mainProvider.name;
|
|
67
|
+
this.attemptTtftTimeoutMs =
|
|
68
|
+
options.attemptTtftTimeoutMs && options.attemptTtftTimeoutMs > 0
|
|
69
|
+
? options.attemptTtftTimeoutMs
|
|
70
|
+
: DEFAULT_FALLBACK_ATTEMPT_TTFT_MS;
|
|
71
|
+
this.attemptChatTimeoutMs =
|
|
72
|
+
options.attemptChatTimeoutMs && options.attemptChatTimeoutMs > 0
|
|
73
|
+
? options.attemptChatTimeoutMs
|
|
74
|
+
: DEFAULT_FALLBACK_CHAT_ATTEMPT_MS;
|
|
75
|
+
this.degradeTtlMs = options.degradeTtlMs && options.degradeTtlMs > 0
|
|
76
|
+
? options.degradeTtlMs
|
|
77
|
+
: this.attemptTtftTimeoutMs;
|
|
78
|
+
this.now = options.now ?? Date.now;
|
|
24
79
|
this.chain = [{ provider: mainProvider, model: mainModel, defaultModel: mainProvider.defaultModel }];
|
|
25
80
|
for (const fb of fallbackModels) {
|
|
26
81
|
const fbProvider = providerMap.get(fb.provider);
|
|
@@ -32,55 +87,211 @@ export class FallbackProvider {
|
|
|
32
87
|
}
|
|
33
88
|
this.defaultModel = mainModel;
|
|
34
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* 本次调用的尝试顺序:未降级节点保持原相对顺序在前,降级节点在后。
|
|
92
|
+
*
|
|
93
|
+
* @returns 带原下标与「是否仍有下一候选」的尝试序列
|
|
94
|
+
*/
|
|
95
|
+
attemptOrder() {
|
|
96
|
+
const nowTs = this.now();
|
|
97
|
+
const active = [];
|
|
98
|
+
const degraded = [];
|
|
99
|
+
for (let i = 0; i < this.chain.length; i++) {
|
|
100
|
+
const item = { entry: this.chain[i], index: i };
|
|
101
|
+
const until = this.degradedUntil.get(i);
|
|
102
|
+
if (until !== undefined && until > nowTs)
|
|
103
|
+
degraded.push(item);
|
|
104
|
+
else {
|
|
105
|
+
// 过期条目顺手清理,避免 Map 长期增长
|
|
106
|
+
if (until !== undefined)
|
|
107
|
+
this.degradedUntil.delete(i);
|
|
108
|
+
active.push(item);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return [...active, ...degraded];
|
|
112
|
+
}
|
|
113
|
+
/** 标记节点失败(进入降级窗口) */
|
|
114
|
+
markDegraded(index) {
|
|
115
|
+
this.degradedUntil.set(index, this.now() + this.degradeTtlMs);
|
|
116
|
+
}
|
|
117
|
+
/** 标记节点恢复(成功一次即解除降级) */
|
|
118
|
+
markHealthy(index) {
|
|
119
|
+
this.degradedUntil.delete(index);
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* 仅在「节点健康类」错误上降级
|
|
123
|
+
*
|
|
124
|
+
* 401/403(鉴权)、400(上下文超长/请求非法)、billing 等是**请求级**问题,
|
|
125
|
+
* 换节点不会变好;把它们算作节点故障会让主节点被无谓地排后,
|
|
126
|
+
* 并在下一次调用里静默换到别的模型(计费/缓存口径随之错位)。
|
|
127
|
+
*
|
|
128
|
+
* @param index - 节点下标
|
|
129
|
+
* @param err - 该节点的错误
|
|
130
|
+
*/
|
|
131
|
+
degradeIfNodeUnhealthy(index, err) {
|
|
132
|
+
const { reason } = classifyError(err);
|
|
133
|
+
if (reason === 'timeout' || reason === 'network' || reason === 'server' || reason === 'rate_limit') {
|
|
134
|
+
this.markDegraded(index);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* 由备选节点服务时留下一行可检索的提示
|
|
139
|
+
*
|
|
140
|
+
* 换模型对调用方不可见(响应里的 model 由各 provider 自填,用量/计费口径仍在主模型名下),
|
|
141
|
+
* 若不显式记一笔,「为什么这次的回答风格/质量变了」将无从追查。
|
|
142
|
+
*
|
|
143
|
+
* @param entry - 实际服务的节点
|
|
144
|
+
* @param pos - 其在本次尝试顺序中的位置(0 = 主节点)
|
|
145
|
+
*/
|
|
146
|
+
warnIfServedByFallback(entry, pos) {
|
|
147
|
+
if (pos === 0)
|
|
148
|
+
return;
|
|
149
|
+
console.warn(`[FallbackProvider] served by fallback node ${pos}: ${entry.provider.name}/${entry.model} ` +
|
|
150
|
+
`(primary ${this.name}/${this.defaultModel} unavailable or degraded)`);
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* 在预算内等待一个 promise;超时则 reject 预算耗尽标记
|
|
154
|
+
*
|
|
155
|
+
* @param work - 待等待的 promise
|
|
156
|
+
* @param budgetMs - 预算;undefined 表示不限(最后一个候选)
|
|
157
|
+
* @param onTimeout - 预算耗尽时的清理动作(abort 子 signal)
|
|
158
|
+
* @returns work 的结果
|
|
159
|
+
*/
|
|
160
|
+
async withBudget(work, budgetMs, onTimeout) {
|
|
161
|
+
if (budgetMs === undefined)
|
|
162
|
+
return work;
|
|
163
|
+
let timer;
|
|
164
|
+
const budget = new Promise((_resolve, reject) => {
|
|
165
|
+
timer = setTimeout(() => {
|
|
166
|
+
onTimeout();
|
|
167
|
+
reject(ATTEMPT_BUDGET_EXHAUSTED);
|
|
168
|
+
}, budgetMs);
|
|
169
|
+
});
|
|
170
|
+
try {
|
|
171
|
+
return await Promise.race([work, budget]);
|
|
172
|
+
}
|
|
173
|
+
finally {
|
|
174
|
+
if (timer)
|
|
175
|
+
clearTimeout(timer);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
35
178
|
async chat(request) {
|
|
179
|
+
const order = this.attemptOrder();
|
|
36
180
|
let lastError;
|
|
37
|
-
for (
|
|
181
|
+
for (let pos = 0; pos < order.length; pos++) {
|
|
182
|
+
const { entry, index } = order[pos];
|
|
183
|
+
const hasNext = pos < order.length - 1;
|
|
184
|
+
// 子 controller:预算耗尽时只取消本次尝试,不波及父级 signal
|
|
185
|
+
const attempt = new AbortController();
|
|
186
|
+
const onParentAbort = () => attempt.abort();
|
|
187
|
+
if (request.signal?.aborted)
|
|
188
|
+
attempt.abort();
|
|
189
|
+
else
|
|
190
|
+
request.signal?.addEventListener('abort', onParentAbort, { once: true });
|
|
38
191
|
try {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
192
|
+
const response = await this.withBudget(entry.provider.chat({ ...request, model: entry.model, signal: attempt.signal }),
|
|
193
|
+
// chat 是非流式:预算覆盖整段响应,与 stream 的「首块」语义不同(见 DEFAULT_FALLBACK_CHAT_ATTEMPT_MS)
|
|
194
|
+
hasNext ? this.attemptChatTimeoutMs : undefined, () => attempt.abort());
|
|
195
|
+
this.markHealthy(index);
|
|
196
|
+
this.warnIfServedByFallback(entry, pos);
|
|
197
|
+
return response;
|
|
43
198
|
}
|
|
44
199
|
catch (err) {
|
|
200
|
+
if (request.signal?.aborted)
|
|
201
|
+
throw err;
|
|
202
|
+
if (err === ATTEMPT_BUDGET_EXHAUSTED) {
|
|
203
|
+
this.markDegraded(index);
|
|
204
|
+
// 消息显式带 "timeout":classifyError 依文本归类时才能落到 timeout
|
|
205
|
+
//(否则落 unknown → 直接中止,丢掉了「换节点后再试一次」的机会)
|
|
206
|
+
lastError = new Error(`FallbackProvider: ${entry.provider.name}/${entry.model} response timeout after ${this.attemptChatTimeoutMs}ms (no response received)`);
|
|
207
|
+
attachAttemptBudgetExhausted(lastError, this.attemptChatTimeoutMs);
|
|
208
|
+
console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} attempt budget exhausted (${this.attemptChatTimeoutMs}ms), trying next`);
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
45
211
|
lastError = err;
|
|
212
|
+
this.degradeIfNodeUnhealthy(index, err);
|
|
46
213
|
const msg = err instanceof Error ? err.message : String(err);
|
|
47
214
|
console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} failed: ${msg}`);
|
|
48
215
|
}
|
|
216
|
+
finally {
|
|
217
|
+
request.signal?.removeEventListener('abort', onParentAbort);
|
|
218
|
+
}
|
|
49
219
|
}
|
|
50
220
|
throw lastError ?? new Error('All fallback providers exhausted');
|
|
51
221
|
}
|
|
52
222
|
/**
|
|
53
223
|
* 流式生成回复
|
|
54
224
|
*
|
|
55
|
-
*
|
|
225
|
+
* 每个节点在独立首块预算内等待首个 chunk;超时切下一节点。
|
|
56
226
|
*
|
|
57
227
|
* 注意:一旦某个 provider 成功产出第一个 chunk,后续就完全委托给该 provider。
|
|
58
228
|
* 如果该 provider 在流中途抛异常,错误会直接传递给调用方,不会触发回退。
|
|
59
229
|
* 这是流式生成的固有限制——已产出的内容无法撤回。
|
|
230
|
+
* 流中途的静默由外层 idle watchdog 负责(它会 abort 父 signal)。
|
|
60
231
|
*/
|
|
61
232
|
async *stream(request) {
|
|
233
|
+
const order = this.attemptOrder();
|
|
62
234
|
let lastError;
|
|
63
|
-
for (
|
|
235
|
+
for (let pos = 0; pos < order.length; pos++) {
|
|
236
|
+
const { entry, index } = order[pos];
|
|
237
|
+
const hasNext = pos < order.length - 1;
|
|
238
|
+
// 子 controller:预算耗尽时取消本次尝试的 HTTP,不波及父级 signal
|
|
239
|
+
const attempt = new AbortController();
|
|
240
|
+
const onParentAbort = () => attempt.abort();
|
|
241
|
+
if (request.signal?.aborted)
|
|
242
|
+
attempt.abort();
|
|
243
|
+
else
|
|
244
|
+
request.signal?.addEventListener('abort', onParentAbort, { once: true });
|
|
245
|
+
let gen;
|
|
246
|
+
let first;
|
|
64
247
|
// try/catch 仅覆盖初始化阶段(创建 stream + 获取首个 chunk)。
|
|
65
248
|
// 一旦首个 chunk 成功产出,后续 yield* 在 try/catch 之外执行,
|
|
66
249
|
// 确保流中途错误直接向上传播,不会被吞掉后静默回退。
|
|
67
|
-
let gen;
|
|
68
|
-
let first;
|
|
69
250
|
try {
|
|
70
|
-
gen = entry.provider.stream({ ...request, model: entry.model });
|
|
71
|
-
first = await gen.next();
|
|
251
|
+
gen = entry.provider.stream({ ...request, model: entry.model, signal: attempt.signal });
|
|
252
|
+
first = await this.withBudget(gen.next(), hasNext ? this.attemptTtftTimeoutMs : undefined, () => attempt.abort());
|
|
72
253
|
}
|
|
73
254
|
catch (err) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
255
|
+
// 放弃被预算掐断的尝试:abort 底层 HTTP 并回收 generator
|
|
256
|
+
void gen?.return?.(undefined)?.catch(() => undefined);
|
|
257
|
+
request.signal?.removeEventListener('abort', onParentAbort);
|
|
258
|
+
// 父级中止:不是本节点的问题,直接上抛(不降级、不切下一个)
|
|
259
|
+
if (request.signal?.aborted)
|
|
260
|
+
throw err;
|
|
261
|
+
if (err === ATTEMPT_BUDGET_EXHAUSTED) {
|
|
262
|
+
this.markDegraded(index);
|
|
263
|
+
lastError = new Error(`FallbackProvider: ${entry.provider.name}/${entry.model} first-chunk timeout after ${this.attemptTtftTimeoutMs}ms (no chunk received)`);
|
|
264
|
+
attachAttemptBudgetExhausted(lastError, this.attemptTtftTimeoutMs);
|
|
265
|
+
console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} attempt budget exhausted (${this.attemptTtftTimeoutMs}ms), trying next`);
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
lastError = err;
|
|
269
|
+
this.degradeIfNodeUnhealthy(index, err);
|
|
270
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
271
|
+
console.warn(`[FallbackProvider] ${entry.provider.name}/${entry.model} stream failed: ${msg}`);
|
|
272
|
+
}
|
|
77
273
|
continue;
|
|
78
274
|
}
|
|
79
|
-
if (first.done)
|
|
275
|
+
if (first.done) {
|
|
276
|
+
// 空流:本轮不算成功,交给下一节点(与旧行为一致)
|
|
277
|
+
request.signal?.removeEventListener('abort', onParentAbort);
|
|
278
|
+
this.markDegraded(index);
|
|
80
279
|
continue;
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
280
|
+
}
|
|
281
|
+
this.markHealthy(index);
|
|
282
|
+
this.warnIfServedByFallback(entry, pos);
|
|
283
|
+
// 首个 chunk 成功,完全委托给当前 provider(无 try/catch 包裹)。
|
|
284
|
+
// finally 里必须 abort:消费者 break / 提前 return 时 gen.return() 打断不了
|
|
285
|
+
// 挂在 reader.read() 上的 await,只有 abort 子 signal 才能真正关掉底层 HTTP。
|
|
286
|
+
// 对已完成/正常的流 abort 是无害的(provider 侧 signal 已不再被等待)。
|
|
287
|
+
try {
|
|
288
|
+
yield first.value;
|
|
289
|
+
yield* gen;
|
|
290
|
+
}
|
|
291
|
+
finally {
|
|
292
|
+
attempt.abort();
|
|
293
|
+
request.signal?.removeEventListener('abort', onParentAbort);
|
|
294
|
+
}
|
|
84
295
|
return;
|
|
85
296
|
}
|
|
86
297
|
throw lastError ?? new Error('All fallback providers exhausted');
|
|
@@ -116,3 +327,32 @@ export class FallbackProvider {
|
|
|
116
327
|
return infos;
|
|
117
328
|
}
|
|
118
329
|
}
|
|
330
|
+
/**
|
|
331
|
+
* 「单节点预算耗尽」标记(观测用,与 STREAM_DIAG 同属 Symbol 通道,不影响错误分类)
|
|
332
|
+
*
|
|
333
|
+
* 定义在 harness(产生方);`integration/providers/wire-dump` 作为外层读取它落盘。
|
|
334
|
+
*/
|
|
335
|
+
export const ATTEMPT_BUDGET_EXHAUSTED_DIAG = Symbol.for('@octopi-agent/engine.provider.attemptBudgetExhausted');
|
|
336
|
+
/**
|
|
337
|
+
* 在错误对象上标记「单节点预算耗尽」
|
|
338
|
+
*
|
|
339
|
+
* @param err - 待标记错误
|
|
340
|
+
* @param budgetMs - 被耗尽的预算
|
|
341
|
+
*/
|
|
342
|
+
export function attachAttemptBudgetExhausted(err, budgetMs) {
|
|
343
|
+
if (typeof err !== 'object' || err === null)
|
|
344
|
+
return;
|
|
345
|
+
err[ATTEMPT_BUDGET_EXHAUSTED_DIAG] = budgetMs;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* 读取「单节点预算耗尽」标记
|
|
349
|
+
*
|
|
350
|
+
* @param err - 任意错误
|
|
351
|
+
* @returns 被耗尽的预算毫秒数;未标记时 undefined
|
|
352
|
+
*/
|
|
353
|
+
export function readAttemptBudgetExhausted(err) {
|
|
354
|
+
if (typeof err !== 'object' || err === null)
|
|
355
|
+
return undefined;
|
|
356
|
+
const v = err[ATTEMPT_BUDGET_EXHAUSTED_DIAG];
|
|
357
|
+
return typeof v === 'number' ? v : undefined;
|
|
358
|
+
}
|
|
@@ -107,7 +107,30 @@ export type SecurityBlockedEvent = {
|
|
|
107
107
|
}>;
|
|
108
108
|
};
|
|
109
109
|
};
|
|
110
|
+
/**
|
|
111
|
+
* 模型调用即将重试(用户可见)
|
|
112
|
+
*
|
|
113
|
+
* 背景:一次 52 分钟的会话停滞期间前端零反馈(发送看门狗 10s 内即被首个事件清掉),
|
|
114
|
+
* 用户只看到 runStatus 停在 waiting。该事件给重试窗口提供可观测锚点:
|
|
115
|
+
* 每次重试必须能被看见,且带「第几次 / 还剩几次 / 为什么 / 等多久」。
|
|
116
|
+
*/
|
|
117
|
+
export type ModelRetryEvent = {
|
|
118
|
+
type: 'model_retry';
|
|
119
|
+
timestamp: number;
|
|
120
|
+
data: {
|
|
121
|
+
/** 本次是第几次重试(从 1 起) */
|
|
122
|
+
attempt: number;
|
|
123
|
+
/** 该错误原因允许的最大重试次数 */
|
|
124
|
+
maxAttempts: number;
|
|
125
|
+
/** 分类后的错误原因(timeout / network / server / rate_limit …) */
|
|
126
|
+
reason: string;
|
|
127
|
+
/** 重试前等待时长 */
|
|
128
|
+
delayMs: number;
|
|
129
|
+
/** 错误摘要(截断,供 UI 直接展示) */
|
|
130
|
+
error: string;
|
|
131
|
+
};
|
|
132
|
+
};
|
|
110
133
|
/** Harness 层扩展事件 */
|
|
111
|
-
export type HarnessLoopExtension = BudgetExceededEvent | BudgetControlEvent | RunGuardRecoveredEvent | RunGuardStoppedEvent | RunGuardMetricsEvent | SecurityBlockedEvent;
|
|
134
|
+
export type HarnessLoopExtension = BudgetExceededEvent | BudgetControlEvent | RunGuardRecoveredEvent | RunGuardStoppedEvent | RunGuardMetricsEvent | SecurityBlockedEvent | ModelRetryEvent;
|
|
112
135
|
/** reliability 包装后的完整事件流 */
|
|
113
136
|
export type HarnessLoopEvent = AgentLoopEvent | HarnessLoopExtension;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 模型调用错误的重试上限(单一事实源)
|
|
3
|
+
*
|
|
4
|
+
* 为什么集中在一处:生产链路由 `AgentBuilder` 注入 `DefaultErrorStrategy`,
|
|
5
|
+
* `runAgentWithReliability` 的兜底分支只在未注入时生效。两处各写一套数字,
|
|
6
|
+
* 很容易出现「改了一处、线上还是旧策略」的漂移。
|
|
7
|
+
*
|
|
8
|
+
* ## 为什么 timeout 保持 3 次(2026-10-27 修正)
|
|
9
|
+
*
|
|
10
|
+
* 曾经把它降到 1 次,理由是「一次 timeout 已耗掉整个 watchdog 窗口,再赌 3 轮就是
|
|
11
|
+
* 52 分钟黑屏」。那是把**症状**当成了**病因**:52 分钟来自「单次尝试耗时过长」
|
|
12
|
+
* 叠加「全程零反馈」,不是来自重试次数。砍次数只是让最坏时长变小,代价是
|
|
13
|
+
* 削弱了上游抖动时的自愈能力——这正是「让系统看起来好了」而不是修好了。
|
|
14
|
+
*
|
|
15
|
+
* 真正的修复在别处:
|
|
16
|
+
* - 单次尝试有界:回退链单节点预算(`fallbackAttemptTtftMs`)使有备选时 90s 即切换
|
|
17
|
+
* - 重试过程可见:`engine.model_retry` 事件 + WebUI 角标,用户知道在第几次、还要等多久
|
|
18
|
+
* - 重试过程可中止:退避等待响应 abort,用户不必等满整个序列
|
|
19
|
+
*
|
|
20
|
+
* 三者俱备后,3 次重试是安全的:用户全程看得到、随时能停。
|
|
21
|
+
*
|
|
22
|
+
* @module harness/run/reliability/model-error-policy
|
|
23
|
+
*/
|
|
24
|
+
/** 各错误原因的最大重试次数(不含首次尝试) */
|
|
25
|
+
export declare const MODEL_ERROR_MAX_ATTEMPTS: Readonly<Record<string, number>>;
|
|
26
|
+
/**
|
|
27
|
+
* 查询某错误原因允许的重试次数
|
|
28
|
+
*
|
|
29
|
+
* @param reason - `classifyError` 给出的错误原因
|
|
30
|
+
* @returns 最大重试次数;不可重试的原因返回 0
|
|
31
|
+
*/
|
|
32
|
+
export declare function maxAttemptsFor(reason: string): number;
|