niceeval 0.10.3-canary.7 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/agents/types.d.ts +5 -4
  2. package/dist/context/turn-errors.d.ts +27 -23
  3. package/dist/i18n/en.d.ts +14 -0
  4. package/dist/i18n/en.js +15 -1
  5. package/dist/i18n/zh-CN.d.ts +15 -1
  6. package/dist/i18n/zh-CN.js +15 -1
  7. package/dist/o11y/derive.js +28 -24
  8. package/dist/o11y/types.d.ts +8 -5
  9. package/dist/report/components/attempt-detail/UsageTable.js +4 -7
  10. package/dist/report/components/attempt-detail/compute.d.ts +2 -2
  11. package/dist/report/components/attempt-detail/compute.js +2 -6
  12. package/dist/report/components/attempt-detail/faces.js +7 -7
  13. package/dist/report/components/attempt-detail/index.js +0 -3
  14. package/dist/report/components/entity-lists/EvalList.js +0 -0
  15. package/dist/report/components/metric-views/compute.js +1 -1
  16. package/dist/report/model/types.d.ts +3 -4
  17. package/dist/results/locator.js +0 -0
  18. package/dist/results/select.d.ts +6 -0
  19. package/dist/results/select.js +8 -0
  20. package/dist/runner/feedback/sink.d.ts +26 -1
  21. package/dist/runner/fingerprint.d.ts +23 -0
  22. package/dist/runner/types.d.ts +105 -7
  23. package/dist/sandbox/errors.d.ts +29 -0
  24. package/dist/sandbox/resolve.d.ts +9 -0
  25. package/dist/shared/failure-class.d.ts +91 -0
  26. package/dist/types.d.ts +1 -0
  27. package/dist/util.d.ts +3 -2
  28. package/dist/util.js +31 -5
  29. package/docs-site/zh/explanation/runner.mdx +35 -0
  30. package/docs-site/zh/reference/cli.mdx +3 -3
  31. package/docs-site/zh/reference/events.mdx +4 -4
  32. package/docs-site/zh/troubleshooting/debugging.mdx +2 -2
  33. package/docs-site/zh/tutorials/viewing-results.mdx +4 -5
  34. package/package.json +4 -12
  35. package/src/agents/ai-sdk.test.ts +26 -0
  36. package/src/agents/ai-sdk.ts +7 -4
  37. package/src/agents/index.ts +5 -3
  38. package/src/agents/langgraph.test.ts +30 -0
  39. package/src/agents/langgraph.ts +5 -2
  40. package/src/agents/openai-compat.test.ts +35 -0
  41. package/src/agents/openai-compat.ts +16 -4
  42. package/src/agents/sdk-streams.test.ts +66 -0
  43. package/src/agents/sdk-streams.ts +3 -1
  44. package/src/agents/types.ts +5 -4
  45. package/src/cli.ts +55 -14
  46. package/src/context/context.test.ts +34 -0
  47. package/src/context/context.ts +14 -5
  48. package/src/context/send-retry.test.ts +86 -0
  49. package/src/context/send-retry.ts +37 -12
  50. package/src/context/session.ts +24 -0
  51. package/src/context/turn-errors.test.ts +124 -17
  52. package/src/context/turn-errors.ts +60 -50
  53. package/src/define.ts +5 -0
  54. package/src/i18n/en.ts +20 -1
  55. package/src/i18n/zh-CN.ts +20 -1
  56. package/src/index.ts +8 -0
  57. package/src/o11y/cost.ts +4 -2
  58. package/src/o11y/derive.test.ts +40 -0
  59. package/src/o11y/derive.ts +28 -22
  60. package/src/o11y/otlp/sandbox-receiver.test.ts +201 -0
  61. package/src/o11y/otlp/sandbox-receiver.ts +73 -27
  62. package/src/o11y/parsers/bub.test.ts +30 -0
  63. package/src/o11y/parsers/bub.ts +5 -2
  64. package/src/o11y/parsers/codex.test.ts +19 -0
  65. package/src/o11y/parsers/codex.ts +5 -2
  66. package/src/o11y/types.ts +8 -5
  67. package/src/report/components/attempt-detail/UsageTable.tsx +4 -6
  68. package/src/report/components/attempt-detail/attempt-components.test.tsx +5 -8
  69. package/src/report/components/attempt-detail/compute.ts +2 -7
  70. package/src/report/components/attempt-detail/faces.ts +7 -7
  71. package/src/report/components/attempt-detail/index.tsx +0 -3
  72. package/src/report/components/entity-lists/EvalList.tsx +0 -0
  73. package/src/report/components/metric-views/compute.ts +1 -1
  74. package/src/report/model/types.ts +3 -4
  75. package/src/results/format.ts +9 -2
  76. package/src/results/index.ts +2 -0
  77. package/src/results/locator.ts +0 -0
  78. package/src/results/open.ts +132 -21
  79. package/src/results/select.ts +12 -0
  80. package/src/results/skipped-notice.ts +0 -0
  81. package/src/runner/attempt.test.ts +116 -0
  82. package/src/runner/attempt.ts +89 -8
  83. package/src/runner/discover.ts +21 -3
  84. package/src/runner/feedback/coordinator.ts +24 -2
  85. package/src/runner/feedback/eval-conclusions.ts +6 -3
  86. package/src/runner/feedback/human.test.ts +300 -6
  87. package/src/runner/feedback/human.ts +132 -30
  88. package/src/runner/feedback/json.test.ts +127 -2
  89. package/src/runner/feedback/json.ts +51 -3
  90. package/src/runner/feedback/reducer.test.ts +328 -29
  91. package/src/runner/feedback/reducer.ts +84 -9
  92. package/src/runner/feedback/sink.ts +39 -1
  93. package/src/runner/fingerprint.ts +49 -19
  94. package/src/runner/gate-lease.test.ts +510 -0
  95. package/src/runner/gate-lease.ts +350 -0
  96. package/src/runner/lock.test.ts +454 -0
  97. package/src/runner/lock.ts +288 -0
  98. package/src/runner/report.test.ts +1 -0
  99. package/src/runner/run.test.ts +2044 -9
  100. package/src/runner/run.ts +825 -61
  101. package/src/runner/teardown-registry.ts +20 -78
  102. package/src/runner/types.ts +103 -7
  103. package/src/sandbox/errors.test.ts +72 -0
  104. package/src/sandbox/errors.ts +83 -0
  105. package/src/sandbox/keep-registry.ts +22 -46
  106. package/src/sandbox/resolve.test.ts +100 -0
  107. package/src/sandbox/resolve.ts +84 -27
  108. package/src/shared/entry-file-store.test.ts +149 -0
  109. package/src/shared/entry-file-store.ts +117 -0
  110. package/src/shared/failure-class.test.ts +137 -0
  111. package/src/shared/failure-class.ts +175 -0
  112. package/src/show/index.ts +5 -6
  113. package/src/show/render.test.ts +116 -15
  114. package/src/show/render.ts +124 -35
  115. package/src/types.ts +9 -0
  116. package/src/util.ts +31 -4
  117. package/src/view/app/App.tsx +5 -1
  118. package/src/view/client-dist/app.js +1 -1
  119. package/src/view/data.ts +5 -9
  120. package/src/view/shared/types.ts +7 -1
  121. package/src/view/view-report.test.ts +54 -0
@@ -9,6 +9,7 @@ import {
9
9
  type SendRetryDeps,
10
10
  type TurnLens,
11
11
  } from "./send-retry.ts";
12
+ import { failureClassOf, type FailureClass } from "../shared/failure-class.ts";
12
13
  import type { Turn } from "../types.ts";
13
14
 
14
15
  // 直调执行体,不用真实 setTimeout 睡眠——sleep 注入即返回(受控时钟),random 注入固定值,
@@ -265,6 +266,91 @@ describe("sendWithTurnRetry · adapter 分类器接入", () => {
265
266
  });
266
267
  });
267
268
 
269
+ describe("sendWithTurnRetry · 终局失败携带分类浮出(止损闸的进料)", () => {
270
+ const tunnelDown = { retryable: false, scope: "experiment", reason: "tunnel_down" } as const;
271
+
272
+ it("thrown 形态:终局失败的分类标在浮出的错误上,经 failureClassOf 读得到", async () => {
273
+ const original = new Error("connect ECONNREFUSED tunnel.example:443");
274
+ const promise = sendWithTurnRetry(
275
+ async () => {
276
+ throw original;
277
+ },
278
+ identityLens,
279
+ baseDeps({ experimentClassifier: () => tunnelDown }),
280
+ );
281
+ await expect(promise).rejects.toBe(original); // 浮出的仍是原始错误对象,不被包装替换
282
+ expect(failureClassOf(original)).toEqual(tunnelDown);
283
+ });
284
+
285
+ it("turn-failed 形态:分类经 onFinalFailure 回执转交调用方(Turn 上不留字段)", async () => {
286
+ const seen: { cls: FailureClass; turn?: Turn }[] = [];
287
+ const result = await sendWithTurnRetry(
288
+ async () => failedTurn("connect ECONNREFUSED tunnel.example:443"),
289
+ identityLens,
290
+ baseDeps({
291
+ experimentClassifier: () => tunnelDown,
292
+ onFinalFailure: (cls, failure) => {
293
+ seen.push({ cls, turn: failure.type === "turn-failed" ? failure.turn : undefined });
294
+ },
295
+ }),
296
+ );
297
+ expect(seen).toEqual([{ cls: tunnelDown, turn: result }]);
298
+ expect(failureClassOf(result)).toBeUndefined();
299
+ });
300
+
301
+ it("被重试吸收的失败不外泄:重试后成功时没有任何终局分类回执", async () => {
302
+ const seen: FailureClass[] = [];
303
+ let calls = 0;
304
+ const result = await sendWithTurnRetry(
305
+ async () => {
306
+ calls++;
307
+ return calls === 1 ? failedTurn("too many requests, retry later") : completedTurn();
308
+ },
309
+ identityLens,
310
+ baseDeps({
311
+ experimentClassifier: () => ({ retryable: true, reason: "tunnel_flaky", scope: "experiment" }),
312
+ onFinalFailure: (cls) => seen.push(cls),
313
+ }),
314
+ );
315
+ expect(result.status).toBe("completed");
316
+ expect(seen).toEqual([]);
317
+ });
318
+
319
+ it("重试耗尽时 scope 随失败携带浮出,回执报的是带耗尽摘要的那个 Turn", async () => {
320
+ const seen: { cls: FailureClass; turn?: Turn }[] = [];
321
+ const result = await sendWithTurnRetry(
322
+ async () => failedTurn("too many requests, retry later"),
323
+ identityLens,
324
+ baseDeps({
325
+ experimentClassifier: () => ({ retryable: true, reason: "tunnel_flaky", scope: "experiment" }),
326
+ onFinalFailure: (cls, failure) => {
327
+ seen.push({ cls, turn: failure.type === "turn-failed" ? failure.turn : undefined });
328
+ },
329
+ }),
330
+ );
331
+ expect(seen).toHaveLength(1);
332
+ expect(seen[0].cls).toEqual({ retryable: true, reason: "tunnel_flaky", scope: "experiment" });
333
+ expect(seen[0].turn).toBe(result);
334
+ expect((result.events[0] as { message: string }).message).toContain("tunnel_flaky");
335
+ });
336
+
337
+ it("实验分类器排在 adapter 之前:两者同时认领时携带的是实验的 scope", async () => {
338
+ const original = new Error("connect ECONNREFUSED tunnel.example:443");
339
+ const promise = sendWithTurnRetry(
340
+ async () => {
341
+ throw original;
342
+ },
343
+ identityLens,
344
+ baseDeps({
345
+ experimentClassifier: () => tunnelDown,
346
+ classifier: () => ({ retryable: true, reason: "network" }),
347
+ }),
348
+ );
349
+ await expect(promise).rejects.toBe(original);
350
+ expect(failureClassOf(original)).toEqual(tunnelDown);
351
+ });
352
+ });
353
+
268
354
  describe("sendWithTurnRetry · 中断", () => {
269
355
  it("退避睡眠期间 signal abort:干净打断,不等满整段延迟,槽位仍被收回", async () => {
270
356
  const ac = new AbortController();
@@ -6,7 +6,8 @@
6
6
 
7
7
  import type { Turn } from "../types.ts";
8
8
  import { t } from "../i18n/index.ts";
9
- import { resolveTurnErrorClass, type TurnErrorClass, type TurnErrorClassifier, type TurnFailure } from "./turn-errors.ts";
9
+ import { attachFailureClass, type AttemptFailureClassifier, type FailureClass } from "../shared/failure-class.ts";
10
+ import { resolveTurnFailureClass, type TurnErrorClassifier, type TurnFailure } from "./turn-errors.ts";
10
11
 
11
12
  /** 单次 send 调用封顶的尝试次数(首次 + 至多 3 次重试)。 */
12
13
  export const SEND_MAX_ATTEMPTS = 4;
@@ -47,6 +48,14 @@ export interface TurnLens<T> {
47
48
  export interface SendRetryDeps {
48
49
  /** adapter 声明的分类器(可选),undefined 回落保守兜底。 */
49
50
  classifier?: TurnErrorClassifier;
51
+ /** 实验声明的分类器(`ExperimentDef.classifyFailure`,可选),排在 adapter 之前询问。 */
52
+ experimentClassifier?: AttemptFailureClassifier;
53
+ /**
54
+ * 终局失败(不重试 / 重试耗尽)的分类回执:被重试吸收的失败不回调——只有真正浮出的失败
55
+ * 携带分类,止损闸消费的空间轴由此抵达 attempt 封口。`turn-failed` 形态的失败没有错误对象
56
+ * 可标记(错误在 `expectOk()` 才铸造),经这条回执转交调用方。
57
+ */
58
+ onFinalFailure?: (cls: FailureClass, failure: TurnFailure) => void;
50
59
  /** attempt 级预算,持续扣减(调用方在 attempt 生命周期内只创建一份并跨多次 send 复用)。 */
51
60
  budget: AttemptRetryBudget;
52
61
  /** 省略时不释放槽位(测试 / 无并发闸场景)。 */
@@ -93,7 +102,7 @@ function appendToLastErrorEvent(turn: Turn, suffix: string): Turn | undefined {
93
102
 
94
103
  /**
95
104
  * turn 级重试执行体:反复调 `callOnce()`(每次都是原样重发同一个 `TurnInput`),对失败结果
96
- * 走三道分类链(见 turn-errors.ts 的 `resolveTurnErrorClass`),可重试则退避后重试,否则把
105
+ * 走分类链(见 turn-errors.ts 的 `resolveTurnFailureClass`),可重试则退避后重试,否则把
97
106
  * 失败原样(或带耗尽摘要)浮出。
98
107
  */
99
108
  export async function sendWithTurnRetry<T>(
@@ -116,10 +125,15 @@ export async function sendWithTurnRetry<T>(
116
125
  failure = { type: "thrown", error: e };
117
126
  }
118
127
 
119
- const cls = resolveTurnErrorClass(failure, deps.classifier);
120
- if (!cls.retryable) return finalize(failure, result, lens);
121
- if (sendAttempt >= SEND_MAX_ATTEMPTS) return finalize(failure, result, lens, { layer: "send", cls });
122
- if (deps.budget.remaining <= 0) return finalize(failure, result, lens, { layer: "attempt", cls });
128
+ const cls = resolveTurnFailureClass(failure, {
129
+ experiment: deps.experimentClassifier,
130
+ adapter: deps.classifier,
131
+ });
132
+ // 终局失败才携带分类浮出:被吸收的失败尝试不留痕(见 architecture.md「重试执行体」),
133
+ // 它的 scope 永远到不了止损闸。
134
+ if (!cls.retryable) return finalize(failure, result, lens, deps, cls);
135
+ if (sendAttempt >= SEND_MAX_ATTEMPTS) return finalize(failure, result, lens, deps, cls, { layer: "send", cls });
136
+ if (deps.budget.remaining <= 0) return finalize(failure, result, lens, deps, cls, { layer: "attempt", cls });
123
137
 
124
138
  deps.budget.remaining -= 1;
125
139
  const delayMs = BASE_DELAY_MS * 2 ** (sendAttempt - 1) * random();
@@ -143,15 +157,22 @@ export async function sendWithTurnRetry<T>(
143
157
  }
144
158
  }
145
159
 
146
- /** 循环收口:未耗尽的非重试失败原样浮出;耗尽时按耗尽层追加摘要文本。 */
160
+ /**
161
+ * 循环收口:未耗尽的非重试失败原样浮出;耗尽时按耗尽层追加摘要文本。两条路径都先把终局分类
162
+ * 挂到浮出的失败上(`thrown` 形态标在错误对象上,沿 cause 链可读;两种形态都经 `onFinalFailure`
163
+ * 回执),抛出点自己声明过分类的错误不被覆盖。
164
+ */
147
165
  function finalize<T>(
148
166
  failure: TurnFailure,
149
167
  result: T | undefined,
150
168
  lens: TurnLens<T>,
151
- exhausted?: { layer: "send" | "attempt"; cls: Extract<TurnErrorClass, { retryable: true }> },
169
+ deps: SendRetryDeps,
170
+ cls: FailureClass,
171
+ exhausted?: { layer: "send" | "attempt"; cls: Extract<FailureClass, { retryable: true }> },
152
172
  ): T {
153
173
  if (!exhausted) {
154
- if (failure.type === "thrown") throw failure.error;
174
+ deps.onFinalFailure?.(cls, failure);
175
+ if (failure.type === "thrown") throw attachFailureClass(failure.error, cls);
155
176
  return result as T;
156
177
  }
157
178
  const suffix =
@@ -160,11 +181,15 @@ function finalize<T>(
160
181
  : t("session.turnRetryBudgetExhausted", { maxRetries: ATTEMPT_MAX_RETRIES, reason: exhausted.cls.reason });
161
182
 
162
183
  if (failure.type === "thrown") {
184
+ deps.onFinalFailure?.(cls, failure);
163
185
  const e = failure.error;
164
- // adapter 抛出的 Error 仍可能被上层保留、复用或记录;不要原地篡改它。
165
- if (e instanceof Error) throw new Error(e.message + suffix, { cause: e });
166
- throw e;
186
+ // adapter 抛出的 Error 仍可能被上层保留、复用或记录;不要原地篡改它(分类标记挂在外层)。
187
+ if (e instanceof Error) throw attachFailureClass(new Error(e.message + suffix, { cause: e }), cls);
188
+ throw attachFailureClass(e, cls);
167
189
  }
190
+ // 追加摘要产出的是一个新的 Turn 对象;回执必须报浮出的那个,分类才登记在调用方拿到的 Turn 上。
168
191
  const withSuffix = appendToLastErrorEvent(failure.turn, suffix);
192
+ const finalTurn = withSuffix ?? failure.turn;
193
+ deps.onFinalFailure?.(cls, { type: "turn-failed", turn: finalTurn });
169
194
  return withSuffix ? lens.set(result as T, withSuffix) : (result as T);
170
195
  }
@@ -12,6 +12,8 @@ import {
12
12
  type AttemptRetryBudget,
13
13
  type ConcurrencySlot,
14
14
  } from "./send-retry.ts";
15
+ import type { AttemptFailureClassifier, FailureClass } from "../shared/failure-class.ts";
16
+ import type { TurnFailure } from "./turn-errors.ts";
15
17
  import { recordFact } from "../shared/facts.ts";
16
18
 
17
19
  /**
@@ -144,6 +146,11 @@ export interface SessionDeps {
144
146
  * architecture.md「退避与槽位」)。省略时退避不释放槽位(测试 / 无并发闸场景)。
145
147
  */
146
148
  concurrencySlot?: ConcurrencySlot;
149
+ /**
150
+ * 实验声明的失败分类器(`ExperimentDef.classifyFailure`):turn 链上排在 adapter 分类器
151
+ * 之前询问(决议序见 docs/feature/error-classification/architecture.md「分类链」)。
152
+ */
153
+ experimentClassifier?: AttemptFailureClassifier;
147
154
  /** 仅供确定性单测注入:turn 重试执行体的随机数与睡眠(生产路径省略,走真实退避)。 */
148
155
  retryRandom?: () => number;
149
156
  retrySleep?: (ms: number, signal: AbortSignal) => Promise<void>;
@@ -194,6 +201,17 @@ export class SessionManager {
194
201
  return downgradeCoverage(this.agentCoverage, turn.coverage);
195
202
  }
196
203
 
204
+ /**
205
+ * 终局失败 Turn 的分类:失败 Turn 本身不是错误(作者不调 `expectOk()` 就不算失败),分类
206
+ * 因此不能挂在 Turn 上,只在 `expectOk()` 铸造 `TurnFailed` 时随错误浮出——这里按 Turn 身份
207
+ * 登记,`makeTurnHandle` 取用。被重试吸收的失败 Turn 从不外泄,也就不会被登记。
208
+ */
209
+ resolveTurnFailureClass(turn: Turn): FailureClass | undefined {
210
+ return this.turnFailureClasses.get(turn);
211
+ }
212
+
213
+ private readonly turnFailureClasses = new WeakMap<Turn, FailureClass>();
214
+
197
215
  async send(
198
216
  session: RunSession,
199
217
  text: string,
@@ -271,6 +289,12 @@ export class SessionManager {
271
289
  // 因此能被 Effect interruption 干净打断,不新增超时语义。
272
290
  const retryDeps = {
273
291
  classifier: this.deps.agent.classifyTurnError,
292
+ experimentClassifier: this.deps.experimentClassifier,
293
+ // 终局失败的分类落账:thrown 形态由执行体标在错误对象上,turn-failed 形态在这里按 Turn
294
+ // 身份登记,`expectOk()` 铸造 TurnFailed 时取出随错误浮出(止损闸的消费点在 attempt 封口)。
295
+ onFinalFailure: (cls: FailureClass, failure: TurnFailure) => {
296
+ if (failure.type === "turn-failed") this.turnFailureClasses.set(failure.turn, cls);
297
+ },
274
298
  budget: this.retryBudget,
275
299
  slot: this.deps.concurrencySlot,
276
300
  reportRetry: (message: string) => ctx.progress({ message }),
@@ -1,13 +1,18 @@
1
1
  // cases: docs/engineering/testing/unit/eval.md
2
- import { describe, expect, it } from "vitest";
2
+ import { describe, expect, it, vi } from "vitest";
3
3
  import {
4
4
  classifyTurnError,
5
5
  hasAgentEvidence,
6
- resolveTurnErrorClass,
6
+ resolveTurnFailureClass,
7
7
  turnErrorText,
8
- type TurnErrorClass,
9
8
  type TurnFailure,
10
9
  } from "./turn-errors.ts";
10
+ import {
11
+ EvalFatalError,
12
+ ExperimentFatalError,
13
+ type AttemptFailureInfo,
14
+ type FailureClass,
15
+ } from "../shared/failure-class.ts";
11
16
  import type { StreamEvent, Turn } from "../types.ts";
12
17
 
13
18
  function turnFailed(events: StreamEvent[]): TurnFailure {
@@ -87,30 +92,43 @@ describe("classifyTurnError(保守兜底) · thrown 形态", () => {
87
92
  });
88
93
  });
89
94
 
90
- describe("resolveTurnErrorClass · 三道分类链", () => {
95
+ describe("resolveTurnFailureClass · 五道分类链", () => {
91
96
  it("adapter 分类器返回结果时优先生效,自定义 reason 原样透出(不被塞进内建词表)", () => {
92
97
  const failure = turnFailed([errorEvent("ACME_QUEUE_FULL: too many concurrent runs")]);
93
- const result = resolveTurnErrorClass(failure, () => ({ retryable: true, reason: "acme_queue_full" }));
98
+ const result = resolveTurnFailureClass(failure, { adapter: () => ({ retryable: true, reason: "acme_queue_full" }) });
94
99
  expect(result).toEqual({ retryable: true, reason: "acme_queue_full" });
95
100
  });
96
101
 
97
102
  it("adapter 分类器返回 undefined 时回落保守兜底", () => {
98
103
  const failure = turnFailed([errorEvent("too many requests, back off")]);
99
- const result = resolveTurnErrorClass(failure, () => undefined);
104
+ const result = resolveTurnFailureClass(failure, { adapter: () => undefined });
100
105
  expect(result).toEqual({ retryable: true, reason: "rate_limit" });
101
106
  });
102
107
 
103
- it("adapter 分类器抛错按不可重试处理并被吞掉——不回落兜底,即使兜底本会判可重试", () => {
108
+ it("adapter 分类器抛错按 undefined 回落:错误被吞掉,链继续走到兜底", () => {
104
109
  const failure = turnFailed([errorEvent("too many requests, back off")]);
105
- const result = resolveTurnErrorClass(failure, () => {
106
- throw new Error("classifier bug");
110
+ const result = resolveTurnFailureClass(failure, {
111
+ adapter: () => {
112
+ throw new Error("classifier bug");
113
+ },
114
+ });
115
+ expect(result).toEqual({ retryable: true, reason: "rate_limit" });
116
+ });
117
+
118
+ it("实验分类器抛错同样按 undefined 回落,后续通道照常认领(不掩盖原始失败)", () => {
119
+ const failure = turnFailed([errorEvent("ACME_QUEUE_FULL")]);
120
+ const result = resolveTurnFailureClass(failure, {
121
+ experiment: () => {
122
+ throw new Error("classifier bug");
123
+ },
124
+ adapter: () => ({ retryable: true, reason: "acme_queue_full" }),
107
125
  });
108
- expect(result).toEqual({ retryable: false });
126
+ expect(result).toEqual({ retryable: true, reason: "acme_queue_full" });
109
127
  });
110
128
 
111
129
  it("没有 adapter 分类器时直接走保守兜底", () => {
112
130
  const failure = turnFailed([errorEvent("too many requests, back off")]);
113
- expect(resolveTurnErrorClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
131
+ expect(resolveTurnFailureClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
114
132
  });
115
133
 
116
134
  describe("受理证据门:失败 Turn 带 agent 产出事件时强制降级为不可重试", () => {
@@ -119,9 +137,9 @@ describe("resolveTurnErrorClass · 三道分类链", () => {
119
137
  ["thinking", { type: "thinking", text: "let me think" } satisfies StreamEvent],
120
138
  ["action.called", { type: "action.called", name: "bash", callId: "c1", input: {} } satisfies StreamEvent],
121
139
  ["action.result", { type: "action.result", callId: "c1", output: "", status: "completed" } satisfies StreamEvent],
122
- ])("%s 事件出现时,文本再像限流也不重试", (_label, evidenceEvent) => {
140
+ ])("%s 事件出现时,文本再像限流也不重试(reason 原样留着给人读)", (_label, evidenceEvent) => {
123
141
  const failure = turnFailed([evidenceEvent, errorEvent("Concurrency limit exceeded, please retry later")]);
124
- expect(resolveTurnErrorClass(failure)).toEqual({ retryable: false });
142
+ expect(resolveTurnFailureClass(failure)).toEqual({ retryable: false, reason: "rate_limit" });
125
143
  });
126
144
 
127
145
  it("adapter 分类器判可重试同样被否决(执行体的否决权压过分类器)", () => {
@@ -129,13 +147,24 @@ describe("resolveTurnErrorClass · 三道分类链", () => {
129
147
  { type: "action.called", name: "bash", callId: "c1", input: {} },
130
148
  errorEvent("ACME_QUEUE_FULL"),
131
149
  ]);
132
- const result = resolveTurnErrorClass(failure, () => ({ retryable: true, reason: "acme_queue_full" }));
133
- expect(result).toEqual({ retryable: false });
150
+ const result = resolveTurnFailureClass(failure, { adapter: () => ({ retryable: true, reason: "acme_queue_full" }) });
151
+ expect(result).toEqual({ retryable: false, reason: "acme_queue_full" });
152
+ });
153
+
154
+ it("只裁时间轴:分类器给的 scope 原样保留,证据门不触碰空间轴", () => {
155
+ const failure = turnFailed([
156
+ { type: "action.called", name: "bash", callId: "c1", input: {} },
157
+ errorEvent("tunnel flaky, retry later"),
158
+ ]);
159
+ const result = resolveTurnFailureClass(failure, {
160
+ experiment: () => ({ retryable: true, reason: "tunnel_flaky", scope: "experiment" }),
161
+ });
162
+ expect(result).toEqual({ retryable: false, reason: "tunnel_flaky", scope: "experiment" });
134
163
  });
135
164
 
136
165
  it("thrown 形态没有 Turn 可查,不受证据门影响", () => {
137
166
  const failure: TurnFailure = { type: "thrown", error: new Error("too many requests") };
138
- expect(resolveTurnErrorClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
167
+ expect(resolveTurnFailureClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
139
168
  });
140
169
 
141
170
  it("不可重试分类结果不受证据门触碰(门只降级可重试的判断)", () => {
@@ -143,12 +172,90 @@ describe("resolveTurnErrorClass · 三道分类链", () => {
143
172
  { type: "action.called", name: "bash", callId: "c1", input: {} },
144
173
  errorEvent("stream reset mid-response"),
145
174
  ]);
146
- const result: TurnErrorClass = resolveTurnErrorClass(failure);
175
+ const result: FailureClass = resolveTurnFailureClass(failure);
147
176
  expect(result).toEqual({ retryable: false });
148
177
  });
149
178
  });
150
179
  });
151
180
 
181
+ describe("resolveTurnFailureClass · 决议序(先非 undefined 定案)", () => {
182
+ // 区分力场景:同一条失败两个通道都认领——adapter 只能给时间轴(连接错误的通用形状),
183
+ // 只有实验作者认得这个 host 是全实验共享的隧道。实验分类器排在 adapter 之前,
184
+ // scope 才赢得下来(裁决见 memory/failure-chain-experiment-before-adapter.md)。
185
+ const tunnelFailure = turnFailed([errorEvent("connect ECONNREFUSED nowledge.trycloudflare.com:443")]);
186
+ const experimentClassifier = () => ({ retryable: false, scope: "experiment", reason: "tunnel_down" }) as const;
187
+ const adapterClassifier = () => ({ retryable: true, reason: "network" }) as const;
188
+
189
+ it("实验分类器与 adapter 同时认领时,实验的 scope 声明胜出", () => {
190
+ const result = resolveTurnFailureClass(tunnelFailure, {
191
+ experiment: experimentClassifier,
192
+ adapter: adapterClassifier,
193
+ });
194
+ expect(result).toEqual({ retryable: false, scope: "experiment", reason: "tunnel_down" });
195
+ });
196
+
197
+ it("实验分类器认领后不再询问 adapter", () => {
198
+ const adapter = vi.fn(adapterClassifier);
199
+ resolveTurnFailureClass(tunnelFailure, { experiment: experimentClassifier, adapter });
200
+ expect(adapter).not.toHaveBeenCalled();
201
+ });
202
+
203
+ it("实验分类器不认(返回 undefined)时才轮到 adapter,adapter 的时间轴答案生效", () => {
204
+ const result = resolveTurnFailureClass(tunnelFailure, {
205
+ experiment: () => undefined,
206
+ adapter: adapterClassifier,
207
+ });
208
+ expect(result).toEqual({ retryable: true, reason: "network" });
209
+ });
210
+
211
+ it("实验分类器读到的 text 与报错文案同源,phase 是 agent.run,cause 是那个失败 Turn", () => {
212
+ const turn: Turn = { status: "failed", events: [errorEvent("connect ECONNREFUSED nowledge.trycloudflare.com:443")] };
213
+ const seen: AttemptFailureInfo[] = [];
214
+ resolveTurnFailureClass(
215
+ { type: "turn-failed", turn },
216
+ {
217
+ experiment: (failure) => {
218
+ seen.push(failure);
219
+ return undefined;
220
+ },
221
+ },
222
+ );
223
+ expect(seen).toHaveLength(1);
224
+ expect(seen[0].phase).toBe("agent.run");
225
+ expect(seen[0].text).toBe(turnErrorText(turn));
226
+ expect(seen[0].cause).toBe(turn);
227
+ });
228
+
229
+ it("thrown 形态下实验分类器读到抛出的错误本身,text 是错误链串接", () => {
230
+ const error = new Error("send failed", { cause: new Error("connect ECONNREFUSED tunnel.example:443") });
231
+ const seen: { text: string; cause: unknown }[] = [];
232
+ resolveTurnFailureClass({ type: "thrown", error }, {
233
+ experiment: (failure) => {
234
+ seen.push(failure);
235
+ return undefined;
236
+ },
237
+ });
238
+ expect(seen[0].cause).toBe(error);
239
+ expect(seen[0].text).toBe("send failed · connect ECONNREFUSED tunnel.example:443");
240
+ });
241
+
242
+ it("抛出点携带的分类优先级最高:糖衣类命中即定,任何分类器都不被询问", () => {
243
+ const experiment = vi.fn(experimentClassifier);
244
+ const adapter = vi.fn(adapterClassifier);
245
+ const failure: TurnFailure = { type: "thrown", error: new ExperimentFatalError("tunnel probe failed") };
246
+ const result = resolveTurnFailureClass(failure, { experiment, adapter });
247
+ expect(result).toEqual({ retryable: false, scope: "experiment" });
248
+ expect(experiment).not.toHaveBeenCalled();
249
+ expect(adapter).not.toHaveBeenCalled();
250
+ });
251
+
252
+ it("糖衣类被上层包装再抛(cause 链)也照样命中,声明不丢失", () => {
253
+ const wrapped = new Error("adapter wrapped", { cause: new EvalFatalError("fixture missing") });
254
+ const result = resolveTurnFailureClass({ type: "thrown", error: wrapped });
255
+ expect(result).toEqual({ retryable: false, scope: "eval" });
256
+ });
257
+ });
258
+
152
259
  describe("hasAgentEvidence", () => {
153
260
  it("空事件流(无 error 事件也无产出事件)判 false", () => {
154
261
  expect(hasAgentEvidence({ status: "failed", events: [] })).toBe(false);
@@ -1,20 +1,18 @@
1
- // turn 级瞬时错误分类:把一次 send 失败(抛出 / 返回 failed Turn)按重试安全性归类。
2
- // 判据全文见 docs/feature/error-classification/README.md「分类」;这里只落三道分类链里
3
- // 「保守兜底」与「受理证据门」两道的实现,adapter 分类器本身不在这个文件(它是 Agent 的可选字段,
4
- // 挂载在 src/agents/types.ts)。执行体的重试时序在 send-retry.ts,不在这里——本模块只回答
5
- // 「这次失败能不能安全重发」,不碰次数/退避/槽位。
1
+ // turn 失败分类链:把一次 send 失败(抛出 / 返回 failed Turn)归成一份 `FailureClass`。
2
+ // 判据全文见 docs/feature/error-classification/README.md「分类」;两轴词表、糖衣类与守卫在
3
+ // src/shared/failure-class.ts(全仓单源),这里只落 turn 这条链——五道里的「保守兜底」与
4
+ // 「受理证据门」两道的实现在本文件,抛出点声明走守卫、实验分类器挂在 ExperimentDef、adapter
5
+ // 分类器挂在 Agent(src/agents/types.ts),本文件只按序询问它们。执行体的重试时序在
6
+ // send-retry.ts:本模块只回答「这次失败能不能安全重发、波及多远」,不碰次数/退避/槽位。
6
7
 
7
8
  import type { Turn } from "../types.ts";
8
-
9
- /**
10
- * 一次 send 失败的分类结果:`retryable` 是执行体唯一消费的决策轴;`reason` 是开放词表的
11
- * 细分诊断,只进 activity 与耗尽摘要,不参与策略。内建兜底产出 reason `"rate_limit"` /
12
- * `"network"`;adapter 分类器可自造词。`retryable: true` 时 `reason` 必填——可重试的失败
13
- * 一定会出现在 activity 行与可能的耗尽摘要里,那里需要一个给人读的词。
14
- */
15
- export type TurnErrorClass =
16
- | { readonly retryable: true; readonly reason: string }
17
- | { readonly retryable: false; readonly reason?: string };
9
+ import {
10
+ callClassifier,
11
+ errorChainText,
12
+ failureClassOf,
13
+ type AttemptFailureClassifier,
14
+ type FailureClass,
15
+ } from "../shared/failure-class.ts";
18
16
 
19
17
  /** 一次 send 失败的两种浮出形态:`send()` 抛出异常,或返回 `status: "failed"` 的 Turn。 */
20
18
  export type TurnFailure =
@@ -22,10 +20,10 @@ export type TurnFailure =
22
20
  | { readonly type: "turn-failed"; readonly turn: Turn };
23
21
 
24
22
  /**
25
- * adapter 可选分类器:返回 `undefined` 表示「不认识,交给保守兜底」。分类器必须快、纯、
26
- * 不抛错——执行体按「抛错等价于不可重试」处理,自身错误被吞掉,不会掩盖原始失败。
23
+ * adapter 可选分类器:返回 `undefined` 表示「不认识,交给后续链路」。分类器必须快、纯、
24
+ * 不抛错——抛错按 `undefined` 回落处理,自身错误被吞掉,不会掩盖原始失败。
27
25
  */
28
- export type TurnErrorClassifier = (failure: TurnFailure) => TurnErrorClass | undefined;
26
+ export type TurnErrorClassifier = (failure: TurnFailure) => FailureClass | undefined;
29
27
 
30
28
  /**
31
29
  * 失败 Turn 的错误摘要:取 `events` 里最后一个 `type: "error"` 事件的 message。
@@ -41,23 +39,14 @@ export function turnErrorText(turn: Turn): string | undefined {
41
39
  return undefined;
42
40
  }
43
41
 
44
- /** `thrown` 形态的错误文本:沿错误链(含 `cause`)取 message,串接成一段供分类器与摘要读的文本。 */
45
- function thrownErrorText(error: unknown): string {
46
- const parts: string[] = [];
47
- let current: unknown = error;
48
- for (let depth = 0; depth < 5 && current != null; depth++) {
49
- const message = current instanceof Error ? current.message : String(current);
50
- if (message) parts.push(message);
51
- current = current instanceof Error ? (current as { cause?: unknown }).cause : undefined;
52
- }
53
- return parts.join(" · ");
54
- }
55
-
56
42
  /** 两种 `TurnFailure` 形态统一取「给人读也给分类器看」的那段文本。 */
57
43
  export function turnFailureText(failure: TurnFailure): string {
58
- return failure.type === "thrown" ? thrownErrorText(failure.error) : (turnErrorText(failure.turn) ?? "");
44
+ return failure.type === "thrown" ? errorChainText(failure.error) : (turnErrorText(failure.turn) ?? "");
59
45
  }
60
46
 
47
+ /** turn 失败在生命周期词表里的归属:adapter send 期间打开的那一段。 */
48
+ const TURN_FAILURE_PHASE = "agent.run" as const;
49
+
61
50
  // 限流关键字 / 明示 retry later → rate_limit;正则形状对齐 sandbox IO 分类器
62
51
  // (src/sandbox/errors.ts 的 classifySandboxIoError),各自实现、不共享模块。
63
52
  const RATE_LIMIT_PATTERN = /too many requests|rate.?limit|\b429\b|retry later|concurrency limit/i;
@@ -68,10 +57,11 @@ const NETWORK_CODE_PATTERN = /^(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ENETUNREACH|EHO
68
57
  const NETWORK_MESSAGE_PATTERN = /getaddrinfo|connection refused|certificate|tls handshake|connect etimedout|connection timeout/i;
69
58
 
70
59
  /**
71
- * 保守兜底分类器:三道分类链里的第二道。对失败文本做正则匹配,认不出的一律 `{ retryable: false }`
72
- * ——宁可判死一个 attempt,不产出不可信的 verdict(判据见 README「分类」)。
60
+ * 保守兜底分类器:turn 链里的第四道。对失败文本做正则匹配,认不出的一律 `{ retryable: false }`
61
+ * ——宁可判死一个 attempt,不产出不可信的 verdict(判据见 README「分类」)。兜底永不给出超出
62
+ * `"attempt"` 的 scope:框架无法从文案证明兄弟必死,扩 scope 只属于携带作者知识的通道。
73
63
  */
74
- export function classifyTurnError(failure: TurnFailure): TurnErrorClass {
64
+ export function classifyTurnError(failure: TurnFailure): FailureClass {
75
65
  const text = turnFailureText(failure);
76
66
  if (RATE_LIMIT_PATTERN.test(text)) return { retryable: true, reason: "rate_limit" };
77
67
  const code = errorCode(failure);
@@ -98,25 +88,45 @@ export function hasAgentEvidence(turn: Turn): boolean {
98
88
  return turn.events.some((e) => AGENT_EVIDENCE_TYPES.has(e.type));
99
89
  }
100
90
 
91
+ /** turn 链上两个可选声明通道;都省略时链退化成「抛出点 → 兜底 → 证据门」。 */
92
+ export interface TurnClassifiers {
93
+ /** 实验作者的 `ExperimentDef.classifyFailure`,按自家坐标识别共享基建死因。 */
94
+ experiment?: AttemptFailureClassifier;
95
+ /** adapter 作者的 `Agent.classifyTurnError`,识别自家协议的错误形状。 */
96
+ adapter?: TurnErrorClassifier;
97
+ }
98
+
101
99
  /**
102
- * 三道分类链的完整决议:adapter 分类器(可选,抛错按不可重试处理并吞掉)→ 保守兜底 →
103
- * 受理证据门(否决权,失败 Turn 带 agent 产出事件时强制降级)。执行体只需要调这一个函数,
104
- * 不必自己拼三道链的顺序。
100
+ * turn 失败分类链的完整决议(五道,先给出非 `undefined` 结果的一道定分类):
101
+ *
102
+ * 1. 抛出点携带的分类(`failureClassOf`,含 cause 链穿透)——作者知识优先级最高;
103
+ * 2. 实验分类器——按自家坐标(host、路径)过滤,特异性高于协议通用形状,排在 adapter 之前
104
+ * 保证「两者同时认领时 scope 赢」(裁决见 memory/failure-chain-experiment-before-adapter.md);
105
+ * 3. adapter 分类器;
106
+ * 4. 保守兜底正则;
107
+ * 5. 受理证据门(执行体的否决权,只裁时间轴):失败 Turn 里已有 agent 产出事件时 `retryable`
108
+ * 强制降为 `false`,`reason` 与 `scope` 原样保留——门裁的是重发安全性,不是波及范围。
109
+ *
110
+ * 分类器抛错按 `undefined` 回落(继续问后续通道),分类是旁路,不得用新错误掩盖原始失败。
105
111
  */
106
- export function resolveTurnErrorClass(failure: TurnFailure, adapterClassifier?: TurnErrorClassifier): TurnErrorClass {
107
- let cls: TurnErrorClass | undefined;
108
- if (adapterClassifier) {
109
- try {
110
- cls = adapterClassifier(failure);
111
- } catch {
112
- // 分类器抛错按不可重试处理:分类是旁路,不得用新错误掩盖原始失败,也不回落到兜底
113
- // (自造分类器都判断不了的形状,交给通用正则复判没有意义)。
114
- return { retryable: false };
115
- }
116
- }
117
- const resolved = cls ?? classifyTurnError(failure);
112
+ export function resolveTurnFailureClass(failure: TurnFailure, classifiers: TurnClassifiers = {}): FailureClass {
113
+ const declared = failure.type === "thrown" ? failureClassOf(failure.error) : undefined;
114
+ const resolved = declared ?? classifyByChain(failure, classifiers);
118
115
  if (resolved.retryable && failure.type === "turn-failed" && hasAgentEvidence(failure.turn)) {
119
- return { retryable: false };
116
+ return { ...resolved, retryable: false };
120
117
  }
121
118
  return resolved;
122
119
  }
120
+
121
+ function classifyByChain(failure: TurnFailure, classifiers: TurnClassifiers): FailureClass {
122
+ if (classifiers.experiment) {
123
+ const info = {
124
+ phase: TURN_FAILURE_PHASE,
125
+ text: turnFailureText(failure),
126
+ cause: failure.type === "thrown" ? failure.error : failure.turn,
127
+ };
128
+ const cls = callClassifier(classifiers.experiment, info);
129
+ if (cls) return cls;
130
+ }
131
+ return callClassifier(classifiers.adapter, failure) ?? classifyTurnError(failure);
132
+ }
package/src/define.ts CHANGED
@@ -114,6 +114,11 @@ export function defineExperiment(def: ExperimentDef): ExperimentDef {
114
114
  if (def.setup !== undefined && typeof def.setup !== "function") {
115
115
  throw new Error(t("define.experimentSetupNotFunction"));
116
116
  }
117
+ // classifyFailure 是失败分类链上的实验通道(见 runner/types.ts 的 ExperimentDef.classifyFailure):
118
+ // 传成非函数在解析时就报,不等到某条 attempt 撞死才发现这一路声明白写。
119
+ if (def.classifyFailure !== undefined && typeof def.classifyFailure !== "function") {
120
+ throw new Error(t("define.experimentClassifyFailureNotFunction"));
121
+ }
117
122
  // flags 必须可 JSON 序列化(进结果快照的 ExperimentRunInfo.flags):解析时即校验,
118
123
  // 非 JSON 值(函数 / undefined / 循环引用 / bigint)直接报错,不等到落盘才炸。
119
124
  if (def.flags !== undefined) {