niceeval 0.10.3-canary.6 → 0.11.0

package/src/context/turn-errors.test.ts

@@ -1,13 +1,18 @@
 // cases: docs/engineering/testing/unit/eval.md
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import {
   classifyTurnError,
   hasAgentEvidence,
-  resolveTurnErrorClass,
+  resolveTurnFailureClass,
   turnErrorText,
-  type TurnErrorClass,
   type TurnFailure,
 } from "./turn-errors.ts";
+import {
+  EvalFatalError,
+  ExperimentFatalError,
+  type AttemptFailureInfo,
+  type FailureClass,
+} from "../shared/failure-class.ts";
 import type { StreamEvent, Turn } from "../types.ts";
 
 function turnFailed(events: StreamEvent[]): TurnFailure {
@@ -87,30 +92,43 @@ describe("classifyTurnError(保守兜底) · thrown 形态", () => {
   });
 });
 
-describe("resolveTurnErrorClass · 三道分类链", () => {
+describe("resolveTurnFailureClass · 五道分类链", () => {
   it("adapter 分类器返回结果时优先生效,自定义 reason 原样透出(不被塞进内建词表)", () => {
     const failure = turnFailed([errorEvent("ACME_QUEUE_FULL: too many concurrent runs")]);
-    const result = resolveTurnErrorClass(failure, () => ({ retryable: true, reason: "acme_queue_full" }));
+    const result = resolveTurnFailureClass(failure, { adapter: () => ({ retryable: true, reason: "acme_queue_full" }) });
     expect(result).toEqual({ retryable: true, reason: "acme_queue_full" });
   });
 
   it("adapter 分类器返回 undefined 时回落保守兜底", () => {
     const failure = turnFailed([errorEvent("too many requests, back off")]);
-    const result = resolveTurnErrorClass(failure, () => undefined);
+    const result = resolveTurnFailureClass(failure, { adapter: () => undefined });
     expect(result).toEqual({ retryable: true, reason: "rate_limit" });
   });
 
-  it("adapter 分类器抛错按不可重试处理并被吞掉——不回落兜底,即使兜底本会判可重试", () => {
+  it("adapter 分类器抛错按 undefined 回落:错误被吞掉,链继续走到兜底", () => {
     const failure = turnFailed([errorEvent("too many requests, back off")]);
-    const result = resolveTurnErrorClass(failure, () => {
-      throw new Error("classifier bug");
+    const result = resolveTurnFailureClass(failure, {
+      adapter: () => {
+        throw new Error("classifier bug");
+      },
+    });
+    expect(result).toEqual({ retryable: true, reason: "rate_limit" });
+  });
+
+  it("实验分类器抛错同样按 undefined 回落,后续通道照常认领(不掩盖原始失败)", () => {
+    const failure = turnFailed([errorEvent("ACME_QUEUE_FULL")]);
+    const result = resolveTurnFailureClass(failure, {
+      experiment: () => {
+        throw new Error("classifier bug");
+      },
+      adapter: () => ({ retryable: true, reason: "acme_queue_full" }),
     });
-    expect(result).toEqual({ retryable: false });
+    expect(result).toEqual({ retryable: true, reason: "acme_queue_full" });
   });
 
   it("没有 adapter 分类器时直接走保守兜底", () => {
     const failure = turnFailed([errorEvent("too many requests, back off")]);
-    expect(resolveTurnErrorClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
+    expect(resolveTurnFailureClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
   });
 
   describe("受理证据门:失败 Turn 带 agent 产出事件时强制降级为不可重试", () => {
@@ -119,9 +137,9 @@ describe("resolveTurnErrorClass · 三道分类链", () => {
       ["thinking", { type: "thinking", text: "let me think" } satisfies StreamEvent],
       ["action.called", { type: "action.called", name: "bash", callId: "c1", input: {} } satisfies StreamEvent],
       ["action.result", { type: "action.result", callId: "c1", output: "", status: "completed" } satisfies StreamEvent],
-    ])("%s 事件出现时,文本再像限流也不重试", (_label, evidenceEvent) => {
+    ])("%s 事件出现时,文本再像限流也不重试(reason 原样留着给人读)", (_label, evidenceEvent) => {
       const failure = turnFailed([evidenceEvent, errorEvent("Concurrency limit exceeded, please retry later")]);
-      expect(resolveTurnErrorClass(failure)).toEqual({ retryable: false });
+      expect(resolveTurnFailureClass(failure)).toEqual({ retryable: false, reason: "rate_limit" });
     });
 
     it("adapter 分类器判可重试同样被否决(执行体的否决权压过分类器)", () => {
@@ -129,13 +147,24 @@ describe("resolveTurnErrorClass · 三道分类链", () => {
         { type: "action.called", name: "bash", callId: "c1", input: {} },
         errorEvent("ACME_QUEUE_FULL"),
       ]);
-      const result = resolveTurnErrorClass(failure, () => ({ retryable: true, reason: "acme_queue_full" }));
-      expect(result).toEqual({ retryable: false });
+      const result = resolveTurnFailureClass(failure, { adapter: () => ({ retryable: true, reason: "acme_queue_full" }) });
+      expect(result).toEqual({ retryable: false, reason: "acme_queue_full" });
+    });
+
+    it("只裁时间轴:分类器给的 scope 原样保留,证据门不触碰空间轴", () => {
+      const failure = turnFailed([
+        { type: "action.called", name: "bash", callId: "c1", input: {} },
+        errorEvent("tunnel flaky, retry later"),
+      ]);
+      const result = resolveTurnFailureClass(failure, {
+        experiment: () => ({ retryable: true, reason: "tunnel_flaky", scope: "experiment" }),
+      });
+      expect(result).toEqual({ retryable: false, reason: "tunnel_flaky", scope: "experiment" });
     });
 
     it("thrown 形态没有 Turn 可查,不受证据门影响", () => {
       const failure: TurnFailure = { type: "thrown", error: new Error("too many requests") };
-      expect(resolveTurnErrorClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
+      expect(resolveTurnFailureClass(failure)).toEqual({ retryable: true, reason: "rate_limit" });
     });
 
     it("不可重试分类结果不受证据门触碰(门只降级可重试的判断)", () => {
@@ -143,12 +172,90 @@ describe("resolveTurnErrorClass · 三道分类链", () => {
         { type: "action.called", name: "bash", callId: "c1", input: {} },
         errorEvent("stream reset mid-response"),
       ]);
-      const result: TurnErrorClass = resolveTurnErrorClass(failure);
+      const result: FailureClass = resolveTurnFailureClass(failure);
       expect(result).toEqual({ retryable: false });
     });
   });
 });
 
+describe("resolveTurnFailureClass · 决议序(先非 undefined 定案)", () => {
+  // 区分力场景:同一条失败两个通道都认领——adapter 只能给时间轴(连接错误的通用形状),
+  // 只有实验作者认得这个 host 是全实验共享的隧道。实验分类器排在 adapter 之前,
+  // scope 才赢得下来(裁决见 memory/failure-chain-experiment-before-adapter.md)。
+  const tunnelFailure = turnFailed([errorEvent("connect ECONNREFUSED nowledge.trycloudflare.com:443")]);
+  const experimentClassifier = () => ({ retryable: false, scope: "experiment", reason: "tunnel_down" }) as const;
+  const adapterClassifier = () => ({ retryable: true, reason: "network" }) as const;
+
+  it("实验分类器与 adapter 同时认领时,实验的 scope 声明胜出", () => {
+    const result = resolveTurnFailureClass(tunnelFailure, {
+      experiment: experimentClassifier,
+      adapter: adapterClassifier,
+    });
+    expect(result).toEqual({ retryable: false, scope: "experiment", reason: "tunnel_down" });
+  });
+
+  it("实验分类器认领后不再询问 adapter", () => {
+    const adapter = vi.fn(adapterClassifier);
+    resolveTurnFailureClass(tunnelFailure, { experiment: experimentClassifier, adapter });
+    expect(adapter).not.toHaveBeenCalled();
+  });
+
+  it("实验分类器不认(返回 undefined)时才轮到 adapter,adapter 的时间轴答案生效", () => {
+    const result = resolveTurnFailureClass(tunnelFailure, {
+      experiment: () => undefined,
+      adapter: adapterClassifier,
+    });
+    expect(result).toEqual({ retryable: true, reason: "network" });
+  });
+
+  it("实验分类器读到的 text 与报错文案同源,phase 是 agent.run,cause 是那个失败 Turn", () => {
+    const turn: Turn = { status: "failed", events: [errorEvent("connect ECONNREFUSED nowledge.trycloudflare.com:443")] };
+    const seen: AttemptFailureInfo[] = [];
+    resolveTurnFailureClass(
+      { type: "turn-failed", turn },
+      {
+        experiment: (failure) => {
+          seen.push(failure);
+          return undefined;
+        },
+      },
+    );
+    expect(seen).toHaveLength(1);
+    expect(seen[0].phase).toBe("agent.run");
+    expect(seen[0].text).toBe(turnErrorText(turn));
+    expect(seen[0].cause).toBe(turn);
+  });
+
+  it("thrown 形态下实验分类器读到抛出的错误本身,text 是错误链串接", () => {
+    const error = new Error("send failed", { cause: new Error("connect ECONNREFUSED tunnel.example:443") });
+    const seen: { text: string; cause: unknown }[] = [];
+    resolveTurnFailureClass({ type: "thrown", error }, {
+      experiment: (failure) => {
+        seen.push(failure);
+        return undefined;
+      },
+    });
+    expect(seen[0].cause).toBe(error);
+    expect(seen[0].text).toBe("send failed · connect ECONNREFUSED tunnel.example:443");
+  });
+
+  it("抛出点携带的分类优先级最高:糖衣类命中即定,任何分类器都不被询问", () => {
+    const experiment = vi.fn(experimentClassifier);
+    const adapter = vi.fn(adapterClassifier);
+    const failure: TurnFailure = { type: "thrown", error: new ExperimentFatalError("tunnel probe failed") };
+    const result = resolveTurnFailureClass(failure, { experiment, adapter });
+    expect(result).toEqual({ retryable: false, scope: "experiment" });
+    expect(experiment).not.toHaveBeenCalled();
+    expect(adapter).not.toHaveBeenCalled();
+  });
+
+  it("糖衣类被上层包装再抛(cause 链)也照样命中,声明不丢失", () => {
+    const wrapped = new Error("adapter wrapped", { cause: new EvalFatalError("fixture missing") });
+    const result = resolveTurnFailureClass({ type: "thrown", error: wrapped });
+    expect(result).toEqual({ retryable: false, scope: "eval" });
+  });
+});
+
 describe("hasAgentEvidence", () => {
   it("空事件流(无 error 事件也无产出事件)判 false", () => {
     expect(hasAgentEvidence({ status: "failed", events: [] })).toBe(false);

package/src/context/turn-errors.ts

@@ -1,20 +1,18 @@
-// turn 级瞬时错误分类:把一次 send 失败(抛出 / 返回 failed Turn)按重试安全性归类。
-// 判据全文见 docs/feature/error-classification/README.md「分类」;这里只落三道分类链里
-// 「保守兜底」与「受理证据门」两道的实现,adapter 分类器本身不在这个文件(它是 Agent 的可选字段,
-// 挂载在 src/agents/types.ts)。执行体的重试时序在 send-retry.ts,不在这里——本模块只回答
-// 「这次失败能不能安全重发」,不碰次数/退避/槽位。
+// turn 失败分类链:把一次 send 失败(抛出 / 返回 failed Turn)归成一份 `FailureClass`。
+// 判据全文见 docs/feature/error-classification/README.md「分类」;两轴词表、糖衣类与守卫在
+// src/shared/failure-class.ts(全仓单源),这里只落 turn 这条链——五道里的「保守兜底」与
+// 「受理证据门」两道的实现在本文件,抛出点声明走守卫、实验分类器挂在 ExperimentDef、adapter
+// 分类器挂在 Agent(src/agents/types.ts),本文件只按序询问它们。执行体的重试时序在
+// send-retry.ts:本模块只回答「这次失败能不能安全重发、波及多远」,不碰次数/退避/槽位。
 
 import type { Turn } from "../types.ts";
-
-/**
- * 一次 send 失败的分类结果:`retryable` 是执行体唯一消费的决策轴;`reason` 是开放词表的
- * 细分诊断,只进 activity 与耗尽摘要,不参与策略。内建兜底产出 reason `"rate_limit"` /
- * `"network"`;adapter 分类器可自造词。`retryable: true` 时 `reason` 必填——可重试的失败
- * 一定会出现在 activity 行与可能的耗尽摘要里,那里需要一个给人读的词。
- */
-export type TurnErrorClass =
-  | { readonly retryable: true; readonly reason: string }
-  | { readonly retryable: false; readonly reason?: string };
+import {
+  callClassifier,
+  errorChainText,
+  failureClassOf,
+  type AttemptFailureClassifier,
+  type FailureClass,
+} from "../shared/failure-class.ts";
 
 /** 一次 send 失败的两种浮出形态:`send()` 抛出异常,或返回 `status: "failed"` 的 Turn。 */
 export type TurnFailure =
@@ -22,10 +20,10 @@ export type TurnFailure =
   | { readonly type: "turn-failed"; readonly turn: Turn };
 
 /**
- * adapter 可选分类器:返回 `undefined` 表示「不认识,交给保守兜底」。分类器必须快、纯、
- * 不抛错——执行体按「抛错等价于不可重试」处理,自身错误被吞掉,不会掩盖原始失败。
+ * adapter 可选分类器:返回 `undefined` 表示「不认识,交给后续链路」。分类器必须快、纯、
+ * 不抛错——抛错按 `undefined` 回落处理,自身错误被吞掉,不会掩盖原始失败。
  */
-export type TurnErrorClassifier = (failure: TurnFailure) => TurnErrorClass | undefined;
+export type TurnErrorClassifier = (failure: TurnFailure) => FailureClass | undefined;
 
 /**
  * 失败 Turn 的错误摘要:取 `events` 里最后一个 `type: "error"` 事件的 message。
@@ -41,23 +39,14 @@ export function turnErrorText(turn: Turn): string | undefined {
   return undefined;
 }
 
-/** `thrown` 形态的错误文本:沿错误链(含 `cause`)取 message,串接成一段供分类器与摘要读的文本。 */
-function thrownErrorText(error: unknown): string {
-  const parts: string[] = [];
-  let current: unknown = error;
-  for (let depth = 0; depth < 5 && current != null; depth++) {
-    const message = current instanceof Error ? current.message : String(current);
-    if (message) parts.push(message);
-    current = current instanceof Error ? (current as { cause?: unknown }).cause : undefined;
-  }
-  return parts.join(" · ");
-}
-
 /** 两种 `TurnFailure` 形态统一取「给人读也给分类器看」的那段文本。 */
 export function turnFailureText(failure: TurnFailure): string {
-  return failure.type === "thrown" ? thrownErrorText(failure.error) : (turnErrorText(failure.turn) ?? "");
+  return failure.type === "thrown" ? errorChainText(failure.error) : (turnErrorText(failure.turn) ?? "");
 }
 
+/** turn 失败在生命周期词表里的归属:adapter send 期间打开的那一段。 */
+const TURN_FAILURE_PHASE = "agent.run" as const;
+
 // 限流关键字 / 明示 retry later → rate_limit;正则形状对齐 sandbox IO 分类器
 // (src/sandbox/errors.ts 的 classifySandboxIoError),各自实现、不共享模块。
 const RATE_LIMIT_PATTERN = /too many requests|rate.?limit|\b429\b|retry later|concurrency limit/i;
@@ -68,10 +57,11 @@ const NETWORK_CODE_PATTERN = /^(ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ENETUNREACH|EHO
 const NETWORK_MESSAGE_PATTERN = /getaddrinfo|connection refused|certificate|tls handshake|connect etimedout|connection timeout/i;
 
 /**
- * 保守兜底分类器:三道分类链里的第二道。对失败文本做正则匹配,认不出的一律 `{ retryable: false }`
- * ——宁可判死一个 attempt,不产出不可信的 verdict(判据见 README「分类」)。
+ * 保守兜底分类器:turn 链里的第四道。对失败文本做正则匹配,认不出的一律 `{ retryable: false }`
+ * ——宁可判死一个 attempt,不产出不可信的 verdict(判据见 README「分类」)。兜底永不给出超出
+ * `"attempt"` 的 scope:框架无法从文案证明兄弟必死,扩 scope 只属于携带作者知识的通道。
  */
-export function classifyTurnError(failure: TurnFailure): TurnErrorClass {
+export function classifyTurnError(failure: TurnFailure): FailureClass {
   const text = turnFailureText(failure);
   if (RATE_LIMIT_PATTERN.test(text)) return { retryable: true, reason: "rate_limit" };
   const code = errorCode(failure);
@@ -98,25 +88,45 @@ export function hasAgentEvidence(turn: Turn): boolean {
   return turn.events.some((e) => AGENT_EVIDENCE_TYPES.has(e.type));
 }
 
+/** turn 链上两个可选声明通道;都省略时链退化成「抛出点 → 兜底 → 证据门」。 */
+export interface TurnClassifiers {
+  /** 实验作者的 `ExperimentDef.classifyFailure`,按自家坐标识别共享基建死因。 */
+  experiment?: AttemptFailureClassifier;
+  /** adapter 作者的 `Agent.classifyTurnError`,识别自家协议的错误形状。 */
+  adapter?: TurnErrorClassifier;
+}
+
 /**
- * 三道分类链的完整决议:adapter 分类器(可选,抛错按不可重试处理并吞掉)→ 保守兜底 →
- * 受理证据门(否决权,失败 Turn 带 agent 产出事件时强制降级)。执行体只需要调这一个函数,
- * 不必自己拼三道链的顺序。
+ * turn 失败分类链的完整决议(五道,先给出非 `undefined` 结果的一道定分类):
+ *
+ * 1. 抛出点携带的分类(`failureClassOf`,含 cause 链穿透)——作者知识优先级最高;
+ * 2. 实验分类器——按自家坐标(host、路径)过滤,特异性高于协议通用形状,排在 adapter 之前
+ *    保证「两者同时认领时 scope 赢」(裁决见 memory/failure-chain-experiment-before-adapter.md);
+ * 3. adapter 分类器;
+ * 4. 保守兜底正则;
+ * 5. 受理证据门(执行体的否决权,只裁时间轴):失败 Turn 里已有 agent 产出事件时 `retryable`
+ *    强制降为 `false`,`reason` 与 `scope` 原样保留——门裁的是重发安全性,不是波及范围。
+ *
+ * 分类器抛错按 `undefined` 回落(继续问后续通道),分类是旁路,不得用新错误掩盖原始失败。
  */
-export function resolveTurnErrorClass(failure: TurnFailure, adapterClassifier?: TurnErrorClassifier): TurnErrorClass {
-  let cls: TurnErrorClass | undefined;
-  if (adapterClassifier) {
-    try {
-      cls = adapterClassifier(failure);
-    } catch {
-      // 分类器抛错按不可重试处理:分类是旁路,不得用新错误掩盖原始失败,也不回落到兜底
-      // (自造分类器都判断不了的形状,交给通用正则复判没有意义)。
-      return { retryable: false };
-    }
-  }
-  const resolved = cls ?? classifyTurnError(failure);
+export function resolveTurnFailureClass(failure: TurnFailure, classifiers: TurnClassifiers = {}): FailureClass {
+  const declared = failure.type === "thrown" ? failureClassOf(failure.error) : undefined;
+  const resolved = declared ?? classifyByChain(failure, classifiers);
   if (resolved.retryable && failure.type === "turn-failed" && hasAgentEvidence(failure.turn)) {
-    return { retryable: false };
+    return { ...resolved, retryable: false };
   }
   return resolved;
 }
+
+function classifyByChain(failure: TurnFailure, classifiers: TurnClassifiers): FailureClass {
+  if (classifiers.experiment) {
+    const info = {
+      phase: TURN_FAILURE_PHASE,
+      text: turnFailureText(failure),
+      cause: failure.type === "thrown" ? failure.error : failure.turn,
+    };
+    const cls = callClassifier(classifiers.experiment, info);
+    if (cls) return cls;
+  }
+  return callClassifier(classifiers.adapter, failure) ?? classifyTurnError(failure);
+}

package/src/define.ts

@@ -114,6 +114,11 @@ export function defineExperiment(def: ExperimentDef): ExperimentDef {
   if (def.setup !== undefined && typeof def.setup !== "function") {
     throw new Error(t("define.experimentSetupNotFunction"));
   }
+  // classifyFailure 是失败分类链上的实验通道(见 runner/types.ts 的 ExperimentDef.classifyFailure):
+  // 传成非函数在解析时就报,不等到某条 attempt 撞死才发现这一路声明白写。
+  if (def.classifyFailure !== undefined && typeof def.classifyFailure !== "function") {
+    throw new Error(t("define.experimentClassifyFailureNotFunction"));
+  }
   // flags 必须可 JSON 序列化(进结果快照的 ExperimentRunInfo.flags):解析时即校验,
   // 非 JSON 值(函数 / undefined / 循环引用 / bigint)直接报错,不等到落盘才炸。
   if (def.flags !== undefined) {

package/src/i18n/en.ts

@@ -74,6 +74,14 @@ export const en = {
     "experiment {{experimentId}}'s teardown was not triggered by the normal countdown path; it has been executed by the end-of-run sweep instead. Results are unaffected; seeing this line means an unlocated intermittent scheduling issue fired — please record this run in the memory ledger.\n",
   "runner.teardownRegistrationWriteFailed":
     "writing the crash-recovery teardown registration for experiment {{experimentId}} failed: {{message}}. The run continues normally, but a SIGKILL during this run cannot be recovered via `niceeval exp --teardown` or the startup self-heal — check disk space/permissions under .niceeval/teardowns/.\n",
+  "runner.lockTakenOver":
+    "took over an expired case lock for {{experimentId}}/{{evalId}} (previously held by pid {{pid}} on {{host}}; its heartbeat went stale) — that run likely died without releasing it; this run now owns dispatching this case.\n",
+  "runner.gateLeaseTakenOver":
+    "took over an expired concurrency-slot lease for experiment {{experimentId}} (slot {{slot}}, previously held by pid {{pid}} on {{host}}; its heartbeat went stale) — that run likely died without releasing it; this run now owns the slot.\n",
+  "runner.gateLeaseWaiting":
+    "waiting on another run for experiment {{experimentId}}'s concurrency slots: all {{effectiveN}} in use ({{holders}}). Concurrent runs share this experiment's slots, and the smallest maxConcurrency in play wins — this run declared {{declaredN}}. Nothing dispatches until a slot frees up; the other run's slots release when its attempts finish, or 30s after it dies.\n",
+  "runner.dispatchHaltedExperiment": "experiment halted (dispatch-halted): {{message}}\n",
+  "runner.dispatchHaltedEval": "eval halted: {{message}}\n",
   "judge.modelMissing":
     "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
     "  Docs: node_modules/niceeval/docs-site/zh/tutorials/scoring-guide.mdx",
@@ -198,6 +206,8 @@ export const en = {
   "cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
   "cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
   "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site/zh).\n",
+  "cli.init.esmHint":
+    "tip: this project's package.json has no \"type\": \"module\". niceeval runs either way, but CommonJS mode disallows top-level await in config/eval files — add \"type\": \"module\" to package.json for the smoothest path.\n",
   "cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
   "cli.list.header": "Discovered {{count}} evals:\n",
   "cli.noAgent": "No agent specified (use --agent <name>).\n",
@@ -234,6 +244,7 @@ export const en = {
   "define.experimentFlagNotJson": "experiment.flags.{{key}} is not JSON-serializable (functions / undefined / cycles / bigint are not allowed); flags are persisted verbatim into result snapshots and must be plain JSON.",
   "define.experimentLabelInvalid": "experiment.labels.{{key}} must be a string or a finite number; labels are report-side grouping coordinates persisted verbatim into result snapshots.",
   "define.experimentSetupNotFunction": "experiment.setup must be a function ((ctx) => void); use experiment.teardown for cleanup; to prepare the in-sandbox environment per experiment, chain .setup() hooks on the sandbox spec instead.",
+  "define.experimentClassifyFailureNotFunction": "experiment.classifyFailure must be a function ((failure) => FailureClass | undefined); it classifies failures that surface as third-party errors and must return undefined for anything it does not recognize.",
   "define.experimentIdRejected": "defineExperiment does not accept id; ids are derived from file paths.",
   "define.sandboxAgentNameRequired": "defineSandboxAgent requires name.",
   "define.sandboxCreateRequired": "defineSandbox requires a create() function.",
@@ -247,7 +258,8 @@ export const en = {
   "feedback.human.active": "ACTIVE",
   "feedback.human.budgetExhausted": "budget exhausted for {{experimentId}} (spent {{spent}}, unstarted {{unstarted}})",
   "feedback.human.compare": "Compare: niceeval view",
-  "feedback.human.counts": "{{total}} total · {{reused}} reused · {{running}} running · {{queued}} queued · {{completed}} completed",
+  "feedback.human.counts":
+    "{{total}} total · {{reused}} reused · {{running}} running · {{queued}} queued · {{passed}} passed · {{failed}} failed · {{errored}} errored · {{skipped}} skipped",
   "feedback.human.diffHint": "Diff:    niceeval show {{locator}} --diff",
   "feedback.human.evalHint": "Eval:    niceeval show {{locator}} --source",
   "feedback.human.failuresHeader": "FAILURES",
@@ -280,6 +292,15 @@ export const en = {
   "feedback.human.hookFailed": "failed",
   "feedback.human.precheckJudge": "prechecking judge config",
   "feedback.human.precheckJudgeDone": "judge config ok",
+  "feedback.human.countsWithElsewhere":
+    "{{total}} total · {{reused}} reused · {{running}} running · {{elsewhere}} elsewhere · {{queued}} queued · {{passed}} passed · {{failed}} failed · {{errored}} errored · {{skipped}} skipped",
+  "feedback.human.waitingOnAnotherRun": "waiting on another run",
+  "feedback.human.lockWaitDetail": "{{count}} evals · pid {{pid}}",
+  "feedback.human.lockWaitStarted": "waiting on another run · {{experimentId}} ({{count}} evals, pid {{pid}})",
+  "feedback.human.lockWaitResolved": "lock wait resolved · {{experimentId}} ({{summary}}, {{elapsed}})",
+  "feedback.human.lockWaitCarried": "{{count}} carried",
+  "feedback.human.lockWaitDispatched": "{{count}} to run",
+  "feedback.human.lockedRowSuffix": "locked",
   "feedback.phase.sandboxSetup": "sandbox setup",
   "feedback.phase.scoring": "scoring",
   "feedback.phase.teardown": "cleaning up",

package/src/i18n/zh-CN.ts

@@ -71,6 +71,14 @@ export const zhCN = {
     "实验 {{experimentId}} 的 teardown 未被正常计数路径触发,已在运行收尾兜底执行。结果不受影响;这行出现说明命中了一个未定位的调度间歇问题,请把本次运行信息记入 memory 台账。\n",
   "runner.teardownRegistrationWriteFailed":
     "实验 {{experimentId}} 的强杀恢复收尾登记写入失败:{{message}}。本次运行照常继续,但这次运行期间若被 SIGKILL,`niceeval exp --teardown` 与启动自愈都无法找到它——检查 .niceeval/teardowns/ 下的磁盘空间或权限。\n",
+  "runner.lockTakenOver":
+    "接管了 {{experimentId}}/{{evalId}} 的过期用例锁(原持有者 pid {{pid}}@{{host}},心跳已过期)——那次运行大概率没能正常释放它;本次运行现在接手派发这条用例。\n",
+  "runner.gateLeaseTakenOver":
+    "接管了实验 {{experimentId}} 的过期并发名额租约(槽位 {{slot}},原持有者 pid {{pid}}@{{host}},心跳已过期)——那次运行大概率没能正常释放它;本次运行现在接手这个名额。\n",
+  "runner.gateLeaseWaiting":
+    "在等别的运行让出实验 {{experimentId}} 的并发名额:生效的 {{effectiveN}} 个位子全被占着({{holders}})。并行运行共用同一实验的名额,生效值取在场声明里最小的那个——本次运行声明的是 {{declaredN}}。名额腾出来之前不会派发任何 attempt;对方的名额会在它的 attempt 跑完时释放,它若已死则 30s 后过期被接管。\n",
+  "runner.dispatchHaltedExperiment": "实验已止损(dispatch-halted):{{message}}\n",
+  "runner.dispatchHaltedEval": "eval 已止损:{{message}}\n",
   "judge.modelMissing":
     "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定裁判模型(没有内置默认模型)。\n" +
     "  文档:node_modules/niceeval/docs-site/zh/tutorials/scoring-guide.mdx",
@@ -191,6 +199,8 @@ export const zhCN = {
   "cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
   "cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
   "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site/zh)。\n",
+  "cli.init.esmHint":
+    "提示:项目 package.json 没有 \"type\": \"module\"。niceeval 两种形态都能跑,但 CommonJS 形态下 config / eval 文件用不了顶层 await——建议在 package.json 加上 \"type\": \"module\"。\n",
   "cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
   "cli.list.header": "发现 {{count}} 个 eval:\n",
   "cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
@@ -227,6 +237,7 @@ export const zhCN = {
   "define.experimentFlagNotJson": "experiment.flags.{{key}} 不是可 JSON 序列化的值(函数 / undefined / 循环引用 / bigint 不允许);flags 会原样进入结果快照,必须是纯 JSON。",
   "define.experimentLabelInvalid": "experiment.labels.{{key}} 必须是字符串或有限数字;labels 是报告侧的归类坐标,会原样进入结果快照。",
   "define.experimentSetupNotFunction": "experiment.setup 必须是函数((ctx) => void);要清理请挂 experiment.teardown;要按实验准备沙箱内环境请挂 sandbox spec 的 .setup() 钩子链。",
+  "define.experimentClassifyFailureNotFunction": "experiment.classifyFailure 必须是函数((failure) => FailureClass | undefined):它识别以第三方错误形态浮出的失败,认不出的一律返回 undefined 交给后续链路。",
   "define.experimentIdRejected": "defineExperiment 不接受 id —— id 由文件路径推导。",
   "define.sandboxAgentNameRequired": "defineSandboxAgent 需要 name。",
   "define.sandboxCreateRequired": "defineSandbox 需要一个 create() 函数。",
@@ -240,7 +251,8 @@ export const zhCN = {
   "feedback.human.active": "ACTIVE",
   "feedback.human.budgetExhausted": "{{experimentId}} 预算已耗尽(已花 {{spent}},未跑 {{unstarted}})",
   "feedback.human.compare": "Compare: niceeval view",
-  "feedback.human.counts": "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 排队 {{queued}} · 已完成 {{completed}}",
+  "feedback.human.counts":
+    "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 排队 {{queued}} · 通过 {{passed}} · 失败 {{failed}} · 出错 {{errored}} · 跳过 {{skipped}}",
   "feedback.human.diffHint": "Diff:    niceeval show {{locator}} --diff",
   "feedback.human.evalHint": "Eval:    niceeval show {{locator}} --source",
   "feedback.human.failuresHeader": "FAILURES",
@@ -273,6 +285,15 @@ export const zhCN = {
   "feedback.human.hookFailed": "失败",
   "feedback.human.precheckJudge": "预检 judge 配置",
   "feedback.human.precheckJudgeDone": "judge 配置就绪",
+  "feedback.human.countsWithElsewhere":
+    "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 等待中 {{elsewhere}} · 排队 {{queued}} · 通过 {{passed}} · 失败 {{failed}} · 出错 {{errored}} · 跳过 {{skipped}}",
+  "feedback.human.waitingOnAnotherRun": "等待另一个并行 run",
+  "feedback.human.lockWaitDetail": "{{count}} 条用例 · pid {{pid}}",
+  "feedback.human.lockWaitStarted": "等待另一个并行 run · {{experimentId}}({{count}} 条用例,pid {{pid}})",
+  "feedback.human.lockWaitResolved": "等待结束 · {{experimentId}}({{summary}},{{elapsed}})",
+  "feedback.human.lockWaitCarried": "{{count}} 条携入",
+  "feedback.human.lockWaitDispatched": "{{count}} 条自跑",
+  "feedback.human.lockedRowSuffix": "locked",
   "feedback.phase.sandboxSetup": "沙箱预置",
   "feedback.phase.scoring": "评分",
   "feedback.phase.teardown": "清理中",

package/src/index.ts

@@ -5,6 +5,10 @@ export { defineEval, defineScoreEval, defineConfig, defineExperiment } from "./d
 
 export { requireEnv, getEnv, stripComments } from "./util.ts";
 
+// 执行失败分类:抛出点糖衣类(声明死因波及多远)+ 结构守卫。判据、分类链与止损语义见
+// docs/feature/error-classification/README.md;`niceeval/adapter` 复导出同一份词表类型。
+export { ExperimentFatalError, EvalFatalError, failureClassOf } from "./shared/failure-class.ts";
+
 // 类型(eval 作者会用到;跑哪个 agent / 用哪个 sandbox 见对应子路径)
 export type {
   StreamEvent,
@@ -59,6 +63,10 @@ export type {
   TraceSpan,
   SpanKind,
   DerivedFacts,
+  FailureClass,
+  FailureScope,
+  AttemptFailureInfo,
+  AttemptFailureClassifier,
 } from "./types.ts";
 
 export type { ParsedTranscript } from "./o11y/parsers/index.ts";

package/src/o11y/cost.ts

@@ -56,8 +56,10 @@ function lookupBuiltin(model: string): Price | undefined {
 }
 
 /**
- * 按 token 桶 × 单价估算一次运行的美元成本。cache 桶缺专门单价时退回 input 价
- * (cache token 本质也是 input)。无 model / 查不到价(用户表 + 内置快照都没有)→ undefined。
+ * 按 token 桶 × 单价估算一次运行的美元成本。逐桶相加成立的前提是 Usage 桶恒互斥
+ * (inputTokens 不含 cache 命中,归一义务在 adapter,见 docs/feature/results/architecture.md#usage);
+ * cache 桶缺专门单价时退回 input 价——宁可高估,不静默低估。
+ * 无 model / 查不到价(用户表 + 内置快照都没有)→ undefined。
  */
 export function estimateCost(
   model: string | undefined,

package/src/o11y/derive.test.ts

@@ -58,6 +58,46 @@ describe("deriveRunFacts:pending 折叠", () => {
   });
 });
 
+describe("deriveRunFacts:callId 跨轮复用不覆盖", () => {
+  // 回归:adapter 常按轮各自编号(OpenAI 兼容 / transcript 归一复用 c1/c2…)。同一 callId 在其
+  // result 之后再次出现是新的一次调用——旧折叠按 callId 存 Map 会被后一轮覆盖,跨轮聚合断言
+  // (t.calledTool)于是「只扫最后一轮」,前几轮的工具调用被抹掉(定稿见 events.md 不变量 2)。
+  it("同一 callId 完成后在后一轮再次 called,折叠成两条独立调用而非覆盖成一条", () => {
+    const events: StreamEvent[] = [
+      // 第一轮:读 init、读 INDEX(都用 c1/c2 编号)
+      { type: "action.called", callId: "c1", name: "read", input: { path: "init.md" } },
+      { type: "action.result", callId: "c1", output: "init contents", status: "completed" },
+      { type: "action.called", callId: "c2", name: "read", input: { path: "INDEX.md" } },
+      { type: "action.result", callId: "c2", output: "index contents", status: "completed" },
+      { type: "message", role: "assistant", text: "第一轮做完" },
+      // 第二轮:续轮,adapter 从 c1 重新编号
+      { type: "action.called", callId: "c1", name: "write", input: { path: "note.md" } },
+      { type: "action.result", callId: "c1", output: "ok", status: "completed" },
+      { type: "message", role: "assistant", text: "答复" },
+    ];
+    const facts = deriveRunFacts(events);
+    expect(facts.toolCalls).toHaveLength(3); // 两轮共 3 次调用,后一轮不覆盖前一轮
+    expect(facts.toolCalls.map((tc) => tc.originalName)).toEqual(["read", "read", "write"]);
+    // 第一轮的两次 read 都在,且各配对到自己的 result(不是都取最后一条)
+    expect(facts.toolCalls[0]!.input).toEqual({ path: "init.md" });
+    expect(facts.toolCalls[0]!.output).toBe("init contents");
+    expect(facts.toolCalls[1]!.input).toEqual({ path: "INDEX.md" });
+    expect(facts.toolCalls[1]!.output).toBe("index contents");
+  });
+
+  it("subagent 折叠同理:同一 callId 完成后再次 called 起新记录", () => {
+    const events: StreamEvent[] = [
+      { type: "subagent.called", callId: "s1", name: "researcher" },
+      { type: "subagent.completed", callId: "s1", output: "round1", status: "completed" },
+      { type: "subagent.called", callId: "s1", name: "writer" },
+      { type: "subagent.completed", callId: "s1", output: "round2", status: "completed" },
+    ];
+    const facts = deriveRunFacts(events);
+    expect(facts.subagentCalls).toHaveLength(2);
+    expect(facts.subagentCalls.map((sc) => sc.name)).toEqual(["researcher", "writer"]);
+  });
+});
+
 describe("deriveRunFacts:contextInjections 计数", () => {
   it("统计事件流里 context.injected 事件的次数,不与 messageCount 混计", () => {
     const events: StreamEvent[] = [