niceeval 0.6.0 → 0.6.2

package/src/report/report.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/reports/cases.md
 // niceeval/report 计算层的单元测试:全部用内存 fake(Snapshot / AttemptHandle 按
 // niceeval/results 的读取契约手工构造),专门覆盖 docs/feature/reports/architecture.md 点名的坑 ——
 // 两级聚合 vs 平铺、pass@k、examScore 空真、skipped 稀释、scoreboard 固定分母与
@@ -6,11 +7,26 @@
 
 import { describe, expect, it } from "vitest";
 
-import type { AssertionResult, EvalResult, O11ySummary, Verdict } from "../types.ts";
+import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "../types.ts";
+
+/** 结构化 `AttemptError` 的最小构造(测试用):只关心一层 message,operation/code 取中性默认值。 */
+function erroredWith(message: string): AttemptError {
+  return { code: "unexpected-error", message, phase: "eval.run" };
+}
 import type { AttemptHandle, Selection, SelectionWarning, Snapshot } from "../results/index.ts";
 import type { Dimension, MetricCell } from "./types.ts";
-import { costUSD, defineMetric, durationMs, examScore, passRate, tokens, turns } from "./metrics.ts";
-import { flag } from "./flag.ts";
+import {
+  costUSD,
+  defineMetric,
+  durationMs,
+  endToEndPassRate,
+  examScore,
+  executionReliability,
+  taskPassRate,
+  tokens,
+  turns,
+} from "./metrics.ts";
+import { config, flag } from "./flag.ts";
 import { formatMetricValue } from "./format.ts";
 import {
   AttemptList,
@@ -48,7 +64,7 @@ function res(id: string, verdict: Verdict, extra: Partial<EvalResult> = {}): Eva
 }
 
 function softAssertion(name: string, score: number, extra: Partial<AssertionResult> = {}): AssertionResult {
-  return { name, severity: "soft", score, passed: true, ...extra };
+  return { name, severity: "soft", score, outcome: "passed" as const, ...extra } as AssertionResult;
 }
 
 /** 最小合规 O11ySummary,只有 totalTurns 会变;供 turns 指标测试内联挂到 EvalResult.o11y。 */
@@ -136,7 +152,7 @@ function selection(snapshots: Snapshot[], warnings: SelectionWarning[]): Selecti
 }
 
 describe("ExperimentList.data", () => {
-  it("身份、flags、Eval 判定构成、官方两级聚合汇总指标,展开到 Eval 携带 locator 与证据能力", async () => {
+  it("身份、flags、Eval 判定构成、官方两级聚合汇总指标,展开到 Eval 携带全部 Attempt locator", async () => {
     const s = snap({
       experimentId: "group/codex-e2b--mempal",
       agent: "codex",
@@ -146,7 +162,7 @@ describe("ExperimentList.data", () => {
         res("memory/a", "failed", {
           attempt: 0,
           durationMs: 1_000,
-          assertions: [{ name: "gate", severity: "gate", passed: false, score: 0, detail: "wrong" }],
+          assertions: [{ name: "gate", severity: "gate", outcome: "failed" as const, score: 0, detail: "wrong" }],
           usage: { inputTokens: 10, outputTokens: 5, costUSD: 0.1 },
           hasEvents: true,
         }),
@@ -155,10 +171,17 @@ describe("ExperimentList.data", () => {
           durationMs: 2_000,
           usage: { inputTokens: 20, outputTokens: 10, costUSD: 0.2 },
         }),
-        res("memory/b", "errored", { error: "timeout", durationMs: 3_000 }),
+        res("memory/b", "errored", { error: erroredWith("timeout"), durationMs: 3_000 }),
       ],
     });
-    s.experiment = { id: s.experimentId, runs: 2, earlyExit: false, sandbox: "e2b:fast", budget: 2, flags: { cache: true } };
+    s.experiment = {
+      runs: 2,
+      earlyExit: false,
+      sandbox: { provider: "e2b", params: { template: "fast" } },
+      budget: 2,
+      flags: { cache: true },
+      selectedEvalIds: [],
+    };
 
     const items = await ExperimentList.data([s]);
     expect(items).toHaveLength(1);
@@ -173,7 +196,8 @@ describe("ExperimentList.data", () => {
       attempts: 3,
       lastRunAt: "2026-07-10T22:44:00.000Z",
     });
-    // 官方两级聚合口径:memory/a 题内均值 (0+1)/2=0.5,memory/b 题内 0;跨题均值 0.25
+    // 官方端到端两级聚合:memory/a 题内均值 (0+1)/2=0.5;
+    // memory/b errored → 0 → 跨题均值 (0.5+0)/2=0.25
     expect(item.passRate.value).toBeCloseTo(0.25);
     expect(item.cost.samples).toBe(2); // 只有 memory/a 的两次 attempt 报了成本
 
@@ -181,14 +205,13 @@ describe("ExperimentList.data", () => {
     expect(evalA.verdict).toBe("passed"); // 任一轮通过 → 该题通过
     expect(evalA.attempts.map((a) => a.attempt)).toEqual([0, 1]);
     expect(evalA.attempts[0]!.locator).toMatch(/^@1[0-9a-z]{7}$/);
-    expect(evalA.attempts[0]!.capabilities.execution).toBe(true); // attempt0 的 hasEvents:true
-    expect(evalA.attempts[1]!.capabilities.execution).toBe(false);
     // 每次 attempt 独立编码,即使同一道题也不撞车
     expect(evalA.attempts[0]!.locator).not.toBe(evalA.attempts[1]!.locator);
 
     const evalB = item.evalRows.find((e) => e.evalId === "memory/b")!;
     expect(evalB.verdict).toBe("errored");
-    expect(evalB.reason).toBe("timeout");
+    expect(evalB).not.toHaveProperty("reason");
+    expect(evalB.attempts[0]!.error?.message).toBe("timeout");
   });
 
   it("展开数组按 evalId 升序,顶层数组按 experimentId 升序", async () => {
@@ -198,10 +221,29 @@ describe("ExperimentList.data", () => {
     expect(items.map((i) => i.experimentId)).toEqual(["exp/a", "exp/b"]);
     expect(items.find((i) => i.experimentId === "exp/b")!.evalRows.map((e) => e.evalId)).toEqual(["a", "z"]);
   });
+
+  it("默认成功率包含 errored 并据此排序:2 passed / 5 errored = 2/7,不能以条件 100% 排第一", async () => {
+    const unstable = snap({
+      experimentId: "exp/unstable",
+      results: [
+        res("pass/a", "passed"),
+        res("pass/b", "passed"),
+        ...Array.from({ length: 5 }, (_, i) => res(`error/${i}`, "errored", { error: erroredWith("boom") })),
+      ],
+    });
+    const stable = snap({
+      experimentId: "exp/stable",
+      results: [res("pass", "passed"), res("fail", "failed")],
+    });
+
+    const items = await ExperimentList.data([unstable, stable]);
+    expect(items.find((item) => item.experimentId === "exp/unstable")!.passRate.value).toBeCloseTo(2 / 7, 10);
+    expect(items.map((item) => item.experimentId)).toEqual(["exp/stable", "exp/unstable"]);
+  });
 });
 
 describe("EvalList.data", () => {
-  it("每项一个 experimentId + evalId,判定/分数/成本耗时均值/失败原因与展开到 Attempt 都在", async () => {
+  it("每项一个 experimentId + evalId,父项保留判定与题级聚合并展开到含失败事实的 Attempt", async () => {
     const s = snap({
       experimentId: "compare/codex",
       results: [
@@ -209,13 +251,13 @@ describe("EvalList.data", () => {
           attempt: 0,
           durationMs: 40_000,
           usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.02 },
-          assertions: [{ name: 'calledTool("get_weather")', severity: "gate", passed: false, score: 0, detail: "tool was never called" }],
+          assertions: [{ name: 'calledTool("get_weather")', severity: "gate", outcome: "failed" as const, score: 0, expected: "at least one matching call", received: "no tool calls" }],
         }),
         res("weather/brooklyn", "failed", {
           attempt: 1,
           durationMs: 42_000,
           usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.06 },
-          assertions: [{ name: 'calledTool("get_weather")', severity: "gate", passed: false, score: 0, detail: "tool was never called" }],
+          assertions: [{ name: 'calledTool("get_weather")', severity: "gate", outcome: "failed" as const, score: 0, expected: "at least one matching call", received: "no tool calls" }],
         }),
       ],
     });
@@ -225,11 +267,15 @@ describe("EvalList.data", () => {
     expect(item.evalId).toBe("weather/brooklyn");
     expect(item.experimentId).toBe("compare/codex");
     expect(item.verdict).toBe("failed");
-    expect(item.reason).toBe('calledTool("get_weather"): tool was never called');
+    expect(item).not.toHaveProperty("reason");
     expect(item.score.value).toBe(0); // examScore: failed → 0
     expect(item.duration.value).toBeCloseTo(41_000); // 平均耗时
     expect(item.cost.value).toBeCloseTo(0.04); // 平均成本
     expect(item.attempts.map((a) => a.attempt)).toEqual([0, 1]);
+    const assertion = item.attempts[0]!.assertions[0]!;
+    expect(assertion.outcome).toBe("failed");
+    if (assertion.outcome !== "failed") throw new Error("expected failed assertion fixture");
+    expect(assertion.received).toBe("no tool calls");
     expect(item.attempts[0]!.locator).not.toBe(item.attempts[1]!.locator);
   });
 
@@ -243,14 +289,14 @@ describe("EvalList.data", () => {
 });
 
 describe("AttemptList.data", () => {
-  it("每项一个 Attempt:身份、判定、assertions、耗时、成本、locator 与证据能力标记", async () => {
+  it("每项一个 Attempt:身份、判定、assertions、耗时、成本与 locator", async () => {
     const s = snap({
       experimentId: "exp/x",
       results: [
         res("A", "failed", {
           durationMs: 5_000,
           usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.05 },
-          assertions: [{ name: "gate", severity: "gate", passed: false, score: 0, detail: "nope" }],
+          assertions: [{ name: "gate", severity: "gate", outcome: "failed" as const, score: 0, detail: "nope" }],
           hasEvents: true,
           hasSources: true,
         }),
@@ -270,23 +316,37 @@ describe("AttemptList.data", () => {
     });
     expect(item.assertions).toHaveLength(1);
     expect(item.locator).toMatch(/^@1[0-9a-z]{7}$/);
-    // 证据能力:cheap 计算,不调用完整 loadAttemptEvidence——execution 来自 hasEvents,
-    // eval 来自 hasSources,diff 懒加载(fixture 的 diff() 恒返回 null → false)。
-    expect(item.capabilities).toEqual({ eval: true, execution: true, timing: false, diff: false });
   });
 
-  it("redact 作用于 error 与断言 detail/evidence,不作用于身份字段", async () => {
+  it("redact 作用于 error 的 message/cause/stack、diagnostic 的 message/data、断言 detail/evidence,不作用于身份与分类字段", async () => {
     const s = snap({
       experimentId: "exp/x",
       results: [
-        res("A", "errored", { error: "ENOENT /Users/me/repo/tool" }),
+        res("A", "errored", {
+          error: {
+            code: "sandbox-provision-failed",
+            message: "ENOENT /Users/me/repo/tool",
+            phase: "sandbox.create",
+            stack: "Error: ENOENT /Users/me/repo/tool\n    at spawn (/Users/me/repo/src/run.ts:3:1)",
+            cause: { name: "SystemError", code: "ENOENT", message: "no such file /Users/me/repo/tool" },
+          },
+          diagnostics: [
+            {
+              code: "teardown-failed",
+              level: "warning",
+              message: "cleanup left /Users/me/repo/tmp behind",
+              phase: "sandbox.teardown",
+              data: { path: "/Users/me/repo/tmp", attempts: 2 },
+            },
+          ],
+        }),
         res("B", "failed", {
           assertions: [
             {
               name: "includes",
               severity: "gate",
               score: 0,
-              passed: false,
+              outcome: "failed" as const,
               detail: "missing text under /Users/me/repo/src",
               evidence: "checked /Users/me/repo/src/app.ts",
             },
@@ -296,10 +356,25 @@ describe("AttemptList.data", () => {
     });
     const items = await AttemptList.data([s], { redact: (text) => text.replaceAll("/Users/me/repo", "<repo>") });
     const a = items.find((i) => i.evalId === "A")!;
-    expect(a.error).toBe("ENOENT <repo>/tool");
+    // error:自由文本(message/stack/cause.message)全部经钩子
+    expect(a.error?.message).toBe("ENOENT <repo>/tool");
+    expect(a.error?.stack).toBe("Error: ENOENT <repo>/tool\n    at spawn (<repo>/src/run.ts:3:1)");
+    expect(a.error?.cause?.message).toBe("no such file <repo>/tool");
+    // 分类字段(code/operation/cause.name/cause.code)原样保留,不经钩子
+    expect(a.error?.code).toBe("sandbox-provision-failed");
+    expect(a.error?.phase).toBe("sandbox.create");
+    expect(a.error?.cause?.name).toBe("SystemError");
+    expect(a.error?.cause?.code).toBe("ENOENT");
+    // diagnostics:message 与 data 的字符串值经钩子;code/operation/level 原样
+    expect(a.diagnostics).toHaveLength(1);
+    expect(a.diagnostics![0]!.message).toBe("cleanup left <repo>/tmp behind");
+    expect(a.diagnostics![0]!.data).toEqual({ path: "<repo>/tmp", attempts: 2 });
+    expect(a.diagnostics![0]!.code).toBe("teardown-failed");
+    expect(a.diagnostics![0]!.phase).toBe("sandbox.teardown");
     const b = items.find((i) => i.evalId === "B")!;
-    expect(b.assertions[0]!.detail).toBe("missing text under <repo>/src");
-    expect(b.assertions[0]!.evidence).toBe("checked <repo>/src/app.ts");
+    const bAssert = b.assertions[0]!;
+    expect(bAssert.detail).toBe("missing text under <repo>/src");
+    expect(bAssert.outcome !== "unavailable" ? bAssert.evidence : undefined).toBe("checked <repo>/src/app.ts");
     // 身份字段不经 redact
     expect(items.every((i) => i.experimentId === "exp/x")).toBe(true);
   });
@@ -307,7 +382,7 @@ describe("AttemptList.data", () => {
   it("不预设只看失败:passed 的 attempt 同样列出,原样携带 assertions", async () => {
     const s = snap({
       experimentId: "exp/x",
-      results: [res("A", "passed", { assertions: [{ name: "ok", severity: "gate", passed: true, score: 1 }] })],
+      results: [res("A", "passed", { assertions: [{ name: "ok", severity: "gate", outcome: "passed" as const, score: 1 }] })],
     });
     const items = await AttemptList.data([s]);
     expect(items).toHaveLength(1);
@@ -324,10 +399,10 @@ describe("两级聚合引擎", () => {
       experimentId: "exp/x",
       results: [res("A", "passed"), res("B", "failed"), res("B", "failed"), res("B", "failed")],
     });
-    const data = await MetricTable.data([s], { rows: "agent", columns: [passRate] });
+    const data = await MetricTable.data([s], { rows: "agent", columns: [taskPassRate] });
     expect(data.dimension).toBe("agent");
     expect(data.rows).toHaveLength(1);
-    const cell = data.rows[0].cells["pass-rate"];
+    const cell = data.rows[0].cells["task-pass-rate"];
     expect(cell.value).toBe(0.5);
     expect(cell.display).toBe("50%");
     expect(cell.samples).toBe(4);
@@ -347,17 +422,17 @@ describe("两级聚合引擎", () => {
       experimentId: "exp/x",
       results: [res("A", "failed"), res("A", "failed"), res("B", "failed"), res("B", "passed")],
     });
-    const data = await MetricTable.data([s], { rows: "agent", columns: [passAtK, passRate] });
+    const data = await MetricTable.data([s], { rows: "agent", columns: [passAtK, taskPassRate] });
     // A: max(0,0)=0;B: max(0,1)=1 → (0+1)/2
     expect(data.rows[0].cells["pass@k"].value).toBe(0.5);
-    // 对照:默认 mean/mean 的 passRate = (0 + 0.5)/2
-    expect(data.rows[0].cells["pass-rate"].value).toBe(0.25);
+    // 对照:默认 mean/mean 的 taskPassRate = (0 + 0.5)/2
+    expect(data.rows[0].cells["task-pass-rate"].value).toBe(0.25);
   });
 
   it("skipped 是 null:不稀释均值,但计入 total(覆盖率如实)", async () => {
     const s = snap({ experimentId: "exp/x", results: [res("A", "passed"), res("B", "skipped")] });
-    const data = await MetricTable.data([s], { rows: "agent", columns: [passRate] });
-    const cell = data.rows[0].cells["pass-rate"];
+    const data = await MetricTable.data([s], { rows: "agent", columns: [taskPassRate] });
+    const cell = data.rows[0].cells["task-pass-rate"];
     expect(cell.value).toBe(1); // B 整桶为 null,不参与 across,不是 0.5
     expect(cell.samples).toBe(1);
     expect(cell.total).toBe(2);
@@ -366,8 +441,8 @@ describe("两级聚合引擎", () => {
 
   it("全组 null → value null、display 兜底,不编 0;refs 必填(空数组)", async () => {
     const s = snap({ experimentId: "exp/x", results: [res("A", "skipped")] });
-    const data = await MetricTable.data([s], { rows: "agent", columns: [passRate] });
-    const cell = data.rows[0].cells["pass-rate"];
+    const data = await MetricTable.data([s], { rows: "agent", columns: [taskPassRate] });
+    const cell = data.rows[0].cells["task-pass-rate"];
     expect(cell.value).toBeNull();
     expect(cell.display).toBe("—");
     expect(cell.samples).toBe(0);
@@ -402,18 +477,18 @@ describe("两级聚合引擎", () => {
         res("A", "passed", { attempt: 2 }),
       ],
     });
-    const data = await MetricTable.data([s], { rows: byParity, columns: [passRate] });
+    const data = await MetricTable.data([s], { rows: byParity, columns: [taskPassRate] });
     expect(data.dimension).toBe("parity");
-    const byKey = Object.fromEntries(data.rows.map((r) => [r.key, r.cells["pass-rate"].value]));
+    const byKey = Object.fromEntries(data.rows.map((r) => [r.key, r.cells["task-pass-rate"].value]));
     // 同一道题的 attempt 分进两组:even 组内 [1,1] 折成 1,odd 组内 [0] 折成 0
     expect(byKey).toEqual({ even: 1, odd: 0 });
   });
 
   it("同一次计算里指标重名是错误", async () => {
-    const dup = defineMetric({ name: "pass-rate", value: () => 1 });
+    const dup = defineMetric({ name: "task-pass-rate", value: () => 1 });
     const s = snap({ experimentId: "exp/x", results: [res("A", "passed")] });
-    await expect(MetricTable.data([s], { rows: "agent", columns: [passRate, dup] })).rejects.toThrow(
-      /Duplicate metric name "pass-rate"/,
+    await expect(MetricTable.data([s], { rows: "agent", columns: [taskPassRate, dup] })).rejects.toThrow(
+      /Duplicate metric name "task-pass-rate"/,
     );
   });
 
@@ -423,16 +498,16 @@ describe("两级聚合引擎", () => {
     const none = snap({ experimentId: "exp/none", agent: "none", results: [res("A", "skipped", { agent: "none" })] });
     const data = await MetricTable.data([none, bad, good], {
       rows: "agent",
-      columns: [passRate],
-      sort: passRate,
+      columns: [taskPassRate],
+      sort: taskPassRate,
     });
     expect(data.rows.map((r) => r.key)).toEqual(["good", "bad", "none"]);
   });
 
   it("列键是字面量联合:拼错列名编译不过", async () => {
     const s = snap({ experimentId: "exp/x", results: [res("A", "passed")] });
-    const data = await MetricTable.data([s], { rows: "agent", columns: [passRate, costUSD] });
-    const cell: MetricCell = data.rows[0].cells[passRate.name]; // 键锚在指标对象上
+    const data = await MetricTable.data([s], { rows: "agent", columns: [taskPassRate, costUSD] });
+    const cell: MetricCell = data.rows[0].cells[taskPassRate.name]; // 键锚在指标对象上
     expect(cell.value).toBe(1);
     // @ts-expect-error 列里没有这个键 —— 编译期挡住,不是运行时 undefined
     data.rows[0].cells["pass-rat"];
@@ -445,7 +520,7 @@ describe("examScore", () => {
   it("errored(断言空数组)得 0 —— 不因「gate 全过」空真得满分", async () => {
     const s = snap({
       experimentId: "exp/x",
-      results: [res("A", "errored", { assertions: [], error: "adapter crashed" })],
+      results: [res("A", "errored", { assertions: [], error: erroredWith("adapter crashed") })],
     });
     const data = await MetricTable.data([s], { rows: "agent", columns: [examScore] });
     const cell = data.rows[0].cells["exam-score"];
@@ -470,7 +545,7 @@ describe("examScore", () => {
           assertions: [
             softAssertion("judge-1", 0.5),
             softAssertion("judge-2", 1),
-            { name: "includes", severity: "gate", score: 1, passed: true },
+            { name: "includes", severity: "gate", score: 1, outcome: "passed" as const },
           ],
         }),
       ],
@@ -479,7 +554,7 @@ describe("examScore", () => {
       experimentId: "exp/b",
       results: [
         res("B", "passed", {
-          assertions: [{ name: "includes", severity: "gate", score: 1, passed: true }],
+          assertions: [{ name: "includes", severity: "gate", score: 1, outcome: "passed" as const }],
         }),
       ],
     });
@@ -496,6 +571,62 @@ describe("examScore", () => {
   });
 });
 
+// ───────────────────────── 三个通过率指标(docs/feature/reports/library.md「内置指标」)─────────────────────────
+//
+// 同一份 passed / failed / errored / skipped 混合 fixture 上,三个口径必须给出三个不同的数,
+// 任何一处偷懒复用另一个公式都会在这里露馅:
+//   taskPassRate:errored → null 不进分母 → (1 + 0) / 2 = 50%
+//   executionReliability:errored → 0,passed/failed → 1 → (1 + 1 + 0) / 3 ≈ 66.7%
+//   endToEndPassRate:failed/errored → 0 → (1 + 0 + 0) / 3 ≈ 33.3%
+
+describe("taskPassRate / executionReliability / endToEndPassRate", () => {
+  const mixed = () =>
+    snap({
+      experimentId: "exp/mixed",
+      results: [
+        res("A", "passed"),
+        res("B", "failed"),
+        res("C", "errored", { error: erroredWith("adapter crashed") }),
+        res("D", "skipped"),
+      ],
+    });
+
+  it("taskPassRate:errored 记 null 不进分母 —— 基建故障不伪装成 Agent 答错", async () => {
+    const data = await MetricTable.data([mixed()], { rows: "agent", columns: [taskPassRate] });
+    const cell = data.rows[0].cells["task-pass-rate"];
+    expect(cell.value).toBeCloseTo(0.5, 10); // (1 + 0) / 2:errored 与 skipped 都不在分母
+    expect(cell.samples).toBe(2); // errored、skipped 都是 null,不算有效样本
+    expect(cell.total).toBe(4);
+  });
+
+  it("executionReliability:跑到可判定(passed/failed)= 1,errored = 0,skipped → null", async () => {
+    const data = await MetricTable.data([mixed()], { rows: "agent", columns: [executionReliability] });
+    const cell = data.rows[0].cells["execution-reliability"];
+    expect(cell.value).toBeCloseTo(2 / 3, 10);
+    expect(cell.samples).toBe(3); // errored 测得了(= 0),skipped 测不了
+    expect(cell.total).toBe(4);
+  });
+
+  it("endToEndPassRate:failed / errored 都记 0 —— 端到端合成,哪边拖累用前两个拆开看", async () => {
+    const data = await MetricTable.data([mixed()], { rows: "agent", columns: [endToEndPassRate] });
+    const cell = data.rows[0].cells["end-to-end-pass-rate"];
+    expect(cell.value).toBeCloseTo(1 / 3, 10);
+    expect(cell.samples).toBe(3);
+    expect(cell.total).toBe(4);
+  });
+
+  it("全部 errored:taskPassRate 缺数据(null),executionReliability 如实 0 —— 两个指标不互相冒充", async () => {
+    const s = snap({
+      experimentId: "exp/broken",
+      results: [res("A", "errored", { error: erroredWith("boom") })],
+    });
+    const task = await MetricTable.data([s], { rows: "agent", columns: [taskPassRate] });
+    expect(task.rows[0].cells["task-pass-rate"].value).toBeNull(); // 没有一道题真正被答过
+    const reliability = await MetricTable.data([s], { rows: "agent", columns: [executionReliability] });
+    expect(reliability.rows[0].cells["execution-reliability"].value).toBe(0);
+  });
+});
+
 // ───────────────────────── 内置指标口径 ─────────────────────────
 
 describe("内置指标", () => {
@@ -656,7 +787,7 @@ describe("MetricScatter.data", () => {
       points: "experiment",
       series: "agent",
       x: costUSD,
-      y: passRate,
+      y: taskPassRate,
     });
     expect(data.points).toBe("experiment");
     expect(data.series).toBe("agent");
@@ -681,7 +812,7 @@ describe("flag()", () => {
   const withFlags = (id: string, flags: Record<string, unknown> | undefined, verdict: Verdict) =>
     snap({
       experimentId: id,
-      results: [res("A", verdict, { experimentId: id, experiment: { id, flags } })],
+      results: [res("A", verdict, { experimentId: id, experiment: { flags, runs: 1, earlyExit: true, selectedEvalIds: [] } as import("../types.ts").ExperimentRunInfo })],
     });
 
   it("MetricLine.data:x 收 flag、按 experiment 聚合;未声明的作轴 x=null 报数", async () => {
@@ -691,7 +822,7 @@ describe("flag()", () => {
     const data = await MetricLine.data([s1, s2, legacy], {
       x: flag("latencyMs", { label: "Simulated latency", unit: "ms" }),
       series: flag("agents", { label: (v) => `${v} agents` }),
-      y: passRate,
+      y: taskPassRate,
     });
     expect(data.x).toEqual({ key: "latencyMs", label: "Simulated latency", unit: "ms" });
     expect(data.series).toBe("agents");
@@ -715,13 +846,84 @@ describe("flag()", () => {
     const s2 = withFlags("exp/b", { agents: 16 }, "failed");
     const data = await MetricTable.data([s1, s2], {
       rows: flag("agents", { label: (v) => `${v} agents` }),
-      columns: [passRate],
+      columns: [taskPassRate],
     });
     expect(data.dimension).toBe("agents");
     expect(data.rows.map((r) => r.key)).toEqual(["1 agents", "16 agents"]);
   });
 });
 
+// ───────────────────────── config():顶层运行配置当维度与轴 ─────────────────────────
+// flag() 只读 ExperimentDef.flags 的显式声明;model / reasoningEffort / budget / runs 这类
+// 顶层运行配置用 config() 读快照的 ExperimentRunInfo 投影,外加桥接 model / agent 两个键
+// (docs/feature/reports/library.md「维度与 flags」)。
+
+describe("config()", () => {
+  /** 快照声明 ExperimentRunInfo 投影;字段全集以 docs 的接口为准,这里按键名读值。 */
+  const withConfig = (id: string, info: Record<string, unknown> | undefined, verdict: Verdict, model?: string) => {
+    const s = snap({ experimentId: id, model, results: [res("A", verdict, { experimentId: id })] });
+    if (info) s.experiment = info as unknown as (typeof s)["experiment"];
+    return s;
+  };
+
+  it("config 当维度用:按投影值分组;未投影的 experiment 如实归 (unset)", async () => {
+    const high = withConfig("effort/high", { reasoningEffort: "high" }, "passed");
+    const low = withConfig("effort/low", { reasoningEffort: "low" }, "failed");
+    const legacy = withConfig("effort/legacy", undefined, "passed");
+    const data = await MetricTable.data([high, low, legacy], {
+      rows: config("reasoningEffort", { label: "Reasoning effort" }),
+      columns: [taskPassRate],
+    });
+    expect(data.dimension).toBe("reasoningEffort");
+    expect(data.rows.map((r) => r.key)).toEqual(["high", "low", "(unset)"]);
+  });
+
+  it("config 当数值轴用(MetricLine.data 的 x):非数值或未投影 → x=null 不伪造,注脚可数", async () => {
+    const b1 = withConfig("budget/one", { budget: 1 }, "passed");
+    const b2 = withConfig("budget/two", { budget: 2 }, "failed");
+    const none = withConfig("budget/none", { reasoningEffort: "high" }, "passed"); // 没投影 budget
+    const data = await MetricLine.data([b1, b2, none], {
+      x: config("budget", { label: "Budget", unit: "USD" }),
+      y: taskPassRate,
+    });
+    expect(data.x).toEqual({ key: "budget", label: "Budget", unit: "USD" });
+    const one = data.rows.find((r) => r.key === "budget/one")!;
+    expect(one.x).toBe(1);
+    expect(one.y.value).toBe(1);
+    const missing = data.rows.find((r) => r.key === "budget/none")!;
+    expect(missing.x).toBeNull();
+    expect(missing.xDisplay).toBe("");
+    expect(data.rows.filter((r) => r.x === null)).toHaveLength(1);
+  });
+
+  it("model / agent 两个键桥接到快照顶层权威字段,ExperimentRunInfo 不复制这两个事实", async () => {
+    const mini = withConfig("m/mini", { budget: 1 }, "passed", "gpt-mini");
+    const maxi = withConfig("m/maxi", { budget: 1 }, "failed", "gpt-maxi");
+    const data = await MetricTable.data([mini, maxi], {
+      rows: config("model"),
+      columns: [taskPassRate],
+    });
+    expect(data.rows.map((r) => r.key)).toEqual(["gpt-mini", "gpt-maxi"]);
+
+    const byAgent = await MetricTable.data([mini, maxi], { rows: config("agent"), columns: [taskPassRate] });
+    expect(byAgent.rows.map((r) => r.key)).toEqual(["agent-x"]); // res() 默认 agent-x
+  });
+
+  it("label 函数折组名,与 flag() 同一套规则", async () => {
+    const s1 = withConfig("runs/3", { runs: 3 }, "passed");
+    const s2 = withConfig("runs/5", { runs: 5 }, "passed");
+    const data = await MetricTable.data([s1, s2], {
+      rows: config("runs", { label: (v) => `${v} runs` }),
+      columns: [taskPassRate],
+    });
+    expect(data.rows.map((r) => r.key)).toEqual(["3 runs", "5 runs"]);
+  });
+
+  it("空 name 直接报错,与 flag() 同一门槛", () => {
+    expect(() => config("")).toThrow(/non-empty string/);
+  });
+});
+
 // ───────────────────────── MetricTable.data:没有实体下钻 ─────────────────────────
 
 describe("MetricTable.data", () => {
@@ -730,7 +932,7 @@ describe("MetricTable.data", () => {
       experimentId: "exp/x",
       results: [res("A", "passed"), res("B", "failed")],
     });
-    const data = await MetricTable.data([s], { rows: "experiment", columns: [passRate] });
+    const data = await MetricTable.data([s], { rows: "experiment", columns: [taskPassRate] });
     expect(data.rows).toHaveLength(1);
     expect(data.rows[0]!.meta).not.toHaveProperty("subRows");
     // 展开到 Eval 是 ExperimentList 的职责,不在这里验证 —— 见 ExperimentList.data 的测试组。
@@ -745,10 +947,10 @@ describe("MetricTable.data", () => {
 describe("reasonFor", () => {
   it("同一 result 同时含 error、失败 gate、失败 soft → 只显示 error", () => {
     const result = res("A", "errored", {
-      error: "adapter crashed",
+      error: erroredWith("adapter crashed"),
       assertions: [
-        { name: "includes", severity: "gate", score: 0, passed: false, detail: "missing text" },
-        softAssertion("judge", 0.2, { passed: false }),
+        { name: "includes", severity: "gate", score: 0, outcome: "failed" as const, detail: "missing text" },
+        softAssertion("judge", 0.2, { outcome: "failed" as const }),
       ],
     });
     expect(reasonFor(result)).toBe("adapter crashed");
@@ -757,21 +959,21 @@ describe("reasonFor", () => {
   it("skipReason 优先于 gate 断言,但让位给 error", () => {
     const result = res("A", "skipped", {
       skipReason: "missing fixture",
-      assertions: [{ name: "includes", severity: "gate", score: 0, passed: false }],
+      assertions: [{ name: "includes", severity: "gate", score: 0, outcome: "failed" as const }],
     });
     expect(reasonFor(result)).toBe("missing fixture");
   });
 
-  it("多个失败 gate 按原始声明顺序全部保留,拼接成一行;失败 soft 不出现在 reason 里", () => {
+  it("只展示第一条失败 gate，其余 gate 计数；失败 soft 不进入 gate 计数", () => {
     const result = res("A", "failed", {
       assertions: [
-        softAssertion("judge-first", 0.1, { passed: false }), // 声明顺序第一,但 soft 不进 reason
-        { name: "includes", severity: "gate", score: 0, passed: false, detail: "missing text" },
-        { name: "matches", severity: "gate", score: 0, passed: false }, // 无 detail,只有 name
-        softAssertion("judge-last", 0.3, { passed: false }), // soft 依旧不进 reason
+        softAssertion("judge-first", 0.1, { outcome: "failed" as const }), // 声明顺序第一,但 soft 不进 reason
+        { name: "includes", groupPath: ["answer contains required text"], severity: "gate", score: 0, outcome: "failed" as const, expected: "contains Brooklyn", received: "Manhattan" },
+        { name: "matches", severity: "gate", score: 0, outcome: "failed" as const }, // 无 detail,只有 name
+        softAssertion("judge-last", 0.3, { outcome: "failed" as const }), // soft 依旧不进 reason
       ],
     });
-    expect(reasonFor(result)).toBe("includes: missing text, matches");
+    expect(reasonFor(result)).toBe("gate: answer contains required text · includes · expected contains Brooklyn · received Manhattan · +1 more failures");
   });
 
   it("都缺席 → 无原因", () => {
@@ -818,6 +1020,22 @@ describe("RunOverview.data", () => {
     const data = await RunOverview.data(selection([s], [warning]));
     expect(data.warnings).toEqual([warning]);
   });
+
+  it("默认成功率把 errored 记 0:2 passed / 5 errored = 2/7,不是 100% 2/7", async () => {
+    const s = snap({
+      experimentId: "exp/unstable",
+      results: [
+        res("pass/a", "passed"),
+        res("pass/b", "passed"),
+        ...Array.from({ length: 5 }, (_, i) => res(`error/${i}`, "errored", { error: erroredWith("boom") })),
+      ],
+    });
+    const data = await RunOverview.data([s]);
+    expect(data.totals.passRate.value).toBeCloseTo(2 / 7, 10);
+    expect(data.totals.passRate.display).toBe("28.6%");
+    expect(data.totals.passRate.samples).toBe(7);
+    expect(data.totals.passRate.total).toBe(7);
+  });
 });
 
 // ───────────────────────── GroupSummary.data ─────────────────────────
@@ -862,16 +1080,6 @@ describe("GroupSummary.data", () => {
     expect(data.lastRunAt).toBe(expB.startedAt);
   });
 
-  it('跨 experiment 同名 eval 不合并:两个 experiment 各自的 "x" 独立计一票,不是被折成一票', async () => {
-    const a = snap({ experimentId: "exp/a", results: [res("x", "passed")] });
-    const b = snap({ experimentId: "exp/b", results: [res("x", "failed")] });
-    const data = await GroupSummary.data([a, b]);
-    expect(data.experiments).toBe(2);
-    expect(data.evals).toBe(2); // 只按 eval id 折叠(误把两个 experiment 的 "x" 当一道题)会变成 1
-    expect(data.verdicts).toEqual({ passed: 1, failed: 1, errored: 0, skipped: 0 });
-    expect(data.passRate.value).toBeCloseTo(0.5, 10); // 误合并会因为「任一轮过即过」变成 100%
-  });
-
   it("全组没有任何 attempt 报成本 → totalCostUSD null,不编 0", async () => {
     const s = snap({ experimentId: "exp/x", results: [res("A", "passed"), res("B", "failed")] });
     const data = await GroupSummary.data([s]);
@@ -888,30 +1096,12 @@ describe("GroupSummary.data", () => {
     expect(data.passRate.total).toBe(2);
   });
 
-  it("lastRunAt 只从传入的组 Selection 取值,不读全局最新 —— 组外快照更晚也不影响", async () => {
-    const inGroup = snap({
-      experimentId: "exp/a",
-      runStartedAt: "2026-07-01T08:00:00Z",
-      results: [res("A", "passed")],
-    });
-    const outOfGroup = snap({
-      experimentId: "exp/b",
-      runStartedAt: "2026-07-09T08:00:00Z",
-      results: [res("B", "passed")],
-    });
-    const sel = selection([inGroup, outOfGroup], []);
-    const scoped = sel.filter((s) => s.experimentId === "exp/a"); // 只留组内快照,模拟按实验组收窄的 Selection
-    const data = await GroupSummary.data(scoped);
-    expect(data.lastRunAt).toBe(inGroup.startedAt);
-    expect(data.lastRunAt).not.toBe(outOfGroup.startedAt);
-  });
-
   it('MetricTable rows: "experiment" 的 verdicts meta 复用同一份 eval 级统计,行为不变(回归)', async () => {
     const s = snap({
       experimentId: "exp/x",
       results: [res("A", "passed", { attempt: 0 }), res("A", "failed", { attempt: 1 }), res("B", "failed")],
     });
-    const data = await MetricTable.data([s], { rows: "experiment", columns: [passRate] });
+    const data = await MetricTable.data([s], { rows: "experiment", columns: [taskPassRate] });
     // A 两轮折成 passed(任一轮过即过),B 失败:1 passed / 1 failed
     expect(data.rows[0].meta?.verdicts).toEqual({ passed: 1, failed: 1, errored: 0, skipped: 0 });
   });
@@ -931,7 +1121,7 @@ describe("DeltaTable.data", () => {
     });
     const data = await DeltaTable.data([base, plus], {
       pairs: [{ a: "exp/base", b: "exp/plus", label: "memory" }],
-      metrics: [passRate, costUSD],
+      metrics: [taskPassRate, costUSD],
     });
     expect(data.rows).toHaveLength(1);
     const row = data.rows[0];
@@ -939,7 +1129,7 @@ describe("DeltaTable.data", () => {
     expect(row.a).toEqual({ experimentId: "exp/base" });
     expect(row.b).toEqual({ experimentId: "exp/plus" });
 
-    const pass = row.cells["pass-rate"];
+    const pass = row.cells["task-pass-rate"];
     expect(pass.a.value).toBe(0.5);
     expect(pass.b.value).toBe(1);
     expect(pass.delta).toBeCloseTo(0.5);
@@ -953,9 +1143,9 @@ describe("DeltaTable.data", () => {
 
     const flat = await DeltaTable.data([base, base], {
       pairs: [{ a: "exp/base", b: "exp/base", label: "same" }],
-      metrics: [passRate],
+      metrics: [taskPassRate],
     });
-    expect(flat.rows[0].cells["pass-rate"].display).toBe("±0");
+    expect(flat.rows[0].cells["task-pass-rate"].display).toBe("±0");
   });
 
   it("时间轴对比:pairs 的 a/b 收快照键 <experimentId> @ <startedAt>", async () => {
@@ -977,9 +1167,9 @@ describe("DeltaTable.data", () => {
           label: "this week vs last",
         },
       ],
-      metrics: [passRate],
+      metrics: [taskPassRate],
     });
-    const cell = data.rows[0].cells["pass-rate"];
+    const cell = data.rows[0].cells["task-pass-rate"];
     expect(cell.a.value).toBe(0); // 旧快照那份
     expect(cell.b.value).toBe(1); // 新快照那份
     expect(cell.display).toBe("+100%");
@@ -1011,8 +1201,8 @@ describe("身份键去重", () => {
       [older, newer],
       [newer, older],
     ]) {
-      const data = await MetricTable.data(order, { rows: "agent", columns: [passRate] });
-      const cell = data.rows[0].cells["pass-rate"];
+      const data = await MetricTable.data(order, { rows: "agent", columns: [taskPassRate] });
+      const cell = data.rows[0].cells["task-pass-rate"];
       expect(cell.total).toBe(1); // 两份只算一份
       expect(cell.value).toBe(1); // 留的是最新 run 里的 passed,与快照传入顺序无关
 
@@ -1078,10 +1268,10 @@ describe("MetricMatrix.data", () => {
       agent: "b1",
       results: [res("A", "failed", { agent: "b1" })], // b1 没跑 B
     });
-    const data = await MetricMatrix.data([a, b], { rows: "eval", columns: "agent", cell: passRate });
+    const data = await MetricMatrix.data([a, b], { rows: "eval", columns: "agent", cell: taskPassRate });
     expect(data.rows).toBe("eval");
     expect(data.columns).toBe("agent");
-    expect(data.metric.key).toBe("pass-rate");
+    expect(data.metric.key).toBe("task-pass-rate");
     expect(data.cells).toHaveLength(3); // A×a1、B×a1、A×b1;B×b1 不出现
     const find = (row: string, column: string) => data.cells.find((c) => c.row === row && c.column === column);
     expect(find("A", "a1")?.cell.value).toBe(1);
@@ -1090,7 +1280,20 @@ describe("MetricMatrix.data", () => {
     expect(find("B", "b1")).toBeUndefined();
   });
 
-  it("MetricBars.data 就是 MetricMatrix.data 的别名(同一个函数)", () => {
-    expect(MetricBars.data).toBe(MetricMatrix.data);
+  it("MetricBars.data 与 MetricMatrix.data 同一输入给出相同数据(别名契约,不锁函数引用)", async () => {
+    const a = snap({
+      experimentId: "exp/a",
+      agent: "a1",
+      results: [res("A", "passed", { agent: "a1" }), res("B", "failed", { agent: "a1" })],
+    });
+    const b = snap({
+      experimentId: "exp/b",
+      agent: "b1",
+      results: [res("A", "failed", { agent: "b1" })],
+    });
+    const options = { rows: "eval", columns: "agent", cell: taskPassRate } as const;
+    const bars = await MetricBars.data([a, b], options);
+    const matrix = await MetricMatrix.data([a, b], options);
+    expect(bars).toEqual(matrix);
   });
 });