niceeval 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/dist/agents/types.d.ts +5 -1
  2. package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
  3. package/dist/report/built-ins/experiment-comparison.js +13 -0
  4. package/dist/report/built-ins/index.d.ts +1 -1
  5. package/dist/report/built-ins/index.js +1 -1
  6. package/dist/report/components.d.ts +3 -1
  7. package/dist/report/components.js +1 -1
  8. package/dist/report/compute.js +15 -33
  9. package/dist/report/format.d.ts +0 -6
  10. package/dist/report/format.js +1 -18
  11. package/dist/report/index.d.ts +2 -2
  12. package/dist/report/index.js +1 -1
  13. package/dist/report/locale.d.ts +13 -2
  14. package/dist/report/locale.js +26 -3
  15. package/dist/report/metrics.js +1 -1
  16. package/dist/report/react/AttemptList.d.ts +2 -2
  17. package/dist/report/react/AttemptList.js +3 -4
  18. package/dist/report/react/ExperimentList.d.ts +3 -2
  19. package/dist/report/react/ExperimentList.js +59 -7
  20. package/dist/report/react/MetricScatter.js +1 -5
  21. package/dist/report/react/fixtures.js +9 -4
  22. package/dist/report/react/format.d.ts +1 -1
  23. package/dist/report/react/format.js +1 -1
  24. package/dist/report/react/index.d.ts +1 -1
  25. package/dist/report/text/faces.js +71 -33
  26. package/dist/report/types.d.ts +3 -15
  27. package/docs-site/zh/concepts/adapter.mdx +2 -2
  28. package/docs-site/zh/concepts/assert.mdx +11 -10
  29. package/docs-site/zh/concepts/evals.mdx +7 -6
  30. package/docs-site/zh/concepts/overview.mdx +1 -1
  31. package/docs-site/zh/guides/agent-feedback-loop.mdx +9 -7
  32. package/docs-site/zh/guides/fixtures.mdx +3 -1
  33. package/docs-site/zh/guides/official-adapters.mdx +1 -1
  34. package/docs-site/zh/guides/publish-report.mdx +1 -1
  35. package/docs-site/zh/guides/report-components.mdx +13 -13
  36. package/docs-site/zh/guides/results-data.mdx +28 -2
  37. package/docs-site/zh/guides/sandbox-agent.mdx +1 -0
  38. package/docs-site/zh/guides/sandbox-providers.mdx +1 -1
  39. package/docs-site/zh/guides/viewing-results.mdx +6 -5
  40. package/docs-site/zh/guides/write-send.mdx +2 -1
  41. package/docs-site/zh/reference/cli.mdx +3 -3
  42. package/docs-site/zh/reference/define-agent.mdx +3 -1
  43. package/docs-site/zh/reference/events.mdx +2 -2
  44. package/docs-site/zh/reference/expect.mdx +10 -5
  45. package/package.json +1 -1
  46. package/src/agents/types.ts +5 -1
  47. package/src/i18n/en.ts +1 -2
  48. package/src/i18n/zh-CN.ts +1 -1
  49. package/src/report/built-in-user-parity.test.tsx +44 -47
  50. package/src/report/built-ins/experiment-comparison.tsx +19 -0
  51. package/src/report/built-ins/index.ts +1 -1
  52. package/src/report/components.tsx +3 -1
  53. package/src/report/compute.ts +12 -36
  54. package/src/report/dual-render.test.tsx +33 -29
  55. package/src/report/format.ts +1 -16
  56. package/src/report/index.ts +1 -2
  57. package/src/report/locale.ts +26 -3
  58. package/src/report/metrics.ts +1 -1
  59. package/src/report/react/AttemptList.tsx +4 -6
  60. package/src/report/react/ExperimentList.tsx +167 -55
  61. package/src/report/react/MetricScatter.tsx +1 -11
  62. package/src/report/react/enhance.js +39 -0
  63. package/src/report/react/fixtures.ts +10 -4
  64. package/src/report/react/format.ts +1 -1
  65. package/src/report/react/index.tsx +0 -1
  66. package/src/report/react/render.test.tsx +24 -8
  67. package/src/report/react/styles.css +40 -13
  68. package/src/report/report.test.ts +2 -7
  69. package/src/report/text/faces.ts +76 -36
  70. package/src/report/types.ts +3 -15
  71. package/src/results/host-equivalence.test.ts +3 -3
  72. package/src/show/index.ts +8 -15
  73. package/src/show/render.ts +7 -204
  74. package/src/show/show.test.ts +22 -23
  75. package/src/view/data.test.ts +1 -1
  76. package/src/view/data.ts +6 -6
  77. package/src/view/index.ts +1 -1
  78. package/src/view/shared/types.ts +1 -1
  79. package/src/view/view-report.test.ts +5 -5
  80. package/dist/o11y/execution-tree.d.ts +0 -103
  81. package/dist/o11y/otlp/select.d.ts +0 -22
  82. package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +0 -1
  83. package/dist/report/built-ins/cost-pass-rate-comparison.js +0 -17
  84. package/dist/results/annotated-source.d.ts +0 -61
  85. package/dist/results/attempt-evidence.d.ts +0 -69
  86. package/dist/results/attempt-source.d.ts +0 -15
  87. package/src/report/built-ins/cost-pass-rate-comparison.tsx +0 -23
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: "标准事件流参考"
3
3
  sidebarTitle: "事件流"
4
- description: "StreamEvent 的九种事件:每种的字段、什么时候吐、哪些断言消费它。adapter 的核心工作就是产出这条流。"
4
+ description: "StreamEvent 的十种事件:每种的字段、什么时候吐、哪些断言消费它。adapter 的核心工作就是产出这条流。"
5
5
  ---
6
6
 
7
7
  adapter 的 `send` 返回一个 `Turn`,其中 `events: StreamEvent[]` 是**断言的唯一数据源**:`t.calledTool()`、`t.reply`、`toolOrder`、`noFailedActions`……全部从这条流上读。把你的 agent"这一轮做了什么"翻成这条流,整套断言就都能用。
@@ -79,7 +79,7 @@ token 用量或 OTel span 反推得到)。存在时优先于按价格表(`define
79
79
 
80
80
  ## StreamEvent 变体一览
81
81
 
82
- `StreamEvent` 的九种变体,逐字段列出(消费它们的断言 / 使用细节见下面的「事件总表」和「逐事件说明」):
82
+ `StreamEvent` 的十种变体,逐字段列出(消费它们的断言 / 使用细节见下面的「事件总表」和「逐事件说明」):
83
83
 
84
84
  {/* GENERATED:BEGIN stream-events */}
85
85
 
@@ -200,13 +200,18 @@ atLeast(threshold: number): ValueAssertion;
200
200
  ## 自定义 matcher
201
201
 
202
202
  ```ts
203
- import { makeAssertion } from "niceeval/expect";
203
+ import { makeAssertion, type Assertion } from "niceeval/expect";
204
204
 
205
- const validEmail = makeAssertion("valid email", (value) => {
206
- return typeof value === "string" && value.includes("@") ? 1 : 0;
207
- });
205
+ function validEmail(): Assertion {
206
+ return makeAssertion({
207
+ name: "valid email",
208
+ score(value) {
209
+ return typeof value === "string" && value.includes("@") ? 1 : 0;
210
+ },
211
+ });
212
+ }
208
213
 
209
214
  t.check(t.reply, validEmail());
210
215
  ```
211
216
 
212
- 自定义 matcher 应返回 0 1 之间的分数,并给出清晰名称。
217
+ `makeAssertion` 接受单个 spec 对象(`name`、可选 `severity`/`threshold`、`score`),直接返回一个 `Assertion`;按惯例用一个同名函数包一层再调用,而不是把 `makeAssertion` 本身当工厂调用。
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "niceeval",
3
- "version": "0.6.0",
3
+ "version": "0.6.1",
4
4
  "description": "Agent-native eval tool — eval agents, services, functions, and coding-agent fixtures",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -221,7 +221,11 @@ export interface AgentContext {
221
221
  * 具体取值)是两个维度——这里只是「跑的是哪个实验」的稳定标识,不携带条件内容。
222
222
  */
223
223
  readonly experimentId?: string;
224
- /** 仅沙箱型 agent 有(运行器按 --sandbox 备好)。 */
224
+ /**
225
+ * 所有 agent 都有:沙箱型是运行器按项目/experiment 配置备好的真实沙箱句柄,remote 型是
226
+ * `createRemoteSandbox()` 产出的 stub(仅含 `workdir`/`sandboxId`/`otlpHost`/`stop` 等
227
+ * 元信息,其余方法调用即抛错)。
228
+ */
225
229
  readonly sandbox: Sandbox;
226
230
  readonly session: AgentSession;
227
231
  /**
package/src/i18n/en.ts CHANGED
@@ -54,8 +54,7 @@ export const en = {
54
54
  " niceeval exp [group|experiment] [eval-id-prefix…] run experiments\n" +
55
55
  " niceeval show [eval-id-prefix… | @<locator>] read results in the terminal\n" +
56
56
  " bare: current verdicts per experiment (composed across runs), each row\n" +
57
- " with a compact attempt index (locator + evidence capability letters\n" +
58
- " [E,X,⏱,D] + failure reason)\n" +
57
+ " with a compact attempt index (locator + failure reason)\n" +
59
58
  " a single eval id: attempts + assertion details\n" +
60
59
  " @<locator> exactly one attempt: no flag -> compact overview;\n" +
61
60
  " with a flag -> that evidence slice\n" +
package/src/i18n/zh-CN.ts CHANGED
@@ -52,7 +52,7 @@ export const zhCN = {
52
52
  " niceeval exp [组|实验] [eval-id 前缀…] 跑实验\n" +
53
53
  " niceeval show [eval-id 前缀… | @<locator>] 终端读结果\n" +
54
54
  " 裸跑:每个 experiment 的现刻判定(跨 run 合成),每行带紧凑 attempt 索引\n" +
55
- " (locator + 证据 capability 字母 [E,X,⏱,D] + 失败原因)\n" +
55
+ " (locator + 失败原因)\n" +
56
56
  " 单个 eval id:attempt 与断言明细\n" +
57
57
  " @<locator> 精确一个 attempt:无 flag → 紧凑全景;带 flag → 对应证据切面\n" +
58
58
  " --eval 该 attempt 运行时保存的 Eval 源码,断言标回源码行\n" +
@@ -1,15 +1,15 @@
1
- // Phase 4 验收:证明内置报告 CostPassRateComparison 与一份普通用户 --report 文件走的是
1
+ // Phase 4 验收:证明内置报告 ExperimentComparison 与一份普通用户 --report 文件走的是
2
2
  // 完全同一套机制 —— 不是长得像,而是同一条 build → resolveReportTree → validate → render 管线。
3
3
  //
4
4
  // 证据链(plan/built-in-reports-user-parity.md「Phase 4」):
5
- // 1. test/fixtures/report/cost-pass-rate-comparison-public-copy.tsx 只从公开 barrel import,
5
+ // 1. test/fixtures/report/experiment-comparison-public-copy.tsx 只从公开 barrel import,
6
6
  // 与 built-in 逐节点同构(见文件头)。
7
7
  // 2. 同一 ReportContext 下分别 build + resolveReportTree,深比较 resolved 树:组件类型 / 顺序 /
8
8
  // resolved data / props keys 全部结构化相等(不靠 definition 引用相等,两份是独立文件)。
9
9
  // 3. 两份分别过 renderReportToText / renderReportToStaticHtml,比较事实(散点键 / 表格行 /
10
10
  // attempt 深链),不做 HTML 字面 snapshot。
11
11
  // 4. 覆盖 plan 的 9 个必测场景。
12
- // 5. 架构不变量:renderer 源码零 "CostPassRateComparison" / "built-ins" 分支。
12
+ // 5. 架构不变量:renderer 源码零 "ExperimentComparison" / "built-ins" 分支。
13
13
  // 6. show / view 只在「无 --report」分支选内置报告,--report 只换 definition,不碰
14
14
  // Selection / resolve / validate / render。
15
15
 
@@ -22,8 +22,8 @@ import type { AttemptHandle, Results, Selection, Snapshot } from "../results/ind
22
22
  import type { ExperimentListItem, ScatterData } from "./types.ts";
23
23
  import { encodeAttemptLocator } from "../results/locator.ts";
24
24
  import { Col, ExperimentList, defineReport } from "./index.ts";
25
- import { CostPassRateComparison } from "./built-ins/index.ts";
26
- import publicCopy from "../../test/fixtures/report/cost-pass-rate-comparison-public-copy.tsx";
25
+ import { ExperimentComparison } from "./built-ins/index.ts";
26
+ import publicCopy from "../../test/fixtures/report/experiment-comparison-public-copy.tsx";
27
27
  import { renderReportToText } from "./report.ts";
28
28
  import { renderReportToStaticHtml } from "./web.ts";
29
29
  import {
@@ -278,15 +278,15 @@ function expectedLocatorFor(attempt: AttemptHandle): string {
278
278
 
279
279
  describe("resolved 树结构化等价(built-in ≡ 包外 public-copy)", () => {
280
280
  it("两份是独立 ReportDefinition,不是同一引用", () => {
281
- expect(CostPassRateComparison).not.toBe(publicCopy);
282
- expect(typeof CostPassRateComparison.build).toBe("function");
281
+ expect(ExperimentComparison).not.toBe(publicCopy);
282
+ expect(typeof ExperimentComparison.build).toBe("function");
283
283
  expect(typeof publicCopy.build).toBe("function");
284
284
  });
285
285
 
286
286
  for (const { name, ctx } of SCENARIOS) {
287
287
  it(`同一 ReportContext 下 resolved 树逐节点相等 — ${name}`, async () => {
288
288
  const c = ctx();
289
- const builtIn = await resolvedTreeOf(CostPassRateComparison, c);
289
+ const builtIn = await resolvedTreeOf(ExperimentComparison, c);
290
290
  const user = await resolvedTreeOf(publicCopy, ctx()); // 各用一份等价 ctx,证明不依赖共享状态
291
291
 
292
292
  // 组件类型与顺序:<Col> 包 [MetricScatter, ExperimentList],别无它物。
@@ -308,7 +308,7 @@ describe("resolved 树结构化等价(built-in ≡ 包外 public-copy)", () => {
308
308
 
309
309
  describe("必测场景:resolved data 事实", () => {
310
310
  it("场景 1+2:多 experiment/agent,成本与通过率都有值;缺成本的点如实 x=null,实验列表仍出项", async () => {
311
- const tree = await resolvedTreeOf(CostPassRateComparison, richContext());
311
+ const tree = await resolvedTreeOf(ExperimentComparison, richContext());
312
312
  const scatter = scatterOf(tree)!;
313
313
  const experiments = experimentListOf(tree)!;
314
314
 
@@ -342,7 +342,7 @@ describe("必测场景:resolved data 事实", () => {
342
342
  });
343
343
 
344
344
  it("场景 3:恰好一个可画点(另一实验缺成本)", async () => {
345
- const tree = await resolvedTreeOf(CostPassRateComparison, oneDrawableContext());
345
+ const tree = await resolvedTreeOf(ExperimentComparison, oneDrawableContext());
346
346
  const scatter = scatterOf(tree)!;
347
347
  const drawable = scatter.rows.filter((r) => r.x.value !== null && r.y.value !== null);
348
348
  expect(drawable).toHaveLength(1);
@@ -350,20 +350,20 @@ describe("必测场景:resolved data 事实", () => {
350
350
  });
351
351
 
352
352
  it("场景 4:空 Selection —— 散点无 rows、实验列表为空数组", async () => {
353
- const tree = await resolvedTreeOf(CostPassRateComparison, emptyContext());
353
+ const tree = await resolvedTreeOf(ExperimentComparison, emptyContext());
354
354
  expect(scatterOf(tree)!.rows).toEqual([]);
355
355
  expect(experimentListOf(tree)).toEqual([]);
356
356
  });
357
357
 
358
358
  it("场景 4 变体:全 skipped —— 每个点 x/y 都不可测,0 可画点", async () => {
359
- const tree = await resolvedTreeOf(CostPassRateComparison, allSkippedContext());
359
+ const tree = await resolvedTreeOf(ExperimentComparison, allSkippedContext());
360
360
  const scatter = scatterOf(tree)!;
361
361
  expect(scatter.rows.every((r) => r.x.value === null && r.y.value === null)).toBe(true);
362
362
  expect(scatter.rows.filter((r) => r.x.value !== null && r.y.value !== null)).toHaveLength(0);
363
363
  });
364
364
 
365
365
  it("场景 5:failed/errored/skipped 混合 —— ExperimentList 展开区结果摘要正确(eval 级折叠计票)", async () => {
366
- const tree = await resolvedTreeOf(CostPassRateComparison, richContext());
366
+ const tree = await resolvedTreeOf(ExperimentComparison, richContext());
367
367
  const codex = experimentListOf(tree)!.find((e) => e.experimentId === "compare/codex-mid")!;
368
368
  // algebra/x errored、algebra/y skipped、algebra/z passed → eval 级折叠 {passed:1, errored:1, skipped:1}
369
369
  expect(codex.verdicts).toEqual({ passed: 1, failed: 0, errored: 1, skipped: 1 });
@@ -374,7 +374,7 @@ describe("必测场景:resolved data 事实", () => {
374
374
  });
375
375
 
376
376
  it("场景 6:同 experiment 多 eval、多 attempts —— 展开层与 attempt locator 正确", async () => {
377
- const tree = await resolvedTreeOf(CostPassRateComparison, richContext());
377
+ const tree = await resolvedTreeOf(ExperimentComparison, richContext());
378
378
  const bub = experimentListOf(tree)!.find((e) => e.experimentId === "compare/bub-low")!;
379
379
  expect(bub.evalRows.map((e) => e.evalId).sort()).toEqual(["algebra/x", "algebra/y"]);
380
380
  const multi = bub.evalRows.find((e) => e.evalId === "algebra/x")!;
@@ -394,12 +394,12 @@ describe("必测场景:resolved data 事实", () => {
394
394
  describe("built-in 与 public-copy 渲染出的事实相同", () => {
395
395
  for (const { name, ctx } of SCENARIOS) {
396
396
  it(`text / web 事实一致 — ${name}`, async () => {
397
- const builtinText = await renderReportToText(CostPassRateComparison, ctx(), { width: 100 });
397
+ const builtinText = await renderReportToText(ExperimentComparison, ctx(), { width: 100 });
398
398
  const userText = await renderReportToText(publicCopy, ctx(), { width: 100 });
399
399
  // 同一机制、同一 options:text 输出逐字相等(最强的事实级证据)。
400
400
  expect(userText).toBe(builtinText);
401
401
 
402
- const builtinHtml = await renderReportToStaticHtml(CostPassRateComparison, ctx());
402
+ const builtinHtml = await renderReportToStaticHtml(ExperimentComparison, ctx());
403
403
  const userHtml = await renderReportToStaticHtml(publicCopy, ctx());
404
404
  // 事实比较(不做整段 HTML snapshot):散点点键、attempt 深链集合相同。
405
405
  expect(scatterKeysOf(userHtml)).toEqual(scatterKeysOf(builtinHtml));
@@ -411,7 +411,7 @@ describe("built-in 与 public-copy 渲染出的事实相同", () => {
411
411
  const ctx = richContext();
412
412
  // renderReportToStaticHtml 默认 attemptHref = view 的 `#/attempt/@<locator>`(不透明单段路由,
413
413
  // 与 view/data.ts renderReportSlot 不传 attemptHref 时同一条默认路径)。
414
- const builtinHtml = await renderReportToStaticHtml(CostPassRateComparison, ctx);
414
+ const builtinHtml = await renderReportToStaticHtml(ExperimentComparison, ctx);
415
415
  const userHtml = await renderReportToStaticHtml(publicCopy, richContext());
416
416
  const builtinHrefs = hrefsOf(builtinHtml);
417
417
  const userHrefs = hrefsOf(userHtml);
@@ -437,31 +437,28 @@ describe("built-in 与 public-copy 渲染出的事实相同", () => {
437
437
 
438
438
  describe("散点空态 / 单点 / 多点行为", () => {
439
439
  it("0 可画点(空 Selection):text 说 No data、web 走 nre-scatter-empty nre-missing", async () => {
440
- const text = await renderReportToText(CostPassRateComparison, emptyContext(), { width: 100 });
440
+ const text = await renderReportToText(ExperimentComparison, emptyContext(), { width: 100 });
441
441
  expect(text).toContain("No data to plot");
442
442
  expect(text).not.toContain("better → upper right");
443
- const html = await renderReportToStaticHtml(CostPassRateComparison, emptyContext());
443
+ const html = await renderReportToStaticHtml(ExperimentComparison, emptyContext());
444
444
  expect(html).toContain("nre-scatter-empty");
445
445
  expect(html).toContain("nre-missing");
446
446
  });
447
447
 
448
- it("恰好 1 可画点:text/web 都说「至少 2 个实验」而不是画孤点", async () => {
449
- const text = await renderReportToText(CostPassRateComparison, oneDrawableContext(), { width: 100 });
450
- expect(text).toContain("At least 2 experiments needed to compare");
451
- expect(text).not.toContain("better upper right");
452
- const html = await renderReportToStaticHtml(CostPassRateComparison, oneDrawableContext());
453
- expect(html).toContain("nre-scatter-empty");
454
- // 单点不是「缺数据」空态:散点空态本身不带 nre-missing 变体类(与 0 可画点的
455
- // "nre-scatter-empty nre-missing" 区分开)。ExperimentList 展开区自己的缺成本格子
456
- // 也合法渲染 nre-missing,不属于这条散点专属的断言范围。
448
+ it("恰好 1 可画点:text/web 都正常画单点", async () => {
449
+ const text = await renderReportToText(ExperimentComparison, oneDrawableContext(), { width: 100 });
450
+ expect(text).toContain("better upper right");
451
+ const html = await renderReportToStaticHtml(ExperimentComparison, oneDrawableContext());
452
+ expect(html).toContain("nre-scatter-point");
453
+ expect(html).not.toContain("nre-scatter-empty");
457
454
  expect(html).not.toContain("nre-scatter-empty nre-missing");
458
- expect(html).not.toContain("<svg");
455
+ expect(html).toContain("<svg");
459
456
  });
460
457
 
461
458
  it("≥2 可画点:正常绘图(text 有 better → upper right,web 有 svg)", async () => {
462
- const text = await renderReportToText(CostPassRateComparison, richContext(), { width: 100 });
459
+ const text = await renderReportToText(ExperimentComparison, richContext(), { width: 100 });
463
460
  expect(text).toContain("better → upper right");
464
- const html = await renderReportToStaticHtml(CostPassRateComparison, richContext());
461
+ const html = await renderReportToStaticHtml(ExperimentComparison, richContext());
465
462
  expect(html).toContain("<svg");
466
463
  expect(html).not.toContain("nre-scatter-empty");
467
464
  });
@@ -472,7 +469,7 @@ describe("散点空态 / 单点 / 多点行为", () => {
472
469
  describe("场景 7:en / zh-CN 数据 resolve 一次、chrome 分别本地化", () => {
473
470
  it("resolve 一次的同一棵树,en / zh-CN 各渲染一遍:data 事实相同,只有 chrome 文案不同", async () => {
474
471
  // 关键:build + resolveReportTree 只做一次,拿到唯一的 resolved 树;两个 locale 复用它。
475
- const node = (await CostPassRateComparison.build(richContext())) as ReportNode;
472
+ const node = (await ExperimentComparison.build(richContext())) as ReportNode;
476
473
  const resolved = await resolveReportTree(node);
477
474
  validateReportTree(resolved);
478
475
 
@@ -518,7 +515,7 @@ describe("场景 7:en / zh-CN 数据 resolve 一次、chrome 分别本地化", (
518
515
  // 注意:spy 挂在 ExperimentList.data 本身,不是 compute.experimentListData —— components.tsx
519
516
  // 用 `Object.assign(component, { data: experimentListData })` 把函数值复制成一个普通属性,
520
517
  // 那份拷贝在 components.tsx 求值时(模块加载期)就定死了,晚于此的 vi.spyOn(compute, …) патch
521
- // 不到已经复制走的引用。直接调用方(built-ins/cost-pass-rate-comparison.tsx)读的是
518
+ // 不到已经复制走的引用。直接调用方(built-ins/experiment-comparison.tsx)读的是
522
519
  // `ExperimentList.data` 这个属性,所以要在这个属性上打桩。MetricScatter 不受影响:它的
523
520
  // resolve() 住在 components.tsx 里,调用的是 scatterData 这个 import 绑定本身(ESM 实时绑定),
524
521
  // vi.spyOn(compute, "scatterData") 打得中。
@@ -526,7 +523,7 @@ describe("场景 7:en / zh-CN 数据 resolve 一次、chrome 分别本地化", (
526
523
  try {
527
524
  // ExperimentList 没有 resolve 面,.data() 直接在 build() 里 await——计算发生在 build,
528
525
  // 不是 resolveReportTree;MetricScatter 仍是 selection-form,resolve 阶段才调它的 .data()。
529
- const node = (await CostPassRateComparison.build(richContext())) as ReportNode;
526
+ const node = (await ExperimentComparison.build(richContext())) as ReportNode;
530
527
  expect(experimentListSpy).toHaveBeenCalledTimes(1);
531
528
  const resolved = await resolveReportTree(node);
532
529
  expect(scatterSpy).toHaveBeenCalledTimes(1);
@@ -603,38 +600,38 @@ describe("场景 9:最小用户报告只放 <ExperimentList> —— 不计算散
603
600
  describe("架构不变量:renderer 不认识 built-in", () => {
604
601
  const RENDERER_FILES = ["./report.ts", "./web.ts", "./tree.ts"];
605
602
  for (const rel of RENDERER_FILES) {
606
- it(`${rel} 源码里没有 "CostPassRateComparison" / "built-ins" 分支`, () => {
603
+ it(`${rel} 源码里没有 "ExperimentComparison" / "built-ins" 分支`, () => {
607
604
  const src = readFileSync(new URL(rel, import.meta.url), "utf8");
608
- expect(src).not.toContain("CostPassRateComparison");
605
+ expect(src).not.toContain("ExperimentComparison");
609
606
  expect(src).not.toContain("built-ins");
610
607
  });
611
608
  }
612
609
  });
613
610
 
614
- // ═════════════════════════ 8. 宿主选择:show 索引与可替换报告槽分开 ═════════════════════════
611
+ // ═════════════════════════ 8. 宿主选择:show / view 共用默认 definition ═════════════════════════
615
612
 
616
- describe("宿主选择:裸 show 是诊断索引,--report view 走报告 definition", () => {
617
- it("show / view 源码:裸 show 选择专用索引,--report 才装载 definition", () => {
613
+ describe("宿主选择:裸 show / view 共用 ExperimentComparison,--report 替换 definition", () => {
614
+ it("show / view 源码:Selection 先合成,随后选择内置或用户 definition", () => {
618
615
  const showSrc = readFileSync(new URL("../show/index.ts", import.meta.url), "utf8");
619
- expect(showSrc).toMatch(/flags\.report === undefined[\s\S]*showIndexText\(selection, io\.width\)/);
620
- expect(showSrc).toMatch(/const definition = await loadReportFile\(cwd, flags\.report\)/);
621
- expect(showSrc).not.toMatch(/import \{ CostPassRateComparison \}/);
616
+ expect(showSrc).toMatch(/import \{ ExperimentComparison \} from "\.\.\/\.\.\/dist\/report\/built-ins\/index\.js"/);
617
+ expect(showSrc).toMatch(/flags\.report === undefined \? ExperimentComparison : await loadReportFile\(cwd, flags\.report\)/);
618
+ expect(showSrc).not.toContain("showIndexText(selection");
622
619
  // Selection 在报告槽之前、与 report 无关地由 selectCurrentResults 合成。
623
620
  expect(showSrc).toMatch(/selectCurrentResults\(results, \{ experiment: flags\.experiment, patterns \}\)/);
624
621
 
625
622
  const viewSrc = readFileSync(new URL("../view/data.ts", import.meta.url), "utf8");
626
- // view 的报告槽:report 有值走 loadReportFile,否则 CostPassRateComparison;Selection 同样先于报告槽合成。
627
- expect(viewSrc).toMatch(/report\s*\?[\s\S]*loadReportFile[\s\S]*:\s*\(await import\([^)]*built-ins[^)]*\)\)\.CostPassRateComparison/);
623
+ // view 的报告槽:report 有值走 loadReportFile,否则 ExperimentComparison;Selection 同样先于报告槽合成。
624
+ expect(viewSrc).toMatch(/report\s*\?[\s\S]*loadReportFile[\s\S]*:\s*\(await import\([^)]*built-ins[^)]*\)\)\.ExperimentComparison/);
628
625
  expect(viewSrc).toMatch(/selectCurrentResults\(results, \{ experiment: opts\.experiment, patterns \}\)/);
629
626
  });
630
627
 
631
- it("CostPassRateComparison 与 public-copy 作为显式报告时事实完全相同", async () => {
628
+ it("ExperimentComparison 与 public-copy 作为显式报告时事实完全相同", async () => {
632
629
  for (const { ctx } of SCENARIOS) {
633
- const bareText = await renderReportToText(CostPassRateComparison, ctx(), { width: 100 });
630
+ const bareText = await renderReportToText(ExperimentComparison, ctx(), { width: 100 });
634
631
  const reportText = await renderReportToText(publicCopy, ctx(), { width: 100 });
635
632
  expect(reportText).toBe(bareText);
636
633
 
637
- const bareHtml = await renderReportToStaticHtml(CostPassRateComparison, ctx());
634
+ const bareHtml = await renderReportToStaticHtml(ExperimentComparison, ctx());
638
635
  const reportHtml = await renderReportToStaticHtml(publicCopy, ctx());
639
636
  expect(scatterKeysOf(reportHtml)).toEqual(scatterKeysOf(bareHtml));
640
637
  expect(hrefsOf(reportHtml)).toEqual(hrefsOf(bareHtml));
@@ -0,0 +1,19 @@
1
+ // ExperimentComparison:niceeval view 裸跑时的内置 experiment 比较报告。
2
+ //
3
+ // 名字描述整份报告回答的问题,不绑定其中某一张图。报告由成本 × 通过率散点和固定列
4
+ // ExperimentList 组成;它是一份普通 ReportDefinition,没有 renderer 私有通道。
5
+
6
+ import { Col } from "../primitives.tsx";
7
+ import { ExperimentList, MetricScatter } from "../components.tsx";
8
+ import { costUSD, passRate } from "../metrics.ts";
9
+ import { defineReport } from "../report.ts";
10
+
11
+ export const ExperimentComparison = defineReport(async ({ selection }) => {
12
+ const experiments = await ExperimentList.data(selection);
13
+ return (
14
+ <Col>
15
+ <MetricScatter selection={selection} points="experiment" series="agent" x={costUSD} y={passRate} />
16
+ <ExperimentList items={experiments} filter />
17
+ </Col>
18
+ );
19
+ });
@@ -1,2 +1,2 @@
1
1
  // 内置报告目录:一个文件一个按展示内容命名的普通 ReportDefinition,显式导出值,不建字符串 registry。
2
- export { CostPassRateComparison } from "./cost-pass-rate-comparison.tsx";
2
+ export { ExperimentComparison } from "./experiment-comparison.tsx";
@@ -90,6 +90,8 @@ export interface GroupSummaryProps {
90
90
  */
91
91
  export interface ExperimentListProps {
92
92
  items: ExperimentListItem[];
93
+ /** web 面在比较表前显示实验过滤框;text 面忽略。 */
94
+ filter?: boolean;
93
95
  /** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
94
96
  locale?: ReportLocale;
95
97
  className?: string;
@@ -254,7 +256,7 @@ EvalList.displayName = "EvalList";
254
256
 
255
257
  /**
256
258
  * Attempt 列表:实体列表的叶子层,每项一个 Attempt,固定展示判定、断言、error、Judge 评语
257
- * (assertions 的 detail/evidence)与证据引用(locator + 证据能力标记)。它不预设只看失败;
259
+ * (assertions 的 detail/evidence)与证据引用(locator)。它不预设只看失败;
258
260
  * 报告作者过滤 `AttemptListItem[]`、用 `.slice()` 限量,`total` 让渲染面如实报告剩余数量。
259
261
  */
260
262
  export const AttemptList: ReportComponent<AttemptListProps> & { data: typeof attemptListData } = Object.assign(
@@ -11,7 +11,6 @@
11
11
  // - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
12
12
 
13
13
  import type {
14
- AttemptEvidenceCapabilities,
15
14
  AttemptListItem,
16
15
  AttemptLocator,
17
16
  DeltaData,
@@ -248,34 +247,6 @@ export async function tableData<const M extends readonly Metric[]>(
248
247
  // (docs/feature/reports/library.md「实体列表」)。AttemptListItem 是三者共用的叶子形状——
249
248
  // ExperimentList / EvalList 的下钻数组直接复用它,不各自精简一份。
250
249
 
251
- /**
252
- * 一个 attempt 的证据能力标记,只读已有的瘦身摘要位(`hasSources` / `hasEvents` / `hasTrace`)
253
- * 加一次 `diff()` 懒加载——不调用完整的 `loadAttemptEvidence`(那还会额外装配 Eval 源码标注
254
- * 与 ExecutionTree,这里只要四个布尔位)。三位定义与 `AttemptEvidence.capabilities` 完全一致:
255
- *
256
- * - `eval`:`hasSources` 为真。真正的门槛是 `AnnotatedEvalSource` 非空且有内容(源码行或断言
257
- * 非空),但 sources 存在时 `buildAnnotatedEvalSource` 恒产出至少一行源码,断言又永远来自
258
- * `result.assertions`(不需要 IO)——`hasSources` 是这个条件的精确瘦身版,不是近似。
259
- * - `execution`:`hasEvents` 为真,与 `loadAttemptEvidence` 的 `events !== null && events.length > 0`
260
- * 同一个判定,写入面按同一条规则算的 `hasEvents` 就是它的瘦身镜像。
261
- * - `timing`:`hasEvents && hasTrace` 为真——`execution` 成立且这次运行接入过 OTel
262
- * (`ExecutionTree.timingAvailable` = `spans.length > 0`,`hasTrace` 正是这个判定的瘦身镜像)。
263
- * - `diff`:唯一需要 IO 的一位,`attempt.diff()` 非空且至少改动/删除了一个文件——落盘没有
264
- * 与 `hasEvents` 对应的瘦身摘要位,只能懒加载判定;成本是每个 item 一次 `diff()` 读取,
265
- * 不是完整证据装配的四路 Promise.all。
266
- */
267
- async function attemptCapabilities(attempt: AttemptHandle): Promise<AttemptEvidenceCapabilities> {
268
- const result = attempt.result;
269
- const diff = await attempt.diff();
270
- const diffCapable = diff !== null && (Object.keys(diff.generatedFiles).length > 0 || diff.deletedFiles.length > 0);
271
- return {
272
- eval: result.hasSources === true,
273
- execution: result.hasEvents === true,
274
- timing: result.hasEvents === true && result.hasTrace === true,
275
- diff: diffCapable,
276
- };
277
- }
278
-
279
250
  /** 自由文本(error / 断言 detail / evidence)的发布消毒钩子;身份字段(name/severity/loc)不经它。 */
280
251
  function redactAssertions(assertions: AssertionResult[], redact: (text: string) => string): AssertionResult[] {
281
252
  if (assertions.length === 0) return assertions;
@@ -287,7 +258,7 @@ function redactAssertions(assertions: AssertionResult[], redact: (text: string)
287
258
  }
288
259
 
289
260
  /** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
290
- async function attemptListItemOf(item: Item, redact: (text: string) => string): Promise<AttemptListItem> {
261
+ function attemptListItemOf(item: Item, redact: (text: string) => string): AttemptListItem {
291
262
  const result = item.attempt.result;
292
263
  const cost = attemptCostUSD(result);
293
264
  return {
@@ -301,7 +272,6 @@ async function attemptListItemOf(item: Item, redact: (text: string) => string):
301
272
  durationMs: result.durationMs,
302
273
  ...(cost !== null ? { costUSD: cost } : {}),
303
274
  locator: locatorOf(item),
304
- capabilities: await attemptCapabilities(item.attempt),
305
275
  };
306
276
  }
307
277
 
@@ -320,7 +290,7 @@ export async function attemptListData(
320
290
  const { snapshots } = resolveInput(input);
321
291
  const redact = opts?.redact ?? identityRedact;
322
292
  const items = collectItems(snapshots);
323
- return Promise.all(items.map((item) => attemptListItemOf(item, redact)));
293
+ return items.map((item) => attemptListItemOf(item, redact));
324
294
  }
325
295
 
326
296
  /** `EvalList.data(selection)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
@@ -339,7 +309,7 @@ export async function evalListData(input: SnapshotsInput): Promise<EvalListItem[
339
309
  const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
340
310
  const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
341
311
  const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
342
- const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item, identityRedact)));
312
+ const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
343
313
  out.push({
344
314
  evalId: evalIdOf(sorted[0]!),
345
315
  experimentId: experimentIdOf(sorted[0]!),
@@ -370,7 +340,7 @@ export async function experimentListData(input: SnapshotsInput): Promise<Experim
370
340
  const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
371
341
  const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
372
342
  const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
373
- const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item, identityRedact)));
343
+ const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
374
344
  evalRows.push({
375
345
  evalId,
376
346
  verdict,
@@ -400,7 +370,14 @@ export async function experimentListData(input: SnapshotsInput): Promise<Experim
400
370
  evalRows,
401
371
  });
402
372
  }
403
- out.sort((a, b) => a.experimentId.localeCompare(b.experimentId));
373
+ // ExperimentList 是默认实验比较表:初始态按成功率从高到低,缺数据沉底;
374
+ // 同分时按 experiment id 稳定排序。web 增强可临时重排,text 面沿用同一基准顺序。
375
+ out.sort((a, b) => {
376
+ if (a.passRate.value === null && b.passRate.value === null) return a.experimentId.localeCompare(b.experimentId);
377
+ if (a.passRate.value === null) return 1;
378
+ if (b.passRate.value === null) return -1;
379
+ return b.passRate.value - a.passRate.value || a.experimentId.localeCompare(b.experimentId);
380
+ });
404
381
  return out;
405
382
  }
406
383
 
@@ -772,4 +749,3 @@ function deltaDisplay(metric: Metric, delta: number | null): string {
772
749
  const text = displayValue(metric, delta); // 负号由格式化自带
773
750
  return delta > 0 ? `+${text}` : text;
774
751
  }
775
-