niceeval 0.4.5 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/docs-site/zh/concepts/adapter.mdx +6 -6
  2. package/docs-site/zh/concepts/experiment.mdx +7 -7
  3. package/docs-site/zh/concepts/overview.mdx +1 -1
  4. package/docs-site/zh/concepts/tier.mdx +6 -6
  5. package/docs-site/zh/guides/agent-feedback-loop.mdx +63 -20
  6. package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
  7. package/docs-site/zh/guides/custom-reports.mdx +243 -0
  8. package/docs-site/zh/guides/experiments.mdx +3 -3
  9. package/docs-site/zh/guides/report-components.mdx +252 -0
  10. package/docs-site/zh/guides/reporters.mdx +1 -1
  11. package/docs-site/zh/guides/results-data.mdx +224 -0
  12. package/docs-site/zh/guides/runner.mdx +1 -1
  13. package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
  14. package/docs-site/zh/guides/viewing-results.mdx +147 -77
  15. package/docs-site/zh/guides/write-experiment.mdx +9 -9
  16. package/docs-site/zh/guides/write-send.mdx +6 -6
  17. package/docs-site/zh/index.mdx +1 -1
  18. package/docs-site/zh/introduction.mdx +1 -1
  19. package/docs-site/zh/reference/cli.mdx +10 -2
  20. package/docs-site/zh/reference/define-agent.mdx +5 -5
  21. package/docs-site/zh/reference/define-config.mdx +1 -1
  22. package/docs-site/zh/reference/define-eval.mdx +3 -3
  23. package/package.json +3 -2
  24. package/src/agents/ai-sdk.test.ts +1 -1
  25. package/src/agents/ai-sdk.ts +2 -2
  26. package/src/agents/bub.ts +14 -4
  27. package/src/agents/claude-code.ts +1 -1
  28. package/src/agents/codex.ts +2 -2
  29. package/src/agents/streaming.test.ts +1 -1
  30. package/src/agents/types.ts +4 -4
  31. package/src/agents/ui-message-stream.test.ts +1 -1
  32. package/src/cli.ts +104 -14
  33. package/src/context/context.test.ts +1 -1
  34. package/src/context/context.ts +3 -3
  35. package/src/context/session.ts +2 -2
  36. package/src/context/types.ts +2 -2
  37. package/src/i18n/en.ts +29 -7
  38. package/src/i18n/zh-CN.ts +24 -2
  39. package/src/report/aggregate.ts +79 -50
  40. package/src/report/components.tsx +190 -0
  41. package/src/report/compute.ts +149 -84
  42. package/src/report/default-report.tsx +222 -0
  43. package/src/report/dual-face.test.tsx +527 -0
  44. package/src/report/format.ts +29 -0
  45. package/src/report/index.ts +79 -24
  46. package/src/report/load.ts +67 -0
  47. package/src/report/metrics.ts +2 -1
  48. package/src/report/param.ts +18 -0
  49. package/src/report/primitives.tsx +117 -0
  50. package/src/report/react/CaseList.tsx +2 -2
  51. package/src/report/react/DeltaTable.tsx +2 -2
  52. package/src/report/react/MetricBars.tsx +109 -0
  53. package/src/report/react/MetricLine.tsx +200 -0
  54. package/src/report/react/MetricMatrix.tsx +1 -1
  55. package/src/report/react/MetricScatter.tsx +1 -1
  56. package/src/report/react/MetricTable.tsx +1 -1
  57. package/src/report/react/RunOverview.tsx +4 -4
  58. package/src/report/react/Scoreboard.tsx +3 -3
  59. package/src/report/react/cell.tsx +1 -1
  60. package/src/report/react/fixtures.ts +98 -36
  61. package/src/report/react/format.ts +3 -28
  62. package/src/report/react/index.tsx +41 -24
  63. package/src/report/react/render.test.tsx +8 -8
  64. package/src/report/react/styles.css +77 -0
  65. package/src/report/report.test.ts +248 -128
  66. package/src/report/report.ts +69 -0
  67. package/src/report/text/faces.ts +356 -0
  68. package/src/report/text/layout.ts +125 -0
  69. package/src/report/text/plot.ts +127 -0
  70. package/src/report/tree.ts +220 -0
  71. package/src/report/types.ts +75 -27
  72. package/src/report/web.ts +40 -0
  73. package/src/results/copy.ts +95 -41
  74. package/src/results/format.ts +12 -11
  75. package/src/results/index.ts +34 -17
  76. package/src/results/open.ts +201 -81
  77. package/src/results/results.test.ts +595 -191
  78. package/src/results/select.ts +107 -55
  79. package/src/results/types.ts +137 -45
  80. package/src/results/writer.ts +258 -0
  81. package/src/runner/attempt.ts +3 -3
  82. package/src/runner/fingerprint.ts +1 -1
  83. package/src/runner/reporters/artifacts.ts +38 -78
  84. package/src/runner/reporters/braintrust.test.ts +2 -2
  85. package/src/runner/reporters/braintrust.ts +3 -3
  86. package/src/runner/run.ts +1 -1
  87. package/src/runner/types.ts +18 -12
  88. package/src/show/compose.ts +215 -0
  89. package/src/show/index.ts +263 -0
  90. package/src/show/render.ts +456 -0
  91. package/src/show/show.test.ts +505 -0
  92. package/src/view/app/App.tsx +85 -24
  93. package/src/view/app/components/CostScoreChart.tsx +29 -10
  94. package/src/view/app/components/ExperimentTable.tsx +27 -9
  95. package/src/view/app/components/GroupSelector.tsx +1 -1
  96. package/src/view/app/i18n.ts +18 -3
  97. package/src/view/app/lib/attempt-route.test.ts +9 -9
  98. package/src/view/app/lib/attempt-route.ts +5 -5
  99. package/src/view/app/lib/outcome.ts +1 -3
  100. package/src/view/app/lib/rows.ts +95 -15
  101. package/src/view/app/main.tsx +18 -6
  102. package/src/view/app/pages/RunsPage.tsx +4 -7
  103. package/src/view/app/pages/TracesPage.tsx +3 -8
  104. package/src/view/app/types.ts +50 -6
  105. package/src/view/client-dist/app.css +1 -1
  106. package/src/view/client-dist/app.js +20 -20
  107. package/src/view/data.test.ts +254 -0
  108. package/src/view/data.ts +336 -0
  109. package/src/view/index.ts +64 -10
  110. package/src/view/server.ts +31 -9
  111. package/src/view/shared/types.ts +47 -41
  112. package/src/view/styles.css +5 -0
  113. package/src/view/template.html +1 -0
  114. package/src/view/view-report.test.ts +308 -0
  115. package/docs-site/.mintignore +0 -13
  116. package/docs-site/AGENTS.md +0 -46
  117. package/docs-site/concepts/adapter.mdx +0 -287
  118. package/docs-site/concepts/assert.mdx +0 -243
  119. package/docs-site/concepts/drive.mdx +0 -118
  120. package/docs-site/concepts/evals.mdx +0 -108
  121. package/docs-site/concepts/experiment.mdx +0 -37
  122. package/docs-site/concepts/hitl.mdx +0 -83
  123. package/docs-site/concepts/judge.mdx +0 -129
  124. package/docs-site/concepts/overview.mdx +0 -98
  125. package/docs-site/concepts/tier.mdx +0 -42
  126. package/docs-site/docs-ref/00-index.md +0 -23
  127. package/docs-site/docs-ref/01-page-types.md +0 -43
  128. package/docs-site/docs-ref/03-style-rules.md +0 -45
  129. package/docs-site/docs-ref/04-code-examples.md +0 -37
  130. package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
  131. package/docs-site/docs-ref/06-checklists.md +0 -53
  132. package/docs-site/docs.json +0 -256
  133. package/docs-site/example/ai-agent-application.mdx +0 -148
  134. package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
  135. package/docs-site/example/claude-code-codex-skill.mdx +0 -147
  136. package/docs-site/example/showcase.mdx +0 -39
  137. package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
  138. package/docs-site/example/tier1-claude-sdk.mdx +0 -119
  139. package/docs-site/example/tier1-codex-sdk.mdx +0 -111
  140. package/docs-site/example/tier1-langgraph.mdx +0 -119
  141. package/docs-site/example/tier1-pi-sdk.mdx +0 -119
  142. package/docs-site/favicon.svg +0 -5
  143. package/docs-site/github-diff.css +0 -135
  144. package/docs-site/github-diff.js +0 -11
  145. package/docs-site/guides/authoring.mdx +0 -129
  146. package/docs-site/guides/ci-integration.mdx +0 -97
  147. package/docs-site/guides/connect-otel.mdx +0 -209
  148. package/docs-site/guides/connect-your-agent.mdx +0 -221
  149. package/docs-site/guides/dataset-fanout.mdx +0 -98
  150. package/docs-site/guides/experiments.mdx +0 -71
  151. package/docs-site/guides/fixtures.mdx +0 -147
  152. package/docs-site/guides/reporters.mdx +0 -113
  153. package/docs-site/guides/runner.mdx +0 -79
  154. package/docs-site/guides/sandbox-agent.mdx +0 -138
  155. package/docs-site/guides/sandbox-backends.mdx +0 -64
  156. package/docs-site/guides/scoring-guide.mdx +0 -92
  157. package/docs-site/guides/viewing-results.mdx +0 -195
  158. package/docs-site/guides/write-experiment.mdx +0 -81
  159. package/docs-site/guides/write-send.mdx +0 -347
  160. package/docs-site/index.mdx +0 -181
  161. package/docs-site/introduction.mdx +0 -141
  162. package/docs-site/quickstart.mdx +0 -136
  163. package/docs-site/reference/builtin-agents.mdx +0 -161
  164. package/docs-site/reference/capabilities.mdx +0 -76
  165. package/docs-site/reference/cli.mdx +0 -120
  166. package/docs-site/reference/define-agent.mdx +0 -168
  167. package/docs-site/reference/define-config.mdx +0 -41
  168. package/docs-site/reference/define-eval.mdx +0 -160
  169. package/docs-site/reference/events.mdx +0 -131
  170. package/docs-site/reference/expect.mdx +0 -112
  171. package/docs-site/tracker.js +0 -8
  172. package/src/report/react/data.ts +0 -17
  173. package/src/view/aggregate.ts +0 -103
  174. package/src/view/loader.test.ts +0 -61
  175. package/src/view/loader.ts +0 -257
@@ -0,0 +1,527 @@
1
+ // 双面验收:每个官方组件对同一份数据,web 面(renderToStaticMarkup)与 text 面
2
+ // 给出一致判读 —— 排序方向随 better、samples < total 角标、缺数据 — 不补 0、
3
+ // 截断如实报剩余;text 面形态以内联快照锁定(照 report-components.mdx 的示例形态)。
4
+ // 另验收:排版原语的分栏与降级、渲染前树校验、defineComponent 自定义组件、
5
+ // defineReport + renderReportToText / renderReportToStaticHtml 两个宿主入口。
6
+
7
+ import { renderToStaticMarkup } from "react-dom/server";
8
+ import { describe, expect, it } from "vitest";
9
+
10
+ import type { Results, Selection, Snapshot } from "../results/index.ts";
11
+ import {
12
+ CaseList,
13
+ Col,
14
+ DefaultReport,
15
+ DeltaTable,
16
+ MetricBars,
17
+ MetricLine,
18
+ MetricMatrix,
19
+ MetricScatter,
20
+ MetricTable,
21
+ Row,
22
+ RunOverview,
23
+ Scoreboard,
24
+ Section,
25
+ Style,
26
+ Text,
27
+ defineComponent,
28
+ defineReport,
29
+ isReportDefinition,
30
+ renderReportToText,
31
+ } from "./index.ts";
32
+ import { renderReportToStaticHtml } from "./web.ts";
33
+ import { createTextContext, renderNodeToText, validateReportTree } from "./tree.ts";
34
+ import {
35
+ caseListData,
36
+ deltaData,
37
+ lineData,
38
+ matrixData,
39
+ overviewData,
40
+ scatterData,
41
+ scoreboardData,
42
+ tableData,
43
+ } from "./react/fixtures.ts";
44
+
45
+ const ctx = createTextContext({ width: 80 });
46
+ const text = (node: Parameters<typeof renderNodeToText>[0]) => renderNodeToText(node, ctx);
47
+
48
+ // ───────────────────────── 每组件双面一致 ─────────────────────────
49
+
50
+ describe("RunOverview 双面", () => {
51
+ const html = renderToStaticMarkup(<RunOverview data={overviewData} />);
52
+ const term = text(<RunOverview data={overviewData} />);
53
+
54
+ it("text 面形态:头行 + 判决行 + 警告行", () => {
55
+ expect(term).toMatchInlineSnapshot(`
56
+ "2 experiments · 12 evals · 48 attempts · composed from 2 runs · latest 2026-07-01T11:30:00Z
57
+ passed 36 · failed 8 · errored 2 · skipped 2 · no data · 4m 21s
58
+ ! snapshot covers 9 of 12 evals seen in history; re-run \`niceeval exp compare/bub\` for a full snapshot"
59
+ `);
60
+ });
61
+
62
+ it("两面同口径:成本全缺都是 no data 不编 $0;警告都在", () => {
63
+ for (const face of [html, term]) {
64
+ expect(face).toContain("no data");
65
+ expect(face).not.toContain("$0");
66
+ expect(face).toContain("snapshot covers 9 of 12 evals");
67
+ }
68
+ });
69
+ });
70
+
71
+ describe("MetricTable 双面", () => {
72
+ const html = renderToStaticMarkup(<MetricTable data={tableData} />);
73
+ const term = text(<MetricTable data={tableData} />);
74
+
75
+ it("text 面形态:对齐列 + 覆盖率角标 + 缺数据 —", () => {
76
+ expect(term).toMatchInlineSnapshot(`
77
+ "agent pass rate code lines
78
+ codex 50% —
79
+ bub 87% 120 lines 5/6"
80
+ `);
81
+ });
82
+
83
+ it("两面同口径:行序一致(预排即终排)、角标一致、缺数据不补 0", () => {
84
+ // 行序:codex 在 bub 前(数据侧顺序,两面都不重排)
85
+ expect(html.indexOf(">codex<")).toBeLessThan(html.indexOf(">bub<"));
86
+ expect(term.indexOf("codex")).toBeLessThan(term.indexOf("bub"));
87
+ // 覆盖率角标 5/6 两面都在
88
+ expect(html).toContain("5/6");
89
+ expect(term).toContain("5/6");
90
+ // 全 null 格子:web 是 no data 文案,text 是 —,都绝不画 0
91
+ expect(html).toContain("no data");
92
+ expect(term).toContain("—");
93
+ });
94
+ });
95
+
96
+ describe("MetricMatrix 双面", () => {
97
+ const html = renderToStaticMarkup(<MetricMatrix data={matrixData} />);
98
+ const term = text(<MetricMatrix data={matrixData} />);
99
+
100
+ it("text 面形态:稀疏格子 — + 下钻命令指向最值得看的一行", () => {
101
+ expect(term).toMatchInlineSnapshot(`
102
+ "eval bub codex
103
+ algebra/quadratic 100% 0%
104
+ geometry/angles 50% —
105
+
106
+ next: niceeval show geometry/angles"
107
+ `);
108
+ });
109
+
110
+ it("两面同口径:数字一致,稀疏格子不编数", () => {
111
+ for (const value of ["100%", "0%", "50%"]) {
112
+ expect(html).toContain(value);
113
+ expect(term).toContain(value);
114
+ }
115
+ expect(html.match(/nre-td-empty/g)).toHaveLength(1); // web 空格子
116
+ });
117
+ });
118
+
119
+ describe("MetricBars 双面", () => {
120
+ const html = renderToStaticMarkup(<MetricBars data={matrixData} />);
121
+ const term = text(<MetricBars data={matrixData} />);
122
+
123
+ it("text 面形态:一组条 = 一个 row 键,条长即刻度,缺数据 —", () => {
124
+ expect(term).toMatchInlineSnapshot(`
125
+ "algebra/quadratic
126
+ bub ████████████████████ 100%
127
+ codex ░░░░░░░░░░░░░░░░░░░░ 0%
128
+ geometry/angles
129
+ bub ██████████░░░░░░░░░░ 50%
130
+ codex —"
131
+ `);
132
+ });
133
+
134
+ it("两面同口径:组内按值排序(better higher 降序),缺数据不出柱", () => {
135
+ expect(html).toContain("100%");
136
+ expect(html).toContain("50%");
137
+ // web 面:codex 在 geometry/angles 组没有柱(缺数据),柱数 = 3
138
+ expect(html.match(/class="nre-bar"/g)).toHaveLength(3);
139
+ // text 面:codex 行是 —
140
+ expect(term).toContain("codex —");
141
+ });
142
+ });
143
+
144
+ describe("Scoreboard 双面", () => {
145
+ const html = renderToStaticMarkup(<Scoreboard data={scoreboardData} />);
146
+ const term = text(<Scoreboard data={scoreboardData} />);
147
+
148
+ it("text 面形态:总分/满分 + 分科 + missing 注脚 + 权重表", () => {
149
+ expect(term).toMatchInlineSnapshot(`
150
+ "agent total algebra geometry
151
+ bub 78.5/100 14/16 (1 missing) 3/4
152
+ codex 52/100 9/16 1.4/4 (2 missing)
153
+ weights: algebra/ ×2 · others ×1"
154
+ `);
155
+ });
156
+
157
+ it("两面同口径:missing 与权重都如实", () => {
158
+ expect(html).toContain("1 eval missing, scored 0");
159
+ expect(term).toContain("(1 missing)");
160
+ expect(html).toContain("algebra/ ×2");
161
+ expect(term).toContain("algebra/ ×2");
162
+ });
163
+ });
164
+
165
+ describe("MetricScatter 双面", () => {
166
+ const html = renderToStaticMarkup(<MetricScatter data={scatterData} />);
167
+ const term = text(<MetricScatter data={scatterData} />);
168
+
169
+ it("text 面形态:字符坐标图 + 图例 + 缺数据注脚", () => {
170
+ expect(term).toMatchInlineSnapshot(`
171
+ " pass rate ↑
172
+ 90%│B····
173
+ │ ·········
174
+ │ ··········
175
+ │ ·········
176
+ │ ·········
177
+ │ ·········
178
+ │ C ··········
179
+ │ ·········
180
+ 50%│ ····A
181
+ └───────────────────────────────────────────────────────────────────────────→ cost (axis reversed: right = better)
182
+ $10.00 $5.00
183
+
184
+ better → upper right
185
+ A compare/bub-low B compare/bub-high C compare/codex-mid
186
+ 1 point missing data"
187
+ `);
188
+ });
189
+
190
+ it("两面同口径:x 轴反向(lower 好的一端在右)、缺数据点都不画、注脚同数", () => {
191
+ // web:$5(便宜)在 $10 右边(cx 更大)——render.test.tsx 已细测,这里只对注脚
192
+ expect(html).toContain("1 point missing data");
193
+ expect(term).toContain("1 point missing data");
194
+ expect(html).not.toContain('data-key="compare/codex-broken"'); // 注脚 title 如实列缺数据的点,但不画
195
+ // text 图例里也没有缺数据的点
196
+ expect(term).not.toContain("codex-broken");
197
+ });
198
+
199
+ it("点太密时降级坐标表,不硬挤", () => {
200
+ const narrow = renderNodeToText(<MetricScatter data={scatterData} />, createTextContext({ width: 40 }));
201
+ expect(narrow).toContain("experiment");
202
+ expect(narrow).toContain("$5.00");
203
+ expect(narrow).not.toContain("└");
204
+ });
205
+ });
206
+
207
+ describe("MetricLine 双面", () => {
208
+ const html = renderToStaticMarkup(<MetricLine data={lineData} />);
209
+ const term = text(<MetricLine data={lineData} />);
210
+
211
+ it("text 面形态:同系列一个字母、沿 x 排布,y 轴刻度,图例 + 缺数据注脚", () => {
212
+ expect(term).toMatchInlineSnapshot(`
213
+ " pass rate ↑
214
+ 80%│B··················
215
+ │ ·····································
216
+ │ ··················B
217
+
218
+
219
+
220
+ │A··················
221
+ │ ·····································
222
+ 30%│ ··················A
223
+ └───────────────────────────────────────────────────────────────────────────→ Simulated latency
224
+ 100ms 300ms
225
+
226
+ A 1 agents B 16 agents
227
+ 1 point missing data"
228
+ `);
229
+ });
230
+
231
+ it("两面同口径:未声明 param 的点两面都不画、注脚同数", () => {
232
+ expect(html).toContain("1 point missing data");
233
+ expect(term).toContain("1 point missing data");
234
+ expect(html).not.toContain('data-key="ultra/legacy"'); // 注脚 title 如实列缺数据的点,但不画
235
+ });
236
+ });
237
+
238
+ describe("DeltaTable 双面", () => {
239
+ const html = renderToStaticMarkup(<DeltaTable data={deltaData} />);
240
+ const term = text(<DeltaTable data={deltaData} />);
241
+
242
+ it("text 面形态:A → B + Δ,单侧缺数据是 —,Δ=0 是 ±0", () => {
243
+ expect(term).toMatchInlineSnapshot(`
244
+ "pair pass rate cost
245
+ bub 50% → 62% +12pp $0.20 → $0.35 +$0.15
246
+ codex 40% → 40% ±0 — → $0.30 —"
247
+ `);
248
+ });
249
+
250
+ it("两面同口径:Δ 文案一致、缺数据都不硬算", () => {
251
+ for (const piece of ["+12pp", "+$0.15", "±0"]) {
252
+ expect(html).toContain(piece);
253
+ expect(term).toContain(piece);
254
+ }
255
+ expect(html).toContain("nre-delta-missing");
256
+ expect(term).toContain("— → $0.30");
257
+ });
258
+ });
259
+
260
+ describe("CaseList 双面", () => {
261
+ const html = renderToStaticMarkup(<CaseList data={caseListData} />);
262
+ const term = text(<CaseList data={caseListData} />);
263
+
264
+ it("text 面形态:逐条案例 + 下钻命令 + 截断报剩余", () => {
265
+ expect(term).toMatchInlineSnapshot(`
266
+ "✗ algebra/quadratic · compare/bub · failed · 32.0s · $0.12
267
+ roots-correct — expected x=2, got x=3
268
+ judge: sign flipped when substituting into the quadratic formula
269
+ → niceeval show algebra/quadratic
270
+ ✗ geometry/angles · compare/codex · errored · 4.5s
271
+ TypeError: cannot read properties of undefined (reading 'foo')
272
+ → niceeval show geometry/angles
273
+
274
+ (2 more not shown)"
275
+ `);
276
+ });
277
+
278
+ it("两面同口径:失败断言、error、truncated 数一致", () => {
279
+ for (const piece of ["roots-correct", "expected x=2, got x=3", "TypeError", "2 more not shown"]) {
280
+ expect(html).toContain(piece);
281
+ expect(term).toContain(piece);
282
+ }
283
+ });
284
+ });
285
+
286
+ // ───────────────────────── 排版原语 ─────────────────────────
287
+
288
+ describe("排版原语", () => {
289
+ it("Col 纵向堆叠;Section 标题 + 缩进;Text 折行;Style text 面为空", () => {
290
+ const node = (
291
+ <Col>
292
+ <Section title="考试成绩单">
293
+ <Text>algebra 权重 ×2,满分 100。</Text>
294
+ </Section>
295
+ <Style>{`.passbars i { background: #4a7; }`}</Style>
296
+ </Col>
297
+ );
298
+ expect(text(node)).toMatchInlineSnapshot(`
299
+ "考试成绩单
300
+ algebra 权重 ×2,满分 100。"
301
+ `);
302
+ const html = renderToStaticMarkup(node);
303
+ expect(html).toContain("<h2 class=\"nre-section-title\">考试成绩单</h2>");
304
+ expect(html).toContain("<style>.passbars i { background: #4a7; }</style>");
305
+ });
306
+
307
+ it("Row:宽度够时字符分栏,│ 分隔,各栏折到自己的宽度", () => {
308
+ const node = (
309
+ <Row>
310
+ <Text>left column body</Text>
311
+ <Text>right column body</Text>
312
+ </Row>
313
+ );
314
+ const wide = renderNodeToText(node, createTextContext({ width: 60 }));
315
+ expect(wide).toContain("│");
316
+ expect(wide.split("\n")[0]).toContain("left column body");
317
+ expect(wide.split("\n")[0]).toContain("right column body");
318
+ });
319
+
320
+ it("Row:宽度不足降级纵向,不硬挤", () => {
321
+ const node = (
322
+ <Row>
323
+ <Text>left column body</Text>
324
+ <Text>right column body</Text>
325
+ </Row>
326
+ );
327
+ const narrow = renderNodeToText(node, createTextContext({ width: 30 }));
328
+ expect(narrow).not.toContain("│");
329
+ expect(narrow).toContain("left column body\n\nright column body");
330
+ });
331
+ });
332
+
333
+ // ───────────────────────── 树校验与自定义组件 ─────────────────────────
334
+
335
+ describe("渲染前树校验", () => {
336
+ it("字符串 intrinsic(<div>)报错,错误指名组件路径", () => {
337
+ const bad = (
338
+ <Col>
339
+ <Section title="x">
340
+ <div>raw</div>
341
+ </Section>
342
+ </Col>
343
+ );
344
+ expect(() => validateReportTree(bad)).toThrow(
345
+ /Raw HTML <div> has no terminal face; use <Text>, layout primitives, or a defineComponent component\. \(in <Col> > <Section>\)/,
346
+ );
347
+ });
348
+
349
+ it("普通函数组件(组合页面片段)被调用展开继续校验", () => {
350
+ const Fragmented = () => (
351
+ <Col>
352
+ <Text>ok</Text>
353
+ </Col>
354
+ );
355
+ expect(() => validateReportTree(<Fragmented />)).not.toThrow();
356
+ const BadInside = () => <span>nope</span>;
357
+ expect(() => validateReportTree(<BadInside />)).toThrow(/Raw HTML <span>/);
358
+ });
359
+ });
360
+
361
+ describe("defineComponent", () => {
362
+ it("faces 两键必填,少一个面在运行时也拦住(编译期本就不过)", () => {
363
+ // @ts-expect-error 缺 text 面编译不过;运行时同样报错
364
+ expect(() => defineComponent({ web: () => null })).toThrow(/requires both faces/);
365
+ });
366
+
367
+ it("自定义双面组件:同一份数据两面判读一致,text 面拿到宽度", () => {
368
+ interface BarRow {
369
+ key: string;
370
+ ratio: number | null;
371
+ display: string;
372
+ }
373
+ const PassBars = defineComponent<{ rows: BarRow[] }>({
374
+ web({ rows }) {
375
+ return (
376
+ <ul className="passbars">
377
+ {rows.map((r) => (
378
+ <li key={r.key}>
379
+ <span>{r.key}</span>
380
+ <b>{r.ratio === null ? "—" : r.display}</b>
381
+ </li>
382
+ ))}
383
+ </ul>
384
+ );
385
+ },
386
+ text({ rows }, { width }) {
387
+ const bar = (n: number) => "█".repeat(Math.round(n * 10)).padEnd(10, "░");
388
+ expect(width).toBe(80);
389
+ return rows
390
+ .map((r) => `${r.key.padEnd(8)} ${r.ratio === null ? "—".padEnd(10) : bar(r.ratio)} ${r.display}`)
391
+ .join("\n");
392
+ },
393
+ });
394
+ const rows: BarRow[] = [
395
+ { key: "bub", ratio: 0.87, display: "87%" },
396
+ { key: "claude", ratio: null, display: "—" },
397
+ ];
398
+ const term = text(<PassBars rows={rows} />);
399
+ expect(term).toMatchInlineSnapshot(`
400
+ "bub █████████░ 87%
401
+ claude — —"
402
+ `);
403
+ const html = renderToStaticMarkup(<PassBars rows={rows} />);
404
+ expect(html).toContain("87%");
405
+ expect(html).toContain("—"); // 缺数据两面都是 —,不补 0
406
+ });
407
+ });
408
+
409
+ // ───────────────────────── defineReport 与两个宿主入口 ─────────────────────────
410
+
411
+ function fakeContext(): { selection: Selection; results: Results } {
412
+ const snapshot: Snapshot = (() => {
413
+ const summary = {
414
+ agent: "bub",
415
+ startedAt: "2026-07-01T10:00:00Z",
416
+ completedAt: "2026-07-01T10:05:00Z",
417
+ passed: 1,
418
+ failed: 1,
419
+ skipped: 0,
420
+ errored: 0,
421
+ durationMs: 2000,
422
+ results: [] as never[],
423
+ };
424
+ const runDir = { dir: "/results/run-1", summary, attempts: [] as never[] };
425
+ const mk = (id: string, outcome: "passed" | "failed", index: number) => ({
426
+ evalId: id,
427
+ experimentId: "compare/bub",
428
+ result: {
429
+ id,
430
+ agent: "bub",
431
+ outcome,
432
+ attempt: 0,
433
+ startedAt: `2026-07-01T10:0${index}:00Z`,
434
+ durationMs: 1000,
435
+ assertions: [],
436
+ },
437
+ ref: { run: "run-1", result: index },
438
+ runDir,
439
+ events: async () => null,
440
+ trace: async () => null,
441
+ o11y: async () => null,
442
+ diff: async () => null,
443
+ sources: async () => null,
444
+ });
445
+ const attempts = [mk("algebra/x", "passed", 0), mk("algebra/y", "failed", 1)];
446
+ return {
447
+ experimentId: "compare/bub",
448
+ startedAt: "2026-07-01T10:00:00Z",
449
+ agent: "bub",
450
+ schemaVersion: 1,
451
+ evals: attempts.map((a) => ({ id: a.evalId, attempts: [a] })),
452
+ attempts,
453
+ runDir,
454
+ } as unknown as Snapshot;
455
+ })();
456
+ const selection: Selection = {
457
+ snapshots: [snapshot],
458
+ warnings: [],
459
+ filter: () => selection,
460
+ };
461
+ const results = {
462
+ experiments: [{ id: "compare/bub", snapshots: [snapshot], latest: snapshot, evalIds: ["algebra/x", "algebra/y"] }],
463
+ skipped: [],
464
+ runDirs: [snapshot.runDir],
465
+ latest: () => selection,
466
+ } as Results;
467
+ return { selection, results };
468
+ }
469
+
470
+ describe("defineReport + 渲染入口", () => {
471
+ const report = defineReport(async ({ selection }) => (
472
+ <Col>
473
+ <DefaultReport />
474
+ <Section title="考试成绩单">
475
+ <Scoreboard data={await Scoreboard.data(selection, { rows: "agent", subjects: "evalGroup" })} />
476
+ </Section>
477
+ </Col>
478
+ ));
479
+
480
+ it("isReportDefinition 识别产物;非函数报错", () => {
481
+ expect(isReportDefinition(report)).toBe(true);
482
+ expect(isReportDefinition({})).toBe(false);
483
+ // @ts-expect-error 非函数
484
+ expect(() => defineReport(42)).toThrow(/expects a build function/);
485
+ });
486
+
487
+ it("renderReportToText:同一棵树走 text 面,DefaultReport 渲染宿主注入的选集", async () => {
488
+ const out = await renderReportToText(report, fakeContext(), { width: 100 });
489
+ // 官方水位 = show 榜单(viewing-results.mdx):Current verdicts 头 + experiment 榜单 + 失败清单
490
+ expect(out).toContain("Current verdicts · 1 experiment · composed from 1 run");
491
+ expect(out).toContain("experiment");
492
+ expect(out).toContain("evals");
493
+ expect(out).toContain("compare/bub");
494
+ expect(out).toContain("1/2"); // eval 级折叠计票
495
+ expect(out).toContain("50%");
496
+ expect(out).toContain("Failing:");
497
+ expect(out).toContain("✗ algebra/y");
498
+ expect(out).toContain("→ niceeval show algebra/y"); // 失败清单每条自带下钻命令
499
+ // 自己的口径:成绩单
500
+ expect(out).toContain("考试成绩单");
501
+ expect(out).toContain("bub 50/100");
502
+ });
503
+
504
+ it("renderReportToStaticHtml:同一棵树走 web 面,官方组件自动接证据室深链", async () => {
505
+ const html = await renderReportToStaticHtml(report, fakeContext());
506
+ expect(html).toContain("nre-default-report");
507
+ expect(html).toContain("nre-scoreboard");
508
+ expect(html).toContain("考试成绩单");
509
+ // 宿主注入的 attemptHref:CaseList 的下钻是普通 <a>,默认 view 路由
510
+ expect(html).toContain('href="#/attempt/run-1/1"');
511
+ expect(html).not.toContain("<script");
512
+ });
513
+
514
+ it("树里混进 <div>:两个宿主渲染前同一遍校验拦住", async () => {
515
+ const bad = defineReport(() => (
516
+ <Col>
517
+ <div>nope</div>
518
+ </Col>
519
+ ));
520
+ await expect(renderReportToText(bad, fakeContext())).rejects.toThrow(/Raw HTML <div>/);
521
+ await expect(renderReportToStaticHtml(bad, fakeContext())).rejects.toThrow(/Raw HTML <div>/);
522
+ });
523
+
524
+ it("<DefaultReport /> 在宿主之外渲染:一句直说的错误", () => {
525
+ expect(() => renderToStaticMarkup(<DefaultReport />)).toThrow(/renders the host-injected selection/);
526
+ });
527
+ });
@@ -45,3 +45,32 @@ export function formatPlainNumber(value: number): string {
45
45
  const sign = value < 0 ? "-" : "";
46
46
  return sign + trimmed(Math.round(Math.abs(value) * 10) / 10);
47
47
  }
48
+
49
+ // ── 以下是两个渲染面共用的展示格式化:MetricCell 一律自带 display(格式化发生在
50
+ // 计算侧),渲染面不重算;这里只服务 OverviewData 这类携带裸数字的字段。──
51
+
52
+ /** 全 null / 无样本的统一文案。绝不画 0(docs/reports.md「null ≠ 0」)。 */
53
+ export const MISSING_TEXT = "no data";
54
+
55
+ /** 毫秒 → 人读耗时("850ms" / "1.2s" / "4m 20s" / "1h 4m")。 */
56
+ export function formatDurationMs(ms: number): string {
57
+ if (ms < 1000) return `${Math.round(ms)}ms`;
58
+ const s = ms / 1000;
59
+ if (s < 60) return `${s.toFixed(1)}s`;
60
+ const m = Math.floor(s / 60);
61
+ if (m < 60) return `${m}m ${Math.round(s % 60)}s`;
62
+ const h = Math.floor(m / 60);
63
+ return `${h}h ${m % 60}m`;
64
+ }
65
+
66
+ /** 美元金额;小额保留更多位数,不四舍五入成 $0.00 的假零。 */
67
+ export function formatUSD(usd: number): string {
68
+ if (usd === 0) return "$0";
69
+ const digits = Math.abs(usd) >= 0.01 ? 2 : 4;
70
+ return `$${usd.toFixed(digits)}`;
71
+ }
72
+
73
+ /** 0..1 的比率 → 整数百分比。 */
74
+ export function formatPercent(ratio: number): string {
75
+ return `${Math.round(ratio * 100)}%`;
76
+ }
@@ -1,40 +1,95 @@
1
- // niceeval/report —— 报告积木的第二档:指标 × 计算函数,纯数据、零渲染。
2
- // 契约见 docs/reports.md「指标与聚合」「计算函数与数据契约」。
1
+ // niceeval/report —— 报告积木:指标 × 计算函数 × 双面组件 × defineReport。
2
+ // 契约见 docs/reports.md;公开叙事的准绳是 docs-site/zh/guides/custom-reports.mdx
3
+ // 与 report-components.mdx。
3
4
  //
4
- // import 边界即运行时边界:本层的计算函数会经句柄触碰文件系统(懒加载工件),
5
- // 只能进服务端 / 脚本;React 组件(第一档)在 niceeval/report/react,纯渲染。
6
- // 句柄与证据身份(SnapshotHandle / AttemptHandle / AttemptRef)来自 niceeval/results。
5
+ // import 边界即运行时边界:计算函数(挂在组件上的 .data)会经句柄触碰文件系统
6
+ // (懒加载工件),只能进服务端 / 脚本;组件的渲染面纯同步零 IO。text 宿主遍历渲染
7
+ // 不需要 react-dom(renderReportToText);web 宿主的 renderReportToStaticHtml
8
+ // ./web.ts,只有那一侧 import react-dom。写报告文件的项目要装 react(.tsx 编译产物
9
+ // import react/jsx-runtime)。
7
10
 
11
+ // 指标与 param
8
12
  export { defineMetric, passRate, examScore, durationMs, tokens, costUSD } from "./metrics.ts";
13
+ export { param } from "./param.ts";
9
14
 
10
- export { table, matrix, scoreboard, scatter, overview, delta, cases } from "./compute.ts";
15
+ // 报告基座与双面组件基座
16
+ export { defineReport, isReportDefinition, renderReportToText } from "./report.ts";
17
+ export type { ReportContext, ReportDefinition } from "./report.ts";
18
+ export { defineComponent } from "./tree.ts";
19
+ export type {
20
+ ComponentFaces,
21
+ ReportComponent,
22
+ ReportElement,
23
+ ReportNode,
24
+ TextContext,
25
+ TextRenderOptions,
26
+ WebContext,
27
+ } from "./tree.ts";
28
+
29
+ // 排版原语(五个内置双面组件)
30
+ export { Row, Col, Section, Text, Style } from "./primitives.tsx";
31
+ export type { LayoutProps, SectionProps, StyleProps } from "./primitives.tsx";
11
32
 
33
+ // 官方水位整块(零 props 的锚点)
34
+ export { DefaultReport } from "./default-report.tsx";
35
+
36
+ // 官方双面组件(各自带 data 计算函数;MetricBars.data = MetricMatrix.data)
37
+ export {
38
+ CaseList,
39
+ DeltaTable,
40
+ MetricBars,
41
+ MetricLine,
42
+ MetricMatrix,
43
+ MetricScatter,
44
+ MetricTable,
45
+ RunOverview,
46
+ Scoreboard,
47
+ } from "./components.tsx";
12
48
  export type {
13
- TableOptions,
14
- MatrixOptions,
15
- ScoreboardOptions,
16
- ScatterOptions,
17
- OverviewOptions,
18
- DeltaOptions,
49
+ CaseListProps,
50
+ DeltaTableProps,
51
+ MetricLineProps,
52
+ MetricMatrixProps,
53
+ MetricScatterProps,
54
+ MetricTableProps,
55
+ RunOverviewProps,
56
+ ScoreboardProps,
57
+ } from "./components.tsx";
58
+
59
+ // 计算函数的选项类型(函数本体挂在组件上,不做顶层导出)
60
+ export type {
61
+ CaseListDataOptions,
62
+ DeltaDataOptions,
19
63
  DeltaPair,
20
- CasesOptions,
64
+ LineDataOptions,
65
+ MatrixDataOptions,
66
+ ScatterDataOptions,
67
+ ScoreboardDataOptions,
68
+ TableDataOptions,
21
69
  } from "./compute.ts";
22
70
 
71
+ // 数据契约(组件的 data props)
23
72
  export type {
24
73
  Aggregator,
25
- MetricAggregate,
26
- Metric,
27
- Dimension,
28
- MetricColumn,
29
- MetricCell,
30
74
  AttemptRef,
31
- TableData,
75
+ CaseListData,
76
+ DeltaData,
77
+ Dimension,
78
+ DimensionInput,
79
+ LineAxis,
80
+ LineData,
32
81
  MatrixData,
33
- ScoreboardData,
34
- ScatterData,
82
+ Metric,
83
+ MetricAggregate,
84
+ MetricCell,
85
+ MetricColumn,
35
86
  OverviewData,
36
- DeltaData,
37
- CaseListData,
87
+ ParamRef,
88
+ ScatterData,
89
+ ScoreboardData,
90
+ SelectionWarning,
91
+ TableData,
38
92
  } from "./types.ts";
39
93
 
40
- export type { SnapshotHandle, AttemptHandle, RunHandle } from "../results/types.ts";
94
+ // 数据层输入的类型(家在 niceeval/results,这里 re-export 方便写指标 / 报告)
95
+ export type { AttemptHandle, Results, Selection, Snapshot } from "../results/index.ts";