niceeval 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (337) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +321 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/otlp/parse.d.ts +3 -0
  9. package/dist/o11y/otlp/receiver.d.ts +21 -0
  10. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  11. package/dist/o11y/types.d.ts +190 -0
  12. package/dist/report/aggregate.d.ts +57 -0
  13. package/dist/report/aggregate.js +212 -0
  14. package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
  15. package/dist/report/built-ins/experiment-comparison.js +13 -0
  16. package/dist/report/built-ins/index.d.ts +1 -0
  17. package/dist/report/built-ins/index.js +2 -0
  18. package/dist/report/components.d.ts +180 -0
  19. package/dist/report/components.js +132 -0
  20. package/dist/report/compute.d.ts +106 -0
  21. package/dist/report/compute.js +600 -0
  22. package/dist/report/flag.d.ts +6 -0
  23. package/dist/report/flag.js +8 -0
  24. package/dist/report/format.d.ts +26 -0
  25. package/dist/report/format.js +109 -0
  26. package/dist/report/index.d.ts +19 -0
  27. package/dist/report/index.js +30 -0
  28. package/dist/report/load.d.ts +12 -0
  29. package/dist/report/load.js +56 -0
  30. package/dist/report/locale.d.ts +94 -0
  31. package/dist/report/locale.js +183 -0
  32. package/dist/report/metrics.d.ts +21 -0
  33. package/dist/report/metrics.js +115 -0
  34. package/dist/report/primitives.d.ts +64 -0
  35. package/dist/report/primitives.js +113 -0
  36. package/dist/report/react/AttemptList.d.ts +23 -0
  37. package/dist/report/react/AttemptList.js +19 -0
  38. package/dist/report/react/DeltaTable.d.ts +8 -0
  39. package/dist/report/react/DeltaTable.js +24 -0
  40. package/dist/report/react/EvalList.d.ts +10 -0
  41. package/dist/report/react/EvalList.js +0 -0
  42. package/dist/report/react/ExperimentList.d.ts +11 -0
  43. package/dist/report/react/ExperimentList.js +67 -0
  44. package/dist/report/react/GroupSummary.d.ts +8 -0
  45. package/dist/report/react/GroupSummary.js +8 -0
  46. package/dist/report/react/MetricBars.d.ts +10 -0
  47. package/dist/report/react/MetricBars.js +48 -0
  48. package/dist/report/react/MetricLine.d.ts +9 -0
  49. package/dist/report/react/MetricLine.js +68 -0
  50. package/dist/report/react/MetricMatrix.d.ts +10 -0
  51. package/dist/report/react/MetricMatrix.js +23 -0
  52. package/dist/report/react/MetricScatter.d.ts +9 -0
  53. package/dist/report/react/MetricScatter.js +96 -0
  54. package/dist/report/react/MetricTable.d.ts +11 -0
  55. package/dist/report/react/MetricTable.js +26 -0
  56. package/dist/report/react/RunOverview.d.ts +8 -0
  57. package/dist/report/react/RunOverview.js +12 -0
  58. package/dist/report/react/Scoreboard.d.ts +8 -0
  59. package/dist/report/react/Scoreboard.js +26 -0
  60. package/dist/report/react/cell.d.ts +9 -0
  61. package/dist/report/react/cell.js +9 -0
  62. package/dist/report/react/chart-math.d.ts +14 -0
  63. package/dist/report/react/chart-math.js +68 -0
  64. package/dist/report/react/colors.d.ts +14 -0
  65. package/dist/report/react/colors.js +45 -0
  66. package/dist/report/react/fixtures.d.ts +19 -0
  67. package/dist/report/react/fixtures.js +398 -0
  68. package/dist/report/react/format.d.ts +3 -0
  69. package/dist/report/react/format.js +7 -0
  70. package/dist/report/react/index.d.ts +6 -0
  71. package/dist/report/react/index.js +14 -0
  72. package/dist/report/report.d.ts +24 -0
  73. package/dist/report/report.js +49 -0
  74. package/dist/report/text/faces.d.ts +29 -0
  75. package/dist/report/text/faces.js +541 -0
  76. package/dist/report/text/layout.d.ts +75 -0
  77. package/dist/report/text/layout.js +173 -0
  78. package/dist/report/text/plot.d.ts +37 -0
  79. package/dist/report/text/plot.js +81 -0
  80. package/dist/report/text/table.d.ts +8 -0
  81. package/dist/report/text/table.js +86 -0
  82. package/dist/report/tree.d.ts +99 -0
  83. package/dist/report/tree.js +228 -0
  84. package/dist/report/types.d.ts +408 -0
  85. package/dist/report/types.js +4 -0
  86. package/dist/report/web.d.ts +15 -0
  87. package/dist/report/web.js +37 -0
  88. package/dist/results/locator.d.ts +75 -0
  89. package/dist/results/locator.js +0 -0
  90. package/dist/results/select.d.ts +45 -0
  91. package/dist/results/select.js +230 -0
  92. package/dist/results/source-hash.d.ts +9 -0
  93. package/dist/results/types.d.ts +204 -0
  94. package/dist/runner/eval-source.d.ts +22 -0
  95. package/dist/runner/fingerprint.d.ts +23 -0
  96. package/dist/runner/types.d.ts +332 -0
  97. package/dist/sandbox/docker-stream.d.ts +24 -0
  98. package/dist/sandbox/docker.d.ts +84 -0
  99. package/dist/sandbox/e2b.d.ts +31 -0
  100. package/dist/sandbox/errors.d.ts +4 -0
  101. package/dist/sandbox/local-files.d.ts +6 -0
  102. package/dist/sandbox/paths.d.ts +4 -0
  103. package/dist/sandbox/registry.d.ts +13 -0
  104. package/dist/sandbox/resolve.d.ts +40 -0
  105. package/dist/sandbox/retry.d.ts +10 -0
  106. package/dist/sandbox/shell.d.ts +11 -0
  107. package/dist/sandbox/source-files.d.ts +13 -0
  108. package/dist/sandbox/types.d.ts +203 -0
  109. package/dist/sandbox/vercel.d.ts +37 -0
  110. package/dist/scoring/types.d.ts +70 -0
  111. package/dist/shared/aggregate.d.ts +29 -0
  112. package/dist/shared/aggregate.js +55 -0
  113. package/dist/shared/types.d.ts +27 -0
  114. package/dist/shared/verdict.d.ts +27 -0
  115. package/dist/shared/verdict.js +34 -0
  116. package/dist/tty-line.d.ts +6 -0
  117. package/dist/types.d.ts +7 -0
  118. package/dist/util.d.ts +22 -0
  119. package/docs-site/zh/concepts/adapter.mdx +7 -7
  120. package/docs-site/zh/concepts/assert.mdx +19 -18
  121. package/docs-site/zh/concepts/evals.mdx +12 -11
  122. package/docs-site/zh/concepts/experiment.mdx +2 -2
  123. package/docs-site/zh/concepts/judge.mdx +9 -9
  124. package/docs-site/zh/concepts/overview.mdx +3 -3
  125. package/docs-site/zh/concepts/tier.mdx +6 -6
  126. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  127. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  128. package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
  129. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  130. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  131. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  132. package/docs-site/zh/guides/experiments.mdx +7 -1
  133. package/docs-site/zh/guides/fixtures.mdx +4 -2
  134. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  135. package/docs-site/zh/guides/publish-report.mdx +4 -4
  136. package/docs-site/zh/guides/report-components.mdx +194 -55
  137. package/docs-site/zh/guides/reporters.mdx +2 -2
  138. package/docs-site/zh/guides/results-data.mdx +61 -34
  139. package/docs-site/zh/guides/runner.mdx +2 -2
  140. package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
  141. package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
  142. package/docs-site/zh/guides/viewing-results.mdx +200 -79
  143. package/docs-site/zh/guides/write-experiment.mdx +9 -1
  144. package/docs-site/zh/guides/write-send.mdx +4 -3
  145. package/docs-site/zh/index.mdx +4 -3
  146. package/docs-site/zh/introduction.mdx +2 -1
  147. package/docs-site/zh/quickstart.mdx +5 -3
  148. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  149. package/docs-site/zh/reference/capabilities.mdx +1 -1
  150. package/docs-site/zh/reference/cli.mdx +36 -13
  151. package/docs-site/zh/reference/define-agent.mdx +15 -13
  152. package/docs-site/zh/reference/define-config.mdx +1 -1
  153. package/docs-site/zh/reference/define-eval.mdx +3 -3
  154. package/docs-site/zh/reference/events.mdx +9 -3
  155. package/docs-site/zh/reference/expect.mdx +13 -8
  156. package/package.json +10 -6
  157. package/src/agents/bub.ts +129 -42
  158. package/src/agents/claude-code.test.ts +152 -0
  159. package/src/agents/claude-code.ts +127 -13
  160. package/src/agents/codex.test.ts +192 -0
  161. package/src/agents/codex.ts +151 -12
  162. package/src/agents/index.ts +9 -3
  163. package/src/agents/manifest.ts +41 -0
  164. package/src/agents/plugin-config.test.ts +94 -0
  165. package/src/agents/shared.ts +33 -2
  166. package/src/agents/skills.test.ts +214 -0
  167. package/src/agents/skills.ts +304 -0
  168. package/src/agents/streaming.ts +5 -5
  169. package/src/agents/types.ts +59 -3
  170. package/src/cli.ts +74 -47
  171. package/src/context/context.test.ts +20 -2
  172. package/src/context/control-flow.ts +1 -1
  173. package/src/context/session.test.ts +92 -1
  174. package/src/context/session.ts +23 -1
  175. package/src/context/types.ts +4 -4
  176. package/src/define.test.ts +8 -8
  177. package/src/define.ts +11 -11
  178. package/src/expect/index.ts +5 -5
  179. package/src/i18n/en.ts +33 -15
  180. package/src/i18n/zh-CN.ts +37 -21
  181. package/src/index.ts +1 -1
  182. package/src/o11y/execution-tree.test.ts +452 -0
  183. package/src/o11y/execution-tree.ts +367 -0
  184. package/src/o11y/otlp/receiver.ts +1 -1
  185. package/src/o11y/otlp/select.ts +7 -3
  186. package/src/o11y/parsers/claude-code.test.ts +175 -0
  187. package/src/o11y/parsers/claude-code.ts +32 -0
  188. package/src/o11y/types.ts +6 -1
  189. package/src/report/aggregate.ts +26 -20
  190. package/src/report/built-in-user-parity.test.tsx +640 -0
  191. package/src/report/built-ins/experiment-comparison.tsx +19 -0
  192. package/src/report/built-ins/index.ts +2 -0
  193. package/src/report/components.tsx +225 -50
  194. package/src/report/compute.ts +335 -54
  195. package/src/report/dual-render.test.tsx +1129 -0
  196. package/src/report/flag.ts +1 -1
  197. package/src/report/format.ts +35 -2
  198. package/src/report/index.ts +47 -14
  199. package/src/report/load.ts +1 -1
  200. package/src/report/locale.ts +229 -0
  201. package/src/report/metrics.ts +39 -15
  202. package/src/report/primitives.tsx +100 -2
  203. package/src/report/react/AttemptList.tsx +113 -0
  204. package/src/report/react/DeltaTable.tsx +9 -6
  205. package/src/report/react/EvalList.tsx +0 -0
  206. package/src/report/react/ExperimentList.tsx +220 -0
  207. package/src/report/react/GroupSummary.tsx +66 -0
  208. package/src/report/react/MetricBars.tsx +24 -13
  209. package/src/report/react/MetricLine.tsx +21 -21
  210. package/src/report/react/MetricMatrix.tsx +8 -4
  211. package/src/report/react/MetricScatter.tsx +126 -96
  212. package/src/report/react/MetricTable.tsx +115 -15
  213. package/src/report/react/RunOverview.tsx +48 -26
  214. package/src/report/react/Scoreboard.tsx +15 -7
  215. package/src/report/react/cell.tsx +18 -9
  216. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  217. package/src/report/react/colors.ts +24 -15
  218. package/src/report/react/enhance.js +168 -0
  219. package/src/report/react/fixtures.ts +169 -34
  220. package/src/report/react/format.ts +1 -1
  221. package/src/report/react/index.tsx +21 -6
  222. package/src/report/react/render.test.tsx +180 -31
  223. package/src/report/react/styles.css +432 -213
  224. package/src/report/report.test.ts +398 -94
  225. package/src/report/report.ts +29 -11
  226. package/src/report/text/faces.ts +344 -96
  227. package/src/report/text/layout.ts +76 -11
  228. package/src/report/text/table.ts +98 -0
  229. package/src/report/tree.ts +103 -20
  230. package/src/report/types.ts +184 -25
  231. package/src/report/web.ts +46 -19
  232. package/src/results/annotated-source.test.ts +146 -0
  233. package/src/results/annotated-source.ts +128 -0
  234. package/src/results/attempt-evidence.test.ts +158 -0
  235. package/src/results/attempt-evidence.ts +158 -0
  236. package/src/results/attempt-source.ts +48 -0
  237. package/src/results/copy.ts +154 -139
  238. package/src/results/format.ts +54 -47
  239. package/src/results/host-equivalence.test.ts +527 -0
  240. package/src/results/index.ts +56 -14
  241. package/src/results/locator.test.ts +198 -0
  242. package/src/results/locator.ts +0 -0
  243. package/src/results/open.ts +392 -175
  244. package/src/results/results.test.ts +943 -494
  245. package/src/results/select.ts +129 -23
  246. package/src/results/skipped-notice.ts +0 -0
  247. package/src/results/source-hash.ts +28 -0
  248. package/src/results/types.ts +98 -62
  249. package/src/results/writer.ts +304 -176
  250. package/src/runner/attempt.test.ts +170 -0
  251. package/src/runner/attempt.ts +48 -13
  252. package/src/runner/discover.test.ts +65 -0
  253. package/src/runner/discover.ts +6 -2
  254. package/src/runner/eval-source.test.ts +121 -0
  255. package/src/runner/eval-source.ts +45 -0
  256. package/src/runner/fingerprint.ts +54 -1
  257. package/src/runner/remote-sandbox.ts +1 -1
  258. package/src/runner/report.test.ts +6 -8
  259. package/src/runner/report.ts +5 -4
  260. package/src/runner/reporters/artifacts.ts +27 -41
  261. package/src/runner/reporters/braintrust.test.ts +3 -3
  262. package/src/runner/reporters/braintrust.ts +1 -1
  263. package/src/runner/reporters/console.ts +6 -6
  264. package/src/runner/reporters/json.ts +4 -4
  265. package/src/runner/reporters/live.test.ts +56 -0
  266. package/src/runner/reporters/live.ts +36 -10
  267. package/src/runner/reporters/quiet.test.ts +66 -0
  268. package/src/runner/reporters/quiet.ts +49 -0
  269. package/src/runner/reporters/shared.ts +5 -5
  270. package/src/runner/reporters/table.ts +17 -17
  271. package/src/runner/run.ts +61 -102
  272. package/src/runner/types.ts +43 -32
  273. package/src/sandbox/checkpoint.ts +1 -1
  274. package/src/sandbox/docker.ts +15 -2
  275. package/src/sandbox/e2b.ts +11 -5
  276. package/src/sandbox/errors.ts +13 -0
  277. package/src/sandbox/index.ts +3 -3
  278. package/src/sandbox/local-files.ts +1 -1
  279. package/src/sandbox/registry.ts +6 -5
  280. package/src/sandbox/resolve.ts +38 -23
  281. package/src/sandbox/retry.test.ts +68 -0
  282. package/src/sandbox/retry.ts +44 -0
  283. package/src/sandbox/types.ts +24 -24
  284. package/src/sandbox/vercel.ts +17 -3
  285. package/src/scoring/collector.ts +3 -3
  286. package/src/scoring/judge.ts +2 -2
  287. package/src/scoring/scoped.ts +13 -1
  288. package/src/scoring/types.ts +7 -7
  289. package/src/scoring/verdict.ts +4 -4
  290. package/src/shared/aggregate.ts +23 -3
  291. package/src/shared/types.ts +2 -2
  292. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  293. package/src/show/compose.ts +12 -132
  294. package/src/show/index.ts +119 -95
  295. package/src/show/render.ts +462 -159
  296. package/src/show/show.test.ts +471 -253
  297. package/src/source-loc.ts +1 -1
  298. package/src/tty-line.ts +32 -0
  299. package/src/types.ts +2 -2
  300. package/src/util.test.ts +21 -1
  301. package/src/util.ts +5 -1
  302. package/src/view/app/App.tsx +33 -198
  303. package/src/view/app/components/AttemptModal.tsx +6 -6
  304. package/src/view/app/components/CodeView.tsx +2 -2
  305. package/src/view/app/components/CopyControls.tsx +32 -36
  306. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  307. package/src/view/app/components/ui/badge.tsx +1 -1
  308. package/src/view/app/components/ui/dialog.tsx +2 -2
  309. package/src/view/app/i18n.ts +40 -31
  310. package/src/view/app/lib/artifact-url.ts +1 -1
  311. package/src/view/app/lib/attempt-route.test.ts +37 -47
  312. package/src/view/app/lib/attempt-route.ts +26 -26
  313. package/src/view/app/lib/rows.ts +9 -112
  314. package/src/view/app/lib/verdict.ts +25 -0
  315. package/src/view/app/main.tsx +9 -12
  316. package/src/view/app/pages/RunsPage.tsx +4 -4
  317. package/src/view/app/pages/TracesPage.tsx +3 -3
  318. package/src/view/app/shared.ts +0 -1
  319. package/src/view/app/types.ts +6 -51
  320. package/src/view/artifact-serving.test.ts +136 -0
  321. package/src/view/client-dist/app.css +1 -1
  322. package/src/view/client-dist/app.js +21 -21
  323. package/src/view/data.test.ts +214 -143
  324. package/src/view/data.ts +166 -120
  325. package/src/view/index.ts +23 -11
  326. package/src/view/server.ts +52 -17
  327. package/src/view/shared/types.ts +22 -31
  328. package/src/view/styles.css +117 -327
  329. package/src/view/view-report.test.ts +146 -98
  330. package/src/report/default-report.tsx +0 -222
  331. package/src/report/dual-face.test.tsx +0 -527
  332. package/src/report/react/CaseList.tsx +0 -70
  333. package/src/view/app/components/CostScoreChart.tsx +0 -147
  334. package/src/view/app/components/ExperimentTable.tsx +0 -284
  335. package/src/view/app/components/GroupSelector.tsx +0 -61
  336. package/src/view/app/components/primitives.tsx +0 -43
  337. package/src/view/app/lib/outcome.ts +0 -82
@@ -0,0 +1,398 @@
1
+ // 手工 fixture:模拟计算函数(MetricTable.data / MetricMatrix.data / …)的产物。
2
+ // 仅供渲染测试与 scripts/report-react-demo.tsx 使用,不从入口导出。
3
+ // 数字刻意覆盖诚实细节:coverage 角标(samples<total)、全 null 格子、
4
+ // 稀疏矩阵、缺数据的散点、delta 的 null 不硬算、truncated 计数。
5
+ const locator = (s) => s;
6
+ export const passRateColumn = { key: "pass-rate", label: "pass rate", unit: "%", better: "higher" };
7
+ export const codeLinesColumn = { key: "code-lines", label: "code lines", unit: "lines", better: "lower" };
8
+ export const costColumn = { key: "cost", label: "cost", unit: "$", better: "lower" };
9
+ export const overviewData = {
10
+ snapshots: [
11
+ { experimentId: "compare/bub", agent: "bub", model: "gpt-5.4", startedAt: "2026-07-01T10:00:00Z" },
12
+ { experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T11:30:00Z" },
13
+ ],
14
+ totals: {
15
+ evals: 12,
16
+ attempts: 48,
17
+ passed: 36,
18
+ failed: 8,
19
+ errored: 2,
20
+ skipped: 2,
21
+ // 两级聚合口径(computeCell)刻意不等于 36/(36+8+2)≈78% 的 attempt 原始占比:
22
+ // 组件必须原样渲染这个字段,不得从上面四个 verdict 计票现场重算。
23
+ passRate: { value: 0.7, display: "70%", samples: 46, total: 48, refs: [] },
24
+ costUSD: null, // 全部 attempt 都没报成本:null,组件必须显示缺数据而不是 $0
25
+ durationMs: 261_000,
26
+ },
27
+ warnings: [
28
+ {
29
+ kind: "partial-coverage",
30
+ experimentId: "compare/bub",
31
+ covered: 9,
32
+ total: 12,
33
+ message: "snapshot covers 9 of 12 evals seen in history; re-run `niceeval exp compare/bub` for a full snapshot",
34
+ },
35
+ ],
36
+ };
37
+ export const overviewWithCost = {
38
+ ...overviewData,
39
+ totals: { ...overviewData.totals, costUSD: 1.234 },
40
+ warnings: [],
41
+ };
42
+ /** GroupSummary.data 的产物形态:eval 级折叠计票 + 旧 GroupSelector 口径的通过率(包成 MetricCell)。 */
43
+ export const groupSummaryData = {
44
+ experiments: 2,
45
+ // evals = 全部 verdicts 之和(3+1+1+1=6);passRate 分母只数非 skipped 的 5 道
46
+ evals: 6,
47
+ attempts: 9,
48
+ verdicts: { passed: 3, failed: 1, errored: 1, skipped: 1 },
49
+ // 3 passed / (3 + 1 + 1) = 60%;samples=ran=5 < total=evals=6:1 道 skipped 未计入分母的覆盖率角标
50
+ passRate: { value: 0.6, display: "60%", samples: 5, total: 6, refs: [] },
51
+ totalCostUSD: 1.5,
52
+ lastRunAt: "2026-07-01T11:30:00Z",
53
+ };
54
+ export const tableData = {
55
+ dimension: "agent",
56
+ // 行顺序故意不按 passRate 排:组件必须按传入顺序渲染,不重排
57
+ columns: [passRateColumn, codeLinesColumn],
58
+ rows: [
59
+ {
60
+ key: "codex",
61
+ cells: {
62
+ "pass-rate": { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
63
+ // 全 null:一个有效样本都没有 → 缺数据文案,绝不画 0
64
+ "code-lines": { value: null, display: "—", samples: 0, total: 6, refs: [] },
65
+ },
66
+ },
67
+ {
68
+ key: "bub",
69
+ cells: {
70
+ "pass-rate": {
71
+ value: 0.87,
72
+ display: "87%",
73
+ samples: 6,
74
+ total: 6,
75
+ refs: [locator("@1a0a0a0")],
76
+ },
77
+ // samples < total:有 attempt 测不了 → 覆盖率角标 5/6
78
+ "code-lines": { value: 120, display: "120 lines", samples: 5, total: 6, refs: [] },
79
+ },
80
+ },
81
+ ],
82
+ };
83
+ /** rows: "experiment" 的榜单形态:行携带 agent/model 元信息与 eval 级折叠计票。 */
84
+ export const tableDataWithMeta = {
85
+ dimension: "experiment",
86
+ columns: [passRateColumn],
87
+ rows: [
88
+ {
89
+ key: "compare/bub",
90
+ cells: { "pass-rate": { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] } },
91
+ meta: {
92
+ agent: "bub",
93
+ model: "gpt-5.4",
94
+ verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
95
+ },
96
+ },
97
+ {
98
+ key: "compare/codex",
99
+ cells: { "pass-rate": { value: 1, display: "100%", samples: 2, total: 2, refs: [] } },
100
+ meta: {
101
+ agent: "codex",
102
+ verdicts: { passed: 2, failed: 0, errored: 0, skipped: 0 },
103
+ },
104
+ },
105
+ ],
106
+ };
107
+ export const matrixData = {
108
+ rows: "eval",
109
+ columns: "agent",
110
+ metric: passRateColumn,
111
+ // 稀疏:geometry/angles × codex 没有样本,数据里不出现 → 格子空着
112
+ cells: [
113
+ {
114
+ row: "algebra/quadratic",
115
+ column: "bub",
116
+ cell: {
117
+ value: 1,
118
+ display: "100%",
119
+ samples: 2,
120
+ total: 2,
121
+ refs: [locator("@1b3b3b3"), locator("@1b7b7b7")],
122
+ },
123
+ },
124
+ {
125
+ row: "algebra/quadratic",
126
+ column: "codex",
127
+ cell: { value: 0, display: "0%", samples: 3, total: 3, refs: [] },
128
+ },
129
+ {
130
+ row: "geometry/angles",
131
+ column: "bub",
132
+ cell: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
133
+ },
134
+ ],
135
+ };
136
+ export const scoreboardData = {
137
+ dimension: "agent",
138
+ fullMarks: 100,
139
+ weights: [{ prefix: "algebra/", weight: 2 }],
140
+ rows: [
141
+ {
142
+ key: "bub",
143
+ total: { value: 78.5, display: "78.5" },
144
+ subjects: [
145
+ // missing 1:一题没跑、按 0 计——固定分母的如实注脚
146
+ { key: "algebra", earned: 14, possible: 16, evals: 8, missing: 1 },
147
+ { key: "geometry", earned: 3, possible: 4, evals: 4, missing: 0 },
148
+ ],
149
+ },
150
+ {
151
+ key: "codex",
152
+ total: { value: 52, display: "52" },
153
+ subjects: [
154
+ { key: "algebra", earned: 9, possible: 16, evals: 8, missing: 0 },
155
+ { key: "geometry", earned: 1.4, possible: 4, evals: 4, missing: 2 },
156
+ ],
157
+ },
158
+ ],
159
+ };
160
+ export const scatterData = {
161
+ points: "experiment",
162
+ series: "agent",
163
+ x: costColumn, // better: "lower" → 轴反向,便宜的一端在右
164
+ y: passRateColumn,
165
+ rows: [
166
+ {
167
+ key: "compare/bub-low",
168
+ series: "bub",
169
+ x: { value: 5, display: "$5.00", samples: 6, total: 6, refs: [] },
170
+ y: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
171
+ },
172
+ {
173
+ key: "compare/bub-high",
174
+ series: "bub",
175
+ x: { value: 10, display: "$10.00", samples: 6, total: 6, refs: [] },
176
+ y: { value: 0.9, display: "90%", samples: 6, total: 6, refs: [] },
177
+ },
178
+ {
179
+ key: "compare/codex-mid",
180
+ series: "codex",
181
+ x: { value: 7, display: "$7.00", samples: 6, total: 6, refs: [] },
182
+ y: { value: 0.6, display: "60%", samples: 6, total: 6, refs: [] },
183
+ },
184
+ {
185
+ // x 缺数据:这个点不画,注脚报 1 个点缺数据
186
+ key: "compare/codex-broken",
187
+ series: "codex",
188
+ x: { value: null, display: "—", samples: 0, total: 6, refs: [] },
189
+ y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
190
+ },
191
+ ],
192
+ };
193
+ export const lineData = {
194
+ x: { key: "latencyMs", label: "Simulated latency", unit: "ms" },
195
+ series: "agents",
196
+ y: passRateColumn,
197
+ rows: [
198
+ {
199
+ key: "ultra/agents-1-lat-100",
200
+ series: "1 agents",
201
+ x: 100,
202
+ xDisplay: "100ms",
203
+ y: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
204
+ },
205
+ {
206
+ key: "ultra/agents-1-lat-300",
207
+ series: "1 agents",
208
+ x: 300,
209
+ xDisplay: "300ms",
210
+ y: { value: 0.3, display: "30%", samples: 6, total: 6, refs: [] },
211
+ },
212
+ {
213
+ key: "ultra/agents-16-lat-100",
214
+ series: "16 agents",
215
+ x: 100,
216
+ xDisplay: "100ms",
217
+ y: { value: 0.8, display: "80%", samples: 6, total: 6, refs: [] },
218
+ },
219
+ {
220
+ key: "ultra/agents-16-lat-300",
221
+ series: "16 agents",
222
+ x: 300,
223
+ xDisplay: "300ms",
224
+ y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
225
+ },
226
+ {
227
+ // 未声明 flag 的 experiment:作轴不画点,注脚报数
228
+ key: "ultra/legacy",
229
+ series: "1 agents",
230
+ x: null,
231
+ xDisplay: "",
232
+ y: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
233
+ },
234
+ ],
235
+ };
236
+ export const deltaData = {
237
+ columns: [passRateColumn, costColumn],
238
+ rows: [
239
+ {
240
+ key: "bub",
241
+ a: { experimentId: "compare/bub" },
242
+ b: { experimentId: "compare/bub--agents-md" },
243
+ cells: {
244
+ // 通过率 +12pp:better higher → 好(绿)
245
+ "pass-rate": {
246
+ a: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
247
+ b: { value: 0.62, display: "62%", samples: 6, total: 6, refs: [] },
248
+ delta: 0.12,
249
+ display: "+12pp",
250
+ },
251
+ // 成本 +$0.15:better lower → 坏(红)
252
+ cost: {
253
+ a: { value: 0.2, display: "$0.20", samples: 6, total: 6, refs: [] },
254
+ b: { value: 0.35, display: "$0.35", samples: 6, total: 6, refs: [] },
255
+ delta: 0.15,
256
+ display: "+$0.15",
257
+ },
258
+ },
259
+ },
260
+ {
261
+ key: "codex",
262
+ a: { experimentId: "compare/codex" },
263
+ b: { experimentId: "compare/codex--agents-md" },
264
+ cells: {
265
+ "pass-rate": {
266
+ a: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
267
+ b: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
268
+ delta: 0,
269
+ display: "±0",
270
+ },
271
+ // A 侧缺数据 → delta null:显示缺,不硬算
272
+ cost: {
273
+ a: { value: null, display: "—", samples: 0, total: 6, refs: [] },
274
+ b: { value: 0.3, display: "$0.30", samples: 6, total: 6, refs: [] },
275
+ delta: null,
276
+ display: "—",
277
+ },
278
+ },
279
+ },
280
+ ],
281
+ };
282
+ // ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
283
+ /** algebra/quadratic 在 compare/bub 上失败的那次 attempt——两条子失败夹具共用同一条。 */
284
+ const failedAttempt = {
285
+ evalId: "algebra/quadratic",
286
+ experimentId: "compare/bub",
287
+ attempt: 3,
288
+ agent: "bub",
289
+ verdict: "failed",
290
+ assertions: [
291
+ {
292
+ name: "roots-correct",
293
+ severity: "gate",
294
+ score: 0,
295
+ passed: false,
296
+ detail: "expected x=2, got x=3",
297
+ evidence: "judge: sign flipped when substituting into the quadratic formula",
298
+ },
299
+ ],
300
+ durationMs: 32_000,
301
+ costUSD: 0.12,
302
+ locator: locator("@1a4a4a4"),
303
+ };
304
+ const erroredAttempt = {
305
+ evalId: "geometry/angles",
306
+ experimentId: "compare/codex",
307
+ attempt: 0,
308
+ agent: "codex",
309
+ verdict: "errored",
310
+ error: "TypeError: cannot read properties of undefined (reading 'foo')",
311
+ assertions: [],
312
+ durationMs: 4_500,
313
+ locator: locator("@1c1c1c1"),
314
+ };
315
+ const failedRetryAttempt = {
316
+ ...failedAttempt,
317
+ attempt: 4,
318
+ durationMs: 35_000,
319
+ locator: locator("@1b5b5b5"),
320
+ };
321
+ export const attemptListItems = [failedAttempt, erroredAttempt];
322
+ export const evalListItems = [
323
+ {
324
+ evalId: "algebra/quadratic",
325
+ experimentId: "compare/bub",
326
+ verdict: "failed",
327
+ reason: "roots-correct: expected x=2, got x=3",
328
+ score: { value: 0, display: "0%", samples: 1, total: 1, refs: [failedAttempt.locator] },
329
+ duration: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
330
+ cost: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
331
+ attempts: [failedAttempt],
332
+ },
333
+ {
334
+ evalId: "geometry/angles",
335
+ experimentId: "compare/codex",
336
+ verdict: "errored",
337
+ reason: erroredAttempt.error,
338
+ score: { value: 0, display: "0%", samples: 1, total: 1, refs: [erroredAttempt.locator] },
339
+ duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
340
+ cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
341
+ attempts: [erroredAttempt],
342
+ },
343
+ ];
344
+ export const experimentListItems = [
345
+ {
346
+ experimentId: "compare/bub",
347
+ agent: "bub",
348
+ model: "gpt-5.4",
349
+ flags: { memory: true },
350
+ verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
351
+ passRate: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
352
+ cost: { value: 0.12, display: "$0.12", samples: 1, total: 2, refs: [failedAttempt.locator] },
353
+ duration: { value: 32_000, display: "32.0s", samples: 2, total: 2, refs: [] },
354
+ tokens: { value: null, display: "—", samples: 0, total: 2, refs: [] },
355
+ evals: 2,
356
+ attempts: 3,
357
+ lastRunAt: "2026-07-01T10:00:00Z",
358
+ evalRows: [
359
+ {
360
+ evalId: "algebra/quadratic",
361
+ verdict: "failed",
362
+ reason: "roots-correct: expected x=2, got x=3",
363
+ duration: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
364
+ cost: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
365
+ attempts: [failedAttempt, failedRetryAttempt],
366
+ },
367
+ {
368
+ evalId: "algebra/simple",
369
+ verdict: "passed",
370
+ duration: { value: 5_000, display: "5.0s", samples: 1, total: 1, refs: [] },
371
+ cost: { value: 0.02, display: "$0.02", samples: 1, total: 1, refs: [] },
372
+ attempts: [],
373
+ },
374
+ ],
375
+ },
376
+ {
377
+ experimentId: "compare/codex",
378
+ agent: "codex",
379
+ verdicts: { passed: 0, failed: 0, errored: 1, skipped: 0 },
380
+ passRate: { value: 0, display: "0%", samples: 1, total: 1, refs: [] },
381
+ cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
382
+ duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [] },
383
+ tokens: { value: null, display: "—", samples: 0, total: 1, refs: [] },
384
+ evals: 1,
385
+ attempts: 1,
386
+ lastRunAt: "2026-07-01T11:30:00Z",
387
+ evalRows: [
388
+ {
389
+ evalId: "geometry/angles",
390
+ verdict: "errored",
391
+ reason: erroredAttempt.error,
392
+ duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
393
+ cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
394
+ attempts: [erroredAttempt],
395
+ },
396
+ ],
397
+ },
398
+ ];
@@ -0,0 +1,3 @@
1
+ export { MISSING_TEXT, formatDurationMs, formatPercent, formatUSD, verdictMark } from "../format.ts";
2
+ /** 拼 class 名:过滤空值,末尾接使用者透传的 className。 */
3
+ export declare function cx(...parts: (string | undefined | false)[]): string;
@@ -0,0 +1,7 @@
1
+ // web 面的小工具:展示格式化统一住在计算侧的 ../format.ts(两个渲染面同一份),
2
+ // 这里 re-export 并补 class 名拼接。MetricCell 一律自带 display,组件不重算。
3
+ export { MISSING_TEXT, formatDurationMs, formatPercent, formatUSD, verdictMark } from "../format.js";
4
+ /** 拼 class 名:过滤空值,末尾接使用者透传的 className。 */
5
+ export function cx(...parts) {
6
+ return parts.filter(Boolean).join(" ");
7
+ }
@@ -0,0 +1,6 @@
1
+ export { AttemptList, DeltaTable, EvalList, ExperimentList, GroupSummary, MetricBars, MetricLine, MetricMatrix, MetricScatter, MetricTable, RunOverview, Scoreboard, } from "../components.tsx";
2
+ export type { AttemptListProps, DeltaTableProps, EvalListProps, ExperimentListProps, GroupSummaryProps, MetricLineProps, MetricMatrixProps, MetricScatterProps, MetricTableProps, RunOverviewProps, ScoreboardProps, } from "../components.tsx";
3
+ export type { AttemptListItem, AttemptLocator, DeltaData, EvalListItem, ExperimentListEvalRow, ExperimentListItem, GroupSummaryData, LineAxis, LineData, MatrixData, MetricCell, MetricColumn, OverviewData, ScatterData, ScoreboardData, SelectionWarning, TableData, TableRowMeta, } from "../types.ts";
4
+ export { DEFAULT_REPORT_LOCALE, resolveMetricLabel } from "../locale.ts";
5
+ export type { LocalizedLabel, ReportLocale } from "../locale.ts";
6
+ export { NRE_PALETTE, colorClassForKey, colorHexForKey, colorIndexForKey, seriesClassForKey } from "./colors.ts";
@@ -0,0 +1,14 @@
1
+ // niceeval/report/react —— 零件复用的导出点:把某一块指标表嵌进已有内部面板时从这里
2
+ // import。导出的组件与 niceeval/report 是同一批双面组件(web 面即 React 渲染,零 IO、
3
+ // 可进 "use client",不 hydrate 也完整),都带自己的 data 计算函数;那是零件的复用,
4
+ // 不是另一套报告系统 —— 报告的家在官方宿主(--report)。
5
+ //
6
+ // 契约:
7
+ // - 组件只认「算好的可序列化数据」:零 hooks、零数据操作;
8
+ // - 样式随包发布:配套 ./styles.css(nre-* 稳定类名),使用者在其后加载覆盖即可;
9
+ // - 跨块配色一致:维度键 → 稳定散列 → 固定调色板下标(colors.ts)。
10
+ export { AttemptList, DeltaTable, EvalList, ExperimentList, GroupSummary, MetricBars, MetricLine, MetricMatrix, MetricScatter, MetricTable, RunOverview, Scoreboard, } from "../components.js";
11
+ // locale(官方组件 chrome 文案;指标 label 的按 locale 字典也用它解析)
12
+ export { DEFAULT_REPORT_LOCALE, resolveMetricLabel } from "../locale.js";
13
+ // 稳定配色(自定义组件想与官方组件同键同色时用;seriesClassForKey 配 CSS 的 --nre-series)
14
+ export { NRE_PALETTE, colorClassForKey, colorHexForKey, colorIndexForKey, seriesClassForKey } from "./colors.js";
@@ -0,0 +1,24 @@
1
+ import type { Results, Selection } from "../results/types.ts";
2
+ import { type ReportNode, type TextRenderOptions } from "./tree.ts";
3
+ export interface ReportContext {
4
+ /** 宿主按现刻水位规则挑好的 Selection:每个 experiment × eval 取跨快照合成的最新判定,外加结构化挑选
5
+ 警告;show 的默认索引、view 默认报告与两者的 --report 使用同一选择口径。 */
6
+ selection: Selection;
7
+ /** 默认挑法不合口径时,全量数据自己挑(见 docs/feature/results/library.md)。 */
8
+ results: Results;
9
+ }
10
+ declare const REPORT_DEFINITION: unique symbol;
11
+ export interface ReportDefinition {
12
+ build(ctx: ReportContext): ReportNode | Promise<ReportNode>;
13
+ [REPORT_DEFINITION]: true;
14
+ }
15
+ export declare function defineReport(build: (ctx: ReportContext) => ReportNode | Promise<ReportNode>): ReportDefinition;
16
+ /** 宿主装载报告文件时用:默认导出是不是 defineReport 的产物。 */
17
+ export declare function isReportDefinition(value: unknown): value is ReportDefinition;
18
+ /**
19
+ * text 宿主的装载语义:build → 渲染前解析数据组件(唯一的 await 边界)→ 树校验 → 遍历渲染
20
+ * text 面;Selection 有挑选警告时在报告顶部前置一块 "! <message>";报告树里的 RunOverview
21
+ * 已经渲染同一条时不重复。不需要 react-dom。
22
+ */
23
+ export declare function renderReportToText(definition: ReportDefinition, ctx: ReportContext, options?: TextRenderOptions): Promise<string>;
24
+ export {};
@@ -0,0 +1,49 @@
1
+ // defineReport:一份报告 = 一个报告文件(默认导出),两扇门(show / view)共用。
2
+ // 宿主打开结果目录、按官方口径挑好 Selection、注入上下文;报告函数折数据、摆积木,
3
+ // 返回一棵组件树。计算全部发生在报告函数体里(读句柄、await 折数据只在这里合法);
4
+ // 渲染面是纯同步函数 —— 可达百 MB 的 artifact 永远不进渲染路径。
5
+ //
6
+ // renderReportToText 是 text 宿主(show)的装载入口;web 宿主(view)的
7
+ // renderReportToStaticHtml 在 ./web.ts(那一侧才 import react-dom)。宿主接线是下一波,
8
+ // 这两个入口先以内部函数的身份可独立测试。
9
+ import { createTextContext, renderNodeToText, resolveReportTree, validateReportTree, } from "./tree.js";
10
+ const REPORT_DEFINITION = Symbol.for("niceeval.report.definition");
11
+ export function defineReport(build) {
12
+ if (typeof build !== "function") {
13
+ throw new Error("defineReport expects a build function: defineReport(async ({ selection, results }) => <Col>...</Col>).");
14
+ }
15
+ return { build, [REPORT_DEFINITION]: true };
16
+ }
17
+ /** 宿主装载报告文件时用:默认导出是不是 defineReport 的产物。 */
18
+ export function isReportDefinition(value) {
19
+ return (typeof value === "object" &&
20
+ value !== null &&
21
+ value[REPORT_DEFINITION] === true &&
22
+ typeof value.build === "function");
23
+ }
24
+ /**
25
+ * 挑选警告的 text 形态:每条渲染好的 message 前缀 "! ",一行一条(与 RunOverview /
26
+ * overviewText 里 warnings 的 "! <message>" 约定一致)。宿主级前置块——不依赖报告是否
27
+ * 摆了 RunOverview,裸跑 / --report 都在报告顶上如实报残缺,不静默。
28
+ */
29
+ function renderSelectionWarningsText(warnings, _locale) {
30
+ return warnings.map((w) => `! ${w.message}`).join("\n");
31
+ }
32
+ /**
33
+ * text 宿主的装载语义:build → 渲染前解析数据组件(唯一的 await 边界)→ 树校验 → 遍历渲染
34
+ * text 面;Selection 有挑选警告时在报告顶部前置一块 "! <message>";报告树里的 RunOverview
35
+ * 已经渲染同一条时不重复。不需要 react-dom。
36
+ */
37
+ export async function renderReportToText(definition, ctx, options) {
38
+ const node = await definition.build(ctx);
39
+ const resolved = await resolveReportTree(node);
40
+ validateReportTree(resolved);
41
+ const textCtx = createTextContext(options);
42
+ const body = renderNodeToText(resolved, textCtx);
43
+ // RunOverview can render the same Selection warnings as part of the user tree. Keep the
44
+ // host guarantee for reports that omit it, while never printing an identical warning twice.
45
+ const missingWarnings = ctx.selection.warnings.filter((warning) => !body.includes(`! ${warning.message}`));
46
+ return missingWarnings.length > 0
47
+ ? [renderSelectionWarningsText(missingWarnings, textCtx.locale), body].join("\n\n")
48
+ : body;
49
+ }
@@ -0,0 +1,29 @@
1
+ import type { AttemptListItem, DeltaData, EvalListItem, ExperimentListItem, GroupSummaryData, LineData, MatrixData, OverviewData, ScatterData, ScoreboardData, TableData } from "../types.ts";
2
+ import type { TextContext } from "../tree.ts";
3
+ import { type ReportLocale } from "../locale.ts";
4
+ /** 格子的文本形态:缺数据 —,覆盖不全带 samples/total 角标。 */
5
+ export declare function cellText(cell: {
6
+ value: number | null;
7
+ display: string;
8
+ samples: number;
9
+ total: number;
10
+ }): string;
11
+ export declare function overviewText(data: OverviewData, ctx: TextContext): string;
12
+ /**
13
+ * 一至两行:头行是通过率(GroupSummaryData.passRate.display,不重算比例)+ experiment/eval 数 +
14
+ * failed(+ errored,非零才列,与旧 GroupSelector 卡片一致)+ 总成本;第二行(有则加)是最后运行时间。
15
+ * 不依赖固定网格宽度,窄终端自然换行。
16
+ */
17
+ export declare function groupSummaryText(data: GroupSummaryData, ctx: TextContext): string;
18
+ /** verdict 计票的紧凑文案("3 passed / 1 failed"):非零判定逐个列,全部为零如实 —。 */
19
+ export declare function verdictTallyText(verdicts: NonNullable<NonNullable<TableData["rows"][number]["meta"]>["verdicts"]>, locale: ReportLocale): string;
20
+ export declare function tableText(data: TableData, ctx: TextContext): string;
21
+ export declare function matrixText(data: MatrixData, ctx: TextContext): string;
22
+ export declare function barsText(data: MatrixData): string;
23
+ export declare function scoreboardText(data: ScoreboardData, ctx: TextContext): string;
24
+ export declare function scatterText(data: ScatterData, ctx: TextContext): string;
25
+ export declare function lineText(data: LineData, ctx: TextContext): string;
26
+ export declare function deltaText(data: DeltaData, ctx: TextContext): string;
27
+ export declare function experimentListText(items: ExperimentListItem[], ctx: TextContext): string;
28
+ export declare function evalListText(items: EvalListItem[], ctx: TextContext): string;
29
+ export declare function attemptListText(items: AttemptListItem[], total: number | undefined, ctx: TextContext): string;