niceeval 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. package/INDEX.md +23 -23
  2. package/README.zh.md +6 -6
  3. package/dist/agents/types.d.ts +69 -7
  4. package/dist/context/types.d.ts +32 -12
  5. package/dist/i18n/en.d.ts +54 -0
  6. package/dist/i18n/zh-CN.d.ts +57 -3
  7. package/dist/o11y/types.d.ts +16 -2
  8. package/dist/report/aggregate.d.ts +32 -24
  9. package/dist/report/aggregate.js +158 -50
  10. package/dist/report/built-in/index.d.ts +2 -0
  11. package/dist/report/built-in/index.js +8 -0
  12. package/dist/report/components.d.ts +93 -160
  13. package/dist/report/components.js +377 -114
  14. package/dist/report/compute.d.ts +87 -81
  15. package/dist/report/compute.js +597 -417
  16. package/dist/report/flag.d.ts +32 -6
  17. package/dist/report/flag.js +92 -4
  18. package/dist/report/format.d.ts +19 -11
  19. package/dist/report/format.js +30 -13
  20. package/dist/report/index.d.ts +16 -16
  21. package/dist/report/index.js +20 -21
  22. package/dist/report/load.js +3 -2
  23. package/dist/report/locale.d.ts +57 -33
  24. package/dist/report/locale.js +122 -56
  25. package/dist/report/metrics.d.ts +23 -4
  26. package/dist/report/metrics.js +110 -25
  27. package/dist/report/primitives.d.ts +48 -15
  28. package/dist/report/primitives.js +135 -26
  29. package/dist/report/react/AttemptList.d.ts +9 -7
  30. package/dist/report/react/AttemptList.js +17 -10
  31. package/dist/report/react/DeltaTable.js +19 -18
  32. package/dist/report/react/EvalList.d.ts +4 -4
  33. package/dist/report/react/EvalList.js +0 -0
  34. package/dist/report/react/ExperimentComparison.d.ts +10 -0
  35. package/dist/report/react/ExperimentComparison.js +12 -0
  36. package/dist/report/react/ExperimentList.d.ts +4 -3
  37. package/dist/report/react/ExperimentList.js +17 -18
  38. package/dist/report/react/MetricBars.js +5 -4
  39. package/dist/report/react/MetricLine.js +12 -5
  40. package/dist/report/react/MetricMatrix.js +1 -1
  41. package/dist/report/react/MetricScatter.js +59 -28
  42. package/dist/report/react/MetricTable.js +2 -12
  43. package/dist/report/react/ScopeSummary.d.ts +10 -0
  44. package/dist/report/react/ScopeSummary.js +17 -0
  45. package/dist/report/react/Scoreboard.js +6 -6
  46. package/dist/report/react/cell.js +2 -2
  47. package/dist/report/react/chart-math.d.ts +23 -6
  48. package/dist/report/react/chart-math.js +71 -19
  49. package/dist/report/react/fixtures.d.ts +5 -9
  50. package/dist/report/react/fixtures.js +110 -147
  51. package/dist/report/react/index.d.ts +15 -5
  52. package/dist/report/react/index.js +18 -7
  53. package/dist/report/report.d.ts +137 -16
  54. package/dist/report/report.js +259 -28
  55. package/dist/report/text/faces.d.ts +17 -19
  56. package/dist/report/text/faces.js +253 -184
  57. package/dist/report/text/plot.js +1 -1
  58. package/dist/report/text/table.js +38 -7
  59. package/dist/report/tree.d.ts +90 -40
  60. package/dist/report/tree.js +252 -94
  61. package/dist/report/types.d.ts +247 -284
  62. package/dist/report/types.js +4 -3
  63. package/dist/report/web.d.ts +21 -5
  64. package/dist/report/web.js +42 -16
  65. package/dist/results/select.d.ts +38 -16
  66. package/dist/results/select.js +73 -25
  67. package/dist/results/types.d.ts +49 -14
  68. package/dist/runner/feedback/sink.d.ts +110 -0
  69. package/dist/runner/types.d.ts +513 -22
  70. package/dist/sandbox/docker.d.ts +23 -2
  71. package/dist/sandbox/e2b.d.ts +15 -1
  72. package/dist/sandbox/errors.d.ts +30 -3
  73. package/dist/sandbox/io-retry.d.ts +17 -0
  74. package/dist/sandbox/registry.d.ts +2 -0
  75. package/dist/sandbox/resolve.d.ts +18 -5
  76. package/dist/sandbox/retry.d.ts +11 -1
  77. package/dist/sandbox/types.d.ts +39 -5
  78. package/dist/sandbox/vercel.d.ts +7 -1
  79. package/dist/scoring/coverage.d.ts +30 -0
  80. package/dist/scoring/display.d.ts +21 -0
  81. package/dist/scoring/display.js +120 -0
  82. package/dist/scoring/types.d.ts +103 -20
  83. package/dist/shared/aggregate.d.ts +4 -2
  84. package/dist/shared/aggregate.js +8 -7
  85. package/dist/shared/types.d.ts +28 -0
  86. package/dist/tty-line.d.ts +0 -4
  87. package/dist/util.d.ts +23 -0
  88. package/docs-site/zh/README.md +44 -0
  89. package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
  90. package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
  91. package/docs-site/zh/examples/index.mdx +50 -0
  92. package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
  93. package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
  94. package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
  95. package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
  96. package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
  97. package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
  98. package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
  99. package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
  100. package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
  101. package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
  102. package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
  103. package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
  104. package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
  105. package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
  106. package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
  107. package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
  108. package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
  109. package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
  110. package/docs-site/zh/how-to/publish-report.mdx +105 -0
  111. package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
  112. package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
  113. package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
  114. package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
  115. package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
  116. package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
  117. package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
  118. package/docs-site/zh/index.mdx +24 -26
  119. package/docs-site/zh/introduction.mdx +8 -8
  120. package/docs-site/zh/reference/builtin-agents.mdx +32 -5
  121. package/docs-site/zh/reference/capabilities.mdx +8 -8
  122. package/docs-site/zh/reference/cli.mdx +40 -12
  123. package/docs-site/zh/reference/define-agent.mdx +58 -5
  124. package/docs-site/zh/reference/define-config.mdx +1 -1
  125. package/docs-site/zh/reference/define-eval.mdx +42 -9
  126. package/docs-site/zh/reference/events.mdx +3 -3
  127. package/docs-site/zh/reference/expect.mdx +26 -1
  128. package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
  129. package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
  130. package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
  131. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
  132. package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
  133. package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
  134. package/package.json +10 -2
  135. package/src/agents/ai-sdk-otel.test.ts +1 -0
  136. package/src/agents/ai-sdk.test.ts +3 -0
  137. package/src/agents/ai-sdk.ts +3 -0
  138. package/src/agents/bub-install-spec.test.ts +34 -0
  139. package/src/agents/bub-install-spec.ts +32 -0
  140. package/src/agents/bub.ts +31 -32
  141. package/src/agents/claude-code.test.ts +130 -9
  142. package/src/agents/claude-code.ts +76 -4
  143. package/src/agents/codex.test.ts +189 -40
  144. package/src/agents/codex.ts +155 -14
  145. package/src/agents/coding-cli-versions.test.ts +15 -0
  146. package/src/agents/coding-cli-versions.ts +3 -0
  147. package/src/agents/index.ts +13 -2
  148. package/src/agents/langgraph.test.ts +204 -0
  149. package/src/agents/langgraph.ts +495 -0
  150. package/src/agents/marketplace.ts +85 -0
  151. package/src/agents/native-config.test.ts +179 -0
  152. package/src/agents/native-config.ts +267 -0
  153. package/src/agents/openai-compat.test.ts +1 -0
  154. package/src/agents/openai-compat.ts +1 -1
  155. package/src/agents/openclaw.test.ts +31 -0
  156. package/src/agents/openclaw.ts +171 -0
  157. package/src/agents/plugin-config.test.ts +1 -0
  158. package/src/agents/sdk-streams.test.ts +79 -0
  159. package/src/agents/sdk-streams.ts +55 -10
  160. package/src/agents/skills.test.ts +1 -0
  161. package/src/agents/streaming.test.ts +3 -9
  162. package/src/agents/streaming.ts +2 -2
  163. package/src/agents/types.ts +71 -8
  164. package/src/agents/ui-message-stream.test.ts +3 -0
  165. package/src/cli.ts +446 -124
  166. package/src/context/context.test.ts +51 -12
  167. package/src/context/context.ts +162 -30
  168. package/src/context/session.test.ts +2 -1
  169. package/src/context/session.ts +115 -7
  170. package/src/context/types.ts +30 -12
  171. package/src/define.test.ts +13 -8
  172. package/src/define.ts +25 -4
  173. package/src/expect/index.ts +53 -23
  174. package/src/i18n/en.ts +81 -17
  175. package/src/i18n/zh-CN.ts +80 -17
  176. package/src/o11y/cost.test.ts +1 -0
  177. package/src/o11y/execution-tree.test.ts +1 -20
  178. package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
  179. package/src/o11y/otlp/parse.test.ts +1 -0
  180. package/src/o11y/otlp/turn-otel.test.ts +1 -0
  181. package/src/o11y/parsers/bub.test.ts +1 -0
  182. package/src/o11y/parsers/claude-code.test.ts +1 -34
  183. package/src/o11y/parsers/openclaw.test.ts +154 -0
  184. package/src/o11y/parsers/openclaw.ts +310 -0
  185. package/src/o11y/prices.json +746 -311
  186. package/src/o11y/tool-names.test.ts +1 -0
  187. package/src/o11y/types.ts +16 -2
  188. package/src/report/aggregate.ts +178 -61
  189. package/src/report/built-in/index.tsx +9 -0
  190. package/src/report/components.tsx +625 -279
  191. package/src/report/compute.ts +723 -491
  192. package/src/report/dual-render.test.tsx +741 -1024
  193. package/src/report/flag.ts +104 -12
  194. package/src/report/format.ts +32 -12
  195. package/src/report/index.ts +119 -46
  196. package/src/report/load.ts +3 -2
  197. package/src/report/locale.ts +136 -65
  198. package/src/report/metrics.ts +108 -25
  199. package/src/report/primitives.tsx +196 -45
  200. package/src/report/react/AttemptList.tsx +30 -43
  201. package/src/report/react/DeltaTable.tsx +63 -45
  202. package/src/report/react/EvalList.tsx +0 -0
  203. package/src/report/react/ExperimentComparison.tsx +73 -0
  204. package/src/report/react/ExperimentList.tsx +50 -32
  205. package/src/report/react/MetricBars.tsx +5 -4
  206. package/src/report/react/MetricLine.tsx +13 -8
  207. package/src/report/react/MetricMatrix.tsx +2 -2
  208. package/src/report/react/MetricScatter.tsx +86 -34
  209. package/src/report/react/MetricTable.tsx +4 -76
  210. package/src/report/react/ScopeSummary.tsx +86 -0
  211. package/src/report/react/Scoreboard.tsx +28 -10
  212. package/src/report/react/cell.tsx +2 -2
  213. package/src/report/react/chart-math.test.ts +85 -0
  214. package/src/report/react/chart-math.ts +101 -22
  215. package/src/report/react/enhance.js +89 -5
  216. package/src/report/react/fixtures.ts +114 -154
  217. package/src/report/react/index.tsx +24 -39
  218. package/src/report/react/render.test.tsx +138 -158
  219. package/src/report/react/styles.css +243 -82
  220. package/src/report/report.test.ts +779 -841
  221. package/src/report/report.ts +423 -41
  222. package/src/report/text/faces.ts +290 -193
  223. package/src/report/text/plot.ts +1 -1
  224. package/src/report/text/table.ts +44 -7
  225. package/src/report/tree.ts +362 -104
  226. package/src/report/types.ts +261 -271
  227. package/src/report/web.ts +63 -20
  228. package/src/results/annotated-source.test.ts +62 -9
  229. package/src/results/annotated-source.ts +64 -6
  230. package/src/results/attempt-evidence.test.ts +13 -11
  231. package/src/results/attempt-evidence.ts +20 -13
  232. package/src/results/attempt-source.ts +6 -3
  233. package/src/results/copy.ts +150 -60
  234. package/src/results/host-equivalence.test.ts +34 -20
  235. package/src/results/index.ts +12 -4
  236. package/src/results/locator.test.ts +1 -22
  237. package/src/results/open.ts +15 -5
  238. package/src/results/publish.ts +149 -0
  239. package/src/results/results.test.ts +89 -54
  240. package/src/results/select.ts +104 -34
  241. package/src/results/truncate.ts +90 -0
  242. package/src/results/types.ts +43 -14
  243. package/src/results/writer.ts +31 -13
  244. package/src/runner/attempt.test.ts +138 -7
  245. package/src/runner/attempt.ts +603 -104
  246. package/src/runner/discover.test.ts +47 -0
  247. package/src/runner/discover.ts +36 -2
  248. package/src/runner/eval-source.test.ts +1 -27
  249. package/src/runner/feedback/agent.test.ts +504 -0
  250. package/src/runner/feedback/agent.ts +409 -0
  251. package/src/runner/feedback/ci.test.ts +562 -0
  252. package/src/runner/feedback/ci.ts +401 -0
  253. package/src/runner/feedback/coordinator.test.ts +317 -0
  254. package/src/runner/feedback/coordinator.ts +397 -0
  255. package/src/runner/feedback/failure.ts +40 -0
  256. package/src/runner/feedback/human.test.ts +616 -0
  257. package/src/runner/feedback/human.ts +535 -0
  258. package/src/runner/feedback/index.ts +66 -0
  259. package/src/runner/feedback/io.ts +78 -0
  260. package/src/runner/feedback/profile.test.ts +50 -0
  261. package/src/runner/feedback/profile.ts +58 -0
  262. package/src/runner/feedback/reducer.test.ts +395 -0
  263. package/src/runner/feedback/reducer.ts +260 -0
  264. package/src/runner/feedback/renderer.ts +82 -0
  265. package/src/runner/feedback/sink.ts +203 -0
  266. package/src/runner/feedback/testing.ts +106 -0
  267. package/src/runner/ledger.test.ts +230 -0
  268. package/src/runner/ledger.ts +329 -0
  269. package/src/runner/report.test.ts +128 -3
  270. package/src/runner/report.ts +33 -9
  271. package/src/runner/reporters/artifacts.ts +8 -2
  272. package/src/runner/reporters/braintrust.test.ts +8 -7
  273. package/src/runner/reporters/braintrust.ts +9 -2
  274. package/src/runner/reporters/index.ts +2 -2
  275. package/src/runner/reporters/json.test.ts +162 -0
  276. package/src/runner/reporters/json.ts +35 -8
  277. package/src/runner/reporters/shared.ts +1 -5
  278. package/src/runner/run.test.ts +760 -3
  279. package/src/runner/run.ts +243 -37
  280. package/src/runner/sandbox-prep.ts +3 -42
  281. package/src/runner/timing.ts +158 -0
  282. package/src/runner/types.ts +518 -22
  283. package/src/sandbox/checkpoint.test.ts +55 -0
  284. package/src/sandbox/checkpoint.ts +29 -8
  285. package/src/sandbox/cli-commands.ts +407 -0
  286. package/src/sandbox/docker.ts +115 -16
  287. package/src/sandbox/e2b-agent-template.test.ts +56 -0
  288. package/src/sandbox/e2b-agent-template.ts +94 -0
  289. package/src/sandbox/e2b.ts +74 -9
  290. package/src/sandbox/errors.ts +111 -4
  291. package/src/sandbox/index.ts +2 -0
  292. package/src/sandbox/io-retry.test.ts +58 -0
  293. package/src/sandbox/io-retry.ts +45 -0
  294. package/src/sandbox/keep-registry.test.ts +86 -0
  295. package/src/sandbox/keep-registry.ts +142 -0
  296. package/src/sandbox/keep.ts +178 -0
  297. package/src/sandbox/paths.test.ts +1 -0
  298. package/src/sandbox/paths.ts +19 -8
  299. package/src/sandbox/registry.ts +20 -3
  300. package/src/sandbox/resolve.ts +76 -11
  301. package/src/sandbox/retry.test.ts +70 -0
  302. package/src/sandbox/retry.ts +46 -4
  303. package/src/sandbox/types.ts +44 -6
  304. package/src/sandbox/vercel.ts +43 -20
  305. package/src/scoring/collector.ts +60 -17
  306. package/src/scoring/coverage.ts +95 -0
  307. package/src/scoring/diff.ts +81 -0
  308. package/src/scoring/display.test.ts +121 -0
  309. package/src/scoring/display.ts +133 -0
  310. package/src/scoring/evidence.test.ts +189 -0
  311. package/src/scoring/judge.test.ts +142 -0
  312. package/src/scoring/judge.ts +15 -18
  313. package/src/scoring/scoped.ts +217 -50
  314. package/src/scoring/types.ts +117 -20
  315. package/src/scoring/verdict.ts +16 -4
  316. package/src/shared/aggregate.ts +8 -6
  317. package/src/shared/types.ts +31 -0
  318. package/src/show/compose.ts +50 -67
  319. package/src/show/index.ts +127 -56
  320. package/src/show/render.ts +662 -131
  321. package/src/show/report-host.test.ts +188 -0
  322. package/src/show/report-host.ts +375 -0
  323. package/src/show/show.test.ts +320 -54
  324. package/src/tty-line.ts +8 -26
  325. package/src/util.test.ts +1 -0
  326. package/src/util.ts +41 -0
  327. package/src/view/app/App.test.tsx +69 -0
  328. package/src/view/app/App.tsx +144 -48
  329. package/src/view/app/components/AttemptModal.tsx +423 -11
  330. package/src/view/app/components/CodeView.tsx +41 -14
  331. package/src/view/app/components/CopyControls.tsx +2 -2
  332. package/src/view/app/i18n.ts +37 -17
  333. package/src/view/app/lib/attempt-route.test.ts +1 -0
  334. package/src/view/app/lib/verdict.ts +7 -9
  335. package/src/view/app/main.tsx +13 -8
  336. package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
  337. package/src/view/app/types.ts +4 -1
  338. package/src/view/artifact-serving.test.ts +2 -1
  339. package/src/view/client-dist/app.css +1 -1
  340. package/src/view/client-dist/app.js +17 -17
  341. package/src/view/data.test.ts +10 -3
  342. package/src/view/data.ts +155 -49
  343. package/src/view/index.ts +56 -41
  344. package/src/view/server.ts +37 -15
  345. package/src/view/shared/types.ts +34 -5
  346. package/src/view/styles.css +227 -0
  347. package/src/view/view-report.test.ts +167 -62
  348. package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
  349. package/dist/report/built-ins/experiment-comparison.js +0 -13
  350. package/dist/report/built-ins/index.d.ts +0 -1
  351. package/dist/report/built-ins/index.js +0 -2
  352. package/dist/report/react/GroupSummary.d.ts +0 -8
  353. package/dist/report/react/GroupSummary.js +0 -8
  354. package/dist/report/react/RunOverview.d.ts +0 -8
  355. package/dist/report/react/RunOverview.js +0 -12
  356. package/docs-site/zh/example/ai-agent-application.mdx +0 -152
  357. package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
  358. package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
  359. package/docs-site/zh/example/showcase.mdx +0 -39
  360. package/docs-site/zh/guides/publish-report.mdx +0 -91
  361. package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
  362. package/src/report/built-in-user-parity.test.tsx +0 -640
  363. package/src/report/built-ins/experiment-comparison.tsx +0 -19
  364. package/src/report/built-ins/index.ts +0 -2
  365. package/src/report/react/GroupSummary.tsx +0 -66
  366. package/src/report/react/RunOverview.tsx +0 -109
  367. package/src/runner/reporters/console.ts +0 -70
  368. package/src/runner/reporters/live.test.ts +0 -56
  369. package/src/runner/reporters/live.ts +0 -247
  370. package/src/runner/reporters/quiet.test.ts +0 -66
  371. package/src/runner/reporters/quiet.ts +0 -49
  372. package/src/runner/reporters/table.ts +0 -277
  373. /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
  374. /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
  375. /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
  376. /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
  377. /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
  378. /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
@@ -1,1129 +1,846 @@
1
- // 双面验收:每个官方组件对同一份数据,web 面(renderToStaticMarkup)与 text 面
2
- // 给出一致判读 —— 排序方向随 better、samples < total 角标、缺数据 — 不补 0、
3
- // 截断如实报剩余;text 面形态以内联快照锁定( report-components.mdx 的示例形态)。
4
- // 另验收:排版原语的分栏与降级、渲染前树校验、defineComponent 自定义组件、
5
- // defineReport + renderReportToText / renderReportToStaticHtml 两个宿主入口。
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // 管线与双面渲染的单元测试:resolve(spec/data 双形态、记忆化、组合组件、同层并行保序、
3
+ // 非法节点拒绝)、validate(裸字符串、单面组件、Tabs 配对)、装载规范化(defineReport 三种
4
+ // 写法、content/pages 互斥、外壳嵌套、page id、标题回退)、text/web 双面同源、
5
+ // Table 与文本排版原语、FailureList 等价、内建报告等价。
6
6
 
7
7
  import { renderToStaticMarkup } from "react-dom/server";
8
8
  import { describe, expect, it } from "vitest";
9
9
 
10
- import type { EvalResult } from "../types.ts";
11
- import type { Results, Selection, Snapshot } from "../results/index.ts";
12
- import type { GroupSummaryData, ScatterData } from "./types.ts";
13
- import type { AttemptLocator, MetricScatterProps } from "./index.ts";
14
- import { evalLevelStats } from "../shared/verdict.ts";
10
+ import type { EvalResult, Verdict } from "../types.ts";
11
+ import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "../results/index.ts";
12
+ import { makeScope } from "../results/select.ts";
13
+ import {
14
+ createTextContext,
15
+ defineComponent,
16
+ renderNodeToText,
17
+ resolveReportTree,
18
+ validateReportTree,
19
+ ResolveMemo,
20
+ type ReportNode,
21
+ } from "./tree.ts";
22
+ import {
23
+ buildReportMeta,
24
+ defineReport,
25
+ FALLBACK_REPORT_TITLE,
26
+ pickReportPage,
27
+ renderReportToText,
28
+ ReportPageNotFoundError,
29
+ resolveReportTitle,
30
+ } from "./report.ts";
31
+ import { renderReportToStaticHtml } from "./web.ts";
15
32
  import {
16
33
  AttemptList,
17
- Col,
18
- DeltaTable,
19
- EvalList,
34
+ ExperimentComparison,
20
35
  ExperimentList,
21
- GroupSummary,
36
+ FailureList,
22
37
  MetricBars,
23
- MetricLine,
24
38
  MetricMatrix,
25
39
  MetricScatter,
26
40
  MetricTable,
27
- Row,
28
- RunOverview,
29
- Scoreboard,
30
- Section,
31
- Style,
32
- Table,
33
- Text,
34
- bar,
35
- costUSD,
36
- defineComponent,
37
- defineReport,
38
- isReportDefinition,
39
- padEnd,
40
- passRate,
41
- renderReportToText,
42
- stringWidth,
43
- } from "./index.ts";
44
- import { ExperimentComparison } from "./built-ins/index.ts";
45
- import { renderReportToStaticHtml } from "./web.ts";
46
- import { createTextContext, renderNodeToText, validateReportTree } from "./tree.ts";
47
- import {
48
- attemptListItems,
49
- deltaData,
50
- evalListItems,
51
- experimentListItems,
52
- groupSummaryData,
53
- lineData,
54
- matrixData,
55
- overviewData,
56
- scatterData,
57
- scoreboardData,
58
- tableData,
59
- } from "./react/fixtures.ts";
60
-
61
- const ctx = createTextContext({ width: 80 });
62
- const text = (node: Parameters<typeof renderNodeToText>[0]) => renderNodeToText(node, ctx);
63
-
64
- // ───────────────────────── 每组件双面一致 ─────────────────────────
65
-
66
- describe("RunOverview 双面", () => {
67
- const html = renderToStaticMarkup(<RunOverview data={overviewData} />);
68
- const term = text(<RunOverview data={overviewData} />);
69
-
70
- it("text 面形态:头行(含通过率)+ 判定行 + 警告行", () => {
71
- expect(term).toMatchInlineSnapshot(`
72
- "2 experiments · 12 evals · 48 attempts · Pass rate 70% 46/48 · composed from 2 runs · latest 2026-07-01T11:30:00Z
73
- passed 36 · failed 8 · errored 2 · skipped 2 · no data · 4m 21s
74
- ! snapshot covers 9 of 12 evals seen in history; re-run \`niceeval exp compare/bub\` for a full snapshot"
75
- `);
76
- });
77
-
78
- it("两面同口径:通过率显示同一个 MetricCell.display(70%,不是 36/46 现算的 78%);成本全缺都是 no data 不编 $0;警告都在", () => {
79
- for (const face of [html, term]) {
80
- expect(face).toContain("70%");
81
- expect(face).not.toContain("78%");
82
- expect(face).toContain("46/48"); // samples < total:两级聚合的覆盖率角标,两面一致
83
- expect(face).toContain("no data");
84
- expect(face).not.toContain("$0");
85
- expect(face).toContain("snapshot covers 9 of 12 evals");
86
- }
87
- });
88
- });
89
-
90
- // fixtures.overviewData 是手工摆好的终值,只验证渲染面「原样显示 MetricCell,不重算」;
91
- // 下面这组用真实 Selection 走一遍 RunOverview.data(= compute.ts 的 overviewData()),
92
- // 专门验证 totals.passRate 本身的计算口径 —— 三种通过率公式在这个 fixture 上各不相同:
93
- // 两级聚合(唯一官方口径,docs/feature/reports/architecture.md「指标聚合不变量」):eval a 题内 2/3 通过、eval b 题内 1,
94
- // 跨题均值 (2/3 + 1) / 2 = 5/6 ≈ 83.3%
95
- // attempt 原始占比(旧 bug 公式,曾经的 RunOverview 现场重算):3 passed / (3 passed + 1 failed) = 75%
96
- // eval 折叠投票(evalLevelStats,GroupSummary/MetricTable meta 的口径):a、b 都折成 passed → 2/2 = 100%
97
- // 三个数互不相同,任何一处偷懒复用另一个公式都会在这里露馅。
98
- describe("RunOverview.data · passRate 两级聚合口径", () => {
99
- function fakeVaryingAttemptsContext(): { selection: Selection; attempts: { evalId: string; result: { verdict: EvalResult["verdict"] } }[] } {
100
- const dir = "/results/compare_bub/snap-1";
101
- const base = {
102
- experimentId: "compare/bub",
103
- startedAt: "2026-07-01T10:00:00Z",
104
- completedAt: "2026-07-01T10:05:00Z",
105
- agent: "bub",
106
- schemaVersion: 1,
107
- dir,
108
- };
109
- const mk = (evalId: string, verdict: "passed" | "failed" | "skipped", attemptIndex: number, minute: number) => ({
110
- evalId,
111
- experimentId: "compare/bub",
112
- result: {
113
- id: evalId,
114
- agent: "bub",
115
- verdict,
116
- attempt: attemptIndex,
117
- startedAt: `2026-07-01T10:0${minute}:00Z`,
118
- durationMs: 1000,
119
- assertions: [],
120
- },
121
- ref: { snapshot: "compare_bub/snap-1", attempt: `${evalId}/a${attemptIndex}` },
122
- snapshot: base,
123
- events: async () => null,
124
- trace: async () => null,
125
- o11y: async () => null,
126
- diff: async () => null,
127
- sources: async () => null,
128
- });
129
- const attempts = [
130
- // eval a:3 attempts,2 通过 1 失败 → 题内 partial credit 2/3
131
- mk("algebra/a", "passed", 0, 0),
132
- mk("algebra/a", "failed", 1, 1),
133
- mk("algebra/a", "passed", 2, 2),
134
- // eval b:1 attempt 通过 → 题内 1
135
- mk("algebra/b", "passed", 0, 3),
136
- // eval c:1 attempt 跳过 —— 两级聚合与 eval 折叠计票都要把它排除在分母外
137
- mk("algebra/c", "skipped", 0, 4),
138
- ];
139
- const evalIds = [...new Set(attempts.map((a) => a.evalId))];
140
- const snapshot: Snapshot = {
141
- ...base,
142
- evals: evalIds.map((id) => ({ id, attempts: attempts.filter((a) => a.evalId === id) })),
143
- attempts,
144
- } as unknown as Snapshot;
145
- const selection: Selection = { snapshots: [snapshot], warnings: [], filter: () => selection };
146
- return { selection, attempts };
147
- }
148
-
149
- it("totals.passRate = 两级聚合 83.3%,既不等于 attempt 原始占比 75%,也不等于 eval 折叠投票 100%", async () => {
150
- const { selection, attempts } = fakeVaryingAttemptsContext();
151
- const data = await RunOverview.data(selection);
152
-
153
- expect(data.totals.passRate.value).toBeCloseTo(5 / 6, 10);
154
- expect(data.totals.passRate.display).toBe("83.3%");
155
- expect(data.totals.passRate.samples).toBe(4); // 5 attempts - 1 skipped(不进桶)
156
- expect(data.totals.passRate.total).toBe(5);
157
-
158
- // attempt 原始占比(旧 bug 公式):必须与两级聚合不同,证明没有从 passed/failed/errored 现算
159
- const attemptFraction = data.totals.passed / (data.totals.passed + data.totals.failed + data.totals.errored);
160
- expect(attemptFraction).toBeCloseTo(0.75, 10);
161
- expect(attemptFraction).not.toBeCloseTo(data.totals.passRate.value as number, 3);
162
-
163
- // eval 折叠投票(evalLevelStats,GroupSummary 的口径):也必须与两级聚合不同
164
- const stats = evalLevelStats(
165
- attempts.map((a) => ({ verdict: a.result.verdict, key: a.evalId })),
166
- (r) => r.key,
167
- );
168
- expect(stats.passRate).toBeCloseTo(1, 10);
169
- expect(stats.passRate).not.toBeCloseTo(data.totals.passRate.value as number, 3);
170
- });
171
-
172
- it("web 面与 text 面显示同一个 passRate.display,覆盖率角标(4/5)两面一致", async () => {
173
- const { selection } = fakeVaryingAttemptsContext();
174
- const data = await RunOverview.data(selection);
175
- const html = renderToStaticMarkup(<RunOverview data={data} />);
176
- const term = text(<RunOverview data={data} />);
177
- for (const face of [html, term]) {
178
- expect(face).toContain(data.totals.passRate.display);
179
- expect(face).toContain("4/5");
180
- }
181
- });
182
- });
183
-
184
- // ───────────────────────── GroupSummary ─────────────────────────
185
-
186
- describe("GroupSummary 双面", () => {
187
- const html = renderToStaticMarkup(<GroupSummary data={groupSummaryData} />);
188
- const term = text(<GroupSummary data={groupSummaryData} />);
189
-
190
- it("text 面形态:一行头(通过率 + experiment/eval 数 + failed/errored + 总成本)+ 最后运行时间", () => {
191
- expect(term).toMatchInlineSnapshot(`
192
- "Pass rate 60% 5/6 · 2 experiments · 6 evals · failed 1 · errored 1 · $1.50
193
- latest 2026-07-01T11:30:00Z"
194
- `);
195
- });
41
+ ScopeSummary,
42
+ } from "./components.tsx";
43
+ import { Col, Row, Section, Style, Tab, Table, Tabs, Text } from "./primitives.tsx";
44
+ import { attemptListData, metricScatterData } from "./compute.ts";
45
+ import { costUSD, defineMetric, endToEndPassRate } from "./metrics.ts";
46
+ import builtInReport from "./built-in/index.tsx";
47
+
48
+ // ───────────────────────── fake 数据 ─────────────────────────
49
+
50
+ let seq = 0;
51
+
52
+ function res(id: string, verdict: Verdict, extra: Partial<EvalResult> = {}): EvalResult {
53
+ seq += 1;
54
+ return {
55
+ id,
56
+ agent: "agent-x",
57
+ verdict,
58
+ attempt: 0,
59
+ startedAt: `2026-07-01T00:00:00.${String(seq).padStart(6, "0")}Z`,
60
+ durationMs: 1000,
61
+ assertions: [],
62
+ ...extra,
63
+ };
64
+ }
196
65
 
197
- it("两面同口径:通过率显示同一个 MetricCell.display(60%,不是现场重算);总成本、最后运行时间两面一致", () => {
198
- for (const face of [html, term]) {
199
- expect(face).toContain("60%");
200
- expect(face).toContain("$1.50");
201
- expect(face).toContain("2026-07-01T11:30:00Z");
202
- }
203
- // 覆盖率角标:samples(5) < total(6) —— 1 道 eval 是 skipped,没进分母
204
- expect(html).toContain("5/6");
205
- expect(term).toContain("5/6");
206
- });
66
+ let runSeq = 0;
67
+
68
+ function snap(spec: {
69
+ experimentId: string;
70
+ results: EvalResult[];
71
+ agent?: string;
72
+ model?: string;
73
+ name?: Snapshot["name"];
74
+ runStartedAt?: string;
75
+ }): Snapshot {
76
+ runSeq += 1;
77
+ const startedAt = spec.runStartedAt ?? `2026-06-01T00:00:00.${String(runSeq).padStart(3, "0")}Z`;
78
+ const snapshot = {
79
+ experimentId: spec.experimentId,
80
+ startedAt,
81
+ completedAt: startedAt,
82
+ agent: spec.agent ?? "agent-x",
83
+ model: spec.model,
84
+ name: spec.name,
85
+ schemaVersion: 1,
86
+ dir: `/results/exp/snap-${runSeq}`,
87
+ } as Snapshot;
88
+ const attempts: AttemptHandle[] = spec.results.map((r) => ({
89
+ evalId: r.id,
90
+ experimentId: spec.experimentId,
91
+ result: r,
92
+ ref: { snapshot: `exp/snap-${runSeq}`, attempt: `${r.id}/a${r.attempt}` },
93
+ snapshot,
94
+ events: async () => null,
95
+ trace: async () => null,
96
+ o11y: async () => null,
97
+ agentSetup: async () => null,
98
+ diff: async () => null,
99
+ sources: async () => null,
100
+ }));
101
+ const evals = new Map<string, AttemptHandle[]>();
102
+ for (const attempt of attempts) evals.set(attempt.evalId, [...(evals.get(attempt.evalId) ?? []), attempt]);
103
+ snapshot.evals = [...evals.entries()].map(([id, list]) => ({ id, attempts: list }));
104
+ snapshot.attempts = attempts;
105
+ return snapshot;
106
+ }
207
107
 
208
- it("缺成本:两面都显示无数据文案,不编 $0", () => {
209
- const missingCost: GroupSummaryData = { ...groupSummaryData, totalCostUSD: null };
210
- const h = renderToStaticMarkup(<GroupSummary data={missingCost} />);
211
- const t = text(<GroupSummary data={missingCost} />);
212
- for (const face of [h, t]) {
213
- expect(face).toContain("no data");
214
- expect(face).not.toContain("$0");
215
- }
216
- });
108
+ function scopeOf(snapshots: Snapshot[], warnings: ScopeWarning[] = []): Scope {
109
+ return makeScope("current-evals", snapshots, warnings);
110
+ }
217
111
 
218
- it("零失败:web 面整个 errored 片段不渲染,text 面也没有 errored 分段;failed 仍如实显示 0(不是省略整个字段)", () => {
219
- const allPassed: GroupSummaryData = {
220
- ...groupSummaryData,
221
- verdicts: { passed: 6, failed: 0, errored: 0, skipped: 0 },
222
- passRate: { value: 1, display: "100%", samples: 6, total: 6, refs: [] },
112
+ function resultsOf(snapshots: Snapshot[]): Results {
113
+ const byId = new Map<string, Snapshot[]>();
114
+ for (const s of snapshots) byId.set(s.experimentId, [...(byId.get(s.experimentId) ?? []), s]);
115
+ const experiments = [...byId.entries()].map(([id, snaps]) => {
116
+ const sorted = [...snaps].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
117
+ return {
118
+ id,
119
+ snapshots: sorted,
120
+ latest: sorted[0]!,
121
+ evalIds: [...new Set(sorted.flatMap((s) => s.evals.map((e) => e.id)))].sort(),
223
122
  };
224
- const h = renderToStaticMarkup(<GroupSummary data={allPassed} />);
225
- const t = text(<GroupSummary data={allPassed} />);
226
- expect(h).not.toContain("nre-verdict-errored");
227
- expect(h).not.toContain(">errored<");
228
- expect(t).not.toContain("errored");
229
- // failed 计数依旧渲染,哪怕是 0 —— 数据字段本身从不省略,只是 errored 的展示片段在 0 时省略
230
- expect(h).toContain(">failed<");
231
- expect(t).toContain("failed 0");
232
- });
233
-
234
- it("存在错误:两面都显示 errored 计数(与 groupSummaryData 基础 fixture 的 errored:1 一致)", () => {
235
- expect(html).toContain("nre-verdict-errored");
236
- expect(html).toContain(">errored<");
237
- expect(term).toContain("errored 1");
238
- });
239
-
240
- it("zh-CN locale:web 面走中文字典(通过率/失败/错误/总成本/实验数),text 面同理;display 数字不本地化", () => {
241
- const zhHtml = renderToStaticMarkup(<GroupSummary data={groupSummaryData} locale="zh-CN" />);
242
- expect(zhHtml).toContain("通过率");
243
- expect(zhHtml).toContain("失败");
244
- expect(zhHtml).toContain("错误");
245
- expect(zhHtml).toContain("总成本");
246
- expect(zhHtml).toContain("实验数");
247
- expect(zhHtml).toContain("60%"); // display 本身不随 locale 变
248
-
249
- const zhCtx = createTextContext({ width: 80, locale: "zh-CN" });
250
- const zhTerm = renderNodeToText(<GroupSummary data={groupSummaryData} />, zhCtx);
251
- expect(zhTerm).toContain("通过率");
252
- expect(zhTerm).toContain("失败 1");
253
- expect(zhTerm).toContain("错误 1");
254
- expect(zhTerm).toContain("60%");
255
- });
256
- });
257
-
258
- describe("MetricTable 双面", () => {
259
- const html = renderToStaticMarkup(<MetricTable data={tableData} />);
260
- const term = text(<MetricTable data={tableData} />);
261
-
262
- it("text 面形态:对齐列 + 覆盖率角标 + 缺数据 —", () => {
263
- expect(term).toMatchInlineSnapshot(`
264
- "agent pass rate code lines
265
- codex 50% —
266
- bub 87% 120 lines 5/6"
267
- `);
268
- });
269
-
270
- it("两面同口径:行序一致(预排即终排)、角标一致、缺数据不补 0", () => {
271
- // 行序:codex 在 bub 前(数据侧顺序,两面都不重排)
272
- expect(html.indexOf(">codex<")).toBeLessThan(html.indexOf(">bub<"));
273
- expect(term.indexOf("codex")).toBeLessThan(term.indexOf("bub"));
274
- // 覆盖率角标 5/6 两面都在
275
- expect(html).toContain("5/6");
276
- expect(term).toContain("5/6");
277
- // 全 null 格子:web 是 no data 文案,text 是 —,都绝不画 0
278
- expect(html).toContain("no data");
279
- expect(term).toContain("—");
280
- });
281
- });
282
-
283
- describe("MetricMatrix 双面", () => {
284
- const html = renderToStaticMarkup(<MetricMatrix data={matrixData} />);
285
- const term = text(<MetricMatrix data={matrixData} />);
286
-
287
- it("text 面形态:稀疏格子 — + 下钻命令指向最值得看的一行", () => {
288
- expect(term).toMatchInlineSnapshot(`
289
- "eval bub codex
290
- algebra/quadratic 100% 0%
291
- geometry/angles 50% —
292
-
293
- next: niceeval show geometry/angles"
294
- `);
295
123
  });
124
+ return {
125
+ experiments,
126
+ skipped: [],
127
+ latest: () => makeScope("latest-snapshots", experiments.map((e) => e.latest), []),
128
+ current: () => makeScope("current-evals", experiments.map((e) => e.latest), []),
129
+ } as unknown as Results;
130
+ }
296
131
 
297
- it("两面同口径:数字一致,稀疏格子不编数", () => {
298
- for (const value of ["100%", "0%", "50%"]) {
299
- expect(html).toContain(value);
300
- expect(term).toContain(value);
301
- }
302
- expect(html.match(/nre-td-empty/g)).toHaveLength(1); // web 空格子
303
- });
304
- });
132
+ /** 管线便捷入口:resolve + validate + text 渲染,报告声明用最小 meta。 */
133
+ async function renderTreeText(node: ReportNode, scope: Scope, width = 100): Promise<string> {
134
+ const definition = defineReport(node);
135
+ const resolved = await resolveReportTree(pickReportPage(definition).content, {
136
+ scope,
137
+ results: resultsOf(scope.snapshots),
138
+ report: buildReportMeta(definition, scope, "report"),
139
+ memo: new ResolveMemo(),
140
+ });
141
+ validateReportTree(resolved);
142
+ return renderNodeToText(resolved, createTextContext({ width }));
143
+ }
305
144
 
306
- describe("MetricBars 双面", () => {
307
- const html = renderToStaticMarkup(<MetricBars data={matrixData} />);
308
- const term = text(<MetricBars data={matrixData} />);
309
-
310
- it("text 面形态:一组条 = 一个 row 键,条长即刻度,缺数据 —", () => {
311
- expect(term).toMatchInlineSnapshot(`
312
- "algebra/quadratic
313
- bub ████████████████████ 100%
314
- codex ░░░░░░░░░░░░░░░░░░░░ 0%
315
- geometry/angles
316
- bub ██████████░░░░░░░░░░ 50%
317
- codex —"
318
- `);
319
- });
145
+ // ───────────────────────── spec / data 双形态 ─────────────────────────
320
146
 
321
- it("两面同口径:组内按值排序(better higher 降序),缺数据不出柱", () => {
322
- expect(html).toContain("100%");
323
- expect(html).toContain("50%");
324
- // web 面:codex 在 geometry/angles 组没有柱(缺数据),柱数 = 3
325
- expect(html.match(/class="nre-bar /g)).toHaveLength(3);
326
- // text 面:codex 行是 —
327
- expect(term).toContain("codex —");
147
+ describe("spec 形态与 data 形态", () => {
148
+ const scatterScope = () =>
149
+ scopeOf([
150
+ snap({
151
+ experimentId: "cmp/a",
152
+ agent: "bub",
153
+ results: [res("q", "passed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.2 } })],
154
+ }),
155
+ snap({
156
+ experimentId: "cmp/b",
157
+ agent: "codex",
158
+ results: [res("q", "failed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.1 } })],
159
+ }),
160
+ ]);
161
+
162
+ it("spec 形态与「先手工调 *Data 再传 data」严格等价:两棵树渲染深等", async () => {
163
+ const scope = scatterScope();
164
+ const options = { points: "experiment", series: "agent", x: costUSD, y: endToEndPassRate } as const;
165
+ const specText = await renderTreeText(
166
+ <MetricScatter points="experiment" series="agent" x={costUSD} y={endToEndPassRate} />,
167
+ scope,
168
+ );
169
+ const data = await metricScatterData(scope, options);
170
+ const dataText = await renderTreeText(<MetricScatter data={data} />, scope);
171
+ expect(specText).toBe(dataText);
172
+ });
173
+
174
+ it("同一组件同时给 data 与 spec 字段报完整用户反馈,不静默取一边", async () => {
175
+ const scope = scatterScope();
176
+ const data = await metricScatterData(scope, { points: "experiment", x: costUSD, y: endToEndPassRate });
177
+ await expect(
178
+ renderTreeText(
179
+ // @ts-expect-error data 与 spec 字段互斥,类型层已拒绝;这里模拟无类型 JS 输入
180
+ <MetricScatter data={data} points="experiment" x={costUSD} y={endToEndPassRate} />,
181
+ scope,
182
+ ),
183
+ ).rejects.toThrow(/both `data` and spec/);
184
+ });
185
+
186
+ it("input 省略时取宿主注入的 Scope;显式 input 覆盖数据来源", async () => {
187
+ const a = snap({ experimentId: "in/a", results: [res("q", "passed")] });
188
+ const b = snap({ experimentId: "in/b", results: [res("q", "failed")] });
189
+ const scope = scopeOf([a, b]);
190
+ const all = await renderTreeText(<ScopeSummary />, scope);
191
+ expect(all).toContain("2 experiments");
192
+ const narrowed = await renderTreeText(
193
+ <ScopeSummary input={scope.filter((s) => s.experimentId === "in/a")} />,
194
+ scope,
195
+ );
196
+ expect(narrowed).toContain("1 experiment ·");
197
+ // input 也可以是手挑的 Snapshot[](按快照出行)
198
+ const snapshotsOnly = await renderTreeText(
199
+ <MetricTable input={[a]} rows="snapshot" columns={[endToEndPassRate]} />,
200
+ scope,
201
+ 140,
202
+ );
203
+ expect(snapshotsOnly).toContain("in/a @");
204
+ expect(snapshotsOnly).not.toContain("in/b");
328
205
  });
329
- });
330
206
 
331
- describe("Scoreboard 双面", () => {
332
- const html = renderToStaticMarkup(<Scoreboard data={scoreboardData} />);
333
- const term = text(<Scoreboard data={scoreboardData} />);
334
-
335
- it("text 面形态:总分/满分 + 分科 + missing 注脚 + 权重表", () => {
336
- expect(term).toMatchInlineSnapshot(`
337
- "agent total algebra geometry
338
- bub 78.5/100 14/16 (1 missing) 3/4
339
- codex 52/100 9/16 1.4/4 (2 missing)
340
- weights: algebra/ ×2 · others ×1"
341
- `);
342
- });
207
+ it("data 结构校验:字段改名前的旧 JSON 报错且文案含版本漂移提示;round-trip 的同版本 JSON 照常渲染", async () => {
208
+ const scope = scatterScope();
209
+ const table = { dimension: "agent", columns: [], rows: [] }; // 旧形状:dimension 而非 rowDimension
210
+ await expect(renderTreeText(<MetricTable data={table as never} />, scope)).rejects.toThrow(
211
+ /does not match the current TableData shape[\s\S]*different niceeval version/,
212
+ );
343
213
 
344
- it("两面同口径:missing 与权重都如实", () => {
345
- expect(html).toContain("1 eval missing, scored 0");
346
- expect(term).toContain("(1 missing)");
347
- expect(html).toContain("algebra/ ×2");
348
- expect(term).toContain("algebra/ ×2");
214
+ const fresh = await metricScatterData(scope, { points: "experiment", x: costUSD, y: endToEndPassRate });
215
+ const roundTrip = JSON.parse(JSON.stringify(fresh));
216
+ const html = renderToStaticMarkup(<MetricScatter data={roundTrip} />);
217
+ expect(html).toContain("nre-metric-scatter");
349
218
  });
350
219
 
351
- it("一整科完全没跑(不是 missing>0 的部分缺):text 面该格渲染 —,证明走的是 renderTableText 的 null 处理,不是 scoreboardText 自己拼字符串", () => {
352
- const oneSubjectMissing: typeof scoreboardData = {
353
- ...scoreboardData,
354
- rows: [
355
- scoreboardData.rows[0]!,
356
- { key: "solo", total: { value: 14, display: "14" }, subjects: [{ key: "algebra", earned: 14, possible: 16, evals: 8, missing: 0 }] },
357
- ],
220
+ it("text 面散点轴方向跟随 better:成本轴反向(便宜在右)、提示恒为右上;x better 时整图无提示", async () => {
221
+ // cmp/a:cost $0.20、通过;cmp/b:cost $0.10、失败。cost better:"lower" → 轴反向,
222
+ // 便宜的 cmp/b(标记 B)落在贵的 cmp/a(标记 A)右侧;刻度仍显示真实值(右端是更小的 $0.10)。
223
+ const scope = scatterScope();
224
+ const text = await renderTreeText(
225
+ <MetricScatter points="experiment" x={costUSD} y={endToEndPassRate} />,
226
+ scope,
227
+ );
228
+ const markCol = (mark: string): number => {
229
+ const line = text.split("\n").find((l) => l.includes(mark));
230
+ expect(line, `plot row with mark ${mark}`).toBeTruthy();
231
+ return line!.indexOf(mark);
358
232
  };
359
- const soloTerm = text(<Scoreboard data={oneSubjectMissing} />);
360
- const [, , soloLine] = soloTerm.split("\n");
361
- expect(soloLine).toContain(""); // geometry 列:solo 完全没有这一科,cells.subject1 = null → renderTableText 补 —
362
- expect(soloLine).not.toContain("0/4"); // 不补 0,与 <Table> null 契约一致
233
+ expect(markCol("B")).toBeGreaterThan(markCol("A"));
234
+ expect(text).toContain("better upper right");
235
+ expect(text).toContain("$0.10");
236
+ expect(text).toContain("$0.20");
237
+
238
+ // x 未声明 better:轴正向、整图无方向提示(组件不猜「更好」朝哪边)。
239
+ const rawCost = defineMetric({
240
+ name: "rawCost",
241
+ unit: "$",
242
+ value: (attempt) => attempt.result.usage?.costUSD ?? null,
243
+ });
244
+ const noHint = await renderTreeText(
245
+ <MetricScatter points="experiment" x={rawCost} y={endToEndPassRate} />,
246
+ scope,
247
+ );
248
+ expect(noHint).not.toContain("better →");
249
+ // 正向轴:贵的 cmp/a(A)在右。
250
+ const noHintCol = (mark: string): number => {
251
+ const line = noHint.split("\n").find((l) => l.includes(mark));
252
+ expect(line, `plot row with mark ${mark}`).toBeTruthy();
253
+ return line!.indexOf(mark);
254
+ };
255
+ expect(noHintCol("A")).toBeGreaterThan(noHintCol("B"));
363
256
  });
364
257
  });
365
258
 
366
- describe("MetricScatter 双面", () => {
367
- const html = renderToStaticMarkup(<MetricScatter data={scatterData} />);
368
- const term = text(<MetricScatter data={scatterData} />);
369
-
370
- it("text 面形态:字符坐标图 + 图例 + 缺数据注脚", () => {
371
- expect(term).toMatchInlineSnapshot(`
372
- " pass rate ↑
373
- 90%│B····
374
- │ ·········
375
- │ ··········
376
- │ ·········
377
- │ ·········
378
- │ ·········
379
- │ C ··········
380
- │ ·········
381
- 50%│ ····A
382
- └───────────────────────────────────────────────────────────────────────────→ cost (axis reversed: right = better)
383
- $10.00 $5.00
384
-
385
- better → upper right
386
- A compare/bub-low B compare/bub-high C compare/codex-mid
387
- 1 point missing data"
388
- `);
389
- });
259
+ // ───────────────────────── resolve 记忆化 ─────────────────────────
390
260
 
391
- it("两面同口径:x 轴反向(lower 好的一端在右)、缺数据点都不画、注脚同数", () => {
392
- // web:$5(便宜)在 $10 右边(cx 更大)——render.test.tsx 已细测,这里只对注脚
393
- expect(html).toContain("1 point missing data");
394
- expect(term).toContain("1 point missing data");
395
- expect(html).not.toContain('data-key="compare/codex-broken"'); // 注脚 title 如实列缺数据的点,但不画
396
- // text 图例里也没有缺数据的点
397
- expect(term).not.toContain("codex-broken");
398
- });
399
-
400
- it("点太密时降级坐标表,不硬挤", () => {
401
- const narrow = renderNodeToText(<MetricScatter data={scatterData} />, createTextContext({ width: 40 }));
402
- expect(narrow).toContain("experiment");
403
- expect(narrow).toContain("$5.00");
404
- expect(narrow).not.toContain("");
405
- });
406
- });
261
+ describe("resolve 记忆化", () => {
262
+ it("Matrix Bars 同 spec 时计算只发生一次;不同 spec 各自计算", async () => {
263
+ let calls = 0;
264
+ const counted = defineMetric({
265
+ name: "counted",
266
+ value: () => {
267
+ calls += 1;
268
+ return 1;
269
+ },
270
+ });
271
+ const scope = scopeOf([snap({ experimentId: "memo/a", results: [res("q", "passed")] })]);
272
+ await renderTreeText(
273
+ <Col>
274
+ <MetricMatrix rows="eval" columns="agent" cell={counted} />
275
+ <MetricBars rows="eval" columns="agent" cell={counted} />
276
+ </Col>,
277
+ scope,
278
+ );
279
+ expect(calls).toBe(1); // 一个 attempt,矩阵只算一遍
407
280
 
408
- describe("MetricLine 双面", () => {
409
- const html = renderToStaticMarkup(<MetricLine data={lineData} />);
410
- const term = text(<MetricLine data={lineData} />);
411
-
412
- it("text 面形态:同系列一个字母、沿 x 排布,y 轴刻度,图例 + 缺数据注脚", () => {
413
- expect(term).toMatchInlineSnapshot(`
414
- " pass rate ↑
415
- 80%│B··················
416
- │ ·····································
417
- │ ··················B
418
-
419
-
420
-
421
- │A··················
422
- │ ·····································
423
- 30%│ ··················A
424
- └───────────────────────────────────────────────────────────────────────────→ Simulated latency
425
- 100ms 300ms
426
-
427
- A 1 agents B 16 agents
428
- 1 point missing data"
429
- `);
281
+ calls = 0;
282
+ await renderTreeText(
283
+ <Col>
284
+ <MetricMatrix rows="eval" columns="agent" cell={counted} />
285
+ <MetricMatrix rows="eval" columns="model" cell={counted} />
286
+ </Col>,
287
+ scope,
288
+ );
289
+ expect(calls).toBe(2); // spec 不同(columns 维度不同):各自计算
430
290
  });
431
291
 
432
- it("两面同口径:未声明 flag 的点两面都不画、注脚同数", () => {
433
- expect(html).toContain("1 point missing data");
434
- expect(term).toContain("1 point missing data");
435
- expect(html).not.toContain('data-key="ultra/legacy"'); // 注脚 title 如实列缺数据的点,但不画
292
+ it("不同 input 各自计算;字段相同但实例不同的 Metric 各自计算、不报错", async () => {
293
+ let calls = 0;
294
+ const value = () => {
295
+ calls += 1;
296
+ return 1;
297
+ };
298
+ const m1 = defineMetric({ name: "twin", value });
299
+ const m2 = defineMetric({ name: "twin", value: (attempt) => value.call(null) }); // 引用不同的等价定义
300
+ const a = snap({ experimentId: "memo/in-a", results: [res("q", "passed")] });
301
+ const b = snap({ experimentId: "memo/in-b", results: [res("q", "passed")] });
302
+ const scope = scopeOf([a, b]);
303
+ await renderTreeText(
304
+ <Col>
305
+ <MetricMatrix input={[a]} rows="eval" columns="agent" cell={m1} />
306
+ <MetricMatrix input={[b]} rows="eval" columns="agent" cell={m1} />
307
+ <MetricMatrix input={[a]} rows="eval" columns="agent" cell={m2} />
308
+ </Col>,
309
+ scope,
310
+ );
311
+ expect(calls).toBe(3); // 两个 input 各一次 + 不同实例的 Metric 再一次
436
312
  });
437
313
  });
438
314
 
439
- describe("DeltaTable 双面", () => {
440
- const html = renderToStaticMarkup(<DeltaTable data={deltaData} />);
441
- const term = text(<DeltaTable data={deltaData} />);
315
+ // ───────────────────────── 组合组件与树形状 ─────────────────────────
442
316
 
443
- it("text 面形态:A → B + Δ,单侧缺数据是 —,Δ=0 是 ±0", () => {
444
- expect(term).toMatchInlineSnapshot(`
445
- "pair pass rate cost
446
- bub 50% 62% +12pp $0.20 $0.35 +$0.15
447
- codex 40% → 40% ±0 — → $0.30 —"
448
- `);
449
- });
450
-
451
- it("两面同口径:Δ 文案一致、缺数据都不硬算", () => {
452
- for (const piece of ["+12pp", "+$0.15", "±0"]) {
453
- expect(html).toContain(piece);
454
- expect(term).toContain(piece);
455
- }
456
- expect(html).toContain("nre-delta-missing");
457
- expect(term).toContain("— → $0.30");
317
+ describe("组合组件(函数形态)", () => {
318
+ it("resolve 阶段以 (props, ctx) 调用并递归展开;与手写等价树渲染相同;async 可用", async () => {
319
+ const scope = scopeOf([snap({ experimentId: "compose/a", results: [res("q", "passed")] })]);
320
+ const Composed = defineComponent(async (_props: Record<never, never>, ctx) => (
321
+ <Section title="wrapped">
322
+ <ScopeSummary input={ctx.scope} />
323
+ </Section>
324
+ ));
325
+ const composed = await renderTreeText(<Composed />, scope);
326
+ const manual = await renderTreeText(
327
+ <Section title="wrapped">
328
+ <ScopeSummary />
329
+ </Section>,
330
+ scope,
331
+ );
332
+ expect(composed).toBe(manual);
333
+ });
334
+
335
+ it("ctx.results 可自行挑 Snapshot[] 喂 input;ctx.report 携带走完回退链的 title 与当前 pageId", async () => {
336
+ const named = snap({ experimentId: "hist/a", name: "Memory Evals", results: [res("q", "passed")] });
337
+ const scope = scopeOf([named]);
338
+ const Meta = defineComponent((_props: Record<never, never>, ctx) => {
339
+ const history = ctx.results.experiments[0]!.snapshots;
340
+ return (
341
+ <Col>
342
+ <Text>{`title=${typeof ctx.report.title === "string" ? ctx.report.title : "?"} page=${ctx.report.pageId}`}</Text>
343
+ <ScopeSummary input={history} />
344
+ </Col>
345
+ );
346
+ });
347
+ const definition = defineReport({ pages: [{ id: "meta", title: "Meta", content: <Meta /> }] });
348
+ const text = await renderReportToText(definition, { scope, results: resultsOf([named]) }, { pageId: "meta" });
349
+ // 声明没给 title → 唯一快照 name 回退
350
+ expect(text).toContain("title=Memory Evals page=meta");
351
+ });
352
+
353
+ it("同层 sibling 并行取数且输出保持声明顺序:慢 resolve 在前不换位", async () => {
354
+ const order: string[] = [];
355
+ const Slow = defineComponent<{ label: string }, { label: string }>({
356
+ resolve: async (props) => {
357
+ await new Promise((r) => setTimeout(r, 25));
358
+ order.push("slow-resolved");
359
+ return props;
360
+ },
361
+ web: ({ label }) => <p>{label}</p>,
362
+ text: ({ label }) => label,
363
+ });
364
+ const Fast = defineComponent<{ label: string }, { label: string }>({
365
+ resolve: (props) => {
366
+ order.push("fast-resolved");
367
+ return props;
368
+ },
369
+ web: ({ label }) => <p>{label}</p>,
370
+ text: ({ label }) => label,
371
+ });
372
+ const scope = scopeOf([]);
373
+ const text = await renderTreeText(
374
+ <Col>
375
+ <Slow label="first" />
376
+ <Fast label="second" />
377
+ </Col>,
378
+ scope,
379
+ );
380
+ expect(order[0]).toBe("fast-resolved"); // 真并行:快的先完成
381
+ expect(text.indexOf("first")).toBeLessThan(text.indexOf("second")); // 输出仍按声明序
458
382
  });
459
383
  });
460
384
 
461
- describe("AttemptList 双面", () => {
462
- const html = renderToStaticMarkup(<AttemptList items={attemptListItems} />);
463
- const term = text(<AttemptList items={attemptListItems} />);
385
+ describe("ReportNode 形状与非法节点", () => {
386
+ const scope = () => scopeOf([snap({ experimentId: "node/a", results: [res("q", "passed")] })]);
464
387
 
465
- it("两面同口径:判定符 + locator + 断言/error 明细 + 下钻命令一致", () => {
466
- for (const piece of ["roots-correct", "expected x=2, got x=3", "TypeError", "@1a4a4a4", "@1c1c1c1"]) {
467
- expect(html).toContain(piece);
468
- expect(term).toContain(piece);
469
- }
470
- // web 面走证据室路由(#/attempt/@<locator>);text 面只列 locator 本身,不重复整条命令
471
- // (docs/feature/reports/architecture.md「text 输出只在整份报告末尾给一次命令模板」)。
472
- expect(html).toContain('href="#/attempt/@1a4a4a4"');
473
- expect(term).not.toMatch(/\[[EXD⏱,]+\]/);
474
- expect(html).not.toMatch(/\[[EXD⏱,]+\]/);
475
- });
476
-
477
- it("total > items.length 时两面都如实报剩余数量", () => {
478
- const htmlTrunc = renderToStaticMarkup(<AttemptList items={attemptListItems} total={attemptListItems.length + 2} />);
479
- const termTrunc = text(<AttemptList items={attemptListItems} total={attemptListItems.length + 2} />);
480
- expect(htmlTrunc).toContain("2 more not shown");
481
- expect(termTrunc).toContain("2 more not shown");
482
- });
483
-
484
- it("text 面截断超长 assertion evidence,并保留 locator 作为完整证据入口", () => {
485
- const longEvidence = "x".repeat(2_000);
486
- const item = {
487
- ...attemptListItems[0],
488
- assertions: [{ ...attemptListItems[0].assertions[0], evidence: longEvidence }],
489
- };
490
- const out = text(<AttemptList items={[item]} />);
491
- expect(out).toContain(item.locator);
492
- expect(out.replace(/\s+/g, " ")).toContain(`open ${item.locator} for full evidence`);
493
- expect(out).not.toContain(longEvidence);
494
- expect(out.length).toBeLessThan(1_000);
388
+ it("数组 / Fragment 展平保序;null / undefined / boolean 渲染为空", async () => {
389
+ const groups = ["g1", "g2"];
390
+ const text = await renderTreeText(
391
+ <Col>
392
+ {groups.map((g) => (
393
+ <Section key={g} title={g}>
394
+ <Text>{`body of ${g}`}</Text>
395
+ </Section>
396
+ ))}
397
+ <>
398
+ <Text>in fragment</Text>
399
+ </>
400
+ {false && <Text>hidden</Text>}
401
+ {null}
402
+ </Col>,
403
+ scope(),
404
+ );
405
+ expect(text.indexOf("g1")).toBeLessThan(text.indexOf("g2"));
406
+ expect(text).toContain("in fragment");
407
+ expect(text).not.toContain("hidden");
495
408
  });
496
409
 
497
- it("text 面是逐条卡片,不是 renderTableText 的产物:空行分隔每个 attempt、断言明细逐级缩进——共享表格渲染器不产生这种嵌套形状", () => {
498
- const blocks = term.split("\n\n");
499
- expect(blocks).toHaveLength(attemptListItems.length); // 每个 attempt 独立一块,块间空行分隔;表格行之间不留空行
500
- const lines = term.split("\n");
501
- expect(lines.some((l) => /^ \S/.test(l))).toBe(true); // 断言行缩进两格
502
- expect(lines.some((l) => /^ \S/.test(l))).toBe(true); // 断言 detail 再缩进两格,是卡片的层级,不是列
410
+ it("裸字符串在树校验时按完整用户反馈拒绝并指引包 <Text>", async () => {
411
+ await expect(renderTreeText(<Col>{"free text" as unknown as ReportNode}</Col>, scope())).rejects.toThrow(
412
+ /bare string[\s\S]*<Text>/,
413
+ );
503
414
  });
504
- });
505
415
 
506
- describe("EvalList 双面", () => {
507
- const html = renderToStaticMarkup(<EvalList items={evalListItems} />);
508
- const term = text(<EvalList items={evalListItems} />);
509
-
510
- it("两面同口径:身份、判定、展开到 Attempt 的 locator 徽标一致", () => {
511
- for (const piece of ["algebra/quadratic", "compare/bub", "geometry/angles", "compare/codex", "@1a4a4a4", "@1c1c1c1"]) {
512
- expect(html).toContain(piece);
513
- expect(term).toContain(piece);
514
- }
416
+ it("React 组件 / 未包装函数与 HTML intrinsic 在展开遇到时拒绝", async () => {
417
+ const Plain = ({ label }: { label: string }) => <p>{label}</p>;
418
+ await expect(renderTreeText(<Plain label="x" />, scope())).rejects.toThrow(
419
+ /not a report component[\s\S]*defineComponent/,
420
+ );
421
+ await expect(renderTreeText(<div>x</div>, scope())).rejects.toThrow(/raw HTML <div>/);
515
422
  });
516
423
 
517
- it("text 面的 score 行插值正确:不留字面量占位符 {score}(回归——曾把带插值的 locale 模板当纯标签用,漏传 vars)", () => {
518
- expect(term).toContain("score 0%");
519
- expect(term).not.toContain("{score}");
424
+ it("validateReportTree 拒绝缺任一渲染面的组件(无类型 JS 绕过 defineComponent )", () => {
425
+ const single = Object.assign(() => null, {
426
+ [Symbol.for("niceeval.report.faces")]: { web: () => null }, // 缺 text 面
427
+ });
428
+ expect(() => validateReportTree({ type: single, props: {} })).toThrow(/missing its text face/);
429
+ expect(() => validateReportTree({ type: "div", props: {} })).toThrow(/raw HTML <div>/);
520
430
  });
521
431
 
522
- it("text 面是逐条卡片,不是 renderTableText 的产物:空行分隔每个 eval、展开到 attempt 的行缩进——表格没有这层嵌套", () => {
523
- const blocks = term.split("\n\n");
524
- expect(blocks).toHaveLength(evalListItems.length);
525
- expect(term.split("\n").some((l) => /^ \S/.test(l))).toBe(true); // 展开到 attempt 的行缩进两格
432
+ it("defineComponent 对象形态缺 text web 在定义时报完整用户反馈", () => {
433
+ expect(() => defineComponent({ web: () => null } as never)).toThrow(/both faces/);
526
434
  });
527
435
  });
528
436
 
529
- describe("ExperimentList 双面", () => {
530
- const html = renderToStaticMarkup(<ExperimentList items={experimentListItems} />);
531
- const term = text(<ExperimentList items={experimentListItems} />);
437
+ // ───────────────────────── 双面同源 ─────────────────────────
532
438
 
533
- it("两面同口径:身份、Eval 判定构成、展开到 Eval 的 locator 徽标一致", () => {
534
- for (const piece of ["compare/bub", "compare/codex", "algebra/quadratic", "geometry/angles", "@1a4a4a4", "@1c1c1c1"]) {
535
- expect(html).toContain(piece);
536
- expect(term).toContain(piece);
439
+ describe("text/web 双面同源", () => {
440
+ it("双面显示同一份解析终值、覆盖率与 warning;宿主在报告树外统一显示 warning", async () => {
441
+ const s = snap({ experimentId: "dual/a", results: [res("q1", "passed"), res("q2", "skipped")] });
442
+ const warning: ScopeWarning = {
443
+ kind: "unfinished-snapshot",
444
+ experimentId: "dual/a",
445
+ startedAt: s.startedAt,
446
+ dir: s.dir,
447
+ message: "snapshot is incomplete",
448
+ command: "niceeval exp dual/a",
449
+ };
450
+ const scope = scopeOf([s], [warning]);
451
+ const definition = defineReport(<ScopeSummary />);
452
+ const ctx = { scope, results: resultsOf([s]) };
453
+ const text = await renderReportToText(definition, ctx);
454
+ const html = await renderReportToStaticHtml(definition, ctx);
455
+ for (const face of [text, html]) {
456
+ expect(face).toContain("100%"); // endToEndPassRate:skipped 不稀释
457
+ expect(face).toContain("1/2"); // samples/total 覆盖率
458
+ expect(face).toContain("snapshot is incomplete");
537
459
  }
538
- // 官方两级聚合 passRate.display 两面同一个数字,不各自重算
539
- expect(html).toContain("50%");
540
- expect(term).toContain("50%");
541
- });
542
-
543
- it("text 面先给比较表,再按 experiment Eval / Attempt 父子表", () => {
544
- const blocks = term.split("\n\n");
545
- expect(blocks).toHaveLength(experimentListItems.length + 1);
546
- expect(blocks[0]).toContain("Model");
547
- expect(blocks[0]).toContain("Avg");
548
- expect(blocks[1]!.match(/algebra\/quadratic/g)).toHaveLength(1);
549
- expect(blocks[1]).toMatch(/✗ failed\s+algebra\/quadratic[\s\S]*├─ @1a4a4a4[\s\S]*└─ @1b5b5b5/);
460
+ // web 面把警告的 command 渲染为可复制命令
461
+ expect(html).toContain("niceeval exp dual/a");
462
+ expect(html).toContain("nre-warning-command");
463
+ });
464
+
465
+ it("web 排序/过滤只改变浏览状态:有无 filter prop 数值与行集合相同", async () => {
466
+ const scope = scopeOf([snap({ experimentId: "f/a", results: [res("q", "passed")] })]);
467
+ const definition = (filter: boolean) =>
468
+ defineReport(<MetricTable rows="experiment" columns={[endToEndPassRate]} filter={filter} />);
469
+ const ctx = { scope, results: resultsOf(scope.snapshots) };
470
+ const plain = await renderReportToStaticHtml(definition(false), ctx);
471
+ const filtered = await renderReportToStaticHtml(definition(true), ctx);
472
+ const values = (html: string) => html.match(/data-sort-value="[^"]*"/g);
473
+ expect(values(filtered)).toEqual(values(plain));
474
+ });
475
+
476
+ it("ExperimentList text 面保持实体层级:一题两 attempt 只出现一次 Eval 标题;失败摘要只在 Attempt 子行", async () => {
477
+ const s = snap({
478
+ experimentId: "grp/exp-a",
479
+ results: [
480
+ res("algebra/retry", "failed", {
481
+ attempt: 0,
482
+ assertions: [
483
+ {
484
+ name: "equals",
485
+ severity: "gate",
486
+ outcome: "failed",
487
+ score: 0,
488
+ detail: "equals(42)",
489
+ expected: "42",
490
+ received: "41",
491
+ },
492
+ ],
493
+ }),
494
+ res("algebra/retry", "passed", { attempt: 1 }),
495
+ ],
496
+ });
497
+ const text = await renderTreeText(<ExperimentList relativeTo="grp" />, scopeOf([s]), 160);
498
+ expect(text.match(/algebra\/retry/g)).toHaveLength(1); // Eval 父行只出现一次
499
+ expect(text).toContain("├─");
500
+ expect(text).toContain("└─");
501
+ // 失败摘要只在 Attempt 子行(父行不复述)
502
+ expect(text.match(/equals\(42\)/g)).toHaveLength(1);
503
+ // relativeTo:行标签只显示末段
504
+ expect(text).toContain("exp-a");
550
505
  });
551
506
  });
552
507
 
553
- // ───────────────────────── 排版原语 ─────────────────────────
554
-
555
- describe("排版原语", () => {
556
- it("Col 纵向堆叠;Section 标题 + 缩进;Text 折行;Style text 面为空", () => {
557
- const node = (
558
- <Col>
559
- <Section title="考试成绩单">
560
- <Text>algebra 权重 ×2,满分 100。</Text>
561
- </Section>
562
- <Style>{`.passbars i { background: #4a7; }`}</Style>
563
- </Col>
564
- );
565
- expect(text(node)).toMatchInlineSnapshot(`
566
- "考试成绩单
567
- algebra 权重 ×2,满分 100。"
568
- `);
569
- const html = renderToStaticMarkup(node);
570
- expect(html).toContain("<h2 class=\"nre-section-title\">考试成绩单</h2>");
571
- expect(html).toContain("<style>.passbars i { background: #4a7; }</style>");
572
- });
573
-
574
- it("Row:宽度够时字符分栏,│ 分隔,各栏折到自己的宽度", () => {
575
- const node = (
576
- <Row>
577
- <Text>left column body</Text>
578
- <Text>right column body</Text>
579
- </Row>
580
- );
581
- const wide = renderNodeToText(node, createTextContext({ width: 60 }));
582
- expect(wide).toContain("");
583
- expect(wide.split("\n")[0]).toContain("left column body");
584
- expect(wide.split("\n")[0]).toContain("right column body");
585
- });
586
-
587
- it("Row:宽度不足降级纵向,不硬挤", () => {
588
- const node = (
589
- <Row>
590
- <Text>left column body</Text>
591
- <Text>right column body</Text>
592
- </Row>
593
- );
594
- const narrow = renderNodeToText(node, createTextContext({ width: 30 }));
595
- expect(narrow).not.toContain("│");
596
- expect(narrow).toContain("left column body\n\nright column body");
508
+ // ───────────────────────── FailureList ─────────────────────────
509
+
510
+ describe("FailureList", () => {
511
+ it("与手写组合严格等价:failed/errored、开始时间降序、limit 截断且 total 报截断前总数", async () => {
512
+ const s = snap({
513
+ experimentId: "fail/a",
514
+ results: [
515
+ res("q1", "failed", { startedAt: "2026-07-01T01:00:00.000Z" }),
516
+ res("q2", "errored", {
517
+ startedAt: "2026-07-01T03:00:00.000Z",
518
+ error: { code: "x", message: "boom", phase: "eval.run" },
519
+ }),
520
+ res("q3", "failed", { startedAt: "2026-07-01T02:00:00.000Z" }),
521
+ res("q4", "passed", { startedAt: "2026-07-01T04:00:00.000Z" }),
522
+ ],
523
+ });
524
+ const scope = scopeOf([s]);
525
+ const failureText = await renderTreeText(<FailureList limit={2} />, scope);
526
+
527
+ // 手写组合:attemptListData → 过滤 → 排序 → AttemptList data 形态
528
+ const all = await attemptListData(scope);
529
+ const startedAt = new Map(s.attempts.map((a) => [a.evalId, a.result.startedAt ?? ""]));
530
+ const failures = all
531
+ .filter((x) => x.verdict === "failed" || x.verdict === "errored")
532
+ .sort((a, b) => (startedAt.get(b.evalId) ?? "").localeCompare(startedAt.get(a.evalId) ?? ""));
533
+ const manualText = await renderTreeText(<AttemptList data={failures.slice(0, 2)} total={failures.length} />, scope);
534
+ expect(failureText).toBe(manualText);
535
+ expect(failureText).toContain("(1 more not shown)"); // 3 条失败,截断到 2
536
+ // 最近的失败在前
537
+ expect(failureText.indexOf("q2")).toBeLessThan(failureText.indexOf("q3"));
538
+ expect(failureText).not.toContain("q4"); // passed 不进失败清单
539
+ });
540
+
541
+ it("失败数少于 limit 时 total 等于 data 长度,不产出截断文案", async () => {
542
+ const s = snap({ experimentId: "fail/few", results: [res("q1", "failed")] });
543
+ const text = await renderTreeText(<FailureList />, scopeOf([s]));
544
+ expect(text).not.toContain("more not shown");
597
545
  });
598
546
  });
599
547
 
600
- // ───────────────────────── Table(自定义表的标准件)─────────────────────────
601
- //
602
- // 官方的 MetricTable / MetricMatrix / Scoreboard / DeltaTable 的 text 面就建在它上面
603
- // (上面那几个 describe 的内联快照即它的输出);这里验收作者直接用它时的四条契约:
604
- // 显示宽度对齐(中文不撕歪)、右对齐、缺数据 —、超宽折行。
605
-
606
- /** 某个片段的右边缘落在第几显示列 —— 右对齐验收看的就是它。 */
607
- function displayEndOf(line: string, piece: string): number {
608
- const at = line.indexOf(piece);
609
- expect(at).toBeGreaterThanOrEqual(0);
610
- return stringWidth(line.slice(0, at + piece.length));
611
- }
548
+ // ───────────────────────── Table 与排版原语 ─────────────────────────
612
549
 
613
- describe("Table 双面", () => {
614
- const cjkTable = (
615
- <Table
616
- columns={[
617
- { key: "eval", header: "题目" },
618
- { key: "pass", header: "通过率", align: "right" },
619
- { key: "cost", header: "成本", align: "right" },
620
- ]}
621
- rows={[
622
- {
623
- key: "记忆/写缓存",
624
- locator: "@160iuj3h" as AttemptLocator,
625
- cells: { eval: "记忆/写缓存", pass: "87%", cost: "$0.09" },
626
- },
627
- {
628
- key: "浏览/表单填写",
629
- locator: "@1qrdcfq8" as AttemptLocator,
630
- cells: { eval: "浏览/表单填写", pass: null, cost: null },
631
- },
632
- ]}
633
- />
634
- );
635
-
636
- it("中文列宽:按显示宽度对齐(CJK 记 2 列),不是 .length —— String.padEnd 撕歪表的那一步", () => {
637
- const term = text(cjkTable);
638
- expect(term).toMatchInlineSnapshot(`
639
- "题目 通过率 成本 attempt
640
- 记忆/写缓存 87% $0.09 @160iuj3h
641
- 浏览/表单填写 — — @1qrdcfq8"
642
- `);
643
-
644
- const [header, first, second] = term.split("\n");
645
- // 每列的右边缘落在同一显示列:中文格子多一倍显示宽度,补齐必须按显示宽度算
646
- expect(displayEndOf(first, "87%")).toBe(displayEndOf(header, "通过率"));
647
- expect(displayEndOf(second, "—")).toBe(displayEndOf(header, "通过率"));
648
- expect(displayEndOf(first, "$0.09")).toBe(displayEndOf(header, "成本"));
649
- expect(displayEndOf(first, "@160iuj3h")).toBe(displayEndOf(second, "@1qrdcfq8"));
650
- // 上面那几条不是巧合:UTF-16 码元数与显示宽度在中文上就是不同的两个数,
651
- // 拿 .length 补齐(或 String.prototype.padEnd)必然歪
652
- expect("记忆/写缓存".length).toBe(6);
653
- expect(stringWidth("记忆/写缓存")).toBe(11);
654
- });
550
+ describe("Table 与文本排版原语", () => {
551
+ const ctx = () => createTextContext({ width: 100 });
655
552
 
656
- it("align: \"right\":长短不一的数字右边缘对齐;默认列仍是左对齐", () => {
657
- const term = text(
553
+ it("null 单元格与 cells 缺键都渲染成 —,不补 0", () => {
554
+ const text = renderNodeToText(
658
555
  <Table
659
556
  columns={[
660
- { key: "agent", header: "agent" },
661
- { key: "cost", header: "cost", align: "right" },
557
+ { key: "a", header: "A" },
558
+ { key: "b", header: "B", align: "right" },
662
559
  ]}
663
560
  rows={[
664
- { key: "bub", cells: { agent: "bub", cost: "$9.00" } },
665
- { key: "codex", cells: { agent: "codex", cost: "$123.45" } },
561
+ { key: "r1", cells: { a: "x", b: null } },
562
+ { key: "r2", cells: { a: "y" } },
666
563
  ]}
667
564
  />,
565
+ ctx(),
668
566
  );
669
- expect(term).toMatchInlineSnapshot(`
670
- "agent cost
671
- bub $9.00
672
- codex $123.45"
673
- `);
674
- const [, first, second] = term.split("\n");
675
- expect(displayEndOf(first, "$9.00")).toBe(displayEndOf(second, "$123.45"));
676
- // 左对齐列的左边缘对齐(不传 align 就是 left)
677
- expect(first.startsWith("bub")).toBe(true);
678
- expect(second.startsWith("codex")).toBe(true);
567
+ expect(text.match(/—/g)!.length).toBe(2);
568
+ expect(text).not.toMatch(/\b0\b/);
679
569
  });
680
570
 
681
- it("null —,不补 0;cells 里干脆缺这个键也是 —;两面同源", () => {
682
- const node = (
571
+ it("cells 出现未声明的 key 以完整用户反馈报错;列 key 重复报错", () => {
572
+ expect(() =>
573
+ renderNodeToText(
574
+ <Table columns={[{ key: "a", header: "A" }]} rows={[{ key: "r", cells: { ghost: "x" } }]} />,
575
+ ctx(),
576
+ ),
577
+ ).toThrow(/no column declares/);
578
+ expect(() =>
579
+ renderNodeToText(
580
+ <Table
581
+ columns={
582
+ [
583
+ { key: "a", header: "A" },
584
+ { key: "a", header: "A2" },
585
+ ] as never
586
+ }
587
+ rows={[]}
588
+ />,
589
+ ctx(),
590
+ ),
591
+ ).toThrow(/declared twice/);
592
+ });
593
+
594
+ it("带 locator 的行多出 attempt 列;全部无 locator 时不出该列", () => {
595
+ const withLocator = renderNodeToText(
683
596
  <Table
684
- columns={[
685
- { key: "agent", header: "agent" },
686
- { key: "pass", header: "pass", align: "right" },
687
- { key: "cost", header: "cost", align: "right" },
688
- ]}
597
+ columns={[{ key: "a", header: "A" }]}
689
598
  rows={[
690
- { key: "bub", cells: { agent: "bub", pass: "87%", cost: "$0.42" } },
691
- { key: "claude", cells: { agent: "claude", pass: null } }, // cost 键根本不在
599
+ { key: "r1", cells: { a: "x" }, locator: "@1abc0def" as never },
600
+ { key: "r2", cells: { a: "y" } },
692
601
  ]}
693
- />
602
+ />,
603
+ ctx(),
694
604
  );
695
- const term = text(node);
696
- const html = renderToStaticMarkup(node);
697
- expect(term).toMatchInlineSnapshot(`
698
- "agent pass cost
699
- bub 87% $0.42
700
- claude — —"
701
- `);
702
- for (const face of [term, html]) {
703
- expect(face).toContain("—");
704
- expect(face).not.toContain("$0.00");
705
- expect(face).not.toContain("0%");
706
- }
707
- expect(html).toContain('<td class="nre-align-right"><span class="nre-missing">—</span></td>');
708
- });
709
-
710
- it("超宽:先折最宽的左对齐列(数字列不折),压到下限仍放不下就丢列并如实报数", () => {
711
- const wide = (
712
- <Table
713
- columns={[
714
- { key: "eval", header: "eval" },
715
- { key: "reason", header: "reason" },
716
- { key: "cost", header: "cost", align: "right" },
717
- ]}
718
- rows={[
719
- {
720
- key: "weather/brooklyn",
721
- cells: {
722
- eval: "weather/brooklyn",
723
- reason: "gate calledTool(\"get_weather\") — the tool was never called by the agent",
724
- cost: "$0.04",
725
- },
726
- },
727
- ]}
728
- />
605
+ expect(withLocator).toContain("@1abc0def");
606
+ expect(withLocator).toContain("attempt");
607
+ const without = renderNodeToText(
608
+ <Table columns={[{ key: "a", header: "A" }]} rows={[{ key: "r1", cells: { a: "x" } }]} />,
609
+ ctx(),
729
610
  );
730
- const narrow = renderNodeToText(wide, createTextContext({ width: 48 }));
731
- // 折行:一条逻辑行铺成多条物理行,没有一行溢出 48 列
732
- const lines = narrow.split("\n");
733
- expect(lines.length).toBeGreaterThan(2);
734
- for (const line of lines) expect(stringWidth(line)).toBeLessThanOrEqual(48);
735
- // 数字列没被折:$0.04 完整出现在某一行上
736
- expect(narrow).toContain("$0.04");
737
- expect(narrow).toContain("never called");
738
-
739
- // 压到下限还是放不下 → 从右侧丢列,如实报剩余列数(不静默截断)
740
- const tiny = renderNodeToText(wide, createTextContext({ width: 20 }));
741
- expect(tiny).toContain("(1 more column not shown)");
742
- expect(tiny).not.toContain("$0.04");
611
+ expect(without).not.toContain("attempt");
743
612
  });
744
613
 
745
- it("行带 locator:web 面链到证据室,text 面多一列 attempt", () => {
746
- const html = renderToStaticMarkup(cjkTable);
747
- expect(html).toContain('<a class="nre-locator" href="#/attempt/@160iuj3h">@160iuj3h</a>');
748
- expect(html).toContain("<thead>");
749
- expect(html).toContain("<tbody>");
750
- expect(text(cjkTable)).toContain("@160iuj3h");
751
- });
752
- });
753
-
754
- // ───────────────────────── 树校验与自定义组件 ─────────────────────────
755
-
756
- describe("渲染前树校验", () => {
757
- it("字符串 intrinsic(<div>)报错,错误指名组件路径", () => {
758
- const bad = (
759
- <Col>
760
- <Section title="x">
761
- <div>raw</div>
762
- </Section>
763
- </Col>
614
+ it("Row text 面:宽度装得下按显示宽度并排,装不下整块纵向堆叠、内容完整", () => {
615
+ const tree = (
616
+ <Row>
617
+ <Text>left block</Text>
618
+ <Text>right block</Text>
619
+ </Row>
764
620
  );
765
- expect(() => validateReportTree(bad)).toThrow(
766
- /Raw HTML <div> has no terminal face; use <Text>, layout primitives, or a defineComponent component\. \(in <Col> > <Section>\)/,
621
+ const wide = renderNodeToText(tree, createTextContext({ width: 80 }));
622
+ expect(wide.split("\n")[0]).toContain("left block");
623
+ expect(wide.split("\n")[0]).toContain("right block");
624
+ const narrow = renderNodeToText(tree, createTextContext({ width: 20 }));
625
+ expect(narrow).toContain("left block");
626
+ expect(narrow).toContain("right block");
627
+ expect(narrow.split("\n")[0]).not.toContain("right block"); // 纵向堆叠
628
+ });
629
+
630
+ it("Style 在 text 面零输出,web 面吐 <style>(页级全局)", async () => {
631
+ const scope = scopeOf([]);
632
+ const definition = defineReport(
633
+ <Col>
634
+ <Style>{`.nre .x { color: red; }`}</Style>
635
+ <Text>visible</Text>
636
+ </Col>,
767
637
  );
638
+ const hostCtx = { scope, results: resultsOf([]) };
639
+ const text = await renderReportToText(definition, hostCtx);
640
+ expect(text).toBe("visible");
641
+ const html = await renderReportToStaticHtml(definition, hostCtx);
642
+ expect(html).toContain("<style>");
768
643
  });
644
+ });
769
645
 
770
- it("普通函数组件(组合页面片段)被调用展开继续校验", () => {
771
- const Fragmented = () => (
772
- <Col>
773
- <Text>ok</Text>
774
- </Col>
646
+ // ───────────────────────── Tabs ─────────────────────────
647
+
648
+ describe("Tabs", () => {
649
+ it("两面都输出全部 tab 完整内容:web 每 tab 一个 <details> 且仅首个 open;text 按声明序分节不省略", async () => {
650
+ const scope = scopeOf([]);
651
+ const definition = defineReport(
652
+ <Tabs>
653
+ <Tab title="First">
654
+ <Text>alpha body</Text>
655
+ </Tab>
656
+ <Tab title="Second">
657
+ <Text>beta body</Text>
658
+ </Tab>
659
+ </Tabs>,
775
660
  );
776
- expect(() => validateReportTree(<Fragmented />)).not.toThrow();
777
- const BadInside = () => <span>nope</span>;
778
- expect(() => validateReportTree(<BadInside />)).toThrow(/Raw HTML <span>/);
661
+ const hostCtx = { scope, results: resultsOf([]) };
662
+ const html = await renderReportToStaticHtml(definition, hostCtx);
663
+ expect(html.match(/<details/g)).toHaveLength(2);
664
+ expect(html.match(/<details[^>]* open/g)).toHaveLength(1);
665
+ expect(html).toContain("alpha body");
666
+ expect(html).toContain("beta body");
667
+ const text = await renderReportToText(definition, hostCtx);
668
+ expect(text.indexOf("First")).toBeLessThan(text.indexOf("Second"));
669
+ expect(text).toContain("alpha body");
670
+ expect(text).toContain("beta body"); // 不丢第二个 tab
671
+ });
672
+
673
+ it("空 Tabs、普通组件混作直接子节点、游离 Tab 都在树校验期给出完整用户反馈", () => {
674
+ expect(() => validateReportTree(<Tabs>{null}</Tabs>)).toThrow(/at least one <Tab>/);
675
+ expect(() =>
676
+ validateReportTree(
677
+ <Tabs>
678
+ <Text>stray</Text>
679
+ </Tabs>,
680
+ ),
681
+ ).toThrow(/only accepts <Tab>/);
682
+ expect(() =>
683
+ validateReportTree(
684
+ <Col>
685
+ <Tab title="loose">
686
+ <Text>x</Text>
687
+ </Tab>
688
+ </Col>,
689
+ ),
690
+ ).toThrow(/direct child of <Tabs>/);
779
691
  });
780
692
  });
781
693
 
782
- describe("defineComponent", () => {
783
- it("faces 两键必填,少一个面在运行时也拦住(编译期本就不过)", () => {
784
- // @ts-expect-error 缺 text 面编译不过;运行时同样报错
785
- expect(() => defineComponent({ web: () => null })).toThrow(/requires both faces/);
786
- });
694
+ // ───────────────────────── defineReport 装载规范化 ─────────────────────────
787
695
 
788
- // custom-reports.mdx「换形态」的 PassBars 示例,逐字同源:text 面用公开的文本排版工具箱
789
- // (stringWidth / padEnd / bar),不用 String.prototype.padEnd —— 中文名的那一行是护栏。
790
- it("自定义双面组件:同一份数据两面判读一致,text 面拿到宽度;中文行不撕歪", () => {
791
- interface BarRow {
792
- key: string;
793
- ratio: number | null;
794
- display: string;
795
- }
796
- const PassBars = defineComponent<{ rows: BarRow[] }>({
797
- web({ rows }) {
798
- return (
799
- <ul className="passbars">
800
- {rows.map((r) => (
801
- <li key={r.key}>
802
- <span>{r.key}</span>
803
- <b>{r.ratio === null ? "—" : r.display}</b>
804
- </li>
805
- ))}
806
- </ul>
807
- );
808
- },
809
- text({ rows }, { width }) {
810
- expect(width).toBe(80);
811
- const label = Math.max(...rows.map((r) => stringWidth(r.key)));
812
- const barWidth = Math.min(20, width - label - 8);
813
- return rows
814
- .map((r) => {
815
- const chart = r.ratio === null ? padEnd("—", barWidth) : bar(r.ratio, barWidth);
816
- return `${padEnd(r.key, label)} ${chart} ${r.display}`;
817
- })
818
- .join("\n");
819
- },
696
+ describe("defineReport 装载规范化", () => {
697
+ it("三种写法装载出等价的规范化结果:树 {content} pages [{id: report}]", () => {
698
+ const tree = <ScopeSummary />;
699
+ const fromTree = defineReport(tree);
700
+ const fromContent = defineReport({ content: tree });
701
+ const fromPages = defineReport({
702
+ pages: [{ id: "report", title: { en: "Report", "zh-CN": "报告" }, content: tree }],
820
703
  });
821
- const rows: BarRow[] = [
822
- { key: "bub", ratio: 0.87, display: "87%" },
823
- { key: "codex", ratio: 0.8, display: "80%" },
824
- { key: "克劳德", ratio: null, display: "—" },
825
- ];
826
- const term = text(<PassBars rows={rows} />);
827
- expect(term).toMatchInlineSnapshot(`
828
- "bub █████████████████░░░ 87%
829
- codex ████████████████░░░░ 80%
830
- 克劳德 — —"
831
- `);
832
- // 三行的条形都从同一显示列开始:"克劳德" 是 3 个码元、6 个显示列,
833
- // 拿 .padEnd(label) 补齐会把这一行整体左移 3 列
834
- const starts = term.split("\n").map((line, i) => stringWidth(line.slice(0, line.indexOf(["█", "█", "—"][i]))));
835
- expect(new Set(starts).size).toBe(1);
836
-
837
- const html = renderToStaticMarkup(<PassBars rows={rows} />);
838
- expect(html).toContain("87%");
839
- expect(html).toContain("—"); // 缺数据两面都是 —,不补 0
704
+ for (const definition of [fromTree, fromContent, fromPages]) {
705
+ expect(definition.kind).toBe("report");
706
+ expect(definition.pages).toHaveLength(1);
707
+ expect(definition.pages[0]!.id).toBe("report");
708
+ expect(definition.pages[0]!.content).toBe(tree);
709
+ }
710
+ expect(fromTree.pages[0]!.title).toEqual(fromContent.pages[0]!.title);
840
711
  });
841
- });
842
712
 
843
- // ───────────────────────── defineReport 与两个宿主入口 ─────────────────────────
844
-
845
- function fakeContext(): { selection: Selection; results: Results } {
846
- const snapshot: Snapshot = (() => {
847
- const dir = "/results/compare_bub/snap-1";
848
- const base = {
849
- experimentId: "compare/bub",
850
- startedAt: "2026-07-01T10:00:00Z",
851
- completedAt: "2026-07-01T10:05:00Z",
852
- agent: "bub",
853
- schemaVersion: 1,
854
- dir,
855
- };
856
- const mk = (id: string, verdict: "passed" | "failed", index: number) => ({
857
- evalId: id,
858
- experimentId: "compare/bub",
859
- result: {
860
- id,
861
- agent: "bub",
862
- verdict,
863
- attempt: 0,
864
- startedAt: `2026-07-01T10:0${index}:00Z`,
865
- durationMs: 1000,
866
- assertions: [],
867
- },
868
- ref: { snapshot: "compare_bub/snap-1", attempt: `${id}/a0` },
869
- snapshot: base,
870
- events: async () => null,
871
- trace: async () => null,
872
- o11y: async () => null,
873
- diff: async () => null,
874
- sources: async () => null,
875
- });
876
- const attempts = [mk("algebra/x", "passed", 0), mk("algebra/y", "failed", 1)];
877
- return {
878
- ...base,
879
- evals: attempts.map((a) => ({ id: a.evalId, attempts: [a] })),
880
- attempts,
881
- } as unknown as Snapshot;
882
- })();
883
- const selection: Selection = {
884
- snapshots: [snapshot],
885
- warnings: [],
886
- filter: () => selection,
887
- };
888
- const results = {
889
- experiments: [{ id: "compare/bub", snapshots: [snapshot], latest: snapshot, evalIds: ["algebra/x", "algebra/y"] }],
890
- skipped: [],
891
- latest: () => selection,
892
- } as Results;
893
- return { selection, results };
894
- }
895
-
896
- /**
897
- * 三份快照、三个 experiment("compare/bub"、"other/codex"、"solo"),全部通过、无成本——
898
- * 一个多实验 Selection 夹具:ExperimentList 出三项,MetricScatter 三个点都无成本
899
- * (0 可画点,如实走空态)。`filter` 落实真实语义(不像 fakeContext 那样恒等返回自己)。
900
- */
901
- function fakeMultiGroupContext(): { selection: Selection; results: Results } {
902
- const mkSnapshot = (experimentId: string, agent: string, dirSuffix: string): Snapshot => {
903
- const dir = `/results/${dirSuffix}`;
904
- const base = {
905
- experimentId,
906
- startedAt: "2026-07-01T10:00:00Z",
907
- completedAt: "2026-07-01T10:05:00Z",
908
- agent,
909
- schemaVersion: 1,
910
- dir,
911
- };
912
- const attempt = {
913
- evalId: "algebra/x",
914
- experimentId,
915
- result: {
916
- id: "algebra/x",
917
- agent,
918
- verdict: "passed" as const,
919
- attempt: 0,
920
- startedAt: "2026-07-01T10:00:00Z",
921
- durationMs: 1000,
922
- assertions: [],
923
- },
924
- ref: { snapshot: dirSuffix, attempt: "algebra/x/a0" },
925
- snapshot: base,
926
- events: async () => null,
927
- trace: async () => null,
928
- o11y: async () => null,
929
- diff: async () => null,
930
- sources: async () => null,
931
- };
932
- return {
933
- ...base,
934
- evals: [{ id: attempt.evalId, attempts: [attempt] }],
935
- attempts: [attempt],
936
- } as unknown as Snapshot;
937
- };
938
-
939
- const snapshots = [
940
- mkSnapshot("compare/bub", "bub", "compare_bub/snap-1"),
941
- mkSnapshot("other/codex", "codex", "other_codex/snap-1"),
942
- mkSnapshot("solo", "bub", "solo/snap-1"),
943
- ];
944
-
945
- const makeSelection = (snaps: Snapshot[]): Selection => ({
946
- snapshots: snaps,
947
- warnings: [],
948
- filter: (predicate) => makeSelection(snaps.filter(predicate)),
713
+ it("content pages 同时声明或都省略,装载报错且文案含 <ExperimentComparison /> 下一步", () => {
714
+ expect(() => defineReport({ content: <ScopeSummary />, pages: [] } as never)).toThrow(/both "content" and "pages"/);
715
+ expect(() => defineReport({ title: "X" } as never)).toThrow(/<ExperimentComparison \/>/);
949
716
  });
950
- const selection = makeSelection(snapshots);
951
- const results = {
952
- experiments: snapshots.map((s) => ({
953
- id: s.experimentId,
954
- snapshots: [s],
955
- latest: s,
956
- evalIds: ["algebra/x"],
957
- })),
958
- skipped: [],
959
- latest: () => selection,
960
- } as Results;
961
- return { selection, results };
962
- }
963
717
 
964
- describe("defineReport + 渲染入口", () => {
965
- // ExperimentList 没有 selection-form:build() 里直接 await .data(selection) 拿到普通数组
966
- // 再传 items;RunOverview / Scoreboard 同样是预计算的 data 形态。三者都在 build() 里
967
- // await,一棵树验证同一个 renderReportToText/renderReportToStaticHtml 管线消费得动它们。
968
- const report = defineReport(async ({ selection }) => {
969
- const experiments = await ExperimentList.data(selection);
970
- return (
971
- <Col>
972
- <RunOverview data={await RunOverview.data(selection)} />
973
- <ExperimentList items={experiments} />
974
- <Section title="考试成绩单">
975
- <Scoreboard data={await Scoreboard.data(selection, { rows: "agent", subjects: "evalGroup" })} />
976
- </Section>
977
- </Col>
718
+ it("defineReport 产物不是 ReportNode:页里放产物装载报错;树校验同样拒绝", () => {
719
+ const inner = defineReport(<ScopeSummary />);
720
+ expect(() => defineReport({ pages: [{ id: "a", title: "A", content: inner as never }] })).toThrow(
721
+ /shell cannot nest/,
978
722
  );
723
+ expect(() => defineReport(inner as never)).toThrow(/shell cannot nest/);
724
+ expect(() => validateReportTree([inner] as never)).toThrow(/not a report node/);
979
725
  });
980
726
 
981
- it("isReportDefinition 识别产物;非函数报错", () => {
982
- expect(isReportDefinition(report)).toBe(true);
983
- expect(isReportDefinition({})).toBe(false);
984
- // @ts-expect-error 非函数
985
- expect(() => defineReport(42)).toThrow(/expects a build function/);
986
- });
987
-
988
- it("renderReportToText:同一棵树走 text 面,build() 里 await 的实体列表数据原样渲染", async () => {
989
- const out = await renderReportToText(report, fakeContext(), { width: 100 });
990
- // RunOverview(data 形态,预计算)
991
- expect(out).toContain("1 experiment · 2 evals · 2 attempts");
992
- // ExperimentList:主行 + eval 级折叠计票 + 逐 Attempt locator 与失败诊断
993
- expect(out).toMatch(/compare\/bub\s+default\s+bub/);
994
- expect(out).toContain("1 passed / 1 failed");
995
- expect(out).toContain("50%");
996
- expect(out).toMatch(/✗ failed\s+algebra\/y[\s\S]*└─ @[0-9a-z]+/);
997
- // 自己的口径:成绩单
998
- expect(out).toContain("考试成绩单");
999
- expect(out).toContain("bub");
1000
- expect(out).toContain("50/100");
1001
- });
1002
-
1003
- it("renderReportToStaticHtml:同一棵树走 web 面,ExperimentList 自动接证据室深链", async () => {
1004
- const html = await renderReportToStaticHtml(report, fakeContext());
1005
- expect(html).toContain("nre-experiment-list");
1006
- expect(html).toContain("nre-scoreboard");
1007
- expect(html).toContain("考试成绩单");
1008
- // 宿主注入的 attemptHref:ExperimentList 的 locator 徽标是普通 <a>,默认 view 路由(单段 locator)
1009
- expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/);
1010
- expect(html).not.toContain("<script");
727
+ it("重复或非法 page id 装载报错并点名冲突;LocalizedText 全空对象报错", () => {
728
+ expect(() =>
729
+ defineReport({
730
+ pages: [
731
+ { id: "exam", title: "A", content: null },
732
+ { id: "exam", title: "B", content: null },
733
+ ],
734
+ }),
735
+ ).toThrow(/"exam" is declared twice/);
736
+ expect(() => defineReport({ pages: [{ id: "Bad/Id", title: "A", content: null }] })).toThrow(/invalid/);
737
+ expect(() => defineReport({ title: {}, content: null })).toThrow(/no non-empty value/);
738
+ });
739
+
740
+ it("{src} 资产拒绝 .. 路径段、绝对路径与 ~;inline/src 互斥", () => {
741
+ expect(() => defineReport({ content: null, scripts: [{ src: "../x.js" }] })).toThrow(/not allowed/);
742
+ expect(() => defineReport({ content: null, scripts: [{ src: "/abs.js" }] })).toThrow(/not allowed/);
743
+ expect(() => defineReport({ content: null, styles: [{ src: "~/x.css" }] })).toThrow(/not allowed/);
744
+ expect(() => defineReport({ content: null, scripts: [{ src: "./a.js", inline: "x" } as never] })).toThrow(
745
+ /exactly one/,
746
+ );
747
+ expect(() => defineReport({ content: null, scripts: [{ src: "./assets/a.js" }] })).not.toThrow();
1011
748
  });
1012
749
 
1013
- it("RunOverview 已渲染 Selection warning 时,宿主不会在 text/web 顶部重复一遍", async () => {
1014
- const context = fakeContext();
1015
- const message = "snapshot is unfinished; completed attempts may be incomplete";
1016
- context.selection.warnings.push({
1017
- kind: "unfinished-snapshot",
1018
- experimentId: "compare/bub",
1019
- startedAt: "2026-07-01T10:00:00Z",
1020
- dir: "/results/compare_bub/snap-1",
1021
- message,
750
+ it("--page 语义:pickReportPage 缺省第一页,未命中抛 ReportPageNotFoundError 列出可用页", () => {
751
+ const definition = defineReport({
752
+ pages: [
753
+ { id: "overview", title: "Overview", content: null },
754
+ { id: "exam", title: "Exam", content: null },
755
+ ],
1022
756
  });
1023
- const textOut = await renderReportToText(report, context, { width: 100 });
1024
- const htmlOut = await renderReportToStaticHtml(report, context);
1025
- expect(textOut.split(message)).toHaveLength(2);
1026
- expect(htmlOut.split(message)).toHaveLength(2);
1027
- });
1028
-
1029
- it("树里混进 <div>:两个宿主 resolve 后同一遍校验拦住", async () => {
1030
- const bad = defineReport(() => (
1031
- <Col>
1032
- <div>nope</div>
1033
- </Col>
1034
- ));
1035
- await expect(renderReportToText(bad, fakeContext())).rejects.toThrow(/Raw HTML <div>/);
1036
- await expect(renderReportToStaticHtml(bad, fakeContext())).rejects.toThrow(/Raw HTML <div>/);
757
+ expect(pickReportPage(definition).id).toBe("overview");
758
+ expect(pickReportPage(definition, "exam").id).toBe("exam");
759
+ try {
760
+ pickReportPage(definition, "typo");
761
+ expect.unreachable();
762
+ } catch (e) {
763
+ expect(e).toBeInstanceOf(ReportPageNotFoundError);
764
+ expect((e as ReportPageNotFoundError).available).toEqual(["overview", "exam"]);
765
+ }
766
+ // 树形态文件的唯一页 id 是缩写展开出的 report
767
+ const single = defineReport(<ScopeSummary />);
768
+ expect(pickReportPage(single, "report").id).toBe("report");
769
+ });
770
+
771
+ it("标题回退链:def.title → 唯一且相同的快照 name → 内置文案「Eval 运行结果 / Eval Results」;en 相同 zh 不同也落内置文案", () => {
772
+ const named = snap({ experimentId: "t/a", name: "Memory Evals", results: [res("q", "passed")] });
773
+ const definition = defineReport(<ScopeSummary />);
774
+ expect(resolveReportTitle(defineReport({ title: "Custom", content: null }), scopeOf([named]))).toBe("Custom");
775
+ expect(resolveReportTitle(definition, scopeOf([named]))).toBe("Memory Evals");
776
+ expect(resolveReportTitle(definition, scopeOf([snap({ experimentId: "t/b", results: [] })]))).toEqual(
777
+ FALLBACK_REPORT_TITLE,
778
+ );
779
+ const zhA = snap({ experimentId: "t/c", name: { en: "Same", "zh-CN": "一" }, results: [] });
780
+ const zhB = snap({ experimentId: "t/d", name: { en: "Same", "zh-CN": "二" }, results: [] });
781
+ expect(resolveReportTitle(definition, scopeOf([zhA, zhB]))).toEqual(FALLBACK_REPORT_TITLE);
782
+ expect(FALLBACK_REPORT_TITLE).toEqual({ en: "Eval Results", "zh-CN": "Eval 运行结果" });
1037
783
  });
1038
784
 
1039
- it("selection 形态组件裸嵌进 React(不经宿主 resolve):web 面直说未解析", () => {
1040
- const { selection } = fakeContext();
1041
- // selection 形态在类型上合法,但裸调用路径(不过 resolveReportTree)只接收 data 形态,
1042
- // web 面拿不到 data —— 运行时直说,而不是画一张空组件。MetricScatter 是本文件里仍带
1043
- // selection-form 的官方组件(ExperimentList/EvalList/AttemptList 没有 selection-form,
1044
- // 这条契约不适用于它们)
1045
- expect(() =>
1046
- renderToStaticMarkup(<MetricScatter selection={selection} points="experiment" x={costUSD} y={passRate} />),
1047
- ).toThrow(/received unresolved \(selection-form\) props/);
785
+ it("ReportLink.icon { svg: string }:defineReport 接受合法形状;无类型 JS 传其它形状定义时报完整用户反馈", () => {
786
+ const svg = '<svg viewBox="0 0 16 16"><path d="M0 0h16v16z"/></svg>';
787
+ const withIcon = defineReport({
788
+ content: null,
789
+ links: [{ label: "GitHub", href: "https://example.com", icon: { svg } }],
790
+ });
791
+ expect(withIcon.links[0]!.icon).toEqual({ svg });
792
+
793
+ const reactNode = { $$typeof: Symbol.for("react.transitional.element"), type: "svg", props: {} };
794
+ for (const icon of [reactNode, "<svg/>", { svg: 42 }, { svg: "" }]) {
795
+ expect(() =>
796
+ defineReport({
797
+ content: null,
798
+ links: [{ label: "GitHub", href: "https://example.com", icon: icon as never }],
799
+ }),
800
+ ).toThrow(/"icon" must be \{ svg: string \}/);
801
+ }
1048
802
  });
1049
803
  });
1050
804
 
1051
- // ───────────────────────── 数据组件互斥 props(类型层负向)─────────────────────────
1052
- // 只在 pnpm run typecheck 生效(src 纳入 tsconfig);函数从不执行,仅让 tsc 校验每条
1053
- // ts-expect-error 注释对应一处编译错误。直接标注 Props 类型(而非 JSX)——JSX union
1054
- // 属性检查对「缺必填字段」较宽松,直接的类型赋值才严格钉住互斥不变量:同时传 data
1055
- // selection、或两者都不传、或 selection 形态缺必填计算选项,全部编译失败。正向 JSX 用法由
1056
- // ExperimentComparison / defineReport 报告在同一文件里编译验证。
1057
- function metricScatterPropsTypeChecks(selection: Selection, data: ScatterData): void {
1058
- const ok1: MetricScatterProps = { data }; // 合法:data 形态
1059
- const ok2: MetricScatterProps = { selection, points: "experiment", series: "agent", x: costUSD, y: passRate }; // 合法:selection 形态
1060
- // @ts-expect-error 同时传 data 与 selection:非法
1061
- const bad1: MetricScatterProps = { data, selection, points: "experiment", x: costUSD, y: passRate };
1062
- // @ts-expect-error data selection 都不传:非法
1063
- const bad2: MetricScatterProps = { pointHref: () => "/x" };
1064
- // @ts-expect-error selection 形态缺必填的 x / y:非法
1065
- const bad3: MetricScatterProps = { selection, points: "experiment" };
1066
- void ok1;
1067
- void ok2;
1068
- void bad1;
1069
- void bad2;
1070
- void bad3;
1071
- }
1072
-
1073
- void metricScatterPropsTypeChecks;
1074
-
1075
- // ───────────────────────── ExperimentComparison(内置默认报告)─────────────────────────
1076
-
1077
- describe("ExperimentComparison", () => {
1078
- it("是普通 ReportDefinition;text 面 = 成本×通过率散点 + 实验列表,别无它物", async () => {
1079
- expect(isReportDefinition(ExperimentComparison)).toBe(true);
1080
- const out = await renderReportToText(ExperimentComparison, fakeContext(), { width: 100 });
1081
- // 散点:fakeContext 无成本数据 → 0 可画点,显式说明缺哪两个指标(而不是画一张空图)
1082
- expect(out).toContain("No data to plot");
1083
- expect(out).not.toContain("better → upper right");
1084
- // 实验列表主行 + eval 级折叠计票 + 失败诊断(ExperimentList.data 在 build() 里直接 await)
1085
- expect(out).toMatch(/compare\/bub\s+default\s+bub/);
1086
- expect(out).toContain("1 passed / 1 failed");
1087
- expect(out).toContain("50%");
1088
- expect(out).toMatch(/✗ failed\s+algebra\/y[\s\S]*└─ @[0-9a-z]+/);
1089
- // 只有两个直接业务组件:没有 RunOverview / GroupSummary / Section 分组
1090
- expect(out).not.toContain("Current verdicts");
1091
- });
1092
-
1093
- it("web 面:散点空态 + 实验列表 <details> 展开区,无 <script>,无 Section 分组", async () => {
1094
- const html = await renderReportToStaticHtml(ExperimentComparison, fakeContext());
1095
- expect(html).toContain("nre-metric-scatter");
1096
- expect(html).toContain("nre-scatter-empty"); // 0 可画点的空态
1097
- expect(html).toContain('<details class="nre-experiment-entry">');
1098
- expect(html).toContain("nre-experiment-head");
1099
- expect(html).toContain('data-nre-experiment-filter=""');
1100
- expect(html).toContain("nre-experiment-evals");
1101
- expect(html).not.toContain("nre-section");
1102
- expect(html).not.toContain("<script");
1103
- });
1104
-
1105
- it("locale 变体:en / zh-CN 都渲染(chrome 分语言),散点空态两面同一事实", async () => {
1106
- const zhHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
1107
- expect(zhHtml).toContain("成功率"); // passRate 的 zh-CN label(ExperimentList 主行)
1108
- const enHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "en" });
1109
- expect(enHtml).toContain("Pass rate");
1110
- const zhText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
1111
- expect(zhText).toContain("没有可绘制的数据"); // 散点空态 zh
1112
- const enText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "en" });
1113
- expect(enText).toContain("No data to plot");
1114
- });
805
+ // ───────────────────────── 内建报告 ─────────────────────────
806
+
807
+ describe("内建报告", () => {
808
+ it("niceeval/report/built-in 默认导出是普通 defineReport 产物,与用户写的 defineReport(<ExperimentComparison />) 等价", async () => {
809
+ expect(builtInReport.kind).toBe("report");
810
+ expect(builtInReport.pages).toHaveLength(1);
811
+
812
+ const s1 = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
813
+ const s2 = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
814
+ const scope = scopeOf([s1, s2]);
815
+ const ctx = { scope, results: resultsOf([s1, s2]) };
816
+ const builtin = await renderReportToText(builtInReport, ctx, { width: 120 });
817
+ const user = await renderReportToText(defineReport(<ExperimentComparison />), ctx, { width: 120 });
818
+ expect(builtin).toBe(user);
819
+ });
820
+
821
+ it("text 面多组时只输出组索引与单组查看命令;单组时输出散点与实验列表", async () => {
822
+ const g1 = snap({ experimentId: "compare/a", results: [res("q", "passed")] });
823
+ const g2 = snap({ experimentId: "dev/b", results: [res("q", "failed")] });
824
+ const multi = await renderReportToText(
825
+ builtInReport,
826
+ { scope: scopeOf([g1, g2]), results: resultsOf([g1, g2]) },
827
+ { width: 120 },
828
+ );
829
+ expect(multi).toContain("niceeval show --experiment compare");
830
+ expect(multi).toContain("niceeval show --experiment dev");
831
+ expect(multi).not.toContain("Eval / Attempt"); // 不倾倒组内 experiment 明细
1115
832
 
1116
- it("多实验 fixture:每个 experiment 一项,散点如实处理全部无成本", async () => {
1117
- const out = await renderReportToText(ExperimentComparison, fakeMultiGroupContext(), { width: 100 });
1118
- // 三个 experiment 的身份行都出现(experimentId · agent,不再截短成最后一段)
1119
- expect(out).toMatch(/compare\/bub\s+default\s+bub/);
1120
- expect(out).toMatch(/other\/codex\s+default\s+codex/);
1121
- expect(out).toMatch(/solo\s+default\s+bub/);
1122
- // 散点空态(三点都无成本)
1123
- expect(out).toContain("No data to plot");
1124
- // 没有组分 Section 标题(不再按目录前缀分组)
1125
- expect(out).not.toMatch(/^compare$/m);
1126
- const html = await renderReportToStaticHtml(ExperimentComparison, fakeMultiGroupContext());
1127
- expect(html).not.toContain("nre-section");
833
+ const priced = snap({
834
+ experimentId: "compare/priced",
835
+ results: [res("q", "passed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.2 } })],
836
+ });
837
+ const single = await renderReportToText(
838
+ builtInReport,
839
+ { scope: scopeOf([priced]), results: resultsOf([priced]) },
840
+ { width: 140 },
841
+ );
842
+ expect(single).toContain("Eval / Attempt"); // 单组直接进入组内详情
843
+ // 成本轴(better: lower)反向渲染,「更好」恒指向右上;两轴都声明 better 提示在场
844
+ expect(single).toContain("better → upper right");
1128
845
  });
1129
846
  });