niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -0,0 +1,89 @@
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // "validate*Data 递归覆盖到嵌套字段" 行:site-components 四个 validate*Data 的表驱动字段突变覆盖,重点是 ScopeWarning 的四个已登记
3
+ // kind 各自的必填字段,以及未登记 kind 的前向兼容放行路径(与 ScopeWarnings 组件的「未知 kind
4
+ // 单独成组」渲染回退是同一条契约,结构校验不能比渲染逻辑更严)。
5
+
6
+ import { describe, expect, it } from "vitest";
7
+ import { validateCopyFixPromptData, validateHeroData, validateScopeWarningsData, validateTraceWaterfallData } from "./index.tsx";
8
+
9
+ describe("validateHeroData", () => {
10
+ it("合规 literal 通过,latestStartedAt 为 null 合法(空范围不编造当前时间)", () => {
11
+ expect(validateHeroData({ latestStartedAt: null, snapshots: 0 })).toBeNull();
12
+ });
13
+
14
+ it("snapshots 非数字报错", () => {
15
+ expect(validateHeroData({ latestStartedAt: null, snapshots: "0" })).toMatch(/"snapshots"/);
16
+ });
17
+ });
18
+
19
+ describe("validateScopeWarningsData — ScopeWarning 判别联合", () => {
20
+ it("partial-coverage 缺 covered 报错", () => {
21
+ const bad = [{ kind: "partial-coverage", experimentId: "exp/a", total: 6, message: "m", command: "niceeval exp exp/a" }];
22
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.covered"/);
23
+ });
24
+
25
+ it("stale-snapshot 缺 latestStartedAt 报错", () => {
26
+ const bad = [{ kind: "stale-snapshot", experimentId: "exp/a", startedAt: "t1", message: "m", command: "c" }];
27
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.latestStartedAt"/);
28
+ });
29
+
30
+ it("unfinished-snapshot 缺 dir 报错", () => {
31
+ const bad = [{ kind: "unfinished-snapshot", experimentId: "exp/a", startedAt: "t1", message: "m", command: "c" }];
32
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.dir"/);
33
+ });
34
+
35
+ it("unreadable-snapshot 的 reason 不在三态枚举内报错;没有 experimentId 字段本身合法(非实验作用域)", () => {
36
+ const bad = [{ kind: "unreadable-snapshot", dir: "/r/snap", reason: "corrupted", message: "m" }];
37
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.reason"/);
38
+ const ok = [{ kind: "unreadable-snapshot", dir: "/r/snap", reason: "malformed", message: "m" }];
39
+ expect(validateScopeWarningsData(ok)).toBeNull();
40
+ });
41
+
42
+ it("unreadable-snapshot 省略 command 合法(不是每个 reason 都有单条命令)", () => {
43
+ const ok = [{ kind: "unreadable-snapshot", dir: "/r/snap", reason: "incomplete", message: "m" }];
44
+ expect(validateScopeWarningsData(ok)).toBeNull();
45
+ });
46
+
47
+ it("未登记的 kind 只要有 kind/message 就放行——前向兼容,不拒绝未来版本产出的新 kind", () => {
48
+ const future = [{ kind: "future-kind", message: "something new happened" }];
49
+ expect(validateScopeWarningsData(future)).toBeNull();
50
+ });
51
+
52
+ it("未登记的 kind 缺 message 仍报错(两族共用的最小形状)", () => {
53
+ const bad = [{ kind: "future-kind" }];
54
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.message"/);
55
+ });
56
+ });
57
+
58
+ describe("validateCopyFixPromptData", () => {
59
+ it("合规 literal 通过", () => {
60
+ expect(validateCopyFixPromptData({ prompt: "fix it", failures: 1 })).toBeNull();
61
+ });
62
+
63
+ it("failures 非数字报错", () => {
64
+ expect(validateCopyFixPromptData({ prompt: "x", failures: "1" })).toMatch(/"failures"/);
65
+ });
66
+ });
67
+
68
+ describe("validateTraceWaterfallData", () => {
69
+ const validSpan = { name: "turn", kind: "agent", startOffsetMs: 0, durationMs: 100, failed: false };
70
+ const valid = [{ experimentId: "exp/a", evalId: "q1", locator: "@1abcdef2", durationMs: 100, spans: [validSpan] }];
71
+
72
+ it("合规 literal 通过", () => {
73
+ expect(validateTraceWaterfallData(valid)).toBeNull();
74
+ });
75
+
76
+ it("durationMs 为 null 合法(trace 缺失时如实显示缺失)", () => {
77
+ expect(validateTraceWaterfallData([{ ...valid[0], durationMs: null }])).toBeNull();
78
+ });
79
+
80
+ it("spans[i].kind 不在四态枚举内报错", () => {
81
+ const bad = [{ ...valid[0], spans: [{ ...validSpan, kind: "runner" }] }];
82
+ expect(validateTraceWaterfallData(bad)).toMatch(/"data\[0\]\.spans\[0\]\.kind"/);
83
+ });
84
+
85
+ it("spans[i].failed 非布尔报错", () => {
86
+ const bad = [{ ...valid[0], spans: [{ ...validSpan, failed: "true" }] }];
87
+ expect(validateTraceWaterfallData(bad)).toMatch(/"data\[0\]\.spans\[0\]\.failed"/);
88
+ });
89
+ });
@@ -1,13 +1,14 @@
1
1
  // ScopeSummary:一个范围的摘要卡——快照时间窗、experiment / eval / attempt 数、判定计票、
2
- // 端到端成功率与总成本。data 恒携带 eval 级与 attempt 级两份计票;呈现 prop `votes`
3
- // 只选择显示哪一级(默认 "eval"),不改变 data。成功率与总成本只渲染算好的 MetricCell,
2
+ // 端到端通过率与总成本。data 恒携带 eval 级与 attempt 级两份计票;呈现 prop `votes`
3
+ // 只选择显示哪一级(默认 "eval"),不改变 data。通过率与总成本只渲染算好的 MetricCell,
4
4
  // 不现场重算(docs/feature/reports/library/summaries.md「ScopeSummary」)。
5
5
 
6
6
  import type { ReactElement } from "react";
7
- import type { ScopeSummaryData, VerdictTally } from "../types.ts";
8
- import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
9
- import { MetricCellView } from "./cell.tsx";
10
- import { cx } from "./format.ts";
7
+ import type { ScopeSummaryData, VerdictTally } from "../../model/types.ts";
8
+ import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../../model/locale.ts";
9
+ import { formatReportDateTime, formatReportDateTimeRange } from "../../model/format.ts";
10
+ import { MetricCellView } from "../cell.tsx";
11
+ import { cx } from "../shared.ts";
11
12
 
12
13
  function VerdictTallyView({ tally, locale }: { tally: VerdictTally; locale: ReportLocale }): ReactElement {
13
14
  const kinds = (["passed", "failed", "errored", "skipped"] as const).filter((k) => tally[k] > 0);
@@ -36,6 +37,10 @@ export function ScopeSummary({
36
37
  locale?: ReportLocale;
37
38
  }): ReactElement {
38
39
  const tally = votes === "attempt" ? data.attemptVerdicts : data.evalVerdicts;
40
+ const formattedRange =
41
+ data.range.earliestStartedAt !== null && data.range.latestStartedAt !== null
42
+ ? formatReportDateTimeRange(data.range.earliestStartedAt, data.range.latestStartedAt, locale)
43
+ : null;
39
44
  return (
40
45
  <div className={cx("nre", "nre-scope-summary", className)} data-votes={votes}>
41
46
  <dl className="nre-scope-kpis">
@@ -66,7 +71,15 @@ export function ScopeSummary({
66
71
  <div className="nre-scope-kpi">
67
72
  <dt>{localeText(locale, "scopeSummary.totalCost")}</dt>
68
73
  <dd>
69
- <MetricCellView cell={data.totalCostUSD} locale={locale} />
74
+ <MetricCellView cell={data.totalCostUSD} locale={locale} showCoverage={false} />
75
+ {data.totalCostUSD.samples < data.totalCostUSD.total && (
76
+ <small className="nre-scope-kpi-note">
77
+ {localeText(locale, "scopeSummary.costCoverage", {
78
+ samples: data.totalCostUSD.samples,
79
+ total: data.totalCostUSD.total,
80
+ })}
81
+ </small>
82
+ )}
70
83
  </dd>
71
84
  </div>
72
85
  </dl>
@@ -74,11 +87,13 @@ export function ScopeSummary({
74
87
  {data.range.latestStartedAt !== null && (
75
88
  <p className="nre-scope-summary-range">
76
89
  {data.range.earliestStartedAt !== null && data.range.earliestStartedAt !== data.range.latestStartedAt
77
- ? localeText(locale, "scopeSummary.range", {
78
- from: data.range.earliestStartedAt,
79
- to: data.range.latestStartedAt,
90
+ ? localeText(locale, "scopeSummary.runRange", {
91
+ from: formattedRange!.from,
92
+ to: formattedRange!.to,
80
93
  })
81
- : data.range.latestStartedAt}
94
+ : localeText(locale, "scopeSummary.lastRun", {
95
+ time: formatReportDateTime(data.range.latestStartedAt, locale),
96
+ })}
82
97
  </p>
83
98
  )}
84
99
  </div>
@@ -0,0 +1,56 @@
1
+ // 计算函数(*Data):ReportInput → 一份组件数据。ScopeSummary 的 scopeSummaryData 住在这里
2
+ // (docs/feature/reports/library/summaries.md)。ExperimentComparison 是纯组合组件,不产生
3
+ // 独立的 data,没有对应的计算函数。
4
+ //
5
+ // 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
6
+ // - 第一参收 ReportInput = Scope | readonly Snapshot[];warnings 不进组件数据(宿主统一显示);
7
+ // - 聚合前按身份键去重(dedupeAttempts;missing-startedAt 不去重、如实保留、不透出警告);
8
+ // - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露。
9
+
10
+ import type { ReportInput, ScopeSummaryData } from "../../model/types.ts";
11
+ import { collectItems, computeCell, resolveInput } from "../../model/aggregate.ts";
12
+ import { costUSD, defineMetric, endToEndPassRate } from "../../model/metrics.ts";
13
+ import { selectedEvalsOnly, summarizeItems, tallyOf } from "../shared-compute.ts";
14
+
15
+ // ───────────────────────── scopeSummaryData ─────────────────────────
16
+
17
+ /** costUSD 的求和投影:两级都 sum(题内多轮求和 + 跨题求和 = 全量求和),display 走 $。 */
18
+ const totalCostMetric = defineMetric({
19
+ name: "total-cost",
20
+ label: costUSD.label,
21
+ unit: "$",
22
+ value: costUSD.value,
23
+ aggregate: { perEval: "sum", acrossEvals: "sum" },
24
+ });
25
+
26
+ /**
27
+ * `scopeSummaryData(input)`:范围摘要——快照时间窗、experiment / eval / attempt 数、
28
+ * 两级判定计票、端到端通过率与总成本(docs/feature/reports/library/summaries.md)。
29
+ * data 恒携带两级计票;通过率来自官方两级指标引擎,不从任一计票重算。
30
+ */
31
+ export async function scopeSummaryData(input: ReportInput): Promise<ScopeSummaryData> {
32
+ const snapshots = selectedEvalsOnly(resolveInput(input).snapshots);
33
+ const items = collectItems(snapshots);
34
+
35
+ let earliest: string | null = null;
36
+ let latest: string | null = null;
37
+ for (const snapshot of snapshots) {
38
+ if (earliest === null || snapshot.startedAt < earliest) earliest = snapshot.startedAt;
39
+ if (latest === null || snapshot.startedAt > latest) latest = snapshot.startedAt;
40
+ }
41
+
42
+ const stats = summarizeItems(items);
43
+ const attemptVerdicts = tallyOf();
44
+ for (const item of items) attemptVerdicts[item.attempt.result.verdict] += 1;
45
+
46
+ return {
47
+ range: { earliestStartedAt: earliest, latestStartedAt: latest },
48
+ experiments: stats.experiments,
49
+ evals: stats.evals,
50
+ attempts: stats.attempts,
51
+ evalVerdicts: stats.verdicts,
52
+ attemptVerdicts,
53
+ endToEndPassRate: await computeCell(endToEndPassRate, items),
54
+ totalCostUSD: await computeCell(totalCostMetric, items),
55
+ };
56
+ }
@@ -0,0 +1,48 @@
1
+ // ScopeSummary 的 text 面:同一份算好的数据,渲染成终端字符(niceeval show 的形态)。
2
+ // 与 web 面共守诚实契约:排序随 better、samples < total 角标、缺数据 — 不补 0。
3
+ // 零 react、零 IO、纯同步。
4
+
5
+ import type { ScopeSummaryData } from "../../model/types.ts";
6
+ import type { TextContext } from "../../definition/tree.ts";
7
+ import { countText, localeText, resolveLocalizedText } from "../../model/locale.ts";
8
+ import { formatReportDateTime, formatReportDateTimeRange } from "../../model/format.ts";
9
+ import { wrapDisplay } from "../../model/text-layout.ts";
10
+ import { cellText, verdictTallyText, MISSING_MARK } from "../shared-faces.ts";
11
+
12
+ /**
13
+ * 一至两行:头行是端到端通过率(官方 MetricCell,不重算)+ experiment/eval/attempt 数 +
14
+ * `votes` 选中的那级计票 + 总成本;第二行(有则加)是快照时间窗。
15
+ */
16
+ export function scopeSummaryText(data: ScopeSummaryData, votes: "eval" | "attempt", ctx: TextContext): string {
17
+ const locale = ctx.locale;
18
+ const tally = votes === "attempt" ? data.attemptVerdicts : data.evalVerdicts;
19
+ const head = [
20
+ `${localeText(locale, "scopeSummary.passRate")} ${cellText(data.endToEndPassRate, locale)}`,
21
+ countText(locale, "overview.experiments", data.experiments),
22
+ localeText(locale, "overview.evalsCount", { n: data.evals }),
23
+ localeText(locale, "overview.attemptsCount", { n: data.attempts }),
24
+ verdictTallyText(tally, locale),
25
+ `${localeText(locale, "scopeSummary.totalCost")} ${
26
+ data.totalCostUSD.value === null ? MISSING_MARK : resolveLocalizedText(data.totalCostUSD.display, locale)
27
+ }${
28
+ data.totalCostUSD.samples < data.totalCostUSD.total
29
+ ? ` (${localeText(locale, "scopeSummary.costCoverage", {
30
+ samples: data.totalCostUSD.samples,
31
+ total: data.totalCostUSD.total,
32
+ })})`
33
+ : ""
34
+ }`,
35
+ ].join(" · ");
36
+ // 头行拼接的字段较多,窄终端下按显示宽度折行(不截断内容)。
37
+ const lines = wrapDisplay(head, ctx.width);
38
+ if (data.range.latestStartedAt !== null) {
39
+ const from = data.range.earliestStartedAt;
40
+ const to = data.range.latestStartedAt;
41
+ lines.push(
42
+ from !== null && from !== to
43
+ ? localeText(locale, "scopeSummary.runRange", formatReportDateTimeRange(from, to, locale))
44
+ : localeText(locale, "scopeSummary.lastRun", { time: formatReportDateTime(to, locale) }),
45
+ );
46
+ }
47
+ return lines.join("\n");
48
+ }
@@ -0,0 +1,127 @@
1
+ // 官方双面组件的装配点:web 面(./ScopeSummary.tsx 的纯 React 组件)+ text 面(./faces.ts)
2
+ // + resolve 解析面(spec 形态由管线代调配套 ./compute.ts)。ScopeSummary 与 ExperimentComparison
3
+ // 同属汇总族——前者是范围摘要卡,后者是内建报告默认使用的组合件(装配 ScopeSummary +
4
+ // MetricScatter + ExperimentList,不产生自己的 data)。
5
+
6
+ import { defineComponent, type ReportComponent } from "../../definition/tree.ts";
7
+ import { Col } from "../../definition/primitives.tsx";
8
+ import type { ReportInput, ScopeSummaryData, SeriesInput } from "../../model/types.ts";
9
+ import { resolveInput, seriesName } from "../../model/aggregate.ts";
10
+ import { label } from "../../model/flag.ts";
11
+ import { costUSD, endToEndPassRate } from "../../model/metrics.ts";
12
+ import {
13
+ cellProblem,
14
+ isObject,
15
+ makeDataComponent,
16
+ tallyProblem,
17
+ type ChromeProps,
18
+ type DataProps,
19
+ type Validator,
20
+ } from "../shared.ts";
21
+ import { scopeSummaryData } from "./compute.ts";
22
+ import { scopeSummaryText } from "./faces.ts";
23
+ import { ScopeSummary as ScopeSummaryWeb } from "./ScopeSummary.tsx";
24
+ import { MetricScatter } from "../metric-views/index.tsx";
25
+ import { ExperimentList } from "../entity-lists/index.tsx";
26
+
27
+ export const validateScopeSummaryData: Validator = (data) => {
28
+ if (!isObject(data)) return "expected an object";
29
+ if (!isObject(data.range)) return 'missing "range" ({ earliestStartedAt, latestStartedAt })';
30
+ if (!(data.range.earliestStartedAt === null || typeof data.range.earliestStartedAt === "string")) {
31
+ return '"range.earliestStartedAt" must be a string or null';
32
+ }
33
+ if (!(data.range.latestStartedAt === null || typeof data.range.latestStartedAt === "string")) {
34
+ return '"range.latestStartedAt" must be a string or null';
35
+ }
36
+ if (typeof data.experiments !== "number") return '"experiments" must be a number';
37
+ if (typeof data.evals !== "number") return '"evals" must be a number';
38
+ if (typeof data.attempts !== "number") return '"attempts" must be a number';
39
+ const evalVerdictsProblem = tallyProblem(data.evalVerdicts, "evalVerdicts");
40
+ if (evalVerdictsProblem !== null) return evalVerdictsProblem;
41
+ const attemptVerdictsProblem = tallyProblem(data.attemptVerdicts, "attemptVerdicts");
42
+ if (attemptVerdictsProblem !== null) return attemptVerdictsProblem;
43
+ const passRateProblem = cellProblem(data.endToEndPassRate, "endToEndPassRate");
44
+ if (passRateProblem !== null) return passRateProblem;
45
+ return cellProblem(data.totalCostUSD, "totalCostUSD");
46
+ };
47
+
48
+ // ───────────────────────── 概览组件 ─────────────────────────
49
+
50
+ export type ScopeSummaryProps = DataProps<
51
+ ScopeSummaryData,
52
+ Record<never, never>,
53
+ ChromeProps & {
54
+ /** 显示哪一级计票;默认 "eval"。data 恒携带两级,votes 只选择呈现。 */
55
+ votes?: "eval" | "attempt";
56
+ }
57
+ >;
58
+
59
+ /** 范围摘要卡:时间窗、数量、两级计票、端到端通过率与总成本。 */
60
+ export const ScopeSummary = makeDataComponent<
61
+ ScopeSummaryData,
62
+ Record<never, never>,
63
+ ChromeProps & { votes?: "eval" | "attempt" }
64
+ >({
65
+ name: "ScopeSummary",
66
+ dataFnName: "scopeSummaryData",
67
+ shapeName: "ScopeSummaryData",
68
+ dataFn: (input) => scopeSummaryData(input),
69
+ specKeys: [],
70
+ validate: validateScopeSummaryData,
71
+ web: (props, ctx) => <ScopeSummaryWeb {...props} locale={props.locale ?? ctx.locale} />,
72
+ text: (props, ctx) => scopeSummaryText(props.data, props.votes ?? "eval", ctx),
73
+ }) as unknown as ReportComponent<ScopeSummaryProps>;
74
+
75
+ type ComparisonChrome = ChromeProps & {
76
+ /** 透传给散点;缺省跟随缺省 series 解析——按 line 归类时连线(声明了线就画线)。 */
77
+ connect?: boolean;
78
+ };
79
+
80
+ export type ExperimentComparisonProps = ComparisonChrome & {
81
+ input?: ReportInput;
82
+ /** 散点的 series 维度。缺省解析:Scope 内任一实验声明了 label `line` → `label("line")` 并连线;否则 `"agent"`、不连线。 */
83
+ series?: SeriesInput;
84
+ };
85
+
86
+ /** 默认报告识别的归类键:声明了它的实验按线归类并连线(docs/feature/experiments/library.md「labels」)。 */
87
+ const LINE_LABEL_KEY = "line";
88
+
89
+ /**
90
+ * 缺省 series:Scope 内任一快照声明了 labels.line 用 line 维度,否则 agent;显式传入覆盖。
91
+ * connect 缺省跟随最终 series 是否解析为 line(即便是显式传入的)。
92
+ */
93
+ function resolveComparisonSeries(
94
+ input: ReportInput,
95
+ props: { series?: SeriesInput; connect?: boolean },
96
+ ): { series: SeriesInput; connect: boolean } {
97
+ const hasLine = resolveInput(input).snapshots.some((s) => s.experiment?.labels?.[LINE_LABEL_KEY] !== undefined);
98
+ const series = props.series ?? (hasLine ? label(LINE_LABEL_KEY) : "agent");
99
+ return { series, connect: props.connect ?? seriesName(series) === LINE_LABEL_KEY };
100
+ }
101
+
102
+ /**
103
+ * 内建报告的默认组合件:把同一个 input(缺省 ctx.scope)原样透传给 ScopeSummary、
104
+ * 成本 × 端到端通过率的 MetricScatter 与 ExperimentList——组合本身不二次计算或过滤,
105
+ * 也不导出自己的 data 形态;每个叶子组件按自己的公开契约取数,共享计算由 resolve 的
106
+ * 「同引用 input + 深相等 spec」记忆化保证。
107
+ */
108
+ export const ExperimentComparison = defineComponent<ExperimentComparisonProps>((props, ctx) => {
109
+ const input = props.input ?? ctx.scope;
110
+ const { series, connect } = resolveComparisonSeries(input, props);
111
+ return (
112
+ <Col className={props.className}>
113
+ <ScopeSummary input={input} locale={props.locale} />
114
+ <MetricScatter
115
+ input={input}
116
+ points="experiment"
117
+ series={series}
118
+ connect={connect}
119
+ x={costUSD}
120
+ y={endToEndPassRate}
121
+ locale={props.locale}
122
+ />
123
+ <ExperimentList input={input} filter locale={props.locale} />
124
+ </Col>
125
+ );
126
+ });
127
+ ExperimentComparison.displayName = "ExperimentComparison";
@@ -0,0 +1,49 @@
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // "validate*Data 递归覆盖到嵌套字段" 行:summaries 的 validateScopeSummaryData 表驱动字段突变覆盖,range 的 null 边界、两级 tally、
3
+ // 两个 MetricCell 字段各自的嵌套校验。
4
+
5
+ import { describe, expect, it } from "vitest";
6
+ import { validateScopeSummaryData } from "./index.tsx";
7
+
8
+ const validCell = { value: 1, display: "1", samples: 1, total: 1, refs: [] };
9
+ const validTally = { passed: 1, failed: 0, errored: 0, skipped: 0 };
10
+
11
+ const valid = {
12
+ range: { earliestStartedAt: "2026-07-01T00:00:00Z", latestStartedAt: "2026-07-02T00:00:00Z" },
13
+ experiments: 1,
14
+ evals: 6,
15
+ attempts: 6,
16
+ evalVerdicts: validTally,
17
+ attemptVerdicts: validTally,
18
+ endToEndPassRate: validCell,
19
+ totalCostUSD: validCell,
20
+ };
21
+
22
+ describe("validateScopeSummaryData", () => {
23
+ it("合规 literal 通过", () => {
24
+ expect(validateScopeSummaryData(valid)).toBeNull();
25
+ });
26
+
27
+ it("range 两端为 null 合法(空范围不编造当前时间)", () => {
28
+ expect(validateScopeSummaryData({ ...valid, range: { earliestStartedAt: null, latestStartedAt: null } })).toBeNull();
29
+ });
30
+
31
+ it("range.earliestStartedAt 非法类型报错", () => {
32
+ const bad = { ...valid, range: { earliestStartedAt: 123, latestStartedAt: null } };
33
+ expect(validateScopeSummaryData(bad)).toMatch(/"range\.earliestStartedAt"/);
34
+ });
35
+
36
+ it("evalVerdicts 与 attemptVerdicts 分别校验:错的那个才报错", () => {
37
+ const bad = { ...valid, attemptVerdicts: { passed: 1, failed: 0, errored: 0, skipped: "0" } };
38
+ expect(validateScopeSummaryData(bad)).toMatch(/"attemptVerdicts\.skipped"/);
39
+ });
40
+
41
+ it("endToEndPassRate 结构错误定位到嵌套 MetricCell", () => {
42
+ const bad = { ...valid, endToEndPassRate: { value: 1, display: "1", samples: 1, total: 1, refs: "x" } };
43
+ expect(validateScopeSummaryData(bad)).toMatch(/"endToEndPassRate\.refs"/);
44
+ });
45
+
46
+ it("experiments 非数字报错", () => {
47
+ expect(validateScopeSummaryData({ ...valid, experiments: "1" })).toMatch(/"experiments"/);
48
+ });
49
+ });
@@ -0,0 +1,124 @@
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // 分区「Table 与文本排版原语」:Grid 展平规则、columns 校验与 TextGridPlan 的宽度算术
3
+ // 直接对 normalizeGrid / planTextGrid 断言,不经渲染面或 HTML。
4
+
5
+ import { describe, expect, it } from "vitest";
6
+ import { normalizeGrid, planTextGrid, validateGridColumns } from "./grid-layout.ts";
7
+ import type { ReportElement } from "./tree.ts";
8
+
9
+ const FRAGMENT = Symbol.for("react.fragment");
10
+
11
+ function el(type: string, props: Record<string, unknown> = {}): ReportElement {
12
+ return { type, props };
13
+ }
14
+
15
+ describe("validateGridColumns", () => {
16
+ it("接受有限正整数", () => {
17
+ expect(validateGridColumns(1)).toBe(1);
18
+ expect(validateGridColumns(9)).toBe(9);
19
+ });
20
+
21
+ for (const bad of [0, -1, 1.5, NaN, Infinity, -Infinity]) {
22
+ it(`拒绝 ${bad} 并给出完整用户反馈`, () => {
23
+ expect(() => validateGridColumns(bad)).toThrowError(/columns/i);
24
+ try {
25
+ validateGridColumns(bad);
26
+ } catch (err) {
27
+ const message = (err as Error).message;
28
+ expect(message).toContain(JSON.stringify(bad));
29
+ expect(message).toContain("columns={N}");
30
+ }
31
+ });
32
+ }
33
+ });
34
+
35
+ describe("normalizeGrid", () => {
36
+ it("展平数组与 Fragment、跳过空分支,任意 ReportNode 各占一格", () => {
37
+ const a = el("Stat", { label: "a" });
38
+ const b = el("Stat", { label: "b" });
39
+ const c = el("Col", { children: [el("Stat", { label: "c1" }), el("Stat", { label: "c2" })] });
40
+ const children = [
41
+ a,
42
+ null,
43
+ undefined,
44
+ false,
45
+ { type: FRAGMENT, props: { children: [b, null] } },
46
+ c,
47
+ ];
48
+ const normalized = normalizeGrid({ children, columns: 3 });
49
+ expect(normalized.cells.map((cell) => cell.node)).toEqual([a, b, c]);
50
+ // Col 归拢的两个 Stat 仍是同一格,不下钻展开成两格
51
+ expect(normalized.cells).toHaveLength(3);
52
+ });
53
+
54
+ it("全部子节点为空分支时 0 格", () => {
55
+ const normalized = normalizeGrid({ children: [null, undefined, false], columns: 3 });
56
+ expect(normalized.cells).toHaveLength(0);
57
+ });
58
+
59
+ it("variant / density 默认 plain / regular", () => {
60
+ const normalized = normalizeGrid({ children: el("Stat"), columns: 1 });
61
+ expect(normalized.variant).toBe("plain");
62
+ expect(normalized.density).toBe("regular");
63
+ });
64
+
65
+ it("单个裸元素(非数组)也能展平为一格并生成稳定 key", () => {
66
+ const normalized = normalizeGrid({ children: el("Stat", { label: "solo" }), columns: 1 });
67
+ expect(normalized.cells).toHaveLength(1);
68
+ expect(normalized.cells[0].key).toBe("nre-grid-cell-0");
69
+ });
70
+ });
71
+
72
+ describe("planTextGrid", () => {
73
+ // 与 docs/feature/reports/library/layout.md 的运行总览示例逐字核对:Section 缩进后
74
+ // Grid 收到 98 显示列可用宽度,两个 Grid 都必须恰好降到三列且逐行不超过 100(=98+2 缩进)。
75
+ it("目标示例 Grid(columns=6, 6 cell, regular)在 98 可用宽度降为三列,内容宽 28/27/27", () => {
76
+ const plan = planTextGrid({ availableWidth: 98, cellCount: 6, columns: 6, density: "regular" });
77
+ expect(plan.columns).toBe(3);
78
+ expect(plan.contentWidths).toEqual([28, 27, 27]);
79
+ expect(plan.gutter).toBe(2);
80
+ const rowWidth = plan.contentWidths.reduce((sum, w) => sum + w + 4, 0) + plan.gutter * (plan.columns - 1);
81
+ expect(rowWidth).toBe(98);
82
+ });
83
+
84
+ it("目标示例 Grid(columns=9, 9 cell, compact)在 98 可用宽度降为三列,内容宽 28/28/28", () => {
85
+ const plan = planTextGrid({ availableWidth: 98, cellCount: 9, columns: 9, density: "compact" });
86
+ expect(plan.columns).toBe(3);
87
+ expect(plan.contentWidths).toEqual([28, 28, 28]);
88
+ expect(plan.gutter).toBe(1);
89
+ const rowWidth = plan.contentWidths.reduce((sum, w) => sum + w + 4, 0) + plan.gutter * (plan.columns - 1);
90
+ expect(rowWidth).toBe(98);
91
+ });
92
+
93
+ it("继续收窄宽度时降为一列(无条件 fallback,即使内容宽度小于 24)", () => {
94
+ const plan = planTextGrid({ availableWidth: 20, cellCount: 6, columns: 6, density: "regular" });
95
+ expect(plan.columns).toBe(1);
96
+ expect(plan.contentWidths).toEqual([16]); // 20 - BOXED_OVERHEAD_PER_CELL(4)
97
+ });
98
+
99
+ it("columns=1 时无条件单列,不因宽度充裕而尝试更多列", () => {
100
+ const plan = planTextGrid({ availableWidth: 200, cellCount: 6, columns: 1, density: "regular" });
101
+ expect(plan.columns).toBe(1);
102
+ });
103
+
104
+ it("列数不超过 min(declared columns, cellCount)", () => {
105
+ const plan = planTextGrid({ availableWidth: 400, cellCount: 2, columns: 9, density: "regular" });
106
+ expect(plan.columns).toBeLessThanOrEqual(2);
107
+ });
108
+
109
+ it("plain 与 boxed 复用同一份 plan:规划不吃 variant,列数与内容宽只随 columns/density/availableWidth 变化", () => {
110
+ const a = planTextGrid({ availableWidth: 98, cellCount: 6, columns: 6, density: "regular" });
111
+ const b = planTextGrid({ availableWidth: 98, cellCount: 6, columns: 6, density: "regular" });
112
+ expect(a).toEqual(b);
113
+ });
114
+
115
+ it("整除余数从左向右各补一列,行宽不超过 availableWidth", () => {
116
+ const plan = planTextGrid({ availableWidth: 100, cellCount: 4, columns: 4, density: "regular" });
117
+ const rowWidth = plan.contentWidths.reduce((sum, w) => sum + w + 4, 0) + plan.gutter * (plan.columns - 1);
118
+ expect(rowWidth).toBeLessThanOrEqual(100);
119
+ // 余数只补到最左侧若干列,不累积到行尾
120
+ for (let i = 1; i < plan.contentWidths.length; i++) {
121
+ expect(plan.contentWidths[i]).toBeLessThanOrEqual(plan.contentWidths[i - 1]);
122
+ }
123
+ });
124
+ });