niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -0,0 +1,352 @@
1
+ // 官方双面组件的装配点:web 面(./MetricTable.tsx 等纯 React 组件)+ text 面(./faces.ts)
2
+ // + resolve 解析面(spec 形态由管线代调配套 ./compute.ts)。MetricBars 与 MetricMatrix
3
+ // 消费同一份矩阵数据(同一个 metricMatrixData)。
4
+
5
+ import type { ReportComponent } from "../../definition/tree.ts";
6
+ import type {
7
+ DeltaData,
8
+ LineData,
9
+ MatrixData,
10
+ ScatterData,
11
+ ScoreboardData,
12
+ TableData,
13
+ } from "../../model/types.ts";
14
+ import type { AttemptLocator } from "../../../results/locator.ts";
15
+ import {
16
+ arrayProblem,
17
+ cellProblem,
18
+ isLocalizedText,
19
+ isObject,
20
+ makeDataComponent,
21
+ hrefOf,
22
+ type ChromeProps,
23
+ type DataProps,
24
+ type Validator,
25
+ } from "../shared.ts";
26
+ import {
27
+ deltaTableData,
28
+ metricLineData,
29
+ metricMatrixData,
30
+ metricScatterData,
31
+ metricTableData,
32
+ scoreboardData,
33
+ type DeltaTableOptions,
34
+ type MetricLineOptions,
35
+ type MetricMatrixOptions,
36
+ type MetricScatterOptions,
37
+ type MetricTableOptions,
38
+ type ScoreboardOptions,
39
+ } from "./compute.ts";
40
+ import { barsText, deltaText, lineText, matrixText, scatterText, scoreboardText, tableText } from "./faces.ts";
41
+ import { MetricTable as MetricTableWeb } from "./MetricTable.tsx";
42
+ import { MetricMatrix as MetricMatrixWeb } from "./MetricMatrix.tsx";
43
+ import { MetricBars as MetricBarsWeb } from "./MetricBars.tsx";
44
+ import { Scoreboard as ScoreboardWeb } from "./Scoreboard.tsx";
45
+ import { MetricScatter as MetricScatterWeb } from "./MetricScatter.tsx";
46
+ import { MetricLine as MetricLineWeb } from "./MetricLine.tsx";
47
+ import { DeltaTable as DeltaTableWeb } from "./DeltaTable.tsx";
48
+
49
+ /** columns / metric / x / y 共用的 MetricColumn 形状(src/report/model/types.ts)。 */
50
+ function metricColumnProblem(value: unknown, path: string): string | null {
51
+ if (!isObject(value)) return `"${path}" must be a MetricColumn { key, label }`;
52
+ if (typeof value.key !== "string") return `"${path}.key" must be a string`;
53
+ if (!isLocalizedText(value.label)) return `"${path}.label" must be a LocalizedText`;
54
+ return null;
55
+ }
56
+
57
+ export const validateTableData: Validator = (data) => {
58
+ if (!isObject(data)) return "expected an object";
59
+ if (typeof data.rowDimension !== "string") return 'missing "rowDimension" (string)';
60
+ const columnsProblem = arrayProblem(data.columns, "columns", metricColumnProblem);
61
+ if (columnsProblem !== null) return columnsProblem;
62
+ return arrayProblem(data.rows, "rows", (row, path) => {
63
+ if (!isObject(row)) return `"${path}" must be an object`;
64
+ if (typeof row.key !== "string") return `"${path}.key" must be a string`;
65
+ if (!isObject(row.cells)) return `"${path}.cells" must be an object`;
66
+ for (const [metricKey, cell] of Object.entries(row.cells)) {
67
+ const problem = cellProblem(cell, `${path}.cells.${metricKey}`);
68
+ if (problem !== null) return problem;
69
+ }
70
+ return null;
71
+ });
72
+ };
73
+ export const validateMatrixData: Validator = (data) => {
74
+ if (!isObject(data)) return "expected an object";
75
+ if (typeof data.rowDimension !== "string" || typeof data.columnDimension !== "string") {
76
+ return 'missing "rowDimension" / "columnDimension" (string)';
77
+ }
78
+ const metricProblem = metricColumnProblem(data.metric, "metric");
79
+ if (metricProblem !== null) return metricProblem;
80
+ return arrayProblem(data.cells, "cells", (item, path) => {
81
+ if (!isObject(item)) return `"${path}" must be an object`;
82
+ if (typeof item.row !== "string" || typeof item.column !== "string") {
83
+ return `"${path}.row" / "${path}.column" must be strings`;
84
+ }
85
+ return cellProblem(item.cell, `${path}.cell`);
86
+ });
87
+ };
88
+ export const validateScatterData: Validator = (data) => {
89
+ if (!isObject(data)) return "expected an object";
90
+ if (typeof data.pointDimension !== "string") return 'missing "pointDimension" (string)';
91
+ const xColumnProblem = metricColumnProblem(data.x, "x");
92
+ if (xColumnProblem !== null) return xColumnProblem;
93
+ const yColumnProblem = metricColumnProblem(data.y, "y");
94
+ if (yColumnProblem !== null) return yColumnProblem;
95
+ return arrayProblem(data.rows, "rows", (row, path) => {
96
+ if (!isObject(row) || typeof row.key !== "string") return `"${path}" must be an object with a string "key"`;
97
+ const xProblem = cellProblem(row.x, `${path}.x`);
98
+ if (xProblem !== null) return xProblem;
99
+ return cellProblem(row.y, `${path}.y`);
100
+ });
101
+ };
102
+ export const validateLineData: Validator = (data) => {
103
+ if (!isObject(data)) return "expected an object";
104
+ if (!isObject(data.x) || typeof data.x.key !== "string" || !isLocalizedText(data.x.label)) {
105
+ return '"x" must be an axis descriptor { key, label }';
106
+ }
107
+ const yColumnProblem = metricColumnProblem(data.y, "y");
108
+ if (yColumnProblem !== null) return yColumnProblem;
109
+ return arrayProblem(data.rows, "rows", (row, path) => {
110
+ if (!isObject(row) || typeof row.key !== "string") return `"${path}" must be an object with a string "key"`;
111
+ if (!(row.x === null || typeof row.x === "number")) return `"${path}.x" must be a number or null`;
112
+ if (!isLocalizedText(row.xDisplay)) return `"${path}.xDisplay" must be a LocalizedText`;
113
+ return cellProblem(row.y, `${path}.y`);
114
+ });
115
+ };
116
+ function scoreTotalProblem(value: unknown, path: string): string | null {
117
+ if (!isObject(value)) return `"${path}" must be an object { value, display, notRun, unscorable, refs }`;
118
+ if (typeof value.value !== "number") return `"${path}.value" must be a number`;
119
+ if (!isLocalizedText(value.display)) return `"${path}.display" must be a LocalizedText`;
120
+ if (typeof value.notRun !== "number") return `"${path}.notRun" must be a number`;
121
+ if (typeof value.unscorable !== "number") return `"${path}.unscorable" must be a number`;
122
+ if (!Array.isArray(value.refs)) return `"${path}.refs" must be an array`;
123
+ return null;
124
+ }
125
+ export const validateScoreboardData: Validator = (data) => {
126
+ if (!isObject(data)) return "expected an object";
127
+ if (typeof data.rowDimension !== "string") return 'missing "rowDimension" (string)';
128
+ if (!Array.isArray(data.questions)) return 'missing "questions" (array)';
129
+ if (typeof data.fullMarks !== "number") return 'missing "fullMarks" (number)';
130
+ if (typeof data.ignoredEvals !== "number") return 'missing "ignoredEvals" (number)';
131
+ return arrayProblem(data.rows, "rows", (row, path) => {
132
+ if (!isObject(row) || typeof row.key !== "string") return `"${path}" must be an object with a string "key"`;
133
+ const totalProblem = scoreTotalProblem(row.total, `${path}.total`);
134
+ if (totalProblem !== null) return totalProblem;
135
+ return arrayProblem(row.subjects, `${path}.subjects`, (subject, subjectPath) => {
136
+ if (!isObject(subject) || typeof subject.key !== "string") {
137
+ return `"${subjectPath}" must be an object with a string "key"`;
138
+ }
139
+ if (typeof subject.earned !== "number") return `"${subjectPath}.earned" must be a number`;
140
+ if (typeof subject.possible !== "number") return `"${subjectPath}.possible" must be a number`;
141
+ if (typeof subject.questions !== "number") return `"${subjectPath}.questions" must be a number`;
142
+ if (typeof subject.notRun !== "number") return `"${subjectPath}.notRun" must be a number`;
143
+ if (typeof subject.unscorable !== "number") return `"${subjectPath}.unscorable" must be a number`;
144
+ if (!isLocalizedText(subject.display)) return `"${subjectPath}.display" must be a LocalizedText`;
145
+ if (!Array.isArray(subject.refs)) return `"${subjectPath}.refs" must be an array`;
146
+ return null;
147
+ });
148
+ });
149
+ };
150
+ const DELTA_OUTCOMES = ["improved", "regressed", "unchanged", "unavailable"];
151
+ export const validateDeltaData: Validator = (data) => {
152
+ if (!isObject(data)) return "expected an object";
153
+ if (typeof data.byDimension !== "string") return 'missing "byDimension" (string)';
154
+ const columnsProblem = arrayProblem(data.columns, "columns", metricColumnProblem);
155
+ if (columnsProblem !== null) return columnsProblem;
156
+ return arrayProblem(data.rows, "rows", (row, path) => {
157
+ if (!isObject(row) || typeof row.key !== "string") return `"${path}" must be an object with a string "key"`;
158
+ if (!isLocalizedText(row.label)) return `"${path}.label" must be a LocalizedText`;
159
+ if (!isObject(row.a) || typeof row.a.key !== "string") return `"${path}.a" must be an object with a string "key"`;
160
+ if (!isObject(row.b) || typeof row.b.key !== "string") return `"${path}.b" must be an object with a string "key"`;
161
+ if (!isObject(row.cells)) return `"${path}.cells" must be an object`;
162
+ for (const [metricKey, cell] of Object.entries(row.cells)) {
163
+ const cellPath = `${path}.cells.${metricKey}`;
164
+ if (!isObject(cell)) return `"${cellPath}" must be an object { a, b, delta, display, outcome }`;
165
+ const aProblem = cellProblem(cell.a, `${cellPath}.a`);
166
+ if (aProblem !== null) return aProblem;
167
+ const bProblem = cellProblem(cell.b, `${cellPath}.b`);
168
+ if (bProblem !== null) return bProblem;
169
+ if (!(cell.delta === null || typeof cell.delta === "number")) return `"${cellPath}.delta" must be a number or null`;
170
+ if (!isLocalizedText(cell.display)) return `"${cellPath}.display" must be a LocalizedText`;
171
+ if (typeof cell.outcome !== "string" || !DELTA_OUTCOMES.includes(cell.outcome)) {
172
+ return `"${cellPath}.outcome" must be one of ${JSON.stringify(DELTA_OUTCOMES)}`;
173
+ }
174
+ }
175
+ return null;
176
+ });
177
+ };
178
+
179
+ // ───────────────────────── 指标组件 ─────────────────────────
180
+
181
+ export type MetricTableProps = DataProps<
182
+ TableData,
183
+ MetricTableOptions,
184
+ ChromeProps & {
185
+ /** web 面在表格前渲染过滤输入框(enhance.js 接管);无 JS 时表格内容依旧完整。 */
186
+ filter?: boolean;
187
+ attemptHref?: (locator: AttemptLocator) => string;
188
+ }
189
+ >;
190
+
191
+ /** 榜单:一行一个维度值、一列一个指标,回答「谁整体更好」。 */
192
+ export const MetricTable = makeDataComponent<
193
+ TableData,
194
+ MetricTableOptions,
195
+ ChromeProps & { filter?: boolean; attemptHref?: (locator: AttemptLocator) => string }
196
+ >({
197
+ name: "MetricTable",
198
+ dataFnName: "metricTableData",
199
+ shapeName: "TableData",
200
+ dataFn: metricTableData,
201
+ specKeys: ["rows", "columns", "sort", "evals"],
202
+ validate: validateTableData,
203
+ web: (props, ctx) => (
204
+ <MetricTableWeb
205
+ data={props.data}
206
+ filter={props.filter}
207
+ locale={props.locale ?? ctx.locale}
208
+ attemptHref={hrefOf(props, ctx)}
209
+ className={props.className}
210
+ />
211
+ ),
212
+ text: (props, ctx) => tableText(props.data, ctx),
213
+ }) as unknown as ReportComponent<MetricTableProps>;
214
+
215
+ export type MetricMatrixProps = DataProps<
216
+ MatrixData,
217
+ MetricMatrixOptions,
218
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
219
+ >;
220
+ export type MetricBarsProps = MetricMatrixProps;
221
+
222
+ /** 逐题格子:行 × 列两个维度、格子里一个指标,回答「哪道题谁挂了」。 */
223
+ export const MetricMatrix = makeDataComponent<
224
+ MatrixData,
225
+ MetricMatrixOptions,
226
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
227
+ >({
228
+ name: "MetricMatrix",
229
+ dataFnName: "metricMatrixData",
230
+ shapeName: "MatrixData",
231
+ dataFn: metricMatrixData,
232
+ specKeys: ["rows", "columns", "cell", "evals"],
233
+ validate: validateMatrixData,
234
+ web: (props, ctx) => (
235
+ <MetricMatrixWeb data={props.data} locale={props.locale ?? ctx.locale} attemptHref={hrefOf(props, ctx)} className={props.className} />
236
+ ),
237
+ text: (props, ctx) => matrixText(props.data, ctx),
238
+ }) as unknown as ReportComponent<MetricMatrixProps>;
239
+
240
+ /** 分组条形:同一份矩阵数据的另一种摆法;与 MetricMatrix 写同一份 spec 时只计算一次。 */
241
+ export const MetricBars = makeDataComponent<
242
+ MatrixData,
243
+ MetricMatrixOptions,
244
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
245
+ >({
246
+ name: "MetricBars",
247
+ dataFnName: "metricMatrixData",
248
+ shapeName: "MatrixData",
249
+ dataFn: metricMatrixData,
250
+ specKeys: ["rows", "columns", "cell", "evals"],
251
+ validate: validateMatrixData,
252
+ web: (props, ctx) => (
253
+ <MetricBarsWeb data={props.data} locale={props.locale ?? ctx.locale} attemptHref={hrefOf(props, ctx)} className={props.className} />
254
+ ),
255
+ text: (props, ctx) => barsText(props.data, ctx),
256
+ }) as unknown as ReportComponent<MetricBarsProps>;
257
+
258
+ export type ScoreboardProps = DataProps<
259
+ ScoreboardData,
260
+ ScoreboardOptions,
261
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
262
+ >;
263
+
264
+ /** 考试成绩单:总分 + 分科小计,固定分母、notRun / unscorable 分开如实报。 */
265
+ export const Scoreboard = makeDataComponent<
266
+ ScoreboardData,
267
+ ScoreboardOptions,
268
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
269
+ >({
270
+ name: "Scoreboard",
271
+ dataFnName: "scoreboardData",
272
+ shapeName: "ScoreboardData",
273
+ dataFn: scoreboardData,
274
+ specKeys: ["rows", "questions", "subject", "weights", "fullMarks", "score"],
275
+ validate: validateScoreboardData,
276
+ web: (props, ctx) => <ScoreboardWeb data={props.data} locale={props.locale ?? ctx.locale} className={props.className} />,
277
+ text: (props, ctx) => scoreboardText(props.data, ctx),
278
+ }) as unknown as ReportComponent<ScoreboardProps>;
279
+
280
+ type ScatterChrome = ChromeProps & {
281
+ /** series 内按 x 升序成线(呈现,不改变 ScatterData):web 面画折线,text 面在图例给逐段位移摘要;默认 false。 */
282
+ connect?: boolean;
283
+ pointHref?: (row: ScatterData["rows"][number]) => string;
284
+ };
285
+
286
+ export type MetricScatterProps = DataProps<ScatterData, MetricScatterOptions, ScatterChrome>;
287
+
288
+ /** 两个指标之间的取舍:每个点一个维度值,x / y 各一个指标,series 决定颜色和图例归类。 */
289
+ export const MetricScatter = makeDataComponent<ScatterData, MetricScatterOptions, ScatterChrome>({
290
+ name: "MetricScatter",
291
+ dataFnName: "metricScatterData",
292
+ shapeName: "ScatterData",
293
+ dataFn: metricScatterData,
294
+ specKeys: ["points", "series", "x", "y", "evals"],
295
+ validate: validateScatterData,
296
+ web: (props, ctx) => (
297
+ <MetricScatterWeb
298
+ data={props.data}
299
+ connect={props.connect}
300
+ pointHref={props.pointHref}
301
+ locale={props.locale ?? ctx.locale}
302
+ className={props.className}
303
+ />
304
+ ),
305
+ text: (props, ctx) => scatterText(props.data, ctx, { connect: props.connect }),
306
+ }) as unknown as ReportComponent<MetricScatterProps>;
307
+
308
+ export type MetricLineProps = DataProps<
309
+ LineData,
310
+ MetricLineOptions,
311
+ ChromeProps & { pointHref?: (row: LineData["rows"][number]) => string }
312
+ >;
313
+
314
+ /** 趋势线:x 是 NumericAxis(参数轴),点身份 = (series, x)。 */
315
+ export const MetricLine = makeDataComponent<
316
+ LineData,
317
+ MetricLineOptions,
318
+ ChromeProps & { pointHref?: (row: LineData["rows"][number]) => string }
319
+ >({
320
+ name: "MetricLine",
321
+ dataFnName: "metricLineData",
322
+ shapeName: "LineData",
323
+ dataFn: metricLineData,
324
+ specKeys: ["x", "series", "y", "evals"],
325
+ validate: validateLineData,
326
+ web: (props, ctx) => (
327
+ <MetricLineWeb data={props.data} pointHref={props.pointHref} locale={props.locale ?? ctx.locale} className={props.className} />
328
+ ),
329
+ text: (props, ctx) => lineText(props.data, ctx),
330
+ }) as unknown as ReportComponent<MetricLineProps>;
331
+
332
+ export type DeltaTableProps = DataProps<
333
+ DeltaData,
334
+ DeltaTableOptions,
335
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
336
+ >;
337
+
338
+ /** 成对对比:每行一对(字面声明或 pairsByFlag 派生),格子里 A、B、Δ 三个值。 */
339
+ export const DeltaTable = makeDataComponent<
340
+ DeltaData,
341
+ DeltaTableOptions,
342
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
343
+ >({
344
+ name: "DeltaTable",
345
+ dataFnName: "deltaTableData",
346
+ shapeName: "DeltaData",
347
+ dataFn: deltaTableData,
348
+ specKeys: ["by", "pairs", "metrics", "evals"],
349
+ validate: validateDeltaData,
350
+ web: (props, ctx) => <DeltaTableWeb data={props.data} locale={props.locale ?? ctx.locale} className={props.className} />,
351
+ text: (props, ctx) => deltaText(props.data, ctx),
352
+ }) as unknown as ReportComponent<DeltaTableProps>;
@@ -2,7 +2,7 @@
2
2
  // 形态照 docs-site/zh/reference/report-components.mdx 的示例块:y 轴刻度 + │ 边框,
3
3
  // 点用字母标注、图例列在图下;lower-better 的轴反向,「好」的角落恒在右上。
4
4
 
5
- import { padDisplay, padStartDisplay, stringWidth } from "./layout.ts";
5
+ import { padDisplay, padStartDisplay, stringWidth } from "../../model/text-layout.ts";
6
6
 
7
7
  export interface PlotPoint {
8
8
  /** 图上的标记字符(点或系列的字母)。 */
@@ -0,0 +1,211 @@
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // "validate*Data 递归覆盖到嵌套字段" 行:metric-views 六个 validate*Data 的表驱动字段突变覆盖,每个校验函数先证明合规 literal 通过,
3
+ // 再对嵌套字段(MetricColumn / MetricCell / tally)逐个突变,证明报错文案定位到具体坏字段
4
+ // 路径而不是笼统的整份 data 报错。不复制 compute.ts 的聚合逻辑——fixture 是手写的合规
5
+ // literal,不经由真实计算产出。
6
+
7
+ import { describe, expect, it } from "vitest";
8
+ import {
9
+ validateDeltaData,
10
+ validateLineData,
11
+ validateMatrixData,
12
+ validateScatterData,
13
+ validateScoreboardData,
14
+ validateTableData,
15
+ } from "./index.tsx";
16
+
17
+ const validCell = { value: 1, display: "1", samples: 1, total: 1, refs: ["@1abcdef2"] };
18
+ const validColumn = { key: "costUSD", label: "Cost" };
19
+
20
+ describe("validateTableData", () => {
21
+ const valid = {
22
+ rowDimension: "agent",
23
+ columns: [validColumn],
24
+ rows: [{ key: "agent-x", cells: { costUSD: validCell } }],
25
+ };
26
+
27
+ it("合规 literal 通过", () => {
28
+ expect(validateTableData(valid)).toBeNull();
29
+ });
30
+
31
+ it("空 rows 本身合法", () => {
32
+ expect(validateTableData({ ...valid, rows: [] })).toBeNull();
33
+ });
34
+
35
+ it("columns[i] 缺 key 报错定位到该列", () => {
36
+ const bad = { ...valid, columns: [{ label: "Cost" }] };
37
+ expect(validateTableData(bad)).toMatch(/"columns\[0\]\.key"/);
38
+ });
39
+
40
+ it("rows[i].cells.<metric> 缺 samples 报错定位到嵌套 MetricCell 字段", () => {
41
+ const bad = {
42
+ ...valid,
43
+ rows: [{ key: "agent-x", cells: { costUSD: { value: 1, display: "1", total: 1, refs: [] } } }],
44
+ };
45
+ expect(validateTableData(bad)).toMatch(/"rows\[0\]\.cells\.costUSD\.samples"/);
46
+ });
47
+
48
+ it("rows[i].key 非字符串报错", () => {
49
+ const bad = { ...valid, rows: [{ key: 1, cells: {} }] };
50
+ expect(validateTableData(bad)).toMatch(/"rows\[0\]\.key"/);
51
+ });
52
+
53
+ it("旧形状(dimension 而非 rowDimension)整体报错", () => {
54
+ expect(validateTableData({ dimension: "agent", columns: [], rows: [] })).toMatch(/rowDimension/);
55
+ });
56
+ });
57
+
58
+ describe("validateMatrixData", () => {
59
+ const valid = {
60
+ rowDimension: "agent",
61
+ columnDimension: "eval",
62
+ metric: validColumn,
63
+ cells: [{ row: "agent-x", column: "q1", cell: validCell }],
64
+ };
65
+
66
+ it("合规 literal 通过", () => {
67
+ expect(validateMatrixData(valid)).toBeNull();
68
+ });
69
+
70
+ it("metric 缺 label 报错", () => {
71
+ expect(validateMatrixData({ ...valid, metric: { key: "costUSD" } })).toMatch(/"metric\.label"/);
72
+ });
73
+
74
+ it("cells[i].cell 结构错误定位到该格", () => {
75
+ const bad = { ...valid, cells: [{ row: "agent-x", column: "q1", cell: { value: 1 } }] };
76
+ expect(validateMatrixData(bad)).toMatch(/"cells\[0\]\.cell/);
77
+ });
78
+
79
+ it("cells[i].column 非字符串报错", () => {
80
+ const bad = { ...valid, cells: [{ row: "agent-x", column: 1, cell: validCell }] };
81
+ expect(validateMatrixData(bad)).toMatch(/"cells\[0\]\.column"/);
82
+ });
83
+ });
84
+
85
+ describe("validateScatterData", () => {
86
+ const valid = {
87
+ pointDimension: "experiment",
88
+ x: validColumn,
89
+ y: { key: "endToEndPassRate", label: "Pass rate" },
90
+ rows: [{ key: "exp-a", x: validCell, y: validCell }],
91
+ };
92
+
93
+ it("合规 literal 通过", () => {
94
+ expect(validateScatterData(valid)).toBeNull();
95
+ });
96
+
97
+ it("y 轴 MetricColumn 缺 key 报错", () => {
98
+ expect(validateScatterData({ ...valid, y: { label: "Pass rate" } })).toMatch(/"y\.key"/);
99
+ });
100
+
101
+ it("rows[i].y 结构错误定位到该点", () => {
102
+ const bad = { ...valid, rows: [{ key: "exp-a", x: validCell, y: { value: 1 } }] };
103
+ expect(validateScatterData(bad)).toMatch(/"rows\[0\]\.y/);
104
+ });
105
+ });
106
+
107
+ describe("validateLineData", () => {
108
+ const valid = {
109
+ x: { key: "turn", label: "Turn" },
110
+ y: validColumn,
111
+ rows: [{ key: "1", x: 1, y: validCell, xDisplay: "1" }],
112
+ };
113
+
114
+ it("合规 literal 通过", () => {
115
+ expect(validateLineData(valid)).toBeNull();
116
+ });
117
+
118
+ it("x 轴缺 label 报错", () => {
119
+ expect(validateLineData({ ...valid, x: { key: "turn" } })).toMatch(/"x" must be an axis descriptor/);
120
+ });
121
+
122
+ it("rows[i].x 类型错误(非 number|null)报错", () => {
123
+ const bad = { ...valid, rows: [{ key: "1", x: "1", y: validCell, xDisplay: "1" }] };
124
+ expect(validateLineData(bad)).toMatch(/"rows\[0\]\.x"/);
125
+ });
126
+
127
+ it("rows[i].y 缺失报错定位到嵌套 MetricCell", () => {
128
+ const bad = { ...valid, rows: [{ key: "1", x: 1, xDisplay: "1" }] };
129
+ expect(validateLineData(bad)).toMatch(/"rows\[0\]\.y"/);
130
+ });
131
+ });
132
+
133
+ describe("validateScoreboardData", () => {
134
+ const validSubject = {
135
+ key: "security",
136
+ earned: 1,
137
+ possible: 1,
138
+ questions: 1,
139
+ notRun: 0,
140
+ unscorable: 0,
141
+ display: "100%",
142
+ refs: [],
143
+ };
144
+ const valid = {
145
+ rowDimension: "agent",
146
+ questions: ["q1"],
147
+ fullMarks: 100,
148
+ ignoredEvals: 0,
149
+ rows: [
150
+ {
151
+ key: "agent-x",
152
+ total: { value: 100, display: "100%", notRun: 0, unscorable: 0, refs: [] },
153
+ subjects: [validSubject],
154
+ },
155
+ ],
156
+ };
157
+
158
+ it("合规 literal 通过", () => {
159
+ expect(validateScoreboardData(valid)).toBeNull();
160
+ });
161
+
162
+ it("rows[i].total 缺 notRun 报错", () => {
163
+ const bad = {
164
+ ...valid,
165
+ rows: [{ ...valid.rows[0], total: { value: 100, display: "100%", unscorable: 0, refs: [] } }],
166
+ };
167
+ expect(validateScoreboardData(bad)).toMatch(/"rows\[0\]\.total\.notRun"/);
168
+ });
169
+
170
+ it("rows[i].subjects[j] 缺 possible 报错定位到该 subject 下标", () => {
171
+ const bad = {
172
+ ...valid,
173
+ rows: [{ ...valid.rows[0], subjects: [{ ...validSubject, possible: undefined }] }],
174
+ };
175
+ expect(validateScoreboardData(bad)).toMatch(/"rows\[0\]\.subjects\[0\]\.possible"/);
176
+ });
177
+ });
178
+
179
+ describe("validateDeltaData", () => {
180
+ const validDeltaCell = { a: validCell, b: validCell, delta: 0, display: "±0", outcome: "unchanged" };
181
+ const valid = {
182
+ byDimension: "flag",
183
+ columns: [validColumn],
184
+ rows: [{ key: "codex", label: "codex", a: { key: "baseline" }, b: { key: "agents-md" }, cells: { costUSD: validDeltaCell } }],
185
+ };
186
+
187
+ it("合规 literal 通过", () => {
188
+ expect(validateDeltaData(valid)).toBeNull();
189
+ });
190
+
191
+ it("cells.<metric>.outcome 不在枚举内报错", () => {
192
+ const bad = {
193
+ ...valid,
194
+ rows: [{ ...valid.rows[0], cells: { costUSD: { ...validDeltaCell, outcome: "flat" } } }],
195
+ };
196
+ expect(validateDeltaData(bad)).toMatch(/"rows\[0\]\.cells\.costUSD\.outcome"/);
197
+ });
198
+
199
+ it("cells.<metric>.a 结构错误定位到该侧", () => {
200
+ const bad = {
201
+ ...valid,
202
+ rows: [{ ...valid.rows[0], cells: { costUSD: { ...validDeltaCell, a: { value: 1 } } } }],
203
+ };
204
+ expect(validateDeltaData(bad)).toMatch(/"rows\[0\]\.cells\.costUSD\.a/);
205
+ });
206
+
207
+ it("rows[i].label 缺失报错", () => {
208
+ const bad = { ...valid, rows: [{ key: "codex", a: { key: "baseline" }, b: { key: "agents-md" }, cells: {} }] };
209
+ expect(validateDeltaData(bad)).toMatch(/"rows\[0\]\.label"/);
210
+ });
211
+ });