niceeval 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +14 -0
  154. package/dist/shared/aggregate.js +31 -0
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +49 -49
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +71 -57
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +27 -12
  227. package/src/i18n/zh-CN.ts +27 -12
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +124 -106
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +53 -2
  314. package/src/results/select.ts +29 -5
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +29 -0
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +56 -20
  356. package/src/show/render.ts +15 -239
  357. package/src/show/show.test.ts +127 -34
  358. package/src/view/app/App.test.tsx +1 -1
  359. package/src/view/app/App.tsx +119 -47
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +105 -83
  370. package/src/view/index.ts +1 -1
  371. package/src/view/server.ts +32 -2
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +0 -805
  376. package/src/view/view-report.test.ts +141 -22
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -4,15 +4,15 @@
4
4
  // skipped=null、null≠0、Scoreboard 固定分母(notRun/unscorable 分开)、权重最长前缀、
5
5
  // 身份键去重、现刻水位、自定义指标 where/aggregate、evalGroup 完整父路径、verdict 权威、
6
6
  // MetricCell 诚实、缺 artifact 指标、repeatedFailedCommands、实体列表 failureSummary、
7
- // scopeSummaryData 两级计票、experimentComparisonData 分区、pairsByFlag、
7
+ // scopeSummaryData 两级计票、experimentListData/scopeSummaryData 的 selectedEvalIds 投影、pairsByFlag、
8
8
  // MetricLine 点身份、空数组反馈、metricTableData sort。
9
9
 
10
10
  import { describe, expect, it } from "vitest";
11
11
 
12
- import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "../types.ts";
13
- import type { AttemptHandle, Scope, ScopeWarning, Snapshot } from "../results/index.ts";
14
- import { makeScope, selectCurrentResults } from "../results/select.ts";
15
- import type { Results } from "../results/types.ts";
12
+ import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "../../types.ts";
13
+ import type { AttemptHandle, Scope, ScopeWarning, Snapshot } from "../../results/index.ts";
14
+ import { makeScope, selectCurrentResults } from "../../results/select.ts";
15
+ import type { Results } from "../../results/types.ts";
16
16
  import {
17
17
  assistantTurns,
18
18
  costUSD,
@@ -23,23 +23,33 @@ import {
23
23
  executionReliability,
24
24
  repeatedFailedCommands,
25
25
  taskPassRate,
26
- } from "./metrics.ts";
27
- import { flag, numericFlag } from "./flag.ts";
26
+ } from "../model/metrics.ts";
27
+ import { flag, label, numericFlag, numericLabel } from "../model/flag.ts";
28
+ import { scatterText } from "./metric-views/faces.ts";
29
+ import { createTextContext } from "../definition/tree.ts";
30
+ import { colorIndexForKey, colorIndicesForKeys } from "../assets/colors.ts";
31
+ import { attemptListData, evalListData, experimentListData } from "./entity-lists/compute.ts";
32
+ import { validateAttemptListData, validateEvalListData, validateExperimentListData } from "./entity-lists/index.tsx";
28
33
  import {
29
- attemptListData,
30
34
  deltaTableData,
31
- evalListData,
32
- experimentComparisonData,
33
- experimentListData,
34
35
  metricLineData,
35
36
  metricMatrixData,
36
37
  metricScatterData,
37
38
  metricTableData,
38
39
  pairsByFlag,
39
- scopeSummaryData,
40
40
  scoreboardData,
41
- } from "./compute.ts";
42
- import { evalGroupOf } from "./aggregate.ts";
41
+ } from "./metric-views/compute.ts";
42
+ import {
43
+ validateDeltaData,
44
+ validateLineData,
45
+ validateMatrixData,
46
+ validateScatterData,
47
+ validateScoreboardData,
48
+ validateTableData,
49
+ } from "./metric-views/index.tsx";
50
+ import { scopeSummaryData } from "./summaries/compute.ts";
51
+ import { validateScopeSummaryData } from "./summaries/index.tsx";
52
+ import { evalGroupOf } from "../model/aggregate.ts";
43
53
 
44
54
  // ───────────────────────── fake 数据(按 results 读取契约造)─────────────────────────
45
55
 
@@ -183,7 +193,7 @@ describe("两级聚合口径", () => {
183
193
  expect(cells[executionReliability.name]!.value).toBeCloseTo(2 / 3);
184
194
  });
185
195
 
186
- it("2 passed + 5 errored 的默认成功率是 2/7,不是 100%", async () => {
196
+ it("2 passed + 5 errored 的默认通过率是 2/7,不是 100%", async () => {
187
197
  const s = snap({
188
198
  experimentId: "exp/err",
189
199
  results: [
@@ -595,7 +605,7 @@ describe("实体列表 data", () => {
595
605
  expect(byEval.get("list/errored")!.costUSD).toBeNull();
596
606
  });
597
607
 
598
- it("experimentListData:evalVerdicts / endToEndPassRate / costUSD / durationMs / tokens 齐全,默认按端到端成功率降序", async () => {
608
+ it("experimentListData:evalVerdicts / endToEndPassRate / costUSD / durationMs / tokens 齐全,默认按端到端通过率降序", async () => {
599
609
  const winner = snap({ experimentId: "exp/win", results: [res("a", "passed"), res("b", "passed")] });
600
610
  const loser = snap({ experimentId: "exp/lose", results: [res("a", "failed"), res("b", "passed")] });
601
611
  const items = await experimentListData([loser, winner]);
@@ -667,32 +677,59 @@ describe("scopeSummaryData", () => {
667
677
  });
668
678
  });
669
679
 
670
- // ───────────────────────── experimentComparisonData ─────────────────────────
680
+ // ───────────────────────── experimentListData / scopeSummaryData 的 selectedEvalIds 投影 ─────────────────────────
681
+ // ExperimentComparison 已收为普通组合组件(把同一个 input 原样透传给 ScopeSummary /
682
+ // MetricScatter / ExperimentList,不再有自己的 data 形态);经它展开后与直接调用这三个
683
+ // 函数深等的验证挪到 dual-render.test.tsx(需要 resolve 管线)。这里只测三个函数自己的
684
+ // selectedEvalIds 投影契约。
671
685
 
672
- describe("experimentComparisonData", () => {
673
- it("计算前按完整父路径分区,根目录 experiment 单例组;每组与独立调用三个计算函数深等,refs 不跨组", async () => {
686
+ describe("experimentListData / scopeSummaryData 的 selectedEvalIds 投影", () => {
687
+ it("不同深度目录的 experiments 一律进同一份 data,不按父路径分组比较", async () => {
674
688
  const g1a = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
675
689
  const g1b = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
676
690
  const g2 = snap({ experimentId: "bench/long/x", results: [res("q", "passed")] });
677
691
  const solo = snap({ experimentId: "standalone", results: [res("q", "errored", { error: erroredWith("x") })] });
692
+ const items = await experimentListData([g1a, g1b, g2, solo]);
693
+ expect(items.map((e) => e.experimentId).sort()).toEqual([
694
+ "bench/long/x",
695
+ "compare/a",
696
+ "compare/b",
697
+ "standalone",
698
+ ]);
699
+ });
700
+
701
+ it("每个 experiment 只保留自己选择的 eval:A 声明 selectedEvalIds:[q1] 但夹带 q2 attempt,B 只选 q2;q2 不污染 A", async () => {
702
+ const a = snap({
703
+ experimentId: "exp/a",
704
+ results: [res("q1", "passed"), res("q2", "failed")],
705
+ experiment: { runs: 1, earlyExit: false, selectedEvalIds: ["q1"] },
706
+ });
707
+ const b = snap({
708
+ experimentId: "exp/b",
709
+ results: [res("q2", "passed")],
710
+ experiment: { runs: 1, earlyExit: false, selectedEvalIds: ["q2"] },
711
+ });
678
712
 
679
- const data = await experimentComparisonData([g1a, g1b, g2, solo]);
680
- expect(data.groups.map((g) => g.key)).toEqual(["bench/long", "compare", "standalone"]);
713
+ const items = await experimentListData([a, b]);
714
+ const rowA = items.find((e) => e.experimentId === "exp/a")!;
715
+ const rowB = items.find((e) => e.experimentId === "exp/b")!;
716
+ expect(rowA.evals).toBe(1); // 只统计 q1,夹带的 q2 attempt 不计入
717
+ expect(rowA.evalRows.map((r) => r.evalId)).toEqual(["q1"]);
718
+ expect(rowB.evals).toBe(1);
719
+ expect(rowB.evalRows.map((r) => r.evalId)).toEqual(["q2"]);
720
+
721
+ // 汇总口径同样不被污染:全 Scope 只有 2 个 eval(exp/a·q1、exp/b·q2),不是 3 个。
722
+ const summary = await scopeSummaryData([a, b]);
723
+ expect(summary.evals).toBe(2);
724
+ });
681
725
 
682
- const compare = data.groups.find((g) => g.key === "compare")!;
683
- const [summary, scatter, experiments] = await Promise.all([
684
- scopeSummaryData([g1a, g1b]),
685
- metricScatterData([g1a, g1b], { points: "experiment", series: "agent", x: costUSD, y: endToEndPassRate }),
686
- experimentListData([g1a, g1b]),
687
- ]);
688
- expect(compare.summary).toEqual(summary);
689
- expect(compare.scatter).toEqual(scatter);
690
- expect(compare.experiments).toEqual(experiments);
726
+ it("第三方快照缺 experiment 信息时仍可见,按其实际 evals 参与(selectedEvalIdsOf 退化)", async () => {
727
+ const thirdParty = snap({ experimentId: "third-party/exp", results: [res("q", "passed")] });
728
+ expect(thirdParty.experiment).toBeUndefined();
691
729
 
692
- // 组外 attempt 不污染:compare 组的 refs 不含 bench/standalone 的 locator
693
- const soloRefs = data.groups.find((g) => g.key === "standalone")!.summary.endToEndPassRate.refs;
694
- const compareRefs = compare.summary.endToEndPassRate.refs;
695
- expect(compareRefs.some((ref) => soloRefs.includes(ref))).toBe(false);
730
+ const items = await experimentListData([thirdParty]);
731
+ expect(items.map((e) => e.experimentId)).toEqual(["third-party/exp"]);
732
+ expect(items[0]!.evals).toBe(1);
696
733
  });
697
734
  });
698
735
 
@@ -918,22 +955,79 @@ describe("deltaTableData", () => {
918
955
  ];
919
956
  };
920
957
 
921
- it("同可比组 + 删除该 flag 后配置深相等才配对;a 取 baseline(缺省=未声明),label 自动生成,按 (a 末段, 显示键) 字典序", async () => {
958
+ it("input Scope 内删除该 flag 后配置深相等才配对;a 取 baseline(缺省=未声明),label 用完整 a experiment id,按 (a 末段, 显示键) 字典序", async () => {
922
959
  const data = await deltaTableData(matrixSnaps(), {
923
960
  by: "experiment",
924
961
  pairs: pairsByFlag("memory"),
925
962
  metrics: [endToEndPassRate],
926
963
  });
927
964
  expect(data.rows.map((r) => r.label)).toEqual([
928
- "bub · memory=agents-md",
929
- "codex · memory=agents-md",
930
- "codex · memory=mempal",
931
- "gemini · memory=agents-md",
932
- "gemini · memory=mempal",
965
+ "mem/bub · memory=agents-md",
966
+ "mem/codex · memory=agents-md",
967
+ "mem/codex · memory=mempal",
968
+ "mem/gemini · memory=agents-md",
969
+ "mem/gemini · memory=mempal",
933
970
  ]);
934
971
  expect(data.experiments).toBe(8);
935
972
  });
936
973
 
974
+ it("配对边界只是 input Scope,不额外按 experiment id 的目录前缀分组:不同前缀但可比性配置相同就配对", async () => {
975
+ const mk = (id: string, memory?: string) =>
976
+ snap({
977
+ experimentId: id,
978
+ agent: "bub",
979
+ results: [res("q", "passed")],
980
+ experiment: {
981
+ runs: 1,
982
+ earlyExit: false,
983
+ selectedEvalIds: [],
984
+ ...(memory !== undefined ? { flags: { memory } } : {}),
985
+ },
986
+ });
987
+ const data = await deltaTableData([mk("compare/codex"), mk("bench/codex", "agents-md")], {
988
+ by: "experiment",
989
+ pairs: pairsByFlag("memory"),
990
+ metrics: [endToEndPassRate],
991
+ });
992
+ expect(data.rows.map((r) => r.label)).toEqual(["compare/codex · memory=agents-md"]);
993
+ expect(data.rows[0]!.a.key).toBe("compare/codex");
994
+ expect(data.rows[0]!.b.key).toBe("bench/codex");
995
+ });
996
+
997
+ it("排序只看 a 末段与 flag 显示键,不看完整 label 字符串:a 末段相同、完整 id 不同的两个 bucket 不因目录前缀打乱顺序", async () => {
998
+ const mk = (id: string, model: string, memory?: string) =>
999
+ snap({
1000
+ experimentId: id,
1001
+ model,
1002
+ agent: "bub",
1003
+ results: [res("q", "passed")],
1004
+ experiment: {
1005
+ runs: 1,
1006
+ earlyExit: false,
1007
+ selectedEvalIds: [],
1008
+ ...(memory !== undefined ? { flags: { memory } } : {}),
1009
+ },
1010
+ });
1011
+ // 两个 bucket 因 model 不同而彼此不可比(不会互相配对),但都各自派生一对;
1012
+ // 两对的 a 末段都是 "codex",flagKey 分别是 "z-value" 与 "a-value"。
1013
+ const data = await deltaTableData(
1014
+ [
1015
+ mk("alpha/codex", "m1"), // bucket 1 baseline
1016
+ mk("alpha/codex-variant", "m1", "z-value"), // bucket 1 b 侧
1017
+ mk("zeta/codex", "m2"), // bucket 2 baseline
1018
+ mk("zeta/codex-variant", "m2", "a-value"), // bucket 2 b 侧
1019
+ ],
1020
+ { by: "experiment", pairs: pairsByFlag("memory"), metrics: [endToEndPassRate] },
1021
+ );
1022
+ // 只看 a 末段("codex",两对相同)与 flag 显示键("a-value" < "z-value")排序:
1023
+ // zeta 那对(flagKey "a-value")排在 alpha 那对(flagKey "z-value")之前——若误按完整
1024
+ // label 字符串排序,"alpha/codex..." 会先于 "zeta/codex...",顺序会相反。
1025
+ expect(data.rows.map((r) => r.label)).toEqual([
1026
+ "zeta/codex · memory=a-value",
1027
+ "alpha/codex · memory=z-value",
1028
+ ]);
1029
+ });
1030
+
937
1031
  it("可比性配置不同(model 不同)的两实验不配对", async () => {
938
1032
  const base = snap({
939
1033
  experimentId: "mm/a",
@@ -977,13 +1071,13 @@ describe("deltaTableData", () => {
977
1071
  pairs: pairsByFlag("memory", { baseline: "agents-md" }),
978
1072
  metrics: [endToEndPassRate],
979
1073
  });
980
- // a = *--agents-md;b = 未声明(显示键 (missing))与 mempal
1074
+ // a = *--agents-md;b = 未声明(显示键 (missing))与 mempal;label 用完整 a experiment id。
981
1075
  expect(data.rows.map((r) => r.label)).toEqual([
982
- "bub--agents-md · memory=(missing)",
983
- "codex--agents-md · memory=(missing)",
984
- "codex--agents-md · memory=mempal",
985
- "gemini--agents-md · memory=(missing)",
986
- "gemini--agents-md · memory=mempal",
1076
+ "mem/bub--agents-md · memory=(missing)",
1077
+ "mem/codex--agents-md · memory=(missing)",
1078
+ "mem/codex--agents-md · memory=mempal",
1079
+ "mem/gemini--agents-md · memory=(missing)",
1080
+ "mem/gemini--agents-md · memory=mempal",
987
1081
  ]);
988
1082
  });
989
1083
  });
@@ -1009,3 +1103,179 @@ describe("examScore", () => {
1009
1103
  expect(cellOf("crashed").samples).toBe(1); // 0 分是测得的事实,不是缺数据
1010
1104
  });
1011
1105
  });
1106
+
1107
+ // ───────────────────────── labels 维度、series 归类与 connect ─────────────────────────
1108
+
1109
+ describe("labels 维度、series 归类与 connect", () => {
1110
+ const labeled = (
1111
+ experimentId: string,
1112
+ labels: Record<string, string | number> | undefined,
1113
+ verdicts: Verdict[],
1114
+ agent = "agent-x",
1115
+ ) =>
1116
+ snap({
1117
+ experimentId,
1118
+ agent,
1119
+ results: verdicts.map((v, i) => res(`q${i}`, v, { agent })),
1120
+ experiment: {
1121
+ runs: 1,
1122
+ earlyExit: false,
1123
+ selectedEvalIds: verdicts.map((_, i) => `q${i}`),
1124
+ ...(labels ? { labels } : {}),
1125
+ },
1126
+ });
1127
+
1128
+ it("label() 按声明值分组、未声明归 (missing);numericLabel 对字符串值返回 null 不猜序", async () => {
1129
+ const table = await metricTableData(
1130
+ [labeled("m/one", { memory: "mempal" }, ["passed"]), labeled("m/two", undefined, ["failed"])],
1131
+ { rows: label("memory"), columns: [endToEndPassRate] },
1132
+ );
1133
+ expect(table.rows.map((r) => r.key).sort()).toEqual(["(missing)", "mempal"]);
1134
+
1135
+ const line = await metricLineData(
1136
+ [labeled("m/num", { contextK: 32 }, ["passed"]), labeled("m/str", { contextK: "big" }, ["passed"])],
1137
+ { x: numericLabel("contextK"), y: endToEndPassRate },
1138
+ );
1139
+ expect(line.rows.filter((r) => r.x === null)).toHaveLength(1);
1140
+ expect(line.rows.some((r) => r.x === 32)).toBe(true);
1141
+ expect(line.rows.some((r) => r.x === 0)).toBe(false);
1142
+ });
1143
+
1144
+ it("series 数组解析为复合维度:name 以 × 连接、值以 · 连接,缺失成员沿用 (missing)", async () => {
1145
+ const a = labeled("c/one", { memory: "mempal" }, ["passed"], "codex");
1146
+ const b = labeled("c/two", undefined, ["failed"], "codex");
1147
+ const data = await metricScatterData([a, b], {
1148
+ points: "experiment",
1149
+ series: ["agent", label("memory")],
1150
+ x: costUSD,
1151
+ y: endToEndPassRate,
1152
+ });
1153
+ expect(data.seriesDimension).toBe("agent × memory");
1154
+ const byKey = new Map(data.rows.map((r) => [r.key, r.series]));
1155
+ expect(byKey.get("c/one")).toBe("codex · mempal");
1156
+ expect(byKey.get("c/two")).toBe("codex · (missing)");
1157
+ // 单成员数组等价于单维度
1158
+ const single = await metricScatterData([a], { points: "experiment", series: ["agent"], x: costUSD, y: endToEndPassRate });
1159
+ expect(single.seriesDimension).toBe("agent");
1160
+ expect(single.rows[0]!.series).toBe("codex");
1161
+ });
1162
+
1163
+ const cell = (value: number, display: string): import("../model/types.ts").MetricCell => ({
1164
+ value,
1165
+ display,
1166
+ samples: 1,
1167
+ total: 1,
1168
+ refs: [],
1169
+ });
1170
+ const lineScatter = (): import("../model/types.ts").ScatterData => ({
1171
+ pointDimension: "experiment",
1172
+ seriesDimension: "line",
1173
+ x: { key: "costUSD", label: "Cost", unit: "$", better: "lower" },
1174
+ y: { key: "endToEndPassRate", label: "Pass rate", unit: "%", better: "higher" },
1175
+ rows: [
1176
+ { key: "mem/claude-mempal", series: "claude", x: cell(0.55, "$0.55"), y: cell(0.75, "75%") },
1177
+ { key: "mem/claude-baseline", series: "claude", x: cell(0.35, "$0.35"), y: cell(0.5, "50%") },
1178
+ { key: "mem/bub", series: "bub", x: cell(0.09, "$0.09"), y: cell(0.875, "87.5%") },
1179
+ ],
1180
+ });
1181
+
1182
+ it("scatterText:标记按图例顺序分配(series 字典序、series 内 x 升序),标题标注归类维度;默认无箭头", () => {
1183
+ const text = scatterText(lineScatter(), createTextContext({ width: 80 }));
1184
+ expect(text).toContain("grouped by line");
1185
+ // bub < claude:A 给 bub;claude 内按成本升序 baseline(B) → mempal(C)
1186
+ expect(text).toContain("bub A mem/bub");
1187
+ expect(text).toContain("B mem/claude-baseline");
1188
+ expect(text).toContain("C mem/claude-mempal");
1189
+ expect(text).not.toContain("→ C mem/claude-mempal");
1190
+ expect(text).not.toContain("└ Pass rate");
1191
+ });
1192
+
1193
+ it("scatterText connect:图例按 x 升序 → 串联并给逐段位移摘要;单点 series 无箭头无摘要;坐标图内不画折线", () => {
1194
+ const text = scatterText(lineScatter(), createTextContext({ width: 80 }), { connect: true });
1195
+ expect(text).toContain("B mem/claude-baseline → C mem/claude-mempal");
1196
+ expect(text).toContain("└ Pass rate +25pt · Cost +$0.20");
1197
+ expect(text).toContain("A mem/bub");
1198
+ expect(text).not.toContain("A mem/bub →");
1199
+ // 坐标图折线用 · 描画;connect 的 text 投影是图例的位移摘要,网格行(│ 开头)不出现折线点
1200
+ const plotRows = text.split("\n").filter((l) => l.includes("│"));
1201
+ expect(plotRows.length).toBeGreaterThan(0);
1202
+ expect(plotRows.every((l) => !l.includes("·"))).toBe(true);
1203
+ });
1204
+
1205
+ it("同图撞色按图例顺序线性探测空格;无冲突键仍取散列格;超过色板才复用", () => {
1206
+ // 真实回归现场:bub / claude-code / codex 在同一张图里必须三色互异
1207
+ const real = colorIndicesForKeys(["bub", "claude-code", "codex"]);
1208
+ expect(new Set(real.values()).size).toBe(3);
1209
+ // 构造性冲突:找两个散列同格的键,后到者让位,先到者保持散列格(跨图稳定)
1210
+ const pool = Array.from({ length: 40 }, (_, i) => `k${i}`);
1211
+ const byIdx = new Map<number, string[]>();
1212
+ for (const k of pool) {
1213
+ const idx = colorIndexForKey(k);
1214
+ byIdx.set(idx, [...(byIdx.get(idx) ?? []), k]);
1215
+ }
1216
+ const [first, second] = [...byIdx.values()].find((keys) => keys.length >= 2)!;
1217
+ const resolved = colorIndicesForKeys([first, second]);
1218
+ expect(resolved.get(first)).toBe(colorIndexForKey(first));
1219
+ expect(resolved.get(second)).not.toBe(resolved.get(first));
1220
+ // 前 6 个键占满 6 色;第 7 个开始只能复用
1221
+ const seven = colorIndicesForKeys(["s0", "s1", "s2", "s3", "s4", "s5", "s6"]);
1222
+ expect(new Set([...seven.values()].slice(0, 6)).size).toBe(6);
1223
+ });
1224
+ });
1225
+
1226
+ describe("validate*Data 接受真实计算产物(不是只接受手写 literal)", () => {
1227
+ // 手写 literal 通过校验只证明 validator 认得自己设想的形状;真正的把关是喂真实计算产物——
1228
+ // 尤其是省略可选字段(如自定义指标不声明 label)的真实路径,不是校验作者假想的边界。
1229
+ const a = snap({ experimentId: "compare/a", results: [res("q1", "passed"), res("q2", "failed")] });
1230
+ const b = snap({ experimentId: "compare/b", results: [res("q1", "failed"), res("q2", "passed")] });
1231
+ const scope = scopeOf([a, b]);
1232
+ const noLabelMetric = defineMetric({ name: "custom-no-label", value: (attempt) => (attempt.result.verdict === "passed" ? 1 : 0) });
1233
+
1234
+ it("metricTableData:含未声明 label 的自定义指标", async () => {
1235
+ const table = await metricTableData(scope, { rows: "agent", columns: [costUSD, noLabelMetric] });
1236
+ expect(validateTableData(table)).toBeNull();
1237
+ });
1238
+
1239
+ it("metricMatrixData", async () => {
1240
+ const matrix = await metricMatrixData(scope, { rows: "experiment", columns: "eval", cell: endToEndPassRate });
1241
+ expect(validateMatrixData(matrix)).toBeNull();
1242
+ });
1243
+
1244
+ it("metricScatterData:含缺 y 的点(null MetricCell.value)", async () => {
1245
+ const scatter = await metricScatterData(scope, { points: "experiment", x: costUSD, y: noLabelMetric });
1246
+ expect(validateScatterData(scatter)).toBeNull();
1247
+ });
1248
+
1249
+ it("metricLineData", async () => {
1250
+ const line = await metricLineData(scope, { x: numericFlag("budget"), y: endToEndPassRate });
1251
+ expect(validateLineData(line)).toBeNull();
1252
+ });
1253
+
1254
+ it("scoreboardData", async () => {
1255
+ const board = await scoreboardData(scope, { rows: "agent", questions: ["q1", "q2"] });
1256
+ expect(validateScoreboardData(board)).toBeNull();
1257
+ });
1258
+
1259
+ it("deltaTableData(含 pairsByFlag 派生 pair)", async () => {
1260
+ const withFlag = snap({
1261
+ experimentId: "compare/withflag",
1262
+ results: [res("q1", "passed")],
1263
+ experiment: { runs: 1, earlyExit: false, selectedEvalIds: [], flags: { memory: "on" } },
1264
+ });
1265
+ const withoutFlag = snap({ experimentId: "compare/noflag", results: [res("q1", "failed")] });
1266
+ const deltaScope = scopeOf([withFlag, withoutFlag]);
1267
+ const delta = await deltaTableData(deltaScope, { by: "experiment", pairs: pairsByFlag("memory"), metrics: [costUSD] });
1268
+ expect(validateDeltaData(delta)).toBeNull();
1269
+ });
1270
+
1271
+ it("scopeSummaryData", async () => {
1272
+ const summary = await scopeSummaryData(scope);
1273
+ expect(validateScopeSummaryData(summary)).toBeNull();
1274
+ });
1275
+
1276
+ it("experimentListData / evalListData / attemptListData(同一 Scope 三种粒度)", async () => {
1277
+ expect(validateExperimentListData(await experimentListData(scope))).toBeNull();
1278
+ expect(validateEvalListData(await evalListData(scope))).toBeNull();
1279
+ expect(validateAttemptListData(await attemptListData(scope))).toBeNull();
1280
+ });
1281
+ });
@@ -6,30 +6,38 @@
6
6
  // 也是它们展开区里"逐条 attempt"那一层的唯一实现(通过 AttemptRow 导出复用,不重写一遍)。
7
7
 
8
8
  import type { ReactElement } from "react";
9
- import type { AttemptListItem } from "../types.ts";
10
- import type { AttemptLocator } from "../../results/locator.ts";
11
- import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "../locale.ts";
12
- import { colorClassForKey } from "./colors.ts";
13
- import { cx, formatDurationMs, formatUSD, verdictMark } from "./format.ts";
9
+ import type { AttemptListItem } from "../../model/types.ts";
10
+ import type { AttemptLocator } from "../../../results/locator.ts";
11
+ import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "../../model/locale.ts";
12
+ import { colorClassForKey } from "../../assets/colors.ts";
13
+ import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.ts";
14
+ import { cx } from "../shared.ts";
14
15
 
15
- const DEFAULT_ATTEMPT_HREF = (locator: AttemptLocator): string => `#/attempt/${locator}`;
16
-
17
- /** locator + 判定符的普通 <a>,AttemptList/EvalList/ExperimentList 共用。 */
16
+ /**
17
+ * locator + 判定符,AttemptList/EvalList/ExperimentList 共用。没有 target(当前报告没有
18
+ * declare attempt-input page,也没有显式 attemptHref)时是纯文本,不生成空 href 或假链接
19
+ * (docs/feature/reports/architecture.md「Attempt 详情是一张参数化 page」)。
20
+ */
18
21
  export function AttemptLocatorBadge({
19
22
  item,
20
- attemptHref = DEFAULT_ATTEMPT_HREF,
23
+ attemptHref,
21
24
  }: {
22
25
  item: Pick<AttemptListItem, "locator" | "verdict">;
23
26
  attemptHref?: (locator: AttemptLocator) => string;
24
27
  }): ReactElement {
25
- return (
26
- <a
27
- className={cx("nre-locator", `nre-verdict-${item.verdict}`)}
28
- href={attemptHref(item.locator)}
29
- >
28
+ const className = cx("nre-locator", `nre-verdict-${item.verdict}`);
29
+ const content = (
30
+ <>
30
31
  {item.locator}
31
32
  <span className="nre-locator-mark">{verdictMark(item.verdict)}</span>
33
+ </>
34
+ );
35
+ return attemptHref ? (
36
+ <a className={className} href={attemptHref(item.locator)}>
37
+ {content}
32
38
  </a>
39
+ ) : (
40
+ <span className={className}>{content}</span>
33
41
  );
34
42
  }
35
43
 
@@ -44,7 +52,7 @@ export function failureSummaryText(item: Pick<AttemptListItem, "failureSummary"
44
52
  /** 一条 Attempt 的比较卡片;完整 assertions 通过 locator 下钻,不在列表内展开。 */
45
53
  export function AttemptRow({
46
54
  item,
47
- attemptHref = DEFAULT_ATTEMPT_HREF,
55
+ attemptHref,
48
56
  locale = DEFAULT_REPORT_LOCALE,
49
57
  }: {
50
58
  item: AttemptListItem;
@@ -74,7 +82,7 @@ export function AttemptList({
74
82
  data,
75
83
  total,
76
84
  filter = false,
77
- attemptHref = DEFAULT_ATTEMPT_HREF,
85
+ attemptHref,
78
86
  className,
79
87
  locale = DEFAULT_REPORT_LOCALE,
80
88
  }: {
@@ -1,19 +1,18 @@
1
1
  // ExperimentList:实体列表的第一级。web 面是一行一个 experiment 的固定八列比较表
2
- // (Experiment / Model / Agent / Avg duration / End-to-end pass rate / Tokens / Cost / Result),
2
+ // (Experiment / Model / Agent / Avg. time / Pass rate / Tokens / Cost / Results),
3
3
  // 每行用原生 <details> 展开到 Eval 与 Attempt locator。数据完全来自 experimentListData(),
4
4
  // 组件不重算、不推断组边界。
5
5
 
6
6
  import type { ReactElement } from "react";
7
- import type { AttemptLocator } from "../../results/locator.ts";
8
- import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "../types.ts";
9
- import { experimentDisplayName } from "../format.ts";
10
- import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
7
+ import type { AttemptLocator } from "../../../results/locator.ts";
8
+ import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "../../model/types.ts";
9
+ import { experimentDisplayName } from "../../model/format.ts";
10
+ import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../../model/locale.ts";
11
11
  import { AttemptLocatorBadge, failureSummaryText } from "./AttemptList.tsx";
12
- import { MetricCellView } from "./cell.tsx";
13
- import { colorClassForKey } from "./colors.ts";
14
- import { cx, formatDurationMs, formatUSD, verdictMark } from "./format.ts";
15
-
16
- const DEFAULT_ATTEMPT_HREF = (locator: AttemptLocator): string => `#/attempt/${locator}`;
12
+ import { MetricCellView } from "../cell.tsx";
13
+ import { colorClassForKey } from "../../assets/colors.ts";
14
+ import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.ts";
15
+ import { cx } from "../shared.ts";
17
16
 
18
17
  const verdictOrder = ["passed", "failed", "errored", "skipped"] as const;
19
18
 
@@ -39,7 +38,7 @@ function VerdictSummary({ item, locale }: { item: ExperimentListItem; locale: Re
39
38
  const parts = verdictOrder
40
39
  .filter((verdict) => item.evalVerdicts[verdict] > 0)
41
40
  .map((verdict) => `${item.evalVerdicts[verdict]} ${localeText(locale, `verdict.${verdict}`)}`);
42
- return <span className="nre-experiment-pill">{parts.join(" / ") || "—"}</span>;
41
+ return <span className="nre-experiment-pill">{parts.join(" · ") || "—"}</span>;
43
42
  }
44
43
 
45
44
  function ExperimentAttemptRow({
@@ -50,7 +49,7 @@ function ExperimentAttemptRow({
50
49
  }: {
51
50
  attempt: AttemptListItem;
52
51
  last: boolean;
53
- attemptHref: (locator: AttemptLocator) => string;
52
+ attemptHref?: (locator: AttemptLocator) => string;
54
53
  locale: ReportLocale;
55
54
  }): ReactElement {
56
55
  const reason = failureSummaryText(attempt, locale);
@@ -76,7 +75,7 @@ function EvalAttempts({
76
75
  locale,
77
76
  }: {
78
77
  row: ExperimentListEvalRow;
79
- attemptHref: (locator: AttemptLocator) => string;
78
+ attemptHref?: (locator: AttemptLocator) => string;
80
79
  locale: ReportLocale;
81
80
  }): ReactElement {
82
81
  const duration = row.durationMs.value === null ? localeText(locale, "cell.missing") : formatDurationMs(row.durationMs.value);
@@ -128,16 +127,16 @@ function ExperimentRow({
128
127
  relativeTo,
129
128
  }: {
130
129
  item: ExperimentListItem;
131
- attemptHref: (locator: AttemptLocator) => string;
130
+ attemptHref?: (locator: AttemptLocator) => string;
132
131
  locale: ReportLocale;
133
132
  relativeTo?: string;
134
133
  }): ReactElement {
135
134
  return (
136
135
  <details className="nre-experiment-entry">
137
136
  <summary className="nre-experiment-summary">
138
- {/* relativeTo 只影响显示末段;完整 id 仍是排序 / 着色 / 过滤 / 折叠的键 */}
137
+ {/* relativeTo 只影响显示末段;完整 id 仍是排序 / 过滤 / 折叠的键,系列色跟随 agent */}
139
138
  <span className="nre-experiment-name" data-sort-value={item.experimentId}>
140
- <b className={cx("nre-experiment-id", "nre-key", colorClassForKey(item.experimentId))}>
139
+ <b className={cx("nre-experiment-id", "nre-key")}>
141
140
  {experimentDisplayName(item.experimentId, relativeTo)}
142
141
  </b>
143
142
  <small>
@@ -147,7 +146,9 @@ function ExperimentRow({
147
146
  </small>
148
147
  </span>
149
148
  <span data-sort-value={item.model ?? ""}>{item.model ?? localeText(locale, "experimentList.defaultModel")}</span>
150
- <span data-sort-value={item.agent}>{item.agent}</span>
149
+ <span data-sort-value={item.agent}>
150
+ <span className={cx("nre-experiment-agent", "nre-key", colorClassForKey(item.agent))}>{item.agent}</span>
151
+ </span>
151
152
  <span className="nre-num" data-sort-value={item.durationMs.value ?? ""}>
152
153
  <MetricCellView cell={item.durationMs} locale={locale} />
153
154
  </span>
@@ -179,7 +180,7 @@ function ExperimentRow({
179
180
 
180
181
  export function ExperimentList({
181
182
  data,
182
- attemptHref = DEFAULT_ATTEMPT_HREF,
183
+ attemptHref,
183
184
  filter = false,
184
185
  className,
185
186
  locale = DEFAULT_REPORT_LOCALE,
@@ -209,10 +210,12 @@ export function ExperimentList({
209
210
  <button
210
211
  type="button"
211
212
  data-nre-experiment-sort={index}
212
- className={index === 4 ? "nre-sort-desc" : undefined}
213
+ className={cx(index >= 3 && index <= 6 && "nre-num-head", index === 4 && "nre-sort-desc")}
213
214
  key={label}
215
+ title={index === 4 ? localeText(locale, "experimentList.passRateDescription") : undefined}
214
216
  >
215
- {label}
217
+ <span className="nre-sort-label">{label}</span>
218
+ <span className="nre-sort-icon" aria-hidden="true" />
216
219
  </button>
217
220
  ))}
218
221
  </div>