niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -4,15 +4,15 @@
4
4
  // skipped=null、null≠0、Scoreboard 固定分母(notRun/unscorable 分开)、权重最长前缀、
5
5
  // 身份键去重、现刻水位、自定义指标 where/aggregate、evalGroup 完整父路径、verdict 权威、
6
6
  // MetricCell 诚实、缺 artifact 指标、repeatedFailedCommands、实体列表 failureSummary、
7
- // scopeSummaryData 两级计票、experimentComparisonData 分区、pairsByFlag、
7
+ // scopeSummaryData 两级计票、experimentListData/scopeSummaryData 的 selectedEvalIds 投影、pairsByFlag、
8
8
  // MetricLine 点身份、空数组反馈、metricTableData sort。
9
9
 
10
10
  import { describe, expect, it } from "vitest";
11
11
 
12
- import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "../types.ts";
13
- import type { AttemptHandle, Scope, ScopeWarning, Snapshot } from "../results/index.ts";
14
- import { makeScope, selectCurrentResults } from "../results/select.ts";
15
- import type { Results } from "../results/types.ts";
12
+ import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "../../types.ts";
13
+ import type { AttemptHandle, Scope, ScopeWarning, Snapshot } from "../../results/index.ts";
14
+ import { makeScope, selectCurrentResults } from "../../results/select.ts";
15
+ import type { Results } from "../../results/types.ts";
16
16
  import {
17
17
  assistantTurns,
18
18
  costUSD,
@@ -23,23 +23,33 @@ import {
23
23
  executionReliability,
24
24
  repeatedFailedCommands,
25
25
  taskPassRate,
26
- } from "./metrics.ts";
27
- import { flag, numericFlag } from "./flag.ts";
26
+ } from "../model/metrics.ts";
27
+ import { flag, label, numericFlag, numericLabel } from "../model/flag.ts";
28
+ import { scatterText } from "./metric-views/faces.ts";
29
+ import { createTextContext } from "../definition/tree.ts";
30
+ import { colorIndexForKey, colorIndicesForKeys } from "../assets/colors.ts";
31
+ import { attemptListData, evalListData, experimentListData } from "./entity-lists/compute.ts";
32
+ import { validateAttemptListData, validateEvalListData, validateExperimentListData } from "./entity-lists/index.tsx";
28
33
  import {
29
- attemptListData,
30
34
  deltaTableData,
31
- evalListData,
32
- experimentComparisonData,
33
- experimentListData,
34
35
  metricLineData,
35
36
  metricMatrixData,
36
37
  metricScatterData,
37
38
  metricTableData,
38
39
  pairsByFlag,
39
- scopeSummaryData,
40
40
  scoreboardData,
41
- } from "./compute.ts";
42
- import { evalGroupOf } from "./aggregate.ts";
41
+ } from "./metric-views/compute.ts";
42
+ import {
43
+ validateDeltaData,
44
+ validateLineData,
45
+ validateMatrixData,
46
+ validateScatterData,
47
+ validateScoreboardData,
48
+ validateTableData,
49
+ } from "./metric-views/index.tsx";
50
+ import { scopeSummaryData } from "./summaries/compute.ts";
51
+ import { validateScopeSummaryData } from "./summaries/index.tsx";
52
+ import { evalGroupOf } from "../model/aggregate.ts";
43
53
 
44
54
  // ───────────────────────── fake 数据(按 results 读取契约造)─────────────────────────
45
55
 
@@ -183,7 +193,7 @@ describe("两级聚合口径", () => {
183
193
  expect(cells[executionReliability.name]!.value).toBeCloseTo(2 / 3);
184
194
  });
185
195
 
186
- it("2 passed + 5 errored 的默认成功率是 2/7,不是 100%", async () => {
196
+ it("2 passed + 5 errored 的默认通过率是 2/7,不是 100%", async () => {
187
197
  const s = snap({
188
198
  experimentId: "exp/err",
189
199
  results: [
@@ -595,7 +605,7 @@ describe("实体列表 data", () => {
595
605
  expect(byEval.get("list/errored")!.costUSD).toBeNull();
596
606
  });
597
607
 
598
- it("experimentListData:evalVerdicts / endToEndPassRate / costUSD / durationMs / tokens 齐全,默认按端到端成功率降序", async () => {
608
+ it("experimentListData:evalVerdicts / endToEndPassRate / costUSD / durationMs / tokens 齐全,默认按端到端通过率降序", async () => {
599
609
  const winner = snap({ experimentId: "exp/win", results: [res("a", "passed"), res("b", "passed")] });
600
610
  const loser = snap({ experimentId: "exp/lose", results: [res("a", "failed"), res("b", "passed")] });
601
611
  const items = await experimentListData([loser, winner]);
@@ -667,32 +677,59 @@ describe("scopeSummaryData", () => {
667
677
  });
668
678
  });
669
679
 
670
- // ───────────────────────── experimentComparisonData ─────────────────────────
680
+ // ───────────────────────── experimentListData / scopeSummaryData 的 selectedEvalIds 投影 ─────────────────────────
681
+ // ExperimentComparison 已收为普通组合组件(把同一个 input 原样透传给 ScopeSummary /
682
+ // MetricScatter / ExperimentList,不再有自己的 data 形态);经它展开后与直接调用这三个
683
+ // 函数深等的验证挪到 dual-render.test.tsx(需要 resolve 管线)。这里只测三个函数自己的
684
+ // selectedEvalIds 投影契约。
671
685
 
672
- describe("experimentComparisonData", () => {
673
- it("计算前按完整父路径分区,根目录 experiment 单例组;每组与独立调用三个计算函数深等,refs 不跨组", async () => {
686
+ describe("experimentListData / scopeSummaryData 的 selectedEvalIds 投影", () => {
687
+ it("不同深度目录的 experiments 一律进同一份 data,不按父路径分组比较", async () => {
674
688
  const g1a = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
675
689
  const g1b = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
676
690
  const g2 = snap({ experimentId: "bench/long/x", results: [res("q", "passed")] });
677
691
  const solo = snap({ experimentId: "standalone", results: [res("q", "errored", { error: erroredWith("x") })] });
692
+ const items = await experimentListData([g1a, g1b, g2, solo]);
693
+ expect(items.map((e) => e.experimentId).sort()).toEqual([
694
+ "bench/long/x",
695
+ "compare/a",
696
+ "compare/b",
697
+ "standalone",
698
+ ]);
699
+ });
700
+
701
+ it("每个 experiment 只保留自己选择的 eval:A 声明 selectedEvalIds:[q1] 但夹带 q2 attempt,B 只选 q2;q2 不污染 A", async () => {
702
+ const a = snap({
703
+ experimentId: "exp/a",
704
+ results: [res("q1", "passed"), res("q2", "failed")],
705
+ experiment: { runs: 1, earlyExit: false, selectedEvalIds: ["q1"] },
706
+ });
707
+ const b = snap({
708
+ experimentId: "exp/b",
709
+ results: [res("q2", "passed")],
710
+ experiment: { runs: 1, earlyExit: false, selectedEvalIds: ["q2"] },
711
+ });
678
712
 
679
- const data = await experimentComparisonData([g1a, g1b, g2, solo]);
680
- expect(data.groups.map((g) => g.key)).toEqual(["bench/long", "compare", "standalone"]);
713
+ const items = await experimentListData([a, b]);
714
+ const rowA = items.find((e) => e.experimentId === "exp/a")!;
715
+ const rowB = items.find((e) => e.experimentId === "exp/b")!;
716
+ expect(rowA.evals).toBe(1); // 只统计 q1,夹带的 q2 attempt 不计入
717
+ expect(rowA.evalRows.map((r) => r.evalId)).toEqual(["q1"]);
718
+ expect(rowB.evals).toBe(1);
719
+ expect(rowB.evalRows.map((r) => r.evalId)).toEqual(["q2"]);
720
+
721
+ // 汇总口径同样不被污染:全 Scope 只有 2 个 eval(exp/a·q1、exp/b·q2),不是 3 个。
722
+ const summary = await scopeSummaryData([a, b]);
723
+ expect(summary.evals).toBe(2);
724
+ });
681
725
 
682
- const compare = data.groups.find((g) => g.key === "compare")!;
683
- const [summary, scatter, experiments] = await Promise.all([
684
- scopeSummaryData([g1a, g1b]),
685
- metricScatterData([g1a, g1b], { points: "experiment", series: "agent", x: costUSD, y: endToEndPassRate }),
686
- experimentListData([g1a, g1b]),
687
- ]);
688
- expect(compare.summary).toEqual(summary);
689
- expect(compare.scatter).toEqual(scatter);
690
- expect(compare.experiments).toEqual(experiments);
726
+ it("第三方快照缺 experiment 信息时仍可见,按其实际 evals 参与(selectedEvalIdsOf 退化)", async () => {
727
+ const thirdParty = snap({ experimentId: "third-party/exp", results: [res("q", "passed")] });
728
+ expect(thirdParty.experiment).toBeUndefined();
691
729
 
692
- // 组外 attempt 不污染:compare 组的 refs 不含 bench/standalone 的 locator
693
- const soloRefs = data.groups.find((g) => g.key === "standalone")!.summary.endToEndPassRate.refs;
694
- const compareRefs = compare.summary.endToEndPassRate.refs;
695
- expect(compareRefs.some((ref) => soloRefs.includes(ref))).toBe(false);
730
+ const items = await experimentListData([thirdParty]);
731
+ expect(items.map((e) => e.experimentId)).toEqual(["third-party/exp"]);
732
+ expect(items[0]!.evals).toBe(1);
696
733
  });
697
734
  });
698
735
 
@@ -918,22 +955,79 @@ describe("deltaTableData", () => {
918
955
  ];
919
956
  };
920
957
 
921
- it("同可比组 + 删除该 flag 后配置深相等才配对;a 取 baseline(缺省=未声明),label 自动生成,按 (a 末段, 显示键) 字典序", async () => {
958
+ it("input Scope 内删除该 flag 后配置深相等才配对;a 取 baseline(缺省=未声明),label 用完整 a experiment id,按 (a 末段, 显示键) 字典序", async () => {
922
959
  const data = await deltaTableData(matrixSnaps(), {
923
960
  by: "experiment",
924
961
  pairs: pairsByFlag("memory"),
925
962
  metrics: [endToEndPassRate],
926
963
  });
927
964
  expect(data.rows.map((r) => r.label)).toEqual([
928
- "bub · memory=agents-md",
929
- "codex · memory=agents-md",
930
- "codex · memory=mempal",
931
- "gemini · memory=agents-md",
932
- "gemini · memory=mempal",
965
+ "mem/bub · memory=agents-md",
966
+ "mem/codex · memory=agents-md",
967
+ "mem/codex · memory=mempal",
968
+ "mem/gemini · memory=agents-md",
969
+ "mem/gemini · memory=mempal",
933
970
  ]);
934
971
  expect(data.experiments).toBe(8);
935
972
  });
936
973
 
974
+ it("配对边界只是 input Scope,不额外按 experiment id 的目录前缀分组:不同前缀但可比性配置相同就配对", async () => {
975
+ const mk = (id: string, memory?: string) =>
976
+ snap({
977
+ experimentId: id,
978
+ agent: "bub",
979
+ results: [res("q", "passed")],
980
+ experiment: {
981
+ runs: 1,
982
+ earlyExit: false,
983
+ selectedEvalIds: [],
984
+ ...(memory !== undefined ? { flags: { memory } } : {}),
985
+ },
986
+ });
987
+ const data = await deltaTableData([mk("compare/codex"), mk("bench/codex", "agents-md")], {
988
+ by: "experiment",
989
+ pairs: pairsByFlag("memory"),
990
+ metrics: [endToEndPassRate],
991
+ });
992
+ expect(data.rows.map((r) => r.label)).toEqual(["compare/codex · memory=agents-md"]);
993
+ expect(data.rows[0]!.a.key).toBe("compare/codex");
994
+ expect(data.rows[0]!.b.key).toBe("bench/codex");
995
+ });
996
+
997
+ it("排序只看 a 末段与 flag 显示键,不看完整 label 字符串:a 末段相同、完整 id 不同的两个 bucket 不因目录前缀打乱顺序", async () => {
998
+ const mk = (id: string, model: string, memory?: string) =>
999
+ snap({
1000
+ experimentId: id,
1001
+ model,
1002
+ agent: "bub",
1003
+ results: [res("q", "passed")],
1004
+ experiment: {
1005
+ runs: 1,
1006
+ earlyExit: false,
1007
+ selectedEvalIds: [],
1008
+ ...(memory !== undefined ? { flags: { memory } } : {}),
1009
+ },
1010
+ });
1011
+ // 两个 bucket 因 model 不同而彼此不可比(不会互相配对),但都各自派生一对;
1012
+ // 两对的 a 末段都是 "codex",flagKey 分别是 "z-value" 与 "a-value"。
1013
+ const data = await deltaTableData(
1014
+ [
1015
+ mk("alpha/codex", "m1"), // bucket 1 baseline
1016
+ mk("alpha/codex-variant", "m1", "z-value"), // bucket 1 b 侧
1017
+ mk("zeta/codex", "m2"), // bucket 2 baseline
1018
+ mk("zeta/codex-variant", "m2", "a-value"), // bucket 2 b 侧
1019
+ ],
1020
+ { by: "experiment", pairs: pairsByFlag("memory"), metrics: [endToEndPassRate] },
1021
+ );
1022
+ // 只看 a 末段("codex",两对相同)与 flag 显示键("a-value" < "z-value")排序:
1023
+ // zeta 那对(flagKey "a-value")排在 alpha 那对(flagKey "z-value")之前——若误按完整
1024
+ // label 字符串排序,"alpha/codex..." 会先于 "zeta/codex...",顺序会相反。
1025
+ expect(data.rows.map((r) => r.label)).toEqual([
1026
+ "zeta/codex · memory=a-value",
1027
+ "alpha/codex · memory=z-value",
1028
+ ]);
1029
+ });
1030
+
937
1031
  it("可比性配置不同(model 不同)的两实验不配对", async () => {
938
1032
  const base = snap({
939
1033
  experimentId: "mm/a",
@@ -977,13 +1071,13 @@ describe("deltaTableData", () => {
977
1071
  pairs: pairsByFlag("memory", { baseline: "agents-md" }),
978
1072
  metrics: [endToEndPassRate],
979
1073
  });
980
- // a = *--agents-md;b = 未声明(显示键 (missing))与 mempal
1074
+ // a = *--agents-md;b = 未声明(显示键 (missing))与 mempal;label 用完整 a experiment id。
981
1075
  expect(data.rows.map((r) => r.label)).toEqual([
982
- "bub--agents-md · memory=(missing)",
983
- "codex--agents-md · memory=(missing)",
984
- "codex--agents-md · memory=mempal",
985
- "gemini--agents-md · memory=(missing)",
986
- "gemini--agents-md · memory=mempal",
1076
+ "mem/bub--agents-md · memory=(missing)",
1077
+ "mem/codex--agents-md · memory=(missing)",
1078
+ "mem/codex--agents-md · memory=mempal",
1079
+ "mem/gemini--agents-md · memory=(missing)",
1080
+ "mem/gemini--agents-md · memory=mempal",
987
1081
  ]);
988
1082
  });
989
1083
  });
@@ -1009,3 +1103,179 @@ describe("examScore", () => {
1009
1103
  expect(cellOf("crashed").samples).toBe(1); // 0 分是测得的事实,不是缺数据
1010
1104
  });
1011
1105
  });
1106
+
1107
+ // ───────────────────────── labels 维度、series 归类与 connect ─────────────────────────
1108
+
1109
+ describe("labels 维度、series 归类与 connect", () => {
1110
+ const labeled = (
1111
+ experimentId: string,
1112
+ labels: Record<string, string | number> | undefined,
1113
+ verdicts: Verdict[],
1114
+ agent = "agent-x",
1115
+ ) =>
1116
+ snap({
1117
+ experimentId,
1118
+ agent,
1119
+ results: verdicts.map((v, i) => res(`q${i}`, v, { agent })),
1120
+ experiment: {
1121
+ runs: 1,
1122
+ earlyExit: false,
1123
+ selectedEvalIds: verdicts.map((_, i) => `q${i}`),
1124
+ ...(labels ? { labels } : {}),
1125
+ },
1126
+ });
1127
+
1128
+ it("label() 按声明值分组、未声明归 (missing);numericLabel 对字符串值返回 null 不猜序", async () => {
1129
+ const table = await metricTableData(
1130
+ [labeled("m/one", { memory: "mempal" }, ["passed"]), labeled("m/two", undefined, ["failed"])],
1131
+ { rows: label("memory"), columns: [endToEndPassRate] },
1132
+ );
1133
+ expect(table.rows.map((r) => r.key).sort()).toEqual(["(missing)", "mempal"]);
1134
+
1135
+ const line = await metricLineData(
1136
+ [labeled("m/num", { contextK: 32 }, ["passed"]), labeled("m/str", { contextK: "big" }, ["passed"])],
1137
+ { x: numericLabel("contextK"), y: endToEndPassRate },
1138
+ );
1139
+ expect(line.rows.filter((r) => r.x === null)).toHaveLength(1);
1140
+ expect(line.rows.some((r) => r.x === 32)).toBe(true);
1141
+ expect(line.rows.some((r) => r.x === 0)).toBe(false);
1142
+ });
1143
+
1144
+ it("series 数组解析为复合维度:name 以 × 连接、值以 · 连接,缺失成员沿用 (missing)", async () => {
1145
+ const a = labeled("c/one", { memory: "mempal" }, ["passed"], "codex");
1146
+ const b = labeled("c/two", undefined, ["failed"], "codex");
1147
+ const data = await metricScatterData([a, b], {
1148
+ points: "experiment",
1149
+ series: ["agent", label("memory")],
1150
+ x: costUSD,
1151
+ y: endToEndPassRate,
1152
+ });
1153
+ expect(data.seriesDimension).toBe("agent × memory");
1154
+ const byKey = new Map(data.rows.map((r) => [r.key, r.series]));
1155
+ expect(byKey.get("c/one")).toBe("codex · mempal");
1156
+ expect(byKey.get("c/two")).toBe("codex · (missing)");
1157
+ // 单成员数组等价于单维度
1158
+ const single = await metricScatterData([a], { points: "experiment", series: ["agent"], x: costUSD, y: endToEndPassRate });
1159
+ expect(single.seriesDimension).toBe("agent");
1160
+ expect(single.rows[0]!.series).toBe("codex");
1161
+ });
1162
+
1163
+ const cell = (value: number, display: string): import("../model/types.ts").MetricCell => ({
1164
+ value,
1165
+ display,
1166
+ samples: 1,
1167
+ total: 1,
1168
+ refs: [],
1169
+ });
1170
+ const lineScatter = (): import("../model/types.ts").ScatterData => ({
1171
+ pointDimension: "experiment",
1172
+ seriesDimension: "line",
1173
+ x: { key: "costUSD", label: "Cost", unit: "$", better: "lower" },
1174
+ y: { key: "endToEndPassRate", label: "Pass rate", unit: "%", better: "higher" },
1175
+ rows: [
1176
+ { key: "mem/claude-mempal", series: "claude", x: cell(0.55, "$0.55"), y: cell(0.75, "75%") },
1177
+ { key: "mem/claude-baseline", series: "claude", x: cell(0.35, "$0.35"), y: cell(0.5, "50%") },
1178
+ { key: "mem/bub", series: "bub", x: cell(0.09, "$0.09"), y: cell(0.875, "87.5%") },
1179
+ ],
1180
+ });
1181
+
1182
+ it("scatterText:标记按图例顺序分配(series 字典序、series 内 x 升序),标题标注归类维度;默认无箭头", () => {
1183
+ const text = scatterText(lineScatter(), createTextContext({ width: 80 }));
1184
+ expect(text).toContain("grouped by line");
1185
+ // bub < claude:A 给 bub;claude 内按成本升序 baseline(B) → mempal(C)
1186
+ expect(text).toContain("bub A mem/bub");
1187
+ expect(text).toContain("B mem/claude-baseline");
1188
+ expect(text).toContain("C mem/claude-mempal");
1189
+ expect(text).not.toContain("→ C mem/claude-mempal");
1190
+ expect(text).not.toContain("└ Pass rate");
1191
+ });
1192
+
1193
+ it("scatterText connect:图例按 x 升序 → 串联并给逐段位移摘要;单点 series 无箭头无摘要;坐标图内不画折线", () => {
1194
+ const text = scatterText(lineScatter(), createTextContext({ width: 80 }), { connect: true });
1195
+ expect(text).toContain("B mem/claude-baseline → C mem/claude-mempal");
1196
+ expect(text).toContain("└ Pass rate +25pt · Cost +$0.20");
1197
+ expect(text).toContain("A mem/bub");
1198
+ expect(text).not.toContain("A mem/bub →");
1199
+ // 坐标图折线用 · 描画;connect 的 text 投影是图例的位移摘要,网格行(│ 开头)不出现折线点
1200
+ const plotRows = text.split("\n").filter((l) => l.includes("│"));
1201
+ expect(plotRows.length).toBeGreaterThan(0);
1202
+ expect(plotRows.every((l) => !l.includes("·"))).toBe(true);
1203
+ });
1204
+
1205
+ it("同图撞色按图例顺序线性探测空格;无冲突键仍取散列格;超过色板才复用", () => {
1206
+ // 真实回归现场:bub / claude-code / codex 在同一张图里必须三色互异
1207
+ const real = colorIndicesForKeys(["bub", "claude-code", "codex"]);
1208
+ expect(new Set(real.values()).size).toBe(3);
1209
+ // 构造性冲突:找两个散列同格的键,后到者让位,先到者保持散列格(跨图稳定)
1210
+ const pool = Array.from({ length: 40 }, (_, i) => `k${i}`);
1211
+ const byIdx = new Map<number, string[]>();
1212
+ for (const k of pool) {
1213
+ const idx = colorIndexForKey(k);
1214
+ byIdx.set(idx, [...(byIdx.get(idx) ?? []), k]);
1215
+ }
1216
+ const [first, second] = [...byIdx.values()].find((keys) => keys.length >= 2)!;
1217
+ const resolved = colorIndicesForKeys([first, second]);
1218
+ expect(resolved.get(first)).toBe(colorIndexForKey(first));
1219
+ expect(resolved.get(second)).not.toBe(resolved.get(first));
1220
+ // 前 6 个键占满 6 色;第 7 个开始只能复用
1221
+ const seven = colorIndicesForKeys(["s0", "s1", "s2", "s3", "s4", "s5", "s6"]);
1222
+ expect(new Set([...seven.values()].slice(0, 6)).size).toBe(6);
1223
+ });
1224
+ });
1225
+
1226
+ describe("validate*Data 接受真实计算产物(不是只接受手写 literal)", () => {
1227
+ // 手写 literal 通过校验只证明 validator 认得自己设想的形状;真正的把关是喂真实计算产物——
1228
+ // 尤其是省略可选字段(如自定义指标不声明 label)的真实路径,不是校验作者假想的边界。
1229
+ const a = snap({ experimentId: "compare/a", results: [res("q1", "passed"), res("q2", "failed")] });
1230
+ const b = snap({ experimentId: "compare/b", results: [res("q1", "failed"), res("q2", "passed")] });
1231
+ const scope = scopeOf([a, b]);
1232
+ const noLabelMetric = defineMetric({ name: "custom-no-label", value: (attempt) => (attempt.result.verdict === "passed" ? 1 : 0) });
1233
+
1234
+ it("metricTableData:含未声明 label 的自定义指标", async () => {
1235
+ const table = await metricTableData(scope, { rows: "agent", columns: [costUSD, noLabelMetric] });
1236
+ expect(validateTableData(table)).toBeNull();
1237
+ });
1238
+
1239
+ it("metricMatrixData", async () => {
1240
+ const matrix = await metricMatrixData(scope, { rows: "experiment", columns: "eval", cell: endToEndPassRate });
1241
+ expect(validateMatrixData(matrix)).toBeNull();
1242
+ });
1243
+
1244
+ it("metricScatterData:含缺 y 的点(null MetricCell.value)", async () => {
1245
+ const scatter = await metricScatterData(scope, { points: "experiment", x: costUSD, y: noLabelMetric });
1246
+ expect(validateScatterData(scatter)).toBeNull();
1247
+ });
1248
+
1249
+ it("metricLineData", async () => {
1250
+ const line = await metricLineData(scope, { x: numericFlag("budget"), y: endToEndPassRate });
1251
+ expect(validateLineData(line)).toBeNull();
1252
+ });
1253
+
1254
+ it("scoreboardData", async () => {
1255
+ const board = await scoreboardData(scope, { rows: "agent", questions: ["q1", "q2"] });
1256
+ expect(validateScoreboardData(board)).toBeNull();
1257
+ });
1258
+
1259
+ it("deltaTableData(含 pairsByFlag 派生 pair)", async () => {
1260
+ const withFlag = snap({
1261
+ experimentId: "compare/withflag",
1262
+ results: [res("q1", "passed")],
1263
+ experiment: { runs: 1, earlyExit: false, selectedEvalIds: [], flags: { memory: "on" } },
1264
+ });
1265
+ const withoutFlag = snap({ experimentId: "compare/noflag", results: [res("q1", "failed")] });
1266
+ const deltaScope = scopeOf([withFlag, withoutFlag]);
1267
+ const delta = await deltaTableData(deltaScope, { by: "experiment", pairs: pairsByFlag("memory"), metrics: [costUSD] });
1268
+ expect(validateDeltaData(delta)).toBeNull();
1269
+ });
1270
+
1271
+ it("scopeSummaryData", async () => {
1272
+ const summary = await scopeSummaryData(scope);
1273
+ expect(validateScopeSummaryData(summary)).toBeNull();
1274
+ });
1275
+
1276
+ it("experimentListData / evalListData / attemptListData(同一 Scope 三种粒度)", async () => {
1277
+ expect(validateExperimentListData(await experimentListData(scope))).toBeNull();
1278
+ expect(validateEvalListData(await evalListData(scope))).toBeNull();
1279
+ expect(validateAttemptListData(await attemptListData(scope))).toBeNull();
1280
+ });
1281
+ });
@@ -6,30 +6,38 @@
6
6
  // 也是它们展开区里"逐条 attempt"那一层的唯一实现(通过 AttemptRow 导出复用,不重写一遍)。
7
7
 
8
8
  import type { ReactElement } from "react";
9
- import type { AttemptListItem } from "../types.ts";
10
- import type { AttemptLocator } from "../../results/locator.ts";
11
- import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "../locale.ts";
12
- import { colorClassForKey } from "./colors.ts";
13
- import { cx, formatDurationMs, formatUSD, verdictMark } from "./format.ts";
9
+ import type { AttemptListItem } from "../../model/types.ts";
10
+ import type { AttemptLocator } from "../../../results/locator.ts";
11
+ import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "../../model/locale.ts";
12
+ import { colorClassForKey } from "../../assets/colors.ts";
13
+ import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.ts";
14
+ import { cx } from "../shared.ts";
14
15
 
15
- const DEFAULT_ATTEMPT_HREF = (locator: AttemptLocator): string => `#/attempt/${locator}`;
16
-
17
- /** locator + 判定符的普通 <a>,AttemptList/EvalList/ExperimentList 共用。 */
16
+ /**
17
+ * locator + 判定符,AttemptList/EvalList/ExperimentList 共用。没有 target(当前报告没有
18
+ * declare attempt-input page,也没有显式 attemptHref)时是纯文本,不生成空 href 或假链接
19
+ * (docs/feature/reports/architecture.md「Attempt 详情是一张参数化 page」)。
20
+ */
18
21
  export function AttemptLocatorBadge({
19
22
  item,
20
- attemptHref = DEFAULT_ATTEMPT_HREF,
23
+ attemptHref,
21
24
  }: {
22
25
  item: Pick<AttemptListItem, "locator" | "verdict">;
23
26
  attemptHref?: (locator: AttemptLocator) => string;
24
27
  }): ReactElement {
25
- return (
26
- <a
27
- className={cx("nre-locator", `nre-verdict-${item.verdict}`)}
28
- href={attemptHref(item.locator)}
29
- >
28
+ const className = cx("nre-locator", `nre-verdict-${item.verdict}`);
29
+ const content = (
30
+ <>
30
31
  {item.locator}
31
32
  <span className="nre-locator-mark">{verdictMark(item.verdict)}</span>
33
+ </>
34
+ );
35
+ return attemptHref ? (
36
+ <a className={className} href={attemptHref(item.locator)}>
37
+ {content}
32
38
  </a>
39
+ ) : (
40
+ <span className={className}>{content}</span>
33
41
  );
34
42
  }
35
43
 
@@ -44,7 +52,7 @@ export function failureSummaryText(item: Pick<AttemptListItem, "failureSummary"
44
52
  /** 一条 Attempt 的比较卡片;完整 assertions 通过 locator 下钻,不在列表内展开。 */
45
53
  export function AttemptRow({
46
54
  item,
47
- attemptHref = DEFAULT_ATTEMPT_HREF,
55
+ attemptHref,
48
56
  locale = DEFAULT_REPORT_LOCALE,
49
57
  }: {
50
58
  item: AttemptListItem;
@@ -74,7 +82,7 @@ export function AttemptList({
74
82
  data,
75
83
  total,
76
84
  filter = false,
77
- attemptHref = DEFAULT_ATTEMPT_HREF,
85
+ attemptHref,
78
86
  className,
79
87
  locale = DEFAULT_REPORT_LOCALE,
80
88
  }: {