niceeval 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +15 -7
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
  118. package/dist/report/{tree.js → definition/tree.js} +10 -12
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +12 -2
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +15 -1
  154. package/dist/shared/aggregate.js +32 -1
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +77 -64
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +30 -15
  227. package/src/i18n/zh-CN.ts +30 -15
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +136 -112
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +58 -22
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +54 -3
  314. package/src/results/select.ts +30 -6
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +30 -1
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +58 -22
  356. package/src/show/render.ts +16 -240
  357. package/src/show/show.test.ts +128 -35
  358. package/src/view/app/App.test.tsx +8 -7
  359. package/src/view/app/App.tsx +135 -51
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +122 -89
  370. package/src/view/index.ts +3 -13
  371. package/src/view/server.ts +36 -6
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +18 -806
  376. package/src/view/view-report.test.ts +182 -34
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,11 +1,11 @@
1
1
  import { Fragment as _Fragment, jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
2
- import { experimentDisplayName } from "../format.js";
3
- import { DEFAULT_REPORT_LOCALE, localeText } from "../locale.js";
2
+ import { experimentDisplayName } from "../../model/format.js";
3
+ import { DEFAULT_REPORT_LOCALE, localeText } from "../../model/locale.js";
4
4
  import { AttemptLocatorBadge, failureSummaryText } from "./AttemptList.js";
5
- import { MetricCellView } from "./cell.js";
6
- import { colorClassForKey } from "./colors.js";
7
- import { cx, formatDurationMs, formatUSD, verdictMark } from "./format.js";
8
- const DEFAULT_ATTEMPT_HREF = (locator) => `#/attempt/${locator}`;
5
+ import { MetricCellView } from "../cell.js";
6
+ import { colorClassForKey } from "../../assets/colors.js";
7
+ import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.js";
8
+ import { cx } from "../shared.js";
9
9
  const verdictOrder = ["passed", "failed", "errored", "skipped"];
10
10
  function passRateTone(value) {
11
11
  if (value === null)
@@ -31,7 +31,7 @@ function VerdictSummary({ item, locale }) {
31
31
  const parts = verdictOrder
32
32
  .filter((verdict) => item.evalVerdicts[verdict] > 0)
33
33
  .map((verdict) => `${item.evalVerdicts[verdict]} ${localeText(locale, `verdict.${verdict}`)}`);
34
- return _jsx("span", { className: "nre-experiment-pill", children: parts.join(" / ") || "—" });
34
+ return _jsx("span", { className: "nre-experiment-pill", children: parts.join(" · ") || "—" });
35
35
  }
36
36
  function ExperimentAttemptRow({ attempt, last, attemptHref, locale, }) {
37
37
  const reason = failureSummaryText(attempt, locale);
@@ -48,9 +48,9 @@ function Flags({ flags, locale }) {
48
48
  return (_jsxs("div", { className: "nre-experiment-flags", children: [_jsx("span", { children: localeText(locale, "experimentList.flags") }), Object.entries(flags).map(([key, value]) => (_jsxs("b", { children: [key, "=", typeof value === "string" ? value : JSON.stringify(value)] }, key)))] }));
49
49
  }
50
50
  function ExperimentRow({ item, attemptHref, locale, relativeTo, }) {
51
- return (_jsxs("details", { className: "nre-experiment-entry", children: [_jsxs("summary", { className: "nre-experiment-summary", children: [_jsxs("span", { className: "nre-experiment-name", "data-sort-value": item.experimentId, children: [_jsx("b", { className: cx("nre-experiment-id", "nre-key", colorClassForKey(item.experimentId)), children: experimentDisplayName(item.experimentId, relativeTo) }), _jsxs("small", { children: [localeText(locale, "overview.evalsCount", { n: item.evals }), item.attempts > item.evals ? ` · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })}` : "", ` · ${formatDate(item.lastRunAt, locale)}`] })] }), _jsx("span", { "data-sort-value": item.model ?? "", children: item.model ?? localeText(locale, "experimentList.defaultModel") }), _jsx("span", { "data-sort-value": item.agent, children: item.agent }), _jsx("span", { className: "nre-num", "data-sort-value": item.durationMs.value ?? "", children: _jsx(MetricCellView, { cell: item.durationMs, locale: locale }) }), _jsx("span", { className: cx("nre-num", passRateTone(item.endToEndPassRate.value)), "data-sort-value": item.endToEndPassRate.value ?? "", children: _jsx(MetricCellView, { cell: item.endToEndPassRate, locale: locale }) }), _jsx("span", { className: "nre-num", "data-sort-value": item.tokens.value ?? "", children: _jsx(MetricCellView, { cell: item.tokens, locale: locale }) }), _jsx("span", { className: "nre-num", "data-sort-value": item.costUSD.value ?? "", children: _jsx(MetricCellView, { cell: item.costUSD, locale: locale }) }), _jsx("span", { "data-sort-value": item.evalVerdicts.passed, children: _jsx(VerdictSummary, { item: item, locale: locale }) })] }), _jsxs("div", { className: "nre-experiment-detail", children: [_jsx(Flags, { flags: item.flags, locale: locale }), _jsx("ul", { className: "nre-experiment-evals", children: item.evalRows.map((row) => (_jsx(EvalAttempts, { row: row, attemptHref: attemptHref, locale: locale }, row.evalId))) })] })] }));
51
+ return (_jsxs("details", { className: "nre-experiment-entry", children: [_jsxs("summary", { className: "nre-experiment-summary", children: [_jsxs("span", { className: "nre-experiment-name", "data-sort-value": item.experimentId, children: [_jsx("b", { className: cx("nre-experiment-id", "nre-key"), children: experimentDisplayName(item.experimentId, relativeTo) }), _jsxs("small", { children: [localeText(locale, "overview.evalsCount", { n: item.evals }), item.attempts > item.evals ? ` · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })}` : "", ` · ${formatDate(item.lastRunAt, locale)}`] })] }), _jsx("span", { "data-sort-value": item.model ?? "", children: item.model ?? localeText(locale, "experimentList.defaultModel") }), _jsx("span", { "data-sort-value": item.agent, children: _jsx("span", { className: cx("nre-experiment-agent", "nre-key", colorClassForKey(item.agent)), children: item.agent }) }), _jsx("span", { className: "nre-num", "data-sort-value": item.durationMs.value ?? "", children: _jsx(MetricCellView, { cell: item.durationMs, locale: locale }) }), _jsx("span", { className: cx("nre-num", passRateTone(item.endToEndPassRate.value)), "data-sort-value": item.endToEndPassRate.value ?? "", children: _jsx(MetricCellView, { cell: item.endToEndPassRate, locale: locale }) }), _jsx("span", { className: "nre-num", "data-sort-value": item.tokens.value ?? "", children: _jsx(MetricCellView, { cell: item.tokens, locale: locale }) }), _jsx("span", { className: "nre-num", "data-sort-value": item.costUSD.value ?? "", children: _jsx(MetricCellView, { cell: item.costUSD, locale: locale }) }), _jsx("span", { "data-sort-value": item.evalVerdicts.passed, children: _jsx(VerdictSummary, { item: item, locale: locale }) })] }), _jsxs("div", { className: "nre-experiment-detail", children: [_jsx(Flags, { flags: item.flags, locale: locale }), _jsx("ul", { className: "nre-experiment-evals", children: item.evalRows.map((row) => (_jsx(EvalAttempts, { row: row, attemptHref: attemptHref, locale: locale }, row.evalId))) })] })] }));
52
52
  }
53
- export function ExperimentList({ data, attemptHref = DEFAULT_ATTEMPT_HREF, filter = false, className, locale = DEFAULT_REPORT_LOCALE, relativeTo, }) {
53
+ export function ExperimentList({ data, attemptHref, filter = false, className, locale = DEFAULT_REPORT_LOCALE, relativeTo, }) {
54
54
  const labels = [
55
55
  localeText(locale, "experimentList.experiment"),
56
56
  localeText(locale, "table.model"),
@@ -61,6 +61,6 @@ export function ExperimentList({ data, attemptHref = DEFAULT_ATTEMPT_HREF, filte
61
61
  localeText(locale, "experimentList.cost"),
62
62
  localeText(locale, "experimentList.result"),
63
63
  ];
64
- const board = (_jsxs("div", { className: "nre-experiment-table", children: [_jsx("div", { className: "nre-experiment-head", children: labels.map((label, index) => (_jsx("button", { type: "button", "data-nre-experiment-sort": index, className: index === 4 ? "nre-sort-desc" : undefined, children: label }, label))) }), data.length === 0 && _jsx("p", { className: "nre-experiment-list-empty", children: localeText(locale, "attemptList.empty") }), data.map((item) => (_jsx(ExperimentRow, { item: item, attemptHref: attemptHref, locale: locale, relativeTo: relativeTo }, item.experimentId)))] }));
64
+ const board = (_jsxs("div", { className: "nre-experiment-table", children: [_jsx("div", { className: "nre-experiment-head", children: labels.map((label, index) => (_jsxs("button", { type: "button", "data-nre-experiment-sort": index, className: cx(index >= 3 && index <= 6 && "nre-num-head", index === 4 && "nre-sort-desc"), title: index === 4 ? localeText(locale, "experimentList.passRateDescription") : undefined, children: [_jsx("span", { className: "nre-sort-label", children: label }), _jsx("span", { className: "nre-sort-icon", "aria-hidden": "true" })] }, label))) }), data.length === 0 && _jsx("p", { className: "nre-experiment-list-empty", children: localeText(locale, "attemptList.empty") }), data.map((item) => (_jsx(ExperimentRow, { item: item, attemptHref: attemptHref, locale: locale, relativeTo: relativeTo }, item.experimentId)))] }));
65
65
  return (_jsxs("div", { className: cx("nre", "nre-experiment-list", filter && "nre-experiment-table-wrap", className), children: [filter && (_jsx("input", { className: "nre-filter", "data-nre-experiment-filter": "", type: "search", placeholder: localeText(locale, "experimentList.filterPlaceholder") })), board] }));
66
66
  }
@@ -0,0 +1,23 @@
1
+ import type { AttemptListItem, EvalListItem, ExperimentListItem, ReportInput } from "../../model/types.ts";
2
+ import type { EvalResult } from "../../../types.ts";
3
+ /**
4
+ * 一次 attempt 的单行结果摘要(Scoring display 契约):failed 取主失败断言摘要(不含
5
+ * "+N more",N 单独进 moreFailures),errored 取结构化 error 的一层摘要
6
+ * (phase · code · message),passed / skipped 为 null。
7
+ */
8
+ export declare function failureSummaryOf(result: EvalResult): {
9
+ summary: string | null;
10
+ more: number;
11
+ };
12
+ /** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Scope 展平顺序(不重排)。 */
13
+ export declare function attemptListData(input: ReportInput): Promise<AttemptListItem[]>;
14
+ /** `evalListData(input)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
15
+ export declare function evalListData(input: ReportInput): Promise<EvalListItem[]>;
16
+ /**
17
+ * `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始按端到端通过率
18
+ * 从高到低(缺数据沉底,同分按 id)。一行只有一套 agent / model / flags 是输入约束:
19
+ * 宿主注入的 current() Scope 保证每个 experiment 只由可比性配置一致的快照拼成;作者自选
20
+ * Snapshot[] 时若同一 experiment 混入不一致的可比性配置,按完整用户反馈失败并指引——
21
+ * 看跨配置演化用 snapshot 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
22
+ */
23
+ export declare function experimentListData(input: ReportInput): Promise<ExperimentListItem[]>;
@@ -0,0 +1,171 @@
1
+ // 计算函数(*Data):ReportInput → 一份组件数据。实体列表族(ExperimentList / EvalList /
2
+ // AttemptList / FailureList)的 *Data 都住在这里(docs/feature/reports/library/entity-lists.md)。
3
+ //
4
+ // 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
5
+ // - 第一参收 ReportInput = Scope | readonly Snapshot[];warnings 不进组件数据(宿主统一显示);
6
+ // - 聚合前按身份键去重(dedupeAttempts;missing-startedAt 不去重、如实保留、不透出警告);
7
+ // - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露;
8
+ // - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
9
+ import { comparabilityConfigOf, deepEqualJson } from "../../../results/select.js";
10
+ import { foldEvalVerdict } from "../../../shared/verdict.js";
11
+ import { collectItems, computeCell, evalIdOf, experimentIdOf, fullEvalKey, groupItems, locatorOf, resolveInput, } from "../../model/aggregate.js";
12
+ import { attemptCostUSD, costUSD, durationMs, endToEndPassRate, examScore, tokens } from "../../model/metrics.js";
13
+ import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../../../scoring/display.js";
14
+ import { selectedEvalsOnly, summarizeItems } from "../shared-compute.js";
15
+ /**
16
+ * 一次 attempt 的单行结果摘要(Scoring display 契约):failed 取主失败断言摘要(不含
17
+ * "+N more",N 单独进 moreFailures),errored 取结构化 error 的一层摘要
18
+ * (phase · code · message),passed / skipped 为 null。
19
+ */
20
+ export function failureSummaryOf(result) {
21
+ if (result.verdict === "errored" && result.error !== undefined) {
22
+ const parts = [result.error.phase, result.error.code, result.error.message].filter((part) => typeof part === "string" && part.length > 0);
23
+ return { summary: summaryText(parts.join(" · ")), more: 0 };
24
+ }
25
+ if (result.verdict === "failed" || result.verdict === "errored") {
26
+ const primary = primaryAssertionSummary(result.assertions, result.verdict);
27
+ if (primary !== undefined) {
28
+ return {
29
+ summary: compactAssertionSummary({ ...primary, additionalFailures: 0 }),
30
+ more: primary.additionalFailures,
31
+ };
32
+ }
33
+ if (result.verdict === "errored" && result.skipReason !== undefined) {
34
+ return { summary: summaryText(result.skipReason), more: 0 };
35
+ }
36
+ return { summary: null, more: 0 };
37
+ }
38
+ return { summary: null, more: 0 };
39
+ }
40
+ /** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
41
+ async function attemptListItemOf(item) {
42
+ const result = item.attempt.result;
43
+ const { summary, more } = failureSummaryOf(result);
44
+ return {
45
+ experimentId: experimentIdOf(item),
46
+ evalId: evalIdOf(item),
47
+ attempt: result.attempt,
48
+ agent: result.agent,
49
+ verdict: result.verdict,
50
+ failureSummary: summary,
51
+ moreFailures: more,
52
+ examScore: await computeCell(examScore, [item]),
53
+ durationMs: result.durationMs,
54
+ costUSD: attemptCostUSD(result),
55
+ locator: locatorOf(item),
56
+ };
57
+ }
58
+ /** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Scope 展平顺序(不重排)。 */
59
+ export async function attemptListData(input) {
60
+ const { snapshots } = resolveInput(input);
61
+ const items = collectItems(snapshots);
62
+ return Promise.all(items.map((item) => attemptListItemOf(item)));
63
+ }
64
+ /** `evalListData(input)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
65
+ export async function evalListData(input) {
66
+ const { snapshots } = resolveInput(input);
67
+ const items = collectItems(snapshots);
68
+ const groups = new Map();
69
+ for (const item of items) {
70
+ const key = fullEvalKey(item);
71
+ const list = groups.get(key);
72
+ if (list)
73
+ list.push(item);
74
+ else
75
+ groups.set(key, [item]);
76
+ }
77
+ const out = [];
78
+ for (const group of groups.values()) {
79
+ const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
80
+ const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
81
+ const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
82
+ out.push({
83
+ experimentId: experimentIdOf(sorted[0]),
84
+ evalId: evalIdOf(sorted[0]),
85
+ verdict,
86
+ examScore: await computeCell(examScore, sorted),
87
+ durationMs: await computeCell(durationMs, sorted),
88
+ costUSD: await computeCell(costUSD, sorted),
89
+ attempts,
90
+ });
91
+ }
92
+ out.sort((a, b) => a.evalId.localeCompare(b.evalId) || a.experimentId.localeCompare(b.experimentId));
93
+ return out;
94
+ }
95
+ /**
96
+ * `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始按端到端通过率
97
+ * 从高到低(缺数据沉底,同分按 id)。一行只有一套 agent / model / flags 是输入约束:
98
+ * 宿主注入的 current() Scope 保证每个 experiment 只由可比性配置一致的快照拼成;作者自选
99
+ * Snapshot[] 时若同一 experiment 混入不一致的可比性配置,按完整用户反馈失败并指引——
100
+ * 看跨配置演化用 snapshot 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
101
+ */
102
+ export async function experimentListData(input) {
103
+ const { snapshots: rawSnapshots } = resolveInput(input);
104
+ const snapshots = selectedEvalsOnly(rawSnapshots);
105
+ // 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
106
+ const configByExperiment = new Map();
107
+ for (const snapshot of snapshots) {
108
+ const config = comparabilityConfigOf(snapshot);
109
+ const existing = configByExperiment.get(snapshot.experimentId);
110
+ if (existing === undefined) {
111
+ configByExperiment.set(snapshot.experimentId, { snapshot, config });
112
+ }
113
+ else if (!deepEqualJson(existing.config, config)) {
114
+ throw new Error(`experimentListData got inconsistent comparability configs for experiment "${snapshot.experimentId}" ` +
115
+ `(snapshots ${existing.snapshot.startedAt} and ${snapshot.startedAt} differ in agent/model/reasoningEffort/flags/budget/timeoutMs/sandbox). ` +
116
+ "One row shows one configuration — it cannot honestly merge two. To chart evolution across configs, " +
117
+ 'use the "snapshot" dimension or MetricLine; to show the current level, pass results.current() which selects a single config per experiment.');
118
+ }
119
+ }
120
+ const items = collectItems(snapshots);
121
+ const groups = groupItems(items, "experiment");
122
+ const out = [];
123
+ for (const [experimentId, group] of groups) {
124
+ const stats = summarizeItems(group);
125
+ const newest = [...group].sort((a, b) => b.snapshot.startedAt.localeCompare(a.snapshot.startedAt))[0];
126
+ const evalGroups = groupItems(group, "eval");
127
+ const evalRows = [];
128
+ for (const [evalId, evalItems] of evalGroups) {
129
+ const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
130
+ const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
131
+ const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
132
+ evalRows.push({
133
+ evalId,
134
+ verdict,
135
+ durationMs: await computeCell(durationMs, sorted),
136
+ costUSD: await computeCell(costUSD, sorted),
137
+ attempts,
138
+ });
139
+ }
140
+ const experiment = newest.snapshot.experiment ?? newest.attempt.result.experiment;
141
+ const model = newest.attempt.result.model ?? newest.snapshot.model;
142
+ out.push({
143
+ experimentId,
144
+ agent: newest.snapshot.agent || newest.attempt.result.agent,
145
+ ...(model !== undefined ? { model } : {}),
146
+ ...(experiment?.flags ? { flags: experiment.flags } : {}),
147
+ evalVerdicts: stats.verdicts,
148
+ endToEndPassRate: await computeCell(endToEndPassRate, group),
149
+ costUSD: await computeCell(costUSD, group),
150
+ durationMs: await computeCell(durationMs, group),
151
+ tokens: await computeCell(tokens, group),
152
+ evals: stats.evals,
153
+ attempts: stats.attempts,
154
+ lastRunAt: stats.lastRunAt,
155
+ evalRows,
156
+ });
157
+ }
158
+ // 初始态按端到端通过率(endToEndPassRate)从高到低,缺数据沉底;同分按 experiment id 稳定排序。
159
+ out.sort((a, b) => {
160
+ const va = a.endToEndPassRate.value;
161
+ const vb = b.endToEndPassRate.value;
162
+ if (va === null && vb === null)
163
+ return a.experimentId.localeCompare(b.experimentId);
164
+ if (va === null)
165
+ return 1;
166
+ if (vb === null)
167
+ return -1;
168
+ return vb - va || a.experimentId.localeCompare(b.experimentId);
169
+ });
170
+ return out;
171
+ }
@@ -0,0 +1,5 @@
1
+ import type { AttemptListItem, EvalListItem, ExperimentListItem } from "../../model/types.ts";
2
+ import type { TextContext } from "../../definition/tree.ts";
3
+ export declare function experimentListText(items: readonly ExperimentListItem[], ctx: TextContext, relativeTo?: string): string;
4
+ export declare function evalListText(items: readonly EvalListItem[], ctx: TextContext): string;
5
+ export declare function attemptListText(items: readonly AttemptListItem[], total: number | undefined, ctx: TextContext): string;
@@ -0,0 +1,174 @@
1
+ // 实体列表族(ExperimentList / EvalList / AttemptList)的 text 面:同一份算好的数据,
2
+ // 渲染成终端字符(niceeval show 的形态)。三面共用的紧凑标记:`locator✓`(判定符紧跟
3
+ // locator,中间不留空格)。ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的
4
+ // 摘要,不重复整段 niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼
5
+ // `niceeval show <locator>`。零 react、零 IO、纯同步。
6
+ import { experimentDisplayName, fitFailureSummary, formatDurationMs, formatUSD, verdictMark, } from "../../model/format.js";
7
+ import { countText, localeText } from "../../model/locale.js";
8
+ import { stringWidth, wrapDisplay } from "../../model/text-layout.js";
9
+ import { renderTableText } from "../../definition/table-text.js";
10
+ import { cellText, missingText, verdictTallyText, MISSING_MARK } from "../shared-faces.js";
11
+ // ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
12
+ //
13
+ // 三面共用的紧凑标记:`locator✓`(判定符紧跟 locator,中间不留空格)。
14
+ // ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的摘要,不重复整段
15
+ // niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼 `niceeval show <locator>`。
16
+ function locatorBadge(item) {
17
+ return `${item.locator}${verdictMark(item.verdict)}`;
18
+ }
19
+ /**
20
+ * failureSummary + moreFailures 的展示形态:摘要在计算侧已按 Scoring display 契约折好,
21
+ * 这里只加 "+N more failures" 计数与宽度收口,不重算摘要。
22
+ */
23
+ function attemptReasonText(item, locale, maxChars) {
24
+ if (item.failureSummary === null)
25
+ return undefined;
26
+ const withMore = item.moreFailures > 0
27
+ ? `${item.failureSummary} · ${countText(locale, "entityList.moreFailures", item.moreFailures)}`
28
+ : item.failureSummary;
29
+ return fitFailureSummary(withMore, Math.max(24, maxChars));
30
+ }
31
+ // ── ExperimentList ──
32
+ function experimentSummaryTable(items, ctx, relativeTo) {
33
+ const locale = ctx.locale;
34
+ const compact = ctx.width < 100;
35
+ const columns = [
36
+ { key: "experiment", header: compact && locale === "en" ? "Exp." : localeText(locale, "experimentList.experiment") },
37
+ { key: "model", header: localeText(locale, "table.model") },
38
+ { key: "agent", header: localeText(locale, "table.agent") },
39
+ { key: "duration", header: compact && locale === "en" ? "Avg" : localeText(locale, "experimentList.avgDuration"), align: "right" },
40
+ { key: "passRate", header: compact && locale === "en" ? "Pass" : localeText(locale, "experimentList.passRate"), align: "right" },
41
+ { key: "result", header: localeText(locale, "experimentList.result") },
42
+ { key: "tokens", header: localeText(locale, "experimentList.tokens"), align: "right" },
43
+ { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
44
+ ];
45
+ const rows = items.map((item) => ({
46
+ key: item.experimentId,
47
+ cells: {
48
+ experiment: experimentDisplayName(item.experimentId, relativeTo),
49
+ model: item.model ?? localeText(locale, "experimentList.defaultModel"),
50
+ agent: item.agent,
51
+ duration: cellText(item.durationMs, locale),
52
+ passRate: cellText(item.endToEndPassRate, locale),
53
+ result: verdictTallyText(item.evalVerdicts, locale),
54
+ tokens: cellText(item.tokens, locale),
55
+ cost: cellText(item.costUSD, locale),
56
+ },
57
+ }));
58
+ const metadata = items.flatMap((item) => wrapDisplay(`${experimentDisplayName(item.experimentId, relativeTo)}: ${localeText(locale, "overview.evalsCount", { n: item.evals })} · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })} · ${item.lastRunAt}`, Math.max(8, ctx.width - 2)).map((line) => ` ${line}`));
59
+ return [renderTableText({ columns: columns, rows, locale }, ctx), metadata.join("\n")].join("\n");
60
+ }
61
+ function experimentDetailTable(item, ctx, relativeTo) {
62
+ const locale = ctx.locale;
63
+ const columns = [
64
+ { key: "status", header: localeText(locale, "experimentList.status") },
65
+ { key: "entity", header: localeText(locale, "experimentList.evalAttempt") },
66
+ // Result 是可扫读的失败预览,不是证据面:两行放不下的以 … 收口,完整值走 locator 下钻。
67
+ { key: "result", header: localeText(locale, "experimentList.result"), maxLines: 2 },
68
+ { key: "duration", header: localeText(locale, "experimentList.duration"), align: "right" },
69
+ { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
70
+ ];
71
+ // Result 的字符预算 ≈ 两行 × 它能分到的列宽(总宽减其它列的自然宽与列距)。这里只做
72
+ // 粗预算;精确的按宽度收口由列的 maxLines 兜底。
73
+ const statusWidth = Math.max(stringWidth(localeText(locale, "experimentList.status")), ...item.evalRows.map((row) => stringWidth(`${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`)));
74
+ const entityWidth = Math.max(stringWidth(localeText(locale, "experimentList.evalAttempt")), ...item.evalRows.flatMap((row) => [stringWidth(row.evalId), ...row.attempts.map((a) => stringWidth(a.locator) + 3)]));
75
+ const fixedWidth = statusWidth + entityWidth + 8 /* duration */ + 6 /* cost */ + 3 * 4; /* 4 段列距 */
76
+ const resultBudget = Math.max(24, (ctx.width - fixedWidth) * 2);
77
+ const rows = item.evalRows.flatMap((row) => {
78
+ // Eval 父行只承载折叠判定与题级聚合;失败摘要只在 Attempt 子行出现。
79
+ const parent = {
80
+ key: row.evalId,
81
+ cells: {
82
+ status: `${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`,
83
+ entity: row.evalId,
84
+ result: "",
85
+ duration: localeText(locale, "entityList.average", { value: cellText(row.durationMs, locale) }),
86
+ cost: localeText(locale, "entityList.average", { value: cellText(row.costUSD, locale) }),
87
+ },
88
+ };
89
+ const attempts = row.attempts.map((attempt, index) => ({
90
+ key: attempt.locator,
91
+ cells: {
92
+ status: ` ${verdictMark(attempt.verdict)}`,
93
+ entity: `${index === row.attempts.length - 1 ? "└─" : "├─"} ${attempt.locator}`,
94
+ result: attemptReasonText(attempt, locale, resultBudget) ?? MISSING_MARK,
95
+ duration: attempt.verdict === "skipped" && attempt.durationMs === 0 ? null : formatDurationMs(attempt.durationMs),
96
+ cost: attempt.costUSD === null ? null : formatUSD(attempt.costUSD),
97
+ },
98
+ }));
99
+ return [parent, ...attempts];
100
+ });
101
+ const flags = item.flags && Object.keys(item.flags).length > 0
102
+ ? `${localeText(locale, "experimentList.flags")} ${Object.entries(item.flags)
103
+ .map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`)
104
+ .join(" · ")}`
105
+ : undefined;
106
+ return [
107
+ experimentDisplayName(item.experimentId, relativeTo),
108
+ flags,
109
+ renderTableText({ columns: columns, rows, locale }, ctx),
110
+ ]
111
+ .filter(Boolean)
112
+ .join("\n");
113
+ }
114
+ export function experimentListText(items, ctx, relativeTo) {
115
+ if (items.length === 0)
116
+ return localeText(ctx.locale, "attemptList.empty");
117
+ return [
118
+ experimentSummaryTable(items, ctx, relativeTo),
119
+ ...items.map((item) => experimentDetailTable(item, ctx, relativeTo)),
120
+ ].join("\n\n");
121
+ }
122
+ // ── EvalList ──
123
+ function evalListAttemptLine(item, ctx) {
124
+ // 行式列表同守「Result 最多两行」:预算 = 两行终端宽,超出按尾截收口。
125
+ const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - stringWidth(locatorBadge(item)) - 6);
126
+ return ` ${locatorBadge(item)}${reason ? ` · ${reason}` : ""}`;
127
+ }
128
+ export function evalListText(items, ctx) {
129
+ const locale = ctx.locale;
130
+ if (items.length === 0)
131
+ return localeText(locale, "attemptList.empty");
132
+ const blocks = items.map((item) => {
133
+ const identity = `${item.evalId} · ${item.experimentId} · ${localeText(locale, `verdict.${item.verdict}`)}`;
134
+ const summary = [
135
+ localeText(locale, "attemptList.score", { score: cellText(item.examScore, locale) }),
136
+ localeText(locale, "overview.attemptsCount", { n: item.attempts.length }),
137
+ localeText(locale, "entityList.average", {
138
+ value: item.durationMs.value === null ? missingText(locale) : formatDurationMs(item.durationMs.value),
139
+ }),
140
+ localeText(locale, "entityList.average", {
141
+ value: item.costUSD.value === null ? missingText(locale) : formatUSD(item.costUSD.value),
142
+ }),
143
+ ].join(" · ");
144
+ const attemptLines = item.attempts.map((attempt) => evalListAttemptLine(attempt, ctx));
145
+ return [identity, ` ${summary}`, ...attemptLines].join("\n");
146
+ });
147
+ return blocks.join("\n\n");
148
+ }
149
+ // ── AttemptList ──
150
+ /** Attempt 比较卡片:只显示一条主失败摘要(至多两行终端宽);完整 assertions 走 locator 下钻。 */
151
+ function attemptListItemText(item, ctx) {
152
+ const head = [
153
+ `${verdictMark(item.verdict)} ${item.locator}`,
154
+ item.evalId,
155
+ item.experimentId,
156
+ formatDurationMs(item.durationMs),
157
+ ...(item.costUSD !== null ? [formatUSD(item.costUSD)] : []),
158
+ ].join(" · ");
159
+ const lines = [head];
160
+ const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - 4);
161
+ if (reason)
162
+ lines.push(` ${reason}`);
163
+ return lines.join("\n");
164
+ }
165
+ export function attemptListText(items, total, ctx) {
166
+ const locale = ctx.locale;
167
+ if (items.length === 0)
168
+ return localeText(locale, "attemptList.empty");
169
+ const blocks = items.map((item) => attemptListItemText(item, ctx));
170
+ const remaining = (total ?? items.length) - items.length;
171
+ if (remaining > 0)
172
+ blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
173
+ return blocks.join("\n\n");
174
+ }
@@ -0,0 +1,51 @@
1
+ import { type ReportComponent } from "../../definition/tree.ts";
2
+ import type { AttemptListItem, EvalListItem, ExperimentListItem, ReportInput } from "../../model/types.ts";
3
+ import type { AttemptLocator } from "../../../results/locator.ts";
4
+ import type { ReportLocale } from "../../model/locale.ts";
5
+ import { type ChromeProps, type DataProps, type Validator } from "../shared.ts";
6
+ export declare const validateExperimentListData: Validator;
7
+ export declare const validateEvalListData: Validator;
8
+ export declare const validateAttemptListData: Validator;
9
+ interface EntityListChrome extends ChromeProps {
10
+ attemptHref?: (locator: AttemptLocator) => string;
11
+ }
12
+ export type ExperimentListProps = DataProps<readonly ExperimentListItem[], Record<never, never>, EntityListChrome & {
13
+ /** web 面在比较表前显示实验过滤框;text 面忽略。 */
14
+ filter?: boolean;
15
+ /**
16
+ * 可选父路径:两面的行标签去掉与它相同的前缀,只显示 experiment id 末段。自定义报告
17
+ * 显式传入使用;默认 `ExperimentComparison` 不传,完整 id 始终可见。完整 id 仍是
18
+ * 排序 / 着色 / 过滤 / 折叠的键。
19
+ */
20
+ relativeTo?: string;
21
+ }>;
22
+ /** 实验列表:每项一个 experiment,固定八列比较表 + 展开到 Eval / Attempt。 */
23
+ export declare const ExperimentList: ReportComponent<ExperimentListProps>;
24
+ export type EvalListProps = DataProps<readonly EvalListItem[], Record<never, never>, EntityListChrome>;
25
+ /** Eval 列表:每项一个 experimentId + evalId,展开到这道题的 Attempt。 */
26
+ export declare const EvalList: ReportComponent<EvalListProps>;
27
+ export type AttemptListProps = DataProps<readonly AttemptListItem[], Record<never, never>, EntityListChrome & {
28
+ /** 过滤 / 截断前的总数;省略时等于 data 长度。 */
29
+ total?: number;
30
+ /** web 面加过滤输入框(按 experiment、eval、agent、verdict 或摘要文本收窄行);渐进增强,不改变数据与 text 面。 */
31
+ filter?: boolean;
32
+ }>;
33
+ /** Attempt 列表:实体列表的叶子层,每项一次 attempt 的判定、单行摘要与 locator。 */
34
+ export declare const AttemptList: ReportComponent<AttemptListProps>;
35
+ export interface FailureListProps {
36
+ /** 显示的最大条数;默认 20。 */
37
+ limit?: number;
38
+ /** 默认宿主注入的 Scope。 */
39
+ input?: ReportInput;
40
+ attemptHref?: (locator: AttemptLocator) => string;
41
+ locale?: ReportLocale;
42
+ className?: string;
43
+ }
44
+ /**
45
+ * 「现在有哪些失败要处理」的成品组合件:内部就是 attemptListData → 过滤 → AttemptList
46
+ * data 形态,与手写组合严格等价、没有私有能力(docs/feature/reports/library/entity-lists.md)。
47
+ * verdict ∈ failed / errored,按 attempt 开始时间降序(同刻按 locator 字典序),
48
+ * 截断到 limit(默认 20),total 报告截断前总数。
49
+ */
50
+ export declare const FailureList: ReportComponent<FailureListProps>;
51
+ export {};