niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,19 +1,18 @@
1
1
  // ExperimentList:实体列表的第一级。web 面是一行一个 experiment 的固定八列比较表
2
- // (Experiment / Model / Agent / Avg duration / End-to-end pass rate / Tokens / Cost / Result),
2
+ // (Experiment / Model / Agent / Avg. time / Pass rate / Tokens / Cost / Results),
3
3
  // 每行用原生 <details> 展开到 Eval 与 Attempt locator。数据完全来自 experimentListData(),
4
4
  // 组件不重算、不推断组边界。
5
5
 
6
6
  import type { ReactElement } from "react";
7
- import type { AttemptLocator } from "../../results/locator.ts";
8
- import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "../types.ts";
9
- import { experimentDisplayName } from "../format.ts";
10
- import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
7
+ import type { AttemptLocator } from "../../../results/locator.ts";
8
+ import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "../../model/types.ts";
9
+ import { shortestUniqueLabels } from "../../model/format.ts";
10
+ import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../../model/locale.ts";
11
11
  import { AttemptLocatorBadge, failureSummaryText } from "./AttemptList.tsx";
12
- import { MetricCellView } from "./cell.tsx";
13
- import { colorClassForKey } from "./colors.ts";
14
- import { cx, formatDurationMs, formatUSD, verdictMark } from "./format.ts";
15
-
16
- const DEFAULT_ATTEMPT_HREF = (locator: AttemptLocator): string => `#/attempt/${locator}`;
12
+ import { MetricCellView } from "../cell.tsx";
13
+ import { colorClassForKey } from "../../assets/colors.ts";
14
+ import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.ts";
15
+ import { cx } from "../shared.ts";
17
16
 
18
17
  const verdictOrder = ["passed", "failed", "errored", "skipped"] as const;
19
18
 
@@ -39,7 +38,7 @@ function VerdictSummary({ item, locale }: { item: ExperimentListItem; locale: Re
39
38
  const parts = verdictOrder
40
39
  .filter((verdict) => item.evalVerdicts[verdict] > 0)
41
40
  .map((verdict) => `${item.evalVerdicts[verdict]} ${localeText(locale, `verdict.${verdict}`)}`);
42
- return <span className="nre-experiment-pill">{parts.join(" / ") || "—"}</span>;
41
+ return <span className="nre-experiment-pill">{parts.join(" · ") || "—"}</span>;
43
42
  }
44
43
 
45
44
  function ExperimentAttemptRow({
@@ -50,7 +49,7 @@ function ExperimentAttemptRow({
50
49
  }: {
51
50
  attempt: AttemptListItem;
52
51
  last: boolean;
53
- attemptHref: (locator: AttemptLocator) => string;
52
+ attemptHref?: (locator: AttemptLocator) => string;
54
53
  locale: ReportLocale;
55
54
  }): ReactElement {
56
55
  const reason = failureSummaryText(attempt, locale);
@@ -76,7 +75,7 @@ function EvalAttempts({
76
75
  locale,
77
76
  }: {
78
77
  row: ExperimentListEvalRow;
79
- attemptHref: (locator: AttemptLocator) => string;
78
+ attemptHref?: (locator: AttemptLocator) => string;
80
79
  locale: ReportLocale;
81
80
  }): ReactElement {
82
81
  const duration = row.durationMs.value === null ? localeText(locale, "cell.missing") : formatDurationMs(row.durationMs.value);
@@ -123,23 +122,21 @@ function Flags({ flags, locale }: { flags: Record<string, unknown> | undefined;
123
122
 
124
123
  function ExperimentRow({
125
124
  item,
125
+ label,
126
126
  attemptHref,
127
127
  locale,
128
- relativeTo,
129
128
  }: {
130
129
  item: ExperimentListItem;
131
- attemptHref: (locator: AttemptLocator) => string;
130
+ label: string;
131
+ attemptHref?: (locator: AttemptLocator) => string;
132
132
  locale: ReportLocale;
133
- relativeTo?: string;
134
133
  }): ReactElement {
135
134
  return (
136
135
  <details className="nre-experiment-entry">
137
136
  <summary className="nre-experiment-summary">
138
- {/* relativeTo 只影响显示末段;完整 id 仍是排序 / 着色 / 过滤 / 折叠的键 */}
137
+ {/* label 是当前列表里的最短唯一后缀;完整 id 仍是排序 / 过滤 / 折叠的键,系列色跟随 agent */}
139
138
  <span className="nre-experiment-name" data-sort-value={item.experimentId}>
140
- <b className={cx("nre-experiment-id", "nre-key", colorClassForKey(item.experimentId))}>
141
- {experimentDisplayName(item.experimentId, relativeTo)}
142
- </b>
139
+ <b className={cx("nre-experiment-id", "nre-key")}>{label}</b>
143
140
  <small>
144
141
  {localeText(locale, "overview.evalsCount", { n: item.evals })}
145
142
  {item.attempts > item.evals ? ` · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })}` : ""}
@@ -147,7 +144,9 @@ function ExperimentRow({
147
144
  </small>
148
145
  </span>
149
146
  <span data-sort-value={item.model ?? ""}>{item.model ?? localeText(locale, "experimentList.defaultModel")}</span>
150
- <span data-sort-value={item.agent}>{item.agent}</span>
147
+ <span data-sort-value={item.agent}>
148
+ <span className={cx("nre-experiment-agent", "nre-key", colorClassForKey(item.agent))}>{item.agent}</span>
149
+ </span>
151
150
  <span className="nre-num" data-sort-value={item.durationMs.value ?? ""}>
152
151
  <MetricCellView cell={item.durationMs} locale={locale} />
153
152
  </span>
@@ -179,19 +178,18 @@ function ExperimentRow({
179
178
 
180
179
  export function ExperimentList({
181
180
  data,
182
- attemptHref = DEFAULT_ATTEMPT_HREF,
181
+ attemptHref,
183
182
  filter = false,
184
183
  className,
185
184
  locale = DEFAULT_REPORT_LOCALE,
186
- relativeTo,
187
185
  }: {
188
186
  data: readonly ExperimentListItem[];
189
187
  attemptHref?: (locator: AttemptLocator) => string;
190
188
  filter?: boolean;
191
189
  className?: string;
192
190
  locale?: ReportLocale;
193
- relativeTo?: string;
194
191
  }): ReactElement {
192
+ const experimentLabels = shortestUniqueLabels(data.map((item) => item.experimentId));
195
193
  const labels = [
196
194
  localeText(locale, "experimentList.experiment"),
197
195
  localeText(locale, "table.model"),
@@ -209,16 +207,24 @@ export function ExperimentList({
209
207
  <button
210
208
  type="button"
211
209
  data-nre-experiment-sort={index}
212
- className={index === 4 ? "nre-sort-desc" : undefined}
210
+ className={cx(index >= 3 && index <= 6 && "nre-num-head", index === 4 && "nre-sort-desc")}
213
211
  key={label}
212
+ title={index === 4 ? localeText(locale, "experimentList.passRateDescription") : undefined}
214
213
  >
215
- {label}
214
+ <span className="nre-sort-label">{label}</span>
215
+ <span className="nre-sort-icon" aria-hidden="true" />
216
216
  </button>
217
217
  ))}
218
218
  </div>
219
219
  {data.length === 0 && <p className="nre-experiment-list-empty">{localeText(locale, "attemptList.empty")}</p>}
220
220
  {data.map((item) => (
221
- <ExperimentRow key={item.experimentId} item={item} attemptHref={attemptHref} locale={locale} relativeTo={relativeTo} />
221
+ <ExperimentRow
222
+ key={item.experimentId}
223
+ item={item}
224
+ label={experimentLabels.get(item.experimentId) ?? item.experimentId}
225
+ attemptHref={attemptHref}
226
+ locale={locale}
227
+ />
222
228
  ))}
223
229
  </div>
224
230
  );
@@ -0,0 +1,196 @@
1
+ // 计算函数(*Data):ReportInput → 一份组件数据。实体列表族(ExperimentList / EvalList /
2
+ // AttemptList / FailureList)的 *Data 都住在这里(docs/feature/reports/library/entity-lists.md)。
3
+ //
4
+ // 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
5
+ // - 第一参收 ReportInput = Scope | readonly Snapshot[];warnings 不进组件数据(宿主统一显示);
6
+ // - 聚合前按身份键去重(dedupeAttempts;missing-startedAt 不去重、如实保留、不透出警告);
7
+ // - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露;
8
+ // - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
9
+
10
+ import type {
11
+ AttemptListItem,
12
+ EvalListItem,
13
+ ExperimentListEvalRow,
14
+ ExperimentListItem,
15
+ ReportInput,
16
+ } from "../../model/types.ts";
17
+ import type { EvalResult } from "../../../types.ts";
18
+ import type { Snapshot } from "../../../results/types.ts";
19
+ import { comparabilityConfigOf, deepEqualJson } from "../../../results/select.ts";
20
+ import { foldEvalVerdict } from "../../../shared/verdict.ts";
21
+ import {
22
+ collectItems,
23
+ computeCell,
24
+ evalIdOf,
25
+ experimentIdOf,
26
+ fullEvalKey,
27
+ groupItems,
28
+ locatorOf,
29
+ resolveInput,
30
+ type Item,
31
+ } from "../../model/aggregate.ts";
32
+ import { attemptCostUSD, costUSD, durationMs, endToEndPassRate, examScore, tokens } from "../../model/metrics.ts";
33
+ import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../../../scoring/display.ts";
34
+ import { selectedEvalsOnly, summarizeItems } from "../shared-compute.ts";
35
+
36
+ /**
37
+ * 一次 attempt 的单行结果摘要(Scoring display 契约):failed 取主失败断言摘要(不含
38
+ * "+N more",N 单独进 moreFailures),errored 取结构化 error 的一层摘要
39
+ * (phase · code · message),passed / skipped 为 null。
40
+ */
41
+ export function failureSummaryOf(result: EvalResult): { summary: string | null; more: number } {
42
+ if (result.verdict === "errored" && result.error !== undefined) {
43
+ const parts = [result.error.phase, result.error.code, result.error.message].filter(
44
+ (part): part is string => typeof part === "string" && part.length > 0,
45
+ );
46
+ return { summary: summaryText(parts.join(" · ")), more: 0 };
47
+ }
48
+ if (result.verdict === "failed" || result.verdict === "errored") {
49
+ const primary = primaryAssertionSummary(result.assertions, result.verdict);
50
+ if (primary !== undefined) {
51
+ return {
52
+ summary: compactAssertionSummary({ ...primary, additionalFailures: 0 }),
53
+ more: primary.additionalFailures,
54
+ };
55
+ }
56
+ if (result.verdict === "errored" && result.skipReason !== undefined) {
57
+ return { summary: summaryText(result.skipReason), more: 0 };
58
+ }
59
+ return { summary: null, more: 0 };
60
+ }
61
+ return { summary: null, more: 0 };
62
+ }
63
+
64
+ /** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
65
+ async function attemptListItemOf(item: Item): Promise<AttemptListItem> {
66
+ const result = item.attempt.result;
67
+ const { summary, more } = failureSummaryOf(result);
68
+ return {
69
+ experimentId: experimentIdOf(item),
70
+ evalId: evalIdOf(item),
71
+ attempt: result.attempt,
72
+ agent: result.agent,
73
+ verdict: result.verdict,
74
+ failureSummary: summary,
75
+ moreFailures: more,
76
+ examScore: await computeCell(examScore, [item]),
77
+ durationMs: result.durationMs,
78
+ costUSD: attemptCostUSD(result),
79
+ locator: locatorOf(item),
80
+ };
81
+ }
82
+
83
+ /** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Scope 展平顺序(不重排)。 */
84
+ export async function attemptListData(input: ReportInput): Promise<AttemptListItem[]> {
85
+ const { snapshots } = resolveInput(input);
86
+ const items = collectItems(snapshots);
87
+ return Promise.all(items.map((item) => attemptListItemOf(item)));
88
+ }
89
+
90
+ /** `evalListData(input)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
91
+ export async function evalListData(input: ReportInput): Promise<EvalListItem[]> {
92
+ const { snapshots } = resolveInput(input);
93
+ const items = collectItems(snapshots);
94
+ const groups = new Map<string, Item[]>();
95
+ for (const item of items) {
96
+ const key = fullEvalKey(item);
97
+ const list = groups.get(key);
98
+ if (list) list.push(item);
99
+ else groups.set(key, [item]);
100
+ }
101
+ const out: EvalListItem[] = [];
102
+ for (const group of groups.values()) {
103
+ const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
104
+ const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
105
+ const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
106
+ out.push({
107
+ experimentId: experimentIdOf(sorted[0]!),
108
+ evalId: evalIdOf(sorted[0]!),
109
+ verdict,
110
+ examScore: await computeCell(examScore, sorted),
111
+ durationMs: await computeCell(durationMs, sorted),
112
+ costUSD: await computeCell(costUSD, sorted),
113
+ attempts,
114
+ });
115
+ }
116
+ out.sort((a, b) => a.evalId.localeCompare(b.evalId) || a.experimentId.localeCompare(b.experimentId));
117
+ return out;
118
+ }
119
+
120
+ /**
121
+ * `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始按端到端通过率
122
+ * 从高到低(缺数据沉底,同分按 id)。一行只有一套 agent / model / flags 是输入约束:
123
+ * 宿主注入的 current() Scope 保证每个 experiment 只由可比性配置一致的快照拼成;作者自选
124
+ * Snapshot[] 时若同一 experiment 混入不一致的可比性配置,按完整用户反馈失败并指引——
125
+ * 看跨配置演化用 snapshot 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
126
+ */
127
+ export async function experimentListData(input: ReportInput): Promise<ExperimentListItem[]> {
128
+ const { snapshots: rawSnapshots } = resolveInput(input);
129
+ const snapshots = selectedEvalsOnly(rawSnapshots);
130
+
131
+ // 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
132
+ const configByExperiment = new Map<string, { snapshot: Snapshot; config: unknown }>();
133
+ for (const snapshot of snapshots) {
134
+ const config = comparabilityConfigOf(snapshot);
135
+ const existing = configByExperiment.get(snapshot.experimentId);
136
+ if (existing === undefined) {
137
+ configByExperiment.set(snapshot.experimentId, { snapshot, config });
138
+ } else if (!deepEqualJson(existing.config, config)) {
139
+ throw new Error(
140
+ `experimentListData got inconsistent comparability configs for experiment "${snapshot.experimentId}" ` +
141
+ `(snapshots ${existing.snapshot.startedAt} and ${snapshot.startedAt} differ in agent/model/reasoningEffort/flags/budget/timeoutMs/sandbox). ` +
142
+ "One row shows one configuration — it cannot honestly merge two. To chart evolution across configs, " +
143
+ 'use the "snapshot" dimension or MetricLine; to show the current level, pass results.current() which selects a single config per experiment.',
144
+ );
145
+ }
146
+ }
147
+
148
+ const items = collectItems(snapshots);
149
+ const groups = groupItems(items, "experiment");
150
+ const out: ExperimentListItem[] = [];
151
+ for (const [experimentId, group] of groups) {
152
+ const stats = summarizeItems(group);
153
+ const newest = [...group].sort((a, b) => b.snapshot.startedAt.localeCompare(a.snapshot.startedAt))[0]!;
154
+ const evalGroups = groupItems(group, "eval");
155
+ const evalRows: ExperimentListEvalRow[] = [];
156
+ for (const [evalId, evalItems] of evalGroups) {
157
+ const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
158
+ const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
159
+ const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
160
+ evalRows.push({
161
+ evalId,
162
+ verdict,
163
+ durationMs: await computeCell(durationMs, sorted),
164
+ costUSD: await computeCell(costUSD, sorted),
165
+ attempts,
166
+ });
167
+ }
168
+ const experiment = newest.snapshot.experiment ?? newest.attempt.result.experiment;
169
+ const model = newest.attempt.result.model ?? newest.snapshot.model;
170
+ out.push({
171
+ experimentId,
172
+ agent: newest.snapshot.agent || newest.attempt.result.agent,
173
+ ...(model !== undefined ? { model } : {}),
174
+ ...(experiment?.flags ? { flags: experiment.flags } : {}),
175
+ evalVerdicts: stats.verdicts,
176
+ endToEndPassRate: await computeCell(endToEndPassRate, group),
177
+ costUSD: await computeCell(costUSD, group),
178
+ durationMs: await computeCell(durationMs, group),
179
+ tokens: await computeCell(tokens, group),
180
+ evals: stats.evals,
181
+ attempts: stats.attempts,
182
+ lastRunAt: stats.lastRunAt!,
183
+ evalRows,
184
+ });
185
+ }
186
+ // 初始态按端到端通过率(endToEndPassRate)从高到低,缺数据沉底;同分按 experiment id 稳定排序。
187
+ out.sort((a, b) => {
188
+ const va = a.endToEndPassRate.value;
189
+ const vb = b.endToEndPassRate.value;
190
+ if (va === null && vb === null) return a.experimentId.localeCompare(b.experimentId);
191
+ if (va === null) return 1;
192
+ if (vb === null) return -1;
193
+ return vb - va || a.experimentId.localeCompare(b.experimentId);
194
+ });
195
+ return out;
196
+ }
@@ -0,0 +1,208 @@
1
+ // 实体列表族(ExperimentList / EvalList / AttemptList)的 text 面:同一份算好的数据,
2
+ // 渲染成终端字符(niceeval show 的形态)。三面共用的紧凑标记:`locator✓`(判定符紧跟
3
+ // locator,中间不留空格)。ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的
4
+ // 摘要,不重复整段 niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼
5
+ // `niceeval show <locator>`。零 react、零 IO、纯同步。
6
+
7
+ import type { AttemptListItem, EvalListItem, ExperimentListItem } from "../../model/types.ts";
8
+ import type { TextContext } from "../../definition/tree.ts";
9
+ import type { TableColumn, TableRow } from "../../definition/primitives.tsx";
10
+ import {
11
+ fitFailureSummary,
12
+ formatDurationMs,
13
+ formatUSD,
14
+ shortestUniqueLabels,
15
+ verdictMark,
16
+ } from "../../model/format.ts";
17
+ import { countText, localeText, type ReportLocale } from "../../model/locale.ts";
18
+ import { stringWidth, wrapDisplay } from "../../model/text-layout.ts";
19
+ import { renderTableText } from "../../definition/table-text.ts";
20
+ import { cellText, missingText, verdictTallyText, MISSING_MARK } from "../shared-faces.ts";
21
+
22
+ // ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
23
+ //
24
+ // 三面共用的紧凑标记:`locator✓`(判定符紧跟 locator,中间不留空格)。
25
+ // ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的摘要,不重复整段
26
+ // niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼 `niceeval show <locator>`。
27
+
28
+ function locatorBadge(item: { locator: string; verdict: AttemptListItem["verdict"] }): string {
29
+ return `${item.locator}${verdictMark(item.verdict)}`;
30
+ }
31
+
32
+ /**
33
+ * failureSummary + moreFailures 的展示形态:摘要在计算侧已按 Scoring display 契约折好,
34
+ * 这里只加 "+N more failures" 计数与宽度收口,不重算摘要。
35
+ */
36
+ function attemptReasonText(item: AttemptListItem, locale: ReportLocale, maxChars: number): string | undefined {
37
+ if (item.failureSummary === null) return undefined;
38
+ const withMore =
39
+ item.moreFailures > 0
40
+ ? `${item.failureSummary} · ${countText(locale, "entityList.moreFailures", item.moreFailures)}`
41
+ : item.failureSummary;
42
+ return fitFailureSummary(withMore, Math.max(24, maxChars));
43
+ }
44
+
45
+ // ── ExperimentList ──
46
+
47
+ function experimentSummaryTable(
48
+ items: readonly ExperimentListItem[],
49
+ ctx: TextContext,
50
+ labels: Map<string, string>,
51
+ ): string {
52
+ const locale = ctx.locale;
53
+ const compact = ctx.width < 100;
54
+ const columns: TableColumn[] = [
55
+ { key: "experiment", header: compact && locale === "en" ? "Exp." : localeText(locale, "experimentList.experiment") },
56
+ { key: "model", header: localeText(locale, "table.model") },
57
+ { key: "agent", header: localeText(locale, "table.agent") },
58
+ { key: "duration", header: compact && locale === "en" ? "Avg" : localeText(locale, "experimentList.avgDuration"), align: "right" },
59
+ { key: "passRate", header: compact && locale === "en" ? "Pass" : localeText(locale, "experimentList.passRate"), align: "right" },
60
+ { key: "result", header: localeText(locale, "experimentList.result") },
61
+ { key: "tokens", header: localeText(locale, "experimentList.tokens"), align: "right" },
62
+ { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
63
+ ];
64
+ const rows: TableRow[] = items.map((item) => ({
65
+ key: item.experimentId,
66
+ cells: {
67
+ experiment: labels.get(item.experimentId) ?? item.experimentId,
68
+ model: item.model ?? localeText(locale, "experimentList.defaultModel"),
69
+ agent: item.agent,
70
+ duration: cellText(item.durationMs, locale),
71
+ passRate: cellText(item.endToEndPassRate, locale),
72
+ result: verdictTallyText(item.evalVerdicts, locale),
73
+ tokens: cellText(item.tokens, locale),
74
+ cost: cellText(item.costUSD, locale),
75
+ },
76
+ }));
77
+ const metadata = items.flatMap((item) =>
78
+ wrapDisplay(
79
+ `${labels.get(item.experimentId) ?? item.experimentId}: ${localeText(locale, "overview.evalsCount", { n: item.evals })} · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })} · ${item.lastRunAt}`,
80
+ Math.max(8, ctx.width - 2),
81
+ ).map((line) => ` ${line}`),
82
+ );
83
+ return [renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx), metadata.join("\n")].join("\n");
84
+ }
85
+
86
+ function experimentDetailTable(item: ExperimentListItem, ctx: TextContext, label: string): string {
87
+ const locale = ctx.locale;
88
+ const columns: TableColumn[] = [
89
+ { key: "status", header: localeText(locale, "experimentList.status") },
90
+ { key: "entity", header: localeText(locale, "experimentList.evalAttempt") },
91
+ // Result 是可扫读的失败预览,不是证据面:两行放不下的以 … 收口,完整值走 locator 下钻。
92
+ { key: "result", header: localeText(locale, "experimentList.result"), maxLines: 2 },
93
+ { key: "duration", header: localeText(locale, "experimentList.duration"), align: "right" },
94
+ { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
95
+ ];
96
+ // Result 的字符预算 ≈ 两行 × 它能分到的列宽(总宽减其它列的自然宽与列距)。这里只做
97
+ // 粗预算;精确的按宽度收口由列的 maxLines 兜底。
98
+ const statusWidth = Math.max(
99
+ stringWidth(localeText(locale, "experimentList.status")),
100
+ ...item.evalRows.map((row) => stringWidth(`${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`)),
101
+ );
102
+ const entityWidth = Math.max(
103
+ stringWidth(localeText(locale, "experimentList.evalAttempt")),
104
+ ...item.evalRows.flatMap((row) => [stringWidth(row.evalId), ...row.attempts.map((a) => stringWidth(a.locator) + 3)]),
105
+ );
106
+ const fixedWidth = statusWidth + entityWidth + 8 /* duration */ + 6 /* cost */ + 3 * 4; /* 4 段列距 */
107
+ const resultBudget = Math.max(24, (ctx.width - fixedWidth) * 2);
108
+ const rows: TableRow[] = item.evalRows.flatMap((row) => {
109
+ // Eval 父行只承载折叠判定与题级聚合;失败摘要只在 Attempt 子行出现。
110
+ const parent: TableRow = {
111
+ key: row.evalId,
112
+ cells: {
113
+ status: `${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`,
114
+ entity: row.evalId,
115
+ result: "",
116
+ duration: localeText(locale, "entityList.average", { value: cellText(row.durationMs, locale) }),
117
+ cost: localeText(locale, "entityList.average", { value: cellText(row.costUSD, locale) }),
118
+ },
119
+ };
120
+ const attempts: TableRow[] = row.attempts.map((attempt, index) => ({
121
+ key: attempt.locator,
122
+ cells: {
123
+ status: ` ${verdictMark(attempt.verdict)}`,
124
+ entity: `${index === row.attempts.length - 1 ? "└─" : "├─"} ${attempt.locator}`,
125
+ result: attemptReasonText(attempt, locale, resultBudget) ?? MISSING_MARK,
126
+ duration: attempt.verdict === "skipped" && attempt.durationMs === 0 ? null : formatDurationMs(attempt.durationMs),
127
+ cost: attempt.costUSD === null ? null : formatUSD(attempt.costUSD),
128
+ },
129
+ }));
130
+ return [parent, ...attempts];
131
+ });
132
+ const flags = item.flags && Object.keys(item.flags).length > 0
133
+ ? `${localeText(locale, "experimentList.flags")} ${Object.entries(item.flags)
134
+ .map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`)
135
+ .join(" · ")}`
136
+ : undefined;
137
+ return [
138
+ label,
139
+ flags,
140
+ renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx),
141
+ ]
142
+ .filter(Boolean)
143
+ .join("\n");
144
+ }
145
+
146
+ export function experimentListText(items: readonly ExperimentListItem[], ctx: TextContext): string {
147
+ if (items.length === 0) return localeText(ctx.locale, "attemptList.empty");
148
+ const labels = shortestUniqueLabels(items.map((item) => item.experimentId));
149
+ return [
150
+ experimentSummaryTable(items, ctx, labels),
151
+ ...items.map((item) => experimentDetailTable(item, ctx, labels.get(item.experimentId) ?? item.experimentId)),
152
+ ].join("\n\n");
153
+ }
154
+
155
+ // ── EvalList ──
156
+
157
+ function evalListAttemptLine(item: AttemptListItem, ctx: TextContext): string {
158
+ // 行式列表同守「Result 最多两行」:预算 = 两行终端宽,超出按尾截收口。
159
+ const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - stringWidth(locatorBadge(item)) - 6);
160
+ return ` ${locatorBadge(item)}${reason ? ` · ${reason}` : ""}`;
161
+ }
162
+
163
+ export function evalListText(items: readonly EvalListItem[], ctx: TextContext): string {
164
+ const locale = ctx.locale;
165
+ if (items.length === 0) return localeText(locale, "attemptList.empty");
166
+ const blocks = items.map((item) => {
167
+ const identity = `${item.evalId} · ${item.experimentId} · ${localeText(locale, `verdict.${item.verdict}`)}`;
168
+ const summary = [
169
+ localeText(locale, "attemptList.score", { score: cellText(item.examScore, locale) }),
170
+ localeText(locale, "overview.attemptsCount", { n: item.attempts.length }),
171
+ localeText(locale, "entityList.average", {
172
+ value: item.durationMs.value === null ? missingText(locale) : formatDurationMs(item.durationMs.value),
173
+ }),
174
+ localeText(locale, "entityList.average", {
175
+ value: item.costUSD.value === null ? missingText(locale) : formatUSD(item.costUSD.value),
176
+ }),
177
+ ].join(" · ");
178
+ const attemptLines = item.attempts.map((attempt) => evalListAttemptLine(attempt, ctx));
179
+ return [identity, ` ${summary}`, ...attemptLines].join("\n");
180
+ });
181
+ return blocks.join("\n\n");
182
+ }
183
+
184
+ // ── AttemptList ──
185
+
186
+ /** Attempt 比较卡片:只显示一条主失败摘要(至多两行终端宽);完整 assertions 走 locator 下钻。 */
187
+ function attemptListItemText(item: AttemptListItem, ctx: TextContext): string {
188
+ const head = [
189
+ `${verdictMark(item.verdict)} ${item.locator}`,
190
+ item.evalId,
191
+ item.experimentId,
192
+ formatDurationMs(item.durationMs),
193
+ ...(item.costUSD !== null ? [formatUSD(item.costUSD)] : []),
194
+ ].join(" · ");
195
+ const lines = [head];
196
+ const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - 4);
197
+ if (reason) lines.push(` ${reason}`);
198
+ return lines.join("\n");
199
+ }
200
+
201
+ export function attemptListText(items: readonly AttemptListItem[], total: number | undefined, ctx: TextContext): string {
202
+ const locale = ctx.locale;
203
+ if (items.length === 0) return localeText(locale, "attemptList.empty");
204
+ const blocks = items.map((item) => attemptListItemText(item, ctx));
205
+ const remaining = (total ?? items.length) - items.length;
206
+ if (remaining > 0) blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
207
+ return blocks.join("\n\n");
208
+ }