niceeval 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +15 -7
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
  118. package/dist/report/{tree.js → definition/tree.js} +10 -12
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +12 -2
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +15 -1
  154. package/dist/shared/aggregate.js +32 -1
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +77 -64
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +30 -15
  227. package/src/i18n/zh-CN.ts +30 -15
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +136 -112
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +58 -22
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +54 -3
  314. package/src/results/select.ts +30 -6
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +30 -1
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +58 -22
  356. package/src/show/render.ts +16 -240
  357. package/src/show/show.test.ts +128 -35
  358. package/src/view/app/App.test.tsx +8 -7
  359. package/src/view/app/App.tsx +135 -51
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +122 -89
  370. package/src/view/index.ts +3 -13
  371. package/src/view/server.ts +36 -6
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +18 -806
  376. package/src/view/view-report.test.ts +182 -34
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,767 +0,0 @@
1
- // 官方组件的 text 面:同一份算好的数据,渲染成终端字符(niceeval show 的形态)。
2
- // 与 web 面共守诚实契约:排序随 better、samples < total 角标、缺数据 — 不补 0、截断报剩余。
3
- // 零 react、零 IO、纯同步 —— 这是 text 宿主不需要 react-dom 的那一半。
4
- // chrome 文案(注脚、verdict 词、截断提示)经 ctx.locale 查 locale 字典;
5
- // 数据 display 是 LocalizedText,按 LocalizedText 回退规则取值。
6
-
7
- import type {
8
- AttemptListItem,
9
- DeltaData,
10
- EvalListItem,
11
- ExperimentComparisonData,
12
- ExperimentComparisonGroupData,
13
- ExperimentListItem,
14
- HeroData,
15
- LineData,
16
- MatrixData,
17
- MetricCell,
18
- MetricColumn,
19
- ScatterData,
20
- ScopeSummaryData,
21
- ScopeWarning,
22
- ScoreboardData,
23
- TableData,
24
- TraceWaterfallRow,
25
- VerdictTally,
26
- } from "../types.ts";
27
- import type { LocalizedText } from "../locale.ts";
28
- import { groupScopeWarnings } from "../scope-warnings.ts";
29
- import type { TextContext } from "../tree.ts";
30
- import type { TableColumn, TableRow } from "../primitives.tsx";
31
- import {
32
- experimentDisplayName,
33
- fitFailureSummary,
34
- formatDurationMs,
35
- formatMetricValue,
36
- formatPlainNumber,
37
- formatUSD,
38
- verdictMark,
39
- } from "../format.ts";
40
- import {
41
- countText,
42
- localeText,
43
- resolveLocalizedText,
44
- resolveMetricLabel,
45
- type ReportLocale,
46
- } from "../locale.ts";
47
- import { indentBlock, padDisplay, stringWidth, textBar, wrapDisplay } from "./layout.ts";
48
- import { renderTableText } from "./table.ts";
49
- import { renderCharPlot, renderCoordinateTable, type PlotPoint } from "./plot.ts";
50
-
51
- const MISSING_MARK = "—";
52
-
53
- /** 缺数据文案随 locale(en = "no data")。 */
54
- function missingText(locale: ReportLocale): string {
55
- return localeText(locale, "cell.missing");
56
- }
57
-
58
- /** 格子的文本形态:缺数据 —,覆盖不全带 samples/total 角标;display 按 locale 解析。 */
59
- export function cellText(cell: MetricCell, locale: ReportLocale): string {
60
- if (cell.value === null) return MISSING_MARK;
61
- const display = resolveLocalizedText(cell.display, locale);
62
- return cell.samples < cell.total ? `${display} ${cell.samples}/${cell.total}` : display;
63
- }
64
-
65
- /** verdict 计票的紧凑文案("3 passed / 1 failed"):非零判定逐个列,全部为零如实 —。 */
66
- export function verdictTallyText(tally: VerdictTally, locale: ReportLocale): string {
67
- const parts: string[] = [];
68
- for (const kind of ["passed", "failed", "errored", "skipped"] as const) {
69
- if (tally[kind] > 0) parts.push(`${tally[kind]} ${localeText(locale, `verdict.${kind}`)}`);
70
- }
71
- return parts.length > 0 ? parts.join(" / ") : MISSING_MARK;
72
- }
73
-
74
- /** ISO 时间 → "YYYY-MM-DD HH:mm"(本地时区);不可解析原样返回。 */
75
- function formatDateTimeMinute(iso: string): string {
76
- const date = new Date(iso);
77
- if (Number.isNaN(date.valueOf())) return iso;
78
- const pad = (n: number) => String(n).padStart(2, "0");
79
- return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}`;
80
- }
81
-
82
- // ───────────────────────── ScopeSummary ─────────────────────────
83
-
84
- /**
85
- * 一至两行:头行是端到端成功率(官方 MetricCell,不重算)+ experiment/eval/attempt 数 +
86
- * `votes` 选中的那级计票 + 总成本;第二行(有则加)是快照时间窗。
87
- */
88
- export function scopeSummaryText(data: ScopeSummaryData, votes: "eval" | "attempt", ctx: TextContext): string {
89
- const locale = ctx.locale;
90
- const tally = votes === "attempt" ? data.attemptVerdicts : data.evalVerdicts;
91
- const head = [
92
- `${localeText(locale, "scopeSummary.passRate")} ${cellText(data.endToEndPassRate, locale)}`,
93
- countText(locale, "overview.experiments", data.experiments),
94
- localeText(locale, "overview.evalsCount", { n: data.evals }),
95
- localeText(locale, "overview.attemptsCount", { n: data.attempts }),
96
- verdictTallyText(tally, locale),
97
- `${localeText(locale, "scopeSummary.totalCost")} ${cellText(data.totalCostUSD, locale)}`,
98
- ].join(" · ");
99
- const lines = [head];
100
- if (data.range.latestStartedAt !== null) {
101
- const from = data.range.earliestStartedAt;
102
- const to = data.range.latestStartedAt;
103
- lines.push(
104
- from !== null && from !== to
105
- ? localeText(locale, "scopeSummary.range", { from: formatDateTimeMinute(from), to: formatDateTimeMinute(to) })
106
- : formatDateTimeMinute(to),
107
- );
108
- }
109
- return lines.join("\n");
110
- }
111
-
112
- // ───────────────────────── ExperimentComparison ─────────────────────────
113
-
114
- /** 单组详情:标题行 + 缩进的散点与实验列表(与 web 面同一份预分区数据)。 */
115
- function comparisonGroupText(group: ExperimentComparisonGroupData, ctx: TextContext): string {
116
- const inner = Math.max(20, ctx.width - 2);
117
- const innerCtx: TextContext = {
118
- width: inner,
119
- locale: ctx.locale,
120
- attemptCommand: ctx.attemptCommand,
121
- experimentCommand: ctx.experimentCommand,
122
- render: (node, w) => ctx.render(node, w ?? inner),
123
- };
124
- const body = [scatterText(group.scatter, innerCtx), experimentListText(group.experiments, innerCtx, group.key)]
125
- .filter((block) => block.length > 0)
126
- .join("\n\n");
127
- return `${localeText(ctx.locale, "experimentComparison.group")} ${group.key}\n\n${indentBlock(body, " ")}`;
128
- }
129
-
130
- /**
131
- * text 面:命中多个可比组时只输出组索引与可复制的单组查看命令,不倾倒全部组详情;
132
- * 命中单组时省略索引,直接输出该组的散点与实验列表。
133
- */
134
- export function experimentComparisonText(
135
- data: ExperimentComparisonData,
136
- _className: string | undefined,
137
- ctx: TextContext,
138
- ): string {
139
- const locale = ctx.locale;
140
- if (data.groups.length === 0) return localeText(locale, "experimentComparison.empty");
141
- if (data.groups.length === 1) return comparisonGroupText(data.groups[0]!, ctx);
142
-
143
- const columns: TableColumn[] = [
144
- { key: "group", header: localeText(locale, "experimentComparison.group") },
145
- { key: "experiments", header: localeText(locale, "scopeSummary.experiments"), align: "right" },
146
- { key: "evals", header: localeText(locale, "scopeSummary.evals"), align: "right" },
147
- { key: "passRate", header: localeText(locale, "scopeSummary.passRate"), align: "right" },
148
- { key: "results", header: localeText(locale, "experimentComparison.results") },
149
- { key: "cost", header: localeText(locale, "scopeSummary.totalCost"), align: "right" },
150
- { key: "lastRun", header: localeText(locale, "experimentComparison.lastRun") },
151
- ] as unknown as TableColumn[];
152
- const rows: TableRow[] = data.groups.map((group) => ({
153
- key: group.key,
154
- cells: {
155
- group: group.key,
156
- experiments: String(group.summary.experiments),
157
- evals: String(group.summary.evals),
158
- passRate: cellText(group.summary.endToEndPassRate, locale),
159
- results: verdictTallyText(group.summary.evalVerdicts, locale),
160
- cost: group.summary.totalCostUSD.value === null ? null : cellText(group.summary.totalCostUSD, locale),
161
- lastRun: group.summary.range.latestStartedAt === null ? null : formatDateTimeMinute(group.summary.range.latestStartedAt),
162
- },
163
- }));
164
- const table = renderTableText(
165
- { columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale },
166
- ctx,
167
- );
168
- const commands = [
169
- localeText(locale, "experimentComparison.commandsHead"),
170
- ...data.groups.map((group) => ` ${ctx.experimentCommand(group.key)}`),
171
- ].join("\n");
172
- return `${table}\n\n${commands}`;
173
- }
174
-
175
- // ───────────────────────── MetricTable ─────────────────────────
176
-
177
- export function tableText(data: TableData, ctx: TextContext): string {
178
- const locale = ctx.locale;
179
- const columns: TableColumn[] = [
180
- { key: "dimension", header: data.rowDimension },
181
- // 指标列的键用序号,不用 metric name —— 维度名和指标名在同一个命名空间里,可能撞键。
182
- ...data.columns.map((c, i) => ({ key: `metric${i}`, header: resolveMetricLabel(c.label, locale, c.key) })),
183
- ];
184
- const rows: TableRow[] = data.rows.map((row) => {
185
- const cells: Record<string, string | null> = { dimension: row.key };
186
- data.columns.forEach((col, i) => {
187
- const cell = row.cells[col.key];
188
- cells[`metric${i}`] = cell ? cellText(cell, locale) : null;
189
- });
190
- return { key: row.key, cells };
191
- });
192
- return renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx);
193
- }
194
-
195
- // ───────────────────────── MetricMatrix ─────────────────────────
196
-
197
- export function matrixText(data: MatrixData, ctx: TextContext): string {
198
- // 表体全是维度键与 display,没有 chrome 文案;"next:" 是命令提示,不本地化。
199
- const locale = ctx.locale;
200
- const rowKeys: string[] = [];
201
- const columnKeys: string[] = [];
202
- const byPosition = new Map<string, MetricCell>();
203
- for (const entry of data.cells) {
204
- if (!rowKeys.includes(entry.row)) rowKeys.push(entry.row);
205
- if (!columnKeys.includes(entry.column)) columnKeys.push(entry.column);
206
- byPosition.set(JSON.stringify([entry.row, entry.column]), entry.cell);
207
- }
208
- const columns: TableColumn[] = [
209
- { key: "dimension", header: data.rowDimension },
210
- ...columnKeys.map((column, i) => ({ key: `column${i}`, header: column })),
211
- ];
212
- const rows: TableRow[] = rowKeys.map((row) => {
213
- const cells: Record<string, string | null> = { dimension: row };
214
- columnKeys.forEach((column, i) => {
215
- const cell = byPosition.get(JSON.stringify([row, column]));
216
- cells[`column${i}`] = cell ? cellText(cell, locale) : null; // 稀疏格子在文本里以 — 呈现,不编数
217
- });
218
- return { key: row, cells };
219
- });
220
- const table = renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx);
221
-
222
- // 下钻命令:行维度是 eval 时,指向最值得看的一行(先挑有缺格的,再挑按 better 最差的)
223
- if (data.rowDimension !== "eval" || rowKeys.length === 0) return table;
224
- const better = data.metric.better ?? "higher";
225
- let next: string | undefined;
226
- let worst: { key: string; value: number } | undefined;
227
- for (const row of rowKeys) {
228
- let sum = 0;
229
- let count = 0;
230
- for (const column of columnKeys) {
231
- const cell = byPosition.get(JSON.stringify([row, column]));
232
- if (!cell || cell.value === null) {
233
- next ??= row;
234
- continue;
235
- }
236
- sum += cell.value;
237
- count += 1;
238
- }
239
- if (count > 0) {
240
- const value = sum / count;
241
- const isWorse = worst === undefined || (better === "higher" ? value < worst.value : value > worst.value);
242
- if (isWorse) worst = { key: row, value };
243
- }
244
- }
245
- next ??= worst?.key;
246
- return next === undefined ? table : `${table}\n\nnext: niceeval show ${next}`;
247
- }
248
-
249
- // ───────────────────────── MetricBars(矩阵数据的另一种摆法)─────────────────────────
250
-
251
- const BAR_WIDTH = 20;
252
-
253
- export function barsText(data: MatrixData, ctx: TextContext): string {
254
- const locale = ctx.locale;
255
- const groupKeys: string[] = [];
256
- const seriesKeys: string[] = [];
257
- const byPosition = new Map<string, MetricCell>();
258
- for (const entry of data.cells) {
259
- if (!groupKeys.includes(entry.row)) groupKeys.push(entry.row);
260
- if (!seriesKeys.includes(entry.column)) seriesKeys.push(entry.column);
261
- byPosition.set(JSON.stringify([entry.row, entry.column]), entry.cell);
262
- }
263
- const better = data.metric.better ?? "higher";
264
- // 条长刻度:% 的天然域是 [0,1],其余以全图最大值为满条
265
- const values = data.cells.map((c) => c.cell.value).filter((v): v is number => v !== null);
266
- const maxValue = values.length > 0 ? Math.max(...values) : 0;
267
- const ratioOf = (value: number) => (data.metric.unit === "%" ? value : maxValue === 0 ? 0 : value / maxValue);
268
-
269
- const seriesWidth = Math.max(...seriesKeys.map((k) => stringWidth(k)), 0);
270
- const lines: string[] = [];
271
- for (const group of groupKeys) {
272
- lines.push(group);
273
- const entries = seriesKeys.map((series) => ({
274
- series,
275
- cell: byPosition.get(JSON.stringify([group, series])),
276
- }));
277
- // 组内按值排序,方向随 better(缺数据沉底)
278
- entries.sort((a, b) => {
279
- const va = a.cell?.value ?? null;
280
- const vb = b.cell?.value ?? null;
281
- if (va === null && vb === null) return 0;
282
- if (va === null) return 1;
283
- if (vb === null) return -1;
284
- return better === "lower" ? va - vb : vb - va;
285
- });
286
- for (const { series, cell } of entries) {
287
- const label = padDisplay(series, seriesWidth);
288
- if (!cell || cell.value === null) {
289
- lines.push(` ${label} ${MISSING_MARK}`);
290
- continue;
291
- }
292
- lines.push(` ${label} ${textBar(ratioOf(cell.value), BAR_WIDTH)} ${cellText(cell, locale)}`);
293
- }
294
- }
295
- return lines.join("\n");
296
- }
297
-
298
- // ───────────────────────── Scoreboard ─────────────────────────
299
-
300
- export function scoreboardText(data: ScoreboardData, ctx: TextContext): string {
301
- const locale = ctx.locale;
302
- const subjectKeys: string[] = [];
303
- for (const row of data.rows) {
304
- for (const subject of row.subjects) {
305
- if (!subjectKeys.includes(subject.key)) subjectKeys.push(subject.key);
306
- }
307
- }
308
- const columns: TableColumn[] = [
309
- { key: "dimension", header: data.rowDimension },
310
- { key: "total", header: localeText(locale, "scoreboard.totalText") },
311
- ...subjectKeys.map((key, i) => ({ key: `subject${i}`, header: key })),
312
- ];
313
- const rows: TableRow[] = data.rows.map((row) => {
314
- const totalNotes = [
315
- ...(row.total.notRun > 0 ? [localeText(locale, "scoreboard.notRunText", { n: row.total.notRun })] : []),
316
- ...(row.total.unscorable > 0 ? [localeText(locale, "scoreboard.unscorableText", { n: row.total.unscorable })] : []),
317
- ];
318
- const cells: Record<string, string | null> = {
319
- dimension: row.key,
320
- total: `${resolveLocalizedText(row.total.display, locale)}/${data.fullMarks}${totalNotes.length > 0 ? ` ${totalNotes.join(" ")}` : ""}`,
321
- };
322
- subjectKeys.forEach((key, i) => {
323
- const subject = row.subjects.find((s) => s.key === key);
324
- if (!subject) {
325
- cells[`subject${i}`] = null;
326
- return;
327
- }
328
- const score = `${formatPlainNumber(subject.earned)}/${formatPlainNumber(subject.possible)}`;
329
- const notes = [
330
- ...(subject.notRun > 0 ? [localeText(locale, "scoreboard.notRunText", { n: subject.notRun })] : []),
331
- ...(subject.unscorable > 0 ? [localeText(locale, "scoreboard.unscorableText", { n: subject.unscorable })] : []),
332
- ];
333
- cells[`subject${i}`] = notes.length > 0 ? `${score} ${notes.join(" ")}` : score;
334
- });
335
- return { key: row.key, cells };
336
- });
337
- const table = renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx);
338
- const footnotes: string[] = [];
339
- if (data.weights.length > 0) {
340
- const weights = data.weights.map((w) => `${w.prefix} ×${w.weight}`).join(" · ");
341
- footnotes.push(`${localeText(locale, "scoreboard.weights")} ${weights} · ${localeText(locale, "scoreboard.othersWeight")}`);
342
- }
343
- if (data.ignoredEvals > 0) footnotes.push(countText(locale, "scoreboard.ignored", data.ignoredEvals));
344
- return footnotes.length > 0 ? `${table}\n${footnotes.join("\n")}` : table;
345
- }
346
-
347
- // ───────────────────────── MetricScatter ─────────────────────────
348
-
349
- const POINT_MARKS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
350
-
351
- function axisLabel(col: MetricColumn, locale: ReportLocale): string {
352
- return resolveMetricLabel(col.label, locale, col.key);
353
- }
354
-
355
- /**
356
- * 方向提示:轴向已随 better 反正(lower 反向),「更好」恒指向右上,文案恒为
357
- * 「越靠右上越好」;仅当 x、y 都声明 better 时显示——任一轴未声明,组件不猜
358
- * 「更好」朝哪边,整图无提示(返回 undefined)。
359
- */
360
- function betterCornerText(x: MetricColumn, y: MetricColumn, locale: ReportLocale): string | undefined {
361
- if (x.better === undefined || y.better === undefined) return undefined;
362
- return localeText(locale, "scatter.betterUpperRight");
363
- }
364
-
365
- /** 轴标题行:"<x label>(越低越好) × <y label>"——better 方向如实标注,不虚构轴向。 */
366
- function scatterHeading(data: ScatterData, locale: ReportLocale): string {
367
- const withBetter = (col: MetricColumn) => {
368
- const label = axisLabel(col, locale);
369
- if (col.better === undefined) return label;
370
- return `${label}(${localeText(locale, col.better === "lower" ? "table.lowerBetter" : "table.higherBetter")})`;
371
- };
372
- return `${withBetter(data.x)} × ${withBetter(data.y)}`;
373
- }
374
-
375
- export function scatterText(data: ScatterData, ctx: TextContext): string {
376
- const locale = ctx.locale;
377
- const drawable = data.rows.filter((r) => r.x.value !== null && r.y.value !== null);
378
- const missing = data.rows.length - drawable.length;
379
- const footnotes: string[] = [];
380
- if (missing > 0) footnotes.push(countText(locale, "pointsMissing", missing));
381
-
382
- const axes = { x: axisLabel(data.x, locale), y: axisLabel(data.y, locale) };
383
- // 0 个可画点:x/y 指标没有可用数据(与 web 面同一事实)。
384
- if (drawable.length === 0) {
385
- return [localeText(locale, "scatter.noData", axes), ...footnotes].join("\n");
386
- }
387
- // 点太密排不下时降级为坐标表,不硬挤
388
- if (drawable.length > POINT_MARKS.length || ctx.width < 44) {
389
- const table = renderCoordinateTable(
390
- drawable.map((r) => ({
391
- key: r.key,
392
- x: resolveLocalizedText(r.x.display, locale),
393
- y: resolveLocalizedText(r.y.display, locale),
394
- })),
395
- { key: data.pointDimension, x: axes.x, y: axes.y },
396
- );
397
- return [scatterHeading(data, locale), "", table, ...footnotes].join("\n");
398
- }
399
-
400
- const points: PlotPoint[] = drawable.map((r, i) => ({
401
- mark: POINT_MARKS[i],
402
- x: r.x.value as number,
403
- y: r.y.value as number,
404
- }));
405
- // 同系列的点按 x 排序连线
406
- const bySeries = new Map<string, { x: number; y: number }[]>();
407
- for (const r of drawable) {
408
- if (r.series === undefined) continue;
409
- const list = bySeries.get(r.series) ?? [];
410
- list.push({ x: r.x.value as number, y: r.y.value as number });
411
- bySeries.set(r.series, list);
412
- }
413
- for (const list of bySeries.values()) list.sort((a, b) => a.x - b.x);
414
-
415
- const plot = renderCharPlot({
416
- width: ctx.width,
417
- points,
418
- lines: [...bySeries.values()].filter((l) => l.length > 1),
419
- xLabel: axes.x,
420
- yLabel: axes.y,
421
- formatX: (v) => formatMetricValue(v, data.x.unit),
422
- formatY: (v) => formatMetricValue(v, data.y.unit),
423
- // 轴方向跟随 better(与 web 面同规则):lower 反向,higher 与未声明正向。
424
- invertX: data.x.better === "lower",
425
- invertY: data.y.better === "lower",
426
- });
427
- const legend = drawable.map((r, i) => `${POINT_MARKS[i]} ${r.key}`).join(" ");
428
- const hint = betterCornerText(data.x, data.y, locale);
429
- return [scatterHeading(data, locale), plot, "", ...(hint !== undefined ? [hint] : []), legend, ...footnotes].join("\n");
430
- }
431
-
432
- // ───────────────────────── MetricLine ─────────────────────────
433
-
434
- export function lineText(data: LineData, ctx: TextContext): string {
435
- const locale = ctx.locale;
436
- const drawable = data.rows.filter((r) => r.x !== null && r.y.value !== null);
437
- const missing = data.rows.length - drawable.length;
438
- const footnotes: string[] = [];
439
- if (missing > 0) footnotes.push(countText(locale, "pointsMissing", missing));
440
-
441
- if (drawable.length === 0) return [missingText(locale), ...footnotes].join("\n");
442
-
443
- const xLabel = resolveMetricLabel(data.x.label, locale, data.x.key);
444
-
445
- // 系列 → 字母;无系列 = 单系列
446
- const seriesKeys: string[] = [];
447
- for (const r of drawable) {
448
- const key = r.series ?? "";
449
- if (!seriesKeys.includes(key)) seriesKeys.push(key);
450
- }
451
-
452
- if (seriesKeys.length > POINT_MARKS.length || ctx.width < 44) {
453
- const table = renderCoordinateTable(
454
- drawable.map((r) => ({
455
- key: r.series !== undefined ? `${r.series} (${r.key})` : r.key,
456
- x: resolveLocalizedText(r.xDisplay, locale),
457
- y: resolveLocalizedText(r.y.display, locale),
458
- })),
459
- { key: data.seriesDimension ?? xLabel, x: xLabel, y: axisLabel(data.y, locale) },
460
- );
461
- return [table, ...footnotes].join("\n");
462
- }
463
-
464
- const markOf = (r: LineData["rows"][number]) => POINT_MARKS[seriesKeys.indexOf(r.series ?? "")];
465
- const points: PlotPoint[] = drawable.map((r) => ({
466
- mark: markOf(r),
467
- x: r.x as number,
468
- y: r.y.value as number,
469
- }));
470
- const lines = seriesKeys.map((key) =>
471
- drawable
472
- .filter((r) => (r.series ?? "") === key)
473
- .map((r) => ({ x: r.x as number, y: r.y.value as number }))
474
- .sort((a, b) => a.x - b.x),
475
- );
476
-
477
- const plot = renderCharPlot({
478
- width: ctx.width,
479
- points,
480
- lines: lines.filter((l) => l.length > 1),
481
- xLabel,
482
- yLabel: axisLabel(data.y, locale),
483
- formatX: (v) => formatMetricValue(v, data.x.unit),
484
- formatY: (v) => formatMetricValue(v, data.y.unit),
485
- });
486
- const legend = seriesKeys
487
- .map((key, i) => `${POINT_MARKS[i]} ${key === "" ? axisLabel(data.y, locale) : key}`)
488
- .join(" ");
489
- return [plot, "", legend, ...footnotes].join("\n");
490
- }
491
-
492
- // ───────────────────────── DeltaTable ─────────────────────────
493
-
494
- export function deltaText(data: DeltaData, ctx: TextContext): string {
495
- const locale = ctx.locale;
496
- // 0 对不是错误:明确空态并报告配对域实验数(派生形态携带;字面形态缺省按 0)。
497
- if (data.rows.length === 0) {
498
- return localeText(locale, "delta.empty", { experiments: data.experiments ?? 0 });
499
- }
500
- const columns: TableColumn[] = [
501
- { key: "pair", header: localeText(locale, "delta.pairHeader") },
502
- ...data.columns.map((c, i) => ({ key: `metric${i}`, header: resolveMetricLabel(c.label, locale, c.key) })),
503
- ];
504
- const rows: TableRow[] = data.rows.map((row) => {
505
- const cells: Record<string, string | null> = { pair: resolveLocalizedText(row.label, locale) };
506
- data.columns.forEach((col, i) => {
507
- const cell = row.cells[col.key];
508
- if (!cell) {
509
- cells[`metric${i}`] = null;
510
- return;
511
- }
512
- const a = cell.a.value === null ? MISSING_MARK : resolveLocalizedText(cell.a.display, locale);
513
- const b = cell.b.value === null ? MISSING_MARK : resolveLocalizedText(cell.b.display, locale);
514
- cells[`metric${i}`] = `${a} → ${b} ${resolveLocalizedText(cell.display, locale)}`;
515
- });
516
- return { key: row.key, cells };
517
- });
518
- return renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx);
519
- }
520
-
521
- // ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
522
- //
523
- // 三面共用的紧凑标记:`locator✓`(判定符紧跟 locator,中间不留空格)。
524
- // ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的摘要,不重复整段
525
- // niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼 `niceeval show <locator>`。
526
-
527
- function locatorBadge(item: { locator: string; verdict: AttemptListItem["verdict"] }): string {
528
- return `${item.locator}${verdictMark(item.verdict)}`;
529
- }
530
-
531
- /**
532
- * failureSummary + moreFailures 的展示形态:摘要在计算侧已按 Scoring display 契约折好,
533
- * 这里只加 "+N more failures" 计数与宽度收口,不重算摘要。
534
- */
535
- function attemptReasonText(item: AttemptListItem, locale: ReportLocale, maxChars: number): string | undefined {
536
- if (item.failureSummary === null) return undefined;
537
- const withMore =
538
- item.moreFailures > 0
539
- ? `${item.failureSummary} · ${countText(locale, "entityList.moreFailures", item.moreFailures)}`
540
- : item.failureSummary;
541
- return fitFailureSummary(withMore, Math.max(24, maxChars));
542
- }
543
-
544
- // ── ExperimentList ──
545
-
546
- function experimentSummaryTable(
547
- items: readonly ExperimentListItem[],
548
- ctx: TextContext,
549
- relativeTo?: string,
550
- ): string {
551
- const locale = ctx.locale;
552
- const compact = ctx.width < 100;
553
- const columns: TableColumn[] = [
554
- { key: "experiment", header: compact && locale === "en" ? "Exp." : localeText(locale, "experimentList.experiment") },
555
- { key: "model", header: localeText(locale, "table.model") },
556
- { key: "agent", header: localeText(locale, "table.agent") },
557
- { key: "duration", header: compact && locale === "en" ? "Avg" : localeText(locale, "experimentList.avgDuration"), align: "right" },
558
- { key: "passRate", header: compact && locale === "en" ? "Pass" : localeText(locale, "experimentList.passRate"), align: "right" },
559
- { key: "result", header: localeText(locale, "experimentList.result") },
560
- { key: "tokens", header: localeText(locale, "experimentList.tokens"), align: "right" },
561
- { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
562
- ];
563
- const rows: TableRow[] = items.map((item) => ({
564
- key: item.experimentId,
565
- cells: {
566
- experiment: experimentDisplayName(item.experimentId, relativeTo),
567
- model: item.model ?? localeText(locale, "experimentList.defaultModel"),
568
- agent: item.agent,
569
- duration: cellText(item.durationMs, locale),
570
- passRate: cellText(item.endToEndPassRate, locale),
571
- result: verdictTallyText(item.evalVerdicts, locale),
572
- tokens: cellText(item.tokens, locale),
573
- cost: cellText(item.costUSD, locale),
574
- },
575
- }));
576
- const metadata = items.flatMap((item) =>
577
- wrapDisplay(
578
- `${experimentDisplayName(item.experimentId, relativeTo)}: ${localeText(locale, "overview.evalsCount", { n: item.evals })} · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })} · ${item.lastRunAt}`,
579
- Math.max(8, ctx.width - 2),
580
- ).map((line) => ` ${line}`),
581
- );
582
- return [renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx), metadata.join("\n")].join("\n");
583
- }
584
-
585
- function experimentDetailTable(item: ExperimentListItem, ctx: TextContext, relativeTo?: string): string {
586
- const locale = ctx.locale;
587
- const columns: TableColumn[] = [
588
- { key: "status", header: localeText(locale, "experimentList.status") },
589
- { key: "entity", header: localeText(locale, "experimentList.evalAttempt") },
590
- // Result 是可扫读的失败预览,不是证据面:两行放不下的以 … 收口,完整值走 locator 下钻。
591
- { key: "result", header: localeText(locale, "experimentList.result"), maxLines: 2 },
592
- { key: "duration", header: localeText(locale, "experimentList.duration"), align: "right" },
593
- { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
594
- ];
595
- // Result 的字符预算 ≈ 两行 × 它能分到的列宽(总宽减其它列的自然宽与列距)。这里只做
596
- // 粗预算;精确的按宽度收口由列的 maxLines 兜底。
597
- const statusWidth = Math.max(
598
- stringWidth(localeText(locale, "experimentList.status")),
599
- ...item.evalRows.map((row) => stringWidth(`${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`)),
600
- );
601
- const entityWidth = Math.max(
602
- stringWidth(localeText(locale, "experimentList.evalAttempt")),
603
- ...item.evalRows.flatMap((row) => [stringWidth(row.evalId), ...row.attempts.map((a) => stringWidth(a.locator) + 3)]),
604
- );
605
- const fixedWidth = statusWidth + entityWidth + 8 /* duration */ + 6 /* cost */ + 3 * 4; /* 4 段列距 */
606
- const resultBudget = Math.max(24, (ctx.width - fixedWidth) * 2);
607
- const rows: TableRow[] = item.evalRows.flatMap((row) => {
608
- // Eval 父行只承载折叠判定与题级聚合;失败摘要只在 Attempt 子行出现。
609
- const parent: TableRow = {
610
- key: row.evalId,
611
- cells: {
612
- status: `${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`,
613
- entity: row.evalId,
614
- result: "",
615
- duration: localeText(locale, "entityList.average", { value: cellText(row.durationMs, locale) }),
616
- cost: localeText(locale, "entityList.average", { value: cellText(row.costUSD, locale) }),
617
- },
618
- };
619
- const attempts: TableRow[] = row.attempts.map((attempt, index) => ({
620
- key: attempt.locator,
621
- cells: {
622
- status: ` ${verdictMark(attempt.verdict)}`,
623
- entity: `${index === row.attempts.length - 1 ? "└─" : "├─"} ${attempt.locator}`,
624
- result: attemptReasonText(attempt, locale, resultBudget) ?? MISSING_MARK,
625
- duration: attempt.verdict === "skipped" && attempt.durationMs === 0 ? null : formatDurationMs(attempt.durationMs),
626
- cost: attempt.costUSD === null ? null : formatUSD(attempt.costUSD),
627
- },
628
- }));
629
- return [parent, ...attempts];
630
- });
631
- const flags = item.flags && Object.keys(item.flags).length > 0
632
- ? `${localeText(locale, "experimentList.flags")} ${Object.entries(item.flags)
633
- .map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`)
634
- .join(" · ")}`
635
- : undefined;
636
- return [
637
- experimentDisplayName(item.experimentId, relativeTo),
638
- flags,
639
- renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx),
640
- ]
641
- .filter(Boolean)
642
- .join("\n");
643
- }
644
-
645
- export function experimentListText(items: readonly ExperimentListItem[], ctx: TextContext, relativeTo?: string): string {
646
- if (items.length === 0) return localeText(ctx.locale, "attemptList.empty");
647
- return [
648
- experimentSummaryTable(items, ctx, relativeTo),
649
- ...items.map((item) => experimentDetailTable(item, ctx, relativeTo)),
650
- ].join("\n\n");
651
- }
652
-
653
- // ── EvalList ──
654
-
655
- function evalListAttemptLine(item: AttemptListItem, ctx: TextContext): string {
656
- // 行式列表同守「Result 最多两行」:预算 = 两行终端宽,超出按尾截收口。
657
- const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - stringWidth(locatorBadge(item)) - 6);
658
- return ` ${locatorBadge(item)}${reason ? ` · ${reason}` : ""}`;
659
- }
660
-
661
- export function evalListText(items: readonly EvalListItem[], ctx: TextContext): string {
662
- const locale = ctx.locale;
663
- if (items.length === 0) return localeText(locale, "attemptList.empty");
664
- const blocks = items.map((item) => {
665
- const identity = `${item.evalId} · ${item.experimentId} · ${localeText(locale, `verdict.${item.verdict}`)}`;
666
- const summary = [
667
- localeText(locale, "attemptList.score", { score: cellText(item.examScore, locale) }),
668
- localeText(locale, "overview.attemptsCount", { n: item.attempts.length }),
669
- localeText(locale, "entityList.average", {
670
- value: item.durationMs.value === null ? missingText(locale) : formatDurationMs(item.durationMs.value),
671
- }),
672
- localeText(locale, "entityList.average", {
673
- value: item.costUSD.value === null ? missingText(locale) : formatUSD(item.costUSD.value),
674
- }),
675
- ].join(" · ");
676
- const attemptLines = item.attempts.map((attempt) => evalListAttemptLine(attempt, ctx));
677
- return [identity, ` ${summary}`, ...attemptLines].join("\n");
678
- });
679
- return blocks.join("\n\n");
680
- }
681
-
682
- // ── AttemptList ──
683
-
684
- /** Attempt 比较卡片:只显示一条主失败摘要(至多两行终端宽);完整 assertions 走 locator 下钻。 */
685
- function attemptListItemText(item: AttemptListItem, ctx: TextContext): string {
686
- const head = [
687
- `${verdictMark(item.verdict)} ${item.locator}`,
688
- item.evalId,
689
- item.experimentId,
690
- formatDurationMs(item.durationMs),
691
- ...(item.costUSD !== null ? [formatUSD(item.costUSD)] : []),
692
- ].join(" · ");
693
- const lines = [head];
694
- const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - 4);
695
- if (reason) lines.push(` ${reason}`);
696
- return lines.join("\n");
697
- }
698
-
699
- export function attemptListText(items: readonly AttemptListItem[], total: number | undefined, ctx: TextContext): string {
700
- const locale = ctx.locale;
701
- if (items.length === 0) return localeText(locale, "attemptList.empty");
702
- const blocks = items.map((item) => attemptListItemText(item, ctx));
703
- const remaining = (total ?? items.length) - items.length;
704
- if (remaining > 0) blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
705
- return blocks.join("\n\n");
706
- }
707
-
708
- // ───────────────────────── 站点组件(HeroCard / ScopeWarnings / TraceWaterfall)─────────────────────────
709
-
710
- /**
711
- * HeroCard 的 text 面:标题行 + meta 行(最后运行时间;空范围为内置「暂无运行」文案;
712
- * 多快照时标注合成来源),不含品牌行(品牌行是纯 web 件,text 面零输出)。
713
- */
714
- export function heroCardText(title: LocalizedText, data: HeroData, ctx: TextContext): string {
715
- const locale = ctx.locale;
716
- const meta =
717
- data.latestStartedAt === null
718
- ? localeText(locale, "hero.noRuns")
719
- : [
720
- localeText(locale, "hero.lastRun", { time: formatDateTimeMinute(data.latestStartedAt) }),
721
- ...(data.snapshots > 1 ? [localeText(locale, "hero.composedSnapshots", { n: data.snapshots })] : []),
722
- ].join(" · ");
723
- return `${resolveLocalizedText(title, locale)}\n${meta}`;
724
- }
725
-
726
- /**
727
- * ScopeWarnings 的 text 面:按动作聚合(../scope-warnings.ts,与 web 面共用),同构但不折叠——
728
- * 多组时首行 "! <分类计数汇总>";每组一行组头 "! <标题> — <徽标> → <组头命令>",其下缩进
729
- * 逐条原样打印 message(已以下一步收尾,不截断掉尾段)。空警告集零输出。
730
- */
731
- export function scopeWarningsText(warnings: readonly ScopeWarning[], ctx: TextContext): string {
732
- if (warnings.length === 0) return "";
733
- const { summary, groups } = groupScopeWarnings(warnings, ctx.locale);
734
- const lines: string[] = [];
735
- // 汇总行只在多组时打印;单组时组头即汇总,不另起一行(web 面则恒以汇总行作外层 <summary>)。
736
- if (groups.length > 1) lines.push(`! ${summary}`);
737
- for (const group of groups) {
738
- const badges = group.badges.length > 0 ? ` — ${group.badges.map((b) => b.text).join(" · ")}` : "";
739
- const command = group.headCommand !== null ? ` → ${group.headCommand}` : "";
740
- lines.push(`! ${group.title}${badges}${command}`);
741
- for (const w of group.warnings) lines.push(`! ${w.message}`);
742
- }
743
- return lines.join("\n");
744
- }
745
-
746
- /**
747
- * TraceWaterfall 的 text 面:每 attempt 一行——locator、总耗时(缺 trace 如实显示缺失)、
748
- * 顶层 span 计数与失败标记,行尾是可复制的 `--timing` 下钻命令(经宿主注入的
749
- * attemptCommand 通道拼出,携带宿主上下文)。attempt 有选择器,索引终结于可执行命令。
750
- */
751
- export function traceWaterfallText(rows: readonly TraceWaterfallRow[], ctx: TextContext): string {
752
- const locale = ctx.locale;
753
- if (rows.length === 0) return localeText(locale, "traceWaterfall.empty");
754
- return rows
755
- .map((row) => {
756
- const failedSpans = row.spans.filter((span) => span.failed).length;
757
- const parts = [
758
- row.locator,
759
- row.evalId,
760
- row.durationMs === null ? localeText(locale, "traceWaterfall.noTrace") : formatDurationMs(row.durationMs),
761
- countText(locale, "traceWaterfall.spans", row.spans.length),
762
- ...(failedSpans > 0 ? [`✗ ${countText(locale, "traceWaterfall.failedSpans", failedSpans)}`] : []),
763
- ];
764
- return `${parts.join(" · ")} ${ctx.attemptCommand(row.locator)} --timing`;
765
- })
766
- .join("\n");
767
- }