niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,671 +0,0 @@
1
- // 官方组件的 text 面:同一份算好的数据,渲染成终端字符(niceeval show 的形态)。
2
- // 与 web 面共守诚实契约:排序随 better、samples < total 角标、缺数据 — 不补 0、截断报剩余。
3
- // 零 react、零 IO、纯同步 —— 这是 text 宿主不需要 react-dom 的那一半。
4
- // chrome 文案(注脚、verdict 词、截断提示)经 ctx.locale 查 locale 字典;
5
- // 数据 display 是 LocalizedText,按 LocalizedText 回退规则取值。
6
- import { groupScopeWarnings } from "../scope-warnings.js";
7
- import { experimentDisplayName, fitFailureSummary, formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD, verdictMark, } from "../format.js";
8
- import { countText, localeText, resolveLocalizedText, resolveMetricLabel, } from "../locale.js";
9
- import { indentBlock, padDisplay, stringWidth, textBar, wrapDisplay } from "./layout.js";
10
- import { renderTableText } from "./table.js";
11
- import { renderCharPlot, renderCoordinateTable } from "./plot.js";
12
- const MISSING_MARK = "—";
13
- /** 缺数据文案随 locale(en = "no data")。 */
14
- function missingText(locale) {
15
- return localeText(locale, "cell.missing");
16
- }
17
- /** 格子的文本形态:缺数据 —,覆盖不全带 samples/total 角标;display 按 locale 解析。 */
18
- export function cellText(cell, locale) {
19
- if (cell.value === null)
20
- return MISSING_MARK;
21
- const display = resolveLocalizedText(cell.display, locale);
22
- return cell.samples < cell.total ? `${display} ${cell.samples}/${cell.total}` : display;
23
- }
24
- /** verdict 计票的紧凑文案("3 passed / 1 failed"):非零判定逐个列,全部为零如实 —。 */
25
- export function verdictTallyText(tally, locale) {
26
- const parts = [];
27
- for (const kind of ["passed", "failed", "errored", "skipped"]) {
28
- if (tally[kind] > 0)
29
- parts.push(`${tally[kind]} ${localeText(locale, `verdict.${kind}`)}`);
30
- }
31
- return parts.length > 0 ? parts.join(" / ") : MISSING_MARK;
32
- }
33
- /** ISO 时间 → "YYYY-MM-DD HH:mm"(本地时区);不可解析原样返回。 */
34
- function formatDateTimeMinute(iso) {
35
- const date = new Date(iso);
36
- if (Number.isNaN(date.valueOf()))
37
- return iso;
38
- const pad = (n) => String(n).padStart(2, "0");
39
- return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}`;
40
- }
41
- // ───────────────────────── ScopeSummary ─────────────────────────
42
- /**
43
- * 一至两行:头行是端到端成功率(官方 MetricCell,不重算)+ experiment/eval/attempt 数 +
44
- * `votes` 选中的那级计票 + 总成本;第二行(有则加)是快照时间窗。
45
- */
46
- export function scopeSummaryText(data, votes, ctx) {
47
- const locale = ctx.locale;
48
- const tally = votes === "attempt" ? data.attemptVerdicts : data.evalVerdicts;
49
- const head = [
50
- `${localeText(locale, "scopeSummary.passRate")} ${cellText(data.endToEndPassRate, locale)}`,
51
- countText(locale, "overview.experiments", data.experiments),
52
- localeText(locale, "overview.evalsCount", { n: data.evals }),
53
- localeText(locale, "overview.attemptsCount", { n: data.attempts }),
54
- verdictTallyText(tally, locale),
55
- `${localeText(locale, "scopeSummary.totalCost")} ${cellText(data.totalCostUSD, locale)}`,
56
- ].join(" · ");
57
- const lines = [head];
58
- if (data.range.latestStartedAt !== null) {
59
- const from = data.range.earliestStartedAt;
60
- const to = data.range.latestStartedAt;
61
- lines.push(from !== null && from !== to
62
- ? localeText(locale, "scopeSummary.range", { from: formatDateTimeMinute(from), to: formatDateTimeMinute(to) })
63
- : formatDateTimeMinute(to));
64
- }
65
- return lines.join("\n");
66
- }
67
- // ───────────────────────── ExperimentComparison ─────────────────────────
68
- /** 单组详情:标题行 + 缩进的散点与实验列表(与 web 面同一份预分区数据)。 */
69
- function comparisonGroupText(group, ctx) {
70
- const inner = Math.max(20, ctx.width - 2);
71
- const innerCtx = {
72
- width: inner,
73
- locale: ctx.locale,
74
- attemptCommand: ctx.attemptCommand,
75
- experimentCommand: ctx.experimentCommand,
76
- render: (node, w) => ctx.render(node, w ?? inner),
77
- };
78
- const body = [scatterText(group.scatter, innerCtx), experimentListText(group.experiments, innerCtx, group.key)]
79
- .filter((block) => block.length > 0)
80
- .join("\n\n");
81
- return `${localeText(ctx.locale, "experimentComparison.group")} ${group.key}\n\n${indentBlock(body, " ")}`;
82
- }
83
- /**
84
- * text 面:命中多个可比组时只输出组索引与可复制的单组查看命令,不倾倒全部组详情;
85
- * 命中单组时省略索引,直接输出该组的散点与实验列表。
86
- */
87
- export function experimentComparisonText(data, _className, ctx) {
88
- const locale = ctx.locale;
89
- if (data.groups.length === 0)
90
- return localeText(locale, "experimentComparison.empty");
91
- if (data.groups.length === 1)
92
- return comparisonGroupText(data.groups[0], ctx);
93
- const columns = [
94
- { key: "group", header: localeText(locale, "experimentComparison.group") },
95
- { key: "experiments", header: localeText(locale, "scopeSummary.experiments"), align: "right" },
96
- { key: "evals", header: localeText(locale, "scopeSummary.evals"), align: "right" },
97
- { key: "passRate", header: localeText(locale, "scopeSummary.passRate"), align: "right" },
98
- { key: "results", header: localeText(locale, "experimentComparison.results") },
99
- { key: "cost", header: localeText(locale, "scopeSummary.totalCost"), align: "right" },
100
- { key: "lastRun", header: localeText(locale, "experimentComparison.lastRun") },
101
- ];
102
- const rows = data.groups.map((group) => ({
103
- key: group.key,
104
- cells: {
105
- group: group.key,
106
- experiments: String(group.summary.experiments),
107
- evals: String(group.summary.evals),
108
- passRate: cellText(group.summary.endToEndPassRate, locale),
109
- results: verdictTallyText(group.summary.evalVerdicts, locale),
110
- cost: group.summary.totalCostUSD.value === null ? null : cellText(group.summary.totalCostUSD, locale),
111
- lastRun: group.summary.range.latestStartedAt === null ? null : formatDateTimeMinute(group.summary.range.latestStartedAt),
112
- },
113
- }));
114
- const table = renderTableText({ columns: columns, rows, locale }, ctx);
115
- const commands = [
116
- localeText(locale, "experimentComparison.commandsHead"),
117
- ...data.groups.map((group) => ` ${ctx.experimentCommand(group.key)}`),
118
- ].join("\n");
119
- return `${table}\n\n${commands}`;
120
- }
121
- // ───────────────────────── MetricTable ─────────────────────────
122
- export function tableText(data, ctx) {
123
- const locale = ctx.locale;
124
- const columns = [
125
- { key: "dimension", header: data.rowDimension },
126
- // 指标列的键用序号,不用 metric name —— 维度名和指标名在同一个命名空间里,可能撞键。
127
- ...data.columns.map((c, i) => ({ key: `metric${i}`, header: resolveMetricLabel(c.label, locale, c.key) })),
128
- ];
129
- const rows = data.rows.map((row) => {
130
- const cells = { dimension: row.key };
131
- data.columns.forEach((col, i) => {
132
- const cell = row.cells[col.key];
133
- cells[`metric${i}`] = cell ? cellText(cell, locale) : null;
134
- });
135
- return { key: row.key, cells };
136
- });
137
- return renderTableText({ columns: columns, rows, locale }, ctx);
138
- }
139
- // ───────────────────────── MetricMatrix ─────────────────────────
140
- export function matrixText(data, ctx) {
141
- // 表体全是维度键与 display,没有 chrome 文案;"next:" 是命令提示,不本地化。
142
- const locale = ctx.locale;
143
- const rowKeys = [];
144
- const columnKeys = [];
145
- const byPosition = new Map();
146
- for (const entry of data.cells) {
147
- if (!rowKeys.includes(entry.row))
148
- rowKeys.push(entry.row);
149
- if (!columnKeys.includes(entry.column))
150
- columnKeys.push(entry.column);
151
- byPosition.set(JSON.stringify([entry.row, entry.column]), entry.cell);
152
- }
153
- const columns = [
154
- { key: "dimension", header: data.rowDimension },
155
- ...columnKeys.map((column, i) => ({ key: `column${i}`, header: column })),
156
- ];
157
- const rows = rowKeys.map((row) => {
158
- const cells = { dimension: row };
159
- columnKeys.forEach((column, i) => {
160
- const cell = byPosition.get(JSON.stringify([row, column]));
161
- cells[`column${i}`] = cell ? cellText(cell, locale) : null; // 稀疏格子在文本里以 — 呈现,不编数
162
- });
163
- return { key: row, cells };
164
- });
165
- const table = renderTableText({ columns: columns, rows, locale }, ctx);
166
- // 下钻命令:行维度是 eval 时,指向最值得看的一行(先挑有缺格的,再挑按 better 最差的)
167
- if (data.rowDimension !== "eval" || rowKeys.length === 0)
168
- return table;
169
- const better = data.metric.better ?? "higher";
170
- let next;
171
- let worst;
172
- for (const row of rowKeys) {
173
- let sum = 0;
174
- let count = 0;
175
- for (const column of columnKeys) {
176
- const cell = byPosition.get(JSON.stringify([row, column]));
177
- if (!cell || cell.value === null) {
178
- next ??= row;
179
- continue;
180
- }
181
- sum += cell.value;
182
- count += 1;
183
- }
184
- if (count > 0) {
185
- const value = sum / count;
186
- const isWorse = worst === undefined || (better === "higher" ? value < worst.value : value > worst.value);
187
- if (isWorse)
188
- worst = { key: row, value };
189
- }
190
- }
191
- next ??= worst?.key;
192
- return next === undefined ? table : `${table}\n\nnext: niceeval show ${next}`;
193
- }
194
- // ───────────────────────── MetricBars(矩阵数据的另一种摆法)─────────────────────────
195
- const BAR_WIDTH = 20;
196
- export function barsText(data, ctx) {
197
- const locale = ctx.locale;
198
- const groupKeys = [];
199
- const seriesKeys = [];
200
- const byPosition = new Map();
201
- for (const entry of data.cells) {
202
- if (!groupKeys.includes(entry.row))
203
- groupKeys.push(entry.row);
204
- if (!seriesKeys.includes(entry.column))
205
- seriesKeys.push(entry.column);
206
- byPosition.set(JSON.stringify([entry.row, entry.column]), entry.cell);
207
- }
208
- const better = data.metric.better ?? "higher";
209
- // 条长刻度:% 的天然域是 [0,1],其余以全图最大值为满条
210
- const values = data.cells.map((c) => c.cell.value).filter((v) => v !== null);
211
- const maxValue = values.length > 0 ? Math.max(...values) : 0;
212
- const ratioOf = (value) => (data.metric.unit === "%" ? value : maxValue === 0 ? 0 : value / maxValue);
213
- const seriesWidth = Math.max(...seriesKeys.map((k) => stringWidth(k)), 0);
214
- const lines = [];
215
- for (const group of groupKeys) {
216
- lines.push(group);
217
- const entries = seriesKeys.map((series) => ({
218
- series,
219
- cell: byPosition.get(JSON.stringify([group, series])),
220
- }));
221
- // 组内按值排序,方向随 better(缺数据沉底)
222
- entries.sort((a, b) => {
223
- const va = a.cell?.value ?? null;
224
- const vb = b.cell?.value ?? null;
225
- if (va === null && vb === null)
226
- return 0;
227
- if (va === null)
228
- return 1;
229
- if (vb === null)
230
- return -1;
231
- return better === "lower" ? va - vb : vb - va;
232
- });
233
- for (const { series, cell } of entries) {
234
- const label = padDisplay(series, seriesWidth);
235
- if (!cell || cell.value === null) {
236
- lines.push(` ${label} ${MISSING_MARK}`);
237
- continue;
238
- }
239
- lines.push(` ${label} ${textBar(ratioOf(cell.value), BAR_WIDTH)} ${cellText(cell, locale)}`);
240
- }
241
- }
242
- return lines.join("\n");
243
- }
244
- // ───────────────────────── Scoreboard ─────────────────────────
245
- export function scoreboardText(data, ctx) {
246
- const locale = ctx.locale;
247
- const subjectKeys = [];
248
- for (const row of data.rows) {
249
- for (const subject of row.subjects) {
250
- if (!subjectKeys.includes(subject.key))
251
- subjectKeys.push(subject.key);
252
- }
253
- }
254
- const columns = [
255
- { key: "dimension", header: data.rowDimension },
256
- { key: "total", header: localeText(locale, "scoreboard.totalText") },
257
- ...subjectKeys.map((key, i) => ({ key: `subject${i}`, header: key })),
258
- ];
259
- const rows = data.rows.map((row) => {
260
- const totalNotes = [
261
- ...(row.total.notRun > 0 ? [localeText(locale, "scoreboard.notRunText", { n: row.total.notRun })] : []),
262
- ...(row.total.unscorable > 0 ? [localeText(locale, "scoreboard.unscorableText", { n: row.total.unscorable })] : []),
263
- ];
264
- const cells = {
265
- dimension: row.key,
266
- total: `${resolveLocalizedText(row.total.display, locale)}/${data.fullMarks}${totalNotes.length > 0 ? ` ${totalNotes.join(" ")}` : ""}`,
267
- };
268
- subjectKeys.forEach((key, i) => {
269
- const subject = row.subjects.find((s) => s.key === key);
270
- if (!subject) {
271
- cells[`subject${i}`] = null;
272
- return;
273
- }
274
- const score = `${formatPlainNumber(subject.earned)}/${formatPlainNumber(subject.possible)}`;
275
- const notes = [
276
- ...(subject.notRun > 0 ? [localeText(locale, "scoreboard.notRunText", { n: subject.notRun })] : []),
277
- ...(subject.unscorable > 0 ? [localeText(locale, "scoreboard.unscorableText", { n: subject.unscorable })] : []),
278
- ];
279
- cells[`subject${i}`] = notes.length > 0 ? `${score} ${notes.join(" ")}` : score;
280
- });
281
- return { key: row.key, cells };
282
- });
283
- const table = renderTableText({ columns: columns, rows, locale }, ctx);
284
- const footnotes = [];
285
- if (data.weights.length > 0) {
286
- const weights = data.weights.map((w) => `${w.prefix} ×${w.weight}`).join(" · ");
287
- footnotes.push(`${localeText(locale, "scoreboard.weights")} ${weights} · ${localeText(locale, "scoreboard.othersWeight")}`);
288
- }
289
- if (data.ignoredEvals > 0)
290
- footnotes.push(countText(locale, "scoreboard.ignored", data.ignoredEvals));
291
- return footnotes.length > 0 ? `${table}\n${footnotes.join("\n")}` : table;
292
- }
293
- // ───────────────────────── MetricScatter ─────────────────────────
294
- const POINT_MARKS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
295
- function axisLabel(col, locale) {
296
- return resolveMetricLabel(col.label, locale, col.key);
297
- }
298
- /**
299
- * 方向提示:轴向已随 better 反正(lower 反向),「更好」恒指向右上,文案恒为
300
- * 「越靠右上越好」;仅当 x、y 都声明 better 时显示——任一轴未声明,组件不猜
301
- * 「更好」朝哪边,整图无提示(返回 undefined)。
302
- */
303
- function betterCornerText(x, y, locale) {
304
- if (x.better === undefined || y.better === undefined)
305
- return undefined;
306
- return localeText(locale, "scatter.betterUpperRight");
307
- }
308
- /** 轴标题行:"<x label>(越低越好) × <y label>"——better 方向如实标注,不虚构轴向。 */
309
- function scatterHeading(data, locale) {
310
- const withBetter = (col) => {
311
- const label = axisLabel(col, locale);
312
- if (col.better === undefined)
313
- return label;
314
- return `${label}(${localeText(locale, col.better === "lower" ? "table.lowerBetter" : "table.higherBetter")})`;
315
- };
316
- return `${withBetter(data.x)} × ${withBetter(data.y)}`;
317
- }
318
- export function scatterText(data, ctx) {
319
- const locale = ctx.locale;
320
- const drawable = data.rows.filter((r) => r.x.value !== null && r.y.value !== null);
321
- const missing = data.rows.length - drawable.length;
322
- const footnotes = [];
323
- if (missing > 0)
324
- footnotes.push(countText(locale, "pointsMissing", missing));
325
- const axes = { x: axisLabel(data.x, locale), y: axisLabel(data.y, locale) };
326
- // 0 个可画点:x/y 指标没有可用数据(与 web 面同一事实)。
327
- if (drawable.length === 0) {
328
- return [localeText(locale, "scatter.noData", axes), ...footnotes].join("\n");
329
- }
330
- // 点太密排不下时降级为坐标表,不硬挤
331
- if (drawable.length > POINT_MARKS.length || ctx.width < 44) {
332
- const table = renderCoordinateTable(drawable.map((r) => ({
333
- key: r.key,
334
- x: resolveLocalizedText(r.x.display, locale),
335
- y: resolveLocalizedText(r.y.display, locale),
336
- })), { key: data.pointDimension, x: axes.x, y: axes.y });
337
- return [scatterHeading(data, locale), "", table, ...footnotes].join("\n");
338
- }
339
- const points = drawable.map((r, i) => ({
340
- mark: POINT_MARKS[i],
341
- x: r.x.value,
342
- y: r.y.value,
343
- }));
344
- // 同系列的点按 x 排序连线
345
- const bySeries = new Map();
346
- for (const r of drawable) {
347
- if (r.series === undefined)
348
- continue;
349
- const list = bySeries.get(r.series) ?? [];
350
- list.push({ x: r.x.value, y: r.y.value });
351
- bySeries.set(r.series, list);
352
- }
353
- for (const list of bySeries.values())
354
- list.sort((a, b) => a.x - b.x);
355
- const plot = renderCharPlot({
356
- width: ctx.width,
357
- points,
358
- lines: [...bySeries.values()].filter((l) => l.length > 1),
359
- xLabel: axes.x,
360
- yLabel: axes.y,
361
- formatX: (v) => formatMetricValue(v, data.x.unit),
362
- formatY: (v) => formatMetricValue(v, data.y.unit),
363
- // 轴方向跟随 better(与 web 面同规则):lower 反向,higher 与未声明正向。
364
- invertX: data.x.better === "lower",
365
- invertY: data.y.better === "lower",
366
- });
367
- const legend = drawable.map((r, i) => `${POINT_MARKS[i]} ${r.key}`).join(" ");
368
- const hint = betterCornerText(data.x, data.y, locale);
369
- return [scatterHeading(data, locale), plot, "", ...(hint !== undefined ? [hint] : []), legend, ...footnotes].join("\n");
370
- }
371
- // ───────────────────────── MetricLine ─────────────────────────
372
- export function lineText(data, ctx) {
373
- const locale = ctx.locale;
374
- const drawable = data.rows.filter((r) => r.x !== null && r.y.value !== null);
375
- const missing = data.rows.length - drawable.length;
376
- const footnotes = [];
377
- if (missing > 0)
378
- footnotes.push(countText(locale, "pointsMissing", missing));
379
- if (drawable.length === 0)
380
- return [missingText(locale), ...footnotes].join("\n");
381
- const xLabel = resolveMetricLabel(data.x.label, locale, data.x.key);
382
- // 系列 → 字母;无系列 = 单系列
383
- const seriesKeys = [];
384
- for (const r of drawable) {
385
- const key = r.series ?? "";
386
- if (!seriesKeys.includes(key))
387
- seriesKeys.push(key);
388
- }
389
- if (seriesKeys.length > POINT_MARKS.length || ctx.width < 44) {
390
- const table = renderCoordinateTable(drawable.map((r) => ({
391
- key: r.series !== undefined ? `${r.series} (${r.key})` : r.key,
392
- x: resolveLocalizedText(r.xDisplay, locale),
393
- y: resolveLocalizedText(r.y.display, locale),
394
- })), { key: data.seriesDimension ?? xLabel, x: xLabel, y: axisLabel(data.y, locale) });
395
- return [table, ...footnotes].join("\n");
396
- }
397
- const markOf = (r) => POINT_MARKS[seriesKeys.indexOf(r.series ?? "")];
398
- const points = drawable.map((r) => ({
399
- mark: markOf(r),
400
- x: r.x,
401
- y: r.y.value,
402
- }));
403
- const lines = seriesKeys.map((key) => drawable
404
- .filter((r) => (r.series ?? "") === key)
405
- .map((r) => ({ x: r.x, y: r.y.value }))
406
- .sort((a, b) => a.x - b.x));
407
- const plot = renderCharPlot({
408
- width: ctx.width,
409
- points,
410
- lines: lines.filter((l) => l.length > 1),
411
- xLabel,
412
- yLabel: axisLabel(data.y, locale),
413
- formatX: (v) => formatMetricValue(v, data.x.unit),
414
- formatY: (v) => formatMetricValue(v, data.y.unit),
415
- });
416
- const legend = seriesKeys
417
- .map((key, i) => `${POINT_MARKS[i]} ${key === "" ? axisLabel(data.y, locale) : key}`)
418
- .join(" ");
419
- return [plot, "", legend, ...footnotes].join("\n");
420
- }
421
- // ───────────────────────── DeltaTable ─────────────────────────
422
- export function deltaText(data, ctx) {
423
- const locale = ctx.locale;
424
- // 0 对不是错误:明确空态并报告配对域实验数(派生形态携带;字面形态缺省按 0)。
425
- if (data.rows.length === 0) {
426
- return localeText(locale, "delta.empty", { experiments: data.experiments ?? 0 });
427
- }
428
- const columns = [
429
- { key: "pair", header: localeText(locale, "delta.pairHeader") },
430
- ...data.columns.map((c, i) => ({ key: `metric${i}`, header: resolveMetricLabel(c.label, locale, c.key) })),
431
- ];
432
- const rows = data.rows.map((row) => {
433
- const cells = { pair: resolveLocalizedText(row.label, locale) };
434
- data.columns.forEach((col, i) => {
435
- const cell = row.cells[col.key];
436
- if (!cell) {
437
- cells[`metric${i}`] = null;
438
- return;
439
- }
440
- const a = cell.a.value === null ? MISSING_MARK : resolveLocalizedText(cell.a.display, locale);
441
- const b = cell.b.value === null ? MISSING_MARK : resolveLocalizedText(cell.b.display, locale);
442
- cells[`metric${i}`] = `${a} → ${b} ${resolveLocalizedText(cell.display, locale)}`;
443
- });
444
- return { key: row.key, cells };
445
- });
446
- return renderTableText({ columns: columns, rows, locale }, ctx);
447
- }
448
- // ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
449
- //
450
- // 三面共用的紧凑标记:`locator✓`(判定符紧跟 locator,中间不留空格)。
451
- // ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的摘要,不重复整段
452
- // niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼 `niceeval show <locator>`。
453
- function locatorBadge(item) {
454
- return `${item.locator}${verdictMark(item.verdict)}`;
455
- }
456
- /**
457
- * failureSummary + moreFailures 的展示形态:摘要在计算侧已按 Scoring display 契约折好,
458
- * 这里只加 "+N more failures" 计数与宽度收口,不重算摘要。
459
- */
460
- function attemptReasonText(item, locale, maxChars) {
461
- if (item.failureSummary === null)
462
- return undefined;
463
- const withMore = item.moreFailures > 0
464
- ? `${item.failureSummary} · ${countText(locale, "entityList.moreFailures", item.moreFailures)}`
465
- : item.failureSummary;
466
- return fitFailureSummary(withMore, Math.max(24, maxChars));
467
- }
468
- // ── ExperimentList ──
469
- function experimentSummaryTable(items, ctx, relativeTo) {
470
- const locale = ctx.locale;
471
- const compact = ctx.width < 100;
472
- const columns = [
473
- { key: "experiment", header: compact && locale === "en" ? "Exp." : localeText(locale, "experimentList.experiment") },
474
- { key: "model", header: localeText(locale, "table.model") },
475
- { key: "agent", header: localeText(locale, "table.agent") },
476
- { key: "duration", header: compact && locale === "en" ? "Avg" : localeText(locale, "experimentList.avgDuration"), align: "right" },
477
- { key: "passRate", header: compact && locale === "en" ? "Pass" : localeText(locale, "experimentList.passRate"), align: "right" },
478
- { key: "result", header: localeText(locale, "experimentList.result") },
479
- { key: "tokens", header: localeText(locale, "experimentList.tokens"), align: "right" },
480
- { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
481
- ];
482
- const rows = items.map((item) => ({
483
- key: item.experimentId,
484
- cells: {
485
- experiment: experimentDisplayName(item.experimentId, relativeTo),
486
- model: item.model ?? localeText(locale, "experimentList.defaultModel"),
487
- agent: item.agent,
488
- duration: cellText(item.durationMs, locale),
489
- passRate: cellText(item.endToEndPassRate, locale),
490
- result: verdictTallyText(item.evalVerdicts, locale),
491
- tokens: cellText(item.tokens, locale),
492
- cost: cellText(item.costUSD, locale),
493
- },
494
- }));
495
- const metadata = items.flatMap((item) => wrapDisplay(`${experimentDisplayName(item.experimentId, relativeTo)}: ${localeText(locale, "overview.evalsCount", { n: item.evals })} · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })} · ${item.lastRunAt}`, Math.max(8, ctx.width - 2)).map((line) => ` ${line}`));
496
- return [renderTableText({ columns: columns, rows, locale }, ctx), metadata.join("\n")].join("\n");
497
- }
498
- function experimentDetailTable(item, ctx, relativeTo) {
499
- const locale = ctx.locale;
500
- const columns = [
501
- { key: "status", header: localeText(locale, "experimentList.status") },
502
- { key: "entity", header: localeText(locale, "experimentList.evalAttempt") },
503
- // Result 是可扫读的失败预览,不是证据面:两行放不下的以 … 收口,完整值走 locator 下钻。
504
- { key: "result", header: localeText(locale, "experimentList.result"), maxLines: 2 },
505
- { key: "duration", header: localeText(locale, "experimentList.duration"), align: "right" },
506
- { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
507
- ];
508
- // Result 的字符预算 ≈ 两行 × 它能分到的列宽(总宽减其它列的自然宽与列距)。这里只做
509
- // 粗预算;精确的按宽度收口由列的 maxLines 兜底。
510
- const statusWidth = Math.max(stringWidth(localeText(locale, "experimentList.status")), ...item.evalRows.map((row) => stringWidth(`${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`)));
511
- const entityWidth = Math.max(stringWidth(localeText(locale, "experimentList.evalAttempt")), ...item.evalRows.flatMap((row) => [stringWidth(row.evalId), ...row.attempts.map((a) => stringWidth(a.locator) + 3)]));
512
- const fixedWidth = statusWidth + entityWidth + 8 /* duration */ + 6 /* cost */ + 3 * 4; /* 4 段列距 */
513
- const resultBudget = Math.max(24, (ctx.width - fixedWidth) * 2);
514
- const rows = item.evalRows.flatMap((row) => {
515
- // Eval 父行只承载折叠判定与题级聚合;失败摘要只在 Attempt 子行出现。
516
- const parent = {
517
- key: row.evalId,
518
- cells: {
519
- status: `${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`,
520
- entity: row.evalId,
521
- result: "",
522
- duration: localeText(locale, "entityList.average", { value: cellText(row.durationMs, locale) }),
523
- cost: localeText(locale, "entityList.average", { value: cellText(row.costUSD, locale) }),
524
- },
525
- };
526
- const attempts = row.attempts.map((attempt, index) => ({
527
- key: attempt.locator,
528
- cells: {
529
- status: ` ${verdictMark(attempt.verdict)}`,
530
- entity: `${index === row.attempts.length - 1 ? "└─" : "├─"} ${attempt.locator}`,
531
- result: attemptReasonText(attempt, locale, resultBudget) ?? MISSING_MARK,
532
- duration: attempt.verdict === "skipped" && attempt.durationMs === 0 ? null : formatDurationMs(attempt.durationMs),
533
- cost: attempt.costUSD === null ? null : formatUSD(attempt.costUSD),
534
- },
535
- }));
536
- return [parent, ...attempts];
537
- });
538
- const flags = item.flags && Object.keys(item.flags).length > 0
539
- ? `${localeText(locale, "experimentList.flags")} ${Object.entries(item.flags)
540
- .map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`)
541
- .join(" · ")}`
542
- : undefined;
543
- return [
544
- experimentDisplayName(item.experimentId, relativeTo),
545
- flags,
546
- renderTableText({ columns: columns, rows, locale }, ctx),
547
- ]
548
- .filter(Boolean)
549
- .join("\n");
550
- }
551
- export function experimentListText(items, ctx, relativeTo) {
552
- if (items.length === 0)
553
- return localeText(ctx.locale, "attemptList.empty");
554
- return [
555
- experimentSummaryTable(items, ctx, relativeTo),
556
- ...items.map((item) => experimentDetailTable(item, ctx, relativeTo)),
557
- ].join("\n\n");
558
- }
559
- // ── EvalList ──
560
- function evalListAttemptLine(item, ctx) {
561
- // 行式列表同守「Result 最多两行」:预算 = 两行终端宽,超出按尾截收口。
562
- const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - stringWidth(locatorBadge(item)) - 6);
563
- return ` ${locatorBadge(item)}${reason ? ` · ${reason}` : ""}`;
564
- }
565
- export function evalListText(items, ctx) {
566
- const locale = ctx.locale;
567
- if (items.length === 0)
568
- return localeText(locale, "attemptList.empty");
569
- const blocks = items.map((item) => {
570
- const identity = `${item.evalId} · ${item.experimentId} · ${localeText(locale, `verdict.${item.verdict}`)}`;
571
- const summary = [
572
- localeText(locale, "attemptList.score", { score: cellText(item.examScore, locale) }),
573
- localeText(locale, "overview.attemptsCount", { n: item.attempts.length }),
574
- localeText(locale, "entityList.average", {
575
- value: item.durationMs.value === null ? missingText(locale) : formatDurationMs(item.durationMs.value),
576
- }),
577
- localeText(locale, "entityList.average", {
578
- value: item.costUSD.value === null ? missingText(locale) : formatUSD(item.costUSD.value),
579
- }),
580
- ].join(" · ");
581
- const attemptLines = item.attempts.map((attempt) => evalListAttemptLine(attempt, ctx));
582
- return [identity, ` ${summary}`, ...attemptLines].join("\n");
583
- });
584
- return blocks.join("\n\n");
585
- }
586
- // ── AttemptList ──
587
- /** Attempt 比较卡片:只显示一条主失败摘要(至多两行终端宽);完整 assertions 走 locator 下钻。 */
588
- function attemptListItemText(item, ctx) {
589
- const head = [
590
- `${verdictMark(item.verdict)} ${item.locator}`,
591
- item.evalId,
592
- item.experimentId,
593
- formatDurationMs(item.durationMs),
594
- ...(item.costUSD !== null ? [formatUSD(item.costUSD)] : []),
595
- ].join(" · ");
596
- const lines = [head];
597
- const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - 4);
598
- if (reason)
599
- lines.push(` ${reason}`);
600
- return lines.join("\n");
601
- }
602
- export function attemptListText(items, total, ctx) {
603
- const locale = ctx.locale;
604
- if (items.length === 0)
605
- return localeText(locale, "attemptList.empty");
606
- const blocks = items.map((item) => attemptListItemText(item, ctx));
607
- const remaining = (total ?? items.length) - items.length;
608
- if (remaining > 0)
609
- blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
610
- return blocks.join("\n\n");
611
- }
612
- // ───────────────────────── 站点组件(HeroCard / ScopeWarnings / TraceWaterfall)─────────────────────────
613
- /**
614
- * HeroCard 的 text 面:标题行 + meta 行(最后运行时间;空范围为内置「暂无运行」文案;
615
- * 多快照时标注合成来源),不含品牌行(品牌行是纯 web 件,text 面零输出)。
616
- */
617
- export function heroCardText(title, data, ctx) {
618
- const locale = ctx.locale;
619
- const meta = data.latestStartedAt === null
620
- ? localeText(locale, "hero.noRuns")
621
- : [
622
- localeText(locale, "hero.lastRun", { time: formatDateTimeMinute(data.latestStartedAt) }),
623
- ...(data.snapshots > 1 ? [localeText(locale, "hero.composedSnapshots", { n: data.snapshots })] : []),
624
- ].join(" · ");
625
- return `${resolveLocalizedText(title, locale)}\n${meta}`;
626
- }
627
- /**
628
- * ScopeWarnings 的 text 面:按动作聚合(../scope-warnings.ts,与 web 面共用),同构但不折叠——
629
- * 多组时首行 "! <分类计数汇总>";每组一行组头 "! <标题> — <徽标> → <组头命令>",其下缩进
630
- * 逐条原样打印 message(已以下一步收尾,不截断掉尾段)。空警告集零输出。
631
- */
632
- export function scopeWarningsText(warnings, ctx) {
633
- if (warnings.length === 0)
634
- return "";
635
- const { summary, groups } = groupScopeWarnings(warnings, ctx.locale);
636
- const lines = [];
637
- // 汇总行只在多组时打印;单组时组头即汇总,不另起一行(web 面则恒以汇总行作外层 <summary>)。
638
- if (groups.length > 1)
639
- lines.push(`! ${summary}`);
640
- for (const group of groups) {
641
- const badges = group.badges.length > 0 ? ` — ${group.badges.map((b) => b.text).join(" · ")}` : "";
642
- const command = group.headCommand !== null ? ` → ${group.headCommand}` : "";
643
- lines.push(`! ${group.title}${badges}${command}`);
644
- for (const w of group.warnings)
645
- lines.push(`! ${w.message}`);
646
- }
647
- return lines.join("\n");
648
- }
649
- /**
650
- * TraceWaterfall 的 text 面:每 attempt 一行——locator、总耗时(缺 trace 如实显示缺失)、
651
- * 顶层 span 计数与失败标记,行尾是可复制的 `--timing` 下钻命令(经宿主注入的
652
- * attemptCommand 通道拼出,携带宿主上下文)。attempt 有选择器,索引终结于可执行命令。
653
- */
654
- export function traceWaterfallText(rows, ctx) {
655
- const locale = ctx.locale;
656
- if (rows.length === 0)
657
- return localeText(locale, "traceWaterfall.empty");
658
- return rows
659
- .map((row) => {
660
- const failedSpans = row.spans.filter((span) => span.failed).length;
661
- const parts = [
662
- row.locator,
663
- row.evalId,
664
- row.durationMs === null ? localeText(locale, "traceWaterfall.noTrace") : formatDurationMs(row.durationMs),
665
- countText(locale, "traceWaterfall.spans", row.spans.length),
666
- ...(failedSpans > 0 ? [`✗ ${countText(locale, "traceWaterfall.failedSpans", failedSpans)}`] : []),
667
- ];
668
- return `${parts.join(" · ")} ${ctx.attemptCommand(row.locator)} --timing`;
669
- })
670
- .join("\n");
671
- }