niceeval 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +15 -7
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
  118. package/dist/report/{tree.js → definition/tree.js} +10 -12
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +12 -2
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +15 -1
  154. package/dist/shared/aggregate.js +32 -1
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +77 -64
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +30 -15
  227. package/src/i18n/zh-CN.ts +30 -15
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +136 -112
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +58 -22
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +54 -3
  314. package/src/results/select.ts +30 -6
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +30 -1
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +58 -22
  356. package/src/show/render.ts +16 -240
  357. package/src/show/show.test.ts +128 -35
  358. package/src/view/app/App.test.tsx +8 -7
  359. package/src/view/app/App.tsx +135 -51
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +122 -89
  370. package/src/view/index.ts +3 -13
  371. package/src/view/server.ts +36 -6
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +18 -806
  376. package/src/view/view-report.test.ts +182 -34
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -0,0 +1,80 @@
1
+ // 站点组件族(HeroCard / ScopeWarnings / TraceWaterfall)的 text 面:同一份算好的数据,
2
+ // 渲染成终端字符(niceeval show 的形态)。零 react、零 IO、纯同步。
3
+
4
+ import type { HeroData, ScopeWarning, TraceWaterfallRow } from "../../model/types.ts";
5
+ import type { LocalizedText } from "../../model/locale.ts";
6
+ import type { TextContext } from "../../definition/tree.ts";
7
+ import { countText, localeText, resolveLocalizedText } from "../../model/locale.ts";
8
+ import { formatDurationMs } from "../../model/format.ts";
9
+ import { groupScopeWarnings } from "./scope-warnings.ts";
10
+
11
+ /** ISO 时间 → "YYYY-MM-DD HH:mm"(本地时区);不可解析原样返回。 */
12
+ function formatDateTimeMinute(iso: string): string {
13
+ const date = new Date(iso);
14
+ if (Number.isNaN(date.valueOf())) return iso;
15
+ const pad = (n: number) => String(n).padStart(2, "0");
16
+ return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}`;
17
+ }
18
+
19
+ // ───────────────────────── 站点组件(HeroCard / ScopeWarnings / TraceWaterfall)─────────────────────────
20
+
21
+ /**
22
+ * HeroCard 的 text 面:标题行 + meta 行(最后运行时间;空范围为内置「暂无运行」文案;
23
+ * 多快照时标注合成来源),不含品牌行(品牌行是纯 web 件,text 面零输出)。
24
+ */
25
+ export function heroCardText(title: LocalizedText, data: HeroData, ctx: TextContext): string {
26
+ const locale = ctx.locale;
27
+ const meta =
28
+ data.latestStartedAt === null
29
+ ? localeText(locale, "hero.noRuns")
30
+ : [
31
+ localeText(locale, "hero.lastRun", { time: formatDateTimeMinute(data.latestStartedAt) }),
32
+ ...(data.snapshots > 1 ? [localeText(locale, "hero.composedSnapshots", { n: data.snapshots })] : []),
33
+ ].join(" · ");
34
+ return `${resolveLocalizedText(title, locale)}\n${meta}`;
35
+ }
36
+
37
+ /**
38
+ * ScopeWarnings 的 text 面:按动作聚合(../scope-warnings.ts,与 web 面共用),同构但不折叠——
39
+ * 多组时首行 "! <分类计数汇总>";每组一行组头 "! <标题> — <徽标> → <组头命令>",其下缩进
40
+ * 逐条原样打印 message(已以下一步收尾,不截断掉尾段)。空警告集零输出。
41
+ */
42
+ export function scopeWarningsText(warnings: readonly ScopeWarning[], ctx: TextContext): string {
43
+ if (warnings.length === 0) return "";
44
+ const { summary, groups } = groupScopeWarnings(warnings, ctx.locale);
45
+ const lines: string[] = [];
46
+ // 汇总行只在多组时打印;单组时组头即汇总,不另起一行(web 面则恒以汇总行作外层 <summary>)。
47
+ if (groups.length > 1) lines.push(`! ${summary}`);
48
+ for (const group of groups) {
49
+ const badges = group.badges.length > 0 ? ` — ${group.badges.map((b) => b.text).join(" · ")}` : "";
50
+ const command = group.headCommand !== null ? ` → ${group.headCommand}` : "";
51
+ lines.push(`! ${group.title}${badges}${command}`);
52
+ for (const w of group.warnings) lines.push(`! ${w.message}`);
53
+ }
54
+ return lines.join("\n");
55
+ }
56
+
57
+ /**
58
+ * TraceWaterfall 的 text 面:每 attempt 一行——locator、总耗时(缺 trace 如实显示缺失)、
59
+ * 顶层 span 计数与失败标记,行尾是可复制的 `--timing` 下钻命令(经宿主注入的 attemptCommand
60
+ * 通道拼出,携带宿主上下文)。当前报告没有 attempt-input page 时 `ctx.attemptCommand`
61
+ * 不存在,行退化为纯文本,不生成假命令(architecture.md「Attempt 详情是一张参数化 page」)。
62
+ */
63
+ export function traceWaterfallText(rows: readonly TraceWaterfallRow[], ctx: TextContext): string {
64
+ const locale = ctx.locale;
65
+ if (rows.length === 0) return localeText(locale, "traceWaterfall.empty");
66
+ return rows
67
+ .map((row) => {
68
+ const failedSpans = row.spans.filter((span) => span.failed).length;
69
+ const parts = [
70
+ row.locator,
71
+ row.evalId,
72
+ row.durationMs === null ? localeText(locale, "traceWaterfall.noTrace") : formatDurationMs(row.durationMs),
73
+ countText(locale, "traceWaterfall.spans", row.spans.length),
74
+ ...(failedSpans > 0 ? [`✗ ${countText(locale, "traceWaterfall.failedSpans", failedSpans)}`] : []),
75
+ ];
76
+ const line = parts.join(" · ");
77
+ return ctx.attemptCommand ? `${line} ${ctx.attemptCommand(row.locator)} --timing` : line;
78
+ })
79
+ .join("\n");
80
+ }
@@ -0,0 +1,265 @@
1
+ // 官方双面组件的装配点:web 面(./HeroCard.tsx 等纯 React 组件)+ text 面(./faces.ts)+
2
+ // resolve 解析面(spec 形态由管线代调配套 ./compute.ts)。Hero 是组合组件(装配 HeroCard),
3
+ // 不产生自己的 data;PoweredBy 无 props,没有对应的计算函数或 validate。
4
+
5
+ import { defineComponent, type ReportComponent } from "../../definition/tree.ts";
6
+ import type { CopyFixPromptData, HeroData, ScopeWarning, TraceWaterfallRow } from "../../model/types.ts";
7
+ import type { LocalizedText } from "../../model/locale.ts";
8
+ import type { AttemptLocator } from "../../../results/locator.ts";
9
+ import {
10
+ arrayProblem,
11
+ dataShapeError,
12
+ isObject,
13
+ makeDataComponent,
14
+ hrefOf,
15
+ type ChromeProps,
16
+ type DataProps,
17
+ type Validator,
18
+ } from "../shared.ts";
19
+ import { copyFixPromptData, heroData, scopeWarningsData, traceWaterfallData } from "./compute.ts";
20
+ import { heroCardText, scopeWarningsText, traceWaterfallText } from "./faces.ts";
21
+ import { HeroCard as HeroCardWeb } from "./HeroCard.tsx";
22
+ import { PoweredBy as PoweredByWeb } from "./PoweredBy.tsx";
23
+ import { ScopeWarnings as ScopeWarningsWeb } from "./ScopeWarnings.tsx";
24
+ import { CopyFixPrompt as CopyFixPromptWeb } from "./CopyFixPrompt.tsx";
25
+ import { TraceWaterfall as TraceWaterfallWeb } from "./TraceWaterfall.tsx";
26
+
27
+ export const validateHeroData: Validator = (data) => {
28
+ if (!isObject(data)) return "expected an object";
29
+ if (!(data.latestStartedAt === null || typeof data.latestStartedAt === "string")) {
30
+ return '"latestStartedAt" must be a string or null';
31
+ }
32
+ if (typeof data.snapshots !== "number") return '"snapshots" must be a number';
33
+ return null;
34
+ };
35
+
36
+ const UNREADABLE_SNAPSHOT_REASONS = ["incompatible-version", "malformed", "incomplete"];
37
+
38
+ /**
39
+ * ScopeWarning(src/results/types.ts「警告 kind 全集」):按 `kind` 判别的联合,已登记 kind
40
+ * 各自的必填字段单独校验——`unreadable-snapshot` 没有 experimentId(非实验作用域)、
41
+ * `command` 恒可选。未登记的 `kind`(如未来版本新增的 warning)只要求 `kind` / `message`
42
+ * 这份两族共用的最小形状,不拒绝——`ScopeWarnings` 的分组渲染对未识别 kind 有专门的
43
+ * 单独成组回退(message 原样、按 integrity 归位),这条前向兼容路径本身就是契约的一部分,
44
+ * 结构校验不能比渲染逻辑更严。
45
+ */
46
+ function scopeWarningProblem(value: unknown, path: string): string | null {
47
+ if (!isObject(value)) return `"${path}" must be a ScopeWarning object`;
48
+ if (typeof value.kind !== "string") return `"${path}.kind" must be a string`;
49
+ if (typeof value.message !== "string") return `"${path}.message" must be a string`;
50
+ switch (value.kind) {
51
+ case "partial-coverage":
52
+ if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
53
+ if (typeof value.covered !== "number") return `"${path}.covered" must be a number`;
54
+ if (typeof value.total !== "number") return `"${path}.total" must be a number`;
55
+ if (typeof value.command !== "string") return `"${path}.command" must be a string`;
56
+ return null;
57
+ case "stale-snapshot":
58
+ if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
59
+ if (typeof value.startedAt !== "string") return `"${path}.startedAt" must be a string`;
60
+ if (typeof value.latestStartedAt !== "string") return `"${path}.latestStartedAt" must be a string`;
61
+ if (typeof value.command !== "string") return `"${path}.command" must be a string`;
62
+ return null;
63
+ case "unfinished-snapshot":
64
+ if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
65
+ if (typeof value.startedAt !== "string") return `"${path}.startedAt" must be a string`;
66
+ if (typeof value.dir !== "string") return `"${path}.dir" must be a string`;
67
+ if (typeof value.command !== "string") return `"${path}.command" must be a string`;
68
+ return null;
69
+ case "unreadable-snapshot":
70
+ if (typeof value.dir !== "string") return `"${path}.dir" must be a string`;
71
+ if (typeof value.reason !== "string" || !UNREADABLE_SNAPSHOT_REASONS.includes(value.reason)) {
72
+ return `"${path}.reason" must be one of ${JSON.stringify(UNREADABLE_SNAPSHOT_REASONS)}`;
73
+ }
74
+ if (value.command !== undefined && typeof value.command !== "string") {
75
+ return `"${path}.command" must be a string when present`;
76
+ }
77
+ return null;
78
+ default:
79
+ return null;
80
+ }
81
+ }
82
+
83
+ export const validateScopeWarningsData: Validator = (data) => arrayProblem(data, "data", scopeWarningProblem);
84
+
85
+ export const validateCopyFixPromptData: Validator = (data) => {
86
+ if (!isObject(data)) return "expected an object";
87
+ if (typeof data.prompt !== "string") return '"prompt" must be a string';
88
+ if (typeof data.failures !== "number") return '"failures" must be a number';
89
+ return null;
90
+ };
91
+
92
+ const TRACE_SPAN_KINDS = ["agent", "model", "tool", "other"];
93
+
94
+ /** TraceSpanSummary(src/report/model/types.ts):瀑布行里的一个顶层 span 摘要。 */
95
+ function traceSpanSummaryProblem(value: unknown, path: string): string | null {
96
+ if (!isObject(value)) return `"${path}" must be a TraceSpanSummary { name, kind, startOffsetMs, durationMs, failed }`;
97
+ if (typeof value.name !== "string") return `"${path}.name" must be a string`;
98
+ if (typeof value.kind !== "string" || !TRACE_SPAN_KINDS.includes(value.kind)) {
99
+ return `"${path}.kind" must be one of ${JSON.stringify(TRACE_SPAN_KINDS)}`;
100
+ }
101
+ if (typeof value.startOffsetMs !== "number") return `"${path}.startOffsetMs" must be a number`;
102
+ if (typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
103
+ if (typeof value.failed !== "boolean") return `"${path}.failed" must be a boolean`;
104
+ return null;
105
+ }
106
+
107
+ export const validateTraceWaterfallData: Validator = (data) =>
108
+ arrayProblem(data, "data", (row, path) => {
109
+ if (!isObject(row)) return `"${path}" must be an object`;
110
+ if (typeof row.experimentId !== "string") return `"${path}.experimentId" must be a string`;
111
+ if (typeof row.evalId !== "string") return `"${path}.evalId" must be a string`;
112
+ if (typeof row.locator !== "string") return `"${path}.locator" must be a string`;
113
+ if (!(row.durationMs === null || typeof row.durationMs === "number")) {
114
+ return `"${path}.durationMs" must be a number or null`;
115
+ }
116
+ return arrayProblem(row.spans, `${path}.spans`, traceSpanSummaryProblem);
117
+ });
118
+
119
+ // ───────────────────────── 站点组件(Hero / PoweredBy / ScopeWarnings / CopyFixPrompt / TraceWaterfall)─────────────────────────
120
+
121
+ /** `Hero` 的 props:标题缺省取 `ctx.report.title`(回退链后的站点标题)。 */
122
+ export interface HeroProps {
123
+ /** 覆盖标题;省略时取 ctx.report.title(回退链后的站点标题)。 */
124
+ title?: LocalizedText;
125
+ className?: string;
126
+ }
127
+
128
+ /** HeroCard 的 data 校验入口(它不经 makeDataComponent,数据形态是唯一形态)。 */
129
+ const assertHeroData = (data: unknown): HeroData => {
130
+ const problem = validateHeroData(data);
131
+ if (problem !== null) throw dataShapeError("HeroCard", "heroData", "HeroData", problem);
132
+ return data as HeroData;
133
+ };
134
+
135
+ /**
136
+ * `HeroCard`:Hero 的渲染件,双面组件,只收 data 形态——标题输入是站点声明与 Scope 的
137
+ * 合成物,没有单独的 spec 等价形。web 面渲染 hero 标题(h1)、按渲染 locale 格式化的运行
138
+ * meta(latestStartedAt 为 null 时内置「暂无运行」文案)与品牌行(等同 PoweredBy,恒含、
139
+ * 无拆除 prop);text 面输出标题行与 meta 行,不含品牌行
140
+ * (docs/feature/reports/library/site-components.md「HeroCard」)。
141
+ */
142
+ export const HeroCard = defineComponent<HeroCardProps>({
143
+ web: (props, ctx) => {
144
+ assertHeroData(props.data);
145
+ return <HeroCardWeb title={props.title} data={props.data} className={props.className} locale={ctx.locale} />;
146
+ },
147
+ text: (props, ctx) => {
148
+ assertHeroData(props.data);
149
+ return heroCardText(props.title, props.data, ctx);
150
+ },
151
+ });
152
+ HeroCard.displayName = "HeroCard";
153
+
154
+ /** `HeroCard` 的 props:标题 + `heroData()` 的产物,只有 data 形态。 */
155
+ export interface HeroCardProps {
156
+ title: LocalizedText;
157
+ data: HeroData;
158
+ className?: string;
159
+ }
160
+
161
+ /**
162
+ * `Hero`:页首的站点标题区——标题、最后运行时间、快照合成来源,恒含品牌行。官方组合组件,
163
+ * 与手写 `<HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} />`
164
+ * 严格等价、没有私有能力;读 `ctx.report` 意味着输出跟随站点,要站点无关的标题区直接用
165
+ * `HeroCard` 显式传值(docs/feature/reports/library/site-components.md「Hero」)。
166
+ */
167
+ export const Hero = defineComponent<HeroProps>(async ({ title, className }, ctx) => (
168
+ <HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} className={className} />
169
+ ));
170
+ Hero.displayName = "Hero";
171
+
172
+ /**
173
+ * `PoweredBy`:唯一的品牌件,无 props 双面组件。web 面渲染指向 niceeval 官网的一行品牌色
174
+ * 小字(`utm_source=report&utm_medium=powered-by`,`rel` 仅 `noopener` 以保留 Referer);
175
+ * text 面零输出。没有任何配置——品牌契约是「提供一个组件,不给开关」:不想要品牌就不用
176
+ * 这些组件、自己写替代组件(docs/feature/reports/library/site-components.md「PoweredBy」)。
177
+ */
178
+ export const PoweredBy = defineComponent<Record<never, never>>({
179
+ web: () => <PoweredByWeb />,
180
+ text: () => "",
181
+ });
182
+ PoweredBy.displayName = "PoweredBy";
183
+
184
+ /** `ScopeWarnings` 的 props:spec 形态取宿主 Scope 的 warnings,data 形态收 `ScopeWarning[]`。 */
185
+ export type ScopeWarningsProps = DataProps<readonly ScopeWarning[], Record<never, never>, ChromeProps>;
186
+
187
+ /**
188
+ * `ScopeWarnings`:选择警告区,警告的唯一呈现组件。把 Scope 携带的 `ScopeWarning[]`
189
+ * 按「下一步动作」聚合渲染(带 experimentId 的按实验聚合、非实验作用域按 kind 聚合;
190
+ * integrity 组在前);web 面组头带去重后的可复制命令、明细收原生 `<details>`(总条数 ≤ 3
191
+ * 默认展开),text 面同构但不折叠。空警告集与裸 `Snapshot[]` 输入两面零输出
192
+ * (docs/feature/reports/library/site-components.md「ScopeWarnings」)。
193
+ */
194
+ export const ScopeWarnings = makeDataComponent<readonly ScopeWarning[], Record<never, never>, ChromeProps>({
195
+ name: "ScopeWarnings",
196
+ dataFnName: "scopeWarningsData",
197
+ shapeName: "ScopeWarning[]",
198
+ dataFn: (input) => scopeWarningsData(input),
199
+ specKeys: [],
200
+ validate: validateScopeWarningsData,
201
+ web: (props, ctx) =>
202
+ props.data.length === 0 ? null : (
203
+ <ScopeWarningsWeb data={props.data} locale={props.locale ?? ctx.locale} className={props.className} />
204
+ ),
205
+ text: (props, ctx) => scopeWarningsText(props.data, ctx),
206
+ }) as unknown as ReportComponent<ScopeWarningsProps>;
207
+
208
+ /** `CopyFixPrompt` 的 props:spec 形态无选项,data 形态收 `copyFixPromptData()` 的产物。 */
209
+ export type CopyFixPromptProps = DataProps<CopyFixPromptData, Record<never, never>, ChromeProps>;
210
+
211
+ /**
212
+ * `CopyFixPrompt`:把当前范围的全部失败整理成一段可交给 coding agent 的修复 prompt。
213
+ * prompt 在 resolve 阶段算好、烘进静态 HTML,无 JS 时在折叠块里完整可读,「复制」是增强层
214
+ * 行为;`failures` 为 0 时两面零输出;text 面恒零输出——终端里的等价能力是 `show` 的
215
+ * attempt 下钻命令本身(docs/feature/reports/library/site-components.md「CopyFixPrompt」)。
216
+ */
217
+ export const CopyFixPrompt = makeDataComponent<CopyFixPromptData, Record<never, never>, ChromeProps>({
218
+ name: "CopyFixPrompt",
219
+ dataFnName: "copyFixPromptData",
220
+ shapeName: "CopyFixPromptData",
221
+ dataFn: (input) => copyFixPromptData(input),
222
+ specKeys: [],
223
+ validate: validateCopyFixPromptData,
224
+ web: (props, ctx) =>
225
+ props.data.failures === 0 ? null : (
226
+ <CopyFixPromptWeb data={props.data} locale={props.locale ?? ctx.locale} className={props.className} />
227
+ ),
228
+ text: () => "",
229
+ }) as unknown as ReportComponent<CopyFixPromptProps>;
230
+
231
+ /** `TraceWaterfall` 的 props:spec 形态无选项,data 形态收 `traceWaterfallData()` 的产物。 */
232
+ export type TraceWaterfallProps = DataProps<
233
+ readonly TraceWaterfallRow[],
234
+ Record<never, never>,
235
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
236
+ >;
237
+
238
+ /**
239
+ * `TraceWaterfall`:每个 attempt 一行的执行时间瀑布,用 canonical OTel 字段显示被测 agent
240
+ * 的原始 span(agent / model / tool)。web 面静态渲染顶层 span 分解条(失败 span 带失败
241
+ * 标记),行链接 attempt 详情;text 面每 attempt 一行(locator、总耗时、span 计数与失败
242
+ * 标记)+ 可复制的 `--timing` 下钻命令。trace 缺失的行照常出现并如实显示缺失;runner
243
+ * 生命周期节点不进瀑布(docs/feature/reports/library/site-components.md「TraceWaterfall」)。
244
+ */
245
+ export const TraceWaterfall = makeDataComponent<
246
+ readonly TraceWaterfallRow[],
247
+ Record<never, never>,
248
+ ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
249
+ >({
250
+ name: "TraceWaterfall",
251
+ dataFnName: "traceWaterfallData",
252
+ shapeName: "TraceWaterfallRow[]",
253
+ dataFn: (input) => traceWaterfallData(input),
254
+ specKeys: [],
255
+ validate: validateTraceWaterfallData,
256
+ web: (props, ctx) => (
257
+ <TraceWaterfallWeb
258
+ data={props.data}
259
+ attemptHref={hrefOf(props, ctx)}
260
+ locale={props.locale ?? ctx.locale}
261
+ className={props.className}
262
+ />
263
+ ),
264
+ text: (props, ctx) => traceWaterfallText(props.data, ctx),
265
+ }) as unknown as ReportComponent<TraceWaterfallProps>;
@@ -4,9 +4,9 @@
4
4
  // (docs/feature/results/library.md「警告 kind 全集」)经 locale 词典渲染,未登记的 kind
5
5
  // 回退为单独成组、逐条 message 原样。
6
6
 
7
- import type { ScopeWarning } from "../results/types.ts";
8
- import { gapParts } from "../results/select.ts";
9
- import { localeText, type ReportLocale, type ReportMessageKey } from "./locale.ts";
7
+ import type { ScopeWarning } from "../../../results/types.ts";
8
+ import { gapParts } from "../../../results/select.ts";
9
+ import { localeText, type ReportLocale, type ReportMessageKey } from "../../model/locale.ts";
10
10
 
11
11
  /** kind 表登记的类别:integrity(选中集合的分母可能不对)组排在 freshness(可能过期)之前。 */
12
12
  export type WarningCategory = "integrity" | "freshness";
@@ -7,9 +7,9 @@
7
7
 
8
8
  import { describe, expect, it } from "vitest";
9
9
 
10
- import type { AssertionResult, EvalResult, TraceSpan, Verdict } from "../types.ts";
11
- import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "../results/index.ts";
12
- import { makeScope } from "../results/select.ts";
10
+ import type { AssertionResult, EvalResult, TraceSpan, Verdict } from "../../../types.ts";
11
+ import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "../../../results/index.ts";
12
+ import { makeScope } from "../../../results/select.ts";
13
13
  import {
14
14
  createTextContext,
15
15
  defineComponent,
@@ -18,20 +18,15 @@ import {
18
18
  validateReportTree,
19
19
  ResolveMemo,
20
20
  type ReportNode,
21
- } from "./tree.ts";
22
- import { buildReportMeta, defineReport, pickReportPage, renderReportToText, reportTitleText } from "./report.ts";
23
- import { renderReportToStaticHtml } from "./web.ts";
24
- import {
25
- AttemptList,
26
- CopyFixPrompt,
27
- Hero,
28
- HeroCard,
29
- PoweredBy,
30
- ScopeWarnings,
31
- TraceWaterfall,
32
- } from "./components.tsx";
33
- import { Text } from "./primitives.tsx";
34
- import { attemptListData, copyFixPromptData, heroData, traceWaterfallData } from "./compute.ts";
21
+ } from "../../definition/tree.ts";
22
+ import { buildReportMeta, defineReport } from "../../definition/report.ts";
23
+ import { pickReportPage, renderReportToText, reportTitleText } from "../../runtime/text.ts";
24
+ import { renderReportToStaticHtml } from "../../runtime/web.ts";
25
+ import { AttemptList } from "../entity-lists/index.tsx";
26
+ import { CopyFixPrompt, Hero, HeroCard, PoweredBy, ScopeWarnings, TraceWaterfall } from "./index.tsx";
27
+ import { Text } from "../../definition/primitives.tsx";
28
+ import { attemptListData } from "../entity-lists/compute.ts";
29
+ import { copyFixPromptData, heroData, traceWaterfallData } from "./compute.ts";
35
30
 
36
31
  // ───────────────────────── fake 数据(按 results 读取契约造)─────────────────────────
37
32
 
@@ -120,22 +115,29 @@ function hostCtx(scope: Scope) {
120
115
  return { scope, results: resultsOf(scope.snapshots) };
121
116
  }
122
117
 
118
+ /** 站点组件测试固定"宿主已声明 attempt-input page"这条件,验证 attemptCommand/attemptHref 通道本身接得对;
119
+ * 「没有 attempt page 时退化成纯文本」由 report.test.ts / show.test.ts 的报告级测试单独覆盖。 */
120
+ const DEFAULT_TEST_ATTEMPT_COMMAND = (locator: string) => `niceeval show ${locator}`;
121
+ const DEFAULT_TEST_ATTEMPT_HREF = (locator: string) => `#/attempt/${locator}`;
122
+
123
123
  /** 管线便捷入口:装载 → resolve → validate → text 渲染(与 show 同一条管线,不带宿主前置块)。 */
124
124
  async function treeText(node: ReportNode, scope: Scope, width = 120): Promise<string> {
125
125
  const definition = defineReport(node);
126
- const resolved = await resolveReportTree(pickReportPage(definition).content, {
126
+ const page = pickReportPage(definition);
127
+ const resolved = await resolveReportTree(page.content, {
127
128
  scope,
128
129
  results: resultsOf(scope.snapshots),
129
- report: buildReportMeta(definition, scope, "report"),
130
+ report: buildReportMeta(definition, scope),
131
+ page: { id: page.id, input: "scope" },
130
132
  memo: new ResolveMemo(),
131
133
  });
132
134
  validateReportTree(resolved);
133
- return renderNodeToText(resolved, createTextContext({ width }));
135
+ return renderNodeToText(resolved, createTextContext({ width, attemptCommand: DEFAULT_TEST_ATTEMPT_COMMAND }));
134
136
  }
135
137
 
136
138
  /** 管线便捷入口:web 面静态 HTML(经 renderReportToStaticHtml;测试用无警告 Scope,免宿主前置块)。 */
137
139
  async function treeHtml(node: ReportNode, scope: Scope): Promise<string> {
138
- return renderReportToStaticHtml(defineReport(node), hostCtx(scope));
140
+ return renderReportToStaticHtml(defineReport(node), hostCtx(scope), { attemptHref: DEFAULT_TEST_ATTEMPT_HREF });
139
141
  }
140
142
 
141
143
  // ───────────────────────── 警告 fixture(按 ScopeWarning 联合造)─────────────────────────
@@ -0,0 +1,89 @@
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // "validate*Data 递归覆盖到嵌套字段" 行:site-components 四个 validate*Data 的表驱动字段突变覆盖,重点是 ScopeWarning 的四个已登记
3
+ // kind 各自的必填字段,以及未登记 kind 的前向兼容放行路径(与 ScopeWarnings 组件的「未知 kind
4
+ // 单独成组」渲染回退是同一条契约,结构校验不能比渲染逻辑更严)。
5
+
6
+ import { describe, expect, it } from "vitest";
7
+ import { validateCopyFixPromptData, validateHeroData, validateScopeWarningsData, validateTraceWaterfallData } from "./index.tsx";
8
+
9
+ describe("validateHeroData", () => {
10
+ it("合规 literal 通过,latestStartedAt 为 null 合法(空范围不编造当前时间)", () => {
11
+ expect(validateHeroData({ latestStartedAt: null, snapshots: 0 })).toBeNull();
12
+ });
13
+
14
+ it("snapshots 非数字报错", () => {
15
+ expect(validateHeroData({ latestStartedAt: null, snapshots: "0" })).toMatch(/"snapshots"/);
16
+ });
17
+ });
18
+
19
+ describe("validateScopeWarningsData — ScopeWarning 判别联合", () => {
20
+ it("partial-coverage 缺 covered 报错", () => {
21
+ const bad = [{ kind: "partial-coverage", experimentId: "exp/a", total: 6, message: "m", command: "niceeval exp exp/a" }];
22
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.covered"/);
23
+ });
24
+
25
+ it("stale-snapshot 缺 latestStartedAt 报错", () => {
26
+ const bad = [{ kind: "stale-snapshot", experimentId: "exp/a", startedAt: "t1", message: "m", command: "c" }];
27
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.latestStartedAt"/);
28
+ });
29
+
30
+ it("unfinished-snapshot 缺 dir 报错", () => {
31
+ const bad = [{ kind: "unfinished-snapshot", experimentId: "exp/a", startedAt: "t1", message: "m", command: "c" }];
32
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.dir"/);
33
+ });
34
+
35
+ it("unreadable-snapshot 的 reason 不在三态枚举内报错;没有 experimentId 字段本身合法(非实验作用域)", () => {
36
+ const bad = [{ kind: "unreadable-snapshot", dir: "/r/snap", reason: "corrupted", message: "m" }];
37
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.reason"/);
38
+ const ok = [{ kind: "unreadable-snapshot", dir: "/r/snap", reason: "malformed", message: "m" }];
39
+ expect(validateScopeWarningsData(ok)).toBeNull();
40
+ });
41
+
42
+ it("unreadable-snapshot 省略 command 合法(不是每个 reason 都有单条命令)", () => {
43
+ const ok = [{ kind: "unreadable-snapshot", dir: "/r/snap", reason: "incomplete", message: "m" }];
44
+ expect(validateScopeWarningsData(ok)).toBeNull();
45
+ });
46
+
47
+ it("未登记的 kind 只要有 kind/message 就放行——前向兼容,不拒绝未来版本产出的新 kind", () => {
48
+ const future = [{ kind: "future-kind", message: "something new happened" }];
49
+ expect(validateScopeWarningsData(future)).toBeNull();
50
+ });
51
+
52
+ it("未登记的 kind 缺 message 仍报错(两族共用的最小形状)", () => {
53
+ const bad = [{ kind: "future-kind" }];
54
+ expect(validateScopeWarningsData(bad)).toMatch(/"data\[0\]\.message"/);
55
+ });
56
+ });
57
+
58
+ describe("validateCopyFixPromptData", () => {
59
+ it("合规 literal 通过", () => {
60
+ expect(validateCopyFixPromptData({ prompt: "fix it", failures: 1 })).toBeNull();
61
+ });
62
+
63
+ it("failures 非数字报错", () => {
64
+ expect(validateCopyFixPromptData({ prompt: "x", failures: "1" })).toMatch(/"failures"/);
65
+ });
66
+ });
67
+
68
+ describe("validateTraceWaterfallData", () => {
69
+ const validSpan = { name: "turn", kind: "agent", startOffsetMs: 0, durationMs: 100, failed: false };
70
+ const valid = [{ experimentId: "exp/a", evalId: "q1", locator: "@1abcdef2", durationMs: 100, spans: [validSpan] }];
71
+
72
+ it("合规 literal 通过", () => {
73
+ expect(validateTraceWaterfallData(valid)).toBeNull();
74
+ });
75
+
76
+ it("durationMs 为 null 合法(trace 缺失时如实显示缺失)", () => {
77
+ expect(validateTraceWaterfallData([{ ...valid[0], durationMs: null }])).toBeNull();
78
+ });
79
+
80
+ it("spans[i].kind 不在四态枚举内报错", () => {
81
+ const bad = [{ ...valid[0], spans: [{ ...validSpan, kind: "runner" }] }];
82
+ expect(validateTraceWaterfallData(bad)).toMatch(/"data\[0\]\.spans\[0\]\.kind"/);
83
+ });
84
+
85
+ it("spans[i].failed 非布尔报错", () => {
86
+ const bad = [{ ...valid[0], spans: [{ ...validSpan, failed: "true" }] }];
87
+ expect(validateTraceWaterfallData(bad)).toMatch(/"data\[0\]\.spans\[0\]\.failed"/);
88
+ });
89
+ });
@@ -1,13 +1,14 @@
1
1
  // ScopeSummary:一个范围的摘要卡——快照时间窗、experiment / eval / attempt 数、判定计票、
2
- // 端到端成功率与总成本。data 恒携带 eval 级与 attempt 级两份计票;呈现 prop `votes`
3
- // 只选择显示哪一级(默认 "eval"),不改变 data。成功率与总成本只渲染算好的 MetricCell,
2
+ // 端到端通过率与总成本。data 恒携带 eval 级与 attempt 级两份计票;呈现 prop `votes`
3
+ // 只选择显示哪一级(默认 "eval"),不改变 data。通过率与总成本只渲染算好的 MetricCell,
4
4
  // 不现场重算(docs/feature/reports/library/summaries.md「ScopeSummary」)。
5
5
 
6
6
  import type { ReactElement } from "react";
7
- import type { ScopeSummaryData, VerdictTally } from "../types.ts";
8
- import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
9
- import { MetricCellView } from "./cell.tsx";
10
- import { cx } from "./format.ts";
7
+ import type { ScopeSummaryData, VerdictTally } from "../../model/types.ts";
8
+ import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../../model/locale.ts";
9
+ import { formatReportDateTime, formatReportDateTimeRange } from "../../model/format.ts";
10
+ import { MetricCellView } from "../cell.tsx";
11
+ import { cx } from "../shared.ts";
11
12
 
12
13
  function VerdictTallyView({ tally, locale }: { tally: VerdictTally; locale: ReportLocale }): ReactElement {
13
14
  const kinds = (["passed", "failed", "errored", "skipped"] as const).filter((k) => tally[k] > 0);
@@ -36,6 +37,10 @@ export function ScopeSummary({
36
37
  locale?: ReportLocale;
37
38
  }): ReactElement {
38
39
  const tally = votes === "attempt" ? data.attemptVerdicts : data.evalVerdicts;
40
+ const formattedRange =
41
+ data.range.earliestStartedAt !== null && data.range.latestStartedAt !== null
42
+ ? formatReportDateTimeRange(data.range.earliestStartedAt, data.range.latestStartedAt, locale)
43
+ : null;
39
44
  return (
40
45
  <div className={cx("nre", "nre-scope-summary", className)} data-votes={votes}>
41
46
  <dl className="nre-scope-kpis">
@@ -66,7 +71,15 @@ export function ScopeSummary({
66
71
  <div className="nre-scope-kpi">
67
72
  <dt>{localeText(locale, "scopeSummary.totalCost")}</dt>
68
73
  <dd>
69
- <MetricCellView cell={data.totalCostUSD} locale={locale} />
74
+ <MetricCellView cell={data.totalCostUSD} locale={locale} showCoverage={false} />
75
+ {data.totalCostUSD.samples < data.totalCostUSD.total && (
76
+ <small className="nre-scope-kpi-note">
77
+ {localeText(locale, "scopeSummary.costCoverage", {
78
+ samples: data.totalCostUSD.samples,
79
+ total: data.totalCostUSD.total,
80
+ })}
81
+ </small>
82
+ )}
70
83
  </dd>
71
84
  </div>
72
85
  </dl>
@@ -74,11 +87,13 @@ export function ScopeSummary({
74
87
  {data.range.latestStartedAt !== null && (
75
88
  <p className="nre-scope-summary-range">
76
89
  {data.range.earliestStartedAt !== null && data.range.earliestStartedAt !== data.range.latestStartedAt
77
- ? localeText(locale, "scopeSummary.range", {
78
- from: data.range.earliestStartedAt,
79
- to: data.range.latestStartedAt,
90
+ ? localeText(locale, "scopeSummary.runRange", {
91
+ from: formattedRange!.from,
92
+ to: formattedRange!.to,
80
93
  })
81
- : data.range.latestStartedAt}
94
+ : localeText(locale, "scopeSummary.lastRun", {
95
+ time: formatReportDateTime(data.range.latestStartedAt, locale),
96
+ })}
82
97
  </p>
83
98
  )}
84
99
  </div>
@@ -0,0 +1,56 @@
1
+ // 计算函数(*Data):ReportInput → 一份组件数据。ScopeSummary 的 scopeSummaryData 住在这里
2
+ // (docs/feature/reports/library/summaries.md)。ExperimentComparison 是纯组合组件,不产生
3
+ // 独立的 data,没有对应的计算函数。
4
+ //
5
+ // 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
6
+ // - 第一参收 ReportInput = Scope | readonly Snapshot[];warnings 不进组件数据(宿主统一显示);
7
+ // - 聚合前按身份键去重(dedupeAttempts;missing-startedAt 不去重、如实保留、不透出警告);
8
+ // - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露。
9
+
10
+ import type { ReportInput, ScopeSummaryData } from "../../model/types.ts";
11
+ import { collectItems, computeCell, resolveInput } from "../../model/aggregate.ts";
12
+ import { costUSD, defineMetric, endToEndPassRate } from "../../model/metrics.ts";
13
+ import { selectedEvalsOnly, summarizeItems, tallyOf } from "../shared-compute.ts";
14
+
15
+ // ───────────────────────── scopeSummaryData ─────────────────────────
16
+
17
+ /** costUSD 的求和投影:两级都 sum(题内多轮求和 + 跨题求和 = 全量求和),display 走 $。 */
18
+ const totalCostMetric = defineMetric({
19
+ name: "total-cost",
20
+ label: costUSD.label,
21
+ unit: "$",
22
+ value: costUSD.value,
23
+ aggregate: { perEval: "sum", acrossEvals: "sum" },
24
+ });
25
+
26
+ /**
27
+ * `scopeSummaryData(input)`:范围摘要——快照时间窗、experiment / eval / attempt 数、
28
+ * 两级判定计票、端到端通过率与总成本(docs/feature/reports/library/summaries.md)。
29
+ * data 恒携带两级计票;通过率来自官方两级指标引擎,不从任一计票重算。
30
+ */
31
+ export async function scopeSummaryData(input: ReportInput): Promise<ScopeSummaryData> {
32
+ const snapshots = selectedEvalsOnly(resolveInput(input).snapshots);
33
+ const items = collectItems(snapshots);
34
+
35
+ let earliest: string | null = null;
36
+ let latest: string | null = null;
37
+ for (const snapshot of snapshots) {
38
+ if (earliest === null || snapshot.startedAt < earliest) earliest = snapshot.startedAt;
39
+ if (latest === null || snapshot.startedAt > latest) latest = snapshot.startedAt;
40
+ }
41
+
42
+ const stats = summarizeItems(items);
43
+ const attemptVerdicts = tallyOf();
44
+ for (const item of items) attemptVerdicts[item.attempt.result.verdict] += 1;
45
+
46
+ return {
47
+ range: { earliestStartedAt: earliest, latestStartedAt: latest },
48
+ experiments: stats.experiments,
49
+ evals: stats.evals,
50
+ attempts: stats.attempts,
51
+ evalVerdicts: stats.verdicts,
52
+ attemptVerdicts,
53
+ endToEndPassRate: await computeCell(endToEndPassRate, items),
54
+ totalCostUSD: await computeCell(totalCostMetric, items),
55
+ };
56
+ }