niceeval 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +15 -7
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
  118. package/dist/report/{tree.js → definition/tree.js} +10 -12
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +12 -2
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +15 -1
  154. package/dist/shared/aggregate.js +32 -1
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +77 -64
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +30 -15
  227. package/src/i18n/zh-CN.ts +30 -15
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +136 -112
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +58 -22
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +54 -3
  314. package/src/results/select.ts +30 -6
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +30 -1
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +58 -22
  356. package/src/show/render.ts +16 -240
  357. package/src/show/show.test.ts +128 -35
  358. package/src/view/app/App.test.tsx +8 -7
  359. package/src/view/app/App.tsx +135 -51
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +122 -89
  370. package/src/view/index.ts +3 -13
  371. package/src/view/server.ts +36 -6
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +18 -806
  376. package/src/view/view-report.test.ts +182 -34
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,8 +1,8 @@
1
1
  // cases: docs/engineering/unit-tests/reports/cases.md
2
2
  // niceeval view 的报告槽与宿主组合语义(docs/feature/reports/architecture.md「Selection 是计算入口」
3
- // 与裁决记录 6;公开行为准绳 docs-site/zh/how-to/viewing-results.mdx / custom-reports.mdx)。
3
+ // 与裁决记录 6;公开行为准绳 docs-site/zh/tutorials/viewing-results.mdx / custom-reports.mdx)。
4
4
  // 覆盖:
5
- // - 组合语义与 show 对齐:位置前缀收窄报告槽 Scope、--experiment 过滤、匹配不到直说;
5
+ // - 组合语义与 show 对齐:位置前缀收窄报告槽 Scope、--exp 过滤、匹配不到直说;
6
6
  // - 输入语义:位置参数只表示 eval id 前缀(不随文件系统状态改变),结果根走 --results,
7
7
  // 单开一份快照走 --snapshot(文件不可读时失败);
8
8
  // - 报告槽恒在:裸跑填充内建报告,--report 整槽替换;en / zh-CN 双语各渲染一遍;
@@ -13,7 +13,7 @@
13
13
  // fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
14
14
  // 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
15
15
 
16
- import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises";
16
+ import { mkdir, mkdtemp, readdir, readFile, rm, utimes, writeFile } from "node:fs/promises";
17
17
  import { existsSync } from "node:fs";
18
18
  import { tmpdir } from "node:os";
19
19
  import { join, resolve } from "node:path";
@@ -21,11 +21,12 @@ import { afterEach, describe, expect, it } from "vitest";
21
21
  // dist-sourced: this must be the exact class loadViewScan()/data.ts's loadReportFile() throws
22
22
  // (see src/view/data.ts's comment) — a raw-src import would be a structurally-identical but
23
23
  // `instanceof`-incompatible class.
24
- import { ReportLoadError } from "../../dist/report/load.js";
24
+ import { ReportLoadError } from "../../dist/report/runtime/load.js";
25
25
  import { ViewInputError, loadViewScan, type ViewScan } from "./data.ts";
26
26
  import { renderHtml } from "./site.ts";
27
27
  import { buildView, resolveViewInput } from "./index.ts";
28
28
  import { runShow } from "../show/index.ts";
29
+ import { createResultsWriter } from "../results/index.ts";
29
30
  import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
30
31
 
31
32
  const EXAM_REPORT = resolve(__dirname, "../../test/fixtures/report/exam-report.tsx");
@@ -162,10 +163,9 @@ function bareReportHtml(scan: ViewScan): { en: string; "zh-CN": string } {
162
163
  // ───────────────────────── 组合语义(与 show 对齐) ─────────────────────────
163
164
 
164
165
  describe("loadViewScan · 组合语义", () => {
165
- it("位置前缀收窄对全部页生效(含内建 Attempts 页);attempt 详情路由对完整结果根解析,被滤掉的深链仍可达", async () => {
166
+ it("位置前缀收窄作用在有效根上:全部页与证据室一致缩小,被滤掉的 attempt 不烘进页面数据", async () => {
166
167
  const root = await seedRoot();
167
168
  const scan = await loadViewScan(root, { patterns: ["weather"] });
168
- const { viewData } = scan;
169
169
  const reportHtml = bareReportHtml(scan);
170
170
  // 报告页:两实验都只剩 weather/brooklyn 一题,范围外的失败不再出现。
171
171
  expect(reportHtml.en).toContain("compare/bub");
@@ -176,16 +176,20 @@ describe("loadViewScan · 组合语义", () => {
176
176
  expect(pageHtml(scan, "traces").en).not.toContain("fixtures/button");
177
177
  const bare = await loadViewScan(root);
178
178
  expect(pageHtml(bare, "attempts").en).toContain("fixtures/button"); // 不收窄时在场,对照
179
- // attempt 详情路由不是页:viewData.snapshots 全量通道保留,被滤掉的 attempt 深链仍解析成功。
180
- const filteredOut = viewData.snapshots
181
- .flatMap((s) => s.results)
182
- .find((r) => r.id === "fixtures/button");
183
- expect(filteredOut?.locator).toBeTruthy();
184
- const { resolveAttemptLocator } = await import("./app/lib/attempt-route.ts");
185
- expect(resolveAttemptLocator(viewData.snapshots, filteredOut!.locator!)).toBe(filteredOut);
179
+ // 有效根即证据室:被滤掉的 attempt 不在 attemptsByBase / attemptPages.locators 里,
180
+ // 站点管线(site.ts)不会为它生成 attempt/<locator>.html。
181
+ const filteredOut = [...scan.attemptsByBase.values()].find((a) => a.evalId === "fixtures/button");
182
+ expect(filteredOut).toBeUndefined();
183
+ expect(scan.artifactDirs.size).toBe(
184
+ [...scan.artifactDirs.keys()].filter((base) => base.includes("weather/brooklyn")).length,
185
+ );
186
+ // 收窄之内、不在现刻水位的口径差异由 --exp 深链测试覆盖;不收窄时深链照常可达,对照:
187
+ const inBare = [...bare.attemptsByBase.values()].find((a) => a.evalId === "fixtures/button");
188
+ expect(inBare?.locator).toBeTruthy();
189
+ expect(bare.attemptPages!.locators.get(inBare!.locator!)).toBe(inBare);
186
190
  });
187
191
 
188
- it("--experiment 过滤:报告槽 Selection 只留该实验", async () => {
192
+ it("--exp 过滤:报告槽 Selection 只留该实验", async () => {
189
193
  const root = await seedRoot();
190
194
  const reportHtml = bareReportHtml(await loadViewScan(root, { experiment: "compare/codex" }));
191
195
  expect(reportHtml.en).toContain("compare/codex");
@@ -236,13 +240,13 @@ describe("loadViewScan · 默认报告槽(裸跑)", () => {
236
240
  it("报告槽双语渲染:同一棵树按 locale 渲染两遍,chrome 文案分语言、数据不分语言", async () => {
237
241
  const root = await seedRoot();
238
242
  const reportHtml = bareReportHtml(await loadViewScan(root));
239
- expect(reportHtml.en).toContain("End-to-end pass rate"); // ExperimentList 主行(en)
240
- expect(reportHtml["zh-CN"]).toContain("端到端成功率"); // ExperimentList 主行(zh-CN)
243
+ expect(reportHtml.en).toContain("Pass rate"); // ExperimentList 主行(en)
244
+ expect(reportHtml["zh-CN"]).toContain("通过率"); // ExperimentList 主行(zh-CN)
241
245
  for (const html of [reportHtml.en, reportHtml["zh-CN"]]) {
242
246
  expect(html).toContain("compare/bub");
243
- // 失败案例深链进证据室:不透明 AttemptLocator 单段路由 `#/attempt/@<locator>`,
244
- // 不再是旧的两段式 `#/attempt/<snapshot>/<attempt>`。
245
- expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/);
247
+ // 失败案例深链进独立 attempt 文档:不透明 AttemptLocator 编码进文件名
248
+ // (`attempt/<encodeURIComponent(locator)>.html`),不再是旧的 hash 路由。
249
+ expect(html).toMatch(/href="attempt\/%40[0-9a-z]+\.html"/);
246
250
  expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
247
251
  }
248
252
  });
@@ -268,20 +272,24 @@ describe("loadViewScan · 默认报告槽(裸跑)", () => {
268
272
  // ───────────────────────── 报告槽整槽替换 ─────────────────────────
269
273
 
270
274
  describe("loadViewScan · --report 报告槽", () => {
271
- it("报告树渲染为静态 HTML:官方水位 + 自定义摆法 + <Style> 产物 + 证据室深链,零 <script>", async () => {
275
+ it("报告树渲染为静态 HTML:官方水位 + 自定义摆法 + <Style> 产物,零 <script>", async () => {
272
276
  const root = await seedRoot();
273
277
  const scan = await loadViewScan(root, { report: { path: EXAM_REPORT, cwd: root } });
274
278
  const html = slotHtml(scan).en;
275
279
  expect(html).toContain("考试成绩单"); // 自定义 Section
276
280
  expect(html).toContain("nre-"); // 官方组件的稳定类名
277
281
  expect(html).toContain("<style>.exam-note { color: #4a7; }</style>"); // <Style> 随树带走
278
- expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/); // 失败案例深链进证据室(单段 locator 路由)
282
+ // exam-report.tsx 是树形态 defineReport,没有声明 attempt-input page:locator 因此只是
283
+ // 纯文本,不生成假 href(architecture.md「Attempt 详情是一张参数化 page」)。
284
+ expect(html).toMatch(/<span class="nre-locator[^"]*">@[0-9a-z]+/);
285
+ expect(html).not.toMatch(/href="#\/attempt\//);
279
286
  expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
280
287
  // 用户报告同样双语渲染两遍(壳按界面语言摆放)。
281
288
  expect(slotHtml(scan)["zh-CN"]).toContain("考试成绩单");
282
- // 证据室数据契约(__NICEEVAL_VIEW_DATA__)原样保留:快照、locator、skipped 不动。
283
- expect(scan.viewData.snapshots.length).toBeGreaterThan(0);
284
- expect(scan.viewData.snapshots.flatMap((s) => s.results).every((r) => r.locator)).toBe(true);
289
+ // 证据室索引(attemptsByBase,供 artifact/ 与 attempt/<locator>.html 两条出口共用)原样保留,
290
+ // 不受这份没有 attempt-input page 的自定义报告影响;每条都有 locator。
291
+ expect(scan.attemptsByBase.size).toBeGreaterThan(0);
292
+ expect([...scan.attemptsByBase.values()].every((a) => a.locator)).toBe(true);
285
293
  expect(JSON.stringify(scan.viewData)).not.toContain("考试成绩单"); // 报告块不进 viewData
286
294
  });
287
295
 
@@ -314,6 +322,27 @@ describe("loadViewScan · --report 报告槽", () => {
314
322
  expect(text).toContain(needle);
315
323
  expect(html.en).toContain(needle);
316
324
  }
325
+ // Grid/Stat/Section.meta:两面共享同一批 label/value/detail/meta 终值,不要求布局逐字一致。
326
+ for (const needle of ["速览", "人工摘要,非计算结果", "及格线", "60 分", "最高分", "97 分", "compare/codex"]) {
327
+ expect(text).toContain(needle);
328
+ expect(html.en).toContain(needle);
329
+ }
330
+ // web 面是 Grid 结构(稳定 nre-grid/nre-grid-cell class),text 面无 JS 概念、只有内容。
331
+ expect(html.en).toContain("nre-grid");
332
+ expect(html.en).toContain("nre-grid-cell");
333
+ expect(html.en).toContain("nre-stat");
334
+
335
+ // text 面按显示宽度减列但不丢 Stat:极窄终端下两个 Stat 仍都完整可读。
336
+ let narrowText = "";
337
+ const narrowCode = await runShow(root, [], { results: root, report: EXAM_REPORT }, {
338
+ out: (s) => (narrowText += s),
339
+ err: () => {},
340
+ width: 30,
341
+ });
342
+ expect(narrowCode).toBe(0);
343
+ for (const needle of ["及格线", "60 分", "最高分", "97 分"]) {
344
+ expect(narrowText).toContain(needle);
345
+ }
317
346
  });
318
347
 
319
348
  it("报告文件缺失 / 默认导出不是 defineReport 产物:ReportLoadError 直说", async () => {
@@ -346,6 +375,7 @@ describe("loadViewScan · 外壳标题与 ReportLink.icon", () => {
346
375
  ' kind: "report",',
347
376
  ' title: { en: "Memory Evals", "zh-CN": "记忆能力评测" },',
348
377
  ' links: [{ label: "GitHub", href: "https://example.com", icon: { svg: "<svg data-mark></svg>" } }],',
378
+ " head: [],",
349
379
  " scripts: [],",
350
380
  " styles: [],",
351
381
  ' pages: [{ id: "report", title: "Report", content: { $$typeof: Symbol.for("react.transitional.element"), type: Block, props: {}, key: null } }],',
@@ -396,6 +426,7 @@ describe("loadViewScan · 报告文件变更整页重算", () => {
396
426
  "const definition = {",
397
427
  ' kind: "report",',
398
428
  " links: [],",
429
+ " head: [],",
399
430
  " scripts: [],",
400
431
  " styles: [],",
401
432
  ' pages: [{ id: "report", title: "Report", content: { $$typeof: Symbol.for("react.transitional.element"), type: Block, props: {}, key: null } }],',
@@ -457,7 +488,7 @@ describe("buildView · --out 与 --report", () => {
457
488
  const artifactDir = join(root, "compare_codex", "2026-07-09T10-00-00-000Z", "weather", "brooklyn", "a0");
458
489
  await mkdir(artifactDir, { recursive: true });
459
490
  await writeFile(join(artifactDir, "events.json"), "[]", "utf-8");
460
- await writeFile(join(artifactDir, "diff.json"), '{"windows":[]}', "utf-8");
491
+ await writeFile(join(artifactDir, "diff.json"), "[]", "utf-8");
461
492
  await writeFile(join(artifactDir, "o11y.json"), "{}", "utf-8");
462
493
 
463
494
  const out = join(root, "site");
@@ -468,18 +499,41 @@ describe("buildView · --out 与 --report", () => {
468
499
  expect(existsSync(join(exported, "o11y.json"))).toBe(false);
469
500
  });
470
501
 
471
- it("--out 与位置参数 / --experiment 互斥:报错含 copySnapshots + filter 下一步", async () => {
502
+ it("--out 接受收窄,出站的就是收窄到的:被滤掉实验的证据文件与页面数据都不出站", async () => {
472
503
  const root = await seedRoot();
504
+ // 两个实验各写一份 events.json:导出后只有收窄内的那份在站里。
505
+ const bubDir = join(root, "compare_bub", "2026-07-08T10-00-00-000Z", "weather", "brooklyn", "a0");
506
+ const codexDir = join(root, "compare_codex", "2026-07-09T10-00-00-000Z", "weather", "brooklyn", "a0");
507
+ for (const dir of [bubDir, codexDir]) {
508
+ await mkdir(dir, { recursive: true });
509
+ await writeFile(join(dir, "events.json"), "[]", "utf-8");
510
+ }
511
+
473
512
  const out = join(root, "site");
474
- for (const scan of [{ patterns: ["weather"] }, { experiment: "compare/bub" }]) {
475
- const attempt = buildView({ input: root, out, scan });
476
- await expect(attempt).rejects.toBeInstanceOf(ViewInputError);
477
- await expect(
478
- buildView({ input: root, out, scan }),
479
- ).rejects.toThrow(/copySnapshots/);
513
+ await buildView({ input: root, out, scan: { patterns: [], experiment: "compare/codex" } });
514
+ expect(existsSync(join(out, "artifact", "compare_codex/2026-07-09T10-00-00-000Z/weather/brooklyn/a0", "events.json"))).toBe(true);
515
+ // 被滤掉的实验:证据树整目录不出站,页面数据(烘进 HTML 的 viewData 与报告块)也不含它。
516
+ expect(existsSync(join(out, "artifact", "compare_bub"))).toBe(false);
517
+ const html = await readFile(join(out, "index.html"), "utf-8");
518
+ expect(html).not.toContain("compare/bub");
519
+ expect(html).toContain("compare/codex");
520
+ });
521
+
522
+ it("eval 前缀收窄导出:不匹配 attempt 的证据不出站", async () => {
523
+ const root = await seedRoot();
524
+ const weatherDir = join(root, "compare_bub", "2026-07-08T10-00-00-000Z", "weather", "brooklyn", "a0");
525
+ const buttonDir = join(root, "compare_bub", "2026-07-08T10-00-00-000Z", "fixtures", "button", "a0");
526
+ for (const dir of [weatherDir, buttonDir]) {
527
+ await mkdir(dir, { recursive: true });
528
+ await writeFile(join(dir, "events.json"), "[]", "utf-8");
480
529
  }
481
- // 同参数不带 --out 时照常收窄报告槽(不报错)。
482
- await expect(loadViewScan(root, { patterns: ["weather"] })).resolves.toBeTruthy();
530
+
531
+ const out = join(root, "site");
532
+ await buildView({ input: root, out, scan: { patterns: ["weather"] } });
533
+ expect(existsSync(join(out, "artifact", "compare_bub/2026-07-08T10-00-00-000Z/weather/brooklyn/a0", "events.json"))).toBe(true);
534
+ expect(existsSync(join(out, "artifact", "compare_bub/2026-07-08T10-00-00-000Z/fixtures"))).toBe(false);
535
+ const html = await readFile(join(out, "index.html"), "utf-8");
536
+ expect(html).not.toContain("fixtures/button");
483
537
  });
484
538
 
485
539
  it("默认导出(无 --report):报告槽填充 ExperimentComparison,双语块与增强 runtime 恒内联", async () => {
@@ -496,3 +550,97 @@ describe("buildView · --out 与 --report", () => {
496
550
  expect(block).not.toContain("<script");
497
551
  });
498
552
  });
553
+
554
+ // ───────────────────── attempt/<locator>.html:独立静态详情文档 ─────────────────────
555
+
556
+ describe("buildView · attempt/<locator>.html", () => {
557
+ it("收窄后有效根内每个可达 locator 各一份文档,收窄外 locator 不生成文件(cases.md 第 205 行)", async () => {
558
+ const root = await makeRoot();
559
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
560
+ const snapA = await writer.snapshot({ experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
561
+ await snapA.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1000, assertions: [] });
562
+ const snapB = await writer.snapshot({ experimentId: "exp/b", agent: "bub", startedAt: "2026-07-01T09:00:00.000Z" });
563
+ await snapB.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1000, assertions: [] });
564
+ await writer.finish();
565
+
566
+ const scanFull = await loadViewScan(root);
567
+ const scanNarrowed = await loadViewScan(root, { experiment: "exp/a" });
568
+ expect(scanFull.attemptPages!.locators.size).toBe(2);
569
+ expect(scanNarrowed.attemptPages!.locators.size).toBe(1);
570
+ const [keptLocator] = [...scanNarrowed.attemptPages!.locators.keys()];
571
+ const droppedLocator = [...scanFull.attemptPages!.locators.keys()].find((l) => l !== keptLocator)!;
572
+
573
+ const out = join(root, "site");
574
+ await buildView({ input: root, out, scan: { experiment: "exp/a" } });
575
+ const files = await readdir(join(out, "attempt"));
576
+ expect(files).toEqual([`${keptLocator}.html`]);
577
+ expect(existsSync(join(out, "attempt", `${droppedLocator}.html`))).toBe(false);
578
+ });
579
+
580
+ it("直接读取该文档(无 JavaScript 场景)即可见完整内容:身份/verdict/断言/时间树/diagnostics/usage/对话/trace/diff 都在可见的 en 区块里,不藏在 hidden 或 <template> 里(cases.md 第 206 行)", async () => {
581
+ const root = await makeRoot();
582
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
583
+ const snap = await writer.snapshot({ experimentId: "compare/bub", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
584
+ await snap.writeAttempt(
585
+ {
586
+ id: "weather/brooklyn",
587
+ verdict: "failed",
588
+ attempt: 0,
589
+ durationMs: 4200,
590
+ assertions: [
591
+ {
592
+ name: "temperature matches forecast",
593
+ severity: "gate",
594
+ outcome: "failed",
595
+ score: 0,
596
+ expected: "72",
597
+ received: "71",
598
+ loc: { file: "evals/weather.eval.ts", line: 1 },
599
+ },
600
+ ],
601
+ phases: [{ name: "eval.run", durationMs: 4000 }],
602
+ diagnostics: [{ code: "teardown-failed", level: "warning", phase: "sandbox.teardown", message: "container stop timed out" }],
603
+ usage: { inputTokens: 100, outputTokens: 50 },
604
+ },
605
+ {
606
+ sources: [{ path: "evals/weather.eval.ts", content: "export default 1;\n" }],
607
+ events: [
608
+ { type: "message", role: "user", text: "what is the forecast for brooklyn" },
609
+ { type: "message", role: "assistant", text: "72 degrees and sunny" },
610
+ ],
611
+ trace: [{ traceId: "t1", spanId: "s1", name: "turn", kind: "turn", startMs: 0, endMs: 100 }],
612
+ diff: [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }] as never,
613
+ },
614
+ );
615
+ await writer.finish();
616
+
617
+ const out = join(root, "site");
618
+ await buildView({ input: root, out });
619
+ const [locator] = [...(await loadViewScan(root)).attemptPages!.locators.keys()];
620
+ const html = await readFile(join(out, "attempt", `${locator}.html`), "utf-8");
621
+
622
+ // 结构:en 可见(无 hidden 属性),zh-CN 带 hidden——两者都不需要 JS 才能被浏览器正确
623
+ // 处理(hidden 是浏览器原生渲染指令,不是 JS 行为);没有 <template> 把内容整个锁起来。
624
+ expect(html).toMatch(/<div data-nre-locale="en">/);
625
+ expect(html).not.toMatch(/<div data-nre-locale="en"[^>]*\shidden/);
626
+ expect(html).toMatch(/<div data-nre-locale="zh-CN" hidden>/);
627
+ const body = html.split("<body>")[1]!.split("</body>")[0]!;
628
+ expect(body).not.toContain("<template"); // 不像 index.html 那样把内容锁进 <template>
629
+ const enBlock = html.split('<div data-nre-locale="en">')[1]!.split('<div data-nre-locale="zh-CN"')[0]!;
630
+ for (const needle of [
631
+ "weather/brooklyn", // 身份
632
+ "compare/bub",
633
+ "evals/weather.eval.ts", // source
634
+ "temperature matches forecast", // 断言
635
+ "71", // received(fix prompt 里的摘要行)
636
+ "4.0s", // 时间树(eval.run 4000ms)
637
+ "container stop timed out", // diagnostics
638
+ "output tokens", // usage
639
+ "what is the forecast", // 对话
640
+ "nre-attempt-trace", // trace
641
+ "a.txt", // diff 摘要
642
+ ]) {
643
+ expect(enBlock, needle).toContain(needle);
644
+ }
645
+ });
646
+ });
@@ -1,179 +0,0 @@
1
- import { type ReportComponent } from "./tree.ts";
2
- import type { LocalizedText, ReportLocale } from "./locale.ts";
3
- import type { AttemptLocator } from "../results/locator.ts";
4
- import type { AttemptListItem, CopyFixPromptData, DeltaData, EvalListItem, ExperimentComparisonData, ExperimentListItem, HeroData, LineData, MatrixData, ReportInput, ScatterData, ScopeSummaryData, ScopeWarning, ScoreboardData, TableData, TraceWaterfallRow } from "./types.ts";
5
- import { type DeltaTableOptions, type MetricLineOptions, type MetricMatrixOptions, type MetricScatterOptions, type MetricTableOptions, type ScoreboardOptions } from "./compute.ts";
6
- type Never<T> = {
7
- [K in keyof T]?: never;
8
- };
9
- /**
10
- * 官方数据组件的统一 props 组合(docs/feature/reports/library/metric-views.md):
11
- * data 形态(接收配套 *Data 的产物)或 spec 形态(Options 平铺 + 可选 input)。
12
- */
13
- export type DataProps<Data, Options, Presentation> = ({
14
- data: Data;
15
- input?: never;
16
- } & Never<Options> & Presentation) | ({
17
- data?: never;
18
- input?: ReportInput;
19
- } & Options & Presentation);
20
- interface ChromeProps {
21
- /** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
22
- locale?: ReportLocale;
23
- className?: string;
24
- }
25
- export type ScopeSummaryProps = DataProps<ScopeSummaryData, Record<never, never>, ChromeProps & {
26
- /** 显示哪一级计票;默认 "eval"。data 恒携带两级,votes 只选择呈现。 */
27
- votes?: "eval" | "attempt";
28
- }>;
29
- /** 范围摘要卡:时间窗、数量、两级计票、端到端成功率与总成本。 */
30
- export declare const ScopeSummary: ReportComponent<ScopeSummaryProps>;
31
- export type ExperimentComparisonProps = DataProps<ExperimentComparisonData, Record<never, never>, ChromeProps>;
32
- /**
33
- * 内建报告的默认组合件:先把 input 按可比组分区,再为每组分别计算 ScopeSummary、
34
- * 成本 × 端到端成功率散点和 ExperimentList。web 面持有完整组索引并一次聚焦一组;
35
- * text 面命中多个组时只显示组索引与可执行的单组查看命令,命中单组时才输出完整散点与列表。
36
- */
37
- export declare const ExperimentComparison: ReportComponent<ExperimentComparisonProps>;
38
- interface EntityListChrome extends ChromeProps {
39
- attemptHref?: (locator: AttemptLocator) => string;
40
- }
41
- export type ExperimentListProps = DataProps<readonly ExperimentListItem[], Record<never, never>, EntityListChrome & {
42
- /** web 面在比较表前显示实验过滤框;text 面忽略。 */
43
- filter?: boolean;
44
- /**
45
- * 可选父路径:两面的行标签去掉与它相同的前缀,只显示 experiment id 末段(默认
46
- * `ExperimentComparison` 给每组传组键)。完整 id 仍是排序 / 着色 / 过滤 / 折叠的键。
47
- */
48
- relativeTo?: string;
49
- }>;
50
- /** 实验列表:每项一个 experiment,固定八列比较表 + 展开到 Eval / Attempt。 */
51
- export declare const ExperimentList: ReportComponent<ExperimentListProps>;
52
- export type EvalListProps = DataProps<readonly EvalListItem[], Record<never, never>, EntityListChrome>;
53
- /** Eval 列表:每项一个 experimentId + evalId,展开到这道题的 Attempt。 */
54
- export declare const EvalList: ReportComponent<EvalListProps>;
55
- export type AttemptListProps = DataProps<readonly AttemptListItem[], Record<never, never>, EntityListChrome & {
56
- /** 过滤 / 截断前的总数;省略时等于 data 长度。 */
57
- total?: number;
58
- /** web 面加过滤输入框(按 experiment、eval、agent、verdict 或摘要文本收窄行);渐进增强,不改变数据与 text 面。 */
59
- filter?: boolean;
60
- }>;
61
- /** Attempt 列表:实体列表的叶子层,每项一次 attempt 的判定、单行摘要与 locator。 */
62
- export declare const AttemptList: ReportComponent<AttemptListProps>;
63
- export interface FailureListProps {
64
- /** 显示的最大条数;默认 20。 */
65
- limit?: number;
66
- /** 默认宿主注入的 Scope。 */
67
- input?: ReportInput;
68
- attemptHref?: (locator: AttemptLocator) => string;
69
- locale?: ReportLocale;
70
- className?: string;
71
- }
72
- /**
73
- * 「现在有哪些失败要处理」的成品组合件:内部就是 attemptListData → 过滤 → AttemptList
74
- * data 形态,与手写组合严格等价、没有私有能力(docs/feature/reports/library/entity-lists.md)。
75
- * verdict ∈ failed / errored,按 attempt 开始时间降序(同刻按 locator 字典序),
76
- * 截断到 limit(默认 20),total 报告截断前总数。
77
- */
78
- export declare const FailureList: ReportComponent<FailureListProps>;
79
- /** `Hero` 的 props:标题缺省取 `ctx.report.title`(回退链后的站点标题)。 */
80
- export interface HeroProps {
81
- /** 覆盖标题;省略时取 ctx.report.title(回退链后的站点标题)。 */
82
- title?: LocalizedText;
83
- className?: string;
84
- }
85
- /**
86
- * `HeroCard`:Hero 的渲染件,双面组件,只收 data 形态——标题输入是站点声明与 Scope 的
87
- * 合成物,没有单独的 spec 等价形。web 面渲染 hero 标题(h1)、按渲染 locale 格式化的运行
88
- * meta(latestStartedAt 为 null 时内置「暂无运行」文案)与品牌行(等同 PoweredBy,恒含、
89
- * 无拆除 prop);text 面输出标题行与 meta 行,不含品牌行
90
- * (docs/feature/reports/library/site-components.md「HeroCard」)。
91
- */
92
- export declare const HeroCard: ReportComponent<HeroCardProps>;
93
- /** `HeroCard` 的 props:标题 + `heroData()` 的产物,只有 data 形态。 */
94
- export interface HeroCardProps {
95
- title: LocalizedText;
96
- data: HeroData;
97
- className?: string;
98
- }
99
- /**
100
- * `Hero`:页首的站点标题区——标题、最后运行时间、快照合成来源,恒含品牌行。官方组合组件,
101
- * 与手写 `<HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} />`
102
- * 严格等价、没有私有能力;读 `ctx.report` 意味着输出跟随站点,要站点无关的标题区直接用
103
- * `HeroCard` 显式传值(docs/feature/reports/library/site-components.md「Hero」)。
104
- */
105
- export declare const Hero: ReportComponent<HeroProps>;
106
- /**
107
- * `PoweredBy`:唯一的品牌件,无 props 双面组件。web 面渲染指向 niceeval 官网的一行品牌色
108
- * 小字(`utm_source=report&utm_medium=powered-by`,`rel` 仅 `noopener` 以保留 Referer);
109
- * text 面零输出。没有任何配置——品牌契约是「提供一个组件,不给开关」:不想要品牌就不用
110
- * 这些组件、自己写替代组件(docs/feature/reports/library/site-components.md「PoweredBy」)。
111
- */
112
- export declare const PoweredBy: ReportComponent<Record<never, never>>;
113
- /** `ScopeWarnings` 的 props:spec 形态取宿主 Scope 的 warnings,data 形态收 `ScopeWarning[]`。 */
114
- export type ScopeWarningsProps = DataProps<readonly ScopeWarning[], Record<never, never>, ChromeProps>;
115
- /**
116
- * `ScopeWarnings`:选择警告区,警告的唯一呈现组件。把 Scope 携带的 `ScopeWarning[]`
117
- * 按「下一步动作」聚合渲染(带 experimentId 的按实验聚合、非实验作用域按 kind 聚合;
118
- * integrity 组在前);web 面组头带去重后的可复制命令、明细收原生 `<details>`(总条数 ≤ 3
119
- * 默认展开),text 面同构但不折叠。空警告集与裸 `Snapshot[]` 输入两面零输出
120
- * (docs/feature/reports/library/site-components.md「ScopeWarnings」)。
121
- */
122
- export declare const ScopeWarnings: ReportComponent<ScopeWarningsProps>;
123
- /** `CopyFixPrompt` 的 props:spec 形态无选项,data 形态收 `copyFixPromptData()` 的产物。 */
124
- export type CopyFixPromptProps = DataProps<CopyFixPromptData, Record<never, never>, ChromeProps>;
125
- /**
126
- * `CopyFixPrompt`:把当前范围的全部失败整理成一段可交给 coding agent 的修复 prompt。
127
- * prompt 在 resolve 阶段算好、烘进静态 HTML,无 JS 时在折叠块里完整可读,「复制」是增强层
128
- * 行为;`failures` 为 0 时两面零输出;text 面恒零输出——终端里的等价能力是 `show` 的
129
- * attempt 下钻命令本身(docs/feature/reports/library/site-components.md「CopyFixPrompt」)。
130
- */
131
- export declare const CopyFixPrompt: ReportComponent<CopyFixPromptProps>;
132
- /** `TraceWaterfall` 的 props:spec 形态无选项,data 形态收 `traceWaterfallData()` 的产物。 */
133
- export type TraceWaterfallProps = DataProps<readonly TraceWaterfallRow[], Record<never, never>, ChromeProps & {
134
- attemptHref?: (locator: AttemptLocator) => string;
135
- }>;
136
- /**
137
- * `TraceWaterfall`:每个 attempt 一行的执行时间瀑布,用 canonical OTel 字段显示被测 agent
138
- * 的原始 span(agent / model / tool)。web 面静态渲染顶层 span 分解条(失败 span 带失败
139
- * 标记),行链接 attempt 详情;text 面每 attempt 一行(locator、总耗时、span 计数与失败
140
- * 标记)+ 可复制的 `--timing` 下钻命令。trace 缺失的行照常出现并如实显示缺失;runner
141
- * 生命周期节点不进瀑布(docs/feature/reports/library/site-components.md「TraceWaterfall」)。
142
- */
143
- export declare const TraceWaterfall: ReportComponent<TraceWaterfallProps>;
144
- export type MetricTableProps = DataProps<TableData, MetricTableOptions, ChromeProps & {
145
- /** web 面在表格前渲染过滤输入框(enhance.js 接管);无 JS 时表格内容依旧完整。 */
146
- filter?: boolean;
147
- attemptHref?: (locator: AttemptLocator) => string;
148
- }>;
149
- /** 榜单:一行一个维度值、一列一个指标,回答「谁整体更好」。 */
150
- export declare const MetricTable: ReportComponent<MetricTableProps>;
151
- export type MetricMatrixProps = DataProps<MatrixData, MetricMatrixOptions, ChromeProps & {
152
- attemptHref?: (locator: AttemptLocator) => string;
153
- }>;
154
- export type MetricBarsProps = MetricMatrixProps;
155
- /** 逐题格子:行 × 列两个维度、格子里一个指标,回答「哪道题谁挂了」。 */
156
- export declare const MetricMatrix: ReportComponent<MetricMatrixProps>;
157
- /** 分组条形:同一份矩阵数据的另一种摆法;与 MetricMatrix 写同一份 spec 时只计算一次。 */
158
- export declare const MetricBars: ReportComponent<MetricBarsProps>;
159
- export type ScoreboardProps = DataProps<ScoreboardData, ScoreboardOptions, ChromeProps & {
160
- attemptHref?: (locator: AttemptLocator) => string;
161
- }>;
162
- /** 考试成绩单:总分 + 分科小计,固定分母、notRun / unscorable 分开如实报。 */
163
- export declare const Scoreboard: ReportComponent<ScoreboardProps>;
164
- export type MetricScatterProps = DataProps<ScatterData, MetricScatterOptions, ChromeProps & {
165
- pointHref?: (row: ScatterData["rows"][number]) => string;
166
- }>;
167
- /** 两个指标之间的取舍:每个点一个维度值,x / y 各一个指标,series 只决定颜色和分组。 */
168
- export declare const MetricScatter: ReportComponent<MetricScatterProps>;
169
- export type MetricLineProps = DataProps<LineData, MetricLineOptions, ChromeProps & {
170
- pointHref?: (row: LineData["rows"][number]) => string;
171
- }>;
172
- /** 趋势线:x 是 NumericAxis(参数轴),点身份 = (series, x)。 */
173
- export declare const MetricLine: ReportComponent<MetricLineProps>;
174
- export type DeltaTableProps = DataProps<DeltaData, DeltaTableOptions, ChromeProps & {
175
- attemptHref?: (locator: AttemptLocator) => string;
176
- }>;
177
- /** 成对对比:每行一对(字面声明或 pairsByFlag 派生),格子里 A、B、Δ 三个值。 */
178
- export declare const DeltaTable: ReportComponent<DeltaTableProps>;
179
- export {};