niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
package/src/report/web.ts DELETED
@@ -1,77 +0,0 @@
1
- // web 宿主(view --report)的装载入口:同一棵树走 web 面,renderToStaticMarkup 吐静态
2
- // HTML 烘进查看器的报告槽。只有这一侧真正 import react-dom(import 边界即运行时边界),
3
- // 所以本文件不从 niceeval/report 的入口 re-export —— 宿主与测试按源路径 import。
4
-
5
- import type { ReactNode } from "react";
6
- import { renderToStaticMarkup } from "react-dom/server";
7
- import type { AttemptLocator } from "../results/locator.ts";
8
- import type { Scope } from "../results/types.ts";
9
- import {
10
- resolveReportTree,
11
- runWithWebContext,
12
- validateReportTree,
13
- ResolveMemo,
14
- type WebContext,
15
- } from "./tree.ts";
16
- import { DEFAULT_REPORT_LOCALE, type ReportLocale } from "./locale.ts";
17
- import { buildReportMeta, pickReportPage, type ReportDefinition, type ReportHostContext } from "./report.ts";
18
-
19
- export interface StaticHtmlOptions {
20
- /** 渲染哪一页;缺省第一页。未命中抛 ReportPageNotFoundError。 */
21
- pageId?: string;
22
- /** 证据室深链;缺省用 view 的 attempt 路由 `#/attempt/@<locator>`(单段、不透明)。 */
23
- attemptHref?: (locator: AttemptLocator) => string;
24
- /** 官方组件 chrome 文案的 locale;默认 "en"。 */
25
- locale?: ReportLocale;
26
- }
27
-
28
- /**
29
- * web 宿主的装载语义:选页 → resolve(组合展开 + spec 取数,唯一的 await 边界)→
30
- * 树校验(与 text 宿主同一遍)→ 静态渲染 web 面。宿主不在报告树外另设警告通道——
31
- * 挑选警告的呈现件是 `ScopeWarnings` 组件,内建报告每页都放它,自定义报告放不放是
32
- * 作者义务(docs/feature/reports/architecture.md「Scope 是计算入口」)。
33
- */
34
- export async function renderReportToStaticHtml(
35
- definition: ReportDefinition,
36
- ctx: ReportHostContext,
37
- options?: StaticHtmlOptions,
38
- ): Promise<string> {
39
- const page = pickReportPage(definition, options?.pageId);
40
- const meta = buildReportMeta(definition, ctx.scope, page.id);
41
- const resolved = await resolveReportTree(page.content, {
42
- scope: ctx.scope,
43
- results: ctx.results,
44
- report: meta,
45
- memo: new ResolveMemo(),
46
- });
47
- validateReportTree(resolved);
48
- const webCtx: WebContext = {
49
- attemptHref: options?.attemptHref ?? ((locator) => `#/attempt/${locator}`),
50
- locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
51
- };
52
- return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved as ReactNode));
53
- }
54
-
55
- /**
56
- * 渲染一页报告树的 web 面(宿主逐页调用;页选择归宿主):resolve → validate → 静态渲染。
57
- * 挑选警告由页内的 `ScopeWarnings` 组件呈现,宿主不前置任何树外块。
58
- * ctx.report 是宿主规范化后的声明。
59
- */
60
- export async function renderReportTreeToStaticHtml(
61
- tree: import("./tree.ts").ReportNode,
62
- ctx: { scope: Scope; results: import("../results/types.ts").Results; report: import("./report.ts").ReportMeta },
63
- options?: { attemptHref?: (locator: AttemptLocator) => string; locale?: ReportLocale },
64
- ): Promise<string> {
65
- const resolved = await resolveReportTree(tree, {
66
- scope: ctx.scope,
67
- results: ctx.results,
68
- report: ctx.report,
69
- memo: new ResolveMemo(),
70
- });
71
- validateReportTree(resolved);
72
- const webCtx: WebContext = {
73
- attemptHref: options?.attemptHref ?? ((locator) => `#/attempt/${locator}`),
74
- locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
75
- };
76
- return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved as ReactNode));
77
- }
@@ -1,205 +0,0 @@
1
- // cases: docs/engineering/unit-tests/reports/cases.md
2
- // 「外壳、页面与 Tabs」——宿主装载规范化、页索引命令上下文与标题回退链
3
- // (契约:docs/feature/reports/library/shell.md)。
4
- // 页内树的 resolve / render 归报告库测试;这里只测宿主侧的规范化与选择逻辑(纯函数)。
5
-
6
- import { describe, expect, it } from "vitest";
7
- import {
8
- BUILT_IN_PAGE_TITLE,
9
- BUILT_IN_REPORT_TITLE,
10
- HostReportError,
11
- loadHostReport,
12
- localizeText,
13
- localizedTextEquals,
14
- normalizeHostReport,
15
- resolveReportTitle,
16
- showCommand,
17
- } from "./report-host.ts";
18
- import { otherPagesText } from "./render.ts";
19
- // dist-sourced:裸宿主装载的就是这份预编译产物的默认导出(show 与 view 同一条路),
20
- // raw-src import 会是另一份模块实例,引用等同断言必须对着 dist。
21
- import distBuiltInReport from "../../dist/report/built-in/index.js";
22
-
23
- const tree = { kind: "node" }; // 页 content 对宿主是不透明值,规范化不解析树
24
-
25
- describe("裸宿主装载内建报告", () => {
26
- it("缺省(无 --report)装载 niceeval/report/built-in 的默认导出:三页与其 content 同引用", async () => {
27
- const host = await loadHostReport(process.cwd(), undefined);
28
- const builtIn = distBuiltInReport as { pages: readonly { id: string; content: unknown }[] };
29
- expect(host.pages.map((p) => p.id)).toEqual(["report", "attempts", "traces"]);
30
- expect(builtIn.pages.map((p) => p.id)).toEqual(["report", "attempts", "traces"]);
31
- for (let i = 0; i < host.pages.length; i++) {
32
- expect(host.pages[i]!.content).toBe(builtIn.pages[i]!.content); // 同一份默认导出,不是复制品
33
- }
34
- });
35
- });
36
-
37
- describe("装载规范化:外壳 + 非空页列表", () => {
38
- it("content 缩写展开为唯一页 id `report`,页名是内置页名「报告 / Report」", () => {
39
- const report = normalizeHostReport({ kind: "report", content: tree }, "reports/frontier.tsx");
40
- expect(report.pages).toHaveLength(1);
41
- expect(report.pages[0]).toMatchObject({ id: "report", title: BUILT_IN_PAGE_TITLE, content: tree });
42
- expect(report.links).toEqual([]);
43
- });
44
-
45
- it("pages 形态按声明序保留页列表与外壳字段", () => {
46
- const report = normalizeHostReport(
47
- {
48
- kind: "report",
49
- title: { en: "Memory Evals", "zh-CN": "记忆能力评测" },
50
- links: [{ label: "GitHub", href: "https://example.com" }],
51
- footer: "Published nightly.",
52
- pages: [
53
- { id: "overview", title: { en: "Overview", "zh-CN": "总览" }, content: tree },
54
- { id: "exam", title: { en: "Exam", "zh-CN": "成绩单" }, content: tree },
55
- ],
56
- },
57
- "reports/site.tsx",
58
- );
59
- expect(report.pages.map((p) => p.id)).toEqual(["overview", "exam"]);
60
- expect(report.title).toEqual({ en: "Memory Evals", "zh-CN": "记忆能力评测" });
61
- expect(report.footer).toBe("Published nightly.");
62
- });
63
-
64
- it("content 与 pages 恰好声明一个:同给 / 同缺都报错,文案给出 extends: standard 下一步", () => {
65
- for (const bad of [
66
- { kind: "report", content: tree, pages: [{ id: "a", title: "A", content: tree }] },
67
- { kind: "report", title: "T" },
68
- ]) {
69
- expect(() => normalizeHostReport(bad, "reports/site.tsx")).toThrow(HostReportError);
70
- expect(() => normalizeHostReport(bad, "reports/site.tsx")).toThrow(/niceeval\/report\/built-in/);
71
- }
72
- });
73
-
74
- it("空 pages 列表 / 重复 page id / 非法 page id 在装载期报错", () => {
75
- expect(() => normalizeHostReport({ kind: "report", pages: [] }, "r.tsx")).toThrow(/non-empty/);
76
- expect(() =>
77
- normalizeHostReport(
78
- { kind: "report", pages: [{ id: "exam", title: "A", content: tree }, { id: "exam", title: "B", content: tree }] },
79
- "r.tsx",
80
- ),
81
- ).toThrow(/duplicate page id "exam"/);
82
- for (const id of ["Exam", "a/b"]) {
83
- expect(() =>
84
- normalizeHostReport({ kind: "report", pages: [{ id, title: "A", content: tree }] }, "r.tsx"),
85
- ).toThrow(/invalid/);
86
- }
87
- });
88
-
89
- it("默认导出不是 defineReport 产物:完整用户反馈", () => {
90
- expect(() => normalizeHostReport({ some: "object" }, "reports/bad.tsx")).toThrow(
91
- /does not default-export a report/,
92
- );
93
- });
94
-
95
- it("ReportLink.icon 是 { svg: string }:合法形状原样透传;无类型 JS 传其它形状装载报错", () => {
96
- const svg = '<svg viewBox="0 0 16 16"><path d="M0 0h16v16z"/></svg>';
97
- const report = normalizeHostReport(
98
- { kind: "report", content: tree, links: [{ label: "GitHub", href: "https://example.com", icon: { svg } }] },
99
- "reports/site.tsx",
100
- );
101
- expect(report.links[0]!.icon).toEqual({ svg });
102
-
103
- // ReactNode / 组件 / 裸字符串都不是 { svg: string },装载期以完整用户反馈拒绝。
104
- const reactNode = { $$typeof: Symbol.for("react.transitional.element"), type: "svg", props: {} };
105
- for (const icon of [reactNode, "<svg/>", { svg: 42 }, { svg: "" }]) {
106
- expect(() =>
107
- normalizeHostReport(
108
- { kind: "report", content: tree, links: [{ label: "GitHub", href: "https://example.com", icon }] },
109
- "reports/site.tsx",
110
- ),
111
- ).toThrow(/icon" must be \{ svg: string \}/);
112
- }
113
- });
114
-
115
- it("旧 build 函数形态(集成前桥接)恒为单页 report", () => {
116
- const legacy = { build: () => tree };
117
- const report = normalizeHostReport(legacy, "the built-in report");
118
- expect(report.pages.map((p) => p.id)).toEqual(["report"]);
119
- expect(report.pages[0]!.content).toBe(legacy);
120
- });
121
- });
122
-
123
- describe("标题回退链:def.title → 唯一且相同的快照 name → 内置文案「Eval 运行结果 / Eval Results」", () => {
124
- it("def.title 优先", () => {
125
- expect(resolveReportTitle({ en: "T" }, [{ name: "S" }])).toEqual({ en: "T" });
126
- });
127
-
128
- it("无 def.title 时取 Scope 中唯一且相同(深相等,键顺序无关)的非空快照 name", () => {
129
- expect(resolveReportTitle(undefined, [{ name: { en: "S", "zh-CN": "斯" } }, { name: { "zh-CN": "斯", en: "S" } }]))
130
- .toEqual({ en: "S", "zh-CN": "斯" });
131
- expect(resolveReportTitle(undefined, [{}, { name: "Only" }])).toBe("Only");
132
- });
133
-
134
- it("多个不同 name(en 相同、zh-CN 不同也算不同)不随机挑,落内置文案;全无 name 亦然", () => {
135
- expect(
136
- resolveReportTitle(undefined, [{ name: { en: "S", "zh-CN": "甲" } }, { name: { en: "S", "zh-CN": "乙" } }]),
137
- ).toEqual(BUILT_IN_REPORT_TITLE);
138
- expect(resolveReportTitle(undefined, [{}, {}])).toEqual(BUILT_IN_REPORT_TITLE);
139
- expect(resolveReportTitle("", [])).toEqual(BUILT_IN_REPORT_TITLE); // 空串标题不算声明
140
- // 链终点是内置文案(shell.md),不是产品名——品牌位固定 NiceEval,不吃标题。
141
- expect(BUILT_IN_REPORT_TITLE).toEqual({ en: "Eval Results", "zh-CN": "Eval 运行结果" });
142
- });
143
-
144
- it("LocalizedText 深相等按字段值,不看键顺序", () => {
145
- expect(localizedTextEquals({ a: "1", b: "2" }, { b: "2", a: "1" })).toBe(true);
146
- expect(localizedTextEquals({ a: "1" }, { a: "1", b: "2" })).toBe(false);
147
- });
148
- });
149
-
150
- describe("LocalizedText 回退:locale → en → 键字典序第一个非空值", () => {
151
- it("三级回退各自命中", () => {
152
- expect(localizeText({ "zh-CN": "中", en: "E" }, "zh-CN")).toBe("中");
153
- expect(localizeText({ "zh-CN": "中", en: "E" }, "fr")).toBe("E");
154
- expect(localizeText({ "zh-TW": "繁", ja: "日" }, "en")).toBe("日"); // ja < zh-TW 字典序
155
- expect(localizeText("plain", "en")).toBe("plain");
156
- expect(localizeText({}, "en")).toBeUndefined();
157
- });
158
- });
159
-
160
- describe("其余页索引与索引命令上下文", () => {
161
- const report = normalizeHostReport(
162
- {
163
- kind: "report",
164
- title: { en: "Memory Evals", "zh-CN": "记忆能力评测" },
165
- links: [{ label: "GitHub", href: "https://example.com", icon: { svg: "<svg data-mark></svg>" } }],
166
- pages: [
167
- { id: "overview", title: { en: "Overview", "zh-CN": "总览" }, content: tree },
168
- { id: "exam", title: { en: "Exam", "zh-CN": "成绩单" }, content: tree },
169
- ],
170
- },
171
- "reports/site.tsx",
172
- );
173
-
174
- it("只列未渲染的页,索引命令保留当前 --results / --report 与位置参数,复制即可复现下一层视图", () => {
175
- // 渲染的是 overview,其余页索引只含 exam 一行——与「渲染初始页 + 尾部附其余页索引」
176
- // 的新行为一致(docs/feature/reports/show/reports.md Case 2)。
177
- const text = otherPagesText({
178
- otherPages: report.pages.filter((p) => p.id !== "overview").map((p) => ({ id: p.id, title: p.title })),
179
- command: { patterns: [], results: "tmp/published-results", report: "reports/site.tsx" },
180
- locale: "zh-CN",
181
- });
182
- expect(text).toContain("其余页:");
183
- expect(text).toContain("niceeval show --results tmp/published-results --report reports/site.tsx --page exam");
184
- expect(text).toContain("成绩单");
185
- expect(text).not.toContain("总览");
186
- expect(text).not.toContain("--page overview");
187
- });
188
-
189
- it("show 不消费 links:其余页索引不含 icon svg 与 href(icon 是 web 面属性)", () => {
190
- const text = otherPagesText({
191
- otherPages: report.pages.filter((p) => p.id !== "overview").map((p) => ({ id: p.id, title: p.title })),
192
- command: { patterns: [] },
193
- locale: "en",
194
- });
195
- expect(text).toContain("Other pages:");
196
- expect(text).not.toContain("<svg");
197
- expect(text).not.toContain("https://example.com");
198
- });
199
-
200
- it("showCommand 按序携带位置参数与 --exp / --results / --report / --page", () => {
201
- expect(
202
- showCommand({ patterns: ["memory/swelancer"], experiment: "dev-e2b", report: "reports/site.tsx", page: "exam" }),
203
- ).toBe("niceeval show memory/swelancer --exp dev-e2b --report reports/site.tsx --page exam");
204
- });
205
- });
@@ -1,389 +0,0 @@
1
- // 报告装载的宿主联系面:show 与 view 共用(view 从这里 import;两个宿主对同一份
2
- // defineReport 产物必须得到同一个规范化结果,见 docs/feature/reports/library/shell.md)。
3
- //
4
- // 契约(docs/feature/reports/library/shell.md「行为约束」/ architecture.md「外壳与页:装载规范化」):
5
- // - `--report` 文件默认导出恒为 defineReport 产物;裸宿主装载 `niceeval/report/built-in` 的默认导出,
6
- // 走同一条「装载 → resolve → validate → render」管线。
7
- // - 装载规范化唯一产物是「外壳 + 非空页列表」:`defineReport(树)` ≡ `{ content: 树 }` ≡
8
- // `pages: [{ id: "report", title: 内置页名, content: 树 }]`。
9
- // - 标题回退单点:def.title → Scope 中唯一且相同(LocalizedText 深相等)的非空快照 name →
10
- // 内置文案「Eval 运行结果 / Eval Results」。宿主落点是浏览器标题与 show 页索引标题行;
11
- // 页内 hero 标题由 Hero 组件消费同一取值链(ctx.report.title),宿主页头无品牌位。
12
- // - LocalizedText 回退:当前 locale → en → 按 locale 键字典序的第一个非空值。
13
- //
14
- // ⚠ 集成状态:src/report/** 正被并行重写(plan/reports-redesign-implementation.md)。这里把宿主
15
- // 需要的新报告 runtime API 收敛成唯一联系面,并对「新 dist 尚未产出」提供旧 API 桥接
16
- // (旧 ReportDefinition 是 build 函数形态,恒为单页)。集成阶段的对接点(搜索 INTEGRATION):
17
- // 1. `dist/report/built-in/index.js` 默认导出(内建报告入口;恒为新格式,无桥接——
18
- // 它随本包源码一起构建,不存在旧产物)。
19
- // 2. `normalizeReport(definition)`:装载规范化(外壳 + 非空页列表);旧桥接对 build 形态自行包单页。
20
- // 3. `renderReportTreeToText(tree, ctx, options)` / `renderReportTreeToStaticHtml(tree, ctx, options)`:
21
- // 逐页渲染入口(ctx = { scope, results, report });旧桥接调 renderReportToText /
22
- // renderReportToStaticHtml(definition 整体)。
23
-
24
- import type { Results } from "../results/index.ts";
25
- import type { LocalizedText } from "../types.ts";
26
-
27
- /** 页:宿主寻址单位(`--page <id>` / `#/page/<id>`)。content 是报告树(旧桥接下是整个旧 definition)。 */
28
- export interface HostReportPage {
29
- id: string;
30
- title: LocalizedText;
31
- content: unknown;
32
- }
33
-
34
- export interface HostReportLink {
35
- label: LocalizedText;
36
- href: string;
37
- /**
38
- * 可选内联 SVG 字标(shell.md「字段穷尽」):web 面渲染在 label 前,静态导出原样内联;
39
- * 不收组件(外壳声明经 viewData 序列化边界);show 不消费。
40
- */
41
- icon?: { svg: string };
42
- }
43
-
44
- /** `{src}` 与 `{inline}` 两种形态不可同时出现(shell.md「字段穷尽」)。 */
45
- export type HostReportAsset = { src: string; inline?: never } | { inline: string; src?: never };
46
-
47
- /**
48
- * 结构化 head 标签(shell.md「字段穷尽」):白名单闭集(meta/link/script/style),
49
- * attrs 值 true 渲染裸布尔属性、字符串转义后渲染 `key="value"`;script/style 的
50
- * children 原样落进标签。形状与闭合序列在 defineReport 装载期已校验。
51
- */
52
- export type HostHeadTag =
53
- | { tag: "meta" | "link"; attrs: Record<string, string | true>; children?: never }
54
- | { tag: "script" | "style"; attrs?: Record<string, string | true>; children?: string };
55
-
56
- /** 装载规范化产物:外壳 + 非空页列表。show 只消费 title / pages;其余是 web 面属性。 */
57
- export interface HostReport {
58
- title?: LocalizedText;
59
- links: HostReportLink[];
60
- footer?: LocalizedText;
61
- head: HostHeadTag[];
62
- scripts: HostReportAsset[];
63
- styles: HostReportAsset[];
64
- pages: HostReportPage[];
65
- }
66
-
67
- /** 单页缩写展开出的唯一页使用内置页名(shell.md:「报告 / Report」)。 */
68
- export const BUILT_IN_PAGE_TITLE: LocalizedText = { en: "Report", "zh-CN": "报告" };
69
- export const SINGLE_PAGE_ID = "report";
70
-
71
- /** 标题回退链的终点:内置文案「Eval 运行结果 / Eval Results」(shell.md「行为约束」)。 */
72
- export const BUILT_IN_REPORT_TITLE: LocalizedText = { en: "Eval Results", "zh-CN": "Eval 运行结果" };
73
-
74
- /** 规范化报告声明的只读注入(组合组件 ctx.report;ReportMeta,见 library/layout.md)。 */
75
- export interface HostReportMeta {
76
- title: LocalizedText;
77
- links: HostReportLink[];
78
- footer?: LocalizedText;
79
- pages: { id: string; title: LocalizedText }[];
80
- pageId: string;
81
- }
82
-
83
- /** 可预期的装载用户错误(与 ReportLoadError 同待遇:打一句直说问题与下一步,不抛堆栈)。 */
84
- export class HostReportError extends Error {}
85
-
86
- // ───────────────────────── LocalizedText ─────────────────────────
87
-
88
- /**
89
- * LocalizedText 的确定回退(shell.md「行为约束」):当前 locale → en → 按 locale 键字典序的
90
- * 第一个非空值。对象没有任何非空值时返回 undefined(装载期应当已报错,渲染兜底不再抛)。
91
- */
92
- export function localizeText(text: LocalizedText | undefined, locale: string): string | undefined {
93
- if (text === undefined) return undefined;
94
- if (typeof text === "string") return text || undefined;
95
- const exact = text[locale];
96
- if (exact) return exact;
97
- if (text.en) return text.en;
98
- for (const key of Object.keys(text).sort()) {
99
- if (text[key]) return text[key];
100
- }
101
- return undefined;
102
- }
103
-
104
- /** LocalizedText 深相等:按字段值比较,对象键顺序不影响结果(shell.md「标题回退必须确定」)。 */
105
- export function localizedTextEquals(a: LocalizedText, b: LocalizedText): boolean {
106
- if (typeof a === "string" || typeof b === "string") return a === b;
107
- const keysA = Object.keys(a).sort();
108
- const keysB = Object.keys(b).sort();
109
- if (keysA.length !== keysB.length) return false;
110
- return keysA.every((key, i) => key === keysB[i] && a[key] === b[key]);
111
- }
112
-
113
- /** LocalizedText 是否有任何非空值(空串 / 全空对象都算空)。 */
114
- function hasText(text: LocalizedText | undefined): text is LocalizedText {
115
- if (text === undefined) return false;
116
- if (typeof text === "string") return text.length > 0;
117
- return Object.values(text).some((v) => typeof v === "string" && v.length > 0);
118
- }
119
-
120
- /**
121
- * 标题回退单点(show 页索引标题与 view hero / 浏览器标题共用):def.title → Scope 中唯一且相同
122
- * (深相等)的非空快照 name → 内置文案「Eval 运行结果 / Eval Results」。多个不同 name 时
123
- * 不随机挑一个,直接落到内置文案。
124
- */
125
- export function resolveReportTitle(
126
- defTitle: LocalizedText | undefined,
127
- snapshots: readonly { name?: LocalizedText }[],
128
- ): LocalizedText {
129
- if (hasText(defTitle)) return defTitle;
130
- let unique: LocalizedText | undefined;
131
- for (const snapshot of snapshots) {
132
- if (!hasText(snapshot.name)) continue;
133
- if (unique === undefined) {
134
- unique = snapshot.name;
135
- } else if (!localizedTextEquals(unique, snapshot.name)) {
136
- return BUILT_IN_REPORT_TITLE;
137
- }
138
- }
139
- return unique ?? BUILT_IN_REPORT_TITLE;
140
- }
141
-
142
- /** 组合组件 ctx.report 的形状(走完回退链的 title;scripts / styles 不进)。 */
143
- export function reportMetaFor(
144
- report: HostReport,
145
- pageId: string,
146
- snapshots: readonly { name?: LocalizedText }[],
147
- ): HostReportMeta {
148
- return {
149
- title: resolveReportTitle(report.title, snapshots),
150
- links: report.links,
151
- ...(report.footer !== undefined ? { footer: report.footer } : {}),
152
- pages: report.pages.map((p) => ({ id: p.id, title: p.title })),
153
- pageId,
154
- };
155
- }
156
-
157
- // ───────────────────────── 装载与规范化 ─────────────────────────
158
-
159
- /** 旧(集成前)ReportDefinition:build 函数形态,恒为单页。集成阶段删除。 */
160
- interface LegacyReportDefinition {
161
- build: (ctx: unknown) => unknown;
162
- }
163
-
164
- function isLegacyDefinition(value: unknown): value is LegacyReportDefinition {
165
- return (
166
- (typeof value === "object" || typeof value === "function") &&
167
- value !== null &&
168
- typeof (value as Partial<LegacyReportDefinition>).build === "function"
169
- );
170
- }
171
-
172
- const PAGE_ID_PATTERN = /^[a-z0-9-]+$/;
173
-
174
- /**
175
- * 装载期校验 `ReportLink.icon`(shell.md「字段穷尽」):唯一合法形状是 `{ svg: string }`。
176
- * 无类型 JS 传组件 / ReactNode / 裸字符串都在这里被完整用户反馈拒绝——外壳声明要过
177
- * viewData 的序列化边界,ReactNode 过不去,可序列化是外壳契约的一部分。
178
- */
179
- function assertLinkIcon(link: unknown, sourceLabel: string): void {
180
- const icon = (link as { icon?: unknown })?.icon;
181
- if (icon === undefined) return;
182
- const svg = (icon as { svg?: unknown })?.svg;
183
- if (typeof icon !== "object" || icon === null || typeof svg !== "string" || svg.length === 0) {
184
- throw new HostReportError(
185
- `${sourceLabel}: a link "icon" must be { svg: string } — an inline SVG string rendered before the label. ` +
186
- "Components and React nodes are not accepted: the shell declaration crosses a serialization boundary. " +
187
- 'Write e.g. icon: { svg: "<svg …>…</svg>" }.',
188
- );
189
- }
190
- }
191
-
192
- /**
193
- * 装载规范化:把 defineReport 产物折成「外壳 + 非空页列表」。三种入参写法
194
- * (树 / `content:` / `pages:`)展开成同一形状;page id 的唯一性与字符纪律在这里校验
195
- * (shell.md「校验分两期」的装载期部分)。
196
- */
197
- export function normalizeHostReport(definition: unknown, sourceLabel: string): HostReport {
198
- // INTEGRATION(2):新 defineReport 产物(kind === "report")。规范化后的外壳与页列表
199
- // 若由新报告 runtime 导出的 normalizeReport 承担,这里改为直接调它;当前按 docs 的
200
- // 字段穷尽读取产物自带的声明。
201
- if (typeof definition === "object" && definition !== null && (definition as { kind?: unknown }).kind === "report") {
202
- const def = definition as {
203
- title?: LocalizedText;
204
- links?: HostReportLink[];
205
- footer?: LocalizedText;
206
- head?: HostHeadTag[];
207
- scripts?: HostReportAsset[];
208
- styles?: HostReportAsset[];
209
- content?: unknown;
210
- pages?: { id: string; title: LocalizedText; content: unknown }[];
211
- };
212
- const hasContent = def.content !== undefined;
213
- const hasPages = def.pages !== undefined;
214
- if (hasContent === hasPages) {
215
- // content / pages 同缺或同给:装载期完整用户反馈(defineReport 产物恒已折叠,这里只拦
216
- // 手搓 kind:"report" 的无类型输入),下一步指向内建视图的 extends 复用。
217
- throw new HostReportError(
218
- `${sourceLabel}: a report declares exactly one of "content" or "pages" — ` +
219
- (hasContent ? "it declares both. " : "it declares neither. ") +
220
- `To render the built-in report, write extends: standard (import { standard } from "niceeval/report/built-in").`,
221
- );
222
- }
223
- const pages: HostReportPage[] = hasPages
224
- ? def.pages!.map((p) => ({ id: p.id, title: p.title, content: p.content }))
225
- : [{ id: SINGLE_PAGE_ID, title: BUILT_IN_PAGE_TITLE, content: def.content }];
226
- if (pages.length === 0) {
227
- throw new HostReportError(`${sourceLabel}: "pages" must be a non-empty list of report pages.`);
228
- }
229
- const seen = new Set<string>();
230
- for (const page of pages) {
231
- if (!PAGE_ID_PATTERN.test(page.id)) {
232
- throw new HostReportError(
233
- `${sourceLabel}: page id "${page.id}" is invalid. Page ids use lowercase letters, digits and hyphens.`,
234
- );
235
- }
236
- if (seen.has(page.id)) {
237
- throw new HostReportError(`${sourceLabel}: duplicate page id "${page.id}". Page ids must be unique per file.`);
238
- }
239
- seen.add(page.id);
240
- }
241
- for (const link of def.links ?? []) assertLinkIcon(link, sourceLabel);
242
- return {
243
- ...(def.title !== undefined ? { title: def.title } : {}),
244
- links: def.links ?? [],
245
- ...(def.footer !== undefined ? { footer: def.footer } : {}),
246
- head: def.head ?? [],
247
- scripts: def.scripts ?? [],
248
- styles: def.styles ?? [],
249
- pages,
250
- };
251
- }
252
-
253
- // INTEGRATION(2) 桥接:旧 build 函数形态(集成前的 dist);恒为单页,无外壳字段。
254
- if (isLegacyDefinition(definition)) {
255
- return {
256
- links: [],
257
- head: [],
258
- scripts: [],
259
- styles: [],
260
- pages: [{ id: SINGLE_PAGE_ID, title: BUILT_IN_PAGE_TITLE, content: definition }],
261
- };
262
- }
263
-
264
- throw new HostReportError(
265
- `${sourceLabel} does not default-export a report. Export default defineReport(<ExperimentComparison />) (or defineReport({ title, content })) from "niceeval/report".`,
266
- );
267
- }
268
-
269
- /**
270
- * 装载宿主报告:`--report <file>` 走 loadReportFile;缺省(裸 show / 裸 view)装载内建报告的
271
- * 默认导出。两条路都进 normalizeHostReport,同一条装载管线。
272
- */
273
- export async function loadHostReport(
274
- cwd: string,
275
- reportPath: string | undefined,
276
- options?: { freshImport?: boolean },
277
- ): Promise<HostReport> {
278
- if (reportPath !== undefined) {
279
- const { loadReportFile } = await import("../../dist/report/load.js");
280
- const definition = await loadReportFile(cwd, reportPath, options);
281
- return normalizeHostReport(definition, reportPath);
282
- }
283
- return normalizeHostReport(await loadBuiltInDefinition(), "the built-in report");
284
- }
285
-
286
- /** 内建报告即 `niceeval/report/built-in` 的默认导出(dist 预编译产物,`prepare` 时构建)。 */
287
- async function loadBuiltInDefinition(): Promise<unknown> {
288
- const mod = await import("../../dist/report/built-in/index.js");
289
- return mod.default;
290
- }
291
-
292
- // ───────────────────────── 逐页渲染 ─────────────────────────
293
-
294
- export interface HostRenderContext {
295
- /** 宿主按官方现刻水位挑好的 Scope(集成前类型名仍是 Selection;两者同物)。 */
296
- scope: unknown;
297
- results: Results;
298
- report: HostReportMeta;
299
- }
300
-
301
- export interface HostTextRenderOptions {
302
- width?: number;
303
- locale?: string;
304
- /**
305
- * 索引命令的完整上下文(docs/feature/reports/show/reports.md:「索引命令携带完整上下文」):
306
- * 组索引 / 页索引输出的每一条可复制命令都保留当前 --results / --report / --page 与位置参数。
307
- * INTEGRATION(3):由新 text 面消费;旧桥接的 renderReportToText 不认识该字段,原样忽略。
308
- */
309
- commandContext?: HostCommandContext;
310
- }
311
-
312
- export interface HostCommandContext {
313
- patterns: string[];
314
- results?: string;
315
- experiment?: string;
316
- report?: string;
317
- page?: string;
318
- }
319
-
320
- /** 按上下文拼一条可复制的 show 命令(页索引 / 组索引共用的携带规则)。 */
321
- export function showCommand(ctx: HostCommandContext, extra: string[] = []): string {
322
- const parts = ["niceeval show", ...ctx.patterns];
323
- if (ctx.experiment !== undefined) parts.push(`--exp ${ctx.experiment}`);
324
- if (ctx.results !== undefined) parts.push(`--results ${ctx.results}`);
325
- if (ctx.report !== undefined) parts.push(`--report ${ctx.report}`);
326
- if (ctx.page !== undefined) parts.push(`--page ${ctx.page}`);
327
- parts.push(...extra);
328
- return parts.join(" ");
329
- }
330
-
331
- /**
332
- * 渲染一页的 text 面。INTEGRATION(3):新 runtime 的入口是
333
- * `renderReportTreeToText(tree, { scope, results, report }, options)`;旧桥接下 content 是
334
- * 整个旧 definition,调 renderReportToText(definition, { selection, results }, options)。
335
- */
336
- export async function renderHostPageText(
337
- page: HostReportPage,
338
- ctx: HostRenderContext,
339
- options: HostTextRenderOptions,
340
- ): Promise<string> {
341
- const mod = (await import("../../dist/report/report.js")) as Record<string, unknown>;
342
- const renderTree = mod.renderReportTreeToText as
343
- | ((tree: unknown, ctx: HostRenderContext, options: HostTextRenderOptions) => Promise<string>)
344
- | undefined;
345
- if (typeof renderTree === "function" && !isLegacyDefinition(page.content)) {
346
- return renderTree(page.content, ctx, options);
347
- }
348
- if (isLegacyDefinition(page.content)) {
349
- const renderLegacy = mod.renderReportToText as (
350
- definition: unknown,
351
- ctx: { selection: unknown; results: Results },
352
- options: { width?: number; locale?: string },
353
- ) => Promise<string>;
354
- return renderLegacy(
355
- page.content,
356
- { selection: ctx.scope, results: ctx.results },
357
- { ...(options.width !== undefined ? { width: options.width } : {}), ...(options.locale !== undefined ? { locale: options.locale } : {}) },
358
- );
359
- }
360
- throw new HostReportError(
361
- "The installed report runtime cannot render this report page yet (renderReportTreeToText is missing). Rebuild dist/report with `pnpm run build:report`.",
362
- );
363
- }
364
-
365
- /** 渲染一页的 web 面(静态 HTML)。INTEGRATION(3) 同上,web 侧入口在 dist/report/web.js。 */
366
- export async function renderHostPageHtml(
367
- page: HostReportPage,
368
- ctx: HostRenderContext,
369
- options: { locale: string },
370
- ): Promise<string> {
371
- const mod = (await import("../../dist/report/web.js")) as Record<string, unknown>;
372
- const renderTree = mod.renderReportTreeToStaticHtml as
373
- | ((tree: unknown, ctx: HostRenderContext, options: { locale: string }) => Promise<string>)
374
- | undefined;
375
- if (typeof renderTree === "function" && !isLegacyDefinition(page.content)) {
376
- return renderTree(page.content, ctx, options);
377
- }
378
- if (isLegacyDefinition(page.content)) {
379
- const renderLegacy = mod.renderReportToStaticHtml as (
380
- definition: unknown,
381
- ctx: { selection: unknown; results: Results },
382
- options: { locale: string },
383
- ) => Promise<string>;
384
- return renderLegacy(page.content, { selection: ctx.scope, results: ctx.results }, options);
385
- }
386
- throw new HostReportError(
387
- "The installed report runtime cannot render this report page yet (renderReportTreeToStaticHtml is missing). Rebuild dist/report with `pnpm run build:report`.",
388
- );
389
- }