niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
package/src/view/site.ts CHANGED
@@ -8,14 +8,22 @@ import { existsSync } from "node:fs";
8
8
  import { createHash } from "node:crypto";
9
9
  import { dirname, join, resolve } from "node:path";
10
10
  import { loadViewScan, type ResolvedHeadTag, type ViewScan, type ViewScanOptions } from "./data.ts";
11
- import { localizeText } from "../show/report-host.ts";
11
+ import { localizeText } from "../report/runtime/host.ts";
12
+ import type { AttemptHandle } from "../results/index.ts";
13
+ import type { AttemptLocator } from "../results/locator.ts";
12
14
 
13
- /** 站点产物清单里的一个文件:现算内容,或指向结果根内的原文件。 */
15
+ /**
16
+ * 站点产物清单里的一个文件:现算内容(content)、指向结果根内的原文件(file),或延迟到
17
+ * 真正被请求 / 导出时才求值的产出器(lazy)。lazy 专为 `attempt/<locator>.html` 准备——
18
+ * 每个可达 locator 一份,数量可能很大,不能像 index.html 那样在建清单时就全部渲染
19
+ * (architecture.md「管线以 page 实例为单位执行」:本地 server 按请求求值并缓存进当前
20
+ * plan,`writeSite` 在写任何文件前对全部产出器求值一次)。
21
+ */
14
22
  export interface SiteFile {
15
23
  /** 站点相对路径(posix),如 `index.html`、`artifact/<base>/events.json`。 */
16
24
  path: string;
17
25
  contentType: string;
18
- source: { kind: "content"; body: string } | { kind: "file"; abs: string };
26
+ source: { kind: "content"; body: string } | { kind: "file"; abs: string } | { kind: "lazy"; produce: () => Promise<string> };
19
27
  }
20
28
 
21
29
  export interface SitePlan {
@@ -63,15 +71,51 @@ export async function planSite(input?: string, opts: ViewScanOptions = {}): Prom
63
71
  files.set(path, { path, contentType: JSON_TYPE, source: { kind: "content", body: JSON.stringify(sources ?? []) } });
64
72
  }
65
73
  }
74
+
75
+ // attempt/<locator>.html:报告声明了 attempt-input page 时才出现,收窄后有效根内每个可达
76
+ // locator 各一份(view.md「静态导出」)。头资产的相对路径要从 attempt/ 子目录回退一层,
77
+ // 因此单独物化一份 `../` 前缀版本(与 index.html 的根相对版本共用同一份 files 内容寻址,
78
+ // 同一份资产只按内容哈希写一次)。每份文档的 IO/resolve 延迟到真正被请求或导出时才发生——
79
+ // 可达 locator 数量可能很大,不能像 index.html 一样在建清单时就全部渲染
80
+ // (architecture.md「管线以 page 实例为单位执行」)。
81
+ if (scan.attemptPages) {
82
+ const { locators, render } = scan.attemptPages;
83
+ const nestedHeadHtml = await materializeHeadAssets(scan.shellAssets.head, files, "../");
84
+ for (const [locator, handle] of locators) {
85
+ const path = `attempt/${locator}.html`;
86
+ files.set(path, {
87
+ path,
88
+ contentType: HTML_TYPE,
89
+ source: { kind: "lazy", produce: () => renderAttemptDocument(scan, locator, handle, render, nestedHeadHtml) },
90
+ });
91
+ }
92
+ }
93
+
66
94
  return { files, scan };
67
95
  }
68
96
 
69
- /** 把站点产物清单写盘(`--out`)。输入本身已是导出布局(对上次导出的目录重新生成)时原文件不自拷。 */
97
+ /**
98
+ * 把站点产物清单写盘(`--out`)。输入本身已是导出布局(对上次导出的目录重新生成)时原文件不自拷。
99
+ * 先对全部 lazy 产出器求值(任一份 attempt page resolve 失败就在这一步整体抛出,不写入任何
100
+ * 文件——architecture.md「写 writeSite 的整体失败语义」:静态导出不留半套目录),求值期间
101
+ * 头资产可能按内容哈希追加注册进 plan.files(见 planSite 的 `../` 前缀变体),因此写盘前
102
+ * 重新遍历一次清单,不用求值前的快照。
103
+ */
70
104
  export async function writeSite(plan: SitePlan, outDir: string): Promise<void> {
105
+ const lazyBodies = new Map<string, string>();
106
+ await Promise.all(
107
+ [...plan.files.values()]
108
+ .filter((file): file is SiteFile & { source: { kind: "lazy"; produce: () => Promise<string> } } => file.source.kind === "lazy")
109
+ .map(async (file) => {
110
+ lazyBodies.set(file.path, await file.source.produce());
111
+ }),
112
+ );
71
113
  for (const file of plan.files.values()) {
72
114
  const dest = join(outDir, file.path);
73
115
  await mkdir(dirname(dest), { recursive: true });
74
- if (file.source.kind === "content") {
116
+ if (file.source.kind === "lazy") {
117
+ await writeFile(dest, lazyBodies.get(file.path)!, "utf-8");
118
+ } else if (file.source.kind === "content") {
75
119
  await writeFile(dest, file.source.body, "utf-8");
76
120
  } else if (resolve(file.source.abs) !== resolve(dest)) {
77
121
  await copyFile(file.source.abs, dest);
@@ -79,9 +123,14 @@ export async function writeSite(plan: SitePlan, outDir: string): Promise<void> {
79
123
  }
80
124
  }
81
125
 
82
- /** 取清单中一个文件的字节(server 响应体与写盘内容同源;file 类缺失时返回 undefined,由宿主 404)。 */
126
+ /**
127
+ * 取清单中一个文件的字节(server 响应体与写盘内容同源;file 类缺失时返回 undefined,由宿主 404)。
128
+ * lazy 类每次调用都重新求值(不在这里缓存)——server.ts 按「同一路径同一 plan 生命周期内
129
+ * 不重复计算」的语义自行把求值结果写回 plan.files,这里保持纯函数,不持有 plan 状态。
130
+ */
83
131
  export async function readSiteFile(file: SiteFile): Promise<string | Buffer | undefined> {
84
132
  if (file.source.kind === "content") return file.source.body;
133
+ if (file.source.kind === "lazy") return file.source.produce();
85
134
  try {
86
135
  // 原字节读取:assets/ 里的 head 资产可以是二进制(favicon、字体),不能按 utf-8 解码。
87
136
  return await readFile(file.source.abs);
@@ -121,11 +170,14 @@ function renderHeadTagHtml(tag: ResolvedHeadTag, attrs: Record<string, string |
121
170
  }
122
171
 
123
172
  /**
124
- * head 标签落成 HTML,本地 src/href 资产进站点清单:`assets/<sha256><ext>`,
125
- * source 指向原文件(写盘 copyFile、server 原字节下发都二进制安全),标签属性回填该相对路径。
126
- * 外链(http(s)://)原样透传,不进 assets/。
173
+ * head 标签落成 HTML,本地 src/href 资产进站点清单:`assets/<sha256><ext>`(与调用方无关,
174
+ * 同内容同扩展名跨全部文档去重),source 指向原文件(写盘 copyFile、server 原字节下发都
175
+ * 二进制安全)。标签属性回填的路径按 `prefix` 相对当前文档:index.html 在站点根,
176
+ * `prefix` 为空;`attempt/<locator>.html` 低一级,`prefix` 是 `"../"`(view.md「静态导出」
177
+ * 「所有 HTML 都按自身相对位置生成 assets/ / artifact/ 引用」)。外链(http(s)://)原样透传,
178
+ * 不进 assets/,prefix 对它们不生效。
127
179
  */
128
- async function materializeHeadAssets(head: ResolvedHeadTag[], files: Map<string, SiteFile>): Promise<string> {
180
+ async function materializeHeadAssets(head: ResolvedHeadTag[], files: Map<string, SiteFile>, prefix = ""): Promise<string> {
129
181
  const rendered: string[] = [];
130
182
  for (const tag of head) {
131
183
  const attrs = { ...tag.attrs };
@@ -138,7 +190,7 @@ async function materializeHeadAssets(head: ResolvedHeadTag[], files: Map<string,
138
190
  contentType: ASSET_CONTENT_TYPES[tag.localAsset.ext.toLowerCase()] ?? "application/octet-stream",
139
191
  source: { kind: "file", abs: tag.localAsset.abs },
140
192
  });
141
- attrs[tag.localAsset.attr] = path;
193
+ attrs[tag.localAsset.attr] = `${prefix}${path}`;
142
194
  }
143
195
  rendered.push(renderHeadTagHtml(tag, attrs));
144
196
  }
@@ -168,8 +220,8 @@ export async function renderHtml(scan: ViewScan, headHtml = ""): Promise<string>
168
220
  const styles = await readViewAsset("client-dist/app.css");
169
221
  const app = await readViewAsset("client-dist/app.js");
170
222
  const [reportStyles, reportEnhance] = await Promise.all([
171
- readFile(new URL("../report/react/styles.css", import.meta.url), "utf-8"),
172
- readFile(new URL("../report/react/enhance.js", import.meta.url), "utf-8"),
223
+ readFile(new URL("../report/assets/styles.css", import.meta.url), "utf-8"),
224
+ readFile(new URL("../report/assets/enhance.js", import.meta.url), "utf-8"),
173
225
  ]);
174
226
 
175
227
  const shellStyles = scan.shellAssets.styles.map((css) => `\n<style>\n${css}\n</style>`).join("");
@@ -187,10 +239,7 @@ export async function renderHtml(scan: ViewScan, headHtml = ""): Promise<string>
187
239
  const title = localizeText(scan.viewData.report?.title, "en") ?? "Eval Results";
188
240
 
189
241
  return template
190
- .replace(
191
- /<title>[^<]*<\/title>/,
192
- () => `<title>${title.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")}</title>`,
193
- )
242
+ .replace(/<title>[^<]*<\/title>/, () => `<title>${escapeText(title)}</title>`)
194
243
  .replace(
195
244
  TEMPLATE_PLACEHOLDERS.styles,
196
245
  () =>
@@ -204,6 +253,84 @@ export async function renderHtml(scan: ViewScan, headHtml = ""): Promise<string>
204
253
  .replace("</body>", () => `${shellScripts}</body>`);
205
254
  }
206
255
 
256
+ function escapeText(value: string): string {
257
+ return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
258
+ }
259
+
260
+ /**
261
+ * localStorage 里存的界面语言优先,否则按 navigator 语言猜(与 app/i18n.ts 的 detectLocale()
262
+ * 同一套判定规则,故意在这里用一份独立的极小 vanilla 版本——standalone 文档不拉入
263
+ * react/react-dom,不需要为了一次语言判定背上整个 SPA 打包产物)。
264
+ */
265
+ const ATTEMPT_LOCALE_SWAP_SCRIPT = `(function(){try{var s=localStorage.getItem("niceeval:view:locale");var l=s==="zh-CN"||s==="en"?s:((navigator.languages||[navigator.language]).some(function(v){return /^zh/i.test(String(v||"").trim())})?"zh-CN":"en");if(l==="zh-CN"){var en=document.querySelector('[data-nre-locale="en"]');var zh=document.querySelector('[data-nre-locale="zh-CN"]');if(en)en.hidden=true;if(zh){zh.hidden=false;}document.documentElement.lang="zh-CN";}}catch(e){}})();`;
266
+
267
+ /**
268
+ * 一个 locator 的独立 attempt 文档:与 index.html 是「文档」而非「App」——en 内容直接可见
269
+ * (无 JavaScript 时浏览器正常渲染这个 div),zh-CN 变体带 `hidden` 属性(同样不需要 JS 就能
270
+ * 被浏览器正确隐藏,只是不显示,不是不存在),一段极小内联脚本按检测到的界面语言在两者间切换
271
+ * (docs/feature/reports/view.md「静态导出」:基线链接直接指向这份文档,保证无 JavaScript 也能
272
+ * 读完整详情)。不复用 renderHtml/template.html 的 SPA 外壳——那条路径的 #root 要等 React
273
+ * 挂载才有内容,不满足这里「无 JS 仍完整可读」的要求。增强脚本(index.html 里的渐进增强,
274
+ * 拦截 locator 链接后 fetch 这份文档、按同一 `[data-nre-locale]` 选择器取出对应语言的内容
275
+ * 塞进 dialog)与这里的选择器保持同一套约定,不维护第二份提取逻辑。
276
+ */
277
+ async function renderAttemptDocument(
278
+ scan: ViewScan,
279
+ locator: AttemptLocator,
280
+ handle: AttemptHandle,
281
+ render: (locator: AttemptLocator, handle: AttemptHandle) => Promise<{ en: string; "zh-CN": string }>,
282
+ headHtml: string,
283
+ ): Promise<string> {
284
+ const content = await render(locator, handle);
285
+ const [reportStyles, reportEnhance] = await Promise.all([
286
+ readFile(new URL("../report/assets/styles.css", import.meta.url), "utf-8"),
287
+ readFile(new URL("../report/assets/enhance.js", import.meta.url), "utf-8"),
288
+ ]);
289
+ const shellStyles = scan.shellAssets.styles.map((css) => `\n<style>\n${css}\n</style>`).join("");
290
+ const shellScripts = scan.shellAssets.scripts.map((js) => `<script>\n${js}\n</script>\n`).join("");
291
+ const title = `${handle.evalId} · ${handle.experimentId}`;
292
+
293
+ return [
294
+ "<!doctype html>",
295
+ '<html lang="en">',
296
+ "<head>",
297
+ '<meta charset="utf-8">',
298
+ '<meta name="viewport" content="width=device-width, initial-scale=1">',
299
+ `<title>${escapeText(title)}</title>`,
300
+ `<style>\n${reportStyles}\n</style>`,
301
+ `<script>\n${reportEnhance}\n</script>`,
302
+ shellStyles,
303
+ headHtml ? `\n${headHtml}` : "",
304
+ "</head>",
305
+ "<body>",
306
+ `<div data-nre-locale="en">${content.en}</div>`,
307
+ `<div data-nre-locale="zh-CN" hidden>${content["zh-CN"]}</div>`,
308
+ `<script>${ATTEMPT_LOCALE_SWAP_SCRIPT}</script>`,
309
+ shellScripts,
310
+ "</body>",
311
+ "</html>",
312
+ ].join("\n");
313
+ }
314
+
315
+ /**
316
+ * 独立渲染一个 locator 的 attempt 文档,不依赖已建好的站点清单——供本地 server 的「越过收窄」
317
+ * 回退路由使用(docs/engineering/unit-tests/reports/cases.md 第 198/220 行:本地宿主的 attempt
318
+ * 详情路由对完整结果根解析,不受 `--exp`/eval 前缀收窄限制,与 `show @<locator>` 同一套「各自
319
+ * 结果根语义寻址」)。头资产的物化路径由内容哈希确定性算出,这里用一个用后即弃的 Map 重新
320
+ * 算一遍就够——真正的文件已经由主 plan(`planSite` 对 index.html 的物化)写过一次,不需要
321
+ * 在这里重新登记。调用方(server.ts)负责先确认 `scan.attemptPages` 存在且这个 locator
322
+ * 在里面能查到对应的 handle。
323
+ */
324
+ export async function renderStandaloneAttemptDocument(
325
+ scan: ViewScan,
326
+ locator: AttemptLocator,
327
+ handle: AttemptHandle,
328
+ ): Promise<string> {
329
+ const throwaway = new Map<string, SiteFile>();
330
+ const headHtml = await materializeHeadAssets(scan.shellAssets.head, throwaway, "../");
331
+ return renderAttemptDocument(scan, locator, handle, scan.attemptPages!.render, headHtml);
332
+ }
333
+
207
334
  async function readViewAsset(name: string): Promise<string> {
208
335
  return readFile(new URL(name, import.meta.url), "utf-8");
209
336
  }