niceeval 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (337) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +321 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/otlp/parse.d.ts +3 -0
  9. package/dist/o11y/otlp/receiver.d.ts +21 -0
  10. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  11. package/dist/o11y/types.d.ts +190 -0
  12. package/dist/report/aggregate.d.ts +57 -0
  13. package/dist/report/aggregate.js +212 -0
  14. package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
  15. package/dist/report/built-ins/experiment-comparison.js +13 -0
  16. package/dist/report/built-ins/index.d.ts +1 -0
  17. package/dist/report/built-ins/index.js +2 -0
  18. package/dist/report/components.d.ts +180 -0
  19. package/dist/report/components.js +132 -0
  20. package/dist/report/compute.d.ts +106 -0
  21. package/dist/report/compute.js +600 -0
  22. package/dist/report/flag.d.ts +6 -0
  23. package/dist/report/flag.js +8 -0
  24. package/dist/report/format.d.ts +26 -0
  25. package/dist/report/format.js +109 -0
  26. package/dist/report/index.d.ts +19 -0
  27. package/dist/report/index.js +30 -0
  28. package/dist/report/load.d.ts +12 -0
  29. package/dist/report/load.js +56 -0
  30. package/dist/report/locale.d.ts +94 -0
  31. package/dist/report/locale.js +183 -0
  32. package/dist/report/metrics.d.ts +21 -0
  33. package/dist/report/metrics.js +115 -0
  34. package/dist/report/primitives.d.ts +64 -0
  35. package/dist/report/primitives.js +113 -0
  36. package/dist/report/react/AttemptList.d.ts +23 -0
  37. package/dist/report/react/AttemptList.js +19 -0
  38. package/dist/report/react/DeltaTable.d.ts +8 -0
  39. package/dist/report/react/DeltaTable.js +24 -0
  40. package/dist/report/react/EvalList.d.ts +10 -0
  41. package/dist/report/react/EvalList.js +0 -0
  42. package/dist/report/react/ExperimentList.d.ts +11 -0
  43. package/dist/report/react/ExperimentList.js +67 -0
  44. package/dist/report/react/GroupSummary.d.ts +8 -0
  45. package/dist/report/react/GroupSummary.js +8 -0
  46. package/dist/report/react/MetricBars.d.ts +10 -0
  47. package/dist/report/react/MetricBars.js +48 -0
  48. package/dist/report/react/MetricLine.d.ts +9 -0
  49. package/dist/report/react/MetricLine.js +68 -0
  50. package/dist/report/react/MetricMatrix.d.ts +10 -0
  51. package/dist/report/react/MetricMatrix.js +23 -0
  52. package/dist/report/react/MetricScatter.d.ts +9 -0
  53. package/dist/report/react/MetricScatter.js +96 -0
  54. package/dist/report/react/MetricTable.d.ts +11 -0
  55. package/dist/report/react/MetricTable.js +26 -0
  56. package/dist/report/react/RunOverview.d.ts +8 -0
  57. package/dist/report/react/RunOverview.js +12 -0
  58. package/dist/report/react/Scoreboard.d.ts +8 -0
  59. package/dist/report/react/Scoreboard.js +26 -0
  60. package/dist/report/react/cell.d.ts +9 -0
  61. package/dist/report/react/cell.js +9 -0
  62. package/dist/report/react/chart-math.d.ts +14 -0
  63. package/dist/report/react/chart-math.js +68 -0
  64. package/dist/report/react/colors.d.ts +14 -0
  65. package/dist/report/react/colors.js +45 -0
  66. package/dist/report/react/fixtures.d.ts +19 -0
  67. package/dist/report/react/fixtures.js +398 -0
  68. package/dist/report/react/format.d.ts +3 -0
  69. package/dist/report/react/format.js +7 -0
  70. package/dist/report/react/index.d.ts +6 -0
  71. package/dist/report/react/index.js +14 -0
  72. package/dist/report/report.d.ts +24 -0
  73. package/dist/report/report.js +49 -0
  74. package/dist/report/text/faces.d.ts +29 -0
  75. package/dist/report/text/faces.js +541 -0
  76. package/dist/report/text/layout.d.ts +75 -0
  77. package/dist/report/text/layout.js +173 -0
  78. package/dist/report/text/plot.d.ts +37 -0
  79. package/dist/report/text/plot.js +81 -0
  80. package/dist/report/text/table.d.ts +8 -0
  81. package/dist/report/text/table.js +86 -0
  82. package/dist/report/tree.d.ts +99 -0
  83. package/dist/report/tree.js +228 -0
  84. package/dist/report/types.d.ts +408 -0
  85. package/dist/report/types.js +4 -0
  86. package/dist/report/web.d.ts +15 -0
  87. package/dist/report/web.js +37 -0
  88. package/dist/results/locator.d.ts +75 -0
  89. package/dist/results/locator.js +0 -0
  90. package/dist/results/select.d.ts +45 -0
  91. package/dist/results/select.js +230 -0
  92. package/dist/results/source-hash.d.ts +9 -0
  93. package/dist/results/types.d.ts +204 -0
  94. package/dist/runner/eval-source.d.ts +22 -0
  95. package/dist/runner/fingerprint.d.ts +23 -0
  96. package/dist/runner/types.d.ts +332 -0
  97. package/dist/sandbox/docker-stream.d.ts +24 -0
  98. package/dist/sandbox/docker.d.ts +84 -0
  99. package/dist/sandbox/e2b.d.ts +31 -0
  100. package/dist/sandbox/errors.d.ts +4 -0
  101. package/dist/sandbox/local-files.d.ts +6 -0
  102. package/dist/sandbox/paths.d.ts +4 -0
  103. package/dist/sandbox/registry.d.ts +13 -0
  104. package/dist/sandbox/resolve.d.ts +40 -0
  105. package/dist/sandbox/retry.d.ts +10 -0
  106. package/dist/sandbox/shell.d.ts +11 -0
  107. package/dist/sandbox/source-files.d.ts +13 -0
  108. package/dist/sandbox/types.d.ts +203 -0
  109. package/dist/sandbox/vercel.d.ts +37 -0
  110. package/dist/scoring/types.d.ts +70 -0
  111. package/dist/shared/aggregate.d.ts +29 -0
  112. package/dist/shared/aggregate.js +55 -0
  113. package/dist/shared/types.d.ts +27 -0
  114. package/dist/shared/verdict.d.ts +27 -0
  115. package/dist/shared/verdict.js +34 -0
  116. package/dist/tty-line.d.ts +6 -0
  117. package/dist/types.d.ts +7 -0
  118. package/dist/util.d.ts +22 -0
  119. package/docs-site/zh/concepts/adapter.mdx +7 -7
  120. package/docs-site/zh/concepts/assert.mdx +19 -18
  121. package/docs-site/zh/concepts/evals.mdx +12 -11
  122. package/docs-site/zh/concepts/experiment.mdx +2 -2
  123. package/docs-site/zh/concepts/judge.mdx +9 -9
  124. package/docs-site/zh/concepts/overview.mdx +3 -3
  125. package/docs-site/zh/concepts/tier.mdx +6 -6
  126. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  127. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  128. package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
  129. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  130. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  131. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  132. package/docs-site/zh/guides/experiments.mdx +7 -1
  133. package/docs-site/zh/guides/fixtures.mdx +4 -2
  134. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  135. package/docs-site/zh/guides/publish-report.mdx +4 -4
  136. package/docs-site/zh/guides/report-components.mdx +194 -55
  137. package/docs-site/zh/guides/reporters.mdx +2 -2
  138. package/docs-site/zh/guides/results-data.mdx +61 -34
  139. package/docs-site/zh/guides/runner.mdx +2 -2
  140. package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
  141. package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
  142. package/docs-site/zh/guides/viewing-results.mdx +200 -79
  143. package/docs-site/zh/guides/write-experiment.mdx +9 -1
  144. package/docs-site/zh/guides/write-send.mdx +4 -3
  145. package/docs-site/zh/index.mdx +4 -3
  146. package/docs-site/zh/introduction.mdx +2 -1
  147. package/docs-site/zh/quickstart.mdx +5 -3
  148. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  149. package/docs-site/zh/reference/capabilities.mdx +1 -1
  150. package/docs-site/zh/reference/cli.mdx +36 -13
  151. package/docs-site/zh/reference/define-agent.mdx +15 -13
  152. package/docs-site/zh/reference/define-config.mdx +1 -1
  153. package/docs-site/zh/reference/define-eval.mdx +3 -3
  154. package/docs-site/zh/reference/events.mdx +9 -3
  155. package/docs-site/zh/reference/expect.mdx +13 -8
  156. package/package.json +10 -6
  157. package/src/agents/bub.ts +129 -42
  158. package/src/agents/claude-code.test.ts +152 -0
  159. package/src/agents/claude-code.ts +127 -13
  160. package/src/agents/codex.test.ts +192 -0
  161. package/src/agents/codex.ts +151 -12
  162. package/src/agents/index.ts +9 -3
  163. package/src/agents/manifest.ts +41 -0
  164. package/src/agents/plugin-config.test.ts +94 -0
  165. package/src/agents/shared.ts +33 -2
  166. package/src/agents/skills.test.ts +214 -0
  167. package/src/agents/skills.ts +304 -0
  168. package/src/agents/streaming.ts +5 -5
  169. package/src/agents/types.ts +59 -3
  170. package/src/cli.ts +74 -47
  171. package/src/context/context.test.ts +20 -2
  172. package/src/context/control-flow.ts +1 -1
  173. package/src/context/session.test.ts +92 -1
  174. package/src/context/session.ts +23 -1
  175. package/src/context/types.ts +4 -4
  176. package/src/define.test.ts +8 -8
  177. package/src/define.ts +11 -11
  178. package/src/expect/index.ts +5 -5
  179. package/src/i18n/en.ts +33 -15
  180. package/src/i18n/zh-CN.ts +37 -21
  181. package/src/index.ts +1 -1
  182. package/src/o11y/execution-tree.test.ts +452 -0
  183. package/src/o11y/execution-tree.ts +367 -0
  184. package/src/o11y/otlp/receiver.ts +1 -1
  185. package/src/o11y/otlp/select.ts +7 -3
  186. package/src/o11y/parsers/claude-code.test.ts +175 -0
  187. package/src/o11y/parsers/claude-code.ts +32 -0
  188. package/src/o11y/types.ts +6 -1
  189. package/src/report/aggregate.ts +26 -20
  190. package/src/report/built-in-user-parity.test.tsx +640 -0
  191. package/src/report/built-ins/experiment-comparison.tsx +19 -0
  192. package/src/report/built-ins/index.ts +2 -0
  193. package/src/report/components.tsx +225 -50
  194. package/src/report/compute.ts +335 -54
  195. package/src/report/dual-render.test.tsx +1129 -0
  196. package/src/report/flag.ts +1 -1
  197. package/src/report/format.ts +35 -2
  198. package/src/report/index.ts +47 -14
  199. package/src/report/load.ts +1 -1
  200. package/src/report/locale.ts +229 -0
  201. package/src/report/metrics.ts +39 -15
  202. package/src/report/primitives.tsx +100 -2
  203. package/src/report/react/AttemptList.tsx +113 -0
  204. package/src/report/react/DeltaTable.tsx +9 -6
  205. package/src/report/react/EvalList.tsx +0 -0
  206. package/src/report/react/ExperimentList.tsx +220 -0
  207. package/src/report/react/GroupSummary.tsx +66 -0
  208. package/src/report/react/MetricBars.tsx +24 -13
  209. package/src/report/react/MetricLine.tsx +21 -21
  210. package/src/report/react/MetricMatrix.tsx +8 -4
  211. package/src/report/react/MetricScatter.tsx +126 -96
  212. package/src/report/react/MetricTable.tsx +115 -15
  213. package/src/report/react/RunOverview.tsx +48 -26
  214. package/src/report/react/Scoreboard.tsx +15 -7
  215. package/src/report/react/cell.tsx +18 -9
  216. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  217. package/src/report/react/colors.ts +24 -15
  218. package/src/report/react/enhance.js +168 -0
  219. package/src/report/react/fixtures.ts +169 -34
  220. package/src/report/react/format.ts +1 -1
  221. package/src/report/react/index.tsx +21 -6
  222. package/src/report/react/render.test.tsx +180 -31
  223. package/src/report/react/styles.css +432 -213
  224. package/src/report/report.test.ts +398 -94
  225. package/src/report/report.ts +29 -11
  226. package/src/report/text/faces.ts +344 -96
  227. package/src/report/text/layout.ts +76 -11
  228. package/src/report/text/table.ts +98 -0
  229. package/src/report/tree.ts +103 -20
  230. package/src/report/types.ts +184 -25
  231. package/src/report/web.ts +46 -19
  232. package/src/results/annotated-source.test.ts +146 -0
  233. package/src/results/annotated-source.ts +128 -0
  234. package/src/results/attempt-evidence.test.ts +158 -0
  235. package/src/results/attempt-evidence.ts +158 -0
  236. package/src/results/attempt-source.ts +48 -0
  237. package/src/results/copy.ts +154 -139
  238. package/src/results/format.ts +54 -47
  239. package/src/results/host-equivalence.test.ts +527 -0
  240. package/src/results/index.ts +56 -14
  241. package/src/results/locator.test.ts +198 -0
  242. package/src/results/locator.ts +0 -0
  243. package/src/results/open.ts +392 -175
  244. package/src/results/results.test.ts +943 -494
  245. package/src/results/select.ts +129 -23
  246. package/src/results/skipped-notice.ts +0 -0
  247. package/src/results/source-hash.ts +28 -0
  248. package/src/results/types.ts +98 -62
  249. package/src/results/writer.ts +304 -176
  250. package/src/runner/attempt.test.ts +170 -0
  251. package/src/runner/attempt.ts +48 -13
  252. package/src/runner/discover.test.ts +65 -0
  253. package/src/runner/discover.ts +6 -2
  254. package/src/runner/eval-source.test.ts +121 -0
  255. package/src/runner/eval-source.ts +45 -0
  256. package/src/runner/fingerprint.ts +54 -1
  257. package/src/runner/remote-sandbox.ts +1 -1
  258. package/src/runner/report.test.ts +6 -8
  259. package/src/runner/report.ts +5 -4
  260. package/src/runner/reporters/artifacts.ts +27 -41
  261. package/src/runner/reporters/braintrust.test.ts +3 -3
  262. package/src/runner/reporters/braintrust.ts +1 -1
  263. package/src/runner/reporters/console.ts +6 -6
  264. package/src/runner/reporters/json.ts +4 -4
  265. package/src/runner/reporters/live.test.ts +56 -0
  266. package/src/runner/reporters/live.ts +36 -10
  267. package/src/runner/reporters/quiet.test.ts +66 -0
  268. package/src/runner/reporters/quiet.ts +49 -0
  269. package/src/runner/reporters/shared.ts +5 -5
  270. package/src/runner/reporters/table.ts +17 -17
  271. package/src/runner/run.ts +61 -102
  272. package/src/runner/types.ts +43 -32
  273. package/src/sandbox/checkpoint.ts +1 -1
  274. package/src/sandbox/docker.ts +15 -2
  275. package/src/sandbox/e2b.ts +11 -5
  276. package/src/sandbox/errors.ts +13 -0
  277. package/src/sandbox/index.ts +3 -3
  278. package/src/sandbox/local-files.ts +1 -1
  279. package/src/sandbox/registry.ts +6 -5
  280. package/src/sandbox/resolve.ts +38 -23
  281. package/src/sandbox/retry.test.ts +68 -0
  282. package/src/sandbox/retry.ts +44 -0
  283. package/src/sandbox/types.ts +24 -24
  284. package/src/sandbox/vercel.ts +17 -3
  285. package/src/scoring/collector.ts +3 -3
  286. package/src/scoring/judge.ts +2 -2
  287. package/src/scoring/scoped.ts +13 -1
  288. package/src/scoring/types.ts +7 -7
  289. package/src/scoring/verdict.ts +4 -4
  290. package/src/shared/aggregate.ts +23 -3
  291. package/src/shared/types.ts +2 -2
  292. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  293. package/src/show/compose.ts +12 -132
  294. package/src/show/index.ts +119 -95
  295. package/src/show/render.ts +462 -159
  296. package/src/show/show.test.ts +471 -253
  297. package/src/source-loc.ts +1 -1
  298. package/src/tty-line.ts +32 -0
  299. package/src/types.ts +2 -2
  300. package/src/util.test.ts +21 -1
  301. package/src/util.ts +5 -1
  302. package/src/view/app/App.tsx +33 -198
  303. package/src/view/app/components/AttemptModal.tsx +6 -6
  304. package/src/view/app/components/CodeView.tsx +2 -2
  305. package/src/view/app/components/CopyControls.tsx +32 -36
  306. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  307. package/src/view/app/components/ui/badge.tsx +1 -1
  308. package/src/view/app/components/ui/dialog.tsx +2 -2
  309. package/src/view/app/i18n.ts +40 -31
  310. package/src/view/app/lib/artifact-url.ts +1 -1
  311. package/src/view/app/lib/attempt-route.test.ts +37 -47
  312. package/src/view/app/lib/attempt-route.ts +26 -26
  313. package/src/view/app/lib/rows.ts +9 -112
  314. package/src/view/app/lib/verdict.ts +25 -0
  315. package/src/view/app/main.tsx +9 -12
  316. package/src/view/app/pages/RunsPage.tsx +4 -4
  317. package/src/view/app/pages/TracesPage.tsx +3 -3
  318. package/src/view/app/shared.ts +0 -1
  319. package/src/view/app/types.ts +6 -51
  320. package/src/view/artifact-serving.test.ts +136 -0
  321. package/src/view/client-dist/app.css +1 -1
  322. package/src/view/client-dist/app.js +21 -21
  323. package/src/view/data.test.ts +214 -143
  324. package/src/view/data.ts +166 -120
  325. package/src/view/index.ts +23 -11
  326. package/src/view/server.ts +52 -17
  327. package/src/view/shared/types.ts +22 -31
  328. package/src/view/styles.css +117 -327
  329. package/src/view/view-report.test.ts +146 -98
  330. package/src/report/default-report.tsx +0 -222
  331. package/src/report/dual-face.test.tsx +0 -527
  332. package/src/report/react/CaseList.tsx +0 -70
  333. package/src/view/app/components/CostScoreChart.tsx +0 -147
  334. package/src/view/app/components/ExperimentTable.tsx +0 -284
  335. package/src/view/app/components/GroupSelector.tsx +0 -61
  336. package/src/view/app/components/primitives.tsx +0 -43
  337. package/src/view/app/lib/outcome.ts +0 -82
@@ -1,24 +1,32 @@
1
- // niceeval view 的 --report 报告槽与宿主组合语义(docs/reports.md「宿主输入的组合语义」
1
+ // niceeval view 的报告槽与宿主组合语义(docs/feature/reports/architecture.md「Selection 是计算入口」
2
2
  // 与裁决记录 6;公开行为准绳 docs-site/zh/guides/viewing-results.mdx / custom-reports.mdx)。
3
3
  // 覆盖:
4
- // - 组合语义与 show 对齐:位置前缀收窄报告槽选集、--experiment 过滤、匹配不到直说;
4
+ // - 组合语义与 show 对齐:位置前缀收窄报告槽 Selection、--experiment 过滤、匹配不到直说;
5
5
  // - 单文件模式共存:存在的文件路径 → 单文件模式,目录报错直说走 --run,其余按 eval 前缀;
6
- // - 报告槽整槽替换:报告 HTML(含 <Style> 产物与证据室深链)烘进静态块,证据室数据原样保留;
7
- // - --out 静态导出:index.html 含报告块与官方样式,报告 HTML 零 <script>;
6
+ // - 报告槽恒在:裸跑填充 ExperimentComparison,--report 整槽替换;en / zh-CN 双语各渲染一遍;
7
+ // 裸跑 --report <re-export ExperimentComparison 的文件>(等价性);
8
+ // - --out 静态导出:index.html 含两个语言的报告块、官方样式与增强 runtime,报告块零 <script>;
8
9
  // - dev server 装载语义:报告文件变更 → 下次装载整页重算(mtime cache-busting)。
10
+ //
11
+ // fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
12
+ // 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
9
13
 
10
14
  import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises";
11
15
  import { existsSync } from "node:fs";
12
16
  import { tmpdir } from "node:os";
13
17
  import { join, resolve } from "node:path";
14
18
  import { afterEach, describe, expect, it } from "vitest";
15
- import { ReportLoadError } from "../report/load.ts";
19
+ // dist-sourced: this must be the exact class loadViewScan()/data.ts's loadReportFile() throws
20
+ // (see src/view/data.ts's comment) — a raw-src import would be a structurally-identical but
21
+ // `instanceof`-incompatible class.
22
+ import { ReportLoadError } from "../../dist/report/load.js";
16
23
  import { ViewInputError, loadViewScan } from "./data.ts";
17
24
  import { buildView, resolveViewInput } from "./index.ts";
18
25
  import { runShow } from "../show/index.ts";
19
- import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
26
+ import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
20
27
 
21
28
  const EXAM_REPORT = resolve(__dirname, "../../test/fixtures/report/exam-report.tsx");
29
+ const DEFAULT_REPORT_REEXPORT = resolve(__dirname, "../../test/fixtures/report/default-report-reexport.tsx");
22
30
 
23
31
  const roots: string[] = [];
24
32
  async function makeRoot(): Promise<string> {
@@ -30,66 +38,58 @@ afterEach(async () => {
30
38
  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
31
39
  });
32
40
 
33
- function res(over: Partial<EvalResult> & Pick<EvalResult, "id">): EvalResult {
34
- return { agent: "bub", outcome: "passed", attempt: 0, durationMs: 1000, assertions: [], ...over };
41
+ type AttemptFixture = Pick<EvalResult, "id" | "verdict"> & Partial<Pick<EvalResult, "attempt" | "durationMs" | "assertions">>;
42
+
43
+ function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}): AttemptFixture {
44
+ return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
45
+ }
46
+
47
+ interface SnapshotOpts {
48
+ experimentId: string;
49
+ agent?: string;
50
+ startedAt: string;
35
51
  }
36
52
 
37
- function summaryOf(results: EvalResult[], over: Partial<RunSummary> = {}): RunSummary {
38
- const count = (o: EvalResult["outcome"]) => results.filter((r) => r.outcome === o).length;
39
- return {
53
+ /** 写一份新布局快照:snapshot.json + attempt result.json。返回快照目录绝对路径。 */
54
+ async function writeSnapshot(
55
+ root: string,
56
+ expDirName: string,
57
+ snapDirName: string,
58
+ opts: SnapshotOpts,
59
+ results: AttemptFixture[],
60
+ ): Promise<string> {
61
+ const dir = join(root, expDirName, snapDirName);
62
+ await mkdir(dir, { recursive: true });
63
+ const meta = {
40
64
  format: RESULTS_FORMAT,
41
65
  schemaVersion: RESULTS_SCHEMA_VERSION,
42
66
  producer: { name: "niceeval", version: "0.4.6" },
43
- agent: results[0]?.agent ?? "bub",
44
- startedAt: "2026-07-08T10:00:00.000Z",
45
- completedAt: "2026-07-08T10:10:00.000Z",
46
- passed: count("passed"),
47
- failed: count("failed"),
48
- skipped: count("skipped"),
49
- errored: count("errored"),
50
- durationMs: 60_000,
51
- results,
52
- ...over,
67
+ experimentId: opts.experimentId,
68
+ agent: opts.agent ?? "bub",
69
+ startedAt: opts.startedAt,
70
+ completedAt: opts.startedAt,
53
71
  };
54
- }
55
-
56
- async function writeRun(root: string, dirName: string, summary: RunSummary): Promise<string> {
57
- const dir = join(root, dirName);
58
- await mkdir(dir, { recursive: true });
59
- await writeFile(join(dir, "summary.json"), JSON.stringify(summary, null, 2), "utf-8");
72
+ await writeFile(join(dir, "snapshot.json"), JSON.stringify(meta, null, 2), "utf-8");
73
+ for (const r of results) {
74
+ const attemptDir = join(dir, r.id, `a${r.attempt ?? 0}`);
75
+ await mkdir(attemptDir, { recursive: true });
76
+ await writeFile(join(attemptDir, "result.json"), JSON.stringify(r, null, 2), "utf-8");
77
+ }
60
78
  return dir;
61
79
  }
62
80
 
63
81
  /** 两个实验、weather 通过 + button 失败:报告槽/证据室/深链断言都要用到失败案例。 */
64
82
  async function seedRoot(): Promise<string> {
65
83
  const root = await makeRoot();
66
- await writeRun(
67
- root,
68
- "2026-07-08T10-00-00-000Z",
69
- summaryOf(
70
- [
71
- res({ id: "weather/brooklyn", experimentId: "compare/bub", startedAt: "2026-07-08T10:00:01.000Z" }),
72
- res({
73
- id: "fixtures/button",
74
- experimentId: "compare/bub",
75
- outcome: "failed",
76
- startedAt: "2026-07-08T10:00:02.000Z",
77
- assertions: [
78
- { name: 'fileChanged("Button.tsx")', severity: "gate", score: 0, passed: false },
79
- ],
80
- }),
81
- ],
82
- { startedAt: "2026-07-08T10:00:00.000Z" },
83
- ),
84
- );
85
- await writeRun(
86
- root,
87
- "2026-07-09T10-00-00-000Z",
88
- summaryOf(
89
- [res({ id: "weather/brooklyn", experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-09T10:00:01.000Z" })],
90
- { agent: "codex", startedAt: "2026-07-09T10:00:00.000Z" },
91
- ),
92
- );
84
+ await writeSnapshot(root, "compare_bub", "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", agent: "bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
85
+ res("weather/brooklyn", "passed"),
86
+ res("fixtures/button", "failed", {
87
+ assertions: [{ name: 'fileChanged("Button.tsx")', severity: "gate", score: 0, passed: false }],
88
+ }),
89
+ ]);
90
+ await writeSnapshot(root, "compare_codex", "2026-07-09T10-00-00-000Z", { experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-09T10:00:00.000Z" }, [
91
+ res("weather/brooklyn", "passed"),
92
+ ]);
93
93
  return root;
94
94
  }
95
95
 
@@ -98,7 +98,7 @@ async function seedRoot(): Promise<string> {
98
98
  describe("resolveViewInput · 位置参数语义", () => {
99
99
  it("存在的文件路径 → 单文件模式;其余位置参数按 eval id 前缀", async () => {
100
100
  const root = await seedRoot();
101
- const file = join(root, "2026-07-08T10-00-00-000Z", "summary.json");
101
+ const file = join(root, "compare_bub", "2026-07-08T10-00-00-000Z", "snapshot.json");
102
102
  expect(resolveViewInput(root, [file])).toEqual({ input: file, patterns: [] });
103
103
  expect(resolveViewInput(root, ["weather", "fixtures/button"])).toEqual({
104
104
  patterns: ["weather", "fixtures/button"],
@@ -108,13 +108,13 @@ describe("resolveViewInput · 位置参数语义", () => {
108
108
 
109
109
  it("目录位置参数:报错直说走 --run(位置参数留给 eval id 前缀)", async () => {
110
110
  const root = await seedRoot();
111
- expect(() => resolveViewInput(root, ["2026-07-08T10-00-00-000Z"])).toThrow(ViewInputError);
112
- expect(() => resolveViewInput(root, ["2026-07-08T10-00-00-000Z"])).toThrow(/--run/);
111
+ expect(() => resolveViewInput(root, ["compare_bub"])).toThrow(ViewInputError);
112
+ expect(() => resolveViewInput(root, ["compare_bub"])).toThrow(/--run/);
113
113
  });
114
114
 
115
115
  it("单文件模式不与其它位置参数或 --run 混用,歧义报错直说", async () => {
116
116
  const root = await seedRoot();
117
- const file = join(root, "2026-07-08T10-00-00-000Z", "summary.json");
117
+ const file = join(root, "compare_bub", "2026-07-08T10-00-00-000Z", "snapshot.json");
118
118
  expect(() => resolveViewInput(root, [file, "weather"])).toThrow(/exactly one path/);
119
119
  expect(() => resolveViewInput(root, [file], root)).toThrow(/--run/);
120
120
  });
@@ -133,24 +133,23 @@ describe("resolveViewInput · 位置参数语义", () => {
133
133
  // ───────────────────────── 组合语义(与 show 对齐) ─────────────────────────
134
134
 
135
135
  describe("loadViewScan · 组合语义", () => {
136
- it("位置前缀收窄报告槽选集(榜单/overview);证据室快照不收窄,深链恒可达", async () => {
136
+ it("位置前缀收窄报告槽 Selection;证据室快照不收窄,深链恒可达", async () => {
137
137
  const root = await seedRoot();
138
- const { viewData } = await loadViewScan(root, { patterns: ["weather"] });
139
- // 报告槽:两实验都只剩 weather/brooklyn 一题,全过。
140
- expect(viewData.overview.totals.evals).toBe(1);
141
- expect(viewData.overview.totals.attempts).toBe(2); // 2 个实验各 1 attempt
142
- for (const row of viewData.table.rows) {
143
- expect(row.cells["pass-rate"]!.value).toBe(1);
144
- }
138
+ const { viewData, reportHtml } = await loadViewScan(root, { patterns: ["weather"] });
139
+ // 报告槽:两实验都只剩 weather/brooklyn 一题,范围外的失败不再出现。
140
+ expect(reportHtml.en).toContain("compare/bub");
141
+ expect(reportHtml.en).toContain("compare/codex");
142
+ expect(reportHtml.en).not.toContain("fixtures/button");
145
143
  // 证据室:快照明细仍含 fixtures/button(attempt 深链在收窄下也能解析)。
146
144
  const allIds = viewData.snapshots.flatMap((s) => s.results.map((r) => r.id));
147
145
  expect(allIds).toContain("fixtures/button");
148
146
  });
149
147
 
150
- it("--experiment 过滤:选集只留该实验", async () => {
148
+ it("--experiment 过滤:报告槽 Selection 只留该实验", async () => {
151
149
  const root = await seedRoot();
152
- const { viewData } = await loadViewScan(root, { experiment: "compare/codex" });
153
- expect(viewData.table.rows.map((r) => r.key)).toEqual(["compare/codex"]);
150
+ const { reportHtml } = await loadViewScan(root, { experiment: "compare/codex" });
151
+ expect(reportHtml.en).toContain("compare/codex");
152
+ expect(reportHtml.en).not.toContain("compare/bub");
154
153
  });
155
154
 
156
155
  it("前缀/实验匹配不到:直说,不渲染空页面", async () => {
@@ -160,11 +159,51 @@ describe("loadViewScan · 组合语义", () => {
160
159
  await expect(loadViewScan(root, { experiment: "nosuch" })).rejects.toBeInstanceOf(ViewInputError);
161
160
  });
162
161
 
163
- it("全部缺省:默认行为不变(results.latest() 口径,不走合成)", async () => {
162
+ it("全部缺省:报告槽使用现刻水位口径( show 相同的合成规则)", async () => {
163
+ const root = await seedRoot();
164
+ const { reportHtml } = await loadViewScan(root);
165
+ // 默认报告的榜单含两个实验;官方组件的稳定类名在场。
166
+ expect(reportHtml.en).toContain("compare/bub");
167
+ expect(reportHtml.en).toContain("compare/codex");
168
+ expect(reportHtml.en).toContain("nre-");
169
+ });
170
+ });
171
+
172
+ // ─────────────────── 报告槽恒在:裸跑 ≡ --report <ExperimentComparison> ───────────────────
173
+
174
+ describe("loadViewScan · 默认报告槽(裸跑)", () => {
175
+ it("裸跑产出的报告槽 HTML 与 --report <re-export ExperimentComparison 的文件> 完全一致(双语)", async () => {
176
+ const root = await seedRoot();
177
+ const bare = await loadViewScan(root);
178
+ const viaReport = await loadViewScan(root, {
179
+ report: { path: DEFAULT_REPORT_REEXPORT, cwd: root },
180
+ });
181
+ expect(bare.reportHtml.en).toBe(viaReport.reportHtml.en);
182
+ expect(bare.reportHtml["zh-CN"]).toBe(viaReport.reportHtml["zh-CN"]);
183
+ });
184
+
185
+ it("报告槽双语渲染:同一棵树按 locale 渲染两遍,chrome 文案分语言、数据不分语言", async () => {
186
+ const root = await seedRoot();
187
+ const { reportHtml } = await loadViewScan(root);
188
+ expect(reportHtml.en).toContain("Pass rate"); // ExperimentList 主行(en)
189
+ expect(reportHtml["zh-CN"]).toContain("成功率"); // ExperimentList 主行(zh-CN,passRate 的 zh label)
190
+ for (const html of [reportHtml.en, reportHtml["zh-CN"]]) {
191
+ expect(html).toContain("compare/bub");
192
+ // 失败案例深链进证据室:不透明 AttemptLocator 单段路由 `#/attempt/@<locator>`,
193
+ // 不再是旧的两段式 `#/attempt/<snapshot>/<attempt>`。
194
+ expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/);
195
+ expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
196
+ }
197
+ });
198
+
199
+ it("失败清单与警告住在报告槽里:ExperimentList 列出失败,壳的 viewData 不携带统计产物", async () => {
164
200
  const root = await seedRoot();
165
201
  const { viewData, reportHtml } = await loadViewScan(root);
166
- expect(reportHtml).toBeUndefined();
167
- expect(viewData.table.rows.map((r) => r.key).sort()).toEqual(["compare/bub", "compare/codex"]);
202
+ expect(reportHtml.en).toContain("fixtures/button");
203
+ // viewData 只有证据室数据:快照 + skipped + 壳元信息。
204
+ expect(viewData).not.toHaveProperty("overview");
205
+ expect(viewData).not.toHaveProperty("table");
206
+ expect(viewData).not.toHaveProperty("overall");
168
207
  });
169
208
  });
170
209
 
@@ -174,31 +213,33 @@ describe("loadViewScan · --report 报告槽", () => {
174
213
  it("报告树渲染为静态 HTML:官方水位 + 自定义摆法 + <Style> 产物 + 证据室深链,零 <script>", async () => {
175
214
  const root = await seedRoot();
176
215
  const scan = await loadViewScan(root, { report: { path: EXAM_REPORT, cwd: root } });
177
- const html = scan.reportHtml!;
216
+ const html = scan.reportHtml.en;
178
217
  expect(html).toContain("考试成绩单"); // 自定义 Section
179
218
  expect(html).toContain("nre-"); // 官方组件的稳定类名
180
219
  expect(html).toContain("<style>.exam-note { color: #4a7; }</style>"); // <Style> 随树带走
181
- expect(html).toContain("#/attempt/2026-07-08T10-00-00-000Z/"); // 失败案例深链进证据室
220
+ expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/); // 失败案例深链进证据室(单段 locator 路由)
182
221
  expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
183
- // 证据室数据契约(__NICEEVAL_VIEW_DATA__)原样保留:快照、attemptRef、skipped 不动。
222
+ // 用户报告同样双语渲染两遍(壳按界面语言摆放)
223
+ expect(scan.reportHtml["zh-CN"]).toContain("考试成绩单");
224
+ // 证据室数据契约(__NICEEVAL_VIEW_DATA__)原样保留:快照、locator、skipped 不动。
184
225
  expect(scan.viewData.snapshots.length).toBeGreaterThan(0);
185
- expect(scan.viewData.snapshots.flatMap((s) => s.results).every((r) => r.attemptRef)).toBe(true);
226
+ expect(scan.viewData.snapshots.flatMap((s) => s.results).every((r) => r.locator)).toBe(true);
186
227
  expect(JSON.stringify(scan.viewData)).not.toContain("考试成绩单"); // 报告块不进 viewData
187
228
  });
188
229
 
189
- it("位置前缀对 --report 生效:收窄注入选集,报告只见范围内的 eval", async () => {
230
+ it("位置前缀对 --report 生效:收窄注入 Selection,报告只见范围内的 eval", async () => {
190
231
  const root = await seedRoot();
191
232
  const scan = await loadViewScan(root, {
192
233
  patterns: ["weather"],
193
234
  report: { path: EXAM_REPORT, cwd: root },
194
235
  });
195
236
  // 范围外的失败(fixtures/button)不再出现在报告里;范围内的实验行都在。
196
- expect(scan.reportHtml).not.toContain("fixtures/button");
197
- expect(scan.reportHtml).toContain("compare/bub");
198
- expect(scan.reportHtml).toContain("compare/codex");
237
+ expect(scan.reportHtml.en).not.toContain("fixtures/button");
238
+ expect(scan.reportHtml.en).toContain("compare/bub");
239
+ expect(scan.reportHtml.en).toContain("compare/codex");
199
240
  });
200
241
 
201
- it("show --report 与 view --report 吃同一个报告文件,判决口径一致", async () => {
242
+ it("show --report 与 view --report 吃同一个报告文件,判定口径一致", async () => {
202
243
  const root = await seedRoot();
203
244
  let text = "";
204
245
  const code = await runShow(root, [], { run: root, report: EXAM_REPORT }, {
@@ -211,7 +252,7 @@ describe("loadViewScan · --report 报告槽", () => {
211
252
  // 同一棵树的两个面:同一份失败清单与同一个自定义 Section。
212
253
  for (const needle of ["考试成绩单", "compare/bub", "compare/codex", "fixtures/button"]) {
213
254
  expect(text).toContain(needle);
214
- expect(scan.reportHtml).toContain(needle);
255
+ expect(scan.reportHtml.en).toContain(needle);
215
256
  }
216
257
  });
217
258
 
@@ -258,51 +299,58 @@ describe("loadViewScan · 报告文件变更整页重算", () => {
258
299
  const path = join(root, "report.mjs");
259
300
  await writeFile(path, reportSource("FIRST_RENDER"), "utf-8");
260
301
  const first = await loadViewScan(root, { report: { path, cwd: root } });
261
- expect(first.reportHtml).toContain("FIRST_RENDER");
302
+ expect(first.reportHtml.en).toContain("FIRST_RENDER");
262
303
 
263
304
  await writeFile(path, reportSource("SECOND_RENDER"), "utf-8");
264
305
  // mtime 精度兜底:显式把 mtime 拨到未来,确保与首次装载可区分。
265
306
  const future = new Date(Date.now() + 5000);
266
307
  await utimes(path, future, future);
267
308
  const second = await loadViewScan(root, { report: { path, cwd: root } });
268
- expect(second.reportHtml).toContain("SECOND_RENDER");
269
- expect(second.reportHtml).not.toContain("FIRST_RENDER");
309
+ expect(second.reportHtml.en).toContain("SECOND_RENDER");
310
+ expect(second.reportHtml.en).not.toContain("FIRST_RENDER");
270
311
  });
271
312
  });
272
313
 
273
314
  // ───────────────────────── --out 静态导出 ─────────────────────────
274
315
 
275
316
  describe("buildView · --out 与 --report", () => {
276
- it("报告页为首页报告槽,证据室同站:index.html 含报告块与官方样式,工件照常复制", async () => {
317
+ it("报告页为首页报告槽,证据室同站:index.html 含报告块与官方样式,artifact 照常复制", async () => {
277
318
  const root = await seedRoot();
278
- // 给 weather attempt 一份 events 工件,验证证据室工件照常进导出。
279
- const artifactDir = join(root, "2026-07-09T10-00-00-000Z", "artifacts", "weather");
319
+ // 给 weather attempt 一份 events artifact,验证证据室 artifact 照常进导出。
320
+ // 本快照跑出的条目:artifact 目录 = <snapshot.dir>/<evalId>/a<n>,不需要声明字段。
321
+ const artifactDir = join(root, "compare_codex", "2026-07-09T10-00-00-000Z", "weather", "brooklyn", "a0");
280
322
  await mkdir(artifactDir, { recursive: true });
281
323
  await writeFile(join(artifactDir, "events.json"), "[]", "utf-8");
282
- const summaryPath = join(root, "2026-07-09T10-00-00-000Z", "summary.json");
283
- const summary = JSON.parse(await readFile(summaryPath, "utf-8")) as RunSummary;
284
- summary.results[0]!.artifactsDir = "artifacts/weather";
285
- await writeFile(summaryPath, JSON.stringify(summary), "utf-8");
286
324
 
287
325
  const out = join(root, "site");
288
326
  await buildView({ input: root, out, scan: { report: { path: EXAM_REPORT, cwd: root } } });
289
327
 
290
328
  const html = await readFile(join(out, "index.html"), "utf-8");
291
- expect(html).toContain('<template id="niceeval-report">');
329
+ // 双语两个 <template> 静态块都在,壳按界面语言摆放。
330
+ expect(html).toContain('<template id="niceeval-report-en">');
331
+ expect(html).toContain('<template id="niceeval-report-zh-CN">');
292
332
  expect(html).toContain("考试成绩单");
293
333
  expect(html).toContain("nre-"); // 官方组件样式(report/react/styles.css)随页注入
294
334
  // 报告块本体零 <script>:静态块起于 <template>,内部只有标记与 <style>。
295
- const block = html.split('<template id="niceeval-report">')[1]!.split("</template>")[0]!;
335
+ const block = html.split('<template id="niceeval-report-en">')[1]!.split("</template>")[0]!;
296
336
  expect(block).not.toContain("<script");
297
- // 证据室同站:工件按 /artifact/<base>/ 布局复制。
298
- expect(existsSync(join(out, "artifact", "2026-07-09T10-00-00-000Z/artifacts/weather", "events.json"))).toBe(true);
337
+ // 证据室同站: artifact 按 /artifact/<base>/ 布局复制。
338
+ expect(
339
+ existsSync(join(out, "artifact", "compare_codex/2026-07-09T10-00-00-000Z/weather/brooklyn/a0", "events.json")),
340
+ ).toBe(true);
299
341
  });
300
342
 
301
- it("默认导出(无 --report)不含报告块,行为不变", async () => {
343
+ it("默认导出(无 --report):报告槽填充 ExperimentComparison,双语块与增强 runtime 恒内联", async () => {
302
344
  const root = await seedRoot();
303
345
  const out = join(root, "site");
304
346
  await buildView({ input: root, out });
305
347
  const html = await readFile(join(out, "index.html"), "utf-8");
306
- expect(html).not.toContain('<template id="niceeval-report">');
348
+ expect(html).toContain('<template id="niceeval-report-en">');
349
+ expect(html).toContain('<template id="niceeval-report-zh-CN">');
350
+ expect(html).toContain("nre-"); // 官方组件的稳定类名(KPI / 榜单 / 散点来自 nre 组件)
351
+ // 渐进增强 runtime(排序 / 过滤 / tooltip)恒内联;报告块本体仍零 <script>。
352
+ expect(html).toContain("__nreEnhanced");
353
+ const block = html.split('<template id="niceeval-report-en">')[1]!.split("</template>")[0]!;
354
+ expect(block).not.toContain("<script");
307
355
  });
308
356
  });
@@ -1,222 +0,0 @@
1
- // DefaultReport:官方水位整块 —— 官方两扇门裸跑时渲染的就是它,零 props、纯声明。
2
- // 「渲染面纯同步」与它不冲突,靠的是一个数据事实:官方水位(overview、榜单、失败清单)
3
- // 只读瘦身条目、不碰任何懒加载工件,宿主对着已挑好的选集总是把这份数据备好、经上下文
4
- // 注入(renderReportToText / renderReportToStaticHtml 里 prepareDefaultReportData)。
5
- // 它渲染的口径钉死为宿主注入的那份选集 —— 零 props 意味着没有跟随的通道,这是锚点语义:
6
- // 官方口径与自定义口径并排对照。默认无特权:数据全部来自公开计算函数。
7
- //
8
- // text 面即 `niceeval show` 的榜单(docs-site/zh/guides/viewing-results.mdx 的示例块是
9
- // 行为规范):`Current verdicts` 头标注合成自几个 run、experiment 表带 eval 级折叠的
10
- // evals 列、Failing 清单每条带判决时间与下钻命令。
11
-
12
- import { basename } from "node:path";
13
- import type { Selection } from "../results/index.ts";
14
- import { foldEvalOutcome } from "../shared/outcome.ts";
15
- import { defineComponent } from "./tree.ts";
16
- import type { CaseListData, OverviewData, TableData } from "./types.ts";
17
- import { caseListData, overviewData, tableData } from "./compute.ts";
18
- import { attemptCostUSD, costUSD, durationMs, passRate } from "./metrics.ts";
19
- import { RunOverview, MetricTable, CaseList } from "./components.tsx";
20
- import { cellText } from "./text/faces.ts";
21
- import { renderAlignedRows } from "./text/layout.ts";
22
-
23
- /** 榜单的合成标注与 eval 级折叠(text 面用;与 verdicts/cases 出自同一选集)。 */
24
- export interface VerdictBoardData {
25
- /** 判决合成自几个物理 run。 */
26
- composedFromRuns: number;
27
- /** 参与合成的最新 run(目录名,即时间戳)。 */
28
- latestRun?: string;
29
- /** experiment → eval 级折叠计票(evals 列的 13/15)。 */
30
- tallies: Record<string, { passedEvals: number; totalEvals: number }>;
31
- /** eval 级折叠后失败/出错的题,带判决时间;新失败在前。 */
32
- failing: {
33
- evalId: string;
34
- experimentId: string;
35
- outcome: "failed" | "errored";
36
- /** 最新 attempt 的第一条失败断言("gate calledTool(...)")或错误摘要。 */
37
- reason?: string;
38
- /** 判决产生的时刻(最新 attempt 的 startedAt,缺失退快照时刻)。 */
39
- verdictAt?: string;
40
- }[];
41
- /** limit 之外还有几条,如实报。 */
42
- failingTruncated: number;
43
- }
44
-
45
- export interface DefaultReportData {
46
- overview: OverviewData;
47
- /** 现刻榜单:experiment × (passRate, costUSD, durationMs)。 */
48
- verdicts: TableData;
49
- cases: CaseListData;
50
- board: VerdictBoardData;
51
- }
52
-
53
- /** 失败清单的出厂截断;完整清单自己摆 <CaseList>(截断如实报剩余)。 */
54
- const DEFAULT_CASE_LIMIT = 10;
55
-
56
- function buildBoard(selection: Selection): VerdictBoardData {
57
- const tallies: VerdictBoardData["tallies"] = {};
58
- const failing: VerdictBoardData["failing"] = [];
59
- const runs = new Set<string>();
60
- let latestRunDir = "";
61
-
62
- for (const snapshot of selection.snapshots) {
63
- let passedEvals = 0;
64
- for (const ev of snapshot.evals) {
65
- for (const attempt of ev.attempts) {
66
- runs.add(attempt.runDir.dir);
67
- if (attempt.runDir.dir > latestRunDir) latestRunDir = attempt.runDir.dir;
68
- }
69
- const outcome = foldEvalOutcome(ev.attempts.map((a) => a.result));
70
- if (outcome === "passed") {
71
- passedEvals += 1;
72
- continue;
73
- }
74
- if (outcome !== "failed" && outcome !== "errored") continue;
75
- const latest = ev.attempts.reduce((a, b) => (b.result.attempt >= a.result.attempt ? b : a));
76
- const failedAssertion = latest.result.assertions.find((a) => !a.passed);
77
- const reason = failedAssertion
78
- ? `${failedAssertion.severity} ${failedAssertion.name}`
79
- : latest.result.error !== undefined
80
- ? `error: ${latest.result.error}`
81
- : undefined;
82
- failing.push({
83
- evalId: ev.id,
84
- experimentId: snapshot.experimentId,
85
- outcome,
86
- ...(reason !== undefined ? { reason } : {}),
87
- ...(latest.result.startedAt !== undefined || snapshot.startedAt
88
- ? { verdictAt: latest.result.startedAt ?? snapshot.startedAt }
89
- : {}),
90
- });
91
- }
92
- tallies[snapshot.experimentId] = { passedEvals, totalEvals: snapshot.evals.length };
93
- }
94
-
95
- failing.sort((a, b) => (b.verdictAt ?? "").localeCompare(a.verdictAt ?? ""));
96
- const shown = failing.slice(0, DEFAULT_CASE_LIMIT);
97
- return {
98
- composedFromRuns: runs.size,
99
- ...(latestRunDir ? { latestRun: basename(latestRunDir) } : {}),
100
- tallies,
101
- failing: shown,
102
- failingTruncated: failing.length - shown.length,
103
- };
104
- }
105
-
106
- /** 宿主渲染前备好官方水位:只读瘦身条目,代价可忽略。 */
107
- export async function prepareDefaultReportData(selection: Selection): Promise<DefaultReportData> {
108
- return {
109
- overview: await overviewData(selection),
110
- verdicts: await tableData(selection, {
111
- rows: "experiment",
112
- columns: [passRate, costUSD, durationMs],
113
- sort: passRate,
114
- }),
115
- cases: await caseListData(selection, { limit: DEFAULT_CASE_LIMIT }),
116
- board: buildBoard(selection),
117
- };
118
- }
119
-
120
- let activeData: DefaultReportData | null = null;
121
-
122
- /** 宿主(与渲染入口)用:在注入好的官方水位下同步渲染。 */
123
- export function runWithDefaultReportData<T>(data: DefaultReportData, fn: () => T): T {
124
- const prev = activeData;
125
- activeData = data;
126
- try {
127
- return fn();
128
- } finally {
129
- activeData = prev;
130
- }
131
- }
132
-
133
- function requireData(): DefaultReportData {
134
- if (!activeData) {
135
- throw new Error(
136
- "<DefaultReport /> renders the host-injected selection; render the report via " +
137
- "`niceeval show --report` / `niceeval view --report` (or renderReportToText / renderReportToStaticHtml). " +
138
- "Outside a host, compose the same blocks yourself: RunOverview, MetricTable, CaseList.",
139
- );
140
- }
141
- return activeData;
142
- }
143
-
144
- /** 判决时间的相对标注(text 面的 "41s ago";渲染面纯同步,Date.now 不算 IO)。 */
145
- function agoText(iso: string | undefined, now: number): string {
146
- if (!iso) return "";
147
- const ms = now - Date.parse(iso);
148
- if (!Number.isFinite(ms) || ms < 10_000) return "just now";
149
- const seconds = Math.floor(ms / 1000);
150
- if (seconds < 120) return `${seconds}s ago`;
151
- const minutes = Math.floor(seconds / 60);
152
- if (minutes < 120) return `${minutes}m ago`;
153
- const hours = Math.floor(minutes / 60);
154
- if (hours < 48) return `${hours}h ago`;
155
- return `${Math.floor(hours / 24)}d ago`;
156
- }
157
-
158
- /** show 榜单(viewing-results.mdx「榜单」块):头 + experiment 表 + Failing 清单。 */
159
- function boardText(data: DefaultReportData): string {
160
- const { verdicts, board, overview } = data;
161
- const experiments = verdicts.rows.length;
162
- const head = [
163
- "Current verdicts",
164
- `${experiments} ${experiments === 1 ? "experiment" : "experiments"}`,
165
- `composed from ${board.composedFromRuns} ${board.composedFromRuns === 1 ? "run" : "runs"}`,
166
- ...(board.latestRun ? [`latest ${board.latestRun}`] : []),
167
- ].join(" · ");
168
- const headBlock = [head, ...overview.warnings.map((w) => `! ${w.message}`)].join("\n");
169
-
170
- // 表:experiment | evals(eval 级折叠计票)| pass | cost | duration(顺序随 verdicts 预排)
171
- const header = [verdicts.dimension, "evals", "pass", "cost", "duration"];
172
- const rows = verdicts.rows.map((row) => {
173
- const tally = board.tallies[row.key];
174
- const cells = row.cells as Record<string, TableData["rows"][number]["cells"][string]>;
175
- return [
176
- row.key,
177
- tally ? `${tally.passedEvals}/${tally.totalEvals}` : "—",
178
- cells[passRate.name] ? cellText(cells[passRate.name]) : "—",
179
- cells[costUSD.name] ? cellText(cells[costUSD.name]) : "—",
180
- cells[durationMs.name] ? cellText(cells[durationMs.name]) : "—",
181
- ];
182
- });
183
- const table = renderAlignedRows([header, ...rows]);
184
-
185
- const blocks = [headBlock, table];
186
- if (board.failing.length > 0) {
187
- const now = Date.now();
188
- const aligned = renderAlignedRows(
189
- board.failing.map((f) => [
190
- `✗ ${f.evalId}`,
191
- f.experimentId,
192
- f.reason ?? f.outcome,
193
- agoText(f.verdictAt, now),
194
- ]),
195
- ).split("\n");
196
- const lines: string[] = ["Failing:"];
197
- board.failing.forEach((f, i) => {
198
- lines.push(` ${aligned[i]}`);
199
- lines.push(` → niceeval show ${f.evalId}`);
200
- });
201
- if (board.failingTruncated > 0) lines.push(` (${board.failingTruncated} more not shown)`);
202
- blocks.push(lines.join("\n"));
203
- }
204
- return blocks.join("\n\n");
205
- }
206
-
207
- export const DefaultReport = defineComponent<Record<string, never>>({
208
- web() {
209
- const data = requireData();
210
- return (
211
- <div className="nre nre-default-report">
212
- <RunOverview data={data.overview} />
213
- <MetricTable data={data.verdicts} />
214
- {data.cases.rows.length > 0 && <CaseList data={data.cases} />}
215
- </div>
216
- );
217
- },
218
- text() {
219
- return boardText(requireData());
220
- },
221
- });
222
- DefaultReport.displayName = "DefaultReport";