niceeval 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (337) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +321 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/otlp/parse.d.ts +3 -0
  9. package/dist/o11y/otlp/receiver.d.ts +21 -0
  10. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  11. package/dist/o11y/types.d.ts +190 -0
  12. package/dist/report/aggregate.d.ts +57 -0
  13. package/dist/report/aggregate.js +212 -0
  14. package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
  15. package/dist/report/built-ins/experiment-comparison.js +13 -0
  16. package/dist/report/built-ins/index.d.ts +1 -0
  17. package/dist/report/built-ins/index.js +2 -0
  18. package/dist/report/components.d.ts +180 -0
  19. package/dist/report/components.js +132 -0
  20. package/dist/report/compute.d.ts +106 -0
  21. package/dist/report/compute.js +600 -0
  22. package/dist/report/flag.d.ts +6 -0
  23. package/dist/report/flag.js +8 -0
  24. package/dist/report/format.d.ts +26 -0
  25. package/dist/report/format.js +109 -0
  26. package/dist/report/index.d.ts +19 -0
  27. package/dist/report/index.js +30 -0
  28. package/dist/report/load.d.ts +12 -0
  29. package/dist/report/load.js +56 -0
  30. package/dist/report/locale.d.ts +94 -0
  31. package/dist/report/locale.js +183 -0
  32. package/dist/report/metrics.d.ts +21 -0
  33. package/dist/report/metrics.js +115 -0
  34. package/dist/report/primitives.d.ts +64 -0
  35. package/dist/report/primitives.js +113 -0
  36. package/dist/report/react/AttemptList.d.ts +23 -0
  37. package/dist/report/react/AttemptList.js +19 -0
  38. package/dist/report/react/DeltaTable.d.ts +8 -0
  39. package/dist/report/react/DeltaTable.js +24 -0
  40. package/dist/report/react/EvalList.d.ts +10 -0
  41. package/dist/report/react/EvalList.js +0 -0
  42. package/dist/report/react/ExperimentList.d.ts +11 -0
  43. package/dist/report/react/ExperimentList.js +67 -0
  44. package/dist/report/react/GroupSummary.d.ts +8 -0
  45. package/dist/report/react/GroupSummary.js +8 -0
  46. package/dist/report/react/MetricBars.d.ts +10 -0
  47. package/dist/report/react/MetricBars.js +48 -0
  48. package/dist/report/react/MetricLine.d.ts +9 -0
  49. package/dist/report/react/MetricLine.js +68 -0
  50. package/dist/report/react/MetricMatrix.d.ts +10 -0
  51. package/dist/report/react/MetricMatrix.js +23 -0
  52. package/dist/report/react/MetricScatter.d.ts +9 -0
  53. package/dist/report/react/MetricScatter.js +96 -0
  54. package/dist/report/react/MetricTable.d.ts +11 -0
  55. package/dist/report/react/MetricTable.js +26 -0
  56. package/dist/report/react/RunOverview.d.ts +8 -0
  57. package/dist/report/react/RunOverview.js +12 -0
  58. package/dist/report/react/Scoreboard.d.ts +8 -0
  59. package/dist/report/react/Scoreboard.js +26 -0
  60. package/dist/report/react/cell.d.ts +9 -0
  61. package/dist/report/react/cell.js +9 -0
  62. package/dist/report/react/chart-math.d.ts +14 -0
  63. package/dist/report/react/chart-math.js +68 -0
  64. package/dist/report/react/colors.d.ts +14 -0
  65. package/dist/report/react/colors.js +45 -0
  66. package/dist/report/react/fixtures.d.ts +19 -0
  67. package/dist/report/react/fixtures.js +398 -0
  68. package/dist/report/react/format.d.ts +3 -0
  69. package/dist/report/react/format.js +7 -0
  70. package/dist/report/react/index.d.ts +6 -0
  71. package/dist/report/react/index.js +14 -0
  72. package/dist/report/report.d.ts +24 -0
  73. package/dist/report/report.js +49 -0
  74. package/dist/report/text/faces.d.ts +29 -0
  75. package/dist/report/text/faces.js +541 -0
  76. package/dist/report/text/layout.d.ts +75 -0
  77. package/dist/report/text/layout.js +173 -0
  78. package/dist/report/text/plot.d.ts +37 -0
  79. package/dist/report/text/plot.js +81 -0
  80. package/dist/report/text/table.d.ts +8 -0
  81. package/dist/report/text/table.js +86 -0
  82. package/dist/report/tree.d.ts +99 -0
  83. package/dist/report/tree.js +228 -0
  84. package/dist/report/types.d.ts +408 -0
  85. package/dist/report/types.js +4 -0
  86. package/dist/report/web.d.ts +15 -0
  87. package/dist/report/web.js +37 -0
  88. package/dist/results/locator.d.ts +75 -0
  89. package/dist/results/locator.js +0 -0
  90. package/dist/results/select.d.ts +45 -0
  91. package/dist/results/select.js +230 -0
  92. package/dist/results/source-hash.d.ts +9 -0
  93. package/dist/results/types.d.ts +204 -0
  94. package/dist/runner/eval-source.d.ts +22 -0
  95. package/dist/runner/fingerprint.d.ts +23 -0
  96. package/dist/runner/types.d.ts +332 -0
  97. package/dist/sandbox/docker-stream.d.ts +24 -0
  98. package/dist/sandbox/docker.d.ts +84 -0
  99. package/dist/sandbox/e2b.d.ts +31 -0
  100. package/dist/sandbox/errors.d.ts +4 -0
  101. package/dist/sandbox/local-files.d.ts +6 -0
  102. package/dist/sandbox/paths.d.ts +4 -0
  103. package/dist/sandbox/registry.d.ts +13 -0
  104. package/dist/sandbox/resolve.d.ts +40 -0
  105. package/dist/sandbox/retry.d.ts +10 -0
  106. package/dist/sandbox/shell.d.ts +11 -0
  107. package/dist/sandbox/source-files.d.ts +13 -0
  108. package/dist/sandbox/types.d.ts +203 -0
  109. package/dist/sandbox/vercel.d.ts +37 -0
  110. package/dist/scoring/types.d.ts +70 -0
  111. package/dist/shared/aggregate.d.ts +29 -0
  112. package/dist/shared/aggregate.js +55 -0
  113. package/dist/shared/types.d.ts +27 -0
  114. package/dist/shared/verdict.d.ts +27 -0
  115. package/dist/shared/verdict.js +34 -0
  116. package/dist/tty-line.d.ts +6 -0
  117. package/dist/types.d.ts +7 -0
  118. package/dist/util.d.ts +22 -0
  119. package/docs-site/zh/concepts/adapter.mdx +7 -7
  120. package/docs-site/zh/concepts/assert.mdx +19 -18
  121. package/docs-site/zh/concepts/evals.mdx +12 -11
  122. package/docs-site/zh/concepts/experiment.mdx +2 -2
  123. package/docs-site/zh/concepts/judge.mdx +9 -9
  124. package/docs-site/zh/concepts/overview.mdx +3 -3
  125. package/docs-site/zh/concepts/tier.mdx +6 -6
  126. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  127. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  128. package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
  129. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  130. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  131. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  132. package/docs-site/zh/guides/experiments.mdx +7 -1
  133. package/docs-site/zh/guides/fixtures.mdx +4 -2
  134. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  135. package/docs-site/zh/guides/publish-report.mdx +4 -4
  136. package/docs-site/zh/guides/report-components.mdx +194 -55
  137. package/docs-site/zh/guides/reporters.mdx +2 -2
  138. package/docs-site/zh/guides/results-data.mdx +61 -34
  139. package/docs-site/zh/guides/runner.mdx +2 -2
  140. package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
  141. package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
  142. package/docs-site/zh/guides/viewing-results.mdx +200 -79
  143. package/docs-site/zh/guides/write-experiment.mdx +9 -1
  144. package/docs-site/zh/guides/write-send.mdx +4 -3
  145. package/docs-site/zh/index.mdx +4 -3
  146. package/docs-site/zh/introduction.mdx +2 -1
  147. package/docs-site/zh/quickstart.mdx +5 -3
  148. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  149. package/docs-site/zh/reference/capabilities.mdx +1 -1
  150. package/docs-site/zh/reference/cli.mdx +36 -13
  151. package/docs-site/zh/reference/define-agent.mdx +15 -13
  152. package/docs-site/zh/reference/define-config.mdx +1 -1
  153. package/docs-site/zh/reference/define-eval.mdx +3 -3
  154. package/docs-site/zh/reference/events.mdx +9 -3
  155. package/docs-site/zh/reference/expect.mdx +13 -8
  156. package/package.json +10 -6
  157. package/src/agents/bub.ts +129 -42
  158. package/src/agents/claude-code.test.ts +152 -0
  159. package/src/agents/claude-code.ts +127 -13
  160. package/src/agents/codex.test.ts +192 -0
  161. package/src/agents/codex.ts +151 -12
  162. package/src/agents/index.ts +9 -3
  163. package/src/agents/manifest.ts +41 -0
  164. package/src/agents/plugin-config.test.ts +94 -0
  165. package/src/agents/shared.ts +33 -2
  166. package/src/agents/skills.test.ts +214 -0
  167. package/src/agents/skills.ts +304 -0
  168. package/src/agents/streaming.ts +5 -5
  169. package/src/agents/types.ts +59 -3
  170. package/src/cli.ts +74 -47
  171. package/src/context/context.test.ts +20 -2
  172. package/src/context/control-flow.ts +1 -1
  173. package/src/context/session.test.ts +92 -1
  174. package/src/context/session.ts +23 -1
  175. package/src/context/types.ts +4 -4
  176. package/src/define.test.ts +8 -8
  177. package/src/define.ts +11 -11
  178. package/src/expect/index.ts +5 -5
  179. package/src/i18n/en.ts +33 -15
  180. package/src/i18n/zh-CN.ts +37 -21
  181. package/src/index.ts +1 -1
  182. package/src/o11y/execution-tree.test.ts +452 -0
  183. package/src/o11y/execution-tree.ts +367 -0
  184. package/src/o11y/otlp/receiver.ts +1 -1
  185. package/src/o11y/otlp/select.ts +7 -3
  186. package/src/o11y/parsers/claude-code.test.ts +175 -0
  187. package/src/o11y/parsers/claude-code.ts +32 -0
  188. package/src/o11y/types.ts +6 -1
  189. package/src/report/aggregate.ts +26 -20
  190. package/src/report/built-in-user-parity.test.tsx +640 -0
  191. package/src/report/built-ins/experiment-comparison.tsx +19 -0
  192. package/src/report/built-ins/index.ts +2 -0
  193. package/src/report/components.tsx +225 -50
  194. package/src/report/compute.ts +335 -54
  195. package/src/report/dual-render.test.tsx +1129 -0
  196. package/src/report/flag.ts +1 -1
  197. package/src/report/format.ts +35 -2
  198. package/src/report/index.ts +47 -14
  199. package/src/report/load.ts +1 -1
  200. package/src/report/locale.ts +229 -0
  201. package/src/report/metrics.ts +39 -15
  202. package/src/report/primitives.tsx +100 -2
  203. package/src/report/react/AttemptList.tsx +113 -0
  204. package/src/report/react/DeltaTable.tsx +9 -6
  205. package/src/report/react/EvalList.tsx +0 -0
  206. package/src/report/react/ExperimentList.tsx +220 -0
  207. package/src/report/react/GroupSummary.tsx +66 -0
  208. package/src/report/react/MetricBars.tsx +24 -13
  209. package/src/report/react/MetricLine.tsx +21 -21
  210. package/src/report/react/MetricMatrix.tsx +8 -4
  211. package/src/report/react/MetricScatter.tsx +126 -96
  212. package/src/report/react/MetricTable.tsx +115 -15
  213. package/src/report/react/RunOverview.tsx +48 -26
  214. package/src/report/react/Scoreboard.tsx +15 -7
  215. package/src/report/react/cell.tsx +18 -9
  216. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  217. package/src/report/react/colors.ts +24 -15
  218. package/src/report/react/enhance.js +168 -0
  219. package/src/report/react/fixtures.ts +169 -34
  220. package/src/report/react/format.ts +1 -1
  221. package/src/report/react/index.tsx +21 -6
  222. package/src/report/react/render.test.tsx +180 -31
  223. package/src/report/react/styles.css +432 -213
  224. package/src/report/report.test.ts +398 -94
  225. package/src/report/report.ts +29 -11
  226. package/src/report/text/faces.ts +344 -96
  227. package/src/report/text/layout.ts +76 -11
  228. package/src/report/text/table.ts +98 -0
  229. package/src/report/tree.ts +103 -20
  230. package/src/report/types.ts +184 -25
  231. package/src/report/web.ts +46 -19
  232. package/src/results/annotated-source.test.ts +146 -0
  233. package/src/results/annotated-source.ts +128 -0
  234. package/src/results/attempt-evidence.test.ts +158 -0
  235. package/src/results/attempt-evidence.ts +158 -0
  236. package/src/results/attempt-source.ts +48 -0
  237. package/src/results/copy.ts +154 -139
  238. package/src/results/format.ts +54 -47
  239. package/src/results/host-equivalence.test.ts +527 -0
  240. package/src/results/index.ts +56 -14
  241. package/src/results/locator.test.ts +198 -0
  242. package/src/results/locator.ts +0 -0
  243. package/src/results/open.ts +392 -175
  244. package/src/results/results.test.ts +943 -494
  245. package/src/results/select.ts +129 -23
  246. package/src/results/skipped-notice.ts +0 -0
  247. package/src/results/source-hash.ts +28 -0
  248. package/src/results/types.ts +98 -62
  249. package/src/results/writer.ts +304 -176
  250. package/src/runner/attempt.test.ts +170 -0
  251. package/src/runner/attempt.ts +48 -13
  252. package/src/runner/discover.test.ts +65 -0
  253. package/src/runner/discover.ts +6 -2
  254. package/src/runner/eval-source.test.ts +121 -0
  255. package/src/runner/eval-source.ts +45 -0
  256. package/src/runner/fingerprint.ts +54 -1
  257. package/src/runner/remote-sandbox.ts +1 -1
  258. package/src/runner/report.test.ts +6 -8
  259. package/src/runner/report.ts +5 -4
  260. package/src/runner/reporters/artifacts.ts +27 -41
  261. package/src/runner/reporters/braintrust.test.ts +3 -3
  262. package/src/runner/reporters/braintrust.ts +1 -1
  263. package/src/runner/reporters/console.ts +6 -6
  264. package/src/runner/reporters/json.ts +4 -4
  265. package/src/runner/reporters/live.test.ts +56 -0
  266. package/src/runner/reporters/live.ts +36 -10
  267. package/src/runner/reporters/quiet.test.ts +66 -0
  268. package/src/runner/reporters/quiet.ts +49 -0
  269. package/src/runner/reporters/shared.ts +5 -5
  270. package/src/runner/reporters/table.ts +17 -17
  271. package/src/runner/run.ts +61 -102
  272. package/src/runner/types.ts +43 -32
  273. package/src/sandbox/checkpoint.ts +1 -1
  274. package/src/sandbox/docker.ts +15 -2
  275. package/src/sandbox/e2b.ts +11 -5
  276. package/src/sandbox/errors.ts +13 -0
  277. package/src/sandbox/index.ts +3 -3
  278. package/src/sandbox/local-files.ts +1 -1
  279. package/src/sandbox/registry.ts +6 -5
  280. package/src/sandbox/resolve.ts +38 -23
  281. package/src/sandbox/retry.test.ts +68 -0
  282. package/src/sandbox/retry.ts +44 -0
  283. package/src/sandbox/types.ts +24 -24
  284. package/src/sandbox/vercel.ts +17 -3
  285. package/src/scoring/collector.ts +3 -3
  286. package/src/scoring/judge.ts +2 -2
  287. package/src/scoring/scoped.ts +13 -1
  288. package/src/scoring/types.ts +7 -7
  289. package/src/scoring/verdict.ts +4 -4
  290. package/src/shared/aggregate.ts +23 -3
  291. package/src/shared/types.ts +2 -2
  292. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  293. package/src/show/compose.ts +12 -132
  294. package/src/show/index.ts +119 -95
  295. package/src/show/render.ts +462 -159
  296. package/src/show/show.test.ts +471 -253
  297. package/src/source-loc.ts +1 -1
  298. package/src/tty-line.ts +32 -0
  299. package/src/types.ts +2 -2
  300. package/src/util.test.ts +21 -1
  301. package/src/util.ts +5 -1
  302. package/src/view/app/App.tsx +33 -198
  303. package/src/view/app/components/AttemptModal.tsx +6 -6
  304. package/src/view/app/components/CodeView.tsx +2 -2
  305. package/src/view/app/components/CopyControls.tsx +32 -36
  306. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  307. package/src/view/app/components/ui/badge.tsx +1 -1
  308. package/src/view/app/components/ui/dialog.tsx +2 -2
  309. package/src/view/app/i18n.ts +40 -31
  310. package/src/view/app/lib/artifact-url.ts +1 -1
  311. package/src/view/app/lib/attempt-route.test.ts +37 -47
  312. package/src/view/app/lib/attempt-route.ts +26 -26
  313. package/src/view/app/lib/rows.ts +9 -112
  314. package/src/view/app/lib/verdict.ts +25 -0
  315. package/src/view/app/main.tsx +9 -12
  316. package/src/view/app/pages/RunsPage.tsx +4 -4
  317. package/src/view/app/pages/TracesPage.tsx +3 -3
  318. package/src/view/app/shared.ts +0 -1
  319. package/src/view/app/types.ts +6 -51
  320. package/src/view/artifact-serving.test.ts +136 -0
  321. package/src/view/client-dist/app.css +1 -1
  322. package/src/view/client-dist/app.js +21 -21
  323. package/src/view/data.test.ts +214 -143
  324. package/src/view/data.ts +166 -120
  325. package/src/view/index.ts +23 -11
  326. package/src/view/server.ts +52 -17
  327. package/src/view/shared/types.ts +22 -31
  328. package/src/view/styles.css +117 -327
  329. package/src/view/view-report.test.ts +146 -98
  330. package/src/report/default-report.tsx +0 -222
  331. package/src/report/dual-face.test.tsx +0 -527
  332. package/src/report/react/CaseList.tsx +0 -70
  333. package/src/view/app/components/CostScoreChart.tsx +0 -147
  334. package/src/view/app/components/ExperimentTable.tsx +0 -284
  335. package/src/view/app/components/GroupSelector.tsx +0 -61
  336. package/src/view/app/components/primitives.tsx +0 -43
  337. package/src/view/app/lib/outcome.ts +0 -82
@@ -1,15 +1,21 @@
1
- // view 数据层(data.ts)的单测:loader 收编到 openResults、统计收编到官方计算函数之后,
2
- // 守护三件事——skipped 三种原因如实进 viewData(producer 感知的 npx 提示)、榜单是
3
- // results.latest() 口径(不再跨历史合并)、跨快照去重让 Runs/Traces 不被 --resume 复印件灌票。
1
+ // view 数据层(data.ts)的单测:loader 收编到 openResults、统计整体住进报告槽之后,
2
+ // 守护三件事——skipped 三种原因如实进 viewData(producer 感知的 npx 提示)、报告槽是
3
+ // 现刻水位口径(裸跑经 selectCurrentResults 跨快照合成每 experiment × eval 的最新判定,
4
+ // 与 show 同一函数;en / zh-CN 双语渲染)、跨快照去重让 Runs/Traces 不被 --resume 复印件
5
+ // 灌票。viewData 只携带证据室数据(快照明细 + skipped + 壳元信息),不再有 overview /
6
+ // table / overall 统计产物。
4
7
  // 另含 loadLatestResultsPerEval 的续跑携带语义(从旧 loader.test.ts 移植,口径不变)。
8
+ //
9
+ // fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
10
+ // 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
5
11
 
6
12
  import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
7
13
  import { tmpdir } from "node:os";
8
14
  import { join } from "node:path";
9
15
  import { afterEach, describe, expect, it } from "vitest";
10
16
  import { IncompatibleResultsError, ViewInputError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
11
- import { createRunWriter } from "../results/index.ts";
12
- import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
17
+ import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
18
+ import { encodeAttemptLocator } from "../results/locator.ts";
13
19
 
14
20
  const roots: string[] = [];
15
21
  async function makeRoot(): Promise<string> {
@@ -21,52 +27,90 @@ afterEach(async () => {
21
27
  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
22
28
  });
23
29
 
24
- function res(over: Partial<EvalResult> & Pick<EvalResult, "id" | "agent">): EvalResult {
25
- return { outcome: "passed", attempt: 0, durationMs: 1000, assertions: [], ...over };
30
+ type AttemptFixture = Pick<EvalResult, "id" | "verdict"> &
31
+ Partial<Pick<EvalResult, "attempt" | "durationMs" | "assertions" | "fingerprint" | "startedAt" | "artifactBase" | "hasEvents">>;
32
+
33
+ function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}): AttemptFixture {
34
+ return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
26
35
  }
27
36
 
28
- function summaryOf(results: EvalResult[], over: Partial<RunSummary> = {}): RunSummary {
29
- const count = (o: EvalResult["outcome"]) => results.filter((r) => r.outcome === o).length;
30
- return {
31
- format: RESULTS_FORMAT,
32
- schemaVersion: RESULTS_SCHEMA_VERSION,
33
- producer: { name: "niceeval", version: "0.4.0" },
34
- agent: results[0]?.agent ?? "agent",
35
- startedAt: "2026-07-01T08:00:00.000Z",
36
- completedAt: "2026-07-01T08:10:00.000Z",
37
- passed: count("passed"),
38
- failed: count("failed"),
39
- skipped: count("skipped"),
40
- errored: count("errored"),
41
- durationMs: 60_000,
42
- results,
43
- ...over,
44
- };
37
+ /** 实验目录名的清洗:与 docs/feature/results/architecture.md 一致(/ 与非 [\w.@-] 换成 _) */
38
+ function cleanDirName(id: string): string {
39
+ return id.replace(/[^\w.@-]/g, "_");
45
40
  }
46
41
 
47
- async function writeRun(root: string, dirName: string, summary: unknown): Promise<string> {
48
- const dir = join(root, dirName);
42
+ interface SnapshotOpts {
43
+ experimentId: string;
44
+ agent?: string;
45
+ model?: string;
46
+ startedAt: string;
47
+ completedAt?: string;
48
+ schemaVersion?: number;
49
+ producer?: { name: string; version?: string };
50
+ }
51
+
52
+ /** 写一份新布局快照:snapshot.json + 各 attempt 的 result.json。返回快照目录绝对路径。 */
53
+ async function writeSnapshot(
54
+ root: string,
55
+ expDirName: string,
56
+ snapDirName: string,
57
+ opts: SnapshotOpts,
58
+ results: AttemptFixture[],
59
+ ): Promise<string> {
60
+ const dir = join(root, expDirName, snapDirName);
49
61
  await mkdir(dir, { recursive: true });
50
- await writeFile(join(dir, "summary.json"), typeof summary === "string" ? summary : JSON.stringify(summary), "utf-8");
62
+ const meta = {
63
+ format: RESULTS_FORMAT,
64
+ schemaVersion: opts.schemaVersion ?? RESULTS_SCHEMA_VERSION,
65
+ producer: opts.producer ?? { name: "niceeval", version: "0.4.0" },
66
+ experimentId: opts.experimentId,
67
+ agent: opts.agent ?? "agent",
68
+ ...(opts.model !== undefined ? { model: opts.model } : {}),
69
+ startedAt: opts.startedAt,
70
+ ...(opts.completedAt !== undefined ? { completedAt: opts.completedAt } : {}),
71
+ };
72
+ await writeFile(join(dir, "snapshot.json"), JSON.stringify(meta, null, 2), "utf-8");
73
+ for (const r of results) {
74
+ const attemptDir = join(dir, r.id, `a${r.attempt ?? 0}`);
75
+ await mkdir(attemptDir, { recursive: true });
76
+ await writeFile(join(attemptDir, "result.json"), JSON.stringify(r, null, 2), "utf-8");
77
+ }
51
78
  return dir;
52
79
  }
53
80
 
54
81
  describe("loadViewScan · skipped 三种原因进 viewData", () => {
55
82
  it("incompatible-version / malformed / incomplete 都进 skippedRuns;niceeval 落盘拼 npx 命令,第三方如实报名字不拼", async () => {
56
83
  const root = await makeRoot();
57
- // 正常 run,页面照常渲染。
58
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([res({ id: "q1", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-01T08:01:00.000Z" })]));
84
+ // 正常快照,页面照常渲染。
85
+ await writeSnapshot(root, "exp_a", "2026-07-01T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
86
+ res("q1", "passed"),
87
+ ]);
59
88
  // 版本不同:niceeval 写的。
60
- await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
89
+ await writeSnapshot(
90
+ root,
91
+ "exp_b",
92
+ "2026-07-02T08-00-00-000Z",
93
+ { experimentId: "exp/b", startedAt: "2026-07-02T08:00:00.000Z", schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } },
94
+ [],
95
+ );
61
96
  // 版本不同:第三方 harness 写的。
62
- await writeRun(root, "2026-07-03T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "otherharness", version: "1.2.3" } }));
97
+ await writeSnapshot(
98
+ root,
99
+ "exp_c",
100
+ "2026-07-03T08-00-00-000Z",
101
+ { experimentId: "exp/c", startedAt: "2026-07-03T08:00:00.000Z", schemaVersion: 999, producer: { name: "otherharness", version: "1.2.3" } },
102
+ [],
103
+ );
63
104
  // 坏 JSON。
64
- await writeRun(root, "2026-07-04T08-00-00-000Z", "{not json");
65
- // incomplete:有 attempt 工件、没有 summary(run 中途 crash)
66
- await mkdir(join(root, "2026-07-05T08-00-00-000Z", "q1", "bub", "default", "a0"), { recursive: true });
67
- await writeFile(join(root, "2026-07-05T08-00-00-000Z", "q1", "bub", "default", "a0", "events.json"), "[]", "utf-8");
105
+ const malformedDir = join(root, "exp_d", "2026-07-04T08-00-00-000Z");
106
+ await mkdir(malformedDir, { recursive: true });
107
+ await writeFile(join(malformedDir, "snapshot.json"), "{not json", "utf-8");
108
+ // incomplete:有 attempt 落盘、没有 snapshot.json(快照目录建好、元数据没写完的极窄窗口)
109
+ const incompleteDir = join(root, "exp_e", "2026-07-05T08-00-00-000Z");
110
+ await mkdir(join(incompleteDir, "q1", "a0"), { recursive: true });
111
+ await writeFile(join(incompleteDir, "q1", "a0", "events.json"), "[]", "utf-8");
68
112
 
69
- const { viewData } = await loadViewScan(root);
113
+ const { viewData, reportHtml } = await loadViewScan(root);
70
114
  const byReason = new Map(viewData.skippedRuns!.map((s) => [s.dir, s]));
71
115
  expect(viewData.skippedRuns).toHaveLength(4);
72
116
 
@@ -85,171 +129,190 @@ describe("loadViewScan · skipped 三种原因进 viewData", () => {
85
129
  const incomplete = [...byReason.values()].find((s) => s.reason === "incomplete")!;
86
130
  expect(incomplete.dir).toContain("2026-07-05T08-00-00-000Z");
87
131
 
88
- // 正常 run 照常进榜单,坏 run 不拖垮整页。
89
- expect(viewData.table.rows.map((r) => r.key)).toEqual(["exp/a"]);
132
+ // 正常快照照常进报告槽(默认报告的榜单),坏快照不拖垮整页。
133
+ expect(reportHtml.en).toContain("exp/a");
134
+ expect(viewData.snapshots.map((s) => s.experimentId)).toEqual(["exp/a"]);
90
135
  });
91
136
 
92
137
  it("单文件模式指向版本不同的报告:抛 IncompatibleResultsError(CLI 打印提示退出)", async () => {
93
138
  const root = await makeRoot();
94
- const dir = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
95
- await expect(loadViewScan(join(dir, "summary.json"))).rejects.toBeInstanceOf(IncompatibleResultsError);
139
+ const dir = await writeSnapshot(
140
+ root,
141
+ "exp_b",
142
+ "2026-07-02T08-00-00-000Z",
143
+ { experimentId: "exp/b", startedAt: "2026-07-02T08:00:00.000Z", schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } },
144
+ [],
145
+ );
146
+ await expect(loadViewScan(join(dir, "snapshot.json"))).rejects.toBeInstanceOf(IncompatibleResultsError);
96
147
  });
97
148
  });
98
149
 
99
- describe("loadViewScan · 榜单是 latest 口径,不再跨历史合并", () => {
100
- it("同一实验两次 run:榜单格子只反映最新快照;partial-coverage 警告经 overview.warnings 透传", async () => {
150
+ describe("loadViewScan · 报告槽是现刻水位口径,裸跑与局部收窄合成规则一致", () => {
151
+ it("同一实验两次快照:报告槽跨快照补齐每 eval 的最新判定,不残缺;历史快照仍供证据室", async () => {
101
152
  const root = await makeRoot();
102
153
  // 周一全量:q1 失败、q2 通过。
103
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
104
- res({ id: "q1", agent: "bub", experimentId: "exp/a", outcome: "failed", startedAt: "2026-07-01T08:01:00.000Z" }),
105
- res({ id: "q2", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-01T08:02:00.000Z" }),
106
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
107
- // 周二只补跑 q1(通过):latest 快照只盖 1/2 道题。
108
- await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
109
- res({ id: "q1", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-02T08:01:00.000Z" }),
110
- ], { startedAt: "2026-07-02T08:00:00.000Z" }));
154
+ await writeSnapshot(root, "exp_a", "2026-07-01T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
155
+ res("q1", "failed"),
156
+ res("q2", "passed"),
157
+ ]);
158
+ // 周二只补跑 q1(通过):latest 快照只盖 1/2 道题,但现刻水位从周一补齐 q2。
159
+ await writeSnapshot(root, "exp_a", "2026-07-02T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
160
+ res("q1", "passed"),
161
+ ]);
111
162
 
112
- const { viewData } = await loadViewScan(root);
113
- const row = viewData.table.rows.find((r) => r.key === "exp/a")!;
114
- // 旧的 aggregateRows 会把两次 run 揉在一起(q1 一败一过);latest 口径只看周二快照:全过。
115
- expect(row.cells["pass-rate"]!.value).toBe(1);
116
- expect(row.cells["pass-rate"]!.refs.length).toBe(1);
163
+ const { viewData, reportHtml } = await loadViewScan(root);
164
+ // 现刻水位:q1 取周二(更新,通过),q2 取周一(仅此一次,通过)—— 两题全过(2/2 = 100%),
165
+ // 不是「只看周二快照的 1/1」。深链分别指向各自的贡献快照(AttemptLocator 由身份元组——
166
+ // 含快照 startedAt——确定性派生,两个不同快照的 q1/q2 编出两个不同的 locator)
167
+ expect(reportHtml.en).toContain("100%");
168
+ const q1Locator = encodeAttemptLocator({
169
+ experimentId: "exp/a",
170
+ snapshotStartedAt: "2026-07-02T08:00:00.000Z",
171
+ evalId: "q1",
172
+ attempt: 0,
173
+ });
174
+ const q2Locator = encodeAttemptLocator({
175
+ experimentId: "exp/a",
176
+ snapshotStartedAt: "2026-07-01T08:00:00.000Z",
177
+ evalId: "q2",
178
+ attempt: 0,
179
+ });
180
+ expect(reportHtml.en).toContain(`#/attempt/${q1Locator}`); // q1 来自周二
181
+ expect(reportHtml.en).toContain(`#/attempt/${q2Locator}`); // q2 来自周一
182
+
183
+ // 两题都有真实判定,不是伪残缺:报告槽不再出 partial-coverage 警告。
184
+ expect(reportHtml.en).not.toContain('data-kind="partial-coverage"');
117
185
 
118
- // 快照元信息:latest 标记 + 每行可标注判决时间。
186
+ // 证据室的 latest 标记仍按 results.latest() 口径(周二快照),与报告槽 Selection 无关。
119
187
  const latest = viewData.snapshots.filter((s) => s.latest);
120
188
  expect(latest).toHaveLength(1);
121
189
  expect(latest[0]!.startedAt).toBe("2026-07-02T08:00:00.000Z");
122
- expect(viewData.composedRuns).toBe(1);
123
190
  expect(viewData.lastRunAt).toBe("2026-07-02T08:00:00.000Z");
124
-
125
- // 挑选警告随 OverviewData 进前端,不静默。
126
- const kinds = viewData.overview.warnings.map((w) => w.kind);
127
- expect(kinds).toContain("partial-coverage");
128
- const partial = viewData.overview.warnings.find((w) => w.kind === "partial-coverage")!;
129
- expect(partial.message).toContain("1 of 2");
191
+ // 合成 Selection 的 attempts 来自周一、周二两个物理 run。
192
+ expect(viewData.composedRuns).toBe(2);
130
193
 
131
194
  // 历史快照仍在(Runs / Traces 吃全部),但不是 latest。
132
195
  const historical = viewData.snapshots.filter((s) => !s.latest);
133
196
  expect(historical).toHaveLength(1);
134
197
  expect(historical[0]!.results.map((r) => r.id).sort()).toEqual(["q1", "q2"]);
135
198
  });
136
-
137
- it("缺 experimentId 的落盘:合成键快照 synthetic: true,synthetic-experiment-id 警告透传", async () => {
138
- const root = await makeRoot();
139
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
140
- res({ id: "q1", agent: "bub", model: "gpt-5", startedAt: "2026-07-01T08:01:00.000Z" }),
141
- ]));
142
- const { viewData } = await loadViewScan(root);
143
- expect(viewData.snapshots[0]!.synthetic).toBe(true);
144
- expect(viewData.snapshots[0]!.experimentId).toBe("bub/gpt-5");
145
- expect(viewData.overview.warnings.map((w) => w.kind)).toContain("synthetic-experiment-id");
146
- });
147
199
  });
148
200
 
149
201
  describe("loadViewScan · 跨快照去重(--resume 携带的复印件只算一次)", () => {
150
- it("同一 attempt 存在于两份落盘:只保留最新 run 里的那份,attemptRef 落在最新落盘,artifactBase 沿用原 run 工件", async () => {
202
+ it("同一 attempt 存在于两份落盘:只保留最新快照里的那份,locator 落在最新落盘,artifactBase 沿用原快照 artifact", async () => {
151
203
  const root = await makeRoot();
152
- const original = res({
153
- id: "q1",
154
- agent: "bub",
155
- experimentId: "exp/a",
156
- startedAt: "2026-07-01T08:01:00.000Z",
157
- artifactsDir: "artifacts/q1/bub/default/a0",
158
- hasEvents: true,
159
- });
160
- const oldRun = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([original], { startedAt: "2026-07-01T08:00:00.000Z" }));
161
- await mkdir(join(oldRun, "artifacts/q1/bub/default/a0"), { recursive: true });
162
- await writeFile(join(oldRun, "artifacts/q1/bub/default/a0/events.json"), "[]", "utf-8");
163
- // 携带条目:artifactsDir 清空,artifactBase 指向原 run 的工件目录(runner 的 carriedResults 形状)。
164
- const carried = { ...original, artifactsDir: undefined, artifactBase: "2026-07-01T08-00-00-000Z/artifacts/q1/bub/default/a0" };
165
- await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
166
- carried,
167
- res({ id: "q2", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-02T08:02:00.000Z", artifactsDir: "artifacts/q2/bub/default/a0" }),
168
- ], { startedAt: "2026-07-02T08:00:00.000Z" }));
204
+ const oldDir = await writeSnapshot(
205
+ root,
206
+ "exp_a",
207
+ "2026-07-01T08-00-00-000Z",
208
+ { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" },
209
+ [res("q1", "passed", { hasEvents: true })],
210
+ );
211
+ await writeFile(join(oldDir, "q1", "a0", "events.json"), "[]", "utf-8");
212
+ // 携带条目:startedAt 锚定原快照, artifactBase 指向原快照的 attempt 目录(root 相对)
213
+ await writeSnapshot(root, "exp_a", "2026-07-02T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
214
+ res("q1", "passed", {
215
+ hasEvents: true,
216
+ startedAt: "2026-07-01T08:00:00.000Z",
217
+ artifactBase: "exp_a/2026-07-01T08-00-00-000Z/q1/a0",
218
+ }),
219
+ res("q2", "passed"),
220
+ ]);
169
221
 
170
222
  const { viewData, artifactDirs } = await loadViewScan(root);
171
223
  // 全部快照(Runs/Traces 的数据面)里 q1 只出现一次:复印件不灌票。
172
224
  const allResults = viewData.snapshots.flatMap((s) => s.results);
173
225
  expect(allResults.filter((r) => r.id === "q1")).toHaveLength(1);
174
226
  const q1 = allResults.find((r) => r.id === "q1")!;
175
- expect(q1.attemptRef).toEqual({ run: "2026-07-02T08-00-00-000Z", result: 0 }); // 证据身份跟着最新落盘走
176
- expect(q1.artifactBase).toBe("2026-07-01T08-00-00-000Z/artifacts/q1/bub/default/a0"); // 工件仍指原 run
177
- // 静态导出(--out)能把携带条目的工件一并带走。
178
- expect(artifactDirs.get("2026-07-01T08-00-00-000Z/artifacts/q1/bub/default/a0")).toBe(join(oldRun, "artifacts/q1/bub/default/a0"));
227
+ // 证据身份(locator)跟着最新落盘走:身份元组里的 snapshotStartedAt 是新快照的 startedAt。
228
+ expect(q1.locator).toBe(
229
+ encodeAttemptLocator({ experimentId: "exp/a", snapshotStartedAt: "2026-07-02T08:00:00.000Z", evalId: "q1", attempt: 0 }),
230
+ );
231
+ expect(q1.artifactBase).toBe("exp_a/2026-07-01T08-00-00-000Z/q1/a0"); // artifact 仍指原快照
232
+ // 静态导出(--out)能把携带条目的 artifact 一并带走。
233
+ expect(artifactDirs.get("exp_a/2026-07-01T08-00-00-000Z/q1/a0")).toBe(join(oldDir, "q1", "a0"));
179
234
  // 条目全被吸走的旧快照不再出现。
180
235
  expect(viewData.snapshots.filter((s) => !s.latest)).toHaveLength(0);
181
236
  });
182
237
  });
183
238
 
184
- describe("loadViewScan · createRunWriter 写出的落盘直接可读(writer/reader 同一契约)", () => {
185
- it("writer roundtrip:默认报告、快照与 attemptRef 都从写入面产物算出", async () => {
239
+ describe("loadViewScan · 新布局落盘直接可读(写入面 / 读取面同一契约)", () => {
240
+ it("快照与 locator 都从落盘产物算出", async () => {
186
241
  const root = await makeRoot();
187
- const writer = await createRunWriter(root, { producer: { name: "niceeval", version: "0.4.6" } });
188
- const snap = writer.snapshot({
189
- experiment: "compare/bub",
190
- agent: "bub",
191
- model: "gpt-5",
192
- startedAt: "2026-07-03T08:00:00.000Z",
193
- });
194
- await snap.writeAttempt(
195
- { id: "q1", outcome: "passed", attempt: 0, durationMs: 1200, assertions: [], estimatedCostUSD: 0.5 },
196
- { events: [{ type: "run.started", t: 0 } as never] },
242
+ await writeSnapshot(
243
+ root,
244
+ "compare_bub",
245
+ "2026-07-03T08-00-00-000Z",
246
+ { experimentId: "compare/bub", agent: "bub", model: "gpt-5", startedAt: "2026-07-03T08:00:00.000Z" },
247
+ [
248
+ res("q1", "passed", { hasEvents: true }),
249
+ res("q2", "failed"),
250
+ ],
197
251
  );
198
- await snap.writeAttempt({ id: "q2", outcome: "failed", attempt: 0, durationMs: 800, assertions: [] });
199
- await writer.finish();
200
252
 
201
- const { viewData } = await loadViewScan(root);
202
- expect(viewData.table.rows.map((r) => r.key)).toEqual(["compare/bub"]);
203
- expect(viewData.table.rows[0]!.cells["pass-rate"]!.value).toBe(0.5);
204
- expect(viewData.overview.totals.evals).toBe(2);
253
+ const { viewData, reportHtml } = await loadViewScan(root);
254
+ // 报告槽(ExperimentComparison 的 web 面):Experiment 工作台行 + 官方通过率格子(1 过 1 败 = 50%)
255
+ expect(reportHtml.en).toContain("compare/bub");
256
+ expect(reportHtml.en).toContain("50%");
205
257
  const snapshot = viewData.snapshots[0]!;
206
258
  expect(snapshot.latest).toBe(true);
207
259
  expect(snapshot.agent).toBe("bub");
208
- expect(snapshot.results.every((r) => r.attemptRef?.run === snapshot.run)).toBe(true);
209
- // 写入面拆出的工件,读取面拼回相对 view 根的 artifactBase。
260
+ expect(snapshot.run).toBe("compare_bub/2026-07-03T08-00-00-000Z");
261
+ // 每条结果的 locator 都能由身份元组(experimentId/快照 startedAt/evalId/attempt 下标)独立复算,
262
+ // 证明它不是随手塞的占位值,而是真从落盘产物(snapshot.json + result.json)算出来的。
263
+ expect(snapshot.results.every((r) => r.locator === encodeAttemptLocator({
264
+ experimentId: snapshot.experimentId,
265
+ snapshotStartedAt: snapshot.startedAt,
266
+ evalId: r.id,
267
+ attempt: r.attempt,
268
+ }))).toBe(true);
269
+ // 本快照跑出的条目落盘没有 artifactBase 字段:读取面按 `${ref.snapshot}/${ref.attempt}` 现算,
270
+ // ref.snapshot 恒等于 snapshot.run,ref.attempt 恒是 `${evalId}/a${attempt}`。
210
271
  const q1 = snapshot.results.find((r) => r.id === "q1")!;
211
272
  expect(q1.hasEvents).toBe(true);
212
- expect(q1.artifactBase).toBe(`${snapshot.run}/${q1.artifactsDir}`);
273
+ expect(q1.artifactBase).toBe(`${snapshot.run}/q1/a0`);
213
274
  });
214
275
  });
215
276
 
216
277
  describe("loadLatestResultsPerEval(续跑携带基线,口径与旧 loader 一致)", () => {
217
- it("部分补跑 run 只遮蔽它跑过的 eval,其它 eval 仍取自更早的全量 run", async () => {
278
+ it("部分补跑快照只遮蔽它跑过的 eval,其它 eval 仍取自更早的全量快照", async () => {
218
279
  const root = await makeRoot();
219
- await writeRun(root, "2026-01-01T00-00-00", summaryOf([
220
- res({ id: "e1", agent: "a", experimentId: "exp/a" }),
221
- res({ id: "e2", agent: "a", experimentId: "exp/a", outcome: "errored" }),
222
- res({ id: "e1", agent: "a", experimentId: "exp/b" }),
223
- ], { startedAt: "2026-01-01T00:00:00.000Z" }));
280
+ await writeSnapshot(root, "exp_a", "2026-01-01T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-01T00:00:00.000Z" }, [
281
+ res("e1", "passed"),
282
+ res("e2", "errored"),
283
+ ]);
284
+ await writeSnapshot(root, "exp_b", "2026-01-01T00-00-00-000Z", { experimentId: "exp/b", agent: "a", startedAt: "2026-01-01T00:00:00.000Z" }, [
285
+ res("e1", "passed"),
286
+ ]);
224
287
  // 部分补跑:只重跑了 exp/a 的 e2
225
- await writeRun(root, "2026-01-02T00-00-00", summaryOf([
226
- res({ id: "e2", agent: "a", experimentId: "exp/a" }),
227
- ], { startedAt: "2026-01-02T00:00:00.000Z" }));
288
+ await writeSnapshot(root, "exp_a", "2026-01-02T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-02T00:00:00.000Z" }, [
289
+ res("e2", "passed"),
290
+ ]);
228
291
 
229
292
  const results = await loadLatestResultsPerEval(root);
230
- const byKey = new Map(results.map((r) => [`${r.experimentId}|${r.id}`, r.outcome]));
231
- expect(byKey.get("exp/a|e1")).toBe("passed"); // 来自旧全量 run,没被部分 run 冲掉
232
- expect(byKey.get("exp/a|e2")).toBe("passed"); // 来自补跑 run(最新)
293
+ const byKey = new Map(results.map((r) => [`${r.experimentId}|${r.id}`, r.verdict]));
294
+ expect(byKey.get("exp/a|e1")).toBe("passed"); // 来自旧全量快照,没被部分快照冲掉
295
+ expect(byKey.get("exp/a|e2")).toBe("passed"); // 来自补跑快照(最新)
233
296
  expect(byKey.get("exp/b|e1")).toBe("passed");
234
297
  expect(results).toHaveLength(3);
235
298
  });
236
299
 
237
- it("同 (experiment, eval) 多 attempt 整批取自含它的最新 run,不跨 run 混装;artifactBase 已拼好", async () => {
300
+ it("同 (experiment, eval) 多 attempt 整批取自含它的最新快照,不跨快照混装;artifactBase 已拼好", async () => {
238
301
  const root = await makeRoot();
239
- await writeRun(root, "2026-01-01T00-00-00", summaryOf([
240
- res({ id: "e1", agent: "a", experimentId: "exp/a", fingerprint: "old" }),
241
- ], { startedAt: "2026-01-01T00:00:00.000Z" }));
242
- await writeRun(root, "2026-01-02T00-00-00", summaryOf([
243
- res({ id: "e1", agent: "a", experimentId: "exp/a", outcome: "failed", fingerprint: "new", artifactsDir: "artifacts/e1/a0" }),
244
- res({ id: "e1", agent: "a", experimentId: "exp/a", attempt: 1, fingerprint: "new" }),
245
- ], { startedAt: "2026-01-02T00:00:00.000Z" }));
302
+ await writeSnapshot(root, "exp_a", "2026-01-01T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-01T00:00:00.000Z" }, [
303
+ res("e1", "passed", { fingerprint: "old" }),
304
+ ]);
305
+ await writeSnapshot(root, "exp_a", "2026-01-02T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-02T00:00:00.000Z" }, [
306
+ res("e1", "failed", { fingerprint: "new" }),
307
+ res("e1", "passed", { attempt: 1, fingerprint: "new" }),
308
+ ]);
246
309
 
247
310
  const results = await loadLatestResultsPerEval(root);
248
311
  expect(results).toHaveLength(2);
249
312
  expect(results.every((r) => r.fingerprint === "new")).toBe(true);
250
- // runner 携带条目时依赖 artifactBase(相对结果根)可解析,view 才找得回工件。
251
- const withArtifacts = results.find((r) => r.artifactsDir)!;
252
- expect(withArtifacts.artifactBase).toBe("2026-01-02T00-00-00/artifacts/e1/a0");
313
+ // runner 携带条目时依赖 artifactBase(相对结果根)可解析,view 才找得回 artifact。
314
+ const withArtifact = results.find((r) => r.attempt === 0)!;
315
+ expect(withArtifact.artifactBase).toBe("exp_a/2026-01-02T00-00-00-000Z/e1/a0");
253
316
  });
254
317
  });
255
318
 
@@ -262,15 +325,23 @@ describe("loadViewScan · 零可读结果直说,不渲染空页面", () => {
262
325
 
263
326
  it("全被跳过:错误逐条列目录与原因,niceeval 落盘的 schemaVersion 场景给出可跑的 npx 命令", async () => {
264
327
  const root = await makeRoot();
265
- await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
266
- await writeRun(root, "2026-07-04T08-00-00-000Z", "{not json");
328
+ await writeSnapshot(
329
+ root,
330
+ "exp_b",
331
+ "2026-07-02T08-00-00-000Z",
332
+ { experimentId: "exp/b", startedAt: "2026-07-02T08:00:00.000Z", schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } },
333
+ [],
334
+ );
335
+ const malformedDir = join(root, "exp_d", "2026-07-04T08-00-00-000Z");
336
+ await mkdir(malformedDir, { recursive: true });
337
+ await writeFile(join(malformedDir, "snapshot.json"), "{not json", "utf-8");
267
338
 
268
339
  const err = await loadViewScan(root).then(
269
340
  () => { throw new Error("expected ViewInputError"); },
270
341
  (e) => e as Error,
271
342
  );
272
343
  expect(err).toBeInstanceOf(ViewInputError);
273
- expect(err.message).toContain("2 run directories were skipped");
344
+ expect(err.message).toContain("2 snapshot directories were skipped");
274
345
  expect(err.message).toContain("incompatible-version, schemaVersion 999");
275
346
  expect(err.message).toContain("npx niceeval@9.9.9 view ");
276
347
  expect(err.message).toContain("malformed");