niceeval 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (337) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +321 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/otlp/parse.d.ts +3 -0
  9. package/dist/o11y/otlp/receiver.d.ts +21 -0
  10. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  11. package/dist/o11y/types.d.ts +190 -0
  12. package/dist/report/aggregate.d.ts +57 -0
  13. package/dist/report/aggregate.js +212 -0
  14. package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
  15. package/dist/report/built-ins/experiment-comparison.js +13 -0
  16. package/dist/report/built-ins/index.d.ts +1 -0
  17. package/dist/report/built-ins/index.js +2 -0
  18. package/dist/report/components.d.ts +180 -0
  19. package/dist/report/components.js +132 -0
  20. package/dist/report/compute.d.ts +106 -0
  21. package/dist/report/compute.js +600 -0
  22. package/dist/report/flag.d.ts +6 -0
  23. package/dist/report/flag.js +8 -0
  24. package/dist/report/format.d.ts +26 -0
  25. package/dist/report/format.js +109 -0
  26. package/dist/report/index.d.ts +19 -0
  27. package/dist/report/index.js +30 -0
  28. package/dist/report/load.d.ts +12 -0
  29. package/dist/report/load.js +56 -0
  30. package/dist/report/locale.d.ts +94 -0
  31. package/dist/report/locale.js +183 -0
  32. package/dist/report/metrics.d.ts +21 -0
  33. package/dist/report/metrics.js +115 -0
  34. package/dist/report/primitives.d.ts +64 -0
  35. package/dist/report/primitives.js +113 -0
  36. package/dist/report/react/AttemptList.d.ts +23 -0
  37. package/dist/report/react/AttemptList.js +19 -0
  38. package/dist/report/react/DeltaTable.d.ts +8 -0
  39. package/dist/report/react/DeltaTable.js +24 -0
  40. package/dist/report/react/EvalList.d.ts +10 -0
  41. package/dist/report/react/EvalList.js +0 -0
  42. package/dist/report/react/ExperimentList.d.ts +11 -0
  43. package/dist/report/react/ExperimentList.js +67 -0
  44. package/dist/report/react/GroupSummary.d.ts +8 -0
  45. package/dist/report/react/GroupSummary.js +8 -0
  46. package/dist/report/react/MetricBars.d.ts +10 -0
  47. package/dist/report/react/MetricBars.js +48 -0
  48. package/dist/report/react/MetricLine.d.ts +9 -0
  49. package/dist/report/react/MetricLine.js +68 -0
  50. package/dist/report/react/MetricMatrix.d.ts +10 -0
  51. package/dist/report/react/MetricMatrix.js +23 -0
  52. package/dist/report/react/MetricScatter.d.ts +9 -0
  53. package/dist/report/react/MetricScatter.js +96 -0
  54. package/dist/report/react/MetricTable.d.ts +11 -0
  55. package/dist/report/react/MetricTable.js +26 -0
  56. package/dist/report/react/RunOverview.d.ts +8 -0
  57. package/dist/report/react/RunOverview.js +12 -0
  58. package/dist/report/react/Scoreboard.d.ts +8 -0
  59. package/dist/report/react/Scoreboard.js +26 -0
  60. package/dist/report/react/cell.d.ts +9 -0
  61. package/dist/report/react/cell.js +9 -0
  62. package/dist/report/react/chart-math.d.ts +14 -0
  63. package/dist/report/react/chart-math.js +68 -0
  64. package/dist/report/react/colors.d.ts +14 -0
  65. package/dist/report/react/colors.js +45 -0
  66. package/dist/report/react/fixtures.d.ts +19 -0
  67. package/dist/report/react/fixtures.js +398 -0
  68. package/dist/report/react/format.d.ts +3 -0
  69. package/dist/report/react/format.js +7 -0
  70. package/dist/report/react/index.d.ts +6 -0
  71. package/dist/report/react/index.js +14 -0
  72. package/dist/report/report.d.ts +24 -0
  73. package/dist/report/report.js +49 -0
  74. package/dist/report/text/faces.d.ts +29 -0
  75. package/dist/report/text/faces.js +541 -0
  76. package/dist/report/text/layout.d.ts +75 -0
  77. package/dist/report/text/layout.js +173 -0
  78. package/dist/report/text/plot.d.ts +37 -0
  79. package/dist/report/text/plot.js +81 -0
  80. package/dist/report/text/table.d.ts +8 -0
  81. package/dist/report/text/table.js +86 -0
  82. package/dist/report/tree.d.ts +99 -0
  83. package/dist/report/tree.js +228 -0
  84. package/dist/report/types.d.ts +408 -0
  85. package/dist/report/types.js +4 -0
  86. package/dist/report/web.d.ts +15 -0
  87. package/dist/report/web.js +37 -0
  88. package/dist/results/locator.d.ts +75 -0
  89. package/dist/results/locator.js +0 -0
  90. package/dist/results/select.d.ts +45 -0
  91. package/dist/results/select.js +230 -0
  92. package/dist/results/source-hash.d.ts +9 -0
  93. package/dist/results/types.d.ts +204 -0
  94. package/dist/runner/eval-source.d.ts +22 -0
  95. package/dist/runner/fingerprint.d.ts +23 -0
  96. package/dist/runner/types.d.ts +332 -0
  97. package/dist/sandbox/docker-stream.d.ts +24 -0
  98. package/dist/sandbox/docker.d.ts +84 -0
  99. package/dist/sandbox/e2b.d.ts +31 -0
  100. package/dist/sandbox/errors.d.ts +4 -0
  101. package/dist/sandbox/local-files.d.ts +6 -0
  102. package/dist/sandbox/paths.d.ts +4 -0
  103. package/dist/sandbox/registry.d.ts +13 -0
  104. package/dist/sandbox/resolve.d.ts +40 -0
  105. package/dist/sandbox/retry.d.ts +10 -0
  106. package/dist/sandbox/shell.d.ts +11 -0
  107. package/dist/sandbox/source-files.d.ts +13 -0
  108. package/dist/sandbox/types.d.ts +203 -0
  109. package/dist/sandbox/vercel.d.ts +37 -0
  110. package/dist/scoring/types.d.ts +70 -0
  111. package/dist/shared/aggregate.d.ts +29 -0
  112. package/dist/shared/aggregate.js +55 -0
  113. package/dist/shared/types.d.ts +27 -0
  114. package/dist/shared/verdict.d.ts +27 -0
  115. package/dist/shared/verdict.js +34 -0
  116. package/dist/tty-line.d.ts +6 -0
  117. package/dist/types.d.ts +7 -0
  118. package/dist/util.d.ts +22 -0
  119. package/docs-site/zh/concepts/adapter.mdx +7 -7
  120. package/docs-site/zh/concepts/assert.mdx +19 -18
  121. package/docs-site/zh/concepts/evals.mdx +12 -11
  122. package/docs-site/zh/concepts/experiment.mdx +2 -2
  123. package/docs-site/zh/concepts/judge.mdx +9 -9
  124. package/docs-site/zh/concepts/overview.mdx +3 -3
  125. package/docs-site/zh/concepts/tier.mdx +6 -6
  126. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  127. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  128. package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
  129. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  130. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  131. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  132. package/docs-site/zh/guides/experiments.mdx +7 -1
  133. package/docs-site/zh/guides/fixtures.mdx +4 -2
  134. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  135. package/docs-site/zh/guides/publish-report.mdx +4 -4
  136. package/docs-site/zh/guides/report-components.mdx +194 -55
  137. package/docs-site/zh/guides/reporters.mdx +2 -2
  138. package/docs-site/zh/guides/results-data.mdx +61 -34
  139. package/docs-site/zh/guides/runner.mdx +2 -2
  140. package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
  141. package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
  142. package/docs-site/zh/guides/viewing-results.mdx +200 -79
  143. package/docs-site/zh/guides/write-experiment.mdx +9 -1
  144. package/docs-site/zh/guides/write-send.mdx +4 -3
  145. package/docs-site/zh/index.mdx +4 -3
  146. package/docs-site/zh/introduction.mdx +2 -1
  147. package/docs-site/zh/quickstart.mdx +5 -3
  148. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  149. package/docs-site/zh/reference/capabilities.mdx +1 -1
  150. package/docs-site/zh/reference/cli.mdx +36 -13
  151. package/docs-site/zh/reference/define-agent.mdx +15 -13
  152. package/docs-site/zh/reference/define-config.mdx +1 -1
  153. package/docs-site/zh/reference/define-eval.mdx +3 -3
  154. package/docs-site/zh/reference/events.mdx +9 -3
  155. package/docs-site/zh/reference/expect.mdx +13 -8
  156. package/package.json +10 -6
  157. package/src/agents/bub.ts +129 -42
  158. package/src/agents/claude-code.test.ts +152 -0
  159. package/src/agents/claude-code.ts +127 -13
  160. package/src/agents/codex.test.ts +192 -0
  161. package/src/agents/codex.ts +151 -12
  162. package/src/agents/index.ts +9 -3
  163. package/src/agents/manifest.ts +41 -0
  164. package/src/agents/plugin-config.test.ts +94 -0
  165. package/src/agents/shared.ts +33 -2
  166. package/src/agents/skills.test.ts +214 -0
  167. package/src/agents/skills.ts +304 -0
  168. package/src/agents/streaming.ts +5 -5
  169. package/src/agents/types.ts +59 -3
  170. package/src/cli.ts +74 -47
  171. package/src/context/context.test.ts +20 -2
  172. package/src/context/control-flow.ts +1 -1
  173. package/src/context/session.test.ts +92 -1
  174. package/src/context/session.ts +23 -1
  175. package/src/context/types.ts +4 -4
  176. package/src/define.test.ts +8 -8
  177. package/src/define.ts +11 -11
  178. package/src/expect/index.ts +5 -5
  179. package/src/i18n/en.ts +33 -15
  180. package/src/i18n/zh-CN.ts +37 -21
  181. package/src/index.ts +1 -1
  182. package/src/o11y/execution-tree.test.ts +452 -0
  183. package/src/o11y/execution-tree.ts +367 -0
  184. package/src/o11y/otlp/receiver.ts +1 -1
  185. package/src/o11y/otlp/select.ts +7 -3
  186. package/src/o11y/parsers/claude-code.test.ts +175 -0
  187. package/src/o11y/parsers/claude-code.ts +32 -0
  188. package/src/o11y/types.ts +6 -1
  189. package/src/report/aggregate.ts +26 -20
  190. package/src/report/built-in-user-parity.test.tsx +640 -0
  191. package/src/report/built-ins/experiment-comparison.tsx +19 -0
  192. package/src/report/built-ins/index.ts +2 -0
  193. package/src/report/components.tsx +225 -50
  194. package/src/report/compute.ts +335 -54
  195. package/src/report/dual-render.test.tsx +1129 -0
  196. package/src/report/flag.ts +1 -1
  197. package/src/report/format.ts +35 -2
  198. package/src/report/index.ts +47 -14
  199. package/src/report/load.ts +1 -1
  200. package/src/report/locale.ts +229 -0
  201. package/src/report/metrics.ts +39 -15
  202. package/src/report/primitives.tsx +100 -2
  203. package/src/report/react/AttemptList.tsx +113 -0
  204. package/src/report/react/DeltaTable.tsx +9 -6
  205. package/src/report/react/EvalList.tsx +0 -0
  206. package/src/report/react/ExperimentList.tsx +220 -0
  207. package/src/report/react/GroupSummary.tsx +66 -0
  208. package/src/report/react/MetricBars.tsx +24 -13
  209. package/src/report/react/MetricLine.tsx +21 -21
  210. package/src/report/react/MetricMatrix.tsx +8 -4
  211. package/src/report/react/MetricScatter.tsx +126 -96
  212. package/src/report/react/MetricTable.tsx +115 -15
  213. package/src/report/react/RunOverview.tsx +48 -26
  214. package/src/report/react/Scoreboard.tsx +15 -7
  215. package/src/report/react/cell.tsx +18 -9
  216. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  217. package/src/report/react/colors.ts +24 -15
  218. package/src/report/react/enhance.js +168 -0
  219. package/src/report/react/fixtures.ts +169 -34
  220. package/src/report/react/format.ts +1 -1
  221. package/src/report/react/index.tsx +21 -6
  222. package/src/report/react/render.test.tsx +180 -31
  223. package/src/report/react/styles.css +432 -213
  224. package/src/report/report.test.ts +398 -94
  225. package/src/report/report.ts +29 -11
  226. package/src/report/text/faces.ts +344 -96
  227. package/src/report/text/layout.ts +76 -11
  228. package/src/report/text/table.ts +98 -0
  229. package/src/report/tree.ts +103 -20
  230. package/src/report/types.ts +184 -25
  231. package/src/report/web.ts +46 -19
  232. package/src/results/annotated-source.test.ts +146 -0
  233. package/src/results/annotated-source.ts +128 -0
  234. package/src/results/attempt-evidence.test.ts +158 -0
  235. package/src/results/attempt-evidence.ts +158 -0
  236. package/src/results/attempt-source.ts +48 -0
  237. package/src/results/copy.ts +154 -139
  238. package/src/results/format.ts +54 -47
  239. package/src/results/host-equivalence.test.ts +527 -0
  240. package/src/results/index.ts +56 -14
  241. package/src/results/locator.test.ts +198 -0
  242. package/src/results/locator.ts +0 -0
  243. package/src/results/open.ts +392 -175
  244. package/src/results/results.test.ts +943 -494
  245. package/src/results/select.ts +129 -23
  246. package/src/results/skipped-notice.ts +0 -0
  247. package/src/results/source-hash.ts +28 -0
  248. package/src/results/types.ts +98 -62
  249. package/src/results/writer.ts +304 -176
  250. package/src/runner/attempt.test.ts +170 -0
  251. package/src/runner/attempt.ts +48 -13
  252. package/src/runner/discover.test.ts +65 -0
  253. package/src/runner/discover.ts +6 -2
  254. package/src/runner/eval-source.test.ts +121 -0
  255. package/src/runner/eval-source.ts +45 -0
  256. package/src/runner/fingerprint.ts +54 -1
  257. package/src/runner/remote-sandbox.ts +1 -1
  258. package/src/runner/report.test.ts +6 -8
  259. package/src/runner/report.ts +5 -4
  260. package/src/runner/reporters/artifacts.ts +27 -41
  261. package/src/runner/reporters/braintrust.test.ts +3 -3
  262. package/src/runner/reporters/braintrust.ts +1 -1
  263. package/src/runner/reporters/console.ts +6 -6
  264. package/src/runner/reporters/json.ts +4 -4
  265. package/src/runner/reporters/live.test.ts +56 -0
  266. package/src/runner/reporters/live.ts +36 -10
  267. package/src/runner/reporters/quiet.test.ts +66 -0
  268. package/src/runner/reporters/quiet.ts +49 -0
  269. package/src/runner/reporters/shared.ts +5 -5
  270. package/src/runner/reporters/table.ts +17 -17
  271. package/src/runner/run.ts +61 -102
  272. package/src/runner/types.ts +43 -32
  273. package/src/sandbox/checkpoint.ts +1 -1
  274. package/src/sandbox/docker.ts +15 -2
  275. package/src/sandbox/e2b.ts +11 -5
  276. package/src/sandbox/errors.ts +13 -0
  277. package/src/sandbox/index.ts +3 -3
  278. package/src/sandbox/local-files.ts +1 -1
  279. package/src/sandbox/registry.ts +6 -5
  280. package/src/sandbox/resolve.ts +38 -23
  281. package/src/sandbox/retry.test.ts +68 -0
  282. package/src/sandbox/retry.ts +44 -0
  283. package/src/sandbox/types.ts +24 -24
  284. package/src/sandbox/vercel.ts +17 -3
  285. package/src/scoring/collector.ts +3 -3
  286. package/src/scoring/judge.ts +2 -2
  287. package/src/scoring/scoped.ts +13 -1
  288. package/src/scoring/types.ts +7 -7
  289. package/src/scoring/verdict.ts +4 -4
  290. package/src/shared/aggregate.ts +23 -3
  291. package/src/shared/types.ts +2 -2
  292. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  293. package/src/show/compose.ts +12 -132
  294. package/src/show/index.ts +119 -95
  295. package/src/show/render.ts +462 -159
  296. package/src/show/show.test.ts +471 -253
  297. package/src/source-loc.ts +1 -1
  298. package/src/tty-line.ts +32 -0
  299. package/src/types.ts +2 -2
  300. package/src/util.test.ts +21 -1
  301. package/src/util.ts +5 -1
  302. package/src/view/app/App.tsx +33 -198
  303. package/src/view/app/components/AttemptModal.tsx +6 -6
  304. package/src/view/app/components/CodeView.tsx +2 -2
  305. package/src/view/app/components/CopyControls.tsx +32 -36
  306. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  307. package/src/view/app/components/ui/badge.tsx +1 -1
  308. package/src/view/app/components/ui/dialog.tsx +2 -2
  309. package/src/view/app/i18n.ts +40 -31
  310. package/src/view/app/lib/artifact-url.ts +1 -1
  311. package/src/view/app/lib/attempt-route.test.ts +37 -47
  312. package/src/view/app/lib/attempt-route.ts +26 -26
  313. package/src/view/app/lib/rows.ts +9 -112
  314. package/src/view/app/lib/verdict.ts +25 -0
  315. package/src/view/app/main.tsx +9 -12
  316. package/src/view/app/pages/RunsPage.tsx +4 -4
  317. package/src/view/app/pages/TracesPage.tsx +3 -3
  318. package/src/view/app/shared.ts +0 -1
  319. package/src/view/app/types.ts +6 -51
  320. package/src/view/artifact-serving.test.ts +136 -0
  321. package/src/view/client-dist/app.css +1 -1
  322. package/src/view/client-dist/app.js +21 -21
  323. package/src/view/data.test.ts +214 -143
  324. package/src/view/data.ts +166 -120
  325. package/src/view/index.ts +23 -11
  326. package/src/view/server.ts +52 -17
  327. package/src/view/shared/types.ts +22 -31
  328. package/src/view/styles.css +117 -327
  329. package/src/view/view-report.test.ts +146 -98
  330. package/src/report/default-report.tsx +0 -222
  331. package/src/report/dual-face.test.tsx +0 -527
  332. package/src/report/react/CaseList.tsx +0 -70
  333. package/src/view/app/components/CostScoreChart.tsx +0 -147
  334. package/src/view/app/components/ExperimentTable.tsx +0 -284
  335. package/src/view/app/components/GroupSelector.tsx +0 -61
  336. package/src/view/app/components/primitives.tsx +0 -43
  337. package/src/view/app/lib/outcome.ts +0 -82
@@ -1,24 +1,34 @@
1
- // niceeval/results 的单测:临时目录里构造最小 summary.json / 工件 fixture,覆盖定稿契约
2
- // (docs/results-lib.md):分层读取、懒加载回退、skipped 三种原因、latest() 三种警告、
3
- // Selection.filter 修剪、dedupeAttempts 身份键、writer roundtripcopySnapshots 补记,
4
- // 以及 Artifacts 报告器(writer 薄壳)与直写时代逐字节等价的守护。
5
- // 读取面 fixture 的目录名/工件路径手写(不 import 库的路径函数),让测试独立于实现充当格式基准。
6
-
7
- import { afterEach, describe, expect, it } from "vitest";
8
- import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
1
+ // niceeval/results 的单测:临时目录里构造最小 snapshot.json / result.json / artifact fixture,
2
+ // 覆盖定稿契约(docs/feature/results/library.md、docs/feature/results/architecture.md):分层读取(含快照级字段注入)
3
+ // 懒加载与 artifactBase 回退、skipped 三种原因、unfinished-snapshotlatest() 三种警告、
4
+ // Selection.filter 修剪、dedupeAttempts 身份键、writer(独占目录、并发快照互不干扰、
5
+ // snapshot.json 键形状、writeAttempt/writeAttemptFor、finish 幂等)、copySnapshots(布局、
6
+ // knownEvalIds 补记、has* 重算)。
7
+ // 读取面 fixture 的目录名/artifact 路径手写(不 import 库的路径函数),让测试独立于实现充当格式基准。
8
+
9
+ import { afterEach, describe, expect, it, vi } from "vitest";
10
+ import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
9
11
  import { tmpdir } from "node:os";
10
- import { join } from "node:path";
12
+ import { basename, join } from "node:path";
11
13
  import {
12
14
  RESULTS_FORMAT,
13
15
  RESULTS_SCHEMA_VERSION,
14
16
  copySnapshots,
15
- createRunWriter,
17
+ createResultsWriter,
16
18
  dedupeAttempts,
17
19
  openResults,
20
+ resolveLocator,
21
+ loadAnnotatedEvalSource,
22
+ LocatorNotFoundError,
23
+ MalformedLocatorError,
24
+ LocatorCollisionError,
25
+ encodeAttemptLocator,
26
+ type AttemptHandle,
18
27
  type EvalResult,
19
- type RunSummary,
28
+ type Results,
29
+ type Snapshot,
30
+ type SnapshotMeta,
20
31
  } from "./index.ts";
21
- import { Artifacts } from "../runner/reporters/artifacts.ts";
22
32
 
23
33
  // ───────────────────────── fixture 工具 ─────────────────────────
24
34
 
@@ -32,38 +42,36 @@ afterEach(async () => {
32
42
  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
33
43
  });
34
44
 
35
- function res(over: Partial<EvalResult> & Pick<EvalResult, "id" | "agent">): EvalResult {
36
- return { outcome: "passed", attempt: 1, durationMs: 1000, assertions: [], ...over };
37
- }
38
-
39
- function summaryOf(results: EvalResult[], over: Partial<RunSummary> = {}): RunSummary {
40
- const count = (o: EvalResult["outcome"]) => results.filter((r) => r.outcome === o).length;
45
+ function meta(over: { experimentId: string; agent: string; startedAt: string } & Partial<SnapshotMeta>): SnapshotMeta {
41
46
  return {
42
47
  format: RESULTS_FORMAT,
43
48
  schemaVersion: RESULTS_SCHEMA_VERSION,
44
49
  producer: { name: "niceeval", version: "0.3.0" },
45
- agent: results[0]?.agent ?? "agent",
46
- startedAt: "2026-07-01T08:00:00.000Z",
47
- completedAt: "2026-07-01T08:10:00.000Z",
48
- passed: count("passed"),
49
- failed: count("failed"),
50
- skipped: count("skipped"),
51
- errored: count("errored"),
52
- durationMs: 60_000,
53
- results,
54
50
  ...over,
55
51
  };
56
52
  }
57
53
 
58
- async function writeRun(root: string, dirName: string, summary: unknown): Promise<string> {
59
- const dir = join(root, dirName);
54
+ function record(over: { id: string; attempt: number } & Record<string, unknown>): EvalResult {
55
+ return { verdict: "passed", durationMs: 1000, assertions: [], ...over } as unknown as EvalResult;
56
+ }
57
+
58
+ async function writeSnapshot(root: string, expDir: string, snapDirName: string, m: SnapshotMeta): Promise<string> {
59
+ const dir = join(root, expDir, snapDirName);
60
60
  await mkdir(dir, { recursive: true });
61
- await writeFile(join(dir, "summary.json"), typeof summary === "string" ? summary : JSON.stringify(summary, null, 2), "utf-8");
61
+ await writeFile(join(dir, "snapshot.json"), JSON.stringify(m, null, 2), "utf-8");
62
62
  return dir;
63
63
  }
64
64
 
65
- async function writeArtifact(runDir: string, relDir: string, file: string, data: unknown): Promise<string> {
66
- const dir = join(runDir, relDir);
65
+ async function writeResultFile(snapDir: string, relAttemptDir: string, r: unknown): Promise<string> {
66
+ const dir = join(snapDir, relAttemptDir);
67
+ await mkdir(dir, { recursive: true });
68
+ const path = join(dir, "result.json");
69
+ await writeFile(path, JSON.stringify(r, null, 2), "utf-8");
70
+ return path;
71
+ }
72
+
73
+ async function writeArtifactFile(snapDir: string, relAttemptDir: string, file: string, data: unknown): Promise<string> {
74
+ const dir = join(snapDir, relAttemptDir);
67
75
  await mkdir(dir, { recursive: true });
68
76
  const path = join(dir, file);
69
77
  await writeFile(path, JSON.stringify(data), "utf-8");
@@ -82,184 +90,222 @@ async function exists(path: string): Promise<boolean> {
82
90
  // ───────────────────────── 分层读取 ─────────────────────────
83
91
 
84
92
  describe("openResults · 实验 → 快照 → eval → attempt 分层", () => {
85
- it("一个 run 装两个 experiment:切成两个快照;实验按 id 字典序,快照最新在前,latest = snapshots[0]", async () => {
86
- const root = await makeRoot();
87
- const monday = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
88
- res({ id: "algebra/q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", startedAt: "2026-07-01T08:01:00.000Z" }),
89
- res({ id: "algebra/q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", attempt: 2, outcome: "failed" }),
90
- res({ id: "algebra/q2", agent: "bub", model: "gpt-5", experimentId: "compare/bub" }),
91
- res({ id: "algebra/q1", agent: "codex", model: "o3", experimentId: "compare/codex" }),
92
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
93
- const tuesday = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
94
- res({ id: "algebra/q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub" }),
95
- ], { startedAt: "2026-07-02T08:00:00.000Z" }));
93
+ it("字典序实验、最新快照在前;eval 分组、attempt 平铺;快照级字段注入进 attempt.result(缺才补)", async () => {
94
+ const root = await makeRoot();
95
+ const bubDir = await writeSnapshot(
96
+ root,
97
+ "compare_bub",
98
+ "2026-07-01T08-00-00-000Z-a1b2",
99
+ meta({
100
+ experimentId: "compare/bub",
101
+ agent: "bub",
102
+ model: "gpt-5",
103
+ startedAt: "2026-07-01T08:00:00.000Z",
104
+ completedAt: "2026-07-01T08:10:00.000Z",
105
+ }),
106
+ );
107
+ await writeResultFile(bubDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" }));
108
+ await writeResultFile(bubDir, "algebra/q1/a2", record({ id: "algebra/q1", attempt: 2, verdict: "failed" }));
109
+ await writeResultFile(bubDir, "algebra/q2/a1", record({ id: "algebra/q2", attempt: 1 }));
110
+
111
+ await writeSnapshot(
112
+ root,
113
+ "compare_codex",
114
+ "2026-07-01T08-05-00-000Z-c3d4",
115
+ meta({ experimentId: "compare/codex", agent: "codex", model: "o3", startedAt: "2026-07-01T08:05:00.000Z" }),
116
+ );
96
117
 
97
118
  const results = await openResults(root);
98
119
  expect(results.skipped).toHaveLength(0);
99
- expect(results.runDirs.map((r) => r.dir)).toEqual([tuesday, monday]); // 新→旧
100
120
  expect(results.experiments.map((e) => e.id)).toEqual(["compare/bub", "compare/codex"]); // 字典序
101
121
 
102
122
  const bub = results.experiments[0];
103
- expect(bub.snapshots.map((s) => s.startedAt)).toEqual(["2026-07-02T08:00:00.000Z", "2026-07-01T08:00:00.000Z"]);
123
+ expect(bub.snapshots).toHaveLength(1);
104
124
  expect(bub.latest).toBe(bub.snapshots[0]);
105
- expect(bub.evalIds).toEqual(["algebra/q1", "algebra/q2"]); // 本地历史并集
106
-
107
- // eval 分组:attempt 挂在题下面;attempts 平铺 = evals 逐题展开。
108
- const mondayBub = bub.snapshots[1];
109
- expect(mondayBub.agent).toBe("bub");
110
- expect(mondayBub.model).toBe("gpt-5");
111
- expect(mondayBub.producer).toEqual({ name: "niceeval", version: "0.3.0" });
112
- expect(mondayBub.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
113
- expect(mondayBub.runDir.dir).toBe(monday);
114
- expect(mondayBub.evals.map((e) => e.id)).toEqual(["algebra/q1", "algebra/q2"]);
115
- expect(mondayBub.evals[0].attempts).toHaveLength(2);
116
- expect(mondayBub.attempts).toHaveLength(3);
117
-
118
- // attempt 直达字段与证据引用。
119
- const attempt = mondayBub.evals[0].attempts[0];
125
+ expect(bub.evalIds).toEqual(["algebra/q1", "algebra/q2"]);
126
+
127
+ const snap = bub.latest;
128
+ expect(snap.agent).toBe("bub");
129
+ expect(snap.model).toBe("gpt-5");
130
+ expect(snap.producer).toEqual({ name: "niceeval", version: "0.3.0" });
131
+ expect(snap.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
132
+ expect(snap.dir).toBe(bubDir);
133
+ expect(snap.completedAt).toBe("2026-07-01T08:10:00.000Z");
134
+ expect(snap.evals.map((e) => e.id)).toEqual(["algebra/q1", "algebra/q2"]);
135
+ expect(snap.evals[0].attempts).toHaveLength(2);
136
+ expect(snap.attempts).toHaveLength(3);
137
+
138
+ const attempt = snap.evals[0].attempts[0];
120
139
  expect(attempt.evalId).toBe("algebra/q1");
121
140
  expect(attempt.experimentId).toBe("compare/bub");
122
- expect(attempt.ref).toEqual({ run: "2026-07-01T08-00-00-000Z", result: 0 });
141
+ expect(attempt.ref).toEqual({ snapshot: "compare_bub/2026-07-01T08-00-00-000Z-a1b2", attempt: "algebra/q1/a1" });
142
+ // 快照级字段注入(record 没写 agent/model/experimentId)。
143
+ expect(attempt.result.agent).toBe("bub");
144
+ expect(attempt.result.model).toBe("gpt-5");
145
+ expect(attempt.result.experimentId).toBe("compare/bub");
146
+ // 缺才补:条目自带的 startedAt 优先。
123
147
  expect(attempt.result.startedAt).toBe("2026-07-01T08:01:00.000Z");
124
- });
125
-
126
- it("summary.snapshots 元数据:快照级 startedAt 覆盖顶层,knownEvalIds 并进 exp.evalIds", async () => {
127
- const root = await makeRoot();
128
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
129
- res({ id: "q1", agent: "bub", experimentId: "mid/a" }),
130
- res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
131
- ], {
132
- startedAt: "2026-07-01T08:00:00.000Z",
133
- snapshots: {
134
- "mid/b": { startedAt: "2026-06-20T08:00:00.000Z", knownEvalIds: ["q1", "q2", "q3"] },
135
- },
136
- }));
137
-
138
- const results = await openResults(root);
139
- const a = results.experiments.find((e) => e.id === "mid/a")!;
140
- const b = results.experiments.find((e) => e.id === "mid/b")!;
141
- expect(a.latest.startedAt).toBe("2026-07-01T08:00:00.000Z");
142
- expect(b.latest.startedAt).toBe("2026-06-20T08:00:00.000Z");
143
- expect(b.latest.knownEvalIds).toEqual(["q1", "q2", "q3"]);
144
- expect(b.evalIds).toEqual(["q1", "q2", "q3"]); // 本地覆盖 ∪ 携带的 knownEvalIds
145
- expect(a.evalIds).toEqual(["q1"]);
148
+ // 第二个 attempt 没写 startedAt,补快照的。
149
+ expect(snap.evals[0].attempts[1].result.startedAt).toBe("2026-07-01T08:00:00.000Z");
146
150
  });
147
151
  });
148
152
 
149
153
  // ───────────────────────── 懒加载与回退 ─────────────────────────
150
154
 
151
155
  describe("AttemptHandle · 懒加载", () => {
152
- it("缺文件返回 null 不抛错;读过一次即记忆化;artifactsDir 优先、artifactBase 回退;原 run 清理后如实 null", async () => {
153
- const root = await makeRoot();
154
- // run:携带条目的工件真身。
155
- const oldRun = await writeRun(root, "2026-06-30T08-00-00-000Z", summaryOf([
156
- res({ id: "q3", agent: "bub", model: "gpt-5", experimentId: "e", artifactsDir: "q3/bub/gpt-5/e/a1", hasEvents: true }),
157
- ], { startedAt: "2026-06-30T08:00:00.000Z" }));
158
- await writeArtifact(oldRun, "q3/bub/gpt-5/e/a1", "events.json", [{ type: "message", text: "old" }]);
159
-
160
- const runDir = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
161
- res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "e", artifactsDir: "q1/bub/gpt-5/e/a1", hasEvents: true }),
162
- res({ id: "q2", agent: "bub", model: "gpt-5", experimentId: "e" }),
163
- // --resume 携带条目:本 run 没有工件,artifactBase(相对结果根)指向原 run。
164
- res({ id: "q3", agent: "bub", model: "gpt-5", experimentId: "e", artifactBase: "2026-06-30T08-00-00-000Z/q3/bub/gpt-5/e/a1", hasEvents: true }),
165
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
166
- const eventsPath = await writeArtifact(runDir, "q1/bub/gpt-5/e/a1", "events.json", [{ type: "message", text: "hi" }]);
156
+ it("缺文件返回 null;读一次记忆化;attempt 目录优先、artifactBase 回退;原快照清理后如实 null", async () => {
157
+ const root = await makeRoot();
158
+ const oldSnap = await writeSnapshot(
159
+ root,
160
+ "e",
161
+ "2026-06-30T08-00-00-000Z-xxxx",
162
+ meta({ experimentId: "e", agent: "bub", startedAt: "2026-06-30T08:00:00.000Z", completedAt: "2026-06-30T08:10:00.000Z" }),
163
+ );
164
+ await writeResultFile(oldSnap, "q3/a1", record({ id: "q3", attempt: 1, hasEvents: true }));
165
+ await writeArtifactFile(oldSnap, "q3/a1", "events.json", [{ type: "message", text: "old" }]);
166
+
167
+ const newSnap = await writeSnapshot(
168
+ root,
169
+ "e",
170
+ "2026-07-01T08-00-00-000Z-yyyy",
171
+ meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }),
172
+ );
173
+ await writeResultFile(newSnap, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true }));
174
+ const eventsPath = await writeArtifactFile(newSnap, "q1/a1", "events.json", [{ type: "message", text: "hi" }]);
175
+ await writeResultFile(newSnap, "q2/a1", record({ id: "q2", attempt: 1 }));
176
+ await writeResultFile(
177
+ newSnap,
178
+ "q3/a1",
179
+ record({
180
+ id: "q3",
181
+ attempt: 1,
182
+ startedAt: "2026-06-30T08:01:00.000Z",
183
+ artifactBase: "e/2026-06-30T08-00-00-000Z-xxxx/q3/a1",
184
+ hasEvents: true,
185
+ }),
186
+ );
167
187
 
168
188
  const results = await openResults(root);
169
189
  const snap = results.experiments[0].latest;
170
- const [q1, q2, q3] = snap.evals.map((e) => e.attempts[0]);
190
+ const q1 = snap.evals.find((e) => e.id === "q1")!.attempts[0];
191
+ const q2 = snap.evals.find((e) => e.id === "q2")!.attempts[0];
192
+ const q3 = snap.evals.find((e) => e.id === "q3")!.attempts[0];
171
193
 
172
194
  const events = await q1.events();
173
- expect(events).toHaveLength(1);
174
- // summary 只有 hasEvents/hasTrace/hasSources 三个标记,o11y/diff 没有标记 —— 全靠方法语义吸收。
195
+ expect(events).toEqual([{ type: "message", text: "hi" }]);
196
+ // result.json 只有 hasEvents/hasTrace/hasSources 三个标记,o11y/diff 没有标记 —— 全靠方法语义吸收。
175
197
  expect(await q1.trace()).toBeNull();
176
198
  expect(await q1.o11y()).toBeNull();
177
199
  expect(await q1.diff()).toBeNull();
178
200
  expect(await q1.sources()).toBeNull();
179
201
 
180
- // 记忆化:同一 handle 读一次缓存,文件删掉后再读仍返回同一份数据。
181
202
  await rm(eventsPath);
182
- expect(await q1.events()).toBe(events);
203
+ expect(await q1.events()).toBe(events); // 记忆化:同一 handle 不重新读盘
183
204
 
184
- // 条目没有 artifactsDir 也没有 artifactBase:不猜路径,全部 null。
185
- expect(await q2.events()).toBeNull();
205
+ expect(await q2.events()).toBeNull(); // artifactBase,不猜路径
186
206
 
187
- // 携带条目经 artifactBase 回退读到原 run 的工件;ref 指条目所在的落盘(新 run)。
188
- expect(await q3.events()).toEqual([{ type: "message", text: "old" }]);
189
- expect(q3.ref.run).toBe("2026-07-01T08-00-00-000Z");
207
+ expect(await q3.events()).toEqual([{ type: "message", text: "old" }]); // artifactBase 回退到原快照
208
+ expect(q3.ref.snapshot).toBe("e/2026-07-01T08-00-00-000Z-yyyy"); // ref 指条目所在的落盘(新快照)
190
209
 
191
- // run 被清理:回退落空,如实返回 null(新句柄,不吃上面的记忆化)
192
- await rm(oldRun, { recursive: true });
210
+ await rm(oldSnap, { recursive: true });
193
211
  const reopened = await openResults(root);
194
212
  const q3Again = reopened.experiments[0].latest.evals.find((e) => e.id === "q3")!.attempts[0];
195
- expect(await q3Again.events()).toBeNull();
213
+ expect(await q3Again.events()).toBeNull(); // 原快照清理后如实返回 null(新句柄,不吃上面的记忆化)
196
214
  });
197
215
  });
198
216
 
199
217
  // ───────────────────────── skipped 三种原因 ─────────────────────────
200
218
 
201
219
  describe("openResults · skipped", () => {
202
- it("版本不匹配带 schemaVersion 与完整 producer;坏 JSON malformed;无 summary 有工件记 incomplete;无关 JSON 静默;legacy 无信封按 1 读", async () => {
203
- const root = await makeRoot();
204
- const incompatible = await writeRun(root, "2026-07-03T08-00-00-000Z", {
205
- format: RESULTS_FORMAT,
206
- schemaVersion: RESULTS_SCHEMA_VERSION + 1,
207
- producer: { name: "other-harness", version: "9.9.9" },
208
- startedAt: "2026-07-03T08:00:00.000Z",
209
- results: [],
210
- });
211
- await writeRun(root, "2026-07-04T08-00-00-000Z", "not json {");
212
- await writeRun(root, "unrelated", { hello: 1 });
213
- await writeRun(root, "alien-results", { results: [] }); // 只沾一个键:无关 JSON,不进 skipped
214
- // crash 没收尾:有 attempt 工件、没有 summary.json。
215
- const crashed = join(root, "2026-07-05T08-00-00-000Z");
216
- await writeArtifact(crashed, "q1/bub/default/a1", "events.json", [{ type: "message" }]);
217
- // 空目录:既无 summary 也无工件,静默忽略。
218
- await mkdir(join(root, "2026-07-06T08-00-00-000Z"), { recursive: true });
219
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([res({ id: "q1", agent: "bub", experimentId: "e" })]));
220
- // legacy:引入版本号之前的存量报告,没有 format 信封,按 schemaVersion 1 读。
221
- const legacy = summaryOf([res({ id: "q1", agent: "bub", experimentId: "old" })], { startedAt: "2026-06-01T08:00:00.000Z" });
222
- delete legacy.format;
223
- delete legacy.schemaVersion;
224
- delete legacy.producer;
225
- await writeRun(root, "2026-06-01T08-00-00-000Z", legacy);
220
+ it("incompatible(v3 summary.json / 无信封 legacy)、malformed(坏 JSON)、incomplete(有 attempt snapshot.json);无关 JSON 静默", async () => {
221
+ const root = await makeRoot();
222
+
223
+ // v2/v3 的 summary.json 带 format + schemaVersion(≠ 4),自然落进 incompatible 档。
224
+ const v3Dir = join(root, "old-exp", "2026-06-01T08-00-00-000Z");
225
+ await mkdir(v3Dir, { recursive: true });
226
+ await writeFile(
227
+ join(v3Dir, "summary.json"),
228
+ JSON.stringify({
229
+ format: RESULTS_FORMAT,
230
+ schemaVersion: 3,
231
+ producer: { name: "niceeval", version: "0.4.6" },
232
+ agent: "bub",
233
+ startedAt: "2026-06-01T08:00:00.000Z",
234
+ completedAt: "2026-06-01T08:10:00.000Z",
235
+ passed: 1,
236
+ failed: 0,
237
+ skipped: 0,
238
+ errored: 0,
239
+ durationMs: 1000,
240
+ results: [],
241
+ }),
242
+ "utf-8",
243
+ );
244
+
245
+ // legacy:引入版本信封之前的存量报告,无 format,按 schemaVersion 1 读。
246
+ const legacyDir = join(root, "legacy-exp", "2026-05-01T08-00-00-000Z");
247
+ await mkdir(legacyDir, { recursive: true });
248
+ await writeFile(
249
+ join(legacyDir, "summary.json"),
250
+ JSON.stringify({ agent: "bub", startedAt: "2026-05-01T08:00:00.000Z", completedAt: "2026-05-01T08:10:00.000Z", results: [] }),
251
+ "utf-8",
252
+ );
253
+
254
+ // malformed:坏 JSON。
255
+ const badDir = join(root, "bad-exp", "2026-07-02T08-00-00-000Z-zzzz");
256
+ await mkdir(badDir, { recursive: true });
257
+ await writeFile(join(badDir, "snapshot.json"), "not json {", "utf-8");
258
+
259
+ // incomplete:有 result.json,没有 snapshot.json —— crash 没收尾。
260
+ const crashDir = join(root, "crash-exp", "2026-07-03T08-00-00-000Z-wwww");
261
+ await writeResultFile(crashDir, "q1/a1", record({ id: "q1", attempt: 1 }));
262
+
263
+ // 直接存在 summary.json 但内容与 niceeval 无关(不满足 legacy 启发式)→ not-a-report,静默忽略,
264
+ // 且不连累父目录被判 incomplete。
265
+ const alienDir = join(root, "alien-exp", "2026-07-06T08-00-00-000Z-alien");
266
+ await mkdir(alienDir, { recursive: true });
267
+ await writeFile(join(alienDir, "summary.json"), JSON.stringify({ hello: 1 }), "utf-8");
268
+
269
+ // 完全无关的空目录:静默忽略。
270
+ await mkdir(join(root, "unrelated"), { recursive: true });
271
+ await writeFile(join(root, "unrelated", "hello.json"), JSON.stringify({ hello: 1 }), "utf-8");
272
+
273
+ // 一份正常快照,确认不受干扰。
274
+ const okDir = await writeSnapshot(root, "ok-exp", "2026-07-04T08-00-00-000Z-oooo", meta({ experimentId: "ok", agent: "bub", startedAt: "2026-07-04T08:00:00.000Z" }));
275
+ await writeResultFile(okDir, "q1/a1", record({ id: "q1", attempt: 1 }));
226
276
 
227
277
  const results = await openResults(root);
228
- expect(results.runDirs).toHaveLength(2);
229
- expect(results.skipped).toHaveLength(3);
278
+ expect(results.experiments.map((e) => e.id)).toEqual(["ok"]);
279
+ expect(results.skipped).toHaveLength(4);
230
280
 
231
- const versionSkip = results.skipped.find((s) => s.reason === "incompatible-version")!;
232
- expect(versionSkip.dir).toBe(incompatible);
233
- expect(versionSkip.schemaVersion).toBe(RESULTS_SCHEMA_VERSION + 1);
234
- // 完整 producer:第三方 harness 的名字如实报出,消费方才能做对「要不要拼 npx 提示」的分支。
235
- expect(versionSkip.producer).toEqual({ name: "other-harness", version: "9.9.9" });
281
+ const v3Skip = results.skipped.find((s) => s.dir === v3Dir)!;
282
+ expect(v3Skip.reason).toBe("incompatible-version");
283
+ expect(v3Skip.schemaVersion).toBe(3);
284
+ expect(v3Skip.producer).toEqual({ name: "niceeval", version: "0.4.6" });
236
285
 
237
- expect(results.skipped.find((s) => s.reason === "malformed")!.detail).toBe("invalid JSON");
238
- expect(results.skipped.find((s) => s.reason === "incomplete")!.dir).toBe(crashed);
286
+ const legacySkip = results.skipped.find((s) => s.dir === legacyDir)!;
287
+ expect(legacySkip.reason).toBe("incompatible-version");
288
+ expect(legacySkip.schemaVersion).toBe(1);
239
289
 
240
- const legacyExp = results.experiments.find((e) => e.id === "old")!;
241
- expect(legacyExp.latest.schemaVersion).toBe(1);
242
- expect(legacyExp.latest.producer).toBeUndefined();
290
+ expect(results.skipped.find((s) => s.dir === badDir)!.reason).toBe("malformed");
291
+ expect(results.skipped.find((s) => s.dir === crashDir)!.reason).toBe("incomplete");
292
+ expect(results.skipped.find((s) => s.dir === alienDir)).toBeUndefined();
243
293
  });
244
294
  });
245
295
 
246
- // ───────────────────────── latest() 选集与警告 ─────────────────────────
296
+ // ───────────────────────── latest() Selection 与警告 ─────────────────────────
247
297
 
248
298
  describe("results.latest() · Selection", () => {
249
299
  it("每个实验取最新快照;experiments 前缀过滤同 CLI 语义(尾斜杠等价)", async () => {
250
300
  const root = await makeRoot();
251
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
252
- res({ id: "q1", agent: "bub", experimentId: "mid/a" }),
253
- res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
254
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
255
- const tuesday = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
256
- res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
257
- ], { startedAt: "2026-07-02T08:00:00.000Z" }));
301
+ await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
302
+ await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
303
+ const tuesday = await writeSnapshot(root, "mid_b", "s2", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
258
304
 
259
305
  const results = await openResults(root);
260
306
  const latest = results.latest();
261
307
  expect(latest.snapshots.map((s) => s.experimentId)).toEqual(["mid/a", "mid/b"]);
262
- expect(latest.snapshots[1].runDir.dir).toBe(tuesday);
308
+ expect(latest.snapshots[1].dir).toBe(tuesday);
263
309
 
264
310
  expect(results.latest({ experiments: "mid/a" }).snapshots).toHaveLength(1);
265
311
  expect(results.latest({ experiments: "mid/" }).snapshots).toHaveLength(2);
@@ -270,14 +316,13 @@ describe("results.latest() · Selection", () => {
270
316
 
271
317
  it("partial-coverage:最新快照覆盖 < 已知并集;结构化字段 + 渲染好的英文 message", async () => {
272
318
  const root = await makeRoot();
273
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
274
- res({ id: "algebra/q1", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
275
- res({ id: "algebra/q2", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
276
- res({ id: "algebra/q3", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
277
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
278
- await writeRun(root, "2026-07-05T08-00-00-000Z", summaryOf([
279
- res({ id: "algebra/q1", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
280
- ], { startedAt: "2026-07-05T08:00:00.000Z" }));
319
+ const mondayDir = await writeSnapshot(root, "midterm", "2026-07-01T08-00-00-000Z", meta({ experimentId: "midterm/bub-gpt-5.4", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
320
+ await writeResultFile(mondayDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1 }));
321
+ await writeResultFile(mondayDir, "algebra/q2/a1", record({ id: "algebra/q2", attempt: 1 }));
322
+ await writeResultFile(mondayDir, "algebra/q3/a1", record({ id: "algebra/q3", attempt: 1 }));
323
+
324
+ const fridayDir = await writeSnapshot(root, "midterm", "2026-07-05T08-00-00-000Z", meta({ experimentId: "midterm/bub-gpt-5.4", agent: "bub", startedAt: "2026-07-05T08:00:00.000Z", completedAt: "2026-07-05T08:10:00.000Z" }));
325
+ await writeResultFile(fridayDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1 }));
281
326
 
282
327
  const latest = (await openResults(root)).latest();
283
328
  expect(latest.snapshots).toHaveLength(1);
@@ -288,56 +333,41 @@ describe("results.latest() · Selection", () => {
288
333
  );
289
334
  });
290
335
 
291
- it("stale-snapshot:早于选集中最新落盘即触发(无阈值),message 带人话时距", async () => {
336
+ it("stale-snapshot:早于 Selection 中最新落盘即触发(无阈值),message 带人话时距", async () => {
292
337
  const root = await makeRoot();
293
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
294
- res({ id: "q1", agent: "bub", experimentId: "mid/a" }),
295
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
296
- await writeRun(root, "2026-07-05T08-00-00-000Z", summaryOf([
297
- res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
298
- ], { startedAt: "2026-07-05T08:00:00.000Z" }));
338
+ await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
339
+ await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-05T08:00:00.000Z", completedAt: "2026-07-05T08:10:00.000Z" }));
299
340
 
300
341
  const latest = (await openResults(root)).latest();
301
342
  const stale = latest.warnings.filter((w) => w.kind === "stale-snapshot");
302
343
  expect(stale).toHaveLength(1);
303
- expect(stale[0]).toMatchObject({
304
- experimentId: "mid/a",
305
- startedAt: "2026-07-01T08:00:00.000Z",
306
- latestStartedAt: "2026-07-05T08:00:00.000Z",
307
- });
344
+ expect(stale[0]).toMatchObject({ experimentId: "mid/a", startedAt: "2026-07-01T08:00:00.000Z", latestStartedAt: "2026-07-05T08:00:00.000Z" });
308
345
  expect(stale[0].message).toContain("predates the latest run in this selection by 4 days");
309
346
  });
310
347
 
311
- it("synthetic-experiment-id:落盘缺 experimentId 以 <agent>/<model> 合成键(无 model 用 default)", async () => {
348
+ it("unfinished-snapshot:选中快照缺 completedAt", async () => {
312
349
  const root = await makeRoot();
313
- const runDir = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
314
- res({ id: "q1", agent: "codex" }),
315
- ]));
350
+ const dir = await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }));
316
351
 
317
- const results = await openResults(root);
318
- expect(results.experiments[0].id).toBe("codex/default");
319
- expect(results.experiments[0].latest.synthetic).toBe(true);
320
-
321
- const latest = results.latest();
322
- const synthetic = latest.warnings.find((w) => w.kind === "synthetic-experiment-id")!;
323
- expect(synthetic).toMatchObject({ experimentId: "codex/default", runDir });
324
- expect(synthetic.message).toContain("without experimentId");
325
- // 合成键拼不出可执行的 niceeval exp 命令:partial 提示退化成中性说法(此处无 partial,仅验证不炸)。
352
+ const latest = (await openResults(root)).latest();
353
+ const warn = latest.warnings.find((w) => w.kind === "unfinished-snapshot")!;
354
+ expect(warn).toMatchObject({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir });
355
+ expect(warn.message).toContain("has no completedAt");
326
356
  });
327
357
 
328
358
  it("Selection.filter 只删不换:快照删减,幸存实验的警告保留、其余丢弃", async () => {
329
359
  const root = await makeRoot();
330
- // 两个实验都制造 partial-coverage。
331
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
332
- res({ id: "q1", agent: "bub", experimentId: "mid/a" }),
333
- res({ id: "q2", agent: "bub", experimentId: "mid/a" }),
334
- res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
335
- res({ id: "q2", agent: "codex", experimentId: "mid/b" }),
336
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
337
- await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
338
- res({ id: "q1", agent: "bub", experimentId: "mid/a" }),
339
- res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
340
- ], { startedAt: "2026-07-02T08:00:00.000Z" }));
360
+ const aDir = await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
361
+ await writeResultFile(aDir, "q1/a1", record({ id: "q1", attempt: 1 }));
362
+ await writeResultFile(aDir, "q2/a1", record({ id: "q2", attempt: 1 }));
363
+ const bDir = await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
364
+ await writeResultFile(bDir, "q1/a1", record({ id: "q1", attempt: 1 }));
365
+ await writeResultFile(bDir, "q2/a1", record({ id: "q2", attempt: 1 }));
366
+
367
+ const aDir2 = await writeSnapshot(root, "mid_a", "s2", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
368
+ await writeResultFile(aDir2, "q1/a1", record({ id: "q1", attempt: 1 }));
369
+ const bDir2 = await writeSnapshot(root, "mid_b", "s2", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
370
+ await writeResultFile(bDir2, "q1/a1", record({ id: "q1", attempt: 1 }));
341
371
 
342
372
  const latest = (await openResults(root)).latest();
343
373
  expect(latest.warnings.filter((w) => w.kind === "partial-coverage")).toHaveLength(2);
@@ -345,7 +375,7 @@ describe("results.latest() · Selection", () => {
345
375
  const filtered = latest.filter((s) => s.experimentId !== "mid/b");
346
376
  expect(filtered.snapshots.map((s) => s.experimentId)).toEqual(["mid/a"]);
347
377
  expect(filtered.warnings.map((w) => w.experimentId)).toEqual(["mid/a"]);
348
- // 原选集不被改动。
378
+ // 原 Selection 不被改动。
349
379
  expect(latest.snapshots).toHaveLength(2);
350
380
  expect(latest.warnings).toHaveLength(2);
351
381
  });
@@ -353,30 +383,50 @@ describe("results.latest() · Selection", () => {
353
383
 
354
384
  // ───────────────────────── 身份键去重 ─────────────────────────
355
385
 
356
- describe("dedupeAttempts", () => {
357
- it("按 (experimentId, evalId, attempt, startedAt) 去重,保留最新 run 目录的那份;缺 startedAt 不去重并出 missing-startedAt", async () => {
358
- const root = await makeRoot();
359
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
360
- res({ id: "q1", agent: "bub", experimentId: "e", startedAt: "2026-07-01T08:01:00.000Z" }),
361
- res({ id: "q2", agent: "bub", experimentId: "e", outcome: "failed", startedAt: "2026-07-01T08:02:00.000Z" }),
362
- res({ id: "q3", agent: "bub", experimentId: "e" }),
363
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
364
- const tuesday = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
365
- res({ id: "q1", agent: "bub", experimentId: "e", startedAt: "2026-07-01T08:01:00.000Z" }), // resume 原样合入
366
- res({ id: "q2", agent: "bub", experimentId: "e", startedAt: "2026-07-02T08:02:00.000Z" }), // 重跑,新 startedAt
367
- res({ id: "q3", agent: "bub", experimentId: "e" }),
368
- ], { startedAt: "2026-07-02T08:00:00.000Z" }));
386
+ function fakeSnapshot(over: { experimentId: string; startedAt: string; dir: string }): Snapshot {
387
+ return {
388
+ agent: "bub",
389
+ producer: { name: "niceeval" },
390
+ schemaVersion: RESULTS_SCHEMA_VERSION,
391
+ evals: [],
392
+ attempts: [],
393
+ ...over,
394
+ };
395
+ }
369
396
 
370
- const results = await openResults(root);
371
- const all = results.experiments[0].snapshots.flatMap((s) => s.attempts);
372
- expect(all).toHaveLength(6);
397
+ function fakeAttempt(snapshot: Snapshot, result: EvalResult): AttemptHandle {
398
+ return {
399
+ evalId: result.id,
400
+ experimentId: snapshot.experimentId,
401
+ result,
402
+ ref: { snapshot: "x/y", attempt: `${result.id}/a${result.attempt}` },
403
+ snapshot,
404
+ events: async () => null,
405
+ trace: async () => null,
406
+ o11y: async () => null,
407
+ agentSetup: async () => null,
408
+ diff: async () => null,
409
+ sources: async () => null,
410
+ };
411
+ }
373
412
 
374
- const { attempts, warnings } = dedupeAttempts(all);
375
- // q1 合并成一条(取新 run ref 落在最新落盘);q2 两条(两次真实运行);q3 两条(缺 startedAt 不敢去重)
413
+ describe("dedupeAttempts", () => {
414
+ it("按 (experimentId, evalId, attempt, startedAt) 去重,保留最新快照里的那份;缺 startedAt 不去重并出 missing-startedAt", () => {
415
+ const monday = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir: "/tmp/e/monday" });
416
+ const tuesday = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-02T08:00:00.000Z", dir: "/tmp/e/tuesday" });
417
+
418
+ const a1 = fakeAttempt(monday, record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" }));
419
+ const a1Resumed = fakeAttempt(tuesday, record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" })); // resume 原样合入
420
+ const a2Mon = fakeAttempt(monday, record({ id: "q2", attempt: 1, startedAt: "2026-07-01T08:02:00.000Z" }));
421
+ const a2Tue = fakeAttempt(tuesday, record({ id: "q2", attempt: 1, startedAt: "2026-07-02T08:02:00.000Z" })); // 重跑,新 startedAt
422
+ const a3Mon = fakeAttempt(monday, record({ id: "q3", attempt: 1 })); // 缺 startedAt(携带条目缺锚的极端情况)
423
+ const a3Tue = fakeAttempt(tuesday, record({ id: "q3", attempt: 1 }));
424
+
425
+ const { attempts, warnings } = dedupeAttempts([a1, a2Mon, a3Mon, a1Resumed, a2Tue, a3Tue]);
376
426
  expect(attempts).toHaveLength(5);
377
427
  const q1 = attempts.filter((a) => a.evalId === "q1");
378
428
  expect(q1).toHaveLength(1);
379
- expect(q1[0].runDir.dir).toBe(tuesday);
429
+ expect(q1[0].snapshot).toBe(tuesday); // 保留最新快照
380
430
  expect(attempts.filter((a) => a.evalId === "q2")).toHaveLength(2);
381
431
  expect(attempts.filter((a) => a.evalId === "q3")).toHaveLength(2);
382
432
 
@@ -386,53 +436,44 @@ describe("dedupeAttempts", () => {
386
436
  });
387
437
  });
388
438
 
389
- // ───────────────────────── writer roundtrip ─────────────────────────
439
+ // ───────────────────────── writer ─────────────────────────
390
440
 
391
- describe("createRunWriter", () => {
392
- it("writeAttempt + snapshot 声明写出 openResults 读回逐字段相等;工件懒加载;knownEvalIds 进分母", async () => {
441
+ describe("createResultsWriter", () => {
442
+ it("snapshot() 建目录(独占)+ snapshot.json(无 completedAt);writeAttempt artifact + 回填 has*;finish 补 completedAt 并幂等", async () => {
393
443
  const root = await makeRoot();
394
- const writer = await createRunWriter(root, { producer: { name: "my-harness", version: "1.0.0" } });
395
- expect(writer.dir.startsWith(root)).toBe(true);
444
+ const writer = createResultsWriter(root, { producer: { name: "my-harness", version: "1.0.0" } });
396
445
 
397
- const snapA = writer.snapshot({
398
- experiment: "compare/a",
446
+ const snapA = await writer.snapshot({
447
+ experimentId: "compare/a",
399
448
  agent: "bub",
400
449
  model: "gpt-5",
401
450
  startedAt: "2026-07-01T08:00:00.000Z",
402
451
  knownEvalIds: ["q1", "q2", "q3"],
403
452
  });
404
- const events = [{ type: "message", role: "assistant", text: "hi" }] as never[];
405
- const o11y = { toolCalls: 2 } as never;
453
+ expect(snapA.dir.startsWith(root)).toBe(true);
454
+ const before = JSON.parse(await readFile(join(snapA.dir, "snapshot.json"), "utf-8"));
455
+ expect(before.completedAt).toBeUndefined();
456
+ expect(Object.keys(before)).toEqual(["format", "schemaVersion", "producer", "experimentId", "agent", "model", "startedAt", "knownEvalIds"]);
457
+
458
+ const events = [{ type: "message", text: "hi" }] as never[];
459
+ const o11yData = { toolCalls: 2 } as never;
406
460
  await snapA.writeAttempt(
407
- {
408
- id: "q1",
409
- outcome: "passed",
410
- attempt: 1,
411
- durationMs: 100,
412
- assertions: [],
413
- usage: { inputTokens: 10, outputTokens: 5 },
414
- estimatedCostUSD: 0.25,
415
- },
416
- { events, o11y },
461
+ { id: "q1", verdict: "passed", attempt: 1, durationMs: 100, assertions: [], usage: { inputTokens: 10, outputTokens: 5 }, estimatedCostUSD: 0.25 },
462
+ { events, o11y: o11yData },
417
463
  );
418
- await snapA.writeAttempt({ id: "q2", outcome: "failed", attempt: 1, durationMs: 50, assertions: [] });
464
+ await snapA.writeAttempt({ id: "q2", verdict: "failed", attempt: 1, durationMs: 50, assertions: [] });
419
465
 
420
- const snapB = writer.snapshot({ experiment: "compare/b", agent: "codex", startedAt: "2026-07-02T09:00:00.000Z" });
421
- await snapB.writeAttempt(
422
- { id: "q1", outcome: "passed", attempt: 1, durationMs: 80, assertions: [] },
423
- { diff: { generatedFiles: { "a.txt": "1" }, deletedFiles: [] } },
424
- );
466
+ const snapB = await writer.snapshot({ experimentId: "compare/b", agent: "codex", startedAt: "2026-07-02T09:00:00.000Z" });
467
+ await snapB.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 80, assertions: [] }, { diff: { generatedFiles: { "a.txt": "1" }, deletedFiles: [] } });
468
+
469
+ await writer.finish();
470
+ expect(writer.snapshotDirs().map((s) => s.experimentId).sort()).toEqual(["compare/a", "compare/b"]);
425
471
 
426
- const summary = await writer.finish();
427
- // summary 从已写 attempt 推导:计数永远和条目一致;版本元数据注入。
428
- expect(summary.format).toBe(RESULTS_FORMAT);
429
- expect(summary.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
430
- expect(summary.producer).toEqual({ name: "my-harness", version: "1.0.0" });
431
- expect(summary.passed).toBe(2);
432
- expect(summary.failed).toBe(1);
433
- expect(summary.startedAt).toBe("2026-07-01T08:00:00.000Z"); // 最早的快照时刻
434
- expect(summary.usage).toEqual({ inputTokens: 10, outputTokens: 5 });
435
- expect(summary.estimatedCostUSD).toBe(0.25);
472
+ const after = JSON.parse(await readFile(join(snapA.dir, "snapshot.json"), "utf-8"));
473
+ expect(typeof after.completedAt).toBe("string");
474
+ expect(Object.keys(after)).toEqual(["format", "schemaVersion", "producer", "experimentId", "agent", "model", "startedAt", "completedAt", "knownEvalIds"]);
475
+
476
+ await expect(writer.finish()).rejects.toThrow(/already called/);
436
477
 
437
478
  const results = await openResults(root);
438
479
  expect(results.skipped).toHaveLength(0);
@@ -441,22 +482,17 @@ describe("createRunWriter", () => {
441
482
  const a = results.experiments[0].latest;
442
483
  expect(a.agent).toBe("bub");
443
484
  expect(a.model).toBe("gpt-5");
444
- expect(a.startedAt).toBe("2026-07-01T08:00:00.000Z");
445
- expect(a.producer).toEqual({ name: "my-harness", version: "1.0.0" });
446
- expect(a.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
447
485
  expect(a.knownEvalIds).toEqual(["q1", "q2", "q3"]);
448
486
  expect(results.experiments[0].evalIds).toEqual(["q1", "q2", "q3"]);
449
487
 
450
- // 快照级字段注入进条目(agent/model/experimentId/startedAt),attempt 级字段原样读回。
451
488
  const q1 = a.evals.find((e) => e.id === "q1")!.attempts[0];
452
- expect(q1.experimentId).toBe("compare/a");
453
489
  expect(q1.result).toMatchObject({
454
490
  id: "q1",
455
491
  agent: "bub",
456
492
  model: "gpt-5",
457
493
  experimentId: "compare/a",
458
494
  startedAt: "2026-07-01T08:00:00.000Z",
459
- outcome: "passed",
495
+ verdict: "passed",
460
496
  durationMs: 100,
461
497
  usage: { inputTokens: 10, outputTokens: 5 },
462
498
  estimatedCostUSD: 0.25,
@@ -465,137 +501,122 @@ describe("createRunWriter", () => {
465
501
  hasSources: false,
466
502
  });
467
503
  expect(await q1.events()).toEqual(events);
468
- expect(await q1.o11y()).toEqual(o11y);
504
+ expect(await q1.o11y()).toEqual(o11yData);
469
505
  expect(await q1.trace()).toBeNull();
470
- expect(await q1.diff()).toBeNull();
471
506
 
472
- // 第二个快照:自己的 startedAt(≠ 顶层)经快照元数据读回;diff 工件可达。
473
507
  const b = results.experiments[1].latest;
474
- expect(b.agent).toBe("codex");
475
508
  expect(b.model).toBeUndefined();
476
- expect(b.startedAt).toBe("2026-07-02T09:00:00.000Z");
477
509
  expect(await b.attempts[0].diff()).toEqual({ generatedFiles: { "a.txt": "1" }, deletedFiles: [] });
478
510
 
479
- // knownEvalIds 是残缺检测的分母:compare/a 只写了 2/3 → partial-coverage。
480
511
  const partial = results.latest().warnings.find((w) => w.kind === "partial-coverage")!;
481
512
  expect(partial).toMatchObject({ experimentId: "compare/a", covered: 2, total: 3 });
482
513
  });
483
514
 
484
- it("没走到 finish() 的目录 = skipped(\"incomplete\"):有工件、无 summary,reader 不读半份落盘", async () => {
515
+ it("agentSetup:落成 agent-setup.json(不内联进 result.json),懒加载读回;没装扩展的 attempt null;copySnapshots 能带上", async () => {
485
516
  const root = await makeRoot();
486
- const writer = await createRunWriter(root, { producer: { name: "my-harness" } });
487
- const snap = writer.snapshot({ experiment: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
488
- await snap.writeAttempt(
489
- { id: "q1", outcome: "passed", attempt: 1, durationMs: 10, assertions: [] },
490
- { events: [{ type: "message" }] as never[] },
491
- );
492
- // crash:没有 finish()。
517
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "0.12.0" } });
518
+ const manifest = {
519
+ skills: [{ kind: "repo" as const, source: "Effect-TS/skills", ref: "8f3c1a2", skills: ["effect"] }],
520
+ nativePlugins: [
521
+ {
522
+ agent: "claude-code" as const,
523
+ marketplace: { name: "acme", source: "acme/claude-code-plugins", ref: "v1.3.0" },
524
+ name: "safe-shell",
525
+ resolvedVersion: "1.3.0",
526
+ },
527
+ ],
528
+ mcpServers: [{ name: "browser", command: "npx", args: ["-y", "@modelcontextprotocol/server-browser"] }],
529
+ };
530
+
531
+ const snap = await writer.snapshot({
532
+ experimentId: "skill-ab/claude-effect",
533
+ agent: "claude-code",
534
+ startedAt: "2026-07-11T08:00:00.000Z",
535
+ });
536
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 10, assertions: [] }, { agentSetup: manifest });
537
+ await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 1, durationMs: 10, assertions: [] });
538
+ await writer.finish();
539
+
540
+ // 文件名是磁盘侧的 kebab;判决记录里不内联 manifest(它是 artifact,不是判决的一部分)
541
+ const attemptDir = join(snap.dir, "q1", "a1");
542
+ expect(await exists(join(attemptDir, "agent-setup.json"))).toBe(true);
543
+ expect(JSON.parse(await readFile(join(attemptDir, "result.json"), "utf-8")).agentSetup).toBeUndefined();
544
+ expect(await exists(join(snap.dir, "q2", "a1", "agent-setup.json"))).toBe(false);
545
+
493
546
  const results = await openResults(root);
494
- expect(results.experiments).toHaveLength(0);
495
- expect(results.skipped).toEqual([{ dir: writer.dir, reason: "incomplete" }]);
547
+ const [q1, q2] = results.experiments[0].latest.attempts;
548
+ expect(await q1.agentSetup()).toEqual(manifest);
549
+ expect(await q2.agentSetup()).toBeNull();
550
+
551
+ const dest = join(await makeRoot(), "published");
552
+ await copySnapshots(results.latest(), dest, { artifacts: ["agentSetup"] });
553
+ const copied = join(dest, "skill-ab_claude-effect", basename(snap.dir), "q1", "a1", "agent-setup.json");
554
+ expect(JSON.parse(await readFile(copied, "utf-8"))).toEqual(manifest);
496
555
  });
497
556
 
498
- it("snapshot() startedAt 必填(运行时也拦):身份键与去重以它为锚", async () => {
557
+ it("snapshot() experimentId/agent/startedAt 抛可执行错误", async () => {
499
558
  const root = await makeRoot();
500
- const writer = await createRunWriter(root, { producer: { name: "x" } });
501
- expect(() => writer.snapshot({ experiment: "e", agent: "a", startedAt: "" })).toThrow(/startedAt/);
559
+ const writer = createResultsWriter(root, { producer: { name: "x" } });
560
+ await expect(writer.snapshot({ experimentId: "", agent: "a", startedAt: "t" })).rejects.toThrow(/experimentId/);
561
+ await expect(writer.snapshot({ experimentId: "e", agent: "", startedAt: "t" })).rejects.toThrow(/agent/);
562
+ await expect(writer.snapshot({ experimentId: "e", agent: "a", startedAt: "" })).rejects.toThrow(/startedAt/);
502
563
  });
503
- });
504
564
 
505
- // ───────────────────────── copySnapshots ─────────────────────────
506
-
507
- describe("copySnapshots", () => {
508
- it("按指定工件复制;summary 重建保留版本元数据;补记 knownEvalIds 让发布目录重算出同样的残缺警告", async () => {
509
- const root = await makeRoot();
510
- const monday = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
511
- res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", artifactsDir: "q1/bub/gpt-5/compare_bub/a1", hasEvents: true, hasTrace: true, startedAt: "2026-07-01T08:01:00.000Z" }),
512
- res({ id: "q2", agent: "bub", model: "gpt-5", experimentId: "compare/bub", startedAt: "2026-07-01T08:02:00.000Z" }),
513
- res({ id: "q1", agent: "codex", model: "o3", experimentId: "compare/codex", artifactsDir: "q1/codex/o3/compare_codex/a1", hasEvents: true, outcome: "failed", startedAt: "2026-07-01T08:03:00.000Z" }),
514
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
515
- await writeArtifact(monday, "q1/bub/gpt-5/compare_bub/a1", "events.json", [{ n: 1 }]);
516
- await writeArtifact(monday, "q1/bub/gpt-5/compare_bub/a1", "trace.json", [{ name: "turn" }]);
517
- await writeArtifact(monday, "q1/bub/gpt-5/compare_bub/a1", "diff.json", { generatedFiles: {}, deletedFiles: [] });
518
- await writeArtifact(monday, "q1/codex/o3/compare_codex/a1", "events.json", [{ n: 2 }]);
519
- // 周五只重跑了 compare/bub 的 q1:它的最新快照残缺。
520
- const friday = await writeRun(root, "2026-07-05T08-00-00-000Z", summaryOf([
521
- res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", artifactsDir: "q1/bub/gpt-5/compare_bub/a1", hasEvents: true, startedAt: "2026-07-05T08:01:00.000Z" }),
522
- ], { startedAt: "2026-07-05T08:00:00.000Z" }));
523
- await writeArtifact(friday, "q1/bub/gpt-5/compare_bub/a1", "events.json", [{ n: 1 }, { n: 2 }]);
524
-
525
- const results = await openResults(root);
526
- const dest = join(await makeRoot(), "site/data/run");
527
- const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events", "sources"] });
528
-
529
- expect(copied.warnings).toHaveLength(0);
530
- expect(copied.summary.format).toBe(RESULTS_FORMAT);
531
- expect(copied.summary.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
532
- expect(copied.summary.producer?.version).toBe("0.3.0");
533
- expect(copied.summary.results).toHaveLength(2);
534
- // 补记的覆盖事实:复制时刻该实验已知的 eval 并集。
535
- expect(copied.summary.snapshots?.["compare/bub"].knownEvalIds).toEqual(["q1", "q2"]);
536
- expect(copied.summary.snapshots?.["compare/codex"].knownEvalIds).toEqual(["q1"]);
537
-
538
- // 磁盘:只有选中的工件种类被复制;存在标记按目标目录重算。
539
- expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/events.json"))).toBe(true);
540
- expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/trace.json"))).toBe(false);
541
- expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/diff.json"))).toBe(false);
542
- const bubEntry = copied.summary.results.find((r) => r.experimentId === "compare/bub")!;
543
- expect(bubEntry.startedAt).toBe("2026-07-05T08:01:00.000Z"); // 最新快照的那份
544
- expect(bubEntry.hasEvents).toBe(true);
545
- expect(bubEntry.hasTrace).toBe(false);
565
+ it("同一 writer 内同 experimentId 重复声明:复用同一个 SnapshotWriter,knownEvalIds 取并集", async () => {
566
+ const root = await makeRoot();
567
+ const writer = createResultsWriter(root, { producer: { name: "x" } });
568
+ const s1 = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q1", "q2"] });
569
+ const s2 = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q2", "q3"] });
570
+ expect(s2).toBe(s1);
571
+ await writer.finish();
572
+ const written = JSON.parse(await readFile(join(s1.dir, "snapshot.json"), "utf-8"));
573
+ expect(written.knownEvalIds).toEqual(["q1", "q2", "q3"]);
574
+ });
546
575
 
547
- // 发布目录上重新 openResults().latest():残缺警告被同一套机制重新算出来,不靠发布者转述。
548
- const republished = await openResults(dest);
549
- expect(republished.experiments.find((e) => e.id === "compare/bub")!.evalIds).toEqual(["q1", "q2"]);
550
- const partial = republished.latest().warnings.find((w) => w.kind === "partial-coverage")!;
551
- expect(partial).toMatchObject({ experimentId: "compare/bub", covered: 1, total: 2 });
552
- // 快照各自的 startedAt 也随行(compare/bub 周五、compare/codex 周一)
553
- expect(republished.experiments.find((e) => e.id === "compare/bub")!.latest.startedAt).toBe("2026-07-05T08:00:00.000Z");
554
- expect(republished.experiments.find((e) => e.id === "compare/codex")!.latest.startedAt).toBe("2026-07-01T08:00:00.000Z");
555
- // 工件懒加载在发布目录同样成立。
556
- const bubAttempt = republished.experiments.find((e) => e.id === "compare/bub")!.latest.attempts[0];
557
- expect(await bubAttempt.events()).toHaveLength(2);
558
- expect(await bubAttempt.trace()).toBeNull();
559
- });
560
-
561
- it("目标目录非空即报错(不静默覆盖、不合并);artifacts 非法值报错;同实验重复快照落同一目录出 warning", async () => {
562
- const root = await makeRoot();
563
- await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
564
- res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "e", artifactsDir: "q1/bub/gpt-5/e/a1", startedAt: "2026-07-01T08:01:00.000Z" }),
565
- ], { startedAt: "2026-07-01T08:00:00.000Z" }));
566
- await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
567
- res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "e", artifactsDir: "q1/bub/gpt-5/e/a1", startedAt: "2026-07-01T08:01:00.000Z" }),
568
- ], { startedAt: "2026-07-02T08:00:00.000Z" }));
576
+ it("同一毫秒并发声明不同 experimentId 的快照:互不干扰,各自独立目录", async () => {
577
+ const root = await makeRoot();
578
+ const writer = createResultsWriter(root, { producer: { name: "x" } });
579
+ const now = "2026-07-01T08:00:00.000Z";
580
+ const [a, b, c] = await Promise.all([
581
+ writer.snapshot({ experimentId: "e/a", agent: "bub", startedAt: now }),
582
+ writer.snapshot({ experimentId: "e/b", agent: "bub", startedAt: now }),
583
+ writer.snapshot({ experimentId: "e/c", agent: "bub", startedAt: now }),
584
+ ]);
585
+ expect(new Set([a.dir, b.dir, c.dir]).size).toBe(3);
586
+ await writer.finish();
569
587
  const results = await openResults(root);
570
-
571
- const occupied = await makeRoot();
572
- await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
573
- await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
574
-
575
- await expect(
576
- copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] }),
577
- ).rejects.toThrow(/Unknown artifact kind/);
578
-
579
- await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
580
-
581
- // 同一 experiment 的两个快照(未走 latest 的手工数组):同键 attempt 落同一目录 → 保留最新 + warning。
582
- const both = results.experiments[0].snapshots;
583
- expect(both).toHaveLength(2);
584
- const dest2 = join(await makeRoot(), "run2");
585
- const collided = await copySnapshots(both, dest2);
586
- expect(collided.warnings).toHaveLength(1);
587
- expect(collided.warnings[0]).toMatch(/multiple attempts map to/);
588
- expect(collided.summary.results).toHaveLength(1);
588
+ expect(results.experiments.map((e) => e.id).sort()).toEqual(["e/a", "e/b", "e/c"]);
589
589
  });
590
- });
591
590
 
592
- // ───────────────────────── Artifacts 报告器 = writer 薄壳 ─────────────────────────
591
+ it("快照目录独占创建:撞名换随机后缀重试直到成功(EEXIST 不会覆盖已有目录)", async () => {
592
+ const root = await makeRoot();
593
+ vi.useFakeTimers();
594
+ vi.setSystemTime(new Date("2026-07-01T08:00:00.000Z"));
595
+ let call = 0;
596
+ const randomSpy = vi.spyOn(Math, "random").mockImplementation(() => {
597
+ call += 1;
598
+ return call <= 4 ? 0 : 0.03; // 第一次 randomSuffix() 产出 "aaaa",第二次产出 "bbbb"
599
+ });
600
+ try {
601
+ const collidingDir = join(root, "e", "2026-07-01T08-00-00-000Z-aaaa");
602
+ await mkdir(collidingDir, { recursive: true });
603
+
604
+ const writer = createResultsWriter(root, { producer: { name: "x" } });
605
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
606
+ expect(snap.dir).not.toBe(collidingDir);
607
+ expect(snap.dir.endsWith("-bbbb")).toBe(true);
608
+ expect(await exists(join(snap.dir, "snapshot.json"))).toBe(true);
609
+ // 被占用的目录没有被写入 snapshot.json —— 独占创建不会覆盖已有内容。
610
+ expect(await exists(join(collidingDir, "snapshot.json"))).toBe(false);
611
+ } finally {
612
+ randomSpy.mockRestore();
613
+ vi.useRealTimers();
614
+ }
615
+ });
593
616
 
594
- describe("Artifacts reporter(writer 薄壳)", () => {
595
- it("落盘行为与 runner 直写时代逐字节等价:summary.json 键序/瘦身/携带条目原样,工件文件按需紧凑写", async () => {
617
+ it("writeAttemptFor:按 EvalResult.experimentId 懒建快照;正常条目拆 artifact,携带条目原样保留 startedAt/artifactBase/has*", async () => {
596
618
  const root = await makeRoot();
597
- const rep = Artifacts(root);
598
- await rep.onRunStart?.([], {} as never);
619
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "0.12.0" } });
599
620
 
600
621
  const fresh: EvalResult = {
601
622
  id: "algebra/q1",
@@ -603,12 +624,12 @@ describe("Artifacts reporter(writer 薄壳)", () => {
603
624
  experiment: { id: "compare/bub", flags: { style: "concise" } },
604
625
  agent: "bub",
605
626
  model: "gpt-5.4",
606
- outcome: "passed",
627
+ verdict: "passed",
607
628
  fingerprint: "abc",
608
629
  attempt: 1,
609
630
  startedAt: "2026-07-01T08:01:00.000Z",
610
631
  durationMs: 1234,
611
- assertions: [{ kind: "contains", ok: true } as never],
632
+ assertions: [],
612
633
  usage: { inputTokens: 10, outputTokens: 5 },
613
634
  estimatedCostUSD: 0.5,
614
635
  events: [{ type: "message", role: "assistant", text: "hi" } as never],
@@ -618,146 +639,574 @@ describe("Artifacts reporter(writer 薄壳)", () => {
618
639
  diff: { generatedFiles: { "a.txt": "1" }, deletedFiles: [] },
619
640
  rawTranscript: "raw",
620
641
  };
621
- const noArtifacts: EvalResult = {
622
- id: "algebra/q2",
623
- agent: "bub",
624
- outcome: "failed",
625
- attempt: 1,
626
- durationMs: 10,
627
- assertions: [],
628
- events: [],
629
- };
630
- // --resume 携带条目:artifactBase 指向原 run,has* 真值原样携带,不得重算或编造 artifactsDir。
631
642
  const carried: EvalResult = {
632
643
  id: "algebra/q3",
633
644
  experimentId: "compare/bub",
634
645
  agent: "bub",
635
646
  model: "gpt-5.4",
636
- outcome: "passed",
647
+ verdict: "passed",
637
648
  attempt: 1,
638
649
  startedAt: "2026-06-30T08:01:00.000Z",
639
650
  durationMs: 99,
640
651
  assertions: [],
641
- artifactBase: "2026-06-30T08-00-00-000Z/algebra/q3/bub/gpt-5.4/compare_bub/a1",
652
+ artifactBase: "compare_bub/2026-06-30T08-00-00-000Z-xxxx/algebra/q3/a1",
642
653
  hasEvents: true,
643
654
  hasTrace: false,
644
655
  hasSources: true,
645
656
  };
646
657
 
647
- await rep.onEvalComplete?.(fresh);
648
- await rep.onEvalComplete?.(noArtifacts);
649
- await rep.onRunComplete?.({
650
- name: "demo",
658
+ await writer.writeAttemptFor(fresh);
659
+ await writer.writeAttemptFor(carried);
660
+ await writer.finish();
661
+
662
+ const dirs = writer.snapshotDirs();
663
+ expect(dirs).toHaveLength(1);
664
+ const snapDir = dirs[0].dir;
665
+
666
+ const freshRecord = JSON.parse(await readFile(join(snapDir, "algebra/q1/a1/result.json"), "utf-8"));
667
+ for (const key of ["agent", "model", "experimentId", "experiment", "events", "sources", "o11y", "trace", "diff", "rawTranscript"]) {
668
+ expect(freshRecord).not.toHaveProperty(key);
669
+ }
670
+ // startedAt 是 attempt 级事实(每条各异,view 靠它显示「何时跑的」),正常条目也原样落盘。
671
+ expect(freshRecord.startedAt).toBe("2026-07-01T08:01:00.000Z");
672
+ expect(freshRecord.hasEvents).toBe(true);
673
+ expect(freshRecord.hasTrace).toBe(true);
674
+ expect(freshRecord.hasSources).toBe(true);
675
+ expect(await readFile(join(snapDir, "algebra/q1/a1/events.json"), "utf-8")).toBe('[{"type":"message","role":"assistant","text":"hi"}]');
676
+ expect(await readFile(join(snapDir, "algebra/q1/a1/o11y.json"), "utf-8")).toBe('{"toolCalls":2}');
677
+
678
+ const carriedRecord = JSON.parse(await readFile(join(snapDir, "algebra/q3/a1/result.json"), "utf-8"));
679
+ expect(carriedRecord.startedAt).toBe("2026-06-30T08:01:00.000Z");
680
+ expect(carriedRecord.artifactBase).toBe("compare_bub/2026-06-30T08-00-00-000Z-xxxx/algebra/q3/a1");
681
+ expect(carriedRecord.hasEvents).toBe(true);
682
+ expect(carriedRecord.hasTrace).toBe(false);
683
+ expect(carriedRecord.hasSources).toBe(true);
684
+ expect(carriedRecord).not.toHaveProperty("agent");
685
+ expect(carriedRecord).not.toHaveProperty("experimentId");
686
+ expect(await exists(join(snapDir, "algebra/q3/a1/events.json"))).toBe(false); // 携带条目不写 artifact 文件
687
+
688
+ const meta = JSON.parse(await readFile(join(snapDir, "snapshot.json"), "utf-8"));
689
+ expect(meta.experimentId).toBe("compare/bub");
690
+ expect(meta.agent).toBe("bub");
691
+ expect(meta.model).toBe("gpt-5.4");
692
+ });
693
+
694
+ it("writeAttemptFor:result.experimentId 缺失时抛可执行错误(v4 布局按实验分目录)", async () => {
695
+ const root = await makeRoot();
696
+ const writer = createResultsWriter(root, { producer: { name: "x" } });
697
+ await expect(
698
+ writer.writeAttemptFor({ id: "q1", agent: "bub", verdict: "passed", attempt: 1, durationMs: 1, assertions: [] }),
699
+ ).rejects.toThrow(/experimentId/);
700
+ });
701
+ });
702
+
703
+ // ───────────────────────── copySnapshots ─────────────────────────
704
+
705
+ describe("copySnapshots", () => {
706
+ it("产物是标准结果根目录(快照目录名原样保留);按指定 artifact 复制;补记 knownEvalIds;has* 按目标目录重算", async () => {
707
+ const root = await makeRoot();
708
+ const monday = await writeSnapshot(
709
+ root,
710
+ "compare_bub",
711
+ "2026-07-01T08-00-00-000Z-mon1",
712
+ meta({ experimentId: "compare/bub", agent: "bub", model: "gpt-5", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }),
713
+ );
714
+ await writeResultFile(monday, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true, hasTrace: true }));
715
+ await writeArtifactFile(monday, "q1/a1", "events.json", [{ n: 1 }]);
716
+ await writeArtifactFile(monday, "q1/a1", "trace.json", [{ name: "turn" }]);
717
+ await writeArtifactFile(monday, "q1/a1", "diff.json", { generatedFiles: {}, deletedFiles: [] });
718
+ await writeResultFile(monday, "q2/a1", record({ id: "q2", attempt: 1 }));
719
+
720
+ // 周五只重跑了 q1:最新快照残缺。
721
+ const friday = await writeSnapshot(
722
+ root,
723
+ "compare_bub",
724
+ "2026-07-05T08-00-00-000Z-fri1",
725
+ meta({ experimentId: "compare/bub", agent: "bub", model: "gpt-5", startedAt: "2026-07-05T08:00:00.000Z", completedAt: "2026-07-05T08:10:00.000Z" }),
726
+ );
727
+ await writeResultFile(friday, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true }));
728
+ await writeArtifactFile(friday, "q1/a1", "events.json", [{ n: 1 }, { n: 2 }]);
729
+
730
+ const results = await openResults(root);
731
+ const dest = join(await makeRoot(), "site/data/run");
732
+ const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events"] });
733
+
734
+ expect(copied.warnings).toHaveLength(0);
735
+ expect(copied.dir).toBe(dest);
736
+
737
+ const destSnapDir = join(dest, "compare_bub", "2026-07-05T08-00-00-000Z-fri1"); // 快照目录名原样保留
738
+ expect(await exists(join(destSnapDir, "snapshot.json"))).toBe(true);
739
+ expect(await exists(join(destSnapDir, "q1/a1/events.json"))).toBe(true);
740
+ expect(await exists(join(destSnapDir, "q1/a1/trace.json"))).toBe(false); // 未选中的 artifact 种类不复制
741
+
742
+ const destMeta = JSON.parse(await readFile(join(destSnapDir, "snapshot.json"), "utf-8"));
743
+ expect(destMeta.knownEvalIds).toEqual(["q1", "q2"]); // 补记:复制时刻该实验已知的 eval 并集
744
+ expect(destMeta.completedAt).toBe("2026-07-05T08:10:00.000Z");
745
+ expect(destMeta.producer).toEqual({ name: "niceeval", version: "0.3.0" });
746
+
747
+ const destRecord = JSON.parse(await readFile(join(destSnapDir, "q1/a1/result.json"), "utf-8"));
748
+ expect(destRecord.hasEvents).toBe(true);
749
+ expect(destRecord.hasTrace).toBe(false); // 没选中 trace,目标按实际复制重算(不沿用源的 true)
750
+ expect(destRecord).not.toHaveProperty("artifactBase");
751
+ expect(destRecord).not.toHaveProperty("agent"); // 快照级字段不重复
752
+
753
+ // 发布目录上重新 openResults().latest():残缺警告被同一套机制重新算出来,不靠发布者转述。
754
+ const republished = await openResults(dest);
755
+ expect(republished.experiments[0].evalIds).toEqual(["q1", "q2"]);
756
+ const partial = republished.latest().warnings.find((w) => w.kind === "partial-coverage")!;
757
+ expect(partial).toMatchObject({ experimentId: "compare/bub", covered: 1, total: 2 });
758
+ });
759
+
760
+ it("目标目录非空即报错;artifacts 非法值报错;无快照报错;同实验多快照选中 → 取最新 + warning", async () => {
761
+ const root = await makeRoot();
762
+ await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z-a", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
763
+ const tuesday = await writeSnapshot(root, "e", "2026-07-02T08-00-00-000Z-b", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
764
+ await writeResultFile(tuesday, "q1/a1", record({ id: "q1", attempt: 1 }));
765
+
766
+ const results = await openResults(root);
767
+
768
+ const occupied = await makeRoot();
769
+ await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
770
+ await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
771
+
772
+ await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] })).rejects.toThrow(/Unknown artifact kind/);
773
+
774
+ await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
775
+
776
+ // 手工传入同一 experiment 的两个快照(未走 latest 去重):只带最新,记 warning。
777
+ const dest2 = join(await makeRoot(), "run2");
778
+ const collided = await copySnapshots(results.experiments[0].snapshots, dest2);
779
+ expect(collided.warnings).toHaveLength(1);
780
+ expect(collided.warnings[0]).toMatch(/multiple snapshots selected/);
781
+ const destDirs = await readdir(join(dest2, "e"));
782
+ expect(destDirs).toEqual([basename(tuesday)]);
783
+ });
784
+ });
785
+
786
+ // ───────────────────────── AttemptLocator 集成 ─────────────────────────
787
+
788
+ describe("AttemptLocator · 落盘 / 读取 / 携带 / 撞车", () => {
789
+ it("非携带条目由 writer 按身份算出 locator 并落盘;确定性(同身份重开两次相同);resolveLocator 能找到", async () => {
790
+ const root = await makeRoot();
791
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
792
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
793
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
794
+ await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
795
+ await writer.finish();
796
+
797
+ const record1 = JSON.parse(await readFile(join(snap.dir, "q1/a0/result.json"), "utf-8"));
798
+ expect(record1.locator).toMatch(/^@1[0-9a-z]{7}$/);
799
+ expect(record1.locator).toBe(
800
+ encodeAttemptLocator({ experimentId: "e", snapshotStartedAt: "2026-07-01T08:00:00.000Z", evalId: "q1", attempt: 0 }),
801
+ );
802
+
803
+ const record2 = JSON.parse(await readFile(join(snap.dir, "q2/a0/result.json"), "utf-8"));
804
+ expect(record2.locator).not.toBe(record1.locator); // 不同 evalId → 不同 locator
805
+
806
+ const resultsA = await openResults(root);
807
+ const resultsB = await openResults(root); // 独立重开一次:身份不变,locator 必须一致
808
+ const q1a = resultsA.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
809
+ const q1b = resultsB.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
810
+ expect(q1a.locator).toBe(record1.locator);
811
+ expect(q1b.locator).toBe(record1.locator);
812
+
813
+ expect(resolveLocator(resultsA, record1.locator).evalId).toBe("q1");
814
+ });
815
+
816
+ it("携带条目(--resume 合入)原样复制原 locator,不按新快照的 startedAt 重算", async () => {
817
+ const root = await makeRoot();
818
+ const writer1 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
819
+ await writer1.writeAttemptFor({
820
+ id: "q1",
821
+ experimentId: "e",
651
822
  agent: "bub",
823
+ verdict: "passed",
824
+ attempt: 0,
652
825
  startedAt: "2026-07-01T08:00:00.000Z",
653
- completedAt: "2026-07-01T08:10:00.000Z",
654
- passed: 2,
655
- failed: 1,
656
- skipped: 0,
657
- errored: 0,
658
- durationMs: 600000,
659
- usage: { inputTokens: 10, outputTokens: 5 },
660
- estimatedCostUSD: 0.5,
661
- results: [carried, fresh, noArtifacts],
826
+ durationMs: 1,
827
+ assertions: [],
828
+ });
829
+ await writer1.finish();
830
+
831
+ const opened1 = await openResults(root);
832
+ const original = opened1.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
833
+ const originalLocator = original.locator!;
834
+
835
+ // 第二轮:carry 合入 q1(artifactBase 指回第一轮的快照),locator 从上一轮读回的记录里原样带过来。
836
+ // q2(真正新跑的)先写:snapshot() 的 startedAt 由「该实验首条落盘结果的 attempt 时刻」锚定
837
+ // (writer.ts 的注释),让第二轮快照的真实 startedAt("07-02")明确不同于原快照("07-01")——
838
+ // 这样如果 locator 被错误地按「当前快照」重算,会得到一个可判别的不同字符串。
839
+ const writer2 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
840
+ await writer2.writeAttemptFor({
841
+ id: "q2",
842
+ experimentId: "e",
843
+ agent: "bub",
844
+ verdict: "passed",
845
+ attempt: 0,
846
+ startedAt: "2026-07-02T08:00:00.000Z",
847
+ durationMs: 1,
848
+ assertions: [],
849
+ });
850
+ const carried: EvalResult = {
851
+ ...original.result,
852
+ experimentId: "e",
853
+ agent: "bub",
854
+ artifactBase: `${original.ref.snapshot}/${original.ref.attempt}`,
855
+ };
856
+ await writer2.writeAttemptFor(carried);
857
+ await writer2.finish();
858
+
859
+ const opened2 = await openResults(root);
860
+ const newest = opened2.experiments[0].latest;
861
+ const carriedAttempt = newest.evals.find((e) => e.id === "q1")!.attempts[0];
862
+ expect(carriedAttempt.locator).toBe(originalLocator);
863
+ expect(resolveLocator(opened2, originalLocator).evalId).toBe("q1");
864
+
865
+ // 反证:如果按「新快照的 startedAt」重算,会得到一个不同的字符串——证明确实是原样复制,不是重算。
866
+ const wronglyRecomputed = encodeAttemptLocator({
867
+ experimentId: "e",
868
+ snapshotStartedAt: newest.startedAt,
869
+ evalId: "q1",
870
+ attempt: 0,
662
871
  });
872
+ expect(originalLocator).not.toBe(wronglyRecomputed);
873
+ });
663
874
 
664
- const dir = rep.outputDir();
665
- const version = (JSON.parse(await readFile(new URL("../../package.json", import.meta.url), "utf-8")) as { version: string }).version;
666
-
667
- // 基线由改造前的实现捕获(runner 直写时代的真实输出),薄壳必须逐字节还原它。
668
- const expected = `{
669
- "format": "niceeval.results",
670
- "schemaVersion": ${RESULTS_SCHEMA_VERSION},
671
- "producer": {
672
- "name": "niceeval",
673
- "version": "${version}"
674
- },
675
- "name": "demo",
676
- "agent": "bub",
677
- "startedAt": "2026-07-01T08:00:00.000Z",
678
- "completedAt": "2026-07-01T08:10:00.000Z",
679
- "passed": 2,
680
- "failed": 1,
681
- "skipped": 0,
682
- "errored": 0,
683
- "durationMs": 600000,
684
- "usage": {
685
- "inputTokens": 10,
686
- "outputTokens": 5
687
- },
688
- "estimatedCostUSD": 0.5,
689
- "results": [
690
- {
691
- "id": "algebra/q3",
692
- "experimentId": "compare/bub",
693
- "agent": "bub",
694
- "model": "gpt-5.4",
695
- "outcome": "passed",
696
- "attempt": 1,
697
- "startedAt": "2026-06-30T08:01:00.000Z",
698
- "durationMs": 99,
699
- "assertions": [],
700
- "artifactBase": "2026-06-30T08-00-00-000Z/algebra/q3/bub/gpt-5.4/compare_bub/a1",
701
- "hasEvents": true,
702
- "hasTrace": false,
703
- "hasSources": true
704
- },
705
- {
706
- "id": "algebra/q1",
707
- "experimentId": "compare/bub",
708
- "experiment": {
709
- "id": "compare/bub",
710
- "flags": {
711
- "style": "concise"
712
- }
713
- },
714
- "agent": "bub",
715
- "model": "gpt-5.4",
716
- "outcome": "passed",
717
- "fingerprint": "abc",
718
- "attempt": 1,
719
- "startedAt": "2026-07-01T08:01:00.000Z",
720
- "durationMs": 1234,
721
- "assertions": [
722
- {
723
- "kind": "contains",
724
- "ok": true
725
- }
726
- ],
727
- "usage": {
728
- "inputTokens": 10,
729
- "outputTokens": 5
730
- },
731
- "estimatedCostUSD": 0.5,
732
- "artifactsDir": "algebra/q1/bub/gpt-5.4/compare_bub/a1",
733
- "hasTrace": true,
734
- "hasEvents": true,
735
- "hasSources": true
736
- },
737
- {
738
- "id": "algebra/q2",
739
- "agent": "bub",
740
- "outcome": "failed",
741
- "attempt": 1,
742
- "durationMs": 10,
743
- "assertions": [],
744
- "artifactsDir": "algebra/q2/bub/default/a1",
745
- "hasTrace": false,
746
- "hasEvents": false,
747
- "hasSources": false
748
- }
749
- ],
750
- "outputDir": "${dir}"
751
- }`;
752
- expect(await readFile(join(dir, "summary.json"), "utf-8")).toBe(expected);
753
-
754
- // 工件:紧凑 JSON,按需生成(q2 全空不落文件;q1 五类都在)
755
- const q1Dir = join(dir, "algebra/q1/bub/gpt-5.4/compare_bub/a1");
756
- expect(await readFile(join(q1Dir, "events.json"), "utf-8")).toBe('[{"type":"message","role":"assistant","text":"hi"}]');
757
- expect(await readFile(join(q1Dir, "sources.json"), "utf-8")).toBe('[{"path":"evals/a.ts","content":"x"}]');
758
- expect(await readFile(join(q1Dir, "trace.json"), "utf-8")).toBe('[{"name":"turn","kind":"turn"}]');
759
- expect(await readFile(join(q1Dir, "o11y.json"), "utf-8")).toBe('{"toolCalls":2}');
760
- expect(await readFile(join(q1Dir, "diff.json"), "utf-8")).toBe('{"generatedFiles":{"a.txt":"1"},"deletedFiles":[]}');
761
- expect(await exists(join(dir, "algebra/q2/bub/default/a1/events.json"))).toBe(false);
875
+ it("resolveLocator:malformed not-found 是两种可判别的错误", async () => {
876
+ const root = await makeRoot();
877
+ await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z-a", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }));
878
+ const results = await openResults(root);
879
+
880
+ expect(() => resolveLocator(results, "not-a-locator")).toThrow(MalformedLocatorError);
881
+ expect(() => resolveLocator(results, "@1nosuch1")).toThrow(LocatorNotFoundError); // 语法合法(7 位 body),索引里没有
882
+ });
883
+
884
+ it("两个不同身份的 attempt 撞出同一个 locator 字符串:openResults() 抛 LocatorCollisionError,不静默覆盖", async () => {
885
+ const root = await makeRoot();
886
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
887
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
888
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
889
+ await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
890
+ await writer.finish();
891
+
892
+ // 人为制造撞车:把 q2 的 locator 改成和 q1 一样(真实哈希撞车不可复现,这里直接模拟其效果——
893
+ // 索引建立逻辑只关心「同一 locator 字符串映射到身份三元组不同的两个 attempt」)。
894
+ const q1Path = join(snap.dir, "q1/a0/result.json");
895
+ const q2Path = join(snap.dir, "q2/a0/result.json");
896
+ const q1Record = JSON.parse(await readFile(q1Path, "utf-8"));
897
+ const q2Record = JSON.parse(await readFile(q2Path, "utf-8"));
898
+ q2Record.locator = q1Record.locator;
899
+ await writeFile(q2Path, JSON.stringify(q2Record), "utf-8");
900
+
901
+ await expect(openResults(root)).rejects.toThrow(LocatorCollisionError);
902
+ });
903
+
904
+ it(" experiment:同 evalId/attempt 在不同 experiment 下产出不同 locator,resolveLocator 精确定位到各自的 experiment", async () => {
905
+ const root = await makeRoot();
906
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
907
+ const snapA = await writer.snapshot({ experimentId: "compare/bub", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
908
+ const snapB = await writer.snapshot({ experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z" });
909
+ await snapA.writeAttempt({ id: "algebra/q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
910
+ await snapB.writeAttempt({ id: "algebra/q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
911
+ await writer.finish();
912
+
913
+ const results = await openResults(root);
914
+ const a = results.experiments.find((e) => e.id === "compare/bub")!.latest.evals[0]!.attempts[0]!;
915
+ const b = results.experiments.find((e) => e.id === "compare/codex")!.latest.evals[0]!.attempts[0]!;
916
+ // 同 evalId、同 attempt 序号,只有 experimentId 不同 → locator 必须不同(身份元组含 experimentId)。
917
+ expect(a.locator).not.toBe(b.locator);
918
+
919
+ expect(resolveLocator(results, a.locator!).experimentId).toBe("compare/bub");
920
+ expect(resolveLocator(results, b.locator!).experimentId).toBe("compare/codex");
921
+ });
922
+
923
+ it("同一 evalId 的不同 attempt 序号产出不同 locator,resolveLocator 各自精确定位到对应 attempt", async () => {
924
+ const root = await makeRoot();
925
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
926
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
927
+ await snap.writeAttempt({ id: "q1", verdict: "failed", attempt: 0, durationMs: 1, assertions: [] });
928
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 1, assertions: [] });
929
+ await writer.finish();
930
+
931
+ const results = await openResults(root);
932
+ const attempts = results.experiments[0]!.latest.evals.find((e) => e.id === "q1")!.attempts;
933
+ expect(attempts).toHaveLength(2);
934
+ const [a0, a1] = attempts;
935
+ expect(a0!.locator).not.toBe(a1!.locator);
936
+ expect(resolveLocator(results, a0!.locator!).result).toMatchObject({ attempt: 0, verdict: "failed" });
937
+ expect(resolveLocator(results, a1!.locator!).result).toMatchObject({ attempt: 1, verdict: "passed" });
938
+ });
939
+
940
+ it("历史快照(非 latest)的 attempt 依然被建进索引,resolveLocator 能定位到旧快照里的那份", async () => {
941
+ const root = await makeRoot();
942
+ const writer1 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
943
+ const monday = await writer1.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
944
+ await monday.writeAttempt({ id: "q1", verdict: "failed", attempt: 0, durationMs: 1, assertions: [] });
945
+ await writer1.finish();
946
+
947
+ const writer2 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
948
+ const friday = await writer2.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-05T08:00:00.000Z" });
949
+ await friday.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
950
+ await writer2.finish();
951
+
952
+ const results = await openResults(root);
953
+ const exp = results.experiments[0]!;
954
+ expect(exp.snapshots).toHaveLength(2); // 两次快照都在(忠实磁盘,不合并/不丢弃历史)
955
+ const oldAttempt = exp.snapshots.find((s) => s.startedAt === "2026-07-01T08:00:00.000Z")!.attempts[0]!;
956
+ const newAttempt = exp.snapshots.find((s) => s.startedAt === "2026-07-05T08:00:00.000Z")!.attempts[0]!;
957
+ expect(oldAttempt.locator).not.toBe(newAttempt.locator); // 不同 startedAt → 不同身份 → 不同 locator
958
+
959
+ expect(resolveLocator(results, oldAttempt.locator!).result.verdict).toBe("failed");
960
+ expect(resolveLocator(results, newAttempt.locator!).result.verdict).toBe("passed");
961
+ });
962
+
963
+ it("手工构造的 Results(未经 openResults())上调 resolveLocator:索引查不到,统一按 not-found 处理,不抛意外错误", () => {
964
+ const snapshot = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir: "/tmp/e/s1" });
965
+ const attempt = fakeAttempt(snapshot, record({ id: "q1", attempt: 0 }));
966
+ snapshot.attempts = [attempt];
967
+ snapshot.evals = [{ id: "q1", attempts: [attempt] }];
968
+ const handMadeResults: Results = {
969
+ experiments: [{ id: "e", snapshots: [snapshot], latest: snapshot, evalIds: ["q1"] }],
970
+ skipped: [],
971
+ // filter() 本测试不调用,用不到,给个占位实现即可满足 Selection 接口。
972
+ latest: () => ({ snapshots: [snapshot], warnings: [], filter: () => { throw new Error("not implemented"); } }),
973
+ };
974
+ // 这份 locator 语法合法、甚至真的对应 handMadeResults 里那个 attempt 的身份,
975
+ // 但 handMadeResults 没经过 openResults(),locatorIndexByResults 里查不到它 —— 空索引,not-found。
976
+ const syntacticallyValidLocator = encodeAttemptLocator({
977
+ experimentId: "e",
978
+ snapshotStartedAt: "2026-07-01T08:00:00.000Z",
979
+ evalId: "q1",
980
+ attempt: 0,
981
+ });
982
+ expect(() => resolveLocator(handMadeResults, syntacticallyValidLocator)).toThrow(LocatorNotFoundError);
983
+ });
984
+
985
+ it("copySnapshots:普通(非 sources)attempt 的 locator 原样复制,目标结果根上 resolveLocator 依然命中", async () => {
986
+ const root = await makeRoot();
987
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
988
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
989
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
990
+ await snap.writeAttempt({ id: "q1", verdict: "failed", attempt: 1, durationMs: 1, assertions: [] });
991
+ await writer.finish();
992
+
993
+ const results = await openResults(root);
994
+ const [a0, a1] = results.experiments[0]!.latest.evals[0]!.attempts;
995
+ const locator0 = a0!.locator!;
996
+ const locator1 = a1!.locator!;
997
+
998
+ const dest = join(await makeRoot(), "published");
999
+ await copySnapshots(results.latest(), dest, { artifacts: [] });
1000
+
1001
+ const destResults = await openResults(dest);
1002
+ expect(resolveLocator(destResults, locator0).result.attempt).toBe(0);
1003
+ expect(resolveLocator(destResults, locator1).result.attempt).toBe(1);
1004
+ });
1005
+ });
1006
+
1007
+ // ───────────────────────── sources 去重仓库 ─────────────────────────
1008
+
1009
+ describe("sources · 快照级去重仓库", () => {
1010
+ it("两个 attempt 共享字节相同的 eval 源码:去重仓库只落一份 blob(文件数 = 1)", async () => {
1011
+ const root = await makeRoot();
1012
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1013
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
1014
+ const content = "export default { test() {} };\n";
1015
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: [{ path: "evals/shared.eval.ts", content }] });
1016
+ await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: [{ path: "evals/shared.eval.ts", content }] });
1017
+ await writer.finish();
1018
+
1019
+ const storeFiles = await readdir(join(snap.dir, "sources"));
1020
+ expect(storeFiles).toHaveLength(1);
1021
+ });
1022
+
1023
+ it("两个 attempt 的 eval 源码内容不同:各自落一份 blob(文件数 = 2)", async () => {
1024
+ const root = await makeRoot();
1025
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1026
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
1027
+ await snap.writeAttempt(
1028
+ { id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] },
1029
+ { sources: [{ path: "evals/a.eval.ts", content: "export default { test() {} };\n" }] },
1030
+ );
1031
+ await snap.writeAttempt(
1032
+ { id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] },
1033
+ { sources: [{ path: "evals/b.eval.ts", content: "export default { test() { /* different */ } };\n" }] },
1034
+ );
1035
+ await writer.finish();
1036
+
1037
+ const storeFiles = await readdir(join(snap.dir, "sources"));
1038
+ expect(storeFiles).toHaveLength(2);
1039
+ });
1040
+
1041
+ it("经真实 --resume carry 流程(writeAttemptFor 的 artifactBase 分支)携带的 attempt,其 sources() 引用在新快照里依然能解到原快照内容", async () => {
1042
+ const root = await makeRoot();
1043
+ const content = "export default { test() {} };\n";
1044
+ const writer1 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1045
+ await writer1.writeAttemptFor({
1046
+ id: "q1",
1047
+ experimentId: "e",
1048
+ agent: "bub",
1049
+ verdict: "passed",
1050
+ attempt: 0,
1051
+ startedAt: "2026-07-01T08:00:00.000Z",
1052
+ durationMs: 1,
1053
+ assertions: [],
1054
+ sources: [{ path: "evals/q1.eval.ts", content }],
1055
+ });
1056
+ await writer1.finish();
1057
+
1058
+ const opened1 = await openResults(root);
1059
+ const original = opened1.experiments[0]!.latest.evals.find((e) => e.id === "q1")!.attempts[0]!;
1060
+ expect(await original.sources()).toEqual([{ path: "evals/q1.eval.ts", content }]);
1061
+
1062
+ // 第二轮:q2 是真正新跑的(锚定新快照的 startedAt 明确晚于原快照),q1 是 carry 合入 ——
1063
+ // artifactBase 指回第一轮的快照,与 locator carry 测试同一套构造手法。
1064
+ const writer2 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1065
+ await writer2.writeAttemptFor({
1066
+ id: "q2",
1067
+ experimentId: "e",
1068
+ agent: "bub",
1069
+ verdict: "passed",
1070
+ attempt: 0,
1071
+ startedAt: "2026-07-02T08:00:00.000Z",
1072
+ durationMs: 1,
1073
+ assertions: [],
1074
+ });
1075
+ const carried: EvalResult = {
1076
+ ...original.result,
1077
+ experimentId: "e",
1078
+ agent: "bub",
1079
+ artifactBase: `${original.ref.snapshot}/${original.ref.attempt}`,
1080
+ };
1081
+ await writer2.writeAttemptFor(carried);
1082
+ await writer2.finish();
1083
+
1084
+ const opened2 = await openResults(root);
1085
+ const carriedAttempt = opened2.experiments[0]!.latest.evals.find((e) => e.id === "q1")!.attempts[0]!;
1086
+ // 新快照下没有为携带条目重新写 sources.json/blob(carry 分支不写 artifact),
1087
+ // sources() 必须靠 artifactBase 回退到原快照的去重仓库才能解出内容。
1088
+ expect(await carriedAttempt.sources()).toEqual([{ path: "evals/q1.eval.ts", content }]);
1089
+ });
1090
+
1091
+ it("同一快照内相同内容只落一份 blob;不同内容各一份;attempt.sources() 各自读回正确内容", async () => {
1092
+ const root = await makeRoot();
1093
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1094
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
1095
+ const shared = [{ path: "evals/shared.eval.ts", content: "export default { test() {} };\n" }];
1096
+ const other = [{ path: "evals/other.eval.ts", content: "export default { test() { /* different */ } };\n" }];
1097
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
1098
+ await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
1099
+ await snap.writeAttempt({ id: "q3", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: other });
1100
+ await writer.finish();
1101
+
1102
+ const storeFiles = await readdir(join(snap.dir, "sources"));
1103
+ expect(storeFiles).toHaveLength(2); // 三份引用,内容只两种 → 两个 blob
1104
+
1105
+ // attempt 级 sources.json 只是引用(小,不含 content),不是全量内容。
1106
+ const q1Ref = JSON.parse(await readFile(join(snap.dir, "q1/a0/sources.json"), "utf-8"));
1107
+ expect(q1Ref).toEqual([{ path: "evals/shared.eval.ts", sha256: expect.any(String) }]);
1108
+ expect(JSON.stringify(q1Ref)).not.toContain("export default");
1109
+
1110
+ const results = await openResults(root);
1111
+ const evalById = (id: string) => results.experiments[0].latest.evals.find((e) => e.id === id)!.attempts[0];
1112
+ await expect(evalById("q1").sources()).resolves.toEqual(shared);
1113
+ await expect(evalById("q2").sources()).resolves.toEqual(shared);
1114
+ await expect(evalById("q3").sources()).resolves.toEqual(other);
1115
+ });
1116
+
1117
+ it("携带条目(artifactBase 回退)的 sources() 仍能解到原快照的去重仓库", async () => {
1118
+ const root = await makeRoot();
1119
+ const oldSnap = await writeSnapshot(
1120
+ root,
1121
+ "e",
1122
+ "2026-06-30T08-00-00-000Z-xxxx",
1123
+ meta({ experimentId: "e", agent: "bub", startedAt: "2026-06-30T08:00:00.000Z", completedAt: "2026-06-30T08:10:00.000Z" }),
1124
+ );
1125
+ await writeResultFile(oldSnap, "q1/a0", record({ id: "q1", attempt: 0, hasSources: true }));
1126
+ await mkdir(join(oldSnap, "sources"), { recursive: true });
1127
+ await writeFile(join(oldSnap, "sources", "abc123.json"), JSON.stringify({ content: "export default {};\n" }), "utf-8");
1128
+ await writeArtifactFile(oldSnap, "q1/a0", "sources.json", [{ path: "evals/q1.eval.ts", sha256: "abc123" }]);
1129
+
1130
+ const newSnap = await writeSnapshot(
1131
+ root,
1132
+ "e",
1133
+ "2026-07-01T08-00-00-000Z-yyyy",
1134
+ meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }),
1135
+ );
1136
+ await writeResultFile(
1137
+ newSnap,
1138
+ "q1/a0",
1139
+ record({ id: "q1", attempt: 0, startedAt: "2026-06-30T08:01:00.000Z", artifactBase: "e/2026-06-30T08-00-00-000Z-xxxx/q1/a0", hasSources: true }),
1140
+ );
1141
+
1142
+ const results = await openResults(root);
1143
+ const carried = results.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
1144
+ expect(await carried.sources()).toEqual([{ path: "evals/q1.eval.ts", content: "export default {};\n" }]);
1145
+ });
1146
+
1147
+ it("copySnapshots:sources 引用与去重仓库一起复制,内容按目的地重新去重(同一份不重复落盘)", async () => {
1148
+ const root = await makeRoot();
1149
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1150
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
1151
+ const shared = [{ path: "evals/shared.eval.ts", content: "export default { test() {} };\n" }];
1152
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
1153
+ await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
1154
+ await writer.finish();
1155
+
1156
+ const originalLocator = JSON.parse(await readFile(join(snap.dir, "q1/a0/result.json"), "utf-8")).locator;
1157
+
1158
+ const results = await openResults(root);
1159
+ const dest = join(await makeRoot(), "published");
1160
+ await copySnapshots(results.latest(), dest, { artifacts: ["sources"] });
1161
+
1162
+ const destSnapDir = join(dest, "e", basename(snap.dir));
1163
+ const destStoreFiles = await readdir(join(destSnapDir, "sources"));
1164
+ expect(destStoreFiles).toHaveLength(1); // 复制后在目的地重新按内容去重,仍只一份
1165
+
1166
+ const destResults = await openResults(dest);
1167
+ const q1 = destResults.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
1168
+ expect(await q1.sources()).toEqual(shared);
1169
+ expect(q1.result.locator).toBe(originalLocator); // locator 随 result.json 原样复制,不重算
1170
+ });
1171
+ });
1172
+
1173
+ // ───────────────────────── loadAnnotatedEvalSource(端到端打通) ─────────────────────────
1174
+
1175
+ describe("loadAnnotatedEvalSource · discovery 捕获 → 去重存储 → 检索 → 标注 打通链路", () => {
1176
+ it("给一个真实落盘的 attempt,取回 sources() 内容并按 loc 标注断言", async () => {
1177
+ const root = await makeRoot();
1178
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1179
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
1180
+ const content = "import { defineEval } from \"niceeval\";\nexport default defineEval({\n test() {},\n});\n";
1181
+ const assertions = [
1182
+ { name: "check-1", passed: true, severity: "gate", score: 1, loc: { file: "evals/a.eval.ts", line: 3 } },
1183
+ { name: "no-loc", passed: false, severity: "soft", score: 0 },
1184
+ ] as unknown as EvalResult["assertions"];
1185
+ await snap.writeAttempt(
1186
+ { id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions },
1187
+ { sources: [{ path: "evals/a.eval.ts", content }] },
1188
+ );
1189
+ await writer.finish();
1190
+
1191
+ const results = await openResults(root);
1192
+ const attempt = results.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
1193
+ const annotated = await loadAnnotatedEvalSource(attempt);
1194
+ expect(annotated).not.toBeNull();
1195
+ expect(annotated!.sourcePath).toBe("evals/a.eval.ts");
1196
+ expect(annotated!.lines[2]!.assertions.map((a) => a.name)).toEqual(["check-1"]);
1197
+ expect(annotated!.unmapped.map((a) => a.name)).toEqual(["no-loc"]);
1198
+ expect(annotated!.summary).toMatchObject({ totalAssertions: 2, mappedAssertions: 1, unmappedAssertions: 1 });
1199
+ });
1200
+
1201
+ it("没有 sources() 时返回 null,不伪造空文档", async () => {
1202
+ const root = await makeRoot();
1203
+ const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
1204
+ const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
1205
+ await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
1206
+ await writer.finish();
1207
+
1208
+ const results = await openResults(root);
1209
+ const attempt = results.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
1210
+ expect(await loadAnnotatedEvalSource(attempt)).toBeNull();
762
1211
  });
763
1212
  });