niceeval 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. package/INDEX.md +23 -23
  2. package/README.zh.md +6 -6
  3. package/dist/agents/types.d.ts +69 -7
  4. package/dist/context/types.d.ts +32 -12
  5. package/dist/i18n/en.d.ts +54 -0
  6. package/dist/i18n/zh-CN.d.ts +57 -3
  7. package/dist/o11y/types.d.ts +16 -2
  8. package/dist/report/aggregate.d.ts +32 -24
  9. package/dist/report/aggregate.js +158 -50
  10. package/dist/report/built-in/index.d.ts +2 -0
  11. package/dist/report/built-in/index.js +8 -0
  12. package/dist/report/components.d.ts +93 -160
  13. package/dist/report/components.js +377 -114
  14. package/dist/report/compute.d.ts +87 -81
  15. package/dist/report/compute.js +597 -417
  16. package/dist/report/flag.d.ts +32 -6
  17. package/dist/report/flag.js +92 -4
  18. package/dist/report/format.d.ts +19 -11
  19. package/dist/report/format.js +30 -13
  20. package/dist/report/index.d.ts +16 -16
  21. package/dist/report/index.js +20 -21
  22. package/dist/report/load.js +3 -2
  23. package/dist/report/locale.d.ts +57 -33
  24. package/dist/report/locale.js +122 -56
  25. package/dist/report/metrics.d.ts +23 -4
  26. package/dist/report/metrics.js +110 -25
  27. package/dist/report/primitives.d.ts +48 -15
  28. package/dist/report/primitives.js +135 -26
  29. package/dist/report/react/AttemptList.d.ts +9 -7
  30. package/dist/report/react/AttemptList.js +17 -10
  31. package/dist/report/react/DeltaTable.js +19 -18
  32. package/dist/report/react/EvalList.d.ts +4 -4
  33. package/dist/report/react/EvalList.js +0 -0
  34. package/dist/report/react/ExperimentComparison.d.ts +10 -0
  35. package/dist/report/react/ExperimentComparison.js +12 -0
  36. package/dist/report/react/ExperimentList.d.ts +4 -3
  37. package/dist/report/react/ExperimentList.js +17 -18
  38. package/dist/report/react/MetricBars.js +5 -4
  39. package/dist/report/react/MetricLine.js +12 -5
  40. package/dist/report/react/MetricMatrix.js +1 -1
  41. package/dist/report/react/MetricScatter.js +59 -28
  42. package/dist/report/react/MetricTable.js +2 -12
  43. package/dist/report/react/ScopeSummary.d.ts +10 -0
  44. package/dist/report/react/ScopeSummary.js +17 -0
  45. package/dist/report/react/Scoreboard.js +6 -6
  46. package/dist/report/react/cell.js +2 -2
  47. package/dist/report/react/chart-math.d.ts +23 -6
  48. package/dist/report/react/chart-math.js +71 -19
  49. package/dist/report/react/fixtures.d.ts +5 -9
  50. package/dist/report/react/fixtures.js +110 -147
  51. package/dist/report/react/index.d.ts +15 -5
  52. package/dist/report/react/index.js +18 -7
  53. package/dist/report/report.d.ts +137 -16
  54. package/dist/report/report.js +259 -28
  55. package/dist/report/text/faces.d.ts +17 -19
  56. package/dist/report/text/faces.js +253 -184
  57. package/dist/report/text/plot.js +1 -1
  58. package/dist/report/text/table.js +38 -7
  59. package/dist/report/tree.d.ts +90 -40
  60. package/dist/report/tree.js +252 -94
  61. package/dist/report/types.d.ts +247 -284
  62. package/dist/report/types.js +4 -3
  63. package/dist/report/web.d.ts +21 -5
  64. package/dist/report/web.js +42 -16
  65. package/dist/results/select.d.ts +38 -16
  66. package/dist/results/select.js +73 -25
  67. package/dist/results/types.d.ts +49 -14
  68. package/dist/runner/feedback/sink.d.ts +110 -0
  69. package/dist/runner/types.d.ts +513 -22
  70. package/dist/sandbox/docker.d.ts +23 -2
  71. package/dist/sandbox/e2b.d.ts +15 -1
  72. package/dist/sandbox/errors.d.ts +30 -3
  73. package/dist/sandbox/io-retry.d.ts +17 -0
  74. package/dist/sandbox/registry.d.ts +2 -0
  75. package/dist/sandbox/resolve.d.ts +18 -5
  76. package/dist/sandbox/retry.d.ts +11 -1
  77. package/dist/sandbox/types.d.ts +39 -5
  78. package/dist/sandbox/vercel.d.ts +7 -1
  79. package/dist/scoring/coverage.d.ts +30 -0
  80. package/dist/scoring/display.d.ts +21 -0
  81. package/dist/scoring/display.js +120 -0
  82. package/dist/scoring/types.d.ts +103 -20
  83. package/dist/shared/aggregate.d.ts +4 -2
  84. package/dist/shared/aggregate.js +8 -7
  85. package/dist/shared/types.d.ts +28 -0
  86. package/dist/tty-line.d.ts +0 -4
  87. package/dist/util.d.ts +23 -0
  88. package/docs-site/zh/README.md +44 -0
  89. package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
  90. package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
  91. package/docs-site/zh/examples/index.mdx +50 -0
  92. package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
  93. package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
  94. package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
  95. package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
  96. package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
  97. package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
  98. package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
  99. package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
  100. package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
  101. package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
  102. package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
  103. package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
  104. package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
  105. package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
  106. package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
  107. package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
  108. package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
  109. package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
  110. package/docs-site/zh/how-to/publish-report.mdx +105 -0
  111. package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
  112. package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
  113. package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
  114. package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
  115. package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
  116. package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
  117. package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
  118. package/docs-site/zh/index.mdx +24 -26
  119. package/docs-site/zh/introduction.mdx +8 -8
  120. package/docs-site/zh/reference/builtin-agents.mdx +32 -5
  121. package/docs-site/zh/reference/capabilities.mdx +8 -8
  122. package/docs-site/zh/reference/cli.mdx +40 -12
  123. package/docs-site/zh/reference/define-agent.mdx +58 -5
  124. package/docs-site/zh/reference/define-config.mdx +1 -1
  125. package/docs-site/zh/reference/define-eval.mdx +42 -9
  126. package/docs-site/zh/reference/events.mdx +3 -3
  127. package/docs-site/zh/reference/expect.mdx +26 -1
  128. package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
  129. package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
  130. package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
  131. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
  132. package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
  133. package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
  134. package/package.json +10 -2
  135. package/src/agents/ai-sdk-otel.test.ts +1 -0
  136. package/src/agents/ai-sdk.test.ts +3 -0
  137. package/src/agents/ai-sdk.ts +3 -0
  138. package/src/agents/bub-install-spec.test.ts +34 -0
  139. package/src/agents/bub-install-spec.ts +32 -0
  140. package/src/agents/bub.ts +31 -32
  141. package/src/agents/claude-code.test.ts +130 -9
  142. package/src/agents/claude-code.ts +76 -4
  143. package/src/agents/codex.test.ts +189 -40
  144. package/src/agents/codex.ts +155 -14
  145. package/src/agents/coding-cli-versions.test.ts +15 -0
  146. package/src/agents/coding-cli-versions.ts +3 -0
  147. package/src/agents/index.ts +13 -2
  148. package/src/agents/langgraph.test.ts +204 -0
  149. package/src/agents/langgraph.ts +495 -0
  150. package/src/agents/marketplace.ts +85 -0
  151. package/src/agents/native-config.test.ts +179 -0
  152. package/src/agents/native-config.ts +267 -0
  153. package/src/agents/openai-compat.test.ts +1 -0
  154. package/src/agents/openai-compat.ts +1 -1
  155. package/src/agents/openclaw.test.ts +31 -0
  156. package/src/agents/openclaw.ts +171 -0
  157. package/src/agents/plugin-config.test.ts +1 -0
  158. package/src/agents/sdk-streams.test.ts +79 -0
  159. package/src/agents/sdk-streams.ts +55 -10
  160. package/src/agents/skills.test.ts +1 -0
  161. package/src/agents/streaming.test.ts +3 -9
  162. package/src/agents/streaming.ts +2 -2
  163. package/src/agents/types.ts +71 -8
  164. package/src/agents/ui-message-stream.test.ts +3 -0
  165. package/src/cli.ts +446 -124
  166. package/src/context/context.test.ts +51 -12
  167. package/src/context/context.ts +162 -30
  168. package/src/context/session.test.ts +2 -1
  169. package/src/context/session.ts +115 -7
  170. package/src/context/types.ts +30 -12
  171. package/src/define.test.ts +13 -8
  172. package/src/define.ts +25 -4
  173. package/src/expect/index.ts +53 -23
  174. package/src/i18n/en.ts +81 -17
  175. package/src/i18n/zh-CN.ts +80 -17
  176. package/src/o11y/cost.test.ts +1 -0
  177. package/src/o11y/execution-tree.test.ts +1 -20
  178. package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
  179. package/src/o11y/otlp/parse.test.ts +1 -0
  180. package/src/o11y/otlp/turn-otel.test.ts +1 -0
  181. package/src/o11y/parsers/bub.test.ts +1 -0
  182. package/src/o11y/parsers/claude-code.test.ts +1 -34
  183. package/src/o11y/parsers/openclaw.test.ts +154 -0
  184. package/src/o11y/parsers/openclaw.ts +310 -0
  185. package/src/o11y/prices.json +746 -311
  186. package/src/o11y/tool-names.test.ts +1 -0
  187. package/src/o11y/types.ts +16 -2
  188. package/src/report/aggregate.ts +178 -61
  189. package/src/report/built-in/index.tsx +9 -0
  190. package/src/report/components.tsx +625 -279
  191. package/src/report/compute.ts +723 -491
  192. package/src/report/dual-render.test.tsx +741 -1024
  193. package/src/report/flag.ts +104 -12
  194. package/src/report/format.ts +32 -12
  195. package/src/report/index.ts +119 -46
  196. package/src/report/load.ts +3 -2
  197. package/src/report/locale.ts +136 -65
  198. package/src/report/metrics.ts +108 -25
  199. package/src/report/primitives.tsx +196 -45
  200. package/src/report/react/AttemptList.tsx +30 -43
  201. package/src/report/react/DeltaTable.tsx +63 -45
  202. package/src/report/react/EvalList.tsx +0 -0
  203. package/src/report/react/ExperimentComparison.tsx +73 -0
  204. package/src/report/react/ExperimentList.tsx +50 -32
  205. package/src/report/react/MetricBars.tsx +5 -4
  206. package/src/report/react/MetricLine.tsx +13 -8
  207. package/src/report/react/MetricMatrix.tsx +2 -2
  208. package/src/report/react/MetricScatter.tsx +86 -34
  209. package/src/report/react/MetricTable.tsx +4 -76
  210. package/src/report/react/ScopeSummary.tsx +86 -0
  211. package/src/report/react/Scoreboard.tsx +28 -10
  212. package/src/report/react/cell.tsx +2 -2
  213. package/src/report/react/chart-math.test.ts +85 -0
  214. package/src/report/react/chart-math.ts +101 -22
  215. package/src/report/react/enhance.js +89 -5
  216. package/src/report/react/fixtures.ts +114 -154
  217. package/src/report/react/index.tsx +24 -39
  218. package/src/report/react/render.test.tsx +138 -158
  219. package/src/report/react/styles.css +243 -82
  220. package/src/report/report.test.ts +779 -841
  221. package/src/report/report.ts +423 -41
  222. package/src/report/text/faces.ts +290 -193
  223. package/src/report/text/plot.ts +1 -1
  224. package/src/report/text/table.ts +44 -7
  225. package/src/report/tree.ts +362 -104
  226. package/src/report/types.ts +261 -271
  227. package/src/report/web.ts +63 -20
  228. package/src/results/annotated-source.test.ts +62 -9
  229. package/src/results/annotated-source.ts +64 -6
  230. package/src/results/attempt-evidence.test.ts +13 -11
  231. package/src/results/attempt-evidence.ts +20 -13
  232. package/src/results/attempt-source.ts +6 -3
  233. package/src/results/copy.ts +150 -60
  234. package/src/results/host-equivalence.test.ts +34 -20
  235. package/src/results/index.ts +12 -4
  236. package/src/results/locator.test.ts +1 -22
  237. package/src/results/open.ts +15 -5
  238. package/src/results/publish.ts +149 -0
  239. package/src/results/results.test.ts +89 -54
  240. package/src/results/select.ts +104 -34
  241. package/src/results/truncate.ts +90 -0
  242. package/src/results/types.ts +43 -14
  243. package/src/results/writer.ts +31 -13
  244. package/src/runner/attempt.test.ts +138 -7
  245. package/src/runner/attempt.ts +603 -104
  246. package/src/runner/discover.test.ts +47 -0
  247. package/src/runner/discover.ts +36 -2
  248. package/src/runner/eval-source.test.ts +1 -27
  249. package/src/runner/feedback/agent.test.ts +504 -0
  250. package/src/runner/feedback/agent.ts +409 -0
  251. package/src/runner/feedback/ci.test.ts +562 -0
  252. package/src/runner/feedback/ci.ts +401 -0
  253. package/src/runner/feedback/coordinator.test.ts +317 -0
  254. package/src/runner/feedback/coordinator.ts +397 -0
  255. package/src/runner/feedback/failure.ts +40 -0
  256. package/src/runner/feedback/human.test.ts +616 -0
  257. package/src/runner/feedback/human.ts +535 -0
  258. package/src/runner/feedback/index.ts +66 -0
  259. package/src/runner/feedback/io.ts +78 -0
  260. package/src/runner/feedback/profile.test.ts +50 -0
  261. package/src/runner/feedback/profile.ts +58 -0
  262. package/src/runner/feedback/reducer.test.ts +395 -0
  263. package/src/runner/feedback/reducer.ts +260 -0
  264. package/src/runner/feedback/renderer.ts +82 -0
  265. package/src/runner/feedback/sink.ts +203 -0
  266. package/src/runner/feedback/testing.ts +106 -0
  267. package/src/runner/ledger.test.ts +230 -0
  268. package/src/runner/ledger.ts +329 -0
  269. package/src/runner/report.test.ts +128 -3
  270. package/src/runner/report.ts +33 -9
  271. package/src/runner/reporters/artifacts.ts +8 -2
  272. package/src/runner/reporters/braintrust.test.ts +8 -7
  273. package/src/runner/reporters/braintrust.ts +9 -2
  274. package/src/runner/reporters/index.ts +2 -2
  275. package/src/runner/reporters/json.test.ts +162 -0
  276. package/src/runner/reporters/json.ts +35 -8
  277. package/src/runner/reporters/shared.ts +1 -5
  278. package/src/runner/run.test.ts +760 -3
  279. package/src/runner/run.ts +243 -37
  280. package/src/runner/sandbox-prep.ts +3 -42
  281. package/src/runner/timing.ts +158 -0
  282. package/src/runner/types.ts +518 -22
  283. package/src/sandbox/checkpoint.test.ts +55 -0
  284. package/src/sandbox/checkpoint.ts +29 -8
  285. package/src/sandbox/cli-commands.ts +407 -0
  286. package/src/sandbox/docker.ts +115 -16
  287. package/src/sandbox/e2b-agent-template.test.ts +56 -0
  288. package/src/sandbox/e2b-agent-template.ts +94 -0
  289. package/src/sandbox/e2b.ts +74 -9
  290. package/src/sandbox/errors.ts +111 -4
  291. package/src/sandbox/index.ts +2 -0
  292. package/src/sandbox/io-retry.test.ts +58 -0
  293. package/src/sandbox/io-retry.ts +45 -0
  294. package/src/sandbox/keep-registry.test.ts +86 -0
  295. package/src/sandbox/keep-registry.ts +142 -0
  296. package/src/sandbox/keep.ts +178 -0
  297. package/src/sandbox/paths.test.ts +1 -0
  298. package/src/sandbox/paths.ts +19 -8
  299. package/src/sandbox/registry.ts +20 -3
  300. package/src/sandbox/resolve.ts +76 -11
  301. package/src/sandbox/retry.test.ts +70 -0
  302. package/src/sandbox/retry.ts +46 -4
  303. package/src/sandbox/types.ts +44 -6
  304. package/src/sandbox/vercel.ts +43 -20
  305. package/src/scoring/collector.ts +60 -17
  306. package/src/scoring/coverage.ts +95 -0
  307. package/src/scoring/diff.ts +81 -0
  308. package/src/scoring/display.test.ts +121 -0
  309. package/src/scoring/display.ts +133 -0
  310. package/src/scoring/evidence.test.ts +189 -0
  311. package/src/scoring/judge.test.ts +142 -0
  312. package/src/scoring/judge.ts +15 -18
  313. package/src/scoring/scoped.ts +217 -50
  314. package/src/scoring/types.ts +117 -20
  315. package/src/scoring/verdict.ts +16 -4
  316. package/src/shared/aggregate.ts +8 -6
  317. package/src/shared/types.ts +31 -0
  318. package/src/show/compose.ts +50 -67
  319. package/src/show/index.ts +127 -56
  320. package/src/show/render.ts +662 -131
  321. package/src/show/report-host.test.ts +188 -0
  322. package/src/show/report-host.ts +375 -0
  323. package/src/show/show.test.ts +320 -54
  324. package/src/tty-line.ts +8 -26
  325. package/src/util.test.ts +1 -0
  326. package/src/util.ts +41 -0
  327. package/src/view/app/App.test.tsx +69 -0
  328. package/src/view/app/App.tsx +144 -48
  329. package/src/view/app/components/AttemptModal.tsx +423 -11
  330. package/src/view/app/components/CodeView.tsx +41 -14
  331. package/src/view/app/components/CopyControls.tsx +2 -2
  332. package/src/view/app/i18n.ts +37 -17
  333. package/src/view/app/lib/attempt-route.test.ts +1 -0
  334. package/src/view/app/lib/verdict.ts +7 -9
  335. package/src/view/app/main.tsx +13 -8
  336. package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
  337. package/src/view/app/types.ts +4 -1
  338. package/src/view/artifact-serving.test.ts +2 -1
  339. package/src/view/client-dist/app.css +1 -1
  340. package/src/view/client-dist/app.js +17 -17
  341. package/src/view/data.test.ts +10 -3
  342. package/src/view/data.ts +155 -49
  343. package/src/view/index.ts +56 -41
  344. package/src/view/server.ts +37 -15
  345. package/src/view/shared/types.ts +34 -5
  346. package/src/view/styles.css +227 -0
  347. package/src/view/view-report.test.ts +167 -62
  348. package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
  349. package/dist/report/built-ins/experiment-comparison.js +0 -13
  350. package/dist/report/built-ins/index.d.ts +0 -1
  351. package/dist/report/built-ins/index.js +0 -2
  352. package/dist/report/react/GroupSummary.d.ts +0 -8
  353. package/dist/report/react/GroupSummary.js +0 -8
  354. package/dist/report/react/RunOverview.d.ts +0 -8
  355. package/dist/report/react/RunOverview.js +0 -12
  356. package/docs-site/zh/example/ai-agent-application.mdx +0 -152
  357. package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
  358. package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
  359. package/docs-site/zh/example/showcase.mdx +0 -39
  360. package/docs-site/zh/guides/publish-report.mdx +0 -91
  361. package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
  362. package/src/report/built-in-user-parity.test.tsx +0 -640
  363. package/src/report/built-ins/experiment-comparison.tsx +0 -19
  364. package/src/report/built-ins/index.ts +0 -2
  365. package/src/report/react/GroupSummary.tsx +0 -66
  366. package/src/report/react/RunOverview.tsx +0 -109
  367. package/src/runner/reporters/console.ts +0 -70
  368. package/src/runner/reporters/live.test.ts +0 -56
  369. package/src/runner/reporters/live.ts +0 -247
  370. package/src/runner/reporters/quiet.test.ts +0 -66
  371. package/src/runner/reporters/quiet.ts +0 -49
  372. package/src/runner/reporters/table.ts +0 -277
  373. /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
  374. /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
  375. /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
  376. /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
  377. /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
  378. /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
@@ -7,20 +7,38 @@
7
7
  // knownEvalIds(复制时刻该实验已知的 eval 并集),发布目录上重新 openResults().latest(),
8
8
  // 残缺警告被同一套机制重新算出来,不靠发布者转述。
9
9
 
10
- import { copyFile, mkdir, readdir, stat, writeFile } from "node:fs/promises";
10
+ import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
11
11
  import { basename, dirname, join, resolve } from "node:path";
12
- import type { EvalResult } from "../types.ts";
12
+ import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
13
13
  import { RESULTS_FORMAT } from "../types.ts";
14
14
  import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, experimentDirOf } from "./format.ts";
15
15
  import { experimentOfSnapshot } from "./open.ts";
16
16
  import { isNewerSnapshot } from "./select.ts";
17
17
  import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
18
- import type { ArtifactKind, AttemptHandle, Selection, Snapshot, SnapshotMeta } from "./types.ts";
19
- import { ARTIFACT_KINDS } from "./types.ts";
18
+ import {
19
+ PUBLISH_FILE_MAX_BYTES,
20
+ redactEvents,
21
+ redactExperimentInfo,
22
+ redactJsonValue,
23
+ redactResultRecord,
24
+ redactSpans,
25
+ type Redactor,
26
+ } from "./publish.ts";
27
+ import type { ArtifactKind, AttemptHandle, Scope, Snapshot, SnapshotMeta } from "./types.ts";
28
+
29
+ /** 缺省携带的 artifact:events / trace / o11y / agentSetup / sources;diff 不截断、可达百 MB,缺省不带。 */
30
+ const DEFAULT_PUBLISH_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "sources"];
31
+ const VALID_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "diff", "sources"];
20
32
 
21
33
  export interface CopySnapshotsOptions {
22
- /** 要带上的 artifact 种类;省略 = 全部六类。diff 可达百 MB,发布时常见地不带;o11y 只有几 KB,报告用 turns 这类 artifact 档指标(见 docs/feature/reports/library.md「内置指标」)时记得带上。 */
34
+ /** 要带上的 artifact 种类;缺省带 events / trace / o11y / agentSetup / sources,不带 diff。 */
23
35
  artifacts?: ArtifactKind[];
36
+ /**
37
+ * 发布消毒(必填,没有隐式默认):传函数逐值消毒自由文本字段,或显式传 `false` 声明
38
+ * 「这批数据可以原文发布」——「不消毒」必须是写在代码里的选择,不是忘了传参数的副作用。
39
+ * 结构字段(格式、判定、身份、路径、哈希)永不经过 redactor。
40
+ */
41
+ redact: Redactor | false;
24
42
  }
25
43
 
26
44
  export interface CopySnapshotsResult {
@@ -32,25 +50,31 @@ export interface CopySnapshotsResult {
32
50
 
33
51
  /**
34
52
  * 把选中快照复制成 `destDir` 下的一个标准结果根目录(`<experiment-dir>/<源快照目录名>/`,
35
- * 快照目录名原样保留,身份不变)。输入收 Selection 或手工挑的 Snapshot[]
53
+ * 快照目录名原样保留,身份不变)。输入收 Scope 或手工挑的 Snapshot[]
36
54
  * (与 Reports 计算函数同一输入约定)。目标目录非空即报错,不静默覆盖、不合并。
37
55
  */
38
56
  export async function copySnapshots(
39
- selection: Selection | Snapshot[],
57
+ scope: Scope | readonly Snapshot[],
40
58
  destDir: string,
41
- opts?: CopySnapshotsOptions,
59
+ opts: CopySnapshotsOptions,
42
60
  ): Promise<CopySnapshotsResult> {
43
- const selected = Array.isArray(selection) ? selection : selection.snapshots;
61
+ const selected = Array.isArray(scope) ? (scope as readonly Snapshot[]) : (scope as Scope).snapshots;
44
62
  if (selected.length === 0) {
45
63
  throw new Error(
46
64
  "copySnapshots got no snapshots to copy. Check the experiments filter, or pass snapshots from openResults().latest().",
47
65
  );
48
66
  }
49
- const kinds = opts?.artifacts ?? [...ARTIFACT_KINDS];
67
+ if (opts?.redact === undefined) {
68
+ throw new Error(
69
+ 'copySnapshots requires an explicit "redact" option: pass a (text) => string sanitizer, or the literal false to declare this data safe to publish verbatim.',
70
+ );
71
+ }
72
+ const redactor: Redactor | undefined = opts.redact === false ? undefined : opts.redact;
73
+ const kinds = opts.artifacts ?? [...DEFAULT_PUBLISH_ARTIFACTS];
50
74
  for (const kind of kinds) {
51
- if (!ARTIFACT_KINDS.includes(kind)) {
75
+ if (!VALID_ARTIFACTS.includes(kind)) {
52
76
  throw new Error(
53
- `Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${ARTIFACT_KINDS.join(", ")}.`,
77
+ `Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${VALID_ARTIFACTS.join(", ")}.`,
54
78
  );
55
79
  }
56
80
  }
@@ -68,101 +92,167 @@ export async function copySnapshots(
68
92
  continue;
69
93
  }
70
94
  warnings.push(
71
- `warning: multiple snapshots selected for experiment "${snapshot.experimentId}"; kept the newest one, dropped the rest. Dedupe with Selection.filter() or pick a single snapshot per experiment before copySnapshots to avoid this.`,
95
+ `warning: multiple snapshots selected for experiment "${snapshot.experimentId}"; kept the newest one, dropped the rest. Dedupe with Scope.filter() or pick a single snapshot per experiment before copySnapshots to avoid this.`,
72
96
  );
73
97
  if (isNewerSnapshot(snapshot, existing)) byExperiment.set(snapshot.experimentId, snapshot);
74
98
  }
75
99
 
76
- await mkdir(dest, { recursive: true });
100
+ // 发布前整文件预检:先规划并序列化全部目标文件(消毒发生在序列化后、写盘前),任一文件
101
+ // 超过 PUBLISH_FILE_MAX_BYTES 就整体失败,不留半成品目标目录。
102
+ const planned: PlannedFile[] = [];
77
103
  for (const snapshot of byExperiment.values()) {
78
- await copyOneSnapshot(snapshot, selected, dest, kinds);
104
+ planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds, redactor)));
105
+ }
106
+ const oversized = planned.filter((f) => f.bytes.byteLength > PUBLISH_FILE_MAX_BYTES);
107
+ if (oversized.length > 0) {
108
+ const lines = oversized.map(
109
+ (f) =>
110
+ ` ${f.source ?? f.path}: ${f.bytes.byteLength} bytes (limit ${PUBLISH_FILE_MAX_BYTES}). Exclude that artifact kind from "artifacts", or regenerate the history with the current writer.`,
111
+ );
112
+ throw new Error(`copySnapshots publish precheck failed; ${oversized.length} file(s) exceed the 50 MiB publish budget:\n${lines.join("\n")}`);
113
+ }
114
+
115
+ await mkdir(dest, { recursive: true });
116
+ for (const file of planned) {
117
+ await mkdir(dirname(file.path), { recursive: true });
118
+ await writeFile(file.path, file.bytes);
79
119
  }
80
120
 
81
121
  return { dir: dest, warnings };
82
122
  }
83
123
 
84
- async function copyOneSnapshot(snapshot: Snapshot, selected: Snapshot[], destRoot: string, kinds: ArtifactKind[]): Promise<void> {
124
+ interface PlannedFile {
125
+ /** 目标绝对路径。 */
126
+ path: string;
127
+ /** 序列化(且已消毒)后的字节。 */
128
+ bytes: Buffer;
129
+ /** 源文件路径(预检报错定位用);重建的 json 没有单一源。 */
130
+ source?: string;
131
+ }
132
+
133
+ async function planOneSnapshot(
134
+ snapshot: Snapshot,
135
+ selected: Snapshot[],
136
+ destRoot: string,
137
+ kinds: ArtifactKind[],
138
+ redactor: Redactor | undefined,
139
+ ): Promise<PlannedFile[]> {
85
140
  const destSnapDir = join(destRoot, experimentDirOf(snapshot.experimentId), basename(snapshot.dir));
86
- await mkdir(destSnapDir, { recursive: true });
141
+ const planned: PlannedFile[] = [];
87
142
 
88
143
  // sources 的去重仓库(sources/<sha256>.json)是快照级的:同一份源码被多个 attempt 引用时,
89
- // 复制到目的地也只应该有一份——这个 Set 记录本快照已经落过盘的 hash,整快照的 attempt 共享。
90
- const copiedSourceHashes = new Set<string>();
144
+ // 复制到目的地也只应该有一份——这个 Set 记录本快照已经规划过的 hash,整快照的 attempt 共享。
145
+ const plannedSourceHashes = new Set<string>();
91
146
  for (const attempt of snapshot.attempts) {
92
- await copyOneAttempt(attempt, destSnapDir, kinds, copiedSourceHashes);
147
+ planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes, redactor)));
93
148
  }
94
149
 
95
150
  const knownEvalIds = experimentOfSnapshot(snapshot)?.evalIds ?? fallbackUnion(selected, snapshot.experimentId);
151
+ const experiment =
152
+ snapshot.experiment !== undefined && redactor
153
+ ? (redactExperimentInfo(snapshot.experiment as unknown as Record<string, unknown>, redactor) as unknown as SnapshotMeta["experiment"])
154
+ : snapshot.experiment;
96
155
  const meta: SnapshotMeta = {
97
156
  format: RESULTS_FORMAT,
98
157
  schemaVersion: snapshot.schemaVersion,
99
158
  producer: snapshot.producer,
100
159
  experimentId: snapshot.experimentId,
101
- ...(snapshot.experiment !== undefined ? { experiment: snapshot.experiment } : {}),
160
+ ...(experiment !== undefined ? { experiment } : {}),
102
161
  agent: snapshot.agent,
103
162
  ...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
104
163
  startedAt: snapshot.startedAt,
105
164
  ...(snapshot.completedAt !== undefined ? { completedAt: snapshot.completedAt } : {}),
106
165
  ...(knownEvalIds.length ? { knownEvalIds } : {}),
107
- ...(snapshot.name !== undefined ? { name: snapshot.name } : {}),
166
+ ...(snapshot.name !== undefined
167
+ ? { name: redactor && typeof snapshot.name === "string" ? redactor(snapshot.name) : snapshot.name }
168
+ : {}),
169
+ // 发布根标记只声明流程,不证明结果:applied = 消毒函数对全部自由文本字段跑过;
170
+ // none = 作者显式的原文发布声明。view --out 的防呆据此分级。
171
+ publish: { redaction: redactor ? "applied" : "none" },
108
172
  };
109
- await writeFile(join(destSnapDir, SNAPSHOT_FILE), JSON.stringify(meta, null, 2), "utf-8");
173
+ planned.push({ path: join(destSnapDir, SNAPSHOT_FILE), bytes: Buffer.from(JSON.stringify(meta, null, 2), "utf-8") });
174
+ return planned;
110
175
  }
111
176
 
112
- async function copyOneAttempt(
177
+ async function planOneAttempt(
113
178
  attempt: AttemptHandle,
114
179
  destSnapDir: string,
115
180
  kinds: ArtifactKind[],
116
- copiedSourceHashes: Set<string>,
117
- ): Promise<void> {
181
+ plannedSourceHashes: Set<string>,
182
+ redactor: Redactor | undefined,
183
+ ): Promise<PlannedFile[]> {
118
184
  const destAttemptDir = join(destSnapDir, attempt.ref.attempt);
119
- await mkdir(destAttemptDir, { recursive: true });
185
+ const planned: PlannedFile[] = [];
120
186
 
121
187
  // sources 是唯一「两层」的 artifact(attempt 级引用 + 快照级去重仓库),不能像其它四类那样
122
- // 单文件 copyFile 原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的
123
- // attempt.sources() 拿到完整内容,再在目的地按内容哈希重新去重落盘,天然吸收 artifactBase
124
- // 回退链(携带条目复制后,原快照可能不在目的地里,必须此刻把内容落到自己脚下)。
188
+ // 单文件原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的 attempt.sources()
189
+ // 拿到完整内容,消毒后按(消毒后)内容哈希重新去重落盘——发布根里引用与内容永远一致。
125
190
  const genericKinds = kinds.filter((k) => k !== "sources");
126
191
  const files = await findArtifactFiles(attempt, genericKinds);
127
- await Promise.all(files.map(({ kind, source }) => copyFile(source, join(destAttemptDir, artifactFileOf(kind)))));
128
192
  const copied = new Set(files.map((f) => f.kind));
193
+ for (const { kind, source } of files) {
194
+ const raw = await readFile(source);
195
+ const bytes = redactor ? redactArtifactBytes(kind, raw, redactor) : raw;
196
+ planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes, source });
197
+ }
129
198
 
130
199
  if (kinds.includes("sources")) {
131
- const wroteSources = await copySources(attempt, destSnapDir, destAttemptDir, copiedSourceHashes);
132
- if (wroteSources) copied.add("sources");
200
+ const sources = await attempt.sources();
201
+ if (sources && sources.length > 0) {
202
+ copied.add("sources");
203
+ const destStoreDir = join(destSnapDir, "sources");
204
+ const refs: { path: string; sha256: string }[] = [];
205
+ for (const src of sources) {
206
+ const content = redactor ? redactor(src.content) : src.content;
207
+ const sha256 = hashEvalSource(normalizeEvalSource(content));
208
+ refs.push({ path: src.path, sha256 });
209
+ if (!plannedSourceHashes.has(sha256)) {
210
+ planned.push({
211
+ path: join(destStoreDir, `${sha256}.json`),
212
+ bytes: Buffer.from(JSON.stringify({ content }), "utf-8"),
213
+ });
214
+ plannedSourceHashes.add(sha256);
215
+ }
216
+ }
217
+ planned.push({
218
+ path: join(destAttemptDir, artifactFileOf("sources")),
219
+ bytes: Buffer.from(JSON.stringify(refs), "utf-8"),
220
+ });
221
+ }
133
222
  }
134
223
 
135
- const record = slimForCopy(attempt.result, copied);
136
- await writeFile(join(destAttemptDir, RESULT_FILE), JSON.stringify(record, null, 2), "utf-8");
224
+ let record = slimForCopy(attempt.result, copied);
225
+ if (redactor) record = redactResultRecord(record, redactor);
226
+ planned.push({ path: join(destAttemptDir, RESULT_FILE), bytes: Buffer.from(JSON.stringify(record, null, 2), "utf-8") });
227
+ return planned;
137
228
  }
138
229
 
139
- /**
140
- * sources 的复制:经 attempt.sources() 拿到已解引用的完整内容(null / 空数组 = 没有,不写任何
141
- * 文件,与其它 artifact「空数据不落文件」的约定一致),按内容 sha256 写进目的快照的
142
- * `sources/<sha256>.json`(同一快照内已经写过的 hash 不重写),attempt 目录下落一份引用。
143
- */
144
- async function copySources(
145
- attempt: AttemptHandle,
146
- destSnapDir: string,
147
- destAttemptDir: string,
148
- copiedSourceHashes: Set<string>,
149
- ): Promise<boolean> {
150
- const sources = await attempt.sources();
151
- if (!sources || sources.length === 0) return false;
152
-
153
- const destStoreDir = join(destSnapDir, "sources");
154
- const refs: { path: string; sha256: string }[] = [];
155
- for (const src of sources) {
156
- const sha256 = hashEvalSource(normalizeEvalSource(src.content));
157
- refs.push({ path: src.path, sha256 });
158
- if (!copiedSourceHashes.has(sha256)) {
159
- await mkdir(destStoreDir, { recursive: true });
160
- await writeFile(join(destStoreDir, `${sha256}.json`), JSON.stringify({ content: src.content }), "utf-8");
161
- copiedSourceHashes.add(sha256);
230
+ /** 单个 artifact 文件的消毒:按种类解析 JSON、走各自的自由文本标注,重新序列化。 */
231
+ function redactArtifactBytes(kind: ArtifactKind, raw: Buffer, redactor: Redactor): Buffer {
232
+ try {
233
+ const parsed = JSON.parse(raw.toString("utf-8")) as unknown;
234
+ let next: unknown;
235
+ switch (kind) {
236
+ case "events":
237
+ next = redactEvents(parsed as StreamEvent[], redactor);
238
+ break;
239
+ case "trace":
240
+ next = redactSpans(parsed as TraceSpan[], redactor);
241
+ break;
242
+ // diff before/after/内容与 o11y 的命令 / 错误 / URL 都是自由文本;路径键为结构字段。
243
+ case "diff":
244
+ case "o11y":
245
+ case "agentSetup":
246
+ next = redactJsonValue(parsed, redactor);
247
+ break;
248
+ default:
249
+ next = parsed;
162
250
  }
251
+ return Buffer.from(JSON.stringify(next), "utf-8");
252
+ } catch {
253
+ // 解析不了的按原字节透传(malformed 数据不该在发布路径上被静默改写)。
254
+ return raw;
163
255
  }
164
- await writeFile(join(destAttemptDir, artifactFileOf("sources")), JSON.stringify(refs), "utf-8");
165
- return true;
166
256
  }
167
257
 
168
258
  /** 目标目录非空即报错:盘上不该出现「我没写的东西被动过」的惊讶;发布脚本要幂等就自己先清目录。 */
@@ -1,3 +1,4 @@
1
+ // cases: docs/engineering/unit-tests/results/cases.md
1
2
  // show / view 宿主等价契约(设计契约:plan/show-view-equivalence.md 完成定义;
2
3
  // docs/feature/reports/architecture.md「Selection 是计算入口」;docs/feature/results/library.md「选择快照」)。
3
4
  //
@@ -23,7 +24,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
23
24
  import { tmpdir } from "node:os";
24
25
  import { join } from "node:path";
25
26
  import { openResults } from "./index.ts";
26
- import type { Selection, SelectionWarning } from "./index.ts";
27
+ import type { Scope, ScopeWarning } from "./index.ts";
27
28
  import { selectCurrentResults, type ResultScope } from "./select.ts";
28
29
  import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
29
30
  import { runShow, type ShowFlags } from "../show/index.ts";
@@ -131,7 +132,7 @@ interface NormSelection {
131
132
  experiments: NormExperiment[];
132
133
  }
133
134
 
134
- function normalizeWarning(w: SelectionWarning): NormWarning {
135
+ function normalizeWarning(w: ScopeWarning): NormWarning {
135
136
  switch (w.kind) {
136
137
  case "partial-coverage":
137
138
  return { kind: w.kind, experimentId: w.experimentId, covered: w.covered, total: w.total };
@@ -143,7 +144,7 @@ function normalizeWarning(w: SelectionWarning): NormWarning {
143
144
  }
144
145
  }
145
146
 
146
- function normalizeSelection(selection: Selection): NormSelection {
147
+ function normalizeSelection(selection: Scope): NormSelection {
147
148
  return {
148
149
  warnings: selection.warnings.map(normalizeWarning),
149
150
  experiments: selection.snapshots.map((snapshot) => ({
@@ -166,7 +167,7 @@ function normalizeSelection(selection: Selection): NormSelection {
166
167
  async function showText(root: string, patterns: string[], flags: ShowFlags = {}): Promise<string> {
167
168
  let out = "";
168
169
  let err = "";
169
- const code = await runShow(root, patterns, { run: root, ...flags }, {
170
+ const code = await runShow(root, patterns, { results: root, ...flags }, {
170
171
  out: (s) => (out += s),
171
172
  err: (s) => (err += s),
172
173
  width: 120,
@@ -178,8 +179,8 @@ async function showText(root: string, patterns: string[], flags: ShowFlags = {})
178
179
 
179
180
  /** view 的 web 面:loadViewScan → 报告槽 HTML(en)。 */
180
181
  async function viewHtml(root: string, opts: ViewScanOptions = {}): Promise<string> {
181
- const { reportHtml } = await loadViewScan(root, opts);
182
- return reportHtml.en;
182
+ const { reportPages } = await loadViewScan(root, opts);
183
+ return reportPages.map((page) => page.html.en).join("\n");
183
184
  }
184
185
 
185
186
  /** 两个宿主构造给选择器的 scope 完全同形:验证读源无误,避免"我以为它们一样"。 */
@@ -339,7 +340,7 @@ describe("selectCurrentResults · 现刻水位结构化身份", () => {
339
340
  const rootA = await makeRoot();
340
341
  const rootB = await makeRoot();
341
342
  // 失败 eval 让 id 现于两面(通过 eval 只进折叠子行,不便断言);qa / qb 各根独有。
342
- const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0, passed: false, detail: "boom" }] };
343
+ const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0, outcome: "failed" as const, detail: "boom" }] };
343
344
  await writeSnapshot(rootA, "2026-07-01T08-00-00-000Z", { experimentId: "onlyA/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [res("qa", "failed", boom)]);
344
345
  await writeSnapshot(rootB, "2026-07-02T08-00-00-000Z", { experimentId: "onlyB/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [res("qb", "failed", boom)]);
345
346
  const normB = normalizeSelection(selectCurrentResults(await openResults(rootB)));
@@ -389,7 +390,7 @@ async function seedPartialRerun(): Promise<string> {
389
390
  const root = await makeRoot();
390
391
  await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
391
392
  res("q1", "passed"),
392
- res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0, passed: false, detail: "file was not modified" }] }),
393
+ res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0, outcome: "failed" as const, detail: "file was not modified" }] }),
393
394
  ]);
394
395
  await writeSnapshot(root, "2026-07-02T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
395
396
  res("q1", "passed"),
@@ -404,7 +405,7 @@ async function seedPartialCoverage(): Promise<string> {
404
405
  root,
405
406
  "2026-07-01T08-00-00-000Z",
406
407
  { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q1", "q2"] },
407
- [res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, passed: false, detail: "boom" }] })],
408
+ [res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] })],
408
409
  );
409
410
  return root;
410
411
  }
@@ -415,9 +416,10 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
415
416
  const text = await showText(root, []);
416
417
  const html = await viewHtml(root);
417
418
  for (const face of [text, html]) {
418
- expect(face).toContain("compare/bub");
419
419
  expect(face).toContain("q2"); // 周一补齐的失败 eval,若宿主回退 results.latest() 就会消失
420
420
  }
421
+ expect(text).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
422
+ expect(html).toContain('data-sort-value="compare/bub"');
421
423
  expect(text).toContain("1 passed / 1 failed");
422
424
  expect(html).toContain("50%");
423
425
  // 现刻水位覆盖齐全 → 两面都不出 partial-coverage(text: "verdicts cover"; html: data-kind)。
@@ -431,7 +433,7 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
431
433
  it("位置前缀收窄:两扇门都只见范围内的 eval,范围外一致排除", async () => {
432
434
  const root = await makeRoot();
433
435
  await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
434
- res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, passed: false, detail: "boom" }] }),
436
+ res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
435
437
  res("weather/queens", "passed"),
436
438
  res("algebra/quadratic", "passed"),
437
439
  ]);
@@ -451,20 +453,32 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
451
453
  await writeFile(
452
454
  reportPath,
453
455
  [
456
+ "// 手搭新 defineReport 产物(kind + brand + 规范化页列表);组合组件从 ctx.scope 读注入范围。",
454
457
  'const FACES = Symbol.for("niceeval.report.faces");',
455
- "const Echo = (props) => Echo[FACES].web(props);",
456
- "Echo[FACES] = {",
458
+ 'const COMPOSE = Symbol.for("niceeval.report.compose");',
459
+ "const Leaf = (props) => Leaf[FACES].web(props);",
460
+ "Leaf[FACES] = {",
457
461
  ' web: (props) => "EVALS[" + props.ids + "]",',
458
462
  ' text: (props) => "EVALS[" + props.ids + "]",',
459
463
  "};",
464
+ "const Echo = () => { throw new Error(\"compose only\"); };",
465
+ "Echo[COMPOSE] = (_props, ctx) => ({",
466
+ ' $$typeof: Symbol.for("react.transitional.element"),',
467
+ " type: Leaf,",
468
+ ' props: { ids: ctx.scope.snapshots.flatMap((s) => s.evals.map((e) => e.id)).sort().join(",") },',
469
+ " key: null,",
470
+ "});",
460
471
  "export default {",
472
+ ' kind: "report",',
461
473
  ' [Symbol.for("niceeval.report.definition")]: true,',
462
- " build: (ctx) => ({",
463
- ' $$typeof: Symbol.for("react.transitional.element"),',
464
- " type: Echo,",
465
- ' props: { ids: ctx.selection.snapshots.flatMap((s) => s.evals.map((e) => e.id)).sort().join(",") },',
466
- " key: null,",
467
- " }),",
474
+ " links: [],",
475
+ " scripts: [],",
476
+ " styles: [],",
477
+ " pages: [{",
478
+ ' id: "report",',
479
+ ' title: "Report",',
480
+ ' content: { $$typeof: Symbol.for("react.transitional.element"), type: Echo, props: {}, key: null },',
481
+ " }],",
468
482
  "};",
469
483
  "",
470
484
  ].join("\n"),
@@ -497,7 +511,7 @@ function hasPartialCoverageHtml(html: string): boolean {
497
511
  it("通过计数一致:同一 fixture 两面反映 3 过 1 败", async () => {
498
512
  const root = await makeRoot();
499
513
  await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
500
- res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0, passed: false, detail: "boom" }] }),
514
+ res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
501
515
  res("q2", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
502
516
  res("q3", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
503
517
  res("q4", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
@@ -1,6 +1,6 @@
1
1
  // niceeval/results —— 实验结果数据的读写库(定稿见 docs/feature/results/library.md、docs/feature/results/architecture.md)。
2
2
  //
3
- // 读:openResults(实验 → 快照 → eval → attempt 分层、skipped、latest() Selection);
3
+ // 读:openResults(实验 → 快照 → eval → attempt 分层、skipped、latest() / current() Scope);
4
4
  // 写:createResultsWriter(快照声明 + attempt 增量落盘 + finish 补 completedAt);
5
5
  // 发布:copySnapshots(格式感知复制 + knownEvalIds 补记);
6
6
  // 身份:dedupeAttempts(跨快照聚合前按 (experimentId, evalId, attempt, startedAt) 去重)、isNewerSnapshot。
@@ -29,9 +29,11 @@ export {
29
29
  } from "./locator.ts";
30
30
  export {
31
31
  buildAnnotatedEvalSource,
32
+ deriveSendAnnotations,
32
33
  type AnnotatedEvalSource,
33
34
  type AnnotatedEvalSourceSummary,
34
35
  type AnnotatedSourceLine,
36
+ type SendAnnotation,
35
37
  } from "./annotated-source.ts";
36
38
  export { loadAnnotatedEvalSource } from "./attempt-source.ts";
37
39
  export {
@@ -42,7 +44,13 @@ export {
42
44
  } from "./attempt-evidence.ts";
43
45
  export { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
44
46
  export { groupIncompatibleVersionSkips, type SkippedVersionGroup } from "./skipped-notice.ts";
45
- export { dedupeAttempts, isNewerSnapshot } from "./select.ts";
47
+ export {
48
+ comparabilityConfigOf,
49
+ dedupeAttempts,
50
+ deepEqualJson,
51
+ isNewerSnapshot,
52
+ type ComparabilityConfig,
53
+ } from "./select.ts";
46
54
  export { copySnapshots, type CopySnapshotsOptions, type CopySnapshotsResult } from "./copy.ts";
47
55
  export {
48
56
  createResultsWriter,
@@ -71,8 +79,8 @@ export {
71
79
  type Eval,
72
80
  type Experiment,
73
81
  type Results,
74
- type Selection,
75
- type SelectionWarning,
82
+ type Scope,
83
+ type ScopeWarning,
76
84
  type SkippedDir,
77
85
  type Snapshot,
78
86
  type SnapshotMeta,
@@ -1,3 +1,4 @@
1
+ // cases: docs/engineering/unit-tests/results/cases.md
1
2
  // AttemptLocator 单测(定稿见 docs/feature/results/library.md「按 locator 寻址一个 attempt」、docs/concepts.md「Attempt 定位符」):
2
3
  // 确定性编码、字段敏感(任一身份字段变化都换 locator)、@ 前缀与定长 base36 格式、decode 的语法
3
4
  // 校验分支、大批量下的实际无撞车、buildLocatorIndex 的撞车检测(用可注入的 encode 强制制造撞车,
@@ -79,10 +80,6 @@ describe("encodeAttemptLocator · 确定性与格式", () => {
79
80
  });
80
81
 
81
82
  describe("decodeAttemptLocator · 语法校验", () => {
82
- it("接受一个真实编码出来的 locator", () => {
83
- expect(decodeAttemptLocator(encodeAttemptLocator(id()))).toEqual({ valid: true, scheme: 1 });
84
- });
85
-
86
83
  it.each([
87
84
  ["", "空字符串"],
88
85
  ["7k2m9qz", "缺 @ 前缀"],
@@ -149,24 +146,6 @@ describe("buildLocatorIndex", () => {
149
146
  expect(err.message).toContain("collision");
150
147
  });
151
148
 
152
- it("撞车中若第三个身份与已记录身份相同,不重复抛错(只有真正不同的身份对才算撞车)", () => {
153
- const sameEncode = (() => {
154
- let calls = 0;
155
- return (_identity: AttemptIdentity) => {
156
- calls++;
157
- return `@1samesame` as ReturnType<typeof encodeAttemptLocator>;
158
- };
159
- })();
160
- const identity = id();
161
- const index = buildLocatorIndex(
162
- [
163
- { identity, handle: "first" },
164
- { identity, handle: "second" },
165
- ],
166
- sameEncode,
167
- );
168
- expect(index.size).toBe(1);
169
- });
170
149
  });
171
150
 
172
151
  describe("resolveAttemptLocator", () => {
@@ -11,8 +11,9 @@ import { basename, dirname, join, relative, resolve, sep } from "node:path";
11
11
  import type { EvalResult } from "../types.ts";
12
12
  import type { O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
13
13
  import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
14
+ import { deriveDiffData } from "../scoring/diff.ts";
14
15
  import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, classifySnapshot } from "./format.ts";
15
- import { isNewerSnapshot, selectLatest } from "./select.ts";
16
+ import { isNewerSnapshot, selectCurrentResults, selectLatest } from "./select.ts";
16
17
  import {
17
18
  encodeAttemptLocator,
18
19
  resolveAttemptLocator,
@@ -26,7 +27,7 @@ import type {
26
27
  Eval,
27
28
  Experiment,
28
29
  Results,
29
- Selection,
30
+ Scope,
30
31
  SkippedDir,
31
32
  Snapshot,
32
33
  SnapshotMeta,
@@ -170,13 +171,17 @@ export async function openResults(dir: string): Promise<Results> {
170
171
  }
171
172
 
172
173
  function makeResults(experiments: Experiment[], skipped: SkippedDir[]): Results {
173
- return {
174
+ const results: Results = {
174
175
  experiments,
175
176
  skipped,
176
- latest(opts?: { experiments?: string | string[] }): Selection {
177
+ latest(opts?: { experiments?: string | string[] }): Scope {
177
178
  return selectLatest(experiments, opts);
178
179
  },
180
+ current(opts?: { experiments?: string | string[] }): Scope {
181
+ return selectCurrentResults(results, { experiment: opts?.experiments });
182
+ },
179
183
  };
184
+ return results;
180
185
  }
181
186
 
182
187
  // ───────────────────────── 目录扫描 ─────────────────────────
@@ -360,7 +365,12 @@ function makeAttempt(snapshot: Snapshot, snapshotDir: string, attemptDir: string
360
365
  trace: lazyArtifact<TraceSpan[]>(candidates, "trace", record.trace),
361
366
  o11y: lazyArtifact<O11ySummary>(candidates, "o11y", record.o11y),
362
367
  agentSetup: lazyArtifact<AgentSetupManifest>(candidates, "agentSetup", record.agentSetup),
363
- diff: lazyArtifact<DiffData>(candidates, "diff", record.diff),
368
+ diff: (() => {
369
+ // diff.json 落盘的是逐窗口 delta 序列(DiffArtifact);文件级视图在读取面派生。
370
+ const raw = lazyArtifact<import("../types.ts").DiffArtifact>(candidates, "diff", record.diff);
371
+ let memo: Promise<DiffData | null> | undefined;
372
+ return () => (memo ??= raw().then((windows) => (windows === null ? null : deriveDiffData(windows))));
373
+ })(),
364
374
  sources: lazySources(candidates, candidateSnapshotRoots, record.sources),
365
375
  };
366
376
  }