niceeval 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. package/INDEX.md +23 -23
  2. package/README.zh.md +6 -6
  3. package/dist/agents/types.d.ts +69 -7
  4. package/dist/context/types.d.ts +32 -12
  5. package/dist/i18n/en.d.ts +54 -0
  6. package/dist/i18n/zh-CN.d.ts +57 -3
  7. package/dist/o11y/types.d.ts +16 -2
  8. package/dist/report/aggregate.d.ts +32 -24
  9. package/dist/report/aggregate.js +158 -50
  10. package/dist/report/built-in/index.d.ts +2 -0
  11. package/dist/report/built-in/index.js +8 -0
  12. package/dist/report/components.d.ts +93 -160
  13. package/dist/report/components.js +377 -114
  14. package/dist/report/compute.d.ts +87 -81
  15. package/dist/report/compute.js +597 -417
  16. package/dist/report/flag.d.ts +32 -6
  17. package/dist/report/flag.js +92 -4
  18. package/dist/report/format.d.ts +19 -11
  19. package/dist/report/format.js +30 -13
  20. package/dist/report/index.d.ts +16 -16
  21. package/dist/report/index.js +20 -21
  22. package/dist/report/load.js +3 -2
  23. package/dist/report/locale.d.ts +57 -33
  24. package/dist/report/locale.js +122 -56
  25. package/dist/report/metrics.d.ts +23 -4
  26. package/dist/report/metrics.js +110 -25
  27. package/dist/report/primitives.d.ts +48 -15
  28. package/dist/report/primitives.js +135 -26
  29. package/dist/report/react/AttemptList.d.ts +9 -7
  30. package/dist/report/react/AttemptList.js +17 -10
  31. package/dist/report/react/DeltaTable.js +19 -18
  32. package/dist/report/react/EvalList.d.ts +4 -4
  33. package/dist/report/react/EvalList.js +0 -0
  34. package/dist/report/react/ExperimentComparison.d.ts +10 -0
  35. package/dist/report/react/ExperimentComparison.js +12 -0
  36. package/dist/report/react/ExperimentList.d.ts +4 -3
  37. package/dist/report/react/ExperimentList.js +17 -18
  38. package/dist/report/react/MetricBars.js +5 -4
  39. package/dist/report/react/MetricLine.js +12 -5
  40. package/dist/report/react/MetricMatrix.js +1 -1
  41. package/dist/report/react/MetricScatter.js +59 -28
  42. package/dist/report/react/MetricTable.js +2 -12
  43. package/dist/report/react/ScopeSummary.d.ts +10 -0
  44. package/dist/report/react/ScopeSummary.js +17 -0
  45. package/dist/report/react/Scoreboard.js +6 -6
  46. package/dist/report/react/cell.js +2 -2
  47. package/dist/report/react/chart-math.d.ts +23 -6
  48. package/dist/report/react/chart-math.js +71 -19
  49. package/dist/report/react/fixtures.d.ts +5 -9
  50. package/dist/report/react/fixtures.js +110 -147
  51. package/dist/report/react/index.d.ts +15 -5
  52. package/dist/report/react/index.js +18 -7
  53. package/dist/report/report.d.ts +137 -16
  54. package/dist/report/report.js +259 -28
  55. package/dist/report/text/faces.d.ts +17 -19
  56. package/dist/report/text/faces.js +253 -184
  57. package/dist/report/text/plot.js +1 -1
  58. package/dist/report/text/table.js +38 -7
  59. package/dist/report/tree.d.ts +90 -40
  60. package/dist/report/tree.js +252 -94
  61. package/dist/report/types.d.ts +247 -284
  62. package/dist/report/types.js +4 -3
  63. package/dist/report/web.d.ts +21 -5
  64. package/dist/report/web.js +42 -16
  65. package/dist/results/select.d.ts +38 -16
  66. package/dist/results/select.js +73 -25
  67. package/dist/results/types.d.ts +49 -14
  68. package/dist/runner/feedback/sink.d.ts +110 -0
  69. package/dist/runner/types.d.ts +513 -22
  70. package/dist/sandbox/docker.d.ts +23 -2
  71. package/dist/sandbox/e2b.d.ts +15 -1
  72. package/dist/sandbox/errors.d.ts +30 -3
  73. package/dist/sandbox/io-retry.d.ts +17 -0
  74. package/dist/sandbox/registry.d.ts +2 -0
  75. package/dist/sandbox/resolve.d.ts +18 -5
  76. package/dist/sandbox/retry.d.ts +11 -1
  77. package/dist/sandbox/types.d.ts +39 -5
  78. package/dist/sandbox/vercel.d.ts +7 -1
  79. package/dist/scoring/coverage.d.ts +30 -0
  80. package/dist/scoring/display.d.ts +21 -0
  81. package/dist/scoring/display.js +120 -0
  82. package/dist/scoring/types.d.ts +103 -20
  83. package/dist/shared/aggregate.d.ts +4 -2
  84. package/dist/shared/aggregate.js +8 -7
  85. package/dist/shared/types.d.ts +28 -0
  86. package/dist/tty-line.d.ts +0 -4
  87. package/dist/util.d.ts +23 -0
  88. package/docs-site/zh/README.md +44 -0
  89. package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
  90. package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
  91. package/docs-site/zh/examples/index.mdx +50 -0
  92. package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
  93. package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
  94. package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
  95. package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
  96. package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
  97. package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
  98. package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
  99. package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
  100. package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
  101. package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
  102. package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
  103. package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
  104. package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
  105. package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
  106. package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
  107. package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
  108. package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
  109. package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
  110. package/docs-site/zh/how-to/publish-report.mdx +105 -0
  111. package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
  112. package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
  113. package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
  114. package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
  115. package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
  116. package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
  117. package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
  118. package/docs-site/zh/index.mdx +24 -26
  119. package/docs-site/zh/introduction.mdx +8 -8
  120. package/docs-site/zh/reference/builtin-agents.mdx +32 -5
  121. package/docs-site/zh/reference/capabilities.mdx +8 -8
  122. package/docs-site/zh/reference/cli.mdx +40 -12
  123. package/docs-site/zh/reference/define-agent.mdx +58 -5
  124. package/docs-site/zh/reference/define-config.mdx +1 -1
  125. package/docs-site/zh/reference/define-eval.mdx +42 -9
  126. package/docs-site/zh/reference/events.mdx +3 -3
  127. package/docs-site/zh/reference/expect.mdx +26 -1
  128. package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
  129. package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
  130. package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
  131. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
  132. package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
  133. package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
  134. package/package.json +10 -2
  135. package/src/agents/ai-sdk-otel.test.ts +1 -0
  136. package/src/agents/ai-sdk.test.ts +3 -0
  137. package/src/agents/ai-sdk.ts +3 -0
  138. package/src/agents/bub-install-spec.test.ts +34 -0
  139. package/src/agents/bub-install-spec.ts +32 -0
  140. package/src/agents/bub.ts +31 -32
  141. package/src/agents/claude-code.test.ts +130 -9
  142. package/src/agents/claude-code.ts +76 -4
  143. package/src/agents/codex.test.ts +189 -40
  144. package/src/agents/codex.ts +155 -14
  145. package/src/agents/coding-cli-versions.test.ts +15 -0
  146. package/src/agents/coding-cli-versions.ts +3 -0
  147. package/src/agents/index.ts +13 -2
  148. package/src/agents/langgraph.test.ts +204 -0
  149. package/src/agents/langgraph.ts +495 -0
  150. package/src/agents/marketplace.ts +85 -0
  151. package/src/agents/native-config.test.ts +179 -0
  152. package/src/agents/native-config.ts +267 -0
  153. package/src/agents/openai-compat.test.ts +1 -0
  154. package/src/agents/openai-compat.ts +1 -1
  155. package/src/agents/openclaw.test.ts +31 -0
  156. package/src/agents/openclaw.ts +171 -0
  157. package/src/agents/plugin-config.test.ts +1 -0
  158. package/src/agents/sdk-streams.test.ts +79 -0
  159. package/src/agents/sdk-streams.ts +55 -10
  160. package/src/agents/skills.test.ts +1 -0
  161. package/src/agents/streaming.test.ts +3 -9
  162. package/src/agents/streaming.ts +2 -2
  163. package/src/agents/types.ts +71 -8
  164. package/src/agents/ui-message-stream.test.ts +3 -0
  165. package/src/cli.ts +446 -124
  166. package/src/context/context.test.ts +51 -12
  167. package/src/context/context.ts +162 -30
  168. package/src/context/session.test.ts +2 -1
  169. package/src/context/session.ts +115 -7
  170. package/src/context/types.ts +30 -12
  171. package/src/define.test.ts +13 -8
  172. package/src/define.ts +25 -4
  173. package/src/expect/index.ts +53 -23
  174. package/src/i18n/en.ts +81 -17
  175. package/src/i18n/zh-CN.ts +80 -17
  176. package/src/o11y/cost.test.ts +1 -0
  177. package/src/o11y/execution-tree.test.ts +1 -20
  178. package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
  179. package/src/o11y/otlp/parse.test.ts +1 -0
  180. package/src/o11y/otlp/turn-otel.test.ts +1 -0
  181. package/src/o11y/parsers/bub.test.ts +1 -0
  182. package/src/o11y/parsers/claude-code.test.ts +1 -34
  183. package/src/o11y/parsers/openclaw.test.ts +154 -0
  184. package/src/o11y/parsers/openclaw.ts +310 -0
  185. package/src/o11y/prices.json +746 -311
  186. package/src/o11y/tool-names.test.ts +1 -0
  187. package/src/o11y/types.ts +16 -2
  188. package/src/report/aggregate.ts +178 -61
  189. package/src/report/built-in/index.tsx +9 -0
  190. package/src/report/components.tsx +625 -279
  191. package/src/report/compute.ts +723 -491
  192. package/src/report/dual-render.test.tsx +741 -1024
  193. package/src/report/flag.ts +104 -12
  194. package/src/report/format.ts +32 -12
  195. package/src/report/index.ts +119 -46
  196. package/src/report/load.ts +3 -2
  197. package/src/report/locale.ts +136 -65
  198. package/src/report/metrics.ts +108 -25
  199. package/src/report/primitives.tsx +196 -45
  200. package/src/report/react/AttemptList.tsx +30 -43
  201. package/src/report/react/DeltaTable.tsx +63 -45
  202. package/src/report/react/EvalList.tsx +0 -0
  203. package/src/report/react/ExperimentComparison.tsx +73 -0
  204. package/src/report/react/ExperimentList.tsx +50 -32
  205. package/src/report/react/MetricBars.tsx +5 -4
  206. package/src/report/react/MetricLine.tsx +13 -8
  207. package/src/report/react/MetricMatrix.tsx +2 -2
  208. package/src/report/react/MetricScatter.tsx +86 -34
  209. package/src/report/react/MetricTable.tsx +4 -76
  210. package/src/report/react/ScopeSummary.tsx +86 -0
  211. package/src/report/react/Scoreboard.tsx +28 -10
  212. package/src/report/react/cell.tsx +2 -2
  213. package/src/report/react/chart-math.test.ts +85 -0
  214. package/src/report/react/chart-math.ts +101 -22
  215. package/src/report/react/enhance.js +89 -5
  216. package/src/report/react/fixtures.ts +114 -154
  217. package/src/report/react/index.tsx +24 -39
  218. package/src/report/react/render.test.tsx +138 -158
  219. package/src/report/react/styles.css +243 -82
  220. package/src/report/report.test.ts +779 -841
  221. package/src/report/report.ts +423 -41
  222. package/src/report/text/faces.ts +290 -193
  223. package/src/report/text/plot.ts +1 -1
  224. package/src/report/text/table.ts +44 -7
  225. package/src/report/tree.ts +362 -104
  226. package/src/report/types.ts +261 -271
  227. package/src/report/web.ts +63 -20
  228. package/src/results/annotated-source.test.ts +62 -9
  229. package/src/results/annotated-source.ts +64 -6
  230. package/src/results/attempt-evidence.test.ts +13 -11
  231. package/src/results/attempt-evidence.ts +20 -13
  232. package/src/results/attempt-source.ts +6 -3
  233. package/src/results/copy.ts +150 -60
  234. package/src/results/host-equivalence.test.ts +34 -20
  235. package/src/results/index.ts +12 -4
  236. package/src/results/locator.test.ts +1 -22
  237. package/src/results/open.ts +15 -5
  238. package/src/results/publish.ts +149 -0
  239. package/src/results/results.test.ts +89 -54
  240. package/src/results/select.ts +104 -34
  241. package/src/results/truncate.ts +90 -0
  242. package/src/results/types.ts +43 -14
  243. package/src/results/writer.ts +31 -13
  244. package/src/runner/attempt.test.ts +138 -7
  245. package/src/runner/attempt.ts +603 -104
  246. package/src/runner/discover.test.ts +47 -0
  247. package/src/runner/discover.ts +36 -2
  248. package/src/runner/eval-source.test.ts +1 -27
  249. package/src/runner/feedback/agent.test.ts +504 -0
  250. package/src/runner/feedback/agent.ts +409 -0
  251. package/src/runner/feedback/ci.test.ts +562 -0
  252. package/src/runner/feedback/ci.ts +401 -0
  253. package/src/runner/feedback/coordinator.test.ts +317 -0
  254. package/src/runner/feedback/coordinator.ts +397 -0
  255. package/src/runner/feedback/failure.ts +40 -0
  256. package/src/runner/feedback/human.test.ts +616 -0
  257. package/src/runner/feedback/human.ts +535 -0
  258. package/src/runner/feedback/index.ts +66 -0
  259. package/src/runner/feedback/io.ts +78 -0
  260. package/src/runner/feedback/profile.test.ts +50 -0
  261. package/src/runner/feedback/profile.ts +58 -0
  262. package/src/runner/feedback/reducer.test.ts +395 -0
  263. package/src/runner/feedback/reducer.ts +260 -0
  264. package/src/runner/feedback/renderer.ts +82 -0
  265. package/src/runner/feedback/sink.ts +203 -0
  266. package/src/runner/feedback/testing.ts +106 -0
  267. package/src/runner/ledger.test.ts +230 -0
  268. package/src/runner/ledger.ts +329 -0
  269. package/src/runner/report.test.ts +128 -3
  270. package/src/runner/report.ts +33 -9
  271. package/src/runner/reporters/artifacts.ts +8 -2
  272. package/src/runner/reporters/braintrust.test.ts +8 -7
  273. package/src/runner/reporters/braintrust.ts +9 -2
  274. package/src/runner/reporters/index.ts +2 -2
  275. package/src/runner/reporters/json.test.ts +162 -0
  276. package/src/runner/reporters/json.ts +35 -8
  277. package/src/runner/reporters/shared.ts +1 -5
  278. package/src/runner/run.test.ts +760 -3
  279. package/src/runner/run.ts +243 -37
  280. package/src/runner/sandbox-prep.ts +3 -42
  281. package/src/runner/timing.ts +158 -0
  282. package/src/runner/types.ts +518 -22
  283. package/src/sandbox/checkpoint.test.ts +55 -0
  284. package/src/sandbox/checkpoint.ts +29 -8
  285. package/src/sandbox/cli-commands.ts +407 -0
  286. package/src/sandbox/docker.ts +115 -16
  287. package/src/sandbox/e2b-agent-template.test.ts +56 -0
  288. package/src/sandbox/e2b-agent-template.ts +94 -0
  289. package/src/sandbox/e2b.ts +74 -9
  290. package/src/sandbox/errors.ts +111 -4
  291. package/src/sandbox/index.ts +2 -0
  292. package/src/sandbox/io-retry.test.ts +58 -0
  293. package/src/sandbox/io-retry.ts +45 -0
  294. package/src/sandbox/keep-registry.test.ts +86 -0
  295. package/src/sandbox/keep-registry.ts +142 -0
  296. package/src/sandbox/keep.ts +178 -0
  297. package/src/sandbox/paths.test.ts +1 -0
  298. package/src/sandbox/paths.ts +19 -8
  299. package/src/sandbox/registry.ts +20 -3
  300. package/src/sandbox/resolve.ts +76 -11
  301. package/src/sandbox/retry.test.ts +70 -0
  302. package/src/sandbox/retry.ts +46 -4
  303. package/src/sandbox/types.ts +44 -6
  304. package/src/sandbox/vercel.ts +43 -20
  305. package/src/scoring/collector.ts +60 -17
  306. package/src/scoring/coverage.ts +95 -0
  307. package/src/scoring/diff.ts +81 -0
  308. package/src/scoring/display.test.ts +121 -0
  309. package/src/scoring/display.ts +133 -0
  310. package/src/scoring/evidence.test.ts +189 -0
  311. package/src/scoring/judge.test.ts +142 -0
  312. package/src/scoring/judge.ts +15 -18
  313. package/src/scoring/scoped.ts +217 -50
  314. package/src/scoring/types.ts +117 -20
  315. package/src/scoring/verdict.ts +16 -4
  316. package/src/shared/aggregate.ts +8 -6
  317. package/src/shared/types.ts +31 -0
  318. package/src/show/compose.ts +50 -67
  319. package/src/show/index.ts +127 -56
  320. package/src/show/render.ts +662 -131
  321. package/src/show/report-host.test.ts +188 -0
  322. package/src/show/report-host.ts +375 -0
  323. package/src/show/show.test.ts +320 -54
  324. package/src/tty-line.ts +8 -26
  325. package/src/util.test.ts +1 -0
  326. package/src/util.ts +41 -0
  327. package/src/view/app/App.test.tsx +69 -0
  328. package/src/view/app/App.tsx +144 -48
  329. package/src/view/app/components/AttemptModal.tsx +423 -11
  330. package/src/view/app/components/CodeView.tsx +41 -14
  331. package/src/view/app/components/CopyControls.tsx +2 -2
  332. package/src/view/app/i18n.ts +37 -17
  333. package/src/view/app/lib/attempt-route.test.ts +1 -0
  334. package/src/view/app/lib/verdict.ts +7 -9
  335. package/src/view/app/main.tsx +13 -8
  336. package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
  337. package/src/view/app/types.ts +4 -1
  338. package/src/view/artifact-serving.test.ts +2 -1
  339. package/src/view/client-dist/app.css +1 -1
  340. package/src/view/client-dist/app.js +17 -17
  341. package/src/view/data.test.ts +10 -3
  342. package/src/view/data.ts +155 -49
  343. package/src/view/index.ts +56 -41
  344. package/src/view/server.ts +37 -15
  345. package/src/view/shared/types.ts +34 -5
  346. package/src/view/styles.css +227 -0
  347. package/src/view/view-report.test.ts +167 -62
  348. package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
  349. package/dist/report/built-ins/experiment-comparison.js +0 -13
  350. package/dist/report/built-ins/index.d.ts +0 -1
  351. package/dist/report/built-ins/index.js +0 -2
  352. package/dist/report/react/GroupSummary.d.ts +0 -8
  353. package/dist/report/react/GroupSummary.js +0 -8
  354. package/dist/report/react/RunOverview.d.ts +0 -8
  355. package/dist/report/react/RunOverview.js +0 -12
  356. package/docs-site/zh/example/ai-agent-application.mdx +0 -152
  357. package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
  358. package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
  359. package/docs-site/zh/example/showcase.mdx +0 -39
  360. package/docs-site/zh/guides/publish-report.mdx +0 -91
  361. package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
  362. package/src/report/built-in-user-parity.test.tsx +0 -640
  363. package/src/report/built-ins/experiment-comparison.tsx +0 -19
  364. package/src/report/built-ins/index.ts +0 -2
  365. package/src/report/react/GroupSummary.tsx +0 -66
  366. package/src/report/react/RunOverview.tsx +0 -109
  367. package/src/runner/reporters/console.ts +0 -70
  368. package/src/runner/reporters/live.test.ts +0 -56
  369. package/src/runner/reporters/live.ts +0 -247
  370. package/src/runner/reporters/quiet.test.ts +0 -66
  371. package/src/runner/reporters/quiet.ts +0 -49
  372. package/src/runner/reporters/table.ts +0 -277
  373. /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
  374. /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
  375. /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
  376. /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
  377. /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
  378. /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
@@ -1,11 +1,12 @@
1
- // niceeval show 终端宿主的测试(行为规范:docs-site/zh/guides/viewing-results.mdx;
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // niceeval show 终端宿主的测试(行为规范:docs-site/zh/how-to/viewing-results.mdx;
2
3
  // 组合语义:docs/feature/reports/architecture.md「Selection 是计算入口」)。覆盖:
3
4
  // - 榜单合成口径:每 experiment × eval 取最新判定,局部重跑从更早快照补齐,头部标注合成自几个快照;
4
5
  // - 前缀过滤收窄 Selection,覆盖警告分母 = 已知并集 ∩ 范围;
5
6
  // - --history 时间轴只列真实执行,resume 携带的复印件不占行;
6
7
  // - --report 装载(合法 / 非法默认导出 / 文件缺失)、位置前缀收窄注入 Selection、attemptCommand 下钻;
7
8
  // - 互斥:--history 与 --report;
8
- // - 单 eval 详情、`@<locator>` 精确定位与 --eval / --execution / --diff 证据切面的输出形态。
9
+ // - 单 eval 详情、`@<locator>` 精确定位与 --source / --execution / --diff 证据切面的输出形态。
9
10
  //
10
11
  // fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
11
12
  // 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API(避免与并行重写的
@@ -17,9 +18,9 @@ import { tmpdir } from "node:os";
17
18
  import { join } from "node:path";
18
19
  import { createHash } from "node:crypto";
19
20
  import { openResults } from "../results/index.ts";
20
- import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
21
+ import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type TimingNode, type TraceSpan, type Verdict } from "../types.ts";
21
22
  import { selectCurrentResults } from "../results/select.ts";
22
- import { evalHistory } from "./compose.ts";
23
+ import { attemptHistory } from "./compose.ts";
23
24
  import { runShow, type ShowFlags } from "./index.ts";
24
25
  import { stringWidth } from "../report/text/layout.ts";
25
26
 
@@ -52,7 +53,7 @@ type AttemptFixture = Pick<EvalResult, "id" | "verdict"> &
52
53
  Partial<
53
54
  Pick<
54
55
  EvalResult,
55
- "attempt" | "durationMs" | "assertions" | "estimatedCostUSD" | "usage" | "error" | "startedAt" | "artifactBase" | "hasEvents" | "hasTrace"
56
+ "attempt" | "durationMs" | "assertions" | "estimatedCostUSD" | "usage" | "error" | "diagnostics" | "startedAt" | "artifactBase" | "hasEvents" | "hasTrace" | "phases"
56
57
  >
57
58
  >;
58
59
 
@@ -112,7 +113,7 @@ interface Captured {
112
113
  async function show(root: string, patterns: string[], flags: ShowFlags = {}, width = 100): Promise<Captured> {
113
114
  let out = "";
114
115
  let err = "";
115
- const code = await runShow(root, patterns, { run: root, ...flags }, {
116
+ const code = await runShow(root, patterns, { results: root, ...flags }, {
116
117
  out: (s) => (out += s),
117
118
  err: (s) => (err += s),
118
119
  width,
@@ -135,7 +136,7 @@ async function seedComposedRoot(): Promise<string> {
135
136
  name: 'fileChanged("src/components/Button.tsx")',
136
137
  severity: "gate",
137
138
  score: 0,
138
- passed: false,
139
+ outcome: "failed" as const,
139
140
  detail: "file was not modified",
140
141
  },
141
142
  ],
@@ -151,12 +152,12 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
151
152
  const root = await seedComposedRoot();
152
153
  const { out, code } = await show(root, []);
153
154
  expect(code).toBe(0);
154
- expect(out).toContain("No data to plot Cost × Pass rate");
155
+ expect(out).toContain("No data to plot Cost × End-to-end pass rate");
155
156
  expect(out).toContain("1 point missing data");
156
157
  expect(out).not.toContain("needed to compare");
157
158
  expect(out).not.toContain("COMPARISON");
158
- expect(out).toMatch(/compare\/bub\s+default\s+bub\s+1s\s+50%/);
159
- expect(out).toContain("1 passed / 1 failed");
159
+ expect(out).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
160
+ expect(out).toMatch(/1 passed[\s\S]*?\/ 1\s+failed/);
160
161
  expect(out).toMatch(/✗ failed\s+fixtures\/button[\s\S]*└─ @1[0-9a-z]{7}[\s\S]*fileChanged/);
161
162
  expect(out).toMatch(/✓ passed\s+weather\/brooklyn[\s\S]*└─ @1[0-9a-z]{7}/);
162
163
  expect(out).not.toMatch(/\[[EXD⏱,]+\]/);
@@ -168,9 +169,9 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
168
169
  try {
169
170
  const { out, code } = await show(root, []);
170
171
  expect(code).toBe(0);
171
- expect(out).toContain("预估成本 × 成功率 没有可绘制的数据");
172
+ expect(out).toContain("成本 × 端到端成功率 没有可绘制的数据");
172
173
  expect(out).toContain("1 通过 / 1 失败");
173
- expect(out).toContain("compare/bub");
174
+ expect(out).toMatch(/\bbub\s+默认\s+bub\s+1s\s+50%/);
174
175
  } finally {
175
176
  process.env.NICEEVAL_LANG = "en";
176
177
  }
@@ -198,10 +199,11 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
198
199
  ]);
199
200
  const { out, code } = await show(root, []);
200
201
  expect(code).toBe(0);
202
+ // 成本轴(better: lower)反向,「更好」恒指向右上;两轴都声明 better → 提示在场
201
203
  expect(out).toContain("better → upper right");
202
204
  expect(out).toContain("A compare/a B compare/b");
203
- expect(out).toMatch(/compare\/b\s+large\s+claude\s+1s\s+100%/);
204
- expect(out).toMatch(/compare\/a\s+mini\s+codex\s+1s\s+50%/);
205
+ expect(out).toMatch(/\bb\s+large\s+claude\s+1s\s+100%/);
206
+ expect(out).toMatch(/\ba\s+mini\s+codex\s+1s\s+50%/);
205
207
  expect(out).toMatch(/✓ passed\s+q1[\s\S]*└─ @1[0-9a-z]{7}/);
206
208
  expect(out).toMatch(/✗ failed\s+q2[\s\S]*└─ @1[0-9a-z]{7}/);
207
209
  expect(out).not.toContain("needed to compare");
@@ -280,15 +282,15 @@ describe("单 eval 详情", () => {
280
282
  name: 'calledTool("get_weather")',
281
283
  severity: "gate",
282
284
  score: 0,
283
- passed: false,
285
+ outcome: "failed" as const,
284
286
  detail: "tool was never called",
285
287
  },
286
- { name: "succeeded()", severity: "gate", score: 1, passed: true },
288
+ { name: "succeeded()", severity: "gate", score: 1, outcome: "passed" as const },
287
289
  {
288
290
  name: 'judge("回答基于实时数据")',
289
291
  severity: "soft",
290
292
  score: 0.2,
291
- passed: false,
293
+ outcome: "failed" as const,
292
294
  detail: "reply invents a temperature without any tool call",
293
295
  },
294
296
  ],
@@ -311,7 +313,7 @@ describe("单 eval 详情", () => {
311
313
  // 每 experiment 一行都带紧凑索引(locator + 失败原因),详情块再补一条精确的 attempt locator
312
314
  expect(out).toMatch(/✗ failed\s+2 attempts.*@1[0-9a-z]{7}.*gate calledTool/);
313
315
  expect(out).toMatch(/attempt locator: @1[0-9a-z]{7}/);
314
- expect(out).toMatch(/next: niceeval show @1[0-9a-z]{7} \[--eval\|--execution\|--diff\]/);
316
+ expect(out).toMatch(/next: niceeval show @1[0-9a-z]{7} \[--source\|--execution\|--diff\]/);
315
317
  });
316
318
  });
317
319
 
@@ -341,68 +343,94 @@ describe("--history 时间轴", () => {
341
343
  res("weather/brooklyn", "failed", {
342
344
  attempt: 1,
343
345
  estimatedCostUSD: 0.04,
344
- assertions: [{ name: 'calledTool("get_weather")', severity: "gate", score: 0, passed: false }],
346
+ assertions: [{ name: 'calledTool("get_weather")', severity: "gate", score: 0, outcome: "failed" as const }],
345
347
  }),
346
348
  ],
347
349
  );
348
350
  return root;
349
351
  }
350
352
 
351
- it("时间轴只列真实执行:复印件不占行,新失败带断言", async () => {
353
+ it("逐 attempt 分节:复印件按身份键去重不占行,startedAt 升序,行带摘要 / 成本 / locator", async () => {
352
354
  const root = await seedHistoryRoot();
353
355
  const results = await openResults(root);
354
356
  const exp = results.experiments.find((e) => e.id === "compare/bub")!;
355
- const rows = evalHistory(exp, "weather/brooklyn");
356
- // 快照2 里复印件被识别,真实执行只有:快照1 passed + 快照2failed(新 attempt)
357
+ const rows = attemptHistory(exp, "weather/brooklyn");
358
+ // 快照2 里复印件被识别(与快照1 的真实执行同身份键),历次 attempt = 快照1passed +
359
+ // 快照2 的 failed(新 attempt);startedAt 升序,旧的在前。
357
360
  expect(rows).toHaveLength(2);
358
- expect(rows[0]).toMatchObject({ verdict: "failed", attempts: 1, costUSD: 0.04 });
359
- expect(rows[0].failedAssertion).toBe('gate calledTool("get_weather")');
360
- expect(rows[1]).toMatchObject({ verdict: "passed", attempts: 1, costUSD: 0.03 });
361
+ expect(rows[0]).toMatchObject({ verdict: "passed", costUSD: 0.03 });
362
+ expect(rows[0].summary).toBeUndefined();
363
+ expect(rows[1]).toMatchObject({ verdict: "failed", costUSD: 0.04 });
364
+ expect(rows[1].summary).toContain('calledTool("get_weather")');
365
+ expect(rows[1].locator).toMatch(/^@/);
361
366
 
362
367
  const { out, code } = await show(root, ["weather/brooklyn"], { history: true });
363
368
  expect(code).toBe(0);
364
- expect(out).toContain("compare/bub · 2 runs · passed 1/2");
365
- expect(out).toContain("2026-07-09T10-00");
366
- expect(out).toContain("2026-07-07T09-00");
367
- // 复印件那份 passed 判定只出现一行(快照1),不在快照2 再占一行
369
+ // 分节头:experimentId + evalId 一起出现,计数按 attempt。
370
+ expect(out).toContain("weather/brooklyn · compare/bub · 2 attempts · passed 1/2");
371
+ // 复印件那份 passed 判定只出现一行,不在快照2 再占一行;升序:passed 行在 failed 行之前。
368
372
  expect(out.match(/✓ passed/g)).toHaveLength(1);
373
+ expect(out.indexOf("✓ passed")).toBeLessThan(out.indexOf("✗ failed"));
374
+ // 每行携带可复制的 locator。
375
+ expect(out).toMatch(/@\w+/);
369
376
  });
370
377
 
371
- it("裸 --history:每个 experiment per-run 通过率序列", async () => {
378
+ it("裸 --history:对 Scope 中每个 experimentId + evalId 分节,不再输出快照级序列", async () => {
372
379
  const root = await seedHistoryRoot();
373
380
  const { out, code } = await show(root, [], { history: true });
374
381
  expect(code).toBe(0);
375
- expect(out).toContain("compare/bub · 2 runs");
376
- // 每个快照一行(快照2 折叠含携带的 passed 复印件,任一轮通过 → passed)
377
- expect(out.match(/1\/1 passed/g)).toHaveLength(2);
382
+ expect(out).toContain("weather/brooklyn · compare/bub · 2 attempts · passed 1/2");
383
+ // 不是「每个快照一行」的通过率序列。
384
+ expect(out).not.toContain("1/1 passed");
385
+ });
386
+
387
+ it("--history 与 --page 组合是用法矛盾:直说", async () => {
388
+ const root = await seedHistoryRoot();
389
+ const { err, code } = await show(root, [], { history: true, page: "report" });
390
+ expect(code).toBe(1);
391
+ expect(err).toContain("--page");
378
392
  });
379
393
  });
380
394
 
381
395
  // ───────────────────────── --report 装载与组合语义 ─────────────────────────
382
396
 
383
397
  describe("--report 装载", () => {
384
- /** 不经 niceeval 包也能造出合法报告:判别与双面组件都锚在 Symbol.for 上。 */
398
+ /**
399
+ * 不经 niceeval 包也能造出合法报告:判别、组合组件与双面组件都锚在 Symbol.for 上
400
+ * (docs/feature/reports/library/shell.md「defineReport 产物」、layout.md「自定义组件」)。
401
+ * Custom 是双面组件,读 props 直接渲染;Wrapper 是组合组件,用 ComposeContext 的 scope
402
+ * 算出 evals / locator 再装配进 Custom —— 对应 defineReport(<Wrapper />) 的展开形态。
403
+ */
385
404
  async function writeReportFile(dir: string): Promise<string> {
386
405
  const path = join(dir, "report.mjs");
387
406
  await writeFile(
388
407
  path,
389
408
  [
390
409
  'const FACES = Symbol.for("niceeval.report.faces");',
410
+ 'const COMPOSE = Symbol.for("niceeval.report.compose");',
411
+ 'const DEFINITION = Symbol.for("niceeval.report.definition");',
391
412
  "const Custom = () => null;",
392
413
  "Custom[FACES] = {",
393
414
  " web: () => null,",
394
415
  " text: (props, ctx) => `CUSTOM ${props.evals} · drill ${ctx.attemptCommand(props.locator)}`,",
395
416
  "};",
396
- "export default {",
397
- ' [Symbol.for("niceeval.report.definition")]: true,',
398
- " build: (ctx) => ({",
399
- " type: Custom,",
400
- " props: {",
401
- " evals: ctx.selection.snapshots.flatMap((s) => s.evals.map((e) => e.id)).sort().join(\",\"),",
402
- " locator: ctx.selection.snapshots[0].evals[0].attempts[0].locator,",
403
- " },",
404
- " }),",
417
+ "const Wrapper = () => null;",
418
+ "Wrapper[COMPOSE] = (props, ctx) => ({",
419
+ " type: Custom,",
420
+ " props: {",
421
+ " evals: ctx.scope.snapshots.flatMap((s) => s.evals.map((e) => e.id)).sort().join(\",\"),",
422
+ " locator: ctx.scope.snapshots[0].evals[0].attempts[0].locator,",
423
+ " },",
424
+ "});",
425
+ "const definition = {",
426
+ ' kind: "report",',
427
+ " links: [],",
428
+ " scripts: [],",
429
+ " styles: [],",
430
+ ' pages: [{ id: "report", title: "Report", content: { type: Wrapper, props: {} } }],',
405
431
  "};",
432
+ "Object.defineProperty(definition, DEFINITION, { value: true });",
433
+ "export default definition;",
406
434
  "",
407
435
  ].join("\n"),
408
436
  "utf-8",
@@ -472,6 +500,29 @@ describe("--report 装载", () => {
472
500
  expect(out).not.toContain("CUSTOM");
473
501
  expect(out).toContain("no diff recorded"); // fixture 没有 diff artifact:如实说缺
474
502
  });
503
+
504
+ it("--page 命中单页定义的唯一页 id `report`:与直接渲染等价", async () => {
505
+ const root = await seedComposedRoot();
506
+ const report = await writeReportFile(root);
507
+ const bare = await show(root, [], { report });
508
+ const paged = await show(root, [], { report, page: "report" });
509
+ expect(paged.code).toBe(0);
510
+ expect(paged.out).toBe(bare.out);
511
+ });
512
+
513
+ it("--page 未命中:按用法错误非零退出并列出可用页 id(内建报告同样成立)", async () => {
514
+ const root = await seedComposedRoot();
515
+ const report = await writeReportFile(root);
516
+ const miss = await show(root, [], { report, page: "typo" });
517
+ expect(miss.code).toBe(1);
518
+ expect(miss.err).toContain(`page "typo" not found in ${report}`);
519
+ expect(miss.err).toContain("Available pages: report");
520
+
521
+ const builtin = await show(root, [], { page: "typo" });
522
+ expect(builtin.code).toBe(1);
523
+ expect(builtin.err).toContain('page "typo" not found in the built-in report');
524
+ expect(builtin.err).toContain("Available pages: report");
525
+ });
475
526
  });
476
527
 
477
528
  // ───────────────────────── 证据切面:--execution ─────────────────────────
@@ -540,18 +591,18 @@ describe("--execution", () => {
540
591
  });
541
592
  });
542
593
 
543
- // ───────────────────────── 证据切面:--eval ─────────────────────────
594
+ // ───────────────────────── 证据切面:--source ─────────────────────────
544
595
 
545
- describe("--eval", () => {
596
+ describe("--source", () => {
546
597
  it("evalSource === null 时如实说源码未捕获", async () => {
547
598
  const root = await makeRoot();
548
599
  await writeSnapshot(
549
600
  root,
550
601
  "2026-07-08T10-00-00-000Z",
551
602
  { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" },
552
- [res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, passed: false }] })],
603
+ [res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const }] })],
553
604
  );
554
- const { out, code } = await show(root, ["weather/brooklyn"], { eval: true });
605
+ const { out, code } = await show(root, ["weather/brooklyn"], { source: true });
555
606
  expect(code).toBe(0);
556
607
  expect(out).toContain("eval source unavailable for this attempt");
557
608
  });
@@ -569,11 +620,11 @@ describe("--eval", () => {
569
620
  name: 'calledTool("get_weather")',
570
621
  severity: "gate",
571
622
  score: 0,
572
- passed: false,
623
+ outcome: "failed" as const,
573
624
  detail: "tool was never called",
574
625
  loc: { file: "evals/weather/brooklyn.eval.ts", line: 2 },
575
626
  },
576
- { name: "unlocated()", severity: "soft", score: 0.5, passed: false, detail: "no loc on this one" },
627
+ { name: "unlocated()", severity: "soft", score: 0.5, outcome: "failed" as const, detail: "no loc on this one" },
577
628
  ],
578
629
  }),
579
630
  ],
@@ -585,7 +636,7 @@ describe("--eval", () => {
585
636
  await mkdir(join(dir, "sources"), { recursive: true });
586
637
  await writeFile(join(dir, "sources", `${sha}.json`), JSON.stringify({ content }), "utf-8");
587
638
 
588
- const { out, code } = await show(root, ["weather/brooklyn"], { eval: true });
639
+ const { out, code } = await show(root, ["weather/brooklyn"], { source: true });
589
640
  expect(code).toBe(0);
590
641
  expect(out).toContain(`eval source: evals/weather/brooklyn.eval.ts · sha256:${sha.slice(0, 8)}`);
591
642
  // 源码行的缩进是语义的一部分,渲染必须原样保留(不按词重排折叠掉这两个空格)。
@@ -599,7 +650,7 @@ describe("--eval", () => {
599
650
  it("源码行标注保留 group 与值断言的 expected/received", async () => {
600
651
  const root = await makeRoot();
601
652
  const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
602
- res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, passed: false, evidence: "1", group: "Issue 15193", loc: { file: "evals/manager.eval.ts", line: 1 } }] }),
653
+ res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, outcome: "failed" as const, expected: "4", received: "1", groupPath: ["Issue 15193"], loc: { file: "evals/manager.eval.ts", line: 1 } }] }),
603
654
  ]);
604
655
  const attemptDir = join(dir, "manager/a0");
605
656
  const content = "t.check(actual, equals(expected));\n";
@@ -607,7 +658,7 @@ describe("--eval", () => {
607
658
  await writeFile(join(attemptDir, "sources.json"), JSON.stringify([{ path: "evals/manager.eval.ts", sha256: sha }]), "utf-8");
608
659
  await mkdir(join(dir, "sources"), { recursive: true });
609
660
  await writeFile(join(dir, "sources", `${sha}.json`), JSON.stringify({ content }), "utf-8");
610
- const { out } = await show(root, ["manager"], { eval: true });
661
+ const { out } = await show(root, ["manager"], { source: true });
611
662
  expect(out).toContain("gate · Issue 15193 · equals(4) · expected 4 · received 1");
612
663
  });
613
664
  });
@@ -618,7 +669,7 @@ describe("show @<locator>", () => {
618
669
  it("默认页直接解释失败断言的 group、期望值、实际值和源码位置", async () => {
619
670
  const root = await makeRoot();
620
671
  await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
621
- res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, passed: false, evidence: "1", group: "Issue 15193", loc: { file: "evals/manager.eval.ts", line: 40, column: 11 } }] }),
672
+ res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, outcome: "failed" as const, expected: "4", received: "1", groupPath: ["Issue 15193"], loc: { file: "evals/manager.eval.ts", line: 40, column: 11 } }] }),
622
673
  ]);
623
674
  const results = await openResults(root);
624
675
  const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
@@ -649,6 +700,41 @@ describe("show @<locator>", () => {
649
700
  expect(out).not.toContain("available:");
650
701
  });
651
702
 
703
+ it("errored attempt 首页展开结构化 error(phase/code/message/cause)与 diagnostics", async () => {
704
+ const root = await makeRoot();
705
+ await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/claude-e2b", startedAt: "2026-07-08T10:00:00.000Z" }, [
706
+ res("agent-029", "errored", {
707
+ error: {
708
+ code: "sandbox-rate-limit",
709
+ message: "E2B sandbox allocation failed after 5 attempts",
710
+ phase: "sandbox.create",
711
+ cause: { name: "RateLimitError", message: "too many concurrent sandboxes" },
712
+ },
713
+ diagnostics: [
714
+ { code: "teardown-failed", level: "warning", message: "container stop timed out", phase: "sandbox.teardown" },
715
+ ],
716
+ }),
717
+ ]);
718
+ const results = await openResults(root);
719
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
720
+
721
+ const { out, code } = await show(root, [locator]);
722
+ expect(code).toBe(0);
723
+ expect(out).toContain(`${locator} · agent-029 · compare/claude-e2b · errored`);
724
+ // 结构化 error 块:phase 直接展示闭集点分名,code/message/cause 各一行(见 docs/feature/reports/show.md)
725
+ expect(out).toContain("error:");
726
+ expect(out).toContain("phase: sandbox.create");
727
+ expect(out).toContain("code: sandbox-rate-limit");
728
+ expect(out).toContain("message: E2B sandbox allocation failed after 5 attempts");
729
+ expect(out).toContain("cause: RateLimitError · too many concurrent sandboxes");
730
+ // attempt 级 diagnostics 块(与 verdict 独立)
731
+ expect(out).toContain("diagnostics:");
732
+ expect(out).toContain("warning · sandbox.teardown · teardown-failed");
733
+ expect(out).toContain("container stop timed out");
734
+ // errored 且无断言:不打印空的 assertions 汇总行
735
+ expect(out).not.toContain("assertions:");
736
+ });
737
+
652
738
  it("available 只逐行列出实际存在的证据命令,不打印能力字母或合并式 next", async () => {
653
739
  const root = await makeRoot();
654
740
  const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
@@ -666,7 +752,7 @@ describe("show @<locator>", () => {
666
752
 
667
753
  const { out, code } = await show(root, [locator]);
668
754
  expect(code).toBe(0);
669
- expect(out).toContain(`available:\n niceeval show ${locator} --eval\n niceeval show ${locator} --execution`);
755
+ expect(out).toContain(`available:\n niceeval show ${locator} --source\n niceeval show ${locator} --execution`);
670
756
  expect(out).not.toContain(`niceeval show ${locator} --diff`);
671
757
  expect(out).not.toContain("evidence:");
672
758
  expect(out).not.toContain("next:");
@@ -688,6 +774,186 @@ describe("show @<locator>", () => {
688
774
  expect(out).not.toContain("available:");
689
775
  });
690
776
 
777
+ it("单独 --timing 进入完整时间树,不回落到 attempt 首页", async () => {
778
+ const root = await makeRoot();
779
+ await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
780
+ res("weather/brooklyn", "passed", {
781
+ durationMs: 50_000,
782
+ phases: [
783
+ { name: "eval.run", durationMs: 42_000 },
784
+ { name: "sandbox.stop", durationMs: 800 },
785
+ ],
786
+ }),
787
+ ]);
788
+ const results = await openResults(root);
789
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
790
+
791
+ const { out, code } = await show(root, [locator], { timing: true });
792
+ expect(code).toBe(0);
793
+ expect(out).toContain("total 50.0s");
794
+ expect(out).toContain("eval.run 42.0s");
795
+ expect(out).toContain("teardown (not counted in total):");
796
+ expect(out).toContain("sandbox.stop 800ms");
797
+ expect(out).not.toContain("assertions:");
798
+ expect(out).not.toContain("available:");
799
+ });
800
+
801
+ it("--timing 在 80 个 detail node 内与 full 等价,81 个开始原位省略", async () => {
802
+ const root = await makeRoot();
803
+ const children: TimingNode[] = Array.from({ length: 81 }, (_, index) => ({
804
+ id: `command-${String(index).padStart(3, "0")}`,
805
+ kind: "command",
806
+ label: "tool",
807
+ startOffsetMs: index * 10,
808
+ durationMs: 81 - index,
809
+ command: { display: `tool --item=${index}` },
810
+ }));
811
+ await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
812
+ res("weather/brooklyn", "passed", {
813
+ durationMs: 50_000,
814
+ phases: [
815
+ { name: "sandbox.create", durationMs: 1_000 },
816
+ { name: "eval.run", durationMs: 42_000, children },
817
+ { name: "sandbox.stop", durationMs: 800 },
818
+ ],
819
+ }),
820
+ ]);
821
+ const results = await openResults(root);
822
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
823
+
824
+ const bounded = await show(root, [locator], { timing: "summary" });
825
+ const full = await show(root, [locator], { timing: "full" });
826
+ expect(bounded.out).toContain("sandbox.create");
827
+ expect(bounded.out).toContain("eval.run");
828
+ expect(bounded.out).toContain("sandbox.stop");
829
+ expect(bounded.out).toMatch(/… 1 nodes omitted/);
830
+ expect(bounded.out).toContain(`niceeval show ${locator} --timing=full`);
831
+ expect(full.out).not.toContain("nodes omitted");
832
+ expect(full.out).toContain("tool --item=0");
833
+ expect(full.out).toContain("tool --item=80");
834
+
835
+ // 去掉第 81 个后,默认投影不再省略,detail 行与 full 完全一致。
836
+ children.pop();
837
+ const root80 = await makeRoot();
838
+ await writeSnapshot(root80, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
839
+ res("weather/brooklyn", "passed", {
840
+ durationMs: 50_000,
841
+ phases: [{ name: "eval.run", durationMs: 42_000, children }],
842
+ }),
843
+ ]);
844
+ const results80 = await openResults(root80);
845
+ const locator80 = results80.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
846
+ const bounded80 = await show(root80, [locator80], { timing: "summary" });
847
+ const full80 = await show(root80, [locator80], { timing: "full" });
848
+ expect(bounded80.out).toBe(full80.out);
849
+ });
850
+
851
+ it("默认投影保留失败路径/慢点/首尾,omission 报失败数且不虚构 children 合计", async () => {
852
+ const root = await makeRoot();
853
+ const children: TimingNode[] = Array.from({ length: 120 }, (_, index) => ({
854
+ id: `node-${String(index).padStart(3, "0")}`,
855
+ kind: "provider",
856
+ label: `provider-step-${index}`,
857
+ startOffsetMs: index * 100,
858
+ durationMs: index === 60 ? 99_000 : index + 1,
859
+ ...(index === 55 ? { failed: true as const } : {}),
860
+ }));
861
+ await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
862
+ res("weather/brooklyn", "errored", {
863
+ durationMs: 120_000,
864
+ error: { code: "provider-failed", message: "boom", phase: "sandbox.create" },
865
+ phases: [{ name: "sandbox.create", durationMs: 120_000, failed: true, children }],
866
+ }),
867
+ ]);
868
+ const results = await openResults(root);
869
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
870
+ const { out } = await show(root, [locator], { timing: "summary" });
871
+
872
+ expect(out).toContain("provider-step-0");
873
+ expect(out).toContain("provider-step-55");
874
+ expect(out).toContain("provider-step-60");
875
+ expect(out).toContain("provider-step-119");
876
+ expect(out).not.toContain("combined");
877
+ });
878
+
879
+ it("operation 使用 producer label;full 按 parentSpanId 展开唯一关联 OTel 树", async () => {
880
+ const root = await makeRoot();
881
+ const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
882
+ res("weather/brooklyn", "passed", {
883
+ durationMs: 50_000,
884
+ hasTrace: true,
885
+ phases: [{
886
+ name: "workspace.diff",
887
+ durationMs: 5_000,
888
+ children: [{
889
+ id: "operation-1",
890
+ kind: "operation",
891
+ label: "export workspace diff · 1 window · 3,302 files",
892
+ startOffsetMs: 100,
893
+ durationMs: 4_000,
894
+ children: [{
895
+ id: "turn-1",
896
+ kind: "turn",
897
+ label: "s1/t1",
898
+ startOffsetMs: 200,
899
+ durationMs: 3_000,
900
+ traceId: "trace-1",
901
+ }],
902
+ }],
903
+ }],
904
+ }),
905
+ ]);
906
+ const trace: TraceSpan[] = [
907
+ { traceId: "trace-1", spanId: "child", parentSpanId: "root", name: "chat", kind: "model", startMs: 1_100, endMs: 2_000 },
908
+ { traceId: "trace-1", spanId: "root", name: "codex.exec", kind: "agent", startMs: 1_000, endMs: 3_000 },
909
+ ];
910
+ await writeFile(join(dir, "weather/brooklyn/a0/trace.json"), JSON.stringify(trace), "utf-8");
911
+ const results = await openResults(root);
912
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
913
+ const { out } = await show(root, [locator], { timing: "full" });
914
+
915
+ expect(out).toContain("operation · export workspace diff · 1 window · 3,302 files");
916
+ expect(out).toMatch(/agent · codex\.exec[\s\S]*model · chat/);
917
+ expect(out).not.toContain("git show ×");
918
+ });
919
+
920
+ it("attempt 首页 timing 只列大头,但保留变慢的 telemetry 阶段", async () => {
921
+ const root = await makeRoot();
922
+ const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
923
+ res("weather/brooklyn", "passed", {
924
+ durationMs: 100_000,
925
+ hasEvents: true,
926
+ phases: [
927
+ { name: "sandbox.queue", durationMs: 0 },
928
+ { name: "sandbox.create", durationMs: 3_000 },
929
+ { name: "workspace.baseline", durationMs: 310 },
930
+ { name: "telemetry.configure", durationMs: 358 },
931
+ { name: "eval.run", durationMs: 45_000 },
932
+ { name: "telemetry.collect", durationMs: 35_300 },
933
+ { name: "sandbox.stop", durationMs: 8_500 },
934
+ ],
935
+ }),
936
+ ]);
937
+ await writeFile(
938
+ join(dir, "weather/brooklyn/a0/events.json"),
939
+ JSON.stringify([{ type: "message", role: "assistant", text: "done" }]),
940
+ "utf-8",
941
+ );
942
+ const results = await openResults(root);
943
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
944
+
945
+ const { out } = await show(root, [locator]);
946
+ const timingLine = out.split("\n").find((line) => line.startsWith("timing:"));
947
+ expect(out).toContain("1 AI message");
948
+ expect(timingLine).toContain("sandbox.queue 0ms");
949
+ expect(timingLine).toContain("sandbox.create 3.0s");
950
+ expect(timingLine).toContain("eval.run 45.0s");
951
+ expect(timingLine).toContain("telemetry.collect 35.3s");
952
+ expect(timingLine).toContain("teardown +8.5s");
953
+ expect(timingLine).not.toContain("workspace.baseline");
954
+ expect(timingLine).not.toContain("telemetry.configure");
955
+ });
956
+
691
957
  it("语法不对的 locator 报「not a valid attempt locator」,退出码 1,不崩", async () => {
692
958
  const root = await makeRoot();
693
959
  await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
package/src/tty-line.ts CHANGED
@@ -1,32 +1,14 @@
1
- // 独立诊断行的统一终端出口。
1
+ // 独立诊断行的 bootstrap 终端出口:只服务 run 启动前的 argv/config 错误,以及崩溃兜底/
2
+ // 尚无活跃 coordinator 时的兜底(见 src/runner/feedback/sink.ts 的 no-active-coordinator 分支)。
2
3
  //
3
- // live 表格(src/runner/reporters/live.ts)重画靠 \x1B[{drawnLines}A 回跳到上一帧起点,
4
- // 这个回跳量的前提是"stderr/stdout 这块屏幕在两帧之间只有它自己在写"。sandbox teardown
5
- // 失败、budget 不可执行、reporter 抛错这类跟 attempt 进度无关的独立诊断行,如果在 live
6
- // 表格激活期间直接 process.stderr.write / console.log,就会插进两次 draw() 之间——下一帧
7
- // 按旧的 drawnLines 回跳,已经够不到表格真正的起点,清行重画变成往下多写一份,且这个偏移
8
- // 被记进新的 drawnLines,此后每帧都在错位的基础上继续错位,越滚越多(见
9
- // memory/live-raw-stderr-write-desyncs-redraw.md)。
10
- //
11
- // 所以任何不经过 Reporter.progress() / onEvalComplete() 的独立诊断行,写之前都要先广播
12
- // beforeExternalTerminalWrite(),live.ts 借此机会把已画内容清掉、drawnLines 归零,下一次
13
- // draw() 就会把这行诊断消息之下当成全新起点重画,不会累积。
14
-
15
- const listeners = new Set<() => void>();
16
-
17
- /** live.ts 用来订阅"即将有一行独立诊断消息落地"。返回取消订阅函数。 */
18
- export function onBeforeExternalTerminalWrite(fn: () => void): () => void {
19
- listeners.add(fn);
20
- return () => listeners.delete(fn);
21
- }
22
-
23
- /** 任何要绕开 Reporter 直接往终端打一行独立诊断消息的地方,写之前都先调用这个。 */
24
- export function beforeExternalTerminalWrite(): void {
25
- for (const fn of listeners) fn();
26
- }
4
+ // run 激活之后的全部反馈(dashboard、durable event、heartbeat)
5
+ // src/runner/feedback/coordinator.ts 统一排序;它通过 sink.ts 的模块级 coordinator 栈直接接管
6
+ // reportXxx() 调用的目的地(显式路由,不是广播-订阅),所以这里不再需要"写之前先广播、订阅方
7
+ // 自己清屏"的机制——那套机制是为已删除的 live.ts ANSI 回跳重画设计的(
8
+ // memory/live-raw-stderr-write-desyncs-redraw.md),新 coordinator 的 clear → append → redraw
9
+ // 顺序由 sink.ts 的显式路由保证,不依赖本文件。
27
10
 
28
11
  /** process.stderr.write 的替代:文本需自带换行(沿用现有 i18n 字符串的约定)。 */
29
12
  export function writeStderrLine(text: string): void {
30
- beforeExternalTerminalWrite();
31
13
  process.stderr.write(text);
32
14
  }
package/src/util.test.ts CHANGED
@@ -1,3 +1,4 @@
1
+ // cases: docs/engineering/unit-tests/scoring/cases.md
1
2
  import { describe, expect, it } from "vitest";
2
3
  import { brief, formatThrown, upsertManagedBlock } from "./util.ts";
3
4