niceeval 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +15 -7
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
  118. package/dist/report/{tree.js → definition/tree.js} +10 -12
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +12 -2
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +15 -1
  154. package/dist/shared/aggregate.js +32 -1
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +77 -64
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +30 -15
  227. package/src/i18n/zh-CN.ts +30 -15
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +136 -112
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +58 -22
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +54 -3
  314. package/src/results/select.ts +30 -6
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +30 -1
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +58 -22
  356. package/src/show/render.ts +16 -240
  357. package/src/show/show.test.ts +128 -35
  358. package/src/view/app/App.test.tsx +8 -7
  359. package/src/view/app/App.tsx +135 -51
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +122 -89
  370. package/src/view/index.ts +3 -13
  371. package/src/view/server.ts +36 -6
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +18 -806
  376. package/src/view/view-report.test.ts +182 -34
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,10 +1,10 @@
1
1
  // show 的文本渲染:单 eval 详情、--history 时间轴、三个证据切面(transcript / trace / diff)。
2
- // 输出形态照 docs-site/zh/how-to/viewing-results.mdx 的示例块;长内容一律截断,
2
+ // 输出形态照 docs-site/zh/tutorials/viewing-results.mdx 的示例块;长内容一律截断,
3
3
  // 但截断永远如实标注剩余数量和原始 artifact 路径 —— 输出对上下文窗口友好,事实源留在盘上。
4
4
  // 全部纯函数(时间经 now 显式传入),证据数据由调用方 await 好了递进来。
5
5
 
6
6
  import { join, relative } from "node:path";
7
- import type { AssertionResult, DiffData, EvalResult, TimingNode, TraceSpan, Verdict } from "../types.ts";
7
+ import type { AssertionResult, DiffData, EvalResult, LocalizedText, TimingNode, TraceSpan, Verdict } from "../types.ts";
8
8
  import type { AttemptEvidence, AttemptHandle, Snapshot } from "../results/index.ts";
9
9
  import type { AnnotatedSourceLine, SendAnnotation } from "../results/index.ts";
10
10
  import { groupIncompatibleVersionSkips } from "../results/index.ts";
@@ -12,11 +12,12 @@ import type { SkippedDir } from "../results/index.ts";
12
12
  import type { ExecutionNode } from "../o11y/execution-tree.ts";
13
13
  import { foldEvalVerdict } from "../shared/verdict.ts";
14
14
  import { summaryText } from "../scoring/display.ts";
15
- import { attemptCostUSD } from "../report/metrics.ts";
16
- import { formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD } from "../report/format.ts";
17
- import { indentBlock, padDisplay, renderAlignedRows, wrapDisplay } from "../report/text/layout.ts";
15
+ import { attemptCostUSD } from "../report/model/metrics.ts";
16
+ import { formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD } from "../report/model/format.ts";
17
+ import { indentBlock, padDisplay, renderAlignedRows, wrapDisplay } from "../report/model/text-layout.ts";
18
18
  import type { AttemptHistoryRow } from "./compose.ts";
19
- import { localizeText, showCommand, type HostCommandContext, type HostReport } from "./report-host.ts";
19
+ import { localizeText, type HostCommandContext } from "../report/runtime/host.ts";
20
+ import { showCommand } from "./command.ts";
20
21
 
21
22
  const MISSING = "—";
22
23
 
@@ -113,7 +114,7 @@ export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHa
113
114
  /**
114
115
  * 详情块 / eval-id 前缀证据切面默认挑最新一次失败的 attempt;没有失败挑最新一次。
115
116
  * 精确选某一次 attempt 走 `@<locator>`(`resolveLocator`),不再有数字 `--attempt`——
116
- * --experiment 已在 Selection 合成时收窄。
117
+ * --exp 已在 Selection 合成时收窄。
117
118
  */
118
119
  export function pickDetailAttempt(attempts: AttemptHandle[]): AttemptHandle | undefined {
119
120
  if (attempts.length === 0) return undefined;
@@ -158,74 +159,6 @@ export function assertionLine(a: AssertionResult): string {
158
159
  return head;
159
160
  }
160
161
 
161
- /** 字段行:首行 `<label>: <值首行>`,值的其余行(如 CommandResult 的 output tail)缩进原样展开。 */
162
- function fieldLines(label: string, value: string): string[] {
163
- const [first, ...rest] = value.split("\n");
164
- return [` ${label}: ${first ?? ""}`, ...rest.map((line) => ` ${line}`)];
165
- }
166
-
167
- /** 每条的通用行组(见 docs/feature/scoring/library/display.md「通用渲染规则」):
168
- * 首行 `severity · <标题>`(有分组时标题是分组路径,随后 assertion: 行给检查方式),
169
- * 之后按有则显示的顺序列 expected / received / score / reason / source。 */
170
- function assertionRecordLines(a: AssertionResult): string[] {
171
- const group = groupTitle(a);
172
- const optional = a.optional ? "optional · " : "";
173
- const title = group ?? a.name;
174
- const lines: string[] = [];
175
- const scoreSuffix =
176
- a.outcome !== "unavailable" && (a.severity === "soft" || a.threshold !== undefined)
177
- ? ` ${scoreText(a.score)}${a.threshold !== undefined ? ` / ${scoreText(a.threshold)}` : ""}`
178
- : "";
179
- lines.push(` ${a.severity} · ${optional}${title}${scoreSuffix}`);
180
- const detailLine = a.detail ?? (group ? a.name : undefined);
181
- if (detailLine !== undefined && detailLine !== title) lines.push(` assertion: ${detailLine}`);
182
- if (a.outcome === "unavailable") {
183
- lines.push(` reason: ${a.reason}`);
184
- } else {
185
- if (a.expected !== undefined) lines.push(...fieldLines("expected", a.expected));
186
- if (a.received !== undefined) lines.push(...fieldLines("received", a.received));
187
- if (a.evidence !== undefined && a.evidence !== a.received) lines.push(...fieldLines("evidence", a.evidence));
188
- }
189
- if (a.loc) lines.push(` source: ${a.loc.file}:${a.loc.line}${a.loc.column ? `:${a.loc.column}` : ""}`);
190
- return lines;
191
- }
192
-
193
- /**
194
- * 默认 Attempt 页的首要诊断:按结果分节,只逐条列出需要看的(见 display.md)——
195
- * `failures:`(gate 失败,含 --strict 下改判的 soft)、`soft below threshold:`、
196
- * `scores:`(无阈值 judge 的纯打分)、`unavailable:`(证据评不了的,带 reason)。
197
- * 全部通过时这些节整体省略(返回 undefined),只留计数行。
198
- */
199
- export function failureDiagnostics(assertions: AssertionResult[], width: number, strict?: boolean): string | undefined {
200
- const failures = assertions.filter(
201
- (a) => a.outcome === "failed" && (a.severity === "gate" || strict),
202
- );
203
- const softBelow = assertions.filter(
204
- (a) => a.outcome === "failed" && a.severity === "soft" && !strict,
205
- );
206
- const scores = assertions.filter(
207
- (a) => a.outcome === "passed" && a.severity === "soft" && a.threshold === undefined && a.score < 1,
208
- );
209
- const unavailableOnes = assertions.filter((a) => a.outcome === "unavailable");
210
- const lines: string[] = [];
211
- const section = (title: string, items: AssertionResult[]) => {
212
- if (items.length === 0) return;
213
- if (lines.length > 0) lines.push("");
214
- lines.push(title);
215
- for (const a of items) lines.push(...assertionRecordLines(a));
216
- };
217
- section("failures:", failures);
218
- section("soft below threshold:", softBelow);
219
- section("scores:", scores);
220
- section("unavailable:", unavailableOnes);
221
- if (lines.length === 0) return undefined;
222
- return lines.flatMap((line) => {
223
- if (line === "") return [line];
224
- const indent = line.length - line.trimStart().length;
225
- return wrapDisplay(line.trimStart(), Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`);
226
- }).join("\n");
227
- }
228
-
229
162
  /** 详情块头部:`attempt 3 · compare/codex-gpt-5.4 · failed · 41s · 12.3k tokens · $0.04`。 */
230
163
  export function attemptHeader(attempt: AttemptHandle): string {
231
164
  const r = attempt.result;
@@ -242,12 +175,13 @@ export function attemptHeader(attempt: AttemptHandle): string {
242
175
  }
243
176
 
244
177
  // ───────────────────────── AttemptEvidence 共用 ─────────────────────────
245
- // evalSourceText / executionText / attemptOverviewText(--eval / --execution / 默认全景)
246
- // 与 evalDetailText 的紧凑索引列共用的小件:locator 头、失败原因、
247
- // 断言计票摘要。三个证据 renderer 与全景面都只消费同一份 AttemptEvidence,不各自读
248
- // artifact 或重新判定 capability(loadAttemptEvidence 已经算好)
178
+ // evalSourceText / executionText / timingText / diffText(--source / --execution / --timing / --diff
179
+ // 四个证据切面)与 evalDetailText 的紧凑索引列共用的小件:locator 头、失败原因、断言计票摘要。
180
+ // 四个证据 renderer 都只消费同一份 AttemptEvidence,不各自读 artifact 或重新判定 capability
181
+ // (loadAttemptEvidence 已经算好)。无证据 flag 的默认页改由 niceeval/report 的 attempt-input
182
+ // page 管线渲染(docs/feature/reports/show/attempt.md),不在这里。
249
183
 
250
- /** `--eval` / `--execution` / 全景块的头行:`@<locator> · <evalId> · <experimentId> · <verdict>`。 */
184
+ /** 证据切面的头行:`@<locator> · <evalId> · <experimentId> · <verdict>`。 */
251
185
  export function attemptEvidenceHeader(evidence: AttemptEvidence): string {
252
186
  return [evidence.locator, evidence.identity.evalId, evidence.identity.experimentId, evidence.result.verdict].join(" · ");
253
187
  }
@@ -427,7 +361,7 @@ export function attemptHistoryText(opts: {
427
361
  * 拼接这段(单页定义没有「其余页」段);`otherPages` 不含被渲染的那一页。
428
362
  */
429
363
  export function otherPagesText(opts: {
430
- otherPages: { id: string; title: HostReport["pages"][number]["title"] }[];
364
+ otherPages: { id: string; title: LocalizedText }[];
431
365
  command: HostCommandContext;
432
366
  locale: string;
433
367
  }): string {
@@ -619,7 +553,7 @@ function execBody(
619
553
  /**
620
554
  * action / subagent 节点渲染成两行(call + result)——节点模型把 call+result 合并成一个
621
555
  * ExecutionNode(按 callId),但分两行读更符合「先看调用、再看结果」的阅读顺序,
622
- * 与 docs-site/zh/how-to/agent-feedback-loop.mdx 的示例一致。
556
+ * 与 docs-site/zh/tutorials/agent-feedback-loop.mdx 的示例一致。
623
557
  */
624
558
  function executionNodeLines(node: ExecutionNode, originMs: number, timingAvailable: boolean, width: number): string[] {
625
559
  const time = node.kind !== "telemetry" && node.span ? relSeconds(node.span.startMs, originMs) : undefined;
@@ -753,10 +687,6 @@ export function executionText(
753
687
  return `${header}\n\n${lines.join("\n")}\n\n${tail.join("\n")}`;
754
688
  }
755
689
 
756
- // ───────────────────────── 默认全景:AttemptEvidence 紧凑总览 ─────────────────────────
757
-
758
- const MAX_OVERVIEW_DIFF_NAMES = 5;
759
-
760
690
  /** net 效果的单字母标记(A/M/D;none = 动过但净无变化,标 ±)。 */
761
691
  function netLetter(net: string): string {
762
692
  switch (net) {
@@ -771,15 +701,6 @@ function netLetter(net: string): string {
771
701
  }
772
702
  }
773
703
 
774
- function overviewDiffLine(diff: DiffData): string {
775
- const names = Object.entries(diff.files)
776
- .sort(([a], [b]) => a.localeCompare(b))
777
- .map(([p, f]) => `${netLetter(f.net)} ${p}`);
778
- const shown = names.slice(0, MAX_OVERVIEW_DIFF_NAMES);
779
- const more = names.length > shown.length ? ` · +${names.length - shown.length} more` : "";
780
- return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed by agent · ${shown.join(" · ")}${more}`;
781
- }
782
-
783
704
  /** 有界行 diff(公共前后缀修剪):对单区域编辑精确,复杂编辑给出上界近似。 */
784
705
  function lineDelta(before: string | undefined, after: string | undefined): { adds: number; dels: number } {
785
706
  const a = before === undefined ? [] : before.split("\n");
@@ -791,131 +712,6 @@ function lineDelta(before: string | undefined, after: string | undefined): { add
791
712
  return { adds: b.length - prefix - suffix, dels: a.length - prefix - suffix };
792
713
  }
793
714
 
794
- /**
795
- * `niceeval show @<locator>` 不带证据 flag 时的默认面:紧凑全景,只给摘要——Eval 断言计票、
796
- * 执行事件计数、可选 OTel 时间指示、工作区 diff 摘要,不复现 `--eval` 的完整源码、
797
- * `--execution` 的完整事件流或 `--diff` 的完整文件列表(那些内容各自的证据 flag 才给)。
798
- */
799
- /** lifecycle operation → 首页可读标签:点换空格,与 docs/feature/reports/show.md 的
800
- * `phase: sandbox provision` 字面一致(不引入第二套本地化标签,show 首页用英文原样呈现)。 */
801
- function operationWords(operation: string): string {
802
- return operation.replace(/\./g, " ");
803
- }
804
-
805
- /**
806
- * errored attempt 的结构化 `error:` 块(见 docs/feature/reports/show.md「errored attempt 的首页」)。
807
- * 先展开 phase(= operation 的可读形态)/ code / message / cause,stack 放在块后、保持原始换行。
808
- * 字段来自结构化 `AttemptError`;非 errored(`r.error === undefined`)返回 undefined。
809
- */
810
- function renderErrorBlock(r: EvalResult): string | undefined {
811
- const err = r.error;
812
- if (err === undefined) return undefined;
813
- const lines = ["error:", ` phase: ${err.phase}`, ` code: ${err.code}`, ` message: ${err.message}`];
814
- if (err.cause) {
815
- const c = err.cause;
816
- const causeText = c.name ? `${c.name} · ${c.message}` : c.message;
817
- lines.push(` cause: ${causeText}`);
818
- }
819
- // stack 放在 error 块之后、保持原始换行(见 docs);没有 stack(如 timeout / turn-failed)就不加空块。
820
- if (err.stack && err.stack.trim() !== "") return `${lines.join("\n")}\n\n${err.stack.replace(/\n+$/, "")}`;
821
- return lines.join("\n");
822
- }
823
-
824
- /**
825
- * attempt 级诊断块(见 docs/feature/reports/show.md「diagnostics」)。每条一行标头
826
- * `<level> · <operation> · <code>`,message 缩进在下一行;`count > 1` 时补 `(N occurrences)`。
827
- * 诊断的 level 与 verdict 无关 —— passed / failed / errored 都可能带一条 cleanup / teardown
828
- * warning。没有诊断返回 undefined。
829
- */
830
- function renderAttemptDiagnostics(r: EvalResult): string | undefined {
831
- if (!r.diagnostics || r.diagnostics.length === 0) return undefined;
832
- const lines = ["diagnostics:"];
833
- for (const d of r.diagnostics) {
834
- lines.push(` ${d.level} · ${d.phase} · ${d.code}`);
835
- const occ = d.count && d.count > 1 ? ` (${d.count} occurrences)` : "";
836
- lines.push(` ${d.message}${occ}`);
837
- }
838
- return lines.join("\n");
839
- }
840
-
841
- export function attemptOverviewText(
842
- evidence: AttemptEvidence,
843
- opts: { header: string; artifactPath?: string; width: number },
844
- ): string {
845
- const { header, artifactPath } = opts;
846
- const r = evidence.result;
847
-
848
- const metaParts = [`snapshot ${evidence.identity.snapshotStartedAt}`, `attempt ${evidence.identity.attempt + 1}`, formatDurationMs(r.durationMs)];
849
- if (r.usage) metaParts.push(`${formatMetricValue(r.usage.inputTokens + r.usage.outputTokens)} tokens`);
850
- const cost = attemptCostUSD(r);
851
- if (cost !== null) metaParts.push(formatUSD(cost));
852
-
853
- const blocks: string[] = [[header, metaParts.join(" · ")].join("\n")];
854
-
855
- // errored attempt 的首页不靠 trace 就要能解释基础设施错误(见 docs/feature/reports/show.md
856
- // 「errored attempt 的首页」):先展开结构化 error(phase/operation/code/message/cause + stack),
857
- // 断言块对没有断言的 errored attempt 省略(它在评分之前就挂了)。
858
- const errorBlock = renderErrorBlock(r);
859
- if (errorBlock) blocks.push(errorBlock);
860
-
861
- // errored 且没有任何断言时不打印空的 "assertions: 0 passed" —— 主因已经在 error 块里说清楚了。
862
- if (r.assertions.length > 0 || r.error === undefined) {
863
- blocks.push(
864
- [
865
- assertionSummaryLine(r.assertions),
866
- evidence.evalSource
867
- ? `eval source: ${evidence.evalSource.sourcePath} · sha256:${evidence.evalSource.sourceSha256.slice(0, 8)}…`
868
- : "eval source: unavailable (not captured for this attempt)",
869
- ].join("\n"),
870
- );
871
- }
872
-
873
- const diagnostics = failureDiagnostics(r.assertions, opts.width);
874
- if (diagnostics) blocks.push(diagnostics);
875
-
876
- // attempt 级诊断(teardown/cleanup 等,与 verdict 独立;passed/failed/errored 都可能有)。
877
- const attemptDiag = renderAttemptDiagnostics(r);
878
- if (attemptDiag) blocks.push(attemptDiag);
879
-
880
- if (evidence.execution) {
881
- const nodes = evidence.execution.nodes.filter((node) => node.kind !== "telemetry");
882
- const skillLoads = nodes.filter((n) => n.kind === "skill.loaded").length;
883
- const toolCalls = nodes.filter((n) => n.kind === "action").length;
884
- const aiMessages = nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
885
- const counted = (count: number, singular: string, plural = `${singular}s`) =>
886
- `${count} ${count === 1 ? singular : plural}`;
887
- const execLines = [
888
- `execution: ${counted(nodes.length, "event")} · ${counted(skillLoads, "skill load")} · ${counted(toolCalls, "tool call")} · ${counted(aiMessages, "AI message")}`,
889
- ];
890
- const timingLine = overviewTimingLine(r);
891
- if (timingLine) execLines.push(timingLine);
892
- blocks.push(execLines.join("\n"));
893
- } else {
894
- blocks.push("execution: unavailable (no events recorded for this attempt)");
895
- }
896
-
897
- if (evidence.diff && evidence.capabilities.diff) {
898
- blocks.push(overviewDiffLine(evidence.diff));
899
- } else if (evidence.diff) {
900
- blocks.push("changes: diff unavailable · this attempt did not produce workspace file changes");
901
- } else {
902
- blocks.push("changes: diff unavailable · no workspace diff was recorded for this attempt");
903
- }
904
-
905
- const tail: string[] = [];
906
- if (artifactPath) tail.push(`artifacts: ${artifactPath}/`);
907
- const available = [
908
- evidence.capabilities.source ? `niceeval show ${evidence.locator} --source` : undefined,
909
- evidence.capabilities.execution ? `niceeval show ${evidence.locator} --execution` : undefined,
910
- evidence.capabilities.timing ? `niceeval show ${evidence.locator} --timing` : undefined,
911
- evidence.capabilities.diff ? `niceeval show ${evidence.locator} --diff` : undefined,
912
- ].filter((command): command is string => command !== undefined);
913
- if (available.length > 0) tail.push(`available:\n${available.map((command) => ` ${command}`).join("\n")}`);
914
- if (tail.length > 0) blocks.push(tail.join("\n"));
915
-
916
- return blocks.join("\n\n");
917
- }
918
-
919
715
  // ───────────────────────── 证据切面:diff ─────────────────────────
920
716
 
921
717
  const MAX_DIFF_LINES = 200;
@@ -1013,26 +809,6 @@ function windowHunk(c: { status: string; before?: string; after?: string }): str
1013
809
 
1014
810
  const CLOSING_PHASE_NAMES = new Set(["eval.teardown", "agent.teardown", "sandbox.teardown", "sandbox.suspend", "sandbox.stop"]);
1015
811
 
1016
- // 首页只保留能回答「大头在哪」的阶段。分类账锚点与 telemetry bookkeeping 在很短时
1017
- // 留给 --timing 完整树;一旦自身变成慢点或失败,仍必须抬到首页,不能被静默藏掉。
1018
- const OVERVIEW_DETAIL_PHASE_NAMES = new Set(["workspace.baseline", "telemetry.configure", "telemetry.collect"]);
1019
- const OVERVIEW_DETAIL_THRESHOLD_MS = 1_000;
1020
-
1021
- /** phases 主链摘要行(首页 `timing:`):各主链阶段耗时点隔,收尾段合计以 `teardown +N` 单列。 */
1022
- export function overviewTimingLine(r: EvalResult): string | undefined {
1023
- if (!r.phases || r.phases.length === 0) return undefined;
1024
- const main = r.phases.filter(
1025
- (p) =>
1026
- !CLOSING_PHASE_NAMES.has(p.name) &&
1027
- (p.failed === true || !OVERVIEW_DETAIL_PHASE_NAMES.has(p.name) || p.durationMs >= OVERVIEW_DETAIL_THRESHOLD_MS),
1028
- );
1029
- const closing = r.phases.filter((p) => CLOSING_PHASE_NAMES.has(p.name));
1030
- const parts = main.map((p) => `${p.name} ${formatDurationMs(p.durationMs)}${p.failed ? " ✗" : ""}`);
1031
- const closingMs = closing.reduce((sum, p) => sum + p.durationMs, 0);
1032
- if (closingMs > 0) parts.push(`teardown +${formatDurationMs(closingMs)}`);
1033
- return `timing: ${parts.join(" · ")}`;
1034
- }
1035
-
1036
812
  const TIMING_DETAIL_NODE_BUDGET = 80;
1037
813
 
1038
814
  interface DiagnosticTimingNode {
@@ -1,5 +1,5 @@
1
1
  // cases: docs/engineering/unit-tests/reports/cases.md
2
- // niceeval show 终端宿主的测试(行为规范:docs-site/zh/how-to/viewing-results.mdx;
2
+ // niceeval show 终端宿主的测试(行为规范:docs-site/zh/tutorials/viewing-results.mdx;
3
3
  // 组合语义:docs/feature/reports/architecture.md「Selection 是计算入口」)。覆盖:
4
4
  // - 榜单合成口径:每 experiment × eval 取最新判定,局部重跑从更早快照补齐,头部标注合成自几个快照;
5
5
  // - 前缀过滤收窄 Selection,覆盖警告分母 = 已知并集 ∩ 范围;
@@ -22,7 +22,7 @@ import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type TimingNod
22
22
  import { selectCurrentResults } from "../results/select.ts";
23
23
  import { attemptHistory } from "./compose.ts";
24
24
  import { runShow, type ShowFlags } from "./index.ts";
25
- import { stringWidth } from "../report/text/layout.ts";
25
+ import { stringWidth } from "../report/model/text-layout.ts";
26
26
 
27
27
  // ───────────────────────── fixture 工具 ─────────────────────────
28
28
 
@@ -152,12 +152,12 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
152
152
  const root = await seedComposedRoot();
153
153
  const { out, code } = await show(root, []);
154
154
  expect(code).toBe(0);
155
- expect(out).toContain("No data to plot Cost × End-to-end pass rate");
155
+ expect(out).toContain("No data to plot Cost × Pass rate");
156
156
  expect(out).toContain("1 point missing data");
157
157
  expect(out).not.toContain("needed to compare");
158
158
  expect(out).not.toContain("COMPARISON");
159
159
  expect(out).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
160
- expect(out).toMatch(/1 passed[\s\S]*?\/ 1\s+failed/);
160
+ expect(out).toMatch(/1 passed[\s\S]*?· 1\s+failed/);
161
161
  expect(out).toMatch(/✗ failed\s+fixtures\/button[\s\S]*└─ @1[0-9a-z]{7}[\s\S]*fileChanged/);
162
162
  expect(out).toMatch(/✓ passed\s+weather\/brooklyn[\s\S]*└─ @1[0-9a-z]{7}/);
163
163
  expect(out).not.toMatch(/\[[EXD⏱,]+\]/);
@@ -220,8 +220,8 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
220
220
  try {
221
221
  const { out, code } = await show(root, []);
222
222
  expect(code).toBe(0);
223
- expect(out).toContain("成本 × 端到端成功率 没有可绘制的数据");
224
- expect(out).toContain("1 通过 / 1 失败");
223
+ expect(out).toContain("成本 × 通过率 没有可绘制的数据");
224
+ expect(out).toContain("1 通过 · 1 失败");
225
225
  expect(out).toMatch(/\bbub\s+默认\s+bub\s+1s\s+50%/);
226
226
  // 页首 Hero 与尾部页索引同样跟随 locale(内置文案与页名)。
227
227
  expect(out.split("\n")[0]).toBe("Eval 运行结果");
@@ -259,7 +259,10 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
259
259
  expect(code).toBe(0);
260
260
  // 成本轴(better: lower)反向,「更好」恒指向右上;两轴都声明 better → 提示在场
261
261
  expect(out).toContain("better → upper right");
262
- expect(out).toContain("A compare/a B compare/b");
262
+ // line 声明:按 agent 归类、不连线;图例一行一个 series(显示键字典序),标记按图例顺序分配
263
+ expect(out).toContain("grouped by agent");
264
+ expect(out).toContain("claude A compare/b");
265
+ expect(out).toContain("codex B compare/a");
263
266
  expect(out).toMatch(/\bb\s+large\s+claude\s+1s\s+100%/);
264
267
  expect(out).toMatch(/\ba\s+mini\s+codex\s+1s\s+50%/);
265
268
  expect(out).toMatch(/✓ passed\s+q1[\s\S]*└─ @1[0-9a-z]{7}/);
@@ -480,12 +483,23 @@ describe("--report 装载", () => {
480
483
  " locator: ctx.scope.snapshots[0].evals[0].attempts[0].locator,",
481
484
  " },",
482
485
  "});",
486
+ "const AttemptPage = () => null;",
487
+ "AttemptPage[FACES] = {",
488
+ " resolve: (props, ctx) => ({ locator: ctx.page.locator, verdict: ctx.page.evidence.result.verdict }),",
489
+ " web: () => null,",
490
+ " text: (props) => `CUSTOM ATTEMPT PAGE · ${props.locator} · ${props.verdict}`,",
491
+ "};",
483
492
  "const definition = {",
484
493
  ' kind: "report",',
485
494
  " links: [],",
486
495
  " scripts: [],",
487
496
  " styles: [],",
488
- ' pages: [{ id: "report", title: "Report", content: { type: Wrapper, props: {} } }],',
497
+ " pages: [",
498
+ ' { id: "report", title: "Report", input: "scope", navigation: true, content: { type: Wrapper, props: {} } },',
499
+ // 报告需要声明一张 attempt-input page,attemptCommand 生成器才存在(architecture.md
500
+ // 「Attempt 详情是一张参数化 page」);navigation:false 不进「其余页」索引,单页断言不受影响。
501
+ ' { id: "attempt", title: "Attempt", input: "attempt", navigation: false, content: { type: AttemptPage, props: {} } },',
502
+ " ],",
489
503
  "};",
490
504
  "Object.defineProperty(definition, DEFINITION, { value: true });",
491
505
  "export default definition;",
@@ -592,7 +606,7 @@ describe("--report 装载", () => {
592
606
  expect(out).toContain("CUSTOM weather/brooklyn ·");
593
607
  });
594
608
 
595
- it("--experiment 让 Selection 只留该实验", async () => {
609
+ it("--exp 让 Selection 只留该实验", async () => {
596
610
  const root = await seedComposedRoot();
597
611
  await writeSnapshot(
598
612
  root,
@@ -660,6 +674,75 @@ describe("--report 装载", () => {
660
674
  expect(builtin.err).toContain('page "typo" not found in the built-in report');
661
675
  expect(builtin.err).toContain("Available pages: report, attempts, traces");
662
676
  });
677
+
678
+ /** 只有一张 scope-input page,没有声明 attempt-input page。 */
679
+ async function writeReportFileNoAttemptPage(dir: string): Promise<string> {
680
+ const path = join(dir, "no-attempt-page.mjs");
681
+ await writeFile(
682
+ path,
683
+ [
684
+ 'const FACES = Symbol.for("niceeval.report.faces");',
685
+ 'const DEFINITION = Symbol.for("niceeval.report.definition");',
686
+ "const Overview = () => null;",
687
+ "Overview[FACES] = { web: () => null, text: () => \"OVERVIEW\" };",
688
+ "const definition = {",
689
+ ' kind: "report",',
690
+ " links: [],",
691
+ " scripts: [],",
692
+ " styles: [],",
693
+ ' pages: [{ id: "report", title: "Report", input: "scope", navigation: true, content: { type: Overview, props: {} } }],',
694
+ "};",
695
+ "Object.defineProperty(definition, DEFINITION, { value: true });",
696
+ "export default definition;",
697
+ "",
698
+ ].join("\n"),
699
+ "utf-8",
700
+ );
701
+ return path;
702
+ }
703
+
704
+ it("show @<locator> --report <file>:选中该定义自己声明的 attempt-input page,不是内建固定首页(docs/engineering/unit-tests/reports/cases.md 第 202 行)", async () => {
705
+ const root = await seedComposedRoot();
706
+ const report = await writeReportFile(root);
707
+ const results = await openResults(root);
708
+ const evals = results.experiments[0]!.snapshots.flatMap((s) => s.evals);
709
+ const locator = evals.find((e) => e.id === "weather/brooklyn")!.attempts[0]!.locator!;
710
+
711
+ const { out, code } = await show(root, [locator], { report });
712
+ expect(code).toBe(0);
713
+ expect(out).toBe(`CUSTOM ATTEMPT PAGE · ${locator} · passed\n`);
714
+ // 不是内建 standard 的固定首页文案(AttemptSummary/AttemptAssessment 等)
715
+ expect(out).not.toContain("niceeval show");
716
+ });
717
+
718
+ it("自定义报告没有 attempt-input page 时,裸 show @<locator> --report <file> 报完整用户反馈,指引三种解决路径,不回退到内建详情(第 204 行)", async () => {
719
+ const root = await seedComposedRoot();
720
+ const report = await writeReportFileNoAttemptPage(root);
721
+ const results = await openResults(root);
722
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
723
+
724
+ const { err, code } = await show(root, [locator], { report });
725
+ expect(code).toBe(1);
726
+ expect(err).toContain(report);
727
+ expect(err).toContain("has no attempt-input page");
728
+ expect(err).toContain("extends: standard");
729
+ expect(err).toContain("standardAttemptPage");
730
+ expect(err).toContain('input: "attempt"');
731
+ // 不静默回退渲染内建 standard 的详情页(那会让用户以为自定义报告本来就有这页)
732
+ expect(err).not.toContain("Eval Results");
733
+ });
734
+
735
+ it("--source/--execution/--diff 直接读 AttemptEvidence,不经 page content:没有 attempt page 的自定义报告下仍正常工作(第 203 行)", async () => {
736
+ const root = await seedComposedRoot();
737
+ const report = await writeReportFileNoAttemptPage(root);
738
+ const results = await openResults(root);
739
+ const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
740
+
741
+ const { out, code } = await show(root, [locator], { report, diff: true });
742
+ expect(code).toBe(0);
743
+ expect(out).toContain(locator);
744
+ expect(out).not.toContain("has no attempt-input page");
745
+ });
663
746
  });
664
747
 
665
748
  // ───────────────────────── 证据切面:--execution ─────────────────────────
@@ -812,7 +895,7 @@ describe("show @<locator>", () => {
812
895
  const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
813
896
  const { out } = await show(root, [locator]);
814
897
  expect(out).toContain("gate · Issue 15193");
815
- expect(out).toContain("assertion: equals(4)");
898
+ expect(out).toContain("equals(4)");
816
899
  expect(out).toContain("expected: 4");
817
900
  expect(out).toContain("received: 1");
818
901
  expect(out).toContain("source: evals/manager.eval.ts:40:11");
@@ -829,10 +912,13 @@ describe("show @<locator>", () => {
829
912
 
830
913
  const { out, code } = await show(root, [locator]);
831
914
  expect(code).toBe(0);
832
- expect(out).toContain(`${locator} · weather/brooklyn · compare/bub · passed`);
833
- expect(out).toContain("attempt 1 · ");
834
- expect(out).toContain("execution: unavailable (no events recorded for this attempt)");
835
- expect(out).toContain("changes: diff unavailable");
915
+ // 首行不做整段精确匹配:startedAt 按本机时区格式化,时间部分跨时区不稳定,只断言与时区无关的片段。
916
+ expect(out).toContain(`${locator} · weather/brooklyn · compare/bub · attempt 1 · ✓ passed`);
917
+ expect(out).toContain("· 1.0s");
918
+ // 没有 events/diff 的组件零输出(不是"execution: unavailable"这类占位文案)——
919
+ // 与「Attempt 详情组件族:非空/空证据矩阵」的单元测试同一条契约,这里是集成层确认。
920
+ expect(out).not.toContain("execution:");
921
+ expect(out).not.toContain("changes:");
836
922
  expect(out).not.toContain("evidence:");
837
923
  expect(out).not.toContain("available:");
838
924
  });
@@ -857,22 +943,22 @@ describe("show @<locator>", () => {
857
943
 
858
944
  const { out, code } = await show(root, [locator]);
859
945
  expect(code).toBe(0);
860
- expect(out).toContain(`${locator} · agent-029 · compare/claude-e2b · errored`);
861
- // 结构化 error 块:phase 直接展示闭集点分名,code/message/cause 各一行(见 docs/feature/reports/show.md)
862
- expect(out).toContain("error:");
946
+ // 同上:跳过 startedAt 的时区敏感部分,只断言与时区无关的片段。
947
+ expect(out).toContain(`${locator} · agent-029 · compare/claude-e2b · attempt 1 · ! errored`);
948
+ expect(out).toContain("· 1.0s");
949
+ // 结构化 error 块:code 折进 error: 首行,phase/message/cause 各一行(AttemptError)
950
+ expect(out).toContain("error: sandbox-rate-limit");
863
951
  expect(out).toContain("phase: sandbox.create");
864
- expect(out).toContain("code: sandbox-rate-limit");
865
952
  expect(out).toContain("message: E2B sandbox allocation failed after 5 attempts");
866
953
  expect(out).toContain("cause: RateLimitError · too many concurrent sandboxes");
867
- // attempt 级 diagnostics ( verdict 独立)
868
- expect(out).toContain("diagnostics:");
869
- expect(out).toContain("warning · sandbox.teardown · teardown-failed");
870
- expect(out).toContain("container stop timed out");
871
- // errored 且无断言:不打印空的 assertions 汇总行
954
+ // attempt 级 diagnostics 按 lifecycle phase 分组(AttemptDiagnostics),标题即 phase
955
+ expect(out).toContain("sandbox.teardown:");
956
+ expect(out).toContain("warning · teardown-failed container stop timed out");
957
+ // errored 且无断言:AttemptAssertions 零输出,不打印空的 assertions 汇总行
872
958
  expect(out).not.toContain("assertions:");
873
959
  });
874
960
 
875
- it("available 只逐行列出实际存在的证据命令,不打印能力字母或合并式 next", async () => {
961
+ it("证据下钻命令散落在各自区块行尾(带 locator),不存在的能力不生成假命令,也没有合并式 available/next 尾块", async () => {
876
962
  const root = await makeRoot();
877
963
  const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
878
964
  res("weather/brooklyn", "failed", { hasEvents: true }),
@@ -889,9 +975,14 @@ describe("show @<locator>", () => {
889
975
 
890
976
  const { out, code } = await show(root, [locator]);
891
977
  expect(code).toBe(0);
892
- expect(out).toContain(`available:\n niceeval show ${locator} --source\n niceeval show ${locator} --execution`);
978
+ // AttemptSource / AttemptConversation 各自行尾带命令(不是集中列在一个 available: 块里)
979
+ expect(out).toContain(`niceeval show ${locator} --source`);
980
+ expect(out).toContain(`niceeval show ${locator} --execution`);
981
+ // 没有 diff/timing 能力(fixture 没写 diff.json,也没有 phases)时不生成假命令
893
982
  expect(out).not.toContain(`niceeval show ${locator} --diff`);
983
+ expect(out).not.toContain(`niceeval show ${locator} --timing`);
894
984
  expect(out).not.toContain("evidence:");
985
+ expect(out).not.toContain("available:");
895
986
  expect(out).not.toContain("next:");
896
987
  });
897
988
 
@@ -1054,7 +1145,7 @@ describe("show @<locator>", () => {
1054
1145
  expect(out).not.toContain("git show ×");
1055
1146
  });
1056
1147
 
1057
- it("attempt 首页 timing 只列大头,但保留变慢的 telemetry 阶段", async () => {
1148
+ it("attempt 首页 timing 逐阶段一行,收尾段单独归拢(AttemptTimeline 不做单行宽度收窄,--timing 才是完整分解)", async () => {
1058
1149
  const root = await makeRoot();
1059
1150
  const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
1060
1151
  res("weather/brooklyn", "passed", {
@@ -1080,15 +1171,17 @@ describe("show @<locator>", () => {
1080
1171
  const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
1081
1172
 
1082
1173
  const { out } = await show(root, [locator]);
1083
- const timingLine = out.split("\n").find((line) => line.startsWith("timing:"));
1084
- expect(out).toContain("1 AI message");
1085
- expect(timingLine).toContain("sandbox.queue 0ms");
1086
- expect(timingLine).toContain("sandbox.create 3.0s");
1087
- expect(timingLine).toContain("eval.run 45.0s");
1088
- expect(timingLine).toContain("telemetry.collect 35.3s");
1089
- expect(timingLine).toContain("teardown +8.5s");
1090
- expect(timingLine).not.toContain("workspace.baseline");
1091
- expect(timingLine).not.toContain("telemetry.configure");
1174
+ // 每个主链阶段各占一行(不再是旧格式的单行拼接,那条规则本身是为单行可读服务的,
1175
+ // 多行块没有这个约束了),收尾段(sandbox.stop)单独归拢在 teardown: 之下。
1176
+ expect(out).toContain("timing: 1m 24s");
1177
+ expect(out).toContain("sandbox.queue 0ms");
1178
+ expect(out).toContain("sandbox.create 3.0s");
1179
+ expect(out).toContain("workspace.baseline 310ms");
1180
+ expect(out).toContain("telemetry.configure 358ms");
1181
+ expect(out).toContain("eval.run 45.0s");
1182
+ expect(out).toContain("telemetry.collect 35.3s");
1183
+ expect(out).toContain("teardown:");
1184
+ expect(out).toContain("sandbox.stop 8.5s");
1092
1185
  });
1093
1186
 
1094
1187
  it("语法不对的 locator 报「not a valid attempt locator」,退出码 1,不崩", async () => {
@@ -22,11 +22,11 @@ const reportPages = {
22
22
  };
23
23
 
24
24
  function dataWithShell(report: ViewData["report"]): ViewData {
25
- return { composedRuns: 1, snapshots: [], ...(report !== undefined ? { report } : {}) };
25
+ return { composedRuns: 1, ...(report !== undefined ? { report } : {}) };
26
26
  }
27
27
 
28
- describe("外壳:宿主无品牌位与 hero,导航只有报告页", () => {
29
- it("宿主导航壳 DOM 无品牌节点、无 hero 区:品牌与 hero 是页内组件,不归宿主", () => {
28
+ describe("外壳:宿主恒有 NiceEval 品牌位、无 hero,导航只有报告页", () => {
29
+ it("宿主页头有恒定品牌字标、无 hero 区:hero 是页内组件不归宿主,品牌归宿主", () => {
30
30
  const html = renderToStaticMarkup(
31
31
  <App
32
32
  data={dataWithShell({
@@ -38,11 +38,12 @@ describe("外壳:宿主无品牌位与 hero,导航只有报告页", () => {
38
38
  reportPages={reportPages}
39
39
  />,
40
40
  );
41
- // 宿主 DOM 无品牌节点:没有字标、没有 Powered by、没有任何官网品牌链接。
42
- expect(html).not.toContain('class="brand"');
43
- expect(html).not.toContain("NiceEval");
41
+ // 宿主页头恒有 NiceEval 品牌字标,外链官网;报告定义不能覆盖或移除。
42
+ expect(html).toContain('class="brand"');
43
+ expect(html).toContain("NiceEval");
44
+ expect(html).toContain("niceeval.com");
45
+ // 品牌位是页头字标,不是 hero 的 Powered by 行——后者是页内组件,不落宿主壳。
44
46
  expect(html).not.toContain("Powered by");
45
- expect(html).not.toContain("niceeval.com");
46
47
  // 宿主无 hero 区:标题只落浏览器 <title>(useEffect,静态渲染不执行),不落任何宿主节点。
47
48
  expect(html).not.toContain('class="hero"');
48
49
  expect(html).not.toContain("<h1");