niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,7 +1,7 @@
1
1
  // sandbox 域类型:Sandbox 接口、provider spec(可辨识联合)、命令与文件 IO 的形状。
2
2
  // 「在哪里跑、如何隔离」的全部契约在这里;provider 实现见本目录各文件,分发见 resolve.ts。
3
3
 
4
- import type { Cleanup, ScopedFeedback } from "../shared/types.ts";
4
+ import type { ScopedFeedback } from "../shared/types.ts";
5
5
 
6
6
  export interface CommandResult {
7
7
  stdout: string;
@@ -76,15 +76,16 @@ export type SandboxRuntime = "node20" | "node24";
76
76
  * **执行顺序**:沙箱就绪 → `sandbox.setup` 钩子 → workspace 上传 / git 基线 / `eval.setup` →
77
77
  * `agent.setup` → `agent.tracing.configure` → 逐轮 `send`。`sandbox.setup` 特意排在 git
78
78
  * 基线之前——它的改动会被提交进基线,不会被误算进 agent 产出的 diff。收尾按 LIFO:
79
- * agent 级 cleanup / `agent.teardown` 先跑,`sandbox.teardown` 钩子最后跑(沙箱销毁前)。
79
+ * `agent.teardown` 先跑,`sandbox.teardown` 钩子最后跑(沙箱销毁前)。
80
80
  *
81
81
  * **多钩子**:`.setup(a).setup(b)` 按追加顺序依次执行(a 先 b 后);`.teardown(x).teardown(y)`
82
82
  * 按追加的**逆序**执行(y 先 x 后)。每次调用都返回一个新 spec(不改变原对象),可继续链式。
83
+ * `setup` 链中途抛错时后续 `setup` 不再执行,`teardown` 链仍完整走完。
83
84
  *
84
85
  * **失败语义**:`setup` 钩子抛错按执行错误计——与 `eval.setup` / `agent.setup` 抛错走同一条
85
- * 路径,不新增错误分类;但不阻断该 attempt 已进入的收尾(已跑过的 setup 返回的 cleanup、
86
- * `teardown` 钩子仍会在 finally 里跑)。`teardown` 钩子报错只作诊断(吞掉 / 记 log),不改变
87
- * 已产出的结果——与 `agent.teardown` 现状一致。
86
+ * 路径,不新增错误分类;但不阻断该 attempt 已进入的收尾(已挂载的 `teardown` 钩子仍会在
87
+ * finally 里跑)。`teardown` 钩子报错只作诊断(吞掉 / 记 log),不改变已产出的结果——与
88
+ * `agent.teardown` 现状一致。
88
89
  */
89
90
  export interface SandboxHooks<Self> {
90
91
  /** 已挂载的 setup 钩子,按追加顺序保存(内部读取,一般用不到)。 */
@@ -109,11 +110,12 @@ export interface SandboxHookContext extends ScopedFeedback {
109
110
  readonly signal: AbortSignal;
110
111
  }
111
112
 
112
- /** 沙箱级生命周期钩子(`.setup()` / `.teardown()` 链式挂载);`setup` 可返回 cleanup 闭包。 */
113
+ /** 沙箱级生命周期钩子(`.setup()` / `.teardown()` 链式挂载);`setup` 不返回值——要把 `setup`
114
+ * 创建的句柄传给 `teardown`,以 `sandbox` 实例为键存取(见 {@link SandboxHooks})。 */
113
115
  export type SandboxHook = (
114
116
  sandbox: Sandbox,
115
117
  ctx: SandboxHookContext,
116
- ) => void | Cleanup | Promise<void | Cleanup>;
118
+ ) => void | Promise<void>;
117
119
 
118
120
  /**
119
121
  * Sandbox 的「数据结构」定义 —— 与 agent 一样可带参数(见 docs/feature/sandbox/library.md)。
@@ -170,12 +172,12 @@ export type SandboxSpec = DockerSandboxSpec | VercelSandboxSpec | E2BSandboxSpec
170
172
  export type SandboxOption = SandboxSpec;
171
173
 
172
174
  export interface CommandOptions {
173
- /** 追加/覆盖本命令的环境变量(与沙箱默认环境叠加,不清空默认值;各 provider 会保留自己固定的 `PATH` 等变量,不保证能被这里覆盖)。 */
175
+ /** 追加/覆盖本命令的环境变量( Sandbox 默认环境叠加,不清空默认值;各 provider 会保留自己固定的 `PATH` 等变量,不保证能被这里覆盖)。 */
174
176
  env?: Record<string, string>;
175
177
  /** 本命令的工作目录;省略时落到 `Sandbox.workdir`。相对路径按 workdir 解析,绝对路径原样使用。 */
176
178
  cwd?: string;
177
179
  /**
178
- * 把本命令的输出也送进沙箱的「原生日志流」(于是 `docker logs` / Docker UI 的 Logs
180
+ * 把本命令的输出也送进 Sandbox 的「原生日志流」(于是 `docker logs` / Docker UI 的 Logs
179
181
  * 标签页能实时看到它)。给 agent 命令(codex exec / bub run / claude)开它,就能在容器
180
182
  * 日志里看到 agent 的【原始输出】。provider 各自实现(docker:tee 到 PID1 tail 的文件;
181
183
  * 不支持的 provider 忽略)—— 日志怎么浮现是 provider 的事,adapter 只声明意图。
@@ -190,7 +192,7 @@ export interface CommandOptions {
190
192
  /** `onStdout` 的 stderr 对应物;完整 stderr 仍保留在 `CommandResult`。 */
191
193
  onStderr?: (chunk: string) => void | Promise<void>;
192
194
  /**
193
- * 以 root 跑本命令。默认 `false` —— 命令以沙箱的标准**非 root** 用户跑(agent 的自然环境)。
195
+ * 以 root 跑本命令。默认 `false` —— 命令以 Sandbox 的标准**非 root** 用户跑(agent 的自然环境)。
194
196
  * 给 setup 阶段装系统依赖用(`apt-get install …`、`pip install --break-system-packages …`)。
195
197
  *
196
198
  * 语义跨 provider 一致:"本命令以 root 跑,否则以标准非 root 用户跑"。各 provider 映射到自己的原生机制
@@ -202,7 +204,7 @@ export interface CommandOptions {
202
204
  }
203
205
 
204
206
  export interface Sandbox {
205
- /** 沙箱内项目/工作区根目录的绝对路径(agent 命令的默认 cwd,也是 git baseline 提交的位置)。各方法的相对路径都以此为基准解析,省略 `cwd`/`targetDir` 时也落到这里。 */
207
+ /** Sandbox 内项目/工作区根目录的绝对路径(agent 命令的默认 cwd,也是 git baseline 提交的位置)。各方法的相对路径都以此为基准解析,省略 `cwd`/`targetDir` 时也落到这里。 */
206
208
  readonly workdir: string;
207
209
  /**
208
210
  * 执行单个命令,`args` 作为独立 argv 传递、不经 shell 解释(无 `&&`、管道、通配符展开)。
@@ -214,29 +216,29 @@ export interface Sandbox {
214
216
  * 需要拼多条命令或做条件判断时用它。
215
217
  */
216
218
  runShell(script: string, opts?: CommandOptions): Promise<CommandResult>;
217
- /** 读取沙箱内文件的文本内容(UTF-8)。文件不存在时抛错,不返回空字符串——需要容错请自行 `.catch()`。 */
219
+ /** 读取 Sandbox 内文件的文本内容(UTF-8)。文件不存在时抛错,不返回空字符串——需要容错请自行 `.catch()`。 */
218
220
  readFile(path: string): Promise<string>;
219
- /** 检查沙箱内路径是否存在。跨 provider 语义不完全一致:仅保证对普通文件可靠,对目录路径的行为不同 provider 不保证一致。 */
221
+ /** 检查 Sandbox 内路径是否存在。跨 provider 语义不完全一致:仅保证对普通文件可靠,对目录路径的行为不同 provider 不保证一致。 */
220
222
  fileExists(path: string): Promise<boolean>;
221
223
  /**
222
224
  * 一次 shell 往返读全部源码文件(按扩展名收、按目录/文件名忽略)。
223
- * 取代每个 eval 目录里手写的 find + 逐文件 readFile。
225
+ * 取代每个评估用例目录里手写的 find + 逐文件 readFile。
224
226
  */
225
227
  readSourceFiles(opts?: ReadSourceFilesOptions): Promise<SourceFiles>;
226
228
  /** 写入若干文本文件(内容已在内存里的字符串);是 `uploadFiles` 的文本特化,省略 `targetDir` 落到 workdir。 */
227
229
  writeFiles(files: Record<string, string>, targetDir?: string): Promise<void>;
228
230
  /** 批量写入若干文件,内容可以是文本或二进制 Buffer;省略 `targetDir` 落到 workdir。 */
229
231
  uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void>;
230
- /** 把本地磁盘上的一个目录整体上传进沙箱(递归读取本地文件后按 `uploadFiles` 写入);`opts.ignore` 是排除规则,省略 `targetDir` 落到 workdir。 */
232
+ /** 把本地磁盘上的一个目录整体上传进 Sandbox(递归读取本地文件后按 `uploadFiles` 写入);`opts.ignore` 是排除规则,省略 `targetDir` 落到 workdir。 */
231
233
  uploadDirectory(localDir: string, targetDir?: string, opts?: { ignore?: string[] }): Promise<void>;
232
- /** 销毁沙箱占用的计算资源(容器/microVM)。调用后沙箱不可再用;是否可安全重复调用因 provider 而异,不要依赖这一点。 */
234
+ /** 销毁 Sandbox 占用的计算资源(容器/microVM)。调用后 Sandbox 不可再用;是否可安全重复调用因 provider 而异,不要依赖这一点。 */
233
235
  stop(): Promise<void>;
234
- /** 本沙箱的稳定标识(各 provider 原生 ID,如 Docker 容器 ID 前缀);用于跨调用关联同一沙箱的会话状态,也用于日志展示。 */
236
+ /** 本 Sandbox 的稳定标识(各 provider 原生 ID,如 Docker 容器 ID 前缀);用于跨调用关联同一 Sandbox 的会话状态,也用于日志展示。 */
235
237
  readonly sandboxId: string;
236
238
  /**
237
239
  * 本地 OTLP 接收器的目标 host。
238
- * - `string`:沙箱内可通过该 hostname 回连宿主 OTLP 端口(如 docker 的 `host.docker.internal`)。
239
- * - `null`:沙箱运行在远程云端(如 e2b/vercel),无法访问宿主本地端口 → 跳过 tracing。
240
+ * - `string`:Sandbox 内可通过该 hostname 回连宿主 OTLP 端口(如 docker 的 `host.docker.internal`)。
241
+ * - `null`:Sandbox 运行在远程云端(如 e2b/vercel),无法访问宿主本地端口 → 跳过 tracing。
240
242
  * 可通过环境变量 `NICEEVAL_OTLP_HOST` 强制覆盖(如配置 tunnel 时)。
241
243
  */
242
244
  readonly otlpHost: string | null;
@@ -248,13 +250,13 @@ export interface Sandbox {
248
250
  appendLog?(line: string): Promise<void>;
249
251
 
250
252
  /**
251
- * 从沙箱内任意路径读取文件,返回二进制 Buffer。
253
+ * Sandbox 内任意路径读取文件,返回二进制 Buffer。
252
254
  * 对应各 provider:Docker getArchive / Vercel readFileToBuffer / e2b files.read(bytes) / …
253
255
  */
254
256
  downloadFile(path: string): Promise<Buffer>;
255
257
 
256
258
  /**
257
- * 向沙箱内任意路径写入文件(二进制)。
259
+ * 向 Sandbox 内任意路径写入文件(二进制)。
258
260
  * 对应各 provider:Docker putArchive / Vercel fs.writeFile(Buffer) / e2b files.write / …
259
261
  */
260
262
  uploadFile(path: string, content: Buffer): Promise<void>;
@@ -35,7 +35,8 @@ describe("primaryAssertionSummary", () => {
35
35
  });
36
36
  expect(assertionSummaryLines(summary!)).toEqual([
37
37
  "gate: Issue 15193: selected proposal matches the accepted proposal",
38
- "equals(4) · expected 4 · received 3 · +1 more failures",
38
+ "equals(4) · expected 4 · received 3",
39
+ "+1 more failures",
39
40
  ]);
40
41
  expect(compactAssertionSummary(summary!)).toBe(
41
42
  "gate: Issue 15193: selected proposal matches the accepted proposal · equals(4) · expected 4 · received 3 · +1 more failures",
@@ -78,8 +79,33 @@ describe("primaryAssertionSummary", () => {
78
79
  expect(summary.received!.length).toBeLessThanOrEqual(240);
79
80
  expect(summary.received).toMatch(/…$/);
80
81
  const lines = assertionSummaryLines(summary);
81
- expect(lines).toHaveLength(2);
82
82
  expect(lines.every((line) => !line.includes("\n"))).toBe(true);
83
+ // matcher · expected 放不下大值 received 时各自另起一行
84
+ expect(lines).toEqual([
85
+ "gate: includes(/updateTag/)",
86
+ expect.stringContaining("expected matches /updateTag/"),
87
+ expect.stringMatching(/^received: .*…$/),
88
+ ]);
89
+ });
90
+
91
+ it("received 大值单独截断一行时,+N more failures 仍是独立尾行、不与被截断的值粘连", () => {
92
+ const assertions: AssertionResult[] = [
93
+ {
94
+ name: "includes(/updateTag/)",
95
+ severity: "gate",
96
+ outcome: "failed",
97
+ score: 0,
98
+ expected: "matches /updateTag/",
99
+ received: `// app/actions/posts.ts\n'use server';\n${"const source = 1;\n".repeat(80)}`,
100
+ },
101
+ { name: "matches(schema)", severity: "gate", outcome: "failed", score: 0 },
102
+ ];
103
+
104
+ const summary = primaryAssertionSummary(assertions, "failed")!;
105
+ const lines = assertionSummaryLines(summary);
106
+ expect(lines.at(-1)).toBe("+1 more failures");
107
+ const receivedLine = lines.find((line) => line.startsWith("received:"))!;
108
+ expect(receivedLine).not.toContain("more failures");
83
109
  });
84
110
  });
85
111
 
@@ -9,6 +9,13 @@ import type { AssertionResult, PrimaryAssertionSummary, Verdict } from "./types.
9
9
  */
10
10
  const SUMMARY_TEXT_MAX_CHARS = 240;
11
11
 
12
+ /**
13
+ * Human/Agent 永久行的单行事实预算,与 SUMMARY_TEXT_MAX_CHARS(单值上限)分开计:一条
14
+ * `matcher · expected · received` 拼起来很容易超过一屏宽,这里给的是「一行」的上限,不依赖
15
+ * 终端 columns——agent profile 的 handoff 不是 TTY,不能按运行时宽度变化。
16
+ */
17
+ const DETAIL_LINE_MAX_CHARS = 100;
18
+
12
19
  /** 摘要面的单值收口:折单行 + 240 字符上限。任何把断言事实放进「行」里的面共用这一条。 */
13
20
  export function summaryText(value: string): string {
14
21
  const singleLine = value.replace(/\s+/g, " ").trim();
@@ -80,11 +87,37 @@ export function assertionSummaryDetail(summary: PrimaryAssertionSummary): string
80
87
  return parts.length > 0 ? parts.join(" · ") : undefined;
81
88
  }
82
89
 
83
- /** Human/Agent 的至多两层文本。 */
84
- export function assertionSummaryLines(summary: PrimaryAssertionSummary): [string] | [string, string] {
85
- const head = `${summary.severity}: ${summary.assertion}`;
86
- const detail = assertionSummaryDetail(summary);
87
- return detail === undefined ? [head] : [head, detail];
90
+ /**
91
+ * Human/Agent 永久行的排版:标题独占一行;`matcher · expected`(含 score/threshold/reason)
92
+ * 独占下一行;`received` 能跟这行拼在一起仍在 DETAIL_LINE_MAX_CHARS 内就合并,放不下就单独
93
+ * 再起一行并硬截断。`+N more failures` 永远是独立尾行,不参与截断,也不拼进被截断的值——
94
+ * 截断处的 `…` 后面只会是值本身,不会被人误读成计数的一部分。
95
+ */
96
+ export function assertionSummaryLines(summary: PrimaryAssertionSummary): string[] {
97
+ const lines: string[] = [`${summary.severity}: ${summary.assertion}`];
98
+
99
+ const factParts: string[] = [];
100
+ if (summary.matcher !== undefined) factParts.push(summary.matcher);
101
+ if (summary.expected !== undefined) factParts.push(`expected ${summary.expected}`);
102
+ if (summary.score !== undefined) factParts.push(`score ${summary.score}`);
103
+ if (summary.threshold !== undefined) factParts.push(`threshold ${summary.threshold}`);
104
+ if (summary.reason !== undefined) factParts.push(`reason ${summary.reason}`);
105
+ const facts = factParts.length > 0 ? factParts.join(" · ") : undefined;
106
+
107
+ if (summary.received !== undefined) {
108
+ const combined = facts !== undefined ? `${facts} · received ${summary.received}` : `received ${summary.received}`;
109
+ if (combined.length <= DETAIL_LINE_MAX_CHARS) {
110
+ lines.push(combined);
111
+ } else {
112
+ if (facts !== undefined) lines.push(facts);
113
+ lines.push(shrinkTo(`received: ${summary.received}`, DETAIL_LINE_MAX_CHARS));
114
+ }
115
+ } else if (facts !== undefined) {
116
+ lines.push(facts);
117
+ }
118
+
119
+ if (summary.additionalFailures > 0) lines.push(`+${summary.additionalFailures} more failures`);
120
+ return lines;
88
121
  }
89
122
 
90
123
  /** 比较列表的单元格投影;无 group 时不重复 `gate:` 前缀。 */
@@ -22,11 +22,11 @@ export interface ValueAssertion {
22
22
  /** 期望条件的有界文本描述(如 `contains "Brooklyn"`),失败时进 AssertionResult.expected。 */
23
23
  readonly expected?: string;
24
24
  score(value: unknown): number | Promise<number>;
25
- /** 转成硬门槛断言:未达阈值(省略 threshold 则按 score > 0 判定)整条 eval 判为 failed。返回新实例,不改原对象。 */
25
+ /** 转成硬门槛断言:未达阈值(省略 threshold 则按 score > 0 判定)整条评估用例判为 failed。返回新实例,不改原对象。 */
26
26
  gate(threshold?: number): ValueAssertion;
27
27
  /**
28
- * 转成软阈值断言:未达 threshold 时该条记为 failed,但默认不拖累整条 eval 的 verdict;
29
- * `--strict` 运行下,软阈值失败也会把整条 eval 的 verdict 计为 failed。返回新实例,不改原对象。
28
+ * 转成软阈值断言:未达 threshold 时该条记为 failed,但默认不拖累整条评估用例的 verdict;
29
+ * `--strict` 运行下,软阈值失败也会把整条评估用例的 verdict 计为 failed。返回新实例,不改原对象。
30
30
  */
31
31
  atLeast(threshold: number): ValueAssertion;
32
32
  /**
@@ -0,0 +1,52 @@
1
+ // cases: docs/engineering/unit-tests/experiments-runner/cases.md
2
+ import { describe, expect, it } from "vitest";
3
+ import { matchExperimentSelector } from "./aggregate.ts";
4
+
5
+ describe("matchExperimentSelector", () => {
6
+ const ids = [
7
+ "compare/bub-gpt-5.6-luna",
8
+ "compare/codex-gpt-5.6-luna",
9
+ "compare/codex-gpt-5.6-luna--agents-md",
10
+ "compare/codex-gpt-5.6-luna--mempal",
11
+ "dev-e2b/bub-e2b",
12
+ "dev/bub-gpt-5.4-mini",
13
+ "top-level",
14
+ ];
15
+
16
+ it("selects an entire group by exact directory match", () => {
17
+ expect(matchExperimentSelector(ids, "compare")).toEqual([
18
+ "compare/bub-gpt-5.6-luna",
19
+ "compare/codex-gpt-5.6-luna",
20
+ "compare/codex-gpt-5.6-luna--agents-md",
21
+ "compare/codex-gpt-5.6-luna--mempal",
22
+ ]);
23
+ });
24
+
25
+ it("an exact id match wins even when it is a prefix of sibling variants", () => {
26
+ expect(matchExperimentSelector(ids, "compare/codex-gpt-5.6-luna")).toEqual([
27
+ "compare/codex-gpt-5.6-luna",
28
+ ]);
29
+ });
30
+
31
+ it("falls back to a filename prefix within an exact-matched directory to select a family", () => {
32
+ expect(matchExperimentSelector(ids, "compare/codex")).toEqual([
33
+ "compare/codex-gpt-5.6-luna",
34
+ "compare/codex-gpt-5.6-luna--agents-md",
35
+ "compare/codex-gpt-5.6-luna--mempal",
36
+ ]);
37
+ });
38
+
39
+ it("requires an exact directory segment; a group-like prefix does not leak across groups", () => {
40
+ expect(matchExperimentSelector(ids, "dev")).toEqual(["dev/bub-gpt-5.4-mini"]);
41
+ expect(matchExperimentSelector(ids, "dev")).not.toContain("dev-e2b/bub-e2b");
42
+ });
43
+
44
+ it("matches a top-level (groupless) experiment by exact id", () => {
45
+ expect(matchExperimentSelector(ids, "top-level")).toEqual(["top-level"]);
46
+ });
47
+
48
+ it("returns empty when nothing matches", () => {
49
+ expect(matchExperimentSelector(ids, "dev-e3b")).toEqual([]);
50
+ expect(matchExperimentSelector(ids, "compare/nope")).toEqual([]);
51
+ });
52
+ });
@@ -55,6 +55,35 @@ export function evalPrefixPredicate(evals?: string | string[]): (id: string) =>
55
55
  return (id) => prefixes.some((prefix) => id.startsWith(prefix));
56
56
  }
57
57
 
58
+ /**
59
+ * 实验选择器解析:`niceeval exp` 位置参数与 `--exp` 共用同一条规则(docs/feature/experiments/cli.md
60
+ * 「实验选择器怎样解析」)。按序:①精确 id 优先,即使它同时是同目录内其它文件名的前缀;
61
+ * ②目录段精确前缀(含更深层目录),选中该目录下全部实验;③以上都不命中且选择器形如
62
+ * `目录/文件名前缀` 时,目录段精确匹配后按文件名段前缀选中一族配置——目录段永远精确,不跨目录
63
+ * 误配,文件名段裸前缀,与 `evalPrefixPredicate` 同一条"共享前缀即家族"的逻辑。
64
+ */
65
+ export function matchExperimentSelector(ids: readonly string[], selector: string): string[] {
66
+ const exact = ids.find((id) => id === selector);
67
+ if (exact !== undefined) return [exact];
68
+ const dirMatches = ids.filter((id) => id.startsWith(selector + "/"));
69
+ if (dirMatches.length > 0) return dirMatches;
70
+ const lastSlash = selector.lastIndexOf("/");
71
+ if (lastSlash === -1) return [];
72
+ const dir = selector.slice(0, lastSlash);
73
+ const namePrefix = selector.slice(lastSlash + 1);
74
+ if (namePrefix === "") return [];
75
+ return ids.filter((id) => id.startsWith(dir + "/") && id.slice(dir.length + 1).startsWith(namePrefix));
76
+ }
77
+
78
+ /**
79
+ * 零命中时展示的可浏览路径清单(docs/feature/experiments/cli.md「零命中」):按 id 的顶层目录段
80
+ * 去重、排序,带尾斜杠;没有目录段的顶层 experiment id 原样列出(它本身就是一个可选择的目标,
81
+ * 不是一段路径)。这是运行选择的展示投影,不是报告分组。
82
+ */
83
+ export function browsableExperimentPaths(ids: readonly string[]): string[] {
84
+ return [...new Set(ids.map((id) => (id.includes("/") ? `${id.split("/")[0]}/` : id)))].sort();
85
+ }
86
+
58
87
  /** 无 experimentId 时的兜底标签。 */
59
88
  export function fallbackExperimentLabel(result: {
60
89
  experimentId?: string;
@@ -30,7 +30,12 @@ export interface SourceArtifact {
30
30
  content: string;
31
31
  }
32
32
 
33
- /** 通用清理闭包(setup 返回值 / teardown 的形状),异步同步皆可,统一在 finally 里执行。 */
33
+ /**
34
+ * 内部收尾闭包类型:供运行器内部的清理注册表使用(如 `postSetup` 钩子内部累积待收尾动作),
35
+ * 不出现在任何公开的生命周期钩子签名里——`SandboxHook`/`AgentSetup`/`AgentTeardown` 等公开
36
+ * 钩子一律 `void | Promise<void>`,setup 不返回值,收尾靠成对的 teardown(见
37
+ * docs/runner.md「环境预置不进运行器,但按顺序调它」)。
38
+ */
34
39
  export type Cleanup = () => Promise<void> | void;
35
40
 
36
41
  /** `ScopedFeedback.progress` 的入参:此刻正在做什么(短命状态,可被后续更新覆盖)。 */
@@ -0,0 +1,34 @@
1
+ // cases: docs/engineering/unit-tests/reports/cases.md
2
+ // 「外壳、页面与 Tabs」——show 专属的索引命令拼装(showCommand)与其余页索引文案(otherPagesText)。
3
+
4
+ import { describe, expect, it } from "vitest";
5
+ import { showCommand } from "./command.ts";
6
+ import { otherPagesText } from "./render.ts";
7
+
8
+ describe("其余页索引与索引命令上下文", () => {
9
+ const pages = [
10
+ { id: "overview", title: { en: "Overview", "zh-CN": "总览" } },
11
+ { id: "exam", title: { en: "Exam", "zh-CN": "成绩单" } },
12
+ ];
13
+
14
+ it("只列未渲染的页,索引命令保留当前 --results / --report 与位置参数,复制即可复现下一层视图", () => {
15
+ // 渲染的是 overview,其余页索引只含 exam 一行——与「渲染初始页 + 尾部附其余页索引」
16
+ // 的行为一致(docs/feature/reports/show/reports.md Case 2)。
17
+ const text = otherPagesText({
18
+ otherPages: pages.filter((p) => p.id !== "overview"),
19
+ command: { patterns: [], results: "tmp/published-results", report: "reports/site.tsx" },
20
+ locale: "zh-CN",
21
+ });
22
+ expect(text).toContain("其余页:");
23
+ expect(text).toContain("niceeval show --results tmp/published-results --report reports/site.tsx --page exam");
24
+ expect(text).toContain("成绩单");
25
+ expect(text).not.toContain("总览");
26
+ expect(text).not.toContain("--page overview");
27
+ });
28
+
29
+ it("showCommand 按序携带位置参数与 --exp / --results / --report / --page", () => {
30
+ expect(
31
+ showCommand({ patterns: ["memory/swelancer"], experiment: "dev-e2b", report: "reports/site.tsx", page: "exam" }),
32
+ ).toBe("niceeval show memory/swelancer --exp dev-e2b --report reports/site.tsx --page exam");
33
+ });
34
+ });
@@ -0,0 +1,16 @@
1
+ // show 专属的可复制命令拼装:页索引 / 组索引共用的携带规则(docs/feature/reports/show/reports.md
2
+ // 「索引命令携带完整上下文」)。这不是两宿主共用的报告 runtime 部分——view 走网页路由,不生成
3
+ // 终端命令——所以住在 show 自己这一侧,不进 src/report/runtime/host.ts。
4
+
5
+ import type { HostCommandContext } from "../report/runtime/host.ts";
6
+
7
+ /** 按上下文拼一条可复制的 show 命令(页索引 / 组索引共用的携带规则)。 */
8
+ export function showCommand(ctx: HostCommandContext, extra: string[] = []): string {
9
+ const parts = ["niceeval show", ...ctx.patterns];
10
+ if (ctx.experiment !== undefined) parts.push(`--exp ${ctx.experiment}`);
11
+ if (ctx.results !== undefined) parts.push(`--results ${ctx.results}`);
12
+ if (ctx.report !== undefined) parts.push(`--report ${ctx.report}`);
13
+ if (ctx.page !== undefined) parts.push(`--page ${ctx.page}`);
14
+ parts.push(...extra);
15
+ return parts.join(" ");
16
+ }
@@ -6,7 +6,7 @@
6
6
  // 现刻水位 Scope(两个宿主共用)住在 ../results/select.ts;本文件只留 show 独有的时间轴计算。
7
7
  // 数据只消费 niceeval/results 的读取面。
8
8
 
9
- import { attemptCostUSD } from "../report/metrics.ts";
9
+ import { attemptCostUSD } from "../report/model/metrics.ts";
10
10
  import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../scoring/display.ts";
11
11
  import type { EvalResult, Verdict } from "../types.ts";
12
12
  import type { AttemptHandle, Experiment } from "../results/index.ts";
package/src/show/index.ts CHANGED
@@ -4,7 +4,8 @@
4
4
  // 位置参数 = eval id 前缀,或 `@<locator>`(精确指名单个 attempt,见 results/locator.ts):
5
5
  // 裸跑 / 多 eval 前缀 内建报告(niceeval/report/built-in 默认导出)的 text 面(单 eval 前缀仍进入详情)
6
6
  // 恰好一个 eval 单 eval 详情(attempt / 断言明细,宿主本体)
7
- // @<locator> 精确 attempt:无证据 flag → 紧凑全景;带 flag 对应证据切面
7
+ // @<locator> 精确 attempt:无证据 flag → 当前 report attempt-input page(内建 standard
8
+ // 或 --report 自定义);带 flag → 对应证据切面(宿主本体,不经报告管线)
8
9
  // --source / --execution / --diff[=路径] 证据切面(宿主本体):出现即走证据室,不渲染报告槽
9
10
  // --history 执行时间轴(逐 experimentId + evalId 分节),与 --report 互斥
10
11
  // --report <文件> 整槽换成用户报告;位置前缀 / --results / --exp 先收窄 Scope 再注入
@@ -28,26 +29,25 @@ import {
28
29
  // against — `instanceof` is keyed by declaration site, so a raw src copy and the compiled
29
30
  // dist copy of "the same" class are two different types. The package-owned report runtime
30
31
  // ships as precompiled ESM (dist/report/**, built by `pnpm run build:report`); all report
31
- // loading/rendering goes through ./report-host.ts (the single contact surface with it).
32
- import { ReportLoadError } from "../../dist/report/load.js";
32
+ // loading/rendering goes through ../report/runtime/host.ts (the shared contact surface).
33
+ import { ReportLoadError } from "../../dist/report/runtime/load.js";
33
34
  import { detectLocale, t } from "../i18n/index.ts";
34
35
  import { foldEvalVerdict } from "../shared/verdict.ts";
35
36
  import { selectCurrentResults, filterExperiments } from "../results/select.ts";
36
37
  import { evalPrefixPredicate } from "../shared/aggregate.ts";
37
38
  import { attemptHistory } from "./compose.ts";
38
39
  import {
40
+ buildHostReportMeta,
39
41
  HostReportError,
40
42
  loadHostReport,
41
- reportMetaFor,
42
43
  renderHostPageText,
43
44
  type HostCommandContext,
44
- } from "./report-host.ts";
45
+ } from "../report/runtime/host.ts";
45
46
  import {
46
47
  attemptArtifactsPath,
47
48
  attemptEvidenceHeader,
48
49
  attemptHistoryText,
49
50
  attemptIndexLine,
50
- attemptOverviewText,
51
51
  attemptsOfEval,
52
52
  diffText,
53
53
  evalDetailText,
@@ -96,8 +96,9 @@ function clampWidth(columns: number | undefined): number {
96
96
  return Math.max(40, Math.min(columns as number, 160));
97
97
  }
98
98
 
99
- // --report 的装载与规范化住在 ./report-host.ts(两个宿主共用的唯一联系面)
100
- export { loadHostReport, normalizeHostReport, resolveReportTitle, localizeText } from "./report-host.ts";
99
+ // --report 的装载住在 ../report/runtime/host.ts(两个宿主共用的中性联系面);规范化本身是
100
+ // `defineReport` 自己的职责,不在宿主层重复。
101
+ export { loadHostReport, localizeText } from "../report/runtime/host.ts";
101
102
 
102
103
  export async function runShow(
103
104
  cwd: string,
@@ -163,8 +164,7 @@ async function show(
163
164
  // `@<locator>` 位置参数:身份直达单个 attempt,与 eval id 前缀匹配完全不同的语义
164
165
  // (`@` 打头对 eval id 天然无歧义,见 locator.ts),必须在下面的前缀匹配逻辑之前分流掉,
165
166
  // 不然 "@1x7f3q" 会被当成一个谁都匹配不到的 eval id 前缀,报「no eval match」这种文不对题的
166
- // 错误。这一步只解析并渲染出「当前 show 对单个已解析 attempt 能渲染的东西」(单 eval 详情 /
167
- // 三个证据切面)——真正的 `--source`/`--execution`/`--diff` 统一 attempt 全景是后续阶段。
167
+ // 错误。
168
168
  const locatorArg = patterns.find((p) => p.startsWith(ATTEMPT_LOCATOR_PREFIX));
169
169
  if (locatorArg !== undefined) {
170
170
  if (patterns.length !== 1) {
@@ -199,7 +199,35 @@ async function show(
199
199
  io.out(blocks.join("\n\n") + "\n");
200
200
  return;
201
201
  }
202
- io.out(attemptOverviewText(attemptEvidence, { header, artifactPath, width: io.width }) + "\n");
202
+ // 无证据 flag:选中当前 report definition 里唯一的 attempt-input page,注入这份 evidence,
203
+ // 走与其它 page 完全相同的 resolve → validate → render 管线(docs/feature/reports/show/attempt.md;
204
+ // docs/feature/reports/library/attempt-detail.md「在 show 与 view 怎样渲染」)。不带 --report
205
+ // 时装载内建 standard,其中就带这张 page;--report 指向的自定义报告没有声明 attempt-input page
206
+ // 时报完整用户反馈,不回退到内建详情(三条解决路径都在错误文案里给出)。
207
+ const report = await loadHostReport(cwd, flags.report);
208
+ const attemptPage = report.pages.find((p) => p.input === "attempt");
209
+ if (attemptPage === undefined) {
210
+ const sourceLabel = flags.report ?? "the built-in report";
211
+ throw new ShowError(
212
+ `error: ${sourceLabel} has no attempt-input page — "${locatorArg}" cannot be opened without one. ` +
213
+ `Add one: use \`extends: standard\` (inherits its attempt page), import { standardAttemptPage } from ` +
214
+ `"niceeval/report/built-in" and add it to your pages list, or declare your own \`input: "attempt"\` page.\n`,
215
+ );
216
+ }
217
+ const locale = detectLocale();
218
+ const selection = selectCurrentResults(results, {});
219
+ const meta = await buildHostReportMeta(report, selection);
220
+ const text = await renderHostPageText(
221
+ attemptPage,
222
+ {
223
+ scope: selection,
224
+ results,
225
+ report: meta,
226
+ page: { id: attemptPage.id, input: "attempt", locator: attempt.locator!, evidence: attemptEvidence },
227
+ },
228
+ { width: io.width, locale },
229
+ );
230
+ io.out(text + "\n");
203
231
  return;
204
232
  }
205
233
 
@@ -322,16 +350,23 @@ async function show(
322
350
  };
323
351
  const sourceLabel = flags.report ?? "the built-in report";
324
352
 
325
- // 初始页 = --page 指定的页,缺省第一页(docs/feature/reports/show/reports.md Case 2);
326
- // 本地宿主只 resolve 被打开的这一页——其余页只留 id / title,不触发取数(见 shell.md
327
- // 「行为约束」「本地宿主只 resolve 被打开的页」)。
328
- let page = report.pages[0];
353
+ // 初始页 = --page 指定的页,缺省第一张可导航页(docs/feature/reports/show/reports.md
354
+ // Case 2);本地宿主只 resolve 被打开的这一页——其余页只留 id / title,不触发取数(见
355
+ // shell.md「行为约束」「本地宿主只 resolve 被打开的页」)。navigation:false 的页(参数化
356
+ // attempt 详情)不参与缺省选择,也不能被 --page 直接打开——没有 locator 不能拿 Scope 强行
357
+ // resolve(architecture.md「Attempt 详情是一张参数化 page」)。
358
+ let page = report.pages.find((p) => p.navigation !== false) ?? report.pages[0];
329
359
  if (flags.page !== undefined) {
330
360
  const hit = report.pages.find((p) => p.id === flags.page);
331
361
  if (!hit) {
332
362
  // 用法错误:列出可用页 id(docs/feature/reports/show/reports.md Case 1/2 的报错样例)。
333
363
  throw new ShowError(
334
- `error: page "${flags.page}" not found in ${sourceLabel}. Available pages: ${report.pages.map((p) => p.id).join(", ")}\n`,
364
+ `error: page "${flags.page}" not found in ${sourceLabel}. Available pages: ${report.pages.filter((p) => p.navigation !== false).map((p) => p.id).join(", ")}\n`,
365
+ );
366
+ }
367
+ if (hit.input === "attempt") {
368
+ throw new ShowError(
369
+ `error: page "${hit.id}" in ${sourceLabel} is an attempt-input page and needs a locator — it cannot be opened with --page directly. Use niceeval show @<locator> instead.\n`,
335
370
  );
336
371
  }
337
372
  page = hit;
@@ -339,10 +374,10 @@ async function show(
339
374
 
340
375
  // attemptCommand 留给渲染管线的默认值:AttemptLocator 已经是可直接 `niceeval show @<locator>`
341
376
  // 的真实 CLI 语法,不需要再反查 eval id 拼一条近似命令。
342
- const meta = reportMetaFor(report, page.id, selection.snapshots);
377
+ const meta = await buildHostReportMeta(report, selection);
343
378
  const text = await renderHostPageText(
344
379
  page,
345
- { scope: selection, results, report: meta },
380
+ { scope: selection, results, report: meta, page: { id: page.id, input: "scope" } },
346
381
  {
347
382
  width: io.width,
348
383
  locale,
@@ -350,8 +385,9 @@ async function show(
350
385
  },
351
386
  );
352
387
 
353
- // 页数大于一时尾部附「其余页」索引(只列未渲染的页,不倾倒内容);单页定义没有这段。
354
- const remaining = report.pages.filter((p) => p.id !== page.id);
388
+ // 页数大于一时尾部附「其余页」索引(只列未渲染、且可导航的页,不倾倒内容);单页定义
389
+ // 没有这段;隐藏的 attempt page 不出现在「其余页」里。
390
+ const remaining = report.pages.filter((p) => p.id !== page.id && p.navigation !== false);
355
391
  if (remaining.length === 0) {
356
392
  io.out(text + "\n");
357
393
  return;