niceeval 0.5.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (344) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +317 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/execution-tree.d.ts +103 -0
  9. package/dist/o11y/otlp/parse.d.ts +3 -0
  10. package/dist/o11y/otlp/receiver.d.ts +21 -0
  11. package/dist/o11y/otlp/select.d.ts +22 -0
  12. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  13. package/dist/o11y/types.d.ts +190 -0
  14. package/dist/report/aggregate.d.ts +57 -0
  15. package/dist/report/aggregate.js +212 -0
  16. package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
  17. package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
  18. package/dist/report/built-ins/index.d.ts +1 -0
  19. package/dist/report/built-ins/index.js +2 -0
  20. package/dist/report/components.d.ts +178 -0
  21. package/dist/report/components.js +132 -0
  22. package/dist/report/compute.d.ts +106 -0
  23. package/dist/report/compute.js +618 -0
  24. package/dist/report/flag.d.ts +6 -0
  25. package/dist/report/flag.js +8 -0
  26. package/dist/report/format.d.ts +32 -0
  27. package/dist/report/format.js +126 -0
  28. package/dist/report/index.d.ts +19 -0
  29. package/dist/report/index.js +30 -0
  30. package/dist/report/load.d.ts +12 -0
  31. package/dist/report/load.js +56 -0
  32. package/dist/report/locale.d.ts +83 -0
  33. package/dist/report/locale.js +160 -0
  34. package/dist/report/metrics.d.ts +21 -0
  35. package/dist/report/metrics.js +115 -0
  36. package/dist/report/primitives.d.ts +64 -0
  37. package/dist/report/primitives.js +113 -0
  38. package/dist/report/react/AttemptList.d.ts +23 -0
  39. package/dist/report/react/AttemptList.js +20 -0
  40. package/dist/report/react/DeltaTable.d.ts +8 -0
  41. package/dist/report/react/DeltaTable.js +24 -0
  42. package/dist/report/react/EvalList.d.ts +10 -0
  43. package/dist/report/react/EvalList.js +0 -0
  44. package/dist/report/react/ExperimentList.d.ts +10 -0
  45. package/dist/report/react/ExperimentList.js +15 -0
  46. package/dist/report/react/GroupSummary.d.ts +8 -0
  47. package/dist/report/react/GroupSummary.js +8 -0
  48. package/dist/report/react/MetricBars.d.ts +10 -0
  49. package/dist/report/react/MetricBars.js +48 -0
  50. package/dist/report/react/MetricLine.d.ts +9 -0
  51. package/dist/report/react/MetricLine.js +68 -0
  52. package/dist/report/react/MetricMatrix.d.ts +10 -0
  53. package/dist/report/react/MetricMatrix.js +23 -0
  54. package/dist/report/react/MetricScatter.d.ts +9 -0
  55. package/dist/report/react/MetricScatter.js +100 -0
  56. package/dist/report/react/MetricTable.d.ts +11 -0
  57. package/dist/report/react/MetricTable.js +26 -0
  58. package/dist/report/react/RunOverview.d.ts +8 -0
  59. package/dist/report/react/RunOverview.js +12 -0
  60. package/dist/report/react/Scoreboard.d.ts +8 -0
  61. package/dist/report/react/Scoreboard.js +26 -0
  62. package/dist/report/react/cell.d.ts +9 -0
  63. package/dist/report/react/cell.js +9 -0
  64. package/dist/report/react/chart-math.d.ts +14 -0
  65. package/dist/report/react/chart-math.js +68 -0
  66. package/dist/report/react/colors.d.ts +14 -0
  67. package/dist/report/react/colors.js +45 -0
  68. package/dist/report/react/fixtures.d.ts +19 -0
  69. package/dist/report/react/fixtures.js +393 -0
  70. package/dist/report/react/format.d.ts +3 -0
  71. package/dist/report/react/format.js +7 -0
  72. package/dist/report/react/index.d.ts +6 -0
  73. package/dist/report/react/index.js +14 -0
  74. package/dist/report/report.d.ts +24 -0
  75. package/dist/report/report.js +49 -0
  76. package/dist/report/text/faces.d.ts +29 -0
  77. package/dist/report/text/faces.js +503 -0
  78. package/dist/report/text/layout.d.ts +75 -0
  79. package/dist/report/text/layout.js +173 -0
  80. package/dist/report/text/plot.d.ts +37 -0
  81. package/dist/report/text/plot.js +81 -0
  82. package/dist/report/text/table.d.ts +8 -0
  83. package/dist/report/text/table.js +86 -0
  84. package/dist/report/tree.d.ts +99 -0
  85. package/dist/report/tree.js +228 -0
  86. package/dist/report/types.d.ts +420 -0
  87. package/dist/report/types.js +4 -0
  88. package/dist/report/web.d.ts +15 -0
  89. package/dist/report/web.js +37 -0
  90. package/dist/results/annotated-source.d.ts +61 -0
  91. package/dist/results/attempt-evidence.d.ts +69 -0
  92. package/dist/results/attempt-source.d.ts +15 -0
  93. package/dist/results/locator.d.ts +75 -0
  94. package/dist/results/locator.js +0 -0
  95. package/dist/results/select.d.ts +45 -0
  96. package/dist/results/select.js +230 -0
  97. package/dist/results/source-hash.d.ts +9 -0
  98. package/dist/results/types.d.ts +204 -0
  99. package/dist/runner/eval-source.d.ts +22 -0
  100. package/dist/runner/fingerprint.d.ts +23 -0
  101. package/dist/runner/types.d.ts +332 -0
  102. package/dist/sandbox/docker-stream.d.ts +24 -0
  103. package/dist/sandbox/docker.d.ts +84 -0
  104. package/dist/sandbox/e2b.d.ts +31 -0
  105. package/dist/sandbox/errors.d.ts +4 -0
  106. package/dist/sandbox/local-files.d.ts +6 -0
  107. package/dist/sandbox/paths.d.ts +4 -0
  108. package/dist/sandbox/registry.d.ts +13 -0
  109. package/dist/sandbox/resolve.d.ts +40 -0
  110. package/dist/sandbox/retry.d.ts +10 -0
  111. package/dist/sandbox/shell.d.ts +11 -0
  112. package/dist/sandbox/source-files.d.ts +13 -0
  113. package/dist/sandbox/types.d.ts +203 -0
  114. package/dist/sandbox/vercel.d.ts +37 -0
  115. package/dist/scoring/types.d.ts +70 -0
  116. package/dist/shared/aggregate.d.ts +29 -0
  117. package/dist/shared/aggregate.js +55 -0
  118. package/dist/shared/types.d.ts +27 -0
  119. package/dist/shared/verdict.d.ts +27 -0
  120. package/dist/shared/verdict.js +34 -0
  121. package/dist/tty-line.d.ts +6 -0
  122. package/dist/types.d.ts +7 -0
  123. package/dist/util.d.ts +22 -0
  124. package/docs-site/zh/concepts/adapter.mdx +9 -4
  125. package/docs-site/zh/concepts/assert.mdx +8 -8
  126. package/docs-site/zh/concepts/evals.mdx +5 -5
  127. package/docs-site/zh/concepts/experiment.mdx +3 -1
  128. package/docs-site/zh/concepts/judge.mdx +9 -9
  129. package/docs-site/zh/concepts/overview.mdx +2 -2
  130. package/docs-site/zh/concepts/tier.mdx +6 -6
  131. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  132. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  133. package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
  134. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  135. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  136. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  137. package/docs-site/zh/guides/experiments.mdx +7 -0
  138. package/docs-site/zh/guides/fixtures.mdx +1 -1
  139. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  140. package/docs-site/zh/guides/publish-report.mdx +4 -4
  141. package/docs-site/zh/guides/report-components.mdx +194 -55
  142. package/docs-site/zh/guides/reporters.mdx +2 -2
  143. package/docs-site/zh/guides/results-data.mdx +35 -34
  144. package/docs-site/zh/guides/runner.mdx +2 -2
  145. package/docs-site/zh/guides/sandbox-agent.mdx +10 -5
  146. package/docs-site/zh/guides/sandbox-providers.mdx +102 -0
  147. package/docs-site/zh/guides/viewing-results.mdx +199 -79
  148. package/docs-site/zh/guides/write-experiment.mdx +23 -1
  149. package/docs-site/zh/guides/write-send.mdx +2 -2
  150. package/docs-site/zh/index.mdx +4 -3
  151. package/docs-site/zh/introduction.mdx +2 -1
  152. package/docs-site/zh/quickstart.mdx +5 -3
  153. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  154. package/docs-site/zh/reference/capabilities.mdx +1 -1
  155. package/docs-site/zh/reference/cli.mdx +36 -13
  156. package/docs-site/zh/reference/define-agent.mdx +24 -12
  157. package/docs-site/zh/reference/define-config.mdx +1 -1
  158. package/docs-site/zh/reference/define-eval.mdx +3 -3
  159. package/docs-site/zh/reference/events.mdx +7 -1
  160. package/docs-site/zh/reference/expect.mdx +3 -3
  161. package/package.json +10 -6
  162. package/src/agents/bub.ts +134 -43
  163. package/src/agents/claude-code.test.ts +152 -0
  164. package/src/agents/claude-code.ts +127 -13
  165. package/src/agents/codex.test.ts +192 -0
  166. package/src/agents/codex.ts +151 -12
  167. package/src/agents/index.ts +11 -3
  168. package/src/agents/manifest.ts +41 -0
  169. package/src/agents/plugin-config.test.ts +94 -0
  170. package/src/agents/shared.ts +33 -2
  171. package/src/agents/skills.test.ts +214 -0
  172. package/src/agents/skills.ts +304 -0
  173. package/src/agents/streaming.ts +5 -5
  174. package/src/agents/types.ts +62 -2
  175. package/src/cli.ts +74 -47
  176. package/src/context/context.test.ts +20 -2
  177. package/src/context/context.ts +3 -0
  178. package/src/context/control-flow.ts +1 -1
  179. package/src/context/session.test.ts +92 -1
  180. package/src/context/session.ts +26 -1
  181. package/src/context/types.ts +4 -4
  182. package/src/define.test.ts +82 -0
  183. package/src/define.ts +72 -11
  184. package/src/expect/index.ts +5 -5
  185. package/src/i18n/en.ts +36 -15
  186. package/src/i18n/zh-CN.ts +39 -21
  187. package/src/index.ts +1 -1
  188. package/src/o11y/execution-tree.test.ts +452 -0
  189. package/src/o11y/execution-tree.ts +367 -0
  190. package/src/o11y/otlp/receiver.ts +1 -1
  191. package/src/o11y/otlp/select.ts +7 -3
  192. package/src/o11y/parsers/claude-code.test.ts +175 -0
  193. package/src/o11y/parsers/claude-code.ts +32 -0
  194. package/src/o11y/types.ts +6 -1
  195. package/src/report/aggregate.ts +26 -20
  196. package/src/report/built-in-user-parity.test.tsx +643 -0
  197. package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
  198. package/src/report/built-ins/index.ts +2 -0
  199. package/src/report/components.tsx +223 -50
  200. package/src/report/compute.ts +358 -53
  201. package/src/report/dual-render.test.tsx +1125 -0
  202. package/src/report/flag.ts +1 -1
  203. package/src/report/format.ts +50 -2
  204. package/src/report/index.ts +48 -14
  205. package/src/report/load.ts +1 -1
  206. package/src/report/locale.ts +206 -0
  207. package/src/report/metrics.ts +39 -15
  208. package/src/report/primitives.tsx +100 -2
  209. package/src/report/react/AttemptList.tsx +115 -0
  210. package/src/report/react/DeltaTable.tsx +9 -6
  211. package/src/report/react/EvalList.tsx +0 -0
  212. package/src/report/react/ExperimentList.tsx +108 -0
  213. package/src/report/react/GroupSummary.tsx +66 -0
  214. package/src/report/react/MetricBars.tsx +24 -13
  215. package/src/report/react/MetricLine.tsx +21 -21
  216. package/src/report/react/MetricMatrix.tsx +8 -4
  217. package/src/report/react/MetricScatter.tsx +136 -96
  218. package/src/report/react/MetricTable.tsx +115 -15
  219. package/src/report/react/RunOverview.tsx +48 -26
  220. package/src/report/react/Scoreboard.tsx +15 -7
  221. package/src/report/react/cell.tsx +18 -9
  222. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  223. package/src/report/react/colors.ts +24 -15
  224. package/src/report/react/enhance.js +129 -0
  225. package/src/report/react/fixtures.ts +163 -34
  226. package/src/report/react/format.ts +1 -1
  227. package/src/report/react/index.tsx +22 -6
  228. package/src/report/react/render.test.tsx +164 -31
  229. package/src/report/react/styles.css +405 -213
  230. package/src/report/report.test.ts +403 -94
  231. package/src/report/report.ts +29 -11
  232. package/src/report/text/faces.ts +303 -95
  233. package/src/report/text/layout.ts +76 -11
  234. package/src/report/text/table.ts +98 -0
  235. package/src/report/tree.ts +103 -20
  236. package/src/report/types.ts +196 -25
  237. package/src/report/web.ts +46 -19
  238. package/src/results/annotated-source.test.ts +146 -0
  239. package/src/results/annotated-source.ts +128 -0
  240. package/src/results/attempt-evidence.test.ts +158 -0
  241. package/src/results/attempt-evidence.ts +158 -0
  242. package/src/results/attempt-source.ts +48 -0
  243. package/src/results/copy.ts +154 -139
  244. package/src/results/format.ts +54 -47
  245. package/src/results/host-equivalence.test.ts +527 -0
  246. package/src/results/index.ts +56 -14
  247. package/src/results/locator.test.ts +198 -0
  248. package/src/results/locator.ts +0 -0
  249. package/src/results/open.ts +392 -175
  250. package/src/results/results.test.ts +943 -494
  251. package/src/results/select.ts +129 -23
  252. package/src/results/skipped-notice.ts +0 -0
  253. package/src/results/source-hash.ts +28 -0
  254. package/src/results/types.ts +98 -62
  255. package/src/results/writer.ts +304 -176
  256. package/src/runner/attempt.test.ts +170 -0
  257. package/src/runner/attempt.ts +97 -15
  258. package/src/runner/discover.test.ts +65 -0
  259. package/src/runner/discover.ts +6 -2
  260. package/src/runner/eval-source.test.ts +121 -0
  261. package/src/runner/eval-source.ts +45 -0
  262. package/src/runner/fingerprint.ts +54 -1
  263. package/src/runner/remote-sandbox.ts +1 -1
  264. package/src/runner/report.test.ts +6 -8
  265. package/src/runner/report.ts +5 -4
  266. package/src/runner/reporters/artifacts.ts +27 -41
  267. package/src/runner/reporters/braintrust.test.ts +3 -3
  268. package/src/runner/reporters/braintrust.ts +1 -1
  269. package/src/runner/reporters/console.ts +6 -6
  270. package/src/runner/reporters/json.ts +4 -4
  271. package/src/runner/reporters/live.test.ts +56 -0
  272. package/src/runner/reporters/live.ts +36 -10
  273. package/src/runner/reporters/quiet.test.ts +66 -0
  274. package/src/runner/reporters/quiet.ts +49 -0
  275. package/src/runner/reporters/shared.ts +5 -5
  276. package/src/runner/reporters/table.ts +17 -17
  277. package/src/runner/run.ts +61 -102
  278. package/src/runner/types.ts +43 -32
  279. package/src/sandbox/checkpoint.ts +1 -1
  280. package/src/sandbox/docker.ts +15 -2
  281. package/src/sandbox/e2b.ts +11 -5
  282. package/src/sandbox/errors.ts +13 -0
  283. package/src/sandbox/index.ts +3 -3
  284. package/src/sandbox/local-files.ts +1 -1
  285. package/src/sandbox/registry.ts +6 -5
  286. package/src/sandbox/resolve.ts +38 -23
  287. package/src/sandbox/retry.test.ts +68 -0
  288. package/src/sandbox/retry.ts +44 -0
  289. package/src/sandbox/types.ts +69 -28
  290. package/src/sandbox/vercel.ts +17 -3
  291. package/src/scoring/collector.ts +3 -3
  292. package/src/scoring/judge.ts +2 -2
  293. package/src/scoring/scoped.ts +13 -1
  294. package/src/scoring/types.ts +7 -7
  295. package/src/scoring/verdict.ts +4 -4
  296. package/src/shared/aggregate.ts +23 -3
  297. package/src/shared/types.ts +2 -2
  298. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  299. package/src/show/compose.ts +12 -132
  300. package/src/show/index.ts +125 -94
  301. package/src/show/render.ts +660 -160
  302. package/src/show/show.test.ts +472 -253
  303. package/src/source-loc.ts +1 -1
  304. package/src/tty-line.ts +32 -0
  305. package/src/types.ts +2 -2
  306. package/src/util.test.ts +21 -1
  307. package/src/util.ts +5 -1
  308. package/src/view/app/App.tsx +33 -198
  309. package/src/view/app/components/AttemptModal.tsx +6 -6
  310. package/src/view/app/components/CodeView.tsx +2 -2
  311. package/src/view/app/components/CopyControls.tsx +32 -36
  312. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  313. package/src/view/app/components/ui/badge.tsx +1 -1
  314. package/src/view/app/components/ui/dialog.tsx +2 -2
  315. package/src/view/app/i18n.ts +40 -31
  316. package/src/view/app/lib/artifact-url.ts +1 -1
  317. package/src/view/app/lib/attempt-route.test.ts +37 -47
  318. package/src/view/app/lib/attempt-route.ts +26 -26
  319. package/src/view/app/lib/rows.ts +9 -112
  320. package/src/view/app/lib/verdict.ts +25 -0
  321. package/src/view/app/main.tsx +9 -12
  322. package/src/view/app/pages/RunsPage.tsx +4 -4
  323. package/src/view/app/pages/TracesPage.tsx +3 -3
  324. package/src/view/app/shared.ts +0 -1
  325. package/src/view/app/types.ts +6 -51
  326. package/src/view/artifact-serving.test.ts +136 -0
  327. package/src/view/client-dist/app.css +1 -1
  328. package/src/view/client-dist/app.js +21 -21
  329. package/src/view/data.test.ts +214 -143
  330. package/src/view/data.ts +166 -120
  331. package/src/view/index.ts +23 -11
  332. package/src/view/server.ts +52 -17
  333. package/src/view/shared/types.ts +22 -31
  334. package/src/view/styles.css +117 -327
  335. package/src/view/view-report.test.ts +146 -98
  336. package/docs-site/zh/guides/sandbox-backends.mdx +0 -75
  337. package/src/report/default-report.tsx +0 -222
  338. package/src/report/dual-face.test.tsx +0 -527
  339. package/src/report/react/CaseList.tsx +0 -70
  340. package/src/view/app/components/CostScoreChart.tsx +0 -147
  341. package/src/view/app/components/ExperimentTable.tsx +0 -284
  342. package/src/view/app/components/GroupSelector.tsx +0 -61
  343. package/src/view/app/components/primitives.tsx +0 -43
  344. package/src/view/app/lib/outcome.ts +0 -82
@@ -1,140 +1,20 @@
1
- // show 宿主的选集合成与时间轴口径(docs-site/zh/guides/viewing-results.mdx 是行为规范)。
1
+ // show 专属的跨 run 时间轴口径(--history;docs-site/zh/guides/viewing-results.mdx 是行为规范)。
2
2
  //
3
- // 「现刻水位」= 每个 experiment × eval 取时间上最新的那份判决,跨 run 合成:
4
- // results.latest() 只挑「每实验最新快照」,带 eval 前缀的局部重跑会产出残缺快照;
5
- // 榜单承诺「不会因为一次局部重跑变残缺」,所以宿主在实验的全部历史快照上逐 eval
6
- // 向更早的 run 补齐,再把合成好的选集注入报告槽——内置默认报告与 --report 吃同一份,
7
- // 官方榜单口径 = 宿主注入口径。本文件只消费 niceeval/results 的读取面。
3
+ // 现刻水位 Selection(两个宿主共用)住在 ../results/select.ts selectCurrentResults;
4
+ // 本文件只留 show 独有的时间轴计算:每个快照 / 每次真实执行一行,resume 携带的复印件不占行。
5
+ // 数据只消费 niceeval/results 的读取面。
8
6
 
9
- import { foldEvalOutcome } from "../shared/outcome.ts";
10
- import { evalPrefixPredicate } from "../report/aggregate.ts";
7
+ import { foldEvalVerdict } from "../shared/verdict.ts";
11
8
  import { attemptCostUSD } from "../report/metrics.ts";
12
- import { makeSelection } from "../results/select.ts";
13
- import type { ResultOutcome } from "../types.ts";
14
- import type {
15
- AttemptHandle,
16
- Eval,
17
- Experiment,
18
- Results,
19
- Selection,
20
- SelectionWarning,
21
- Snapshot,
22
- } from "../results/index.ts";
23
-
24
- export interface ComposeOptions {
25
- /** experiment id 前缀(--experiment),与 latest({ experiments }) 同一分段匹配语义。 */
26
- experiment?: string;
27
- /** eval id 前缀(位置参数),收窄选集覆盖的 eval;覆盖警告分母 = 已知并集 ∩ 范围。 */
28
- patterns?: string[];
29
- }
30
-
31
- /** --experiment 的实验过滤,同 results.latest({ experiments }) 的分段前缀语义。 */
32
- export function filterExperiments(experiments: Experiment[], prefix?: string): Experiment[] {
33
- if (prefix === undefined) return experiments;
34
- const p = prefix.replace(/\/+$/, "");
35
- return experiments.filter((exp) => exp.id === p || exp.id.startsWith(p + "/"));
36
- }
37
-
38
- /**
39
- * 合成「现刻水位」选集:每个实验一份合成快照,快照里每道题的判决取该题最后一次
40
- * 出现的快照(--resume 携带的复印件身份与原判决相同,取到哪份内容都一致)。
41
- * 警告随选集重算:partial-coverage 的分母 = 已知并集 ∩ 范围;stale / synthetic
42
- * 与 results.latest() 同口径。
43
- */
44
- export function composeShowSelection(results: Results, opts: ComposeOptions = {}): Selection {
45
- const match =
46
- opts.patterns && opts.patterns.length > 0 ? evalPrefixPredicate(opts.patterns) : () => true;
47
- const experiments = filterExperiments(results.experiments, opts.experiment);
48
-
49
- const snapshots: Snapshot[] = [];
50
- const warnings: SelectionWarning[] = [];
51
-
52
- for (const exp of experiments) {
53
- // 逐题取最新:快照按最新在前,首个出现即最新判决
54
- const taken = new Map<string, { ev: Eval; snapshot: Snapshot }>();
55
- for (const snapshot of exp.snapshots) {
56
- for (const ev of snapshot.evals) {
57
- if (!match(ev.id) || taken.has(ev.id)) continue;
58
- taken.set(ev.id, { ev, snapshot });
59
- }
60
- }
61
- if (taken.size === 0) continue;
62
-
63
- const picks = [...taken.values()].sort((a, b) => a.ev.id.localeCompare(b.ev.id));
64
- let startedAt = "";
65
- let newest: Snapshot = picks[0].snapshot;
66
- for (const pick of picks) {
67
- if (pick.snapshot.startedAt > startedAt) {
68
- startedAt = pick.snapshot.startedAt;
69
- newest = pick.snapshot;
70
- }
71
- }
72
- const evals = picks.map((p) => p.ev);
73
- const base = exp.latest;
74
- snapshots.push({
75
- experimentId: exp.id,
76
- startedAt,
77
- agent: base.agent,
78
- ...(base.model !== undefined ? { model: base.model } : {}),
79
- ...(base.producer ? { producer: base.producer } : {}),
80
- schemaVersion: base.schemaVersion,
81
- evals,
82
- attempts: evals.flatMap((ev) => ev.attempts),
83
- runDir: newest.runDir,
84
- ...(base.synthetic ? { synthetic: true } : {}),
85
- ...(base.knownEvalIds ? { knownEvalIds: [...base.knownEvalIds] } : {}),
86
- });
87
-
88
- // 残缺检测:跨 run 补齐后仍缺,只可能是历史上见过(或 knownEvalIds 声明过)
89
- // 却从未在可读落盘里出现的题 —— 分母收窄到范围内,不让范围外的缺口刷屏。
90
- const total = exp.evalIds.filter(match).length;
91
- if (evals.length < total) {
92
- const hint = base.synthetic
93
- ? "re-run the experiment for a full snapshot"
94
- : `re-run \`niceeval exp ${exp.id}\` for a full snapshot`;
95
- warnings.push({
96
- kind: "partial-coverage",
97
- experimentId: exp.id,
98
- covered: evals.length,
99
- total,
100
- message: `verdicts cover ${evals.length} of ${total} evals seen in history; ${hint}`,
101
- });
102
- }
103
- }
104
-
105
- let latestStartedAt = "";
106
- for (const snapshot of snapshots) {
107
- if (snapshot.startedAt > latestStartedAt) latestStartedAt = snapshot.startedAt;
108
- }
109
- for (const snapshot of snapshots) {
110
- if (snapshot.startedAt < latestStartedAt) {
111
- warnings.push({
112
- kind: "stale-snapshot",
113
- experimentId: snapshot.experimentId,
114
- startedAt: snapshot.startedAt,
115
- latestStartedAt,
116
- message: `verdicts for "${snapshot.experimentId}" were produced at ${snapshot.startedAt}, before the latest run in this selection (${latestStartedAt})`,
117
- });
118
- }
119
- if (snapshot.synthetic) {
120
- warnings.push({
121
- kind: "synthetic-experiment-id",
122
- experimentId: snapshot.experimentId,
123
- runDir: snapshot.runDir.dir,
124
- message: `run "${snapshot.runDir.dir}" has results without experimentId; grouped as "${snapshot.experimentId}" by agent/model`,
125
- });
126
- }
127
- }
128
-
129
- return makeSelection(snapshots, warnings);
130
- }
9
+ import type { Verdict } from "../types.ts";
10
+ import type { AttemptHandle, Experiment } from "../results/index.ts";
131
11
 
132
12
  // ───────────────────────── 时间轴(--history)─────────────────────────
133
13
 
134
14
  export interface EvalHistoryRow {
135
15
  /** 该次真实执行所在 run(快照)的时刻。 */
136
16
  startedAt: string;
137
- outcome: ResultOutcome;
17
+ verdict: Verdict;
138
18
  attempts: number;
139
19
  costUSD: number | null;
140
20
  /** 最新一次 attempt 的第一条失败断言("gate calledTool(...)")。 */
@@ -151,7 +31,7 @@ function attemptKey(attempt: AttemptHandle): string | undefined {
151
31
 
152
32
  /**
153
33
  * 单 eval 的跨 run 时间轴:每次真实执行一行,新→旧。--resume 携带的复印件
154
- * (身份键与原判决相同的条目)不占行 —— 否则趋势会被复印件灌满假数据。
34
+ * (身份键与原判定相同的条目)不占行 —— 否则趋势会被复印件灌满假数据。
155
35
  */
156
36
  export function evalHistory(exp: Experiment, evalId: string): EvalHistoryRow[] {
157
37
  const rows: EvalHistoryRow[] = [];
@@ -171,7 +51,7 @@ export function evalHistory(exp: Experiment, evalId: string): EvalHistoryRow[] {
171
51
  seen.add(key);
172
52
  fresh.push(attempt);
173
53
  }
174
- if (fresh.length === 0) continue; // 纯复印件:判决在更早的行里已经出现过
54
+ if (fresh.length === 0) continue; // 纯复印件:判定在更早的行里已经出现过
175
55
  let cost: number | null = null;
176
56
  for (const attempt of fresh) {
177
57
  const c = attemptCostUSD(attempt.result);
@@ -181,7 +61,7 @@ export function evalHistory(exp: Experiment, evalId: string): EvalHistoryRow[] {
181
61
  const failed = latest.result.assertions.find((a) => !a.passed);
182
62
  rows.push({
183
63
  startedAt: snapshot.startedAt,
184
- outcome: foldEvalOutcome(fresh.map((a) => a.result)),
64
+ verdict: foldEvalVerdict(fresh.map((a) => a.result)),
185
65
  attempts: fresh.length,
186
66
  costUSD: cost,
187
67
  ...(failed ? { failedAssertion: `${failed.severity} ${failed.name}` } : {}),
@@ -203,7 +83,7 @@ export function experimentHistory(exp: Experiment): ExperimentHistoryRow[] {
203
83
  return exp.snapshots.map((snapshot) => {
204
84
  let passed = 0;
205
85
  for (const ev of snapshot.evals) {
206
- if (foldEvalOutcome(ev.attempts.map((a) => a.result)) === "passed") passed += 1;
86
+ if (foldEvalVerdict(ev.attempts.map((a) => a.result)) === "passed") passed += 1;
207
87
  }
208
88
  let cost: number | null = null;
209
89
  for (const attempt of snapshot.attempts) {
package/src/show/index.ts CHANGED
@@ -1,53 +1,72 @@
1
1
  // niceeval show —— 终端宿主(行为规范:docs-site/zh/guides/viewing-results.mdx;
2
- // 宿主组合语义:docs/reports.md「宿主输入的组合语义」)。
2
+ // 宿主组合语义:docs/feature/reports/architecture.md「Selection 是计算入口」)。
3
3
  //
4
- // 位置参数 = eval id 前缀(选「看哪些 eval」),flag 选「看哪个切面」:
5
- // 裸跑 / 前缀 报告槽 —— 内置默认报告的 text 面(show show --report <内置默认报告>)
4
+ // 位置参数 = eval id 前缀,或 `@<locator>`(精确指名单个 attempt,见 results/locator.ts):
5
+ // 裸跑 / 前缀 show 专用的紧凑 attempt 索引( eval 前缀仍进入详情)
6
6
  // 恰好一个 eval 单 eval 详情(attempt / 断言明细,宿主本体)
7
- // --transcript / --trace / --diff[=路径] 证据切面(宿主本体):出现即走证据室,不渲染报告槽
7
+ // @<locator> 精确 attempt:无证据 flag 紧凑全景;带 flag → 对应证据切面
8
+ // --eval / --execution / --diff[=路径] 证据切面(宿主本体):出现即走证据室,不渲染报告槽
8
9
  // --history 跨 run 时间轴(内置趋势视图),与 --report 互斥
9
- // --report <文件> 整槽换成用户报告;位置前缀 / --run / --experiment 先收窄选集再注入
10
- // --run <目录> 结果根换成该目录;--experiment 选集只留该实验;--attempt 指定详情/证据的 attempt
10
+ // --report <文件> 整槽换成用户报告;位置前缀 / --run / --experiment 先收窄 Selection 再注入
11
+ // --run <目录> 结果根换成该目录;--experiment Selection 只留该实验
11
12
  //
12
- // 数据全部走 niceeval/results 的读取面(openResults + 合成选集),不自己爬目录。
13
+ // 数据全部走 niceeval/results 的读取面(openResults + 合成 Selection + loadAttemptEvidence),
14
+ // 不自己爬目录、不自己重算证据 capability。
13
15
 
14
16
  import { existsSync } from "node:fs";
15
17
  import { join, resolve } from "node:path";
16
- import { openResults, type AttemptRef, type Results } from "../results/index.ts";
17
- import { defineReport, renderReportToText, type ReportDefinition } from "../report/report.ts";
18
- import { ReportLoadError, loadReportFile } from "../report/load.ts";
19
- import { DefaultReport } from "../report/default-report.tsx";
20
- import { t } from "../i18n/index.ts";
21
18
  import {
22
- composeShowSelection,
23
- evalHistory,
24
- experimentHistory,
25
- filterExperiments,
26
- } from "./compose.ts";
19
+ openResults,
20
+ resolveLocator,
21
+ loadAttemptEvidence,
22
+ ATTEMPT_LOCATOR_PREFIX,
23
+ LocatorNotFoundError,
24
+ MalformedLocatorError,
25
+ } from "../results/index.ts";
26
+ // report.ts / load.ts have no JSX of their own, but ReportDefinition / ReportLoadError must
27
+ // come from the SAME module instance CostPassRateComparison (built-ins, .tsx) is built
28
+ // against — `unique symbol` branding and `instanceof` are keyed by declaration site, so a raw
29
+ // src copy and the compiled dist copy of "the same" ReportDefinition are, to TypeScript and to
30
+ // `instanceof`, two different types. The package-owned report runtime ships as precompiled
31
+ // ESM (dist/report/**, built by `pnpm run build:report`, immune to a consumer's cwd/tsconfig
32
+ // when niceeval is linked in — see tsconfig.report-build.json); show pulls all of it from
33
+ // there, not just the .tsx-touching pieces.
34
+ import { renderReportToText } from "../../dist/report/report.js";
35
+ import { ReportLoadError, loadReportFile } from "../../dist/report/load.js";
36
+ import { detectLocale, t } from "../i18n/index.ts";
37
+ import { foldEvalVerdict } from "../shared/verdict.ts";
38
+ import { selectCurrentResults, filterExperiments } from "../results/select.ts";
39
+ import { evalHistory, experimentHistory } from "./compose.ts";
27
40
  import {
28
41
  attemptArtifactsPath,
42
+ attemptEvidenceHeader,
43
+ attemptIndexLine,
44
+ attemptOverviewText,
29
45
  attemptsOfEval,
46
+ cheapCapabilities,
30
47
  diffText,
31
- displayAttemptNumber,
32
48
  evalDetailText,
33
49
  evalHistoryText,
50
+ evalSourceText,
51
+ executionText,
34
52
  experimentHistoryText,
35
53
  pickDetailAttempt,
36
- traceText,
37
- transcriptText,
54
+ skippedRunsText,
55
+ showIndexText,
56
+ verdictReasonLine,
38
57
  } from "./render.ts";
39
58
 
40
59
  export interface ShowFlags {
41
- transcript?: boolean;
42
- trace?: boolean;
60
+ /** 该 attempt 运行时保存的 Eval 源码,断言标回源码行(证据切面)。 */
61
+ eval?: boolean;
62
+ /** 该 attempt 的标准执行事件流 + OTel enrichment(证据切面)。 */
63
+ execution?: boolean;
43
64
  /** --diff(文件级摘要)。 */
44
65
  diff?: boolean;
45
66
  /** --diff=<路径>(单个文件的完整改动;路径必须 = 连写,位置参数永远留给 eval id 前缀)。 */
46
67
  diffPath?: string;
47
68
  history?: boolean;
48
69
  experiment?: string;
49
- /** 人看的 1 计序号(详情块显示的 attempt 3 就传 3)。 */
50
- attempt?: number;
51
70
  run?: string;
52
71
  report?: string;
53
72
  }
@@ -63,33 +82,13 @@ export interface ShowIO {
63
82
  /** 可预期的用户错误:打一句英文直说问题与下一步,退出码 1,不抛堆栈。 */
64
83
  class ShowError extends Error {}
65
84
 
66
- /** 内置默认报告:报告槽的出厂填充 —— `niceeval show` ≡ `show --report <这一份>`。 */
67
- const builtinDefaultReport: ReportDefinition = defineReport(() => ({
68
- type: DefaultReport,
69
- props: {},
70
- }));
71
-
72
85
  function clampWidth(columns: number | undefined): number {
73
86
  if (!Number.isFinite(columns) || (columns ?? 0) <= 0) return 80;
74
87
  return Math.max(40, Math.min(columns as number, 160));
75
88
  }
76
89
 
77
90
  // --report 的装载移到中性模块(两个宿主共用),show 的导出面与错误行为不变。
78
- export { loadReportFile } from "../report/load.ts";
79
-
80
- /** 报告里的下钻命令:AttemptRef → `niceeval show <eval id>`(查不到时退 view 深链)。 */
81
- function makeAttemptCommand(results: Results): (ref: AttemptRef) => string {
82
- const byRef = new Map<string, string>();
83
- for (const run of results.runDirs) {
84
- for (const attempt of run.attempts) {
85
- byRef.set(`${attempt.ref.run}/${attempt.ref.result}`, attempt.evalId);
86
- }
87
- }
88
- return (ref) => {
89
- const id = byRef.get(`${ref.run}/${ref.result}`);
90
- return id !== undefined ? `niceeval show ${id}` : `niceeval view "#/attempt/${ref.run}/${ref.result}"`;
91
- };
92
- }
91
+ export { loadReportFile } from "../../dist/report/load.js";
93
92
 
94
93
  export async function runShow(
95
94
  cwd: string,
@@ -122,9 +121,9 @@ async function show(
122
121
  flags: ShowFlags,
123
122
  io: { out: (s: string) => void; err: (s: string) => void; width: number; now: number },
124
123
  ): Promise<void> {
125
- const evidence = flags.transcript === true || flags.trace === true || flags.diff === true || flags.diffPath !== undefined;
124
+ const evidence = flags.eval === true || flags.execution === true || flags.diff === true || flags.diffPath !== undefined;
126
125
 
127
- // 组合语义矩阵(docs/reports.md):--history 与 --report 互斥,先于任何 IO 报出来。
126
+ // 组合语义矩阵(docs/feature/reports/show.md「选择结果范围」):--history 与 --report 互斥,先于任何 IO 报出来。
128
127
  if (flags.history && flags.report !== undefined) {
129
128
  throw new ShowError(t("cli.show.historyReportConflict"));
130
129
  }
@@ -136,13 +135,47 @@ async function show(
136
135
 
137
136
  const results = await openResults(root);
138
137
  if (results.experiments.length === 0) {
139
- const skipped =
140
- results.skipped.length > 0
141
- ? `\n${results.skipped.map((s) => ` skipped ${s.dir} (${s.reason})`).join("\n")}\n`
142
- : "";
138
+ const skipped = results.skipped.length > 0 ? `\n${skippedRunsText(results.skipped, root, cwd)}\n` : "";
143
139
  throw new ShowError(t("cli.show.noResults", { root }) + skipped);
144
140
  }
145
141
 
142
+ // `@<locator>` 位置参数:身份直达单个 attempt,与 eval id 前缀匹配完全不同的语义
143
+ // (`@` 打头对 eval id 天然无歧义,见 locator.ts),必须在下面的前缀匹配逻辑之前分流掉,
144
+ // 不然 "@1x7f3q" 会被当成一个谁都匹配不到的 eval id 前缀,报「no eval match」这种文不对题的
145
+ // 错误。这一步只解析并渲染出「当前 show 对单个已解析 attempt 能渲染的东西」(单 eval 详情 /
146
+ // 三个证据切面)——真正的 `--eval`/`--execution`/`--diff` 统一 attempt 全景是后续阶段。
147
+ const locatorArg = patterns.find((p) => p.startsWith(ATTEMPT_LOCATOR_PREFIX));
148
+ if (locatorArg !== undefined) {
149
+ if (patterns.length !== 1) {
150
+ throw new ShowError(
151
+ `An attempt locator ("${locatorArg}") must be the only positional argument; got ${patterns.length}: ${patterns.join(", ")}.`,
152
+ );
153
+ }
154
+ let attempt;
155
+ try {
156
+ attempt = resolveLocator(results, locatorArg);
157
+ } catch (e) {
158
+ if (e instanceof MalformedLocatorError) throw new ShowError(t("cli.show.locatorMalformed", { message: e.message }));
159
+ if (e instanceof LocatorNotFoundError) throw new ShowError(t("cli.show.locatorNotFound", { message: e.message }));
160
+ throw e;
161
+ }
162
+ const attemptEvidence = await loadAttemptEvidence(attempt);
163
+ const header = attemptEvidenceHeader(attemptEvidence);
164
+ const artifactPath = attemptArtifactsPath(attempt, cwd);
165
+ if (evidence) {
166
+ const blocks: string[] = [];
167
+ if (flags.eval) blocks.push(evalSourceText(attemptEvidence, { header, artifactPath, width: io.width }));
168
+ if (flags.execution) blocks.push(executionText(attemptEvidence, { header, artifactPath, width: io.width }));
169
+ if (flags.diff || flags.diffPath !== undefined) {
170
+ blocks.push(diffText({ header, diff: attemptEvidence.diff, artifactPath, file: flags.diffPath }));
171
+ }
172
+ io.out(blocks.join("\n\n") + "\n");
173
+ return;
174
+ }
175
+ io.out(attemptOverviewText(attemptEvidence, { header, artifactPath, width: io.width }) + "\n");
176
+ return;
177
+ }
178
+
146
179
  if (flags.experiment !== undefined && filterExperiments(results.experiments, flags.experiment).length === 0) {
147
180
  throw new ShowError(
148
181
  t("cli.show.noExperimentMatch", {
@@ -152,7 +185,7 @@ async function show(
152
185
  );
153
186
  }
154
187
 
155
- const selection = composeShowSelection(results, { experiment: flags.experiment, patterns });
188
+ const selection = selectCurrentResults(results, { experiment: flags.experiment, patterns });
156
189
  const matchedEvalIds = [
157
190
  ...new Set(selection.snapshots.flatMap((s) => s.evals.map((e) => e.id))),
158
191
  ].sort();
@@ -169,31 +202,36 @@ async function show(
169
202
  // 证据切面是宿主本体:出现即走证据室,不渲染报告槽(与默认报告同规则)。
170
203
  if (evidence) {
171
204
  if (matchedEvalIds.length !== 1) {
172
- throw new ShowError(t("cli.show.evidenceNeedsEval", { matched: matchedEvalIds.length }));
205
+ // 撞多个 eval 时不止说「有几个」,直接给紧凑索引(locator + capability 字母 + 失败原因)
206
+ // 让 agent 一步摘到 `@<locator>`,不必再跑一轮 `show <eval id>` 才知道选谁。
207
+ const index = matchedEvalIds
208
+ .map((evalId) => {
209
+ const attempts = attemptsOfEval(selection.snapshots, evalId);
210
+ const rep = pickDetailAttempt(attempts);
211
+ const verdict = foldEvalVerdict(attempts.map((a) => a.result));
212
+ return attemptIndexLine({
213
+ evalId,
214
+ verdict,
215
+ locator: rep?.locator,
216
+ capabilities: rep ? cheapCapabilities(rep.result) : { eval: false, execution: false, timing: false, diff: false },
217
+ reason: rep ? verdictReasonLine(rep.result) : undefined,
218
+ });
219
+ })
220
+ .join("\n");
221
+ throw new ShowError(t("cli.show.evidenceNeedsEval", { matched: matchedEvalIds.length, index }));
173
222
  }
174
223
  const evalId = matchedEvalIds[0];
175
224
  const attempts = attemptsOfEval(selection.snapshots, evalId);
176
- const picked = pickDetailAttempt(attempts, flags.attempt);
177
- if (!picked) {
178
- throw new ShowError(
179
- t("cli.show.attemptNotFound", {
180
- attempt: flags.attempt ?? "?",
181
- evalId,
182
- available: attempts.map((a) => displayAttemptNumber(a)).join(", ") || "(none)",
183
- }),
184
- );
185
- }
186
- const header = `attempt ${displayAttemptNumber(picked)} · ${picked.experimentId} · ${picked.result.outcome}`;
225
+ const picked = pickDetailAttempt(attempts);
226
+ if (!picked) throw new Error(`internal error: eval "${evalId}" matched by selection but has no attempts`);
227
+ const attemptEvidence = await loadAttemptEvidence(picked);
228
+ const header = attemptEvidenceHeader(attemptEvidence);
187
229
  const artifactPath = attemptArtifactsPath(picked, cwd);
188
230
  const blocks: string[] = [];
189
- if (flags.transcript) {
190
- blocks.push(transcriptText({ header, events: await picked.events(), artifactPath, width: io.width }));
191
- }
192
- if (flags.trace) {
193
- blocks.push(traceText({ header, spans: await picked.trace(), artifactPath, width: io.width }));
194
- }
231
+ if (flags.eval) blocks.push(evalSourceText(attemptEvidence, { header, artifactPath, width: io.width }));
232
+ if (flags.execution) blocks.push(executionText(attemptEvidence, { header, artifactPath, width: io.width }));
195
233
  if (flags.diff || flags.diffPath !== undefined) {
196
- blocks.push(diffText({ header, diff: await picked.diff(), artifactPath, file: flags.diffPath }));
234
+ blocks.push(diffText({ header, diff: attemptEvidence.diff, artifactPath, file: flags.diffPath }));
197
235
  }
198
236
  io.out(blocks.join("\n\n") + "\n");
199
237
  return;
@@ -219,20 +257,13 @@ async function show(
219
257
  return;
220
258
  }
221
259
 
222
- // 单 eval 详情(宿主本体);--report 在场时报告槽优先,前缀只用来收窄选集。
260
+ // 单 eval 详情(宿主本体);--report 在场时报告槽优先,前缀只用来收窄 Selection。
261
+ // 挑哪个 attempt 展开明细不再收数字 --attempt——pickDetailAttempt 的默认启发式
262
+ // (最新一次失败,没有失败挑最新一次)是唯一路径;精确选某一次走 `@<locator>`。
223
263
  if (flags.report === undefined && patterns.length > 0 && matchedEvalIds.length === 1) {
224
264
  const evalId = matchedEvalIds[0];
225
265
  const attempts = attemptsOfEval(selection.snapshots, evalId);
226
- const detail = pickDetailAttempt(attempts, flags.attempt);
227
- if (flags.attempt !== undefined && !detail) {
228
- throw new ShowError(
229
- t("cli.show.attemptNotFound", {
230
- attempt: flags.attempt,
231
- evalId,
232
- available: attempts.map((a) => displayAttemptNumber(a)).join(", ") || "(none)",
233
- }),
234
- );
235
- }
266
+ const detail = pickDetailAttempt(attempts);
236
267
  io.out(
237
268
  evalDetailText({
238
269
  evalId,
@@ -246,18 +277,18 @@ async function show(
246
277
  return;
247
278
  }
248
279
 
249
- if (flags.attempt !== undefined) {
250
- // --attempt 只对单 eval 的详情/证据生效;报告槽/榜单下无从对位。
251
- throw new ShowError(t("cli.show.attemptNeedsEval"));
280
+ // 裸 show 是终端诊断索引(docs/feature/reports/show.md),不是 web 默认分析报告的 text 面。
281
+ // --report 才进入可替换报告槽;view 仍可选择 CostPassRateComparison 作为自己的默认面。
282
+ // locale = CLI 界面语言(NICEEVAL_LANG / LC_* / LANG 检测):报告 chrome 文案跟随
283
+ // 终端语言(docs/feature/reports/library.md「locale:渲染面的语言」);Locale 与 ReportLocale 同为
284
+ // "en" | "zh-CN",直接传递。
285
+ if (flags.report === undefined) {
286
+ io.out(showIndexText(selection, io.width) + "\n");
287
+ return;
252
288
  }
253
-
254
- // 报告槽:--report 整槽替换,否则内置默认报告(同一条渲染路径)。
255
- const definition =
256
- flags.report !== undefined ? await loadReportFile(cwd, flags.report) : builtinDefaultReport;
257
- const text = await renderReportToText(
258
- definition,
259
- { selection, results },
260
- { width: io.width, attemptCommand: makeAttemptCommand(results) },
261
- );
289
+ const definition = await loadReportFile(cwd, flags.report);
290
+ // attemptCommand 留给 renderReportToText 的默认值:AttemptLocator 已经是可直接 `niceeval show
291
+ // @<locator>` 的真实 CLI 语法,不需要再反查 eval id 拼一条近似命令(见 tree.ts 的默认实现)。
292
+ const text = await renderReportToText(definition, { selection, results }, { width: io.width, locale: detectLocale() });
262
293
  io.out(text + "\n");
263
294
  }