niceeval 0.5.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (344) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +317 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/execution-tree.d.ts +103 -0
  9. package/dist/o11y/otlp/parse.d.ts +3 -0
  10. package/dist/o11y/otlp/receiver.d.ts +21 -0
  11. package/dist/o11y/otlp/select.d.ts +22 -0
  12. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  13. package/dist/o11y/types.d.ts +190 -0
  14. package/dist/report/aggregate.d.ts +57 -0
  15. package/dist/report/aggregate.js +212 -0
  16. package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
  17. package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
  18. package/dist/report/built-ins/index.d.ts +1 -0
  19. package/dist/report/built-ins/index.js +2 -0
  20. package/dist/report/components.d.ts +178 -0
  21. package/dist/report/components.js +132 -0
  22. package/dist/report/compute.d.ts +106 -0
  23. package/dist/report/compute.js +618 -0
  24. package/dist/report/flag.d.ts +6 -0
  25. package/dist/report/flag.js +8 -0
  26. package/dist/report/format.d.ts +32 -0
  27. package/dist/report/format.js +126 -0
  28. package/dist/report/index.d.ts +19 -0
  29. package/dist/report/index.js +30 -0
  30. package/dist/report/load.d.ts +12 -0
  31. package/dist/report/load.js +56 -0
  32. package/dist/report/locale.d.ts +83 -0
  33. package/dist/report/locale.js +160 -0
  34. package/dist/report/metrics.d.ts +21 -0
  35. package/dist/report/metrics.js +115 -0
  36. package/dist/report/primitives.d.ts +64 -0
  37. package/dist/report/primitives.js +113 -0
  38. package/dist/report/react/AttemptList.d.ts +23 -0
  39. package/dist/report/react/AttemptList.js +20 -0
  40. package/dist/report/react/DeltaTable.d.ts +8 -0
  41. package/dist/report/react/DeltaTable.js +24 -0
  42. package/dist/report/react/EvalList.d.ts +10 -0
  43. package/dist/report/react/EvalList.js +0 -0
  44. package/dist/report/react/ExperimentList.d.ts +10 -0
  45. package/dist/report/react/ExperimentList.js +15 -0
  46. package/dist/report/react/GroupSummary.d.ts +8 -0
  47. package/dist/report/react/GroupSummary.js +8 -0
  48. package/dist/report/react/MetricBars.d.ts +10 -0
  49. package/dist/report/react/MetricBars.js +48 -0
  50. package/dist/report/react/MetricLine.d.ts +9 -0
  51. package/dist/report/react/MetricLine.js +68 -0
  52. package/dist/report/react/MetricMatrix.d.ts +10 -0
  53. package/dist/report/react/MetricMatrix.js +23 -0
  54. package/dist/report/react/MetricScatter.d.ts +9 -0
  55. package/dist/report/react/MetricScatter.js +100 -0
  56. package/dist/report/react/MetricTable.d.ts +11 -0
  57. package/dist/report/react/MetricTable.js +26 -0
  58. package/dist/report/react/RunOverview.d.ts +8 -0
  59. package/dist/report/react/RunOverview.js +12 -0
  60. package/dist/report/react/Scoreboard.d.ts +8 -0
  61. package/dist/report/react/Scoreboard.js +26 -0
  62. package/dist/report/react/cell.d.ts +9 -0
  63. package/dist/report/react/cell.js +9 -0
  64. package/dist/report/react/chart-math.d.ts +14 -0
  65. package/dist/report/react/chart-math.js +68 -0
  66. package/dist/report/react/colors.d.ts +14 -0
  67. package/dist/report/react/colors.js +45 -0
  68. package/dist/report/react/fixtures.d.ts +19 -0
  69. package/dist/report/react/fixtures.js +393 -0
  70. package/dist/report/react/format.d.ts +3 -0
  71. package/dist/report/react/format.js +7 -0
  72. package/dist/report/react/index.d.ts +6 -0
  73. package/dist/report/react/index.js +14 -0
  74. package/dist/report/report.d.ts +24 -0
  75. package/dist/report/report.js +49 -0
  76. package/dist/report/text/faces.d.ts +29 -0
  77. package/dist/report/text/faces.js +503 -0
  78. package/dist/report/text/layout.d.ts +75 -0
  79. package/dist/report/text/layout.js +173 -0
  80. package/dist/report/text/plot.d.ts +37 -0
  81. package/dist/report/text/plot.js +81 -0
  82. package/dist/report/text/table.d.ts +8 -0
  83. package/dist/report/text/table.js +86 -0
  84. package/dist/report/tree.d.ts +99 -0
  85. package/dist/report/tree.js +228 -0
  86. package/dist/report/types.d.ts +420 -0
  87. package/dist/report/types.js +4 -0
  88. package/dist/report/web.d.ts +15 -0
  89. package/dist/report/web.js +37 -0
  90. package/dist/results/annotated-source.d.ts +61 -0
  91. package/dist/results/attempt-evidence.d.ts +69 -0
  92. package/dist/results/attempt-source.d.ts +15 -0
  93. package/dist/results/locator.d.ts +75 -0
  94. package/dist/results/locator.js +0 -0
  95. package/dist/results/select.d.ts +45 -0
  96. package/dist/results/select.js +230 -0
  97. package/dist/results/source-hash.d.ts +9 -0
  98. package/dist/results/types.d.ts +204 -0
  99. package/dist/runner/eval-source.d.ts +22 -0
  100. package/dist/runner/fingerprint.d.ts +23 -0
  101. package/dist/runner/types.d.ts +332 -0
  102. package/dist/sandbox/docker-stream.d.ts +24 -0
  103. package/dist/sandbox/docker.d.ts +84 -0
  104. package/dist/sandbox/e2b.d.ts +31 -0
  105. package/dist/sandbox/errors.d.ts +4 -0
  106. package/dist/sandbox/local-files.d.ts +6 -0
  107. package/dist/sandbox/paths.d.ts +4 -0
  108. package/dist/sandbox/registry.d.ts +13 -0
  109. package/dist/sandbox/resolve.d.ts +40 -0
  110. package/dist/sandbox/retry.d.ts +10 -0
  111. package/dist/sandbox/shell.d.ts +11 -0
  112. package/dist/sandbox/source-files.d.ts +13 -0
  113. package/dist/sandbox/types.d.ts +203 -0
  114. package/dist/sandbox/vercel.d.ts +37 -0
  115. package/dist/scoring/types.d.ts +70 -0
  116. package/dist/shared/aggregate.d.ts +29 -0
  117. package/dist/shared/aggregate.js +55 -0
  118. package/dist/shared/types.d.ts +27 -0
  119. package/dist/shared/verdict.d.ts +27 -0
  120. package/dist/shared/verdict.js +34 -0
  121. package/dist/tty-line.d.ts +6 -0
  122. package/dist/types.d.ts +7 -0
  123. package/dist/util.d.ts +22 -0
  124. package/docs-site/zh/concepts/adapter.mdx +9 -4
  125. package/docs-site/zh/concepts/assert.mdx +8 -8
  126. package/docs-site/zh/concepts/evals.mdx +5 -5
  127. package/docs-site/zh/concepts/experiment.mdx +3 -1
  128. package/docs-site/zh/concepts/judge.mdx +9 -9
  129. package/docs-site/zh/concepts/overview.mdx +2 -2
  130. package/docs-site/zh/concepts/tier.mdx +6 -6
  131. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  132. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  133. package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
  134. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  135. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  136. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  137. package/docs-site/zh/guides/experiments.mdx +7 -0
  138. package/docs-site/zh/guides/fixtures.mdx +1 -1
  139. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  140. package/docs-site/zh/guides/publish-report.mdx +4 -4
  141. package/docs-site/zh/guides/report-components.mdx +194 -55
  142. package/docs-site/zh/guides/reporters.mdx +2 -2
  143. package/docs-site/zh/guides/results-data.mdx +35 -34
  144. package/docs-site/zh/guides/runner.mdx +2 -2
  145. package/docs-site/zh/guides/sandbox-agent.mdx +10 -5
  146. package/docs-site/zh/guides/sandbox-providers.mdx +102 -0
  147. package/docs-site/zh/guides/viewing-results.mdx +199 -79
  148. package/docs-site/zh/guides/write-experiment.mdx +23 -1
  149. package/docs-site/zh/guides/write-send.mdx +2 -2
  150. package/docs-site/zh/index.mdx +4 -3
  151. package/docs-site/zh/introduction.mdx +2 -1
  152. package/docs-site/zh/quickstart.mdx +5 -3
  153. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  154. package/docs-site/zh/reference/capabilities.mdx +1 -1
  155. package/docs-site/zh/reference/cli.mdx +36 -13
  156. package/docs-site/zh/reference/define-agent.mdx +24 -12
  157. package/docs-site/zh/reference/define-config.mdx +1 -1
  158. package/docs-site/zh/reference/define-eval.mdx +3 -3
  159. package/docs-site/zh/reference/events.mdx +7 -1
  160. package/docs-site/zh/reference/expect.mdx +3 -3
  161. package/package.json +10 -6
  162. package/src/agents/bub.ts +134 -43
  163. package/src/agents/claude-code.test.ts +152 -0
  164. package/src/agents/claude-code.ts +127 -13
  165. package/src/agents/codex.test.ts +192 -0
  166. package/src/agents/codex.ts +151 -12
  167. package/src/agents/index.ts +11 -3
  168. package/src/agents/manifest.ts +41 -0
  169. package/src/agents/plugin-config.test.ts +94 -0
  170. package/src/agents/shared.ts +33 -2
  171. package/src/agents/skills.test.ts +214 -0
  172. package/src/agents/skills.ts +304 -0
  173. package/src/agents/streaming.ts +5 -5
  174. package/src/agents/types.ts +62 -2
  175. package/src/cli.ts +74 -47
  176. package/src/context/context.test.ts +20 -2
  177. package/src/context/context.ts +3 -0
  178. package/src/context/control-flow.ts +1 -1
  179. package/src/context/session.test.ts +92 -1
  180. package/src/context/session.ts +26 -1
  181. package/src/context/types.ts +4 -4
  182. package/src/define.test.ts +82 -0
  183. package/src/define.ts +72 -11
  184. package/src/expect/index.ts +5 -5
  185. package/src/i18n/en.ts +36 -15
  186. package/src/i18n/zh-CN.ts +39 -21
  187. package/src/index.ts +1 -1
  188. package/src/o11y/execution-tree.test.ts +452 -0
  189. package/src/o11y/execution-tree.ts +367 -0
  190. package/src/o11y/otlp/receiver.ts +1 -1
  191. package/src/o11y/otlp/select.ts +7 -3
  192. package/src/o11y/parsers/claude-code.test.ts +175 -0
  193. package/src/o11y/parsers/claude-code.ts +32 -0
  194. package/src/o11y/types.ts +6 -1
  195. package/src/report/aggregate.ts +26 -20
  196. package/src/report/built-in-user-parity.test.tsx +643 -0
  197. package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
  198. package/src/report/built-ins/index.ts +2 -0
  199. package/src/report/components.tsx +223 -50
  200. package/src/report/compute.ts +358 -53
  201. package/src/report/dual-render.test.tsx +1125 -0
  202. package/src/report/flag.ts +1 -1
  203. package/src/report/format.ts +50 -2
  204. package/src/report/index.ts +48 -14
  205. package/src/report/load.ts +1 -1
  206. package/src/report/locale.ts +206 -0
  207. package/src/report/metrics.ts +39 -15
  208. package/src/report/primitives.tsx +100 -2
  209. package/src/report/react/AttemptList.tsx +115 -0
  210. package/src/report/react/DeltaTable.tsx +9 -6
  211. package/src/report/react/EvalList.tsx +0 -0
  212. package/src/report/react/ExperimentList.tsx +108 -0
  213. package/src/report/react/GroupSummary.tsx +66 -0
  214. package/src/report/react/MetricBars.tsx +24 -13
  215. package/src/report/react/MetricLine.tsx +21 -21
  216. package/src/report/react/MetricMatrix.tsx +8 -4
  217. package/src/report/react/MetricScatter.tsx +136 -96
  218. package/src/report/react/MetricTable.tsx +115 -15
  219. package/src/report/react/RunOverview.tsx +48 -26
  220. package/src/report/react/Scoreboard.tsx +15 -7
  221. package/src/report/react/cell.tsx +18 -9
  222. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  223. package/src/report/react/colors.ts +24 -15
  224. package/src/report/react/enhance.js +129 -0
  225. package/src/report/react/fixtures.ts +163 -34
  226. package/src/report/react/format.ts +1 -1
  227. package/src/report/react/index.tsx +22 -6
  228. package/src/report/react/render.test.tsx +164 -31
  229. package/src/report/react/styles.css +405 -213
  230. package/src/report/report.test.ts +403 -94
  231. package/src/report/report.ts +29 -11
  232. package/src/report/text/faces.ts +303 -95
  233. package/src/report/text/layout.ts +76 -11
  234. package/src/report/text/table.ts +98 -0
  235. package/src/report/tree.ts +103 -20
  236. package/src/report/types.ts +196 -25
  237. package/src/report/web.ts +46 -19
  238. package/src/results/annotated-source.test.ts +146 -0
  239. package/src/results/annotated-source.ts +128 -0
  240. package/src/results/attempt-evidence.test.ts +158 -0
  241. package/src/results/attempt-evidence.ts +158 -0
  242. package/src/results/attempt-source.ts +48 -0
  243. package/src/results/copy.ts +154 -139
  244. package/src/results/format.ts +54 -47
  245. package/src/results/host-equivalence.test.ts +527 -0
  246. package/src/results/index.ts +56 -14
  247. package/src/results/locator.test.ts +198 -0
  248. package/src/results/locator.ts +0 -0
  249. package/src/results/open.ts +392 -175
  250. package/src/results/results.test.ts +943 -494
  251. package/src/results/select.ts +129 -23
  252. package/src/results/skipped-notice.ts +0 -0
  253. package/src/results/source-hash.ts +28 -0
  254. package/src/results/types.ts +98 -62
  255. package/src/results/writer.ts +304 -176
  256. package/src/runner/attempt.test.ts +170 -0
  257. package/src/runner/attempt.ts +97 -15
  258. package/src/runner/discover.test.ts +65 -0
  259. package/src/runner/discover.ts +6 -2
  260. package/src/runner/eval-source.test.ts +121 -0
  261. package/src/runner/eval-source.ts +45 -0
  262. package/src/runner/fingerprint.ts +54 -1
  263. package/src/runner/remote-sandbox.ts +1 -1
  264. package/src/runner/report.test.ts +6 -8
  265. package/src/runner/report.ts +5 -4
  266. package/src/runner/reporters/artifacts.ts +27 -41
  267. package/src/runner/reporters/braintrust.test.ts +3 -3
  268. package/src/runner/reporters/braintrust.ts +1 -1
  269. package/src/runner/reporters/console.ts +6 -6
  270. package/src/runner/reporters/json.ts +4 -4
  271. package/src/runner/reporters/live.test.ts +56 -0
  272. package/src/runner/reporters/live.ts +36 -10
  273. package/src/runner/reporters/quiet.test.ts +66 -0
  274. package/src/runner/reporters/quiet.ts +49 -0
  275. package/src/runner/reporters/shared.ts +5 -5
  276. package/src/runner/reporters/table.ts +17 -17
  277. package/src/runner/run.ts +61 -102
  278. package/src/runner/types.ts +43 -32
  279. package/src/sandbox/checkpoint.ts +1 -1
  280. package/src/sandbox/docker.ts +15 -2
  281. package/src/sandbox/e2b.ts +11 -5
  282. package/src/sandbox/errors.ts +13 -0
  283. package/src/sandbox/index.ts +3 -3
  284. package/src/sandbox/local-files.ts +1 -1
  285. package/src/sandbox/registry.ts +6 -5
  286. package/src/sandbox/resolve.ts +38 -23
  287. package/src/sandbox/retry.test.ts +68 -0
  288. package/src/sandbox/retry.ts +44 -0
  289. package/src/sandbox/types.ts +69 -28
  290. package/src/sandbox/vercel.ts +17 -3
  291. package/src/scoring/collector.ts +3 -3
  292. package/src/scoring/judge.ts +2 -2
  293. package/src/scoring/scoped.ts +13 -1
  294. package/src/scoring/types.ts +7 -7
  295. package/src/scoring/verdict.ts +4 -4
  296. package/src/shared/aggregate.ts +23 -3
  297. package/src/shared/types.ts +2 -2
  298. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  299. package/src/show/compose.ts +12 -132
  300. package/src/show/index.ts +125 -94
  301. package/src/show/render.ts +660 -160
  302. package/src/show/show.test.ts +472 -253
  303. package/src/source-loc.ts +1 -1
  304. package/src/tty-line.ts +32 -0
  305. package/src/types.ts +2 -2
  306. package/src/util.test.ts +21 -1
  307. package/src/util.ts +5 -1
  308. package/src/view/app/App.tsx +33 -198
  309. package/src/view/app/components/AttemptModal.tsx +6 -6
  310. package/src/view/app/components/CodeView.tsx +2 -2
  311. package/src/view/app/components/CopyControls.tsx +32 -36
  312. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  313. package/src/view/app/components/ui/badge.tsx +1 -1
  314. package/src/view/app/components/ui/dialog.tsx +2 -2
  315. package/src/view/app/i18n.ts +40 -31
  316. package/src/view/app/lib/artifact-url.ts +1 -1
  317. package/src/view/app/lib/attempt-route.test.ts +37 -47
  318. package/src/view/app/lib/attempt-route.ts +26 -26
  319. package/src/view/app/lib/rows.ts +9 -112
  320. package/src/view/app/lib/verdict.ts +25 -0
  321. package/src/view/app/main.tsx +9 -12
  322. package/src/view/app/pages/RunsPage.tsx +4 -4
  323. package/src/view/app/pages/TracesPage.tsx +3 -3
  324. package/src/view/app/shared.ts +0 -1
  325. package/src/view/app/types.ts +6 -51
  326. package/src/view/artifact-serving.test.ts +136 -0
  327. package/src/view/client-dist/app.css +1 -1
  328. package/src/view/client-dist/app.js +21 -21
  329. package/src/view/data.test.ts +214 -143
  330. package/src/view/data.ts +166 -120
  331. package/src/view/index.ts +23 -11
  332. package/src/view/server.ts +52 -17
  333. package/src/view/shared/types.ts +22 -31
  334. package/src/view/styles.css +117 -327
  335. package/src/view/view-report.test.ts +146 -98
  336. package/docs-site/zh/guides/sandbox-backends.mdx +0 -75
  337. package/src/report/default-report.tsx +0 -222
  338. package/src/report/dual-face.test.tsx +0 -527
  339. package/src/report/react/CaseList.tsx +0 -70
  340. package/src/view/app/components/CostScoreChart.tsx +0 -147
  341. package/src/view/app/components/ExperimentTable.tsx +0 -284
  342. package/src/view/app/components/GroupSelector.tsx +0 -61
  343. package/src/view/app/components/primitives.tsx +0 -43
  344. package/src/view/app/lib/outcome.ts +0 -82
package/src/source-loc.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  // 源码位置回溯:在 `t.send` / 断言落地时(collector.record / SessionManager.send)抓一次
2
2
  // 调用栈,挑出**第一帧不属于 niceeval 自身**的位置——也就是用户 eval 里那一行。view 据此把
3
- // 运行结果(回复 / 分数 / 判决)叠回真实源码行,渲染成 github-diff 式代码视图。
3
+ // 运行结果(回复 / 分数 / 判定)叠回真实源码行,渲染成 github-diff 式代码视图。
4
4
  //
5
5
  // 为什么靠栈而不是改 API:这样 `t` 的表面一个字不用动,作者照常写 `t.judge.autoevals...`,
6
6
  // 位置在底层免费拿到。tsx 直接跑 .ts,栈里就是真实 .ts 路径+行号(ESM 下是 file:// URL)。
@@ -0,0 +1,32 @@
1
+ // 独立诊断行的统一终端出口。
2
+ //
3
+ // live 表格(src/runner/reporters/live.ts)重画靠 \x1B[{drawnLines}A 回跳到上一帧起点,
4
+ // 这个回跳量的前提是"stderr/stdout 这块屏幕在两帧之间只有它自己在写"。sandbox teardown
5
+ // 失败、budget 不可执行、reporter 抛错这类跟 attempt 进度无关的独立诊断行,如果在 live
6
+ // 表格激活期间直接 process.stderr.write / console.log,就会插进两次 draw() 之间——下一帧
7
+ // 按旧的 drawnLines 回跳,已经够不到表格真正的起点,清行重画变成往下多写一份,且这个偏移
8
+ // 被记进新的 drawnLines,此后每帧都在错位的基础上继续错位,越滚越多(见
9
+ // memory/live-raw-stderr-write-desyncs-redraw.md)。
10
+ //
11
+ // 所以任何不经过 Reporter.progress() / onEvalComplete() 的独立诊断行,写之前都要先广播
12
+ // beforeExternalTerminalWrite(),live.ts 借此机会把已画内容清掉、drawnLines 归零,下一次
13
+ // draw() 就会把这行诊断消息之下当成全新起点重画,不会累积。
14
+
15
+ const listeners = new Set<() => void>();
16
+
17
+ /** live.ts 用来订阅"即将有一行独立诊断消息落地"。返回取消订阅函数。 */
18
+ export function onBeforeExternalTerminalWrite(fn: () => void): () => void {
19
+ listeners.add(fn);
20
+ return () => listeners.delete(fn);
21
+ }
22
+
23
+ /** 任何要绕开 Reporter 直接往终端打一行独立诊断消息的地方,写之前都先调用这个。 */
24
+ export function beforeExternalTerminalWrite(): void {
25
+ for (const fn of listeners) fn();
26
+ }
27
+
28
+ /** process.stderr.write 的替代:文本需自带换行(沿用现有 i18n 字符串的约定)。 */
29
+ export function writeStderrLine(text: string): void {
30
+ beforeExternalTerminalWrite();
31
+ process.stderr.write(text);
32
+ }
package/src/types.ts CHANGED
@@ -4,9 +4,9 @@
4
4
  // 模块代码统一 `import type { … } from "../types.ts"`,不必记住每个类型的家:
5
5
  // · shared/types.ts 跨域原子(JsonValue / Severity / SourceLoc / Cleanup / LocalizedText)
6
6
  // · o11y/types.ts 标准事件流 / DerivedFacts / TraceSpan / Usage / O11ySummary
7
- // · sandbox/types.ts Sandbox 接口 / 后端 spec / 命令与文件 IO
7
+ // · sandbox/types.ts Sandbox 接口 / provider spec / 命令与文件 IO
8
8
  // · agents/types.ts Agent / Adapter 契约 / 会话 / tracing 导出
9
- // · scoring/types.ts 断言(值级 / 记录 / 结果)/ ScoringContext / JudgeConfig
9
+ // · scoring/types.ts 断言( / 记录 / 结果)/ ScoringContext / JudgeConfig
10
10
  // · context/types.ts TestContext(t)与子句柄(turn / session / sandbox 视图)
11
11
  // · runner/types.ts EvalResult / RunSummary / Reporter / eval・experiment・config 定义 / 调度编排
12
12
  //
package/src/util.test.ts CHANGED
@@ -1,5 +1,25 @@
1
1
  import { describe, expect, it } from "vitest";
2
- import { formatThrown, upsertManagedBlock } from "./util.ts";
2
+ import { brief, formatThrown, upsertManagedBlock } from "./util.ts";
3
+
4
+ describe("brief", () => {
5
+ // JSON.stringify(undefined) 返回值 undefined(不是字符串 "undefined"),不特殊处理会让
6
+ // 下游 `s.length` 抛 TypeError——真实复现见 previewCheckedValue 断言 undefined 字段值时
7
+ // (2026-07-13 codex native plugin e2e,resolvedVersion 缺失时 equals() 的 gate 预览崩溃)。
8
+ it("does not throw on undefined and returns a readable placeholder", () => {
9
+ expect(() => brief(undefined)).not.toThrow();
10
+ expect(brief(undefined)).toBe("undefined");
11
+ });
12
+
13
+ it("still handles ordinary values (string passthrough, object JSON-stringified)", () => {
14
+ expect(brief("hello")).toBe("hello");
15
+ expect(brief({ a: 1 })).toBe('{"a":1}');
16
+ });
17
+
18
+ it("truncates long output with an ellipsis", () => {
19
+ const long = "x".repeat(300);
20
+ expect(brief(long, 10)).toBe(`${"x".repeat(10)}…`);
21
+ });
22
+ });
3
23
 
4
24
  describe("upsertManagedBlock", () => {
5
25
  const begin = "<!-- BEGIN:x -->";
package/src/util.ts CHANGED
@@ -57,7 +57,11 @@ export function upsertManagedBlock(source: string, begin: string, end: string, c
57
57
  export function brief(value: unknown, max = 200): string {
58
58
  let s: string;
59
59
  try {
60
- s = typeof value === "string" ? value : JSON.stringify(value);
60
+ // `JSON.stringify` undefined / function / symbol 返回**值** undefined,不是字符串
61
+ // "undefined"——不兜底会让 s 变成非字符串,下面 `s.length` 直接抛
62
+ // `Cannot read properties of undefined (reading 'length')`(2026-07-13 native plugin
63
+ // e2e 断言 `equals(undefined 的字段)` 时真实复现,不是假设场景)。
64
+ s = typeof value === "string" ? value : (JSON.stringify(value) ?? String(value));
61
65
  } catch {
62
66
  s = String(value);
63
67
  }
@@ -1,29 +1,20 @@
1
1
  import { useCallback, useEffect, useMemo, useRef, useState } from "react";
2
2
  import { detectLocale, makeTranslator, persistLocale, setDocumentLocale } from "./i18n.ts";
3
3
  import type { MessageKey } from "./i18n.ts";
4
- import type { Locale, LocalizedText, SortKey, SortState, Tab, ViewData, ViewResult, ViewRow } from "./types.ts";
5
- import { CELL_KEYS, buildGroupMap, buildRows, compareRows, flattenAttempts, resultFromUrl } from "./lib/rows.ts";
6
- import { formatAttemptHash, parseAttemptHash, resolveAttemptRef, unresolvedAttemptWarning } from "./lib/attempt-route.ts";
7
- import { formatCost, formatDateTime, formatDuration } from "./lib/format.ts";
8
- import { Metric } from "./components/primitives.tsx";
9
- import { GroupSelector } from "./components/GroupSelector.tsx";
10
- import { CostScoreChart } from "./components/CostScoreChart.tsx";
11
- import { ExperimentTable } from "./components/ExperimentTable.tsx";
4
+ import type { Locale, LocalizedText, ReportSlotHtml, Tab, ViewData, ViewResult } from "./types.ts";
5
+ import { flattenAttempts, resultFromUrl } from "./lib/rows.ts";
6
+ import { parseAttemptHash, resolveAttemptLocator, unresolvedAttemptWarning } from "./lib/attempt-route.ts";
7
+ import { formatDateTime } from "./lib/format.ts";
12
8
  import { CopyFixPrompt } from "./components/CopyControls.tsx";
9
+ import { SkippedRunsBanner } from "./components/SkippedRunsBanner.tsx";
13
10
  import { AttemptModal } from "./components/AttemptModal.tsx";
14
11
  import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs.tsx";
15
12
  import { RunsView } from "./pages/RunsPage.tsx";
16
13
  import { TracesView } from "./pages/TracesPage.tsx";
17
14
 
18
- // 报告槽只有一个:默认装官方榜单(Experiments tab);--report 整槽替换成用户报告。
19
- // 证据室(Runs / Traces / AttemptModal)是 view 本体,两种模式下原样保留。
15
+ // 首页 = 报告槽:恒在恒默认,裸跑填充内置默认报告,--report 整槽替换(两种填充走同一个 tab)
16
+ // 证据室(Runs / Traces / AttemptModal)是 view 本体,任何填充下原样保留。
20
17
  export const navItems: { id: Tab; label: MessageKey }[] = [
21
- { id: "experiments", label: "nav.experiments" },
22
- { id: "runs", label: "nav.runs" },
23
- { id: "traces", label: "nav.traces" },
24
- ];
25
-
26
- export const reportNavItems: { id: Tab; label: MessageKey }[] = [
27
18
  { id: "report", label: "nav.report" },
28
19
  { id: "runs", label: "nav.runs" },
29
20
  { id: "traces", label: "nav.traces" },
@@ -36,34 +27,25 @@ export function localizedText(text: LocalizedText | undefined, locale: Locale):
36
27
  return text[locale] ?? text.en ?? Object.values(text)[0];
37
28
  }
38
29
 
39
- /** 初始 URL → 直接打开的 attempt:先认 #/attempt/<run>/<result> 深链,回退旧版 ?modal= 参数。 */
30
+ /** 初始 URL → 直接打开的 attempt:先认 #/attempt/@<locator> 深链,回退旧版 ?modal= 参数。 */
40
31
  function modalResultFromLocation(snapshots: ViewData["snapshots"]): ViewResult | null {
41
- const ref = parseAttemptHash(location.hash);
42
- if (ref) {
43
- const found = resolveAttemptRef(snapshots, ref);
32
+ const locator = parseAttemptHash(location.hash);
33
+ if (locator) {
34
+ const found = resolveAttemptLocator(snapshots, locator);
44
35
  if (found) return found;
45
- // 定位不到(run 不在、下标越界、旧格式数据):不开空 modal,页面照常渲染。
36
+ // 定位不到(locator 不在、快照未加载、旧格式数据):不开空 modal,页面照常渲染。
46
37
  console.warn(unresolvedAttemptWarning(location.hash));
47
38
  return null;
48
39
  }
49
40
  return resultFromUrl(snapshots);
50
41
  }
51
42
 
52
- export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string }) {
43
+ export function App({ data, reportHtml }: { data: ViewData; reportHtml: ReportSlotHtml }) {
53
44
  const snapshots = data.snapshots ?? [];
54
- const hasReport = reportHtml !== undefined;
55
- const rows = useMemo(() => buildRows(data), [data]);
56
45
  const attempts = useMemo(() => flattenAttempts(snapshots), [snapshots]);
57
46
  const [locale, setLocale] = useState<Locale>(() => detectLocale());
58
47
  const t = useMemo(() => makeTranslator(locale), [locale]);
59
- const [tab, setTab] = useState<Tab>(hasReport ? "report" : "experiments");
60
- const [sort, setSort] = useState<SortState>({ key: "passRate", dir: -1 });
61
- const [query, setQuery] = useState("");
62
- const [openRows, setOpenRows] = useState<Set<string>>(() => new Set());
63
- const [selectedGroup, setSelectedGroup] = useState(() => {
64
- const groups = [...new Set(rows.map((r) => r.group).filter(Boolean))].sort();
65
- return groups[0] ?? null;
66
- });
48
+ const [tab, setTab] = useState<Tab>("report");
67
49
  const [modalResult, setModalResult] = useState<ViewResult | null>(() => modalResultFromLocation(snapshots));
68
50
  // 当前 modal 的 hash 历史条目前面是否还有本页条目(本页 push 的 / 前进键回到的):
69
51
  // true → UI 关闭走 history.back(),前进键还能重新打开;false(深链直接落地)→ 原地抹 hash,
@@ -75,25 +57,6 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
75
57
  persistLocale(locale);
76
58
  }, [locale]);
77
59
 
78
- const openModal = useCallback((result: ViewResult) => {
79
- setModalResult(result);
80
- const ref = result.attemptRef;
81
- // 旧格式烘焙的数据没有 attemptRef:modal 照常打开,只是这条 attempt 产不出可分享链接。
82
- if (!ref) return;
83
- try {
84
- // pushState/replaceState 不触发 hashchange,不会和下面的监听器重复开合。
85
- if (parseAttemptHash(location.hash)) {
86
- // 已经在某条 attempt 深链上(防御浏览器导航竞态):替换当前条目,不叠历史。
87
- history.replaceState(null, "", formatAttemptHash(ref));
88
- } else {
89
- history.pushState(null, "", formatAttemptHash(ref));
90
- modalOwnsHistory.current = true;
91
- }
92
- } catch {
93
- // file:// 等受限环境可能拒绝写 history;URL 同步失败不影响打开 modal。
94
- }
95
- }, []);
96
-
97
60
  const closeModal = useCallback(() => {
98
61
  setModalResult(null);
99
62
  if (modalOwnsHistory.current) {
@@ -109,16 +72,16 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
109
72
  }
110
73
  }, []);
111
74
 
112
- // 浏览器前进/后退、手改 hash、页内 attempt 链接统一从 hashchange 开合 modal。
75
+ // 浏览器前进/后退、手改 hash、页内 attempt 链接(含报告槽里的深链)统一从 hashchange 开合 modal。
113
76
  useEffect(() => {
114
77
  const onHashChange = () => {
115
- const ref = parseAttemptHash(location.hash);
116
- if (!ref) {
78
+ const locator = parseAttemptHash(location.hash);
79
+ if (!locator) {
117
80
  modalOwnsHistory.current = false;
118
81
  setModalResult(null);
119
82
  return;
120
83
  }
121
- const found = resolveAttemptRef(snapshots, ref);
84
+ const found = resolveAttemptLocator(snapshots, locator);
122
85
  if (!found) {
123
86
  console.warn(unresolvedAttemptWarning(location.hash));
124
87
  setModalResult(null);
@@ -132,48 +95,6 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
132
95
  return () => window.removeEventListener("hashchange", onHashChange);
133
96
  }, [snapshots]);
134
97
 
135
- const groupMap = useMemo<Map<string, ViewRow[]>>(() => buildGroupMap(rows), [rows]);
136
- const pool = selectedGroup ? groupMap.get(selectedGroup) ?? [] : rows;
137
- const filtered = useMemo(() => {
138
- const q = query.trim().toLowerCase();
139
- return pool
140
- .filter((row: ViewRow) => {
141
- if (!q) return true;
142
- return [
143
- row.label,
144
- row.group || "",
145
- row.experimentId || "",
146
- row.agent,
147
- row.model || "",
148
- ...(row.results ?? []).map((r: ViewResult) => r.id),
149
- ]
150
- .join(" ")
151
- .toLowerCase()
152
- .includes(q);
153
- })
154
- .sort((a: ViewRow, b: ViewRow) => compareRows(a, b, sort.key) * sort.dir);
155
- }, [pool, query, sort]);
156
-
157
- const setSortKey = (key: SortKey) => {
158
- setSort((prev) =>
159
- prev.key === key ? { key, dir: prev.dir === 1 ? -1 : 1 } : { key, dir: key === "experiment" || key === "agent" ? 1 : -1 },
160
- );
161
- };
162
-
163
- const toggleRow = (key: string) => {
164
- setOpenRows((prev) => {
165
- const next = new Set(prev);
166
- if (next.has(key)) next.delete(key);
167
- else next.add(key);
168
- return next;
169
- });
170
- };
171
-
172
- // hero 的官方数字:整体通过率是 MetricTable.data 常量维度的单行格子,其余来自 RunOverview.data。
173
- const overallPassRate = data.overall?.rows[0]?.cells[CELL_KEYS.passRate];
174
- const totals = data.overview?.totals;
175
- const warnings = data.overview?.warnings ?? [];
176
-
177
98
  return (
178
99
  <Tabs value={tab} onValueChange={(v) => setTab(v as Tab)}>
179
100
  <header className="topbar">
@@ -182,7 +103,7 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
182
103
  <span>NiceEval</span>
183
104
  </a>
184
105
  <TabsList aria-label={t("nav.label")}>
185
- {(hasReport ? reportNavItems : navItems).map((item) => (
106
+ {navItems.map((item) => (
186
107
  <TabsTrigger key={item.id} value={item.id}>
187
108
  {t(item.label)}
188
109
  </TabsTrigger>
@@ -211,117 +132,31 @@ export function App({ data, reportHtml }: { data: ViewData; reportHtml?: string
211
132
  <b>{t("hero.lastRun")}</b> {data.lastRunAt ? formatDateTime(data.lastRunAt, locale) : t("hero.noRuns")}
212
133
  </span>
213
134
  {data.composedRuns > 0 ? (
214
- // 榜单是跨 run 合成的现刻水位,表头如实标注合成来源(几个 run)。
135
+ // 报告槽是跨 run 合成的现刻水位,hero 如实标注合成来源(几个 run)。
215
136
  <span>{t("hero.composedFrom", { count: data.composedRuns })}</span>
216
137
  ) : null}
217
138
  </div>
218
139
  </section>
219
140
 
220
- {/* 官方水位 KPI 与挑选警告属于报告槽:--report 整槽替换后由用户报告自己决定
221
- 摆不摆(<DefaultReport /> / <RunOverview /> 都能把它们摆回来)。 */}
222
- {!hasReport && (
223
- <section className="summary" aria-label="Run summary">
224
- <Metric label={t("metric.passRate")} value={overallPassRate?.display ?? "—"} />
225
- <Metric label={t("metric.evalResults")} value={String(totals?.evals ?? 0)} />
226
- <Metric label={t("metric.duration")} value={formatDuration(totals?.durationMs ?? 0)} />
227
- <Metric label={t("metric.cost")} value={formatCost(totals?.costUSD ?? undefined)} />
228
- </section>
229
- )}
230
-
231
- {!hasReport && warnings.length > 0 && (
232
- // 选集警告(partial-coverage / stale-snapshot / synthetic-experiment-id):
233
- // message 是挑选器渲染好的英文句子,原样打;data-kind 供样式与测试定位。
234
- <section className="incompatible-banner selection-warnings" role="alert">
235
- <b>{t("banner.warningsTitle")}</b>
236
- <ul>
237
- {warnings.map((w, i) => (
238
- <li key={`${w.kind}-${i}`} data-kind={w.kind}>
239
- <span className="ib-meta">{w.message}</span>
240
- </li>
241
- ))}
242
- </ul>
243
- </section>
244
- )}
141
+ <SkippedRunsBanner skippedRuns={data.skippedRuns ?? []} t={t} />
245
142
 
246
- {(data.skippedRuns?.length ?? 0) > 0 && (
247
- <section className="incompatible-banner" role="alert">
248
- <b>{t("banner.skippedTitle")}</b>
249
- <ul>
250
- {data.skippedRuns!.map((run) => (
251
- <li key={run.dir} data-reason={run.reason}>
252
- <span className="ib-dir">{run.dir}</span>
253
- <span className="ib-meta">
254
- {run.reason === "malformed"
255
- ? t("banner.skipped.malformed", { detail: run.detail ?? "?" })
256
- : run.reason === "incomplete"
257
- ? t("banner.skipped.incomplete")
258
- : run.producerName && run.producerName !== "niceeval"
259
- ? // 第三方 harness 写的落盘:如实报名字和版本,不拼 npx 命令。
260
- t("banner.skipped.incompatibleForeign", {
261
- name: run.producerName,
262
- version: run.producerVersion ?? "?",
263
- schemaVersion: run.schemaVersion,
264
- })
265
- : t("banner.skipped.incompatible", {
266
- producer: run.producerVersion ?? "?",
267
- schemaVersion: run.schemaVersion,
268
- })}
269
- </span>
270
- {run.command ? <code>{run.command}</code> : null}
271
- </li>
272
- ))}
273
- </ul>
274
- </section>
275
- )}
276
-
277
- {hasReport && (
278
- <TabsContent value="report" id="tab-report">
279
- {/* 报告槽:server 侧渲染好的静态 HTML(含 <Style> 产物),这里只摆放。
280
- attempt 深链是普通 <a href="#/attempt/…">,经 hashchange 打开证据室弹窗。 */}
281
- <div className="report-slot" dangerouslySetInnerHTML={{ __html: reportHtml }} />
282
- </TabsContent>
283
- )}
284
-
285
- {!hasReport && (
286
- <TabsContent value="experiments" id="tab-experiments">
287
- <div className="section-head">
288
- <h2>{t("section.experiments")}</h2>
289
- </div>
290
- <GroupSelector groupMap={groupMap} selectedGroup={selectedGroup} onSelect={setSelectedGroup} t={t} />
143
+ <TabsContent value="report" id="tab-report">
144
+ {/* 壳区:报告槽上方靠右的批量修复 prompt 按钮。失败清单从 viewData.snapshots
145
+ 现算(latest 口径),默认报告与 --report 两种填充下都在。 */}
291
146
  <div className="section-sub-head">
292
- <span className="group-detail-label">{selectedGroup ?? ""}</span>
147
+ <span className="group-detail-label" />
293
148
  <div className="controls">
294
- <input
295
- className="search"
296
- type="search"
297
- placeholder={t("search.experiments")}
298
- autoComplete="off"
299
- value={query}
300
- onChange={(e) => setQuery(e.target.value)}
301
- />
302
- <CopyFixPrompt rows={filtered} t={t} />
149
+ <CopyFixPrompt snapshots={snapshots} t={t} />
303
150
  </div>
304
151
  </div>
305
- {rows.length ? (
306
- <>
307
- <CostScoreChart rows={filtered} t={t} />
308
- <ExperimentTable
309
- rows={filtered}
310
- sort={sort}
311
- setSortKey={setSortKey}
312
- openRows={openRows}
313
- toggleRow={toggleRow}
314
- openModal={openModal}
315
- t={t}
316
- />
317
- </>
318
- ) : (
319
- <div className="empty">
320
- {t("empty.summary")}
321
- </div>
322
- )}
152
+ {/* 报告槽:server 侧渲染好的静态 HTML(含 <Style> 产物),按当前界面语言摆放
153
+ 对应语言的块;Selection 警告由报告槽内的 RunOverview 呈现,壳不设第二条通道。
154
+ attempt 深链是普通 <a href="#/attempt/…">,经 hashchange 打开证据室弹窗。 */}
155
+ <div
156
+ className="report-slot"
157
+ dangerouslySetInnerHTML={{ __html: reportHtml[locale] || reportHtml.en }}
158
+ />
323
159
  </TabsContent>
324
- )}
325
160
 
326
161
  <TabsContent value="runs">
327
162
  <RunsView attempts={attempts} t={t} />
@@ -3,7 +3,7 @@ import type { ArtifactLoadState, T } from "../shared.ts";
3
3
  import type { ViewResult } from "../types.ts";
4
4
  import { artifactUrl } from "../lib/artifact-url.ts";
5
5
  import { asEvents, asSources } from "../lib/guards.ts";
6
- import { outcomeClass, outcomeLabel } from "../lib/outcome.ts";
6
+ import { verdictClass, verdictLabel } from "../lib/verdict.ts";
7
7
  import { CodeView, NoSourceBody } from "./CodeView.tsx";
8
8
  import { CopyAttemptPrompt } from "./CopyControls.tsx";
9
9
  import { LazyArtifact } from "./LazyArtifact.tsx";
@@ -15,13 +15,13 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
15
15
  const base = result.artifactBase;
16
16
  const [data, setData] = useState<ArtifactLoadState>({ sources: null, events: null, status: "loading" });
17
17
 
18
- // Esc / 焦点陷阱 / 背景滚动锁 / 点遮罩关闭 都交给 Radix Dialog;这里只保留工件拉取。
18
+ // Esc / 焦点陷阱 / 背景滚动锁 / 点遮罩关闭 都交给 Radix Dialog;这里只保留 artifact 拉取。
19
19
  useEffect(() => {
20
20
  if (!base) { setData({ sources: null, events: null, status: "none" }); return; }
21
21
  let alive = true;
22
22
  const grab = (name: string, has?: boolean): Promise<unknown> =>
23
23
  has
24
- ? fetch(artifactUrl(`${base}/${name}`))
24
+ ? fetch( artifactUrl(`${base}/${name}`))
25
25
  .then((r) => (r.ok ? r.json() : null))
26
26
  .catch(() => null)
27
27
  : Promise.resolve(null);
@@ -31,7 +31,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
31
31
  return () => { alive = false; };
32
32
  }, [base, result.hasSources, result.hasEvents]);
33
33
 
34
- const outcome = result.outcome;
34
+ const verdict = result.verdict;
35
35
  const hasCode = Boolean(data.sources?.length);
36
36
 
37
37
  return (
@@ -39,7 +39,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
39
39
  <DialogContent aria-describedby={undefined}>
40
40
  <div className="flex min-w-0 shrink-0 items-center justify-between gap-3 border-b border-line px-[18px] pb-[11px] pt-[13px]">
41
41
  <div className="flex min-w-0 flex-col gap-[3px]">
42
- <Badge tone={outcomeClass(outcome)}>{outcomeLabel(outcome, t)}</Badge>
42
+ <Badge tone={verdictClass(verdict)}>{verdictLabel(verdict, t)}</Badge>
43
43
  <DialogTitle asChild>
44
44
  <span className="truncate text-sm font-[640] text-text">{result.id}</span>
45
45
  </DialogTitle>
@@ -59,7 +59,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
59
59
  {hasCode ? (
60
60
  <CodeView sources={data.sources ?? []} events={data.events || []} assertions={allAssertions} t={t} />
61
61
  ) : data.status !== "loading" ? (
62
- // hasSources 为真却取不到 → 源码捕获过,是工件文件在当前托管里缺失;和「从未捕获」分开提示。
62
+ // hasSources 为真却取不到 → 源码捕获过,是 artifact 文件在当前托管里缺失;和「从未捕获」分开提示。
63
63
  <NoSourceBody
64
64
  assertions={allAssertions}
65
65
  events={data.events || []}
@@ -6,7 +6,7 @@ import { highlightTs, indexAsserts, indexTurns, locKey } from "../lib/transcript
6
6
  import { formatScore } from "../lib/format.ts";
7
7
  import { InputBlock, ToolBlock, Transcript } from "./Transcript.tsx";
8
8
 
9
- /** soft 断言没过阈值不影响 outcome,颜色上跟 gate 失败(红)区分开,用 warn(黄)。 */
9
+ /** soft 断言没过阈值不影响 verdict,颜色上跟 gate 失败(红)区分开,用 warn(黄)。 */
10
10
  function assertTone(a: Assertion): "good" | "warn" | "bad" {
11
11
  if (a.passed) return "good";
12
12
  return a.severity === "soft" ? "warn" : "bad";
@@ -118,7 +118,7 @@ export function CodeLine({
118
118
  }) {
119
119
  const hasReply = !!turn;
120
120
  const hasAsserts = !!(asserts && asserts.length);
121
- // 只有 gate 断言没过才算这一行真的"fail";只剩 soft 断言没过阈值时是"warn"(不影响 outcome)。
121
+ // 只有 gate 断言没过才算这一行真的"fail";只剩 soft 断言没过阈值时是"warn"(不影响 verdict)。
122
122
  const status = hasAsserts
123
123
  ? asserts?.every((a: Assertion) => a.passed)
124
124
  ? "pass"
@@ -1,62 +1,47 @@
1
1
  import React, { useState } from "react";
2
2
  import { Check, Copy } from "lucide-react";
3
3
  import type { T } from "../shared.ts";
4
- import type { ViewResult, ViewRow } from "../types.ts";
5
- import { failingAssertions, reasonFor } from "../lib/outcome.ts";
6
-
7
- export function CopyReason({ text, t }: { text: string; t: T }) {
8
- const [copied, setCopied] = useState(false);
9
- const copy = async (event: React.MouseEvent<HTMLButtonElement>) => {
10
- event.stopPropagation();
11
- try {
12
- await copyText(text);
13
- setCopied(true);
14
- setTimeout(() => setCopied(false), 1200);
15
- } catch {
16
- setCopied(false);
17
- }
18
- };
19
- return (
20
- <button className={`copy-reason${copied ? " is-copied" : ""}`} onClick={copy} aria-label={t("action.copyReason")} title={t("action.copyReason")}>
21
- {copied ? <Check aria-hidden="true" /> : <Copy aria-hidden="true" />}
22
- </button>
23
- );
24
- }
4
+ import type { ViewResult, ViewSnapshot } from "../types.ts";
5
+ import { snapshotLabel } from "../lib/rows.ts";
6
+ import { failingAssertions, reasonFor } from "../lib/verdict.ts";
25
7
 
26
8
  /** 修复 prompt 的一条失败条目;路径均相对 view 输入根(默认 `.niceeval/`)。 */
27
9
  export interface FixPromptEntry {
28
10
  experiment: string;
29
11
  evalId: string;
30
- outcome: string;
12
+ verdict: string;
31
13
  reason: string;
32
14
  artifactBase?: string;
33
- summaryPath?: string;
15
+ resultPath?: string;
34
16
  }
35
17
 
36
18
  export function toFixPromptEntry(r: ViewResult, experimentLabel: string): FixPromptEntry {
37
19
  return {
38
20
  experiment: r.experimentId ?? experimentLabel,
39
21
  evalId: r.id,
40
- outcome: r.outcome,
22
+ verdict: r.verdict,
41
23
  reason: reasonFor(r, failingAssertions(r)),
42
24
  artifactBase: r.artifactBase,
43
- summaryPath: r.attemptRef ? `${r.attemptRef.run}/summary.json` : undefined,
25
+ // result.json 与其它 artifact(events.json 等)同目录落盘,resultPath 因此复用同一个
26
+ // artifactBase(不再靠 attemptRef 的 {snapshot, attempt} 两段拼——locator 是不透明的,
27
+ // 浏览器端解不出磁盘路径)。
28
+ resultPath: r.artifactBase ? `${r.artifactBase}/result.json` : undefined,
44
29
  };
45
30
  }
46
31
 
47
32
  /**
48
33
  * 学 Next.js 16.3 的「Copy prompt」:把失败打包成一段可直接粘给 coding agent 的修复
49
- * prompt——失败清单 + 工件路径 + 先读随包文档 / 判断缺陷在哪一侧 / 重跑验证的步骤。
34
+ * prompt——失败清单 + artifact 路径 + 先读随包文档 / 判断缺陷在哪一侧 / 重跑验证的步骤。
50
35
  * prompt 面向 agent,固定英文;按钮文案走界面 i18n。
51
36
  */
52
37
  export function buildFixPrompt(entries: FixPromptEntry[]): string {
53
38
  const failures = entries
54
39
  .map((e, i) =>
55
40
  [
56
- `${i + 1}. eval "${e.evalId}" [experiment ${e.experiment}] — ${e.outcome}`,
41
+ `${i + 1}. eval "${e.evalId}" [experiment ${e.experiment}] — ${e.verdict}`,
57
42
  e.reason ? ` reason: ${e.reason}` : null,
58
43
  e.artifactBase ? ` artifacts: ${e.artifactBase}/` : null,
59
- e.summaryPath ? ` summary: ${e.summaryPath}` : null,
44
+ e.resultPath ? ` result: ${e.resultPath}` : null,
60
45
  ]
61
46
  .filter(Boolean)
62
47
  .join("\n"),
@@ -74,18 +59,29 @@ export function buildFixPrompt(entries: FixPromptEntry[]): string {
74
59
  "2. The paths above are relative to the results directory (default `.niceeval/`). For each failure, read `events.json` in its artifacts directory — the full agent transcript including tool calls — plus `trace.json` (execution trace) and `diff.json` (workspace diff) when present, to see what actually happened.",
75
60
  "3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
76
61
  `4. Re-run: \`npx niceeval exp ${experiments || "<experiment>"} <eval-id-prefix>\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
77
- "5. Read the new `summary.json` path the CLI prints and confirm these failures are gone.",
62
+ "5. Run `npx niceeval show` and confirm these failures are gone.",
78
63
  ].join("\n");
79
64
  }
80
65
 
81
- export function CopyFixPrompt({ rows, t }: { rows: ViewRow[]; t: T }) {
66
+ /**
67
+ * 报告槽同款口径的失败清单:每个 experiment 最新一次快照(latest 标记;快照明细已在
68
+ * server 侧跨快照去重)里的 failed / errored attempt,从 viewData.snapshots 现算——
69
+ * 默认报告与 --report 两种填充下按钮都在,不依赖任何统计产物。
70
+ */
71
+ export function fixPromptEntries(snapshots: ViewSnapshot[]): FixPromptEntry[] {
72
+ return snapshots
73
+ .filter((s) => s.latest)
74
+ .flatMap((snapshot) =>
75
+ snapshot.results
76
+ .filter((r: ViewResult) => r.verdict === "failed" || r.verdict === "errored")
77
+ .map((r: ViewResult) => toFixPromptEntry(r, snapshotLabel(snapshot))),
78
+ );
79
+ }
80
+
81
+ export function CopyFixPrompt({ snapshots, t }: { snapshots: ViewSnapshot[]; t: T }) {
82
82
  const [copied, setCopied] = useState(false);
83
83
 
84
- const entries = rows.flatMap((row: ViewRow) =>
85
- (row.results ?? [])
86
- .filter((r: ViewResult) => r.outcome === "failed" || r.outcome === "errored")
87
- .map((r: ViewResult) => toFixPromptEntry(r, row.label)),
88
- );
84
+ const entries = fixPromptEntries(snapshots);
89
85
 
90
86
  if (!entries.length) return null;
91
87
 
@@ -111,7 +107,7 @@ export function CopyFixPrompt({ rows, t }: { rows: ViewRow[]; t: T }) {
111
107
  /** attempt 弹窗里的单条版:只打包当前 attempt 的失败,供逐条转交 agent。 */
112
108
  export function CopyAttemptPrompt({ result, t }: { result: ViewResult; t: T }) {
113
109
  const [copied, setCopied] = useState(false);
114
- if (result.outcome !== "failed" && result.outcome !== "errored") return null;
110
+ if (result.verdict !== "failed" && result.verdict !== "errored") return null;
115
111
 
116
112
  const copy = async (event: React.MouseEvent<HTMLButtonElement>) => {
117
113
  event.stopPropagation();