niceeval 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +15 -7
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
  118. package/dist/report/{tree.js → definition/tree.js} +10 -12
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +12 -2
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +15 -1
  154. package/dist/shared/aggregate.js +32 -1
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +77 -64
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +30 -15
  227. package/src/i18n/zh-CN.ts +30 -15
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +136 -112
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +58 -22
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +54 -3
  314. package/src/results/select.ts +30 -6
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +30 -1
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +58 -22
  356. package/src/show/render.ts +16 -240
  357. package/src/show/show.test.ts +128 -35
  358. package/src/view/app/App.test.tsx +8 -7
  359. package/src/view/app/App.tsx +135 -51
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +122 -89
  370. package/src/view/index.ts +3 -13
  371. package/src/view/server.ts +36 -6
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +18 -806
  376. package/src/view/view-report.test.ts +182 -34
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -1,496 +0,0 @@
1
- import { Fragment, useCallback, useEffect, useMemo, useRef, useState } from "react";
2
- import type { ArtifactLoadState, T } from "../shared.ts";
3
- import type { Assertion, Span, ViewResult } from "../types.ts";
4
- import { artifactUrl } from "../lib/artifact-url.ts";
5
- import { asEvents, asSources, asSpans } from "../lib/guards.ts";
6
- import { verdictClass, verdictLabel } from "../lib/verdict.ts";
7
- import { AssertDetail, assertTone, CodeView, NoSourceBody } from "./CodeView.tsx";
8
- import { formatScore } from "../lib/format.ts";
9
- import { CopyAttemptPrompt } from "./CopyControls.tsx";
10
- import { Trace } from "./Trace.tsx";
11
- import { Dialog, DialogClose, DialogContent, DialogTitle } from "./ui/dialog.tsx";
12
- import { Badge } from "./ui/badge.tsx";
13
-
14
- export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClose: () => void; t: T }) {
15
- const allAssertions = result.assertions || [];
16
- const base = result.artifactBase;
17
- const [data, setData] = useState<ArtifactLoadState>({ sources: null, events: null, status: "loading" });
18
- const trace = useTraceSpans(base, result.hasTrace);
19
-
20
- // Esc / 焦点陷阱 / 背景滚动锁 / 点遮罩关闭 都交给 Radix Dialog;这里只保留 artifact 拉取。
21
- useEffect(() => {
22
- if (!base) { setData({ sources: null, events: null, status: "none" }); return; }
23
- let alive = true;
24
- const grab = (name: string, has?: boolean): Promise<unknown> =>
25
- has
26
- ? fetch( artifactUrl(`${base}/${name}`))
27
- .then((r) => (r.ok ? r.json() : null))
28
- .catch(() => null)
29
- : Promise.resolve(null);
30
- Promise.all([grab("sources.json", result.hasSources), grab("events.json", result.hasEvents)]).then(([sources, events]) => {
31
- if (alive) setData({ sources: asSources(sources), events: asEvents(events), status: "ready" });
32
- });
33
- return () => { alive = false; };
34
- }, [base, result.hasSources, result.hasEvents]);
35
-
36
- const verdict = result.verdict;
37
- const hasCode = Boolean(data.sources?.length);
38
-
39
- // 区块顺序对齐 docs/feature/reports/view.md「Attempt 详情」:判定与断言(errored 的结构化错误
40
- // 就是判定原因,归这一区)→ 紧凑时间树 → diagnostics → usage → 源码/对话 → trace 入口。
41
- return (
42
- <Dialog open onOpenChange={(o) => { if (!o) onClose(); }}>
43
- <DialogContent aria-describedby={undefined}>
44
- <div className="flex min-w-0 shrink-0 items-center justify-between gap-3 border-b border-line px-7 pb-3 pt-4">
45
- <div className="flex min-w-0 flex-col gap-[3px]">
46
- <Badge tone={verdictClass(verdict)}>{verdictLabel(verdict, t)}</Badge>
47
- <DialogTitle asChild>
48
- <span className="truncate text-sm font-[640] text-text">{result.id}</span>
49
- </DialogTitle>
50
- {result.description ? <span className="truncate text-xs text-muted">{result.description}</span> : null}
51
- </div>
52
- <CopyAttemptPrompt result={result} t={t} />
53
- <DialogClose
54
- aria-label={t("action.close")}
55
- className="grid h-7 w-7 shrink-0 place-items-center rounded-md border border-transparent text-sm text-muted transition-colors hover:border-line hover:bg-panel-2 hover:text-text"
56
- >
57
- x
58
- </DialogClose>
59
- </div>
60
- <div className="flex-1 overflow-y-auto px-7 pb-7 pt-5">
61
- {data.status !== "loading" && !hasCode ? <AssertionSection assertions={allAssertions} t={t} /> : null}
62
- {result.error ? (
63
- <div className="modal-error">
64
- <ErrorDetailBlock error={result.error} />
65
- </div>
66
- ) : null}
67
- {result.phases && result.phases.length > 0 ? <PhaseTimingBlock phases={result.phases} trace={trace} t={t} /> : null}
68
- {result.diagnostics && result.diagnostics.length > 0 ? (
69
- <AttemptDiagnostics diagnostics={result.diagnostics} t={t} />
70
- ) : null}
71
- <UsageDiffLine result={result} />
72
- {data.status === "loading" ? <div className="conv-loading">{t("trace.loading")}</div> : null}
73
- {hasCode ? (
74
- <CodeView sources={data.sources ?? []} events={data.events || []} assertions={allAssertions} t={t} />
75
- ) : data.status !== "loading" ? (
76
- // hasSources 为真却取不到 → 源码捕获过,是 artifact 文件在当前托管里缺失;和「从未捕获」分开提示。
77
- // 断言明细此时由断言区(detailMode)内嵌呈现,这里只保留说明与会话流,不再重复 checks。
78
- <NoSourceBody
79
- assertions={[]}
80
- events={data.events || []}
81
- message={t(result.hasSources && base ? "code.sourceUnavailable" : "code.noSource")}
82
- t={t}
83
- />
84
- ) : null}
85
- {trace.available ? (
86
- <FullTraceEntry trace={trace} t={t} />
87
- ) : data.status !== "loading" ? (
88
- <div className="mt-3 text-xs text-muted">
89
- {t("trace.enableHint")}
90
- <a href={t("trace.enableHintUrl")} target="_blank" rel="noreferrer" className="underline">
91
- {t("trace.enableHintLink")}
92
- </a>
93
- </div>
94
- ) : null}
95
- </div>
96
- </DialogContent>
97
- </Dialog>
98
- );
99
- }
100
-
101
- /* ───────────────────────── 判定与断言 ───────────────────────── */
102
-
103
- /** 断言的源码位置标注(fallback 下无代码可跳,纯文本)。 */
104
- function locLabel(a: Assertion): React.ReactNode {
105
- const loc = a.loc;
106
- if (!loc) return null;
107
- const short = loc.file.split("/").pop() ?? loc.file;
108
- return (
109
- <span className="assert-loc" title={`${loc.file}:${loc.line}`}>
110
- {short}:{loc.line}
111
- </span>
112
- );
113
- }
114
-
115
- /**
116
- * 断言区 = 源码视图的 fallback(见 docs/feature/reports/view.md「Attempt 详情」):断言明细
117
- * 单点住在源码行里,只有源码不可用(未捕获或 artifact 缺失)时这里才出现,呈现同一份事实——
118
- * failed / unavailable 与没过阈值的 soft 每条一行、一次展开明细,第一条 failed 默认展开;
119
- * passed 按 group 收进默认折叠区,只显示数量。
120
- */
121
- function AssertionSection({ assertions, t }: { assertions: Assertion[]; t: T }) {
122
- if (!assertions.length) return null;
123
- const attention = assertions.filter((a) => a.outcome !== "passed");
124
- const passed = assertions.filter((a) => a.outcome === "passed");
125
- const firstFailed = attention.findIndex((a) => a.outcome === "failed");
126
- const openIdx = firstFailed === -1 ? 0 : firstFailed;
127
- const groups = new Map<string, Assertion[]>();
128
- for (const a of passed) {
129
- const g = a.groupPath?.join(" > ") ?? "";
130
- groups.set(g, [...(groups.get(g) ?? []), a]);
131
- }
132
- return (
133
- <div className="attempt-asserts">
134
- <div className="modal-sect">{t("code.checks")}</div>
135
- {attention.length ? (
136
- <div className="attempt-asserts-open">
137
- {attention.map((a, i) => (
138
- <FailureRow key={i} a={a} defaultOpen={i === openIdx} t={t} />
139
- ))}
140
- </div>
141
- ) : null}
142
- {passed.length ? (
143
- <details className="asserts-passed">
144
- <summary>{t("assert.passedCollapsed", { count: passed.length })}</summary>
145
- {[...groups.entries()].map(([g, list]) => (
146
- <details key={g || "·"} className="asserts-passed-group">
147
- <summary>
148
- {g || "—"} · {list.length}
149
- </summary>
150
- <AssertDetail asserts={list.map(stripGroupPath)} t={t} anchor={locLabel} />
151
- </details>
152
- ))}
153
- </details>
154
- ) : null}
155
- </div>
156
- );
157
- }
158
-
159
- /** 折叠区已按 group 分组,行内不再重复组前缀。 */
160
- function stripGroupPath(a: Assertion): Assertion {
161
- const { groupPath: _gp, ...rest } = a;
162
- return rest as Assertion;
163
- }
164
-
165
- /** fallback 断言区里的一行:标题(徽章/名字/matcher/分数/位置),明细一次展开可见。 */
166
- function FailureRow({ a, defaultOpen, t }: { a: Assertion; defaultOpen: boolean; t: T }) {
167
- return (
168
- <details className="fail-row" open={defaultOpen || undefined}>
169
- <summary className="fail-sum">
170
- <span className={`abadge ${assertTone(a)}`}>
171
- {a.outcome === "unavailable" ? t("assert.unavailable") : t("assert.fail")}
172
- </span>
173
- <span className="fail-name">
174
- {a.groupPath?.length ? `${a.groupPath.join(" > ")} · ` : ""}
175
- {a.name}
176
- </span>
177
- {a.detail && a.detail !== a.name ? <span className="fail-matcher">{a.detail}</span> : null}
178
- {a.optional ? <span className="assert-sev">{t("assert.optional")}</span> : null}
179
- {a.severity === "soft" ? <span className="assert-sev">{t("assert.soft")}</span> : null}
180
- {a.outcome !== "unavailable" && a.threshold !== undefined ? (
181
- <span className="assert-score">
182
- {formatScore(a.score)} / {formatScore(a.threshold)}
183
- </span>
184
- ) : null}
185
- {locLabel(a)}
186
- </summary>
187
- <div className="fail-body">
188
- {a.outcome === "unavailable" ? <div className="assert-reason">{a.reason}</div> : null}
189
- {a.outcome !== "unavailable" && a.expected !== undefined ? (
190
- <div className="assert-reason">expected: {a.expected}</div>
191
- ) : null}
192
- {a.outcome !== "unavailable" && a.received !== undefined ? (
193
- <div className="assert-reason">received: {a.received}</div>
194
- ) : null}
195
- {a.outcome !== "unavailable" && a.evidence ? (
196
- <details className="assert-evidence">
197
- <summary>{t("assert.evidence")}</summary>
198
- <pre className="assert-evidence-pre">{a.evidence}</pre>
199
- </details>
200
- ) : null}
201
- </div>
202
- </details>
203
- );
204
- }
205
-
206
- /* ───────────────────────── 结构化错误与 diagnostics ───────────────────────── */
207
-
208
- /** errored attempt 的结构化 error 明细(见 docs/feature/reports/view.md「结构化错误」)。结构化
209
- * `AttemptError`:operation / code / message + 可选 cause / stack。字段标签是低层技术标识,与终端
210
- * `niceeval show` 一样保持英文,不进 view 的 i18n 词典。 */
211
- function ErrorDetailBlock({ error }: { error: NonNullable<ViewResult["error"]> }) {
212
- const rows: [string, string][] = [
213
- ["phase", error.phase],
214
- ["code", error.code],
215
- ["message", error.message],
216
- ];
217
- if (error.cause) rows.push(["cause", error.cause.name ? `${error.cause.name} · ${error.cause.message}` : error.cause.message]);
218
- const stack = error.stack?.replace(/\n+$/, "") ?? "";
219
- return (
220
- <div>
221
- <dl className="grid grid-cols-[max-content_1fr] gap-x-3 gap-y-1 text-xs">
222
- {rows.map(([k, v]) => (
223
- <Fragment key={k}>
224
- <dt className="text-muted">{k}</dt>
225
- <dd className="min-w-0 break-words">{v}</dd>
226
- </Fragment>
227
- ))}
228
- </dl>
229
- {stack ? (
230
- <pre className="mt-2 overflow-x-auto whitespace-pre-wrap text-[11px] text-muted">{stack}</pre>
231
- ) : null}
232
- </div>
233
- );
234
- }
235
-
236
- /** attempt 级诊断,按 lifecycle 阶段分组(teardown/cleanup 等,与 verdict 独立;
237
- * 见 docs/feature/reports/view.md「Attempt 详情」)。 */
238
- function AttemptDiagnostics({ diagnostics, t }: { diagnostics: NonNullable<ViewResult["diagnostics"]>; t: T }) {
239
- const groups = new Map<string, typeof diagnostics>();
240
- for (const d of diagnostics) {
241
- const list = groups.get(d.phase) ?? [];
242
- groups.set(d.phase, [...list, d]);
243
- }
244
- return (
245
- <div className="mt-3">
246
- <div className="modal-sect">{t("attempt.diagnostics")}</div>
247
- {[...groups.entries()].map(([phase, list]) => (
248
- <div key={phase} className="mt-1">
249
- <div className="text-[11px] text-muted">{phase}</div>
250
- <ul className="mt-0.5 space-y-1 pl-3">
251
- {list.map((d, i) => (
252
- <li key={i} className="text-xs">
253
- <span className="text-muted">
254
- {d.level} · {d.code}
255
- </span>
256
- <div className="min-w-0 break-words">
257
- {d.message}
258
- {d.count && d.count > 1 ? ` (${d.count} occurrences)` : ""}
259
- </div>
260
- </li>
261
- ))}
262
- </ul>
263
- </div>
264
- ))}
265
- </div>
266
- );
267
- }
268
-
269
- /* ───────────────────────── 统一时间树 ───────────────────────── */
270
-
271
- const CLOSING_PHASES = new Set(["eval.teardown", "agent.teardown", "sandbox.teardown", "sandbox.suspend", "sandbox.stop"]);
272
-
273
- function fmtMs(ms: number): string {
274
- if (ms >= 60_000) return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
275
- if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
276
- return `${ms}ms`;
277
- }
278
-
279
- type PhaseEntry = NonNullable<ViewResult["phases"]>[number];
280
- type TimingChild = NonNullable<PhaseEntry["children"]>[number];
281
-
282
- /** trace.json 的共享按需装载:时间树的 turn 挂接与底部 trace 入口用同一份,只 fetch 一次。 */
283
- interface TraceHook {
284
- available: boolean;
285
- status: "idle" | "loading" | "ready" | "failed";
286
- spans: Span[] | null;
287
- load: () => void;
288
- }
289
-
290
- function useTraceSpans(base: string | undefined, hasTrace: boolean | undefined): TraceHook {
291
- const [state, setState] = useState<{ spans: Span[] | null; status: TraceHook["status"] }>({ spans: null, status: "idle" });
292
- const started = useRef(false);
293
- const available = Boolean(base && hasTrace);
294
- const load = useCallback(() => {
295
- if (started.current || !base) return;
296
- started.current = true;
297
- setState({ spans: null, status: "loading" });
298
- fetch(artifactUrl(`${base}/trace.json`))
299
- .then((r) => (r.ok ? r.json() : Promise.reject(new Error(`HTTP ${r.status}`))))
300
- .then((body) => setState({ spans: asSpans(body) ?? [], status: "ready" }))
301
- .catch(() => setState({ spans: null, status: "failed" }));
302
- }, [base]);
303
- return useMemo(() => ({ available, load, ...state }), [available, load, state]);
304
- }
305
-
306
- const seriesClass = (i: number, failed?: true): string => (failed ? "phase-seg-bad" : `phase-seg-${(i % 6) + 1}`);
307
-
308
- /**
309
- * 统一时间树(见 docs/feature/reports/view.md「Attempt 详情」):整区一个折叠块,收合时只占
310
- * 一行(标题 + 总耗时),有失败 phase 的 attempt 默认展开;展开后主链先画一条分解条,列表里
311
- * 每个 phase 的 children(hook / 沙箱命令 / session/turn)默认收合、可逐个展开。
312
- * 收尾段单列,不计入总耗时。
313
- */
314
- function PhaseTimingBlock({ phases, trace, t }: { phases: NonNullable<ViewResult["phases"]>; trace: TraceHook; t: T }) {
315
- const main = phases.filter((p) => !CLOSING_PHASES.has(p.name));
316
- const closing = phases.filter((p) => CLOSING_PHASES.has(p.name));
317
- const total = main.reduce((sum, p) => sum + p.durationMs, 0);
318
- const anyFailed = phases.some((p) => p.failed);
319
- const [open, setOpen] = useState(anyFailed);
320
- return (
321
- <details className="attempt-timing" open={open} onToggle={(e) => setOpen(e.currentTarget.open)}>
322
- <summary className="timing-sum">
323
- <span className="modal-sect">{t("attempt.timing")}</span>
324
- <span className="phase-dur num">{fmtMs(total)}</span>
325
- {anyFailed ? <span className="phase-fail">✗</span> : null}
326
- </summary>
327
- {open ? (
328
- <>
329
- {total > 0 ? (
330
- <div className="phase-bar" aria-hidden="true">
331
- {main.map((p, i) => (
332
- <div
333
- key={i}
334
- className={`phase-seg ${seriesClass(i, p.failed)}`}
335
- style={{ flexGrow: Math.max(p.durationMs, total / 200) }}
336
- title={`${p.name} ${fmtMs(p.durationMs)}`}
337
- />
338
- ))}
339
- </div>
340
- ) : null}
341
- <ul className="phase-list">
342
- {main.map((p, i) => (
343
- <PhaseRow key={i} phase={p} dotClass={seriesClass(i, p.failed)} trace={trace} t={t} />
344
- ))}
345
- </ul>
346
- {closing.length > 0 ? (
347
- <div className="phase-closing">
348
- <div className="phase-closing-head">{t("attempt.teardown")}</div>
349
- <ul className="phase-list">
350
- {closing.map((p, i) => (
351
- <PhaseRow key={i} phase={p} trace={trace} t={t} />
352
- ))}
353
- </ul>
354
- </div>
355
- ) : null}
356
- </>
357
- ) : null}
358
- </details>
359
- );
360
- }
361
-
362
- function PhaseRow({ phase, dotClass, trace, t }: { phase: PhaseEntry; dotClass?: string; trace: TraceHook; t: T }) {
363
- const kids = phase.children ?? [];
364
- // 失败 phase 默认展开:失败线索直接给出,其余保持收合(契约:children 默认收合)。
365
- const [open, setOpen] = useState(Boolean(phase.failed));
366
- const head = (
367
- <>
368
- {dotClass ? <span className={`phase-dot ${dotClass}`} /> : <span className="phase-dot phase-dot-none" />}
369
- <span className="phase-name">{phase.name}</span>
370
- <span className="phase-dur num">{fmtMs(phase.durationMs)}</span>
371
- {phase.failed ? <span className="phase-fail">✗</span> : null}
372
- {kids.length > 0 ? <span className="phase-kids num">{countNodes(kids)}</span> : null}
373
- </>
374
- );
375
- if (kids.length === 0) return <li className="phase-row">{head}</li>;
376
- return (
377
- <li>
378
- <details className="phase-details" open={open} onToggle={(e) => setOpen(e.currentTarget.open)}>
379
- <summary className="phase-row">{head}</summary>
380
- {open ? (
381
- <ul className="timing-kids">
382
- {kids.map((node) => (
383
- <TimingNodeRow key={node.id} node={node} trace={trace} t={t} />
384
- ))}
385
- </ul>
386
- ) : null}
387
- </details>
388
- </li>
389
- );
390
- }
391
-
392
- function countNodes(nodes: TimingChild[]): number {
393
- return nodes.reduce((sum, n) => sum + 1 + countNodes(n.children ?? []), 0);
394
- }
395
-
396
- function TimingNodeRow({ node, trace, t }: { node: TimingChild; trace: TraceHook; t: T }) {
397
- const kids = node.children ?? [];
398
- // turn 带 traceId 且这次运行有 trace.json 时,展开即挂接同一轮的 agent/model/tool spans。
399
- const hooksTrace = node.kind === "turn" && Boolean(node.traceId) && trace.available;
400
- const [open, setOpen] = useState(Boolean(node.failed) && (kids.length > 0 || hooksTrace));
401
- const label =
402
- node.kind === "command" && node.command ? `shell · ${node.command.display}` : node.kind === "turn" ? `turn ${node.label}` : node.label;
403
- const head = (
404
- <>
405
- <span className="tnode-label" title={label}>{label}</span>
406
- <span className="phase-dur num">{fmtMs(node.durationMs)}</span>
407
- {node.failed ? <span className="phase-fail">✗</span> : null}
408
- </>
409
- );
410
- if (kids.length === 0 && !hooksTrace) return <li className="tnode-row">{head}</li>;
411
- return (
412
- <li>
413
- <details className="phase-details" open={open} onToggle={(e) => setOpen(e.currentTarget.open)}>
414
- <summary className="tnode-row">{head}</summary>
415
- {open ? (
416
- <>
417
- {kids.length > 0 ? (
418
- <ul className="timing-kids">
419
- {kids.map((child) => (
420
- <TimingNodeRow key={child.id} node={child} trace={trace} t={t} />
421
- ))}
422
- </ul>
423
- ) : null}
424
- {hooksTrace ? <TurnSpans traceId={node.traceId!} trace={trace} t={t} /> : null}
425
- </>
426
- ) : null}
427
- </details>
428
- </li>
429
- );
430
- }
431
-
432
- function TurnSpans({ traceId, trace, t }: { traceId: string; trace: TraceHook; t: T }) {
433
- useEffect(() => { trace.load(); }, [trace]);
434
- if (trace.status === "idle" || trace.status === "loading") return <div className="tnode-meta">{t("trace.loading")}</div>;
435
- if (trace.status === "failed") return <div className="tnode-meta">{t("trace.loadFailed")}</div>;
436
- const spans = (trace.spans ?? []).filter((s) => s.traceId === traceId);
437
- if (!spans.length) return <div className="tnode-meta">{t("trace.noSpans")}</div>;
438
- return (
439
- <div className="tnode-trace">
440
- <Trace spans={spans} t={t} />
441
- </div>
442
- );
443
- }
444
-
445
- /** 底部 trace 入口:被测 agent 的完整原始 span 瀑布,与时间树共享同一次 trace.json 拉取。 */
446
- function FullTraceEntry({ trace, t }: { trace: TraceHook; t: T }) {
447
- const [open, setOpen] = useState(false);
448
- return (
449
- <details
450
- className="trace-details"
451
- open={open}
452
- onToggle={(e) => {
453
- const isOpen = e.currentTarget.open;
454
- setOpen(isOpen);
455
- if (isOpen) trace.load();
456
- }}
457
- >
458
- <summary>{t("trace.timing")}</summary>
459
- <div className="trace-slot">
460
- {trace.status === "failed" ? (
461
- <div className="trace-span-meta">{t("trace.loadFailed")}</div>
462
- ) : trace.status !== "ready" ? (
463
- <div className="trace-span-meta">{t("trace.loading")}</div>
464
- ) : (
465
- <Trace spans={trace.spans ?? []} t={t} />
466
- )}
467
- </div>
468
- </details>
469
- );
470
- }
471
-
472
- /* ───────────────────────── usage 行 ───────────────────────── */
473
-
474
- /** usage / sandbox 元信息 chips(见 docs/feature/reports/view.md「Attempt 详情」)。 */
475
- function UsageDiffLine({ result }: { result: ViewResult }) {
476
- const chips: React.ReactNode[] = [];
477
- if (result.usage) {
478
- const tok = result.usage.inputTokens + result.usage.outputTokens;
479
- chips.push(
480
- <span className="meta-chip" key="usage">
481
- usage <b>{tok.toLocaleString()} tok</b>
482
- {result.usage.costUSD !== undefined ? <> · ${result.usage.costUSD.toFixed(4)}</> : null}
483
- </span>,
484
- );
485
- }
486
- if (result.sandbox) {
487
- chips.push(
488
- <span className="meta-chip" key="sandbox">
489
- sandbox <b>{result.sandbox.provider}</b> · {result.sandbox.sandboxId}
490
- {result.sandbox.kept ? " · kept" : ""}
491
- </span>,
492
- );
493
- }
494
- if (chips.length === 0) return null;
495
- return <div className="meta-chips">{chips}</div>;
496
- }
@@ -1,142 +0,0 @@
1
- // @vitest-environment jsdom
2
- // cases: docs/engineering/unit-tests/reports/cases.md
3
- // 「Attempt 详情(view 证据室)」分区——
4
- // 源码视图是判定与断言的单点:带 loc 的 send 行可点开查看该轮回复(assistant 文本 / thinking),
5
- // 失败断言行默认展开、展开面直接给 matcher 与 expected / received 的值。
6
- // 这组测试在 jsdom 里跑真实点击,守住「统一站点管线之后弹窗证据链仍然可用」的行为面
7
- // (数据侧的字节奇偶由 src/view/site-parity.test.ts 守护,两头合起来 = 线上站点可用)。
8
-
9
- import { act } from "react";
10
- import { createRoot, type Root } from "react-dom/client";
11
- import { afterEach, beforeAll, describe, expect, it } from "vitest";
12
- import { CodeView } from "./CodeView.tsx";
13
- import { makeTranslator } from "../i18n.ts";
14
- import type { Assertion, CodeSource, TranscriptEvent } from "../types.ts";
15
-
16
- declare global {
17
- // eslint-disable-next-line no-var
18
- var IS_REACT_ACT_ENVIRONMENT: boolean;
19
- }
20
-
21
- beforeAll(() => {
22
- globalThis.IS_REACT_ACT_ENVIRONMENT = true;
23
- });
24
-
25
- const t = makeTranslator("en");
26
-
27
- const FILE = "evals/a.eval.ts";
28
- const SOURCE: CodeSource = {
29
- path: FILE,
30
- content: [
31
- "import { defineEval } from \"niceeval\";",
32
- "",
33
- "await t.send(\"do the task\");",
34
- "",
35
- "t.check(source, includes(/use cache/));",
36
- ].join("\n"),
37
- };
38
-
39
- const EVENTS: TranscriptEvent[] = [
40
- { type: "message", role: "user", text: "do the task", loc: { file: FILE, line: 3 } },
41
- { type: "thinking", text: "THINKING_MARKER" },
42
- { type: "message", role: "assistant", text: "REPLY_TEXT_MARKER" },
43
- ];
44
-
45
- const FAILED_ASSERT: Assertion = {
46
- name: "Catalog reads use cache",
47
- detail: "includes(/use cache/)",
48
- severity: "gate",
49
- outcome: "failed",
50
- score: 0,
51
- expected: "/use cache/",
52
- received: "RECEIVED_VALUE_MARKER",
53
- loc: { file: FILE, line: 5 },
54
- };
55
-
56
- let container: HTMLElement | undefined;
57
- let root: Root | undefined;
58
-
59
- function render(ui: React.ReactElement): HTMLElement {
60
- container = document.createElement("div");
61
- document.body.appendChild(container);
62
- root = createRoot(container);
63
- act(() => root!.render(ui));
64
- return container;
65
- }
66
-
67
- afterEach(() => {
68
- if (root) act(() => root!.unmount());
69
- container?.remove();
70
- root = undefined;
71
- container = undefined;
72
- });
73
-
74
- function click(el: Element): void {
75
- act(() => {
76
- el.dispatchEvent(new MouseEvent("click", { bubbles: true }));
77
- });
78
- }
79
-
80
- /** 按行号取源码行的 DOM 节点(.code-line 里第一个 .ln 是行号)。 */
81
- function lineEl(host: HTMLElement, n: number): Element {
82
- const row = [...host.querySelectorAll(".code-line")].find((el) => el.querySelector(".ln")?.textContent === String(n));
83
- expect(row, `source line ${n}`).toBeTruthy();
84
- return row!;
85
- }
86
-
87
- describe("CodeView · send 行的回复展开", () => {
88
- it("带 loc 的 send 行可点开:回复面板显示 assistant 文本与 thinking;再点收起", () => {
89
- const host = render(<CodeView sources={[SOURCE]} events={EVENTS} assertions={[]} t={t} />);
90
-
91
- const sendLine = lineEl(host, 3);
92
- expect(sendLine.className).toContain("line-send");
93
- // 初始未展开:回复不可见。
94
- expect(host.textContent).not.toContain("REPLY_TEXT_MARKER");
95
-
96
- click(sendLine);
97
- expect(host.querySelector(".reply-panel")).toBeTruthy();
98
- expect(host.textContent).toContain("REPLY_TEXT_MARKER");
99
- expect(host.textContent).toContain("THINKING_MARKER");
100
-
101
- click(lineEl(host, 3));
102
- expect(host.textContent).not.toContain("REPLY_TEXT_MARKER");
103
- });
104
-
105
- it("send 轮没有任何回复事件时,展开面如实显示「无回复」而不是空白", () => {
106
- const onlySend: TranscriptEvent[] = [EVENTS[0]!];
107
- const host = render(<CodeView sources={[SOURCE]} events={onlySend} assertions={[]} t={t} />);
108
- click(lineEl(host, 3));
109
- expect(host.querySelector(".reply-empty")?.textContent).toBe(t("code.noReply"));
110
- });
111
- });
112
-
113
- describe("CodeView · 断言行的明细展开", () => {
114
- it("第一条失败断言默认展开:matcher 与 expected / received 的值直接可见,点行可收起", () => {
115
- const host = render(<CodeView sources={[SOURCE]} events={EVENTS} assertions={[FAILED_ASSERT]} t={t} />);
116
-
117
- const assertLine = lineEl(host, 5);
118
- expect(assertLine.className).toContain("line-fail");
119
- // 默认展开(第一条 failed):不点任何东西就能看到为什么失败。
120
- expect(host.textContent).toContain("includes(/use cache/)");
121
- expect(host.textContent).toContain("/use cache/");
122
- expect(host.textContent).toContain("RECEIVED_VALUE_MARKER");
123
-
124
- click(assertLine);
125
- expect(host.textContent).not.toContain("RECEIVED_VALUE_MARKER");
126
- });
127
-
128
- it("passed 断言行不默认展开,点开后显示明细", () => {
129
- const passed: Assertion = { ...FAILED_ASSERT, outcome: "passed", score: 1, loc: { file: FILE, line: 5 } };
130
- delete (passed as { expected?: string }).expected;
131
- delete (passed as { received?: string }).received;
132
- const host = render(<CodeView sources={[SOURCE]} events={EVENTS} assertions={[passed]} t={t} />);
133
-
134
- const assertLine = lineEl(host, 5);
135
- expect(assertLine.className).toContain("line-pass");
136
- expect(host.querySelector(".line-detail")).toBeNull();
137
-
138
- click(assertLine);
139
- expect(host.querySelector(".line-detail")).toBeTruthy();
140
- expect(host.textContent).toContain("includes(/use cache/)");
141
- });
142
- });