niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -3,8 +3,8 @@
3
3
  // (docs/feature/reports/library/built-in.md)。新增内建视图 = 新文件 + 新具名导出,
4
4
  // 不需要注册表,也不改变装载管线。
5
5
 
6
- import { standard } from "./standard.tsx";
6
+ import { standard, standardAttemptPage } from "./standard.tsx";
7
7
 
8
- export { standard };
8
+ export { standard, standardAttemptPage };
9
9
 
10
10
  export default standard;
@@ -1,14 +1,17 @@
1
- // standard —— 内建视图之一:报告 / Attempts / 追踪三页的通用结果站,没有任何私有钩子
2
- // (docs/feature/reports/library/built-in.md)。裸 `niceeval show` 与 `niceeval view`
3
- // 装载的就是它:一份普通 defineReport,与用户的 --report 文件同层、走同一条
1
+ // standard —— 内建视图之一:报告 / Attempts / 追踪三页的通用结果站 + 一张隐藏的 attempt
2
+ // 详情页,没有任何私有钩子(docs/feature/reports/library/built-in.md)。裸 `niceeval show` 与
3
+ // `niceeval view` 装载的就是它:一份普通 defineReport,与用户的 --report 文件同层、走同一条
4
4
  // 装载 → resolve → validate → render 管线。「builtin」不是装载逻辑里的类别,
5
5
  // 只是宿主默认拿哪个值的事实;用户报告经 defineReport({ extends: standard, … }) 整站复用。
6
6
  //
7
- // 三页站点:report(Hero + 警告 + 批量修复 prompt + 实验对比)、attempts(带过滤的
8
- // attempt 全列表)、traces(执行瀑布)。裸宿主导航上能看到的一切内容都在这份定义里;
9
- // 宿主保留的只有机器(docs/feature/reports/architecture.md「宿主保留的只有机器」)。
7
+ // 四页:report(Hero + 警告 + 批量修复 prompt + 实验对比)、attempts(带过滤的 attempt
8
+ // 全列表)、traces(执行瀑布)三页进导航;第四张 standardAttemptPage 以 locator 为输入、
9
+ // 不进导航,locator 深链打开它,content 就是公开的 AttemptDetail 组合组件。裸宿主导航上能
10
+ // 看到的一切内容都在这份定义里;宿主保留的只有机器(docs/feature/reports/architecture.md
11
+ // 「宿主保留的只有机器」)。
10
12
 
11
13
  import {
14
+ AttemptDetail,
12
15
  AttemptList,
13
16
  Col,
14
17
  CopyFixPrompt,
@@ -19,6 +22,14 @@ import {
19
22
  defineReport,
20
23
  } from "../index.ts";
21
24
 
25
+ export const standardAttemptPage = {
26
+ id: "attempt",
27
+ title: "Attempt",
28
+ input: "attempt",
29
+ navigation: false,
30
+ content: <AttemptDetail />,
31
+ } as const;
32
+
22
33
  export const standard = defineReport({
23
34
  pages: [
24
35
  {
@@ -55,5 +66,6 @@ export const standard = defineReport({
55
66
  </Col>
56
67
  ),
57
68
  },
69
+ standardAttemptPage,
58
70
  ],
59
71
  });
@@ -0,0 +1,67 @@
1
+ // AttemptAssertions:全量 assertion,非 passed 默认展开、passed 按 group 折叠计数。
2
+ // 没有 assertion 时零输出(docs/feature/reports/library/attempt-detail.md)。
3
+
4
+ import type { ReactElement } from "react";
5
+ import type { AttemptAssertionsData } from "../../model/types.ts";
6
+ import type { AssertionResult } from "../../../types.ts";
7
+ import { cx } from "../shared.ts";
8
+
9
+ function assertTone(a: AssertionResult): "good" | "warn" | "bad" | "na" {
10
+ if (a.outcome === "unavailable") return "na";
11
+ if (a.outcome === "passed") return "good";
12
+ return a.severity === "soft" ? "warn" : "bad";
13
+ }
14
+
15
+ function AssertionRow({ a }: { a: AssertionResult }): ReactElement {
16
+ return (
17
+ <details className="nre-assertion-row" open>
18
+ <summary>
19
+ <span className={`nre-assertion-badge nre-tone-${assertTone(a)}`}>
20
+ {a.outcome === "unavailable" ? "unavailable" : a.outcome}
21
+ </span>
22
+ <span className="nre-assertion-name">
23
+ {a.groupPath?.length ? `${a.groupPath.join(" > ")} · ` : ""}
24
+ {a.name}
25
+ </span>
26
+ {a.detail && a.detail !== a.name ? <span className="nre-assertion-detail">{a.detail}</span> : null}
27
+ </summary>
28
+ <div className="nre-assertion-body">
29
+ {a.outcome === "unavailable" ? <div>{a.reason}</div> : null}
30
+ {a.outcome !== "unavailable" && a.expected !== undefined ? <div>expected: {a.expected}</div> : null}
31
+ {a.outcome !== "unavailable" && a.received !== undefined ? <div>received: {a.received}</div> : null}
32
+ </div>
33
+ </details>
34
+ );
35
+ }
36
+
37
+ export function AttemptAssertions({
38
+ data,
39
+ className,
40
+ }: {
41
+ data: AttemptAssertionsData | null;
42
+ className?: string;
43
+ }): ReactElement | null {
44
+ if (data === null) return null;
45
+ return (
46
+ <div className={cx("nre", "nre-attempt-assertions", className)}>
47
+ {data.attention.map((a, i) => (
48
+ <AssertionRow key={i} a={a} />
49
+ ))}
50
+ {data.passedGroups.length > 0 ? (
51
+ <details className="nre-assertions-passed">
52
+ <summary>passed · {data.passedGroups.reduce((n, g) => n + g.items.length, 0)}</summary>
53
+ {data.passedGroups.map(({ group, items }) => (
54
+ <details key={group || "·"} className="nre-assertions-passed-group">
55
+ <summary>
56
+ {group || "—"} · {items.length}
57
+ </summary>
58
+ {items.map((a, i) => (
59
+ <AssertionRow key={i} a={a} />
60
+ ))}
61
+ </details>
62
+ ))}
63
+ </details>
64
+ ) : null}
65
+ </div>
66
+ );
67
+ }
@@ -0,0 +1,158 @@
1
+ // AttemptConversation:标准事件流按轮组织的完整分轮事件卡。没有 events 时零输出
2
+ // (docs/feature/reports/library/attempt-detail.md)。
3
+
4
+ import type { ReactElement, ReactNode } from "react";
5
+ import type { AttemptConversationData, AttemptConversationReply, AttemptConversationRound } from "../../model/types.ts";
6
+ import type { JsonValue, ToolName } from "../../../types.ts";
7
+ import { cx } from "../shared.ts";
8
+
9
+ const TOOL_VERB: Partial<Record<ToolName, string>> = {
10
+ shell: "Bash",
11
+ file_read: "Read",
12
+ file_write: "Write",
13
+ file_edit: "Edit",
14
+ web_fetch: "Fetch",
15
+ web_search: "Search",
16
+ glob: "Glob",
17
+ grep: "Grep",
18
+ list_dir: "List",
19
+ agent_task: "Task",
20
+ };
21
+
22
+ function compact(value: JsonValue | undefined, max = 140): string {
23
+ if (value === undefined || value === null) return "";
24
+ const text = typeof value === "string" ? value : JSON.stringify(value);
25
+ const oneLine = text.replace(/\s+/g, " ").trim();
26
+ return oneLine.length > max ? `${oneLine.slice(0, max)}…` : oneLine;
27
+ }
28
+
29
+ function toolPrimaryArg(input: JsonValue): string {
30
+ if (typeof input === "string") return compact(input, 80);
31
+ if (input === null || Array.isArray(input) || typeof input !== "object") return "";
32
+ for (const key of ["command", "cmd", "path", "file", "file_path", "pattern", "query", "url", "prompt", "description"]) {
33
+ const value = input[key];
34
+ if (typeof value === "string" && value) return compact(value, 80);
35
+ if (key === "command" && Array.isArray(value)) return compact(value.filter((item) => typeof item === "string").join(" "), 80);
36
+ }
37
+ return compact(input, 80);
38
+ }
39
+
40
+ function ReplyRow({ reply }: { reply: AttemptConversationReply }): ReactNode {
41
+ switch (reply.kind) {
42
+ case "assistant":
43
+ return (
44
+ <div className="nre-conv-assistant">
45
+ <span className="nre-conv-role">assistant</span>
46
+ <div className="nre-conv-text">{reply.text}</div>
47
+ </div>
48
+ );
49
+ case "user":
50
+ return (
51
+ <div className="nre-conv-user">
52
+ <span className="nre-conv-role">user</span>
53
+ <div className="nre-conv-text">{reply.text}</div>
54
+ </div>
55
+ );
56
+ case "thinking":
57
+ return (
58
+ <details className="nre-conv-thinking">
59
+ <summary>thinking</summary>
60
+ <div className="nre-conv-text">{reply.text}</div>
61
+ </details>
62
+ );
63
+ case "error":
64
+ return <div className="nre-conv-error">! {reply.text}</div>;
65
+ case "skill":
66
+ return (
67
+ <div className="nre-conv-skill">
68
+ <span className="nre-conv-role">skill loaded</span> {reply.skill}
69
+ </div>
70
+ );
71
+ case "tool":
72
+ const verb = (reply.tool ? TOOL_VERB[reply.tool] : undefined) ?? reply.name;
73
+ const arg = toolPrimaryArg(reply.input);
74
+ const preview = compact(reply.output);
75
+ return (
76
+ <details className="nre-conv-tool">
77
+ <summary>
78
+ <span className={cx("nre-conv-tool-dot", reply.status ? `nre-conv-tool-${reply.status}` : "nre-conv-tool-pending")} />
79
+ <span className="nre-conv-tool-name" title={arg ? `${verb}(${arg})` : verb}>
80
+ {arg ? `${verb}(${arg})` : verb}
81
+ </span>
82
+ <span className="nre-conv-tool-preview">
83
+ {reply.status ?? "pending"}
84
+ {preview ? ` · ${preview}` : ""}
85
+ </span>
86
+ </summary>
87
+ <pre className="nre-conv-tool-io">{JSON.stringify(reply.input, null, 2)}</pre>
88
+ {reply.output !== undefined ? <pre className="nre-conv-tool-io">{JSON.stringify(reply.output, null, 2)}</pre> : null}
89
+ </details>
90
+ );
91
+ case "subagent":
92
+ return (
93
+ <details className="nre-conv-subagent">
94
+ <summary>
95
+ subagent {reply.name}
96
+ {reply.status ? ` · ${reply.status}` : ""}
97
+ </summary>
98
+ {reply.output !== undefined ? <pre className="nre-conv-tool-io">{JSON.stringify(reply.output, null, 2)}</pre> : null}
99
+ </details>
100
+ );
101
+ case "input":
102
+ return <div className="nre-conv-input">input requested{reply.request.prompt ? `: ${reply.request.prompt}` : ""}</div>;
103
+ case "compaction":
104
+ return <div className="nre-conv-compaction">compaction{reply.reason ? `: ${reply.reason}` : ""}</div>;
105
+ case "raw":
106
+ return (
107
+ <details className="nre-conv-raw">
108
+ <summary>unrecognized event</summary>
109
+ <pre className="nre-conv-tool-io">{JSON.stringify(reply.raw, null, 2)}</pre>
110
+ </details>
111
+ );
112
+ }
113
+ }
114
+
115
+ /** AttemptSource 复用同一份回复 renderer,把一轮执行挂回对应的 send 源码行。 */
116
+ export function ConversationReplies({ replies }: { replies: AttemptConversationReply[] }): ReactElement {
117
+ return (
118
+ <div className="nre-conv-replies">
119
+ {replies.map((reply, i) => (
120
+ <ReplyRow key={i} reply={reply} />
121
+ ))}
122
+ </div>
123
+ );
124
+ }
125
+
126
+ function RoundCard({ round, index }: { round: AttemptConversationRound; index: number }): ReactElement {
127
+ return (
128
+ <div className="nre-conv-round">
129
+ <div className="nre-conv-round-head">
130
+ round {index + 1}
131
+ {round.loc ? (
132
+ <span className="nre-conv-round-loc" title={`${round.loc.file}:${round.loc.line}`}>
133
+ {round.loc.file.split("/").pop()}:{round.loc.line}
134
+ </span>
135
+ ) : null}
136
+ </div>
137
+ {round.sentText ? <div className="nre-conv-sent">{round.sentText}</div> : null}
138
+ <ConversationReplies replies={round.replies} />
139
+ </div>
140
+ );
141
+ }
142
+
143
+ export function AttemptConversation({
144
+ data,
145
+ className,
146
+ }: {
147
+ data: AttemptConversationData | null;
148
+ className?: string;
149
+ }): ReactElement | null {
150
+ if (data === null) return null;
151
+ return (
152
+ <div className={cx("nre", "nre-attempt-conversation", className)}>
153
+ {data.rounds.map((round, i) => (
154
+ <RoundCard key={i} round={round} index={i} />
155
+ ))}
156
+ </div>
157
+ );
158
+ }
@@ -0,0 +1,38 @@
1
+ // AttemptDiagnostics:lifecycle 分组的 diagnostics。没有 diagnostics 时零输出
2
+ // (docs/feature/reports/library/attempt-detail.md)。
3
+
4
+ import type { ReactElement } from "react";
5
+ import type { AttemptDiagnosticsData } from "../../model/types.ts";
6
+ import { cx } from "../shared.ts";
7
+
8
+ export function AttemptDiagnostics({
9
+ data,
10
+ className,
11
+ }: {
12
+ data: AttemptDiagnosticsData | null;
13
+ className?: string;
14
+ }): ReactElement | null {
15
+ if (data === null) return null;
16
+ return (
17
+ <div className={cx("nre", "nre-attempt-diagnostics", className)}>
18
+ {data.groups.map(({ phase, items }) => (
19
+ <div key={phase} className="nre-diagnostics-group">
20
+ <div className="nre-diagnostics-phase">{phase}</div>
21
+ <ul>
22
+ {items.map((d, i) => (
23
+ <li key={i} className={`nre-diagnostics-item nre-tone-${d.level === "error" ? "bad" : "warn"}`}>
24
+ <span className="nre-diagnostics-meta">
25
+ {d.level} · {d.code}
26
+ </span>
27
+ <div>
28
+ {d.message}
29
+ {d.count && d.count > 1 ? ` (${d.count} occurrences)` : ""}
30
+ </div>
31
+ </li>
32
+ ))}
33
+ </ul>
34
+ </div>
35
+ ))}
36
+ </div>
37
+ );
38
+ }
@@ -0,0 +1,35 @@
1
+ // AttemptDiff:generated / modified / deleted 文件摘要与展开的行变化统计。没有变更时零输出
2
+ // (docs/feature/reports/library/attempt-detail.md)。
3
+
4
+ import type { ReactElement } from "react";
5
+ import type { AttemptDiffData, AttemptDiffFileEntry } from "../../model/types.ts";
6
+ import { cx } from "../shared.ts";
7
+
8
+ const NET_LABEL: Record<AttemptDiffFileEntry["net"], string> = { added: "generated", modified: "modified", deleted: "deleted" };
9
+
10
+ function FileRow({ file }: { file: AttemptDiffFileEntry }): ReactElement {
11
+ return (
12
+ <li className={cx("nre-diff-file", `nre-diff-${file.net}`)}>
13
+ <span className="nre-diff-net">{NET_LABEL[file.net]}</span>
14
+ <span className="nre-diff-path">{file.path}</span>
15
+ {file.binary ? (
16
+ <span className="nre-diff-lines">binary</span>
17
+ ) : (
18
+ <span className="nre-diff-lines">
19
+ +{file.lines.added} / -{file.lines.deleted}
20
+ </span>
21
+ )}
22
+ </li>
23
+ );
24
+ }
25
+
26
+ export function AttemptDiff({ data, className }: { data: AttemptDiffData | null; className?: string }): ReactElement | null {
27
+ if (data === null) return null;
28
+ return (
29
+ <ul className={cx("nre", "nre-attempt-diff", className)}>
30
+ {data.files.map((f) => (
31
+ <FileRow key={f.path} file={f} />
32
+ ))}
33
+ </ul>
34
+ );
35
+ }
@@ -0,0 +1,31 @@
1
+ // AttemptError:结构化 error 明细。没有 error 时零输出(docs/feature/reports/library/attempt-detail.md)。
2
+ // 字段标签是低层技术标识,与终端 `niceeval show` 一样保持英文,不进 view 的 i18n 词典。
3
+
4
+ import type { ReactElement } from "react";
5
+ import { Fragment } from "react";
6
+ import type { AttemptErrorData } from "../../model/types.ts";
7
+ import { cx } from "../shared.ts";
8
+
9
+ export function AttemptError({ data, className }: { data: AttemptErrorData | null; className?: string }): ReactElement | null {
10
+ if (data === null) return null;
11
+ const rows: [string, string][] = [
12
+ ["phase", data.phase],
13
+ ["code", data.code],
14
+ ["message", data.message],
15
+ ];
16
+ if (data.cause) rows.push(["cause", data.cause.name ? `${data.cause.name} · ${data.cause.message}` : data.cause.message]);
17
+ const stack = data.stack?.replace(/\n+$/, "") ?? "";
18
+ return (
19
+ <div className={cx("nre", "nre-attempt-error", className)}>
20
+ <dl className="nre-attempt-error-fields">
21
+ {rows.map(([k, v]) => (
22
+ <Fragment key={k}>
23
+ <dt>{k}</dt>
24
+ <dd>{v}</dd>
25
+ </Fragment>
26
+ ))}
27
+ </dl>
28
+ {stack ? <pre className="nre-attempt-error-stack">{stack}</pre> : null}
29
+ </div>
30
+ );
31
+ }
@@ -0,0 +1,26 @@
1
+ // AttemptFixPrompt:单条 attempt 的复制修复 prompt。passed 或没有可操作失败时零输出
2
+ // (docs/feature/reports/library/attempt-detail.md)。与 CopyFixPrompt 同一套增强约定:
3
+ // prompt 在 resolve 阶段算好、烘进静态 HTML,复制按钮是 enhance.js 的增强行为。
4
+
5
+ import type { ReactElement } from "react";
6
+ import type { AttemptFixPromptData } from "../../model/types.ts";
7
+ import { cx } from "../shared.ts";
8
+
9
+ export function AttemptFixPrompt({
10
+ data,
11
+ className,
12
+ }: {
13
+ data: AttemptFixPromptData | null;
14
+ className?: string;
15
+ }): ReactElement | null {
16
+ if (data === null) return null;
17
+ return (
18
+ <details className={cx("nre", "nre-attempt-fix-prompt", className)}>
19
+ <summary className="nre-attempt-fix-prompt-summary">Fix prompt</summary>
20
+ <button type="button" className="nre-attempt-fix-prompt-copy" data-nre-copy={data.prompt}>
21
+ Copy fix prompt
22
+ </button>
23
+ <pre className="nre-attempt-fix-prompt-text">{data.prompt}</pre>
24
+ </details>
25
+ );
26
+ }
@@ -0,0 +1,224 @@
1
+ // AttemptSource:GitHub diff 式带标注 eval 源码。send / assertion 行按状态着色,点击
2
+ // 原生 details 在调用点展开完整回复与 assertion 细节。没有 source 时零输出
3
+ // (docs/feature/reports/library/attempt-detail.md)。
4
+
5
+ import type { ReactElement, ReactNode } from "react";
6
+ import type { AttemptSourceData, AttemptSourceLineData, AttemptSourceTurn } from "../../model/types.ts";
7
+ import type { AssertionResult } from "../../../types.ts";
8
+ import { cx } from "../shared.ts";
9
+ import { ConversationReplies } from "./AttemptConversation.tsx";
10
+
11
+ const TS_HL_RE =
12
+ /(\/\/[^\n]*)|(\/\*[^]*?\*\/)|(`(?:\\.|[^`\\])*`|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|\b(import|from|export|default|const|let|var|async|await|function|return|if|else|for|of|in|new|class|extends|typeof|void|true|false|null|undefined)\b|\b(\d[\d_.]*)\b|([A-Za-z_$][\w$]*)(?=\s*\()/g;
13
+
14
+ /** 逐行零依赖 TS 高亮;token class 是稳定的 web 展示语义,不改源码文本。 */
15
+ function highlightTs(line: string): ReactNode[] {
16
+ const out: ReactNode[] = [];
17
+ let last = 0;
18
+ let i = 0;
19
+ let match: RegExpExecArray | null;
20
+ TS_HL_RE.lastIndex = 0;
21
+ while ((match = TS_HL_RE.exec(line))) {
22
+ if (match.index > last) out.push(line.slice(last, match.index));
23
+ const tokenClass =
24
+ match[1] || match[2] ? "tok-comment" : match[3] ? "tok-str" : match[4] ? "tok-kw" : match[5] ? "tok-num" : "tok-fn";
25
+ out.push(
26
+ <span key={i++} className={tokenClass}>
27
+ {match[0]}
28
+ </span>,
29
+ );
30
+ last = match.index + match[0].length;
31
+ if (match[0].length === 0) TS_HL_RE.lastIndex++;
32
+ }
33
+ if (last < line.length) out.push(line.slice(last));
34
+ return out;
35
+ }
36
+
37
+ function assertTone(a: AssertionResult): "good" | "warn" | "bad" | "na" {
38
+ if (a.outcome === "unavailable") return "na";
39
+ if (a.outcome === "passed") return "good";
40
+ return a.severity === "soft" ? "warn" : "bad";
41
+ }
42
+
43
+ function lineTone(line: AttemptSourceLineData): "good" | "warn" | "bad" | "na" | undefined {
44
+ if (line.assertions.length === 0) return undefined;
45
+ if (line.assertions.some((a) => assertTone(a) === "bad")) return "bad";
46
+ if (line.assertions.some((a) => assertTone(a) === "warn")) return "warn";
47
+ if (line.assertions.some((a) => assertTone(a) === "na")) return "na";
48
+ return "good";
49
+ }
50
+
51
+ function formatDuration(ms: number): string {
52
+ return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`;
53
+ }
54
+
55
+ function thresholdScores(assertions: AssertionResult[]): string[] {
56
+ const scores: string[] = [];
57
+ for (const assertion of assertions) {
58
+ if (assertion.outcome !== "unavailable" && assertion.threshold !== undefined) {
59
+ scores.push(`${assertion.score}/${assertion.threshold}`);
60
+ }
61
+ }
62
+ return scores;
63
+ }
64
+
65
+ /** 行号位标记(与产品站示例卡同语言:图标顶替行号,不加独立状态列);内联 SVG,零图标依赖。 */
66
+ const MARK_ICONS: Record<"send" | "good" | "bad" | "warn" | "na", ReactElement> = {
67
+ send: <path d="M21 15a4 4 0 0 1-4 4H8l-5 3V7a4 4 0 0 1 4-4h10a4 4 0 0 1 4 4z" />,
68
+ good: (
69
+ <>
70
+ <circle cx="12" cy="12" r="10" />
71
+ <path d="m9 12 2 2 4-4" />
72
+ </>
73
+ ),
74
+ bad: (
75
+ <>
76
+ <circle cx="12" cy="12" r="10" />
77
+ <path d="m15 9-6 6" />
78
+ <path d="m9 9 6 6" />
79
+ </>
80
+ ),
81
+ warn: (
82
+ <>
83
+ <circle cx="12" cy="12" r="10" />
84
+ <line x1="12" x2="12" y1="8" y2="12" />
85
+ <line x1="12" x2="12.01" y1="16" y2="16" />
86
+ </>
87
+ ),
88
+ na: (
89
+ <>
90
+ <circle cx="12" cy="12" r="10" />
91
+ <path d="M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3" />
92
+ <path d="M12 17h.01" />
93
+ </>
94
+ ),
95
+ };
96
+
97
+ function LineNo({ line, tone, send }: { line: number; tone: ReturnType<typeof lineTone>; send: boolean }): ReactElement {
98
+ const kind = tone ?? (send ? "send" : null);
99
+ if (kind === null) return <span className="nre-source-ln">{line}</span>;
100
+ const label = kind === "bad" ? "failed" : kind === "warn" ? "soft failed" : kind === "good" ? "passed" : kind === "na" ? "unavailable" : "send";
101
+ return (
102
+ <span className="nre-source-ln nre-source-ln-mark" role="img" aria-label={label} title={label}>
103
+ <svg viewBox="0 0 24 24" aria-hidden="true">{MARK_ICONS[kind]}</svg>
104
+ </span>
105
+ );
106
+ }
107
+
108
+ function LineSummary({ line, tone }: { line: AttemptSourceLineData; tone: ReturnType<typeof lineTone> }): ReactElement {
109
+ const interactive = line.assertions.length > 0 || line.sends.length > 0 || line.turns.length > 0;
110
+ const scores = thresholdScores(line.assertions);
111
+ return (
112
+ <span className="nre-source-line-summary">
113
+ <LineNo line={line.line} tone={tone} send={line.sends.length > 0 || line.turns.length > 0} />
114
+ <code className="nre-source-text">{highlightTs(line.text)}</code>
115
+ <span className="nre-source-line-meta">
116
+ {scores.length > 0 ? <span className="nre-source-score-badge">{scores.join(", ")}</span> : null}
117
+ {interactive ? <span className="nre-source-chevron">›</span> : null}
118
+ </span>
119
+ </span>
120
+ );
121
+ }
122
+
123
+ function TurnDetail({ turn, showMeta = false, showSent = false }: { turn: AttemptSourceTurn; showMeta?: boolean; showSent?: boolean }): ReactElement {
124
+ return (
125
+ <div className={cx("nre-source-turn", `nre-source-turn-${turn.status}`)}>
126
+ {showMeta ? (
127
+ <div className="nre-source-turn-head">
128
+ <span>{turn.label}</span>
129
+ <span>{turn.status}</span>
130
+ {turn.durationMs === undefined ? null : <span>{formatDuration(turn.durationMs)}</span>}
131
+ </div>
132
+ ) : null}
133
+ {showSent && turn.sentText ? <div className="nre-conv-sent">{turn.sentText}</div> : null}
134
+ <ConversationReplies replies={turn.replies} />
135
+ </div>
136
+ );
137
+ }
138
+
139
+ function AssertionDetail({ assertion }: { assertion: AssertionResult }): ReactElement {
140
+ return (
141
+ <div className={`nre-assertion-row nre-tone-${assertTone(assertion)}`}>
142
+ <div className="nre-source-assertion-head">
143
+ <span className="nre-assertion-badge">{assertion.outcome}</span>
144
+ <span className="nre-assertion-name">{assertion.name}</span>
145
+ {assertion.outcome !== "unavailable" ? <span className="nre-source-assertion-score">{assertion.score}</span> : null}
146
+ </div>
147
+ {assertion.detail ? <div className="nre-assertion-detail">{assertion.detail}</div> : null}
148
+ {assertion.outcome === "unavailable" ? (
149
+ <div className="nre-assertion-body">reason: {assertion.reason}</div>
150
+ ) : assertion.expected !== undefined || assertion.received !== undefined || assertion.evidence !== undefined ? (
151
+ <div className="nre-assertion-body">
152
+ {assertion.expected !== undefined ? <span>expected: {assertion.expected}</span> : null}
153
+ {assertion.received !== undefined ? <span>received: {assertion.received}</span> : null}
154
+ {assertion.evidence !== undefined ? <span>evidence: {assertion.evidence}</span> : null}
155
+ </div>
156
+ ) : null}
157
+ </div>
158
+ );
159
+ }
160
+
161
+ export function AttemptSource({ data, className }: { data: AttemptSourceData | null; className?: string }): ReactElement | null {
162
+ if (data === null) return null;
163
+ const firstAttentionLine = data.lines.find((line) => {
164
+ const tone = lineTone(line);
165
+ return tone === "bad" || tone === "warn" || tone === "na";
166
+ })?.line;
167
+ return (
168
+ <div className={cx("nre", "nre-attempt-source", className)}>
169
+ <div className="nre-attempt-source-head">{data.sourcePath}</div>
170
+ <div className="nre-attempt-source-lines">
171
+ {data.lines.map((line) => {
172
+ const tone = lineTone(line);
173
+ const interactive = line.assertions.length > 0 || line.sends.length > 0 || line.turns.length > 0;
174
+ const lineClass = cx(
175
+ "nre-source-line",
176
+ tone ? `nre-tone-${tone}` : undefined,
177
+ line.sends.length > 0 || line.turns.length > 0 ? "nre-source-line-send" : undefined,
178
+ );
179
+ if (!interactive) {
180
+ return (
181
+ <div key={line.line} className={lineClass}>
182
+ <LineSummary line={line} tone={tone} />
183
+ </div>
184
+ );
185
+ }
186
+ return (
187
+ <details key={line.line} className={lineClass} open={line.line === firstAttentionLine}>
188
+ <summary>
189
+ <LineSummary line={line} tone={tone} />
190
+ </summary>
191
+ <div className="nre-source-line-detail">
192
+ {line.turns.map((turn, i) => (
193
+ <TurnDetail key={`${turn.label}-${i}`} turn={turn} />
194
+ ))}
195
+ {line.assertions.map((assertion, i) => (
196
+ <AssertionDetail key={i} assertion={assertion} />
197
+ ))}
198
+ </div>
199
+ </details>
200
+ );
201
+ })}
202
+ </div>
203
+ {data.unmapped.length > 0 ? (
204
+ <div className="nre-attempt-source-unmapped">
205
+ <div className="nre-attempt-source-unmapped-head">Other assertions</div>
206
+ {data.unmapped.map((a, i) => (
207
+ <div key={i} className={`nre-assertion-row nre-tone-${assertTone(a)}`}>
208
+ {a.groupPath?.length ? `${a.groupPath.join(" > ")} · ` : ""}
209
+ {a.name}
210
+ </div>
211
+ ))}
212
+ </div>
213
+ ) : null}
214
+ {data.unlocatedTurns.length > 0 ? (
215
+ <div className="nre-attempt-source-unlocated">
216
+ <div className="nre-attempt-source-unmapped-head">Other conversation</div>
217
+ {data.unlocatedTurns.map((turn, i) => (
218
+ <TurnDetail key={`${turn.label}-${i}`} turn={turn} showMeta showSent />
219
+ ))}
220
+ </div>
221
+ ) : null}
222
+ </div>
223
+ );
224
+ }