niceeval 0.9.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (432) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +297 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +239 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +547 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +175 -0
  54. package/dist/report/components/entity-lists/index.d.ts +49 -0
  55. package/dist/report/components/entity-lists/index.js +169 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
  126. package/dist/report/{format.js → model/format.js} +79 -9
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +151 -24
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/e2b.d.ts +0 -1
  150. package/dist/sandbox/types.d.ts +23 -21
  151. package/dist/scoring/display.d.ts +7 -2
  152. package/dist/scoring/display.js +42 -4
  153. package/dist/scoring/types.d.ts +3 -3
  154. package/dist/shared/aggregate.d.ts +14 -0
  155. package/dist/shared/aggregate.js +31 -0
  156. package/dist/shared/types.d.ts +6 -1
  157. package/docs-site/images/logo-dark.svg +7 -0
  158. package/docs-site/images/logo.svg +6 -5
  159. package/docs-site/zh/README.md +8 -7
  160. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  161. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  162. package/docs-site/zh/examples/index.mdx +3 -3
  163. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
  164. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  167. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  168. package/docs-site/zh/explanation/adapter.mdx +19 -19
  169. package/docs-site/zh/explanation/assert.mdx +9 -9
  170. package/docs-site/zh/explanation/drive.mdx +4 -4
  171. package/docs-site/zh/explanation/evals.mdx +8 -8
  172. package/docs-site/zh/explanation/experiment.mdx +8 -8
  173. package/docs-site/zh/explanation/hitl.mdx +13 -13
  174. package/docs-site/zh/explanation/judge.mdx +3 -3
  175. package/docs-site/zh/explanation/overview.mdx +7 -7
  176. package/docs-site/zh/explanation/runner.mdx +17 -13
  177. package/docs-site/zh/explanation/tier.mdx +6 -6
  178. package/docs-site/zh/index.mdx +14 -14
  179. package/docs-site/zh/introduction.mdx +13 -16
  180. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  181. package/docs-site/zh/reference/capabilities.mdx +7 -27
  182. package/docs-site/zh/reference/cli.mdx +36 -33
  183. package/docs-site/zh/reference/define-agent.mdx +32 -31
  184. package/docs-site/zh/reference/define-config.mdx +6 -6
  185. package/docs-site/zh/reference/define-eval.mdx +33 -20
  186. package/docs-site/zh/reference/events.mdx +4 -4
  187. package/docs-site/zh/reference/expect.mdx +3 -3
  188. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  189. package/docs-site/zh/reference/report-components.mdx +195 -121
  190. package/docs-site/zh/reference/results-data.mdx +13 -13
  191. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  192. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  193. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  194. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  195. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  196. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  197. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  198. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  199. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  200. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  201. package/docs-site/zh/tutorials/experiments.mdx +102 -0
  202. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  203. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  204. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  205. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  207. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  208. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  209. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
  210. package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
  211. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  212. package/package.json +10 -7
  213. package/src/agents/bub.ts +21 -8
  214. package/src/agents/claude-code.ts +23 -10
  215. package/src/agents/codex.test.ts +22 -7
  216. package/src/agents/codex.ts +22 -9
  217. package/src/agents/index.ts +2 -2
  218. package/src/agents/openai-compat.ts +1 -1
  219. package/src/agents/post-setup.ts +45 -22
  220. package/src/agents/streaming.ts +1 -1
  221. package/src/agents/types.ts +20 -17
  222. package/src/agents/ui-message-stream.test.ts +10 -0
  223. package/src/agents/ui-message-stream.ts +12 -1
  224. package/src/cli.ts +75 -59
  225. package/src/context/types.ts +21 -21
  226. package/src/define.ts +13 -0
  227. package/src/i18n/en.ts +27 -12
  228. package/src/i18n/zh-CN.ts +27 -12
  229. package/src/index.ts +2 -0
  230. package/src/o11y/prices.json +176 -99
  231. package/src/report/{react → assets}/colors.ts +22 -0
  232. package/src/report/{react → assets}/enhance.js +1 -33
  233. package/src/report/assets/styles.css +1785 -0
  234. package/src/report/built-in/index.tsx +2 -2
  235. package/src/report/built-in/standard.tsx +18 -6
  236. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  237. package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
  238. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  239. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  240. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  241. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  242. package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
  243. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  244. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  245. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  246. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  247. package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
  248. package/src/report/components/attempt-detail/compute.ts +336 -0
  249. package/src/report/components/attempt-detail/faces.ts +260 -0
  250. package/src/report/components/attempt-detail/index.tsx +600 -0
  251. package/src/report/components/attempt-detail/validate.test.ts +252 -0
  252. package/src/report/{react → components}/cell.tsx +6 -3
  253. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  254. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  255. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  256. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
  257. package/src/report/components/entity-lists/compute.ts +196 -0
  258. package/src/report/components/entity-lists/faces.ts +208 -0
  259. package/src/report/components/entity-lists/index.tsx +238 -0
  260. package/src/report/components/entity-lists/validate.test.ts +114 -0
  261. package/src/report/{react → components}/fixtures.ts +2 -2
  262. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  263. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  264. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  265. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
  267. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  268. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  269. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  270. package/src/report/components/metric-views/faces.ts +423 -0
  271. package/src/report/components/metric-views/index.tsx +352 -0
  272. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  273. package/src/report/components/metric-views/validate.test.ts +211 -0
  274. package/src/report/{react → components}/render.test.tsx +66 -26
  275. package/src/report/components/shared-compute.ts +59 -0
  276. package/src/report/components/shared-faces.ts +30 -0
  277. package/src/report/components/shared.ts +192 -0
  278. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  279. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  280. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  281. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  282. package/src/report/components/site-components/compute.ts +144 -0
  283. package/src/report/components/site-components/faces.ts +80 -0
  284. package/src/report/components/site-components/index.tsx +265 -0
  285. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  286. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  287. package/src/report/components/site-components/validate.test.ts +89 -0
  288. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  289. package/src/report/components/summaries/compute.ts +56 -0
  290. package/src/report/components/summaries/faces.ts +48 -0
  291. package/src/report/components/summaries/index.tsx +127 -0
  292. package/src/report/components/summaries/validate.test.ts +49 -0
  293. package/src/report/definition/grid-layout.test.ts +124 -0
  294. package/src/report/definition/grid-layout.ts +146 -0
  295. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  296. package/src/report/{report.ts → definition/report.ts} +98 -165
  297. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  298. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  299. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  300. package/src/report/index.ts +128 -66
  301. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  302. package/src/report/{flag.ts → model/flag.ts} +39 -1
  303. package/src/report/{format.ts → model/format.ts} +89 -10
  304. package/src/report/{locale.ts → model/locale.ts} +34 -36
  305. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  306. package/src/report/{types.ts → model/types.ts} +159 -22
  307. package/src/report/react/index.tsx +54 -26
  308. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
  309. package/src/report/runtime/host.test.ts +44 -0
  310. package/src/report/runtime/host.ts +138 -0
  311. package/src/report/{load.ts → runtime/load.ts} +1 -1
  312. package/src/report/runtime/text.ts +192 -0
  313. package/src/report/runtime/web.ts +106 -0
  314. package/src/results/attempt-evidence.ts +5 -1
  315. package/src/results/host-equivalence.test.ts +53 -2
  316. package/src/results/select.ts +29 -5
  317. package/src/runner/attempt.test.ts +48 -1
  318. package/src/runner/attempt.ts +62 -42
  319. package/src/runner/cleanup-timeout.test.ts +16 -0
  320. package/src/runner/cleanup-timeout.ts +23 -0
  321. package/src/runner/discover.ts +1 -2
  322. package/src/runner/eval-selection.test.ts +187 -0
  323. package/src/runner/eval-selection.ts +109 -0
  324. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  325. package/src/runner/experiment-cleanup-registry.ts +39 -0
  326. package/src/runner/experiment-labels.test.ts +71 -0
  327. package/src/runner/feedback/agent.test.ts +33 -1
  328. package/src/runner/feedback/agent.ts +12 -16
  329. package/src/runner/feedback/ci.test.ts +35 -1
  330. package/src/runner/feedback/ci.ts +16 -18
  331. package/src/runner/feedback/coordinator.ts +28 -0
  332. package/src/runner/feedback/human.test.ts +108 -1
  333. package/src/runner/feedback/human.ts +71 -21
  334. package/src/runner/feedback/reducer.test.ts +64 -0
  335. package/src/runner/feedback/reducer.ts +26 -0
  336. package/src/runner/feedback/sink.ts +40 -1
  337. package/src/runner/fingerprint.ts +2 -1
  338. package/src/runner/report.test.ts +14 -0
  339. package/src/runner/report.ts +3 -1
  340. package/src/runner/run.test.ts +390 -12
  341. package/src/runner/run.ts +259 -26
  342. package/src/runner/sandbox-selection.test.ts +13 -3
  343. package/src/runner/sandbox-selection.ts +4 -2
  344. package/src/runner/types.ts +160 -24
  345. package/src/sandbox/docker.ts +7 -0
  346. package/src/sandbox/e2b-reconcile.test.ts +125 -0
  347. package/src/sandbox/e2b.ts +47 -12
  348. package/src/sandbox/types.ts +23 -21
  349. package/src/scoring/display.test.ts +28 -2
  350. package/src/scoring/display.ts +38 -5
  351. package/src/scoring/types.ts +3 -3
  352. package/src/shared/aggregate.test.ts +52 -0
  353. package/src/shared/aggregate.ts +29 -0
  354. package/src/shared/types.ts +6 -1
  355. package/src/show/command.test.ts +34 -0
  356. package/src/show/command.ts +16 -0
  357. package/src/show/compose.ts +1 -1
  358. package/src/show/index.ts +56 -20
  359. package/src/show/render.ts +15 -239
  360. package/src/show/show.test.ts +127 -34
  361. package/src/view/app/App.test.tsx +1 -1
  362. package/src/view/app/App.tsx +119 -47
  363. package/src/view/app/i18n.ts +7 -147
  364. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  365. package/src/view/app/lib/attempt-dialog.ts +69 -0
  366. package/src/view/app/main.tsx +0 -1
  367. package/src/view/app/types.ts +3 -86
  368. package/src/view/artifact-serving.test.ts +22 -38
  369. package/src/view/client-dist/app.css +1 -1
  370. package/src/view/client-dist/app.js +15 -22
  371. package/src/view/data.test.ts +48 -36
  372. package/src/view/data.ts +105 -83
  373. package/src/view/index.ts +1 -1
  374. package/src/view/server.ts +32 -2
  375. package/src/view/shared/types.ts +6 -31
  376. package/src/view/site-parity.test.ts +24 -1
  377. package/src/view/site.ts +144 -17
  378. package/src/view/styles.css +0 -805
  379. package/src/view/view-report.test.ts +143 -22
  380. package/dist/report/components.d.ts +0 -179
  381. package/dist/report/components.js +0 -544
  382. package/dist/report/compute.d.ts +0 -139
  383. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  384. package/dist/report/react/ExperimentComparison.js +0 -12
  385. package/dist/report/react/MetricScatter.d.ts +0 -9
  386. package/dist/report/react/format.d.ts +0 -3
  387. package/dist/report/react/format.js +0 -7
  388. package/dist/report/text/faces.d.ts +0 -45
  389. package/dist/report/text/faces.js +0 -671
  390. package/dist/report/web.d.ts +0 -32
  391. package/dist/report/web.js +0 -48
  392. package/docs-site/zh/how-to/authoring.mdx +0 -162
  393. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  394. package/docs-site/zh/how-to/experiments.mdx +0 -86
  395. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  396. package/src/report/components.tsx +0 -925
  397. package/src/report/react/ExperimentComparison.tsx +0 -73
  398. package/src/report/react/format.ts +0 -9
  399. package/src/report/react/styles.css +0 -945
  400. package/src/report/text/faces.ts +0 -767
  401. package/src/report/web.ts +0 -77
  402. package/src/show/report-host.test.ts +0 -205
  403. package/src/show/report-host.ts +0 -389
  404. package/src/view/app/components/AttemptModal.tsx +0 -496
  405. package/src/view/app/components/CodeView.test.tsx +0 -142
  406. package/src/view/app/components/CodeView.tsx +0 -310
  407. package/src/view/app/components/CopyControls.tsx +0 -106
  408. package/src/view/app/components/Trace.tsx +0 -100
  409. package/src/view/app/components/Transcript.tsx +0 -157
  410. package/src/view/app/components/ui/badge.tsx +0 -21
  411. package/src/view/app/lib/artifact-url.ts +0 -17
  412. package/src/view/app/lib/attempt-route.test.ts +0 -80
  413. package/src/view/app/lib/attempt-route.ts +0 -52
  414. package/src/view/app/lib/format.ts +0 -70
  415. package/src/view/app/lib/guards.test.ts +0 -108
  416. package/src/view/app/lib/guards.ts +0 -71
  417. package/src/view/app/lib/rows.ts +0 -22
  418. package/src/view/app/lib/transcript-data.tsx +0 -151
  419. package/src/view/app/lib/verdict.ts +0 -23
  420. package/src/view/app/shared.ts +0 -8
  421. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  422. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  423. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  424. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  425. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  426. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  427. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  428. /package/dist/report/{types.js → model/types.js} +0 -0
  429. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  430. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  431. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  432. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -0,0 +1,336 @@
1
+ // Attempt 详情组件族的计算函数(docs/feature/reports/library/attempt-detail.md)。每个
2
+ // `attempt*Data(evidence)` 都是纯同步派生:evidence 已经由 loadAttemptEvidence 一次性
3
+ // 装配好全部证据,这里只做适合展示与序列化的取舍,不读文件、不 fetch、不重复调用
4
+ // attempt.events() / trace() / diff()。
5
+ //
6
+ // 与 compute.ts(Scope → *Data)不同,这一族的输入恒为单个 AttemptEvidence,函数签名
7
+ // 因此不是 async——没有 IO 就没有理由返回 Promise。
8
+
9
+ import type { AttemptEvidence } from "../../../results/attempt-evidence.ts";
10
+ import type {
11
+ AttemptAssertionsData,
12
+ AttemptConversationData,
13
+ AttemptConversationReply,
14
+ AttemptConversationRound,
15
+ AttemptDiagnosticsData,
16
+ AttemptDiffData,
17
+ AttemptDiffFileEntry,
18
+ AttemptErrorData,
19
+ AttemptFixPromptData,
20
+ AttemptSourceData,
21
+ AttemptSourceTurn,
22
+ AttemptSummaryData,
23
+ AttemptTimelineData,
24
+ AttemptTraceData,
25
+ AttemptUsageData,
26
+ } from "../../model/types.ts";
27
+ import type { AssertionResult, DiagnosticRecord, JsonValue, StreamEvent } from "../../../types.ts";
28
+ import { attemptCostUSD } from "../../model/metrics.ts";
29
+ import { failureSummaryOf } from "../entity-lists/compute.ts";
30
+
31
+ // ───────────────────────── AttemptSummary(恒非空) ─────────────────────────
32
+
33
+ export function attemptSummaryData(evidence: AttemptEvidence): AttemptSummaryData {
34
+ const { result } = evidence;
35
+ return {
36
+ locator: evidence.locator,
37
+ identity: evidence.identity,
38
+ verdict: result.verdict,
39
+ startedAt: result.startedAt,
40
+ durationMs: result.durationMs,
41
+ costUSD: attemptCostUSD(result),
42
+ capabilities: evidence.capabilities,
43
+ };
44
+ }
45
+
46
+ // ───────────────────────── AttemptError ─────────────────────────
47
+
48
+ export function attemptErrorData(evidence: AttemptEvidence): AttemptErrorData | null {
49
+ const err = evidence.result.error;
50
+ return err ?? null;
51
+ }
52
+
53
+ // ───────────────────────── AttemptAssertions ─────────────────────────
54
+
55
+ export function attemptAssertionsData(evidence: AttemptEvidence): AttemptAssertionsData | null {
56
+ const assertions = evidence.result.assertions;
57
+ if (!assertions || assertions.length === 0) return null;
58
+ const attention = assertions.filter((a) => a.outcome !== "passed");
59
+ const passed = assertions.filter((a) => a.outcome === "passed");
60
+ const groups = new Map<string, AssertionResult[]>();
61
+ for (const a of passed) {
62
+ const key = a.groupPath?.join(" > ") ?? "";
63
+ const list = groups.get(key);
64
+ if (list) list.push(a);
65
+ else groups.set(key, [a]);
66
+ }
67
+ return { attention, passedGroups: [...groups.entries()].map(([group, items]) => ({ group, items })) };
68
+ }
69
+
70
+ // ───────────────────────── AttemptSource ─────────────────────────
71
+
72
+ export function attemptSourceData(evidence: AttemptEvidence): AttemptSourceData | null {
73
+ if (!evidence.capabilities.source || evidence.evalSource === null) return null;
74
+ const { sourcePath, lines, unmapped, summary } = evidence.evalSource;
75
+ const projectedLines = lines.map((line) => ({
76
+ ...line,
77
+ turns: line.sends.map<AttemptSourceTurn>((send) => ({
78
+ label: send.label,
79
+ status: send.status,
80
+ ...(send.durationMs === undefined ? {} : { durationMs: send.durationMs }),
81
+ sentText: "",
82
+ replies: [],
83
+ })),
84
+ }));
85
+ const usedTurns = new Map<number, number>();
86
+ const unlocatedTurns: AttemptSourceTurn[] = [];
87
+ const conversation = attemptConversationData(evidence);
88
+
89
+ for (const [roundIndex, round] of (conversation?.rounds ?? []).entries()) {
90
+ const status = round.replies.some(
91
+ (reply) =>
92
+ reply.kind === "error" ||
93
+ ((reply.kind === "tool" || reply.kind === "subagent") && reply.status === "failed"),
94
+ )
95
+ ? "failed"
96
+ : round.replies.some((reply) => reply.kind === "input")
97
+ ? "waiting"
98
+ : "completed";
99
+ const fallback: AttemptSourceTurn = {
100
+ label: `t${roundIndex + 1}`,
101
+ status,
102
+ sentText: round.sentText,
103
+ replies: round.replies,
104
+ };
105
+ const loc = round.loc;
106
+ if (!loc || loc.file !== sourcePath || loc.line < 1 || loc.line > projectedLines.length) {
107
+ unlocatedTurns.push(fallback);
108
+ continue;
109
+ }
110
+
111
+ const line = projectedLines[loc.line - 1]!;
112
+ const turnIndex = usedTurns.get(loc.line) ?? 0;
113
+ usedTurns.set(loc.line, turnIndex + 1);
114
+ const annotated = line.turns[turnIndex];
115
+ if (annotated) {
116
+ annotated.sentText = round.sentText;
117
+ annotated.replies = round.replies;
118
+ } else {
119
+ line.turns.push(fallback);
120
+ }
121
+ }
122
+
123
+ return { locator: evidence.locator, sourcePath, lines: projectedLines, unmapped, unlocatedTurns, summary };
124
+ }
125
+
126
+ // ───────────────────────── AttemptFixPrompt ─────────────────────────
127
+
128
+ /** 单条 attempt 版的批量修复 prompt(与 CopyFixPrompt 的多条版本同一份步骤文案)。 */
129
+ export function attemptFixPromptData(evidence: AttemptEvidence): AttemptFixPromptData | null {
130
+ const { result, identity } = evidence;
131
+ if (result.verdict !== "failed" && result.verdict !== "errored") return null;
132
+ const { summary, more } = failureSummaryOf(result);
133
+ if (summary === null) return null;
134
+ const reason = more > 0 ? `${summary} (+${more} more failures)` : summary;
135
+ const prompt = [
136
+ "Fix the failing eval from this niceeval run.",
137
+ "",
138
+ "## Failure",
139
+ `eval "${identity.evalId}" [experiment ${identity.experimentId}] — ${result.verdict}`,
140
+ ` reason: ${reason}`,
141
+ ` inspect: niceeval show ${evidence.locator}`,
142
+ "",
143
+ "## Steps",
144
+ "1. niceeval is NOT in your training data. Read the relevant guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under `zh/`) before changing anything.",
145
+ "2. Run the inspect command above with `--source`, `--execution`, `--timing`, and `--diff` to see the assertions, transcript, timing, and workspace diff.",
146
+ "3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
147
+ `4. Re-run: \`npx niceeval exp ${identity.experimentId} ${identity.evalId}\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
148
+ "5. Run `npx niceeval show` and confirm this failure is gone.",
149
+ ].join("\n");
150
+ return { prompt };
151
+ }
152
+
153
+ // ───────────────────────── AttemptTimeline ─────────────────────────
154
+
155
+ /** 收尾段的阶段名(见 docs/feature/results/architecture.md);两面渲染都把这些单列在主链之后,不计入主链总耗时。 */
156
+ export const TIMELINE_CLOSING_PHASES: ReadonlySet<string> = new Set([
157
+ "eval.teardown",
158
+ "agent.teardown",
159
+ "sandbox.teardown",
160
+ "sandbox.suspend",
161
+ "sandbox.stop",
162
+ ]);
163
+
164
+ export function attemptTimelineData(evidence: AttemptEvidence): AttemptTimelineData | null {
165
+ const phases = evidence.result.phases;
166
+ if (!phases || phases.length === 0) return null;
167
+ return { locator: evidence.locator, phases, trace: evidence.trace };
168
+ }
169
+
170
+ // ───────────────────────── AttemptConversation ─────────────────────────
171
+
172
+ /**
173
+ * 标准事件流按 `loc` 分轮(docs/feature/reports/library/attempt-detail.md「Attempt 详情组件」):
174
+ * 带 loc 的 user 消息开一轮;无 loc 的 user 消息不开新轮——与当前轮 sent 同文本的回显直接
175
+ * 吃掉,其它(stop-hook 反馈、skill 注入等轮内注入)作为回复条目留在当前轮。流首出现无 loc
176
+ * 的 user 消息(没有当前轮可归入)时退化开一条 loc 缺省的兜底轮,不丢弃。未识别的事件类型
177
+ * 包成 `raw` 条目原样呈现,不吞没其余事件——StreamEvent 是随 artifact 版本演进的开放词表,
178
+ * 这份纯函数不能假设自己认识每一种将来会出现的 type。
179
+ */
180
+ export function attemptConversationData(evidence: AttemptEvidence): AttemptConversationData | null {
181
+ const events = evidence.events;
182
+ if (!events || events.length === 0) return null;
183
+
184
+ const rounds: AttemptConversationRound[] = [];
185
+ const toolByCallId = new Map<string, Extract<AttemptConversationReply, { kind: "tool" }>>();
186
+ const subagentByCallId = new Map<string, Extract<AttemptConversationReply, { kind: "subagent" }>>();
187
+ let current: AttemptConversationRound | null = null;
188
+
189
+ for (const ev of events) {
190
+ if (ev.type === "message" && ev.role === "user") {
191
+ if (!ev.loc && current) {
192
+ if (current.replies.length === 0 && (ev.text || "") === current.sentText) continue;
193
+ current.replies.push({ kind: "user", text: ev.text || "" });
194
+ continue;
195
+ }
196
+ current = { loc: ev.loc, sentText: ev.text || "", replies: [] };
197
+ rounds.push(current);
198
+ continue;
199
+ }
200
+ if (!current) {
201
+ // 流首没有开轮的用户消息就先来了其它事件(旧 artifact 的边界情况):开一条无 loc 兜底轮。
202
+ current = { sentText: "", replies: [] };
203
+ rounds.push(current);
204
+ }
205
+ current.replies.push(...conversationReplyOf(ev, toolByCallId, subagentByCallId));
206
+ }
207
+
208
+ return { locator: evidence.locator, rounds };
209
+ }
210
+
211
+ /** 单条事件 → 0 或 1 条回复条目;action.result/subagent.completed 只更新已有条目,不新增。 */
212
+ function conversationReplyOf(
213
+ ev: StreamEvent,
214
+ toolByCallId: Map<string, Extract<AttemptConversationReply, { kind: "tool" }>>,
215
+ subagentByCallId: Map<string, Extract<AttemptConversationReply, { kind: "subagent" }>>,
216
+ ): AttemptConversationReply[] {
217
+ switch (ev.type) {
218
+ case "message":
219
+ // role === "user" 已在主循环处理(开轮 / 回显吃掉 / 轮内注入),这里只剩 assistant。
220
+ return [{ kind: "assistant", text: ev.text }];
221
+ case "thinking":
222
+ return [{ kind: "thinking", text: ev.text }];
223
+ case "error":
224
+ return [{ kind: "error", text: ev.message }];
225
+ case "skill.loaded":
226
+ return [{ kind: "skill", skill: ev.skill }];
227
+ case "input.requested":
228
+ return [{ kind: "input", request: ev.request }];
229
+ case "compaction":
230
+ return [{ kind: "compaction", reason: ev.reason }];
231
+ case "action.called": {
232
+ const reply: Extract<AttemptConversationReply, { kind: "tool" }> = {
233
+ kind: "tool",
234
+ callId: ev.callId,
235
+ name: ev.name,
236
+ tool: ev.tool,
237
+ input: ev.input,
238
+ };
239
+ toolByCallId.set(ev.callId, reply);
240
+ return [reply];
241
+ }
242
+ case "action.result": {
243
+ const tool = toolByCallId.get(ev.callId);
244
+ if (tool) {
245
+ tool.output = ev.output;
246
+ tool.status = ev.status;
247
+ }
248
+ return [];
249
+ }
250
+ case "subagent.called": {
251
+ const reply: Extract<AttemptConversationReply, { kind: "subagent" }> = {
252
+ kind: "subagent",
253
+ callId: ev.callId,
254
+ name: ev.name,
255
+ remoteUrl: ev.remoteUrl,
256
+ };
257
+ subagentByCallId.set(ev.callId, reply);
258
+ return [reply];
259
+ }
260
+ case "subagent.completed": {
261
+ const subagent = subagentByCallId.get(ev.callId);
262
+ if (subagent) {
263
+ subagent.output = ev.output;
264
+ subagent.status = ev.status;
265
+ }
266
+ return [];
267
+ }
268
+ default: {
269
+ // 穷尽性检查在这里刻意不做:StreamEvent 是随 artifact 版本演进的开放词表,未识别的
270
+ // 事件(将来的新 type,或第三方 harness 的自定义变体)包成 raw 原样呈现,不静默丢弃、
271
+ // 也不因为一个不认识的条目让整个装配失败。
272
+ return [{ kind: "raw", raw: ev as unknown as JsonValue }];
273
+ }
274
+ }
275
+ }
276
+
277
+ // ───────────────────────── AttemptDiagnostics ─────────────────────────
278
+
279
+ export function attemptDiagnosticsData(evidence: AttemptEvidence): AttemptDiagnosticsData | null {
280
+ const diagnostics = evidence.result.diagnostics;
281
+ if (!diagnostics || diagnostics.length === 0) return null;
282
+ const groups = new Map<string, DiagnosticRecord[]>();
283
+ for (const d of diagnostics) {
284
+ const list = groups.get(d.phase);
285
+ if (list) list.push(d);
286
+ else groups.set(d.phase, [d]);
287
+ }
288
+ return { groups: [...groups.entries()].map(([phase, items]) => ({ phase, items })) };
289
+ }
290
+
291
+ // ───────────────────────── AttemptUsage ─────────────────────────
292
+
293
+ export function attemptUsageData(evidence: AttemptEvidence): AttemptUsageData | null {
294
+ const usage = evidence.result.usage;
295
+ if (!usage) return null;
296
+ return { usage, costUSD: attemptCostUSD(evidence.result) };
297
+ }
298
+
299
+ // ───────────────────────── AttemptTrace ─────────────────────────
300
+
301
+ export function attemptTraceData(evidence: AttemptEvidence): AttemptTraceData | null {
302
+ const spans = evidence.trace;
303
+ if (!spans || spans.length === 0) return null;
304
+ return { locator: evidence.locator, spans };
305
+ }
306
+
307
+ // ───────────────────────── AttemptDiff ─────────────────────────
308
+
309
+ /** 有界行 diff(公共前后缀修剪):对单区域编辑精确,复杂编辑给出上界近似;与 `niceeval show --diff` 同一算法。 */
310
+ function lineDelta(before: string | undefined, after: string | undefined): { added: number; deleted: number } {
311
+ const a = before === undefined ? [] : before.split("\n");
312
+ const b = after === undefined ? [] : after.split("\n");
313
+ let prefix = 0;
314
+ while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
315
+ let suffix = 0;
316
+ while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix]) suffix++;
317
+ return { added: b.length - prefix - suffix, deleted: a.length - prefix - suffix };
318
+ }
319
+
320
+ export function attemptDiffData(evidence: AttemptEvidence): AttemptDiffData | null {
321
+ if (!evidence.capabilities.diff || evidence.diff === null) return null;
322
+ const diff = evidence.diff;
323
+ const files: AttemptDiffFileEntry[] = [];
324
+ for (const [path, summary] of Object.entries(diff.files).sort(([a], [b]) => a.localeCompare(b))) {
325
+ if (summary.net === "none") continue;
326
+ const windows = diff.windows.filter((w) => w.changes[path] !== undefined).map((w) => w.window);
327
+ if (summary.binary) {
328
+ files.push({ path, net: summary.net, lines: { added: 0, deleted: 0 }, binary: true, windows });
329
+ continue;
330
+ }
331
+ const before = summary.net === "added" ? undefined : diff.windows.find((w) => w.changes[path]?.before !== undefined)?.changes[path]?.before;
332
+ const after = summary.net === "deleted" ? undefined : diff.get(path);
333
+ files.push({ path, net: summary.net, lines: lineDelta(before, after), windows });
334
+ }
335
+ return files.length > 0 ? { locator: evidence.locator, files } : null;
336
+ }
@@ -0,0 +1,260 @@
1
+ // Attempt 详情组件族的 text 面(docs/feature/reports/library/attempt-detail.md「在 show 与
2
+ // view 怎样渲染」):与 web 面共享同一次 resolve 产出的 data 事实(verdict、计数、能力位、
3
+ // 引用),允许把大块内容折成摘要 + 专用证据命令,但不得改变判定、计数或引用。
4
+ // 零 react、零 IO、纯同步——text 宿主不需要 react-dom 的那一半。
5
+
6
+ import type {
7
+ AttemptAssertionsData,
8
+ AttemptConversationData,
9
+ AttemptConversationReply,
10
+ AttemptDiagnosticsData,
11
+ AttemptDiffData,
12
+ AttemptErrorData,
13
+ AttemptFixPromptData,
14
+ AttemptSourceData,
15
+ AttemptSummaryData,
16
+ AttemptTimelineData,
17
+ AttemptTraceData,
18
+ AttemptUsageData,
19
+ } from "../../model/types.ts";
20
+ import type { AssertionResult, TimingNode } from "../../../types.ts";
21
+ import type { AttemptLocator } from "../../../results/locator.ts";
22
+ import type { TextContext } from "../../definition/tree.ts";
23
+ import { localeText } from "../../model/locale.ts";
24
+ import { formatDurationMs, formatMetricValue, formatReportDateTime, formatUSD, verdictMark } from "../../model/format.ts";
25
+ import { TIMELINE_CLOSING_PHASES } from "./compute.ts";
26
+ import { summaryText } from "../../../scoring/display.ts";
27
+
28
+ /**
29
+ * 证据切面下钻命令:`ctx.attemptCommand` 由宿主按当前 report 是否有 attempt-input page
30
+ * 注入(见 report.ts DEFAULT_ATTEMPT_COMMAND);没有时不生成假命令,调用方直接省略这一段
31
+ * (与 traceWaterfallText 同一套退化规则,architecture.md「Attempt 详情是一张参数化 page」)。
32
+ */
33
+ function evidenceCommand(ctx: TextContext, locator: AttemptLocator, flag: string): string | undefined {
34
+ return ctx.attemptCommand ? `${ctx.attemptCommand(locator)} ${flag}` : undefined;
35
+ }
36
+
37
+ // ───────────────────────── AttemptSummary ─────────────────────────
38
+
39
+ export function attemptSummaryText(data: AttemptSummaryData, ctx: TextContext): string {
40
+ const locale = ctx.locale;
41
+ const parts = [
42
+ data.locator,
43
+ data.identity.evalId,
44
+ data.identity.experimentId,
45
+ localeText(locale, "attemptSummary.attempt", { n: data.identity.attempt + 1 }),
46
+ `${verdictMark(data.verdict)} ${localeText(locale, `verdict.${data.verdict}`)}`,
47
+ ];
48
+ if (data.startedAt !== undefined) parts.push(formatReportDateTime(data.startedAt, locale));
49
+ parts.push(formatDurationMs(data.durationMs));
50
+ if (data.costUSD !== null) parts.push(formatUSD(data.costUSD));
51
+ return parts.join(" · ");
52
+ }
53
+
54
+ // ───────────────────────── AttemptError ─────────────────────────
55
+
56
+ export function attemptErrorText(data: AttemptErrorData | null, _ctx: TextContext): string {
57
+ if (data === null) return "";
58
+ // message/cause 折单行加上限(同 assertionLine 的 summaryText 规则);stack 是唯一没有替代
59
+ // 查看入口的自由文本,原样保留多行,不折。
60
+ const lines = [`error: ${data.code}`, ` phase: ${data.phase}`, ` message: ${summaryText(data.message)}`];
61
+ if (data.cause) {
62
+ const causeMessage = summaryText(data.cause.message);
63
+ lines.push(` cause: ${data.cause.name ? `${data.cause.name} · ${causeMessage}` : causeMessage}`);
64
+ }
65
+ const stack = data.stack?.replace(/\n+$/, "");
66
+ return stack ? `${lines.join("\n")}\n\n${stack}` : lines.join("\n");
67
+ }
68
+
69
+ // ───────────────────────── AttemptAssertions ─────────────────────────
70
+
71
+ /** `loc` 是断言在 eval 源码里的调用点,独立于整份源码是否被捕获(`AttemptSource` 的能力位);
72
+ * 失败断言只要带 loc 就给可复制定位的源码锚,不必等 AttemptSource 可用才显示。 */
73
+ function locAnchor(loc: { file: string; line: number; column?: number }): string {
74
+ return `${loc.file}:${loc.line}${loc.column ? `:${loc.column}` : ""}`;
75
+ }
76
+
77
+ function assertionLine(a: AssertionResult): string {
78
+ const group = a.groupPath && a.groupPath.length > 0 ? `${a.groupPath.join(" > ")} · ` : "";
79
+ if (a.outcome === "unavailable") return `◌ unavailable · ${group}${a.name} — ${a.reason}`;
80
+ const mark = a.outcome === "passed" ? "✓" : "✗";
81
+ const detail = a.detail && a.detail !== a.name ? `: ${a.detail}` : "";
82
+ // expected/received 折单行 + 加字符上限(与 scoring 摘要面同一条 summaryText 规则):
83
+ // received 常常就是被检查的整份文本(如 includes() 对全文匹配),不收口会把整份源码/工具
84
+ // 输出灌进这一行;完整值仍在 web 面(可展开的完整结构化细节)与 --source 里原样可查。
85
+ const evidence = [
86
+ a.expected !== undefined ? `expected: ${summaryText(a.expected)}` : undefined,
87
+ a.received !== undefined ? `received: ${summaryText(a.received)}` : undefined,
88
+ a.outcome === "failed" && a.loc ? `source: ${locAnchor(a.loc)}` : undefined,
89
+ ].filter((part): part is string => part !== undefined);
90
+ const evidenceSuffix = evidence.length > 0 ? ` · ${evidence.join(" · ")}` : "";
91
+ return `${mark} ${a.severity} · ${group}${a.name}${detail}${evidenceSuffix}`;
92
+ }
93
+
94
+ export function attemptAssertionsText(data: AttemptAssertionsData | null, _ctx: TextContext): string {
95
+ if (data === null) return "";
96
+ const lines = data.attention.map(assertionLine);
97
+ for (const { group, items } of data.passedGroups) {
98
+ lines.push(`✓ passed · ${group || "(ungrouped)"} · ${items.length}`);
99
+ }
100
+ return lines.join("\n");
101
+ }
102
+
103
+ // ───────────────────────── AttemptSource ─────────────────────────
104
+
105
+ export function attemptSourceText(data: AttemptSourceData | null, ctx: TextContext): string {
106
+ if (data === null) return "";
107
+ const command = evidenceCommand(ctx, data.locator, "--source");
108
+ const headerParts = [`${data.sourcePath} · ${data.summary.annotatedLines}/${data.summary.totalLines} lines annotated`];
109
+ if (command) headerParts.push(command);
110
+ const hasConversation = data.unlocatedTurns.length > 0 || data.lines.some((line) => line.turns.length > 0);
111
+ const executionCommand = hasConversation ? evidenceCommand(ctx, data.locator, "--execution") : null;
112
+ if (executionCommand) headerParts.push(executionCommand);
113
+ // 源码锚由 assertionLine 自己按 a.loc 拼(与 AttemptAssertions 共用同一份逻辑);这里只负责
114
+ // 挑出非 passed 的条目,不重复算锚点。
115
+ const failed = data.lines.flatMap((line) => line.assertions.filter((a) => a.outcome !== "passed"));
116
+ const lines = failed.map((a) => ` ${assertionLine(a)}`);
117
+ return [headerParts.join(" · "), ...lines].join("\n");
118
+ }
119
+
120
+ // ───────────────────────── AttemptFixPrompt ─────────────────────────
121
+
122
+ /** text 面零输出:终端已有 attemptSummaryText 里的 locator,直接跑 `niceeval show @<locator>` 即可;这里不重复整段 prompt。 */
123
+ export function attemptFixPromptText(_data: AttemptFixPromptData | null, _ctx: TextContext): string {
124
+ return "";
125
+ }
126
+
127
+ // ───────────────────────── AttemptTimeline ─────────────────────────
128
+
129
+ function fmtMs(ms: number): string {
130
+ if (ms >= 60_000) return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
131
+ if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
132
+ return `${ms}ms`;
133
+ }
134
+
135
+ function countTimingNodes(nodes: readonly TimingNode[]): number {
136
+ return nodes.reduce((sum, n) => sum + 1 + countTimingNodes(n.children ?? []), 0);
137
+ }
138
+
139
+ export function attemptTimelineText(data: AttemptTimelineData | null, ctx: TextContext): string {
140
+ if (data === null) return "";
141
+ const main = data.phases.filter((p) => !TIMELINE_CLOSING_PHASES.has(p.name));
142
+ const closing = data.phases.filter((p) => TIMELINE_CLOSING_PHASES.has(p.name));
143
+ const total = main.reduce((sum, p) => sum + p.durationMs, 0);
144
+ const anyFailed = data.phases.some((p) => p.failed);
145
+ const command = evidenceCommand(ctx, data.locator, "--timing");
146
+ const head = [`timing: ${fmtMs(total)}${anyFailed ? " ✗" : ""}`];
147
+ if (command) head.push(command);
148
+ const lines = [head.join(" · ")];
149
+ for (const p of main) {
150
+ const kids = p.children ?? [];
151
+ const suffix = kids.length > 0 ? ` (${countTimingNodes(kids)} children collapsed)` : "";
152
+ lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}${suffix}`);
153
+ }
154
+ if (closing.length > 0) {
155
+ lines.push(" teardown:");
156
+ for (const p of closing) lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}`);
157
+ }
158
+ return lines.join("\n");
159
+ }
160
+
161
+ // ───────────────────────── AttemptConversation ─────────────────────────
162
+
163
+ // 每条回复的自由文本(消息正文、发起请求的 prompt、未识别事件的原始 payload)都可能是任意大小的
164
+ // 块——system prompt、完整工具输出等——同一条 summaryText 规则折单行加上限;完整内容仍在
165
+ // --execution 里原样可查。
166
+ function replySummary(reply: AttemptConversationReply): string {
167
+ switch (reply.kind) {
168
+ case "assistant":
169
+ return `assistant: ${summaryText(reply.text)}`;
170
+ case "user":
171
+ return `user: ${summaryText(reply.text)}`;
172
+ case "thinking":
173
+ return `thinking: ${summaryText(reply.text)}`;
174
+ case "error":
175
+ return `error: ${summaryText(reply.text)}`;
176
+ case "tool":
177
+ return `tool ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
178
+ case "skill":
179
+ return `skill loaded: ${reply.skill}`;
180
+ case "subagent":
181
+ return `subagent ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
182
+ case "input":
183
+ return `input requested${reply.request.prompt ? `: ${summaryText(reply.request.prompt)}` : ""}`;
184
+ case "compaction":
185
+ return `compaction${reply.reason ? `: ${summaryText(reply.reason)}` : ""}`;
186
+ case "raw":
187
+ return `unrecognized event: ${summaryText(JSON.stringify(reply.raw))}`;
188
+ }
189
+ }
190
+
191
+ export function attemptConversationText(data: AttemptConversationData | null, ctx: TextContext): string {
192
+ if (data === null) return "";
193
+ const command = evidenceCommand(ctx, data.locator, "--execution");
194
+ const head = [`conversation: ${data.rounds.length} round${data.rounds.length === 1 ? "" : "s"}`];
195
+ if (command) head.push(command);
196
+ const lines = [head.join(" · ")];
197
+ data.rounds.forEach((round, i) => {
198
+ lines.push(` round ${i + 1}${round.sentText ? `: ${summaryText(round.sentText)}` : ""}`);
199
+ for (const reply of round.replies) lines.push(` ${replySummary(reply)}`);
200
+ });
201
+ return lines.join("\n");
202
+ }
203
+
204
+ // ───────────────────────── AttemptDiagnostics ─────────────────────────
205
+
206
+ export function attemptDiagnosticsText(data: AttemptDiagnosticsData | null, _ctx: TextContext): string {
207
+ if (data === null) return "";
208
+ const lines: string[] = [];
209
+ for (const { phase, items } of data.groups) {
210
+ lines.push(`${phase}:`);
211
+ for (const d of items) {
212
+ const count = d.count && d.count > 1 ? ` (${d.count} occurrences)` : "";
213
+ lines.push(` ${d.level} · ${d.code} — ${summaryText(d.message)}${count}`);
214
+ }
215
+ }
216
+ return lines.join("\n");
217
+ }
218
+
219
+ // ───────────────────────── AttemptUsage ─────────────────────────
220
+
221
+ export function attemptUsageText(data: AttemptUsageData | null, _ctx: TextContext): string {
222
+ if (data === null) return "";
223
+ const { usage } = data;
224
+ const total = usage.inputTokens + usage.outputTokens;
225
+ const parts = [`usage: ${formatMetricValue(total)} tokens (${formatMetricValue(usage.inputTokens)} in / ${formatMetricValue(usage.outputTokens)} out)`];
226
+ if (usage.cacheReadTokens !== undefined) parts.push(`cache read ${formatMetricValue(usage.cacheReadTokens)}`);
227
+ if (usage.cacheWriteTokens !== undefined) parts.push(`cache write ${formatMetricValue(usage.cacheWriteTokens)}`);
228
+ if (usage.requests !== undefined) parts.push(`${usage.requests} requests`);
229
+ if (data.costUSD !== null) parts.push(formatUSD(data.costUSD));
230
+ return parts.join(" · ");
231
+ }
232
+
233
+ // ───────────────────────── AttemptTrace ─────────────────────────
234
+
235
+ export function attemptTraceText(data: AttemptTraceData | null, ctx: TextContext): string {
236
+ if (data === null) return "";
237
+ const command = evidenceCommand(ctx, data.locator, "--timing");
238
+ const head = [`trace: ${data.spans.length} span${data.spans.length === 1 ? "" : "s"}`];
239
+ if (command) head.push(command);
240
+ return head.join(" · ");
241
+ }
242
+
243
+ // ───────────────────────── AttemptDiff ─────────────────────────
244
+
245
+ function netLetter(net: "added" | "modified" | "deleted"): string {
246
+ return net === "added" ? "A" : net === "deleted" ? "D" : "M";
247
+ }
248
+
249
+ export function attemptDiffText(data: AttemptDiffData | null, ctx: TextContext): string {
250
+ if (data === null) return "";
251
+ const command = evidenceCommand(ctx, data.locator, "--diff");
252
+ const head = [`changes: ${data.files.length} file${data.files.length === 1 ? "" : "s"} changed by agent`];
253
+ if (command) head.push(command);
254
+ const lines = [head.join(" · ")];
255
+ for (const f of data.files) {
256
+ const delta = f.binary ? "binary" : `+${f.lines.added}/-${f.lines.deleted}`;
257
+ lines.push(` ${netLetter(f.net)} ${f.path} (${delta})`);
258
+ }
259
+ return lines.join("\n");
260
+ }