niceeval 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +15 -7
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
  118. package/dist/report/{tree.js → definition/tree.js} +10 -12
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +12 -2
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +15 -1
  154. package/dist/shared/aggregate.js +32 -1
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +77 -64
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +30 -15
  227. package/src/i18n/zh-CN.ts +30 -15
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +136 -112
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +58 -22
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +54 -3
  314. package/src/results/select.ts +30 -6
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +30 -1
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +58 -22
  356. package/src/show/render.ts +16 -240
  357. package/src/show/show.test.ts +128 -35
  358. package/src/view/app/App.test.tsx +8 -7
  359. package/src/view/app/App.tsx +135 -51
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +122 -89
  370. package/src/view/index.ts +3 -13
  371. package/src/view/server.ts +36 -6
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +18 -806
  376. package/src/view/view-report.test.ts +182 -34
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
@@ -0,0 +1,287 @@
1
+ // Attempt 详情组件族的计算函数(docs/feature/reports/library/attempt-detail.md)。每个
2
+ // `attempt*Data(evidence)` 都是纯同步派生:evidence 已经由 loadAttemptEvidence 一次性
3
+ // 装配好全部证据,这里只做适合展示与序列化的取舍,不读文件、不 fetch、不重复调用
4
+ // attempt.events() / trace() / diff()。
5
+ //
6
+ // 与 compute.ts(Scope → *Data)不同,这一族的输入恒为单个 AttemptEvidence,函数签名
7
+ // 因此不是 async——没有 IO 就没有理由返回 Promise。
8
+
9
+ import type { AttemptEvidence } from "../../../results/attempt-evidence.ts";
10
+ import type {
11
+ AttemptAssertionsData,
12
+ AttemptConversationData,
13
+ AttemptConversationReply,
14
+ AttemptConversationRound,
15
+ AttemptDiagnosticsData,
16
+ AttemptDiffData,
17
+ AttemptDiffFileEntry,
18
+ AttemptErrorData,
19
+ AttemptFixPromptData,
20
+ AttemptSourceData,
21
+ AttemptSummaryData,
22
+ AttemptTimelineData,
23
+ AttemptTraceData,
24
+ AttemptUsageData,
25
+ } from "../../model/types.ts";
26
+ import type { AssertionResult, DiagnosticRecord, JsonValue, StreamEvent } from "../../../types.ts";
27
+ import { attemptCostUSD } from "../../model/metrics.ts";
28
+ import { failureSummaryOf } from "../entity-lists/compute.ts";
29
+
30
+ // ───────────────────────── AttemptSummary(恒非空) ─────────────────────────
31
+
32
+ export function attemptSummaryData(evidence: AttemptEvidence): AttemptSummaryData {
33
+ const { result } = evidence;
34
+ return {
35
+ locator: evidence.locator,
36
+ identity: evidence.identity,
37
+ verdict: result.verdict,
38
+ startedAt: result.startedAt,
39
+ durationMs: result.durationMs,
40
+ costUSD: attemptCostUSD(result),
41
+ capabilities: evidence.capabilities,
42
+ };
43
+ }
44
+
45
+ // ───────────────────────── AttemptError ─────────────────────────
46
+
47
+ export function attemptErrorData(evidence: AttemptEvidence): AttemptErrorData | null {
48
+ const err = evidence.result.error;
49
+ return err ?? null;
50
+ }
51
+
52
+ // ───────────────────────── AttemptAssertions ─────────────────────────
53
+
54
+ export function attemptAssertionsData(evidence: AttemptEvidence): AttemptAssertionsData | null {
55
+ const assertions = evidence.result.assertions;
56
+ if (!assertions || assertions.length === 0) return null;
57
+ const attention = assertions.filter((a) => a.outcome !== "passed");
58
+ const passed = assertions.filter((a) => a.outcome === "passed");
59
+ const groups = new Map<string, AssertionResult[]>();
60
+ for (const a of passed) {
61
+ const key = a.groupPath?.join(" > ") ?? "";
62
+ const list = groups.get(key);
63
+ if (list) list.push(a);
64
+ else groups.set(key, [a]);
65
+ }
66
+ return { attention, passedGroups: [...groups.entries()].map(([group, items]) => ({ group, items })) };
67
+ }
68
+
69
+ // ───────────────────────── AttemptSource ─────────────────────────
70
+
71
+ export function attemptSourceData(evidence: AttemptEvidence): AttemptSourceData | null {
72
+ if (!evidence.capabilities.source || evidence.evalSource === null) return null;
73
+ const { sourcePath, lines, unmapped, summary } = evidence.evalSource;
74
+ return { locator: evidence.locator, sourcePath, lines, unmapped, summary };
75
+ }
76
+
77
+ // ───────────────────────── AttemptFixPrompt ─────────────────────────
78
+
79
+ /** 单条 attempt 版的批量修复 prompt(与 CopyFixPrompt 的多条版本同一份步骤文案)。 */
80
+ export function attemptFixPromptData(evidence: AttemptEvidence): AttemptFixPromptData | null {
81
+ const { result, identity } = evidence;
82
+ if (result.verdict !== "failed" && result.verdict !== "errored") return null;
83
+ const { summary, more } = failureSummaryOf(result);
84
+ if (summary === null) return null;
85
+ const reason = more > 0 ? `${summary} (+${more} more failures)` : summary;
86
+ const prompt = [
87
+ "Fix the failing eval from this niceeval run.",
88
+ "",
89
+ "## Failure",
90
+ `eval "${identity.evalId}" [experiment ${identity.experimentId}] — ${result.verdict}`,
91
+ ` reason: ${reason}`,
92
+ ` inspect: niceeval show ${evidence.locator}`,
93
+ "",
94
+ "## Steps",
95
+ "1. niceeval is NOT in your training data. Read the relevant guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under `zh/`) before changing anything.",
96
+ "2. Run the inspect command above with `--source`, `--execution`, `--timing`, and `--diff` to see the assertions, transcript, timing, and workspace diff.",
97
+ "3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
98
+ `4. Re-run: \`npx niceeval exp ${identity.experimentId} ${identity.evalId}\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
99
+ "5. Run `npx niceeval show` and confirm this failure is gone.",
100
+ ].join("\n");
101
+ return { prompt };
102
+ }
103
+
104
+ // ───────────────────────── AttemptTimeline ─────────────────────────
105
+
106
+ /** 收尾段的阶段名(见 docs/feature/results/architecture.md);两面渲染都把这些单列在主链之后,不计入主链总耗时。 */
107
+ export const TIMELINE_CLOSING_PHASES: ReadonlySet<string> = new Set([
108
+ "eval.teardown",
109
+ "agent.teardown",
110
+ "sandbox.teardown",
111
+ "sandbox.suspend",
112
+ "sandbox.stop",
113
+ ]);
114
+
115
+ export function attemptTimelineData(evidence: AttemptEvidence): AttemptTimelineData | null {
116
+ const phases = evidence.result.phases;
117
+ if (!phases || phases.length === 0) return null;
118
+ return { locator: evidence.locator, phases, trace: evidence.trace };
119
+ }
120
+
121
+ // ───────────────────────── AttemptConversation ─────────────────────────
122
+
123
+ /**
124
+ * 标准事件流按 `loc` 分轮(docs/feature/reports/library/attempt-detail.md「Attempt 详情组件」):
125
+ * 带 loc 的 user 消息开一轮;无 loc 的 user 消息不开新轮——与当前轮 sent 同文本的回显直接
126
+ * 吃掉,其它(stop-hook 反馈、skill 注入等轮内注入)作为回复条目留在当前轮。流首出现无 loc
127
+ * 的 user 消息(没有当前轮可归入)时退化开一条 loc 缺省的兜底轮,不丢弃。未识别的事件类型
128
+ * 包成 `raw` 条目原样呈现,不吞没其余事件——StreamEvent 是随 artifact 版本演进的开放词表,
129
+ * 这份纯函数不能假设自己认识每一种将来会出现的 type。
130
+ */
131
+ export function attemptConversationData(evidence: AttemptEvidence): AttemptConversationData | null {
132
+ const events = evidence.events;
133
+ if (!events || events.length === 0) return null;
134
+
135
+ const rounds: AttemptConversationRound[] = [];
136
+ const toolByCallId = new Map<string, Extract<AttemptConversationReply, { kind: "tool" }>>();
137
+ const subagentByCallId = new Map<string, Extract<AttemptConversationReply, { kind: "subagent" }>>();
138
+ let current: AttemptConversationRound | null = null;
139
+
140
+ for (const ev of events) {
141
+ if (ev.type === "message" && ev.role === "user") {
142
+ if (!ev.loc && current) {
143
+ if (current.replies.length === 0 && (ev.text || "") === current.sentText) continue;
144
+ current.replies.push({ kind: "user", text: ev.text || "" });
145
+ continue;
146
+ }
147
+ current = { loc: ev.loc, sentText: ev.text || "", replies: [] };
148
+ rounds.push(current);
149
+ continue;
150
+ }
151
+ if (!current) {
152
+ // 流首没有开轮的用户消息就先来了其它事件(旧 artifact 的边界情况):开一条无 loc 兜底轮。
153
+ current = { sentText: "", replies: [] };
154
+ rounds.push(current);
155
+ }
156
+ current.replies.push(...conversationReplyOf(ev, toolByCallId, subagentByCallId));
157
+ }
158
+
159
+ return { locator: evidence.locator, rounds };
160
+ }
161
+
162
+ /** 单条事件 → 0 或 1 条回复条目;action.result/subagent.completed 只更新已有条目,不新增。 */
163
+ function conversationReplyOf(
164
+ ev: StreamEvent,
165
+ toolByCallId: Map<string, Extract<AttemptConversationReply, { kind: "tool" }>>,
166
+ subagentByCallId: Map<string, Extract<AttemptConversationReply, { kind: "subagent" }>>,
167
+ ): AttemptConversationReply[] {
168
+ switch (ev.type) {
169
+ case "message":
170
+ // role === "user" 已在主循环处理(开轮 / 回显吃掉 / 轮内注入),这里只剩 assistant。
171
+ return [{ kind: "assistant", text: ev.text }];
172
+ case "thinking":
173
+ return [{ kind: "thinking", text: ev.text }];
174
+ case "error":
175
+ return [{ kind: "error", text: ev.message }];
176
+ case "skill.loaded":
177
+ return [{ kind: "skill", skill: ev.skill }];
178
+ case "input.requested":
179
+ return [{ kind: "input", request: ev.request }];
180
+ case "compaction":
181
+ return [{ kind: "compaction", reason: ev.reason }];
182
+ case "action.called": {
183
+ const reply: Extract<AttemptConversationReply, { kind: "tool" }> = {
184
+ kind: "tool",
185
+ callId: ev.callId,
186
+ name: ev.name,
187
+ tool: ev.tool,
188
+ input: ev.input,
189
+ };
190
+ toolByCallId.set(ev.callId, reply);
191
+ return [reply];
192
+ }
193
+ case "action.result": {
194
+ const tool = toolByCallId.get(ev.callId);
195
+ if (tool) {
196
+ tool.output = ev.output;
197
+ tool.status = ev.status;
198
+ }
199
+ return [];
200
+ }
201
+ case "subagent.called": {
202
+ const reply: Extract<AttemptConversationReply, { kind: "subagent" }> = {
203
+ kind: "subagent",
204
+ callId: ev.callId,
205
+ name: ev.name,
206
+ remoteUrl: ev.remoteUrl,
207
+ };
208
+ subagentByCallId.set(ev.callId, reply);
209
+ return [reply];
210
+ }
211
+ case "subagent.completed": {
212
+ const subagent = subagentByCallId.get(ev.callId);
213
+ if (subagent) {
214
+ subagent.output = ev.output;
215
+ subagent.status = ev.status;
216
+ }
217
+ return [];
218
+ }
219
+ default: {
220
+ // 穷尽性检查在这里刻意不做:StreamEvent 是随 artifact 版本演进的开放词表,未识别的
221
+ // 事件(将来的新 type,或第三方 harness 的自定义变体)包成 raw 原样呈现,不静默丢弃、
222
+ // 也不因为一个不认识的条目让整个装配失败。
223
+ return [{ kind: "raw", raw: ev as unknown as JsonValue }];
224
+ }
225
+ }
226
+ }
227
+
228
+ // ───────────────────────── AttemptDiagnostics ─────────────────────────
229
+
230
+ export function attemptDiagnosticsData(evidence: AttemptEvidence): AttemptDiagnosticsData | null {
231
+ const diagnostics = evidence.result.diagnostics;
232
+ if (!diagnostics || diagnostics.length === 0) return null;
233
+ const groups = new Map<string, DiagnosticRecord[]>();
234
+ for (const d of diagnostics) {
235
+ const list = groups.get(d.phase);
236
+ if (list) list.push(d);
237
+ else groups.set(d.phase, [d]);
238
+ }
239
+ return { groups: [...groups.entries()].map(([phase, items]) => ({ phase, items })) };
240
+ }
241
+
242
+ // ───────────────────────── AttemptUsage ─────────────────────────
243
+
244
+ export function attemptUsageData(evidence: AttemptEvidence): AttemptUsageData | null {
245
+ const usage = evidence.result.usage;
246
+ if (!usage) return null;
247
+ return { usage, costUSD: attemptCostUSD(evidence.result) };
248
+ }
249
+
250
+ // ───────────────────────── AttemptTrace ─────────────────────────
251
+
252
+ export function attemptTraceData(evidence: AttemptEvidence): AttemptTraceData | null {
253
+ const spans = evidence.trace;
254
+ if (!spans || spans.length === 0) return null;
255
+ return { locator: evidence.locator, spans };
256
+ }
257
+
258
+ // ───────────────────────── AttemptDiff ─────────────────────────
259
+
260
+ /** 有界行 diff(公共前后缀修剪):对单区域编辑精确,复杂编辑给出上界近似;与 `niceeval show --diff` 同一算法。 */
261
+ function lineDelta(before: string | undefined, after: string | undefined): { added: number; deleted: number } {
262
+ const a = before === undefined ? [] : before.split("\n");
263
+ const b = after === undefined ? [] : after.split("\n");
264
+ let prefix = 0;
265
+ while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
266
+ let suffix = 0;
267
+ while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix]) suffix++;
268
+ return { added: b.length - prefix - suffix, deleted: a.length - prefix - suffix };
269
+ }
270
+
271
+ export function attemptDiffData(evidence: AttemptEvidence): AttemptDiffData | null {
272
+ if (!evidence.capabilities.diff || evidence.diff === null) return null;
273
+ const diff = evidence.diff;
274
+ const files: AttemptDiffFileEntry[] = [];
275
+ for (const [path, summary] of Object.entries(diff.files).sort(([a], [b]) => a.localeCompare(b))) {
276
+ if (summary.net === "none") continue;
277
+ const windows = diff.windows.filter((w) => w.changes[path] !== undefined).map((w) => w.window);
278
+ if (summary.binary) {
279
+ files.push({ path, net: summary.net, lines: { added: 0, deleted: 0 }, binary: true, windows });
280
+ continue;
281
+ }
282
+ const before = summary.net === "added" ? undefined : diff.windows.find((w) => w.changes[path]?.before !== undefined)?.changes[path]?.before;
283
+ const after = summary.net === "deleted" ? undefined : diff.get(path);
284
+ files.push({ path, net: summary.net, lines: lineDelta(before, after), windows });
285
+ }
286
+ return files.length > 0 ? { locator: evidence.locator, files } : null;
287
+ }
@@ -0,0 +1,257 @@
1
+ // Attempt 详情组件族的 text 面(docs/feature/reports/library/attempt-detail.md「在 show 与
2
+ // view 怎样渲染」):与 web 面共享同一次 resolve 产出的 data 事实(verdict、计数、能力位、
3
+ // 引用),允许把大块内容折成摘要 + 专用证据命令,但不得改变判定、计数或引用。
4
+ // 零 react、零 IO、纯同步——text 宿主不需要 react-dom 的那一半。
5
+
6
+ import type {
7
+ AttemptAssertionsData,
8
+ AttemptConversationData,
9
+ AttemptConversationReply,
10
+ AttemptDiagnosticsData,
11
+ AttemptDiffData,
12
+ AttemptErrorData,
13
+ AttemptFixPromptData,
14
+ AttemptSourceData,
15
+ AttemptSummaryData,
16
+ AttemptTimelineData,
17
+ AttemptTraceData,
18
+ AttemptUsageData,
19
+ } from "../../model/types.ts";
20
+ import type { AssertionResult, TimingNode } from "../../../types.ts";
21
+ import type { AttemptLocator } from "../../../results/locator.ts";
22
+ import type { TextContext } from "../../definition/tree.ts";
23
+ import { localeText } from "../../model/locale.ts";
24
+ import { formatDurationMs, formatMetricValue, formatReportDateTime, formatUSD, verdictMark } from "../../model/format.ts";
25
+ import { TIMELINE_CLOSING_PHASES } from "./compute.ts";
26
+ import { summaryText } from "../../../scoring/display.ts";
27
+
28
+ /**
29
+ * 证据切面下钻命令:`ctx.attemptCommand` 由宿主按当前 report 是否有 attempt-input page
30
+ * 注入(见 report.ts DEFAULT_ATTEMPT_COMMAND);没有时不生成假命令,调用方直接省略这一段
31
+ * (与 traceWaterfallText 同一套退化规则,architecture.md「Attempt 详情是一张参数化 page」)。
32
+ */
33
+ function evidenceCommand(ctx: TextContext, locator: AttemptLocator, flag: string): string | undefined {
34
+ return ctx.attemptCommand ? `${ctx.attemptCommand(locator)} ${flag}` : undefined;
35
+ }
36
+
37
+ // ───────────────────────── AttemptSummary ─────────────────────────
38
+
39
+ export function attemptSummaryText(data: AttemptSummaryData, ctx: TextContext): string {
40
+ const locale = ctx.locale;
41
+ const parts = [
42
+ data.locator,
43
+ data.identity.evalId,
44
+ data.identity.experimentId,
45
+ localeText(locale, "attemptSummary.attempt", { n: data.identity.attempt + 1 }),
46
+ `${verdictMark(data.verdict)} ${localeText(locale, `verdict.${data.verdict}`)}`,
47
+ ];
48
+ if (data.startedAt !== undefined) parts.push(formatReportDateTime(data.startedAt, locale));
49
+ parts.push(formatDurationMs(data.durationMs));
50
+ if (data.costUSD !== null) parts.push(formatUSD(data.costUSD));
51
+ return parts.join(" · ");
52
+ }
53
+
54
+ // ───────────────────────── AttemptError ─────────────────────────
55
+
56
+ export function attemptErrorText(data: AttemptErrorData | null, _ctx: TextContext): string {
57
+ if (data === null) return "";
58
+ // message/cause 折单行加上限(同 assertionLine 的 summaryText 规则);stack 是唯一没有替代
59
+ // 查看入口的自由文本,原样保留多行,不折。
60
+ const lines = [`error: ${data.code}`, ` phase: ${data.phase}`, ` message: ${summaryText(data.message)}`];
61
+ if (data.cause) {
62
+ const causeMessage = summaryText(data.cause.message);
63
+ lines.push(` cause: ${data.cause.name ? `${data.cause.name} · ${causeMessage}` : causeMessage}`);
64
+ }
65
+ const stack = data.stack?.replace(/\n+$/, "");
66
+ return stack ? `${lines.join("\n")}\n\n${stack}` : lines.join("\n");
67
+ }
68
+
69
+ // ───────────────────────── AttemptAssertions ─────────────────────────
70
+
71
+ /** `loc` 是断言在 eval 源码里的调用点,独立于整份源码是否被捕获(`AttemptSource` 的能力位);
72
+ * 失败断言只要带 loc 就给可复制定位的源码锚,不必等 AttemptSource 可用才显示。 */
73
+ function locAnchor(loc: { file: string; line: number; column?: number }): string {
74
+ return `${loc.file}:${loc.line}${loc.column ? `:${loc.column}` : ""}`;
75
+ }
76
+
77
+ function assertionLine(a: AssertionResult): string {
78
+ const group = a.groupPath && a.groupPath.length > 0 ? `${a.groupPath.join(" > ")} · ` : "";
79
+ if (a.outcome === "unavailable") return `◌ unavailable · ${group}${a.name} — ${a.reason}`;
80
+ const mark = a.outcome === "passed" ? "✓" : "✗";
81
+ const detail = a.detail && a.detail !== a.name ? `: ${a.detail}` : "";
82
+ // expected/received 折单行 + 加字符上限(与 scoring 摘要面同一条 summaryText 规则):
83
+ // received 常常就是被检查的整份文本(如 includes() 对全文匹配),不收口会把整份源码/工具
84
+ // 输出灌进这一行;完整值仍在 web 面(可展开的完整结构化细节)与 --source 里原样可查。
85
+ const evidence = [
86
+ a.expected !== undefined ? `expected: ${summaryText(a.expected)}` : undefined,
87
+ a.received !== undefined ? `received: ${summaryText(a.received)}` : undefined,
88
+ a.outcome === "failed" && a.loc ? `source: ${locAnchor(a.loc)}` : undefined,
89
+ ].filter((part): part is string => part !== undefined);
90
+ const evidenceSuffix = evidence.length > 0 ? ` · ${evidence.join(" · ")}` : "";
91
+ return `${mark} ${a.severity} · ${group}${a.name}${detail}${evidenceSuffix}`;
92
+ }
93
+
94
+ export function attemptAssertionsText(data: AttemptAssertionsData | null, _ctx: TextContext): string {
95
+ if (data === null) return "";
96
+ const lines = data.attention.map(assertionLine);
97
+ for (const { group, items } of data.passedGroups) {
98
+ lines.push(`✓ passed · ${group || "(ungrouped)"} · ${items.length}`);
99
+ }
100
+ return lines.join("\n");
101
+ }
102
+
103
+ // ───────────────────────── AttemptSource ─────────────────────────
104
+
105
+ export function attemptSourceText(data: AttemptSourceData | null, ctx: TextContext): string {
106
+ if (data === null) return "";
107
+ const command = evidenceCommand(ctx, data.locator, "--source");
108
+ const headerParts = [`${data.sourcePath} · ${data.summary.annotatedLines}/${data.summary.totalLines} lines annotated`];
109
+ if (command) headerParts.push(command);
110
+ // 源码锚由 assertionLine 自己按 a.loc 拼(与 AttemptAssertions 共用同一份逻辑);这里只负责
111
+ // 挑出非 passed 的条目,不重复算锚点。
112
+ const failed = data.lines.flatMap((line) => line.assertions.filter((a) => a.outcome !== "passed"));
113
+ const lines = failed.map((a) => ` ${assertionLine(a)}`);
114
+ return [headerParts.join(" · "), ...lines].join("\n");
115
+ }
116
+
117
+ // ───────────────────────── AttemptFixPrompt ─────────────────────────
118
+
119
+ /** text 面零输出:终端已有 attemptSummaryText 里的 locator,直接跑 `niceeval show @<locator>` 即可;这里不重复整段 prompt。 */
120
+ export function attemptFixPromptText(_data: AttemptFixPromptData | null, _ctx: TextContext): string {
121
+ return "";
122
+ }
123
+
124
+ // ───────────────────────── AttemptTimeline ─────────────────────────
125
+
126
+ function fmtMs(ms: number): string {
127
+ if (ms >= 60_000) return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
128
+ if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
129
+ return `${ms}ms`;
130
+ }
131
+
132
+ function countTimingNodes(nodes: readonly TimingNode[]): number {
133
+ return nodes.reduce((sum, n) => sum + 1 + countTimingNodes(n.children ?? []), 0);
134
+ }
135
+
136
+ export function attemptTimelineText(data: AttemptTimelineData | null, ctx: TextContext): string {
137
+ if (data === null) return "";
138
+ const main = data.phases.filter((p) => !TIMELINE_CLOSING_PHASES.has(p.name));
139
+ const closing = data.phases.filter((p) => TIMELINE_CLOSING_PHASES.has(p.name));
140
+ const total = main.reduce((sum, p) => sum + p.durationMs, 0);
141
+ const anyFailed = data.phases.some((p) => p.failed);
142
+ const command = evidenceCommand(ctx, data.locator, "--timing");
143
+ const head = [`timing: ${fmtMs(total)}${anyFailed ? " ✗" : ""}`];
144
+ if (command) head.push(command);
145
+ const lines = [head.join(" · ")];
146
+ for (const p of main) {
147
+ const kids = p.children ?? [];
148
+ const suffix = kids.length > 0 ? ` (${countTimingNodes(kids)} children collapsed)` : "";
149
+ lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}${suffix}`);
150
+ }
151
+ if (closing.length > 0) {
152
+ lines.push(" teardown:");
153
+ for (const p of closing) lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}`);
154
+ }
155
+ return lines.join("\n");
156
+ }
157
+
158
+ // ───────────────────────── AttemptConversation ─────────────────────────
159
+
160
+ // 每条回复的自由文本(消息正文、发起请求的 prompt、未识别事件的原始 payload)都可能是任意大小的
161
+ // 块——system prompt、完整工具输出等——同一条 summaryText 规则折单行加上限;完整内容仍在
162
+ // --execution 里原样可查。
163
+ function replySummary(reply: AttemptConversationReply): string {
164
+ switch (reply.kind) {
165
+ case "assistant":
166
+ return `assistant: ${summaryText(reply.text)}`;
167
+ case "user":
168
+ return `user: ${summaryText(reply.text)}`;
169
+ case "thinking":
170
+ return `thinking: ${summaryText(reply.text)}`;
171
+ case "error":
172
+ return `error: ${summaryText(reply.text)}`;
173
+ case "tool":
174
+ return `tool ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
175
+ case "skill":
176
+ return `skill loaded: ${reply.skill}`;
177
+ case "subagent":
178
+ return `subagent ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
179
+ case "input":
180
+ return `input requested${reply.request.prompt ? `: ${summaryText(reply.request.prompt)}` : ""}`;
181
+ case "compaction":
182
+ return `compaction${reply.reason ? `: ${summaryText(reply.reason)}` : ""}`;
183
+ case "raw":
184
+ return `unrecognized event: ${summaryText(JSON.stringify(reply.raw))}`;
185
+ }
186
+ }
187
+
188
+ export function attemptConversationText(data: AttemptConversationData | null, ctx: TextContext): string {
189
+ if (data === null) return "";
190
+ const command = evidenceCommand(ctx, data.locator, "--execution");
191
+ const head = [`conversation: ${data.rounds.length} round${data.rounds.length === 1 ? "" : "s"}`];
192
+ if (command) head.push(command);
193
+ const lines = [head.join(" · ")];
194
+ data.rounds.forEach((round, i) => {
195
+ lines.push(` round ${i + 1}${round.sentText ? `: ${summaryText(round.sentText)}` : ""}`);
196
+ for (const reply of round.replies) lines.push(` ${replySummary(reply)}`);
197
+ });
198
+ return lines.join("\n");
199
+ }
200
+
201
+ // ───────────────────────── AttemptDiagnostics ─────────────────────────
202
+
203
+ export function attemptDiagnosticsText(data: AttemptDiagnosticsData | null, _ctx: TextContext): string {
204
+ if (data === null) return "";
205
+ const lines: string[] = [];
206
+ for (const { phase, items } of data.groups) {
207
+ lines.push(`${phase}:`);
208
+ for (const d of items) {
209
+ const count = d.count && d.count > 1 ? ` (${d.count} occurrences)` : "";
210
+ lines.push(` ${d.level} · ${d.code} — ${summaryText(d.message)}${count}`);
211
+ }
212
+ }
213
+ return lines.join("\n");
214
+ }
215
+
216
+ // ───────────────────────── AttemptUsage ─────────────────────────
217
+
218
+ export function attemptUsageText(data: AttemptUsageData | null, _ctx: TextContext): string {
219
+ if (data === null) return "";
220
+ const { usage } = data;
221
+ const total = usage.inputTokens + usage.outputTokens;
222
+ const parts = [`usage: ${formatMetricValue(total)} tokens (${formatMetricValue(usage.inputTokens)} in / ${formatMetricValue(usage.outputTokens)} out)`];
223
+ if (usage.cacheReadTokens !== undefined) parts.push(`cache read ${formatMetricValue(usage.cacheReadTokens)}`);
224
+ if (usage.cacheWriteTokens !== undefined) parts.push(`cache write ${formatMetricValue(usage.cacheWriteTokens)}`);
225
+ if (usage.requests !== undefined) parts.push(`${usage.requests} requests`);
226
+ if (data.costUSD !== null) parts.push(formatUSD(data.costUSD));
227
+ return parts.join(" · ");
228
+ }
229
+
230
+ // ───────────────────────── AttemptTrace ─────────────────────────
231
+
232
+ export function attemptTraceText(data: AttemptTraceData | null, ctx: TextContext): string {
233
+ if (data === null) return "";
234
+ const command = evidenceCommand(ctx, data.locator, "--timing");
235
+ const head = [`trace: ${data.spans.length} span${data.spans.length === 1 ? "" : "s"}`];
236
+ if (command) head.push(command);
237
+ return head.join(" · ");
238
+ }
239
+
240
+ // ───────────────────────── AttemptDiff ─────────────────────────
241
+
242
+ function netLetter(net: "added" | "modified" | "deleted"): string {
243
+ return net === "added" ? "A" : net === "deleted" ? "D" : "M";
244
+ }
245
+
246
+ export function attemptDiffText(data: AttemptDiffData | null, ctx: TextContext): string {
247
+ if (data === null) return "";
248
+ const command = evidenceCommand(ctx, data.locator, "--diff");
249
+ const head = [`changes: ${data.files.length} file${data.files.length === 1 ? "" : "s"} changed by agent`];
250
+ if (command) head.push(command);
251
+ const lines = [head.join(" · ")];
252
+ for (const f of data.files) {
253
+ const delta = f.binary ? "binary" : `+${f.lines.added}/-${f.lines.deleted}`;
254
+ lines.push(` ${netLetter(f.net)} ${f.path} (${delta})`);
255
+ }
256
+ return lines.join("\n");
257
+ }