niceeval 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/INDEX.md +31 -33
  2. package/dist/agents/types.d.ts +20 -17
  3. package/dist/context/types.d.ts +21 -21
  4. package/dist/i18n/en.d.ts +10 -2
  5. package/dist/i18n/zh-CN.d.ts +14 -6
  6. package/dist/o11y/execution-tree.d.ts +103 -0
  7. package/dist/o11y/otlp/select.d.ts +22 -0
  8. package/dist/report/{react → assets}/colors.d.ts +8 -0
  9. package/dist/report/{react → assets}/colors.js +23 -0
  10. package/dist/report/built-in/index.d.ts +2 -2
  11. package/dist/report/built-in/index.js +2 -2
  12. package/dist/report/built-in/standard.d.ts +8 -1
  13. package/dist/report/built-in/standard.js +17 -7
  14. package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
  15. package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
  16. package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
  17. package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
  18. package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
  19. package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
  20. package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
  21. package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
  22. package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
  23. package/dist/report/components/attempt-detail/AttemptError.js +16 -0
  24. package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
  25. package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
  26. package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
  27. package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
  28. package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
  29. package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
  30. package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
  31. package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
  32. package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
  33. package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
  34. package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
  35. package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
  36. package/dist/report/components/attempt-detail/compute.d.ts +24 -0
  37. package/dist/report/components/attempt-detail/compute.js +254 -0
  38. package/dist/report/components/attempt-detail/faces.d.ts +14 -0
  39. package/dist/report/components/attempt-detail/faces.js +235 -0
  40. package/dist/report/components/attempt-detail/index.d.ts +38 -0
  41. package/dist/report/components/attempt-detail/index.js +527 -0
  42. package/dist/report/{react → components}/cell.d.ts +5 -3
  43. package/dist/report/{react → components}/cell.js +3 -3
  44. package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
  45. package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
  46. package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
  47. package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
  48. package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
  49. package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
  50. package/dist/report/components/entity-lists/compute.d.ts +23 -0
  51. package/dist/report/components/entity-lists/compute.js +171 -0
  52. package/dist/report/components/entity-lists/faces.d.ts +5 -0
  53. package/dist/report/components/entity-lists/faces.js +174 -0
  54. package/dist/report/components/entity-lists/index.d.ts +51 -0
  55. package/dist/report/components/entity-lists/index.js +165 -0
  56. package/dist/report/{react → components}/fixtures.d.ts +2 -2
  57. package/dist/report/{react → components}/fixtures.js +1 -1
  58. package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
  59. package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
  60. package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
  61. package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
  62. package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
  63. package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
  64. package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
  65. package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
  66. package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
  67. package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
  68. package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
  69. package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
  70. package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
  71. package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
  72. package/dist/report/components/metric-views/compute.d.ts +89 -0
  73. package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
  74. package/dist/report/components/metric-views/faces.d.ts +13 -0
  75. package/dist/report/components/metric-views/faces.js +381 -0
  76. package/dist/report/components/metric-views/index.d.ts +50 -0
  77. package/dist/report/components/metric-views/index.js +273 -0
  78. package/dist/report/{text → components/metric-views}/plot.js +1 -1
  79. package/dist/report/components/shared-compute.d.ts +22 -0
  80. package/dist/report/components/shared-compute.js +47 -0
  81. package/dist/report/components/shared-faces.d.ts +9 -0
  82. package/dist/report/components/shared-faces.js +26 -0
  83. package/dist/report/components/shared.d.ts +68 -0
  84. package/dist/report/components/shared.js +125 -0
  85. package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
  86. package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
  87. package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
  88. package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
  89. package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
  90. package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
  91. package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
  92. package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
  93. package/dist/report/components/site-components/compute.d.ts +28 -0
  94. package/dist/report/components/site-components/compute.js +132 -0
  95. package/dist/report/components/site-components/faces.d.ts +21 -0
  96. package/dist/report/components/site-components/faces.js +75 -0
  97. package/dist/report/components/site-components/index.d.ts +74 -0
  98. package/dist/report/components/site-components/index.js +220 -0
  99. package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
  100. package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
  101. package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
  102. package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
  103. package/dist/report/components/summaries/compute.d.ts +7 -0
  104. package/dist/report/components/summaries/compute.js +51 -0
  105. package/dist/report/components/summaries/faces.d.ts +7 -0
  106. package/dist/report/components/summaries/faces.js +38 -0
  107. package/dist/report/components/summaries/index.d.ts +27 -0
  108. package/dist/report/components/summaries/index.js +78 -0
  109. package/dist/report/definition/grid-layout.d.ts +50 -0
  110. package/dist/report/definition/grid-layout.js +89 -0
  111. package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
  112. package/dist/report/{primitives.js → definition/primitives.js} +101 -10
  113. package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
  114. package/dist/report/{report.js → definition/report.js} +58 -90
  115. package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
  116. package/dist/report/{text/table.js → definition/table-text.js} +2 -2
  117. package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
  118. package/dist/report/{tree.js → definition/tree.js} +9 -11
  119. package/dist/report/index.d.ts +33 -18
  120. package/dist/report/index.js +23 -14
  121. package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
  122. package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
  123. package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
  124. package/dist/report/{flag.js → model/flag.js} +35 -0
  125. package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
  126. package/dist/report/{format.js → model/format.js} +48 -4
  127. package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
  128. package/dist/report/{locale.js → model/locale.js} +31 -33
  129. package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
  130. package/dist/report/{metrics.js → model/metrics.js} +2 -2
  131. package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
  132. package/dist/report/react/index.d.ts +32 -21
  133. package/dist/report/react/index.js +33 -21
  134. package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
  135. package/dist/report/{load.js → runtime/load.js} +1 -1
  136. package/dist/report/runtime/text.d.ts +67 -0
  137. package/dist/report/runtime/text.js +114 -0
  138. package/dist/report/runtime/web.d.ts +35 -0
  139. package/dist/report/runtime/web.js +70 -0
  140. package/dist/results/annotated-source.d.ts +87 -0
  141. package/dist/results/attempt-evidence.d.ts +74 -0
  142. package/dist/results/attempt-source.d.ts +15 -0
  143. package/dist/results/select.d.ts +11 -1
  144. package/dist/results/select.js +27 -5
  145. package/dist/runner/eval-selection.d.ts +26 -0
  146. package/dist/runner/feedback/sink.d.ts +26 -1
  147. package/dist/runner/types.d.ts +149 -23
  148. package/dist/sandbox/docker.d.ts +6 -0
  149. package/dist/sandbox/types.d.ts +23 -21
  150. package/dist/scoring/display.d.ts +7 -2
  151. package/dist/scoring/display.js +42 -4
  152. package/dist/scoring/types.d.ts +3 -3
  153. package/dist/shared/aggregate.d.ts +14 -0
  154. package/dist/shared/aggregate.js +31 -0
  155. package/dist/shared/types.d.ts +6 -1
  156. package/docs-site/images/logo-dark.svg +7 -0
  157. package/docs-site/images/logo.svg +6 -5
  158. package/docs-site/zh/README.md +8 -7
  159. package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
  160. package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
  161. package/docs-site/zh/examples/index.mdx +3 -3
  162. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
  163. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  164. package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
  165. package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
  166. package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
  167. package/docs-site/zh/explanation/adapter.mdx +19 -19
  168. package/docs-site/zh/explanation/assert.mdx +9 -9
  169. package/docs-site/zh/explanation/drive.mdx +4 -4
  170. package/docs-site/zh/explanation/evals.mdx +8 -8
  171. package/docs-site/zh/explanation/experiment.mdx +8 -8
  172. package/docs-site/zh/explanation/hitl.mdx +13 -13
  173. package/docs-site/zh/explanation/judge.mdx +3 -3
  174. package/docs-site/zh/explanation/overview.mdx +7 -7
  175. package/docs-site/zh/explanation/runner.mdx +11 -11
  176. package/docs-site/zh/explanation/tier.mdx +6 -6
  177. package/docs-site/zh/index.mdx +14 -14
  178. package/docs-site/zh/introduction.mdx +13 -16
  179. package/docs-site/zh/reference/builtin-agents.mdx +61 -28
  180. package/docs-site/zh/reference/capabilities.mdx +7 -27
  181. package/docs-site/zh/reference/cli.mdx +35 -32
  182. package/docs-site/zh/reference/define-agent.mdx +32 -31
  183. package/docs-site/zh/reference/define-config.mdx +6 -6
  184. package/docs-site/zh/reference/define-eval.mdx +33 -20
  185. package/docs-site/zh/reference/events.mdx +4 -4
  186. package/docs-site/zh/reference/expect.mdx +3 -3
  187. package/docs-site/zh/reference/official-adapters.mdx +17 -17
  188. package/docs-site/zh/reference/report-components.mdx +193 -121
  189. package/docs-site/zh/reference/results-data.mdx +13 -13
  190. package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
  191. package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
  192. package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
  193. package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
  194. package/docs-site/zh/tutorials/authoring.mdx +211 -0
  195. package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
  196. package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
  197. package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
  198. package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
  199. package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
  200. package/docs-site/zh/tutorials/experiments.mdx +103 -0
  201. package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
  202. package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
  203. package/docs-site/zh/tutorials/quickstart.mdx +18 -18
  204. package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
  205. package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
  206. package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
  207. package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
  208. package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +49 -49
  209. package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
  210. package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
  211. package/package.json +8 -4
  212. package/src/agents/bub.ts +21 -8
  213. package/src/agents/claude-code.ts +23 -10
  214. package/src/agents/codex.test.ts +22 -7
  215. package/src/agents/codex.ts +22 -9
  216. package/src/agents/index.ts +2 -2
  217. package/src/agents/openai-compat.ts +1 -1
  218. package/src/agents/post-setup.ts +45 -22
  219. package/src/agents/streaming.ts +1 -1
  220. package/src/agents/types.ts +20 -17
  221. package/src/agents/ui-message-stream.test.ts +10 -0
  222. package/src/agents/ui-message-stream.ts +12 -1
  223. package/src/cli.ts +71 -57
  224. package/src/context/types.ts +21 -21
  225. package/src/define.ts +13 -0
  226. package/src/i18n/en.ts +27 -12
  227. package/src/i18n/zh-CN.ts +27 -12
  228. package/src/index.ts +2 -0
  229. package/src/report/{react → assets}/colors.ts +22 -0
  230. package/src/report/{react → assets}/enhance.js +1 -33
  231. package/src/report/{react → assets}/styles.css +124 -106
  232. package/src/report/built-in/index.tsx +2 -2
  233. package/src/report/built-in/standard.tsx +18 -6
  234. package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
  235. package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
  236. package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
  237. package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
  238. package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
  239. package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
  240. package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
  241. package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
  242. package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
  243. package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
  244. package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
  245. package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
  246. package/src/report/components/attempt-detail/compute.ts +287 -0
  247. package/src/report/components/attempt-detail/faces.ts +257 -0
  248. package/src/report/components/attempt-detail/index.tsx +583 -0
  249. package/src/report/components/attempt-detail/validate.test.ts +235 -0
  250. package/src/report/{react → components}/cell.tsx +6 -3
  251. package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
  252. package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
  253. package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
  254. package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
  255. package/src/report/components/entity-lists/compute.ts +196 -0
  256. package/src/report/components/entity-lists/faces.ts +207 -0
  257. package/src/report/components/entity-lists/index.tsx +241 -0
  258. package/src/report/components/entity-lists/validate.test.ts +114 -0
  259. package/src/report/{react → components}/fixtures.ts +2 -2
  260. package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
  261. package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
  262. package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
  263. package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
  264. package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
  265. package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
  266. package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
  267. package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
  268. package/src/report/components/metric-views/faces.ts +423 -0
  269. package/src/report/components/metric-views/index.tsx +352 -0
  270. package/src/report/{text → components/metric-views}/plot.ts +1 -1
  271. package/src/report/components/metric-views/validate.test.ts +211 -0
  272. package/src/report/{react → components}/render.test.tsx +47 -17
  273. package/src/report/components/shared-compute.ts +59 -0
  274. package/src/report/components/shared-faces.ts +30 -0
  275. package/src/report/components/shared.ts +192 -0
  276. package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
  277. package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
  278. package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
  279. package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
  280. package/src/report/components/site-components/compute.ts +144 -0
  281. package/src/report/components/site-components/faces.ts +80 -0
  282. package/src/report/components/site-components/index.tsx +265 -0
  283. package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
  284. package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
  285. package/src/report/components/site-components/validate.test.ts +89 -0
  286. package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
  287. package/src/report/components/summaries/compute.ts +56 -0
  288. package/src/report/components/summaries/faces.ts +48 -0
  289. package/src/report/components/summaries/index.tsx +127 -0
  290. package/src/report/components/summaries/validate.test.ts +49 -0
  291. package/src/report/definition/grid-layout.test.ts +124 -0
  292. package/src/report/definition/grid-layout.ts +146 -0
  293. package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
  294. package/src/report/{report.ts → definition/report.ts} +98 -165
  295. package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
  296. package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
  297. package/src/report/{tree.ts → definition/tree.ts} +56 -20
  298. package/src/report/index.ts +126 -66
  299. package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
  300. package/src/report/{flag.ts → model/flag.ts} +39 -1
  301. package/src/report/{format.ts → model/format.ts} +59 -5
  302. package/src/report/{locale.ts → model/locale.ts} +34 -36
  303. package/src/report/{metrics.ts → model/metrics.ts} +3 -3
  304. package/src/report/{types.ts → model/types.ts} +143 -22
  305. package/src/report/react/index.tsx +52 -26
  306. package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
  307. package/src/report/runtime/host.test.ts +44 -0
  308. package/src/report/runtime/host.ts +138 -0
  309. package/src/report/{load.ts → runtime/load.ts} +1 -1
  310. package/src/report/runtime/text.ts +192 -0
  311. package/src/report/runtime/web.ts +106 -0
  312. package/src/results/attempt-evidence.ts +5 -1
  313. package/src/results/host-equivalence.test.ts +53 -2
  314. package/src/results/select.ts +29 -5
  315. package/src/runner/attempt.test.ts +48 -1
  316. package/src/runner/attempt.ts +62 -42
  317. package/src/runner/cleanup-timeout.test.ts +16 -0
  318. package/src/runner/cleanup-timeout.ts +23 -0
  319. package/src/runner/discover.ts +1 -2
  320. package/src/runner/eval-selection.test.ts +187 -0
  321. package/src/runner/eval-selection.ts +109 -0
  322. package/src/runner/experiment-cleanup-registry.test.ts +89 -0
  323. package/src/runner/experiment-cleanup-registry.ts +39 -0
  324. package/src/runner/experiment-labels.test.ts +71 -0
  325. package/src/runner/feedback/agent.test.ts +33 -1
  326. package/src/runner/feedback/agent.ts +12 -16
  327. package/src/runner/feedback/ci.test.ts +35 -1
  328. package/src/runner/feedback/ci.ts +16 -18
  329. package/src/runner/feedback/coordinator.ts +28 -0
  330. package/src/runner/feedback/human.test.ts +108 -1
  331. package/src/runner/feedback/human.ts +71 -21
  332. package/src/runner/feedback/reducer.test.ts +64 -0
  333. package/src/runner/feedback/reducer.ts +26 -0
  334. package/src/runner/feedback/sink.ts +40 -1
  335. package/src/runner/fingerprint.ts +2 -1
  336. package/src/runner/report.test.ts +14 -0
  337. package/src/runner/report.ts +3 -1
  338. package/src/runner/run.test.ts +390 -12
  339. package/src/runner/run.ts +259 -26
  340. package/src/runner/sandbox-selection.test.ts +13 -3
  341. package/src/runner/sandbox-selection.ts +4 -2
  342. package/src/runner/types.ts +158 -23
  343. package/src/sandbox/docker.ts +7 -0
  344. package/src/sandbox/e2b.ts +12 -13
  345. package/src/sandbox/types.ts +23 -21
  346. package/src/scoring/display.test.ts +28 -2
  347. package/src/scoring/display.ts +38 -5
  348. package/src/scoring/types.ts +3 -3
  349. package/src/shared/aggregate.test.ts +52 -0
  350. package/src/shared/aggregate.ts +29 -0
  351. package/src/shared/types.ts +6 -1
  352. package/src/show/command.test.ts +34 -0
  353. package/src/show/command.ts +16 -0
  354. package/src/show/compose.ts +1 -1
  355. package/src/show/index.ts +56 -20
  356. package/src/show/render.ts +15 -239
  357. package/src/show/show.test.ts +127 -34
  358. package/src/view/app/App.test.tsx +1 -1
  359. package/src/view/app/App.tsx +119 -47
  360. package/src/view/app/i18n.ts +7 -147
  361. package/src/view/app/lib/attempt-dialog.test.ts +65 -0
  362. package/src/view/app/lib/attempt-dialog.ts +69 -0
  363. package/src/view/app/main.tsx +0 -1
  364. package/src/view/app/types.ts +3 -86
  365. package/src/view/artifact-serving.test.ts +22 -38
  366. package/src/view/client-dist/app.css +1 -1
  367. package/src/view/client-dist/app.js +15 -22
  368. package/src/view/data.test.ts +48 -36
  369. package/src/view/data.ts +105 -83
  370. package/src/view/index.ts +1 -1
  371. package/src/view/server.ts +32 -2
  372. package/src/view/shared/types.ts +6 -31
  373. package/src/view/site-parity.test.ts +24 -1
  374. package/src/view/site.ts +144 -17
  375. package/src/view/styles.css +0 -805
  376. package/src/view/view-report.test.ts +141 -22
  377. package/dist/report/components.d.ts +0 -179
  378. package/dist/report/components.js +0 -544
  379. package/dist/report/compute.d.ts +0 -139
  380. package/dist/report/react/ExperimentComparison.d.ts +0 -10
  381. package/dist/report/react/ExperimentComparison.js +0 -12
  382. package/dist/report/react/MetricScatter.d.ts +0 -9
  383. package/dist/report/react/format.d.ts +0 -3
  384. package/dist/report/react/format.js +0 -7
  385. package/dist/report/text/faces.d.ts +0 -45
  386. package/dist/report/text/faces.js +0 -671
  387. package/dist/report/web.d.ts +0 -32
  388. package/dist/report/web.js +0 -48
  389. package/docs-site/zh/how-to/authoring.mdx +0 -162
  390. package/docs-site/zh/how-to/custom-reports.mdx +0 -414
  391. package/docs-site/zh/how-to/experiments.mdx +0 -86
  392. package/docs-site/zh/how-to/write-experiment.mdx +0 -164
  393. package/src/report/components.tsx +0 -925
  394. package/src/report/react/ExperimentComparison.tsx +0 -73
  395. package/src/report/react/format.ts +0 -9
  396. package/src/report/text/faces.ts +0 -767
  397. package/src/report/web.ts +0 -77
  398. package/src/show/report-host.test.ts +0 -205
  399. package/src/show/report-host.ts +0 -389
  400. package/src/view/app/components/AttemptModal.tsx +0 -496
  401. package/src/view/app/components/CodeView.test.tsx +0 -142
  402. package/src/view/app/components/CodeView.tsx +0 -310
  403. package/src/view/app/components/CopyControls.tsx +0 -106
  404. package/src/view/app/components/Trace.tsx +0 -100
  405. package/src/view/app/components/Transcript.tsx +0 -157
  406. package/src/view/app/components/ui/badge.tsx +0 -21
  407. package/src/view/app/lib/artifact-url.ts +0 -17
  408. package/src/view/app/lib/attempt-route.test.ts +0 -80
  409. package/src/view/app/lib/attempt-route.ts +0 -52
  410. package/src/view/app/lib/format.ts +0 -70
  411. package/src/view/app/lib/guards.test.ts +0 -108
  412. package/src/view/app/lib/guards.ts +0 -71
  413. package/src/view/app/lib/rows.ts +0 -22
  414. package/src/view/app/lib/transcript-data.tsx +0 -151
  415. package/src/view/app/lib/verdict.ts +0 -23
  416. package/src/view/app/shared.ts +0 -8
  417. /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
  418. /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
  419. /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
  420. /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
  421. /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
  422. /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
  423. /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
  424. /package/dist/report/{types.js → model/types.js} +0 -0
  425. /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
  426. /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
  427. /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
  428. /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
package/src/view/app/components/ui/badge.tsx DELETED
@@ -1,21 +0,0 @@
1
- import type { ReactNode } from "react";
2
- import { cn } from "../../lib/cn.ts";
3
-
4
- /**
5
- * 状态小标签:边框 + 浅底都取自当前文字色,tone(.good/.bad/.warn/.infra-err)决定颜色。
6
- * 取代旧的 .modal-verdict / 内联状态 chip。
7
- */
8
- export function Badge({ tone, className, children }: { tone?: string; className?: string; children: ReactNode }) {
9
- return (
10
- <span
11
- className={cn(
12
- "inline-block w-fit rounded-[5px] border border-current bg-current/10 px-[7px] py-px",
13
- "text-[10px] font-bold uppercase tracking-[0.05em]",
14
- tone,
15
- className,
16
- )}
17
- >
18
- {children}
19
- </span>
20
- );
21
- }
package/src/view/app/lib/artifact-url.ts DELETED
@@ -1,17 +0,0 @@
1
- // artifact fetch 的 URL:以「页面所在目录」为基底的路径 `<页面目录>/artifact/<rel>`。
2
- // 本地 dev server(server.ts 的 /artifact/ 路由)和目录式静态导出(buildView 拷到 <out>/artifact/)
3
- // 共用同一布局:artifact/ 恒为 index.html 的同级目录。基底不能交给浏览器的相对解析——
4
- // 静态托管常把 <dir>/index.html 服务在无尾斜杠的 <dir> 路径上(反代 rewrite、cleanUrls),
5
- // 此时相对路径 `artifact/...` 会解析到上一级目录断链。这里自己算目录:pathname 末段带 `.`
6
- // 视为文件名去掉(直接打开 .../index.html),否则整个 pathname 就是目录(含无尾斜杠形态)。
7
- export function artifactUrl(rel: string): string {
8
- const tail = "artifact/" + rel.split("/").map(encodeURIComponent).join("/");
9
- if (typeof location === "undefined") return tail; // 非浏览器环境(测试直调)保持相对形态
10
- return pageDir(location.pathname) + tail;
11
- }
12
-
13
- /** 页面 pathname → 它所在目录(恒以 `/` 结尾)。 */
14
- export function pageDir(pathname: string): string {
15
- if (/\.[^/]*$/.test(pathname)) return pathname.replace(/[^/]*$/, "");
16
- return pathname.endsWith("/") ? pathname : pathname + "/";
17
- }
package/src/view/app/lib/attempt-route.test.ts DELETED
@@ -1,80 +0,0 @@
1
- // cases: docs/engineering/unit-tests/reports/cases.md
2
- // #/attempt/@<locator> 深链的纯函数单测:解析 / 格式化往返、坏输入、按 locator 定位。
3
-
4
- import { describe, expect, it } from "vitest";
5
- import { formatAttemptHash, parseAttemptHash, resolveAttemptLocator } from "./attempt-route.ts";
6
- import type { AttemptLocator, ViewResult, ViewSnapshot } from "../types.ts";
7
-
8
- const LOCATOR_A = "@1abc123" as AttemptLocator;
9
- const LOCATOR_B = "@1zzz999" as AttemptLocator;
10
-
11
- const attempt = (locator: AttemptLocator | undefined, index: number): ViewResult => ({
12
- id: `demo/eval-${index}`,
13
- agent: "demo-agent",
14
- verdict: "passed",
15
- attempt: 0,
16
- durationMs: 1,
17
- assertions: [],
18
- ...(locator !== undefined ? { locator } : {}),
19
- });
20
-
21
- // 快照只有 results 参与定位,其余元信息字段与路由无关。
22
- const snap = (results: ViewResult[]): ViewSnapshot => ({ results }) as ViewSnapshot;
23
-
24
- describe("parseAttemptHash / formatAttemptHash", () => {
25
- it("parses the canonical `@<scheme><body>` locator shape", () => {
26
- expect(parseAttemptHash("#/attempt/@1abc123")).toBe("@1abc123");
27
- });
28
-
29
- it("round-trips through formatAttemptHash", () => {
30
- for (const locator of [LOCATOR_A, LOCATOR_B, "@10000000" as AttemptLocator]) {
31
- expect(parseAttemptHash(formatAttemptHash(locator))).toBe(locator);
32
- }
33
- });
34
-
35
- it("formatAttemptHash matches the report tree's DEFAULT_WEB_CONTEXT.attemptHref format exactly", () => {
36
- // src/report/tree.ts: attemptHref: (locator) => `#/attempt/${locator}` — locator 本身自带 "@" 前缀。
37
- expect(formatAttemptHash(LOCATOR_A)).toBe(`#/attempt/${LOCATOR_A}`);
38
- });
39
-
40
- it("rejects non-attempt hashes and malformed shapes", () => {
41
- for (const hash of [
42
- "",
43
- "#",
44
- "#/",
45
- "#tab-experiments", // 页内锚点不归这条路由
46
- "#/compare/a/b",
47
- "#/attempt",
48
- "#/attempt/",
49
- "#/attempt/exp/snap/eval/a0", // 旧的两段式路由,不再被接受
50
- "#/attempt/nolocatorprefix",
51
- "#/attempt/@", // 只有前缀,没有 scheme + body
52
- "#/attempt/@ABC123", // 大写不合法(base36 恒小写)
53
- "#/attempt/@1abc 123", // 含空格
54
- "#/attempt/@1abc/123", // 含 "/"
55
- ]) {
56
- expect(parseAttemptHash(hash), hash).toBeNull();
57
- }
58
- });
59
- });
60
-
61
- describe("resolveAttemptLocator", () => {
62
- const snapshots = [
63
- snap([attempt(LOCATOR_A, 0), attempt(LOCATOR_B, 1)]),
64
- snap([attempt("@1other01" as AttemptLocator, 2)]),
65
- ];
66
-
67
- it("finds the attempt whose injected locator matches", () => {
68
- expect(resolveAttemptLocator(snapshots, LOCATOR_A)).toBe(snapshots[0]!.results[0]);
69
- expect(resolveAttemptLocator(snapshots, LOCATOR_B)).toBe(snapshots[0]!.results[1]);
70
- });
71
-
72
- it("returns null for unknown locators", () => {
73
- expect(resolveAttemptLocator(snapshots, "@1nosuch1" as AttemptLocator)).toBeNull();
74
- });
75
-
76
- it("returns null when results predate locator injection (old baked data)", () => {
77
- const legacy = attempt(undefined, 0);
78
- expect(resolveAttemptLocator([snap([legacy])], LOCATOR_A)).toBeNull();
79
- });
80
- });
package/src/view/app/lib/attempt-route.ts DELETED
@@ -1,52 +0,0 @@
1
- // attempt 级 hash 深链:`#/attempt/@<locator>`(docs/feature/reports/view.md「打开与收窄」)。
2
- // 路由参数就是不透明的 AttemptLocator(src/results/locator.ts),由 loader 注入到每条
3
- // result 上;这里只做纯解析 / 格式化 / 匹配,不碰 location / history,方便单测。与报告槽
4
- // DEFAULT_WEB_CONTEXT.attemptHref(src/report/tree.ts)同一格式,两条深链来源互通。
5
- // hash 目前只有这一种路由:tab 切换是纯组件 state,旧版 modal 深链走 ?modal= 查询参数,互不占用。
6
- //
7
- // AttemptLocator 的编码/解码本体住在 src/results/locator.ts,但那个模块顶层 import 了
8
- // node:crypto(encodeAttemptLocator 用于生成 locator),不能被这个浏览器打包的 app/ 目录
9
- // 静态 import——这里只需要「像不像一个 locator」的轻量语法校验,不需要真校验 scheme/body
10
- // 长度(那是 reader 建索引时的事,view 前端拿到的 locator 恒来自可信的 loader 注入)。
11
-
12
- import type { AttemptLocator, ViewResult, ViewSnapshot } from "../types.ts";
13
-
14
- export const ATTEMPT_HASH_PREFIX = "#/attempt/";
15
-
16
- /** locator 串的最小形状:`@` + 至少一个 base36 字符(scheme 字符 + body)。 */
17
- const LOCATOR_SHAPE = /^@[0-9a-z]+$/;
18
-
19
- /** AttemptLocator → 可分享的 hash:locator 本身就是路由参数,原样拼在前缀后面,不需要分段编码。 */
20
- export function formatAttemptHash(locator: AttemptLocator): string {
21
- return `${ATTEMPT_HASH_PREFIX}${locator}`;
22
- }
23
-
24
- /**
25
- * hash → AttemptLocator;不是本路由 / 形状不像 locator 返回 null(由调用方决定 warn 与否)。
26
- * 只做前缀 + 粗粒度字符集校验,不重新实现 decodeAttemptLocator 的 scheme/body 长度校验——
27
- * 那份权威校验属于 src/results/locator.ts,这里刻意保持浏览器打包安全(不引入 node:crypto)。
28
- */
29
- export function parseAttemptHash(hash: string): AttemptLocator | null {
30
- if (!hash.startsWith(ATTEMPT_HASH_PREFIX)) return null;
31
- const rest = hash.slice(ATTEMPT_HASH_PREFIX.length);
32
- if (!LOCATOR_SHAPE.test(rest)) return null;
33
- return rest as AttemptLocator;
34
- }
35
-
36
- /** 在全部快照(含历史)里找 locator 指向的 attempt;旧格式烘焙的数据没有 locator,自然找不到。 */
37
- export function resolveAttemptLocator(snapshots: ViewSnapshot[], locator: AttemptLocator): ViewResult | null {
38
- for (const snapshot of snapshots) {
39
- for (const result of snapshot.results ?? []) {
40
- if (result.locator === locator) return result;
41
- }
42
- }
43
- return null;
44
- }
45
-
46
- /** 深链定位不到时的提示(console.warn 用,英文);页面照常渲染,不开空 modal。 */
47
- export function unresolvedAttemptWarning(hash: string): string {
48
- return (
49
- `[niceeval view] Ignoring attempt link "${hash}": no matching attempt in this view ` +
50
- `(snapshot not loaded, attempt not found, or the data was baked without a locator).`
51
- );
52
- }
package/src/view/app/lib/format.ts DELETED
@@ -1,70 +0,0 @@
1
- import type { ViewUsage } from "../types.ts";
2
-
3
- // 通过率/耗时/成本的展示口径与 server 共用一份实现,见 src/shared/format.ts(CLI 表格同用)。
4
- export { formatCost, formatDuration, formatPercent } from "../../../shared/format.ts";
5
-
6
- export function prettyJson(value: unknown): string {
7
- if (typeof value === "string") return value;
8
- try {
9
- return JSON.stringify(value, null, 2);
10
- } catch {
11
- return String(value);
12
- }
13
- }
14
-
15
- export function previewText(value: string): string {
16
- const str = String(value);
17
- const trimmed = str.trim();
18
- // object 结果会被 prettyJson 成多行,只取首行会剩一个 "{";JSON 压回单行再做预览。
19
- if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
20
- try {
21
- return JSON.stringify(JSON.parse(trimmed));
22
- } catch {
23
- // 不是合法 JSON,按普通多行文本取首个非空行
24
- }
25
- }
26
- return str.split("\n").find((line) => line.trim()) || "";
27
- }
28
-
29
- export function truncate(value: unknown, n: number): string {
30
- const str = String(value);
31
- return str.length > n ? str.slice(0, n) + " ... [+" + (str.length - n) + " chars]" : str;
32
- }
33
-
34
- export function formatConfigValue(value: unknown): string {
35
- if (value === null) return "null";
36
- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return String(value);
37
- return JSON.stringify(value);
38
- }
39
-
40
- export function totalTokens(usage?: ViewUsage): number {
41
- return (usage?.inputTokens || 0) + (usage?.outputTokens || 0) + (usage?.cacheReadTokens || 0) + (usage?.cacheWriteTokens || 0);
42
- }
43
-
44
- /** 断言 / judge 分数本就是 0–1,直接展示原值(去掉末尾零),不转百分比。pass-rate 之类的「比率」仍用 formatPercent。 */
45
- export function formatScore(value?: number): string {
46
- if (typeof value !== "number" || !Number.isFinite(value)) return "0";
47
- return String(Number(value.toFixed(2)));
48
- }
49
-
50
- export function formatTokens(value: number): string {
51
- if (!Number.isFinite(value) || value <= 0) return "0";
52
- if (value >= 1000000) return (value / 1000000).toFixed(2) + "M";
53
- if (value >= 1000) return (value / 1000).toFixed(1) + "k";
54
- return String(Math.round(value));
55
- }
56
-
57
- /** 日期按 locale 格式化;不传 locale 时跟随浏览器设置。server 不再预格式化日期,都走这里。 */
58
- export function formatDateTime(iso?: string, locale?: string): string {
59
- if (!iso) return "-";
60
- const date = new Date(iso);
61
- if (Number.isNaN(date.getTime())) return "-";
62
- return date.toLocaleString(locale ?? [], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit" });
63
- }
64
-
65
- export function formatClock(iso?: string): string {
66
- if (!iso) return "";
67
- const date = new Date(iso);
68
- if (Number.isNaN(date.getTime())) return "";
69
- return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" });
70
- }
package/src/view/app/lib/guards.test.ts DELETED
@@ -1,108 +0,0 @@
1
- // cases: docs/engineering/unit-tests/reports/cases.md
2
- // 「Attempt 详情」:事件流按条目校验、按条目容错;skill.loaded 是一等回复条目。
3
- // bug: memory/view-unknown-event-type-drops-whole-transcript.md
4
- import { describe, expect, it } from "vitest";
5
- import { asEvents } from "./guards.ts";
6
- import { indexTurns, locKey } from "./transcript-data.tsx";
7
- import type { TranscriptEvent } from "../types.ts";
8
-
9
- const send: TranscriptEvent = {
10
- type: "message",
11
- role: "user",
12
- text: "Migrate every route",
13
- loc: { file: "evals/m.eval.ts", line: 37, column: 8 },
14
- };
15
-
16
- describe("asEvents 按条目容错", () => {
17
- it("含 skill.loaded 的 events 数组不被判空,全部事件保留", () => {
18
- const raw = [
19
- send,
20
- { type: "skill.loaded", skill: "pdf-export", callId: "c1" },
21
- { type: "message", role: "assistant", text: "done" },
22
- ];
23
- const events = asEvents(raw);
24
- expect(events).toHaveLength(3);
25
- });
26
-
27
- it("未知事件类型包成 view.raw 原样保留,其余事件照常", () => {
28
- const unknown = { type: "future.event", payload: 1 };
29
- const events = asEvents([send, unknown, { type: "thinking", text: "hm" }]);
30
- expect(events?.map((e) => e.type)).toEqual(["message", "view.raw", "thinking"]);
31
- expect(events?.[1]).toEqual({ type: "view.raw", raw: unknown });
32
- });
33
-
34
- it("形状不合的已知类型同样包成 view.raw,不静默丢", () => {
35
- const malformed = { type: "message", role: "assistant" }; // 缺 text
36
- const events = asEvents([malformed]);
37
- expect(events).toEqual([{ type: "view.raw", raw: malformed }]);
38
- });
39
-
40
- it("非对象条目丢弃;非数组载荷整体拒绝", () => {
41
- expect(asEvents(["junk", 42, send])).toHaveLength(1);
42
- expect(asEvents({ events: [] })).toBeNull();
43
- expect(asEvents("nope")).toBeNull();
44
- });
45
- });
46
-
47
- describe("skill.loaded 聚合进 send 的回复", () => {
48
- it("indexTurns 聚出 kind: skill 回复并保留 Skill 名", () => {
49
- const turns = indexTurns([
50
- send,
51
- { type: "skill.loaded", skill: "pdf-export", callId: "c1" },
52
- { type: "message", role: "assistant", text: "done" },
53
- ]);
54
- const turn = turns.byKey.get(locKey("evals/m.eval.ts", 37));
55
- expect(turn?.replies).toEqual([
56
- { kind: "skill", skill: "pdf-export" },
57
- { kind: "text", text: "done" },
58
- ]);
59
- });
60
-
61
- it("view.raw 条目聚成 kind: raw 回复,原始载荷不丢", () => {
62
- const turns = indexTurns([send, { type: "view.raw", raw: { type: "future.event", payload: 1 } }]);
63
- const turn = turns.byKey.get(locKey("evals/m.eval.ts", 37));
64
- expect(turn?.replies).toEqual([{ kind: "raw", raw: { type: "future.event", payload: 1 } }]);
65
- });
66
- });
67
-
68
- describe("轮归属按 loc 判定,无 loc 的 user 消息不开新轮", () => {
69
- it("send 后紧跟的同文本无 loc 回显被吃掉,回复仍全部聚到 send 行", () => {
70
- const turns = indexTurns([
71
- send,
72
- { type: "message", role: "user", text: send.text },
73
- { type: "thinking", text: "plan" },
74
- { type: "message", role: "assistant", text: "done" },
75
- ]);
76
- const turn = turns.byKey.get(locKey("evals/m.eval.ts", 37));
77
- expect(turn?.replies).toEqual([
78
- { kind: "thinking", text: "plan" },
79
- { kind: "text", text: "done" },
80
- ]);
81
- expect(turns.noloc).toHaveLength(0);
82
- });
83
-
84
- it("轮中段的 stop-hook 反馈成为 kind: user 回复,其后的 assistant 回复不脱轮", () => {
85
- const turns = indexTurns([
86
- send,
87
- { type: "message", role: "user", text: send.text },
88
- { type: "message", role: "assistant", text: "first" },
89
- { type: "message", role: "user", text: "Stop hook feedback: save notes" },
90
- { type: "message", role: "assistant", text: "saved" },
91
- ]);
92
- const turn = turns.byKey.get(locKey("evals/m.eval.ts", 37));
93
- expect(turn?.replies).toEqual([
94
- { kind: "text", text: "first" },
95
- { kind: "user", text: "Stop hook feedback: save notes" },
96
- { kind: "text", text: "saved" },
97
- ]);
98
- });
99
-
100
- it("流首无 loc 的 user 消息(旧工件)仍开 noloc 轮", () => {
101
- const turns = indexTurns([
102
- { type: "message", role: "user", text: "hi" },
103
- { type: "message", role: "assistant", text: "hello" },
104
- ]);
105
- expect(turns.noloc).toHaveLength(1);
106
- expect(turns.noloc[0]?.replies).toEqual([{ kind: "text", text: "hello" }]);
107
- });
108
- });
package/src/view/app/lib/guards.ts DELETED
@@ -1,71 +0,0 @@
1
- import type { CodeSource, KnownTranscriptEvent, ObjectRecord, Span, TranscriptEvent } from "../types.ts";
2
-
3
- export function asSources(value: unknown): CodeSource[] | null {
4
- if (!Array.isArray(value)) return null;
5
- return value.every(isCodeSource) ? value : null;
6
- }
7
-
8
- export function isCodeSource(value: unknown): value is CodeSource {
9
- return isObjectRecord(value) && typeof value.path === "string" && typeof value.content === "string";
10
- }
11
-
12
- export function asEvents(value: unknown): TranscriptEvent[] | null {
13
- if (!Array.isArray(value)) return null;
14
- // 事件词汇会演进(skill.loaded 就是先例):未识别或缺字段的条目不整体判空、
15
- // 也不静默丢弃——包成 view.raw 原样展示,让新词汇在界面上可被发现、后续补一等呈现。
16
- // 全有全无判空会让源码视图的 send 行连回复入口都消失;只有非对象条目没有可展示的结构才丢。
17
- const out: TranscriptEvent[] = [];
18
- for (const item of value) {
19
- if (isKnownTranscriptEvent(item)) out.push(item);
20
- else if (isObjectRecord(item)) out.push({ type: "view.raw", raw: item });
21
- }
22
- return out;
23
- }
24
-
25
- export function asSpans(value: unknown): Span[] | null {
26
- if (!Array.isArray(value)) return null;
27
- return value.every(isSpan) ? value : null;
28
- }
29
-
30
- export function isKnownTranscriptEvent(value: unknown): value is KnownTranscriptEvent {
31
- if (!isObjectRecord(value) || typeof value.type !== "string") return false;
32
- switch (value.type) {
33
- case "message":
34
- return (value.role === "assistant" || value.role === "user") && typeof value.text === "string";
35
- case "action.called":
36
- return typeof value.callId === "string" && typeof value.name === "string";
37
- case "action.result":
38
- return typeof value.callId === "string";
39
- case "subagent.called":
40
- return typeof value.callId === "string" && typeof value.name === "string";
41
- case "subagent.completed":
42
- return typeof value.callId === "string";
43
- case "skill.loaded":
44
- return typeof value.skill === "string";
45
- case "input.requested":
46
- return isObjectRecord(value.request);
47
- case "thinking":
48
- return typeof value.text === "string";
49
- case "compaction":
50
- return true;
51
- case "error":
52
- return typeof value.message === "string";
53
- default:
54
- return false;
55
- }
56
- }
57
-
58
- export function isSpan(value: unknown): value is Span {
59
- return (
60
- isObjectRecord(value) &&
61
- typeof value.traceId === "string" &&
62
- typeof value.spanId === "string" &&
63
- typeof value.name === "string" &&
64
- typeof value.startMs === "number" &&
65
- typeof value.endMs === "number"
66
- );
67
- }
68
-
69
- export function isObjectRecord(value: unknown): value is ObjectRecord {
70
- return !!value && typeof value === "object" && !Array.isArray(value);
71
- }
package/src/view/app/lib/rows.ts DELETED
@@ -1,22 +0,0 @@
1
- // viewData 快照明细 → attempt 深链解析的纯拼接,零聚合口径:统计与列表整体住在报告页的
2
- // 静态 HTML 里(报告组件的官方计算函数),前端不拼任何榜单行。Attempts / Traces 是内建报告
3
- // 的普通页(AttemptList / TraceWaterfall 组件),不再由前端平铺快照明细。
4
-
5
- import type { ViewResult, ViewSnapshot } from "../types.ts";
6
-
7
- /** 旧版 ?modal= 深链的只读回退:在全部快照里按 (eval id, experimentId, attempt) 定位。 */
8
- export function resultFromUrl(snapshots: ViewSnapshot[]): ViewResult | null {
9
- const p = new URLSearchParams(location.search);
10
- const id = p.get("modal");
11
- if (!id) return null;
12
- const exp = p.get("exp");
13
- const attempt = parseInt(p.get("a") ?? "0", 10);
14
- for (const snapshot of snapshots) {
15
- for (const result of snapshot.results) {
16
- if (result.id === id && (!exp || result.experimentId === exp) && result.attempt === attempt) {
17
- return result;
18
- }
19
- }
20
- }
21
- return null;
22
- }
package/src/view/app/lib/transcript-data.tsx DELETED
@@ -1,151 +0,0 @@
1
- import type { ReactNode } from "react";
2
- import type { Assertion, Indexed, IndexedTurns, SourceTurn, TranscriptEvent, ViewJson } from "../types.ts";
3
- import type { ToolBlockCall } from "../shared.ts";
4
- import { isObjectRecord } from "./guards.ts";
5
- import { prettyJson } from "./format.ts";
6
-
7
- // ───────────────────────── 源码对齐的代码视图(github-diff 式)─────────────────────────
8
- // 拿 sources.json(eval 源码)+ events.json(带 loc 的 send),把每条 send / 断言的运行结果
9
- // 叠回真实源码行:send 行折叠→展开看回复;断言行绿(过)/红(不过),judge 行带分数,展开看 CoT。
10
-
11
- export function locKey(file: string, line: number): string {
12
- return `${file}:${line}`;
13
- }
14
-
15
- /** events → 按 send 的 loc 聚成「轮」:每轮含 sent 文本 + 后续 thinking/assistant/tool 回复。 */
16
- export function indexTurns(events: TranscriptEvent[]): IndexedTurns {
17
- const byKey = new Map<string, SourceTurn>();
18
- const noloc: SourceTurn[] = [];
19
- // callId → tool 回复,跨轮共享:HITL(审批门)下 action.called 在 send 轮、
20
- // action.result 在 respond 轮才到,只在当轮找会把结果丢掉。
21
- const toolByCallId = new Map<string, Extract<SourceTurn["replies"][number], { kind: "tool" }>>();
22
- let cur: SourceTurn | null = null;
23
- for (const ev of events || []) {
24
- if (ev.type === "message" && ev.role === "user") {
25
- // 同一条 send 会在流里出现两次:runner 在 send 时记带 loc 的一条,agent 原生
26
- // transcript 又回显同文本、无 loc 的一条。无 loc 的 user 消息不开新轮——回显
27
- // (与当前轮 sent 同文本且回复还没开始)直接吃掉,其它(stop-hook 反馈、skill
28
- // 注入等轮内注入)作为回复条目留在当前轮;否则回复全被回显轮抢走,
29
- // 带 loc 的 send 行只剩「(无回复)」。
30
- if (!ev.loc && cur) {
31
- if (cur.replies.length === 0 && (ev.text || "") === cur.sent) continue;
32
- cur.replies.push({ kind: "user", text: ev.text || "" });
33
- continue;
34
- }
35
- cur = { loc: ev.loc, sent: ev.text || "", replies: [] };
36
- if (ev.loc) byKey.set(locKey(ev.loc.file, ev.loc.line), cur);
37
- else noloc.push(cur);
38
- } else if (!cur) {
39
- continue;
40
- } else if (ev.type === "message" && ev.role === "assistant") {
41
- cur.replies.push({ kind: "text", text: ev.text || "" });
42
- } else if (ev.type === "thinking") {
43
- cur.replies.push({ kind: "thinking", text: ev.text || "" });
44
- } else if (ev.type === "action.called") {
45
- const reply = { kind: "tool" as const, ev };
46
- toolByCallId.set(ev.callId, reply);
47
- cur.replies.push(reply);
48
- } else if (ev.type === "action.result") {
49
- const tool = toolByCallId.get(ev.callId);
50
- if (tool) tool.result = ev;
51
- } else if (ev.type === "skill.loaded") {
52
- cur.replies.push({ kind: "skill", skill: ev.skill });
53
- } else if (ev.type === "view.raw") {
54
- cur.replies.push({ kind: "raw", raw: ev.raw });
55
- } else if (ev.type === "input.requested") {
56
- cur.replies.push({ kind: "input", ev });
57
- } else if (ev.type === "error") {
58
- cur.replies.push({ kind: "error", text: ev.message || "error" });
59
- }
60
- }
61
- return { byKey, noloc };
62
- }
63
-
64
- /** assertions → 按 loc 聚到行。有 loc 的进 byKey,没 loc 的进 noloc(底部兜底列)。 */
65
- export function indexAsserts(assertions: Assertion[]): Indexed<Assertion> {
66
- const byKey = new Map<string, Assertion[]>();
67
- const noloc: Assertion[] = [];
68
- for (const a of assertions || []) {
69
- if (a.loc) {
70
- const k = locKey(a.loc.file, a.loc.line);
71
- if (!byKey.has(k)) byKey.set(k, []);
72
- byKey.get(k)?.push(a);
73
- } else {
74
- noloc.push(a);
75
- }
76
- }
77
- return { byKey, noloc };
78
- }
79
-
80
- export const TS_HL_RE =
81
- /(\/\/[^\n]*)|(\/\*[^]*?\*\/)|(`(?:\\.|[^`\\])*`|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|\b(import|from|export|default|const|let|var|async|await|function|return|if|else|for|of|in|new|class|extends|typeof|void|true|false|null|undefined)\b|\b(\d[\d_.]*)\b|([A-Za-z_$][\w$]*)(?=\s*\()/g;
82
-
83
- /** 轻量 TS 着色(逐行,零依赖):注释 / 字符串 / 关键字 / 数字 / 函数名。 */
84
- export function highlightTs(line: string): ReactNode[] {
85
- const out: ReactNode[] = [];
86
- let last = 0;
87
- let i = 0;
88
- let m: RegExpExecArray | null;
89
- TS_HL_RE.lastIndex = 0;
90
- while ((m = TS_HL_RE.exec(line))) {
91
- if (m.index > last) out.push(line.slice(last, m.index));
92
- const cls = m[1] || m[2] ? "tok-comment" : m[3] ? "tok-str" : m[4] ? "tok-kw" : m[5] ? "tok-num" : m[6] ? "tok-fn" : null;
93
- out.push(cls ? <span key={i++} className={cls}>{m[0]}</span> : m[0]);
94
- last = m.index + m[0].length;
95
- if (m[0].length === 0) TS_HL_RE.lastIndex++;
96
- }
97
- if (last < line.length) out.push(line.slice(last));
98
- return out;
99
- }
100
-
101
- export const TOOL_VERB: Record<string, string> = {
102
- file_read: "Read",
103
- file_write: "Write",
104
- file_edit: "Edit",
105
- shell: "Bash",
106
- web_fetch: "Fetch",
107
- web_search: "Search",
108
- glob: "Glob",
109
- grep: "Grep",
110
- list_dir: "List",
111
- agent_task: "Task",
112
- };
113
-
114
- export function toolPrimaryArg(call: ToolBlockCall): string {
115
- const input = call.input;
116
- if (typeof input === "string") return input;
117
- if (!isObjectRecord(input)) return "";
118
- if (call.tool === "shell") {
119
- const command = input.command ?? input.cmd;
120
- if (typeof command === "string") return command;
121
- if (Array.isArray(command)) return command.filter((x: ViewJson) => typeof x === "string").join(" ");
122
- }
123
- for (const key of ["path", "file", "file_path", "filename", "pattern", "query", "url", "uri", "prompt", "description", "command", "remoteUrl"]) {
124
- const value = input[key];
125
- if (typeof value === "string" && value) return value;
126
- }
127
- // 领域工具(get_weather / calculate…)的入参 key 不在上面的通用名单里,
128
- // 兜底给紧凑 JSON——入参是断言的一等材料,不能因为 key 认不出就不显示。
129
- const keys = Object.keys(input);
130
- if (keys.length > 0) {
131
- try {
132
- const compact = JSON.stringify(input);
133
- return compact.length > 200 ? compact.slice(0, 200) + "…" : compact;
134
- } catch {
135
- return "";
136
- }
137
- }
138
- return "";
139
- }
140
-
141
- export function resultBody(output: ViewJson | undefined): string {
142
- if (output == null) return "";
143
- if (typeof output === "string") return output;
144
- if (isObjectRecord(output)) {
145
- for (const key of ["output", "stdout", "content", "text", "result", "body"]) {
146
- const value = output[key];
147
- if (typeof value === "string") return value;
148
- }
149
- }
150
- return prettyJson(output);
151
- }
package/src/view/app/lib/verdict.ts DELETED
@@ -1,23 +0,0 @@
1
- import type { Verdict, ViewResult } from "../types.ts";
2
- import type { T } from "../shared.ts";
3
- import { compactAssertionSummary, primaryAssertionSummary } from "../../../scoring/display.ts";
4
- import type { Verdict as CoreVerdict } from "../../../scoring/types.ts";
5
-
6
- export function verdictClass(verdict: Verdict): string {
7
- return verdict === "passed" ? "good" : verdict === "errored" ? "infra-err" : verdict === "failed" ? "bad" : "warn";
8
- }
9
-
10
- export function verdictLabel(verdict: Verdict, t: T): string {
11
- if (verdict === "passed") return t("status.pass");
12
- if (verdict === "failed") return t("status.fail");
13
- if (verdict === "errored") return t("status.error");
14
- if (verdict === "skipped") return t("status.skipped");
15
- return verdict || "—";
16
- }
17
-
18
- export function reasonFor(result: ViewResult): string {
19
- if (result.error) return result.error.message;
20
- if (result.skipReason) return result.skipReason;
21
- const summary = primaryAssertionSummary(result.assertions ?? [], result.verdict as CoreVerdict);
22
- return summary ? compactAssertionSummary(summary) : "";
23
- }
package/src/view/app/shared.ts DELETED
@@ -1,8 +0,0 @@
1
- import type { Vars } from "../../i18n/core.ts";
2
- import type { MessageKey } from "./i18n.ts";
3
- import type { CodeSource, TranscriptEvent, ViewJson } from "./types.ts";
4
-
5
- export type T = (key: MessageKey, vars?: Vars) => string;
6
- export type ArtifactLoadState =
7
- | { sources: CodeSource[] | null; events: TranscriptEvent[] | null; status: "loading" | "ready" | "none" };
8
- export type ToolBlockCall = { tool?: string; name: string; input: ViewJson };
/package/dist/report/{types.js → model/types.js} RENAMED
File without changes