niceeval 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (337) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +321 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/otlp/parse.d.ts +3 -0
  9. package/dist/o11y/otlp/receiver.d.ts +21 -0
  10. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  11. package/dist/o11y/types.d.ts +190 -0
  12. package/dist/report/aggregate.d.ts +57 -0
  13. package/dist/report/aggregate.js +212 -0
  14. package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
  15. package/dist/report/built-ins/experiment-comparison.js +13 -0
  16. package/dist/report/built-ins/index.d.ts +1 -0
  17. package/dist/report/built-ins/index.js +2 -0
  18. package/dist/report/components.d.ts +180 -0
  19. package/dist/report/components.js +132 -0
  20. package/dist/report/compute.d.ts +106 -0
  21. package/dist/report/compute.js +600 -0
  22. package/dist/report/flag.d.ts +6 -0
  23. package/dist/report/flag.js +8 -0
  24. package/dist/report/format.d.ts +26 -0
  25. package/dist/report/format.js +109 -0
  26. package/dist/report/index.d.ts +19 -0
  27. package/dist/report/index.js +30 -0
  28. package/dist/report/load.d.ts +12 -0
  29. package/dist/report/load.js +56 -0
  30. package/dist/report/locale.d.ts +94 -0
  31. package/dist/report/locale.js +183 -0
  32. package/dist/report/metrics.d.ts +21 -0
  33. package/dist/report/metrics.js +115 -0
  34. package/dist/report/primitives.d.ts +64 -0
  35. package/dist/report/primitives.js +113 -0
  36. package/dist/report/react/AttemptList.d.ts +23 -0
  37. package/dist/report/react/AttemptList.js +19 -0
  38. package/dist/report/react/DeltaTable.d.ts +8 -0
  39. package/dist/report/react/DeltaTable.js +24 -0
  40. package/dist/report/react/EvalList.d.ts +10 -0
  41. package/dist/report/react/EvalList.js +0 -0
  42. package/dist/report/react/ExperimentList.d.ts +11 -0
  43. package/dist/report/react/ExperimentList.js +67 -0
  44. package/dist/report/react/GroupSummary.d.ts +8 -0
  45. package/dist/report/react/GroupSummary.js +8 -0
  46. package/dist/report/react/MetricBars.d.ts +10 -0
  47. package/dist/report/react/MetricBars.js +48 -0
  48. package/dist/report/react/MetricLine.d.ts +9 -0
  49. package/dist/report/react/MetricLine.js +68 -0
  50. package/dist/report/react/MetricMatrix.d.ts +10 -0
  51. package/dist/report/react/MetricMatrix.js +23 -0
  52. package/dist/report/react/MetricScatter.d.ts +9 -0
  53. package/dist/report/react/MetricScatter.js +96 -0
  54. package/dist/report/react/MetricTable.d.ts +11 -0
  55. package/dist/report/react/MetricTable.js +26 -0
  56. package/dist/report/react/RunOverview.d.ts +8 -0
  57. package/dist/report/react/RunOverview.js +12 -0
  58. package/dist/report/react/Scoreboard.d.ts +8 -0
  59. package/dist/report/react/Scoreboard.js +26 -0
  60. package/dist/report/react/cell.d.ts +9 -0
  61. package/dist/report/react/cell.js +9 -0
  62. package/dist/report/react/chart-math.d.ts +14 -0
  63. package/dist/report/react/chart-math.js +68 -0
  64. package/dist/report/react/colors.d.ts +14 -0
  65. package/dist/report/react/colors.js +45 -0
  66. package/dist/report/react/fixtures.d.ts +19 -0
  67. package/dist/report/react/fixtures.js +398 -0
  68. package/dist/report/react/format.d.ts +3 -0
  69. package/dist/report/react/format.js +7 -0
  70. package/dist/report/react/index.d.ts +6 -0
  71. package/dist/report/react/index.js +14 -0
  72. package/dist/report/report.d.ts +24 -0
  73. package/dist/report/report.js +49 -0
  74. package/dist/report/text/faces.d.ts +29 -0
  75. package/dist/report/text/faces.js +541 -0
  76. package/dist/report/text/layout.d.ts +75 -0
  77. package/dist/report/text/layout.js +173 -0
  78. package/dist/report/text/plot.d.ts +37 -0
  79. package/dist/report/text/plot.js +81 -0
  80. package/dist/report/text/table.d.ts +8 -0
  81. package/dist/report/text/table.js +86 -0
  82. package/dist/report/tree.d.ts +99 -0
  83. package/dist/report/tree.js +228 -0
  84. package/dist/report/types.d.ts +408 -0
  85. package/dist/report/types.js +4 -0
  86. package/dist/report/web.d.ts +15 -0
  87. package/dist/report/web.js +37 -0
  88. package/dist/results/locator.d.ts +75 -0
  89. package/dist/results/locator.js +0 -0
  90. package/dist/results/select.d.ts +45 -0
  91. package/dist/results/select.js +230 -0
  92. package/dist/results/source-hash.d.ts +9 -0
  93. package/dist/results/types.d.ts +204 -0
  94. package/dist/runner/eval-source.d.ts +22 -0
  95. package/dist/runner/fingerprint.d.ts +23 -0
  96. package/dist/runner/types.d.ts +332 -0
  97. package/dist/sandbox/docker-stream.d.ts +24 -0
  98. package/dist/sandbox/docker.d.ts +84 -0
  99. package/dist/sandbox/e2b.d.ts +31 -0
  100. package/dist/sandbox/errors.d.ts +4 -0
  101. package/dist/sandbox/local-files.d.ts +6 -0
  102. package/dist/sandbox/paths.d.ts +4 -0
  103. package/dist/sandbox/registry.d.ts +13 -0
  104. package/dist/sandbox/resolve.d.ts +40 -0
  105. package/dist/sandbox/retry.d.ts +10 -0
  106. package/dist/sandbox/shell.d.ts +11 -0
  107. package/dist/sandbox/source-files.d.ts +13 -0
  108. package/dist/sandbox/types.d.ts +203 -0
  109. package/dist/sandbox/vercel.d.ts +37 -0
  110. package/dist/scoring/types.d.ts +70 -0
  111. package/dist/shared/aggregate.d.ts +29 -0
  112. package/dist/shared/aggregate.js +55 -0
  113. package/dist/shared/types.d.ts +27 -0
  114. package/dist/shared/verdict.d.ts +27 -0
  115. package/dist/shared/verdict.js +34 -0
  116. package/dist/tty-line.d.ts +6 -0
  117. package/dist/types.d.ts +7 -0
  118. package/dist/util.d.ts +22 -0
  119. package/docs-site/zh/concepts/adapter.mdx +7 -7
  120. package/docs-site/zh/concepts/assert.mdx +19 -18
  121. package/docs-site/zh/concepts/evals.mdx +12 -11
  122. package/docs-site/zh/concepts/experiment.mdx +2 -2
  123. package/docs-site/zh/concepts/judge.mdx +9 -9
  124. package/docs-site/zh/concepts/overview.mdx +3 -3
  125. package/docs-site/zh/concepts/tier.mdx +6 -6
  126. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  127. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  128. package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
  129. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  130. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  131. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  132. package/docs-site/zh/guides/experiments.mdx +7 -1
  133. package/docs-site/zh/guides/fixtures.mdx +4 -2
  134. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  135. package/docs-site/zh/guides/publish-report.mdx +4 -4
  136. package/docs-site/zh/guides/report-components.mdx +194 -55
  137. package/docs-site/zh/guides/reporters.mdx +2 -2
  138. package/docs-site/zh/guides/results-data.mdx +61 -34
  139. package/docs-site/zh/guides/runner.mdx +2 -2
  140. package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
  141. package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
  142. package/docs-site/zh/guides/viewing-results.mdx +200 -79
  143. package/docs-site/zh/guides/write-experiment.mdx +9 -1
  144. package/docs-site/zh/guides/write-send.mdx +4 -3
  145. package/docs-site/zh/index.mdx +4 -3
  146. package/docs-site/zh/introduction.mdx +2 -1
  147. package/docs-site/zh/quickstart.mdx +5 -3
  148. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  149. package/docs-site/zh/reference/capabilities.mdx +1 -1
  150. package/docs-site/zh/reference/cli.mdx +36 -13
  151. package/docs-site/zh/reference/define-agent.mdx +15 -13
  152. package/docs-site/zh/reference/define-config.mdx +1 -1
  153. package/docs-site/zh/reference/define-eval.mdx +3 -3
  154. package/docs-site/zh/reference/events.mdx +9 -3
  155. package/docs-site/zh/reference/expect.mdx +13 -8
  156. package/package.json +10 -6
  157. package/src/agents/bub.ts +129 -42
  158. package/src/agents/claude-code.test.ts +152 -0
  159. package/src/agents/claude-code.ts +127 -13
  160. package/src/agents/codex.test.ts +192 -0
  161. package/src/agents/codex.ts +151 -12
  162. package/src/agents/index.ts +9 -3
  163. package/src/agents/manifest.ts +41 -0
  164. package/src/agents/plugin-config.test.ts +94 -0
  165. package/src/agents/shared.ts +33 -2
  166. package/src/agents/skills.test.ts +214 -0
  167. package/src/agents/skills.ts +304 -0
  168. package/src/agents/streaming.ts +5 -5
  169. package/src/agents/types.ts +59 -3
  170. package/src/cli.ts +74 -47
  171. package/src/context/context.test.ts +20 -2
  172. package/src/context/control-flow.ts +1 -1
  173. package/src/context/session.test.ts +92 -1
  174. package/src/context/session.ts +23 -1
  175. package/src/context/types.ts +4 -4
  176. package/src/define.test.ts +8 -8
  177. package/src/define.ts +11 -11
  178. package/src/expect/index.ts +5 -5
  179. package/src/i18n/en.ts +33 -15
  180. package/src/i18n/zh-CN.ts +37 -21
  181. package/src/index.ts +1 -1
  182. package/src/o11y/execution-tree.test.ts +452 -0
  183. package/src/o11y/execution-tree.ts +367 -0
  184. package/src/o11y/otlp/receiver.ts +1 -1
  185. package/src/o11y/otlp/select.ts +7 -3
  186. package/src/o11y/parsers/claude-code.test.ts +175 -0
  187. package/src/o11y/parsers/claude-code.ts +32 -0
  188. package/src/o11y/types.ts +6 -1
  189. package/src/report/aggregate.ts +26 -20
  190. package/src/report/built-in-user-parity.test.tsx +640 -0
  191. package/src/report/built-ins/experiment-comparison.tsx +19 -0
  192. package/src/report/built-ins/index.ts +2 -0
  193. package/src/report/components.tsx +225 -50
  194. package/src/report/compute.ts +335 -54
  195. package/src/report/dual-render.test.tsx +1129 -0
  196. package/src/report/flag.ts +1 -1
  197. package/src/report/format.ts +35 -2
  198. package/src/report/index.ts +47 -14
  199. package/src/report/load.ts +1 -1
  200. package/src/report/locale.ts +229 -0
  201. package/src/report/metrics.ts +39 -15
  202. package/src/report/primitives.tsx +100 -2
  203. package/src/report/react/AttemptList.tsx +113 -0
  204. package/src/report/react/DeltaTable.tsx +9 -6
  205. package/src/report/react/EvalList.tsx +0 -0
  206. package/src/report/react/ExperimentList.tsx +220 -0
  207. package/src/report/react/GroupSummary.tsx +66 -0
  208. package/src/report/react/MetricBars.tsx +24 -13
  209. package/src/report/react/MetricLine.tsx +21 -21
  210. package/src/report/react/MetricMatrix.tsx +8 -4
  211. package/src/report/react/MetricScatter.tsx +126 -96
  212. package/src/report/react/MetricTable.tsx +115 -15
  213. package/src/report/react/RunOverview.tsx +48 -26
  214. package/src/report/react/Scoreboard.tsx +15 -7
  215. package/src/report/react/cell.tsx +18 -9
  216. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  217. package/src/report/react/colors.ts +24 -15
  218. package/src/report/react/enhance.js +168 -0
  219. package/src/report/react/fixtures.ts +169 -34
  220. package/src/report/react/format.ts +1 -1
  221. package/src/report/react/index.tsx +21 -6
  222. package/src/report/react/render.test.tsx +180 -31
  223. package/src/report/react/styles.css +432 -213
  224. package/src/report/report.test.ts +398 -94
  225. package/src/report/report.ts +29 -11
  226. package/src/report/text/faces.ts +344 -96
  227. package/src/report/text/layout.ts +76 -11
  228. package/src/report/text/table.ts +98 -0
  229. package/src/report/tree.ts +103 -20
  230. package/src/report/types.ts +184 -25
  231. package/src/report/web.ts +46 -19
  232. package/src/results/annotated-source.test.ts +146 -0
  233. package/src/results/annotated-source.ts +128 -0
  234. package/src/results/attempt-evidence.test.ts +158 -0
  235. package/src/results/attempt-evidence.ts +158 -0
  236. package/src/results/attempt-source.ts +48 -0
  237. package/src/results/copy.ts +154 -139
  238. package/src/results/format.ts +54 -47
  239. package/src/results/host-equivalence.test.ts +527 -0
  240. package/src/results/index.ts +56 -14
  241. package/src/results/locator.test.ts +198 -0
  242. package/src/results/locator.ts +0 -0
  243. package/src/results/open.ts +392 -175
  244. package/src/results/results.test.ts +943 -494
  245. package/src/results/select.ts +129 -23
  246. package/src/results/skipped-notice.ts +0 -0
  247. package/src/results/source-hash.ts +28 -0
  248. package/src/results/types.ts +98 -62
  249. package/src/results/writer.ts +304 -176
  250. package/src/runner/attempt.test.ts +170 -0
  251. package/src/runner/attempt.ts +48 -13
  252. package/src/runner/discover.test.ts +65 -0
  253. package/src/runner/discover.ts +6 -2
  254. package/src/runner/eval-source.test.ts +121 -0
  255. package/src/runner/eval-source.ts +45 -0
  256. package/src/runner/fingerprint.ts +54 -1
  257. package/src/runner/remote-sandbox.ts +1 -1
  258. package/src/runner/report.test.ts +6 -8
  259. package/src/runner/report.ts +5 -4
  260. package/src/runner/reporters/artifacts.ts +27 -41
  261. package/src/runner/reporters/braintrust.test.ts +3 -3
  262. package/src/runner/reporters/braintrust.ts +1 -1
  263. package/src/runner/reporters/console.ts +6 -6
  264. package/src/runner/reporters/json.ts +4 -4
  265. package/src/runner/reporters/live.test.ts +56 -0
  266. package/src/runner/reporters/live.ts +36 -10
  267. package/src/runner/reporters/quiet.test.ts +66 -0
  268. package/src/runner/reporters/quiet.ts +49 -0
  269. package/src/runner/reporters/shared.ts +5 -5
  270. package/src/runner/reporters/table.ts +17 -17
  271. package/src/runner/run.ts +61 -102
  272. package/src/runner/types.ts +43 -32
  273. package/src/sandbox/checkpoint.ts +1 -1
  274. package/src/sandbox/docker.ts +15 -2
  275. package/src/sandbox/e2b.ts +11 -5
  276. package/src/sandbox/errors.ts +13 -0
  277. package/src/sandbox/index.ts +3 -3
  278. package/src/sandbox/local-files.ts +1 -1
  279. package/src/sandbox/registry.ts +6 -5
  280. package/src/sandbox/resolve.ts +38 -23
  281. package/src/sandbox/retry.test.ts +68 -0
  282. package/src/sandbox/retry.ts +44 -0
  283. package/src/sandbox/types.ts +24 -24
  284. package/src/sandbox/vercel.ts +17 -3
  285. package/src/scoring/collector.ts +3 -3
  286. package/src/scoring/judge.ts +2 -2
  287. package/src/scoring/scoped.ts +13 -1
  288. package/src/scoring/types.ts +7 -7
  289. package/src/scoring/verdict.ts +4 -4
  290. package/src/shared/aggregate.ts +23 -3
  291. package/src/shared/types.ts +2 -2
  292. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  293. package/src/show/compose.ts +12 -132
  294. package/src/show/index.ts +119 -95
  295. package/src/show/render.ts +462 -159
  296. package/src/show/show.test.ts +471 -253
  297. package/src/source-loc.ts +1 -1
  298. package/src/tty-line.ts +32 -0
  299. package/src/types.ts +2 -2
  300. package/src/util.test.ts +21 -1
  301. package/src/util.ts +5 -1
  302. package/src/view/app/App.tsx +33 -198
  303. package/src/view/app/components/AttemptModal.tsx +6 -6
  304. package/src/view/app/components/CodeView.tsx +2 -2
  305. package/src/view/app/components/CopyControls.tsx +32 -36
  306. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  307. package/src/view/app/components/ui/badge.tsx +1 -1
  308. package/src/view/app/components/ui/dialog.tsx +2 -2
  309. package/src/view/app/i18n.ts +40 -31
  310. package/src/view/app/lib/artifact-url.ts +1 -1
  311. package/src/view/app/lib/attempt-route.test.ts +37 -47
  312. package/src/view/app/lib/attempt-route.ts +26 -26
  313. package/src/view/app/lib/rows.ts +9 -112
  314. package/src/view/app/lib/verdict.ts +25 -0
  315. package/src/view/app/main.tsx +9 -12
  316. package/src/view/app/pages/RunsPage.tsx +4 -4
  317. package/src/view/app/pages/TracesPage.tsx +3 -3
  318. package/src/view/app/shared.ts +0 -1
  319. package/src/view/app/types.ts +6 -51
  320. package/src/view/artifact-serving.test.ts +136 -0
  321. package/src/view/client-dist/app.css +1 -1
  322. package/src/view/client-dist/app.js +21 -21
  323. package/src/view/data.test.ts +214 -143
  324. package/src/view/data.ts +166 -120
  325. package/src/view/index.ts +23 -11
  326. package/src/view/server.ts +52 -17
  327. package/src/view/shared/types.ts +22 -31
  328. package/src/view/styles.css +117 -327
  329. package/src/view/view-report.test.ts +146 -98
  330. package/src/report/default-report.tsx +0 -222
  331. package/src/report/dual-face.test.tsx +0 -527
  332. package/src/report/react/CaseList.tsx +0 -70
  333. package/src/view/app/components/CostScoreChart.tsx +0 -147
  334. package/src/view/app/components/ExperimentTable.tsx +0 -284
  335. package/src/view/app/components/GroupSelector.tsx +0 -61
  336. package/src/view/app/components/primitives.tsx +0 -43
  337. package/src/view/app/lib/outcome.ts +0 -82
@@ -1,26 +1,28 @@
1
1
  // defineReport:一份报告 = 一个报告文件(默认导出),两扇门(show / view)共用。
2
- // 宿主打开结果目录、按官方口径挑好选集、注入上下文;报告函数折数据、摆积木,
2
+ // 宿主打开结果目录、按官方口径挑好 Selection、注入上下文;报告函数折数据、摆积木,
3
3
  // 返回一棵组件树。计算全部发生在报告函数体里(读句柄、await 折数据只在这里合法);
4
- // 渲染面是纯同步函数 —— 可达百 MB 的工件永远不进渲染路径。
4
+ // 渲染面是纯同步函数 —— 可达百 MB 的 artifact 永远不进渲染路径。
5
5
  //
6
6
  // renderReportToText 是 text 宿主(show)的装载入口;web 宿主(view)的
7
7
  // renderReportToStaticHtml 在 ./web.ts(那一侧才 import react-dom)。宿主接线是下一波,
8
8
  // 这两个入口先以内部函数的身份可独立测试。
9
9
 
10
- import type { Results, Selection } from "../results/index.ts";
10
+ import type { Results, Selection, SelectionWarning } from "../results/types.ts";
11
11
  import {
12
12
  createTextContext,
13
13
  renderNodeToText,
14
+ resolveReportTree,
14
15
  validateReportTree,
15
16
  type ReportNode,
16
17
  type TextRenderOptions,
17
18
  } from "./tree.ts";
18
- import { prepareDefaultReportData, runWithDefaultReportData } from "./default-report.tsx";
19
+ import type { ReportLocale } from "./locale.ts";
19
20
 
20
21
  export interface ReportContext {
21
- /** results.latest() 挑好的选集:现刻水位快照 + 结构化挑选警告,同默认报告口径。 */
22
+ /** 宿主按现刻水位规则挑好的 Selection:每个 experiment × eval 取跨快照合成的最新判定,外加结构化挑选
23
+ 警告;show 的默认索引、view 默认报告与两者的 --report 使用同一选择口径。 */
22
24
  selection: Selection;
23
- /** 默认挑法不合口径时,全量数据自己挑(见 docs/results-lib.md)。 */
25
+ /** 默认挑法不合口径时,全量数据自己挑(见 docs/feature/results/library.md)。 */
24
26
  results: Results;
25
27
  }
26
28
 
@@ -53,8 +55,18 @@ export function isReportDefinition(value: unknown): value is ReportDefinition {
53
55
  }
54
56
 
55
57
  /**
56
- * text 宿主的装载语义:build 渲染前树校验 备好官方水位(DefaultReport 的数据)
57
- * 遍历渲染 text 面。不需要 react-dom。
58
+ * 挑选警告的 text 形态:每条渲染好的 message 前缀 "! ",一行一条( RunOverview /
59
+ * overviewText warnings "! <message>" 约定一致)。宿主级前置块——不依赖报告是否
60
+ * 摆了 RunOverview,裸跑 / --report 都在报告顶上如实报残缺,不静默。
61
+ */
62
+ function renderSelectionWarningsText(warnings: SelectionWarning[], _locale: ReportLocale): string {
63
+ return warnings.map((w) => `! ${w.message}`).join("\n");
64
+ }
65
+
66
+ /**
67
+ * text 宿主的装载语义:build → 渲染前解析数据组件(唯一的 await 边界)→ 树校验 → 遍历渲染
68
+ * text 面;Selection 有挑选警告时在报告顶部前置一块 "! <message>";报告树里的 RunOverview
69
+ * 已经渲染同一条时不重复。不需要 react-dom。
58
70
  */
59
71
  export async function renderReportToText(
60
72
  definition: ReportDefinition,
@@ -62,8 +74,14 @@ export async function renderReportToText(
62
74
  options?: TextRenderOptions,
63
75
  ): Promise<string> {
64
76
  const node = await definition.build(ctx);
65
- validateReportTree(node);
66
- const defaultData = await prepareDefaultReportData(ctx.selection);
77
+ const resolved = await resolveReportTree(node);
78
+ validateReportTree(resolved);
67
79
  const textCtx = createTextContext(options);
68
- return runWithDefaultReportData(defaultData, () => renderNodeToText(node, textCtx));
80
+ const body = renderNodeToText(resolved, textCtx);
81
+ // RunOverview can render the same Selection warnings as part of the user tree. Keep the
82
+ // host guarantee for reports that omit it, while never printing an identical warning twice.
83
+ const missingWarnings = ctx.selection.warnings.filter((warning) => !body.includes(`! ${warning.message}`));
84
+ return missingWarnings.length > 0
85
+ ? [renderSelectionWarningsText(missingWarnings, textCtx.locale), body].join("\n\n")
86
+ : body;
69
87
  }