niceeval 0.6.2 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/INDEX.md +23 -23
  2. package/README.zh.md +6 -6
  3. package/dist/agents/types.d.ts +2 -2
  4. package/dist/i18n/zh-CN.d.ts +3 -3
  5. package/dist/report/aggregate.d.ts +32 -26
  6. package/dist/report/aggregate.js +157 -76
  7. package/dist/report/built-in/index.d.ts +2 -0
  8. package/dist/report/built-in/index.js +8 -0
  9. package/dist/report/components.d.ts +91 -164
  10. package/dist/report/components.js +377 -114
  11. package/dist/report/compute.d.ts +86 -73
  12. package/dist/report/compute.js +592 -432
  13. package/dist/report/flag.d.ts +28 -17
  14. package/dist/report/flag.js +86 -16
  15. package/dist/report/format.d.ts +11 -11
  16. package/dist/report/format.js +17 -15
  17. package/dist/report/index.d.ts +16 -17
  18. package/dist/report/index.js +20 -22
  19. package/dist/report/load.js +3 -2
  20. package/dist/report/locale.d.ts +49 -34
  21. package/dist/report/locale.js +106 -58
  22. package/dist/report/metrics.d.ts +10 -3
  23. package/dist/report/metrics.js +46 -12
  24. package/dist/report/primitives.d.ts +42 -15
  25. package/dist/report/primitives.js +135 -26
  26. package/dist/report/react/AttemptList.d.ts +10 -8
  27. package/dist/report/react/AttemptList.js +18 -10
  28. package/dist/report/react/DeltaTable.js +19 -18
  29. package/dist/report/react/EvalList.d.ts +3 -3
  30. package/dist/report/react/EvalList.js +0 -0
  31. package/dist/report/react/ExperimentComparison.d.ts +4 -2
  32. package/dist/report/react/ExperimentComparison.js +5 -4
  33. package/dist/report/react/ExperimentList.d.ts +3 -3
  34. package/dist/report/react/ExperimentList.js +16 -15
  35. package/dist/report/react/MetricBars.js +5 -4
  36. package/dist/report/react/MetricLine.js +12 -5
  37. package/dist/report/react/MetricMatrix.js +1 -1
  38. package/dist/report/react/MetricScatter.js +54 -17
  39. package/dist/report/react/MetricTable.js +2 -12
  40. package/dist/report/react/ScopeSummary.d.ts +10 -0
  41. package/dist/report/react/ScopeSummary.js +17 -0
  42. package/dist/report/react/Scoreboard.js +6 -6
  43. package/dist/report/react/cell.js +2 -2
  44. package/dist/report/react/fixtures.d.ts +5 -9
  45. package/dist/report/react/fixtures.js +105 -149
  46. package/dist/report/react/index.d.ts +15 -5
  47. package/dist/report/react/index.js +18 -7
  48. package/dist/report/report.d.ts +137 -20
  49. package/dist/report/report.js +261 -34
  50. package/dist/report/text/faces.d.ts +17 -19
  51. package/dist/report/text/faces.js +225 -157
  52. package/dist/report/text/plot.js +1 -1
  53. package/dist/report/text/table.js +2 -2
  54. package/dist/report/tree.d.ts +90 -40
  55. package/dist/report/tree.js +252 -94
  56. package/dist/report/types.d.ts +245 -300
  57. package/dist/report/types.js +4 -3
  58. package/dist/report/web.d.ts +21 -5
  59. package/dist/report/web.js +42 -16
  60. package/dist/results/select.d.ts +38 -16
  61. package/dist/results/select.js +73 -25
  62. package/dist/results/types.d.ts +38 -14
  63. package/dist/shared/aggregate.d.ts +3 -2
  64. package/dist/shared/aggregate.js +5 -4
  65. package/docs-site/zh/README.md +44 -0
  66. package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
  67. package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
  68. package/docs-site/zh/examples/index.mdx +50 -0
  69. package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
  70. package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
  71. package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
  72. package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
  73. package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
  74. package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
  75. package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
  76. package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
  77. package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
  78. package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
  79. package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
  80. package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
  81. package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
  82. package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
  83. package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
  84. package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
  85. package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
  86. package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
  87. package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
  88. package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
  89. package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
  90. package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
  91. package/docs-site/zh/index.mdx +23 -25
  92. package/docs-site/zh/introduction.mdx +8 -8
  93. package/docs-site/zh/reference/builtin-agents.mdx +5 -5
  94. package/docs-site/zh/reference/capabilities.mdx +6 -6
  95. package/docs-site/zh/reference/cli.mdx +9 -7
  96. package/docs-site/zh/reference/define-agent.mdx +1 -1
  97. package/docs-site/zh/reference/events.mdx +3 -3
  98. package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
  99. package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
  100. package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
  101. package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
  102. package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
  103. package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
  104. package/package.json +6 -2
  105. package/src/agents/index.ts +2 -2
  106. package/src/agents/openai-compat.ts +1 -1
  107. package/src/agents/streaming.ts +2 -2
  108. package/src/agents/types.ts +3 -3
  109. package/src/cli.ts +42 -23
  110. package/src/context/context.ts +1 -1
  111. package/src/context/session.test.ts +1 -1
  112. package/src/context/session.ts +1 -1
  113. package/src/i18n/en.ts +18 -16
  114. package/src/i18n/zh-CN.ts +16 -15
  115. package/src/report/aggregate.ts +175 -87
  116. package/src/report/built-in/index.tsx +9 -0
  117. package/src/report/components.tsx +625 -285
  118. package/src/report/compute.ts +717 -515
  119. package/src/report/dual-render.test.tsx +738 -1148
  120. package/src/report/flag.ts +97 -33
  121. package/src/report/format.ts +18 -22
  122. package/src/report/index.ts +113 -58
  123. package/src/report/load.ts +3 -2
  124. package/src/report/locale.ts +120 -69
  125. package/src/report/metrics.ts +42 -12
  126. package/src/report/primitives.tsx +190 -45
  127. package/src/report/react/AttemptList.tsx +32 -20
  128. package/src/report/react/DeltaTable.tsx +63 -45
  129. package/src/report/react/EvalList.tsx +0 -0
  130. package/src/report/react/ExperimentComparison.tsx +12 -7
  131. package/src/report/react/ExperimentList.tsx +38 -26
  132. package/src/report/react/MetricBars.tsx +5 -4
  133. package/src/report/react/MetricLine.tsx +13 -8
  134. package/src/report/react/MetricMatrix.tsx +2 -2
  135. package/src/report/react/MetricScatter.tsx +74 -20
  136. package/src/report/react/MetricTable.tsx +4 -76
  137. package/src/report/react/ScopeSummary.tsx +86 -0
  138. package/src/report/react/Scoreboard.tsx +28 -10
  139. package/src/report/react/cell.tsx +2 -2
  140. package/src/report/react/enhance.js +57 -5
  141. package/src/report/react/fixtures.ts +109 -156
  142. package/src/report/react/index.tsx +24 -39
  143. package/src/report/react/render.test.tsx +139 -104
  144. package/src/report/react/styles.css +181 -91
  145. package/src/report/report.test.ts +761 -1031
  146. package/src/report/report.ts +425 -47
  147. package/src/report/text/faces.ts +257 -164
  148. package/src/report/text/plot.ts +1 -1
  149. package/src/report/text/table.ts +2 -2
  150. package/src/report/tree.ts +362 -104
  151. package/src/report/types.ts +257 -287
  152. package/src/report/web.ts +63 -20
  153. package/src/results/attempt-evidence.test.ts +4 -4
  154. package/src/results/attempt-evidence.ts +5 -5
  155. package/src/results/copy.ts +6 -6
  156. package/src/results/host-equivalence.test.ts +26 -14
  157. package/src/results/index.ts +10 -4
  158. package/src/results/open.ts +8 -4
  159. package/src/results/results.test.ts +4 -3
  160. package/src/results/select.ts +104 -34
  161. package/src/results/types.ts +36 -14
  162. package/src/runner/feedback/human.test.ts +1 -1
  163. package/src/runner/run.ts +1 -1
  164. package/src/sandbox/cli-commands.ts +2 -2
  165. package/src/scoring/judge.test.ts +1 -1
  166. package/src/shared/aggregate.ts +5 -4
  167. package/src/show/compose.ts +50 -67
  168. package/src/show/index.ts +107 -56
  169. package/src/show/render.ts +43 -27
  170. package/src/show/report-host.test.ts +188 -0
  171. package/src/show/report-host.ts +375 -0
  172. package/src/show/show.test.ts +86 -36
  173. package/src/view/app/App.test.tsx +69 -0
  174. package/src/view/app/App.tsx +144 -48
  175. package/src/view/app/components/AttemptModal.tsx +324 -63
  176. package/src/view/app/components/CodeView.tsx +10 -4
  177. package/src/view/app/i18n.ts +31 -17
  178. package/src/view/app/lib/artifact-url.ts +14 -3
  179. package/src/view/app/main.tsx +13 -8
  180. package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
  181. package/src/view/app/types.ts +4 -1
  182. package/src/view/artifact-serving.test.ts +21 -1
  183. package/src/view/client-dist/app.css +1 -1
  184. package/src/view/client-dist/app.js +6 -6
  185. package/src/view/data.test.ts +9 -3
  186. package/src/view/data.ts +145 -49
  187. package/src/view/index.ts +48 -44
  188. package/src/view/server.ts +35 -15
  189. package/src/view/shared/types.ts +34 -5
  190. package/src/view/styles.css +224 -0
  191. package/src/view/view-report.test.ts +161 -57
  192. package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
  193. package/dist/report/built-ins/experiment-comparison.js +0 -119
  194. package/dist/report/built-ins/index.d.ts +0 -2
  195. package/dist/report/built-ins/index.js +0 -2
  196. package/dist/report/react/GroupSummary.d.ts +0 -8
  197. package/dist/report/react/GroupSummary.js +0 -8
  198. package/dist/report/react/RunOverview.d.ts +0 -8
  199. package/dist/report/react/RunOverview.js +0 -12
  200. package/docs-site/zh/example/ai-agent-application.mdx +0 -152
  201. package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
  202. package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
  203. package/docs-site/zh/example/showcase.mdx +0 -39
  204. package/src/report/built-in-user-parity.test.tsx +0 -597
  205. package/src/report/built-ins/experiment-comparison.tsx +0 -179
  206. package/src/report/built-ins/index.ts +0 -7
  207. package/src/report/react/GroupSummary.tsx +0 -66
  208. package/src/report/react/RunOverview.tsx +0 -109
  209. /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
  210. /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
  211. /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
  212. /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
  213. /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
  214. /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
  215. /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
  216. /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
  217. /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
  218. /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
@@ -1,10 +1,11 @@
1
1
  // Scoreboard:考试成绩单——总分 + 分科小计。
2
- // 固定分母的口径不藏:没跑的题挣 0 分但留在分母里,科目行如实报 missing;
3
- // weights 是「实际生效的权重表」,渲染出来让成绩单可审计(docs/feature/reports/library.md「Scoreboard」)
2
+ // 固定分母的口径不藏:没跑的题挣 0 分但留在分母里(notRun),跑了但测不了的题同样按 0 分
3
+ // (unscorable),两个计数分开显示,连同 refs 一起,不合并成一个笼统的缺失数;
4
+ // weights 是「实际生效的权重表」,渲染出来让成绩单可审计;题集之外被忽略的 eval 数在注脚。
4
5
 
5
6
  import type { ReactElement } from "react";
6
7
  import type { ScoreboardData } from "../types.ts";
7
- import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "../locale.ts";
8
+ import { DEFAULT_REPORT_LOCALE, countText, localeText, resolveLocalizedText, type ReportLocale } from "../locale.ts";
8
9
  import { colorClassForKey } from "./colors.ts";
9
10
  import { cx } from "./format.ts";
10
11
 
@@ -31,7 +32,7 @@ export function Scoreboard({
31
32
  <thead>
32
33
  <tr>
33
34
  <th scope="col" className="nre-dimension">
34
- {data.dimension}
35
+ {data.rowDimension}
35
36
  </th>
36
37
  <th scope="col" className="nre-total-col">
37
38
  {localeText(locale, "scoreboard.total")}
@@ -51,7 +52,16 @@ export function Scoreboard({
51
52
  <th scope="row" className={cx("nre-row-key", "nre-key", colorClassForKey(row.key))}>
52
53
  {row.key}
53
54
  </th>
54
- <td className="nre-total">{row.total.display}</td>
55
+ <td className="nre-total">
56
+ {resolveLocalizedText(row.total.display, locale)}
57
+ {/* 两种 0 分分开注脚:没去考(notRun)与考了判不了(unscorable) */}
58
+ {row.total.notRun > 0 && (
59
+ <span className="nre-total-not-run">{countText(locale, "scoreboard.notRun", row.total.notRun)}</span>
60
+ )}
61
+ {row.total.unscorable > 0 && (
62
+ <span className="nre-total-unscorable">{countText(locale, "scoreboard.unscorable", row.total.unscorable)}</span>
63
+ )}
64
+ </td>
55
65
  {subjectKeys.map((key) => {
56
66
  const subject = row.subjects.find((s) => s.key === key);
57
67
  if (!subject) return <td key={key} className="nre-td-empty" />;
@@ -60,16 +70,20 @@ export function Scoreboard({
60
70
  <span
61
71
  className="nre-subject-score"
62
72
  title={localeText(locale, "scoreboard.subjectTitle", {
63
- evals: subject.evals,
73
+ questions: subject.questions,
64
74
  earned: subject.earned,
65
75
  possible: subject.possible,
66
76
  })}
67
77
  >
68
- {subject.earned}/{subject.possible}
78
+ {resolveLocalizedText(subject.display, locale)}
69
79
  </span>
70
- {/* 固定分母的如实注脚:没跑、按 0 计的题数 */}
71
- {subject.missing > 0 && (
72
- <span className="nre-subject-missing">{countText(locale, "scoreboard.missing", subject.missing)}</span>
80
+ {subject.notRun > 0 && (
81
+ <span className="nre-subject-not-run">{localeText(locale, "scoreboard.notRunText", { n: subject.notRun })}</span>
82
+ )}
83
+ {subject.unscorable > 0 && (
84
+ <span className="nre-subject-unscorable">
85
+ {localeText(locale, "scoreboard.unscorableText", { n: subject.unscorable })}
86
+ </span>
73
87
  )}
74
88
  </td>
75
89
  );
@@ -90,6 +104,10 @@ export function Scoreboard({
90
104
  ))}
91
105
  {data.weights.length > 0 && <span className="nre-weight-rest">{localeText(locale, "scoreboard.othersWeight")}</span>}
92
106
  </p>
107
+ {/* 题集之外的 eval:忽略但如实报数 */}
108
+ {data.ignoredEvals > 0 && (
109
+ <p className="nre-scoreboard-ignored">{countText(locale, "scoreboard.ignored", data.ignoredEvals)}</p>
110
+ )}
93
111
  </section>
94
112
  );
95
113
  }
@@ -5,7 +5,7 @@
5
5
  import type { ReactElement } from "react";
6
6
  import type { MetricCell } from "../types.ts";
7
7
  import type { AttemptLocator } from "../../results/locator.ts";
8
- import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
8
+ import { DEFAULT_REPORT_LOCALE, localeText, resolveLocalizedText, type ReportLocale } from "../locale.ts";
9
9
 
10
10
  export function MetricCellView({
11
11
  cell,
@@ -32,7 +32,7 @@ export function MetricCellView({
32
32
  className="nre-value"
33
33
  title={localeText(locale, "cell.measuredTitle", { samples: cell.samples, total: cell.total })}
34
34
  >
35
- {cell.display}
35
+ {resolveLocalizedText(cell.display, locale)}
36
36
  </span>
37
37
  {/* samples < total:有 attempt 测不了这个指标,覆盖率角标如实标出 */}
38
38
  {cell.samples < cell.total && (
@@ -1,15 +1,20 @@
1
1
  // niceeval 报告的渐进增强 runtime:纯 vanilla JS、零依赖、IIFE、幂等。
2
- // 只作用于 .nre DOM 与 data-nre-* 属性;四个行为——实验组切换、表格排序、行过滤、SVG 点 tooltip。
3
- // 静态 HTML JS 时内容完整可读是硬约束:排序有数据侧预排、tooltip 退化为原生
4
- // <title>、过滤输入框静默无功能。全部经 document 级事件委托绑定,重复注入本文件
5
- // 只在首次生效(window.__nreEnhanced 守卫),DOM 被搬动(如 view 把 <template> 内容
6
- // 摆进报告槽)也无需重新绑定。
2
+ // 只作用于 .nre DOM 与 data-nre-* 属性;六个行为——实验组切换、Tabs 单选切换、表格排序、
3
+ // 行过滤、SVG tooltip、警告命令复制。全部只改浏览状态,不改数据、指标口径或初始 HTML 数值。
4
+ // 静态 HTML 无 JS 时内容完整可读是硬约束:排序有数据侧预排、tooltip 退化为原生 <title>、
5
+ // 过滤输入框静默无功能、Tabs 退化为原生 <details> 手风琴、命令块退化为点击全选。
6
+ // 全部经 document 级事件委托绑定,重复注入本文件只在首次生效(window.__nreEnhanced 守卫),
7
+ // DOM 被搬动(如 view 把 <template> 内容摆进报告槽)也无需重新绑定。
7
8
 
8
9
  (function () {
9
10
  "use strict";
10
11
  if (typeof window === "undefined" || window.__nreEnhanced) return;
11
12
  window.__nreEnhanced = true;
12
13
 
14
+ // 根类 nre-js:styles.css 用它把仅增强态的布局(单选 tab 条、复制指针)限定在 JS 在场时。
15
+ // 挂在 documentElement 上与报告块位置无关,块之后被搬进槽位也不需要补标记。
16
+ document.documentElement.classList.add("nre-js");
17
+
13
18
  function closest(target, selector) {
14
19
  return target && target.closest ? target.closest(selector) : null;
15
20
  }
@@ -46,6 +51,53 @@
46
51
  selectExperimentGroup(control);
47
52
  });
48
53
 
54
+ // ───────────────────────── Tabs:[data-nre-tabs] 单选切换 ─────────────────────────
55
+ // 静态 HTML 每 tab 一个 <details> 且仅首个 open;点击 summary 时接管原生 toggle:
56
+ // 打开所点 tab、收起同组其余,点已开的 tab 保持打开(单选语义,恒有一个面板可见)。
57
+ // 只切换 open 状态,不触碰 tab 内任何数据;键盘 Enter/Space 走 summary 的原生激活(即 click)。
58
+
59
+ document.addEventListener("click", function (e) {
60
+ var title = closest(e.target, "[data-nre-tabs] > details > summary");
61
+ if (!title) return;
62
+ e.preventDefault();
63
+ var tab = title.parentNode;
64
+ var group = tab.parentNode;
65
+ for (var i = 0; i < group.children.length; i++) {
66
+ var child = group.children[i];
67
+ if (child.tagName === "DETAILS") child.open = child === tab;
68
+ }
69
+ });
70
+
71
+ // ───────────────────────── 复制:[data-nre-copy](宿主警告块的命令) ─────────────────────────
72
+ // 点击把 data-nre-copy 携带的完整命令写进剪贴板,成功后短暂打上 data-nre-copied
73
+ // (styles.css 显示 ✓);剪贴板不可用时退化为全选该块文本,用户手动复制。
74
+ // 块内文本与属性值恒不变,复制的是数据侧已写好的命令原文。
75
+
76
+ document.addEventListener("click", function (e) {
77
+ var block = closest(e.target, "[data-nre-copy]");
78
+ if (!block) return;
79
+ var command = block.getAttribute("data-nre-copy") || "";
80
+ function mark() {
81
+ block.setAttribute("data-nre-copied", "");
82
+ setTimeout(function () {
83
+ block.removeAttribute("data-nre-copied");
84
+ }, 1500);
85
+ }
86
+ function selectFallback() {
87
+ var selection = window.getSelection();
88
+ if (!selection) return;
89
+ var range = document.createRange();
90
+ range.selectNodeContents(block);
91
+ selection.removeAllRanges();
92
+ selection.addRange(range);
93
+ }
94
+ if (navigator.clipboard && navigator.clipboard.writeText) {
95
+ navigator.clipboard.writeText(command).then(mark, selectFallback);
96
+ } else {
97
+ selectFallback();
98
+ }
99
+ });
100
+
49
101
  // ───────────────────────── 排序:th[data-nre-sort] ─────────────────────────
50
102
  // 点击按该列排序 tbody 行(td/th 的 data-sort-value,数值优先、退回字符串;
51
103
  // 空值恒沉底),再点反向;方向指示由 th 上的 nre-sort-asc / nre-sort-desc 类驱动。
@@ -1,4 +1,4 @@
1
- // 手工 fixture:模拟计算函数(MetricTable.data / MetricMatrix.data / …)的产物。
1
+ // 手工 fixture:模拟计算函数(metricTableData / metricMatrixData / …)的产物。
2
2
  // 仅供渲染测试与 scripts/report-react-demo.tsx 使用,不从入口导出。
3
3
  // 数字刻意覆盖诚实细节:coverage 角标(samples<total)、全 null 格子、
4
4
  // 稀疏矩阵、缺数据的散点、delta 的 null 不硬算、truncated 计数。
@@ -8,12 +8,11 @@ import type {
8
8
  DeltaData,
9
9
  EvalListItem,
10
10
  ExperimentListItem,
11
- GroupSummaryData,
12
11
  LineData,
13
12
  MatrixData,
14
13
  MetricColumn,
15
- OverviewData,
16
14
  ScatterData,
15
+ ScopeSummaryData,
17
16
  ScoreboardData,
18
17
  TableData,
19
18
  } from "../types.ts";
@@ -25,56 +24,21 @@ export const passRateColumn: MetricColumn = { key: "task-pass-rate", label: "pas
25
24
  export const codeLinesColumn: MetricColumn = { key: "code-lines", label: "code lines", unit: "lines", better: "lower" };
26
25
  export const costColumn: MetricColumn = { key: "cost", label: "cost", unit: "$", better: "lower" };
27
26
 
28
- export const overviewData: OverviewData = {
29
- snapshots: [
30
- { experimentId: "compare/bub", agent: "bub", model: "gpt-5.4", startedAt: "2026-07-01T10:00:00Z" },
31
- { experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T11:30:00Z" },
32
- ],
33
- totals: {
34
- evals: 12,
35
- attempts: 48,
36
- passed: 36,
37
- failed: 8,
38
- errored: 2,
39
- skipped: 2,
40
- // 两级聚合口径(computeCell)刻意不等于 36/(36+8+2)≈78% 的 attempt 原始占比:
41
- // 组件必须原样渲染这个字段,不得从上面四个 verdict 计票现场重算。
42
- passRate: { value: 0.7, display: "70%", samples: 46, total: 48, refs: [] },
43
- costUSD: null, // 全部 attempt 都没报成本:null,组件必须显示缺数据而不是 $0
44
- durationMs: 261_000,
45
- },
46
- warnings: [
47
- {
48
- kind: "partial-coverage",
49
- experimentId: "compare/bub",
50
- covered: 9,
51
- total: 12,
52
- message: "snapshot covers 9 of 12 evals seen in history; re-run `niceeval exp compare/bub` for a full snapshot",
53
- },
54
- ],
55
- };
56
-
57
- export const overviewWithCost: OverviewData = {
58
- ...overviewData,
59
- totals: { ...overviewData.totals, costUSD: 1.234 },
60
- warnings: [],
61
- };
62
-
63
- /** GroupSummary.data 的产物形态:eval 级折叠计票 + 旧 GroupSelector 口径的通过率(包成 MetricCell)。 */
64
- export const groupSummaryData: GroupSummaryData = {
27
+ /** scopeSummaryData 的产物形态:两级计票恒随行,成功率与总成本是官方 MetricCell。 */
28
+ export const scopeSummaryData: ScopeSummaryData = {
29
+ range: { earliestStartedAt: "2026-07-01T10:00:00Z", latestStartedAt: "2026-07-01T11:30:00Z" },
65
30
  experiments: 2,
66
- // evals = 全部 verdicts 之和(3+1+1+1=6);passRate 分母只数非 skipped 的 5 道
67
31
  evals: 6,
68
32
  attempts: 9,
69
- verdicts: { passed: 3, failed: 1, errored: 1, skipped: 1 },
70
- // 3 passed / (3 + 1 + 1) = 60%;samples=ran=5 < total=evals=6:1 道 skipped 未计入分母的覆盖率角标
71
- passRate: { value: 0.6, display: "60%", samples: 5, total: 6, refs: [] },
72
- totalCostUSD: 1.5,
73
- lastRunAt: "2026-07-01T11:30:00Z",
33
+ evalVerdicts: { passed: 3, failed: 1, errored: 1, skipped: 1 },
34
+ attemptVerdicts: { passed: 4, failed: 3, errored: 1, skipped: 1 },
35
+ // 两级聚合口径,刻意不等于任一计票的比例:组件必须原样渲染,不重算
36
+ endToEndPassRate: { value: 0.6, display: "60%", samples: 8, total: 9, refs: [] },
37
+ totalCostUSD: { value: 1.5, display: "$1.50", samples: 8, total: 9, refs: [] },
74
38
  };
75
39
 
76
- export const tableData: TableData<"task-pass-rate" | "code-lines"> = {
77
- dimension: "agent",
40
+ export const tableData: TableData = {
41
+ rowDimension: "agent",
78
42
  // 行顺序故意不按 passRate 排:组件必须按传入顺序渲染,不重排
79
43
  columns: [passRateColumn, codeLinesColumn],
80
44
  rows: [
@@ -94,7 +58,7 @@ export const tableData: TableData<"task-pass-rate" | "code-lines"> = {
94
58
  display: "87%",
95
59
  samples: 6,
96
60
  total: 6,
97
- refs: [locator("@1a0a0a0")],
61
+ refs: [locator("@1a0a0a0a")],
98
62
  },
99
63
  // samples < total:有 attempt 测不了 → 覆盖率角标 5/6
100
64
  "code-lines": { value: 120, display: "120 lines", samples: 5, total: 6, refs: [] },
@@ -103,34 +67,9 @@ export const tableData: TableData<"task-pass-rate" | "code-lines"> = {
103
67
  ],
104
68
  };
105
69
 
106
- /** rows: "experiment" 的榜单形态:行携带 agent/model 元信息与 eval 级折叠计票。 */
107
- export const tableDataWithMeta: TableData<"task-pass-rate"> = {
108
- dimension: "experiment",
109
- columns: [passRateColumn],
110
- rows: [
111
- {
112
- key: "compare/bub",
113
- cells: { "task-pass-rate": { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] } },
114
- meta: {
115
- agent: "bub",
116
- model: "gpt-5.4",
117
- verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
118
- },
119
- },
120
- {
121
- key: "compare/codex",
122
- cells: { "task-pass-rate": { value: 1, display: "100%", samples: 2, total: 2, refs: [] } },
123
- meta: {
124
- agent: "codex",
125
- verdicts: { passed: 2, failed: 0, errored: 0, skipped: 0 },
126
- },
127
- },
128
- ],
129
- };
130
-
131
70
  export const matrixData: MatrixData = {
132
- rows: "eval",
133
- columns: "agent",
71
+ rowDimension: "eval",
72
+ columnDimension: "agent",
134
73
  metric: passRateColumn,
135
74
  // 稀疏:geometry/angles × codex 没有样本,数据里不出现 → 格子空着
136
75
  cells: [
@@ -142,7 +81,7 @@ export const matrixData: MatrixData = {
142
81
  display: "100%",
143
82
  samples: 2,
144
83
  total: 2,
145
- refs: [locator("@1b3b3b3"), locator("@1b7b7b7")],
84
+ refs: [locator("@1b3b3b3b"), locator("@1b7b7b7b")],
146
85
  },
147
86
  },
148
87
  {
@@ -159,34 +98,41 @@ export const matrixData: MatrixData = {
159
98
  };
160
99
 
161
100
  export const scoreboardData: ScoreboardData = {
162
- dimension: "agent",
101
+ rowDimension: "agent",
102
+ questions: [
103
+ "algebra/quadratic",
104
+ "algebra/simple",
105
+ "geometry/angles",
106
+ "geometry/area",
107
+ ],
163
108
  fullMarks: 100,
164
109
  weights: [{ prefix: "algebra/", weight: 2 }],
110
+ ignoredEvals: 1,
165
111
  rows: [
166
112
  {
167
113
  key: "bub",
168
- total: { value: 78.5, display: "78.5" },
114
+ // notRun 1:一题没跑、按 0 计;unscorable 0 —— 两种 0 分分开计数
115
+ total: { value: 78.5, display: "78.5", notRun: 1, unscorable: 0, refs: [locator("@1a0a0a0a")] },
169
116
  subjects: [
170
- // missing 1:一题没跑、按 0 计——固定分母的如实注脚
171
- { key: "algebra", earned: 14, possible: 16, evals: 8, missing: 1 },
172
- { key: "geometry", earned: 3, possible: 4, evals: 4, missing: 0 },
117
+ { key: "algebra", earned: 14, possible: 16, questions: 8, notRun: 1, unscorable: 0, display: "14/16 (87.5%)", refs: [] },
118
+ { key: "geometry", earned: 3, possible: 4, questions: 4, notRun: 0, unscorable: 0, display: "3/4 (75%)", refs: [] },
173
119
  ],
174
120
  },
175
121
  {
176
122
  key: "codex",
177
- total: { value: 52, display: "52" },
123
+ total: { value: 52, display: "52", notRun: 0, unscorable: 2, refs: [] },
178
124
  subjects: [
179
- { key: "algebra", earned: 9, possible: 16, evals: 8, missing: 0 },
180
- { key: "geometry", earned: 1.4, possible: 4, evals: 4, missing: 2 },
125
+ { key: "algebra", earned: 9, possible: 16, questions: 8, notRun: 0, unscorable: 0, display: "9/16 (56.3%)", refs: [] },
126
+ { key: "geometry", earned: 1.4, possible: 4, questions: 4, notRun: 0, unscorable: 2, display: "1.4/4 (35%)", refs: [] },
181
127
  ],
182
128
  },
183
129
  ],
184
130
  };
185
131
 
186
132
  export const scatterData: ScatterData = {
187
- points: "experiment",
188
- series: "agent",
189
- x: costColumn, // better: "lower" → 轴反向,便宜的一端在右
133
+ pointDimension: "experiment",
134
+ seriesDimension: "agent",
135
+ x: costColumn, // better: "lower" → 轴反向(贵在左、便宜在右);两轴都有 better → 提示「越靠右上越好」
190
136
  y: passRateColumn,
191
137
  rows: [
192
138
  {
@@ -219,82 +165,88 @@ export const scatterData: ScatterData = {
219
165
 
220
166
  export const lineData: LineData = {
221
167
  x: { key: "latencyMs", label: "Simulated latency", unit: "ms" },
222
- series: "agents",
168
+ seriesDimension: "agents",
223
169
  y: passRateColumn,
224
170
  rows: [
225
171
  {
226
- key: "ultra/agents-1-lat-100",
172
+ key: "100",
227
173
  series: "1 agents",
228
174
  x: 100,
229
175
  xDisplay: "100ms",
230
176
  y: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
231
177
  },
232
178
  {
233
- key: "ultra/agents-1-lat-300",
179
+ key: "300",
234
180
  series: "1 agents",
235
181
  x: 300,
236
182
  xDisplay: "300ms",
237
183
  y: { value: 0.3, display: "30%", samples: 6, total: 6, refs: [] },
238
184
  },
239
185
  {
240
- key: "ultra/agents-16-lat-100",
186
+ key: "100",
241
187
  series: "16 agents",
242
188
  x: 100,
243
189
  xDisplay: "100ms",
244
190
  y: { value: 0.8, display: "80%", samples: 6, total: 6, refs: [] },
245
191
  },
246
192
  {
247
- key: "ultra/agents-16-lat-300",
193
+ key: "300",
248
194
  series: "16 agents",
249
195
  x: 300,
250
196
  xDisplay: "300ms",
251
197
  y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
252
198
  },
253
199
  {
254
- // 未声明 flag 的 experiment:作轴不画点,注脚报数
255
- key: "ultra/legacy",
200
+ // 未声明数值 flag 的 attempt:不伪造 x 值,注脚报数
201
+ key: "null",
256
202
  series: "1 agents",
257
203
  x: null,
258
- xDisplay: "",
204
+ xDisplay: "",
259
205
  y: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
260
206
  },
261
207
  ],
262
208
  };
263
209
 
264
- export const deltaData: DeltaData<"task-pass-rate" | "cost"> = {
210
+ export const deltaData: DeltaData = {
211
+ byDimension: "experiment",
265
212
  columns: [passRateColumn, costColumn],
266
213
  rows: [
267
214
  {
268
- key: "bub",
269
- a: { experimentId: "compare/bub" },
270
- b: { experimentId: "compare/bub--agents-md" },
215
+ key: "compare/bub → compare/bub--agents-md",
216
+ label: "bub",
217
+ a: { key: "compare/bub" },
218
+ b: { key: "compare/bub--agents-md" },
271
219
  cells: {
272
- // 通过率 +12pp:better higher → (绿)
220
+ // 通过率 +12pp:better higher → improved(绿)
273
221
  "task-pass-rate": {
274
222
  a: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
275
223
  b: { value: 0.62, display: "62%", samples: 6, total: 6, refs: [] },
276
224
  delta: 0.12,
277
225
  display: "+12pp",
226
+ outcome: "improved",
278
227
  },
279
- // 成本 +$0.15:better lower → (红)
228
+ // 成本 +$0.15:better lower → regressed(红)
280
229
  cost: {
281
230
  a: { value: 0.2, display: "$0.20", samples: 6, total: 6, refs: [] },
282
231
  b: { value: 0.35, display: "$0.35", samples: 6, total: 6, refs: [] },
283
232
  delta: 0.15,
284
233
  display: "+$0.15",
234
+ outcome: "regressed",
285
235
  },
286
236
  },
287
237
  },
288
238
  {
289
- key: "codex",
290
- a: { experimentId: "compare/codex" },
291
- b: { experimentId: "compare/codex--agents-md" },
239
+ key: "compare/codex → compare/codex--agents-md",
240
+ label: "codex",
241
+ a: { key: "compare/codex" },
242
+ b: { key: "compare/codex--agents-md" },
292
243
  cells: {
293
244
  "task-pass-rate": {
294
245
  a: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
295
246
  b: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
296
247
  delta: 0,
297
248
  display: "±0",
249
+ outcome: "unchanged",
298
250
  },
299
251
  // A 侧缺数据 → delta null:显示缺,不硬算
300
252
  cost: {
@@ -302,6 +254,7 @@ export const deltaData: DeltaData<"task-pass-rate" | "cost"> = {
302
254
  b: { value: 0.3, display: "$0.30", samples: 6, total: 6, refs: [] },
303
255
  delta: null,
304
256
  display: "—",
257
+ outcome: "unavailable",
305
258
  },
306
259
  },
307
260
  },
@@ -312,75 +265,75 @@ export const deltaData: DeltaData<"task-pass-rate" | "cost"> = {
312
265
 
313
266
  /** algebra/quadratic 在 compare/bub 上失败的那次 attempt——两条子失败夹具共用同一条。 */
314
267
  const failedAttempt: AttemptListItem = {
315
- evalId: "algebra/quadratic",
316
268
  experimentId: "compare/bub",
269
+ evalId: "algebra/quadratic",
317
270
  attempt: 3,
318
271
  agent: "bub",
319
272
  verdict: "failed",
320
- assertions: [
321
- {
322
- name: "roots-correct",
323
- severity: "gate",
324
- score: 0,
325
- outcome: "failed" as const,
326
- expected: "x=2",
327
- received: "x=3",
328
- evidence: "judge: sign flipped when substituting into the quadratic formula",
329
- },
330
- ],
273
+ // 已按 Scoring display 契约折好的单行摘要;渲染面只做宽度截断,不重算
274
+ failureSummary: "gate: roots-correct · expected x=2 · received x=3",
275
+ moreFailures: 1,
276
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1a4a4a4a")] },
331
277
  durationMs: 32_000,
332
278
  costUSD: 0.12,
333
- locator: locator("@1a4a4a4"),
279
+ locator: locator("@1a4a4a4a"),
334
280
  };
335
281
 
336
282
  const erroredAttempt: AttemptListItem = {
337
- evalId: "geometry/angles",
338
283
  experimentId: "compare/codex",
284
+ evalId: "geometry/angles",
339
285
  attempt: 0,
340
286
  agent: "codex",
341
287
  verdict: "errored",
342
- // 结构化 error:列表只渲染 message 一层摘要;cause/stack diagnostics 是下钻详情,随数据携带
343
- error: {
344
- code: "unexpected-error",
345
- message: "TypeError: cannot read properties of undefined (reading 'foo')",
346
- phase: "eval.run",
347
- stack: "TypeError: cannot read properties of undefined (reading 'foo')\n at run (adapter.ts:42:7)",
348
- cause: { name: "TypeError", message: "cannot read properties of undefined (reading 'foo')" },
349
- },
350
- diagnostics: [
351
- { code: "sandbox-teardown-failed", level: "warning", message: "sandbox teardown timed out", phase: "sandbox.teardown" },
352
- ],
353
- assertions: [],
288
+ // errored:结构化 error 的一层摘要(phase · code · message)
289
+ failureSummary: "eval.run · unexpected-error · TypeError: cannot read properties of undefined (reading 'foo')",
290
+ moreFailures: 0,
291
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1c1c1c1c")] },
354
292
  durationMs: 4_500,
355
- locator: locator("@1c1c1c1"),
293
+ costUSD: null,
294
+ locator: locator("@1c1c1c1c"),
356
295
  };
357
296
 
358
297
  const failedRetryAttempt: AttemptListItem = {
359
298
  ...failedAttempt,
360
299
  attempt: 4,
361
300
  durationMs: 35_000,
362
- locator: locator("@1b5b5b5"),
301
+ locator: locator("@1b5b5b5b"),
302
+ };
303
+
304
+ const passedAttempt: AttemptListItem = {
305
+ experimentId: "compare/bub",
306
+ evalId: "algebra/simple",
307
+ attempt: 0,
308
+ agent: "bub",
309
+ verdict: "passed",
310
+ failureSummary: null,
311
+ moreFailures: 0,
312
+ examScore: { value: 1, display: "100%", samples: 1, total: 1, refs: [locator("@1d2d2d2d")] },
313
+ durationMs: 5_000,
314
+ costUSD: 0.02,
315
+ locator: locator("@1d2d2d2d"),
363
316
  };
364
317
 
365
318
  export const attemptListItems: AttemptListItem[] = [failedAttempt, erroredAttempt];
366
319
 
367
320
  export const evalListItems: EvalListItem[] = [
368
321
  {
369
- evalId: "algebra/quadratic",
370
322
  experimentId: "compare/bub",
323
+ evalId: "algebra/quadratic",
371
324
  verdict: "failed",
372
- score: { value: 0, display: "0%", samples: 1, total: 1, refs: [failedAttempt.locator] },
373
- duration: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
374
- cost: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
325
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [failedAttempt.locator] },
326
+ durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
327
+ costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
375
328
  attempts: [failedAttempt],
376
329
  },
377
330
  {
378
- evalId: "geometry/angles",
379
331
  experimentId: "compare/codex",
332
+ evalId: "geometry/angles",
380
333
  verdict: "errored",
381
- score: { value: 0, display: "0%", samples: 1, total: 1, refs: [erroredAttempt.locator] },
382
- duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
383
- cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
334
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [erroredAttempt.locator] },
335
+ durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
336
+ costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
384
337
  attempts: [erroredAttempt],
385
338
  },
386
339
  ];
@@ -391,10 +344,10 @@ export const experimentListItems: ExperimentListItem[] = [
391
344
  agent: "bub",
392
345
  model: "gpt-5.4",
393
346
  flags: { memory: true },
394
- verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
395
- passRate: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
396
- cost: { value: 0.12, display: "$0.12", samples: 1, total: 2, refs: [failedAttempt.locator] },
397
- duration: { value: 32_000, display: "32.0s", samples: 2, total: 2, refs: [] },
347
+ evalVerdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
348
+ endToEndPassRate: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
349
+ costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 2, refs: [failedAttempt.locator] },
350
+ durationMs: { value: 32_000, display: "32.0s", samples: 2, total: 2, refs: [] },
398
351
  tokens: { value: null, display: "—", samples: 0, total: 2, refs: [] },
399
352
  evals: 2,
400
353
  attempts: 3,
@@ -403,26 +356,26 @@ export const experimentListItems: ExperimentListItem[] = [
403
356
  {
404
357
  evalId: "algebra/quadratic",
405
358
  verdict: "failed",
406
- duration: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
407
- cost: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
359
+ durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
360
+ costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
408
361
  attempts: [failedAttempt, failedRetryAttempt],
409
362
  },
410
363
  {
411
364
  evalId: "algebra/simple",
412
365
  verdict: "passed",
413
- duration: { value: 5_000, display: "5.0s", samples: 1, total: 1, refs: [] },
414
- cost: { value: 0.02, display: "$0.02", samples: 1, total: 1, refs: [] },
415
- attempts: [],
366
+ durationMs: { value: 5_000, display: "5.0s", samples: 1, total: 1, refs: [] },
367
+ costUSD: { value: 0.02, display: "$0.02", samples: 1, total: 1, refs: [] },
368
+ attempts: [passedAttempt],
416
369
  },
417
370
  ],
418
371
  },
419
372
  {
420
373
  experimentId: "compare/codex",
421
374
  agent: "codex",
422
- verdicts: { passed: 0, failed: 0, errored: 1, skipped: 0 },
423
- passRate: { value: 0, display: "0%", samples: 1, total: 1, refs: [] },
424
- cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
425
- duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [] },
375
+ evalVerdicts: { passed: 0, failed: 0, errored: 1, skipped: 0 },
376
+ endToEndPassRate: { value: 0, display: "0%", samples: 1, total: 1, refs: [] },
377
+ costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
378
+ durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [] },
426
379
  tokens: { value: null, display: "—", samples: 0, total: 1, refs: [] },
427
380
  evals: 1,
428
381
  attempts: 1,
@@ -431,8 +384,8 @@ export const experimentListItems: ExperimentListItem[] = [
431
384
  {
432
385
  evalId: "geometry/angles",
433
386
  verdict: "errored",
434
- duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
435
- cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
387
+ durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
388
+ costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
436
389
  attempts: [erroredAttempt],
437
390
  },
438
391
  ],