niceeval 0.6.2 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/INDEX.md +23 -23
  2. package/README.zh.md +6 -6
  3. package/dist/agents/types.d.ts +2 -2
  4. package/dist/i18n/zh-CN.d.ts +3 -3
  5. package/dist/report/aggregate.d.ts +32 -26
  6. package/dist/report/aggregate.js +157 -76
  7. package/dist/report/built-in/index.d.ts +2 -0
  8. package/dist/report/built-in/index.js +8 -0
  9. package/dist/report/components.d.ts +91 -164
  10. package/dist/report/components.js +377 -114
  11. package/dist/report/compute.d.ts +86 -73
  12. package/dist/report/compute.js +592 -432
  13. package/dist/report/flag.d.ts +28 -17
  14. package/dist/report/flag.js +86 -16
  15. package/dist/report/format.d.ts +11 -11
  16. package/dist/report/format.js +17 -15
  17. package/dist/report/index.d.ts +16 -17
  18. package/dist/report/index.js +20 -22
  19. package/dist/report/load.js +3 -2
  20. package/dist/report/locale.d.ts +49 -34
  21. package/dist/report/locale.js +106 -58
  22. package/dist/report/metrics.d.ts +10 -3
  23. package/dist/report/metrics.js +46 -12
  24. package/dist/report/primitives.d.ts +42 -15
  25. package/dist/report/primitives.js +135 -26
  26. package/dist/report/react/AttemptList.d.ts +10 -8
  27. package/dist/report/react/AttemptList.js +18 -10
  28. package/dist/report/react/DeltaTable.js +19 -18
  29. package/dist/report/react/EvalList.d.ts +3 -3
  30. package/dist/report/react/EvalList.js +0 -0
  31. package/dist/report/react/ExperimentComparison.d.ts +4 -2
  32. package/dist/report/react/ExperimentComparison.js +5 -4
  33. package/dist/report/react/ExperimentList.d.ts +3 -3
  34. package/dist/report/react/ExperimentList.js +16 -15
  35. package/dist/report/react/MetricBars.js +5 -4
  36. package/dist/report/react/MetricLine.js +12 -5
  37. package/dist/report/react/MetricMatrix.js +1 -1
  38. package/dist/report/react/MetricScatter.js +54 -17
  39. package/dist/report/react/MetricTable.js +2 -12
  40. package/dist/report/react/ScopeSummary.d.ts +10 -0
  41. package/dist/report/react/ScopeSummary.js +17 -0
  42. package/dist/report/react/Scoreboard.js +6 -6
  43. package/dist/report/react/cell.js +2 -2
  44. package/dist/report/react/fixtures.d.ts +5 -9
  45. package/dist/report/react/fixtures.js +105 -149
  46. package/dist/report/react/index.d.ts +15 -5
  47. package/dist/report/react/index.js +18 -7
  48. package/dist/report/report.d.ts +137 -20
  49. package/dist/report/report.js +261 -34
  50. package/dist/report/text/faces.d.ts +17 -19
  51. package/dist/report/text/faces.js +225 -157
  52. package/dist/report/text/plot.js +1 -1
  53. package/dist/report/text/table.js +2 -2
  54. package/dist/report/tree.d.ts +90 -40
  55. package/dist/report/tree.js +252 -94
  56. package/dist/report/types.d.ts +245 -300
  57. package/dist/report/types.js +4 -3
  58. package/dist/report/web.d.ts +21 -5
  59. package/dist/report/web.js +42 -16
  60. package/dist/results/select.d.ts +38 -16
  61. package/dist/results/select.js +73 -25
  62. package/dist/results/types.d.ts +38 -14
  63. package/dist/shared/aggregate.d.ts +3 -2
  64. package/dist/shared/aggregate.js +5 -4
  65. package/docs-site/zh/README.md +44 -0
  66. package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
  67. package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
  68. package/docs-site/zh/examples/index.mdx +50 -0
  69. package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
  70. package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
  71. package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
  72. package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
  73. package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
  74. package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
  75. package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
  76. package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
  77. package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
  78. package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
  79. package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
  80. package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
  81. package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
  82. package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
  83. package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
  84. package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
  85. package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
  86. package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
  87. package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
  88. package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
  89. package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
  90. package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
  91. package/docs-site/zh/index.mdx +23 -25
  92. package/docs-site/zh/introduction.mdx +8 -8
  93. package/docs-site/zh/reference/builtin-agents.mdx +5 -5
  94. package/docs-site/zh/reference/capabilities.mdx +6 -6
  95. package/docs-site/zh/reference/cli.mdx +9 -7
  96. package/docs-site/zh/reference/define-agent.mdx +1 -1
  97. package/docs-site/zh/reference/events.mdx +3 -3
  98. package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
  99. package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
  100. package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
  101. package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
  102. package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
  103. package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
  104. package/package.json +6 -2
  105. package/src/agents/index.ts +2 -2
  106. package/src/agents/openai-compat.ts +1 -1
  107. package/src/agents/streaming.ts +2 -2
  108. package/src/agents/types.ts +3 -3
  109. package/src/cli.ts +42 -23
  110. package/src/context/context.ts +1 -1
  111. package/src/context/session.test.ts +1 -1
  112. package/src/context/session.ts +1 -1
  113. package/src/i18n/en.ts +18 -16
  114. package/src/i18n/zh-CN.ts +16 -15
  115. package/src/report/aggregate.ts +175 -87
  116. package/src/report/built-in/index.tsx +9 -0
  117. package/src/report/components.tsx +625 -285
  118. package/src/report/compute.ts +717 -515
  119. package/src/report/dual-render.test.tsx +738 -1148
  120. package/src/report/flag.ts +97 -33
  121. package/src/report/format.ts +18 -22
  122. package/src/report/index.ts +113 -58
  123. package/src/report/load.ts +3 -2
  124. package/src/report/locale.ts +120 -69
  125. package/src/report/metrics.ts +42 -12
  126. package/src/report/primitives.tsx +190 -45
  127. package/src/report/react/AttemptList.tsx +32 -20
  128. package/src/report/react/DeltaTable.tsx +63 -45
  129. package/src/report/react/EvalList.tsx +0 -0
  130. package/src/report/react/ExperimentComparison.tsx +12 -7
  131. package/src/report/react/ExperimentList.tsx +38 -26
  132. package/src/report/react/MetricBars.tsx +5 -4
  133. package/src/report/react/MetricLine.tsx +13 -8
  134. package/src/report/react/MetricMatrix.tsx +2 -2
  135. package/src/report/react/MetricScatter.tsx +74 -20
  136. package/src/report/react/MetricTable.tsx +4 -76
  137. package/src/report/react/ScopeSummary.tsx +86 -0
  138. package/src/report/react/Scoreboard.tsx +28 -10
  139. package/src/report/react/cell.tsx +2 -2
  140. package/src/report/react/enhance.js +57 -5
  141. package/src/report/react/fixtures.ts +109 -156
  142. package/src/report/react/index.tsx +24 -39
  143. package/src/report/react/render.test.tsx +139 -104
  144. package/src/report/react/styles.css +181 -91
  145. package/src/report/report.test.ts +761 -1031
  146. package/src/report/report.ts +425 -47
  147. package/src/report/text/faces.ts +257 -164
  148. package/src/report/text/plot.ts +1 -1
  149. package/src/report/text/table.ts +2 -2
  150. package/src/report/tree.ts +362 -104
  151. package/src/report/types.ts +257 -287
  152. package/src/report/web.ts +63 -20
  153. package/src/results/attempt-evidence.test.ts +4 -4
  154. package/src/results/attempt-evidence.ts +5 -5
  155. package/src/results/copy.ts +6 -6
  156. package/src/results/host-equivalence.test.ts +26 -14
  157. package/src/results/index.ts +10 -4
  158. package/src/results/open.ts +8 -4
  159. package/src/results/results.test.ts +4 -3
  160. package/src/results/select.ts +104 -34
  161. package/src/results/types.ts +36 -14
  162. package/src/runner/feedback/human.test.ts +1 -1
  163. package/src/runner/run.ts +1 -1
  164. package/src/sandbox/cli-commands.ts +2 -2
  165. package/src/scoring/judge.test.ts +1 -1
  166. package/src/shared/aggregate.ts +5 -4
  167. package/src/show/compose.ts +50 -67
  168. package/src/show/index.ts +107 -56
  169. package/src/show/render.ts +43 -27
  170. package/src/show/report-host.test.ts +188 -0
  171. package/src/show/report-host.ts +375 -0
  172. package/src/show/show.test.ts +86 -36
  173. package/src/view/app/App.test.tsx +69 -0
  174. package/src/view/app/App.tsx +144 -48
  175. package/src/view/app/components/AttemptModal.tsx +324 -63
  176. package/src/view/app/components/CodeView.tsx +10 -4
  177. package/src/view/app/i18n.ts +31 -17
  178. package/src/view/app/lib/artifact-url.ts +14 -3
  179. package/src/view/app/main.tsx +13 -8
  180. package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
  181. package/src/view/app/types.ts +4 -1
  182. package/src/view/artifact-serving.test.ts +21 -1
  183. package/src/view/client-dist/app.css +1 -1
  184. package/src/view/client-dist/app.js +6 -6
  185. package/src/view/data.test.ts +9 -3
  186. package/src/view/data.ts +145 -49
  187. package/src/view/index.ts +48 -44
  188. package/src/view/server.ts +35 -15
  189. package/src/view/shared/types.ts +34 -5
  190. package/src/view/styles.css +224 -0
  191. package/src/view/view-report.test.ts +161 -57
  192. package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
  193. package/dist/report/built-ins/experiment-comparison.js +0 -119
  194. package/dist/report/built-ins/index.d.ts +0 -2
  195. package/dist/report/built-ins/index.js +0 -2
  196. package/dist/report/react/GroupSummary.d.ts +0 -8
  197. package/dist/report/react/GroupSummary.js +0 -8
  198. package/dist/report/react/RunOverview.d.ts +0 -8
  199. package/dist/report/react/RunOverview.js +0 -12
  200. package/docs-site/zh/example/ai-agent-application.mdx +0 -152
  201. package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
  202. package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
  203. package/docs-site/zh/example/showcase.mdx +0 -39
  204. package/src/report/built-in-user-parity.test.tsx +0 -597
  205. package/src/report/built-ins/experiment-comparison.tsx +0 -179
  206. package/src/report/built-ins/index.ts +0 -7
  207. package/src/report/react/GroupSummary.tsx +0 -66
  208. package/src/report/react/RunOverview.tsx +0 -109
  209. /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
  210. /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
  211. /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
  212. /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
  213. /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
  214. /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
  215. /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
  216. /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
  217. /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
  218. /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
@@ -1,4 +1,4 @@
1
- // 手工 fixture:模拟计算函数(MetricTable.data / MetricMatrix.data / …)的产物。
1
+ // 手工 fixture:模拟计算函数(metricTableData / metricMatrixData / …)的产物。
2
2
  // 仅供渲染测试与 scripts/report-react-demo.tsx 使用,不从入口导出。
3
3
  // 数字刻意覆盖诚实细节:coverage 角标(samples<total)、全 null 格子、
4
4
  // 稀疏矩阵、缺数据的散点、delta 的 null 不硬算、truncated 计数。
@@ -6,53 +6,20 @@ const locator = (s) => s;
6
6
  export const passRateColumn = { key: "task-pass-rate", label: "pass rate", unit: "%", better: "higher" };
7
7
  export const codeLinesColumn = { key: "code-lines", label: "code lines", unit: "lines", better: "lower" };
8
8
  export const costColumn = { key: "cost", label: "cost", unit: "$", better: "lower" };
9
- export const overviewData = {
10
- snapshots: [
11
- { experimentId: "compare/bub", agent: "bub", model: "gpt-5.4", startedAt: "2026-07-01T10:00:00Z" },
12
- { experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T11:30:00Z" },
13
- ],
14
- totals: {
15
- evals: 12,
16
- attempts: 48,
17
- passed: 36,
18
- failed: 8,
19
- errored: 2,
20
- skipped: 2,
21
- // 两级聚合口径(computeCell)刻意不等于 36/(36+8+2)≈78% 的 attempt 原始占比:
22
- // 组件必须原样渲染这个字段,不得从上面四个 verdict 计票现场重算。
23
- passRate: { value: 0.7, display: "70%", samples: 46, total: 48, refs: [] },
24
- costUSD: null, // 全部 attempt 都没报成本:null,组件必须显示缺数据而不是 $0
25
- durationMs: 261_000,
26
- },
27
- warnings: [
28
- {
29
- kind: "partial-coverage",
30
- experimentId: "compare/bub",
31
- covered: 9,
32
- total: 12,
33
- message: "snapshot covers 9 of 12 evals seen in history; re-run `niceeval exp compare/bub` for a full snapshot",
34
- },
35
- ],
36
- };
37
- export const overviewWithCost = {
38
- ...overviewData,
39
- totals: { ...overviewData.totals, costUSD: 1.234 },
40
- warnings: [],
41
- };
42
- /** GroupSummary.data 的产物形态:eval 级折叠计票 + 旧 GroupSelector 口径的通过率(包成 MetricCell)。 */
43
- export const groupSummaryData = {
9
+ /** scopeSummaryData 的产物形态:两级计票恒随行,成功率与总成本是官方 MetricCell。 */
10
+ export const scopeSummaryData = {
11
+ range: { earliestStartedAt: "2026-07-01T10:00:00Z", latestStartedAt: "2026-07-01T11:30:00Z" },
44
12
  experiments: 2,
45
- // evals = 全部 verdicts 之和(3+1+1+1=6);passRate 分母只数非 skipped 的 5 道
46
13
  evals: 6,
47
14
  attempts: 9,
48
- verdicts: { passed: 3, failed: 1, errored: 1, skipped: 1 },
49
- // 3 passed / (3 + 1 + 1) = 60%;samples=ran=5 < total=evals=6:1 道 skipped 未计入分母的覆盖率角标
50
- passRate: { value: 0.6, display: "60%", samples: 5, total: 6, refs: [] },
51
- totalCostUSD: 1.5,
52
- lastRunAt: "2026-07-01T11:30:00Z",
15
+ evalVerdicts: { passed: 3, failed: 1, errored: 1, skipped: 1 },
16
+ attemptVerdicts: { passed: 4, failed: 3, errored: 1, skipped: 1 },
17
+ // 两级聚合口径,刻意不等于任一计票的比例:组件必须原样渲染,不重算
18
+ endToEndPassRate: { value: 0.6, display: "60%", samples: 8, total: 9, refs: [] },
19
+ totalCostUSD: { value: 1.5, display: "$1.50", samples: 8, total: 9, refs: [] },
53
20
  };
54
21
  export const tableData = {
55
- dimension: "agent",
22
+ rowDimension: "agent",
56
23
  // 行顺序故意不按 passRate 排:组件必须按传入顺序渲染,不重排
57
24
  columns: [passRateColumn, codeLinesColumn],
58
25
  rows: [
@@ -72,7 +39,7 @@ export const tableData = {
72
39
  display: "87%",
73
40
  samples: 6,
74
41
  total: 6,
75
- refs: [locator("@1a0a0a0")],
42
+ refs: [locator("@1a0a0a0a")],
76
43
  },
77
44
  // samples < total:有 attempt 测不了 → 覆盖率角标 5/6
78
45
  "code-lines": { value: 120, display: "120 lines", samples: 5, total: 6, refs: [] },
@@ -80,33 +47,9 @@ export const tableData = {
80
47
  },
81
48
  ],
82
49
  };
83
- /** rows: "experiment" 的榜单形态:行携带 agent/model 元信息与 eval 级折叠计票。 */
84
- export const tableDataWithMeta = {
85
- dimension: "experiment",
86
- columns: [passRateColumn],
87
- rows: [
88
- {
89
- key: "compare/bub",
90
- cells: { "task-pass-rate": { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] } },
91
- meta: {
92
- agent: "bub",
93
- model: "gpt-5.4",
94
- verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
95
- },
96
- },
97
- {
98
- key: "compare/codex",
99
- cells: { "task-pass-rate": { value: 1, display: "100%", samples: 2, total: 2, refs: [] } },
100
- meta: {
101
- agent: "codex",
102
- verdicts: { passed: 2, failed: 0, errored: 0, skipped: 0 },
103
- },
104
- },
105
- ],
106
- };
107
50
  export const matrixData = {
108
- rows: "eval",
109
- columns: "agent",
51
+ rowDimension: "eval",
52
+ columnDimension: "agent",
110
53
  metric: passRateColumn,
111
54
  // 稀疏:geometry/angles × codex 没有样本,数据里不出现 → 格子空着
112
55
  cells: [
@@ -118,7 +61,7 @@ export const matrixData = {
118
61
  display: "100%",
119
62
  samples: 2,
120
63
  total: 2,
121
- refs: [locator("@1b3b3b3"), locator("@1b7b7b7")],
64
+ refs: [locator("@1b3b3b3b"), locator("@1b7b7b7b")],
122
65
  },
123
66
  },
124
67
  {
@@ -134,33 +77,40 @@ export const matrixData = {
134
77
  ],
135
78
  };
136
79
  export const scoreboardData = {
137
- dimension: "agent",
80
+ rowDimension: "agent",
81
+ questions: [
82
+ "algebra/quadratic",
83
+ "algebra/simple",
84
+ "geometry/angles",
85
+ "geometry/area",
86
+ ],
138
87
  fullMarks: 100,
139
88
  weights: [{ prefix: "algebra/", weight: 2 }],
89
+ ignoredEvals: 1,
140
90
  rows: [
141
91
  {
142
92
  key: "bub",
143
- total: { value: 78.5, display: "78.5" },
93
+ // notRun 1:一题没跑、按 0 计;unscorable 0 —— 两种 0 分分开计数
94
+ total: { value: 78.5, display: "78.5", notRun: 1, unscorable: 0, refs: [locator("@1a0a0a0a")] },
144
95
  subjects: [
145
- // missing 1:一题没跑、按 0 计——固定分母的如实注脚
146
- { key: "algebra", earned: 14, possible: 16, evals: 8, missing: 1 },
147
- { key: "geometry", earned: 3, possible: 4, evals: 4, missing: 0 },
96
+ { key: "algebra", earned: 14, possible: 16, questions: 8, notRun: 1, unscorable: 0, display: "14/16 (87.5%)", refs: [] },
97
+ { key: "geometry", earned: 3, possible: 4, questions: 4, notRun: 0, unscorable: 0, display: "3/4 (75%)", refs: [] },
148
98
  ],
149
99
  },
150
100
  {
151
101
  key: "codex",
152
- total: { value: 52, display: "52" },
102
+ total: { value: 52, display: "52", notRun: 0, unscorable: 2, refs: [] },
153
103
  subjects: [
154
- { key: "algebra", earned: 9, possible: 16, evals: 8, missing: 0 },
155
- { key: "geometry", earned: 1.4, possible: 4, evals: 4, missing: 2 },
104
+ { key: "algebra", earned: 9, possible: 16, questions: 8, notRun: 0, unscorable: 0, display: "9/16 (56.3%)", refs: [] },
105
+ { key: "geometry", earned: 1.4, possible: 4, questions: 4, notRun: 0, unscorable: 2, display: "1.4/4 (35%)", refs: [] },
156
106
  ],
157
107
  },
158
108
  ],
159
109
  };
160
110
  export const scatterData = {
161
- points: "experiment",
162
- series: "agent",
163
- x: costColumn, // better: "lower" → 轴反向,便宜的一端在右
111
+ pointDimension: "experiment",
112
+ seriesDimension: "agent",
113
+ x: costColumn, // better: "lower" → 轴反向(贵在左、便宜在右);两轴都有 better → 提示「越靠右上越好」
164
114
  y: passRateColumn,
165
115
  rows: [
166
116
  {
@@ -192,81 +142,87 @@ export const scatterData = {
192
142
  };
193
143
  export const lineData = {
194
144
  x: { key: "latencyMs", label: "Simulated latency", unit: "ms" },
195
- series: "agents",
145
+ seriesDimension: "agents",
196
146
  y: passRateColumn,
197
147
  rows: [
198
148
  {
199
- key: "ultra/agents-1-lat-100",
149
+ key: "100",
200
150
  series: "1 agents",
201
151
  x: 100,
202
152
  xDisplay: "100ms",
203
153
  y: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
204
154
  },
205
155
  {
206
- key: "ultra/agents-1-lat-300",
156
+ key: "300",
207
157
  series: "1 agents",
208
158
  x: 300,
209
159
  xDisplay: "300ms",
210
160
  y: { value: 0.3, display: "30%", samples: 6, total: 6, refs: [] },
211
161
  },
212
162
  {
213
- key: "ultra/agents-16-lat-100",
163
+ key: "100",
214
164
  series: "16 agents",
215
165
  x: 100,
216
166
  xDisplay: "100ms",
217
167
  y: { value: 0.8, display: "80%", samples: 6, total: 6, refs: [] },
218
168
  },
219
169
  {
220
- key: "ultra/agents-16-lat-300",
170
+ key: "300",
221
171
  series: "16 agents",
222
172
  x: 300,
223
173
  xDisplay: "300ms",
224
174
  y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
225
175
  },
226
176
  {
227
- // 未声明 flag 的 experiment:作轴不画点,注脚报数
228
- key: "ultra/legacy",
177
+ // 未声明数值 flag 的 attempt:不伪造 x 值,注脚报数
178
+ key: "null",
229
179
  series: "1 agents",
230
180
  x: null,
231
- xDisplay: "",
181
+ xDisplay: "",
232
182
  y: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
233
183
  },
234
184
  ],
235
185
  };
236
186
  export const deltaData = {
187
+ byDimension: "experiment",
237
188
  columns: [passRateColumn, costColumn],
238
189
  rows: [
239
190
  {
240
- key: "bub",
241
- a: { experimentId: "compare/bub" },
242
- b: { experimentId: "compare/bub--agents-md" },
191
+ key: "compare/bub → compare/bub--agents-md",
192
+ label: "bub",
193
+ a: { key: "compare/bub" },
194
+ b: { key: "compare/bub--agents-md" },
243
195
  cells: {
244
- // 通过率 +12pp:better higher → (绿)
196
+ // 通过率 +12pp:better higher → improved(绿)
245
197
  "task-pass-rate": {
246
198
  a: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
247
199
  b: { value: 0.62, display: "62%", samples: 6, total: 6, refs: [] },
248
200
  delta: 0.12,
249
201
  display: "+12pp",
202
+ outcome: "improved",
250
203
  },
251
- // 成本 +$0.15:better lower → (红)
204
+ // 成本 +$0.15:better lower → regressed(红)
252
205
  cost: {
253
206
  a: { value: 0.2, display: "$0.20", samples: 6, total: 6, refs: [] },
254
207
  b: { value: 0.35, display: "$0.35", samples: 6, total: 6, refs: [] },
255
208
  delta: 0.15,
256
209
  display: "+$0.15",
210
+ outcome: "regressed",
257
211
  },
258
212
  },
259
213
  },
260
214
  {
261
- key: "codex",
262
- a: { experimentId: "compare/codex" },
263
- b: { experimentId: "compare/codex--agents-md" },
215
+ key: "compare/codex → compare/codex--agents-md",
216
+ label: "codex",
217
+ a: { key: "compare/codex" },
218
+ b: { key: "compare/codex--agents-md" },
264
219
  cells: {
265
220
  "task-pass-rate": {
266
221
  a: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
267
222
  b: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
268
223
  delta: 0,
269
224
  display: "±0",
225
+ outcome: "unchanged",
270
226
  },
271
227
  // A 侧缺数据 → delta null:显示缺,不硬算
272
228
  cost: {
@@ -274,6 +230,7 @@ export const deltaData = {
274
230
  b: { value: 0.3, display: "$0.30", samples: 6, total: 6, refs: [] },
275
231
  delta: null,
276
232
  display: "—",
233
+ outcome: "unavailable",
277
234
  },
278
235
  },
279
236
  },
@@ -282,71 +239,70 @@ export const deltaData = {
282
239
  // ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
283
240
  /** algebra/quadratic 在 compare/bub 上失败的那次 attempt——两条子失败夹具共用同一条。 */
284
241
  const failedAttempt = {
285
- evalId: "algebra/quadratic",
286
242
  experimentId: "compare/bub",
243
+ evalId: "algebra/quadratic",
287
244
  attempt: 3,
288
245
  agent: "bub",
289
246
  verdict: "failed",
290
- assertions: [
291
- {
292
- name: "roots-correct",
293
- severity: "gate",
294
- score: 0,
295
- outcome: "failed",
296
- expected: "x=2",
297
- received: "x=3",
298
- evidence: "judge: sign flipped when substituting into the quadratic formula",
299
- },
300
- ],
247
+ // 已按 Scoring display 契约折好的单行摘要;渲染面只做宽度截断,不重算
248
+ failureSummary: "gate: roots-correct · expected x=2 · received x=3",
249
+ moreFailures: 1,
250
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1a4a4a4a")] },
301
251
  durationMs: 32_000,
302
252
  costUSD: 0.12,
303
- locator: locator("@1a4a4a4"),
253
+ locator: locator("@1a4a4a4a"),
304
254
  };
305
255
  const erroredAttempt = {
306
- evalId: "geometry/angles",
307
256
  experimentId: "compare/codex",
257
+ evalId: "geometry/angles",
308
258
  attempt: 0,
309
259
  agent: "codex",
310
260
  verdict: "errored",
311
- // 结构化 error:列表只渲染 message 一层摘要;cause/stack diagnostics 是下钻详情,随数据携带
312
- error: {
313
- code: "unexpected-error",
314
- message: "TypeError: cannot read properties of undefined (reading 'foo')",
315
- phase: "eval.run",
316
- stack: "TypeError: cannot read properties of undefined (reading 'foo')\n at run (adapter.ts:42:7)",
317
- cause: { name: "TypeError", message: "cannot read properties of undefined (reading 'foo')" },
318
- },
319
- diagnostics: [
320
- { code: "sandbox-teardown-failed", level: "warning", message: "sandbox teardown timed out", phase: "sandbox.teardown" },
321
- ],
322
- assertions: [],
261
+ // errored:结构化 error 的一层摘要(phase · code · message)
262
+ failureSummary: "eval.run · unexpected-error · TypeError: cannot read properties of undefined (reading 'foo')",
263
+ moreFailures: 0,
264
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1c1c1c1c")] },
323
265
  durationMs: 4_500,
324
- locator: locator("@1c1c1c1"),
266
+ costUSD: null,
267
+ locator: locator("@1c1c1c1c"),
325
268
  };
326
269
  const failedRetryAttempt = {
327
270
  ...failedAttempt,
328
271
  attempt: 4,
329
272
  durationMs: 35_000,
330
- locator: locator("@1b5b5b5"),
273
+ locator: locator("@1b5b5b5b"),
274
+ };
275
+ const passedAttempt = {
276
+ experimentId: "compare/bub",
277
+ evalId: "algebra/simple",
278
+ attempt: 0,
279
+ agent: "bub",
280
+ verdict: "passed",
281
+ failureSummary: null,
282
+ moreFailures: 0,
283
+ examScore: { value: 1, display: "100%", samples: 1, total: 1, refs: [locator("@1d2d2d2d")] },
284
+ durationMs: 5_000,
285
+ costUSD: 0.02,
286
+ locator: locator("@1d2d2d2d"),
331
287
  };
332
288
  export const attemptListItems = [failedAttempt, erroredAttempt];
333
289
  export const evalListItems = [
334
290
  {
335
- evalId: "algebra/quadratic",
336
291
  experimentId: "compare/bub",
292
+ evalId: "algebra/quadratic",
337
293
  verdict: "failed",
338
- score: { value: 0, display: "0%", samples: 1, total: 1, refs: [failedAttempt.locator] },
339
- duration: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
340
- cost: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
294
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [failedAttempt.locator] },
295
+ durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
296
+ costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
341
297
  attempts: [failedAttempt],
342
298
  },
343
299
  {
344
- evalId: "geometry/angles",
345
300
  experimentId: "compare/codex",
301
+ evalId: "geometry/angles",
346
302
  verdict: "errored",
347
- score: { value: 0, display: "0%", samples: 1, total: 1, refs: [erroredAttempt.locator] },
348
- duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
349
- cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
303
+ examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [erroredAttempt.locator] },
304
+ durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
305
+ costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
350
306
  attempts: [erroredAttempt],
351
307
  },
352
308
  ];
@@ -356,10 +312,10 @@ export const experimentListItems = [
356
312
  agent: "bub",
357
313
  model: "gpt-5.4",
358
314
  flags: { memory: true },
359
- verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
360
- passRate: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
361
- cost: { value: 0.12, display: "$0.12", samples: 1, total: 2, refs: [failedAttempt.locator] },
362
- duration: { value: 32_000, display: "32.0s", samples: 2, total: 2, refs: [] },
315
+ evalVerdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
316
+ endToEndPassRate: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
317
+ costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 2, refs: [failedAttempt.locator] },
318
+ durationMs: { value: 32_000, display: "32.0s", samples: 2, total: 2, refs: [] },
363
319
  tokens: { value: null, display: "—", samples: 0, total: 2, refs: [] },
364
320
  evals: 2,
365
321
  attempts: 3,
@@ -368,26 +324,26 @@ export const experimentListItems = [
368
324
  {
369
325
  evalId: "algebra/quadratic",
370
326
  verdict: "failed",
371
- duration: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
372
- cost: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
327
+ durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
328
+ costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
373
329
  attempts: [failedAttempt, failedRetryAttempt],
374
330
  },
375
331
  {
376
332
  evalId: "algebra/simple",
377
333
  verdict: "passed",
378
- duration: { value: 5_000, display: "5.0s", samples: 1, total: 1, refs: [] },
379
- cost: { value: 0.02, display: "$0.02", samples: 1, total: 1, refs: [] },
380
- attempts: [],
334
+ durationMs: { value: 5_000, display: "5.0s", samples: 1, total: 1, refs: [] },
335
+ costUSD: { value: 0.02, display: "$0.02", samples: 1, total: 1, refs: [] },
336
+ attempts: [passedAttempt],
381
337
  },
382
338
  ],
383
339
  },
384
340
  {
385
341
  experimentId: "compare/codex",
386
342
  agent: "codex",
387
- verdicts: { passed: 0, failed: 0, errored: 1, skipped: 0 },
388
- passRate: { value: 0, display: "0%", samples: 1, total: 1, refs: [] },
389
- cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
390
- duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [] },
343
+ evalVerdicts: { passed: 0, failed: 0, errored: 1, skipped: 0 },
344
+ endToEndPassRate: { value: 0, display: "0%", samples: 1, total: 1, refs: [] },
345
+ costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
346
+ durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [] },
391
347
  tokens: { value: null, display: "—", samples: 0, total: 1, refs: [] },
392
348
  evals: 1,
393
349
  attempts: 1,
@@ -396,8 +352,8 @@ export const experimentListItems = [
396
352
  {
397
353
  evalId: "geometry/angles",
398
354
  verdict: "errored",
399
- duration: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
400
- cost: { value: null, display: "—", samples: 0, total: 1, refs: [] },
355
+ durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
356
+ costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
401
357
  attempts: [erroredAttempt],
402
358
  },
403
359
  ],
@@ -1,6 +1,16 @@
1
- export { AttemptList, DeltaTable, EvalList, ExperimentList, GroupSummary, MetricBars, MetricLine, MetricMatrix, MetricScatter, MetricTable, RunOverview, Scoreboard, } from "../components.tsx";
2
- export type { AttemptListProps, DeltaTableProps, EvalListProps, ExperimentListProps, GroupSummaryProps, MetricLineProps, MetricMatrixProps, MetricScatterProps, MetricTableProps, RunOverviewProps, ScoreboardProps, } from "../components.tsx";
3
- export type { AttemptListItem, AttemptLocator, DeltaData, EvalListItem, ExperimentListEvalRow, ExperimentListItem, GroupSummaryData, LineAxis, LineData, MatrixData, MetricCell, MetricColumn, OverviewData, ScatterData, ScoreboardData, SelectionWarning, TableData, TableRowMeta, } from "../types.ts";
4
- export { DEFAULT_REPORT_LOCALE, resolveMetricLabel } from "../locale.ts";
5
- export type { LocalizedLabel, ReportLocale } from "../locale.ts";
1
+ export { AttemptList } from "./AttemptList.tsx";
2
+ export { EvalList } from "./EvalList.tsx";
3
+ export { ExperimentList } from "./ExperimentList.tsx";
4
+ export { ScopeSummary } from "./ScopeSummary.tsx";
5
+ export { ExperimentComparisonView as ExperimentComparison } from "./ExperimentComparison.tsx";
6
+ export { MetricTable } from "./MetricTable.tsx";
7
+ export { MetricMatrix } from "./MetricMatrix.tsx";
8
+ export { MetricBars } from "./MetricBars.tsx";
9
+ export { MetricScatter } from "./MetricScatter.tsx";
10
+ export { MetricLine } from "./MetricLine.tsx";
11
+ export { DeltaTable } from "./DeltaTable.tsx";
12
+ export { Scoreboard } from "./Scoreboard.tsx";
13
+ export type { AttemptListItem, AttemptLocator, DeltaData, EvalListItem, ExperimentComparisonData, ExperimentComparisonGroupData, ExperimentListEvalRow, ExperimentListItem, LineData, MatrixData, MetricCell, MetricColumn, ScatterData, ScopeSummaryData, ScopeWarning, ScoreboardData, TableData, VerdictTally, } from "../types.ts";
14
+ export { DEFAULT_REPORT_LOCALE, resolveLocalizedText, resolveMetricLabel } from "../locale.ts";
15
+ export type { LocalizedText, ReportLocale } from "../locale.ts";
6
16
  export { NRE_PALETTE, colorClassForKey, colorHexForKey, colorIndexForKey, seriesClassForKey } from "./colors.ts";
@@ -1,14 +1,25 @@
1
- // niceeval/report/react —— 零件复用的导出点:把某一块指标表嵌进已有内部面板时从这里
2
- // import。导出的组件与 niceeval/report 是同一批双面组件(web 面即 React 渲染,零 IO、
3
- // 可进 "use client",不 hydrate 也完整),都带自己的 data 计算函数;那是零件的复用,
4
- // 不是另一套报告系统 —— 报告的家在官方宿主(--report)。
1
+ // niceeval/report/react —— 纯 web 渲染面的导出点:把某一块指标表嵌进已有 React 页面时
2
+ // 从这里 import。组件只收算好的可序列化 `data`(data 形态),不含任何读盘 / artifact
3
+ // 计算代码;计算函数、spec 形态与组合组件只住在 niceeval/report。
5
4
  //
6
5
  // 契约:
7
6
  // - 组件只认「算好的可序列化数据」:零 hooks、零数据操作;
8
7
  // - 样式随包发布:配套 ./styles.css(nre-* 稳定类名),使用者在其后加载覆盖即可;
8
+ // - 渐进增强脚本 ./enhance.js 可选加载,初始静态 HTML 无 JS 完整可读;
9
9
  // - 跨块配色一致:维度键 → 稳定散列 → 固定调色板下标(colors.ts)。
10
- export { AttemptList, DeltaTable, EvalList, ExperimentList, GroupSummary, MetricBars, MetricLine, MetricMatrix, MetricScatter, MetricTable, RunOverview, Scoreboard, } from "../components.js";
11
- // locale(官方组件 chrome 文案;指标 label 的按 locale 字典也用它解析)
12
- export { DEFAULT_REPORT_LOCALE, resolveMetricLabel } from "../locale.js";
10
+ export { AttemptList } from "./AttemptList.js";
11
+ export { EvalList } from "./EvalList.js";
12
+ export { ExperimentList } from "./ExperimentList.js";
13
+ export { ScopeSummary } from "./ScopeSummary.js";
14
+ export { ExperimentComparisonView as ExperimentComparison } from "./ExperimentComparison.js";
15
+ export { MetricTable } from "./MetricTable.js";
16
+ export { MetricMatrix } from "./MetricMatrix.js";
17
+ export { MetricBars } from "./MetricBars.js";
18
+ export { MetricScatter } from "./MetricScatter.js";
19
+ export { MetricLine } from "./MetricLine.js";
20
+ export { DeltaTable } from "./DeltaTable.js";
21
+ export { Scoreboard } from "./Scoreboard.js";
22
+ // locale(官方组件 chrome 文案;LocalizedText 的按 locale 解析也用它)
23
+ export { DEFAULT_REPORT_LOCALE, resolveLocalizedText, resolveMetricLabel } from "../locale.js";
13
24
  // 稳定配色(自定义组件想与官方组件同键同色时用;seriesClassForKey 配 CSS 的 --nre-series)
14
25
  export { NRE_PALETTE, colorClassForKey, colorHexForKey, colorIndexForKey, seriesClassForKey } from "./colors.js";