niceeval 0.6.2 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/INDEX.md +23 -23
  2. package/README.zh.md +6 -6
  3. package/dist/agents/types.d.ts +2 -2
  4. package/dist/i18n/zh-CN.d.ts +3 -3
  5. package/dist/report/aggregate.d.ts +32 -26
  6. package/dist/report/aggregate.js +157 -76
  7. package/dist/report/built-in/index.d.ts +2 -0
  8. package/dist/report/built-in/index.js +8 -0
  9. package/dist/report/components.d.ts +91 -164
  10. package/dist/report/components.js +377 -114
  11. package/dist/report/compute.d.ts +86 -73
  12. package/dist/report/compute.js +592 -432
  13. package/dist/report/flag.d.ts +28 -17
  14. package/dist/report/flag.js +86 -16
  15. package/dist/report/format.d.ts +11 -11
  16. package/dist/report/format.js +17 -15
  17. package/dist/report/index.d.ts +16 -17
  18. package/dist/report/index.js +20 -22
  19. package/dist/report/load.js +3 -2
  20. package/dist/report/locale.d.ts +49 -34
  21. package/dist/report/locale.js +106 -58
  22. package/dist/report/metrics.d.ts +10 -3
  23. package/dist/report/metrics.js +46 -12
  24. package/dist/report/primitives.d.ts +42 -15
  25. package/dist/report/primitives.js +135 -26
  26. package/dist/report/react/AttemptList.d.ts +10 -8
  27. package/dist/report/react/AttemptList.js +18 -10
  28. package/dist/report/react/DeltaTable.js +19 -18
  29. package/dist/report/react/EvalList.d.ts +3 -3
  30. package/dist/report/react/EvalList.js +0 -0
  31. package/dist/report/react/ExperimentComparison.d.ts +4 -2
  32. package/dist/report/react/ExperimentComparison.js +5 -4
  33. package/dist/report/react/ExperimentList.d.ts +3 -3
  34. package/dist/report/react/ExperimentList.js +16 -15
  35. package/dist/report/react/MetricBars.js +5 -4
  36. package/dist/report/react/MetricLine.js +12 -5
  37. package/dist/report/react/MetricMatrix.js +1 -1
  38. package/dist/report/react/MetricScatter.js +54 -17
  39. package/dist/report/react/MetricTable.js +2 -12
  40. package/dist/report/react/ScopeSummary.d.ts +10 -0
  41. package/dist/report/react/ScopeSummary.js +17 -0
  42. package/dist/report/react/Scoreboard.js +6 -6
  43. package/dist/report/react/cell.js +2 -2
  44. package/dist/report/react/fixtures.d.ts +5 -9
  45. package/dist/report/react/fixtures.js +105 -149
  46. package/dist/report/react/index.d.ts +15 -5
  47. package/dist/report/react/index.js +18 -7
  48. package/dist/report/report.d.ts +137 -20
  49. package/dist/report/report.js +261 -34
  50. package/dist/report/text/faces.d.ts +17 -19
  51. package/dist/report/text/faces.js +225 -157
  52. package/dist/report/text/plot.js +1 -1
  53. package/dist/report/text/table.js +2 -2
  54. package/dist/report/tree.d.ts +90 -40
  55. package/dist/report/tree.js +252 -94
  56. package/dist/report/types.d.ts +245 -300
  57. package/dist/report/types.js +4 -3
  58. package/dist/report/web.d.ts +21 -5
  59. package/dist/report/web.js +42 -16
  60. package/dist/results/select.d.ts +38 -16
  61. package/dist/results/select.js +73 -25
  62. package/dist/results/types.d.ts +38 -14
  63. package/dist/shared/aggregate.d.ts +3 -2
  64. package/dist/shared/aggregate.js +5 -4
  65. package/docs-site/zh/README.md +44 -0
  66. package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
  67. package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
  68. package/docs-site/zh/examples/index.mdx +50 -0
  69. package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
  70. package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
  71. package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
  72. package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
  73. package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
  74. package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
  75. package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
  76. package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
  77. package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
  78. package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
  79. package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
  80. package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
  81. package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
  82. package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
  83. package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
  84. package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
  85. package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
  86. package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
  87. package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
  88. package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
  89. package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
  90. package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
  91. package/docs-site/zh/index.mdx +23 -25
  92. package/docs-site/zh/introduction.mdx +8 -8
  93. package/docs-site/zh/reference/builtin-agents.mdx +5 -5
  94. package/docs-site/zh/reference/capabilities.mdx +6 -6
  95. package/docs-site/zh/reference/cli.mdx +9 -7
  96. package/docs-site/zh/reference/define-agent.mdx +1 -1
  97. package/docs-site/zh/reference/events.mdx +3 -3
  98. package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
  99. package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
  100. package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
  101. package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
  102. package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
  103. package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
  104. package/package.json +6 -2
  105. package/src/agents/index.ts +2 -2
  106. package/src/agents/openai-compat.ts +1 -1
  107. package/src/agents/streaming.ts +2 -2
  108. package/src/agents/types.ts +3 -3
  109. package/src/cli.ts +42 -23
  110. package/src/context/context.ts +1 -1
  111. package/src/context/session.test.ts +1 -1
  112. package/src/context/session.ts +1 -1
  113. package/src/i18n/en.ts +18 -16
  114. package/src/i18n/zh-CN.ts +16 -15
  115. package/src/report/aggregate.ts +175 -87
  116. package/src/report/built-in/index.tsx +9 -0
  117. package/src/report/components.tsx +625 -285
  118. package/src/report/compute.ts +717 -515
  119. package/src/report/dual-render.test.tsx +738 -1148
  120. package/src/report/flag.ts +97 -33
  121. package/src/report/format.ts +18 -22
  122. package/src/report/index.ts +113 -58
  123. package/src/report/load.ts +3 -2
  124. package/src/report/locale.ts +120 -69
  125. package/src/report/metrics.ts +42 -12
  126. package/src/report/primitives.tsx +190 -45
  127. package/src/report/react/AttemptList.tsx +32 -20
  128. package/src/report/react/DeltaTable.tsx +63 -45
  129. package/src/report/react/EvalList.tsx +0 -0
  130. package/src/report/react/ExperimentComparison.tsx +12 -7
  131. package/src/report/react/ExperimentList.tsx +38 -26
  132. package/src/report/react/MetricBars.tsx +5 -4
  133. package/src/report/react/MetricLine.tsx +13 -8
  134. package/src/report/react/MetricMatrix.tsx +2 -2
  135. package/src/report/react/MetricScatter.tsx +74 -20
  136. package/src/report/react/MetricTable.tsx +4 -76
  137. package/src/report/react/ScopeSummary.tsx +86 -0
  138. package/src/report/react/Scoreboard.tsx +28 -10
  139. package/src/report/react/cell.tsx +2 -2
  140. package/src/report/react/enhance.js +57 -5
  141. package/src/report/react/fixtures.ts +109 -156
  142. package/src/report/react/index.tsx +24 -39
  143. package/src/report/react/render.test.tsx +139 -104
  144. package/src/report/react/styles.css +181 -91
  145. package/src/report/report.test.ts +761 -1031
  146. package/src/report/report.ts +425 -47
  147. package/src/report/text/faces.ts +257 -164
  148. package/src/report/text/plot.ts +1 -1
  149. package/src/report/text/table.ts +2 -2
  150. package/src/report/tree.ts +362 -104
  151. package/src/report/types.ts +257 -287
  152. package/src/report/web.ts +63 -20
  153. package/src/results/attempt-evidence.test.ts +4 -4
  154. package/src/results/attempt-evidence.ts +5 -5
  155. package/src/results/copy.ts +6 -6
  156. package/src/results/host-equivalence.test.ts +26 -14
  157. package/src/results/index.ts +10 -4
  158. package/src/results/open.ts +8 -4
  159. package/src/results/results.test.ts +4 -3
  160. package/src/results/select.ts +104 -34
  161. package/src/results/types.ts +36 -14
  162. package/src/runner/feedback/human.test.ts +1 -1
  163. package/src/runner/run.ts +1 -1
  164. package/src/sandbox/cli-commands.ts +2 -2
  165. package/src/scoring/judge.test.ts +1 -1
  166. package/src/shared/aggregate.ts +5 -4
  167. package/src/show/compose.ts +50 -67
  168. package/src/show/index.ts +107 -56
  169. package/src/show/render.ts +43 -27
  170. package/src/show/report-host.test.ts +188 -0
  171. package/src/show/report-host.ts +375 -0
  172. package/src/show/show.test.ts +86 -36
  173. package/src/view/app/App.test.tsx +69 -0
  174. package/src/view/app/App.tsx +144 -48
  175. package/src/view/app/components/AttemptModal.tsx +324 -63
  176. package/src/view/app/components/CodeView.tsx +10 -4
  177. package/src/view/app/i18n.ts +31 -17
  178. package/src/view/app/lib/artifact-url.ts +14 -3
  179. package/src/view/app/main.tsx +13 -8
  180. package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
  181. package/src/view/app/types.ts +4 -1
  182. package/src/view/artifact-serving.test.ts +21 -1
  183. package/src/view/client-dist/app.css +1 -1
  184. package/src/view/client-dist/app.js +6 -6
  185. package/src/view/data.test.ts +9 -3
  186. package/src/view/data.ts +145 -49
  187. package/src/view/index.ts +48 -44
  188. package/src/view/server.ts +35 -15
  189. package/src/view/shared/types.ts +34 -5
  190. package/src/view/styles.css +224 -0
  191. package/src/view/view-report.test.ts +161 -57
  192. package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
  193. package/dist/report/built-ins/experiment-comparison.js +0 -119
  194. package/dist/report/built-ins/index.d.ts +0 -2
  195. package/dist/report/built-ins/index.js +0 -2
  196. package/dist/report/react/GroupSummary.d.ts +0 -8
  197. package/dist/report/react/GroupSummary.js +0 -8
  198. package/dist/report/react/RunOverview.d.ts +0 -8
  199. package/dist/report/react/RunOverview.js +0 -12
  200. package/docs-site/zh/example/ai-agent-application.mdx +0 -152
  201. package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
  202. package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
  203. package/docs-site/zh/example/showcase.mdx +0 -39
  204. package/src/report/built-in-user-parity.test.tsx +0 -597
  205. package/src/report/built-ins/experiment-comparison.tsx +0 -179
  206. package/src/report/built-ins/index.ts +0 -7
  207. package/src/report/react/GroupSummary.tsx +0 -66
  208. package/src/report/react/RunOverview.tsx +0 -109
  209. /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
  210. /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
  211. /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
  212. /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
  213. /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
  214. /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
  215. /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
  216. /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
  217. /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
  218. /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
@@ -1,18 +1,10 @@
1
1
  import { jsx as _jsx } from "react/jsx-runtime";
2
- // 官方双面组件的装配点:web 面(./react/ 的纯 React 组件)+ text 面(./text/faces.ts)
3
- // + 挂在组件上的 data 计算函数。faces 两键必填 —— 配对是结构义务,不是配对表;
4
- // MetricBars.data 就是 MetricMatrix.data(同一份矩阵数据的另一种摆法),别名显式化。
5
- //
6
- // 官方组件在宿主里自动接上证据室:web 面的 attemptHref 缺省取 ctx.attemptHref
7
- // (宿主注入的证据室深链);显式传 prop 可覆盖(嵌进自己应用时自定去处)。三个实体列表
8
- // (ExperimentList / EvalList / AttemptList)没有这个覆盖口子——它们的 locator 徽标恒经
9
- // ctx.attemptHref / ctx.attemptCommand,不接受 attemptHref/attemptCommand prop(docs/feature/reports/library.md
10
- // 「嵌入自己的 React 页面」的函数签名没有这个参数),证据室深链在这三个组件上不是可选行为。
11
- import { defineComponent, isHostWebContextActive } from "./tree.js";
12
- import { attemptListData, deltaData, evalListData, experimentListData, groupSummaryData, lineData, matrixData, overviewData, scatterData, scoreboardData, tableData, } from "./compute.js";
13
- import { attemptListText, barsText, deltaText, evalListText, experimentListText, groupSummaryText, lineText, matrixText, overviewText, scatterText, scoreboardText, tableText, } from "./text/faces.js";
14
- import { RunOverview as RunOverviewWeb } from "./react/RunOverview.js";
15
- import { GroupSummary as GroupSummaryWeb } from "./react/GroupSummary.js";
2
+ import { defineComponent, isHostWebContextActive, memoFetchOf, } from "./tree.js";
3
+ import { attemptListData, deltaTableData, evalListData, experimentComparisonData, experimentListData, metricLineData, metricMatrixData, metricScatterData, metricTableData, scopeSummaryData, scoreboardData, } from "./compute.js";
4
+ import { collectItems, locatorOf, resolveInput } from "./aggregate.js";
5
+ import { attemptListText, deltaText, evalListText, experimentComparisonText, experimentListText, barsText, lineText, matrixText, scatterText, scoreboardText, scopeSummaryText, tableText, } from "./text/faces.js";
6
+ import { ScopeSummary as ScopeSummaryWeb } from "./react/ScopeSummary.js";
7
+ import { ExperimentComparisonView } from "./react/ExperimentComparison.js";
16
8
  import { ExperimentList as ExperimentListWeb } from "./react/ExperimentList.js";
17
9
  import { EvalList as EvalListWeb } from "./react/EvalList.js";
18
10
  import { AttemptList as AttemptListWeb } from "./react/AttemptList.js";
@@ -23,110 +15,381 @@ import { Scoreboard as ScoreboardWeb } from "./react/Scoreboard.js";
23
15
  import { MetricScatter as MetricScatterWeb } from "./react/MetricScatter.js";
24
16
  import { MetricLine as MetricLineWeb } from "./react/MetricLine.js";
25
17
  import { DeltaTable as DeltaTableWeb } from "./react/DeltaTable.js";
26
- // ───────────────────────── 装配 ─────────────────────────
27
- const SCATTER_UNRESOLVED_MESSAGE = "MetricScatter received unresolved (selection-form) props outside the report host pipeline. " +
28
- "Render through defineReport + niceeval show/view (or renderReportToText/renderReportToStaticHtml), " +
29
- "or precompute with `await MetricScatter.data(selection, options)` and pass the result as `data`.";
30
- /** 渲染面(web / text)只吃已解析的数据形态;selection 形态漏解析时直说,而不是画一张空组件。 */
31
- function requireScatterData(props) {
32
- if (props.data === undefined)
33
- throw new Error(SCATTER_UNRESOLVED_MESSAGE);
34
- return props.data;
18
+ // ───────────────────────── data 结构校验(版本漂移防线)─────────────────────────
19
+ function isObject(value) {
20
+ return typeof value === "object" && value !== null && !Array.isArray(value);
35
21
  }
36
- /** 页头 KPI 条:何时跑的、几个配置、几道题、通过率、总成本;Selection 的警告随行显示在条内。 */
37
- export const RunOverview = Object.assign(defineComponent({
38
- web: (props, ctx) => _jsx(RunOverviewWeb, { ...props, locale: props.locale ?? ctx.locale }),
39
- text: ({ data }, ctx) => overviewText(data, ctx),
40
- }), { data: overviewData });
41
- RunOverview.displayName = "RunOverview";
42
- /**
43
- * 组摘要:一组 experiment(典型用法是同一 `<Section>` 内的全部 experiment)的紧凑统计——
44
- * 通过率(旧 GroupSelector 卡片口径)、experiment/eval/attempt 数、eval 级折叠计票、
45
- * 总成本、最后运行时间。
46
- */
47
- export const GroupSummary = Object.assign(defineComponent({
48
- web: (props, ctx) => _jsx(GroupSummaryWeb, { ...props, locale: props.locale ?? ctx.locale }),
49
- text: ({ data }, ctx) => groupSummaryText(data, ctx),
50
- }), { data: groupSummaryData });
51
- GroupSummary.displayName = "GroupSummary";
52
- /**
53
- * 实验列表:每项一个 experiment,固定展示身份、配置(flags)、Eval 判定构成和官方两级聚合
54
- * 汇总指标;展开到这个 experiment Eval,每道题内联该题全部 Attempt locator 徽标。
55
- * 没有 selection-form:`ExperimentList.data(selection)` 产出普通数组,过滤由报告作者对
56
- * 数组调用 `.filter()`。
57
- */
58
- export const ExperimentList = Object.assign(defineComponent({
59
- web: (props, ctx) => _jsx(ExperimentListWeb, { ...props, locale: props.locale ?? ctx.locale, attemptHref: ctx.attemptHref }),
60
- text: ({ items, relativeTo }, ctx) => experimentListText(items, ctx, relativeTo),
61
- }), { data: experimentListData });
62
- ExperimentList.displayName = "ExperimentList";
22
+ function isCell(value) {
23
+ return (isObject(value) &&
24
+ "value" in value &&
25
+ "display" in value &&
26
+ typeof value.samples === "number" &&
27
+ typeof value.total === "number" &&
28
+ Array.isArray(value.refs));
29
+ }
30
+ function isTally(value) {
31
+ return (isObject(value) &&
32
+ typeof value.passed === "number" &&
33
+ typeof value.failed === "number" &&
34
+ typeof value.errored === "number" &&
35
+ typeof value.skipped === "number");
36
+ }
37
+ function dataShapeError(component, dataFnName, shape, problem) {
38
+ return new Error(`<${component}> received data that does not match the current ${shape} shape: ${problem}. ` +
39
+ `It may have been computed by a different niceeval version (component data carries no schemaVersion; the support window is same-version write and read). ` +
40
+ `Recompute it with ${dataFnName}() from this niceeval version, then re-render.`);
41
+ }
42
+ const validateTableData = (data) => {
43
+ if (!isObject(data))
44
+ return "expected an object";
45
+ if (typeof data.rowDimension !== "string")
46
+ return 'missing "rowDimension" (string)';
47
+ if (!Array.isArray(data.columns))
48
+ return 'missing "columns" (array)';
49
+ if (!Array.isArray(data.rows))
50
+ return 'missing "rows" (array)';
51
+ for (const row of data.rows) {
52
+ if (!isObject(row) || typeof row.key !== "string" || !isObject(row.cells)) {
53
+ return 'each row needs { key, cells }';
54
+ }
55
+ }
56
+ return null;
57
+ };
58
+ const validateMatrixData = (data) => {
59
+ if (!isObject(data))
60
+ return "expected an object";
61
+ if (typeof data.rowDimension !== "string" || typeof data.columnDimension !== "string") {
62
+ return 'missing "rowDimension" / "columnDimension" (string)';
63
+ }
64
+ if (!isObject(data.metric))
65
+ return 'missing "metric" (MetricColumn)';
66
+ if (!Array.isArray(data.cells))
67
+ return 'missing "cells" (array)';
68
+ return null;
69
+ };
70
+ const validateScatterData = (data) => {
71
+ if (!isObject(data))
72
+ return "expected an object";
73
+ if (typeof data.pointDimension !== "string")
74
+ return 'missing "pointDimension" (string)';
75
+ if (!isObject(data.x) || !isObject(data.y))
76
+ return 'missing "x" / "y" (MetricColumn)';
77
+ if (!Array.isArray(data.rows))
78
+ return 'missing "rows" (array)';
79
+ for (const row of data.rows) {
80
+ if (!isObject(row) || typeof row.key !== "string" || !isCell(row.x) || !isCell(row.y)) {
81
+ return "each row needs { key, x: MetricCell, y: MetricCell }";
82
+ }
83
+ }
84
+ return null;
85
+ };
86
+ const validateLineData = (data) => {
87
+ if (!isObject(data))
88
+ return "expected an object";
89
+ if (!isObject(data.x) || typeof data.x.key !== "string")
90
+ return 'missing "x" axis descriptor';
91
+ if (!isObject(data.y))
92
+ return 'missing "y" (MetricColumn)';
93
+ if (!Array.isArray(data.rows))
94
+ return 'missing "rows" (array)';
95
+ return null;
96
+ };
97
+ const validateScoreboardData = (data) => {
98
+ if (!isObject(data))
99
+ return "expected an object";
100
+ if (typeof data.rowDimension !== "string")
101
+ return 'missing "rowDimension" (string)';
102
+ if (!Array.isArray(data.questions))
103
+ return 'missing "questions" (array)';
104
+ if (typeof data.fullMarks !== "number")
105
+ return 'missing "fullMarks" (number)';
106
+ if (typeof data.ignoredEvals !== "number")
107
+ return 'missing "ignoredEvals" (number)';
108
+ if (!Array.isArray(data.rows))
109
+ return 'missing "rows" (array)';
110
+ for (const row of data.rows) {
111
+ if (!isObject(row) || !isObject(row.total) || typeof row.total.notRun !== "number") {
112
+ return 'each row needs { key, total: { value, display, notRun, unscorable, refs }, subjects }';
113
+ }
114
+ }
115
+ return null;
116
+ };
117
+ const validateDeltaData = (data) => {
118
+ if (!isObject(data))
119
+ return "expected an object";
120
+ if (typeof data.byDimension !== "string")
121
+ return 'missing "byDimension" (string)';
122
+ if (!Array.isArray(data.columns) || !Array.isArray(data.rows))
123
+ return 'missing "columns" / "rows" (array)';
124
+ for (const row of data.rows) {
125
+ if (!isObject(row) || row.label === undefined || !isObject(row.a) || !isObject(row.b)) {
126
+ return "each row needs { key, label, a, b, cells }";
127
+ }
128
+ }
129
+ return null;
130
+ };
131
+ const validateScopeSummaryData = (data) => {
132
+ if (!isObject(data))
133
+ return "expected an object";
134
+ if (!isObject(data.range))
135
+ return 'missing "range" ({ earliestStartedAt, latestStartedAt })';
136
+ if (!isTally(data.evalVerdicts) || !isTally(data.attemptVerdicts)) {
137
+ return 'missing "evalVerdicts" / "attemptVerdicts" tallies';
138
+ }
139
+ if (!isCell(data.endToEndPassRate) || !isCell(data.totalCostUSD)) {
140
+ return 'missing "endToEndPassRate" / "totalCostUSD" (MetricCell)';
141
+ }
142
+ return null;
143
+ };
144
+ const validateComparisonData = (data) => {
145
+ if (!isObject(data))
146
+ return "expected an object";
147
+ if (!Array.isArray(data.groups))
148
+ return 'missing "groups" (array)';
149
+ for (const group of data.groups) {
150
+ if (!isObject(group) || typeof group.key !== "string" || validateScopeSummaryData(group.summary) !== null) {
151
+ return "each group needs { key, summary, scatter, experiments }";
152
+ }
153
+ }
154
+ return null;
155
+ };
156
+ const validateExperimentListData = (data) => {
157
+ if (!Array.isArray(data))
158
+ return "expected an array of ExperimentListItem";
159
+ for (const item of data) {
160
+ if (!isObject(item) || typeof item.experimentId !== "string" || !isTally(item.evalVerdicts) || !isCell(item.endToEndPassRate)) {
161
+ return "each item needs { experimentId, evalVerdicts, endToEndPassRate, costUSD, durationMs, tokens, evalRows, … }";
162
+ }
163
+ }
164
+ return null;
165
+ };
166
+ const validateEvalListData = (data) => {
167
+ if (!Array.isArray(data))
168
+ return "expected an array of EvalListItem";
169
+ for (const item of data) {
170
+ if (!isObject(item) || typeof item.evalId !== "string" || !isCell(item.examScore) || !Array.isArray(item.attempts)) {
171
+ return "each item needs { experimentId, evalId, verdict, examScore, durationMs, costUSD, attempts }";
172
+ }
173
+ }
174
+ return null;
175
+ };
176
+ const validateAttemptListData = (data) => {
177
+ if (!Array.isArray(data))
178
+ return "expected an array of AttemptListItem";
179
+ for (const item of data) {
180
+ if (!isObject(item) || typeof item.evalId !== "string" || !("failureSummary" in item) || !("costUSD" in item)) {
181
+ return "each item needs { experimentId, evalId, verdict, failureSummary, moreFailures, examScore, durationMs, costUSD, locator }";
182
+ }
183
+ }
184
+ return null;
185
+ };
186
+ function makeDataComponent(def) {
187
+ const assertData = (data) => {
188
+ const problem = def.validate(data);
189
+ if (problem !== null)
190
+ throw dataShapeError(def.name, def.dataFnName, def.shapeName, problem);
191
+ return data;
192
+ };
193
+ const resolve = async (props, ctx) => {
194
+ const givenSpec = def.specKeys.filter((key) => props[key] !== undefined);
195
+ if (props.data !== undefined) {
196
+ if (givenSpec.length > 0 || props.input !== undefined) {
197
+ const extras = [...givenSpec, ...(props.input !== undefined ? ["input"] : [])];
198
+ throw new Error(`<${def.name}> got both \`data\` and spec field${extras.length > 1 ? "s" : ""} (${extras.join(", ")}) — the two data sources are exclusive and niceeval will not silently pick one. ` +
199
+ `Keep \`data\` (precomputed with ${def.dataFnName}()) and drop the spec fields, or drop \`data\` and let the pipeline compute from the spec.`);
200
+ }
201
+ assertData(props.data);
202
+ return props;
203
+ }
204
+ const options = {};
205
+ for (const key of givenSpec)
206
+ options[key] = props[key];
207
+ const input = props.input ?? ctx.input;
208
+ const data = await memoFetchOf(ctx)(def.dataFn, input, options, () => def.dataFn(input, options));
209
+ const rest = { ...props };
210
+ delete rest.input;
211
+ for (const key of def.specKeys)
212
+ delete rest[key];
213
+ return { ...rest, data };
214
+ };
215
+ const component = defineComponent({
216
+ resolve,
217
+ web: (props, ctx) => {
218
+ assertData(props.data);
219
+ return def.web(props, ctx);
220
+ },
221
+ text: (props, ctx) => {
222
+ assertData(props.data);
223
+ return def.text(props, ctx);
224
+ },
225
+ });
226
+ component.displayName = def.name;
227
+ return component;
228
+ }
229
+ /** 宿主内缺省接证据室,显式 prop 覆盖;宿主外不传 attemptHref 就是纯展示。 */
230
+ function hrefOf(props, ctx) {
231
+ return props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined);
232
+ }
233
+ /** 范围摘要卡:时间窗、数量、两级计票、端到端成功率与总成本。 */
234
+ export const ScopeSummary = makeDataComponent({
235
+ name: "ScopeSummary",
236
+ dataFnName: "scopeSummaryData",
237
+ shapeName: "ScopeSummaryData",
238
+ dataFn: (input) => scopeSummaryData(input),
239
+ specKeys: [],
240
+ validate: validateScopeSummaryData,
241
+ web: (props, ctx) => _jsx(ScopeSummaryWeb, { ...props, locale: props.locale ?? ctx.locale }),
242
+ text: (props, ctx) => scopeSummaryText(props.data, props.votes ?? "eval", ctx),
243
+ });
63
244
  /**
64
- * Eval 列表:每项一个 `experimentId + evalId`,固定展示判定、Attempt 数、分数、成本、耗时
65
- * 与失败原因;展开到这道题的 Attempt(与 AttemptList 同一份 AttemptListItem)。
245
+ * 内建报告的默认组合件:先把 input 按可比组分区,再为每组分别计算 ScopeSummary、
246
+ * 成本 × 端到端成功率散点和 ExperimentList。web 面持有完整组索引并一次聚焦一组;
247
+ * text 面命中多个组时只显示组索引与可执行的单组查看命令,命中单组时才输出完整散点与列表。
66
248
  */
67
- export const EvalList = Object.assign(defineComponent({
68
- web: (props, ctx) => _jsx(EvalListWeb, { ...props, locale: props.locale ?? ctx.locale, attemptHref: ctx.attemptHref }),
69
- text: ({ items }, ctx) => evalListText(items, ctx),
70
- }), { data: evalListData });
71
- EvalList.displayName = "EvalList";
249
+ export const ExperimentComparison = makeDataComponent({
250
+ name: "ExperimentComparison",
251
+ dataFnName: "experimentComparisonData",
252
+ shapeName: "ExperimentComparisonData",
253
+ dataFn: (input) => experimentComparisonData(input),
254
+ specKeys: [],
255
+ validate: validateComparisonData,
256
+ web: (props, ctx) => (_jsx(ExperimentComparisonView, { data: props.data, locale: props.locale ?? ctx.locale, className: props.className, attemptHref: isHostWebContextActive() ? ctx.attemptHref : undefined })),
257
+ text: (props, ctx) => experimentComparisonText(props.data, props.className, ctx),
258
+ });
259
+ /** 实验列表:每项一个 experiment,固定八列比较表 + 展开到 Eval / Attempt。 */
260
+ export const ExperimentList = makeDataComponent({
261
+ name: "ExperimentList",
262
+ dataFnName: "experimentListData",
263
+ shapeName: "ExperimentListItem[]",
264
+ dataFn: (input, options) => experimentListData(input, options),
265
+ specKeys: ["redact"],
266
+ validate: validateExperimentListData,
267
+ web: (props, ctx) => (_jsx(ExperimentListWeb, { data: props.data, filter: props.filter, relativeTo: props.relativeTo, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx) ?? ctx.attemptHref, className: props.className })),
268
+ text: (props, ctx) => experimentListText(props.data, ctx, props.relativeTo),
269
+ });
270
+ /** Eval 列表:每项一个 experimentId + evalId,展开到这道题的 Attempt。 */
271
+ export const EvalList = makeDataComponent({
272
+ name: "EvalList",
273
+ dataFnName: "evalListData",
274
+ shapeName: "EvalListItem[]",
275
+ dataFn: (input, options) => evalListData(input, options),
276
+ specKeys: ["redact"],
277
+ validate: validateEvalListData,
278
+ web: (props, ctx) => (_jsx(EvalListWeb, { data: props.data, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx) ?? ctx.attemptHref, className: props.className })),
279
+ text: (props, ctx) => evalListText(props.data, ctx),
280
+ });
281
+ /** Attempt 列表:实体列表的叶子层,每项一次 attempt 的判定、单行摘要与 locator。 */
282
+ export const AttemptList = makeDataComponent({
283
+ name: "AttemptList",
284
+ dataFnName: "attemptListData",
285
+ shapeName: "AttemptListItem[]",
286
+ dataFn: (input, options) => attemptListData(input, options),
287
+ specKeys: ["redact"],
288
+ validate: validateAttemptListData,
289
+ web: (props, ctx) => (_jsx(AttemptListWeb, { data: props.data, total: props.total, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx) ?? ctx.attemptHref, className: props.className })),
290
+ text: (props, ctx) => attemptListText(props.data, props.total, ctx),
291
+ });
72
292
  /**
73
- * Attempt 列表:实体列表的叶子层,每项一个 Attempt,固定展示判定、一条主失败断言摘要或
74
- * error 一层摘要与证据引用(locator)。完整 assertions / evidence 经 locator 下钻。它不预设只看失败;
75
- * 报告作者过滤 `AttemptListItem[]`、用 `.slice()` 限量,`total` 让渲染面如实报告剩余数量。
293
+ * 「现在有哪些失败要处理」的成品组合件:内部就是 attemptListData → 过滤 → AttemptList
294
+ * data 形态,与手写组合严格等价、没有私有能力(docs/feature/reports/library/entity-lists.md)
295
+ * verdict failed / errored,按 attempt 开始时间降序(同刻按 locator 字典序),
296
+ * 截断到 limit(默认 20),total 报告截断前总数。
76
297
  */
77
- export const AttemptList = Object.assign(defineComponent({
78
- web: (props, ctx) => _jsx(AttemptListWeb, { ...props, locale: props.locale ?? ctx.locale, attemptHref: ctx.attemptHref }),
79
- text: ({ items, total }, ctx) => attemptListText(items, total, ctx),
80
- }), { data: attemptListData });
81
- AttemptList.displayName = "AttemptList";
298
+ export const FailureList = defineComponent(async (props, ctx) => {
299
+ const input = props.input ?? ctx.scope;
300
+ const all = await attemptListData(input, props.redact !== undefined ? { redact: props.redact } : undefined);
301
+ // attempt 开始时间不在列表条目里(它不是列表展示字段);从同一 input 的读取面按 locator 对回。
302
+ const startedAtByLocator = new Map();
303
+ for (const item of collectItems(resolveInput(input).snapshots)) {
304
+ startedAtByLocator.set(locatorOf(item), item.attempt.result.startedAt ?? "");
305
+ }
306
+ const failures = all
307
+ .filter((item) => item.verdict === "failed" || item.verdict === "errored")
308
+ .sort((a, b) => {
309
+ const ta = startedAtByLocator.get(a.locator) ?? "";
310
+ const tb = startedAtByLocator.get(b.locator) ?? "";
311
+ if (ta !== tb)
312
+ return ta < tb ? 1 : -1; // 最近的失败在前
313
+ return a.locator < b.locator ? -1 : a.locator > b.locator ? 1 : 0;
314
+ });
315
+ const limit = props.limit ?? 20;
316
+ return (_jsx(AttemptList, { data: failures.slice(0, limit), total: failures.length, attemptHref: props.attemptHref, locale: props.locale, className: props.className }));
317
+ });
318
+ FailureList.displayName = "FailureList";
82
319
  /** 榜单:一行一个维度值、一列一个指标,回答「谁整体更好」。 */
83
- export const MetricTable = Object.assign(defineComponent({
84
- web: (props, ctx) => (_jsx(MetricTableWeb, { ...props, locale: props.locale ?? ctx.locale, attemptHref: props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined) })),
85
- text: ({ data }, ctx) => tableText(data, ctx),
86
- }), { data: tableData });
87
- MetricTable.displayName = "MetricTable";
320
+ export const MetricTable = makeDataComponent({
321
+ name: "MetricTable",
322
+ dataFnName: "metricTableData",
323
+ shapeName: "TableData",
324
+ dataFn: metricTableData,
325
+ specKeys: ["rows", "columns", "sort", "evals"],
326
+ validate: validateTableData,
327
+ web: (props, ctx) => (_jsx(MetricTableWeb, { data: props.data, filter: props.filter, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx), className: props.className })),
328
+ text: (props, ctx) => tableText(props.data, ctx),
329
+ });
88
330
  /** 逐题格子:行 × 列两个维度、格子里一个指标,回答「哪道题谁挂了」。 */
89
- export const MetricMatrix = Object.assign(defineComponent({
90
- web: (props, ctx) => (_jsx(MetricMatrixWeb, { ...props, locale: props.locale ?? ctx.locale, attemptHref: props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined) })),
91
- text: ({ data }, ctx) => matrixText(data, ctx),
92
- }), { data: matrixData });
93
- MetricMatrix.displayName = "MetricMatrix";
94
- /** 分组条形:同一份矩阵数据的另一种摆法;MetricBars.data 就是 MetricMatrix.data 的别名。 */
95
- export const MetricBars = Object.assign(defineComponent({
96
- web: (props, ctx) => (_jsx(MetricBarsWeb, { ...props, locale: props.locale ?? ctx.locale, attemptHref: props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined) })),
97
- text: ({ data }) => barsText(data),
98
- }), { data: matrixData });
99
- MetricBars.displayName = "MetricBars";
100
- /** 考试成绩单:总分 + 分科小计,固定分母、missing 如实报。 */
101
- export const Scoreboard = Object.assign(defineComponent({
102
- web: (props, ctx) => _jsx(ScoreboardWeb, { ...props, locale: props.locale ?? ctx.locale }),
103
- text: ({ data }, ctx) => scoreboardText(data, ctx),
104
- }), { data: scoreboardData });
105
- Scoreboard.displayName = "Scoreboard";
106
- /** 质量 × 成本 frontier:每个点一个配置、两个指标各占一轴,「好」的角落恒在右上。 */
107
- export const MetricScatter = Object.assign(defineComponent({
108
- resolve: async (props) => {
109
- if ("data" in props && props.data !== undefined)
110
- return props;
111
- const { selection, points, series, x, y, ...rest } = props;
112
- return { ...rest, data: await scatterData(selection, { points, series, x, y }) };
113
- },
114
- web: (props, ctx) => {
115
- requireScatterData(props);
116
- return _jsx(MetricScatterWeb, { ...props, locale: props.locale ?? ctx.locale });
117
- },
118
- text: (props, ctx) => scatterText(requireScatterData(props), ctx),
119
- }), { data: scatterData });
120
- MetricScatter.displayName = "MetricScatter";
121
- /** 趋势线:x experiment 声明的 flag(flag()),同系列按 x 排序连线。 */
122
- export const MetricLine = Object.assign(defineComponent({
123
- web: (props, ctx) => _jsx(MetricLineWeb, { ...props, locale: props.locale ?? ctx.locale }),
124
- text: ({ data }, ctx) => lineText(data, ctx),
125
- }), { data: lineData });
126
- MetricLine.displayName = "MetricLine";
127
- /** 成对对比:每行一对配置、格子里 A、B、Δ 三个值,涨跌好坏由 better 判定。 */
128
- export const DeltaTable = Object.assign(defineComponent({
129
- web: (props, ctx) => _jsx(DeltaTableWeb, { ...props, locale: props.locale ?? ctx.locale }),
130
- text: ({ data }, ctx) => deltaText(data, ctx),
131
- }), { data: deltaData });
132
- DeltaTable.displayName = "DeltaTable";
331
+ export const MetricMatrix = makeDataComponent({
332
+ name: "MetricMatrix",
333
+ dataFnName: "metricMatrixData",
334
+ shapeName: "MatrixData",
335
+ dataFn: metricMatrixData,
336
+ specKeys: ["rows", "columns", "cell", "evals"],
337
+ validate: validateMatrixData,
338
+ web: (props, ctx) => (_jsx(MetricMatrixWeb, { data: props.data, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx), className: props.className })),
339
+ text: (props, ctx) => matrixText(props.data, ctx),
340
+ });
341
+ /** 分组条形:同一份矩阵数据的另一种摆法;与 MetricMatrix 写同一份 spec 时只计算一次。 */
342
+ export const MetricBars = makeDataComponent({
343
+ name: "MetricBars",
344
+ dataFnName: "metricMatrixData",
345
+ shapeName: "MatrixData",
346
+ dataFn: metricMatrixData,
347
+ specKeys: ["rows", "columns", "cell", "evals"],
348
+ validate: validateMatrixData,
349
+ web: (props, ctx) => (_jsx(MetricBarsWeb, { data: props.data, locale: props.locale ?? ctx.locale, attemptHref: hrefOf(props, ctx), className: props.className })),
350
+ text: (props, ctx) => barsText(props.data, ctx),
351
+ });
352
+ /** 考试成绩单:总分 + 分科小计,固定分母、notRun / unscorable 分开如实报。 */
353
+ export const Scoreboard = makeDataComponent({
354
+ name: "Scoreboard",
355
+ dataFnName: "scoreboardData",
356
+ shapeName: "ScoreboardData",
357
+ dataFn: scoreboardData,
358
+ specKeys: ["rows", "questions", "subject", "weights", "fullMarks", "score"],
359
+ validate: validateScoreboardData,
360
+ web: (props, ctx) => _jsx(ScoreboardWeb, { data: props.data, locale: props.locale ?? ctx.locale, className: props.className }),
361
+ text: (props, ctx) => scoreboardText(props.data, ctx),
362
+ });
363
+ /** 两个指标之间的取舍:每个点一个维度值,x / y 各一个指标,series 只决定颜色和分组。 */
364
+ export const MetricScatter = makeDataComponent({
365
+ name: "MetricScatter",
366
+ dataFnName: "metricScatterData",
367
+ shapeName: "ScatterData",
368
+ dataFn: metricScatterData,
369
+ specKeys: ["points", "series", "x", "y", "evals"],
370
+ validate: validateScatterData,
371
+ web: (props, ctx) => (_jsx(MetricScatterWeb, { data: props.data, pointHref: props.pointHref, locale: props.locale ?? ctx.locale, className: props.className })),
372
+ text: (props, ctx) => scatterText(props.data, ctx),
373
+ });
374
+ /** 趋势线:x 是 NumericAxis(参数轴),点身份 = (series, x)。 */
375
+ export const MetricLine = makeDataComponent({
376
+ name: "MetricLine",
377
+ dataFnName: "metricLineData",
378
+ shapeName: "LineData",
379
+ dataFn: metricLineData,
380
+ specKeys: ["x", "series", "y", "evals"],
381
+ validate: validateLineData,
382
+ web: (props, ctx) => (_jsx(MetricLineWeb, { data: props.data, pointHref: props.pointHref, locale: props.locale ?? ctx.locale, className: props.className })),
383
+ text: (props, ctx) => lineText(props.data, ctx),
384
+ });
385
+ /** 成对对比:每行一对(字面声明或 pairsByFlag 派生),格子里 A、B、Δ 三个值。 */
386
+ export const DeltaTable = makeDataComponent({
387
+ name: "DeltaTable",
388
+ dataFnName: "deltaTableData",
389
+ shapeName: "DeltaData",
390
+ dataFn: deltaTableData,
391
+ specKeys: ["by", "pairs", "metrics", "evals"],
392
+ validate: validateDeltaData,
393
+ web: (props, ctx) => _jsx(DeltaTableWeb, { data: props.data, locale: props.locale ?? ctx.locale, className: props.className }),
394
+ text: (props, ctx) => deltaText(props.data, ctx),
395
+ });