niceeval 0.4.6 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/docs-site/zh/concepts/adapter.mdx +6 -6
  2. package/docs-site/zh/concepts/experiment.mdx +7 -7
  3. package/docs-site/zh/concepts/overview.mdx +1 -1
  4. package/docs-site/zh/concepts/tier.mdx +6 -6
  5. package/docs-site/zh/guides/agent-feedback-loop.mdx +64 -21
  6. package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
  7. package/docs-site/zh/guides/custom-reports.mdx +243 -0
  8. package/docs-site/zh/guides/experiments.mdx +3 -3
  9. package/docs-site/zh/guides/report-components.mdx +252 -0
  10. package/docs-site/zh/guides/reporters.mdx +1 -1
  11. package/docs-site/zh/guides/results-data.mdx +224 -0
  12. package/docs-site/zh/guides/runner.mdx +1 -1
  13. package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
  14. package/docs-site/zh/guides/viewing-results.mdx +147 -77
  15. package/docs-site/zh/guides/write-experiment.mdx +9 -9
  16. package/docs-site/zh/guides/write-send.mdx +6 -6
  17. package/docs-site/zh/index.mdx +1 -1
  18. package/docs-site/zh/introduction.mdx +1 -1
  19. package/docs-site/zh/reference/cli.mdx +11 -3
  20. package/docs-site/zh/reference/define-agent.mdx +5 -5
  21. package/docs-site/zh/reference/define-config.mdx +1 -1
  22. package/docs-site/zh/reference/define-eval.mdx +3 -3
  23. package/package.json +3 -2
  24. package/src/agents/ai-sdk.test.ts +1 -1
  25. package/src/agents/ai-sdk.ts +2 -2
  26. package/src/agents/claude-code.ts +1 -1
  27. package/src/agents/codex.ts +2 -2
  28. package/src/agents/streaming.test.ts +1 -1
  29. package/src/agents/types.ts +4 -4
  30. package/src/agents/ui-message-stream.test.ts +1 -1
  31. package/src/cli.ts +120 -18
  32. package/src/context/context.test.ts +1 -1
  33. package/src/context/context.ts +3 -3
  34. package/src/context/session.ts +2 -2
  35. package/src/context/types.ts +2 -2
  36. package/src/i18n/en.ts +29 -7
  37. package/src/i18n/zh-CN.ts +24 -2
  38. package/src/report/aggregate.ts +79 -50
  39. package/src/report/components.tsx +190 -0
  40. package/src/report/compute.ts +149 -84
  41. package/src/report/default-report.tsx +222 -0
  42. package/src/report/dual-face.test.tsx +527 -0
  43. package/src/report/format.ts +29 -0
  44. package/src/report/index.ts +79 -24
  45. package/src/report/load.ts +67 -0
  46. package/src/report/metrics.ts +2 -1
  47. package/src/report/param.ts +18 -0
  48. package/src/report/primitives.tsx +117 -0
  49. package/src/report/react/CaseList.tsx +2 -2
  50. package/src/report/react/DeltaTable.tsx +2 -2
  51. package/src/report/react/MetricBars.tsx +109 -0
  52. package/src/report/react/MetricLine.tsx +200 -0
  53. package/src/report/react/MetricMatrix.tsx +1 -1
  54. package/src/report/react/MetricScatter.tsx +1 -1
  55. package/src/report/react/MetricTable.tsx +1 -1
  56. package/src/report/react/RunOverview.tsx +4 -4
  57. package/src/report/react/Scoreboard.tsx +3 -3
  58. package/src/report/react/cell.tsx +1 -1
  59. package/src/report/react/fixtures.ts +98 -36
  60. package/src/report/react/format.ts +3 -28
  61. package/src/report/react/index.tsx +41 -24
  62. package/src/report/react/render.test.tsx +8 -8
  63. package/src/report/react/styles.css +77 -0
  64. package/src/report/report.test.ts +248 -128
  65. package/src/report/report.ts +69 -0
  66. package/src/report/text/faces.ts +356 -0
  67. package/src/report/text/layout.ts +125 -0
  68. package/src/report/text/plot.ts +127 -0
  69. package/src/report/tree.ts +220 -0
  70. package/src/report/types.ts +75 -27
  71. package/src/report/web.ts +40 -0
  72. package/src/results/copy.ts +95 -41
  73. package/src/results/format.ts +12 -11
  74. package/src/results/index.ts +34 -17
  75. package/src/results/open.ts +201 -81
  76. package/src/results/results.test.ts +595 -191
  77. package/src/results/select.ts +107 -55
  78. package/src/results/types.ts +137 -45
  79. package/src/results/writer.ts +258 -0
  80. package/src/runner/attempt.ts +3 -3
  81. package/src/runner/fingerprint.ts +1 -1
  82. package/src/runner/reporters/artifacts.ts +38 -78
  83. package/src/runner/reporters/braintrust.test.ts +2 -2
  84. package/src/runner/reporters/braintrust.ts +3 -3
  85. package/src/runner/run.ts +1 -1
  86. package/src/runner/types.ts +18 -12
  87. package/src/show/compose.ts +215 -0
  88. package/src/show/index.ts +263 -0
  89. package/src/show/render.ts +456 -0
  90. package/src/show/show.test.ts +505 -0
  91. package/src/view/app/App.tsx +85 -24
  92. package/src/view/app/components/CostScoreChart.tsx +29 -10
  93. package/src/view/app/components/ExperimentTable.tsx +27 -9
  94. package/src/view/app/components/GroupSelector.tsx +1 -1
  95. package/src/view/app/i18n.ts +18 -3
  96. package/src/view/app/lib/attempt-route.test.ts +9 -9
  97. package/src/view/app/lib/attempt-route.ts +5 -5
  98. package/src/view/app/lib/outcome.ts +1 -3
  99. package/src/view/app/lib/rows.ts +95 -15
  100. package/src/view/app/main.tsx +18 -6
  101. package/src/view/app/pages/RunsPage.tsx +4 -7
  102. package/src/view/app/pages/TracesPage.tsx +3 -8
  103. package/src/view/app/types.ts +50 -6
  104. package/src/view/client-dist/app.css +1 -1
  105. package/src/view/client-dist/app.js +20 -20
  106. package/src/view/data.test.ts +254 -0
  107. package/src/view/data.ts +336 -0
  108. package/src/view/index.ts +64 -10
  109. package/src/view/server.ts +31 -9
  110. package/src/view/shared/types.ts +47 -41
  111. package/src/view/styles.css +5 -0
  112. package/src/view/template.html +1 -0
  113. package/src/view/view-report.test.ts +308 -0
  114. package/docs-site/.mintignore +0 -13
  115. package/docs-site/AGENTS.md +0 -46
  116. package/docs-site/concepts/adapter.mdx +0 -287
  117. package/docs-site/concepts/assert.mdx +0 -243
  118. package/docs-site/concepts/drive.mdx +0 -118
  119. package/docs-site/concepts/evals.mdx +0 -108
  120. package/docs-site/concepts/experiment.mdx +0 -37
  121. package/docs-site/concepts/hitl.mdx +0 -83
  122. package/docs-site/concepts/judge.mdx +0 -129
  123. package/docs-site/concepts/overview.mdx +0 -98
  124. package/docs-site/concepts/tier.mdx +0 -42
  125. package/docs-site/docs-ref/00-index.md +0 -23
  126. package/docs-site/docs-ref/01-page-types.md +0 -43
  127. package/docs-site/docs-ref/03-style-rules.md +0 -45
  128. package/docs-site/docs-ref/04-code-examples.md +0 -37
  129. package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
  130. package/docs-site/docs-ref/06-checklists.md +0 -53
  131. package/docs-site/docs.json +0 -256
  132. package/docs-site/example/ai-agent-application.mdx +0 -148
  133. package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
  134. package/docs-site/example/claude-code-codex-skill.mdx +0 -147
  135. package/docs-site/example/showcase.mdx +0 -39
  136. package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
  137. package/docs-site/example/tier1-claude-sdk.mdx +0 -119
  138. package/docs-site/example/tier1-codex-sdk.mdx +0 -111
  139. package/docs-site/example/tier1-langgraph.mdx +0 -119
  140. package/docs-site/example/tier1-pi-sdk.mdx +0 -119
  141. package/docs-site/favicon.svg +0 -5
  142. package/docs-site/github-diff.css +0 -135
  143. package/docs-site/github-diff.js +0 -11
  144. package/docs-site/guides/authoring.mdx +0 -129
  145. package/docs-site/guides/ci-integration.mdx +0 -97
  146. package/docs-site/guides/connect-otel.mdx +0 -209
  147. package/docs-site/guides/connect-your-agent.mdx +0 -221
  148. package/docs-site/guides/dataset-fanout.mdx +0 -98
  149. package/docs-site/guides/experiments.mdx +0 -71
  150. package/docs-site/guides/fixtures.mdx +0 -147
  151. package/docs-site/guides/reporters.mdx +0 -113
  152. package/docs-site/guides/runner.mdx +0 -79
  153. package/docs-site/guides/sandbox-agent.mdx +0 -138
  154. package/docs-site/guides/sandbox-backends.mdx +0 -64
  155. package/docs-site/guides/scoring-guide.mdx +0 -92
  156. package/docs-site/guides/viewing-results.mdx +0 -195
  157. package/docs-site/guides/write-experiment.mdx +0 -81
  158. package/docs-site/guides/write-send.mdx +0 -347
  159. package/docs-site/index.mdx +0 -181
  160. package/docs-site/introduction.mdx +0 -141
  161. package/docs-site/quickstart.mdx +0 -136
  162. package/docs-site/reference/builtin-agents.mdx +0 -161
  163. package/docs-site/reference/capabilities.mdx +0 -76
  164. package/docs-site/reference/cli.mdx +0 -120
  165. package/docs-site/reference/define-agent.mdx +0 -168
  166. package/docs-site/reference/define-config.mdx +0 -41
  167. package/docs-site/reference/define-eval.mdx +0 -160
  168. package/docs-site/reference/events.mdx +0 -131
  169. package/docs-site/reference/expect.mdx +0 -112
  170. package/docs-site/tracker.js +0 -8
  171. package/src/report/react/data.ts +0 -17
  172. package/src/view/aggregate.ts +0 -103
  173. package/src/view/loader.test.ts +0 -61
  174. package/src/view/loader.ts +0 -257
@@ -1,20 +1,25 @@
1
- // 计算函数:快照 → 一份组件数据。跑在 Node 侧,产物是算好的、可序列化的普通 JSON
2
- // (终值 + 渲染提示,不含公式);前端只做渲染。
1
+ // 计算函数:选集 → 一份组件数据。跑在 Node 侧,产物是算好的、可序列化的普通 JSON
2
+ // (终值 + 渲染提示,不含公式);渲染面(web/text)只做展示。
3
+ //
4
+ // 这些函数不做顶层导出,而是挂在对应组件上(MetricTable.data / Scoreboard.data …,
5
+ // 见 components.tsx):配对打点即发现,泛化名不占顶层导出。
3
6
  //
4
7
  // 共同约定(docs/reports.md「边界与不变量」):
5
- // - 聚合前按身份键去重(dedupeAttempts);
8
+ // - 第一参收 Selection | Snapshot[];收 Selection 时 warnings 随行进 OverviewData;
9
+ // - 聚合前按身份键去重(dedupeAttempts;missing-startedAt 不去重、如实保留、不透出警告);
6
10
  // - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露;
7
11
  // - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
8
12
 
9
- import type { SnapshotHandle } from "../results/types.ts";
10
13
  import type {
11
14
  CaseListData,
12
15
  DeltaData,
13
- Dimension,
16
+ DimensionInput,
17
+ LineData,
14
18
  MatrixData,
15
19
  Metric,
16
20
  MetricCell,
17
21
  OverviewData,
22
+ ParamRef,
18
23
  ScatterData,
19
24
  ScoreboardData,
20
25
  TableData,
@@ -22,41 +27,50 @@ import type {
22
27
  import {
23
28
  applyAggregator,
24
29
  assertUniqueMetricNames,
30
+ collectItems,
25
31
  computeCell,
26
- dedupeAttempts,
27
32
  dimensionKey,
28
33
  dimensionName,
29
34
  displayValue,
30
35
  evalGroupOf,
36
+ evalIdOf,
31
37
  evalPrefixPredicate,
32
38
  evaluateMetric,
33
39
  experimentIdOf,
34
40
  filterItems,
35
41
  groupItems,
42
+ paramAxisValue,
43
+ resolveInput,
44
+ snapshotKeyOf,
36
45
  toColumn,
37
46
  type Item,
47
+ type SnapshotsInput,
38
48
  } from "./aggregate.ts";
39
49
  import { attemptCostUSD, examScore } from "./metrics.ts";
40
- import { formatPlainNumber } from "./format.ts";
50
+ import { formatMetricValue, formatPlainNumber } from "./format.ts";
41
51
 
42
- // ───────────────────────── table ─────────────────────────
52
+ // ───────────────────────── MetricTable.data ─────────────────────────
43
53
 
44
- export interface TableOptions {
45
- /** 行维度。 */
46
- rows: Dimension;
47
- /** 每列一个指标。 */
48
- columns: Metric[];
49
- /** 构建时排序,方向随 better(higher 降序,「好」的一头在上);缺数据行沉底。 */
54
+ export interface TableDataOptions<M extends readonly Metric[]> {
55
+ /** 行维度(内置 / 自定义 / param())。 */
56
+ rows: DimensionInput;
57
+ /** 每列一个指标;列键 = metric.name 的字面量,拼错编译不过。 */
58
+ columns: M;
59
+ /** 构建时排序,方向随 better(higher 降序,「好」的一头在上);缺数据行沉底。两面同口径,预排即终排。 */
50
60
  sort?: Metric;
51
61
  /** eval id 前缀过滤,同 CLI 位置参数语义。 */
52
62
  evals?: string | string[];
53
63
  }
54
64
 
55
- export async function table(snapshots: SnapshotHandle[], opts: TableOptions): Promise<TableData> {
56
- assertUniqueMetricNames(opts.columns, "table columns");
57
- const items = filterItems(dedupeAttempts(snapshots), opts.evals);
65
+ export async function tableData<const M extends readonly Metric[]>(
66
+ input: SnapshotsInput,
67
+ opts: TableDataOptions<M>,
68
+ ): Promise<TableData<M[number]["name"]>> {
69
+ assertUniqueMetricNames(opts.columns, "MetricTable.data columns");
70
+ const { snapshots } = resolveInput(input);
71
+ const items = filterItems(collectItems(snapshots), opts.evals);
58
72
  const groups = groupItems(items, opts.rows);
59
- const rows: TableData["rows"] = [];
73
+ const rows: { key: string; cells: Record<string, MetricCell> }[] = [];
60
74
  const sortCells = new Map<string, MetricCell>();
61
75
  for (const [key, group] of groups) {
62
76
  const cells: Record<string, MetricCell> = {};
@@ -78,19 +92,26 @@ export async function table(snapshots: SnapshotHandle[], opts: TableOptions): Pr
78
92
  return better === "lower" ? va - vb : vb - va;
79
93
  });
80
94
  }
81
- return { dimension: dimensionName(opts.rows), columns: opts.columns.map(toColumn), rows };
95
+ return {
96
+ dimension: dimensionName(opts.rows),
97
+ columns: opts.columns.map(toColumn),
98
+ rows,
99
+ } as TableData<M[number]["name"]>;
82
100
  }
83
101
 
84
- // ───────────────────────── matrix ─────────────────────────
102
+ // ───────────────────────── MetricMatrix.data(= MetricBars.data)─────────────────────────
85
103
 
86
- export interface MatrixOptions {
87
- rows: Dimension;
88
- columns: Dimension;
104
+ export interface MatrixDataOptions {
105
+ rows: DimensionInput;
106
+ columns: DimensionInput;
89
107
  cell: Metric;
108
+ /** eval id 前缀过滤,同 CLI 位置参数语义。 */
109
+ evals?: string | string[];
90
110
  }
91
111
 
92
- export async function matrix(snapshots: SnapshotHandle[], opts: MatrixOptions): Promise<MatrixData> {
93
- const items = dedupeAttempts(snapshots);
112
+ export async function matrixData(input: SnapshotsInput, opts: MatrixDataOptions): Promise<MatrixData> {
113
+ const { snapshots } = resolveInput(input);
114
+ const items = filterItems(collectItems(snapshots), opts.evals);
94
115
  // 稀疏分组:只有真有 attempt 的 (row, column) 组合成格;没有样本的格子不出现
95
116
  const groups = new Map<string, { row: string; column: string; items: Item[] }>();
96
117
  for (const item of items) {
@@ -113,13 +134,13 @@ export async function matrix(snapshots: SnapshotHandle[], opts: MatrixOptions):
113
134
  };
114
135
  }
115
136
 
116
- // ───────────────────────── scoreboard ─────────────────────────
137
+ // ───────────────────────── Scoreboard.data ─────────────────────────
117
138
 
118
- export interface ScoreboardOptions {
119
- /** 给谁打分(被打分的维度)。 */
120
- of: Dimension;
139
+ export interface ScoreboardDataOptions {
140
+ /** 给谁打分(被打分的维度);维度槽与 MetricTable.data 统一叫 rows。 */
141
+ rows: DimensionInput;
121
142
  /** 按什么分科;默认 "evalGroup"(考试里的「科目」)。 */
122
- subjects?: Dimension;
143
+ subjects?: DimensionInput;
123
144
  /** eval id 前缀 → 每题分值;未列默认 1;前缀重叠时最长的生效。 */
124
145
  weights?: Record<string, number>;
125
146
  /** 折算满分;默认 100。 */
@@ -136,29 +157,31 @@ export interface ScoreboardOptions {
136
157
  * 总分 = fullMarks × Σ(题得分 × 题分值) / Σ(题分值) Σ 遍历选中范围内全部题
137
158
  * 没跑到的题挣 0 分但留在分母里,missing 如实报 —— 这是显式的考试契约,不是「null ≠ 0」的例外。
138
159
  */
139
- export async function scoreboard(
140
- snapshots: SnapshotHandle[],
141
- opts: ScoreboardOptions,
160
+ export async function scoreboardData(
161
+ input: SnapshotsInput,
162
+ opts: ScoreboardDataOptions,
142
163
  ): Promise<ScoreboardData> {
164
+ const { snapshots } = resolveInput(input);
143
165
  const fullMarks = opts.fullMarks ?? 100;
144
166
  const scoreMetric = opts.score ?? examScore;
145
- const subjectsDim: Dimension = opts.subjects ?? "evalGroup";
167
+ const subjectsDim: DimensionInput = opts.subjects ?? "evalGroup";
146
168
  const match = evalPrefixPredicate(opts.evals);
147
- const items = filterItems(dedupeAttempts(snapshots), opts.evals);
169
+ const items = filterItems(collectItems(snapshots), opts.evals);
148
170
 
149
171
  // 题集(固定分母):选中范围内、任一快照声明覆盖或实际出现过的全部题
150
172
  const universe = new Set<string>();
151
173
  for (const snapshot of snapshots) {
152
- for (const id of snapshot.evalIds) if (match(id)) universe.add(id);
174
+ for (const e of snapshot.evals) if (match(e.id)) universe.add(e.id);
175
+ for (const id of snapshot.knownEvalIds ?? []) if (match(id)) universe.add(id);
153
176
  }
154
- for (const item of items) universe.add(item.attempt.result.id);
177
+ for (const item of items) universe.add(evalIdOf(item));
155
178
  const sortedUniverse = [...universe].sort();
156
179
 
157
180
  // 每题的科目:先从任一 attempt 解析(自定义 subjects 维度也能算);
158
181
  // 全程无 attempt 的题按内置规则兜底,自定义维度无从计算时如实标 "(unknown)"
159
182
  const subjectByEval = new Map<string, string>();
160
183
  for (const item of items) {
161
- const id = item.attempt.result.id;
184
+ const id = evalIdOf(item);
162
185
  if (!subjectByEval.has(id)) subjectByEval.set(id, dimensionKey(subjectsDim, item));
163
186
  }
164
187
  const subjectOf = (id: string): string => {
@@ -173,20 +196,18 @@ export async function scoreboard(
173
196
  const weights = Object.entries(opts.weights ?? {})
174
197
  .map(([prefix, weight]) => ({ prefix, weight }))
175
198
  .sort((a, b) => b.prefix.length - a.prefix.length);
176
- const weightOf = (id: string): number =>
177
- weights.find((w) => id.startsWith(w.prefix))?.weight ?? 1;
199
+ const weightOf = (id: string): number => weights.find((w) => id.startsWith(w.prefix))?.weight ?? 1;
178
200
 
179
- const groups = groupItems(items, opts.of);
201
+ const groups = groupItems(items, opts.rows);
180
202
  const rows: ScoreboardData["rows"] = [];
181
203
  for (const [key, group] of groups) {
182
- // 题得分:perEval 折叠(同 eval × 快照 内);同题出现在多个快照时取快照级值的均值。
183
- // TODO(results-lib):多快照同题的口径待与选择器对齐;常规姿势(latestPerExperiment)每题只有一个快照。
184
- const perSnapshot = new Map<string, Map<string, number[]>>(); // evalId → snapshot 身份 → 原始值
204
+ // 题得分:perEval 折叠(同 eval × 快照 内);同题出现在多个快照时取快照级值的均值
205
+ const perSnapshot = new Map<string, Map<string, number[]>>(); // evalId → 快照键 → 原始值
185
206
  for (const item of group) {
186
207
  const value = await evaluateMetric(scoreMetric, item.attempt);
187
208
  if (value === null) continue; // 测不了的 attempt 不进题得分;整题无样本 → missing
188
- const id = item.attempt.result.id;
189
- const snapKey = `${item.snapshot.experimentId} @ ${item.snapshot.startedAt}`;
209
+ const id = evalIdOf(item);
210
+ const snapKey = snapshotKeyOf(item.snapshot);
190
211
  let bySnap = perSnapshot.get(id);
191
212
  if (!bySnap) perSnapshot.set(id, (bySnap = new Map()));
192
213
  const bucket = bySnap.get(snapKey);
@@ -208,7 +229,9 @@ export async function scoreboard(
208
229
  for (const id of sortedUniverse) {
209
230
  const subjectKey = subjectOf(id);
210
231
  let subject = subjects.get(subjectKey);
211
- if (!subject) subjects.set(subjectKey, (subject = { key: subjectKey, earned: 0, possible: 0, evals: 0, missing: 0 }));
232
+ if (!subject) {
233
+ subjects.set(subjectKey, (subject = { key: subjectKey, earned: 0, possible: 0, evals: 0, missing: 0 }));
234
+ }
212
235
  const weight = weightOf(id);
213
236
  const got = scoreByEval.get(id);
214
237
  subject.earned += (got ?? 0) * weight;
@@ -226,22 +249,23 @@ export async function scoreboard(
226
249
  rows.push({ key, total: { value, display: formatPlainNumber(value) }, subjects: [...subjects.values()] });
227
250
  }
228
251
 
229
- return { of: dimensionName(opts.of), fullMarks, weights, rows };
252
+ return { dimension: dimensionName(opts.rows), fullMarks, weights, rows };
230
253
  }
231
254
 
232
- // ───────────────────────── scatter ─────────────────────────
255
+ // ───────────────────────── MetricScatter.data ─────────────────────────
233
256
 
234
- export interface ScatterOptions {
257
+ export interface ScatterDataOptions {
235
258
  /** 点维度:每个点 = 该组 attempt 的聚合。 */
236
- points: Dimension;
259
+ points: DimensionInput;
237
260
  /** 可选:同系列的点连成线;省略 = 纯散点。 */
238
- series?: Dimension;
261
+ series?: DimensionInput;
239
262
  x: Metric;
240
263
  y: Metric;
241
264
  }
242
265
 
243
- export async function scatter(snapshots: SnapshotHandle[], opts: ScatterOptions): Promise<ScatterData> {
244
- const items = dedupeAttempts(snapshots);
266
+ export async function scatterData(input: SnapshotsInput, opts: ScatterDataOptions): Promise<ScatterData> {
267
+ const { snapshots } = resolveInput(input);
268
+ const items = collectItems(snapshots);
245
269
  const groups = groupItems(items, opts.points);
246
270
  const rows: ScatterData["rows"] = [];
247
271
  for (const [key, group] of groups) {
@@ -262,18 +286,50 @@ export async function scatter(snapshots: SnapshotHandle[], opts: ScatterOptions)
262
286
  };
263
287
  }
264
288
 
265
- // ───────────────────────── overview ─────────────────────────
289
+ // ───────────────────────── MetricLine.data ─────────────────────────
266
290
 
267
- export interface OverviewOptions {
268
- /** 选择器(latestPerExperiment 等)的警告,原样透传给 RunOverview 渲染。 */
269
- warnings?: string[];
291
+ export interface LineDataOptions {
292
+ /** x 轴:experiment 声明的 param(数值),不解析 experiment 命名。 */
293
+ x: ParamRef;
294
+ y: Metric;
295
+ /** 可选:每个系列一条线(param 或普通维度);省略 = 单系列。 */
296
+ series?: DimensionInput;
270
297
  }
271
298
 
272
- export async function overview(
273
- snapshots: SnapshotHandle[],
274
- opts?: OverviewOptions,
275
- ): Promise<OverviewData> {
276
- const items = dedupeAttempts(snapshots);
299
+ /** 每个点 = 一个 experiment 的聚合;同系列的点按 x 排序连线(排序在组件面,数据保持分组序)。 */
300
+ export async function lineData(input: SnapshotsInput, opts: LineDataOptions): Promise<LineData> {
301
+ const { snapshots } = resolveInput(input);
302
+ const items = collectItems(snapshots);
303
+ const groups = groupItems(items, "experiment");
304
+ const rows: LineData["rows"] = [];
305
+ for (const [key, group] of groups) {
306
+ const x = paramAxisValue(opts.x, group[0]); // param 是 experiment 级声明,组内一致
307
+ rows.push({
308
+ key,
309
+ series: opts.series ? dimensionKey(opts.series, group[0]) : undefined,
310
+ x,
311
+ xDisplay: x === null ? "" : formatMetricValue(x, opts.x.unit),
312
+ y: await computeCell(opts.y, group),
313
+ });
314
+ }
315
+ return {
316
+ x: {
317
+ key: opts.x.name,
318
+ label: typeof opts.x.label === "string" ? opts.x.label : opts.x.name,
319
+ unit: opts.x.unit,
320
+ },
321
+ series: opts.series ? dimensionName(opts.series) : undefined,
322
+ y: toColumn(opts.y),
323
+ rows,
324
+ };
325
+ }
326
+
327
+ // ───────────────────────── RunOverview.data ─────────────────────────
328
+
329
+ /** 选集的 warnings 随行进 OverviewData,RunOverview 直接渲染 —— 诚实不靠使用者记得接线。 */
330
+ export async function overviewData(input: SnapshotsInput): Promise<OverviewData> {
331
+ const { snapshots, warnings } = resolveInput(input);
332
+ const items = collectItems(snapshots);
277
333
  const evalIds = new Set<string>();
278
334
  let passed = 0;
279
335
  let failed = 0;
@@ -281,9 +337,9 @@ export async function overview(
281
337
  let skipped = 0;
282
338
  let durationMs = 0;
283
339
  let costUSD: number | null = null; // 任一 attempt 报了成本才有;全缺 = null,不编 0
284
- for (const { attempt } of items) {
285
- const result = attempt.result;
286
- evalIds.add(result.id);
340
+ for (const item of items) {
341
+ const result = item.attempt.result;
342
+ evalIds.add(evalIdOf(item));
287
343
  switch (result.outcome) {
288
344
  case "passed":
289
345
  passed += 1;
@@ -310,34 +366,41 @@ export async function overview(
310
366
  startedAt: s.startedAt,
311
367
  })),
312
368
  totals: { evals: evalIds.size, attempts: items.length, passed, failed, errored, skipped, costUSD, durationMs },
313
- warnings: [...(opts?.warnings ?? [])],
369
+ warnings: [...warnings],
314
370
  };
315
371
  }
316
372
 
317
- // ───────────────────────── delta ─────────────────────────
373
+ // ───────────────────────── DeltaTable.data ─────────────────────────
318
374
 
319
375
  export interface DeltaPair {
320
- /** 基线侧 experimentId。 */
376
+ /** 基线侧:experiment id,或快照键 "<experimentId> @ <startedAt>"(时间轴对比用手挑的快照数组)。 */
321
377
  a: string;
322
- /** 对比侧 experimentId。 */
378
+ /** 对比侧,同上。 */
323
379
  b: string;
324
380
  label?: string;
325
381
  }
326
382
 
327
- export interface DeltaOptions {
383
+ export interface DeltaDataOptions<M extends readonly Metric[]> {
328
384
  /** 每行一对:B 相对 A。 */
329
385
  pairs: DeltaPair[];
330
- metrics: Metric[];
386
+ metrics: M;
331
387
  }
332
388
 
333
- export async function delta(snapshots: SnapshotHandle[], opts: DeltaOptions): Promise<DeltaData> {
334
- assertUniqueMetricNames(opts.metrics, "delta metrics");
335
- const items = dedupeAttempts(snapshots);
389
+ export async function deltaData<const M extends readonly Metric[]>(
390
+ input: SnapshotsInput,
391
+ opts: DeltaDataOptions<M>,
392
+ ): Promise<DeltaData<M[number]["name"]>> {
393
+ assertUniqueMetricNames(opts.metrics, "DeltaTable.data metrics");
394
+ const { snapshots } = resolveInput(input);
395
+ const items = collectItems(snapshots);
396
+ // 一侧的键既匹配 experiment id 也匹配快照键 —— 与 "snapshot" 维度同一格式,不另造对比语义
397
+ const sideItems = (key: string) =>
398
+ items.filter((item) => experimentIdOf(item) === key || snapshotKeyOf(item.snapshot) === key);
336
399
  const rows: DeltaData["rows"] = [];
337
400
  for (const pair of opts.pairs) {
338
- const aItems = items.filter((item) => experimentIdOf(item) === pair.a);
339
- const bItems = items.filter((item) => experimentIdOf(item) === pair.b);
340
- const cells: DeltaData["rows"][number]["cells"] = {};
401
+ const aItems = sideItems(pair.a);
402
+ const bItems = sideItems(pair.b);
403
+ const cells: Record<string, DeltaData["rows"][number]["cells"][string]> = {};
341
404
  for (const metric of opts.metrics) {
342
405
  const a = await computeCell(metric, aItems);
343
406
  const b = await computeCell(metric, bItems);
@@ -351,18 +414,19 @@ export async function delta(snapshots: SnapshotHandle[], opts: DeltaOptions): Pr
351
414
  cells,
352
415
  });
353
416
  }
354
- return { columns: opts.metrics.map(toColumn), rows };
417
+ return { columns: opts.metrics.map(toColumn), rows } as DeltaData<M[number]["name"]>;
355
418
  }
356
419
 
357
420
  function deltaDisplay(metric: Metric, delta: number | null): string {
358
421
  if (delta === null) return "—"; // 任一侧缺数据:Δ 显示为缺,不硬算
422
+ if (delta === 0) return "±0";
359
423
  const text = displayValue(metric, delta); // 负号由格式化自带
360
424
  return delta > 0 ? `+${text}` : text;
361
425
  }
362
426
 
363
- // ───────────────────────── cases ─────────────────────────
427
+ // ───────────────────────── CaseList.data ─────────────────────────
364
428
 
365
- export interface CasesOptions {
429
+ export interface CaseListDataOptions {
366
430
  /** 要列出的判决;默认 failed + errored。 */
367
431
  outcomes?: ("failed" | "errored")[];
368
432
  /** 超出如实报 truncated,不静默截断。 */
@@ -371,10 +435,11 @@ export interface CasesOptions {
371
435
  redact?: (text: string) => string;
372
436
  }
373
437
 
374
- export async function cases(snapshots: SnapshotHandle[], opts?: CasesOptions): Promise<CaseListData> {
438
+ export async function caseListData(input: SnapshotsInput, opts?: CaseListDataOptions): Promise<CaseListData> {
439
+ const { snapshots } = resolveInput(input);
375
440
  const wanted = new Set<"failed" | "errored">(opts?.outcomes ?? ["failed", "errored"]);
376
441
  const redact = opts?.redact ?? ((text: string) => text);
377
- const selected = dedupeAttempts(snapshots).filter((item) => {
442
+ const selected = collectItems(snapshots).filter((item) => {
378
443
  const outcome = item.attempt.result.outcome;
379
444
  return (outcome === "failed" || outcome === "errored") && wanted.has(outcome);
380
445
  });
@@ -383,7 +448,7 @@ export async function cases(snapshots: SnapshotHandle[], opts?: CasesOptions): P
383
448
  const result = item.attempt.result;
384
449
  const cost = attemptCostUSD(result);
385
450
  return {
386
- eval: result.id,
451
+ eval: evalIdOf(item),
387
452
  experimentId: experimentIdOf(item),
388
453
  agent: result.agent,
389
454
  outcome: result.outcome as "failed" | "errored",
@@ -0,0 +1,222 @@
1
+ // DefaultReport:官方水位整块 —— 官方两扇门裸跑时渲染的就是它,零 props、纯声明。
2
+ // 「渲染面纯同步」与它不冲突,靠的是一个数据事实:官方水位(overview、榜单、失败清单)
3
+ // 只读瘦身条目、不碰任何懒加载工件,宿主对着已挑好的选集总是把这份数据备好、经上下文
4
+ // 注入(renderReportToText / renderReportToStaticHtml 里 prepareDefaultReportData)。
5
+ // 它渲染的口径钉死为宿主注入的那份选集 —— 零 props 意味着没有跟随的通道,这是锚点语义:
6
+ // 官方口径与自定义口径并排对照。默认无特权:数据全部来自公开计算函数。
7
+ //
8
+ // text 面即 `niceeval show` 的榜单(docs-site/zh/guides/viewing-results.mdx 的示例块是
9
+ // 行为规范):`Current verdicts` 头标注合成自几个 run、experiment 表带 eval 级折叠的
10
+ // evals 列、Failing 清单每条带判决时间与下钻命令。
11
+
12
+ import { basename } from "node:path";
13
+ import type { Selection } from "../results/index.ts";
14
+ import { foldEvalOutcome } from "../shared/outcome.ts";
15
+ import { defineComponent } from "./tree.ts";
16
+ import type { CaseListData, OverviewData, TableData } from "./types.ts";
17
+ import { caseListData, overviewData, tableData } from "./compute.ts";
18
+ import { attemptCostUSD, costUSD, durationMs, passRate } from "./metrics.ts";
19
+ import { RunOverview, MetricTable, CaseList } from "./components.tsx";
20
+ import { cellText } from "./text/faces.ts";
21
+ import { renderAlignedRows } from "./text/layout.ts";
22
+
23
+ /** 榜单的合成标注与 eval 级折叠(text 面用;与 verdicts/cases 出自同一选集)。 */
24
+ export interface VerdictBoardData {
25
+ /** 判决合成自几个物理 run。 */
26
+ composedFromRuns: number;
27
+ /** 参与合成的最新 run(目录名,即时间戳)。 */
28
+ latestRun?: string;
29
+ /** experiment → eval 级折叠计票(evals 列的 13/15)。 */
30
+ tallies: Record<string, { passedEvals: number; totalEvals: number }>;
31
+ /** eval 级折叠后失败/出错的题,带判决时间;新失败在前。 */
32
+ failing: {
33
+ evalId: string;
34
+ experimentId: string;
35
+ outcome: "failed" | "errored";
36
+ /** 最新 attempt 的第一条失败断言("gate calledTool(...)")或错误摘要。 */
37
+ reason?: string;
38
+ /** 判决产生的时刻(最新 attempt 的 startedAt,缺失退快照时刻)。 */
39
+ verdictAt?: string;
40
+ }[];
41
+ /** limit 之外还有几条,如实报。 */
42
+ failingTruncated: number;
43
+ }
44
+
45
+ export interface DefaultReportData {
46
+ overview: OverviewData;
47
+ /** 现刻榜单:experiment × (passRate, costUSD, durationMs)。 */
48
+ verdicts: TableData;
49
+ cases: CaseListData;
50
+ board: VerdictBoardData;
51
+ }
52
+
53
+ /** 失败清单的出厂截断;完整清单自己摆 <CaseList>(截断如实报剩余)。 */
54
+ const DEFAULT_CASE_LIMIT = 10;
55
+
56
+ function buildBoard(selection: Selection): VerdictBoardData {
57
+ const tallies: VerdictBoardData["tallies"] = {};
58
+ const failing: VerdictBoardData["failing"] = [];
59
+ const runs = new Set<string>();
60
+ let latestRunDir = "";
61
+
62
+ for (const snapshot of selection.snapshots) {
63
+ let passedEvals = 0;
64
+ for (const ev of snapshot.evals) {
65
+ for (const attempt of ev.attempts) {
66
+ runs.add(attempt.runDir.dir);
67
+ if (attempt.runDir.dir > latestRunDir) latestRunDir = attempt.runDir.dir;
68
+ }
69
+ const outcome = foldEvalOutcome(ev.attempts.map((a) => a.result));
70
+ if (outcome === "passed") {
71
+ passedEvals += 1;
72
+ continue;
73
+ }
74
+ if (outcome !== "failed" && outcome !== "errored") continue;
75
+ const latest = ev.attempts.reduce((a, b) => (b.result.attempt >= a.result.attempt ? b : a));
76
+ const failedAssertion = latest.result.assertions.find((a) => !a.passed);
77
+ const reason = failedAssertion
78
+ ? `${failedAssertion.severity} ${failedAssertion.name}`
79
+ : latest.result.error !== undefined
80
+ ? `error: ${latest.result.error}`
81
+ : undefined;
82
+ failing.push({
83
+ evalId: ev.id,
84
+ experimentId: snapshot.experimentId,
85
+ outcome,
86
+ ...(reason !== undefined ? { reason } : {}),
87
+ ...(latest.result.startedAt !== undefined || snapshot.startedAt
88
+ ? { verdictAt: latest.result.startedAt ?? snapshot.startedAt }
89
+ : {}),
90
+ });
91
+ }
92
+ tallies[snapshot.experimentId] = { passedEvals, totalEvals: snapshot.evals.length };
93
+ }
94
+
95
+ failing.sort((a, b) => (b.verdictAt ?? "").localeCompare(a.verdictAt ?? ""));
96
+ const shown = failing.slice(0, DEFAULT_CASE_LIMIT);
97
+ return {
98
+ composedFromRuns: runs.size,
99
+ ...(latestRunDir ? { latestRun: basename(latestRunDir) } : {}),
100
+ tallies,
101
+ failing: shown,
102
+ failingTruncated: failing.length - shown.length,
103
+ };
104
+ }
105
+
106
+ /** 宿主渲染前备好官方水位:只读瘦身条目,代价可忽略。 */
107
+ export async function prepareDefaultReportData(selection: Selection): Promise<DefaultReportData> {
108
+ return {
109
+ overview: await overviewData(selection),
110
+ verdicts: await tableData(selection, {
111
+ rows: "experiment",
112
+ columns: [passRate, costUSD, durationMs],
113
+ sort: passRate,
114
+ }),
115
+ cases: await caseListData(selection, { limit: DEFAULT_CASE_LIMIT }),
116
+ board: buildBoard(selection),
117
+ };
118
+ }
119
+
120
+ let activeData: DefaultReportData | null = null;
121
+
122
+ /** 宿主(与渲染入口)用:在注入好的官方水位下同步渲染。 */
123
+ export function runWithDefaultReportData<T>(data: DefaultReportData, fn: () => T): T {
124
+ const prev = activeData;
125
+ activeData = data;
126
+ try {
127
+ return fn();
128
+ } finally {
129
+ activeData = prev;
130
+ }
131
+ }
132
+
133
+ function requireData(): DefaultReportData {
134
+ if (!activeData) {
135
+ throw new Error(
136
+ "<DefaultReport /> renders the host-injected selection; render the report via " +
137
+ "`niceeval show --report` / `niceeval view --report` (or renderReportToText / renderReportToStaticHtml). " +
138
+ "Outside a host, compose the same blocks yourself: RunOverview, MetricTable, CaseList.",
139
+ );
140
+ }
141
+ return activeData;
142
+ }
143
+
144
+ /** 判决时间的相对标注(text 面的 "41s ago";渲染面纯同步,Date.now 不算 IO)。 */
145
+ function agoText(iso: string | undefined, now: number): string {
146
+ if (!iso) return "";
147
+ const ms = now - Date.parse(iso);
148
+ if (!Number.isFinite(ms) || ms < 10_000) return "just now";
149
+ const seconds = Math.floor(ms / 1000);
150
+ if (seconds < 120) return `${seconds}s ago`;
151
+ const minutes = Math.floor(seconds / 60);
152
+ if (minutes < 120) return `${minutes}m ago`;
153
+ const hours = Math.floor(minutes / 60);
154
+ if (hours < 48) return `${hours}h ago`;
155
+ return `${Math.floor(hours / 24)}d ago`;
156
+ }
157
+
158
+ /** show 榜单(viewing-results.mdx「榜单」块):头 + experiment 表 + Failing 清单。 */
159
+ function boardText(data: DefaultReportData): string {
160
+ const { verdicts, board, overview } = data;
161
+ const experiments = verdicts.rows.length;
162
+ const head = [
163
+ "Current verdicts",
164
+ `${experiments} ${experiments === 1 ? "experiment" : "experiments"}`,
165
+ `composed from ${board.composedFromRuns} ${board.composedFromRuns === 1 ? "run" : "runs"}`,
166
+ ...(board.latestRun ? [`latest ${board.latestRun}`] : []),
167
+ ].join(" · ");
168
+ const headBlock = [head, ...overview.warnings.map((w) => `! ${w.message}`)].join("\n");
169
+
170
+ // 表:experiment | evals(eval 级折叠计票)| pass | cost | duration(顺序随 verdicts 预排)
171
+ const header = [verdicts.dimension, "evals", "pass", "cost", "duration"];
172
+ const rows = verdicts.rows.map((row) => {
173
+ const tally = board.tallies[row.key];
174
+ const cells = row.cells as Record<string, TableData["rows"][number]["cells"][string]>;
175
+ return [
176
+ row.key,
177
+ tally ? `${tally.passedEvals}/${tally.totalEvals}` : "—",
178
+ cells[passRate.name] ? cellText(cells[passRate.name]) : "—",
179
+ cells[costUSD.name] ? cellText(cells[costUSD.name]) : "—",
180
+ cells[durationMs.name] ? cellText(cells[durationMs.name]) : "—",
181
+ ];
182
+ });
183
+ const table = renderAlignedRows([header, ...rows]);
184
+
185
+ const blocks = [headBlock, table];
186
+ if (board.failing.length > 0) {
187
+ const now = Date.now();
188
+ const aligned = renderAlignedRows(
189
+ board.failing.map((f) => [
190
+ `✗ ${f.evalId}`,
191
+ f.experimentId,
192
+ f.reason ?? f.outcome,
193
+ agoText(f.verdictAt, now),
194
+ ]),
195
+ ).split("\n");
196
+ const lines: string[] = ["Failing:"];
197
+ board.failing.forEach((f, i) => {
198
+ lines.push(` ${aligned[i]}`);
199
+ lines.push(` → niceeval show ${f.evalId}`);
200
+ });
201
+ if (board.failingTruncated > 0) lines.push(` (${board.failingTruncated} more not shown)`);
202
+ blocks.push(lines.join("\n"));
203
+ }
204
+ return blocks.join("\n\n");
205
+ }
206
+
207
+ export const DefaultReport = defineComponent<Record<string, never>>({
208
+ web() {
209
+ const data = requireData();
210
+ return (
211
+ <div className="nre nre-default-report">
212
+ <RunOverview data={data.overview} />
213
+ <MetricTable data={data.verdicts} />
214
+ {data.cases.rows.length > 0 && <CaseList data={data.cases} />}
215
+ </div>
216
+ );
217
+ },
218
+ text() {
219
+ return boardText(requireData());
220
+ },
221
+ });
222
+ DefaultReport.displayName = "DefaultReport";