niceeval 0.4.5 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/docs-site/zh/concepts/adapter.mdx +6 -6
  2. package/docs-site/zh/concepts/experiment.mdx +7 -7
  3. package/docs-site/zh/concepts/overview.mdx +1 -1
  4. package/docs-site/zh/concepts/tier.mdx +6 -6
  5. package/docs-site/zh/guides/agent-feedback-loop.mdx +63 -20
  6. package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
  7. package/docs-site/zh/guides/custom-reports.mdx +243 -0
  8. package/docs-site/zh/guides/experiments.mdx +3 -3
  9. package/docs-site/zh/guides/report-components.mdx +252 -0
  10. package/docs-site/zh/guides/reporters.mdx +1 -1
  11. package/docs-site/zh/guides/results-data.mdx +224 -0
  12. package/docs-site/zh/guides/runner.mdx +1 -1
  13. package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
  14. package/docs-site/zh/guides/viewing-results.mdx +147 -77
  15. package/docs-site/zh/guides/write-experiment.mdx +9 -9
  16. package/docs-site/zh/guides/write-send.mdx +6 -6
  17. package/docs-site/zh/index.mdx +1 -1
  18. package/docs-site/zh/introduction.mdx +1 -1
  19. package/docs-site/zh/reference/cli.mdx +10 -2
  20. package/docs-site/zh/reference/define-agent.mdx +5 -5
  21. package/docs-site/zh/reference/define-config.mdx +1 -1
  22. package/docs-site/zh/reference/define-eval.mdx +3 -3
  23. package/package.json +3 -2
  24. package/src/agents/ai-sdk.test.ts +1 -1
  25. package/src/agents/ai-sdk.ts +2 -2
  26. package/src/agents/bub.ts +14 -4
  27. package/src/agents/claude-code.ts +1 -1
  28. package/src/agents/codex.ts +2 -2
  29. package/src/agents/streaming.test.ts +1 -1
  30. package/src/agents/types.ts +4 -4
  31. package/src/agents/ui-message-stream.test.ts +1 -1
  32. package/src/cli.ts +104 -14
  33. package/src/context/context.test.ts +1 -1
  34. package/src/context/context.ts +3 -3
  35. package/src/context/session.ts +2 -2
  36. package/src/context/types.ts +2 -2
  37. package/src/i18n/en.ts +29 -7
  38. package/src/i18n/zh-CN.ts +24 -2
  39. package/src/report/aggregate.ts +79 -50
  40. package/src/report/components.tsx +190 -0
  41. package/src/report/compute.ts +149 -84
  42. package/src/report/default-report.tsx +222 -0
  43. package/src/report/dual-face.test.tsx +527 -0
  44. package/src/report/format.ts +29 -0
  45. package/src/report/index.ts +79 -24
  46. package/src/report/load.ts +67 -0
  47. package/src/report/metrics.ts +2 -1
  48. package/src/report/param.ts +18 -0
  49. package/src/report/primitives.tsx +117 -0
  50. package/src/report/react/CaseList.tsx +2 -2
  51. package/src/report/react/DeltaTable.tsx +2 -2
  52. package/src/report/react/MetricBars.tsx +109 -0
  53. package/src/report/react/MetricLine.tsx +200 -0
  54. package/src/report/react/MetricMatrix.tsx +1 -1
  55. package/src/report/react/MetricScatter.tsx +1 -1
  56. package/src/report/react/MetricTable.tsx +1 -1
  57. package/src/report/react/RunOverview.tsx +4 -4
  58. package/src/report/react/Scoreboard.tsx +3 -3
  59. package/src/report/react/cell.tsx +1 -1
  60. package/src/report/react/fixtures.ts +98 -36
  61. package/src/report/react/format.ts +3 -28
  62. package/src/report/react/index.tsx +41 -24
  63. package/src/report/react/render.test.tsx +8 -8
  64. package/src/report/react/styles.css +77 -0
  65. package/src/report/report.test.ts +248 -128
  66. package/src/report/report.ts +69 -0
  67. package/src/report/text/faces.ts +356 -0
  68. package/src/report/text/layout.ts +125 -0
  69. package/src/report/text/plot.ts +127 -0
  70. package/src/report/tree.ts +220 -0
  71. package/src/report/types.ts +75 -27
  72. package/src/report/web.ts +40 -0
  73. package/src/results/copy.ts +95 -41
  74. package/src/results/format.ts +12 -11
  75. package/src/results/index.ts +34 -17
  76. package/src/results/open.ts +201 -81
  77. package/src/results/results.test.ts +595 -191
  78. package/src/results/select.ts +107 -55
  79. package/src/results/types.ts +137 -45
  80. package/src/results/writer.ts +258 -0
  81. package/src/runner/attempt.ts +3 -3
  82. package/src/runner/fingerprint.ts +1 -1
  83. package/src/runner/reporters/artifacts.ts +38 -78
  84. package/src/runner/reporters/braintrust.test.ts +2 -2
  85. package/src/runner/reporters/braintrust.ts +3 -3
  86. package/src/runner/run.ts +1 -1
  87. package/src/runner/types.ts +18 -12
  88. package/src/show/compose.ts +215 -0
  89. package/src/show/index.ts +263 -0
  90. package/src/show/render.ts +456 -0
  91. package/src/show/show.test.ts +505 -0
  92. package/src/view/app/App.tsx +85 -24
  93. package/src/view/app/components/CostScoreChart.tsx +29 -10
  94. package/src/view/app/components/ExperimentTable.tsx +27 -9
  95. package/src/view/app/components/GroupSelector.tsx +1 -1
  96. package/src/view/app/i18n.ts +18 -3
  97. package/src/view/app/lib/attempt-route.test.ts +9 -9
  98. package/src/view/app/lib/attempt-route.ts +5 -5
  99. package/src/view/app/lib/outcome.ts +1 -3
  100. package/src/view/app/lib/rows.ts +95 -15
  101. package/src/view/app/main.tsx +18 -6
  102. package/src/view/app/pages/RunsPage.tsx +4 -7
  103. package/src/view/app/pages/TracesPage.tsx +3 -8
  104. package/src/view/app/types.ts +50 -6
  105. package/src/view/client-dist/app.css +1 -1
  106. package/src/view/client-dist/app.js +20 -20
  107. package/src/view/data.test.ts +254 -0
  108. package/src/view/data.ts +336 -0
  109. package/src/view/index.ts +64 -10
  110. package/src/view/server.ts +31 -9
  111. package/src/view/shared/types.ts +47 -41
  112. package/src/view/styles.css +5 -0
  113. package/src/view/template.html +1 -0
  114. package/src/view/view-report.test.ts +308 -0
  115. package/docs-site/.mintignore +0 -13
  116. package/docs-site/AGENTS.md +0 -46
  117. package/docs-site/concepts/adapter.mdx +0 -287
  118. package/docs-site/concepts/assert.mdx +0 -243
  119. package/docs-site/concepts/drive.mdx +0 -118
  120. package/docs-site/concepts/evals.mdx +0 -108
  121. package/docs-site/concepts/experiment.mdx +0 -37
  122. package/docs-site/concepts/hitl.mdx +0 -83
  123. package/docs-site/concepts/judge.mdx +0 -129
  124. package/docs-site/concepts/overview.mdx +0 -98
  125. package/docs-site/concepts/tier.mdx +0 -42
  126. package/docs-site/docs-ref/00-index.md +0 -23
  127. package/docs-site/docs-ref/01-page-types.md +0 -43
  128. package/docs-site/docs-ref/03-style-rules.md +0 -45
  129. package/docs-site/docs-ref/04-code-examples.md +0 -37
  130. package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
  131. package/docs-site/docs-ref/06-checklists.md +0 -53
  132. package/docs-site/docs.json +0 -256
  133. package/docs-site/example/ai-agent-application.mdx +0 -148
  134. package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
  135. package/docs-site/example/claude-code-codex-skill.mdx +0 -147
  136. package/docs-site/example/showcase.mdx +0 -39
  137. package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
  138. package/docs-site/example/tier1-claude-sdk.mdx +0 -119
  139. package/docs-site/example/tier1-codex-sdk.mdx +0 -111
  140. package/docs-site/example/tier1-langgraph.mdx +0 -119
  141. package/docs-site/example/tier1-pi-sdk.mdx +0 -119
  142. package/docs-site/favicon.svg +0 -5
  143. package/docs-site/github-diff.css +0 -135
  144. package/docs-site/github-diff.js +0 -11
  145. package/docs-site/guides/authoring.mdx +0 -129
  146. package/docs-site/guides/ci-integration.mdx +0 -97
  147. package/docs-site/guides/connect-otel.mdx +0 -209
  148. package/docs-site/guides/connect-your-agent.mdx +0 -221
  149. package/docs-site/guides/dataset-fanout.mdx +0 -98
  150. package/docs-site/guides/experiments.mdx +0 -71
  151. package/docs-site/guides/fixtures.mdx +0 -147
  152. package/docs-site/guides/reporters.mdx +0 -113
  153. package/docs-site/guides/runner.mdx +0 -79
  154. package/docs-site/guides/sandbox-agent.mdx +0 -138
  155. package/docs-site/guides/sandbox-backends.mdx +0 -64
  156. package/docs-site/guides/scoring-guide.mdx +0 -92
  157. package/docs-site/guides/viewing-results.mdx +0 -195
  158. package/docs-site/guides/write-experiment.mdx +0 -81
  159. package/docs-site/guides/write-send.mdx +0 -347
  160. package/docs-site/index.mdx +0 -181
  161. package/docs-site/introduction.mdx +0 -141
  162. package/docs-site/quickstart.mdx +0 -136
  163. package/docs-site/reference/builtin-agents.mdx +0 -161
  164. package/docs-site/reference/capabilities.mdx +0 -76
  165. package/docs-site/reference/cli.mdx +0 -120
  166. package/docs-site/reference/define-agent.mdx +0 -168
  167. package/docs-site/reference/define-config.mdx +0 -41
  168. package/docs-site/reference/define-eval.mdx +0 -160
  169. package/docs-site/reference/events.mdx +0 -131
  170. package/docs-site/reference/expect.mdx +0 -112
  171. package/docs-site/tracker.js +0 -8
  172. package/src/report/react/data.ts +0 -17
  173. package/src/view/aggregate.ts +0 -103
  174. package/src/view/loader.test.ts +0 -61
  175. package/src/view/loader.ts +0 -257
@@ -0,0 +1,254 @@
1
+ // view 数据层(data.ts)的单测:loader 收编到 openResults、统计收编到官方计算函数之后,
2
+ // 守护三件事——skipped 三种原因如实进 viewData(producer 感知的 npx 提示)、榜单是
3
+ // results.latest() 口径(不再跨历史合并)、跨快照去重让 Runs/Traces 不被 --resume 复印件灌票。
4
+ // 另含 loadLatestResultsPerEval 的续跑携带语义(从旧 loader.test.ts 移植,口径不变)。
5
+
6
+ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
7
+ import { tmpdir } from "node:os";
8
+ import { join } from "node:path";
9
+ import { afterEach, describe, expect, it } from "vitest";
10
+ import { IncompatibleResultsError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
11
+ import { createRunWriter } from "../results/index.ts";
12
+ import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
13
+
14
+ const roots: string[] = [];
15
+ async function makeRoot(): Promise<string> {
16
+ const root = await mkdtemp(join(tmpdir(), "niceeval-viewdata-"));
17
+ roots.push(root);
18
+ return root;
19
+ }
20
+ afterEach(async () => {
21
+ await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
22
+ });
23
+
24
+ function res(over: Partial<EvalResult> & Pick<EvalResult, "id" | "agent">): EvalResult {
25
+ return { outcome: "passed", attempt: 0, durationMs: 1000, assertions: [], ...over };
26
+ }
27
+
28
+ function summaryOf(results: EvalResult[], over: Partial<RunSummary> = {}): RunSummary {
29
+ const count = (o: EvalResult["outcome"]) => results.filter((r) => r.outcome === o).length;
30
+ return {
31
+ format: RESULTS_FORMAT,
32
+ schemaVersion: RESULTS_SCHEMA_VERSION,
33
+ producer: { name: "niceeval", version: "0.4.0" },
34
+ agent: results[0]?.agent ?? "agent",
35
+ startedAt: "2026-07-01T08:00:00.000Z",
36
+ completedAt: "2026-07-01T08:10:00.000Z",
37
+ passed: count("passed"),
38
+ failed: count("failed"),
39
+ skipped: count("skipped"),
40
+ errored: count("errored"),
41
+ durationMs: 60_000,
42
+ results,
43
+ ...over,
44
+ };
45
+ }
46
+
47
+ async function writeRun(root: string, dirName: string, summary: unknown): Promise<string> {
48
+ const dir = join(root, dirName);
49
+ await mkdir(dir, { recursive: true });
50
+ await writeFile(join(dir, "summary.json"), typeof summary === "string" ? summary : JSON.stringify(summary), "utf-8");
51
+ return dir;
52
+ }
53
+
54
+ describe("loadViewScan · skipped 三种原因进 viewData", () => {
55
+ it("incompatible-version / malformed / incomplete 都进 skippedRuns;niceeval 落盘拼 npx 命令,第三方如实报名字不拼", async () => {
56
+ const root = await makeRoot();
57
+ // 正常 run,页面照常渲染。
58
+ await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([res({ id: "q1", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-01T08:01:00.000Z" })]));
59
+ // 版本不同:niceeval 写的。
60
+ await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
61
+ // 版本不同:第三方 harness 写的。
62
+ await writeRun(root, "2026-07-03T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "otherharness", version: "1.2.3" } }));
63
+ // 坏 JSON。
64
+ await writeRun(root, "2026-07-04T08-00-00-000Z", "{not json");
65
+ // incomplete:有 attempt 工件、没有 summary(run 中途 crash)。
66
+ await mkdir(join(root, "2026-07-05T08-00-00-000Z", "q1", "bub", "default", "a0"), { recursive: true });
67
+ await writeFile(join(root, "2026-07-05T08-00-00-000Z", "q1", "bub", "default", "a0", "events.json"), "[]", "utf-8");
68
+
69
+ const { viewData } = await loadViewScan(root);
70
+ const byReason = new Map(viewData.skippedRuns!.map((s) => [s.dir, s]));
71
+ expect(viewData.skippedRuns).toHaveLength(4);
72
+
73
+ const niceevalSkip = [...byReason.values()].find((s) => s.producerName === "niceeval" && s.reason === "incompatible-version")!;
74
+ expect(niceevalSkip.schemaVersion).toBe(999);
75
+ expect(niceevalSkip.command).toContain("npx niceeval@9.9.9 view ");
76
+
77
+ const foreignSkip = [...byReason.values()].find((s) => s.producerName === "otherharness")!;
78
+ expect(foreignSkip.reason).toBe("incompatible-version");
79
+ expect(foreignSkip.producerVersion).toBe("1.2.3");
80
+ expect(foreignSkip.command).toBeUndefined(); // 第三方版本号拼 npx 是一句错误提示,不拼
81
+
82
+ const malformed = [...byReason.values()].find((s) => s.reason === "malformed")!;
83
+ expect(malformed.detail).toBe("invalid JSON");
84
+
85
+ const incomplete = [...byReason.values()].find((s) => s.reason === "incomplete")!;
86
+ expect(incomplete.dir).toContain("2026-07-05T08-00-00-000Z");
87
+
88
+ // 正常 run 照常进榜单,坏 run 不拖垮整页。
89
+ expect(viewData.table.rows.map((r) => r.key)).toEqual(["exp/a"]);
90
+ });
91
+
92
+ it("单文件模式指向版本不同的报告:抛 IncompatibleResultsError(CLI 打印提示退出)", async () => {
93
+ const root = await makeRoot();
94
+ const dir = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
95
+ await expect(loadViewScan(join(dir, "summary.json"))).rejects.toBeInstanceOf(IncompatibleResultsError);
96
+ });
97
+ });
98
+
99
+ describe("loadViewScan · 榜单是 latest 口径,不再跨历史合并", () => {
100
+ it("同一实验两次 run:榜单格子只反映最新快照;partial-coverage 警告经 overview.warnings 透传", async () => {
101
+ const root = await makeRoot();
102
+ // 周一全量:q1 失败、q2 通过。
103
+ await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
104
+ res({ id: "q1", agent: "bub", experimentId: "exp/a", outcome: "failed", startedAt: "2026-07-01T08:01:00.000Z" }),
105
+ res({ id: "q2", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-01T08:02:00.000Z" }),
106
+ ], { startedAt: "2026-07-01T08:00:00.000Z" }));
107
+ // 周二只补跑 q1(通过):latest 快照只盖 1/2 道题。
108
+ await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
109
+ res({ id: "q1", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-02T08:01:00.000Z" }),
110
+ ], { startedAt: "2026-07-02T08:00:00.000Z" }));
111
+
112
+ const { viewData } = await loadViewScan(root);
113
+ const row = viewData.table.rows.find((r) => r.key === "exp/a")!;
114
+ // 旧的 aggregateRows 会把两次 run 揉在一起(q1 一败一过);latest 口径只看周二快照:全过。
115
+ expect(row.cells["pass-rate"]!.value).toBe(1);
116
+ expect(row.cells["pass-rate"]!.refs.length).toBe(1);
117
+
118
+ // 快照元信息:latest 标记 + 每行可标注判决时间。
119
+ const latest = viewData.snapshots.filter((s) => s.latest);
120
+ expect(latest).toHaveLength(1);
121
+ expect(latest[0]!.startedAt).toBe("2026-07-02T08:00:00.000Z");
122
+ expect(viewData.composedRuns).toBe(1);
123
+ expect(viewData.lastRunAt).toBe("2026-07-02T08:00:00.000Z");
124
+
125
+ // 挑选警告随 OverviewData 进前端,不静默。
126
+ const kinds = viewData.overview.warnings.map((w) => w.kind);
127
+ expect(kinds).toContain("partial-coverage");
128
+ const partial = viewData.overview.warnings.find((w) => w.kind === "partial-coverage")!;
129
+ expect(partial.message).toContain("1 of 2");
130
+
131
+ // 历史快照仍在(Runs / Traces 吃全部),但不是 latest。
132
+ const historical = viewData.snapshots.filter((s) => !s.latest);
133
+ expect(historical).toHaveLength(1);
134
+ expect(historical[0]!.results.map((r) => r.id).sort()).toEqual(["q1", "q2"]);
135
+ });
136
+
137
+ it("缺 experimentId 的落盘:合成键快照 synthetic: true,synthetic-experiment-id 警告透传", async () => {
138
+ const root = await makeRoot();
139
+ await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
140
+ res({ id: "q1", agent: "bub", model: "gpt-5", startedAt: "2026-07-01T08:01:00.000Z" }),
141
+ ]));
142
+ const { viewData } = await loadViewScan(root);
143
+ expect(viewData.snapshots[0]!.synthetic).toBe(true);
144
+ expect(viewData.snapshots[0]!.experimentId).toBe("bub/gpt-5");
145
+ expect(viewData.overview.warnings.map((w) => w.kind)).toContain("synthetic-experiment-id");
146
+ });
147
+ });
148
+
149
+ describe("loadViewScan · 跨快照去重(--resume 携带的复印件只算一次)", () => {
150
+ it("同一 attempt 存在于两份落盘:只保留最新 run 里的那份,attemptRef 落在最新落盘,artifactBase 沿用原 run 工件", async () => {
151
+ const root = await makeRoot();
152
+ const original = res({
153
+ id: "q1",
154
+ agent: "bub",
155
+ experimentId: "exp/a",
156
+ startedAt: "2026-07-01T08:01:00.000Z",
157
+ artifactsDir: "artifacts/q1/bub/default/a0",
158
+ hasEvents: true,
159
+ });
160
+ const oldRun = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([original], { startedAt: "2026-07-01T08:00:00.000Z" }));
161
+ await mkdir(join(oldRun, "artifacts/q1/bub/default/a0"), { recursive: true });
162
+ await writeFile(join(oldRun, "artifacts/q1/bub/default/a0/events.json"), "[]", "utf-8");
163
+ // 携带条目:artifactsDir 清空,artifactBase 指向原 run 的工件目录(runner 的 carriedResults 形状)。
164
+ const carried = { ...original, artifactsDir: undefined, artifactBase: "2026-07-01T08-00-00-000Z/artifacts/q1/bub/default/a0" };
165
+ await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
166
+ carried,
167
+ res({ id: "q2", agent: "bub", experimentId: "exp/a", startedAt: "2026-07-02T08:02:00.000Z", artifactsDir: "artifacts/q2/bub/default/a0" }),
168
+ ], { startedAt: "2026-07-02T08:00:00.000Z" }));
169
+
170
+ const { viewData, artifactDirs } = await loadViewScan(root);
171
+ // 全部快照(Runs/Traces 的数据面)里 q1 只出现一次:复印件不灌票。
172
+ const allResults = viewData.snapshots.flatMap((s) => s.results);
173
+ expect(allResults.filter((r) => r.id === "q1")).toHaveLength(1);
174
+ const q1 = allResults.find((r) => r.id === "q1")!;
175
+ expect(q1.attemptRef).toEqual({ run: "2026-07-02T08-00-00-000Z", result: 0 }); // 证据身份跟着最新落盘走
176
+ expect(q1.artifactBase).toBe("2026-07-01T08-00-00-000Z/artifacts/q1/bub/default/a0"); // 工件仍指原 run
177
+ // 静态导出(--out)能把携带条目的工件一并带走。
178
+ expect(artifactDirs.get("2026-07-01T08-00-00-000Z/artifacts/q1/bub/default/a0")).toBe(join(oldRun, "artifacts/q1/bub/default/a0"));
179
+ // 条目全被吸走的旧快照不再出现。
180
+ expect(viewData.snapshots.filter((s) => !s.latest)).toHaveLength(0);
181
+ });
182
+ });
183
+
184
+ describe("loadViewScan · createRunWriter 写出的落盘直接可读(writer/reader 同一契约)", () => {
185
+ it("writer roundtrip:官方榜单、快照与 attemptRef 都从写入面产物算出", async () => {
186
+ const root = await makeRoot();
187
+ const writer = await createRunWriter(root, { producer: { name: "niceeval", version: "0.4.6" } });
188
+ const snap = writer.snapshot({
189
+ experiment: "compare/bub",
190
+ agent: "bub",
191
+ model: "gpt-5",
192
+ startedAt: "2026-07-03T08:00:00.000Z",
193
+ });
194
+ await snap.writeAttempt(
195
+ { id: "q1", outcome: "passed", attempt: 0, durationMs: 1200, assertions: [], estimatedCostUSD: 0.5 },
196
+ { events: [{ type: "run.started", t: 0 } as never] },
197
+ );
198
+ await snap.writeAttempt({ id: "q2", outcome: "failed", attempt: 0, durationMs: 800, assertions: [] });
199
+ await writer.finish();
200
+
201
+ const { viewData } = await loadViewScan(root);
202
+ expect(viewData.table.rows.map((r) => r.key)).toEqual(["compare/bub"]);
203
+ expect(viewData.table.rows[0]!.cells["pass-rate"]!.value).toBe(0.5);
204
+ expect(viewData.overview.totals.evals).toBe(2);
205
+ const snapshot = viewData.snapshots[0]!;
206
+ expect(snapshot.latest).toBe(true);
207
+ expect(snapshot.agent).toBe("bub");
208
+ expect(snapshot.results.every((r) => r.attemptRef?.run === snapshot.run)).toBe(true);
209
+ // 写入面拆出的工件,读取面拼回相对 view 根的 artifactBase。
210
+ const q1 = snapshot.results.find((r) => r.id === "q1")!;
211
+ expect(q1.hasEvents).toBe(true);
212
+ expect(q1.artifactBase).toBe(`${snapshot.run}/${q1.artifactsDir}`);
213
+ });
214
+ });
215
+
216
+ describe("loadLatestResultsPerEval(续跑携带基线,口径与旧 loader 一致)", () => {
217
+ it("部分补跑 run 只遮蔽它跑过的 eval,其它 eval 仍取自更早的全量 run", async () => {
218
+ const root = await makeRoot();
219
+ await writeRun(root, "2026-01-01T00-00-00", summaryOf([
220
+ res({ id: "e1", agent: "a", experimentId: "exp/a" }),
221
+ res({ id: "e2", agent: "a", experimentId: "exp/a", outcome: "errored" }),
222
+ res({ id: "e1", agent: "a", experimentId: "exp/b" }),
223
+ ], { startedAt: "2026-01-01T00:00:00.000Z" }));
224
+ // 部分补跑:只重跑了 exp/a 的 e2
225
+ await writeRun(root, "2026-01-02T00-00-00", summaryOf([
226
+ res({ id: "e2", agent: "a", experimentId: "exp/a" }),
227
+ ], { startedAt: "2026-01-02T00:00:00.000Z" }));
228
+
229
+ const results = await loadLatestResultsPerEval(root);
230
+ const byKey = new Map(results.map((r) => [`${r.experimentId}|${r.id}`, r.outcome]));
231
+ expect(byKey.get("exp/a|e1")).toBe("passed"); // 来自旧全量 run,没被部分 run 冲掉
232
+ expect(byKey.get("exp/a|e2")).toBe("passed"); // 来自补跑 run(最新)
233
+ expect(byKey.get("exp/b|e1")).toBe("passed");
234
+ expect(results).toHaveLength(3);
235
+ });
236
+
237
+ it("同 (experiment, eval) 多 attempt 整批取自含它的最新 run,不跨 run 混装;artifactBase 已拼好", async () => {
238
+ const root = await makeRoot();
239
+ await writeRun(root, "2026-01-01T00-00-00", summaryOf([
240
+ res({ id: "e1", agent: "a", experimentId: "exp/a", fingerprint: "old" }),
241
+ ], { startedAt: "2026-01-01T00:00:00.000Z" }));
242
+ await writeRun(root, "2026-01-02T00-00-00", summaryOf([
243
+ res({ id: "e1", agent: "a", experimentId: "exp/a", outcome: "failed", fingerprint: "new", artifactsDir: "artifacts/e1/a0" }),
244
+ res({ id: "e1", agent: "a", experimentId: "exp/a", attempt: 1, fingerprint: "new" }),
245
+ ], { startedAt: "2026-01-02T00:00:00.000Z" }));
246
+
247
+ const results = await loadLatestResultsPerEval(root);
248
+ expect(results).toHaveLength(2);
249
+ expect(results.every((r) => r.fingerprint === "new")).toBe(true);
250
+ // runner 携带条目时依赖 artifactBase(相对结果根)可解析,view 才找得回工件。
251
+ const withArtifacts = results.find((r) => r.artifactsDir)!;
252
+ expect(withArtifacts.artifactBase).toBe("2026-01-02T00-00-00/artifacts/e1/a0");
253
+ });
254
+ });
@@ -0,0 +1,336 @@
1
+ // view 的数据层:读取经 niceeval/results 的 openResults(布局/版本知识只住在那),
2
+ // 统计经 niceeval/report 的官方计算函数(RunOverview.data / MetricTable.data 的实现,
3
+ // 从 compute.ts 直接引用 —— 调用侧适配,不经 components.tsx,避免把 react 拉进 CLI 路径)。
4
+ // 这里只做编排:挑选(results.latest();位置前缀 / --experiment / --report 在场时经
5
+ // composeShowSelection 与 show 同口径合成)、快照明细注入(attemptRef / artifactBase)、
6
+ // skipped / warnings 透传、--report 的报告槽渲染(renderReportSlot)。旧 loader
7
+ // (readSummary / loadSummaries / 目录扫描 / 版本判定)与旧聚合(aggregateRows)已删,
8
+ // 见 docs/view.md「用 Reports 积木重建 view」迁移顺序 1–2。
9
+
10
+ import { statSync } from "node:fs";
11
+ import { basename, dirname, join, relative, resolve } from "node:path";
12
+ import { dedupeAttempts, openResults } from "../results/index.ts";
13
+ import type { AttemptHandle, Results, RunDir, Selection, SkippedRun } from "../results/index.ts";
14
+ import { overviewData, tableData } from "../report/compute.ts";
15
+ import { costUSD, durationMs, passRate, tokens } from "../report/metrics.ts";
16
+ import { loadReportFile } from "../report/load.ts";
17
+ import { composeShowSelection, filterExperiments } from "../show/compose.ts";
18
+ import type { EvalResult, RunSummary } from "../types.ts";
19
+ import type { SkippedRunNotice, ViewData, ViewEvalResult, ViewSnapshot } from "./shared/types.ts";
20
+ import { t } from "../i18n/index.ts";
21
+ import { RESULTS_SCHEMA_VERSION } from "../types.ts";
22
+
23
+ /** 榜单列:官方内置指标,与 `niceeval show` 榜单同一批口径(通过率 / 耗时 / tokens / 成本)。 */
24
+ const BOARD_COLUMNS = [passRate, durationMs, tokens, costUSD] as const;
25
+
26
+ /** hero 整体通过率的常量维度:全部选中 attempt 一组,走同一台两级聚合引擎。 */
27
+ const OVERALL_DIMENSION = { name: "overall", of: () => "all" } as const;
28
+
29
+ export interface ViewScan {
30
+ viewData: ViewData;
31
+ /**
32
+ * artifactBase(相对 view 根)→ 宿主机绝对目录。只在 server 端内存里保留;
33
+ * 绝对路径不进 viewData,避免序列化进可分享的静态 HTML(信息泄漏且浏览器端用不到)。
34
+ */
35
+ artifactDirs: Map<string, string>;
36
+ /**
37
+ * --report:报告树经 renderReportToStaticHtml 渲染出的静态 HTML,整槽替换查看器的
38
+ * 报告槽。作为独立静态块烘进页面(与 __NICEEVAL_VIEW_DATA__ 相邻),不进 viewData ——
39
+ * 证据室沿用官方数据契约不动,前端只负责把这块 HTML 摆进报告槽位置,不解析。
40
+ */
41
+ reportHtml?: string;
42
+ }
43
+
44
+ /** view 宿主输入的组合语义(与 show 对齐,docs/reports.md「宿主输入的组合语义」)。 */
45
+ export interface ViewScanOptions {
46
+ /** eval id 前缀(位置参数):收窄报告槽选集;证据室(快照明细)不收窄,深链恒可达。 */
47
+ patterns?: string[];
48
+ /** experiment id 前缀(--experiment):选集只留该实验。 */
49
+ experiment?: string;
50
+ /** --report 报告文件:相对 cwd 的路径。装载失败抛 ReportLoadError(CLI 打印后退出)。 */
51
+ report?: { path: string; cwd: string };
52
+ }
53
+
54
+ /** 可预期的用户输入错误:CLI 打一句英文直说问题与下一步,退出码 1,不抛堆栈。 */
55
+ export class ViewInputError extends Error {}
56
+
57
+ /** 版本不同、按设计直接不兼容的 run;只占位提示,不解析内容。 */
58
+ export interface IncompatibleRun {
59
+ /** run 目录,相对 cwd;直接可拼进 npx 命令。 */
60
+ dir: string;
61
+ schemaVersion: number;
62
+ producer?: RunSummary["producer"];
63
+ }
64
+
65
+ /** 用能读这份报告的 niceeval 版本查看的命令;第三方 harness 的落盘拼不出 npx,返回 undefined。 */
66
+ export function incompatibleViewCommand(run: IncompatibleRun): string | undefined {
67
+ if (run.producer && run.producer.name !== "niceeval") return undefined;
68
+ return `npx niceeval@${run.producer?.version ?? "<version>"} view ${run.dir}`;
69
+ }
70
+
71
+ /** 版本不匹配的完整提示文案;CLI 单文件模式和目录扫描占位共用。 */
72
+ export function incompatibleHint(run: IncompatibleRun): string {
73
+ const command = incompatibleViewCommand(run);
74
+ if (command === undefined) {
75
+ // 第三方 harness:如实报名字和版本,不拼 npx(docs/results-lib.md 的裁决)。
76
+ return t("cli.view.incompatibleForeign", {
77
+ dir: run.dir,
78
+ name: run.producer?.name ?? "?",
79
+ version: run.producer?.version ?? "?",
80
+ schemaVersion: run.schemaVersion,
81
+ supported: RESULTS_SCHEMA_VERSION,
82
+ });
83
+ }
84
+ return t("cli.view.incompatible", {
85
+ dir: run.dir,
86
+ producer: run.producer?.version ?? "?",
87
+ schemaVersion: run.schemaVersion,
88
+ supported: RESULTS_SCHEMA_VERSION,
89
+ command,
90
+ });
91
+ }
92
+
93
+ /** 单文件模式读到版本不同的 summary 时抛出;CLI 捕获后打印提示退出,不当成普通错误堆栈。 */
94
+ export class IncompatibleResultsError extends Error {
95
+ constructor(readonly run: IncompatibleRun) {
96
+ super(incompatibleHint(run));
97
+ this.name = "IncompatibleResultsError";
98
+ }
99
+ }
100
+
101
+ /** 服务/解析工件的根目录:输入是目录就用它,是文件就用其所在目录。 */
102
+ export function viewRoot(input?: string): string {
103
+ const target = resolve(input ?? ".niceeval");
104
+ try {
105
+ return statSync(target).isFile() ? dirname(target) : target;
106
+ } catch {
107
+ return target;
108
+ }
109
+ }
110
+
111
+ /**
112
+ * 读跨历史「每 (experimentId, evalId) 最新一份」的 EvalResult,供续跑携带已通过结果。
113
+ * 只看最近一个 run 不行:部分补跑(位置参数只跑几道题)会把携带基线换成那个部分 run,
114
+ * 之后重跑任何实验都携带不到东西,`exp <组>` 的「补齐缺失」语义随之失效。
115
+ * 同一 (experimentId, evalId) 的多个 attempt 整批取自含它的最新 run,不跨 run 混装。
116
+ * 携带条目要能被 view 找回工件,这里同时把 artifactBase(相对结果根)拼好(runner 依赖它)。
117
+ */
118
+ export async function loadLatestResultsPerEval(root = ".niceeval"): Promise<EvalResult[]> {
119
+ const results = await openResults(root);
120
+ const rootAbs = resolve(root);
121
+ const claimed = new Set<string>();
122
+ const out: EvalResult[] = [];
123
+ for (const run of results.runDirs) {
124
+ // runDirs 已按新→旧排序;同一 run 内先收本轮的 key,收完再整体入 claimed,
125
+ // 保证同 (experiment, eval) 的多 attempt 整批取自同一个 run。
126
+ const takenThisRun = new Set<string>();
127
+ for (const attempt of run.attempts) {
128
+ const key = `${attempt.experimentId}|${attempt.evalId}`;
129
+ if (claimed.has(key)) continue;
130
+ takenThisRun.add(key);
131
+ out.push(annotateResult(attempt, run, rootAbs).annotated);
132
+ }
133
+ for (const key of takenThisRun) claimed.add(key);
134
+ }
135
+ return out;
136
+ }
137
+
138
+ /**
139
+ * `niceeval view` 的数据装载入口:server 每次请求现读现算,`--out` 导出用同一份。
140
+ * 位置前缀 / --experiment / --report 在场时,报告槽选集经 composeShowSelection 合成
141
+ * (与 `niceeval show` 同一口径,两扇门判决不分叉);全部缺省时维持 results.latest(),
142
+ * 默认行为不变。证据室数据(快照明细 / skipped)恒为全量,深链在任何收窄下都可达。
143
+ */
144
+ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}): Promise<ViewScan> {
145
+ const target = resolve(input ?? ".niceeval");
146
+ const root = viewRoot(input);
147
+ const results = await openResults(target);
148
+ assertSingleFileReadable(results, target);
149
+
150
+ const patterns = opts.patterns ?? [];
151
+ const narrowed = patterns.length > 0 || opts.experiment !== undefined || opts.report !== undefined;
152
+
153
+ // 组合语义的输入校验,与 show 同文案:匹配不到直说,不渲染一张空页面。
154
+ if (opts.report && results.experiments.length === 0) {
155
+ throw new ViewInputError(t("cli.show.noResults", { root: target }).trimEnd());
156
+ }
157
+ if (
158
+ opts.experiment !== undefined &&
159
+ results.experiments.length > 0 &&
160
+ filterExperiments(results.experiments, opts.experiment).length === 0
161
+ ) {
162
+ throw new ViewInputError(
163
+ t("cli.show.noExperimentMatch", {
164
+ arg: opts.experiment,
165
+ experiments: results.experiments.map((e) => e.id).join(", "),
166
+ }).trimEnd(),
167
+ );
168
+ }
169
+
170
+ const baseSelection = results.latest();
171
+ const selection = narrowed
172
+ ? composeShowSelection(results, { experiment: opts.experiment, patterns })
173
+ : baseSelection;
174
+
175
+ if (patterns.length > 0 && selection.snapshots.every((s) => s.evals.length === 0)) {
176
+ const known = [
177
+ ...new Set(filterExperiments(results.experiments, opts.experiment).flatMap((e) => e.evalIds)),
178
+ ].sort();
179
+ throw new ViewInputError(
180
+ t("cli.show.noEvalMatch", { patterns: patterns.join(", "), evals: known.join(", ") || "(none)" }).trimEnd(),
181
+ );
182
+ }
183
+
184
+ const [overview, table, overall] = [
185
+ await overviewData(selection),
186
+ await tableData(selection, { rows: "experiment", columns: BOARD_COLUMNS, sort: passRate }),
187
+ await tableData(selection, { rows: OVERALL_DIMENSION, columns: [passRate] }),
188
+ ];
189
+
190
+ // --report:整槽替换。报告吃同一份注入选集,web 面在计算侧静态渲染成 HTML。
191
+ const reportHtml = opts.report
192
+ ? await renderReportSlot(opts.report, results, selection)
193
+ : undefined;
194
+
195
+ // 跨快照按身份键去重:--resume 携带的条目在多份落盘里重复,只保留最新 run 里的那份
196
+ // (与官方计算函数的聚合口径一致,Runs / Traces 的计数因此不被复印件灌票)。
197
+ const artifactDirs = new Map<string, string>();
198
+ // latest 标记恒按 results.latest() 口径打(ViewSnapshot.latest 的声明语义),
199
+ // 不随收窄后的合成选集漂移 —— 榜单行与快照的关联靠它成立。
200
+ const latestSet = new Set(baseSelection.snapshots);
201
+ const allAttempts: AttemptHandle[] = [];
202
+ for (const exp of results.experiments) {
203
+ for (const snap of exp.snapshots) allAttempts.push(...snap.attempts);
204
+ }
205
+ const survivors = new Set(dedupeAttempts(allAttempts).attempts);
206
+
207
+ const snapshots: ViewSnapshot[] = [];
208
+ for (const exp of results.experiments) {
209
+ for (const snap of exp.snapshots) {
210
+ const kept = snap.attempts.filter((a) => survivors.has(a));
211
+ const latest = latestSet.has(snap);
212
+ // 条目全被去重吸走的历史快照不再携带(它的内容原样活在更新的落盘里)。
213
+ if (kept.length === 0 && !latest) continue;
214
+ snapshots.push({
215
+ experimentId: snap.experimentId,
216
+ ...(snap.synthetic ? { synthetic: true } : {}),
217
+ agent: snap.agent,
218
+ ...(snap.model !== undefined ? { model: snap.model } : {}),
219
+ startedAt: snap.startedAt,
220
+ run: basename(snap.runDir.dir),
221
+ latest,
222
+ results: kept.map((a) => {
223
+ const { annotated, base, abs } = annotateResult(a, snap.runDir, root);
224
+ if (base && abs) artifactDirs.set(base, abs);
225
+ return annotated;
226
+ }),
227
+ });
228
+ }
229
+ }
230
+
231
+ const latestRun = results.runDirs[0];
232
+ const viewData: ViewData = {
233
+ ...(latestRun?.summary.name !== undefined ? { name: latestRun.summary.name } : {}),
234
+ ...(latestRun ? { lastRunAt: latestRun.summary.startedAt } : {}),
235
+ // 合成选集的快照是跨 run 拼出来的,来源 run 数从 attempt 的 runDir 数;
236
+ // 默认选集(latest 口径)保持原表达式,行为不变。
237
+ composedRuns: narrowed
238
+ ? new Set(selection.snapshots.flatMap((s) => s.attempts.map((a) => a.runDir.dir))).size
239
+ : new Set(selection.snapshots.map((s) => s.runDir.dir)).size,
240
+ overview,
241
+ table,
242
+ overall,
243
+ snapshots,
244
+ skippedRuns: results.skipped.map(toSkippedNotice),
245
+ };
246
+ return { viewData, artifactDirs, ...(reportHtml !== undefined ? { reportHtml } : {}) };
247
+ }
248
+
249
+ /**
250
+ * 报告槽渲染:装载报告文件(dev server 语义 —— 文件变更下次请求整页重算,经 mtime
251
+ * cache-busting)→ 注入与官方榜单同口径的选集 → web 面 renderToStaticMarkup 成静态 HTML。
252
+ * react-dom 只在 --report 在场时动态加载,默认 CLI 路径不背 react。
253
+ * attemptHref 缺省即 `#/attempt/<run>/<result>`(view 的 attempt 深链路由)。
254
+ */
255
+ async function renderReportSlot(
256
+ report: { path: string; cwd: string },
257
+ results: Results,
258
+ selection: Selection,
259
+ ): Promise<string> {
260
+ const definition = await loadReportFile(report.cwd, report.path, { freshImport: true });
261
+ const { renderReportToStaticHtml } = await import("../report/web.ts");
262
+ return renderReportToStaticHtml(definition, { selection, results });
263
+ }
264
+
265
+ /**
266
+ * 单文件模式(`niceeval view path/to/summary.json`)是用户明确指定的目标:
267
+ * 读不了就让命令失败并给可执行的下一步,不打开一个空页面。目录模式不走这里
268
+ * (读不了的进 skipped,页面顶部横幅展示,单个坏 run 不拖垮整页)。
269
+ */
270
+ function assertSingleFileReadable(results: Results, target: string): void {
271
+ let isFile = false;
272
+ try {
273
+ isFile = statSync(target).isFile();
274
+ } catch {
275
+ return; // 目标不存在:按空结果渲染(还没跑过 eval 不是错误)。
276
+ }
277
+ if (!isFile || results.runDirs.length > 0) return;
278
+ const skip = results.skipped[0];
279
+ if (skip?.reason === "incompatible-version") {
280
+ throw new IncompatibleResultsError({
281
+ dir: relative(process.cwd(), skip.dir) || ".",
282
+ schemaVersion: skip.schemaVersion ?? 0,
283
+ ...(skip.producer ? { producer: skip.producer } : {}),
284
+ });
285
+ }
286
+ if (skip?.reason === "malformed") {
287
+ throw new Error(
288
+ `${target}: ${skip.detail ?? "unreadable report"}. The report may be corrupted; re-run the eval or delete this run directory.`,
289
+ );
290
+ }
291
+ throw new Error(`${target} is not a niceeval summary`);
292
+ }
293
+
294
+ /**
295
+ * 给单条 attempt 注入 view 侧标注:
296
+ * - attemptRef:直接用 niceeval/results 的证据引用(与 Reports 的 MetricCell.refs 同一身份)。
297
+ * - artifactBase:相对 view 根的工件目录(前端据此 fetch trace.json 等)。--resume 携带的
298
+ * 条目 artifactsDir 为空、artifactBase 指向原 run,原样沿用(同一套候选顺序)。
299
+ * 返回新对象,不 mutate 读入的 summary;宿主机绝对路径只回给调用方写进 artifactDirs
300
+ * (server 端内存),不挂到 result 上,避免随 viewData 进静态 HTML。
301
+ */
302
+ function annotateResult(
303
+ attempt: AttemptHandle,
304
+ run: RunDir,
305
+ root: string,
306
+ ): { annotated: ViewEvalResult; base?: string; abs?: string } {
307
+ const r = attempt.result;
308
+ const annotated: ViewEvalResult = { ...r, attemptRef: attempt.ref };
309
+ if (r.artifactsDir) {
310
+ const abs = join(run.dir, r.artifactsDir);
311
+ const base = relative(root, abs).split(/[\\/]/).join("/");
312
+ return { annotated: { ...annotated, artifactBase: base }, base, abs };
313
+ }
314
+ if (r.artifactBase) {
315
+ // 已是相对结果根的路径(携带条目),工件留在原 run 目录里。
316
+ return { annotated, base: r.artifactBase, abs: join(root, r.artifactBase) };
317
+ }
318
+ return { annotated };
319
+ }
320
+
321
+ function toSkippedNotice(skip: SkippedRun): SkippedRunNotice {
322
+ const dir = relative(process.cwd(), skip.dir) || ".";
323
+ const command =
324
+ skip.reason === "incompatible-version" && skip.producer?.name === "niceeval" && skip.producer.version
325
+ ? incompatibleViewCommand({ dir, schemaVersion: skip.schemaVersion ?? 0, producer: skip.producer })
326
+ : undefined;
327
+ return {
328
+ dir,
329
+ reason: skip.reason,
330
+ ...(skip.schemaVersion !== undefined ? { schemaVersion: skip.schemaVersion } : {}),
331
+ ...(skip.producer?.name !== undefined ? { producerName: skip.producer.name } : {}),
332
+ ...(skip.producer?.version !== undefined ? { producerVersion: skip.producer.version } : {}),
333
+ ...(command !== undefined ? { command } : {}),
334
+ ...(skip.detail !== undefined ? { detail: skip.detail } : {}),
335
+ };
336
+ }