niceeval 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +321 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
- package/dist/report/built-ins/experiment-comparison.js +13 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +180 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +600 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +26 -0
- package/dist/report/format.js +109 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +94 -0
- package/dist/report/locale.js +183 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +19 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +11 -0
- package/dist/report/react/ExperimentList.js +67 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +96 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +398 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +541 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +408 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +7 -7
- package/docs-site/zh/concepts/assert.mdx +19 -18
- package/docs-site/zh/concepts/evals.mdx +12 -11
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +3 -3
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +4 -2
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +4 -4
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +61 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
- package/docs-site/zh/guides/viewing-results.mdx +200 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +4 -3
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +15 -13
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +9 -3
- package/docs-site/zh/reference/expect.mdx +13 -8
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +59 -3
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +33 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +640 -0
- package/src/report/built-ins/experiment-comparison.tsx +19 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +225 -50
- package/src/report/compute.ts +335 -54
- package/src/report/dual-render.test.tsx +1129 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +35 -2
- package/src/report/index.ts +47 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +229 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +113 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +220 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +126 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +168 -0
- package/src/report/react/fixtures.ts +169 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +21 -6
- package/src/report/react/render.test.tsx +180 -31
- package/src/report/react/styles.css +432 -213
- package/src/report/report.test.ts +398 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +344 -96
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +184 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +119 -95
- package/src/show/render.ts +462 -159
- package/src/show/show.test.ts +471 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
|
@@ -0,0 +1,1129 @@
|
|
|
1
|
+
// 双面验收:每个官方组件对同一份数据,web 面(renderToStaticMarkup)与 text 面
|
|
2
|
+
// 给出一致判读 —— 排序方向随 better、samples < total 角标、缺数据 — 不补 0、
|
|
3
|
+
// 截断如实报剩余;text 面形态以内联快照锁定(照 report-components.mdx 的示例形态)。
|
|
4
|
+
// 另验收:排版原语的分栏与降级、渲染前树校验、defineComponent 自定义组件、
|
|
5
|
+
// defineReport + renderReportToText / renderReportToStaticHtml 两个宿主入口。
|
|
6
|
+
|
|
7
|
+
import { renderToStaticMarkup } from "react-dom/server";
|
|
8
|
+
import { describe, expect, it } from "vitest";
|
|
9
|
+
|
|
10
|
+
import type { EvalResult } from "../types.ts";
|
|
11
|
+
import type { Results, Selection, Snapshot } from "../results/index.ts";
|
|
12
|
+
import type { GroupSummaryData, ScatterData } from "./types.ts";
|
|
13
|
+
import type { AttemptLocator, MetricScatterProps } from "./index.ts";
|
|
14
|
+
import { evalLevelStats } from "../shared/verdict.ts";
|
|
15
|
+
import {
|
|
16
|
+
AttemptList,
|
|
17
|
+
Col,
|
|
18
|
+
DeltaTable,
|
|
19
|
+
EvalList,
|
|
20
|
+
ExperimentList,
|
|
21
|
+
GroupSummary,
|
|
22
|
+
MetricBars,
|
|
23
|
+
MetricLine,
|
|
24
|
+
MetricMatrix,
|
|
25
|
+
MetricScatter,
|
|
26
|
+
MetricTable,
|
|
27
|
+
Row,
|
|
28
|
+
RunOverview,
|
|
29
|
+
Scoreboard,
|
|
30
|
+
Section,
|
|
31
|
+
Style,
|
|
32
|
+
Table,
|
|
33
|
+
Text,
|
|
34
|
+
bar,
|
|
35
|
+
costUSD,
|
|
36
|
+
defineComponent,
|
|
37
|
+
defineReport,
|
|
38
|
+
isReportDefinition,
|
|
39
|
+
padEnd,
|
|
40
|
+
passRate,
|
|
41
|
+
renderReportToText,
|
|
42
|
+
stringWidth,
|
|
43
|
+
} from "./index.ts";
|
|
44
|
+
import { ExperimentComparison } from "./built-ins/index.ts";
|
|
45
|
+
import { renderReportToStaticHtml } from "./web.ts";
|
|
46
|
+
import { createTextContext, renderNodeToText, validateReportTree } from "./tree.ts";
|
|
47
|
+
import {
|
|
48
|
+
attemptListItems,
|
|
49
|
+
deltaData,
|
|
50
|
+
evalListItems,
|
|
51
|
+
experimentListItems,
|
|
52
|
+
groupSummaryData,
|
|
53
|
+
lineData,
|
|
54
|
+
matrixData,
|
|
55
|
+
overviewData,
|
|
56
|
+
scatterData,
|
|
57
|
+
scoreboardData,
|
|
58
|
+
tableData,
|
|
59
|
+
} from "./react/fixtures.ts";
|
|
60
|
+
|
|
61
|
+
const ctx = createTextContext({ width: 80 });
|
|
62
|
+
const text = (node: Parameters<typeof renderNodeToText>[0]) => renderNodeToText(node, ctx);
|
|
63
|
+
|
|
64
|
+
// ───────────────────────── 每组件双面一致 ─────────────────────────
|
|
65
|
+
|
|
66
|
+
describe("RunOverview 双面", () => {
|
|
67
|
+
const html = renderToStaticMarkup(<RunOverview data={overviewData} />);
|
|
68
|
+
const term = text(<RunOverview data={overviewData} />);
|
|
69
|
+
|
|
70
|
+
it("text 面形态:头行(含通过率)+ 判定行 + 警告行", () => {
|
|
71
|
+
expect(term).toMatchInlineSnapshot(`
|
|
72
|
+
"2 experiments · 12 evals · 48 attempts · Pass rate 70% 46/48 · composed from 2 runs · latest 2026-07-01T11:30:00Z
|
|
73
|
+
passed 36 · failed 8 · errored 2 · skipped 2 · no data · 4m 21s
|
|
74
|
+
! snapshot covers 9 of 12 evals seen in history; re-run \`niceeval exp compare/bub\` for a full snapshot"
|
|
75
|
+
`);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("两面同口径:通过率显示同一个 MetricCell.display(70%,不是 36/46 现算的 78%);成本全缺都是 no data 不编 $0;警告都在", () => {
|
|
79
|
+
for (const face of [html, term]) {
|
|
80
|
+
expect(face).toContain("70%");
|
|
81
|
+
expect(face).not.toContain("78%");
|
|
82
|
+
expect(face).toContain("46/48"); // samples < total:两级聚合的覆盖率角标,两面一致
|
|
83
|
+
expect(face).toContain("no data");
|
|
84
|
+
expect(face).not.toContain("$0");
|
|
85
|
+
expect(face).toContain("snapshot covers 9 of 12 evals");
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
// fixtures.overviewData 是手工摆好的终值,只验证渲染面「原样显示 MetricCell,不重算」;
|
|
91
|
+
// 下面这组用真实 Selection 走一遍 RunOverview.data(= compute.ts 的 overviewData()),
|
|
92
|
+
// 专门验证 totals.passRate 本身的计算口径 —— 三种通过率公式在这个 fixture 上各不相同:
|
|
93
|
+
// 两级聚合(唯一官方口径,docs/feature/reports/architecture.md「指标聚合不变量」):eval a 题内 2/3 通过、eval b 题内 1,
|
|
94
|
+
// 跨题均值 (2/3 + 1) / 2 = 5/6 ≈ 83.3%
|
|
95
|
+
// attempt 原始占比(旧 bug 公式,曾经的 RunOverview 现场重算):3 passed / (3 passed + 1 failed) = 75%
|
|
96
|
+
// eval 折叠投票(evalLevelStats,GroupSummary/MetricTable meta 的口径):a、b 都折成 passed → 2/2 = 100%
|
|
97
|
+
// 三个数互不相同,任何一处偷懒复用另一个公式都会在这里露馅。
|
|
98
|
+
describe("RunOverview.data · passRate 两级聚合口径", () => {
|
|
99
|
+
function fakeVaryingAttemptsContext(): { selection: Selection; attempts: { evalId: string; result: { verdict: EvalResult["verdict"] } }[] } {
|
|
100
|
+
const dir = "/results/compare_bub/snap-1";
|
|
101
|
+
const base = {
|
|
102
|
+
experimentId: "compare/bub",
|
|
103
|
+
startedAt: "2026-07-01T10:00:00Z",
|
|
104
|
+
completedAt: "2026-07-01T10:05:00Z",
|
|
105
|
+
agent: "bub",
|
|
106
|
+
schemaVersion: 1,
|
|
107
|
+
dir,
|
|
108
|
+
};
|
|
109
|
+
const mk = (evalId: string, verdict: "passed" | "failed" | "skipped", attemptIndex: number, minute: number) => ({
|
|
110
|
+
evalId,
|
|
111
|
+
experimentId: "compare/bub",
|
|
112
|
+
result: {
|
|
113
|
+
id: evalId,
|
|
114
|
+
agent: "bub",
|
|
115
|
+
verdict,
|
|
116
|
+
attempt: attemptIndex,
|
|
117
|
+
startedAt: `2026-07-01T10:0${minute}:00Z`,
|
|
118
|
+
durationMs: 1000,
|
|
119
|
+
assertions: [],
|
|
120
|
+
},
|
|
121
|
+
ref: { snapshot: "compare_bub/snap-1", attempt: `${evalId}/a${attemptIndex}` },
|
|
122
|
+
snapshot: base,
|
|
123
|
+
events: async () => null,
|
|
124
|
+
trace: async () => null,
|
|
125
|
+
o11y: async () => null,
|
|
126
|
+
diff: async () => null,
|
|
127
|
+
sources: async () => null,
|
|
128
|
+
});
|
|
129
|
+
const attempts = [
|
|
130
|
+
// eval a:3 attempts,2 通过 1 失败 → 题内 partial credit 2/3
|
|
131
|
+
mk("algebra/a", "passed", 0, 0),
|
|
132
|
+
mk("algebra/a", "failed", 1, 1),
|
|
133
|
+
mk("algebra/a", "passed", 2, 2),
|
|
134
|
+
// eval b:1 attempt 通过 → 题内 1
|
|
135
|
+
mk("algebra/b", "passed", 0, 3),
|
|
136
|
+
// eval c:1 attempt 跳过 —— 两级聚合与 eval 折叠计票都要把它排除在分母外
|
|
137
|
+
mk("algebra/c", "skipped", 0, 4),
|
|
138
|
+
];
|
|
139
|
+
const evalIds = [...new Set(attempts.map((a) => a.evalId))];
|
|
140
|
+
const snapshot: Snapshot = {
|
|
141
|
+
...base,
|
|
142
|
+
evals: evalIds.map((id) => ({ id, attempts: attempts.filter((a) => a.evalId === id) })),
|
|
143
|
+
attempts,
|
|
144
|
+
} as unknown as Snapshot;
|
|
145
|
+
const selection: Selection = { snapshots: [snapshot], warnings: [], filter: () => selection };
|
|
146
|
+
return { selection, attempts };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
it("totals.passRate = 两级聚合 83.3%,既不等于 attempt 原始占比 75%,也不等于 eval 折叠投票 100%", async () => {
|
|
150
|
+
const { selection, attempts } = fakeVaryingAttemptsContext();
|
|
151
|
+
const data = await RunOverview.data(selection);
|
|
152
|
+
|
|
153
|
+
expect(data.totals.passRate.value).toBeCloseTo(5 / 6, 10);
|
|
154
|
+
expect(data.totals.passRate.display).toBe("83.3%");
|
|
155
|
+
expect(data.totals.passRate.samples).toBe(4); // 5 attempts - 1 skipped(不进桶)
|
|
156
|
+
expect(data.totals.passRate.total).toBe(5);
|
|
157
|
+
|
|
158
|
+
// attempt 原始占比(旧 bug 公式):必须与两级聚合不同,证明没有从 passed/failed/errored 现算
|
|
159
|
+
const attemptFraction = data.totals.passed / (data.totals.passed + data.totals.failed + data.totals.errored);
|
|
160
|
+
expect(attemptFraction).toBeCloseTo(0.75, 10);
|
|
161
|
+
expect(attemptFraction).not.toBeCloseTo(data.totals.passRate.value as number, 3);
|
|
162
|
+
|
|
163
|
+
// eval 折叠投票(evalLevelStats,GroupSummary 的口径):也必须与两级聚合不同
|
|
164
|
+
const stats = evalLevelStats(
|
|
165
|
+
attempts.map((a) => ({ verdict: a.result.verdict, key: a.evalId })),
|
|
166
|
+
(r) => r.key,
|
|
167
|
+
);
|
|
168
|
+
expect(stats.passRate).toBeCloseTo(1, 10);
|
|
169
|
+
expect(stats.passRate).not.toBeCloseTo(data.totals.passRate.value as number, 3);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("web 面与 text 面显示同一个 passRate.display,覆盖率角标(4/5)两面一致", async () => {
|
|
173
|
+
const { selection } = fakeVaryingAttemptsContext();
|
|
174
|
+
const data = await RunOverview.data(selection);
|
|
175
|
+
const html = renderToStaticMarkup(<RunOverview data={data} />);
|
|
176
|
+
const term = text(<RunOverview data={data} />);
|
|
177
|
+
for (const face of [html, term]) {
|
|
178
|
+
expect(face).toContain(data.totals.passRate.display);
|
|
179
|
+
expect(face).toContain("4/5");
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
// ───────────────────────── GroupSummary ─────────────────────────
|
|
185
|
+
|
|
186
|
+
describe("GroupSummary 双面", () => {
|
|
187
|
+
const html = renderToStaticMarkup(<GroupSummary data={groupSummaryData} />);
|
|
188
|
+
const term = text(<GroupSummary data={groupSummaryData} />);
|
|
189
|
+
|
|
190
|
+
it("text 面形态:一行头(通过率 + experiment/eval 数 + failed/errored + 总成本)+ 最后运行时间", () => {
|
|
191
|
+
expect(term).toMatchInlineSnapshot(`
|
|
192
|
+
"Pass rate 60% 5/6 · 2 experiments · 6 evals · failed 1 · errored 1 · $1.50
|
|
193
|
+
latest 2026-07-01T11:30:00Z"
|
|
194
|
+
`);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it("两面同口径:通过率显示同一个 MetricCell.display(60%,不是现场重算);总成本、最后运行时间两面一致", () => {
|
|
198
|
+
for (const face of [html, term]) {
|
|
199
|
+
expect(face).toContain("60%");
|
|
200
|
+
expect(face).toContain("$1.50");
|
|
201
|
+
expect(face).toContain("2026-07-01T11:30:00Z");
|
|
202
|
+
}
|
|
203
|
+
// 覆盖率角标:samples(5) < total(6) —— 1 道 eval 是 skipped,没进分母
|
|
204
|
+
expect(html).toContain("5/6");
|
|
205
|
+
expect(term).toContain("5/6");
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
it("缺成本:两面都显示无数据文案,不编 $0", () => {
|
|
209
|
+
const missingCost: GroupSummaryData = { ...groupSummaryData, totalCostUSD: null };
|
|
210
|
+
const h = renderToStaticMarkup(<GroupSummary data={missingCost} />);
|
|
211
|
+
const t = text(<GroupSummary data={missingCost} />);
|
|
212
|
+
for (const face of [h, t]) {
|
|
213
|
+
expect(face).toContain("no data");
|
|
214
|
+
expect(face).not.toContain("$0");
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it("零失败:web 面整个 errored 片段不渲染,text 面也没有 errored 分段;failed 仍如实显示 0(不是省略整个字段)", () => {
|
|
219
|
+
const allPassed: GroupSummaryData = {
|
|
220
|
+
...groupSummaryData,
|
|
221
|
+
verdicts: { passed: 6, failed: 0, errored: 0, skipped: 0 },
|
|
222
|
+
passRate: { value: 1, display: "100%", samples: 6, total: 6, refs: [] },
|
|
223
|
+
};
|
|
224
|
+
const h = renderToStaticMarkup(<GroupSummary data={allPassed} />);
|
|
225
|
+
const t = text(<GroupSummary data={allPassed} />);
|
|
226
|
+
expect(h).not.toContain("nre-verdict-errored");
|
|
227
|
+
expect(h).not.toContain(">errored<");
|
|
228
|
+
expect(t).not.toContain("errored");
|
|
229
|
+
// failed 计数依旧渲染,哪怕是 0 —— 数据字段本身从不省略,只是 errored 的展示片段在 0 时省略
|
|
230
|
+
expect(h).toContain(">failed<");
|
|
231
|
+
expect(t).toContain("failed 0");
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it("存在错误:两面都显示 errored 计数(与 groupSummaryData 基础 fixture 的 errored:1 一致)", () => {
|
|
235
|
+
expect(html).toContain("nre-verdict-errored");
|
|
236
|
+
expect(html).toContain(">errored<");
|
|
237
|
+
expect(term).toContain("errored 1");
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
it("zh-CN locale:web 面走中文字典(通过率/失败/错误/总成本/实验数),text 面同理;display 数字不本地化", () => {
|
|
241
|
+
const zhHtml = renderToStaticMarkup(<GroupSummary data={groupSummaryData} locale="zh-CN" />);
|
|
242
|
+
expect(zhHtml).toContain("通过率");
|
|
243
|
+
expect(zhHtml).toContain("失败");
|
|
244
|
+
expect(zhHtml).toContain("错误");
|
|
245
|
+
expect(zhHtml).toContain("总成本");
|
|
246
|
+
expect(zhHtml).toContain("实验数");
|
|
247
|
+
expect(zhHtml).toContain("60%"); // display 本身不随 locale 变
|
|
248
|
+
|
|
249
|
+
const zhCtx = createTextContext({ width: 80, locale: "zh-CN" });
|
|
250
|
+
const zhTerm = renderNodeToText(<GroupSummary data={groupSummaryData} />, zhCtx);
|
|
251
|
+
expect(zhTerm).toContain("通过率");
|
|
252
|
+
expect(zhTerm).toContain("失败 1");
|
|
253
|
+
expect(zhTerm).toContain("错误 1");
|
|
254
|
+
expect(zhTerm).toContain("60%");
|
|
255
|
+
});
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
describe("MetricTable 双面", () => {
|
|
259
|
+
const html = renderToStaticMarkup(<MetricTable data={tableData} />);
|
|
260
|
+
const term = text(<MetricTable data={tableData} />);
|
|
261
|
+
|
|
262
|
+
it("text 面形态:对齐列 + 覆盖率角标 + 缺数据 —", () => {
|
|
263
|
+
expect(term).toMatchInlineSnapshot(`
|
|
264
|
+
"agent pass rate code lines
|
|
265
|
+
codex 50% —
|
|
266
|
+
bub 87% 120 lines 5/6"
|
|
267
|
+
`);
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
it("两面同口径:行序一致(预排即终排)、角标一致、缺数据不补 0", () => {
|
|
271
|
+
// 行序:codex 在 bub 前(数据侧顺序,两面都不重排)
|
|
272
|
+
expect(html.indexOf(">codex<")).toBeLessThan(html.indexOf(">bub<"));
|
|
273
|
+
expect(term.indexOf("codex")).toBeLessThan(term.indexOf("bub"));
|
|
274
|
+
// 覆盖率角标 5/6 两面都在
|
|
275
|
+
expect(html).toContain("5/6");
|
|
276
|
+
expect(term).toContain("5/6");
|
|
277
|
+
// 全 null 格子:web 是 no data 文案,text 是 —,都绝不画 0
|
|
278
|
+
expect(html).toContain("no data");
|
|
279
|
+
expect(term).toContain("—");
|
|
280
|
+
});
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
describe("MetricMatrix 双面", () => {
|
|
284
|
+
const html = renderToStaticMarkup(<MetricMatrix data={matrixData} />);
|
|
285
|
+
const term = text(<MetricMatrix data={matrixData} />);
|
|
286
|
+
|
|
287
|
+
it("text 面形态:稀疏格子 — + 下钻命令指向最值得看的一行", () => {
|
|
288
|
+
expect(term).toMatchInlineSnapshot(`
|
|
289
|
+
"eval bub codex
|
|
290
|
+
algebra/quadratic 100% 0%
|
|
291
|
+
geometry/angles 50% —
|
|
292
|
+
|
|
293
|
+
next: niceeval show geometry/angles"
|
|
294
|
+
`);
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
it("两面同口径:数字一致,稀疏格子不编数", () => {
|
|
298
|
+
for (const value of ["100%", "0%", "50%"]) {
|
|
299
|
+
expect(html).toContain(value);
|
|
300
|
+
expect(term).toContain(value);
|
|
301
|
+
}
|
|
302
|
+
expect(html.match(/nre-td-empty/g)).toHaveLength(1); // web 空格子
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
describe("MetricBars 双面", () => {
|
|
307
|
+
const html = renderToStaticMarkup(<MetricBars data={matrixData} />);
|
|
308
|
+
const term = text(<MetricBars data={matrixData} />);
|
|
309
|
+
|
|
310
|
+
it("text 面形态:一组条 = 一个 row 键,条长即刻度,缺数据 —", () => {
|
|
311
|
+
expect(term).toMatchInlineSnapshot(`
|
|
312
|
+
"algebra/quadratic
|
|
313
|
+
bub ████████████████████ 100%
|
|
314
|
+
codex ░░░░░░░░░░░░░░░░░░░░ 0%
|
|
315
|
+
geometry/angles
|
|
316
|
+
bub ██████████░░░░░░░░░░ 50%
|
|
317
|
+
codex —"
|
|
318
|
+
`);
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
it("两面同口径:组内按值排序(better higher 降序),缺数据不出柱", () => {
|
|
322
|
+
expect(html).toContain("100%");
|
|
323
|
+
expect(html).toContain("50%");
|
|
324
|
+
// web 面:codex 在 geometry/angles 组没有柱(缺数据),柱数 = 3
|
|
325
|
+
expect(html.match(/class="nre-bar /g)).toHaveLength(3);
|
|
326
|
+
// text 面:codex 行是 —
|
|
327
|
+
expect(term).toContain("codex —");
|
|
328
|
+
});
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
describe("Scoreboard 双面", () => {
|
|
332
|
+
const html = renderToStaticMarkup(<Scoreboard data={scoreboardData} />);
|
|
333
|
+
const term = text(<Scoreboard data={scoreboardData} />);
|
|
334
|
+
|
|
335
|
+
it("text 面形态:总分/满分 + 分科 + missing 注脚 + 权重表", () => {
|
|
336
|
+
expect(term).toMatchInlineSnapshot(`
|
|
337
|
+
"agent total algebra geometry
|
|
338
|
+
bub 78.5/100 14/16 (1 missing) 3/4
|
|
339
|
+
codex 52/100 9/16 1.4/4 (2 missing)
|
|
340
|
+
weights: algebra/ ×2 · others ×1"
|
|
341
|
+
`);
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
it("两面同口径:missing 与权重都如实", () => {
|
|
345
|
+
expect(html).toContain("1 eval missing, scored 0");
|
|
346
|
+
expect(term).toContain("(1 missing)");
|
|
347
|
+
expect(html).toContain("algebra/ ×2");
|
|
348
|
+
expect(term).toContain("algebra/ ×2");
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
it("一整科完全没跑(不是 missing>0 的部分缺):text 面该格渲染 —,证明走的是 renderTableText 的 null 处理,不是 scoreboardText 自己拼字符串", () => {
|
|
352
|
+
const oneSubjectMissing: typeof scoreboardData = {
|
|
353
|
+
...scoreboardData,
|
|
354
|
+
rows: [
|
|
355
|
+
scoreboardData.rows[0]!,
|
|
356
|
+
{ key: "solo", total: { value: 14, display: "14" }, subjects: [{ key: "algebra", earned: 14, possible: 16, evals: 8, missing: 0 }] },
|
|
357
|
+
],
|
|
358
|
+
};
|
|
359
|
+
const soloTerm = text(<Scoreboard data={oneSubjectMissing} />);
|
|
360
|
+
const [, , soloLine] = soloTerm.split("\n");
|
|
361
|
+
expect(soloLine).toContain("—"); // geometry 列:solo 完全没有这一科,cells.subject1 = null → renderTableText 补 —
|
|
362
|
+
expect(soloLine).not.toContain("0/4"); // 不补 0,与 <Table> null 契约一致
|
|
363
|
+
});
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
describe("MetricScatter 双面", () => {
|
|
367
|
+
const html = renderToStaticMarkup(<MetricScatter data={scatterData} />);
|
|
368
|
+
const term = text(<MetricScatter data={scatterData} />);
|
|
369
|
+
|
|
370
|
+
it("text 面形态:字符坐标图 + 图例 + 缺数据注脚", () => {
|
|
371
|
+
expect(term).toMatchInlineSnapshot(`
|
|
372
|
+
" pass rate ↑
|
|
373
|
+
90%│B····
|
|
374
|
+
│ ·········
|
|
375
|
+
│ ··········
|
|
376
|
+
│ ·········
|
|
377
|
+
│ ·········
|
|
378
|
+
│ ·········
|
|
379
|
+
│ C ··········
|
|
380
|
+
│ ·········
|
|
381
|
+
50%│ ····A
|
|
382
|
+
└───────────────────────────────────────────────────────────────────────────→ cost (axis reversed: right = better)
|
|
383
|
+
$10.00 $5.00
|
|
384
|
+
|
|
385
|
+
better → upper right
|
|
386
|
+
A compare/bub-low B compare/bub-high C compare/codex-mid
|
|
387
|
+
1 point missing data"
|
|
388
|
+
`);
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
it("两面同口径:x 轴反向(lower 好的一端在右)、缺数据点都不画、注脚同数", () => {
|
|
392
|
+
// web:$5(便宜)在 $10 右边(cx 更大)——render.test.tsx 已细测,这里只对注脚
|
|
393
|
+
expect(html).toContain("1 point missing data");
|
|
394
|
+
expect(term).toContain("1 point missing data");
|
|
395
|
+
expect(html).not.toContain('data-key="compare/codex-broken"'); // 注脚 title 如实列缺数据的点,但不画
|
|
396
|
+
// text 图例里也没有缺数据的点
|
|
397
|
+
expect(term).not.toContain("codex-broken");
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
it("点太密时降级坐标表,不硬挤", () => {
|
|
401
|
+
const narrow = renderNodeToText(<MetricScatter data={scatterData} />, createTextContext({ width: 40 }));
|
|
402
|
+
expect(narrow).toContain("experiment");
|
|
403
|
+
expect(narrow).toContain("$5.00");
|
|
404
|
+
expect(narrow).not.toContain("└");
|
|
405
|
+
});
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
describe("MetricLine 双面", () => {
|
|
409
|
+
const html = renderToStaticMarkup(<MetricLine data={lineData} />);
|
|
410
|
+
const term = text(<MetricLine data={lineData} />);
|
|
411
|
+
|
|
412
|
+
it("text 面形态:同系列一个字母、沿 x 排布,y 轴刻度,图例 + 缺数据注脚", () => {
|
|
413
|
+
expect(term).toMatchInlineSnapshot(`
|
|
414
|
+
" pass rate ↑
|
|
415
|
+
80%│B··················
|
|
416
|
+
│ ·····································
|
|
417
|
+
│ ··················B
|
|
418
|
+
│
|
|
419
|
+
│
|
|
420
|
+
│
|
|
421
|
+
│A··················
|
|
422
|
+
│ ·····································
|
|
423
|
+
30%│ ··················A
|
|
424
|
+
└───────────────────────────────────────────────────────────────────────────→ Simulated latency
|
|
425
|
+
100ms 300ms
|
|
426
|
+
|
|
427
|
+
A 1 agents B 16 agents
|
|
428
|
+
1 point missing data"
|
|
429
|
+
`);
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
it("两面同口径:未声明 flag 的点两面都不画、注脚同数", () => {
|
|
433
|
+
expect(html).toContain("1 point missing data");
|
|
434
|
+
expect(term).toContain("1 point missing data");
|
|
435
|
+
expect(html).not.toContain('data-key="ultra/legacy"'); // 注脚 title 如实列缺数据的点,但不画
|
|
436
|
+
});
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
describe("DeltaTable 双面", () => {
|
|
440
|
+
const html = renderToStaticMarkup(<DeltaTable data={deltaData} />);
|
|
441
|
+
const term = text(<DeltaTable data={deltaData} />);
|
|
442
|
+
|
|
443
|
+
it("text 面形态:A → B + Δ,单侧缺数据是 —,Δ=0 是 ±0", () => {
|
|
444
|
+
expect(term).toMatchInlineSnapshot(`
|
|
445
|
+
"pair pass rate cost
|
|
446
|
+
bub 50% → 62% +12pp $0.20 → $0.35 +$0.15
|
|
447
|
+
codex 40% → 40% ±0 — → $0.30 —"
|
|
448
|
+
`);
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
it("两面同口径:Δ 文案一致、缺数据都不硬算", () => {
|
|
452
|
+
for (const piece of ["+12pp", "+$0.15", "±0"]) {
|
|
453
|
+
expect(html).toContain(piece);
|
|
454
|
+
expect(term).toContain(piece);
|
|
455
|
+
}
|
|
456
|
+
expect(html).toContain("nre-delta-missing");
|
|
457
|
+
expect(term).toContain("— → $0.30");
|
|
458
|
+
});
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
describe("AttemptList 双面", () => {
|
|
462
|
+
const html = renderToStaticMarkup(<AttemptList items={attemptListItems} />);
|
|
463
|
+
const term = text(<AttemptList items={attemptListItems} />);
|
|
464
|
+
|
|
465
|
+
it("两面同口径:判定符 + locator + 断言/error 明细 + 下钻命令一致", () => {
|
|
466
|
+
for (const piece of ["roots-correct", "expected x=2, got x=3", "TypeError", "@1a4a4a4", "@1c1c1c1"]) {
|
|
467
|
+
expect(html).toContain(piece);
|
|
468
|
+
expect(term).toContain(piece);
|
|
469
|
+
}
|
|
470
|
+
// web 面走证据室路由(#/attempt/@<locator>);text 面只列 locator 本身,不重复整条命令
|
|
471
|
+
// (docs/feature/reports/architecture.md「text 输出只在整份报告末尾给一次命令模板」)。
|
|
472
|
+
expect(html).toContain('href="#/attempt/@1a4a4a4"');
|
|
473
|
+
expect(term).not.toMatch(/\[[EXD⏱,]+\]/);
|
|
474
|
+
expect(html).not.toMatch(/\[[EXD⏱,]+\]/);
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
it("total > items.length 时两面都如实报剩余数量", () => {
|
|
478
|
+
const htmlTrunc = renderToStaticMarkup(<AttemptList items={attemptListItems} total={attemptListItems.length + 2} />);
|
|
479
|
+
const termTrunc = text(<AttemptList items={attemptListItems} total={attemptListItems.length + 2} />);
|
|
480
|
+
expect(htmlTrunc).toContain("2 more not shown");
|
|
481
|
+
expect(termTrunc).toContain("2 more not shown");
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
it("text 面截断超长 assertion evidence,并保留 locator 作为完整证据入口", () => {
|
|
485
|
+
const longEvidence = "x".repeat(2_000);
|
|
486
|
+
const item = {
|
|
487
|
+
...attemptListItems[0],
|
|
488
|
+
assertions: [{ ...attemptListItems[0].assertions[0], evidence: longEvidence }],
|
|
489
|
+
};
|
|
490
|
+
const out = text(<AttemptList items={[item]} />);
|
|
491
|
+
expect(out).toContain(item.locator);
|
|
492
|
+
expect(out.replace(/\s+/g, " ")).toContain(`open ${item.locator} for full evidence`);
|
|
493
|
+
expect(out).not.toContain(longEvidence);
|
|
494
|
+
expect(out.length).toBeLessThan(1_000);
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
it("text 面是逐条卡片,不是 renderTableText 的产物:空行分隔每个 attempt、断言明细逐级缩进——共享表格渲染器不产生这种嵌套形状", () => {
|
|
498
|
+
const blocks = term.split("\n\n");
|
|
499
|
+
expect(blocks).toHaveLength(attemptListItems.length); // 每个 attempt 独立一块,块间空行分隔;表格行之间不留空行
|
|
500
|
+
const lines = term.split("\n");
|
|
501
|
+
expect(lines.some((l) => /^ \S/.test(l))).toBe(true); // 断言行缩进两格
|
|
502
|
+
expect(lines.some((l) => /^ \S/.test(l))).toBe(true); // 断言 detail 再缩进两格,是卡片的层级,不是列
|
|
503
|
+
});
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
describe("EvalList 双面", () => {
|
|
507
|
+
const html = renderToStaticMarkup(<EvalList items={evalListItems} />);
|
|
508
|
+
const term = text(<EvalList items={evalListItems} />);
|
|
509
|
+
|
|
510
|
+
it("两面同口径:身份、判定、展开到 Attempt 的 locator 徽标一致", () => {
|
|
511
|
+
for (const piece of ["algebra/quadratic", "compare/bub", "geometry/angles", "compare/codex", "@1a4a4a4", "@1c1c1c1"]) {
|
|
512
|
+
expect(html).toContain(piece);
|
|
513
|
+
expect(term).toContain(piece);
|
|
514
|
+
}
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
it("text 面的 score 行插值正确:不留字面量占位符 {score}(回归——曾把带插值的 locale 模板当纯标签用,漏传 vars)", () => {
|
|
518
|
+
expect(term).toContain("score 0%");
|
|
519
|
+
expect(term).not.toContain("{score}");
|
|
520
|
+
});
|
|
521
|
+
|
|
522
|
+
it("text 面是逐条卡片,不是 renderTableText 的产物:空行分隔每个 eval、展开到 attempt 的行缩进——表格没有这层嵌套", () => {
|
|
523
|
+
const blocks = term.split("\n\n");
|
|
524
|
+
expect(blocks).toHaveLength(evalListItems.length);
|
|
525
|
+
expect(term.split("\n").some((l) => /^ \S/.test(l))).toBe(true); // 展开到 attempt 的行缩进两格
|
|
526
|
+
});
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
describe("ExperimentList 双面", () => {
|
|
530
|
+
const html = renderToStaticMarkup(<ExperimentList items={experimentListItems} />);
|
|
531
|
+
const term = text(<ExperimentList items={experimentListItems} />);
|
|
532
|
+
|
|
533
|
+
it("两面同口径:身份、Eval 判定构成、展开到 Eval 的 locator 徽标一致", () => {
|
|
534
|
+
for (const piece of ["compare/bub", "compare/codex", "algebra/quadratic", "geometry/angles", "@1a4a4a4", "@1c1c1c1"]) {
|
|
535
|
+
expect(html).toContain(piece);
|
|
536
|
+
expect(term).toContain(piece);
|
|
537
|
+
}
|
|
538
|
+
// 官方两级聚合 passRate.display 两面同一个数字,不各自重算
|
|
539
|
+
expect(html).toContain("50%");
|
|
540
|
+
expect(term).toContain("50%");
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
it("text 面先给比较表,再按 experiment 给 Eval / Attempt 父子表", () => {
|
|
544
|
+
const blocks = term.split("\n\n");
|
|
545
|
+
expect(blocks).toHaveLength(experimentListItems.length + 1);
|
|
546
|
+
expect(blocks[0]).toContain("Model");
|
|
547
|
+
expect(blocks[0]).toContain("Avg");
|
|
548
|
+
expect(blocks[1]!.match(/algebra\/quadratic/g)).toHaveLength(1);
|
|
549
|
+
expect(blocks[1]).toMatch(/✗ failed\s+algebra\/quadratic[\s\S]*├─ @1a4a4a4[\s\S]*└─ @1b5b5b5/);
|
|
550
|
+
});
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
// ───────────────────────── 排版原语 ─────────────────────────
|
|
554
|
+
|
|
555
|
+
describe("排版原语", () => {
|
|
556
|
+
it("Col 纵向堆叠;Section 标题 + 缩进;Text 折行;Style text 面为空", () => {
|
|
557
|
+
const node = (
|
|
558
|
+
<Col>
|
|
559
|
+
<Section title="考试成绩单">
|
|
560
|
+
<Text>algebra 权重 ×2,满分 100。</Text>
|
|
561
|
+
</Section>
|
|
562
|
+
<Style>{`.passbars i { background: #4a7; }`}</Style>
|
|
563
|
+
</Col>
|
|
564
|
+
);
|
|
565
|
+
expect(text(node)).toMatchInlineSnapshot(`
|
|
566
|
+
"考试成绩单
|
|
567
|
+
algebra 权重 ×2,满分 100。"
|
|
568
|
+
`);
|
|
569
|
+
const html = renderToStaticMarkup(node);
|
|
570
|
+
expect(html).toContain("<h2 class=\"nre-section-title\">考试成绩单</h2>");
|
|
571
|
+
expect(html).toContain("<style>.passbars i { background: #4a7; }</style>");
|
|
572
|
+
});
|
|
573
|
+
|
|
574
|
+
it("Row:宽度够时字符分栏,│ 分隔,各栏折到自己的宽度", () => {
|
|
575
|
+
const node = (
|
|
576
|
+
<Row>
|
|
577
|
+
<Text>left column body</Text>
|
|
578
|
+
<Text>right column body</Text>
|
|
579
|
+
</Row>
|
|
580
|
+
);
|
|
581
|
+
const wide = renderNodeToText(node, createTextContext({ width: 60 }));
|
|
582
|
+
expect(wide).toContain("│");
|
|
583
|
+
expect(wide.split("\n")[0]).toContain("left column body");
|
|
584
|
+
expect(wide.split("\n")[0]).toContain("right column body");
|
|
585
|
+
});
|
|
586
|
+
|
|
587
|
+
it("Row:宽度不足降级纵向,不硬挤", () => {
|
|
588
|
+
const node = (
|
|
589
|
+
<Row>
|
|
590
|
+
<Text>left column body</Text>
|
|
591
|
+
<Text>right column body</Text>
|
|
592
|
+
</Row>
|
|
593
|
+
);
|
|
594
|
+
const narrow = renderNodeToText(node, createTextContext({ width: 30 }));
|
|
595
|
+
expect(narrow).not.toContain("│");
|
|
596
|
+
expect(narrow).toContain("left column body\n\nright column body");
|
|
597
|
+
});
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
// ───────────────────────── Table(自定义表的标准件)─────────────────────────
|
|
601
|
+
//
|
|
602
|
+
// 官方的 MetricTable / MetricMatrix / Scoreboard / DeltaTable 的 text 面就建在它上面
|
|
603
|
+
// (上面那几个 describe 的内联快照即它的输出);这里验收作者直接用它时的四条契约:
|
|
604
|
+
// 显示宽度对齐(中文不撕歪)、右对齐、缺数据 —、超宽折行。
|
|
605
|
+
|
|
606
|
+
/** 某个片段的右边缘落在第几显示列 —— 右对齐验收看的就是它。 */
|
|
607
|
+
function displayEndOf(line: string, piece: string): number {
|
|
608
|
+
const at = line.indexOf(piece);
|
|
609
|
+
expect(at).toBeGreaterThanOrEqual(0);
|
|
610
|
+
return stringWidth(line.slice(0, at + piece.length));
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
describe("Table 双面", () => {
|
|
614
|
+
const cjkTable = (
|
|
615
|
+
<Table
|
|
616
|
+
columns={[
|
|
617
|
+
{ key: "eval", header: "题目" },
|
|
618
|
+
{ key: "pass", header: "通过率", align: "right" },
|
|
619
|
+
{ key: "cost", header: "成本", align: "right" },
|
|
620
|
+
]}
|
|
621
|
+
rows={[
|
|
622
|
+
{
|
|
623
|
+
key: "记忆/写缓存",
|
|
624
|
+
locator: "@160iuj3h" as AttemptLocator,
|
|
625
|
+
cells: { eval: "记忆/写缓存", pass: "87%", cost: "$0.09" },
|
|
626
|
+
},
|
|
627
|
+
{
|
|
628
|
+
key: "浏览/表单填写",
|
|
629
|
+
locator: "@1qrdcfq8" as AttemptLocator,
|
|
630
|
+
cells: { eval: "浏览/表单填写", pass: null, cost: null },
|
|
631
|
+
},
|
|
632
|
+
]}
|
|
633
|
+
/>
|
|
634
|
+
);
|
|
635
|
+
|
|
636
|
+
it("中文列宽:按显示宽度对齐(CJK 记 2 列),不是 .length —— String.padEnd 撕歪表的那一步", () => {
|
|
637
|
+
const term = text(cjkTable);
|
|
638
|
+
expect(term).toMatchInlineSnapshot(`
|
|
639
|
+
"题目 通过率 成本 attempt
|
|
640
|
+
记忆/写缓存 87% $0.09 @160iuj3h
|
|
641
|
+
浏览/表单填写 — — @1qrdcfq8"
|
|
642
|
+
`);
|
|
643
|
+
|
|
644
|
+
const [header, first, second] = term.split("\n");
|
|
645
|
+
// 每列的右边缘落在同一显示列:中文格子多一倍显示宽度,补齐必须按显示宽度算
|
|
646
|
+
expect(displayEndOf(first, "87%")).toBe(displayEndOf(header, "通过率"));
|
|
647
|
+
expect(displayEndOf(second, "—")).toBe(displayEndOf(header, "通过率"));
|
|
648
|
+
expect(displayEndOf(first, "$0.09")).toBe(displayEndOf(header, "成本"));
|
|
649
|
+
expect(displayEndOf(first, "@160iuj3h")).toBe(displayEndOf(second, "@1qrdcfq8"));
|
|
650
|
+
// 上面那几条不是巧合:UTF-16 码元数与显示宽度在中文上就是不同的两个数,
|
|
651
|
+
// 拿 .length 补齐(或 String.prototype.padEnd)必然歪
|
|
652
|
+
expect("记忆/写缓存".length).toBe(6);
|
|
653
|
+
expect(stringWidth("记忆/写缓存")).toBe(11);
|
|
654
|
+
});
|
|
655
|
+
|
|
656
|
+
it("align: \"right\":长短不一的数字右边缘对齐;默认列仍是左对齐", () => {
|
|
657
|
+
const term = text(
|
|
658
|
+
<Table
|
|
659
|
+
columns={[
|
|
660
|
+
{ key: "agent", header: "agent" },
|
|
661
|
+
{ key: "cost", header: "cost", align: "right" },
|
|
662
|
+
]}
|
|
663
|
+
rows={[
|
|
664
|
+
{ key: "bub", cells: { agent: "bub", cost: "$9.00" } },
|
|
665
|
+
{ key: "codex", cells: { agent: "codex", cost: "$123.45" } },
|
|
666
|
+
]}
|
|
667
|
+
/>,
|
|
668
|
+
);
|
|
669
|
+
expect(term).toMatchInlineSnapshot(`
|
|
670
|
+
"agent cost
|
|
671
|
+
bub $9.00
|
|
672
|
+
codex $123.45"
|
|
673
|
+
`);
|
|
674
|
+
const [, first, second] = term.split("\n");
|
|
675
|
+
expect(displayEndOf(first, "$9.00")).toBe(displayEndOf(second, "$123.45"));
|
|
676
|
+
// 左对齐列的左边缘对齐(不传 align 就是 left)
|
|
677
|
+
expect(first.startsWith("bub")).toBe(true);
|
|
678
|
+
expect(second.startsWith("codex")).toBe(true);
|
|
679
|
+
});
|
|
680
|
+
|
|
681
|
+
it("null → —,不补 0;cells 里干脆缺这个键也是 —;两面同源", () => {
|
|
682
|
+
const node = (
|
|
683
|
+
<Table
|
|
684
|
+
columns={[
|
|
685
|
+
{ key: "agent", header: "agent" },
|
|
686
|
+
{ key: "pass", header: "pass", align: "right" },
|
|
687
|
+
{ key: "cost", header: "cost", align: "right" },
|
|
688
|
+
]}
|
|
689
|
+
rows={[
|
|
690
|
+
{ key: "bub", cells: { agent: "bub", pass: "87%", cost: "$0.42" } },
|
|
691
|
+
{ key: "claude", cells: { agent: "claude", pass: null } }, // cost 键根本不在
|
|
692
|
+
]}
|
|
693
|
+
/>
|
|
694
|
+
);
|
|
695
|
+
const term = text(node);
|
|
696
|
+
const html = renderToStaticMarkup(node);
|
|
697
|
+
expect(term).toMatchInlineSnapshot(`
|
|
698
|
+
"agent pass cost
|
|
699
|
+
bub 87% $0.42
|
|
700
|
+
claude — —"
|
|
701
|
+
`);
|
|
702
|
+
for (const face of [term, html]) {
|
|
703
|
+
expect(face).toContain("—");
|
|
704
|
+
expect(face).not.toContain("$0.00");
|
|
705
|
+
expect(face).not.toContain("0%");
|
|
706
|
+
}
|
|
707
|
+
expect(html).toContain('<td class="nre-align-right"><span class="nre-missing">—</span></td>');
|
|
708
|
+
});
|
|
709
|
+
|
|
710
|
+
it("超宽:先折最宽的左对齐列(数字列不折),压到下限仍放不下就丢列并如实报数", () => {
|
|
711
|
+
const wide = (
|
|
712
|
+
<Table
|
|
713
|
+
columns={[
|
|
714
|
+
{ key: "eval", header: "eval" },
|
|
715
|
+
{ key: "reason", header: "reason" },
|
|
716
|
+
{ key: "cost", header: "cost", align: "right" },
|
|
717
|
+
]}
|
|
718
|
+
rows={[
|
|
719
|
+
{
|
|
720
|
+
key: "weather/brooklyn",
|
|
721
|
+
cells: {
|
|
722
|
+
eval: "weather/brooklyn",
|
|
723
|
+
reason: "gate calledTool(\"get_weather\") — the tool was never called by the agent",
|
|
724
|
+
cost: "$0.04",
|
|
725
|
+
},
|
|
726
|
+
},
|
|
727
|
+
]}
|
|
728
|
+
/>
|
|
729
|
+
);
|
|
730
|
+
const narrow = renderNodeToText(wide, createTextContext({ width: 48 }));
|
|
731
|
+
// 折行:一条逻辑行铺成多条物理行,没有一行溢出 48 列
|
|
732
|
+
const lines = narrow.split("\n");
|
|
733
|
+
expect(lines.length).toBeGreaterThan(2);
|
|
734
|
+
for (const line of lines) expect(stringWidth(line)).toBeLessThanOrEqual(48);
|
|
735
|
+
// 数字列没被折:$0.04 完整出现在某一行上
|
|
736
|
+
expect(narrow).toContain("$0.04");
|
|
737
|
+
expect(narrow).toContain("never called");
|
|
738
|
+
|
|
739
|
+
// 压到下限还是放不下 → 从右侧丢列,如实报剩余列数(不静默截断)
|
|
740
|
+
const tiny = renderNodeToText(wide, createTextContext({ width: 20 }));
|
|
741
|
+
expect(tiny).toContain("(1 more column not shown)");
|
|
742
|
+
expect(tiny).not.toContain("$0.04");
|
|
743
|
+
});
|
|
744
|
+
|
|
745
|
+
it("行带 locator:web 面链到证据室,text 面多一列 attempt", () => {
|
|
746
|
+
const html = renderToStaticMarkup(cjkTable);
|
|
747
|
+
expect(html).toContain('<a class="nre-locator" href="#/attempt/@160iuj3h">@160iuj3h</a>');
|
|
748
|
+
expect(html).toContain("<thead>");
|
|
749
|
+
expect(html).toContain("<tbody>");
|
|
750
|
+
expect(text(cjkTable)).toContain("@160iuj3h");
|
|
751
|
+
});
|
|
752
|
+
});
|
|
753
|
+
|
|
754
|
+
// ───────────────────────── 树校验与自定义组件 ─────────────────────────
|
|
755
|
+
|
|
756
|
+
describe("渲染前树校验", () => {
|
|
757
|
+
it("字符串 intrinsic(<div>)报错,错误指名组件路径", () => {
|
|
758
|
+
const bad = (
|
|
759
|
+
<Col>
|
|
760
|
+
<Section title="x">
|
|
761
|
+
<div>raw</div>
|
|
762
|
+
</Section>
|
|
763
|
+
</Col>
|
|
764
|
+
);
|
|
765
|
+
expect(() => validateReportTree(bad)).toThrow(
|
|
766
|
+
/Raw HTML <div> has no terminal face; use <Text>, layout primitives, or a defineComponent component\. \(in <Col> > <Section>\)/,
|
|
767
|
+
);
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
it("普通函数组件(组合页面片段)被调用展开继续校验", () => {
|
|
771
|
+
const Fragmented = () => (
|
|
772
|
+
<Col>
|
|
773
|
+
<Text>ok</Text>
|
|
774
|
+
</Col>
|
|
775
|
+
);
|
|
776
|
+
expect(() => validateReportTree(<Fragmented />)).not.toThrow();
|
|
777
|
+
const BadInside = () => <span>nope</span>;
|
|
778
|
+
expect(() => validateReportTree(<BadInside />)).toThrow(/Raw HTML <span>/);
|
|
779
|
+
});
|
|
780
|
+
});
|
|
781
|
+
|
|
782
|
+
describe("defineComponent", () => {
|
|
783
|
+
it("faces 两键必填,少一个面在运行时也拦住(编译期本就不过)", () => {
|
|
784
|
+
// @ts-expect-error 缺 text 面编译不过;运行时同样报错
|
|
785
|
+
expect(() => defineComponent({ web: () => null })).toThrow(/requires both faces/);
|
|
786
|
+
});
|
|
787
|
+
|
|
788
|
+
// custom-reports.mdx「换形态」的 PassBars 示例,逐字同源:text 面用公开的文本排版工具箱
|
|
789
|
+
// (stringWidth / padEnd / bar),不用 String.prototype.padEnd —— 中文名的那一行是护栏。
|
|
790
|
+
it("自定义双面组件:同一份数据两面判读一致,text 面拿到宽度;中文行不撕歪", () => {
|
|
791
|
+
interface BarRow {
|
|
792
|
+
key: string;
|
|
793
|
+
ratio: number | null;
|
|
794
|
+
display: string;
|
|
795
|
+
}
|
|
796
|
+
const PassBars = defineComponent<{ rows: BarRow[] }>({
|
|
797
|
+
web({ rows }) {
|
|
798
|
+
return (
|
|
799
|
+
<ul className="passbars">
|
|
800
|
+
{rows.map((r) => (
|
|
801
|
+
<li key={r.key}>
|
|
802
|
+
<span>{r.key}</span>
|
|
803
|
+
<b>{r.ratio === null ? "—" : r.display}</b>
|
|
804
|
+
</li>
|
|
805
|
+
))}
|
|
806
|
+
</ul>
|
|
807
|
+
);
|
|
808
|
+
},
|
|
809
|
+
text({ rows }, { width }) {
|
|
810
|
+
expect(width).toBe(80);
|
|
811
|
+
const label = Math.max(...rows.map((r) => stringWidth(r.key)));
|
|
812
|
+
const barWidth = Math.min(20, width - label - 8);
|
|
813
|
+
return rows
|
|
814
|
+
.map((r) => {
|
|
815
|
+
const chart = r.ratio === null ? padEnd("—", barWidth) : bar(r.ratio, barWidth);
|
|
816
|
+
return `${padEnd(r.key, label)} ${chart} ${r.display}`;
|
|
817
|
+
})
|
|
818
|
+
.join("\n");
|
|
819
|
+
},
|
|
820
|
+
});
|
|
821
|
+
const rows: BarRow[] = [
|
|
822
|
+
{ key: "bub", ratio: 0.87, display: "87%" },
|
|
823
|
+
{ key: "codex", ratio: 0.8, display: "80%" },
|
|
824
|
+
{ key: "克劳德", ratio: null, display: "—" },
|
|
825
|
+
];
|
|
826
|
+
const term = text(<PassBars rows={rows} />);
|
|
827
|
+
expect(term).toMatchInlineSnapshot(`
|
|
828
|
+
"bub █████████████████░░░ 87%
|
|
829
|
+
codex ████████████████░░░░ 80%
|
|
830
|
+
克劳德 — —"
|
|
831
|
+
`);
|
|
832
|
+
// 三行的条形都从同一显示列开始:"克劳德" 是 3 个码元、6 个显示列,
|
|
833
|
+
// 拿 .padEnd(label) 补齐会把这一行整体左移 3 列
|
|
834
|
+
const starts = term.split("\n").map((line, i) => stringWidth(line.slice(0, line.indexOf(["█", "█", "—"][i]))));
|
|
835
|
+
expect(new Set(starts).size).toBe(1);
|
|
836
|
+
|
|
837
|
+
const html = renderToStaticMarkup(<PassBars rows={rows} />);
|
|
838
|
+
expect(html).toContain("87%");
|
|
839
|
+
expect(html).toContain("—"); // 缺数据两面都是 —,不补 0
|
|
840
|
+
});
|
|
841
|
+
});
|
|
842
|
+
|
|
843
|
+
// ───────────────────────── defineReport 与两个宿主入口 ─────────────────────────
|
|
844
|
+
|
|
845
|
+
function fakeContext(): { selection: Selection; results: Results } {
|
|
846
|
+
const snapshot: Snapshot = (() => {
|
|
847
|
+
const dir = "/results/compare_bub/snap-1";
|
|
848
|
+
const base = {
|
|
849
|
+
experimentId: "compare/bub",
|
|
850
|
+
startedAt: "2026-07-01T10:00:00Z",
|
|
851
|
+
completedAt: "2026-07-01T10:05:00Z",
|
|
852
|
+
agent: "bub",
|
|
853
|
+
schemaVersion: 1,
|
|
854
|
+
dir,
|
|
855
|
+
};
|
|
856
|
+
const mk = (id: string, verdict: "passed" | "failed", index: number) => ({
|
|
857
|
+
evalId: id,
|
|
858
|
+
experimentId: "compare/bub",
|
|
859
|
+
result: {
|
|
860
|
+
id,
|
|
861
|
+
agent: "bub",
|
|
862
|
+
verdict,
|
|
863
|
+
attempt: 0,
|
|
864
|
+
startedAt: `2026-07-01T10:0${index}:00Z`,
|
|
865
|
+
durationMs: 1000,
|
|
866
|
+
assertions: [],
|
|
867
|
+
},
|
|
868
|
+
ref: { snapshot: "compare_bub/snap-1", attempt: `${id}/a0` },
|
|
869
|
+
snapshot: base,
|
|
870
|
+
events: async () => null,
|
|
871
|
+
trace: async () => null,
|
|
872
|
+
o11y: async () => null,
|
|
873
|
+
diff: async () => null,
|
|
874
|
+
sources: async () => null,
|
|
875
|
+
});
|
|
876
|
+
const attempts = [mk("algebra/x", "passed", 0), mk("algebra/y", "failed", 1)];
|
|
877
|
+
return {
|
|
878
|
+
...base,
|
|
879
|
+
evals: attempts.map((a) => ({ id: a.evalId, attempts: [a] })),
|
|
880
|
+
attempts,
|
|
881
|
+
} as unknown as Snapshot;
|
|
882
|
+
})();
|
|
883
|
+
const selection: Selection = {
|
|
884
|
+
snapshots: [snapshot],
|
|
885
|
+
warnings: [],
|
|
886
|
+
filter: () => selection,
|
|
887
|
+
};
|
|
888
|
+
const results = {
|
|
889
|
+
experiments: [{ id: "compare/bub", snapshots: [snapshot], latest: snapshot, evalIds: ["algebra/x", "algebra/y"] }],
|
|
890
|
+
skipped: [],
|
|
891
|
+
latest: () => selection,
|
|
892
|
+
} as Results;
|
|
893
|
+
return { selection, results };
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
/**
|
|
897
|
+
* 三份快照、三个 experiment("compare/bub"、"other/codex"、"solo"),全部通过、无成本——
|
|
898
|
+
* 一个多实验 Selection 夹具:ExperimentList 出三项,MetricScatter 三个点都无成本
|
|
899
|
+
* (0 可画点,如实走空态)。`filter` 落实真实语义(不像 fakeContext 那样恒等返回自己)。
|
|
900
|
+
*/
|
|
901
|
+
function fakeMultiGroupContext(): { selection: Selection; results: Results } {
|
|
902
|
+
const mkSnapshot = (experimentId: string, agent: string, dirSuffix: string): Snapshot => {
|
|
903
|
+
const dir = `/results/${dirSuffix}`;
|
|
904
|
+
const base = {
|
|
905
|
+
experimentId,
|
|
906
|
+
startedAt: "2026-07-01T10:00:00Z",
|
|
907
|
+
completedAt: "2026-07-01T10:05:00Z",
|
|
908
|
+
agent,
|
|
909
|
+
schemaVersion: 1,
|
|
910
|
+
dir,
|
|
911
|
+
};
|
|
912
|
+
const attempt = {
|
|
913
|
+
evalId: "algebra/x",
|
|
914
|
+
experimentId,
|
|
915
|
+
result: {
|
|
916
|
+
id: "algebra/x",
|
|
917
|
+
agent,
|
|
918
|
+
verdict: "passed" as const,
|
|
919
|
+
attempt: 0,
|
|
920
|
+
startedAt: "2026-07-01T10:00:00Z",
|
|
921
|
+
durationMs: 1000,
|
|
922
|
+
assertions: [],
|
|
923
|
+
},
|
|
924
|
+
ref: { snapshot: dirSuffix, attempt: "algebra/x/a0" },
|
|
925
|
+
snapshot: base,
|
|
926
|
+
events: async () => null,
|
|
927
|
+
trace: async () => null,
|
|
928
|
+
o11y: async () => null,
|
|
929
|
+
diff: async () => null,
|
|
930
|
+
sources: async () => null,
|
|
931
|
+
};
|
|
932
|
+
return {
|
|
933
|
+
...base,
|
|
934
|
+
evals: [{ id: attempt.evalId, attempts: [attempt] }],
|
|
935
|
+
attempts: [attempt],
|
|
936
|
+
} as unknown as Snapshot;
|
|
937
|
+
};
|
|
938
|
+
|
|
939
|
+
const snapshots = [
|
|
940
|
+
mkSnapshot("compare/bub", "bub", "compare_bub/snap-1"),
|
|
941
|
+
mkSnapshot("other/codex", "codex", "other_codex/snap-1"),
|
|
942
|
+
mkSnapshot("solo", "bub", "solo/snap-1"),
|
|
943
|
+
];
|
|
944
|
+
|
|
945
|
+
const makeSelection = (snaps: Snapshot[]): Selection => ({
|
|
946
|
+
snapshots: snaps,
|
|
947
|
+
warnings: [],
|
|
948
|
+
filter: (predicate) => makeSelection(snaps.filter(predicate)),
|
|
949
|
+
});
|
|
950
|
+
const selection = makeSelection(snapshots);
|
|
951
|
+
const results = {
|
|
952
|
+
experiments: snapshots.map((s) => ({
|
|
953
|
+
id: s.experimentId,
|
|
954
|
+
snapshots: [s],
|
|
955
|
+
latest: s,
|
|
956
|
+
evalIds: ["algebra/x"],
|
|
957
|
+
})),
|
|
958
|
+
skipped: [],
|
|
959
|
+
latest: () => selection,
|
|
960
|
+
} as Results;
|
|
961
|
+
return { selection, results };
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
describe("defineReport + 渲染入口", () => {
|
|
965
|
+
// ExperimentList 没有 selection-form:build() 里直接 await .data(selection) 拿到普通数组
|
|
966
|
+
// 再传 items;RunOverview / Scoreboard 同样是预计算的 data 形态。三者都在 build() 里
|
|
967
|
+
// await,一棵树验证同一个 renderReportToText/renderReportToStaticHtml 管线消费得动它们。
|
|
968
|
+
const report = defineReport(async ({ selection }) => {
|
|
969
|
+
const experiments = await ExperimentList.data(selection);
|
|
970
|
+
return (
|
|
971
|
+
<Col>
|
|
972
|
+
<RunOverview data={await RunOverview.data(selection)} />
|
|
973
|
+
<ExperimentList items={experiments} />
|
|
974
|
+
<Section title="考试成绩单">
|
|
975
|
+
<Scoreboard data={await Scoreboard.data(selection, { rows: "agent", subjects: "evalGroup" })} />
|
|
976
|
+
</Section>
|
|
977
|
+
</Col>
|
|
978
|
+
);
|
|
979
|
+
});
|
|
980
|
+
|
|
981
|
+
it("isReportDefinition 识别产物;非函数报错", () => {
|
|
982
|
+
expect(isReportDefinition(report)).toBe(true);
|
|
983
|
+
expect(isReportDefinition({})).toBe(false);
|
|
984
|
+
// @ts-expect-error 非函数
|
|
985
|
+
expect(() => defineReport(42)).toThrow(/expects a build function/);
|
|
986
|
+
});
|
|
987
|
+
|
|
988
|
+
it("renderReportToText:同一棵树走 text 面,build() 里 await 的实体列表数据原样渲染", async () => {
|
|
989
|
+
const out = await renderReportToText(report, fakeContext(), { width: 100 });
|
|
990
|
+
// RunOverview(data 形态,预计算)
|
|
991
|
+
expect(out).toContain("1 experiment · 2 evals · 2 attempts");
|
|
992
|
+
// ExperimentList:主行 + eval 级折叠计票 + 逐 Attempt locator 与失败诊断
|
|
993
|
+
expect(out).toMatch(/compare\/bub\s+default\s+bub/);
|
|
994
|
+
expect(out).toContain("1 passed / 1 failed");
|
|
995
|
+
expect(out).toContain("50%");
|
|
996
|
+
expect(out).toMatch(/✗ failed\s+algebra\/y[\s\S]*└─ @[0-9a-z]+/);
|
|
997
|
+
// 自己的口径:成绩单
|
|
998
|
+
expect(out).toContain("考试成绩单");
|
|
999
|
+
expect(out).toContain("bub");
|
|
1000
|
+
expect(out).toContain("50/100");
|
|
1001
|
+
});
|
|
1002
|
+
|
|
1003
|
+
it("renderReportToStaticHtml:同一棵树走 web 面,ExperimentList 自动接证据室深链", async () => {
|
|
1004
|
+
const html = await renderReportToStaticHtml(report, fakeContext());
|
|
1005
|
+
expect(html).toContain("nre-experiment-list");
|
|
1006
|
+
expect(html).toContain("nre-scoreboard");
|
|
1007
|
+
expect(html).toContain("考试成绩单");
|
|
1008
|
+
// 宿主注入的 attemptHref:ExperimentList 的 locator 徽标是普通 <a>,默认 view 路由(单段 locator)
|
|
1009
|
+
expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/);
|
|
1010
|
+
expect(html).not.toContain("<script");
|
|
1011
|
+
});
|
|
1012
|
+
|
|
1013
|
+
it("RunOverview 已渲染 Selection warning 时,宿主不会在 text/web 顶部重复一遍", async () => {
|
|
1014
|
+
const context = fakeContext();
|
|
1015
|
+
const message = "snapshot is unfinished; completed attempts may be incomplete";
|
|
1016
|
+
context.selection.warnings.push({
|
|
1017
|
+
kind: "unfinished-snapshot",
|
|
1018
|
+
experimentId: "compare/bub",
|
|
1019
|
+
startedAt: "2026-07-01T10:00:00Z",
|
|
1020
|
+
dir: "/results/compare_bub/snap-1",
|
|
1021
|
+
message,
|
|
1022
|
+
});
|
|
1023
|
+
const textOut = await renderReportToText(report, context, { width: 100 });
|
|
1024
|
+
const htmlOut = await renderReportToStaticHtml(report, context);
|
|
1025
|
+
expect(textOut.split(message)).toHaveLength(2);
|
|
1026
|
+
expect(htmlOut.split(message)).toHaveLength(2);
|
|
1027
|
+
});
|
|
1028
|
+
|
|
1029
|
+
it("树里混进 <div>:两个宿主 resolve 后同一遍校验拦住", async () => {
|
|
1030
|
+
const bad = defineReport(() => (
|
|
1031
|
+
<Col>
|
|
1032
|
+
<div>nope</div>
|
|
1033
|
+
</Col>
|
|
1034
|
+
));
|
|
1035
|
+
await expect(renderReportToText(bad, fakeContext())).rejects.toThrow(/Raw HTML <div>/);
|
|
1036
|
+
await expect(renderReportToStaticHtml(bad, fakeContext())).rejects.toThrow(/Raw HTML <div>/);
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
it("selection 形态组件裸嵌进 React(不经宿主 resolve):web 面直说未解析", () => {
|
|
1040
|
+
const { selection } = fakeContext();
|
|
1041
|
+
// selection 形态在类型上合法,但裸调用路径(不过 resolveReportTree)只接收 data 形态,
|
|
1042
|
+
// web 面拿不到 data —— 运行时直说,而不是画一张空组件。MetricScatter 是本文件里仍带
|
|
1043
|
+
// selection-form 的官方组件(ExperimentList/EvalList/AttemptList 没有 selection-form,
|
|
1044
|
+
// 这条契约不适用于它们)。
|
|
1045
|
+
expect(() =>
|
|
1046
|
+
renderToStaticMarkup(<MetricScatter selection={selection} points="experiment" x={costUSD} y={passRate} />),
|
|
1047
|
+
).toThrow(/received unresolved \(selection-form\) props/);
|
|
1048
|
+
});
|
|
1049
|
+
});
|
|
1050
|
+
|
|
1051
|
+
// ───────────────────────── 数据组件互斥 props(类型层负向)─────────────────────────
|
|
1052
|
+
// 只在 pnpm run typecheck 生效(src 纳入 tsconfig);函数从不执行,仅让 tsc 校验每条
|
|
1053
|
+
// ts-expect-error 注释对应一处编译错误。直接标注 Props 类型(而非 JSX)——JSX 的 union
|
|
1054
|
+
// 属性检查对「缺必填字段」较宽松,直接的类型赋值才严格钉住互斥不变量:同时传 data 与
|
|
1055
|
+
// selection、或两者都不传、或 selection 形态缺必填计算选项,全部编译失败。正向 JSX 用法由
|
|
1056
|
+
// ExperimentComparison / defineReport 报告在同一文件里编译验证。
|
|
1057
|
+
function metricScatterPropsTypeChecks(selection: Selection, data: ScatterData): void {
|
|
1058
|
+
const ok1: MetricScatterProps = { data }; // 合法:data 形态
|
|
1059
|
+
const ok2: MetricScatterProps = { selection, points: "experiment", series: "agent", x: costUSD, y: passRate }; // 合法:selection 形态
|
|
1060
|
+
// @ts-expect-error 同时传 data 与 selection:非法
|
|
1061
|
+
const bad1: MetricScatterProps = { data, selection, points: "experiment", x: costUSD, y: passRate };
|
|
1062
|
+
// @ts-expect-error data 与 selection 都不传:非法
|
|
1063
|
+
const bad2: MetricScatterProps = { pointHref: () => "/x" };
|
|
1064
|
+
// @ts-expect-error selection 形态缺必填的 x / y:非法
|
|
1065
|
+
const bad3: MetricScatterProps = { selection, points: "experiment" };
|
|
1066
|
+
void ok1;
|
|
1067
|
+
void ok2;
|
|
1068
|
+
void bad1;
|
|
1069
|
+
void bad2;
|
|
1070
|
+
void bad3;
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
void metricScatterPropsTypeChecks;
|
|
1074
|
+
|
|
1075
|
+
// ───────────────────────── ExperimentComparison(内置默认报告)─────────────────────────
|
|
1076
|
+
|
|
1077
|
+
describe("ExperimentComparison", () => {
|
|
1078
|
+
it("是普通 ReportDefinition;text 面 = 成本×通过率散点 + 实验列表,别无它物", async () => {
|
|
1079
|
+
expect(isReportDefinition(ExperimentComparison)).toBe(true);
|
|
1080
|
+
const out = await renderReportToText(ExperimentComparison, fakeContext(), { width: 100 });
|
|
1081
|
+
// 散点:fakeContext 无成本数据 → 0 可画点,显式说明缺哪两个指标(而不是画一张空图)
|
|
1082
|
+
expect(out).toContain("No data to plot");
|
|
1083
|
+
expect(out).not.toContain("better → upper right");
|
|
1084
|
+
// 实验列表主行 + eval 级折叠计票 + 失败诊断(ExperimentList.data 在 build() 里直接 await)
|
|
1085
|
+
expect(out).toMatch(/compare\/bub\s+default\s+bub/);
|
|
1086
|
+
expect(out).toContain("1 passed / 1 failed");
|
|
1087
|
+
expect(out).toContain("50%");
|
|
1088
|
+
expect(out).toMatch(/✗ failed\s+algebra\/y[\s\S]*└─ @[0-9a-z]+/);
|
|
1089
|
+
// 只有两个直接业务组件:没有 RunOverview / GroupSummary / Section 分组
|
|
1090
|
+
expect(out).not.toContain("Current verdicts");
|
|
1091
|
+
});
|
|
1092
|
+
|
|
1093
|
+
it("web 面:散点空态 + 实验列表 <details> 展开区,无 <script>,无 Section 分组", async () => {
|
|
1094
|
+
const html = await renderReportToStaticHtml(ExperimentComparison, fakeContext());
|
|
1095
|
+
expect(html).toContain("nre-metric-scatter");
|
|
1096
|
+
expect(html).toContain("nre-scatter-empty"); // 0 可画点的空态
|
|
1097
|
+
expect(html).toContain('<details class="nre-experiment-entry">');
|
|
1098
|
+
expect(html).toContain("nre-experiment-head");
|
|
1099
|
+
expect(html).toContain('data-nre-experiment-filter=""');
|
|
1100
|
+
expect(html).toContain("nre-experiment-evals");
|
|
1101
|
+
expect(html).not.toContain("nre-section");
|
|
1102
|
+
expect(html).not.toContain("<script");
|
|
1103
|
+
});
|
|
1104
|
+
|
|
1105
|
+
it("locale 变体:en / zh-CN 都渲染(chrome 分语言),散点空态两面同一事实", async () => {
|
|
1106
|
+
const zhHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
|
|
1107
|
+
expect(zhHtml).toContain("成功率"); // passRate 的 zh-CN label(ExperimentList 主行)
|
|
1108
|
+
const enHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "en" });
|
|
1109
|
+
expect(enHtml).toContain("Pass rate");
|
|
1110
|
+
const zhText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
|
|
1111
|
+
expect(zhText).toContain("没有可绘制的数据"); // 散点空态 zh
|
|
1112
|
+
const enText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "en" });
|
|
1113
|
+
expect(enText).toContain("No data to plot");
|
|
1114
|
+
});
|
|
1115
|
+
|
|
1116
|
+
it("多实验 fixture:每个 experiment 一项,散点如实处理全部无成本", async () => {
|
|
1117
|
+
const out = await renderReportToText(ExperimentComparison, fakeMultiGroupContext(), { width: 100 });
|
|
1118
|
+
// 三个 experiment 的身份行都出现(experimentId · agent,不再截短成最后一段)
|
|
1119
|
+
expect(out).toMatch(/compare\/bub\s+default\s+bub/);
|
|
1120
|
+
expect(out).toMatch(/other\/codex\s+default\s+codex/);
|
|
1121
|
+
expect(out).toMatch(/solo\s+default\s+bub/);
|
|
1122
|
+
// 散点空态(三点都无成本)
|
|
1123
|
+
expect(out).toContain("No data to plot");
|
|
1124
|
+
// 没有组分 Section 标题(不再按目录前缀分组)
|
|
1125
|
+
expect(out).not.toMatch(/^compare$/m);
|
|
1126
|
+
const html = await renderReportToStaticHtml(ExperimentComparison, fakeMultiGroupContext());
|
|
1127
|
+
expect(html).not.toContain("nre-section");
|
|
1128
|
+
});
|
|
1129
|
+
});
|