niceeval 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +2 -2
- package/dist/i18n/zh-CN.d.ts +3 -3
- package/dist/report/aggregate.d.ts +32 -26
- package/dist/report/aggregate.js +157 -76
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +91 -164
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +86 -73
- package/dist/report/compute.js +592 -432
- package/dist/report/flag.d.ts +28 -17
- package/dist/report/flag.js +86 -16
- package/dist/report/format.d.ts +11 -11
- package/dist/report/format.js +17 -15
- package/dist/report/index.d.ts +16 -17
- package/dist/report/index.js +20 -22
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +49 -34
- package/dist/report/locale.js +106 -58
- package/dist/report/metrics.d.ts +10 -3
- package/dist/report/metrics.js +46 -12
- package/dist/report/primitives.d.ts +42 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +10 -8
- package/dist/report/react/AttemptList.js +18 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +3 -3
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +4 -2
- package/dist/report/react/ExperimentComparison.js +5 -4
- package/dist/report/react/ExperimentList.d.ts +3 -3
- package/dist/report/react/ExperimentList.js +16 -15
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +54 -17
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +105 -149
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -20
- package/dist/report/report.js +261 -34
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +225 -157
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +2 -2
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +245 -300
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +38 -14
- package/dist/shared/aggregate.d.ts +3 -2
- package/dist/shared/aggregate.js +5 -4
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
- package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
- package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
- package/docs-site/zh/index.mdx +23 -25
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +5 -5
- package/docs-site/zh/reference/capabilities.mdx +6 -6
- package/docs-site/zh/reference/cli.mdx +9 -7
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
- package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
- package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
- package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
- package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +6 -2
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +3 -3
- package/src/cli.ts +42 -23
- package/src/context/context.ts +1 -1
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/i18n/en.ts +18 -16
- package/src/i18n/zh-CN.ts +16 -15
- package/src/report/aggregate.ts +175 -87
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -285
- package/src/report/compute.ts +717 -515
- package/src/report/dual-render.test.tsx +738 -1148
- package/src/report/flag.ts +97 -33
- package/src/report/format.ts +18 -22
- package/src/report/index.ts +113 -58
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +120 -69
- package/src/report/metrics.ts +42 -12
- package/src/report/primitives.tsx +190 -45
- package/src/report/react/AttemptList.tsx +32 -20
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +12 -7
- package/src/report/react/ExperimentList.tsx +38 -26
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +74 -20
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/enhance.js +57 -5
- package/src/report/react/fixtures.ts +109 -156
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +139 -104
- package/src/report/react/styles.css +181 -91
- package/src/report/report.test.ts +761 -1031
- package/src/report/report.ts +425 -47
- package/src/report/text/faces.ts +257 -164
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +2 -2
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +257 -287
- package/src/report/web.ts +63 -20
- package/src/results/attempt-evidence.test.ts +4 -4
- package/src/results/attempt-evidence.ts +5 -5
- package/src/results/copy.ts +6 -6
- package/src/results/host-equivalence.test.ts +26 -14
- package/src/results/index.ts +10 -4
- package/src/results/open.ts +8 -4
- package/src/results/results.test.ts +4 -3
- package/src/results/select.ts +104 -34
- package/src/results/types.ts +36 -14
- package/src/runner/feedback/human.test.ts +1 -1
- package/src/runner/run.ts +1 -1
- package/src/sandbox/cli-commands.ts +2 -2
- package/src/scoring/judge.test.ts +1 -1
- package/src/shared/aggregate.ts +5 -4
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +107 -56
- package/src/show/render.ts +43 -27
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +86 -36
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +324 -63
- package/src/view/app/components/CodeView.tsx +10 -4
- package/src/view/app/i18n.ts +31 -17
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +14 -14
- package/src/view/data.test.ts +9 -3
- package/src/view/data.ts +145 -49
- package/src/view/index.ts +48 -44
- package/src/view/server.ts +35 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +224 -0
- package/src/view/view-report.test.ts +161 -57
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
- package/dist/report/built-ins/experiment-comparison.js +0 -119
- package/dist/report/built-ins/index.d.ts +0 -2
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/src/report/built-in-user-parity.test.tsx +0 -597
- package/src/report/built-ins/experiment-comparison.tsx +0 -179
- package/src/report/built-ins/index.ts +0 -7
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// 手工 fixture:模拟计算函数(
|
|
1
|
+
// 手工 fixture:模拟计算函数(metricTableData / metricMatrixData / …)的产物。
|
|
2
2
|
// 仅供渲染测试与 scripts/report-react-demo.tsx 使用,不从入口导出。
|
|
3
3
|
// 数字刻意覆盖诚实细节:coverage 角标(samples<total)、全 null 格子、
|
|
4
4
|
// 稀疏矩阵、缺数据的散点、delta 的 null 不硬算、truncated 计数。
|
|
@@ -6,53 +6,20 @@ const locator = (s) => s;
|
|
|
6
6
|
export const passRateColumn = { key: "task-pass-rate", label: "pass rate", unit: "%", better: "higher" };
|
|
7
7
|
export const codeLinesColumn = { key: "code-lines", label: "code lines", unit: "lines", better: "lower" };
|
|
8
8
|
export const costColumn = { key: "cost", label: "cost", unit: "$", better: "lower" };
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
{ experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T11:30:00Z" },
|
|
13
|
-
],
|
|
14
|
-
totals: {
|
|
15
|
-
evals: 12,
|
|
16
|
-
attempts: 48,
|
|
17
|
-
passed: 36,
|
|
18
|
-
failed: 8,
|
|
19
|
-
errored: 2,
|
|
20
|
-
skipped: 2,
|
|
21
|
-
// 两级聚合口径(computeCell)刻意不等于 36/(36+8+2)≈78% 的 attempt 原始占比:
|
|
22
|
-
// 组件必须原样渲染这个字段,不得从上面四个 verdict 计票现场重算。
|
|
23
|
-
passRate: { value: 0.7, display: "70%", samples: 46, total: 48, refs: [] },
|
|
24
|
-
costUSD: null, // 全部 attempt 都没报成本:null,组件必须显示缺数据而不是 $0
|
|
25
|
-
durationMs: 261_000,
|
|
26
|
-
},
|
|
27
|
-
warnings: [
|
|
28
|
-
{
|
|
29
|
-
kind: "partial-coverage",
|
|
30
|
-
experimentId: "compare/bub",
|
|
31
|
-
covered: 9,
|
|
32
|
-
total: 12,
|
|
33
|
-
message: "snapshot covers 9 of 12 evals seen in history; re-run `niceeval exp compare/bub` for a full snapshot",
|
|
34
|
-
},
|
|
35
|
-
],
|
|
36
|
-
};
|
|
37
|
-
export const overviewWithCost = {
|
|
38
|
-
...overviewData,
|
|
39
|
-
totals: { ...overviewData.totals, costUSD: 1.234 },
|
|
40
|
-
warnings: [],
|
|
41
|
-
};
|
|
42
|
-
/** GroupSummary.data 的产物形态:eval 级折叠计票 + 旧 GroupSelector 口径的通过率(包成 MetricCell)。 */
|
|
43
|
-
export const groupSummaryData = {
|
|
9
|
+
/** scopeSummaryData 的产物形态:两级计票恒随行,成功率与总成本是官方 MetricCell。 */
|
|
10
|
+
export const scopeSummaryData = {
|
|
11
|
+
range: { earliestStartedAt: "2026-07-01T10:00:00Z", latestStartedAt: "2026-07-01T11:30:00Z" },
|
|
44
12
|
experiments: 2,
|
|
45
|
-
// evals = 全部 verdicts 之和(3+1+1+1=6);passRate 分母只数非 skipped 的 5 道
|
|
46
13
|
evals: 6,
|
|
47
14
|
attempts: 9,
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
15
|
+
evalVerdicts: { passed: 3, failed: 1, errored: 1, skipped: 1 },
|
|
16
|
+
attemptVerdicts: { passed: 4, failed: 3, errored: 1, skipped: 1 },
|
|
17
|
+
// 两级聚合口径,刻意不等于任一计票的比例:组件必须原样渲染,不重算
|
|
18
|
+
endToEndPassRate: { value: 0.6, display: "60%", samples: 8, total: 9, refs: [] },
|
|
19
|
+
totalCostUSD: { value: 1.5, display: "$1.50", samples: 8, total: 9, refs: [] },
|
|
53
20
|
};
|
|
54
21
|
export const tableData = {
|
|
55
|
-
|
|
22
|
+
rowDimension: "agent",
|
|
56
23
|
// 行顺序故意不按 passRate 排:组件必须按传入顺序渲染,不重排
|
|
57
24
|
columns: [passRateColumn, codeLinesColumn],
|
|
58
25
|
rows: [
|
|
@@ -72,7 +39,7 @@ export const tableData = {
|
|
|
72
39
|
display: "87%",
|
|
73
40
|
samples: 6,
|
|
74
41
|
total: 6,
|
|
75
|
-
refs: [locator("@
|
|
42
|
+
refs: [locator("@1a0a0a0a")],
|
|
76
43
|
},
|
|
77
44
|
// samples < total:有 attempt 测不了 → 覆盖率角标 5/6
|
|
78
45
|
"code-lines": { value: 120, display: "120 lines", samples: 5, total: 6, refs: [] },
|
|
@@ -80,33 +47,9 @@ export const tableData = {
|
|
|
80
47
|
},
|
|
81
48
|
],
|
|
82
49
|
};
|
|
83
|
-
/** rows: "experiment" 的榜单形态:行携带 agent/model 元信息与 eval 级折叠计票。 */
|
|
84
|
-
export const tableDataWithMeta = {
|
|
85
|
-
dimension: "experiment",
|
|
86
|
-
columns: [passRateColumn],
|
|
87
|
-
rows: [
|
|
88
|
-
{
|
|
89
|
-
key: "compare/bub",
|
|
90
|
-
cells: { "task-pass-rate": { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] } },
|
|
91
|
-
meta: {
|
|
92
|
-
agent: "bub",
|
|
93
|
-
model: "gpt-5.4",
|
|
94
|
-
verdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
|
|
95
|
-
},
|
|
96
|
-
},
|
|
97
|
-
{
|
|
98
|
-
key: "compare/codex",
|
|
99
|
-
cells: { "task-pass-rate": { value: 1, display: "100%", samples: 2, total: 2, refs: [] } },
|
|
100
|
-
meta: {
|
|
101
|
-
agent: "codex",
|
|
102
|
-
verdicts: { passed: 2, failed: 0, errored: 0, skipped: 0 },
|
|
103
|
-
},
|
|
104
|
-
},
|
|
105
|
-
],
|
|
106
|
-
};
|
|
107
50
|
export const matrixData = {
|
|
108
|
-
|
|
109
|
-
|
|
51
|
+
rowDimension: "eval",
|
|
52
|
+
columnDimension: "agent",
|
|
110
53
|
metric: passRateColumn,
|
|
111
54
|
// 稀疏:geometry/angles × codex 没有样本,数据里不出现 → 格子空着
|
|
112
55
|
cells: [
|
|
@@ -118,7 +61,7 @@ export const matrixData = {
|
|
|
118
61
|
display: "100%",
|
|
119
62
|
samples: 2,
|
|
120
63
|
total: 2,
|
|
121
|
-
refs: [locator("@
|
|
64
|
+
refs: [locator("@1b3b3b3b"), locator("@1b7b7b7b")],
|
|
122
65
|
},
|
|
123
66
|
},
|
|
124
67
|
{
|
|
@@ -134,33 +77,40 @@ export const matrixData = {
|
|
|
134
77
|
],
|
|
135
78
|
};
|
|
136
79
|
export const scoreboardData = {
|
|
137
|
-
|
|
80
|
+
rowDimension: "agent",
|
|
81
|
+
questions: [
|
|
82
|
+
"algebra/quadratic",
|
|
83
|
+
"algebra/simple",
|
|
84
|
+
"geometry/angles",
|
|
85
|
+
"geometry/area",
|
|
86
|
+
],
|
|
138
87
|
fullMarks: 100,
|
|
139
88
|
weights: [{ prefix: "algebra/", weight: 2 }],
|
|
89
|
+
ignoredEvals: 1,
|
|
140
90
|
rows: [
|
|
141
91
|
{
|
|
142
92
|
key: "bub",
|
|
143
|
-
|
|
93
|
+
// notRun 1:一题没跑、按 0 计;unscorable 0 —— 两种 0 分分开计数
|
|
94
|
+
total: { value: 78.5, display: "78.5", notRun: 1, unscorable: 0, refs: [locator("@1a0a0a0a")] },
|
|
144
95
|
subjects: [
|
|
145
|
-
|
|
146
|
-
{ key: "
|
|
147
|
-
{ key: "geometry", earned: 3, possible: 4, evals: 4, missing: 0 },
|
|
96
|
+
{ key: "algebra", earned: 14, possible: 16, questions: 8, notRun: 1, unscorable: 0, display: "14/16 (87.5%)", refs: [] },
|
|
97
|
+
{ key: "geometry", earned: 3, possible: 4, questions: 4, notRun: 0, unscorable: 0, display: "3/4 (75%)", refs: [] },
|
|
148
98
|
],
|
|
149
99
|
},
|
|
150
100
|
{
|
|
151
101
|
key: "codex",
|
|
152
|
-
total: { value: 52, display: "52" },
|
|
102
|
+
total: { value: 52, display: "52", notRun: 0, unscorable: 2, refs: [] },
|
|
153
103
|
subjects: [
|
|
154
|
-
{ key: "algebra", earned: 9, possible: 16,
|
|
155
|
-
{ key: "geometry", earned: 1.4, possible: 4,
|
|
104
|
+
{ key: "algebra", earned: 9, possible: 16, questions: 8, notRun: 0, unscorable: 0, display: "9/16 (56.3%)", refs: [] },
|
|
105
|
+
{ key: "geometry", earned: 1.4, possible: 4, questions: 4, notRun: 0, unscorable: 2, display: "1.4/4 (35%)", refs: [] },
|
|
156
106
|
],
|
|
157
107
|
},
|
|
158
108
|
],
|
|
159
109
|
};
|
|
160
110
|
export const scatterData = {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
x: costColumn, // better: "lower" →
|
|
111
|
+
pointDimension: "experiment",
|
|
112
|
+
seriesDimension: "agent",
|
|
113
|
+
x: costColumn, // better: "lower" → 轴反向(贵在左、便宜在右);两轴都有 better → 提示「越靠右上越好」
|
|
164
114
|
y: passRateColumn,
|
|
165
115
|
rows: [
|
|
166
116
|
{
|
|
@@ -192,81 +142,87 @@ export const scatterData = {
|
|
|
192
142
|
};
|
|
193
143
|
export const lineData = {
|
|
194
144
|
x: { key: "latencyMs", label: "Simulated latency", unit: "ms" },
|
|
195
|
-
|
|
145
|
+
seriesDimension: "agents",
|
|
196
146
|
y: passRateColumn,
|
|
197
147
|
rows: [
|
|
198
148
|
{
|
|
199
|
-
key: "
|
|
149
|
+
key: "100",
|
|
200
150
|
series: "1 agents",
|
|
201
151
|
x: 100,
|
|
202
152
|
xDisplay: "100ms",
|
|
203
153
|
y: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
|
|
204
154
|
},
|
|
205
155
|
{
|
|
206
|
-
key: "
|
|
156
|
+
key: "300",
|
|
207
157
|
series: "1 agents",
|
|
208
158
|
x: 300,
|
|
209
159
|
xDisplay: "300ms",
|
|
210
160
|
y: { value: 0.3, display: "30%", samples: 6, total: 6, refs: [] },
|
|
211
161
|
},
|
|
212
162
|
{
|
|
213
|
-
key: "
|
|
163
|
+
key: "100",
|
|
214
164
|
series: "16 agents",
|
|
215
165
|
x: 100,
|
|
216
166
|
xDisplay: "100ms",
|
|
217
167
|
y: { value: 0.8, display: "80%", samples: 6, total: 6, refs: [] },
|
|
218
168
|
},
|
|
219
169
|
{
|
|
220
|
-
key: "
|
|
170
|
+
key: "300",
|
|
221
171
|
series: "16 agents",
|
|
222
172
|
x: 300,
|
|
223
173
|
xDisplay: "300ms",
|
|
224
174
|
y: { value: 0.7, display: "70%", samples: 6, total: 6, refs: [] },
|
|
225
175
|
},
|
|
226
176
|
{
|
|
227
|
-
//
|
|
228
|
-
key: "
|
|
177
|
+
// 未声明数值 flag 的 attempt:不伪造 x 值,注脚报数
|
|
178
|
+
key: "null",
|
|
229
179
|
series: "1 agents",
|
|
230
180
|
x: null,
|
|
231
|
-
xDisplay: "",
|
|
181
|
+
xDisplay: "—",
|
|
232
182
|
y: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
|
|
233
183
|
},
|
|
234
184
|
],
|
|
235
185
|
};
|
|
236
186
|
export const deltaData = {
|
|
187
|
+
byDimension: "experiment",
|
|
237
188
|
columns: [passRateColumn, costColumn],
|
|
238
189
|
rows: [
|
|
239
190
|
{
|
|
240
|
-
key: "bub",
|
|
241
|
-
|
|
242
|
-
|
|
191
|
+
key: "compare/bub → compare/bub--agents-md",
|
|
192
|
+
label: "bub",
|
|
193
|
+
a: { key: "compare/bub" },
|
|
194
|
+
b: { key: "compare/bub--agents-md" },
|
|
243
195
|
cells: {
|
|
244
|
-
// 通过率 +12pp:better higher →
|
|
196
|
+
// 通过率 +12pp:better higher → improved(绿)
|
|
245
197
|
"task-pass-rate": {
|
|
246
198
|
a: { value: 0.5, display: "50%", samples: 6, total: 6, refs: [] },
|
|
247
199
|
b: { value: 0.62, display: "62%", samples: 6, total: 6, refs: [] },
|
|
248
200
|
delta: 0.12,
|
|
249
201
|
display: "+12pp",
|
|
202
|
+
outcome: "improved",
|
|
250
203
|
},
|
|
251
|
-
// 成本 +$0.15:better lower →
|
|
204
|
+
// 成本 +$0.15:better lower → regressed(红)
|
|
252
205
|
cost: {
|
|
253
206
|
a: { value: 0.2, display: "$0.20", samples: 6, total: 6, refs: [] },
|
|
254
207
|
b: { value: 0.35, display: "$0.35", samples: 6, total: 6, refs: [] },
|
|
255
208
|
delta: 0.15,
|
|
256
209
|
display: "+$0.15",
|
|
210
|
+
outcome: "regressed",
|
|
257
211
|
},
|
|
258
212
|
},
|
|
259
213
|
},
|
|
260
214
|
{
|
|
261
|
-
key: "codex",
|
|
262
|
-
|
|
263
|
-
|
|
215
|
+
key: "compare/codex → compare/codex--agents-md",
|
|
216
|
+
label: "codex",
|
|
217
|
+
a: { key: "compare/codex" },
|
|
218
|
+
b: { key: "compare/codex--agents-md" },
|
|
264
219
|
cells: {
|
|
265
220
|
"task-pass-rate": {
|
|
266
221
|
a: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
|
|
267
222
|
b: { value: 0.4, display: "40%", samples: 6, total: 6, refs: [] },
|
|
268
223
|
delta: 0,
|
|
269
224
|
display: "±0",
|
|
225
|
+
outcome: "unchanged",
|
|
270
226
|
},
|
|
271
227
|
// A 侧缺数据 → delta null:显示缺,不硬算
|
|
272
228
|
cost: {
|
|
@@ -274,6 +230,7 @@ export const deltaData = {
|
|
|
274
230
|
b: { value: 0.3, display: "$0.30", samples: 6, total: 6, refs: [] },
|
|
275
231
|
delta: null,
|
|
276
232
|
display: "—",
|
|
233
|
+
outcome: "unavailable",
|
|
277
234
|
},
|
|
278
235
|
},
|
|
279
236
|
},
|
|
@@ -282,71 +239,70 @@ export const deltaData = {
|
|
|
282
239
|
// ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
|
|
283
240
|
/** algebra/quadratic 在 compare/bub 上失败的那次 attempt——两条子失败夹具共用同一条。 */
|
|
284
241
|
const failedAttempt = {
|
|
285
|
-
evalId: "algebra/quadratic",
|
|
286
242
|
experimentId: "compare/bub",
|
|
243
|
+
evalId: "algebra/quadratic",
|
|
287
244
|
attempt: 3,
|
|
288
245
|
agent: "bub",
|
|
289
246
|
verdict: "failed",
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
score: 0,
|
|
295
|
-
outcome: "failed",
|
|
296
|
-
expected: "x=2",
|
|
297
|
-
received: "x=3",
|
|
298
|
-
evidence: "judge: sign flipped when substituting into the quadratic formula",
|
|
299
|
-
},
|
|
300
|
-
],
|
|
247
|
+
// 已按 Scoring display 契约折好的单行摘要;渲染面只做宽度截断,不重算
|
|
248
|
+
failureSummary: "gate: roots-correct · expected x=2 · received x=3",
|
|
249
|
+
moreFailures: 1,
|
|
250
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1a4a4a4a")] },
|
|
301
251
|
durationMs: 32_000,
|
|
302
252
|
costUSD: 0.12,
|
|
303
|
-
locator: locator("@
|
|
253
|
+
locator: locator("@1a4a4a4a"),
|
|
304
254
|
};
|
|
305
255
|
const erroredAttempt = {
|
|
306
|
-
evalId: "geometry/angles",
|
|
307
256
|
experimentId: "compare/codex",
|
|
257
|
+
evalId: "geometry/angles",
|
|
308
258
|
attempt: 0,
|
|
309
259
|
agent: "codex",
|
|
310
260
|
verdict: "errored",
|
|
311
|
-
//
|
|
312
|
-
error:
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
phase: "eval.run",
|
|
316
|
-
stack: "TypeError: cannot read properties of undefined (reading 'foo')\n at run (adapter.ts:42:7)",
|
|
317
|
-
cause: { name: "TypeError", message: "cannot read properties of undefined (reading 'foo')" },
|
|
318
|
-
},
|
|
319
|
-
diagnostics: [
|
|
320
|
-
{ code: "sandbox-teardown-failed", level: "warning", message: "sandbox teardown timed out", phase: "sandbox.teardown" },
|
|
321
|
-
],
|
|
322
|
-
assertions: [],
|
|
261
|
+
// errored:结构化 error 的一层摘要(phase · code · message)
|
|
262
|
+
failureSummary: "eval.run · unexpected-error · TypeError: cannot read properties of undefined (reading 'foo')",
|
|
263
|
+
moreFailures: 0,
|
|
264
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [locator("@1c1c1c1c")] },
|
|
323
265
|
durationMs: 4_500,
|
|
324
|
-
|
|
266
|
+
costUSD: null,
|
|
267
|
+
locator: locator("@1c1c1c1c"),
|
|
325
268
|
};
|
|
326
269
|
const failedRetryAttempt = {
|
|
327
270
|
...failedAttempt,
|
|
328
271
|
attempt: 4,
|
|
329
272
|
durationMs: 35_000,
|
|
330
|
-
locator: locator("@
|
|
273
|
+
locator: locator("@1b5b5b5b"),
|
|
274
|
+
};
|
|
275
|
+
const passedAttempt = {
|
|
276
|
+
experimentId: "compare/bub",
|
|
277
|
+
evalId: "algebra/simple",
|
|
278
|
+
attempt: 0,
|
|
279
|
+
agent: "bub",
|
|
280
|
+
verdict: "passed",
|
|
281
|
+
failureSummary: null,
|
|
282
|
+
moreFailures: 0,
|
|
283
|
+
examScore: { value: 1, display: "100%", samples: 1, total: 1, refs: [locator("@1d2d2d2d")] },
|
|
284
|
+
durationMs: 5_000,
|
|
285
|
+
costUSD: 0.02,
|
|
286
|
+
locator: locator("@1d2d2d2d"),
|
|
331
287
|
};
|
|
332
288
|
export const attemptListItems = [failedAttempt, erroredAttempt];
|
|
333
289
|
export const evalListItems = [
|
|
334
290
|
{
|
|
335
|
-
evalId: "algebra/quadratic",
|
|
336
291
|
experimentId: "compare/bub",
|
|
292
|
+
evalId: "algebra/quadratic",
|
|
337
293
|
verdict: "failed",
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
294
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
295
|
+
durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
296
|
+
costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
341
297
|
attempts: [failedAttempt],
|
|
342
298
|
},
|
|
343
299
|
{
|
|
344
|
-
evalId: "geometry/angles",
|
|
345
300
|
experimentId: "compare/codex",
|
|
301
|
+
evalId: "geometry/angles",
|
|
346
302
|
verdict: "errored",
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
303
|
+
examScore: { value: 0, display: "0%", samples: 1, total: 1, refs: [erroredAttempt.locator] },
|
|
304
|
+
durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
|
|
305
|
+
costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
350
306
|
attempts: [erroredAttempt],
|
|
351
307
|
},
|
|
352
308
|
];
|
|
@@ -356,10 +312,10 @@ export const experimentListItems = [
|
|
|
356
312
|
agent: "bub",
|
|
357
313
|
model: "gpt-5.4",
|
|
358
314
|
flags: { memory: true },
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
315
|
+
evalVerdicts: { passed: 1, failed: 1, errored: 0, skipped: 0 },
|
|
316
|
+
endToEndPassRate: { value: 0.5, display: "50%", samples: 2, total: 2, refs: [] },
|
|
317
|
+
costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 2, refs: [failedAttempt.locator] },
|
|
318
|
+
durationMs: { value: 32_000, display: "32.0s", samples: 2, total: 2, refs: [] },
|
|
363
319
|
tokens: { value: null, display: "—", samples: 0, total: 2, refs: [] },
|
|
364
320
|
evals: 2,
|
|
365
321
|
attempts: 3,
|
|
@@ -368,26 +324,26 @@ export const experimentListItems = [
|
|
|
368
324
|
{
|
|
369
325
|
evalId: "algebra/quadratic",
|
|
370
326
|
verdict: "failed",
|
|
371
|
-
|
|
372
|
-
|
|
327
|
+
durationMs: { value: 32_000, display: "32.0s", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
328
|
+
costUSD: { value: 0.12, display: "$0.12", samples: 1, total: 1, refs: [failedAttempt.locator] },
|
|
373
329
|
attempts: [failedAttempt, failedRetryAttempt],
|
|
374
330
|
},
|
|
375
331
|
{
|
|
376
332
|
evalId: "algebra/simple",
|
|
377
333
|
verdict: "passed",
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
attempts: [],
|
|
334
|
+
durationMs: { value: 5_000, display: "5.0s", samples: 1, total: 1, refs: [] },
|
|
335
|
+
costUSD: { value: 0.02, display: "$0.02", samples: 1, total: 1, refs: [] },
|
|
336
|
+
attempts: [passedAttempt],
|
|
381
337
|
},
|
|
382
338
|
],
|
|
383
339
|
},
|
|
384
340
|
{
|
|
385
341
|
experimentId: "compare/codex",
|
|
386
342
|
agent: "codex",
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
343
|
+
evalVerdicts: { passed: 0, failed: 0, errored: 1, skipped: 0 },
|
|
344
|
+
endToEndPassRate: { value: 0, display: "0%", samples: 1, total: 1, refs: [] },
|
|
345
|
+
costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
346
|
+
durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [] },
|
|
391
347
|
tokens: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
392
348
|
evals: 1,
|
|
393
349
|
attempts: 1,
|
|
@@ -396,8 +352,8 @@ export const experimentListItems = [
|
|
|
396
352
|
{
|
|
397
353
|
evalId: "geometry/angles",
|
|
398
354
|
verdict: "errored",
|
|
399
|
-
|
|
400
|
-
|
|
355
|
+
durationMs: { value: 4_500, display: "4.5s", samples: 1, total: 1, refs: [erroredAttempt.locator] },
|
|
356
|
+
costUSD: { value: null, display: "—", samples: 0, total: 1, refs: [] },
|
|
401
357
|
attempts: [erroredAttempt],
|
|
402
358
|
},
|
|
403
359
|
],
|
|
@@ -1,6 +1,16 @@
|
|
|
1
|
-
export { AttemptList
|
|
2
|
-
export
|
|
3
|
-
export
|
|
4
|
-
export {
|
|
5
|
-
export
|
|
1
|
+
export { AttemptList } from "./AttemptList.tsx";
|
|
2
|
+
export { EvalList } from "./EvalList.tsx";
|
|
3
|
+
export { ExperimentList } from "./ExperimentList.tsx";
|
|
4
|
+
export { ScopeSummary } from "./ScopeSummary.tsx";
|
|
5
|
+
export { ExperimentComparisonView as ExperimentComparison } from "./ExperimentComparison.tsx";
|
|
6
|
+
export { MetricTable } from "./MetricTable.tsx";
|
|
7
|
+
export { MetricMatrix } from "./MetricMatrix.tsx";
|
|
8
|
+
export { MetricBars } from "./MetricBars.tsx";
|
|
9
|
+
export { MetricScatter } from "./MetricScatter.tsx";
|
|
10
|
+
export { MetricLine } from "./MetricLine.tsx";
|
|
11
|
+
export { DeltaTable } from "./DeltaTable.tsx";
|
|
12
|
+
export { Scoreboard } from "./Scoreboard.tsx";
|
|
13
|
+
export type { AttemptListItem, AttemptLocator, DeltaData, EvalListItem, ExperimentComparisonData, ExperimentComparisonGroupData, ExperimentListEvalRow, ExperimentListItem, LineData, MatrixData, MetricCell, MetricColumn, ScatterData, ScopeSummaryData, ScopeWarning, ScoreboardData, TableData, VerdictTally, } from "../types.ts";
|
|
14
|
+
export { DEFAULT_REPORT_LOCALE, resolveLocalizedText, resolveMetricLabel } from "../locale.ts";
|
|
15
|
+
export type { LocalizedText, ReportLocale } from "../locale.ts";
|
|
6
16
|
export { NRE_PALETTE, colorClassForKey, colorHexForKey, colorIndexForKey, seriesClassForKey } from "./colors.ts";
|
|
@@ -1,14 +1,25 @@
|
|
|
1
|
-
// niceeval/report/react ——
|
|
2
|
-
// import
|
|
3
|
-
//
|
|
4
|
-
// 不是另一套报告系统 —— 报告的家在官方宿主(--report)。
|
|
1
|
+
// niceeval/report/react —— 纯 web 渲染面的导出点:把某一块指标表嵌进已有 React 页面时
|
|
2
|
+
// 从这里 import。组件只收算好的可序列化 `data`(data 形态),不含任何读盘 / artifact
|
|
3
|
+
// 计算代码;计算函数、spec 形态与组合组件只住在 niceeval/report。
|
|
5
4
|
//
|
|
6
5
|
// 契约:
|
|
7
6
|
// - 组件只认「算好的可序列化数据」:零 hooks、零数据操作;
|
|
8
7
|
// - 样式随包发布:配套 ./styles.css(nre-* 稳定类名),使用者在其后加载覆盖即可;
|
|
8
|
+
// - 渐进增强脚本 ./enhance.js 可选加载,初始静态 HTML 无 JS 完整可读;
|
|
9
9
|
// - 跨块配色一致:维度键 → 稳定散列 → 固定调色板下标(colors.ts)。
|
|
10
|
-
export { AttemptList
|
|
11
|
-
|
|
12
|
-
export {
|
|
10
|
+
export { AttemptList } from "./AttemptList.js";
|
|
11
|
+
export { EvalList } from "./EvalList.js";
|
|
12
|
+
export { ExperimentList } from "./ExperimentList.js";
|
|
13
|
+
export { ScopeSummary } from "./ScopeSummary.js";
|
|
14
|
+
export { ExperimentComparisonView as ExperimentComparison } from "./ExperimentComparison.js";
|
|
15
|
+
export { MetricTable } from "./MetricTable.js";
|
|
16
|
+
export { MetricMatrix } from "./MetricMatrix.js";
|
|
17
|
+
export { MetricBars } from "./MetricBars.js";
|
|
18
|
+
export { MetricScatter } from "./MetricScatter.js";
|
|
19
|
+
export { MetricLine } from "./MetricLine.js";
|
|
20
|
+
export { DeltaTable } from "./DeltaTable.js";
|
|
21
|
+
export { Scoreboard } from "./Scoreboard.js";
|
|
22
|
+
// locale(官方组件 chrome 文案;LocalizedText 的按 locale 解析也用它)
|
|
23
|
+
export { DEFAULT_REPORT_LOCALE, resolveLocalizedText, resolveMetricLabel } from "../locale.js";
|
|
13
24
|
// 稳定配色(自定义组件想与官方组件同键同色时用;seriesClassForKey 配 CSS 的 --nre-series)
|
|
14
25
|
export { NRE_PALETTE, colorClassForKey, colorHexForKey, colorIndexForKey, seriesClassForKey } from "./colors.js";
|