niceeval 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.zh.md +11 -0
- package/docs/README.md +2 -1
- package/docs/adapters/collection.md +1 -1
- package/docs/adapters/contract.md +6 -5
- package/docs/adapters/reference/claude-code-otel-telemetry.md +1 -1
- package/docs/assertions.md +0 -1
- package/docs/capabilities-by-construction.md +2 -2
- package/docs/cli.md +4 -2
- package/docs/concepts.md +2 -2
- package/docs/e2e-ci.md +135 -98
- package/docs/eval-authoring.md +3 -3
- package/docs/experiments.md +7 -4
- package/docs/observability.md +9 -5
- package/docs/origin-integration.md +18 -18
- package/docs/references.md +1 -1
- package/docs/reports.md +551 -0
- package/docs/results-format.md +3 -3
- package/docs/results-lib.md +191 -0
- package/docs/runner.md +1 -1
- package/docs/sandbox.md +1 -1
- package/docs/source-map.md +22 -2
- package/docs/tier-sync.md +74 -58
- package/docs/view.md +39 -5
- package/package.json +29 -3
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/claude-code.ts +18 -1
- package/src/agents/codex.ts +3 -2
- package/src/agents/index.ts +16 -0
- package/src/agents/openai-compat.test.ts +56 -0
- package/src/agents/openai-compat.ts +151 -0
- package/src/agents/types.ts +3 -1
- package/src/cli.ts +4 -3
- package/src/context/context.test.ts +78 -1
- package/src/context/context.ts +19 -11
- package/src/context/session.ts +2 -0
- package/src/context/types.ts +11 -5
- package/src/i18n/en.ts +3 -4
- package/src/i18n/zh-CN.ts +2 -3
- package/src/o11y/cost.test.ts +40 -0
- package/src/o11y/cost.ts +27 -5
- package/src/o11y/derive.ts +2 -2
- package/src/o11y/otlp/mappers/claude-code.test.ts +31 -0
- package/src/o11y/otlp/mappers/claude-code.ts +24 -0
- package/src/o11y/otlp/mappers/index.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +127 -0
- package/src/o11y/prices.json +176 -119
- package/src/o11y/types.ts +2 -1
- package/src/report/aggregate.ts +215 -0
- package/src/report/compute.ts +405 -0
- package/src/report/format.ts +47 -0
- package/src/report/index.ts +40 -0
- package/src/report/metrics.ts +96 -0
- package/src/report/react/CaseList.tsx +70 -0
- package/src/report/react/DeltaTable.tsx +79 -0
- package/src/report/react/MetricMatrix.tsx +75 -0
- package/src/report/react/MetricScatter.tsx +217 -0
- package/src/report/react/MetricTable.tsx +71 -0
- package/src/report/react/RunOverview.tsx +87 -0
- package/src/report/react/Scoreboard.tsx +87 -0
- package/src/report/react/cell.tsx +49 -0
- package/src/report/react/colors.ts +42 -0
- package/src/report/react/data.ts +17 -0
- package/src/report/react/fixtures.ts +236 -0
- package/src/report/react/format.ts +34 -0
- package/src/report/react/index.tsx +34 -0
- package/src/report/react/render.test.tsx +303 -0
- package/src/report/react/styles.css +302 -0
- package/src/report/report.test.ts +667 -0
- package/src/report/types.ts +210 -0
- package/src/results/copy.ts +200 -0
- package/src/results/format.ts +75 -0
- package/src/results/index.ts +27 -0
- package/src/results/open.ts +207 -0
- package/src/results/results.test.ts +359 -0
- package/src/results/select.ts +101 -0
- package/src/results/types.ts +98 -0
- package/src/runner/attempt.ts +3 -1
- package/src/runner/report.test.ts +111 -0
- package/src/runner/report.ts +52 -1
- package/src/runner/reporters/artifacts.ts +1 -1
- package/src/runner/reporters/braintrust.test.ts +106 -0
- package/src/runner/reporters/braintrust.ts +197 -0
- package/src/runner/reporters/index.ts +3 -1
- package/src/runner/run.ts +30 -19
- package/src/runner/types.ts +17 -0
- package/src/sandbox/types.ts +3 -3
- package/src/view/app/App.tsx +82 -17
- package/src/view/app/components/CostScoreChart.tsx +127 -0
- package/src/view/app/i18n.ts +10 -1
- package/src/view/app/lib/attempt-route.test.ts +89 -0
- package/src/view/app/lib/attempt-route.ts +52 -0
- package/src/view/app/lib/chart.ts +72 -0
- package/src/view/app/lib/outcome.ts +1 -1
- package/src/view/app/types.ts +4 -5
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/src/view/index.ts +13 -19
- package/src/view/loader.test.ts +61 -0
- package/src/view/loader.ts +35 -12
- package/src/view/shared/types.ts +14 -1
- package/src/view/styles.css +43 -0
- package/src/view/template.html +0 -1
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// niceeval/report —— 报告积木的第二档:指标 × 计算函数,纯数据、零渲染。
|
|
2
|
+
// 契约见 docs/reports.md「指标与聚合」「计算函数与数据契约」。
|
|
3
|
+
//
|
|
4
|
+
// import 边界即运行时边界:本层的计算函数会经句柄触碰文件系统(懒加载工件),
|
|
5
|
+
// 只能进服务端 / 脚本;React 组件(第一档)在 niceeval/report/react,纯渲染。
|
|
6
|
+
// 句柄与证据身份(SnapshotHandle / AttemptHandle / AttemptRef)来自 niceeval/results。
|
|
7
|
+
|
|
8
|
+
export { defineMetric, passRate, examScore, durationMs, tokens, costUSD } from "./metrics.ts";
|
|
9
|
+
|
|
10
|
+
export { table, matrix, scoreboard, scatter, overview, delta, cases } from "./compute.ts";
|
|
11
|
+
|
|
12
|
+
export type {
|
|
13
|
+
TableOptions,
|
|
14
|
+
MatrixOptions,
|
|
15
|
+
ScoreboardOptions,
|
|
16
|
+
ScatterOptions,
|
|
17
|
+
OverviewOptions,
|
|
18
|
+
DeltaOptions,
|
|
19
|
+
DeltaPair,
|
|
20
|
+
CasesOptions,
|
|
21
|
+
} from "./compute.ts";
|
|
22
|
+
|
|
23
|
+
export type {
|
|
24
|
+
Aggregator,
|
|
25
|
+
MetricAggregate,
|
|
26
|
+
Metric,
|
|
27
|
+
Dimension,
|
|
28
|
+
MetricColumn,
|
|
29
|
+
MetricCell,
|
|
30
|
+
AttemptRef,
|
|
31
|
+
TableData,
|
|
32
|
+
MatrixData,
|
|
33
|
+
ScoreboardData,
|
|
34
|
+
ScatterData,
|
|
35
|
+
OverviewData,
|
|
36
|
+
DeltaData,
|
|
37
|
+
CaseListData,
|
|
38
|
+
} from "./types.ts";
|
|
39
|
+
|
|
40
|
+
export type { SnapshotHandle, AttemptHandle, RunHandle } from "../results/types.ts";
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// defineMetric 与内置指标。
|
|
2
|
+
//
|
|
3
|
+
// null ≠ 0:null = 此 attempt 测不了这个指标(不进聚合);0 = 测了,结果是零(照常进)。
|
|
4
|
+
// 哪个 outcome 落哪边必须显式表态,内置指标按 docs/reports.md 的表格:
|
|
5
|
+
//
|
|
6
|
+
// 指标(name) skipped errored failed passed better
|
|
7
|
+
// passRate(pass-rate) null 0 0 1 higher
|
|
8
|
+
// examScore(exam-score) null 0 0 soft 均分 higher
|
|
9
|
+
// durationMs(duration) null 实测 实测 实测 lower
|
|
10
|
+
// tokens(tokens) null 实测;无 usage→null 同左 同左 lower
|
|
11
|
+
// costUSD(cost) null 同上 同左 同左 lower
|
|
12
|
+
|
|
13
|
+
import type { EvalResult } from "../types.ts";
|
|
14
|
+
import type { Metric } from "./types.ts";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* 定义一个指标。内置指标与自定义指标是同一个类型,没有特权;
|
|
18
|
+
* 校验只管「能进计算」的最低要求,聚合语义见 docs/reports.md。
|
|
19
|
+
*/
|
|
20
|
+
export function defineMetric(def: Metric): Metric {
|
|
21
|
+
if (typeof def.name !== "string" || def.name.length === 0) {
|
|
22
|
+
throw new Error("defineMetric: metric name must be a non-empty string.");
|
|
23
|
+
}
|
|
24
|
+
if (typeof def.value !== "function") {
|
|
25
|
+
throw new Error(`defineMetric: metric "${def.name}" must provide a value(attempt) function.`);
|
|
26
|
+
}
|
|
27
|
+
return { ...def };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** 单 attempt 成本:网关实测(usage.costUSD)优先于价格表估算(estimatedCostUSD);都缺 → null,不编 0。 */
|
|
31
|
+
export function attemptCostUSD(result: EvalResult): number | null {
|
|
32
|
+
return result.usage?.costUSD ?? result.estimatedCostUSD ?? null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export const passRate = defineMetric({
|
|
36
|
+
name: "pass-rate",
|
|
37
|
+
label: "Pass rate",
|
|
38
|
+
description: "Share of evals that passed (skipped attempts excluded).",
|
|
39
|
+
better: "higher",
|
|
40
|
+
unit: "%",
|
|
41
|
+
value: (a) =>
|
|
42
|
+
a.result.outcome === "skipped" ? null : a.result.outcome === "passed" ? 1 : 0,
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
export const examScore = defineMetric({
|
|
46
|
+
name: "exam-score",
|
|
47
|
+
label: "Exam score",
|
|
48
|
+
description: "Per-eval score: gates decide pass, soft assertions grade quality.",
|
|
49
|
+
better: "higher",
|
|
50
|
+
unit: "%",
|
|
51
|
+
value(a) {
|
|
52
|
+
const { outcome, assertions } = a.result;
|
|
53
|
+
if (outcome === "skipped") return null;
|
|
54
|
+
// 先按 outcome 分派,再看断言:errored 的断言是空数组,「gate 全过才得分」的
|
|
55
|
+
// 字面实现会让条件空真成立、崩溃反而得满分 —— 交白卷是 0 分,不是缺数据,更不是满分。
|
|
56
|
+
// failed 同理得 0:--strict 下被翻成 failed 的哪怕 soft 分不低也是 0(报告不重新判卷)。
|
|
57
|
+
if (outcome !== "passed") return 0;
|
|
58
|
+
const soft = assertions.filter((x) => x.severity === "soft");
|
|
59
|
+
if (soft.length === 0) return 1;
|
|
60
|
+
return soft.reduce((sum, x) => sum + x.score, 0) / soft.length;
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
export const durationMs = defineMetric({
|
|
65
|
+
name: "duration",
|
|
66
|
+
label: "Duration",
|
|
67
|
+
description: "Wall-clock duration of the attempt.",
|
|
68
|
+
better: "lower",
|
|
69
|
+
unit: "ms",
|
|
70
|
+
value: (a) => (a.result.outcome === "skipped" ? null : a.result.durationMs),
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
export const tokens = defineMetric({
|
|
74
|
+
name: "tokens",
|
|
75
|
+
label: "Tokens",
|
|
76
|
+
description: "Input + output tokens (cache reads/writes excluded).",
|
|
77
|
+
better: "lower",
|
|
78
|
+
unit: "tokens",
|
|
79
|
+
value(a) {
|
|
80
|
+
if (a.result.outcome === "skipped") return null;
|
|
81
|
+
const usage = a.result.usage;
|
|
82
|
+
if (!usage) return null;
|
|
83
|
+
// 只加 input + output:缓存读写量大但便宜,计进去会把缓存热的 agent 画成 token 大户;
|
|
84
|
+
// 花钱多少本来就有 costUSD 负责。
|
|
85
|
+
return usage.inputTokens + usage.outputTokens;
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
export const costUSD = defineMetric({
|
|
90
|
+
name: "cost",
|
|
91
|
+
label: "Cost",
|
|
92
|
+
description: "USD cost per attempt (gateway-measured beats estimated).",
|
|
93
|
+
better: "lower",
|
|
94
|
+
unit: "$",
|
|
95
|
+
value: (a) => (a.result.outcome === "skipped" ? null : attemptCostUSD(a.result)),
|
|
96
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// CaseList:失败案例清单——报告回答完「多少」,这里回答「为什么」。
|
|
2
|
+
// 逐条列出失败断言 / error 摘要 / judge 评语(evidence),每条带 attemptHref 下钻;
|
|
3
|
+
// truncated 如实报「还有 n 条没列」。长文本收进 <details>,零 JS 也能展开。
|
|
4
|
+
|
|
5
|
+
import type { ReactElement } from "react";
|
|
6
|
+
import type { AttemptRef, CaseListData } from "./data.ts";
|
|
7
|
+
import { colorClassForKey } from "./colors.ts";
|
|
8
|
+
import { cx, formatDurationMs, formatUSD } from "./format.ts";
|
|
9
|
+
|
|
10
|
+
export function CaseList({
|
|
11
|
+
data,
|
|
12
|
+
attemptHref,
|
|
13
|
+
className,
|
|
14
|
+
}: {
|
|
15
|
+
data: CaseListData;
|
|
16
|
+
attemptHref?: (ref: AttemptRef) => string;
|
|
17
|
+
className?: string;
|
|
18
|
+
}): ReactElement {
|
|
19
|
+
return (
|
|
20
|
+
<section className={cx("nre", "nre-case-list", className)}>
|
|
21
|
+
{data.rows.length === 0 && <p className="nre-case-empty">No failed or errored attempts</p>}
|
|
22
|
+
<ol className="nre-cases">
|
|
23
|
+
{data.rows.map((row) => (
|
|
24
|
+
<li key={`${row.ref.run}:${row.ref.result}`} className={cx("nre-case", `nre-case-${row.outcome}`)}>
|
|
25
|
+
<div className="nre-case-head">
|
|
26
|
+
<span className={cx("nre-case-outcome", `nre-outcome-${row.outcome}`)}>{row.outcome}</span>
|
|
27
|
+
<span className="nre-case-eval">{row.eval}</span>
|
|
28
|
+
{/* agent 键:稳定散列上色,与其它块同键同色 */}
|
|
29
|
+
<span className={cx("nre-case-agent", "nre-key", colorClassForKey(row.agent))}>{row.agent}</span>
|
|
30
|
+
<span className="nre-case-experiment">{row.experimentId}</span>
|
|
31
|
+
<span className="nre-case-duration">{formatDurationMs(row.durationMs)}</span>
|
|
32
|
+
{row.costUSD !== undefined && <span className="nre-case-cost">{formatUSD(row.costUSD)}</span>}
|
|
33
|
+
{attemptHref && (
|
|
34
|
+
<a className="nre-case-link" href={attemptHref(row.ref)}>
|
|
35
|
+
查看 attempt
|
|
36
|
+
</a>
|
|
37
|
+
)}
|
|
38
|
+
</div>
|
|
39
|
+
|
|
40
|
+
{/* errored 的错误摘要(计算侧已过 redact) */}
|
|
41
|
+
{row.error && <p className="nre-case-error">{row.error}</p>}
|
|
42
|
+
|
|
43
|
+
{row.failedAssertions.length > 0 && (
|
|
44
|
+
<ul className="nre-assertions">
|
|
45
|
+
{row.failedAssertions.map((assertion, j) => (
|
|
46
|
+
<li key={j} className="nre-assertion">
|
|
47
|
+
<span className="nre-assertion-name">{assertion.name}</span>
|
|
48
|
+
<span className="nre-assertion-score">score {assertion.score}</span>
|
|
49
|
+
{/* detail / evidence 可能很长:<details> 收起,不 hydrate 也能展开 */}
|
|
50
|
+
{(assertion.detail || assertion.evidence) && (
|
|
51
|
+
<details className="nre-assertion-more">
|
|
52
|
+
<summary>details</summary>
|
|
53
|
+
{assertion.detail && <p className="nre-assertion-detail">{assertion.detail}</p>}
|
|
54
|
+
{assertion.evidence && (
|
|
55
|
+
<blockquote className="nre-assertion-evidence">{assertion.evidence}</blockquote>
|
|
56
|
+
)}
|
|
57
|
+
</details>
|
|
58
|
+
)}
|
|
59
|
+
</li>
|
|
60
|
+
))}
|
|
61
|
+
</ul>
|
|
62
|
+
)}
|
|
63
|
+
</li>
|
|
64
|
+
))}
|
|
65
|
+
</ol>
|
|
66
|
+
{/* limit 之外还有几条,如实报,不静默截断 */}
|
|
67
|
+
{data.truncated > 0 && <p className="nre-truncated">and {data.truncated} more not shown</p>}
|
|
68
|
+
</section>
|
|
69
|
+
);
|
|
70
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// DeltaTable:成对对比(B 相对 A)。每格三个值:A、B、Δ;
|
|
2
|
+
// Δ 的涨跌好坏由指标的 better 判定并配色(涨不一定是好——成本涨了是坏);
|
|
3
|
+
// 任一侧缺数据时 Δ 显示为缺,不硬算(数据侧已给 delta: null,这里只如实渲染)。
|
|
4
|
+
|
|
5
|
+
import type { ReactElement } from "react";
|
|
6
|
+
import type { DeltaData, MetricColumn } from "./data.ts";
|
|
7
|
+
import { MetricCellView } from "./cell.tsx";
|
|
8
|
+
import { colorClassForKey } from "./colors.ts";
|
|
9
|
+
import { MISSING_TEXT, cx } from "./format.ts";
|
|
10
|
+
|
|
11
|
+
/** Δ 的语义配色 class:好/坏由 better 方向判定,不看正负号本身。 */
|
|
12
|
+
function deltaToneClass(delta: number | null, better: MetricColumn["better"]): string {
|
|
13
|
+
if (delta === null) return "nre-delta-missing";
|
|
14
|
+
if (delta === 0) return "nre-delta-flat";
|
|
15
|
+
if (!better) return "nre-delta-neutral"; // 指标没表态方向,只显示不评判
|
|
16
|
+
const improved = better === "higher" ? delta > 0 : delta < 0;
|
|
17
|
+
return improved ? "nre-delta-good" : "nre-delta-bad";
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function DeltaTable({
|
|
21
|
+
data,
|
|
22
|
+
className,
|
|
23
|
+
}: {
|
|
24
|
+
data: DeltaData;
|
|
25
|
+
className?: string;
|
|
26
|
+
}): ReactElement {
|
|
27
|
+
return (
|
|
28
|
+
<table className={cx("nre", "nre-delta-table", className)}>
|
|
29
|
+
<thead>
|
|
30
|
+
<tr>
|
|
31
|
+
<th scope="col" className="nre-dimension">
|
|
32
|
+
对比(A → B)
|
|
33
|
+
</th>
|
|
34
|
+
{data.columns.map((col) => (
|
|
35
|
+
<th scope="col" key={col.key} className="nre-metric-col">
|
|
36
|
+
{col.label}
|
|
37
|
+
{col.unit && <span className="nre-unit">({col.unit})</span>}
|
|
38
|
+
</th>
|
|
39
|
+
))}
|
|
40
|
+
</tr>
|
|
41
|
+
</thead>
|
|
42
|
+
<tbody>
|
|
43
|
+
{data.rows.map((row) => (
|
|
44
|
+
<tr key={row.key}>
|
|
45
|
+
<th scope="row" className="nre-pair">
|
|
46
|
+
{/* pair 的 label(如 agent 名):稳定散列上色,与 scatter 的线同色 */}
|
|
47
|
+
<span className={cx("nre-row-key", "nre-key", colorClassForKey(row.key))}>{row.key}</span>
|
|
48
|
+
<span className="nre-pair-ids">
|
|
49
|
+
{row.a.experimentId} → {row.b.experimentId}
|
|
50
|
+
</span>
|
|
51
|
+
</th>
|
|
52
|
+
{data.columns.map((col) => {
|
|
53
|
+
const cell = row.cells[col.key];
|
|
54
|
+
if (!cell) return <td key={col.key} className="nre-td-empty" />;
|
|
55
|
+
return (
|
|
56
|
+
<td key={col.key} className="nre-delta-cell">
|
|
57
|
+
{/* A/B 走统一的 MetricCellView:缺数据文案与覆盖率角标同一套 */}
|
|
58
|
+
<span className="nre-delta-side nre-delta-a">
|
|
59
|
+
<span className="nre-delta-tag">A</span>
|
|
60
|
+
<MetricCellView cell={cell.a} />
|
|
61
|
+
</span>
|
|
62
|
+
<span className="nre-delta-side nre-delta-b">
|
|
63
|
+
<span className="nre-delta-tag">B</span>
|
|
64
|
+
<MetricCellView cell={cell.b} />
|
|
65
|
+
</span>
|
|
66
|
+
<span className={cx("nre-delta-side", "nre-delta-d", deltaToneClass(cell.delta, col.better))}>
|
|
67
|
+
<span className="nre-delta-tag">Δ</span>
|
|
68
|
+
{/* 任一侧 null → delta null:显示缺,不硬算 */}
|
|
69
|
+
{cell.delta === null ? <span className="nre-missing">{MISSING_TEXT}</span> : cell.display}
|
|
70
|
+
</span>
|
|
71
|
+
</td>
|
|
72
|
+
);
|
|
73
|
+
})}
|
|
74
|
+
</tr>
|
|
75
|
+
))}
|
|
76
|
+
</tbody>
|
|
77
|
+
</table>
|
|
78
|
+
);
|
|
79
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// MetricMatrix:行 × 列 × 单指标(如 eval × agent × passRate)。
|
|
2
|
+
// cells 是稀疏的——没有样本的格子在数据里不出现,这里就空着,不补 0;
|
|
3
|
+
// 行/列顺序取 cells 里的首次出现顺序(排序是数据侧的事)。
|
|
4
|
+
// cell.refs + attemptHref:「哪道题谁挂了」之后的下一步「给我看那次 attempt」
|
|
5
|
+
// 是格子里的普通 <a>,不 hydrate 也能点。
|
|
6
|
+
|
|
7
|
+
import type { ReactElement } from "react";
|
|
8
|
+
import type { AttemptRef, MatrixData } from "./data.ts";
|
|
9
|
+
import { MetricCellView } from "./cell.tsx";
|
|
10
|
+
import { colorClassForKey } from "./colors.ts";
|
|
11
|
+
import { cx } from "./format.ts";
|
|
12
|
+
|
|
13
|
+
export function MetricMatrix({
|
|
14
|
+
data,
|
|
15
|
+
attemptHref,
|
|
16
|
+
className,
|
|
17
|
+
}: {
|
|
18
|
+
data: MatrixData;
|
|
19
|
+
attemptHref?: (ref: AttemptRef) => string;
|
|
20
|
+
className?: string;
|
|
21
|
+
}): ReactElement {
|
|
22
|
+
// 稀疏 cells → 首次出现顺序的行/列键 + 查找表;组件只整理形状,不碰数值
|
|
23
|
+
const rowKeys: string[] = [];
|
|
24
|
+
const columnKeys: string[] = [];
|
|
25
|
+
// 键用 \u0000 拼接:行/列键都是用户可见 id,普通分隔符可能撞键
|
|
26
|
+
const byPosition = new Map<string, MatrixData["cells"][number]["cell"]>();
|
|
27
|
+
for (const entry of data.cells) {
|
|
28
|
+
if (!rowKeys.includes(entry.row)) rowKeys.push(entry.row);
|
|
29
|
+
if (!columnKeys.includes(entry.column)) columnKeys.push(entry.column);
|
|
30
|
+
byPosition.set(`${entry.row}\u0000${entry.column}`, entry.cell);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return (
|
|
34
|
+
<table className={cx("nre", "nre-metric-matrix", className)}>
|
|
35
|
+
<caption className="nre-matrix-caption">
|
|
36
|
+
{data.metric.label}
|
|
37
|
+
{data.metric.unit && <span className="nre-unit">({data.metric.unit})</span>}
|
|
38
|
+
<span className="nre-matrix-axes">
|
|
39
|
+
{data.rows} × {data.columns}
|
|
40
|
+
</span>
|
|
41
|
+
</caption>
|
|
42
|
+
<thead>
|
|
43
|
+
<tr>
|
|
44
|
+
<th scope="col" className="nre-dimension">
|
|
45
|
+
{data.rows}
|
|
46
|
+
</th>
|
|
47
|
+
{columnKeys.map((column) => (
|
|
48
|
+
// 列键 = 维度键(如 agent):稳定散列上色,与 scatter 的线、DeltaTable 的行同色
|
|
49
|
+
<th scope="col" key={column} className={cx("nre-col-key", "nre-key", colorClassForKey(column))}>
|
|
50
|
+
{column}
|
|
51
|
+
</th>
|
|
52
|
+
))}
|
|
53
|
+
</tr>
|
|
54
|
+
</thead>
|
|
55
|
+
<tbody>
|
|
56
|
+
{rowKeys.map((row) => (
|
|
57
|
+
<tr key={row}>
|
|
58
|
+
<th scope="row" className="nre-row-key">
|
|
59
|
+
{row}
|
|
60
|
+
</th>
|
|
61
|
+
{columnKeys.map((column) => {
|
|
62
|
+
const cell = byPosition.get(`${row}\u0000${column}`);
|
|
63
|
+
return (
|
|
64
|
+
<td key={column} className={cx("nre-td", !cell && "nre-td-empty")}>
|
|
65
|
+
{/* 稀疏格子:没有样本就空着(数据里不存在),不是 0 也不是缺数据文案 */}
|
|
66
|
+
{cell ? <MetricCellView cell={cell} attemptHref={attemptHref} /> : null}
|
|
67
|
+
</td>
|
|
68
|
+
);
|
|
69
|
+
})}
|
|
70
|
+
</tr>
|
|
71
|
+
))}
|
|
72
|
+
</tbody>
|
|
73
|
+
</table>
|
|
74
|
+
);
|
|
75
|
+
}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
// MetricScatter:质量 × 成本 frontier 的积木,内联 SVG、零图表库。
|
|
2
|
+
// 轴向随 better:"lower" 的轴反向画,「好」的角落恒在右上(成本轴 $20 → $0 就是这么来的);
|
|
3
|
+
// 同系列的点按 x 值排序连线,系列名标在线旁;x 或 y 为 null 的点不画,
|
|
4
|
+
// 底部注脚如实报「n 个点缺数据」;hover 信息退化为 SVG <title>,不 hydrate 也在。
|
|
5
|
+
|
|
6
|
+
import type { ReactElement } from "react";
|
|
7
|
+
import type { MetricColumn, ScatterData } from "./data.ts";
|
|
8
|
+
import { colorHexForKey } from "./colors.ts";
|
|
9
|
+
import { MISSING_TEXT, cx } from "./format.ts";
|
|
10
|
+
|
|
11
|
+
// 画布与边距:右侧留白给系列名,底部给 x 轴标签
|
|
12
|
+
const WIDTH = 640;
|
|
13
|
+
const HEIGHT = 400;
|
|
14
|
+
const PLOT = { left: 64, right: WIDTH - 140, top: 24, bottom: HEIGHT - 56 };
|
|
15
|
+
|
|
16
|
+
/** 可画的点:x/y 都有值。组件内部的整理结果,不改数据。 */
|
|
17
|
+
interface DrawablePoint {
|
|
18
|
+
key: string;
|
|
19
|
+
series?: string;
|
|
20
|
+
xValue: number;
|
|
21
|
+
yValue: number;
|
|
22
|
+
xDisplay: string;
|
|
23
|
+
yDisplay: string;
|
|
24
|
+
title: string;
|
|
25
|
+
px: number;
|
|
26
|
+
py: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** 一根轴的线性映射:值 → 像素;better:"lower" 时反向,好的一端固定在右/上。 */
|
|
30
|
+
function axisScale(values: number[], better: MetricColumn["better"], pixelLo: number, pixelHi: number) {
|
|
31
|
+
const lo = Math.min(...values);
|
|
32
|
+
const hi = Math.max(...values);
|
|
33
|
+
// 单值域:铺 ±1 让唯一的点落在正中,而不是除零
|
|
34
|
+
const span = hi - lo || 2;
|
|
35
|
+
const padded = { lo: lo - (hi - lo ? span * 0.08 : 1), hi: hi + (hi - lo ? span * 0.08 : 1) };
|
|
36
|
+
const scale = (v: number) => {
|
|
37
|
+
let t = (v - padded.lo) / (padded.hi - padded.lo);
|
|
38
|
+
if (better === "lower") t = 1 - t; // 反向:值越低越靠「好」的一端
|
|
39
|
+
return pixelLo + t * (pixelHi - pixelLo);
|
|
40
|
+
};
|
|
41
|
+
return { lo, hi, scale };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function MetricScatter({
|
|
45
|
+
data,
|
|
46
|
+
pointHref,
|
|
47
|
+
className,
|
|
48
|
+
}: {
|
|
49
|
+
data: ScatterData;
|
|
50
|
+
pointHref?: (row: ScatterData["rows"][number]) => string;
|
|
51
|
+
className?: string;
|
|
52
|
+
}): ReactElement {
|
|
53
|
+
const missing = data.rows.filter((r) => r.x.value === null || r.y.value === null);
|
|
54
|
+
const drawableRows = data.rows.filter((r) => r.x.value !== null && r.y.value !== null);
|
|
55
|
+
|
|
56
|
+
const missingNote =
|
|
57
|
+
missing.length > 0 ? (
|
|
58
|
+
<p className="nre-scatter-missing" title={missing.map((r) => r.key).join(", ")}>
|
|
59
|
+
{missing.length} {missing.length === 1 ? "point" : "points"} missing data
|
|
60
|
+
</p>
|
|
61
|
+
) : null;
|
|
62
|
+
|
|
63
|
+
// 一张全缺的图不画空坐标系:缺数据文案 + 注脚,与表格的「绝不画 0」同一态度
|
|
64
|
+
if (drawableRows.length === 0) {
|
|
65
|
+
return (
|
|
66
|
+
<figure className={cx("nre", "nre-metric-scatter", className)}>
|
|
67
|
+
<p className="nre-missing">{MISSING_TEXT}</p>
|
|
68
|
+
{missingNote}
|
|
69
|
+
</figure>
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const xScale = axisScale(drawableRows.map((r) => r.x.value as number), data.x.better, PLOT.left, PLOT.right);
|
|
74
|
+
// y 像素轴向下增长:better:"higher" 高值在上 → 映射到 [bottom, top];"lower" 由 axisScale 反向后同样落到上方
|
|
75
|
+
const yScale = axisScale(drawableRows.map((r) => r.y.value as number), data.y.better, PLOT.bottom, PLOT.top);
|
|
76
|
+
|
|
77
|
+
const points: DrawablePoint[] = drawableRows.map((r) => {
|
|
78
|
+
const xValue = r.x.value as number;
|
|
79
|
+
const yValue = r.y.value as number;
|
|
80
|
+
return {
|
|
81
|
+
key: r.key,
|
|
82
|
+
series: r.series,
|
|
83
|
+
xValue,
|
|
84
|
+
yValue,
|
|
85
|
+
xDisplay: r.x.display,
|
|
86
|
+
yDisplay: r.y.display,
|
|
87
|
+
// hover 内容:display 与 samples/total(docs/reports.md 行为清单)
|
|
88
|
+
title: `${r.key}\n${data.x.label}: ${r.x.display}(${r.x.samples}/${r.x.total})\n${data.y.label}: ${r.y.display}(${r.y.samples}/${r.y.total})`,
|
|
89
|
+
px: xScale.scale(xValue),
|
|
90
|
+
py: yScale.scale(yValue),
|
|
91
|
+
};
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
// 同系列的点按 x 值排序连线;无系列的点只画点不连线
|
|
95
|
+
const seriesOrder: string[] = [];
|
|
96
|
+
const bySeries = new Map<string, DrawablePoint[]>();
|
|
97
|
+
for (const p of points) {
|
|
98
|
+
if (p.series === undefined) continue;
|
|
99
|
+
if (!bySeries.has(p.series)) {
|
|
100
|
+
bySeries.set(p.series, []);
|
|
101
|
+
seriesOrder.push(p.series);
|
|
102
|
+
}
|
|
103
|
+
bySeries.get(p.series)!.push(p);
|
|
104
|
+
}
|
|
105
|
+
for (const list of bySeries.values()) list.sort((a, b) => a.xValue - b.xValue);
|
|
106
|
+
|
|
107
|
+
// 轴端刻度打在真实极值点的位置上(值域有 padding,角落不等于极值)
|
|
108
|
+
const xTicks = xScale.lo === xScale.hi ? [xScale.lo] : [xScale.lo, xScale.hi];
|
|
109
|
+
const yTicks = yScale.lo === yScale.hi ? [yScale.lo] : [yScale.lo, yScale.hi];
|
|
110
|
+
const displayFor = (axis: "x" | "y", value: number) =>
|
|
111
|
+
points.find((p) => (axis === "x" ? p.xValue : p.yValue) === value)?.[axis === "x" ? "xDisplay" : "yDisplay"] ??
|
|
112
|
+
String(value);
|
|
113
|
+
|
|
114
|
+
const axisLabel = (col: MetricColumn) => `${col.label}${col.unit ? `(${col.unit})` : ""}`;
|
|
115
|
+
|
|
116
|
+
return (
|
|
117
|
+
<figure className={cx("nre", "nre-metric-scatter", className)}>
|
|
118
|
+
<svg
|
|
119
|
+
className="nre-scatter-svg"
|
|
120
|
+
viewBox={`0 0 ${WIDTH} ${HEIGHT}`}
|
|
121
|
+
role="img"
|
|
122
|
+
aria-label={`${axisLabel(data.x)} × ${axisLabel(data.y)}`}
|
|
123
|
+
>
|
|
124
|
+
{/* 坐标框 */}
|
|
125
|
+
<rect
|
|
126
|
+
className="nre-scatter-plot"
|
|
127
|
+
x={PLOT.left}
|
|
128
|
+
y={PLOT.top}
|
|
129
|
+
width={PLOT.right - PLOT.left}
|
|
130
|
+
height={PLOT.bottom - PLOT.top}
|
|
131
|
+
fill="none"
|
|
132
|
+
stroke="#d4d4d4"
|
|
133
|
+
/>
|
|
134
|
+
{/* 「好」的角落恒在右上:轴向已按 better 反转,这里只是把这句契约写在图上 */}
|
|
135
|
+
<text className="nre-scatter-better-hint" x={PLOT.right - 6} y={PLOT.top + 14} textAnchor="end" fontSize={11} fill="#9ca3af">
|
|
136
|
+
better ↗
|
|
137
|
+
</text>
|
|
138
|
+
|
|
139
|
+
{/* 轴标签 */}
|
|
140
|
+
<text className="nre-scatter-xlabel" x={(PLOT.left + PLOT.right) / 2} y={HEIGHT - 8} textAnchor="middle" fontSize={12} fill="#525252">
|
|
141
|
+
{axisLabel(data.x)}
|
|
142
|
+
</text>
|
|
143
|
+
<text
|
|
144
|
+
className="nre-scatter-ylabel"
|
|
145
|
+
x={16}
|
|
146
|
+
y={(PLOT.top + PLOT.bottom) / 2}
|
|
147
|
+
textAnchor="middle"
|
|
148
|
+
fontSize={12}
|
|
149
|
+
fill="#525252"
|
|
150
|
+
transform={`rotate(-90 16 ${(PLOT.top + PLOT.bottom) / 2})`}
|
|
151
|
+
>
|
|
152
|
+
{axisLabel(data.y)}
|
|
153
|
+
</text>
|
|
154
|
+
|
|
155
|
+
{/* 轴端刻度:标在真实极值的位置上,文案用该点已格式化的 display */}
|
|
156
|
+
{xTicks.map((v) => (
|
|
157
|
+
<text key={`x${v}`} className="nre-scatter-tick" x={xScale.scale(v)} y={PLOT.bottom + 16} textAnchor="middle" fontSize={11} fill="#737373">
|
|
158
|
+
{displayFor("x", v)}
|
|
159
|
+
</text>
|
|
160
|
+
))}
|
|
161
|
+
{yTicks.map((v) => (
|
|
162
|
+
<text key={`y${v}`} className="nre-scatter-tick" x={PLOT.left - 6} y={yScale.scale(v) + 4} textAnchor="end" fontSize={11} fill="#737373">
|
|
163
|
+
{displayFor("y", v)}
|
|
164
|
+
</text>
|
|
165
|
+
))}
|
|
166
|
+
|
|
167
|
+
{/* 系列连线 + 线旁的系列名(标在视觉上最靠右的点旁) */}
|
|
168
|
+
{seriesOrder.map((series) => {
|
|
169
|
+
const list = bySeries.get(series)!;
|
|
170
|
+
const color = colorHexForKey(series);
|
|
171
|
+
const labelAt = list.reduce((a, b) => (b.px > a.px ? b : a));
|
|
172
|
+
return (
|
|
173
|
+
<g key={series} className="nre-scatter-series" data-series={series}>
|
|
174
|
+
{list.length > 1 && (
|
|
175
|
+
<polyline
|
|
176
|
+
className="nre-scatter-line"
|
|
177
|
+
points={list.map((p) => `${p.px},${p.py}`).join(" ")}
|
|
178
|
+
fill="none"
|
|
179
|
+
stroke={color}
|
|
180
|
+
strokeWidth={1.5}
|
|
181
|
+
/>
|
|
182
|
+
)}
|
|
183
|
+
<text className="nre-scatter-series-label" x={labelAt.px + 8} y={labelAt.py + 4} fontSize={12} fill={color}>
|
|
184
|
+
{series}
|
|
185
|
+
</text>
|
|
186
|
+
</g>
|
|
187
|
+
);
|
|
188
|
+
})}
|
|
189
|
+
|
|
190
|
+
{/* 点(带 <title> 的 hover;pointHref 时包普通 <a>,静态导出也能下钻) */}
|
|
191
|
+
{points.map((p, i) => {
|
|
192
|
+
const circle = (
|
|
193
|
+
<circle
|
|
194
|
+
className="nre-scatter-point"
|
|
195
|
+
data-key={p.key}
|
|
196
|
+
cx={p.px}
|
|
197
|
+
cy={p.py}
|
|
198
|
+
r={4.5}
|
|
199
|
+
fill={p.series !== undefined ? colorHexForKey(p.series) : "#525252"}
|
|
200
|
+
>
|
|
201
|
+
<title>{p.title}</title>
|
|
202
|
+
</circle>
|
|
203
|
+
);
|
|
204
|
+
const row = drawableRows[i];
|
|
205
|
+
return pointHref ? (
|
|
206
|
+
<a key={p.key} className="nre-scatter-point-link" href={pointHref(row)}>
|
|
207
|
+
{circle}
|
|
208
|
+
</a>
|
|
209
|
+
) : (
|
|
210
|
+
<g key={p.key}>{circle}</g>
|
|
211
|
+
);
|
|
212
|
+
})}
|
|
213
|
+
</svg>
|
|
214
|
+
{missingNote}
|
|
215
|
+
</figure>
|
|
216
|
+
);
|
|
217
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// MetricTable:行维度 × 指标列。
|
|
2
|
+
// 行按传入顺序渲染——排序发生在计算侧(table() 的 sort 参数),组件不重排;
|
|
3
|
+
// 「点列头重排」是后续的渐进增强,不 hydrate 时以数据侧预排的顺序呈现即完整。
|
|
4
|
+
// 列头以箭头标注 better 方向;samples < total 的格子带覆盖率角标;
|
|
5
|
+
// 一组全 null 渲染成「缺数据」,绝不画 0(逻辑在 MetricCellView)。
|
|
6
|
+
|
|
7
|
+
import type { ReactElement } from "react";
|
|
8
|
+
import type { AttemptRef, TableData } from "./data.ts";
|
|
9
|
+
import { MetricCellView } from "./cell.tsx";
|
|
10
|
+
import { colorClassForKey } from "./colors.ts";
|
|
11
|
+
import { cx } from "./format.ts";
|
|
12
|
+
|
|
13
|
+
export function MetricTable({
|
|
14
|
+
data,
|
|
15
|
+
attemptHref,
|
|
16
|
+
className,
|
|
17
|
+
}: {
|
|
18
|
+
data: TableData;
|
|
19
|
+
attemptHref?: (ref: AttemptRef) => string;
|
|
20
|
+
className?: string;
|
|
21
|
+
}): ReactElement {
|
|
22
|
+
return (
|
|
23
|
+
<table className={cx("nre", "nre-metric-table", className)}>
|
|
24
|
+
<thead>
|
|
25
|
+
<tr>
|
|
26
|
+
<th scope="col" className="nre-dimension">
|
|
27
|
+
{data.dimension}
|
|
28
|
+
</th>
|
|
29
|
+
{data.columns.map((col) => (
|
|
30
|
+
<th scope="col" key={col.key} className="nre-metric-col">
|
|
31
|
+
{col.label}
|
|
32
|
+
{col.unit && <span className="nre-unit">({col.unit})</span>}
|
|
33
|
+
{/* better 方向提示:↑ 越高越好 / ↓ 越低越好 */}
|
|
34
|
+
{col.better && (
|
|
35
|
+
<span
|
|
36
|
+
className="nre-better"
|
|
37
|
+
title={col.better === "higher" ? "higher is better" : "lower is better"}
|
|
38
|
+
>
|
|
39
|
+
{col.better === "higher" ? "↑" : "↓"}
|
|
40
|
+
</span>
|
|
41
|
+
)}
|
|
42
|
+
</th>
|
|
43
|
+
))}
|
|
44
|
+
</tr>
|
|
45
|
+
</thead>
|
|
46
|
+
<tbody>
|
|
47
|
+
{data.rows.map((row) => (
|
|
48
|
+
<tr key={row.key}>
|
|
49
|
+
{/* 行键 = 维度键(如 agent):稳定散列上色,跨块同键同色 */}
|
|
50
|
+
<th scope="row" className={cx("nre-row-key", "nre-key", colorClassForKey(row.key))}>
|
|
51
|
+
{row.key}
|
|
52
|
+
</th>
|
|
53
|
+
{data.columns.map((col) => {
|
|
54
|
+
const cell = row.cells[col.key];
|
|
55
|
+
return (
|
|
56
|
+
<td key={col.key} className="nre-td">
|
|
57
|
+
{cell ? (
|
|
58
|
+
<MetricCellView cell={cell} attemptHref={attemptHref} />
|
|
59
|
+
) : (
|
|
60
|
+
// 数据侧没给这个格子(理论上 table() 不会缺列)——按空处理,不编数
|
|
61
|
+
<span className="nre-empty" />
|
|
62
|
+
)}
|
|
63
|
+
</td>
|
|
64
|
+
);
|
|
65
|
+
})}
|
|
66
|
+
</tr>
|
|
67
|
+
))}
|
|
68
|
+
</tbody>
|
|
69
|
+
</table>
|
|
70
|
+
);
|
|
71
|
+
}
|