niceeval 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +5 -1
- package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
- package/dist/report/built-ins/experiment-comparison.js +13 -0
- package/dist/report/built-ins/index.d.ts +1 -1
- package/dist/report/built-ins/index.js +1 -1
- package/dist/report/components.d.ts +3 -1
- package/dist/report/components.js +1 -1
- package/dist/report/compute.js +15 -33
- package/dist/report/format.d.ts +0 -6
- package/dist/report/format.js +1 -18
- package/dist/report/index.d.ts +2 -2
- package/dist/report/index.js +1 -1
- package/dist/report/locale.d.ts +13 -2
- package/dist/report/locale.js +26 -3
- package/dist/report/metrics.js +1 -1
- package/dist/report/react/AttemptList.d.ts +2 -2
- package/dist/report/react/AttemptList.js +3 -4
- package/dist/report/react/ExperimentList.d.ts +3 -2
- package/dist/report/react/ExperimentList.js +59 -7
- package/dist/report/react/MetricScatter.js +1 -5
- package/dist/report/react/fixtures.js +9 -4
- package/dist/report/react/format.d.ts +1 -1
- package/dist/report/react/format.js +1 -1
- package/dist/report/react/index.d.ts +1 -1
- package/dist/report/text/faces.js +71 -33
- package/dist/report/types.d.ts +3 -15
- package/docs-site/zh/concepts/adapter.mdx +2 -2
- package/docs-site/zh/concepts/assert.mdx +11 -10
- package/docs-site/zh/concepts/evals.mdx +7 -6
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/guides/agent-feedback-loop.mdx +9 -7
- package/docs-site/zh/guides/fixtures.mdx +3 -1
- package/docs-site/zh/guides/official-adapters.mdx +1 -1
- package/docs-site/zh/guides/publish-report.mdx +1 -1
- package/docs-site/zh/guides/report-components.mdx +13 -13
- package/docs-site/zh/guides/results-data.mdx +28 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +1 -0
- package/docs-site/zh/guides/sandbox-providers.mdx +1 -1
- package/docs-site/zh/guides/viewing-results.mdx +6 -5
- package/docs-site/zh/guides/write-send.mdx +2 -1
- package/docs-site/zh/reference/cli.mdx +3 -3
- package/docs-site/zh/reference/define-agent.mdx +3 -1
- package/docs-site/zh/reference/events.mdx +2 -2
- package/docs-site/zh/reference/expect.mdx +10 -5
- package/package.json +1 -1
- package/src/agents/types.ts +5 -1
- package/src/i18n/en.ts +1 -2
- package/src/i18n/zh-CN.ts +1 -1
- package/src/report/built-in-user-parity.test.tsx +44 -47
- package/src/report/built-ins/experiment-comparison.tsx +19 -0
- package/src/report/built-ins/index.ts +1 -1
- package/src/report/components.tsx +3 -1
- package/src/report/compute.ts +12 -36
- package/src/report/dual-render.test.tsx +33 -29
- package/src/report/format.ts +1 -16
- package/src/report/index.ts +1 -2
- package/src/report/locale.ts +26 -3
- package/src/report/metrics.ts +1 -1
- package/src/report/react/AttemptList.tsx +4 -6
- package/src/report/react/ExperimentList.tsx +167 -55
- package/src/report/react/MetricScatter.tsx +1 -11
- package/src/report/react/enhance.js +39 -0
- package/src/report/react/fixtures.ts +10 -4
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +0 -1
- package/src/report/react/render.test.tsx +24 -8
- package/src/report/react/styles.css +40 -13
- package/src/report/report.test.ts +2 -7
- package/src/report/text/faces.ts +76 -36
- package/src/report/types.ts +3 -15
- package/src/results/host-equivalence.test.ts +3 -3
- package/src/show/index.ts +8 -15
- package/src/show/render.ts +7 -204
- package/src/show/show.test.ts +22 -23
- package/src/view/data.test.ts +1 -1
- package/src/view/data.ts +6 -6
- package/src/view/index.ts +1 -1
- package/src/view/shared/types.ts +1 -1
- package/src/view/view-report.test.ts +5 -5
- package/dist/o11y/execution-tree.d.ts +0 -103
- package/dist/o11y/otlp/select.d.ts +0 -22
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +0 -1
- package/dist/report/built-ins/cost-pass-rate-comparison.js +0 -17
- package/dist/results/annotated-source.d.ts +0 -61
- package/dist/results/attempt-evidence.d.ts +0 -69
- package/dist/results/attempt-source.d.ts +0 -15
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +0 -23
|
@@ -41,7 +41,7 @@ import {
|
|
|
41
41
|
renderReportToText,
|
|
42
42
|
stringWidth,
|
|
43
43
|
} from "./index.ts";
|
|
44
|
-
import {
|
|
44
|
+
import { ExperimentComparison } from "./built-ins/index.ts";
|
|
45
45
|
import { renderReportToStaticHtml } from "./web.ts";
|
|
46
46
|
import { createTextContext, renderNodeToText, validateReportTree } from "./tree.ts";
|
|
47
47
|
import {
|
|
@@ -462,7 +462,7 @@ describe("AttemptList 双面", () => {
|
|
|
462
462
|
const html = renderToStaticMarkup(<AttemptList items={attemptListItems} />);
|
|
463
463
|
const term = text(<AttemptList items={attemptListItems} />);
|
|
464
464
|
|
|
465
|
-
it("两面同口径:判定符 + locator +
|
|
465
|
+
it("两面同口径:判定符 + locator + 断言/error 明细 + 下钻命令一致", () => {
|
|
466
466
|
for (const piece of ["roots-correct", "expected x=2, got x=3", "TypeError", "@1a4a4a4", "@1c1c1c1"]) {
|
|
467
467
|
expect(html).toContain(piece);
|
|
468
468
|
expect(term).toContain(piece);
|
|
@@ -470,9 +470,8 @@ describe("AttemptList 双面", () => {
|
|
|
470
470
|
// web 面走证据室路由(#/attempt/@<locator>);text 面只列 locator 本身,不重复整条命令
|
|
471
471
|
// (docs/feature/reports/architecture.md「text 输出只在整份报告末尾给一次命令模板」)。
|
|
472
472
|
expect(html).toContain('href="#/attempt/@1a4a4a4"');
|
|
473
|
-
|
|
474
|
-
expect(
|
|
475
|
-
expect(html).toContain("[E,X,⏱]");
|
|
473
|
+
expect(term).not.toMatch(/\[[EXD⏱,]+\]/);
|
|
474
|
+
expect(html).not.toMatch(/\[[EXD⏱,]+\]/);
|
|
476
475
|
});
|
|
477
476
|
|
|
478
477
|
it("total > items.length 时两面都如实报剩余数量", () => {
|
|
@@ -541,10 +540,13 @@ describe("ExperimentList 双面", () => {
|
|
|
541
540
|
expect(term).toContain("50%");
|
|
542
541
|
});
|
|
543
542
|
|
|
544
|
-
it("text
|
|
543
|
+
it("text 面先给比较表,再按 experiment 给 Eval / Attempt 父子表", () => {
|
|
545
544
|
const blocks = term.split("\n\n");
|
|
546
|
-
expect(blocks).toHaveLength(experimentListItems.length);
|
|
547
|
-
expect(
|
|
545
|
+
expect(blocks).toHaveLength(experimentListItems.length + 1);
|
|
546
|
+
expect(blocks[0]).toContain("Model");
|
|
547
|
+
expect(blocks[0]).toContain("Avg");
|
|
548
|
+
expect(blocks[1]!.match(/algebra\/quadratic/g)).toHaveLength(1);
|
|
549
|
+
expect(blocks[1]).toMatch(/✗ failed\s+algebra\/quadratic[\s\S]*├─ @1a4a4a4[\s\S]*└─ @1b5b5b5/);
|
|
548
550
|
});
|
|
549
551
|
});
|
|
550
552
|
|
|
@@ -987,11 +989,11 @@ describe("defineReport + 渲染入口", () => {
|
|
|
987
989
|
const out = await renderReportToText(report, fakeContext(), { width: 100 });
|
|
988
990
|
// RunOverview(data 形态,预计算)
|
|
989
991
|
expect(out).toContain("1 experiment · 2 evals · 2 attempts");
|
|
990
|
-
// ExperimentList:主行 + eval 级折叠计票 +
|
|
991
|
-
expect(out).
|
|
992
|
+
// ExperimentList:主行 + eval 级折叠计票 + 逐 Attempt locator 与失败诊断
|
|
993
|
+
expect(out).toMatch(/compare\/bub\s+default\s+bub/);
|
|
992
994
|
expect(out).toContain("1 passed / 1 failed");
|
|
993
995
|
expect(out).toContain("50%");
|
|
994
|
-
expect(out).toMatch(/✗ algebra\/y\s
|
|
996
|
+
expect(out).toMatch(/✗ failed\s+algebra\/y[\s\S]*└─ @[0-9a-z]+/);
|
|
995
997
|
// 自己的口径:成绩单
|
|
996
998
|
expect(out).toContain("考试成绩单");
|
|
997
999
|
expect(out).toContain("bub");
|
|
@@ -1051,7 +1053,7 @@ describe("defineReport + 渲染入口", () => {
|
|
|
1051
1053
|
// ts-expect-error 注释对应一处编译错误。直接标注 Props 类型(而非 JSX)——JSX 的 union
|
|
1052
1054
|
// 属性检查对「缺必填字段」较宽松,直接的类型赋值才严格钉住互斥不变量:同时传 data 与
|
|
1053
1055
|
// selection、或两者都不传、或 selection 形态缺必填计算选项,全部编译失败。正向 JSX 用法由
|
|
1054
|
-
//
|
|
1056
|
+
// ExperimentComparison / defineReport 报告在同一文件里编译验证。
|
|
1055
1057
|
function metricScatterPropsTypeChecks(selection: Selection, data: ScatterData): void {
|
|
1056
1058
|
const ok1: MetricScatterProps = { data }; // 合法:data 形态
|
|
1057
1059
|
const ok2: MetricScatterProps = { selection, points: "experiment", series: "agent", x: costUSD, y: passRate }; // 合法:selection 形态
|
|
@@ -1070,56 +1072,58 @@ function metricScatterPropsTypeChecks(selection: Selection, data: ScatterData):
|
|
|
1070
1072
|
|
|
1071
1073
|
void metricScatterPropsTypeChecks;
|
|
1072
1074
|
|
|
1073
|
-
// ─────────────────────────
|
|
1075
|
+
// ───────────────────────── ExperimentComparison(内置默认报告)─────────────────────────
|
|
1074
1076
|
|
|
1075
|
-
describe("
|
|
1077
|
+
describe("ExperimentComparison", () => {
|
|
1076
1078
|
it("是普通 ReportDefinition;text 面 = 成本×通过率散点 + 实验列表,别无它物", async () => {
|
|
1077
|
-
expect(isReportDefinition(
|
|
1078
|
-
const out = await renderReportToText(
|
|
1079
|
+
expect(isReportDefinition(ExperimentComparison)).toBe(true);
|
|
1080
|
+
const out = await renderReportToText(ExperimentComparison, fakeContext(), { width: 100 });
|
|
1079
1081
|
// 散点:fakeContext 无成本数据 → 0 可画点,显式说明缺哪两个指标(而不是画一张空图)
|
|
1080
1082
|
expect(out).toContain("No data to plot");
|
|
1081
1083
|
expect(out).not.toContain("better → upper right");
|
|
1082
1084
|
// 实验列表主行 + eval 级折叠计票 + 失败诊断(ExperimentList.data 在 build() 里直接 await)
|
|
1083
|
-
expect(out).
|
|
1085
|
+
expect(out).toMatch(/compare\/bub\s+default\s+bub/);
|
|
1084
1086
|
expect(out).toContain("1 passed / 1 failed");
|
|
1085
1087
|
expect(out).toContain("50%");
|
|
1086
|
-
expect(out).toMatch(/✗ algebra\/y\s
|
|
1088
|
+
expect(out).toMatch(/✗ failed\s+algebra\/y[\s\S]*└─ @[0-9a-z]+/);
|
|
1087
1089
|
// 只有两个直接业务组件:没有 RunOverview / GroupSummary / Section 分组
|
|
1088
1090
|
expect(out).not.toContain("Current verdicts");
|
|
1089
1091
|
});
|
|
1090
1092
|
|
|
1091
1093
|
it("web 面:散点空态 + 实验列表 <details> 展开区,无 <script>,无 Section 分组", async () => {
|
|
1092
|
-
const html = await renderReportToStaticHtml(
|
|
1094
|
+
const html = await renderReportToStaticHtml(ExperimentComparison, fakeContext());
|
|
1093
1095
|
expect(html).toContain("nre-metric-scatter");
|
|
1094
1096
|
expect(html).toContain("nre-scatter-empty"); // 0 可画点的空态
|
|
1095
|
-
expect(html).toContain('<
|
|
1097
|
+
expect(html).toContain('<details class="nre-experiment-entry">');
|
|
1098
|
+
expect(html).toContain("nre-experiment-head");
|
|
1099
|
+
expect(html).toContain('data-nre-experiment-filter=""');
|
|
1096
1100
|
expect(html).toContain("nre-experiment-evals");
|
|
1097
1101
|
expect(html).not.toContain("nre-section");
|
|
1098
1102
|
expect(html).not.toContain("<script");
|
|
1099
1103
|
});
|
|
1100
1104
|
|
|
1101
1105
|
it("locale 变体:en / zh-CN 都渲染(chrome 分语言),散点空态两面同一事实", async () => {
|
|
1102
|
-
const zhHtml = await renderReportToStaticHtml(
|
|
1106
|
+
const zhHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
|
|
1103
1107
|
expect(zhHtml).toContain("成功率"); // passRate 的 zh-CN label(ExperimentList 主行)
|
|
1104
|
-
const enHtml = await renderReportToStaticHtml(
|
|
1108
|
+
const enHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "en" });
|
|
1105
1109
|
expect(enHtml).toContain("Pass rate");
|
|
1106
|
-
const zhText = await renderReportToText(
|
|
1110
|
+
const zhText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
|
|
1107
1111
|
expect(zhText).toContain("没有可绘制的数据"); // 散点空态 zh
|
|
1108
|
-
const enText = await renderReportToText(
|
|
1112
|
+
const enText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "en" });
|
|
1109
1113
|
expect(enText).toContain("No data to plot");
|
|
1110
1114
|
});
|
|
1111
1115
|
|
|
1112
1116
|
it("多实验 fixture:每个 experiment 一项,散点如实处理全部无成本", async () => {
|
|
1113
|
-
const out = await renderReportToText(
|
|
1117
|
+
const out = await renderReportToText(ExperimentComparison, fakeMultiGroupContext(), { width: 100 });
|
|
1114
1118
|
// 三个 experiment 的身份行都出现(experimentId · agent,不再截短成最后一段)
|
|
1115
|
-
expect(out).
|
|
1116
|
-
expect(out).
|
|
1117
|
-
expect(out).
|
|
1119
|
+
expect(out).toMatch(/compare\/bub\s+default\s+bub/);
|
|
1120
|
+
expect(out).toMatch(/other\/codex\s+default\s+codex/);
|
|
1121
|
+
expect(out).toMatch(/solo\s+default\s+bub/);
|
|
1118
1122
|
// 散点空态(三点都无成本)
|
|
1119
1123
|
expect(out).toContain("No data to plot");
|
|
1120
1124
|
// 没有组分 Section 标题(不再按目录前缀分组)
|
|
1121
1125
|
expect(out).not.toMatch(/^compare$/m);
|
|
1122
|
-
const html = await renderReportToStaticHtml(
|
|
1126
|
+
const html = await renderReportToStaticHtml(ExperimentComparison, fakeMultiGroupContext());
|
|
1123
1127
|
expect(html).not.toContain("nre-section");
|
|
1124
1128
|
});
|
|
1125
1129
|
});
|
package/src/report/format.ts
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
// "%" → 87% "ms" → 1.2s "$" → $0.31 其余 → 1.2k 缩写(带 unit 后缀)
|
|
3
3
|
// metric.display 可整体覆盖;这里只负责默认。
|
|
4
4
|
|
|
5
|
-
import type { AttemptEvidenceCapabilities } from "../results/attempt-evidence.ts";
|
|
6
5
|
import type { AssertionResult, Verdict } from "../types.ts";
|
|
7
6
|
|
|
8
7
|
/** 一位小数、去掉无意义的 ".0" 尾巴。 */
|
|
@@ -78,8 +77,7 @@ export function formatPercent(ratio: number): string {
|
|
|
78
77
|
return `${Math.round(ratio * 100)}%`;
|
|
79
78
|
}
|
|
80
79
|
|
|
81
|
-
// ── 实体列表(ExperimentList / EvalList / AttemptList)
|
|
82
|
-
// web / text 两面对同一个 AttemptListItem 给出同一个标记(docs-site「终端输出形成反馈闭环」)。
|
|
80
|
+
// ── 实体列表(ExperimentList / EvalList / AttemptList)共用的判定符 ──
|
|
83
81
|
|
|
84
82
|
/** passed / failed / errored / skipped 的判定符。 */
|
|
85
83
|
export function verdictMark(verdict: Verdict): string {
|
|
@@ -95,19 +93,6 @@ export function verdictMark(verdict: Verdict): string {
|
|
|
95
93
|
}
|
|
96
94
|
}
|
|
97
95
|
|
|
98
|
-
/**
|
|
99
|
-
* 证据能力标记:`[E,X,⏱]` 式方括号列表,固定顺序 eval → execution → timing → diff,
|
|
100
|
-
* 缺哪个不列哪个;四位全无时返回空串(没有证据可看,不画空 `[]`)。
|
|
101
|
-
*/
|
|
102
|
-
export function capabilityBadge(capabilities: AttemptEvidenceCapabilities): string {
|
|
103
|
-
const marks: string[] = [];
|
|
104
|
-
if (capabilities.eval) marks.push("E");
|
|
105
|
-
if (capabilities.execution) marks.push("X");
|
|
106
|
-
if (capabilities.timing) marks.push("⏱");
|
|
107
|
-
if (capabilities.diff) marks.push("D");
|
|
108
|
-
return marks.length > 0 ? `[${marks.join(",")}]` : "";
|
|
109
|
-
}
|
|
110
|
-
|
|
111
96
|
/**
|
|
112
97
|
* 一个 AttemptListItem 的失败原因摘要,按优先级取第一个在场的:`error` → 未通过的 gate 断言
|
|
113
98
|
* (原始声明顺序,`name`,detail 在场则 `"name: detail"`,多条用「, 」连接)→ 缺席。与
|
package/src/report/index.ts
CHANGED
|
@@ -45,7 +45,7 @@ export {
|
|
|
45
45
|
export type { ColumnAlign } from "./text/layout.ts";
|
|
46
46
|
|
|
47
47
|
// 内置报告(show / view 裸跑时报告槽的出厂填充,一份普通 ReportDefinition,无 renderer 特权)
|
|
48
|
-
export {
|
|
48
|
+
export { ExperimentComparison } from "./built-ins/index.ts";
|
|
49
49
|
|
|
50
50
|
// locale:官方组件 chrome 文案的语言(en / zh-CN);指标 label 可按 locale 给字典
|
|
51
51
|
export { DEFAULT_REPORT_LOCALE, resolveMetricLabel } from "./locale.ts";
|
|
@@ -95,7 +95,6 @@ export type {
|
|
|
95
95
|
// 数据契约(组件的 data props)
|
|
96
96
|
export type {
|
|
97
97
|
Aggregator,
|
|
98
|
-
AttemptEvidenceCapabilities,
|
|
99
98
|
AttemptListItem,
|
|
100
99
|
AttemptLocator,
|
|
101
100
|
DeltaData,
|
package/src/report/locale.ts
CHANGED
|
@@ -32,6 +32,19 @@ const en = {
|
|
|
32
32
|
"table.viewBreakdown": "Per-eval breakdown",
|
|
33
33
|
/** <Table> 的 locator 列表头(行带 locator 时自动追加)。 */
|
|
34
34
|
"table.attempt": "attempt",
|
|
35
|
+
"experimentList.experiment": "Experiment",
|
|
36
|
+
"experimentList.avgDuration": "Avg duration",
|
|
37
|
+
"experimentList.passRate": "Pass rate",
|
|
38
|
+
"experimentList.tokens": "Tokens",
|
|
39
|
+
"experimentList.estimatedCost": "Est. cost",
|
|
40
|
+
"experimentList.result": "Result",
|
|
41
|
+
"experimentList.status": "Status",
|
|
42
|
+
"experimentList.evalAttempt": "Eval / Attempt",
|
|
43
|
+
"experimentList.duration": "Duration",
|
|
44
|
+
"experimentList.cost": "Cost",
|
|
45
|
+
"experimentList.filterPlaceholder": "Filter experiments…",
|
|
46
|
+
"experimentList.defaultModel": "default",
|
|
47
|
+
"experimentList.flags": "Flags",
|
|
35
48
|
/** <Table> 压到下限仍放不下时,从右侧丢列并如实报数。 */
|
|
36
49
|
"table.columnsHidden.one": "({n} more column not shown)",
|
|
37
50
|
"table.columnsHidden.other": "({n} more columns not shown)",
|
|
@@ -60,8 +73,6 @@ const en = {
|
|
|
60
73
|
"scatter.axisReversed": "(axis reversed: right = better)",
|
|
61
74
|
/** 0 个可画点:x/y 指标没有可用数据。 */
|
|
62
75
|
"scatter.noData": "No data to plot {x} × {y}",
|
|
63
|
-
/** 恰好 1 个可画点:比较至少要两个实验。 */
|
|
64
|
-
"scatter.needTwo": "At least 2 experiments needed to compare {x} × {y}",
|
|
65
76
|
"pointsMissing.one": "{n} point missing data",
|
|
66
77
|
"pointsMissing.other": "{n} points missing data",
|
|
67
78
|
|
|
@@ -110,6 +121,19 @@ const zhCN: Record<ReportMessageKey, string> = {
|
|
|
110
121
|
"table.reason": "原因",
|
|
111
122
|
"table.viewBreakdown": "逐题明细",
|
|
112
123
|
"table.attempt": "Attempt",
|
|
124
|
+
"experimentList.experiment": "实验",
|
|
125
|
+
"experimentList.avgDuration": "平均耗时",
|
|
126
|
+
"experimentList.passRate": "成功率",
|
|
127
|
+
"experimentList.tokens": "Tokens",
|
|
128
|
+
"experimentList.estimatedCost": "预估成本",
|
|
129
|
+
"experimentList.result": "结果",
|
|
130
|
+
"experimentList.status": "状态",
|
|
131
|
+
"experimentList.evalAttempt": "题目 / Attempt",
|
|
132
|
+
"experimentList.duration": "耗时",
|
|
133
|
+
"experimentList.cost": "成本",
|
|
134
|
+
"experimentList.filterPlaceholder": "筛选实验…",
|
|
135
|
+
"experimentList.defaultModel": "默认",
|
|
136
|
+
"experimentList.flags": "Flags",
|
|
113
137
|
"table.columnsHidden.one": "(还有 {n} 列未列出)",
|
|
114
138
|
"table.columnsHidden.other": "(还有 {n} 列未列出)",
|
|
115
139
|
|
|
@@ -135,7 +159,6 @@ const zhCN: Record<ReportMessageKey, string> = {
|
|
|
135
159
|
"scatter.betterUpperRight": "更好 → 右上",
|
|
136
160
|
"scatter.axisReversed": "(轴反向:右 = 更好)",
|
|
137
161
|
"scatter.noData": "{x} × {y} 没有可绘制的数据",
|
|
138
|
-
"scatter.needTwo": "至少需要 2 个实验才能比较 {x} × {y}",
|
|
139
162
|
"pointsMissing.one": "{n} 个点缺数据",
|
|
140
163
|
"pointsMissing.other": "{n} 个点缺数据",
|
|
141
164
|
|
package/src/report/metrics.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
//
|
|
14
14
|
// 两档指标(docs/feature/reports/library.md「内置指标」):以上除 turns 外全部只读 attempt.result
|
|
15
15
|
// 的瘦身字段——任何 producer、任何 copySnapshots artifacts 选择都算得出,内置报告
|
|
16
|
-
//
|
|
16
|
+
// ExperimentComparison 只用这一档。turns 读 attempt.o11y()(懒加载 artifact),发布时若
|
|
17
17
|
// o11y 没随行就诚实渲染缺数据「—」,不算 0——报告作者自己摆时心里要有这根弦,内置报告不用它。
|
|
18
18
|
|
|
19
19
|
import type { EvalResult } from "../types.ts";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AttemptList:实体列表的叶子层——每项一个 Attempt,固定展示判定、断言、error、Judge 评语
|
|
2
|
-
// (assertions 的 detail/evidence)与证据引用(locator
|
|
2
|
+
// (assertions 的 detail/evidence)与证据引用(locator)。它不预设只看失败,
|
|
3
3
|
// 报告作者过滤 items、用 .slice() 限量,total 让渲染面如实报告剩余数量,不静默截断。
|
|
4
4
|
// ExperimentList / EvalList 的下钻数组是同一个 AttemptListItem[],这里的渲染逻辑因此
|
|
5
5
|
// 也是它们展开区里"逐条 attempt"那一层的唯一实现(通过 AttemptRow 导出复用,不重写一遍)。
|
|
@@ -9,17 +9,16 @@ import type { AttemptListItem } from "../types.ts";
|
|
|
9
9
|
import type { AttemptLocator } from "../../results/locator.ts";
|
|
10
10
|
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
|
|
11
11
|
import { colorClassForKey } from "./colors.ts";
|
|
12
|
-
import {
|
|
12
|
+
import { cx, formatDurationMs, formatUSD, verdictMark } from "./format.ts";
|
|
13
13
|
|
|
14
|
-
/** locator +
|
|
14
|
+
/** locator + 判定符的普通 <a>,AttemptList/EvalList/ExperimentList 共用。 */
|
|
15
15
|
export function AttemptLocatorBadge({
|
|
16
16
|
item,
|
|
17
17
|
attemptHref,
|
|
18
18
|
}: {
|
|
19
|
-
item: Pick<AttemptListItem, "locator" | "verdict"
|
|
19
|
+
item: Pick<AttemptListItem, "locator" | "verdict">;
|
|
20
20
|
attemptHref: (locator: AttemptLocator) => string;
|
|
21
21
|
}): ReactElement {
|
|
22
|
-
const caps = capabilityBadge(item.capabilities);
|
|
23
22
|
return (
|
|
24
23
|
<a
|
|
25
24
|
className={cx("nre-locator", `nre-verdict-${item.verdict}`)}
|
|
@@ -27,7 +26,6 @@ export function AttemptLocatorBadge({
|
|
|
27
26
|
>
|
|
28
27
|
{item.locator}
|
|
29
28
|
<span className="nre-locator-mark">{verdictMark(item.verdict)}</span>
|
|
30
|
-
{caps && <span className="nre-locator-caps">{caps}</span>}
|
|
31
29
|
</a>
|
|
32
30
|
);
|
|
33
31
|
}
|
|
@@ -1,47 +1,119 @@
|
|
|
1
|
-
// ExperimentList
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// 的 Eval:每道题一行,判定符 + 该题全部 Attempt 的 locator 徽标(内联,不用再点开一层)+
|
|
5
|
-
// 通过时的平均耗时/成本、失败/出错时的原因摘要。
|
|
1
|
+
// ExperimentList:实体列表的第一级。web 面是一行一个 experiment 的固定列比较表;
|
|
2
|
+
// 每行用原生 <details> 展开到 Eval 与 Attempt locator。数据仍完全来自
|
|
3
|
+
// ExperimentList.data(),不恢复旧 ExperimentTable 的混合实体计算层。
|
|
6
4
|
|
|
7
5
|
import type { ReactElement } from "react";
|
|
8
|
-
import type { ExperimentListEvalRow, ExperimentListItem } from "../types.ts";
|
|
9
6
|
import type { AttemptLocator } from "../../results/locator.ts";
|
|
7
|
+
import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "../types.ts";
|
|
8
|
+
import { attemptItemReason } from "../format.ts";
|
|
10
9
|
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
|
|
11
|
-
import { MetricCellView } from "./cell.tsx";
|
|
12
10
|
import { AttemptLocatorBadge } from "./AttemptList.tsx";
|
|
11
|
+
import { MetricCellView } from "./cell.tsx";
|
|
13
12
|
import { colorClassForKey } from "./colors.ts";
|
|
14
13
|
import { cx, formatDurationMs, formatUSD, verdictMark } from "./format.ts";
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
const verdictOrder = ["passed", "failed", "errored", "skipped"] as const;
|
|
16
|
+
|
|
17
|
+
function passRateTone(value: number | null): string | undefined {
|
|
18
|
+
if (value === null) return undefined;
|
|
19
|
+
if (value >= 0.8) return "nre-good";
|
|
20
|
+
if (value >= 0.5) return "nre-warn";
|
|
21
|
+
return "nre-bad";
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function formatDate(value: string, locale: ReportLocale): string {
|
|
25
|
+
const date = new Date(value);
|
|
26
|
+
if (Number.isNaN(date.valueOf())) return value;
|
|
27
|
+
return new Intl.DateTimeFormat(locale === "zh-CN" ? "zh-CN" : "en", {
|
|
28
|
+
month: "short",
|
|
29
|
+
day: "numeric",
|
|
30
|
+
hour: "2-digit",
|
|
31
|
+
minute: "2-digit",
|
|
32
|
+
}).format(date);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function VerdictSummary({ item, locale }: { item: ExperimentListItem; locale: ReportLocale }): ReactElement {
|
|
36
|
+
const parts = verdictOrder
|
|
37
|
+
.filter((verdict) => item.verdicts[verdict] > 0)
|
|
38
|
+
.map((verdict) => `${item.verdicts[verdict]} ${localeText(locale, `verdict.${verdict}`)}`);
|
|
39
|
+
return <span className="nre-experiment-pill">{parts.join(" / ") || "—"}</span>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function ExperimentAttemptRow({
|
|
43
|
+
attempt,
|
|
44
|
+
last,
|
|
18
45
|
attemptHref,
|
|
19
46
|
}: {
|
|
20
|
-
|
|
47
|
+
attempt: AttemptListItem;
|
|
48
|
+
last: boolean;
|
|
21
49
|
attemptHref: (locator: AttemptLocator) => string;
|
|
22
50
|
}): ReactElement {
|
|
51
|
+
const reason = attemptItemReason(attempt);
|
|
23
52
|
return (
|
|
24
|
-
<li className={cx("nre-experiment-
|
|
25
|
-
<span className=
|
|
26
|
-
<span className="nre-eval-id">{row.evalId}</span>
|
|
53
|
+
<li className={cx("nre-experiment-attempt-row", `nre-eval-${attempt.verdict}`)}>
|
|
54
|
+
<span className="nre-attempt-branch" aria-hidden="true">{last ? "└─" : "├─"}</span>
|
|
27
55
|
<span className="nre-eval-attempt-badges">
|
|
28
|
-
{
|
|
29
|
-
<AttemptLocatorBadge key={attempt.locator} item={attempt} attemptHref={attemptHref} />
|
|
30
|
-
))}
|
|
56
|
+
<AttemptLocatorBadge item={attempt} attemptHref={attemptHref} />
|
|
31
57
|
</span>
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
58
|
+
<span className="nre-eval-attempt-metrics">
|
|
59
|
+
{formatDurationMs(attempt.durationMs)}
|
|
60
|
+
{attempt.costUSD !== undefined && <> · {formatUSD(attempt.costUSD)}</>}
|
|
61
|
+
</span>
|
|
62
|
+
<span className="nre-eval-reason">{reason ?? "—"}</span>
|
|
63
|
+
</li>
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function EvalAttempts({
|
|
68
|
+
row,
|
|
69
|
+
attemptHref,
|
|
70
|
+
locale,
|
|
71
|
+
}: {
|
|
72
|
+
row: ExperimentListEvalRow;
|
|
73
|
+
attemptHref: (locator: AttemptLocator) => string;
|
|
74
|
+
locale: ReportLocale;
|
|
75
|
+
}): ReactElement {
|
|
76
|
+
return (
|
|
77
|
+
<li className="nre-experiment-eval">
|
|
78
|
+
<div className={cx("nre-experiment-eval-header", `nre-eval-${row.verdict}`)}>
|
|
79
|
+
<span className={cx("nre-eval-verdict", `nre-verdict-${row.verdict}`)}>{verdictMark(row.verdict)}</span>
|
|
80
|
+
<span className="nre-eval-id">{row.evalId}</span>
|
|
81
|
+
<span className="nre-eval-attempt-count">{localeText(locale, "overview.attemptsCount", { n: row.attempts.length })}</span>
|
|
82
|
+
<span className="nre-eval-summary">
|
|
83
|
+
{row.verdict === "passed"
|
|
84
|
+
? [formatDurationMs(row.duration.value ?? 0), row.cost.value === null ? undefined : formatUSD(row.cost.value)]
|
|
85
|
+
.filter((value): value is string => value !== undefined)
|
|
86
|
+
.join(" · ")
|
|
87
|
+
: row.reason}
|
|
36
88
|
</span>
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
89
|
+
</div>
|
|
90
|
+
<ul className="nre-experiment-attempts">
|
|
91
|
+
{row.attempts.map((attempt, index) => (
|
|
92
|
+
<ExperimentAttemptRow
|
|
93
|
+
key={attempt.locator}
|
|
94
|
+
attempt={attempt}
|
|
95
|
+
last={index === row.attempts.length - 1}
|
|
96
|
+
attemptHref={attemptHref}
|
|
97
|
+
/>
|
|
98
|
+
))}
|
|
99
|
+
</ul>
|
|
40
100
|
</li>
|
|
41
101
|
);
|
|
42
102
|
}
|
|
43
103
|
|
|
44
|
-
function
|
|
104
|
+
function Flags({ flags, locale }: { flags: Record<string, unknown> | undefined; locale: ReportLocale }): ReactElement | null {
|
|
105
|
+
if (!flags || Object.keys(flags).length === 0) return null;
|
|
106
|
+
return (
|
|
107
|
+
<div className="nre-experiment-flags">
|
|
108
|
+
<span>{localeText(locale, "experimentList.flags")}</span>
|
|
109
|
+
{Object.entries(flags).map(([key, value]) => (
|
|
110
|
+
<b key={key}>{key}={String(value)}</b>
|
|
111
|
+
))}
|
|
112
|
+
</div>
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function ExperimentRow({
|
|
45
117
|
item,
|
|
46
118
|
attemptHref,
|
|
47
119
|
locale,
|
|
@@ -51,58 +123,98 @@ function ExperimentCard({
|
|
|
51
123
|
locale: ReportLocale;
|
|
52
124
|
}): ReactElement {
|
|
53
125
|
return (
|
|
54
|
-
<
|
|
55
|
-
<
|
|
56
|
-
<
|
|
57
|
-
<
|
|
58
|
-
|
|
59
|
-
</span>
|
|
60
|
-
<span className="nre-experiment-agent">{item.agent}</span>
|
|
61
|
-
{item.model && <span className="nre-experiment-model">{item.model}</span>}
|
|
62
|
-
<span className="nre-experiment-cell">
|
|
63
|
-
<MetricCellView cell={item.passRate} attemptHref={attemptHref} locale={locale} />
|
|
64
|
-
</span>
|
|
65
|
-
<span className="nre-experiment-cell">
|
|
66
|
-
<MetricCellView cell={item.cost} attemptHref={attemptHref} locale={locale} />
|
|
67
|
-
</span>
|
|
68
|
-
<span className="nre-experiment-cell">
|
|
69
|
-
<MetricCellView cell={item.duration} attemptHref={attemptHref} locale={locale} />
|
|
70
|
-
</span>
|
|
71
|
-
<span className="nre-experiment-meta-sub">
|
|
126
|
+
<details className="nre-experiment-entry">
|
|
127
|
+
<summary className="nre-experiment-summary">
|
|
128
|
+
<span className="nre-experiment-name" data-sort-value={item.experimentId}>
|
|
129
|
+
<b className={cx("nre-experiment-id", "nre-key", colorClassForKey(item.experimentId))}>{item.experimentId}</b>
|
|
130
|
+
<small>
|
|
72
131
|
{localeText(locale, "overview.evalsCount", { n: item.evals })}
|
|
73
132
|
{item.attempts > item.evals ? ` · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })}` : ""}
|
|
74
|
-
{` · ${
|
|
75
|
-
</
|
|
76
|
-
</
|
|
133
|
+
{` · ${formatDate(item.lastRunAt, locale)}`}
|
|
134
|
+
</small>
|
|
135
|
+
</span>
|
|
136
|
+
<span data-sort-value={item.model ?? ""}>{item.model ?? localeText(locale, "experimentList.defaultModel")}</span>
|
|
137
|
+
<span data-sort-value={item.agent}>{item.agent}</span>
|
|
138
|
+
<span className="nre-num" data-sort-value={item.duration.value ?? ""}>
|
|
139
|
+
<MetricCellView cell={item.duration} locale={locale} />
|
|
140
|
+
</span>
|
|
141
|
+
<span className={cx("nre-num", passRateTone(item.passRate.value))} data-sort-value={item.passRate.value ?? ""}>
|
|
142
|
+
<MetricCellView cell={item.passRate} locale={locale} />
|
|
143
|
+
</span>
|
|
144
|
+
<span className="nre-num" data-sort-value={item.tokens.value ?? ""}>
|
|
145
|
+
<MetricCellView cell={item.tokens} locale={locale} />
|
|
146
|
+
</span>
|
|
147
|
+
<span className="nre-num" data-sort-value={item.cost.value ?? ""}>
|
|
148
|
+
<MetricCellView cell={item.cost} locale={locale} />
|
|
149
|
+
</span>
|
|
150
|
+
<span data-sort-value={item.verdicts.passed}><VerdictSummary item={item} locale={locale} /></span>
|
|
151
|
+
</summary>
|
|
152
|
+
<div className="nre-experiment-detail">
|
|
153
|
+
<Flags flags={item.flags} locale={locale} />
|
|
77
154
|
<ul className="nre-experiment-evals">
|
|
78
155
|
{item.evalRows.map((row) => (
|
|
79
|
-
<
|
|
156
|
+
<EvalAttempts key={row.evalId} row={row} attemptHref={attemptHref} locale={locale} />
|
|
80
157
|
))}
|
|
81
158
|
</ul>
|
|
82
|
-
</
|
|
83
|
-
</
|
|
159
|
+
</div>
|
|
160
|
+
</details>
|
|
84
161
|
);
|
|
85
162
|
}
|
|
86
163
|
|
|
87
164
|
export function ExperimentList({
|
|
88
165
|
items,
|
|
89
166
|
attemptHref,
|
|
167
|
+
filter = false,
|
|
90
168
|
className,
|
|
91
169
|
locale = DEFAULT_REPORT_LOCALE,
|
|
92
170
|
}: {
|
|
93
171
|
items: ExperimentListItem[];
|
|
94
172
|
attemptHref: (locator: AttemptLocator) => string;
|
|
173
|
+
filter?: boolean;
|
|
95
174
|
className?: string;
|
|
96
175
|
locale?: ReportLocale;
|
|
97
176
|
}): ReactElement {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
177
|
+
const labels = [
|
|
178
|
+
localeText(locale, "experimentList.experiment"),
|
|
179
|
+
localeText(locale, "table.model"),
|
|
180
|
+
localeText(locale, "table.agent"),
|
|
181
|
+
localeText(locale, "experimentList.avgDuration"),
|
|
182
|
+
localeText(locale, "experimentList.passRate"),
|
|
183
|
+
localeText(locale, "experimentList.tokens"),
|
|
184
|
+
localeText(locale, "experimentList.estimatedCost"),
|
|
185
|
+
localeText(locale, "experimentList.result"),
|
|
186
|
+
];
|
|
187
|
+
const board = (
|
|
188
|
+
<div className="nre-experiment-table">
|
|
189
|
+
<div className="nre-experiment-head">
|
|
190
|
+
{labels.map((label, index) => (
|
|
191
|
+
<button
|
|
192
|
+
type="button"
|
|
193
|
+
data-nre-experiment-sort={index}
|
|
194
|
+
className={index === 4 ? "nre-sort-desc" : undefined}
|
|
195
|
+
key={label}
|
|
196
|
+
>
|
|
197
|
+
{label}
|
|
198
|
+
</button>
|
|
104
199
|
))}
|
|
105
|
-
</
|
|
200
|
+
</div>
|
|
201
|
+
{items.length === 0 && <p className="nre-experiment-list-empty">{localeText(locale, "attemptList.empty")}</p>}
|
|
202
|
+
{items.map((item) => (
|
|
203
|
+
<ExperimentRow key={item.experimentId} item={item} attemptHref={attemptHref} locale={locale} />
|
|
204
|
+
))}
|
|
205
|
+
</div>
|
|
206
|
+
);
|
|
207
|
+
return (
|
|
208
|
+
<div className={cx("nre", "nre-experiment-list", filter && "nre-experiment-table-wrap", className)}>
|
|
209
|
+
{filter && (
|
|
210
|
+
<input
|
|
211
|
+
className="nre-filter"
|
|
212
|
+
data-nre-experiment-filter=""
|
|
213
|
+
type="search"
|
|
214
|
+
placeholder={localeText(locale, "experimentList.filterPlaceholder")}
|
|
215
|
+
/>
|
|
216
|
+
)}
|
|
217
|
+
{board}
|
|
106
218
|
</div>
|
|
107
219
|
);
|
|
108
220
|
}
|
|
@@ -88,16 +88,6 @@ export function MetricScatter({
|
|
|
88
88
|
);
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
-
// 恰好 1 个可画点:成本 × 通过率的对比至少要两个实验,单点不成图 —— 显式说清而不是画个孤点。
|
|
92
|
-
if (drawableRows.length === 1) {
|
|
93
|
-
return (
|
|
94
|
-
<figure className={cx("nre", "nre-metric-scatter", className)}>
|
|
95
|
-
<p className="nre-scatter-empty">{localeText(locale, "scatter.needTwo", { x: xLabel, y: yLabel })}</p>
|
|
96
|
-
{missingNote}
|
|
97
|
-
</figure>
|
|
98
|
-
);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
91
|
const axisLabel = (label: string, col: MetricColumn) => `${label}${col.unit ? `(${col.unit})` : ""}`;
|
|
102
92
|
|
|
103
93
|
const xScale = axisScale(drawableRows.map((r) => r.x.value as number), data.x.better, MARGIN.left, MARGIN.left + PLOT_W);
|
|
@@ -113,7 +103,7 @@ export function MetricScatter({
|
|
|
113
103
|
label: pointLabel(r.key),
|
|
114
104
|
xValue,
|
|
115
105
|
yValue,
|
|
116
|
-
// hover 内容:experiment(点键)+ 系列(series,
|
|
106
|
+
// hover 内容:experiment(点键)+ 系列(series,ExperimentComparison 传的是 agent 维度,
|
|
117
107
|
// 有则加一行;无系列的散点没有这行)+ 两轴 display 与 samples/total(docs/feature/reports/library.md「MetricScatter」行为清单)
|
|
118
108
|
title: `${r.key}${r.series !== undefined ? `\n${r.series}` : ""}\n${xLabel}: ${r.x.display}(${r.x.samples}/${r.x.total})\n${yLabel}: ${r.y.display}(${r.y.samples}/${r.y.total})`,
|
|
119
109
|
px: xScale.scale(xValue),
|