niceeval 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +317 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +178 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +618 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +32 -0
- package/dist/report/format.js +126 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +83 -0
- package/dist/report/locale.js +160 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +20 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +10 -0
- package/dist/report/react/ExperimentList.js +15 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +100 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +393 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +503 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +420 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/annotated-source.d.ts +61 -0
- package/dist/results/attempt-evidence.d.ts +69 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +5 -5
- package/docs-site/zh/concepts/assert.mdx +8 -8
- package/docs-site/zh/concepts/evals.mdx +5 -5
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +2 -2
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +1 -1
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +35 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +11 -11
- package/docs-site/zh/guides/viewing-results.mdx +199 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +2 -2
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +12 -12
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +7 -1
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +54 -2
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +34 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +643 -0
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +223 -50
- package/src/report/compute.ts +358 -53
- package/src/report/dual-render.test.tsx +1125 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +50 -2
- package/src/report/index.ts +48 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +206 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +115 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +108 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +136 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +129 -0
- package/src/report/react/fixtures.ts +163 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +22 -6
- package/src/report/react/render.test.tsx +164 -31
- package/src/report/react/styles.css +405 -213
- package/src/report/report.test.ts +403 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +303 -95
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +196 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +125 -94
- package/src/show/render.ts +660 -160
- package/src/show/show.test.ts +472 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/report/flag.ts
CHANGED
package/src/report/format.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
// unit 驱动的内置格式化(docs/reports.md
|
|
1
|
+
// unit 驱动的内置格式化(docs/feature/reports/library.md「指标」):
|
|
2
2
|
// "%" → 87% "ms" → 1.2s "$" → $0.31 其余 → 1.2k 缩写(带 unit 后缀)
|
|
3
3
|
// metric.display 可整体覆盖;这里只负责默认。
|
|
4
4
|
|
|
5
|
+
import type { AttemptEvidenceCapabilities } from "../results/attempt-evidence.ts";
|
|
6
|
+
import type { AssertionResult, Verdict } from "../types.ts";
|
|
7
|
+
|
|
5
8
|
/** 一位小数、去掉无意义的 ".0" 尾巴。 */
|
|
6
9
|
function trimmed(n: number): string {
|
|
7
10
|
const s = n.toFixed(1);
|
|
@@ -49,7 +52,7 @@ export function formatPlainNumber(value: number): string {
|
|
|
49
52
|
// ── 以下是两个渲染面共用的展示格式化:MetricCell 一律自带 display(格式化发生在
|
|
50
53
|
// 计算侧),渲染面不重算;这里只服务 OverviewData 这类携带裸数字的字段。──
|
|
51
54
|
|
|
52
|
-
/** 全 null / 无样本的统一文案。绝不画 0(docs/reports.md
|
|
55
|
+
/** 全 null / 无样本的统一文案。绝不画 0(docs/feature/reports/architecture.md「指标聚合不变量」)。 */
|
|
53
56
|
export const MISSING_TEXT = "no data";
|
|
54
57
|
|
|
55
58
|
/** 毫秒 → 人读耗时("850ms" / "1.2s" / "4m 20s" / "1h 4m")。 */
|
|
@@ -74,3 +77,48 @@ export function formatUSD(usd: number): string {
|
|
|
74
77
|
export function formatPercent(ratio: number): string {
|
|
75
78
|
return `${Math.round(ratio * 100)}%`;
|
|
76
79
|
}
|
|
80
|
+
|
|
81
|
+
// ── 实体列表(ExperimentList / EvalList / AttemptList)共用的判定符与证据能力位 ──
|
|
82
|
+
// web / text 两面对同一个 AttemptListItem 给出同一个标记(docs-site「终端输出形成反馈闭环」)。
|
|
83
|
+
|
|
84
|
+
/** passed / failed / errored / skipped 的判定符。 */
|
|
85
|
+
export function verdictMark(verdict: Verdict): string {
|
|
86
|
+
switch (verdict) {
|
|
87
|
+
case "passed":
|
|
88
|
+
return "✓";
|
|
89
|
+
case "failed":
|
|
90
|
+
return "✗";
|
|
91
|
+
case "errored":
|
|
92
|
+
return "!";
|
|
93
|
+
case "skipped":
|
|
94
|
+
return "–";
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* 证据能力标记:`[E,X,⏱]` 式方括号列表,固定顺序 eval → execution → timing → diff,
|
|
100
|
+
* 缺哪个不列哪个;四位全无时返回空串(没有证据可看,不画空 `[]`)。
|
|
101
|
+
*/
|
|
102
|
+
export function capabilityBadge(capabilities: AttemptEvidenceCapabilities): string {
|
|
103
|
+
const marks: string[] = [];
|
|
104
|
+
if (capabilities.eval) marks.push("E");
|
|
105
|
+
if (capabilities.execution) marks.push("X");
|
|
106
|
+
if (capabilities.timing) marks.push("⏱");
|
|
107
|
+
if (capabilities.diff) marks.push("D");
|
|
108
|
+
return marks.length > 0 ? `[${marks.join(",")}]` : "";
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* 一个 AttemptListItem 的失败原因摘要,按优先级取第一个在场的:`error` → 未通过的 gate 断言
|
|
113
|
+
* (原始声明顺序,`name`,detail 在场则 `"name: detail"`,多条用「, 」连接)→ 缺席。与
|
|
114
|
+
* compute.ts 的 `reasonFor(EvalResult)` 同一口径,只是输入换成瘦身后的 AttemptListItem——
|
|
115
|
+
* 没有 `skipReason` 字段(AttemptListItem 不携带),skipped 的 attempt 因此没有原因摘要,
|
|
116
|
+
* 这与它们本来就不该出现在「为什么失败」列表里的事实一致。EvalList / ExperimentList 的
|
|
117
|
+
* 逐 attempt 徽标行用它给每个 attempt 自己的原因(而不是复用整道题的代表原因)。
|
|
118
|
+
*/
|
|
119
|
+
export function attemptItemReason(item: { error?: string; assertions: AssertionResult[] }): string | undefined {
|
|
120
|
+
if (item.error !== undefined) return item.error;
|
|
121
|
+
const gates = item.assertions.filter((a) => !a.passed && a.severity === "gate");
|
|
122
|
+
if (gates.length === 0) return undefined;
|
|
123
|
+
return gates.map((a) => (a.detail ? `${a.name}: ${a.detail}` : a.name)).join(", ");
|
|
124
|
+
}
|
package/src/report/index.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
// niceeval/report —— 报告积木:指标 × 计算函数 × 双面组件 × defineReport。
|
|
2
|
-
// 契约见 docs/reports.md;公开叙事的准绳是 docs-site/zh/guides/custom-reports.mdx
|
|
2
|
+
// 契约见 docs/feature/reports/README.md;公开叙事的准绳是 docs-site/zh/guides/custom-reports.mdx
|
|
3
3
|
// 与 report-components.mdx。
|
|
4
4
|
//
|
|
5
5
|
// import 边界即运行时边界:计算函数(挂在组件上的 .data)会经句柄触碰文件系统
|
|
6
|
-
// (
|
|
6
|
+
// (懒加载 artifact),只能进服务端 / 脚本;组件的渲染面纯同步零 IO。text 宿主遍历渲染
|
|
7
7
|
// 不需要 react-dom(renderReportToText);web 宿主的 renderReportToStaticHtml 在
|
|
8
8
|
// ./web.ts,只有那一侧 import react-dom。写报告文件的项目要装 react(.tsx 编译产物
|
|
9
9
|
// import react/jsx-runtime)。
|
|
10
10
|
|
|
11
11
|
// 指标与 flag
|
|
12
|
-
export { defineMetric, passRate, examScore, durationMs, tokens, costUSD } from "./metrics.ts";
|
|
12
|
+
export { defineMetric, passRate, examScore, durationMs, tokens, costUSD, turns } from "./metrics.ts";
|
|
13
13
|
export { flag } from "./flag.ts";
|
|
14
14
|
|
|
15
15
|
// 报告基座与双面组件基座
|
|
@@ -26,17 +26,38 @@ export type {
|
|
|
26
26
|
WebContext,
|
|
27
27
|
} from "./tree.ts";
|
|
28
28
|
|
|
29
|
-
// 排版原语(
|
|
30
|
-
export { Row, Col, Section, Text, Style } from "./primitives.tsx";
|
|
31
|
-
export type { LayoutProps, SectionProps, StyleProps } from "./primitives.tsx";
|
|
29
|
+
// 排版原语(六个内置双面组件)
|
|
30
|
+
export { Row, Col, Section, Text, Style, Table } from "./primitives.tsx";
|
|
31
|
+
export type { LayoutProps, SectionProps, StyleProps, TableColumn, TableProps, TableRow } from "./primitives.tsx";
|
|
32
32
|
|
|
33
|
-
//
|
|
34
|
-
|
|
33
|
+
// 文本排版工具箱:自定义组件的 text 面用的就是官方组件那把尺子。
|
|
34
|
+
// 表格有 <Table> 承担,这里只给表以外的形态用 —— 尤其别拿 String.prototype.padEnd 对齐:
|
|
35
|
+
// 它数 UTF-16 码元不数显示列宽,一带中文列就撕歪。renderAlignedRows 刻意不导出。
|
|
36
|
+
export {
|
|
37
|
+
stringWidth,
|
|
38
|
+
padDisplay as padEnd,
|
|
39
|
+
padStartDisplay as padStart,
|
|
40
|
+
wrapDisplay as wrapText,
|
|
41
|
+
indentBlock as indent,
|
|
42
|
+
textBar as bar,
|
|
43
|
+
joinColumns as columns,
|
|
44
|
+
} from "./text/layout.ts";
|
|
45
|
+
export type { ColumnAlign } from "./text/layout.ts";
|
|
46
|
+
|
|
47
|
+
// 内置报告(show / view 裸跑时报告槽的出厂填充,一份普通 ReportDefinition,无 renderer 特权)
|
|
48
|
+
export { CostPassRateComparison } from "./built-ins/index.ts";
|
|
49
|
+
|
|
50
|
+
// locale:官方组件 chrome 文案的语言(en / zh-CN);指标 label 可按 locale 给字典
|
|
51
|
+
export { DEFAULT_REPORT_LOCALE, resolveMetricLabel } from "./locale.ts";
|
|
52
|
+
export type { LocalizedLabel, ReportLocale } from "./locale.ts";
|
|
35
53
|
|
|
36
54
|
// 官方双面组件(各自带 data 计算函数;MetricBars.data = MetricMatrix.data)
|
|
37
55
|
export {
|
|
38
|
-
|
|
56
|
+
AttemptList,
|
|
39
57
|
DeltaTable,
|
|
58
|
+
EvalList,
|
|
59
|
+
ExperimentList,
|
|
60
|
+
GroupSummary,
|
|
40
61
|
MetricBars,
|
|
41
62
|
MetricLine,
|
|
42
63
|
MetricMatrix,
|
|
@@ -46,8 +67,11 @@ export {
|
|
|
46
67
|
Scoreboard,
|
|
47
68
|
} from "./components.tsx";
|
|
48
69
|
export type {
|
|
49
|
-
|
|
70
|
+
AttemptListProps,
|
|
50
71
|
DeltaTableProps,
|
|
72
|
+
EvalListProps,
|
|
73
|
+
ExperimentListProps,
|
|
74
|
+
GroupSummaryProps,
|
|
51
75
|
MetricLineProps,
|
|
52
76
|
MetricMatrixProps,
|
|
53
77
|
MetricScatterProps,
|
|
@@ -58,7 +82,7 @@ export type {
|
|
|
58
82
|
|
|
59
83
|
// 计算函数的选项类型(函数本体挂在组件上,不做顶层导出)
|
|
60
84
|
export type {
|
|
61
|
-
|
|
85
|
+
AttemptListDataOptions,
|
|
62
86
|
DeltaDataOptions,
|
|
63
87
|
DeltaPair,
|
|
64
88
|
LineDataOptions,
|
|
@@ -71,11 +95,16 @@ export type {
|
|
|
71
95
|
// 数据契约(组件的 data props)
|
|
72
96
|
export type {
|
|
73
97
|
Aggregator,
|
|
74
|
-
|
|
75
|
-
|
|
98
|
+
AttemptEvidenceCapabilities,
|
|
99
|
+
AttemptListItem,
|
|
100
|
+
AttemptLocator,
|
|
76
101
|
DeltaData,
|
|
77
102
|
Dimension,
|
|
78
103
|
DimensionInput,
|
|
104
|
+
EvalListItem,
|
|
105
|
+
ExperimentListEvalRow,
|
|
106
|
+
ExperimentListItem,
|
|
107
|
+
GroupSummaryData,
|
|
79
108
|
LineAxis,
|
|
80
109
|
LineData,
|
|
81
110
|
MatrixData,
|
|
@@ -89,7 +118,12 @@ export type {
|
|
|
89
118
|
ScoreboardData,
|
|
90
119
|
SelectionWarning,
|
|
91
120
|
TableData,
|
|
121
|
+
TableRowMeta,
|
|
92
122
|
} from "./types.ts";
|
|
93
123
|
|
|
94
124
|
// 数据层输入的类型(家在 niceeval/results,这里 re-export 方便写指标 / 报告)
|
|
95
|
-
export type { AttemptHandle, Results, Selection, Snapshot } from "../results/
|
|
125
|
+
export type { AttemptHandle, Results, Selection, Snapshot } from "../results/types.ts";
|
|
126
|
+
|
|
127
|
+
// experiment id 的组键推导(id 的目录前缀,如 `compare/bub-low` 的 `compare`)。
|
|
128
|
+
// 重新导出,让自定义报告能按同一份口径把 experiment 分组,不必自己重写这两行逻辑。
|
|
129
|
+
export { experimentGroupOf } from "../shared/aggregate.ts";
|
package/src/report/load.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// --report 的装载:两个宿主(show / view)共用的中性入口。复用跑用户 .ts 配置的
|
|
2
2
|
// 同一 tsx 加载机制(bin 里已 register)。show 一进程一次装载;view 的 dev server
|
|
3
3
|
// 每次请求现读现渲染 —— ESM 模块缓存永不失效,所以按文件 mtime 做 cache-busting
|
|
4
|
-
// (freshImport):报告文件变更 → 下次请求拿到新模块,整页重算(docs/reports.md
|
|
4
|
+
// (freshImport):报告文件变更 → 下次请求拿到新模块,整页重算(docs/feature/reports/view.md
|
|
5
5
|
// 裁决记录 6)。装载环境坑见 memory/tsx-dynamic-import-require-cycle.md。
|
|
6
6
|
|
|
7
7
|
import { existsSync, statSync } from "node:fs";
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
// 官方组件 chrome 文案的 locale 字典(en / zh-CN 两份)。
|
|
2
|
+
// 只覆盖组件自带的固定文案(verdict 词、缺数据、覆盖率角标、注脚、占位符等);
|
|
3
|
+
// 数据本身(display、维度键、warnings 的 message)不经这里。
|
|
4
|
+
// 刻意不 import src/i18n/(CLI 专用字典,locale 来源与 key 面完全不同);
|
|
5
|
+
// zh-CN 译法与 src/view/app/i18n.ts 现有词保持一致(成功率/平均耗时/预估成本…)。
|
|
6
|
+
|
|
7
|
+
/** 报告渲染的 locale;默认 "en"(text 面缺省即 en,`niceeval show` 输出不变)。 */
|
|
8
|
+
export type ReportLocale = "en" | "zh-CN";
|
|
9
|
+
|
|
10
|
+
export const DEFAULT_REPORT_LOCALE: ReportLocale = "en";
|
|
11
|
+
|
|
12
|
+
const en = {
|
|
13
|
+
"verdict.passed": "passed",
|
|
14
|
+
"verdict.failed": "failed",
|
|
15
|
+
"verdict.errored": "errored",
|
|
16
|
+
"verdict.skipped": "skipped",
|
|
17
|
+
|
|
18
|
+
/** 全 null / 无样本的统一文案,绝不画 0。 */
|
|
19
|
+
"cell.missing": "no data",
|
|
20
|
+
"cell.measuredTitle": "{samples}/{total} attempts measured",
|
|
21
|
+
"cell.noneMeasurableTitle": "0/{total} attempts measurable",
|
|
22
|
+
"cell.coverageTitle": "coverage {samples}/{total}: this metric is null for the remaining attempts",
|
|
23
|
+
|
|
24
|
+
"table.higherBetter": "higher is better",
|
|
25
|
+
"table.lowerBetter": "lower is better",
|
|
26
|
+
"table.model": "Model",
|
|
27
|
+
"table.agent": "Agent",
|
|
28
|
+
"table.verdicts": "Verdicts",
|
|
29
|
+
"table.filterPlaceholder": "Filter rows…",
|
|
30
|
+
"table.eval": "Eval",
|
|
31
|
+
"table.reason": "Reason",
|
|
32
|
+
"table.viewBreakdown": "Per-eval breakdown",
|
|
33
|
+
/** <Table> 的 locator 列表头(行带 locator 时自动追加)。 */
|
|
34
|
+
"table.attempt": "attempt",
|
|
35
|
+
/** <Table> 压到下限仍放不下时,从右侧丢列并如实报数。 */
|
|
36
|
+
"table.columnsHidden.one": "({n} more column not shown)",
|
|
37
|
+
"table.columnsHidden.other": "({n} more columns not shown)",
|
|
38
|
+
|
|
39
|
+
"overview.snapshots": "Snapshots",
|
|
40
|
+
"overview.evals": "Evals",
|
|
41
|
+
"overview.attempts": "attempts",
|
|
42
|
+
"overview.passRate": "Pass rate",
|
|
43
|
+
"overview.totalCost": "Total cost",
|
|
44
|
+
"overview.totalDuration": "Total duration",
|
|
45
|
+
"overview.source": "Source: {n} snapshots",
|
|
46
|
+
"overview.experiments.one": "{n} experiment",
|
|
47
|
+
"overview.experiments.other": "{n} experiments",
|
|
48
|
+
"overview.evalsCount": "{n} evals",
|
|
49
|
+
"overview.attemptsCount": "{n} attempts",
|
|
50
|
+
|
|
51
|
+
/** GroupSummary 的 KPI dt 标签(其余字段复用 verdict.* / overview.* 已有 key)。 */
|
|
52
|
+
"groupSummary.experiments": "Experiments",
|
|
53
|
+
|
|
54
|
+
"composedFrom.one": "composed from {n} run",
|
|
55
|
+
"composedFrom.other": "composed from {n} runs",
|
|
56
|
+
"latestRun": "latest {run}",
|
|
57
|
+
|
|
58
|
+
"scatter.betterHint": "better ↗",
|
|
59
|
+
"scatter.betterUpperRight": "better → upper right",
|
|
60
|
+
"scatter.axisReversed": "(axis reversed: right = better)",
|
|
61
|
+
/** 0 个可画点:x/y 指标没有可用数据。 */
|
|
62
|
+
"scatter.noData": "No data to plot {x} × {y}",
|
|
63
|
+
/** 恰好 1 个可画点:比较至少要两个实验。 */
|
|
64
|
+
"scatter.needTwo": "At least 2 experiments needed to compare {x} × {y}",
|
|
65
|
+
"pointsMissing.one": "{n} point missing data",
|
|
66
|
+
"pointsMissing.other": "{n} points missing data",
|
|
67
|
+
|
|
68
|
+
"attemptList.empty": "No attempts",
|
|
69
|
+
"attemptList.truncated": "and {n} more not shown",
|
|
70
|
+
"attemptList.truncatedText": "({n} more not shown)",
|
|
71
|
+
"attemptList.score": "score {score}",
|
|
72
|
+
"attemptList.details": "details",
|
|
73
|
+
|
|
74
|
+
"scoreboard.total": "Total",
|
|
75
|
+
"scoreboard.totalText": "total",
|
|
76
|
+
"scoreboard.missing.one": "{n} eval missing, scored 0",
|
|
77
|
+
"scoreboard.missing.other": "{n} evals missing, scored 0",
|
|
78
|
+
"scoreboard.missingText": "({n} missing)",
|
|
79
|
+
"scoreboard.weights": "weights:",
|
|
80
|
+
"scoreboard.allWeights": "all evals ×1",
|
|
81
|
+
"scoreboard.othersWeight": "others ×1",
|
|
82
|
+
"scoreboard.subjectTitle": "{evals} evals, weighted {earned} of {possible}",
|
|
83
|
+
|
|
84
|
+
"delta.pairHeader": "pair (A → B)",
|
|
85
|
+
|
|
86
|
+
"board.currentVerdicts": "Current verdicts",
|
|
87
|
+
"board.failing": "Failing:",
|
|
88
|
+
} as const;
|
|
89
|
+
|
|
90
|
+
export type ReportMessageKey = keyof typeof en;
|
|
91
|
+
|
|
92
|
+
const zhCN: Record<ReportMessageKey, string> = {
|
|
93
|
+
"verdict.passed": "通过",
|
|
94
|
+
"verdict.failed": "失败",
|
|
95
|
+
"verdict.errored": "错误",
|
|
96
|
+
"verdict.skipped": "跳过",
|
|
97
|
+
|
|
98
|
+
"cell.missing": "无数据",
|
|
99
|
+
"cell.measuredTitle": "{samples}/{total} 次 attempt 测得",
|
|
100
|
+
"cell.noneMeasurableTitle": "0/{total} 次 attempt 可测",
|
|
101
|
+
"cell.coverageTitle": "覆盖率 {samples}/{total}:其余 attempt 测不了这个指标",
|
|
102
|
+
|
|
103
|
+
"table.higherBetter": "越高越好",
|
|
104
|
+
"table.lowerBetter": "越低越好",
|
|
105
|
+
"table.model": "模型",
|
|
106
|
+
"table.agent": "Agent",
|
|
107
|
+
"table.verdicts": "结果",
|
|
108
|
+
"table.filterPlaceholder": "筛选行…",
|
|
109
|
+
"table.eval": "题目",
|
|
110
|
+
"table.reason": "原因",
|
|
111
|
+
"table.viewBreakdown": "逐题明细",
|
|
112
|
+
"table.attempt": "Attempt",
|
|
113
|
+
"table.columnsHidden.one": "(还有 {n} 列未列出)",
|
|
114
|
+
"table.columnsHidden.other": "(还有 {n} 列未列出)",
|
|
115
|
+
|
|
116
|
+
"overview.snapshots": "快照",
|
|
117
|
+
"overview.evals": "题目",
|
|
118
|
+
"overview.attempts": "尝试",
|
|
119
|
+
"overview.passRate": "通过率",
|
|
120
|
+
"overview.totalCost": "总成本",
|
|
121
|
+
"overview.totalDuration": "总耗时",
|
|
122
|
+
"overview.source": "数据来源:{n} 个快照",
|
|
123
|
+
"overview.experiments.one": "{n} 个实验",
|
|
124
|
+
"overview.experiments.other": "{n} 个实验",
|
|
125
|
+
"overview.evalsCount": "{n} 道题",
|
|
126
|
+
"overview.attemptsCount": "{n} 次 attempt",
|
|
127
|
+
|
|
128
|
+
"groupSummary.experiments": "实验数",
|
|
129
|
+
|
|
130
|
+
"composedFrom.one": "合成自 {n} 个 run",
|
|
131
|
+
"composedFrom.other": "合成自 {n} 个 run",
|
|
132
|
+
"latestRun": "最新 {run}",
|
|
133
|
+
|
|
134
|
+
"scatter.betterHint": "更好 ↗",
|
|
135
|
+
"scatter.betterUpperRight": "更好 → 右上",
|
|
136
|
+
"scatter.axisReversed": "(轴反向:右 = 更好)",
|
|
137
|
+
"scatter.noData": "{x} × {y} 没有可绘制的数据",
|
|
138
|
+
"scatter.needTwo": "至少需要 2 个实验才能比较 {x} × {y}",
|
|
139
|
+
"pointsMissing.one": "{n} 个点缺数据",
|
|
140
|
+
"pointsMissing.other": "{n} 个点缺数据",
|
|
141
|
+
|
|
142
|
+
"attemptList.empty": "没有 attempt",
|
|
143
|
+
"attemptList.truncated": "还有 {n} 条未列出",
|
|
144
|
+
"attemptList.truncatedText": "(还有 {n} 条未列出)",
|
|
145
|
+
"attemptList.score": "得分 {score}",
|
|
146
|
+
"attemptList.details": "详情",
|
|
147
|
+
|
|
148
|
+
"scoreboard.total": "总分",
|
|
149
|
+
"scoreboard.totalText": "总分",
|
|
150
|
+
"scoreboard.missing.one": "{n} 道题缺席,按 0 计",
|
|
151
|
+
"scoreboard.missing.other": "{n} 道题缺席,按 0 计",
|
|
152
|
+
"scoreboard.missingText": "(缺 {n} 题)",
|
|
153
|
+
"scoreboard.weights": "权重:",
|
|
154
|
+
"scoreboard.allWeights": "全部题 ×1",
|
|
155
|
+
"scoreboard.othersWeight": "其余 ×1",
|
|
156
|
+
"scoreboard.subjectTitle": "{evals} 道题,加权得分 {earned}/{possible}",
|
|
157
|
+
|
|
158
|
+
"delta.pairHeader": "对比 (A → B)",
|
|
159
|
+
|
|
160
|
+
"board.currentVerdicts": "现刻判定",
|
|
161
|
+
"board.failing": "失败清单:",
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const dictionaries: Record<ReportLocale, Record<ReportMessageKey, string>> = {
|
|
165
|
+
en,
|
|
166
|
+
"zh-CN": zhCN,
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
/** 查字典 + 简单插值({name} 占位符)。缺 locale 回退 en。 */
|
|
170
|
+
export function localeText(
|
|
171
|
+
locale: ReportLocale,
|
|
172
|
+
key: ReportMessageKey,
|
|
173
|
+
vars?: Record<string, string | number>,
|
|
174
|
+
): string {
|
|
175
|
+
const template = dictionaries[locale]?.[key] ?? dictionaries.en[key];
|
|
176
|
+
if (!vars) return template;
|
|
177
|
+
return template.replace(/\{(\w+)\}/g, (m, name: string) =>
|
|
178
|
+
name in vars ? String(vars[name]) : m,
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** 带单复数的计数文案:n === 1 用 `<base>.one`,其余 `<base>.other`(zh-CN 两键同值)。 */
|
|
183
|
+
export function countText(
|
|
184
|
+
locale: ReportLocale,
|
|
185
|
+
base: "overview.experiments" | "composedFrom" | "pointsMissing" | "scoreboard.missing" | "table.columnsHidden",
|
|
186
|
+
n: number,
|
|
187
|
+
): string {
|
|
188
|
+
return localeText(locale, `${base}.${n === 1 ? "one" : "other"}` as ReportMessageKey, { n });
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** 指标 label 的可本地化形态:字符串,或按 locale 给的字典(缺项回退 en,再回退任一非空值)。 */
|
|
192
|
+
export type LocalizedLabel = string | Partial<Record<ReportLocale, string>>;
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* 按 locale 解析指标 label:字符串原样;字典取当前 locale,缺项回退 en,
|
|
196
|
+
* 再回退字典里任一值;全空回退 fallback(= metric.name)。渲染面(web / text)共用。
|
|
197
|
+
*/
|
|
198
|
+
export function resolveMetricLabel(
|
|
199
|
+
label: LocalizedLabel | undefined,
|
|
200
|
+
locale: ReportLocale,
|
|
201
|
+
fallback: string,
|
|
202
|
+
): string {
|
|
203
|
+
if (label === undefined) return fallback;
|
|
204
|
+
if (typeof label === "string") return label;
|
|
205
|
+
return label[locale] ?? label.en ?? Object.values(label).find((v) => v !== undefined) ?? fallback;
|
|
206
|
+
}
|
package/src/report/metrics.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// defineMetric 与内置指标。
|
|
2
2
|
//
|
|
3
3
|
// null ≠ 0:null = 此 attempt 测不了这个指标(不进聚合);0 = 测了,结果是零(照常进)。
|
|
4
|
-
// 哪个
|
|
4
|
+
// 哪个 verdict 落哪边必须显式表态,内置指标按 docs/feature/reports/library.md「内置指标」的表格:
|
|
5
5
|
//
|
|
6
6
|
// 指标(name) skipped errored failed passed better
|
|
7
7
|
// passRate(pass-rate) null 0 0 1 higher
|
|
@@ -9,13 +9,19 @@
|
|
|
9
9
|
// durationMs(duration) null 实测 实测 实测 lower
|
|
10
10
|
// tokens(tokens) null 实测;无 usage→null 同左 同左 lower
|
|
11
11
|
// costUSD(cost) null 同上 同左 同左 lower
|
|
12
|
+
// turns(turns) null 实测;o11y 缺失→null 同左 同左 lower
|
|
13
|
+
//
|
|
14
|
+
// 两档指标(docs/feature/reports/library.md「内置指标」):以上除 turns 外全部只读 attempt.result
|
|
15
|
+
// 的瘦身字段——任何 producer、任何 copySnapshots artifacts 选择都算得出,内置报告
|
|
16
|
+
// CostPassRateComparison 只用这一档。turns 读 attempt.o11y()(懒加载 artifact),发布时若
|
|
17
|
+
// o11y 没随行就诚实渲染缺数据「—」,不算 0——报告作者自己摆时心里要有这根弦,内置报告不用它。
|
|
12
18
|
|
|
13
19
|
import type { EvalResult } from "../types.ts";
|
|
14
20
|
import type { Metric } from "./types.ts";
|
|
15
21
|
|
|
16
22
|
/**
|
|
17
23
|
* 定义一个指标。内置指标与自定义指标是同一个类型,没有特权;
|
|
18
|
-
* 校验只管「能进计算」的最低要求,聚合语义见 docs/reports.md
|
|
24
|
+
* 校验只管「能进计算」的最低要求,聚合语义见 docs/feature/reports/architecture.md「指标聚合不变量」。
|
|
19
25
|
* name 是 const 泛型:产物的 name 保持字面量,`row.cells[metric.name]` 才有编译期列键。
|
|
20
26
|
*/
|
|
21
27
|
export function defineMetric<const Name extends string>(def: Metric<Name>): Metric<Name> {
|
|
@@ -35,27 +41,27 @@ export function attemptCostUSD(result: EvalResult): number | null {
|
|
|
35
41
|
|
|
36
42
|
export const passRate = defineMetric({
|
|
37
43
|
name: "pass-rate",
|
|
38
|
-
label: "Pass rate",
|
|
44
|
+
label: { en: "Pass rate", "zh-CN": "成功率" },
|
|
39
45
|
description: "Share of evals that passed (skipped attempts excluded).",
|
|
40
46
|
better: "higher",
|
|
41
47
|
unit: "%",
|
|
42
48
|
value: (a) =>
|
|
43
|
-
a.result.
|
|
49
|
+
a.result.verdict === "skipped" ? null : a.result.verdict === "passed" ? 1 : 0,
|
|
44
50
|
});
|
|
45
51
|
|
|
46
52
|
export const examScore = defineMetric({
|
|
47
53
|
name: "exam-score",
|
|
48
|
-
label: "Exam score",
|
|
54
|
+
label: { en: "Exam score", "zh-CN": "考试得分" },
|
|
49
55
|
description: "Per-eval score: gates decide pass, soft assertions grade quality.",
|
|
50
56
|
better: "higher",
|
|
51
57
|
unit: "%",
|
|
52
58
|
value(a) {
|
|
53
|
-
const {
|
|
54
|
-
if (
|
|
55
|
-
// 先按
|
|
59
|
+
const { verdict, assertions } = a.result;
|
|
60
|
+
if (verdict === "skipped") return null;
|
|
61
|
+
// 先按 verdict 分派,再看断言:errored 的断言是空数组,「gate 全过才得分」的
|
|
56
62
|
// 字面实现会让条件空真成立、崩溃反而得满分 —— 交白卷是 0 分,不是缺数据,更不是满分。
|
|
57
63
|
// failed 同理得 0:--strict 下被翻成 failed 的哪怕 soft 分不低也是 0(报告不重新判卷)。
|
|
58
|
-
if (
|
|
64
|
+
if (verdict !== "passed") return 0;
|
|
59
65
|
const soft = assertions.filter((x) => x.severity === "soft");
|
|
60
66
|
if (soft.length === 0) return 1;
|
|
61
67
|
return soft.reduce((sum, x) => sum + x.score, 0) / soft.length;
|
|
@@ -64,21 +70,21 @@ export const examScore = defineMetric({
|
|
|
64
70
|
|
|
65
71
|
export const durationMs = defineMetric({
|
|
66
72
|
name: "duration",
|
|
67
|
-
label: "Duration",
|
|
73
|
+
label: { en: "Duration", "zh-CN": "平均耗时" },
|
|
68
74
|
description: "Wall-clock duration of the attempt.",
|
|
69
75
|
better: "lower",
|
|
70
76
|
unit: "ms",
|
|
71
|
-
value: (a) => (a.result.
|
|
77
|
+
value: (a) => (a.result.verdict === "skipped" ? null : a.result.durationMs),
|
|
72
78
|
});
|
|
73
79
|
|
|
74
80
|
export const tokens = defineMetric({
|
|
75
81
|
name: "tokens",
|
|
76
|
-
label: "Tokens",
|
|
82
|
+
label: { en: "Tokens", "zh-CN": "Tokens" },
|
|
77
83
|
description: "Input + output tokens (cache reads/writes excluded).",
|
|
78
84
|
better: "lower",
|
|
79
85
|
unit: "tokens",
|
|
80
86
|
value(a) {
|
|
81
|
-
if (a.result.
|
|
87
|
+
if (a.result.verdict === "skipped") return null;
|
|
82
88
|
const usage = a.result.usage;
|
|
83
89
|
if (!usage) return null;
|
|
84
90
|
// 只加 input + output:缓存读写量大但便宜,计进去会把缓存热的 agent 画成 token 大户;
|
|
@@ -89,9 +95,27 @@ export const tokens = defineMetric({
|
|
|
89
95
|
|
|
90
96
|
export const costUSD = defineMetric({
|
|
91
97
|
name: "cost",
|
|
92
|
-
label: "Cost",
|
|
98
|
+
label: { en: "Cost", "zh-CN": "预估成本" },
|
|
93
99
|
description: "USD cost per attempt (gateway-measured beats estimated).",
|
|
94
100
|
better: "lower",
|
|
95
101
|
unit: "$",
|
|
96
|
-
value: (a) => (a.result.
|
|
102
|
+
value: (a) => (a.result.verdict === "skipped" ? null : attemptCostUSD(a.result)),
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* 唯一读 artifact(o11y,懒加载)的内置指标——其余五个只读瘦身字段。
|
|
107
|
+
* 发布时若该 attempt 没带 o11y(如 copySnapshots 的 artifacts 选项漏了它),
|
|
108
|
+
* value 如实返回 null,渲染成「—」,不冒充 0——见 docs/feature/reports/library.md「内置指标」。
|
|
109
|
+
*/
|
|
110
|
+
export const turns = defineMetric({
|
|
111
|
+
name: "turns",
|
|
112
|
+
label: { en: "Turns", "zh-CN": "轮次" },
|
|
113
|
+
description: "Total agent turns (assistant replies) per attempt. Reads o11y — “—” if not published alongside this attempt.",
|
|
114
|
+
better: "lower",
|
|
115
|
+
unit: "turns",
|
|
116
|
+
async value(a) {
|
|
117
|
+
if (a.result.verdict === "skipped") return null;
|
|
118
|
+
const o11y = await a.o11y();
|
|
119
|
+
return o11y?.totalTurns ?? null;
|
|
120
|
+
},
|
|
97
121
|
});
|
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
// 排版原语 Row / Col / Section / Text / Style
|
|
1
|
+
// 排版原语 Row / Col / Section / Text / Style / Table:六个内置双面组件,没有特殊机制。
|
|
2
2
|
// web 面是普通 React 渲染;text 面用 ctx.render(child, 子宽) 显式传宽,
|
|
3
3
|
// Row 分栏、宽度不足降级纵向。Style 给自定义组件带样式:web 面吐 <style> 标签,
|
|
4
4
|
// text 面渲染为空 —— 静态导出不打包用户代码,className 引用的 CSS 靠它随树走。
|
|
5
|
+
// Table 是自定义表的标准件,官方表状组件的 text 面也建在它上面(见 ./text/table.ts)。
|
|
5
6
|
|
|
6
7
|
import type { ReactNode } from "react";
|
|
8
|
+
import type { AttemptLocator } from "../results/locator.ts";
|
|
7
9
|
import { defineComponent, type ReportNode } from "./tree.ts";
|
|
8
|
-
import {
|
|
10
|
+
import { localeText, type ReportLocale } from "./locale.ts";
|
|
11
|
+
import { indentBlock, joinColumns, wrapDisplay, type ColumnAlign } from "./text/layout.ts";
|
|
12
|
+
import { renderTableText } from "./text/table.ts";
|
|
9
13
|
|
|
10
14
|
function childArray(children: ReportNode): ReportNode[] {
|
|
11
15
|
if (children === null || children === undefined || typeof children === "boolean") return [];
|
|
@@ -115,3 +119,97 @@ export const Style = defineComponent<StyleProps>({
|
|
|
115
119
|
},
|
|
116
120
|
});
|
|
117
121
|
Style.displayName = "Style";
|
|
122
|
+
|
|
123
|
+
/** 一列的定义:取哪个 cells 键、表头写什么、往哪边对齐。 */
|
|
124
|
+
export interface TableColumn {
|
|
125
|
+
/** 取 `row.cells[key]` 的键。 */
|
|
126
|
+
key: string;
|
|
127
|
+
/** 表头文案,两个面都原样渲染。 */
|
|
128
|
+
header: string;
|
|
129
|
+
/** 对齐方向,默认 `"left"`;`"right"` 按显示宽度右对齐,数字列用。 */
|
|
130
|
+
align?: ColumnAlign;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** 一行的数据:身份键、已格式化的格子、可选的 attempt locator。 */
|
|
134
|
+
export interface TableRow {
|
|
135
|
+
/** 行身份。 */
|
|
136
|
+
key: string;
|
|
137
|
+
/** 已格式化的显示值;`null`(或缺这个键)渲染成 `—`,不补 0。 */
|
|
138
|
+
cells: Record<string, string | null>;
|
|
139
|
+
/** 带上就多一列 attempt:web 面链到证据室,text 面列出 locator。 */
|
|
140
|
+
locator?: AttemptLocator;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export interface TableProps {
|
|
144
|
+
/** 列定义;数组顺序即渲染顺序。 */
|
|
145
|
+
columns: TableColumn[];
|
|
146
|
+
/** 行数据;数组顺序即渲染顺序,组件不重排也不过滤。 */
|
|
147
|
+
rows: TableRow[];
|
|
148
|
+
/** 组件自带文案(attempt 表头、丢列提示)的语言;省略时随宿主。 */
|
|
149
|
+
locale?: ReportLocale;
|
|
150
|
+
/** web 面挂到 `<table>` 上。 */
|
|
151
|
+
className?: string;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const MISSING_MARK = "—";
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* 自定义表的标准件:列由报告作者定,格子是算好的显示值,两个面各自排整齐。
|
|
158
|
+
*
|
|
159
|
+
* text 面列宽按**显示宽度**算(CJK / 全角记 2 列),所以中文列不会撕歪;总宽超过
|
|
160
|
+
* `ctx.width` 时先折最宽的左对齐列(右对齐列是数字,折行读不了),压到下限仍放不下
|
|
161
|
+
* 就从右侧丢列并在表下如实报丢了几列。web 面是 `<table>` + `<thead>` / `<tbody>`,
|
|
162
|
+
* 右对齐落成 `nre-align-right` 类,不用内联样式。
|
|
163
|
+
*
|
|
164
|
+
* 官方的 `MetricTable` / `MetricMatrix` / `Scoreboard` / `DeltaTable` 的 text 面就建在
|
|
165
|
+
* 这个组件上:自定义表和官方表用同一把尺子。
|
|
166
|
+
*/
|
|
167
|
+
export const Table = defineComponent<TableProps>({
|
|
168
|
+
web({ columns, rows, locale, className }, ctx) {
|
|
169
|
+
const chrome = locale ?? ctx.locale;
|
|
170
|
+
const hasLocator = rows.some((row) => row.locator !== undefined);
|
|
171
|
+
const alignClass = (align?: ColumnAlign) => (align === "right" ? "nre-align-right" : undefined);
|
|
172
|
+
return (
|
|
173
|
+
<table className={cx("nre", "nre-table", className)}>
|
|
174
|
+
<thead>
|
|
175
|
+
<tr>
|
|
176
|
+
{columns.map((column) => (
|
|
177
|
+
<th key={column.key} scope="col" className={alignClass(column.align)}>
|
|
178
|
+
{column.header}
|
|
179
|
+
</th>
|
|
180
|
+
))}
|
|
181
|
+
{hasLocator ? <th scope="col">{localeText(chrome, "table.attempt")}</th> : null}
|
|
182
|
+
</tr>
|
|
183
|
+
</thead>
|
|
184
|
+
<tbody>
|
|
185
|
+
{rows.map((row) => (
|
|
186
|
+
<tr key={row.key}>
|
|
187
|
+
{columns.map((column) => {
|
|
188
|
+
const value = row.cells[column.key];
|
|
189
|
+
const missing = value === null || value === undefined;
|
|
190
|
+
return (
|
|
191
|
+
<td key={column.key} className={alignClass(column.align)}>
|
|
192
|
+
{missing ? <span className="nre-missing">{MISSING_MARK}</span> : value}
|
|
193
|
+
</td>
|
|
194
|
+
);
|
|
195
|
+
})}
|
|
196
|
+
{hasLocator ? (
|
|
197
|
+
<td>
|
|
198
|
+
{row.locator ? (
|
|
199
|
+
<a className="nre-locator" href={ctx.attemptHref(row.locator)}>
|
|
200
|
+
{row.locator}
|
|
201
|
+
</a>
|
|
202
|
+
) : (
|
|
203
|
+
<span className="nre-missing">{MISSING_MARK}</span>
|
|
204
|
+
)}
|
|
205
|
+
</td>
|
|
206
|
+
) : null}
|
|
207
|
+
</tr>
|
|
208
|
+
))}
|
|
209
|
+
</tbody>
|
|
210
|
+
</table>
|
|
211
|
+
);
|
|
212
|
+
},
|
|
213
|
+
text: renderTableText,
|
|
214
|
+
});
|
|
215
|
+
Table.displayName = "Table";
|