niceeval 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +2 -2
- package/dist/i18n/zh-CN.d.ts +3 -3
- package/dist/report/aggregate.d.ts +32 -26
- package/dist/report/aggregate.js +157 -76
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +91 -164
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +86 -73
- package/dist/report/compute.js +592 -432
- package/dist/report/flag.d.ts +28 -17
- package/dist/report/flag.js +86 -16
- package/dist/report/format.d.ts +11 -11
- package/dist/report/format.js +17 -15
- package/dist/report/index.d.ts +16 -17
- package/dist/report/index.js +20 -22
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +49 -34
- package/dist/report/locale.js +106 -58
- package/dist/report/metrics.d.ts +10 -3
- package/dist/report/metrics.js +46 -12
- package/dist/report/primitives.d.ts +42 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +10 -8
- package/dist/report/react/AttemptList.js +18 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +3 -3
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +4 -2
- package/dist/report/react/ExperimentComparison.js +5 -4
- package/dist/report/react/ExperimentList.d.ts +3 -3
- package/dist/report/react/ExperimentList.js +16 -15
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +54 -17
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +105 -149
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -20
- package/dist/report/report.js +261 -34
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +225 -157
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +2 -2
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +245 -300
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +38 -14
- package/dist/shared/aggregate.d.ts +3 -2
- package/dist/shared/aggregate.js +5 -4
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +11 -11
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +5 -5
- package/docs-site/zh/{guides → explanation}/runner.mdx +1 -1
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +7 -7
- package/docs-site/zh/{guides → how-to}/authoring.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +18 -18
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/experiments.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/publish-report.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-providers.mdx +1 -1
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +3 -3
- package/docs-site/zh/{guides → how-to}/write-send.mdx +13 -13
- package/docs-site/zh/index.mdx +23 -25
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +5 -5
- package/docs-site/zh/reference/capabilities.mdx +6 -6
- package/docs-site/zh/reference/cli.mdx +9 -7
- package/docs-site/zh/reference/define-agent.mdx +1 -1
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +7 -7
- package/docs-site/zh/{guides → reference}/report-components.mdx +5 -5
- package/docs-site/zh/{guides → reference}/results-data.mdx +5 -5
- package/docs-site/zh/{guides → troubleshooting}/debug-sandbox.mdx +2 -2
- package/docs-site/zh/{guides → troubleshooting}/debugging.mdx +4 -2
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +6 -2
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +3 -3
- package/src/cli.ts +42 -23
- package/src/context/context.ts +1 -1
- package/src/context/session.test.ts +1 -1
- package/src/context/session.ts +1 -1
- package/src/i18n/en.ts +18 -16
- package/src/i18n/zh-CN.ts +16 -15
- package/src/report/aggregate.ts +175 -87
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -285
- package/src/report/compute.ts +717 -515
- package/src/report/dual-render.test.tsx +738 -1148
- package/src/report/flag.ts +97 -33
- package/src/report/format.ts +18 -22
- package/src/report/index.ts +113 -58
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +120 -69
- package/src/report/metrics.ts +42 -12
- package/src/report/primitives.tsx +190 -45
- package/src/report/react/AttemptList.tsx +32 -20
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +12 -7
- package/src/report/react/ExperimentList.tsx +38 -26
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +74 -20
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/enhance.js +57 -5
- package/src/report/react/fixtures.ts +109 -156
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +139 -104
- package/src/report/react/styles.css +181 -91
- package/src/report/report.test.ts +761 -1031
- package/src/report/report.ts +425 -47
- package/src/report/text/faces.ts +257 -164
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +2 -2
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +257 -287
- package/src/report/web.ts +63 -20
- package/src/results/attempt-evidence.test.ts +4 -4
- package/src/results/attempt-evidence.ts +5 -5
- package/src/results/copy.ts +6 -6
- package/src/results/host-equivalence.test.ts +26 -14
- package/src/results/index.ts +10 -4
- package/src/results/open.ts +8 -4
- package/src/results/results.test.ts +4 -3
- package/src/results/select.ts +104 -34
- package/src/results/types.ts +36 -14
- package/src/runner/feedback/human.test.ts +1 -1
- package/src/runner/run.ts +1 -1
- package/src/sandbox/cli-commands.ts +2 -2
- package/src/scoring/judge.test.ts +1 -1
- package/src/shared/aggregate.ts +5 -4
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +107 -56
- package/src/show/render.ts +43 -27
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +86 -36
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +324 -63
- package/src/view/app/components/CodeView.tsx +10 -4
- package/src/view/app/i18n.ts +31 -17
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +14 -14
- package/src/view/data.test.ts +9 -3
- package/src/view/data.ts +145 -49
- package/src/view/index.ts +48 -44
- package/src/view/server.ts +35 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +224 -0
- package/src/view/view-report.test.ts +161 -57
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -39
- package/dist/report/built-ins/experiment-comparison.js +0 -119
- package/dist/report/built-ins/index.d.ts +0 -2
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/src/report/built-in-user-parity.test.tsx +0 -597
- package/src/report/built-ins/experiment-comparison.tsx +0 -179
- package/src/report/built-ins/index.ts +0 -7
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/ci-integration.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/reporters.mdx +0 -0
- /package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +0 -0
package/dist/report/locale.js
CHANGED
|
@@ -1,9 +1,43 @@
|
|
|
1
|
-
// 官方组件 chrome 文案的 locale
|
|
1
|
+
// 官方组件 chrome 文案的 locale 字典与 LocalizedText 解析。
|
|
2
|
+
// ReportLocale 是开放的 BCP 47 标签(数据协议不封语言上限);官方内置文案与
|
|
3
|
+
// MetricCell.display 生成面当前覆盖 en / zh-CN,其它 locale 走 LocalizedText 回退规则。
|
|
2
4
|
// 只覆盖组件自带的固定文案(verdict 词、缺数据、覆盖率角标、注脚、占位符等);
|
|
3
|
-
//
|
|
4
|
-
// 刻意不 import src/i18n/(CLI 专用字典,locale 来源与 key 面完全不同)
|
|
5
|
-
// zh-CN 译法与默认 Reports 契约保持一致(端到端成功率/平均耗时/预估成本…)。
|
|
5
|
+
// 维度键、warnings 的 message 不经这里。
|
|
6
|
+
// 刻意不 import src/i18n/(CLI 专用字典,locale 来源与 key 面完全不同)。
|
|
6
7
|
export const DEFAULT_REPORT_LOCALE = "en";
|
|
8
|
+
/** MetricCell.display 生成面覆盖的 locale 全集;其它 locale 按 LocalizedText 回退取 en。 */
|
|
9
|
+
export const DISPLAY_LOCALES = ["en", "zh-CN"];
|
|
10
|
+
/**
|
|
11
|
+
* LocalizedText 的确定回退:取当前 locale;缺失时取 en;仍缺失时取按 locale 键字典序的
|
|
12
|
+
* 第一个非空值。对象没有任何非空值时报错,不渲染空文案
|
|
13
|
+
* (docs/feature/reports/library/shell.md「行为约束」)。
|
|
14
|
+
*/
|
|
15
|
+
export function resolveLocalizedText(text, locale) {
|
|
16
|
+
if (typeof text === "string")
|
|
17
|
+
return text;
|
|
18
|
+
const direct = text[locale];
|
|
19
|
+
if (direct !== undefined && direct !== "")
|
|
20
|
+
return direct;
|
|
21
|
+
const en = text.en;
|
|
22
|
+
if (en !== undefined && en !== "")
|
|
23
|
+
return en;
|
|
24
|
+
for (const key of Object.keys(text).sort()) {
|
|
25
|
+
const value = text[key];
|
|
26
|
+
if (value !== undefined && value !== "")
|
|
27
|
+
return value;
|
|
28
|
+
}
|
|
29
|
+
throw new Error("LocalizedText object has no non-empty value. Provide at least one locale entry, e.g. { en: \"…\" }.");
|
|
30
|
+
}
|
|
31
|
+
/** LocalizedText 按字段值深相等(键顺序无关;标题回退链的「唯一且相同」判定用)。 */
|
|
32
|
+
export function localizedTextEquals(a, b) {
|
|
33
|
+
if (typeof a === "string" || typeof b === "string")
|
|
34
|
+
return a === b;
|
|
35
|
+
const keysA = Object.keys(a).sort();
|
|
36
|
+
const keysB = Object.keys(b).sort();
|
|
37
|
+
if (keysA.length !== keysB.length)
|
|
38
|
+
return false;
|
|
39
|
+
return keysA.every((key, i) => key === keysB[i] && a[key] === b[key]);
|
|
40
|
+
}
|
|
7
41
|
const en = {
|
|
8
42
|
"verdict.passed": "passed",
|
|
9
43
|
"verdict.failed": "failed",
|
|
@@ -14,6 +48,7 @@ const en = {
|
|
|
14
48
|
"cell.measuredTitle": "{samples}/{total} attempts measured",
|
|
15
49
|
"cell.noneMeasurableTitle": "0/{total} attempts measurable",
|
|
16
50
|
"cell.coverageTitle": "coverage {samples}/{total}: this metric is null for the remaining attempts",
|
|
51
|
+
"cell.missingValue": "(missing)",
|
|
17
52
|
"table.higherBetter": "higher is better",
|
|
18
53
|
"table.lowerBetter": "lower is better",
|
|
19
54
|
"table.model": "Model",
|
|
@@ -29,45 +64,45 @@ const en = {
|
|
|
29
64
|
"experimentList.avgDuration": "Avg duration",
|
|
30
65
|
"experimentList.passRate": "End-to-end pass rate",
|
|
31
66
|
"experimentList.tokens": "Tokens",
|
|
32
|
-
"experimentList.
|
|
67
|
+
"experimentList.cost": "Cost",
|
|
33
68
|
"experimentList.result": "Result",
|
|
34
69
|
"experimentList.status": "Status",
|
|
35
70
|
"experimentList.evalAttempt": "Eval / Attempt",
|
|
36
71
|
"experimentList.duration": "Duration",
|
|
37
|
-
"experimentList.cost": "Cost",
|
|
38
72
|
"experimentList.filterPlaceholder": "Filter experiments…",
|
|
39
73
|
"experimentList.defaultModel": "default",
|
|
40
74
|
"experimentList.flags": "Flags",
|
|
41
75
|
"entityList.average": "{value} avg",
|
|
76
|
+
"entityList.moreFailures.one": "+{n} more failure",
|
|
77
|
+
"entityList.moreFailures.other": "+{n} more failures",
|
|
42
78
|
/** <Table> 压到下限仍放不下时,从右侧丢列并如实报数。 */
|
|
43
79
|
"table.columnsHidden.one": "({n} more column not shown)",
|
|
44
80
|
"table.columnsHidden.other": "({n} more columns not shown)",
|
|
45
|
-
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
81
|
+
/** ScopeSummary 的 KPI 标签。 */
|
|
82
|
+
"scopeSummary.experiments": "Experiments",
|
|
83
|
+
"scopeSummary.evals": "Evals",
|
|
84
|
+
"scopeSummary.attempts": "attempts",
|
|
85
|
+
"scopeSummary.passRate": "End-to-end pass rate",
|
|
86
|
+
"scopeSummary.totalCost": "Total cost",
|
|
87
|
+
"scopeSummary.range": "{from} – {to}",
|
|
88
|
+
"scopeSummary.votesEval": "eval verdicts",
|
|
89
|
+
"scopeSummary.votesAttempt": "attempt verdicts",
|
|
52
90
|
"overview.experiments.one": "{n} experiment",
|
|
53
91
|
"overview.experiments.other": "{n} experiments",
|
|
54
92
|
"overview.evalsCount": "{n} evals",
|
|
55
93
|
"overview.attemptsCount": "{n} attempts",
|
|
56
|
-
|
|
57
|
-
"
|
|
94
|
+
"overview.evals": "Evals",
|
|
95
|
+
"overview.passRate": "End-to-end pass rate",
|
|
96
|
+
"overview.totalCost": "Cost",
|
|
58
97
|
"experimentComparison.groups": "Experiment groups",
|
|
59
98
|
"experimentComparison.group": "Experiment group",
|
|
60
|
-
"experimentComparison.results": "
|
|
99
|
+
"experimentComparison.results": "Eval results",
|
|
61
100
|
"experimentComparison.lastRun": "Last run",
|
|
62
101
|
"experimentComparison.selectGroup": "Select experiment group {group}",
|
|
63
|
-
"experimentComparison.
|
|
102
|
+
"experimentComparison.commandsHead": "Group details:",
|
|
64
103
|
"experimentComparison.empty": "No experiment groups",
|
|
65
|
-
|
|
66
|
-
"composedFrom.other": "composed from {n} runs",
|
|
67
|
-
"latestRun": "latest {run}",
|
|
68
|
-
"scatter.betterHint": "better ↗",
|
|
104
|
+
/** 方向提示唯一文案:轴向已随 better 反正,「更好」恒指向右上(两轴都声明 better 时才显示)。 */
|
|
69
105
|
"scatter.betterUpperRight": "better → upper right",
|
|
70
|
-
"scatter.axisReversed": "(axis reversed: right = better)",
|
|
71
106
|
/** 0 个可画点:x/y 指标没有可用数据。 */
|
|
72
107
|
"scatter.noData": "No data to plot {x} × {y}",
|
|
73
108
|
"pointsMissing.one": "{n} point missing data",
|
|
@@ -80,16 +115,21 @@ const en = {
|
|
|
80
115
|
"attemptList.details": "details",
|
|
81
116
|
"scoreboard.total": "Total",
|
|
82
117
|
"scoreboard.totalText": "total",
|
|
83
|
-
"scoreboard.
|
|
84
|
-
"scoreboard.
|
|
85
|
-
"scoreboard.
|
|
118
|
+
"scoreboard.notRun.one": "{n} eval not run, scored 0",
|
|
119
|
+
"scoreboard.notRun.other": "{n} evals not run, scored 0",
|
|
120
|
+
"scoreboard.unscorable.one": "{n} eval unscorable, scored 0",
|
|
121
|
+
"scoreboard.unscorable.other": "{n} evals unscorable, scored 0",
|
|
122
|
+
"scoreboard.notRunText": "({n} not run)",
|
|
123
|
+
"scoreboard.unscorableText": "({n} unscorable)",
|
|
124
|
+
"scoreboard.ignored.one": "{n} eval outside the question set ignored",
|
|
125
|
+
"scoreboard.ignored.other": "{n} evals outside the question set ignored",
|
|
86
126
|
"scoreboard.weights": "weights:",
|
|
87
127
|
"scoreboard.allWeights": "all evals ×1",
|
|
88
128
|
"scoreboard.othersWeight": "others ×1",
|
|
89
|
-
"scoreboard.subjectTitle": "{
|
|
129
|
+
"scoreboard.subjectTitle": "{questions} evals, weighted {earned} of {possible}",
|
|
90
130
|
"delta.pairHeader": "pair (A → B)",
|
|
91
|
-
"
|
|
92
|
-
"
|
|
131
|
+
"delta.empty": "{experiments} experiments, 0 comparable pairs",
|
|
132
|
+
"tabs.tab": "Tab",
|
|
93
133
|
};
|
|
94
134
|
const zhCN = {
|
|
95
135
|
"verdict.passed": "通过",
|
|
@@ -100,6 +140,7 @@ const zhCN = {
|
|
|
100
140
|
"cell.measuredTitle": "{samples}/{total} 次 attempt 测得",
|
|
101
141
|
"cell.noneMeasurableTitle": "0/{total} 次 attempt 可测",
|
|
102
142
|
"cell.coverageTitle": "覆盖率 {samples}/{total}:其余 attempt 测不了这个指标",
|
|
143
|
+
"cell.missingValue": "(missing)",
|
|
103
144
|
"table.higherBetter": "越高越好",
|
|
104
145
|
"table.lowerBetter": "越低越好",
|
|
105
146
|
"table.model": "模型",
|
|
@@ -114,43 +155,42 @@ const zhCN = {
|
|
|
114
155
|
"experimentList.avgDuration": "平均耗时",
|
|
115
156
|
"experimentList.passRate": "端到端成功率",
|
|
116
157
|
"experimentList.tokens": "Tokens",
|
|
117
|
-
"experimentList.
|
|
158
|
+
"experimentList.cost": "成本",
|
|
118
159
|
"experimentList.result": "结果",
|
|
119
160
|
"experimentList.status": "状态",
|
|
120
161
|
"experimentList.evalAttempt": "题目 / Attempt",
|
|
121
162
|
"experimentList.duration": "耗时",
|
|
122
|
-
"experimentList.cost": "成本",
|
|
123
163
|
"experimentList.filterPlaceholder": "筛选实验…",
|
|
124
164
|
"experimentList.defaultModel": "默认",
|
|
125
165
|
"experimentList.flags": "Flags",
|
|
126
166
|
"entityList.average": "平均 {value}",
|
|
167
|
+
"entityList.moreFailures.one": "+{n} 条其它失败",
|
|
168
|
+
"entityList.moreFailures.other": "+{n} 条其它失败",
|
|
127
169
|
"table.columnsHidden.one": "(还有 {n} 列未列出)",
|
|
128
170
|
"table.columnsHidden.other": "(还有 {n} 列未列出)",
|
|
129
|
-
"
|
|
130
|
-
"
|
|
131
|
-
"
|
|
132
|
-
"
|
|
133
|
-
"
|
|
134
|
-
"
|
|
135
|
-
"
|
|
171
|
+
"scopeSummary.experiments": "实验数",
|
|
172
|
+
"scopeSummary.evals": "题目",
|
|
173
|
+
"scopeSummary.attempts": "次 attempt",
|
|
174
|
+
"scopeSummary.passRate": "端到端成功率",
|
|
175
|
+
"scopeSummary.totalCost": "总成本",
|
|
176
|
+
"scopeSummary.range": "{from} – {to}",
|
|
177
|
+
"scopeSummary.votesEval": "Eval 计票",
|
|
178
|
+
"scopeSummary.votesAttempt": "Attempt 计票",
|
|
136
179
|
"overview.experiments.one": "{n} 个实验",
|
|
137
180
|
"overview.experiments.other": "{n} 个实验",
|
|
138
181
|
"overview.evalsCount": "{n} 道题",
|
|
139
182
|
"overview.attemptsCount": "{n} 次 attempt",
|
|
140
|
-
"
|
|
183
|
+
"overview.evals": "Eval",
|
|
184
|
+
"overview.passRate": "端到端成功率",
|
|
185
|
+
"overview.totalCost": "成本",
|
|
141
186
|
"experimentComparison.groups": "实验组",
|
|
142
187
|
"experimentComparison.group": "实验组",
|
|
143
|
-
"experimentComparison.results": "结果",
|
|
188
|
+
"experimentComparison.results": "Eval 结果",
|
|
144
189
|
"experimentComparison.lastRun": "最后运行",
|
|
145
190
|
"experimentComparison.selectGroup": "选择实验组 {group}",
|
|
146
|
-
"experimentComparison.
|
|
191
|
+
"experimentComparison.commandsHead": "查看组内详情:",
|
|
147
192
|
"experimentComparison.empty": "没有实验组",
|
|
148
|
-
"
|
|
149
|
-
"composedFrom.other": "合成自 {n} 个 run",
|
|
150
|
-
"latestRun": "最新 {run}",
|
|
151
|
-
"scatter.betterHint": "更好 ↗",
|
|
152
|
-
"scatter.betterUpperRight": "更好 → 右上",
|
|
153
|
-
"scatter.axisReversed": "(轴反向:右 = 更好)",
|
|
193
|
+
"scatter.betterUpperRight": "越靠右上越好",
|
|
154
194
|
"scatter.noData": "{x} × {y} 没有可绘制的数据",
|
|
155
195
|
"pointsMissing.one": "{n} 个点缺数据",
|
|
156
196
|
"pointsMissing.other": "{n} 个点缺数据",
|
|
@@ -162,22 +202,27 @@ const zhCN = {
|
|
|
162
202
|
"attemptList.details": "详情",
|
|
163
203
|
"scoreboard.total": "总分",
|
|
164
204
|
"scoreboard.totalText": "总分",
|
|
165
|
-
"scoreboard.
|
|
166
|
-
"scoreboard.
|
|
167
|
-
"scoreboard.
|
|
205
|
+
"scoreboard.notRun.one": "{n} 道题没跑,按 0 计",
|
|
206
|
+
"scoreboard.notRun.other": "{n} 道题没跑,按 0 计",
|
|
207
|
+
"scoreboard.unscorable.one": "{n} 道题测不了,按 0 计",
|
|
208
|
+
"scoreboard.unscorable.other": "{n} 道题测不了,按 0 计",
|
|
209
|
+
"scoreboard.notRunText": "(没跑 {n} 题)",
|
|
210
|
+
"scoreboard.unscorableText": "(测不了 {n} 题)",
|
|
211
|
+
"scoreboard.ignored.one": "题集之外的 {n} 道题已忽略",
|
|
212
|
+
"scoreboard.ignored.other": "题集之外的 {n} 道题已忽略",
|
|
168
213
|
"scoreboard.weights": "权重:",
|
|
169
214
|
"scoreboard.allWeights": "全部题 ×1",
|
|
170
215
|
"scoreboard.othersWeight": "其余 ×1",
|
|
171
|
-
"scoreboard.subjectTitle": "{
|
|
216
|
+
"scoreboard.subjectTitle": "{questions} 道题,加权得分 {earned}/{possible}",
|
|
172
217
|
"delta.pairHeader": "对比 (A → B)",
|
|
173
|
-
"
|
|
174
|
-
"
|
|
218
|
+
"delta.empty": "{experiments} 个实验、0 个可配对",
|
|
219
|
+
"tabs.tab": "Tab",
|
|
175
220
|
};
|
|
176
221
|
const dictionaries = {
|
|
177
222
|
en,
|
|
178
223
|
"zh-CN": zhCN,
|
|
179
224
|
};
|
|
180
|
-
/** 查字典 + 简单插值({name} 占位符)
|
|
225
|
+
/** 查字典 + 简单插值({name} 占位符)。内置词典未覆盖的 locale 回退 en。 */
|
|
181
226
|
export function localeText(locale, key, vars) {
|
|
182
227
|
const template = dictionaries[locale]?.[key] ?? dictionaries.en[key];
|
|
183
228
|
if (!vars)
|
|
@@ -189,13 +234,16 @@ export function countText(locale, base, n) {
|
|
|
189
234
|
return localeText(locale, `${base}.${n === 1 ? "one" : "other"}`, { n });
|
|
190
235
|
}
|
|
191
236
|
/**
|
|
192
|
-
* 按 locale 解析指标 label
|
|
193
|
-
*
|
|
237
|
+
* 按 locale 解析指标 / 列 label:字符串原样;字典按 LocalizedText 回退规则取值;
|
|
238
|
+
* undefined 回退 fallback(= metric.name)。渲染面(web / text)共用。
|
|
194
239
|
*/
|
|
195
240
|
export function resolveMetricLabel(label, locale, fallback) {
|
|
196
241
|
if (label === undefined)
|
|
197
242
|
return fallback;
|
|
198
|
-
|
|
199
|
-
return label;
|
|
200
|
-
|
|
243
|
+
try {
|
|
244
|
+
return resolveLocalizedText(label, locale);
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return fallback;
|
|
248
|
+
}
|
|
201
249
|
}
|
package/dist/report/metrics.d.ts
CHANGED
|
@@ -26,8 +26,15 @@ export declare const durationMs: Metric<"duration">;
|
|
|
26
26
|
export declare const tokens: Metric<"tokens">;
|
|
27
27
|
export declare const costUSD: Metric<"cost">;
|
|
28
28
|
/**
|
|
29
|
-
*
|
|
29
|
+
* 读 artifact(o11y,懒加载)的内置指标之一——其余只读瘦身字段。
|
|
30
30
|
* 发布时若该 attempt 没带 o11y(如 copySnapshots 的 artifacts 选项漏了它),
|
|
31
|
-
* value 如实返回 null,渲染成「—」,不冒充 0
|
|
31
|
+
* value 如实返回 null,渲染成「—」,不冒充 0。名字带限定词:o11y 事件流中的 assistant
|
|
32
|
+
* turn 数与 `t.send` 的 `s<session>/t<turn>` 轮次是两个计数。
|
|
32
33
|
*/
|
|
33
|
-
export declare const
|
|
34
|
+
export declare const assistantTurns: Metric<"assistant-turns">;
|
|
35
|
+
/**
|
|
36
|
+
* 同一 attempt 内同一条 shell 命令的重复失败数:每条命令失败 n 次(n > 1)记 n − 1,求和。
|
|
37
|
+
* 成功执行与只失败一次的命令不计。回答 agent 是否在反复撞同一个已知失败的命令。
|
|
38
|
+
* 读 o11y.json;skipped 与缺 o11y 返回 null(docs/feature/reports/library/metrics.md「内置指标」)。
|
|
39
|
+
*/
|
|
40
|
+
export declare const repeatedFailedCommands: Metric<"repeated-failed-commands">;
|
package/dist/report/metrics.js
CHANGED
|
@@ -12,12 +12,14 @@
|
|
|
12
12
|
// durationMs(duration) null 实测 实测 实测 lower
|
|
13
13
|
// tokens(tokens) null 实测;无 usage→null 同左 同左 lower
|
|
14
14
|
// costUSD(cost) null 同上 同左 同左 lower
|
|
15
|
-
//
|
|
15
|
+
// assistantTurns(assistant-turns) null 实测;o11y 缺失→null 同左 同左 lower
|
|
16
|
+
// repeatedFailedCommands(repeated-failed-commands) null 实测;o11y 缺失→null 同左 同左 lower
|
|
16
17
|
//
|
|
17
|
-
// 两档指标(docs/feature/reports/library.md「内置指标」):以上除
|
|
18
|
-
// 的瘦身字段——任何 producer、任何
|
|
19
|
-
//
|
|
20
|
-
// o11y
|
|
18
|
+
// 两档指标(docs/feature/reports/library/metrics.md「内置指标」):以上除 assistantTurns 与
|
|
19
|
+
// repeatedFailedCommands 外全部只读 attempt.result 的瘦身字段——任何 producer、任何
|
|
20
|
+
// copySnapshots artifacts 选择都算得出,内置报告 ExperimentComparison 只用这一档。
|
|
21
|
+
// 后两个读 attempt.o11y()(懒加载 artifact),发布时若 o11y 没随行就诚实渲染缺数据「—」,
|
|
22
|
+
// 不算 0——报告作者自己摆时心里要有这根弦,内置报告不用它们。
|
|
21
23
|
/**
|
|
22
24
|
* 定义一个指标。内置指标与自定义指标是同一个类型,没有特权;
|
|
23
25
|
* 校验只管「能进计算」的最低要求,聚合语义见 docs/feature/reports/architecture.md「指标聚合不变量」。
|
|
@@ -140,21 +142,22 @@ export const tokens = defineMetric({
|
|
|
140
142
|
});
|
|
141
143
|
export const costUSD = defineMetric({
|
|
142
144
|
name: "cost",
|
|
143
|
-
label: { en: "Cost", "zh-CN": "
|
|
145
|
+
label: { en: "Cost", "zh-CN": "成本" },
|
|
144
146
|
description: "USD cost per attempt (gateway-measured beats estimated).",
|
|
145
147
|
better: "lower",
|
|
146
148
|
unit: "$",
|
|
147
149
|
value: (a) => (a.result.verdict === "skipped" ? null : attemptCostUSD(a.result)),
|
|
148
150
|
});
|
|
149
151
|
/**
|
|
150
|
-
*
|
|
152
|
+
* 读 artifact(o11y,懒加载)的内置指标之一——其余只读瘦身字段。
|
|
151
153
|
* 发布时若该 attempt 没带 o11y(如 copySnapshots 的 artifacts 选项漏了它),
|
|
152
|
-
* value 如实返回 null,渲染成「—」,不冒充 0
|
|
154
|
+
* value 如实返回 null,渲染成「—」,不冒充 0。名字带限定词:o11y 事件流中的 assistant
|
|
155
|
+
* turn 数与 `t.send` 的 `s<session>/t<turn>` 轮次是两个计数。
|
|
153
156
|
*/
|
|
154
|
-
export const
|
|
155
|
-
name: "turns",
|
|
156
|
-
label: { en: "
|
|
157
|
-
description: "
|
|
157
|
+
export const assistantTurns = defineMetric({
|
|
158
|
+
name: "assistant-turns",
|
|
159
|
+
label: { en: "Assistant turns", "zh-CN": "Assistant 轮次" },
|
|
160
|
+
description: "Assistant turns in the o11y event stream per attempt. Reads o11y — “—” if not published alongside this attempt.",
|
|
158
161
|
better: "lower",
|
|
159
162
|
unit: "turns",
|
|
160
163
|
async value(a) {
|
|
@@ -164,3 +167,34 @@ export const turns = defineMetric({
|
|
|
164
167
|
return o11y?.totalTurns ?? null;
|
|
165
168
|
},
|
|
166
169
|
});
|
|
170
|
+
/**
|
|
171
|
+
* 同一 attempt 内同一条 shell 命令的重复失败数:每条命令失败 n 次(n > 1)记 n − 1,求和。
|
|
172
|
+
* 成功执行与只失败一次的命令不计。回答 agent 是否在反复撞同一个已知失败的命令。
|
|
173
|
+
* 读 o11y.json;skipped 与缺 o11y 返回 null(docs/feature/reports/library/metrics.md「内置指标」)。
|
|
174
|
+
*/
|
|
175
|
+
export const repeatedFailedCommands = defineMetric({
|
|
176
|
+
name: "repeated-failed-commands",
|
|
177
|
+
label: { en: "Repeated failed commands", "zh-CN": "重复失败命令" },
|
|
178
|
+
description: "Per attempt: for each shell command failing n > 1 times, count n − 1, summed. Reads o11y — “—” if not published alongside this attempt.",
|
|
179
|
+
better: "lower",
|
|
180
|
+
unit: "cmds",
|
|
181
|
+
async value(a) {
|
|
182
|
+
if (a.result.verdict === "skipped")
|
|
183
|
+
return null;
|
|
184
|
+
const o11y = await a.o11y();
|
|
185
|
+
if (!o11y)
|
|
186
|
+
return null;
|
|
187
|
+
const failures = new Map();
|
|
188
|
+
for (const entry of o11y.shellCommands) {
|
|
189
|
+
const failed = entry.success === false || (entry.success === undefined && entry.exitCode !== undefined && entry.exitCode !== 0);
|
|
190
|
+
if (!failed)
|
|
191
|
+
continue;
|
|
192
|
+
failures.set(entry.command, (failures.get(entry.command) ?? 0) + 1);
|
|
193
|
+
}
|
|
194
|
+
let repeated = 0;
|
|
195
|
+
for (const n of failures.values())
|
|
196
|
+
if (n > 1)
|
|
197
|
+
repeated += n - 1;
|
|
198
|
+
return repeated;
|
|
199
|
+
},
|
|
200
|
+
});
|
|
@@ -1,33 +1,60 @@
|
|
|
1
1
|
import type { AttemptLocator } from "../results/locator.ts";
|
|
2
2
|
import { type ReportNode } from "./tree.ts";
|
|
3
|
-
import { type ReportLocale } from "./locale.ts";
|
|
4
|
-
import {
|
|
3
|
+
import { type LocalizedText, type ReportLocale } from "./locale.ts";
|
|
4
|
+
import type { ColumnAlign } from "./text/layout.ts";
|
|
5
5
|
export interface LayoutProps {
|
|
6
6
|
children?: ReportNode;
|
|
7
7
|
className?: string;
|
|
8
8
|
}
|
|
9
|
-
|
|
9
|
+
export type RowProps = LayoutProps;
|
|
10
|
+
export type ColProps = LayoutProps;
|
|
11
|
+
/** 纵向依次排列:网页是块级堆叠,终端是逐块输出(块间空一行)。两面都按声明序。 */
|
|
10
12
|
export declare const Col: import("./tree.ts").ReportComponent<LayoutProps>;
|
|
11
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* 并排:web 面横排;text 面在可用宽度装得下全部子块时按显示宽度并排(与 `columns` 工具
|
|
15
|
+
* 同一把尺),装不下时整块退化为纵向堆叠——不截断、不隐藏任何子块。
|
|
16
|
+
*/
|
|
12
17
|
export declare const Row: import("./tree.ts").ReportComponent<LayoutProps>;
|
|
13
18
|
export interface SectionProps extends LayoutProps {
|
|
14
|
-
title:
|
|
19
|
+
title: LocalizedText;
|
|
15
20
|
}
|
|
16
21
|
/** 带标题的块:网页是标题层级,终端是标题行加缩进。 */
|
|
17
22
|
export declare const Section: import("./tree.ts").ReportComponent<SectionProps>;
|
|
18
|
-
|
|
19
|
-
|
|
23
|
+
export interface TextProps {
|
|
24
|
+
/** 自由正文原样渲染,不随 locale 自动翻译。 */
|
|
25
|
+
children: string | number;
|
|
26
|
+
className?: string;
|
|
27
|
+
}
|
|
28
|
+
/** 自由文本的显式载体:web 面负责转义,text 面按显示宽度折行。 */
|
|
29
|
+
export declare const Text: import("./tree.ts").ReportComponent<TextProps>;
|
|
20
30
|
export interface StyleProps {
|
|
21
|
-
children
|
|
31
|
+
children: string;
|
|
22
32
|
}
|
|
23
|
-
/**
|
|
33
|
+
/**
|
|
34
|
+
* 注入页级全局 CSS:树位置只决定声明顺序,不限定作用域;text 面零输出。
|
|
35
|
+
* 配置对象形态的报告要全站样式优先用外壳 styles,两条通道注入同一增强层。
|
|
36
|
+
*/
|
|
24
37
|
export declare const Style: import("./tree.ts").ReportComponent<StyleProps>;
|
|
38
|
+
export interface TabsProps extends LayoutProps {
|
|
39
|
+
}
|
|
40
|
+
export interface TabProps extends LayoutProps {
|
|
41
|
+
title: LocalizedText;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* 页内并列视图的可切换块。tab 是页内浏览状态,不是数据边界,也不是宿主寻址单位——
|
|
45
|
+
* 两个渲染面都输出全部 tab 的完整内容:web 静态 HTML 每 tab 一个 <details>(首个默认展开,
|
|
46
|
+
* 渐进增强升级成单选 tab 条,切换不改变数据);text 面按声明序输出带标题分节,不折成索引
|
|
47
|
+
* 也不省略(tab 没有选择器,索引只能是死路)。
|
|
48
|
+
*/
|
|
49
|
+
export declare const Tabs: import("./tree.ts").ReportComponent<TabsProps>;
|
|
50
|
+
/** 只能直接放在 <Tabs> 下;除通用 children / className 外只有 title。不参与路由,没有 id。 */
|
|
51
|
+
export declare const Tab: import("./tree.ts").ReportComponent<TabProps>;
|
|
25
52
|
/** 一列的定义:取哪个 cells 键、表头写什么、往哪边对齐。 */
|
|
26
53
|
export interface TableColumn {
|
|
27
54
|
/** 取 `row.cells[key]` 的键。 */
|
|
28
55
|
key: string;
|
|
29
|
-
/**
|
|
30
|
-
header:
|
|
56
|
+
/** 表头文案,按渲染 locale 选择。 */
|
|
57
|
+
header: LocalizedText;
|
|
31
58
|
/** 对齐方向,默认 `"left"`;`"right"` 按显示宽度右对齐,数字列用。 */
|
|
32
59
|
align?: ColumnAlign;
|
|
33
60
|
/**
|
|
@@ -42,15 +69,15 @@ export interface TableRow {
|
|
|
42
69
|
/** 行身份。 */
|
|
43
70
|
key: string;
|
|
44
71
|
/** 已格式化的显示值;`null`(或缺这个键)渲染成 `—`,不补 0。 */
|
|
45
|
-
cells: Record<string, string | null
|
|
72
|
+
cells: Readonly<Record<string, string | null>>;
|
|
46
73
|
/** 带上就多一列 attempt:web 面链到证据室,text 面列出 locator。 */
|
|
47
74
|
locator?: AttemptLocator;
|
|
48
75
|
}
|
|
49
76
|
export interface TableProps {
|
|
50
|
-
/**
|
|
51
|
-
columns: TableColumn[];
|
|
77
|
+
/** 非空列定义;数组顺序即渲染顺序。 */
|
|
78
|
+
columns: readonly [TableColumn, ...TableColumn[]];
|
|
52
79
|
/** 行数据;数组顺序即渲染顺序,组件不重排也不过滤。 */
|
|
53
|
-
rows: TableRow[];
|
|
80
|
+
rows: readonly TableRow[];
|
|
54
81
|
/** 组件自带文案(attempt 表头、丢列提示)的语言;省略时随宿主。 */
|
|
55
82
|
locale?: ReportLocale;
|
|
56
83
|
/** web 面挂到 `<table>` 上。 */
|