niceeval 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +317 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +178 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +618 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +32 -0
- package/dist/report/format.js +126 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +83 -0
- package/dist/report/locale.js +160 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +20 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +10 -0
- package/dist/report/react/ExperimentList.js +15 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +100 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +393 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +503 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +420 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/annotated-source.d.ts +61 -0
- package/dist/results/attempt-evidence.d.ts +69 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +5 -5
- package/docs-site/zh/concepts/assert.mdx +8 -8
- package/docs-site/zh/concepts/evals.mdx +5 -5
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +2 -2
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +1 -1
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +35 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +11 -11
- package/docs-site/zh/guides/viewing-results.mdx +199 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +2 -2
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +12 -12
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +7 -1
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +54 -2
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +34 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +643 -0
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +223 -50
- package/src/report/compute.ts +358 -53
- package/src/report/dual-render.test.tsx +1125 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +50 -2
- package/src/report/index.ts +48 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +206 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +115 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +108 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +136 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +129 -0
- package/src/report/react/fixtures.ts +163 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +22 -6
- package/src/report/react/render.test.tsx +164 -31
- package/src/report/react/styles.css +405 -213
- package/src/report/report.test.ts +403 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +303 -95
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +196 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +125 -94
- package/src/show/render.ts +660 -160
- package/src/show/show.test.ts +472 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/report/report.ts
CHANGED
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
// defineReport:一份报告 = 一个报告文件(默认导出),两扇门(show / view)共用。
|
|
2
|
-
//
|
|
2
|
+
// 宿主打开结果目录、按官方口径挑好 Selection、注入上下文;报告函数折数据、摆积木,
|
|
3
3
|
// 返回一棵组件树。计算全部发生在报告函数体里(读句柄、await 折数据只在这里合法);
|
|
4
|
-
// 渲染面是纯同步函数 —— 可达百 MB
|
|
4
|
+
// 渲染面是纯同步函数 —— 可达百 MB 的 artifact 永远不进渲染路径。
|
|
5
5
|
//
|
|
6
6
|
// renderReportToText 是 text 宿主(show)的装载入口;web 宿主(view)的
|
|
7
7
|
// renderReportToStaticHtml 在 ./web.ts(那一侧才 import react-dom)。宿主接线是下一波,
|
|
8
8
|
// 这两个入口先以内部函数的身份可独立测试。
|
|
9
9
|
|
|
10
|
-
import type { Results, Selection } from "../results/
|
|
10
|
+
import type { Results, Selection, SelectionWarning } from "../results/types.ts";
|
|
11
11
|
import {
|
|
12
12
|
createTextContext,
|
|
13
13
|
renderNodeToText,
|
|
14
|
+
resolveReportTree,
|
|
14
15
|
validateReportTree,
|
|
15
16
|
type ReportNode,
|
|
16
17
|
type TextRenderOptions,
|
|
17
18
|
} from "./tree.ts";
|
|
18
|
-
import {
|
|
19
|
+
import type { ReportLocale } from "./locale.ts";
|
|
19
20
|
|
|
20
21
|
export interface ReportContext {
|
|
21
|
-
/**
|
|
22
|
+
/** 宿主按现刻水位规则挑好的 Selection:每个 experiment × eval 取跨快照合成的最新判定,外加结构化挑选
|
|
23
|
+
警告;show 的默认索引、view 默认报告与两者的 --report 使用同一选择口径。 */
|
|
22
24
|
selection: Selection;
|
|
23
|
-
/** 默认挑法不合口径时,全量数据自己挑(见 docs/results
|
|
25
|
+
/** 默认挑法不合口径时,全量数据自己挑(见 docs/feature/results/library.md)。 */
|
|
24
26
|
results: Results;
|
|
25
27
|
}
|
|
26
28
|
|
|
@@ -53,8 +55,18 @@ export function isReportDefinition(value: unknown): value is ReportDefinition {
|
|
|
53
55
|
}
|
|
54
56
|
|
|
55
57
|
/**
|
|
56
|
-
* text
|
|
57
|
-
*
|
|
58
|
+
* 挑选警告的 text 形态:每条渲染好的 message 前缀 "! ",一行一条(与 RunOverview /
|
|
59
|
+
* overviewText 里 warnings 的 "! <message>" 约定一致)。宿主级前置块——不依赖报告是否
|
|
60
|
+
* 摆了 RunOverview,裸跑 / --report 都在报告顶上如实报残缺,不静默。
|
|
61
|
+
*/
|
|
62
|
+
function renderSelectionWarningsText(warnings: SelectionWarning[], _locale: ReportLocale): string {
|
|
63
|
+
return warnings.map((w) => `! ${w.message}`).join("\n");
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* text 宿主的装载语义:build → 渲染前解析数据组件(唯一的 await 边界)→ 树校验 → 遍历渲染
|
|
68
|
+
* text 面;Selection 有挑选警告时在报告顶部前置一块 "! <message>";报告树里的 RunOverview
|
|
69
|
+
* 已经渲染同一条时不重复。不需要 react-dom。
|
|
58
70
|
*/
|
|
59
71
|
export async function renderReportToText(
|
|
60
72
|
definition: ReportDefinition,
|
|
@@ -62,8 +74,14 @@ export async function renderReportToText(
|
|
|
62
74
|
options?: TextRenderOptions,
|
|
63
75
|
): Promise<string> {
|
|
64
76
|
const node = await definition.build(ctx);
|
|
65
|
-
|
|
66
|
-
|
|
77
|
+
const resolved = await resolveReportTree(node);
|
|
78
|
+
validateReportTree(resolved);
|
|
67
79
|
const textCtx = createTextContext(options);
|
|
68
|
-
|
|
80
|
+
const body = renderNodeToText(resolved, textCtx);
|
|
81
|
+
// RunOverview can render the same Selection warnings as part of the user tree. Keep the
|
|
82
|
+
// host guarantee for reports that omit it, while never printing an identical warning twice.
|
|
83
|
+
const missingWarnings = ctx.selection.warnings.filter((warning) => !body.includes(`! ${warning.message}`));
|
|
84
|
+
return missingWarnings.length > 0
|
|
85
|
+
? [renderSelectionWarningsText(missingWarnings, textCtx.locale), body].join("\n\n")
|
|
86
|
+
: body;
|
|
69
87
|
}
|
package/src/report/text/faces.ts
CHANGED
|
@@ -2,10 +2,16 @@
|
|
|
2
2
|
// 输出形态照 docs-site/zh/guides/report-components.mdx 的示例块;与 web 面共守
|
|
3
3
|
// 诚实契约:排序随 better、samples < total 角标、缺数据 — 不补 0、截断报剩余。
|
|
4
4
|
// 零 react、零 IO、纯同步 —— 这是 text 宿主不需要 react-dom 的那一半。
|
|
5
|
+
// chrome 文案(注脚、verdict 词、截断提示)经 ctx.locale 查 locale 字典,
|
|
6
|
+
// 默认 en 与历史输出逐字一致;数据(display、键、warnings message)不本地化。
|
|
5
7
|
|
|
6
8
|
import type {
|
|
7
|
-
|
|
9
|
+
AttemptListItem,
|
|
8
10
|
DeltaData,
|
|
11
|
+
EvalListItem,
|
|
12
|
+
ExperimentListEvalRow,
|
|
13
|
+
ExperimentListItem,
|
|
14
|
+
GroupSummaryData,
|
|
9
15
|
LineData,
|
|
10
16
|
MatrixData,
|
|
11
17
|
MetricColumn,
|
|
@@ -15,12 +21,28 @@ import type {
|
|
|
15
21
|
TableData,
|
|
16
22
|
} from "../types.ts";
|
|
17
23
|
import type { TextContext } from "../tree.ts";
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
24
|
+
import type { TableColumn, TableRow } from "../primitives.tsx";
|
|
25
|
+
import {
|
|
26
|
+
attemptItemReason,
|
|
27
|
+
capabilityBadge,
|
|
28
|
+
formatDurationMs,
|
|
29
|
+
formatMetricValue,
|
|
30
|
+
formatPlainNumber,
|
|
31
|
+
formatUSD,
|
|
32
|
+
verdictMark,
|
|
33
|
+
} from "../format.ts";
|
|
34
|
+
import { countText, localeText, resolveMetricLabel, type ReportLocale } from "../locale.ts";
|
|
35
|
+
import { indentBlock, padDisplay, textBar, wrapDisplay } from "./layout.ts";
|
|
36
|
+
import { renderTableText } from "./table.ts";
|
|
20
37
|
import { renderCharPlot, renderCoordinateTable, type PlotPoint } from "./plot.ts";
|
|
21
38
|
|
|
22
39
|
const MISSING_MARK = "—";
|
|
23
40
|
|
|
41
|
+
/** 缺数据文案随 locale(en = "no data",与 MISSING_TEXT 一致)。 */
|
|
42
|
+
function missingText(locale: ReportLocale): string {
|
|
43
|
+
return localeText(locale, "cell.missing");
|
|
44
|
+
}
|
|
45
|
+
|
|
24
46
|
/** 格子的文本形态:缺数据 —,覆盖不全带 samples/total 角标。 */
|
|
25
47
|
export function cellText(cell: { value: number | null; display: string; samples: number; total: number }): string {
|
|
26
48
|
if (cell.value === null) return MISSING_MARK;
|
|
@@ -29,23 +51,27 @@ export function cellText(cell: { value: number | null; display: string; samples:
|
|
|
29
51
|
|
|
30
52
|
// ───────────────────────── RunOverview ─────────────────────────
|
|
31
53
|
|
|
32
|
-
export function overviewText(data: OverviewData): string {
|
|
54
|
+
export function overviewText(data: OverviewData, ctx: TextContext): string {
|
|
33
55
|
const { totals, snapshots } = data;
|
|
56
|
+
const locale = ctx.locale;
|
|
34
57
|
const runs = new Set(snapshots.map((s) => s.startedAt)).size;
|
|
35
58
|
const latest = snapshots.map((s) => s.startedAt).sort().at(-1);
|
|
36
59
|
const head = [
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
60
|
+
countText(locale, "overview.experiments", snapshots.length),
|
|
61
|
+
localeText(locale, "overview.evalsCount", { n: totals.evals }),
|
|
62
|
+
localeText(locale, "overview.attemptsCount", { n: totals.attempts }),
|
|
63
|
+
// 通过率只渲染 computeCell 算好的同一个 MetricCell(cellText 复用缺数据/coverage 角标语义),
|
|
64
|
+
// 不从 passed/failed/errored 现场重算——与 web 面 RunOverview 同一份数据同一种读法。
|
|
65
|
+
`${localeText(locale, "overview.passRate")} ${cellText(totals.passRate)}`,
|
|
66
|
+
countText(locale, "composedFrom", runs),
|
|
67
|
+
...(latest ? [localeText(locale, "latestRun", { run: latest })] : []),
|
|
42
68
|
].join(" · ");
|
|
43
69
|
const tallies = [
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
totals.costUSD === null ?
|
|
70
|
+
`${localeText(locale, "verdict.passed")} ${totals.passed}`,
|
|
71
|
+
`${localeText(locale, "verdict.failed")} ${totals.failed}`,
|
|
72
|
+
`${localeText(locale, "verdict.errored")} ${totals.errored}`,
|
|
73
|
+
`${localeText(locale, "verdict.skipped")} ${totals.skipped}`,
|
|
74
|
+
totals.costUSD === null ? missingText(locale) : formatUSD(totals.costUSD),
|
|
49
75
|
formatDurationMs(totals.durationMs),
|
|
50
76
|
].join(" · ");
|
|
51
77
|
const lines = [head, tallies];
|
|
@@ -53,23 +79,93 @@ export function overviewText(data: OverviewData): string {
|
|
|
53
79
|
return lines.join("\n");
|
|
54
80
|
}
|
|
55
81
|
|
|
82
|
+
// ───────────────────────── GroupSummary ─────────────────────────
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* 一至两行:头行是通过率(GroupSummaryData.passRate.display,不重算比例)+ experiment/eval 数 +
|
|
86
|
+
* failed(+ errored,非零才列,与旧 GroupSelector 卡片一致)+ 总成本;第二行(有则加)是最后运行时间。
|
|
87
|
+
* 不依赖固定网格宽度,窄终端自然换行。
|
|
88
|
+
*/
|
|
89
|
+
export function groupSummaryText(data: GroupSummaryData, ctx: TextContext): string {
|
|
90
|
+
const locale = ctx.locale;
|
|
91
|
+
const head = [
|
|
92
|
+
`${localeText(locale, "overview.passRate")} ${cellText(data.passRate)}`,
|
|
93
|
+
countText(locale, "overview.experiments", data.experiments),
|
|
94
|
+
localeText(locale, "overview.evalsCount", { n: data.evals }),
|
|
95
|
+
`${localeText(locale, "verdict.failed")} ${data.verdicts.failed}`,
|
|
96
|
+
...(data.verdicts.errored > 0 ? [`${localeText(locale, "verdict.errored")} ${data.verdicts.errored}`] : []),
|
|
97
|
+
data.totalCostUSD === null ? missingText(locale) : formatUSD(data.totalCostUSD),
|
|
98
|
+
].join(" · ");
|
|
99
|
+
const lines = [head];
|
|
100
|
+
if (data.lastRunAt) lines.push(localeText(locale, "latestRun", { run: data.lastRunAt }));
|
|
101
|
+
return lines.join("\n");
|
|
102
|
+
}
|
|
103
|
+
|
|
56
104
|
// ───────────────────────── MetricTable ─────────────────────────
|
|
57
105
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
106
|
+
/** verdict 计票的紧凑文案("3 passed / 1 failed"):非零判定逐个列,全部为零如实 —。 */
|
|
107
|
+
export function verdictTallyText(
|
|
108
|
+
verdicts: NonNullable<NonNullable<TableData["rows"][number]["meta"]>["verdicts"]>,
|
|
109
|
+
locale: ReportLocale,
|
|
110
|
+
): string {
|
|
111
|
+
const parts: string[] = [];
|
|
112
|
+
for (const kind of ["passed", "failed", "errored", "skipped"] as const) {
|
|
113
|
+
if (verdicts[kind] > 0) parts.push(`${verdicts[kind]} ${localeText(locale, `verdict.${kind}`)}`);
|
|
114
|
+
}
|
|
115
|
+
return parts.length > 0 ? parts.join(" / ") : MISSING_MARK;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export function tableText(data: TableData, ctx: TextContext): string {
|
|
119
|
+
const locale = ctx.locale;
|
|
120
|
+
// meta 在场时补 Model / Agent / Verdicts 列(rows: "experiment" 的榜单 parity);
|
|
121
|
+
// 列序对齐 view 原生榜单:experiment、model、agent、指标列…、verdicts
|
|
122
|
+
const hasMeta = data.rows.some((row) => row.meta !== undefined);
|
|
123
|
+
const hasModel = data.rows.some((row) => row.meta?.model !== undefined);
|
|
124
|
+
const hasVerdicts = data.rows.some((row) => row.meta?.verdicts !== undefined);
|
|
125
|
+
const columns: TableColumn[] = [
|
|
126
|
+
{ key: "dimension", header: data.dimension },
|
|
127
|
+
...(hasMeta && hasModel ? [{ key: "model", header: localeText(locale, "table.model") }] : []),
|
|
128
|
+
...(hasMeta ? [{ key: "agent", header: localeText(locale, "table.agent") }] : []),
|
|
129
|
+
// 指标列的键用序号,不用 metric name —— 维度名、"model" 这些键都在同一个命名空间里,
|
|
130
|
+
// 指标恰好叫 "agent" 时不能把 meta 列顶掉。
|
|
131
|
+
...data.columns.map((c, i) => ({ key: `metric${i}`, header: resolveMetricLabel(c.label, locale, c.key) })),
|
|
132
|
+
...(hasVerdicts ? [{ key: "verdicts", header: localeText(locale, "table.verdicts") }] : []),
|
|
133
|
+
];
|
|
134
|
+
const rows: TableRow[] = data.rows.map((row) => {
|
|
135
|
+
const cells: Record<string, string | null> = { dimension: row.key };
|
|
136
|
+
if (hasMeta && hasModel) cells.model = row.meta?.model ?? null;
|
|
137
|
+
if (hasMeta) cells.agent = row.meta?.agent ?? null;
|
|
138
|
+
data.columns.forEach((col, i) => {
|
|
63
139
|
const cell = (row.cells as Record<string, TableData["rows"][number]["cells"][string]>)[col.key];
|
|
64
|
-
|
|
65
|
-
})
|
|
66
|
-
|
|
67
|
-
|
|
140
|
+
cells[`metric${i}`] = cell ? cellText(cell) : null;
|
|
141
|
+
});
|
|
142
|
+
if (hasVerdicts) cells.verdicts = row.meta?.verdicts ? verdictTallyText(row.meta.verdicts, locale) : null;
|
|
143
|
+
return { key: row.key, cells };
|
|
144
|
+
});
|
|
145
|
+
const table = renderTableText({ columns, rows, locale }, ctx);
|
|
146
|
+
|
|
147
|
+
// rows: "experiment" 专属的行摘要(eval/attempt 数 + 最后运行时间):表格列已经挤满
|
|
148
|
+
// dimension/model/agent/指标/verdicts,这几个数字挤进行键下面单独一行,与 web 面
|
|
149
|
+
// 「行键下的小 sub-line」同一份信息、同一种「不进列」的取舍。
|
|
150
|
+
const metaLines: string[] = [];
|
|
151
|
+
for (const row of data.rows) {
|
|
152
|
+
if (row.meta?.evals === undefined) continue;
|
|
153
|
+
const parts = [localeText(locale, "overview.evalsCount", { n: row.meta.evals })];
|
|
154
|
+
if (row.meta.attempts !== undefined && row.meta.attempts > row.meta.evals) {
|
|
155
|
+
parts.push(localeText(locale, "overview.attemptsCount", { n: row.meta.attempts }));
|
|
156
|
+
}
|
|
157
|
+
if (row.meta.lastRunAt) parts.push(localeText(locale, "latestRun", { run: row.meta.lastRunAt }));
|
|
158
|
+
metaLines.push(` ${row.key}: ${parts.join(" · ")}`);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const blocks = [table, ...(metaLines.length > 0 ? [metaLines.join("\n")] : [])];
|
|
162
|
+
return blocks.join("\n\n");
|
|
68
163
|
}
|
|
69
164
|
|
|
70
165
|
// ───────────────────────── MetricMatrix ─────────────────────────
|
|
71
166
|
|
|
72
|
-
export function matrixText(data: MatrixData): string {
|
|
167
|
+
export function matrixText(data: MatrixData, ctx: TextContext): string {
|
|
168
|
+
// 表体全是维度键与 display,没有 chrome 文案;"next:" 是命令提示,不本地化。
|
|
73
169
|
const rowKeys: string[] = [];
|
|
74
170
|
const columnKeys: string[] = [];
|
|
75
171
|
const byPosition = new Map<string, MatrixData["cells"][number]["cell"]>();
|
|
@@ -78,15 +174,19 @@ export function matrixText(data: MatrixData): string {
|
|
|
78
174
|
if (!columnKeys.includes(entry.column)) columnKeys.push(entry.column);
|
|
79
175
|
byPosition.set(JSON.stringify([entry.row, entry.column]), entry.cell);
|
|
80
176
|
}
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
177
|
+
const columns: TableColumn[] = [
|
|
178
|
+
{ key: "dimension", header: data.rows },
|
|
179
|
+
...columnKeys.map((column, i) => ({ key: `column${i}`, header: column })),
|
|
180
|
+
];
|
|
181
|
+
const rows: TableRow[] = rowKeys.map((row) => {
|
|
182
|
+
const cells: Record<string, string | null> = { dimension: row };
|
|
183
|
+
columnKeys.forEach((column, i) => {
|
|
85
184
|
const cell = byPosition.get(JSON.stringify([row, column]));
|
|
86
|
-
|
|
87
|
-
})
|
|
88
|
-
|
|
89
|
-
|
|
185
|
+
cells[`column${i}`] = cell ? cellText(cell) : null; // 稀疏格子在文本里以 — 呈现,不编数
|
|
186
|
+
});
|
|
187
|
+
return { key: row, cells };
|
|
188
|
+
});
|
|
189
|
+
const table = renderTableText({ columns, rows, locale: ctx.locale }, ctx);
|
|
90
190
|
|
|
91
191
|
// 下钻命令:行维度是 eval 时,指向最值得看的一行(先挑有缺格的,再挑按 better 最差的)
|
|
92
192
|
if (data.rows !== "eval" || rowKeys.length === 0) return table;
|
|
@@ -166,54 +266,75 @@ export function barsText(data: MatrixData): string {
|
|
|
166
266
|
|
|
167
267
|
// ───────────────────────── Scoreboard ─────────────────────────
|
|
168
268
|
|
|
169
|
-
export function scoreboardText(data: ScoreboardData): string {
|
|
269
|
+
export function scoreboardText(data: ScoreboardData, ctx: TextContext): string {
|
|
270
|
+
const locale = ctx.locale;
|
|
170
271
|
const subjectKeys: string[] = [];
|
|
171
272
|
for (const row of data.rows) {
|
|
172
273
|
for (const subject of row.subjects) {
|
|
173
274
|
if (!subjectKeys.includes(subject.key)) subjectKeys.push(subject.key);
|
|
174
275
|
}
|
|
175
276
|
}
|
|
176
|
-
const
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
277
|
+
const columns: TableColumn[] = [
|
|
278
|
+
{ key: "dimension", header: data.dimension },
|
|
279
|
+
{ key: "total", header: localeText(locale, "scoreboard.totalText") },
|
|
280
|
+
...subjectKeys.map((key, i) => ({ key: `subject${i}`, header: key })),
|
|
281
|
+
];
|
|
282
|
+
const rows: TableRow[] = data.rows.map((row) => {
|
|
283
|
+
const cells: Record<string, string | null> = {
|
|
284
|
+
dimension: row.key,
|
|
285
|
+
total: `${row.total.display}/${data.fullMarks}`,
|
|
286
|
+
};
|
|
287
|
+
subjectKeys.forEach((key, i) => {
|
|
181
288
|
const subject = row.subjects.find((s) => s.key === key);
|
|
182
|
-
if (!subject)
|
|
289
|
+
if (!subject) {
|
|
290
|
+
cells[`subject${i}`] = null;
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
183
293
|
const score = `${formatPlainNumber(subject.earned)}/${formatPlainNumber(subject.possible)}`;
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
294
|
+
cells[`subject${i}`] =
|
|
295
|
+
subject.missing > 0
|
|
296
|
+
? `${score} ${localeText(locale, "scoreboard.missingText", { n: subject.missing })}`
|
|
297
|
+
: score;
|
|
298
|
+
});
|
|
299
|
+
return { key: row.key, cells };
|
|
300
|
+
});
|
|
301
|
+
const table = renderTableText({ columns, rows, locale }, ctx);
|
|
188
302
|
if (data.weights.length === 0) return table;
|
|
189
303
|
// 实际生效的权重表 —— 成绩单可审计
|
|
190
304
|
const weights = data.weights.map((w) => `${w.prefix} ×${w.weight}`).join(" · ");
|
|
191
|
-
return `${table}\
|
|
305
|
+
return `${table}\n${localeText(locale, "scoreboard.weights")} ${weights} · ${localeText(locale, "scoreboard.othersWeight")}`;
|
|
192
306
|
}
|
|
193
307
|
|
|
194
308
|
// ───────────────────────── MetricScatter ─────────────────────────
|
|
195
309
|
|
|
196
310
|
const POINT_MARKS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
|
197
311
|
|
|
198
|
-
function axisLabel(col: MetricColumn): string {
|
|
199
|
-
return col.label;
|
|
312
|
+
function axisLabel(col: MetricColumn, locale: ReportLocale): string {
|
|
313
|
+
return resolveMetricLabel(col.label, locale, col.key);
|
|
200
314
|
}
|
|
201
315
|
|
|
202
316
|
export function scatterText(data: ScatterData, ctx: TextContext): string {
|
|
317
|
+
const locale = ctx.locale;
|
|
203
318
|
const drawable = data.rows.filter((r) => r.x.value !== null && r.y.value !== null);
|
|
204
319
|
const missing = data.rows.length - drawable.length;
|
|
205
320
|
const footnotes: string[] = [];
|
|
206
|
-
if (missing > 0) footnotes.push(
|
|
321
|
+
if (missing > 0) footnotes.push(countText(locale, "pointsMissing", missing));
|
|
207
322
|
|
|
323
|
+
const axes = { x: axisLabel(data.x, locale), y: axisLabel(data.y, locale) };
|
|
324
|
+
// 0 个可画点:x/y 指标没有可用数据(与 web 面同一事实)。
|
|
208
325
|
if (drawable.length === 0) {
|
|
209
|
-
return [
|
|
326
|
+
return [localeText(locale, "scatter.noData", axes), ...footnotes].join("\n");
|
|
327
|
+
}
|
|
328
|
+
// 恰好 1 个可画点:成本 × 通过率的比较至少要两个实验,单点不成图。
|
|
329
|
+
if (drawable.length === 1) {
|
|
330
|
+
return [localeText(locale, "scatter.needTwo", axes), ...footnotes].join("\n");
|
|
210
331
|
}
|
|
211
332
|
|
|
212
333
|
// 点太密排不下时降级为坐标表,不硬挤
|
|
213
334
|
if (drawable.length > POINT_MARKS.length || ctx.width < 44) {
|
|
214
335
|
const table = renderCoordinateTable(
|
|
215
336
|
drawable.map((r) => ({ key: r.key, x: r.x.display, y: r.y.display })),
|
|
216
|
-
{ key: data.points, x: axisLabel(data.x), y: axisLabel(data.y) },
|
|
337
|
+
{ key: data.points, x: axisLabel(data.x, locale), y: axisLabel(data.y, locale) },
|
|
217
338
|
);
|
|
218
339
|
return [table, ...footnotes].join("\n");
|
|
219
340
|
}
|
|
@@ -238,26 +359,27 @@ export function scatterText(data: ScatterData, ctx: TextContext): string {
|
|
|
238
359
|
width: ctx.width,
|
|
239
360
|
points,
|
|
240
361
|
lines: [...bySeries.values()].filter((l) => l.length > 1),
|
|
241
|
-
xLabel: `${axisLabel(data.x)}${invertX ?
|
|
242
|
-
yLabel: axisLabel(data.y),
|
|
362
|
+
xLabel: `${axisLabel(data.x, locale)}${invertX ? ` ${localeText(locale, "scatter.axisReversed")}` : ""}`,
|
|
363
|
+
yLabel: axisLabel(data.y, locale),
|
|
243
364
|
formatX: (v) => formatMetricValue(v, data.x.unit),
|
|
244
365
|
formatY: (v) => formatMetricValue(v, data.y.unit),
|
|
245
366
|
invertX,
|
|
246
367
|
invertY: data.y.better === "lower",
|
|
247
368
|
});
|
|
248
369
|
const legend = drawable.map((r, i) => `${POINT_MARKS[i]} ${r.key}`).join(" ");
|
|
249
|
-
return [plot, "",
|
|
370
|
+
return [plot, "", localeText(locale, "scatter.betterUpperRight"), legend, ...footnotes].join("\n");
|
|
250
371
|
}
|
|
251
372
|
|
|
252
373
|
// ───────────────────────── MetricLine ─────────────────────────
|
|
253
374
|
|
|
254
375
|
export function lineText(data: LineData, ctx: TextContext): string {
|
|
376
|
+
const locale = ctx.locale;
|
|
255
377
|
const drawable = data.rows.filter((r) => r.x !== null && r.y.value !== null);
|
|
256
378
|
const missing = data.rows.length - drawable.length;
|
|
257
379
|
const footnotes: string[] = [];
|
|
258
|
-
if (missing > 0) footnotes.push(
|
|
380
|
+
if (missing > 0) footnotes.push(countText(locale, "pointsMissing", missing));
|
|
259
381
|
|
|
260
|
-
if (drawable.length === 0) return [
|
|
382
|
+
if (drawable.length === 0) return [missingText(locale), ...footnotes].join("\n");
|
|
261
383
|
|
|
262
384
|
// 系列 → 字母;无系列 = 单系列
|
|
263
385
|
const seriesKeys: string[] = [];
|
|
@@ -269,7 +391,7 @@ export function lineText(data: LineData, ctx: TextContext): string {
|
|
|
269
391
|
if (seriesKeys.length > POINT_MARKS.length || ctx.width < 44) {
|
|
270
392
|
const table = renderCoordinateTable(
|
|
271
393
|
drawable.map((r) => ({ key: r.series ? `${r.key} (${r.series})` : r.key, x: r.xDisplay, y: r.y.display })),
|
|
272
|
-
{ key: "experiment", x: data.x.label, y: axisLabel(data.y) },
|
|
394
|
+
{ key: "experiment", x: data.x.label, y: axisLabel(data.y, locale) },
|
|
273
395
|
);
|
|
274
396
|
return [table, ...footnotes].join("\n");
|
|
275
397
|
}
|
|
@@ -292,65 +414,151 @@ export function lineText(data: LineData, ctx: TextContext): string {
|
|
|
292
414
|
points,
|
|
293
415
|
lines: lines.filter((l) => l.length > 1),
|
|
294
416
|
xLabel: data.x.label,
|
|
295
|
-
yLabel: axisLabel(data.y),
|
|
417
|
+
yLabel: axisLabel(data.y, locale),
|
|
296
418
|
formatX: (v) => formatMetricValue(v, data.x.unit),
|
|
297
419
|
formatY: (v) => formatMetricValue(v, data.y.unit),
|
|
298
420
|
invertY: data.y.better === "lower",
|
|
299
421
|
});
|
|
300
422
|
const legend = seriesKeys
|
|
301
|
-
.map((key, i) => `${POINT_MARKS[i]} ${key === "" ? data.y
|
|
423
|
+
.map((key, i) => `${POINT_MARKS[i]} ${key === "" ? axisLabel(data.y, locale) : key}`)
|
|
302
424
|
.join(" ");
|
|
303
425
|
return [plot, "", legend, ...footnotes].join("\n");
|
|
304
426
|
}
|
|
305
427
|
|
|
306
428
|
// ───────────────────────── DeltaTable ─────────────────────────
|
|
307
429
|
|
|
308
|
-
export function deltaText(data: DeltaData): string {
|
|
309
|
-
const
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
430
|
+
export function deltaText(data: DeltaData, ctx: TextContext): string {
|
|
431
|
+
const columns: TableColumn[] = [
|
|
432
|
+
{ key: "pair", header: "pair" },
|
|
433
|
+
...data.columns.map((c, i) => ({ key: `metric${i}`, header: resolveMetricLabel(c.label, ctx.locale, c.key) })),
|
|
434
|
+
];
|
|
435
|
+
const rows: TableRow[] = data.rows.map((row) => {
|
|
436
|
+
const cells: Record<string, string | null> = { pair: row.key };
|
|
437
|
+
data.columns.forEach((col, i) => {
|
|
313
438
|
const cell = (row.cells as Record<string, DeltaData["rows"][number]["cells"][string]>)[col.key];
|
|
314
|
-
if (!cell)
|
|
439
|
+
if (!cell) {
|
|
440
|
+
cells[`metric${i}`] = null;
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
315
443
|
const a = cell.a.value === null ? MISSING_MARK : cell.a.display;
|
|
316
444
|
const b = cell.b.value === null ? MISSING_MARK : cell.b.display;
|
|
317
|
-
|
|
318
|
-
})
|
|
319
|
-
|
|
320
|
-
|
|
445
|
+
cells[`metric${i}`] = `${a} → ${b} ${cell.display}`;
|
|
446
|
+
});
|
|
447
|
+
return { key: row.key, cells };
|
|
448
|
+
});
|
|
449
|
+
return renderTableText({ columns, rows, locale: ctx.locale }, ctx);
|
|
321
450
|
}
|
|
322
451
|
|
|
323
|
-
// ─────────────────────────
|
|
452
|
+
// ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
|
|
453
|
+
//
|
|
454
|
+
// 三面共用的紧凑标记:`locator✓[E,X,⏱]`(判定符紧跟 locator,证据能力方括号紧跟判定符,
|
|
455
|
+
// 中间不留空格)——docs-site/zh/guides/report-components.mdx「终端输出形成反馈闭环」定的形态。
|
|
456
|
+
// ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的原因/耗时摘要,不重复整段
|
|
457
|
+
// niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼 `niceeval show <locator>`——
|
|
458
|
+
// 命令模板只在 AttemptList(叶子层)展示完整断言明细时才值得,不在中间层重复。
|
|
324
459
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
460
|
+
function locatorBadge(item: { locator: string; verdict: AttemptListItem["verdict"]; capabilities: AttemptListItem["capabilities"] }): string {
|
|
461
|
+
return `${item.locator}${verdictMark(item.verdict)}${capabilityBadge(item.capabilities)}`;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// ── ExperimentList ──
|
|
465
|
+
|
|
466
|
+
function experimentListEvalLine(row: ExperimentListEvalRow): string {
|
|
467
|
+
const badges = row.attempts.map(locatorBadge).join(" ");
|
|
468
|
+
const trailer =
|
|
469
|
+
row.verdict === "passed"
|
|
470
|
+
? [formatDurationMs(row.duration.value ?? 0), row.cost.value === null ? undefined : formatUSD(row.cost.value)]
|
|
471
|
+
.filter((s): s is string => s !== undefined)
|
|
472
|
+
.join(" · ")
|
|
473
|
+
: (row.reason ?? "");
|
|
474
|
+
return ` ${verdictMark(row.verdict)} ${row.evalId} ${badges} ${trailer}`;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
export function experimentListText(items: ExperimentListItem[], ctx: TextContext): string {
|
|
478
|
+
const locale = ctx.locale;
|
|
479
|
+
if (items.length === 0) return localeText(locale, "attemptList.empty");
|
|
480
|
+
const blocks = items.map((item) => {
|
|
481
|
+
const identity = item.model ? `${item.experimentId} · ${item.agent} · ${item.model}` : `${item.experimentId} · ${item.agent}`;
|
|
482
|
+
const summary = [
|
|
483
|
+
`${localeText(locale, "overview.passRate")} ${cellText(item.passRate)}`,
|
|
484
|
+
verdictTallyText(item.verdicts, locale),
|
|
485
|
+
localeText(locale, "overview.attemptsCount", { n: item.attempts }),
|
|
486
|
+
formatDurationMs(item.duration.value ?? 0),
|
|
487
|
+
item.cost.value === null ? missingText(locale) : formatUSD(item.cost.value),
|
|
335
488
|
].join(" · ");
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
489
|
+
const evalLines = item.evalRows.map(experimentListEvalLine);
|
|
490
|
+
return [identity, ` ${summary}`, ...evalLines].join("\n");
|
|
491
|
+
});
|
|
492
|
+
return blocks.join("\n\n");
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// ── EvalList ──
|
|
496
|
+
|
|
497
|
+
function evalListAttemptLine(item: AttemptListItem): string {
|
|
498
|
+
const reason = attemptItemReason(item);
|
|
499
|
+
return ` ${locatorBadge(item)}${reason ? ` · ${reason}` : ""}`;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
export function evalListText(items: EvalListItem[], ctx: TextContext): string {
|
|
503
|
+
const locale = ctx.locale;
|
|
504
|
+
if (items.length === 0) return localeText(locale, "attemptList.empty");
|
|
505
|
+
const blocks = items.map((item) => {
|
|
506
|
+
const identity = `${item.evalId} · ${item.experimentId} · ${localeText(locale, `verdict.${item.verdict}`)}`;
|
|
507
|
+
const summary = [
|
|
508
|
+
localeText(locale, "attemptList.score", { score: cellText(item.score) }),
|
|
509
|
+
localeText(locale, "overview.attemptsCount", { n: item.attempts.length }),
|
|
510
|
+
`${formatDurationMs(item.duration.value ?? 0)} avg`,
|
|
511
|
+
item.cost.value === null ? `${missingText(locale)} avg` : `${formatUSD(item.cost.value)} avg`,
|
|
512
|
+
].join(" · ");
|
|
513
|
+
const attemptLines = item.attempts.map(evalListAttemptLine);
|
|
514
|
+
return [identity, ` ${summary}`, ...attemptLines].join("\n");
|
|
515
|
+
});
|
|
516
|
+
return blocks.join("\n\n");
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
// ── AttemptList ──
|
|
520
|
+
|
|
521
|
+
/** 一个 AttemptListItem 的完整 text 卡片:判定符 + locator + 身份 + 耗时/成本 + 证据能力,
|
|
522
|
+
* 然后逐条断言(gate 与 soft 都列,与 web 面的 AttemptRow 同一份材料)。 */
|
|
523
|
+
function attemptListItemText(item: AttemptListItem, ctx: TextContext, locale: ReportLocale): string {
|
|
524
|
+
const head = [
|
|
525
|
+
`${verdictMark(item.verdict)} ${item.locator}`,
|
|
526
|
+
item.evalId,
|
|
527
|
+
item.experimentId,
|
|
528
|
+
formatDurationMs(item.durationMs),
|
|
529
|
+
...(item.costUSD !== undefined ? [formatUSD(item.costUSD)] : []),
|
|
530
|
+
...(capabilityBadge(item.capabilities) ? [capabilityBadge(item.capabilities)] : []),
|
|
531
|
+
].join(" · ");
|
|
532
|
+
const lines = [head];
|
|
533
|
+
if (item.error) {
|
|
534
|
+
lines.push(indentBlock(wrapDisplay(item.error, ctx.width - 4).join("\n"), " "));
|
|
350
535
|
}
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
536
|
+
for (const assertion of item.assertions) {
|
|
537
|
+
const scoreText =
|
|
538
|
+
assertion.threshold !== undefined
|
|
539
|
+
? `${formatPlainNumber(assertion.score)}/${formatPlainNumber(assertion.threshold)}`
|
|
540
|
+
: formatPlainNumber(assertion.score);
|
|
541
|
+
lines.push(
|
|
542
|
+
` ${assertion.severity} ${assertion.name} · ${localeText(locale, `verdict.${assertion.passed ? "passed" : "failed"}`)}${assertion.severity === "soft" ? ` ${scoreText}` : ""}`,
|
|
543
|
+
);
|
|
544
|
+
if (assertion.detail) lines.push(indentBlock(wrapDisplay(assertion.detail, ctx.width - 4).join("\n"), " "));
|
|
545
|
+
if (assertion.evidence) {
|
|
546
|
+
const limit = Math.max(240, ctx.width * 5);
|
|
547
|
+
const evidence =
|
|
548
|
+
assertion.evidence.length <= limit
|
|
549
|
+
? assertion.evidence
|
|
550
|
+
: `${assertion.evidence.slice(0, limit)}… (${assertion.evidence.length - limit} more chars; open ${item.locator} for full evidence)`;
|
|
551
|
+
lines.push(indentBlock(wrapDisplay(evidence, ctx.width - 6).join("\n"), " "));
|
|
552
|
+
}
|
|
354
553
|
}
|
|
355
554
|
return lines.join("\n");
|
|
356
555
|
}
|
|
556
|
+
|
|
557
|
+
export function attemptListText(items: AttemptListItem[], total: number | undefined, ctx: TextContext): string {
|
|
558
|
+
const locale = ctx.locale;
|
|
559
|
+
if (items.length === 0) return localeText(locale, "attemptList.empty");
|
|
560
|
+
const blocks = items.map((item) => attemptListItemText(item, ctx, locale));
|
|
561
|
+
const remaining = (total ?? items.length) - items.length;
|
|
562
|
+
if (remaining > 0) blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
|
|
563
|
+
return blocks.join("\n\n");
|
|
564
|
+
}
|