niceeval 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +321 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
- package/dist/report/built-ins/experiment-comparison.js +13 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +180 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +600 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +26 -0
- package/dist/report/format.js +109 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +94 -0
- package/dist/report/locale.js +183 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +19 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +11 -0
- package/dist/report/react/ExperimentList.js +67 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +96 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +398 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +541 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +408 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +7 -7
- package/docs-site/zh/concepts/assert.mdx +19 -18
- package/docs-site/zh/concepts/evals.mdx +12 -11
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +3 -3
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +4 -2
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +4 -4
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +61 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
- package/docs-site/zh/guides/viewing-results.mdx +200 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +4 -3
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +15 -13
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +9 -3
- package/docs-site/zh/reference/expect.mdx +13 -8
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +59 -3
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +33 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +640 -0
- package/src/report/built-ins/experiment-comparison.tsx +19 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +225 -50
- package/src/report/compute.ts +335 -54
- package/src/report/dual-render.test.tsx +1129 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +35 -2
- package/src/report/index.ts +47 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +229 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +113 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +220 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +126 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +168 -0
- package/src/report/react/fixtures.ts +169 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +21 -6
- package/src/report/react/render.test.tsx +180 -31
- package/src/report/react/styles.css +432 -213
- package/src/report/report.test.ts +398 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +344 -96
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +184 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +119 -95
- package/src/show/render.ts +462 -159
- package/src/show/show.test.ts +471 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
|
@@ -1,24 +1,32 @@
|
|
|
1
|
-
// niceeval view
|
|
1
|
+
// niceeval view 的报告槽与宿主组合语义(docs/feature/reports/architecture.md「Selection 是计算入口」
|
|
2
2
|
// 与裁决记录 6;公开行为准绳 docs-site/zh/guides/viewing-results.mdx / custom-reports.mdx)。
|
|
3
3
|
// 覆盖:
|
|
4
|
-
// - 组合语义与 show
|
|
4
|
+
// - 组合语义与 show 对齐:位置前缀收窄报告槽 Selection、--experiment 过滤、匹配不到直说;
|
|
5
5
|
// - 单文件模式共存:存在的文件路径 → 单文件模式,目录报错直说走 --run,其余按 eval 前缀;
|
|
6
|
-
// -
|
|
7
|
-
//
|
|
6
|
+
// - 报告槽恒在:裸跑填充 ExperimentComparison,--report 整槽替换;en / zh-CN 双语各渲染一遍;
|
|
7
|
+
// 裸跑 ≡ --report <re-export ExperimentComparison 的文件>(等价性);
|
|
8
|
+
// - --out 静态导出:index.html 含两个语言的报告块、官方样式与增强 runtime,报告块零 <script>;
|
|
8
9
|
// - dev server 装载语义:报告文件变更 → 下次装载整页重算(mtime cache-busting)。
|
|
10
|
+
//
|
|
11
|
+
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
12
|
+
// 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
|
|
9
13
|
|
|
10
14
|
import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises";
|
|
11
15
|
import { existsSync } from "node:fs";
|
|
12
16
|
import { tmpdir } from "node:os";
|
|
13
17
|
import { join, resolve } from "node:path";
|
|
14
18
|
import { afterEach, describe, expect, it } from "vitest";
|
|
15
|
-
|
|
19
|
+
// dist-sourced: this must be the exact class loadViewScan()/data.ts's loadReportFile() throws
|
|
20
|
+
// (see src/view/data.ts's comment) — a raw-src import would be a structurally-identical but
|
|
21
|
+
// `instanceof`-incompatible class.
|
|
22
|
+
import { ReportLoadError } from "../../dist/report/load.js";
|
|
16
23
|
import { ViewInputError, loadViewScan } from "./data.ts";
|
|
17
24
|
import { buildView, resolveViewInput } from "./index.ts";
|
|
18
25
|
import { runShow } from "../show/index.ts";
|
|
19
|
-
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type
|
|
26
|
+
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
|
|
20
27
|
|
|
21
28
|
const EXAM_REPORT = resolve(__dirname, "../../test/fixtures/report/exam-report.tsx");
|
|
29
|
+
const DEFAULT_REPORT_REEXPORT = resolve(__dirname, "../../test/fixtures/report/default-report-reexport.tsx");
|
|
22
30
|
|
|
23
31
|
const roots: string[] = [];
|
|
24
32
|
async function makeRoot(): Promise<string> {
|
|
@@ -30,66 +38,58 @@ afterEach(async () => {
|
|
|
30
38
|
await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
|
|
31
39
|
});
|
|
32
40
|
|
|
33
|
-
|
|
34
|
-
|
|
41
|
+
type AttemptFixture = Pick<EvalResult, "id" | "verdict"> & Partial<Pick<EvalResult, "attempt" | "durationMs" | "assertions">>;
|
|
42
|
+
|
|
43
|
+
function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}): AttemptFixture {
|
|
44
|
+
return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface SnapshotOpts {
|
|
48
|
+
experimentId: string;
|
|
49
|
+
agent?: string;
|
|
50
|
+
startedAt: string;
|
|
35
51
|
}
|
|
36
52
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
53
|
+
/** 写一份新布局快照:snapshot.json + 各 attempt 的 result.json。返回快照目录绝对路径。 */
|
|
54
|
+
async function writeSnapshot(
|
|
55
|
+
root: string,
|
|
56
|
+
expDirName: string,
|
|
57
|
+
snapDirName: string,
|
|
58
|
+
opts: SnapshotOpts,
|
|
59
|
+
results: AttemptFixture[],
|
|
60
|
+
): Promise<string> {
|
|
61
|
+
const dir = join(root, expDirName, snapDirName);
|
|
62
|
+
await mkdir(dir, { recursive: true });
|
|
63
|
+
const meta = {
|
|
40
64
|
format: RESULTS_FORMAT,
|
|
41
65
|
schemaVersion: RESULTS_SCHEMA_VERSION,
|
|
42
66
|
producer: { name: "niceeval", version: "0.4.6" },
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
failed: count("failed"),
|
|
48
|
-
skipped: count("skipped"),
|
|
49
|
-
errored: count("errored"),
|
|
50
|
-
durationMs: 60_000,
|
|
51
|
-
results,
|
|
52
|
-
...over,
|
|
67
|
+
experimentId: opts.experimentId,
|
|
68
|
+
agent: opts.agent ?? "bub",
|
|
69
|
+
startedAt: opts.startedAt,
|
|
70
|
+
completedAt: opts.startedAt,
|
|
53
71
|
};
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
72
|
+
await writeFile(join(dir, "snapshot.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
73
|
+
for (const r of results) {
|
|
74
|
+
const attemptDir = join(dir, r.id, `a${r.attempt ?? 0}`);
|
|
75
|
+
await mkdir(attemptDir, { recursive: true });
|
|
76
|
+
await writeFile(join(attemptDir, "result.json"), JSON.stringify(r, null, 2), "utf-8");
|
|
77
|
+
}
|
|
60
78
|
return dir;
|
|
61
79
|
}
|
|
62
80
|
|
|
63
81
|
/** 两个实验、weather 通过 + button 失败:报告槽/证据室/深链断言都要用到失败案例。 */
|
|
64
82
|
async function seedRoot(): Promise<string> {
|
|
65
83
|
const root = await makeRoot();
|
|
66
|
-
await
|
|
67
|
-
|
|
68
|
-
"
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
outcome: "failed",
|
|
76
|
-
startedAt: "2026-07-08T10:00:02.000Z",
|
|
77
|
-
assertions: [
|
|
78
|
-
{ name: 'fileChanged("Button.tsx")', severity: "gate", score: 0, passed: false },
|
|
79
|
-
],
|
|
80
|
-
}),
|
|
81
|
-
],
|
|
82
|
-
{ startedAt: "2026-07-08T10:00:00.000Z" },
|
|
83
|
-
),
|
|
84
|
-
);
|
|
85
|
-
await writeRun(
|
|
86
|
-
root,
|
|
87
|
-
"2026-07-09T10-00-00-000Z",
|
|
88
|
-
summaryOf(
|
|
89
|
-
[res({ id: "weather/brooklyn", experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-09T10:00:01.000Z" })],
|
|
90
|
-
{ agent: "codex", startedAt: "2026-07-09T10:00:00.000Z" },
|
|
91
|
-
),
|
|
92
|
-
);
|
|
84
|
+
await writeSnapshot(root, "compare_bub", "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", agent: "bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
85
|
+
res("weather/brooklyn", "passed"),
|
|
86
|
+
res("fixtures/button", "failed", {
|
|
87
|
+
assertions: [{ name: 'fileChanged("Button.tsx")', severity: "gate", score: 0, passed: false }],
|
|
88
|
+
}),
|
|
89
|
+
]);
|
|
90
|
+
await writeSnapshot(root, "compare_codex", "2026-07-09T10-00-00-000Z", { experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-09T10:00:00.000Z" }, [
|
|
91
|
+
res("weather/brooklyn", "passed"),
|
|
92
|
+
]);
|
|
93
93
|
return root;
|
|
94
94
|
}
|
|
95
95
|
|
|
@@ -98,7 +98,7 @@ async function seedRoot(): Promise<string> {
|
|
|
98
98
|
describe("resolveViewInput · 位置参数语义", () => {
|
|
99
99
|
it("存在的文件路径 → 单文件模式;其余位置参数按 eval id 前缀", async () => {
|
|
100
100
|
const root = await seedRoot();
|
|
101
|
-
const file = join(root, "2026-07-08T10-00-00-000Z", "
|
|
101
|
+
const file = join(root, "compare_bub", "2026-07-08T10-00-00-000Z", "snapshot.json");
|
|
102
102
|
expect(resolveViewInput(root, [file])).toEqual({ input: file, patterns: [] });
|
|
103
103
|
expect(resolveViewInput(root, ["weather", "fixtures/button"])).toEqual({
|
|
104
104
|
patterns: ["weather", "fixtures/button"],
|
|
@@ -108,13 +108,13 @@ describe("resolveViewInput · 位置参数语义", () => {
|
|
|
108
108
|
|
|
109
109
|
it("目录位置参数:报错直说走 --run(位置参数留给 eval id 前缀)", async () => {
|
|
110
110
|
const root = await seedRoot();
|
|
111
|
-
expect(() => resolveViewInput(root, ["
|
|
112
|
-
expect(() => resolveViewInput(root, ["
|
|
111
|
+
expect(() => resolveViewInput(root, ["compare_bub"])).toThrow(ViewInputError);
|
|
112
|
+
expect(() => resolveViewInput(root, ["compare_bub"])).toThrow(/--run/);
|
|
113
113
|
});
|
|
114
114
|
|
|
115
115
|
it("单文件模式不与其它位置参数或 --run 混用,歧义报错直说", async () => {
|
|
116
116
|
const root = await seedRoot();
|
|
117
|
-
const file = join(root, "2026-07-08T10-00-00-000Z", "
|
|
117
|
+
const file = join(root, "compare_bub", "2026-07-08T10-00-00-000Z", "snapshot.json");
|
|
118
118
|
expect(() => resolveViewInput(root, [file, "weather"])).toThrow(/exactly one path/);
|
|
119
119
|
expect(() => resolveViewInput(root, [file], root)).toThrow(/--run/);
|
|
120
120
|
});
|
|
@@ -133,24 +133,23 @@ describe("resolveViewInput · 位置参数语义", () => {
|
|
|
133
133
|
// ───────────────────────── 组合语义(与 show 对齐) ─────────────────────────
|
|
134
134
|
|
|
135
135
|
describe("loadViewScan · 组合语义", () => {
|
|
136
|
-
it("
|
|
136
|
+
it("位置前缀收窄报告槽 Selection;证据室快照不收窄,深链恒可达", async () => {
|
|
137
137
|
const root = await seedRoot();
|
|
138
|
-
const { viewData } = await loadViewScan(root, { patterns: ["weather"] });
|
|
139
|
-
// 报告槽:两实验都只剩 weather/brooklyn
|
|
140
|
-
expect(
|
|
141
|
-
expect(
|
|
142
|
-
|
|
143
|
-
expect(row.cells["pass-rate"]!.value).toBe(1);
|
|
144
|
-
}
|
|
138
|
+
const { viewData, reportHtml } = await loadViewScan(root, { patterns: ["weather"] });
|
|
139
|
+
// 报告槽:两实验都只剩 weather/brooklyn 一题,范围外的失败不再出现。
|
|
140
|
+
expect(reportHtml.en).toContain("compare/bub");
|
|
141
|
+
expect(reportHtml.en).toContain("compare/codex");
|
|
142
|
+
expect(reportHtml.en).not.toContain("fixtures/button");
|
|
145
143
|
// 证据室:快照明细仍含 fixtures/button(attempt 深链在收窄下也能解析)。
|
|
146
144
|
const allIds = viewData.snapshots.flatMap((s) => s.results.map((r) => r.id));
|
|
147
145
|
expect(allIds).toContain("fixtures/button");
|
|
148
146
|
});
|
|
149
147
|
|
|
150
|
-
it("--experiment
|
|
148
|
+
it("--experiment 过滤:报告槽 Selection 只留该实验", async () => {
|
|
151
149
|
const root = await seedRoot();
|
|
152
|
-
const {
|
|
153
|
-
expect(
|
|
150
|
+
const { reportHtml } = await loadViewScan(root, { experiment: "compare/codex" });
|
|
151
|
+
expect(reportHtml.en).toContain("compare/codex");
|
|
152
|
+
expect(reportHtml.en).not.toContain("compare/bub");
|
|
154
153
|
});
|
|
155
154
|
|
|
156
155
|
it("前缀/实验匹配不到:直说,不渲染空页面", async () => {
|
|
@@ -160,11 +159,51 @@ describe("loadViewScan · 组合语义", () => {
|
|
|
160
159
|
await expect(loadViewScan(root, { experiment: "nosuch" })).rejects.toBeInstanceOf(ViewInputError);
|
|
161
160
|
});
|
|
162
161
|
|
|
163
|
-
it("
|
|
162
|
+
it("全部缺省:报告槽使用现刻水位口径(与 show 相同的合成规则)", async () => {
|
|
163
|
+
const root = await seedRoot();
|
|
164
|
+
const { reportHtml } = await loadViewScan(root);
|
|
165
|
+
// 默认报告的榜单含两个实验;官方组件的稳定类名在场。
|
|
166
|
+
expect(reportHtml.en).toContain("compare/bub");
|
|
167
|
+
expect(reportHtml.en).toContain("compare/codex");
|
|
168
|
+
expect(reportHtml.en).toContain("nre-");
|
|
169
|
+
});
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
// ─────────────────── 报告槽恒在:裸跑 ≡ --report <ExperimentComparison> ───────────────────
|
|
173
|
+
|
|
174
|
+
describe("loadViewScan · 默认报告槽(裸跑)", () => {
|
|
175
|
+
it("裸跑产出的报告槽 HTML 与 --report <re-export ExperimentComparison 的文件> 完全一致(双语)", async () => {
|
|
176
|
+
const root = await seedRoot();
|
|
177
|
+
const bare = await loadViewScan(root);
|
|
178
|
+
const viaReport = await loadViewScan(root, {
|
|
179
|
+
report: { path: DEFAULT_REPORT_REEXPORT, cwd: root },
|
|
180
|
+
});
|
|
181
|
+
expect(bare.reportHtml.en).toBe(viaReport.reportHtml.en);
|
|
182
|
+
expect(bare.reportHtml["zh-CN"]).toBe(viaReport.reportHtml["zh-CN"]);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("报告槽双语渲染:同一棵树按 locale 渲染两遍,chrome 文案分语言、数据不分语言", async () => {
|
|
186
|
+
const root = await seedRoot();
|
|
187
|
+
const { reportHtml } = await loadViewScan(root);
|
|
188
|
+
expect(reportHtml.en).toContain("Pass rate"); // ExperimentList 主行(en)
|
|
189
|
+
expect(reportHtml["zh-CN"]).toContain("成功率"); // ExperimentList 主行(zh-CN,passRate 的 zh label)
|
|
190
|
+
for (const html of [reportHtml.en, reportHtml["zh-CN"]]) {
|
|
191
|
+
expect(html).toContain("compare/bub");
|
|
192
|
+
// 失败案例深链进证据室:不透明 AttemptLocator 单段路由 `#/attempt/@<locator>`,
|
|
193
|
+
// 不再是旧的两段式 `#/attempt/<snapshot>/<attempt>`。
|
|
194
|
+
expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/);
|
|
195
|
+
expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it("失败清单与警告住在报告槽里:ExperimentList 列出失败,壳的 viewData 不携带统计产物", async () => {
|
|
164
200
|
const root = await seedRoot();
|
|
165
201
|
const { viewData, reportHtml } = await loadViewScan(root);
|
|
166
|
-
expect(reportHtml).
|
|
167
|
-
|
|
202
|
+
expect(reportHtml.en).toContain("fixtures/button");
|
|
203
|
+
// viewData 只有证据室数据:快照 + skipped + 壳元信息。
|
|
204
|
+
expect(viewData).not.toHaveProperty("overview");
|
|
205
|
+
expect(viewData).not.toHaveProperty("table");
|
|
206
|
+
expect(viewData).not.toHaveProperty("overall");
|
|
168
207
|
});
|
|
169
208
|
});
|
|
170
209
|
|
|
@@ -174,31 +213,33 @@ describe("loadViewScan · --report 报告槽", () => {
|
|
|
174
213
|
it("报告树渲染为静态 HTML:官方水位 + 自定义摆法 + <Style> 产物 + 证据室深链,零 <script>", async () => {
|
|
175
214
|
const root = await seedRoot();
|
|
176
215
|
const scan = await loadViewScan(root, { report: { path: EXAM_REPORT, cwd: root } });
|
|
177
|
-
const html = scan.reportHtml
|
|
216
|
+
const html = scan.reportHtml.en;
|
|
178
217
|
expect(html).toContain("考试成绩单"); // 自定义 Section
|
|
179
218
|
expect(html).toContain("nre-"); // 官方组件的稳定类名
|
|
180
219
|
expect(html).toContain("<style>.exam-note { color: #4a7; }</style>"); // <Style> 随树带走
|
|
181
|
-
expect(html).
|
|
220
|
+
expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/); // 失败案例深链进证据室(单段 locator 路由)
|
|
182
221
|
expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
|
|
183
|
-
//
|
|
222
|
+
// 用户报告同样双语渲染两遍(壳按界面语言摆放)。
|
|
223
|
+
expect(scan.reportHtml["zh-CN"]).toContain("考试成绩单");
|
|
224
|
+
// 证据室数据契约(__NICEEVAL_VIEW_DATA__)原样保留:快照、locator、skipped 不动。
|
|
184
225
|
expect(scan.viewData.snapshots.length).toBeGreaterThan(0);
|
|
185
|
-
expect(scan.viewData.snapshots.flatMap((s) => s.results).every((r) => r.
|
|
226
|
+
expect(scan.viewData.snapshots.flatMap((s) => s.results).every((r) => r.locator)).toBe(true);
|
|
186
227
|
expect(JSON.stringify(scan.viewData)).not.toContain("考试成绩单"); // 报告块不进 viewData
|
|
187
228
|
});
|
|
188
229
|
|
|
189
|
-
it("位置前缀对 --report
|
|
230
|
+
it("位置前缀对 --report 生效:收窄注入 Selection,报告只见范围内的 eval", async () => {
|
|
190
231
|
const root = await seedRoot();
|
|
191
232
|
const scan = await loadViewScan(root, {
|
|
192
233
|
patterns: ["weather"],
|
|
193
234
|
report: { path: EXAM_REPORT, cwd: root },
|
|
194
235
|
});
|
|
195
236
|
// 范围外的失败(fixtures/button)不再出现在报告里;范围内的实验行都在。
|
|
196
|
-
expect(scan.reportHtml).not.toContain("fixtures/button");
|
|
197
|
-
expect(scan.reportHtml).toContain("compare/bub");
|
|
198
|
-
expect(scan.reportHtml).toContain("compare/codex");
|
|
237
|
+
expect(scan.reportHtml.en).not.toContain("fixtures/button");
|
|
238
|
+
expect(scan.reportHtml.en).toContain("compare/bub");
|
|
239
|
+
expect(scan.reportHtml.en).toContain("compare/codex");
|
|
199
240
|
});
|
|
200
241
|
|
|
201
|
-
it("show --report 与 view --report
|
|
242
|
+
it("show --report 与 view --report 吃同一个报告文件,判定口径一致", async () => {
|
|
202
243
|
const root = await seedRoot();
|
|
203
244
|
let text = "";
|
|
204
245
|
const code = await runShow(root, [], { run: root, report: EXAM_REPORT }, {
|
|
@@ -211,7 +252,7 @@ describe("loadViewScan · --report 报告槽", () => {
|
|
|
211
252
|
// 同一棵树的两个面:同一份失败清单与同一个自定义 Section。
|
|
212
253
|
for (const needle of ["考试成绩单", "compare/bub", "compare/codex", "fixtures/button"]) {
|
|
213
254
|
expect(text).toContain(needle);
|
|
214
|
-
expect(scan.reportHtml).toContain(needle);
|
|
255
|
+
expect(scan.reportHtml.en).toContain(needle);
|
|
215
256
|
}
|
|
216
257
|
});
|
|
217
258
|
|
|
@@ -258,51 +299,58 @@ describe("loadViewScan · 报告文件变更整页重算", () => {
|
|
|
258
299
|
const path = join(root, "report.mjs");
|
|
259
300
|
await writeFile(path, reportSource("FIRST_RENDER"), "utf-8");
|
|
260
301
|
const first = await loadViewScan(root, { report: { path, cwd: root } });
|
|
261
|
-
expect(first.reportHtml).toContain("FIRST_RENDER");
|
|
302
|
+
expect(first.reportHtml.en).toContain("FIRST_RENDER");
|
|
262
303
|
|
|
263
304
|
await writeFile(path, reportSource("SECOND_RENDER"), "utf-8");
|
|
264
305
|
// mtime 精度兜底:显式把 mtime 拨到未来,确保与首次装载可区分。
|
|
265
306
|
const future = new Date(Date.now() + 5000);
|
|
266
307
|
await utimes(path, future, future);
|
|
267
308
|
const second = await loadViewScan(root, { report: { path, cwd: root } });
|
|
268
|
-
expect(second.reportHtml).toContain("SECOND_RENDER");
|
|
269
|
-
expect(second.reportHtml).not.toContain("FIRST_RENDER");
|
|
309
|
+
expect(second.reportHtml.en).toContain("SECOND_RENDER");
|
|
310
|
+
expect(second.reportHtml.en).not.toContain("FIRST_RENDER");
|
|
270
311
|
});
|
|
271
312
|
});
|
|
272
313
|
|
|
273
314
|
// ───────────────────────── --out 静态导出 ─────────────────────────
|
|
274
315
|
|
|
275
316
|
describe("buildView · --out 与 --report", () => {
|
|
276
|
-
it("报告页为首页报告槽,证据室同站:index.html
|
|
317
|
+
it("报告页为首页报告槽,证据室同站:index.html 含报告块与官方样式,artifact 照常复制", async () => {
|
|
277
318
|
const root = await seedRoot();
|
|
278
|
-
// 给 weather attempt 一份 events
|
|
279
|
-
|
|
319
|
+
// 给 weather attempt 一份 events artifact,验证证据室 artifact 照常进导出。
|
|
320
|
+
// 本快照跑出的条目:artifact 目录 = <snapshot.dir>/<evalId>/a<n>,不需要声明字段。
|
|
321
|
+
const artifactDir = join(root, "compare_codex", "2026-07-09T10-00-00-000Z", "weather", "brooklyn", "a0");
|
|
280
322
|
await mkdir(artifactDir, { recursive: true });
|
|
281
323
|
await writeFile(join(artifactDir, "events.json"), "[]", "utf-8");
|
|
282
|
-
const summaryPath = join(root, "2026-07-09T10-00-00-000Z", "summary.json");
|
|
283
|
-
const summary = JSON.parse(await readFile(summaryPath, "utf-8")) as RunSummary;
|
|
284
|
-
summary.results[0]!.artifactsDir = "artifacts/weather";
|
|
285
|
-
await writeFile(summaryPath, JSON.stringify(summary), "utf-8");
|
|
286
324
|
|
|
287
325
|
const out = join(root, "site");
|
|
288
326
|
await buildView({ input: root, out, scan: { report: { path: EXAM_REPORT, cwd: root } } });
|
|
289
327
|
|
|
290
328
|
const html = await readFile(join(out, "index.html"), "utf-8");
|
|
291
|
-
|
|
329
|
+
// 双语两个 <template> 静态块都在,壳按界面语言摆放。
|
|
330
|
+
expect(html).toContain('<template id="niceeval-report-en">');
|
|
331
|
+
expect(html).toContain('<template id="niceeval-report-zh-CN">');
|
|
292
332
|
expect(html).toContain("考试成绩单");
|
|
293
333
|
expect(html).toContain("nre-"); // 官方组件样式(report/react/styles.css)随页注入
|
|
294
334
|
// 报告块本体零 <script>:静态块起于 <template>,内部只有标记与 <style>。
|
|
295
|
-
const block = html.split('<template id="niceeval-report">')[1]!.split("</template>")[0]!;
|
|
335
|
+
const block = html.split('<template id="niceeval-report-en">')[1]!.split("</template>")[0]!;
|
|
296
336
|
expect(block).not.toContain("<script");
|
|
297
|
-
//
|
|
298
|
-
expect(
|
|
337
|
+
// 证据室同站: artifact 按 /artifact/<base>/ 布局复制。
|
|
338
|
+
expect(
|
|
339
|
+
existsSync(join(out, "artifact", "compare_codex/2026-07-09T10-00-00-000Z/weather/brooklyn/a0", "events.json")),
|
|
340
|
+
).toBe(true);
|
|
299
341
|
});
|
|
300
342
|
|
|
301
|
-
it("默认导出(无 --report)
|
|
343
|
+
it("默认导出(无 --report):报告槽填充 ExperimentComparison,双语块与增强 runtime 恒内联", async () => {
|
|
302
344
|
const root = await seedRoot();
|
|
303
345
|
const out = join(root, "site");
|
|
304
346
|
await buildView({ input: root, out });
|
|
305
347
|
const html = await readFile(join(out, "index.html"), "utf-8");
|
|
306
|
-
expect(html).
|
|
348
|
+
expect(html).toContain('<template id="niceeval-report-en">');
|
|
349
|
+
expect(html).toContain('<template id="niceeval-report-zh-CN">');
|
|
350
|
+
expect(html).toContain("nre-"); // 官方组件的稳定类名(KPI / 榜单 / 散点来自 nre 组件)
|
|
351
|
+
// 渐进增强 runtime(排序 / 过滤 / tooltip)恒内联;报告块本体仍零 <script>。
|
|
352
|
+
expect(html).toContain("__nreEnhanced");
|
|
353
|
+
const block = html.split('<template id="niceeval-report-en">')[1]!.split("</template>")[0]!;
|
|
354
|
+
expect(block).not.toContain("<script");
|
|
307
355
|
});
|
|
308
356
|
});
|
|
@@ -1,222 +0,0 @@
|
|
|
1
|
-
// DefaultReport:官方水位整块 —— 官方两扇门裸跑时渲染的就是它,零 props、纯声明。
|
|
2
|
-
// 「渲染面纯同步」与它不冲突,靠的是一个数据事实:官方水位(overview、榜单、失败清单)
|
|
3
|
-
// 只读瘦身条目、不碰任何懒加载工件,宿主对着已挑好的选集总是把这份数据备好、经上下文
|
|
4
|
-
// 注入(renderReportToText / renderReportToStaticHtml 里 prepareDefaultReportData)。
|
|
5
|
-
// 它渲染的口径钉死为宿主注入的那份选集 —— 零 props 意味着没有跟随的通道,这是锚点语义:
|
|
6
|
-
// 官方口径与自定义口径并排对照。默认无特权:数据全部来自公开计算函数。
|
|
7
|
-
//
|
|
8
|
-
// text 面即 `niceeval show` 的榜单(docs-site/zh/guides/viewing-results.mdx 的示例块是
|
|
9
|
-
// 行为规范):`Current verdicts` 头标注合成自几个 run、experiment 表带 eval 级折叠的
|
|
10
|
-
// evals 列、Failing 清单每条带判决时间与下钻命令。
|
|
11
|
-
|
|
12
|
-
import { basename } from "node:path";
|
|
13
|
-
import type { Selection } from "../results/index.ts";
|
|
14
|
-
import { foldEvalOutcome } from "../shared/outcome.ts";
|
|
15
|
-
import { defineComponent } from "./tree.ts";
|
|
16
|
-
import type { CaseListData, OverviewData, TableData } from "./types.ts";
|
|
17
|
-
import { caseListData, overviewData, tableData } from "./compute.ts";
|
|
18
|
-
import { attemptCostUSD, costUSD, durationMs, passRate } from "./metrics.ts";
|
|
19
|
-
import { RunOverview, MetricTable, CaseList } from "./components.tsx";
|
|
20
|
-
import { cellText } from "./text/faces.ts";
|
|
21
|
-
import { renderAlignedRows } from "./text/layout.ts";
|
|
22
|
-
|
|
23
|
-
/** 榜单的合成标注与 eval 级折叠(text 面用;与 verdicts/cases 出自同一选集)。 */
|
|
24
|
-
export interface VerdictBoardData {
|
|
25
|
-
/** 判决合成自几个物理 run。 */
|
|
26
|
-
composedFromRuns: number;
|
|
27
|
-
/** 参与合成的最新 run(目录名,即时间戳)。 */
|
|
28
|
-
latestRun?: string;
|
|
29
|
-
/** experiment → eval 级折叠计票(evals 列的 13/15)。 */
|
|
30
|
-
tallies: Record<string, { passedEvals: number; totalEvals: number }>;
|
|
31
|
-
/** eval 级折叠后失败/出错的题,带判决时间;新失败在前。 */
|
|
32
|
-
failing: {
|
|
33
|
-
evalId: string;
|
|
34
|
-
experimentId: string;
|
|
35
|
-
outcome: "failed" | "errored";
|
|
36
|
-
/** 最新 attempt 的第一条失败断言("gate calledTool(...)")或错误摘要。 */
|
|
37
|
-
reason?: string;
|
|
38
|
-
/** 判决产生的时刻(最新 attempt 的 startedAt,缺失退快照时刻)。 */
|
|
39
|
-
verdictAt?: string;
|
|
40
|
-
}[];
|
|
41
|
-
/** limit 之外还有几条,如实报。 */
|
|
42
|
-
failingTruncated: number;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export interface DefaultReportData {
|
|
46
|
-
overview: OverviewData;
|
|
47
|
-
/** 现刻榜单:experiment × (passRate, costUSD, durationMs)。 */
|
|
48
|
-
verdicts: TableData;
|
|
49
|
-
cases: CaseListData;
|
|
50
|
-
board: VerdictBoardData;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/** 失败清单的出厂截断;完整清单自己摆 <CaseList>(截断如实报剩余)。 */
|
|
54
|
-
const DEFAULT_CASE_LIMIT = 10;
|
|
55
|
-
|
|
56
|
-
function buildBoard(selection: Selection): VerdictBoardData {
|
|
57
|
-
const tallies: VerdictBoardData["tallies"] = {};
|
|
58
|
-
const failing: VerdictBoardData["failing"] = [];
|
|
59
|
-
const runs = new Set<string>();
|
|
60
|
-
let latestRunDir = "";
|
|
61
|
-
|
|
62
|
-
for (const snapshot of selection.snapshots) {
|
|
63
|
-
let passedEvals = 0;
|
|
64
|
-
for (const ev of snapshot.evals) {
|
|
65
|
-
for (const attempt of ev.attempts) {
|
|
66
|
-
runs.add(attempt.runDir.dir);
|
|
67
|
-
if (attempt.runDir.dir > latestRunDir) latestRunDir = attempt.runDir.dir;
|
|
68
|
-
}
|
|
69
|
-
const outcome = foldEvalOutcome(ev.attempts.map((a) => a.result));
|
|
70
|
-
if (outcome === "passed") {
|
|
71
|
-
passedEvals += 1;
|
|
72
|
-
continue;
|
|
73
|
-
}
|
|
74
|
-
if (outcome !== "failed" && outcome !== "errored") continue;
|
|
75
|
-
const latest = ev.attempts.reduce((a, b) => (b.result.attempt >= a.result.attempt ? b : a));
|
|
76
|
-
const failedAssertion = latest.result.assertions.find((a) => !a.passed);
|
|
77
|
-
const reason = failedAssertion
|
|
78
|
-
? `${failedAssertion.severity} ${failedAssertion.name}`
|
|
79
|
-
: latest.result.error !== undefined
|
|
80
|
-
? `error: ${latest.result.error}`
|
|
81
|
-
: undefined;
|
|
82
|
-
failing.push({
|
|
83
|
-
evalId: ev.id,
|
|
84
|
-
experimentId: snapshot.experimentId,
|
|
85
|
-
outcome,
|
|
86
|
-
...(reason !== undefined ? { reason } : {}),
|
|
87
|
-
...(latest.result.startedAt !== undefined || snapshot.startedAt
|
|
88
|
-
? { verdictAt: latest.result.startedAt ?? snapshot.startedAt }
|
|
89
|
-
: {}),
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
tallies[snapshot.experimentId] = { passedEvals, totalEvals: snapshot.evals.length };
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
failing.sort((a, b) => (b.verdictAt ?? "").localeCompare(a.verdictAt ?? ""));
|
|
96
|
-
const shown = failing.slice(0, DEFAULT_CASE_LIMIT);
|
|
97
|
-
return {
|
|
98
|
-
composedFromRuns: runs.size,
|
|
99
|
-
...(latestRunDir ? { latestRun: basename(latestRunDir) } : {}),
|
|
100
|
-
tallies,
|
|
101
|
-
failing: shown,
|
|
102
|
-
failingTruncated: failing.length - shown.length,
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/** 宿主渲染前备好官方水位:只读瘦身条目,代价可忽略。 */
|
|
107
|
-
export async function prepareDefaultReportData(selection: Selection): Promise<DefaultReportData> {
|
|
108
|
-
return {
|
|
109
|
-
overview: await overviewData(selection),
|
|
110
|
-
verdicts: await tableData(selection, {
|
|
111
|
-
rows: "experiment",
|
|
112
|
-
columns: [passRate, costUSD, durationMs],
|
|
113
|
-
sort: passRate,
|
|
114
|
-
}),
|
|
115
|
-
cases: await caseListData(selection, { limit: DEFAULT_CASE_LIMIT }),
|
|
116
|
-
board: buildBoard(selection),
|
|
117
|
-
};
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
let activeData: DefaultReportData | null = null;
|
|
121
|
-
|
|
122
|
-
/** 宿主(与渲染入口)用:在注入好的官方水位下同步渲染。 */
|
|
123
|
-
export function runWithDefaultReportData<T>(data: DefaultReportData, fn: () => T): T {
|
|
124
|
-
const prev = activeData;
|
|
125
|
-
activeData = data;
|
|
126
|
-
try {
|
|
127
|
-
return fn();
|
|
128
|
-
} finally {
|
|
129
|
-
activeData = prev;
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function requireData(): DefaultReportData {
|
|
134
|
-
if (!activeData) {
|
|
135
|
-
throw new Error(
|
|
136
|
-
"<DefaultReport /> renders the host-injected selection; render the report via " +
|
|
137
|
-
"`niceeval show --report` / `niceeval view --report` (or renderReportToText / renderReportToStaticHtml). " +
|
|
138
|
-
"Outside a host, compose the same blocks yourself: RunOverview, MetricTable, CaseList.",
|
|
139
|
-
);
|
|
140
|
-
}
|
|
141
|
-
return activeData;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/** 判决时间的相对标注(text 面的 "41s ago";渲染面纯同步,Date.now 不算 IO)。 */
|
|
145
|
-
function agoText(iso: string | undefined, now: number): string {
|
|
146
|
-
if (!iso) return "";
|
|
147
|
-
const ms = now - Date.parse(iso);
|
|
148
|
-
if (!Number.isFinite(ms) || ms < 10_000) return "just now";
|
|
149
|
-
const seconds = Math.floor(ms / 1000);
|
|
150
|
-
if (seconds < 120) return `${seconds}s ago`;
|
|
151
|
-
const minutes = Math.floor(seconds / 60);
|
|
152
|
-
if (minutes < 120) return `${minutes}m ago`;
|
|
153
|
-
const hours = Math.floor(minutes / 60);
|
|
154
|
-
if (hours < 48) return `${hours}h ago`;
|
|
155
|
-
return `${Math.floor(hours / 24)}d ago`;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
/** show 榜单(viewing-results.mdx「榜单」块):头 + experiment 表 + Failing 清单。 */
|
|
159
|
-
function boardText(data: DefaultReportData): string {
|
|
160
|
-
const { verdicts, board, overview } = data;
|
|
161
|
-
const experiments = verdicts.rows.length;
|
|
162
|
-
const head = [
|
|
163
|
-
"Current verdicts",
|
|
164
|
-
`${experiments} ${experiments === 1 ? "experiment" : "experiments"}`,
|
|
165
|
-
`composed from ${board.composedFromRuns} ${board.composedFromRuns === 1 ? "run" : "runs"}`,
|
|
166
|
-
...(board.latestRun ? [`latest ${board.latestRun}`] : []),
|
|
167
|
-
].join(" · ");
|
|
168
|
-
const headBlock = [head, ...overview.warnings.map((w) => `! ${w.message}`)].join("\n");
|
|
169
|
-
|
|
170
|
-
// 表:experiment | evals(eval 级折叠计票)| pass | cost | duration(顺序随 verdicts 预排)
|
|
171
|
-
const header = [verdicts.dimension, "evals", "pass", "cost", "duration"];
|
|
172
|
-
const rows = verdicts.rows.map((row) => {
|
|
173
|
-
const tally = board.tallies[row.key];
|
|
174
|
-
const cells = row.cells as Record<string, TableData["rows"][number]["cells"][string]>;
|
|
175
|
-
return [
|
|
176
|
-
row.key,
|
|
177
|
-
tally ? `${tally.passedEvals}/${tally.totalEvals}` : "—",
|
|
178
|
-
cells[passRate.name] ? cellText(cells[passRate.name]) : "—",
|
|
179
|
-
cells[costUSD.name] ? cellText(cells[costUSD.name]) : "—",
|
|
180
|
-
cells[durationMs.name] ? cellText(cells[durationMs.name]) : "—",
|
|
181
|
-
];
|
|
182
|
-
});
|
|
183
|
-
const table = renderAlignedRows([header, ...rows]);
|
|
184
|
-
|
|
185
|
-
const blocks = [headBlock, table];
|
|
186
|
-
if (board.failing.length > 0) {
|
|
187
|
-
const now = Date.now();
|
|
188
|
-
const aligned = renderAlignedRows(
|
|
189
|
-
board.failing.map((f) => [
|
|
190
|
-
`✗ ${f.evalId}`,
|
|
191
|
-
f.experimentId,
|
|
192
|
-
f.reason ?? f.outcome,
|
|
193
|
-
agoText(f.verdictAt, now),
|
|
194
|
-
]),
|
|
195
|
-
).split("\n");
|
|
196
|
-
const lines: string[] = ["Failing:"];
|
|
197
|
-
board.failing.forEach((f, i) => {
|
|
198
|
-
lines.push(` ${aligned[i]}`);
|
|
199
|
-
lines.push(` → niceeval show ${f.evalId}`);
|
|
200
|
-
});
|
|
201
|
-
if (board.failingTruncated > 0) lines.push(` (${board.failingTruncated} more not shown)`);
|
|
202
|
-
blocks.push(lines.join("\n"));
|
|
203
|
-
}
|
|
204
|
-
return blocks.join("\n\n");
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
export const DefaultReport = defineComponent<Record<string, never>>({
|
|
208
|
-
web() {
|
|
209
|
-
const data = requireData();
|
|
210
|
-
return (
|
|
211
|
-
<div className="nre nre-default-report">
|
|
212
|
-
<RunOverview data={data.overview} />
|
|
213
|
-
<MetricTable data={data.verdicts} />
|
|
214
|
-
{data.cases.rows.length > 0 && <CaseList data={data.cases} />}
|
|
215
|
-
</div>
|
|
216
|
-
);
|
|
217
|
-
},
|
|
218
|
-
text() {
|
|
219
|
-
return boardText(requireData());
|
|
220
|
-
},
|
|
221
|
-
});
|
|
222
|
-
DefaultReport.displayName = "DefaultReport";
|