niceeval 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +69 -7
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +57 -3
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +32 -24
- package/dist/report/aggregate.js +158 -50
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +93 -160
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +87 -81
- package/dist/report/compute.js +597 -417
- package/dist/report/flag.d.ts +32 -6
- package/dist/report/flag.js +92 -4
- package/dist/report/format.d.ts +19 -11
- package/dist/report/format.js +30 -13
- package/dist/report/index.d.ts +16 -16
- package/dist/report/index.js +20 -21
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +57 -33
- package/dist/report/locale.js +122 -56
- package/dist/report/metrics.d.ts +23 -4
- package/dist/report/metrics.js +110 -25
- package/dist/report/primitives.d.ts +48 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +9 -7
- package/dist/report/react/AttemptList.js +17 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +4 -4
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +10 -0
- package/dist/report/react/ExperimentComparison.js +12 -0
- package/dist/report/react/ExperimentList.d.ts +4 -3
- package/dist/report/react/ExperimentList.js +17 -18
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +59 -28
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +110 -147
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -16
- package/dist/report/report.js +259 -28
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +253 -184
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +38 -7
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +247 -284
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +49 -14
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +4 -2
- package/dist/shared/aggregate.js +8 -7
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
- package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
- package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
- package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
- package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
- package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
- package/docs-site/zh/how-to/publish-report.mdx +105 -0
- package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
- package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
- package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
- package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
- package/docs-site/zh/index.mdx +24 -26
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +32 -5
- package/docs-site/zh/reference/capabilities.mdx +8 -8
- package/docs-site/zh/reference/cli.mdx +40 -12
- package/docs-site/zh/reference/define-agent.mdx +58 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
- package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
- package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
- package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +10 -2
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +13 -2
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +71 -8
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +446 -124
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +162 -30
- package/src/context/session.test.ts +2 -1
- package/src/context/session.ts +115 -7
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +81 -17
- package/src/i18n/zh-CN.ts +80 -17
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +178 -61
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -279
- package/src/report/compute.ts +723 -491
- package/src/report/dual-render.test.tsx +741 -1024
- package/src/report/flag.ts +104 -12
- package/src/report/format.ts +32 -12
- package/src/report/index.ts +119 -46
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +136 -65
- package/src/report/metrics.ts +108 -25
- package/src/report/primitives.tsx +196 -45
- package/src/report/react/AttemptList.tsx +30 -43
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +73 -0
- package/src/report/react/ExperimentList.tsx +50 -32
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +86 -34
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +89 -5
- package/src/report/react/fixtures.ts +114 -154
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +138 -158
- package/src/report/react/styles.css +243 -82
- package/src/report/report.test.ts +779 -841
- package/src/report/report.ts +423 -41
- package/src/report/text/faces.ts +290 -193
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +44 -7
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +261 -271
- package/src/report/web.ts +63 -20
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +13 -11
- package/src/results/attempt-evidence.ts +20 -13
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +150 -60
- package/src/results/host-equivalence.test.ts +34 -20
- package/src/results/index.ts +12 -4
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +15 -5
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +89 -54
- package/src/results/select.ts +104 -34
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +43 -14
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +243 -37
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +8 -6
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +127 -56
- package/src/show/render.ts +662 -131
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +320 -54
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +423 -11
- package/src/view/app/components/CodeView.tsx +41 -14
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +37 -17
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +10 -3
- package/src/view/data.ts +155 -49
- package/src/view/index.ts +56 -41
- package/src/view/server.ts +37 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +227 -0
- package/src/view/view-report.test.ts +167 -62
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
- package/dist/report/built-ins/experiment-comparison.js +0 -13
- package/dist/report/built-ins/index.d.ts +0 -1
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/docs-site/zh/guides/publish-report.mdx +0 -91
- package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
- package/src/report/built-in-user-parity.test.tsx +0 -640
- package/src/report/built-ins/experiment-comparison.tsx +0 -19
- package/src/report/built-ins/index.ts +0 -2
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
|
@@ -1,1129 +1,846 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
1
|
+
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
|
+
// 管线与双面渲染的单元测试:resolve(spec/data 双形态、记忆化、组合组件、同层并行保序、
|
|
3
|
+
// 非法节点拒绝)、validate(裸字符串、单面组件、Tabs 配对)、装载规范化(defineReport 三种
|
|
4
|
+
// 写法、content/pages 互斥、外壳嵌套、page id、标题回退)、text/web 双面同源、
|
|
5
|
+
// Table 与文本排版原语、FailureList 等价、内建报告等价。
|
|
6
6
|
|
|
7
7
|
import { renderToStaticMarkup } from "react-dom/server";
|
|
8
8
|
import { describe, expect, it } from "vitest";
|
|
9
9
|
|
|
10
|
-
import type { EvalResult } from "../types.ts";
|
|
11
|
-
import type { Results,
|
|
12
|
-
import
|
|
13
|
-
import
|
|
14
|
-
|
|
10
|
+
import type { EvalResult, Verdict } from "../types.ts";
|
|
11
|
+
import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "../results/index.ts";
|
|
12
|
+
import { makeScope } from "../results/select.ts";
|
|
13
|
+
import {
|
|
14
|
+
createTextContext,
|
|
15
|
+
defineComponent,
|
|
16
|
+
renderNodeToText,
|
|
17
|
+
resolveReportTree,
|
|
18
|
+
validateReportTree,
|
|
19
|
+
ResolveMemo,
|
|
20
|
+
type ReportNode,
|
|
21
|
+
} from "./tree.ts";
|
|
22
|
+
import {
|
|
23
|
+
buildReportMeta,
|
|
24
|
+
defineReport,
|
|
25
|
+
FALLBACK_REPORT_TITLE,
|
|
26
|
+
pickReportPage,
|
|
27
|
+
renderReportToText,
|
|
28
|
+
ReportPageNotFoundError,
|
|
29
|
+
resolveReportTitle,
|
|
30
|
+
} from "./report.ts";
|
|
31
|
+
import { renderReportToStaticHtml } from "./web.ts";
|
|
15
32
|
import {
|
|
16
33
|
AttemptList,
|
|
17
|
-
|
|
18
|
-
DeltaTable,
|
|
19
|
-
EvalList,
|
|
34
|
+
ExperimentComparison,
|
|
20
35
|
ExperimentList,
|
|
21
|
-
|
|
36
|
+
FailureList,
|
|
22
37
|
MetricBars,
|
|
23
|
-
MetricLine,
|
|
24
38
|
MetricMatrix,
|
|
25
39
|
MetricScatter,
|
|
26
40
|
MetricTable,
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
experimentListItems,
|
|
52
|
-
groupSummaryData,
|
|
53
|
-
lineData,
|
|
54
|
-
matrixData,
|
|
55
|
-
overviewData,
|
|
56
|
-
scatterData,
|
|
57
|
-
scoreboardData,
|
|
58
|
-
tableData,
|
|
59
|
-
} from "./react/fixtures.ts";
|
|
60
|
-
|
|
61
|
-
const ctx = createTextContext({ width: 80 });
|
|
62
|
-
const text = (node: Parameters<typeof renderNodeToText>[0]) => renderNodeToText(node, ctx);
|
|
63
|
-
|
|
64
|
-
// ───────────────────────── 每组件双面一致 ─────────────────────────
|
|
65
|
-
|
|
66
|
-
describe("RunOverview 双面", () => {
|
|
67
|
-
const html = renderToStaticMarkup(<RunOverview data={overviewData} />);
|
|
68
|
-
const term = text(<RunOverview data={overviewData} />);
|
|
69
|
-
|
|
70
|
-
it("text 面形态:头行(含通过率)+ 判定行 + 警告行", () => {
|
|
71
|
-
expect(term).toMatchInlineSnapshot(`
|
|
72
|
-
"2 experiments · 12 evals · 48 attempts · Pass rate 70% 46/48 · composed from 2 runs · latest 2026-07-01T11:30:00Z
|
|
73
|
-
passed 36 · failed 8 · errored 2 · skipped 2 · no data · 4m 21s
|
|
74
|
-
! snapshot covers 9 of 12 evals seen in history; re-run \`niceeval exp compare/bub\` for a full snapshot"
|
|
75
|
-
`);
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
it("两面同口径:通过率显示同一个 MetricCell.display(70%,不是 36/46 现算的 78%);成本全缺都是 no data 不编 $0;警告都在", () => {
|
|
79
|
-
for (const face of [html, term]) {
|
|
80
|
-
expect(face).toContain("70%");
|
|
81
|
-
expect(face).not.toContain("78%");
|
|
82
|
-
expect(face).toContain("46/48"); // samples < total:两级聚合的覆盖率角标,两面一致
|
|
83
|
-
expect(face).toContain("no data");
|
|
84
|
-
expect(face).not.toContain("$0");
|
|
85
|
-
expect(face).toContain("snapshot covers 9 of 12 evals");
|
|
86
|
-
}
|
|
87
|
-
});
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
// fixtures.overviewData 是手工摆好的终值,只验证渲染面「原样显示 MetricCell,不重算」;
|
|
91
|
-
// 下面这组用真实 Selection 走一遍 RunOverview.data(= compute.ts 的 overviewData()),
|
|
92
|
-
// 专门验证 totals.passRate 本身的计算口径 —— 三种通过率公式在这个 fixture 上各不相同:
|
|
93
|
-
// 两级聚合(唯一官方口径,docs/feature/reports/architecture.md「指标聚合不变量」):eval a 题内 2/3 通过、eval b 题内 1,
|
|
94
|
-
// 跨题均值 (2/3 + 1) / 2 = 5/6 ≈ 83.3%
|
|
95
|
-
// attempt 原始占比(旧 bug 公式,曾经的 RunOverview 现场重算):3 passed / (3 passed + 1 failed) = 75%
|
|
96
|
-
// eval 折叠投票(evalLevelStats,GroupSummary/MetricTable meta 的口径):a、b 都折成 passed → 2/2 = 100%
|
|
97
|
-
// 三个数互不相同,任何一处偷懒复用另一个公式都会在这里露馅。
|
|
98
|
-
describe("RunOverview.data · passRate 两级聚合口径", () => {
|
|
99
|
-
function fakeVaryingAttemptsContext(): { selection: Selection; attempts: { evalId: string; result: { verdict: EvalResult["verdict"] } }[] } {
|
|
100
|
-
const dir = "/results/compare_bub/snap-1";
|
|
101
|
-
const base = {
|
|
102
|
-
experimentId: "compare/bub",
|
|
103
|
-
startedAt: "2026-07-01T10:00:00Z",
|
|
104
|
-
completedAt: "2026-07-01T10:05:00Z",
|
|
105
|
-
agent: "bub",
|
|
106
|
-
schemaVersion: 1,
|
|
107
|
-
dir,
|
|
108
|
-
};
|
|
109
|
-
const mk = (evalId: string, verdict: "passed" | "failed" | "skipped", attemptIndex: number, minute: number) => ({
|
|
110
|
-
evalId,
|
|
111
|
-
experimentId: "compare/bub",
|
|
112
|
-
result: {
|
|
113
|
-
id: evalId,
|
|
114
|
-
agent: "bub",
|
|
115
|
-
verdict,
|
|
116
|
-
attempt: attemptIndex,
|
|
117
|
-
startedAt: `2026-07-01T10:0${minute}:00Z`,
|
|
118
|
-
durationMs: 1000,
|
|
119
|
-
assertions: [],
|
|
120
|
-
},
|
|
121
|
-
ref: { snapshot: "compare_bub/snap-1", attempt: `${evalId}/a${attemptIndex}` },
|
|
122
|
-
snapshot: base,
|
|
123
|
-
events: async () => null,
|
|
124
|
-
trace: async () => null,
|
|
125
|
-
o11y: async () => null,
|
|
126
|
-
diff: async () => null,
|
|
127
|
-
sources: async () => null,
|
|
128
|
-
});
|
|
129
|
-
const attempts = [
|
|
130
|
-
// eval a:3 attempts,2 通过 1 失败 → 题内 partial credit 2/3
|
|
131
|
-
mk("algebra/a", "passed", 0, 0),
|
|
132
|
-
mk("algebra/a", "failed", 1, 1),
|
|
133
|
-
mk("algebra/a", "passed", 2, 2),
|
|
134
|
-
// eval b:1 attempt 通过 → 题内 1
|
|
135
|
-
mk("algebra/b", "passed", 0, 3),
|
|
136
|
-
// eval c:1 attempt 跳过 —— 两级聚合与 eval 折叠计票都要把它排除在分母外
|
|
137
|
-
mk("algebra/c", "skipped", 0, 4),
|
|
138
|
-
];
|
|
139
|
-
const evalIds = [...new Set(attempts.map((a) => a.evalId))];
|
|
140
|
-
const snapshot: Snapshot = {
|
|
141
|
-
...base,
|
|
142
|
-
evals: evalIds.map((id) => ({ id, attempts: attempts.filter((a) => a.evalId === id) })),
|
|
143
|
-
attempts,
|
|
144
|
-
} as unknown as Snapshot;
|
|
145
|
-
const selection: Selection = { snapshots: [snapshot], warnings: [], filter: () => selection };
|
|
146
|
-
return { selection, attempts };
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
it("totals.passRate = 两级聚合 83.3%,既不等于 attempt 原始占比 75%,也不等于 eval 折叠投票 100%", async () => {
|
|
150
|
-
const { selection, attempts } = fakeVaryingAttemptsContext();
|
|
151
|
-
const data = await RunOverview.data(selection);
|
|
152
|
-
|
|
153
|
-
expect(data.totals.passRate.value).toBeCloseTo(5 / 6, 10);
|
|
154
|
-
expect(data.totals.passRate.display).toBe("83.3%");
|
|
155
|
-
expect(data.totals.passRate.samples).toBe(4); // 5 attempts - 1 skipped(不进桶)
|
|
156
|
-
expect(data.totals.passRate.total).toBe(5);
|
|
157
|
-
|
|
158
|
-
// attempt 原始占比(旧 bug 公式):必须与两级聚合不同,证明没有从 passed/failed/errored 现算
|
|
159
|
-
const attemptFraction = data.totals.passed / (data.totals.passed + data.totals.failed + data.totals.errored);
|
|
160
|
-
expect(attemptFraction).toBeCloseTo(0.75, 10);
|
|
161
|
-
expect(attemptFraction).not.toBeCloseTo(data.totals.passRate.value as number, 3);
|
|
162
|
-
|
|
163
|
-
// eval 折叠投票(evalLevelStats,GroupSummary 的口径):也必须与两级聚合不同
|
|
164
|
-
const stats = evalLevelStats(
|
|
165
|
-
attempts.map((a) => ({ verdict: a.result.verdict, key: a.evalId })),
|
|
166
|
-
(r) => r.key,
|
|
167
|
-
);
|
|
168
|
-
expect(stats.passRate).toBeCloseTo(1, 10);
|
|
169
|
-
expect(stats.passRate).not.toBeCloseTo(data.totals.passRate.value as number, 3);
|
|
170
|
-
});
|
|
171
|
-
|
|
172
|
-
it("web 面与 text 面显示同一个 passRate.display,覆盖率角标(4/5)两面一致", async () => {
|
|
173
|
-
const { selection } = fakeVaryingAttemptsContext();
|
|
174
|
-
const data = await RunOverview.data(selection);
|
|
175
|
-
const html = renderToStaticMarkup(<RunOverview data={data} />);
|
|
176
|
-
const term = text(<RunOverview data={data} />);
|
|
177
|
-
for (const face of [html, term]) {
|
|
178
|
-
expect(face).toContain(data.totals.passRate.display);
|
|
179
|
-
expect(face).toContain("4/5");
|
|
180
|
-
}
|
|
181
|
-
});
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
// ───────────────────────── GroupSummary ─────────────────────────
|
|
185
|
-
|
|
186
|
-
describe("GroupSummary 双面", () => {
|
|
187
|
-
const html = renderToStaticMarkup(<GroupSummary data={groupSummaryData} />);
|
|
188
|
-
const term = text(<GroupSummary data={groupSummaryData} />);
|
|
189
|
-
|
|
190
|
-
it("text 面形态:一行头(通过率 + experiment/eval 数 + failed/errored + 总成本)+ 最后运行时间", () => {
|
|
191
|
-
expect(term).toMatchInlineSnapshot(`
|
|
192
|
-
"Pass rate 60% 5/6 · 2 experiments · 6 evals · failed 1 · errored 1 · $1.50
|
|
193
|
-
latest 2026-07-01T11:30:00Z"
|
|
194
|
-
`);
|
|
195
|
-
});
|
|
41
|
+
ScopeSummary,
|
|
42
|
+
} from "./components.tsx";
|
|
43
|
+
import { Col, Row, Section, Style, Tab, Table, Tabs, Text } from "./primitives.tsx";
|
|
44
|
+
import { attemptListData, metricScatterData } from "./compute.ts";
|
|
45
|
+
import { costUSD, defineMetric, endToEndPassRate } from "./metrics.ts";
|
|
46
|
+
import builtInReport from "./built-in/index.tsx";
|
|
47
|
+
|
|
48
|
+
// ───────────────────────── fake 数据 ─────────────────────────
|
|
49
|
+
|
|
50
|
+
let seq = 0;
|
|
51
|
+
|
|
52
|
+
function res(id: string, verdict: Verdict, extra: Partial<EvalResult> = {}): EvalResult {
|
|
53
|
+
seq += 1;
|
|
54
|
+
return {
|
|
55
|
+
id,
|
|
56
|
+
agent: "agent-x",
|
|
57
|
+
verdict,
|
|
58
|
+
attempt: 0,
|
|
59
|
+
startedAt: `2026-07-01T00:00:00.${String(seq).padStart(6, "0")}Z`,
|
|
60
|
+
durationMs: 1000,
|
|
61
|
+
assertions: [],
|
|
62
|
+
...extra,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
196
65
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
66
|
+
let runSeq = 0;
|
|
67
|
+
|
|
68
|
+
function snap(spec: {
|
|
69
|
+
experimentId: string;
|
|
70
|
+
results: EvalResult[];
|
|
71
|
+
agent?: string;
|
|
72
|
+
model?: string;
|
|
73
|
+
name?: Snapshot["name"];
|
|
74
|
+
runStartedAt?: string;
|
|
75
|
+
}): Snapshot {
|
|
76
|
+
runSeq += 1;
|
|
77
|
+
const startedAt = spec.runStartedAt ?? `2026-06-01T00:00:00.${String(runSeq).padStart(3, "0")}Z`;
|
|
78
|
+
const snapshot = {
|
|
79
|
+
experimentId: spec.experimentId,
|
|
80
|
+
startedAt,
|
|
81
|
+
completedAt: startedAt,
|
|
82
|
+
agent: spec.agent ?? "agent-x",
|
|
83
|
+
model: spec.model,
|
|
84
|
+
name: spec.name,
|
|
85
|
+
schemaVersion: 1,
|
|
86
|
+
dir: `/results/exp/snap-${runSeq}`,
|
|
87
|
+
} as Snapshot;
|
|
88
|
+
const attempts: AttemptHandle[] = spec.results.map((r) => ({
|
|
89
|
+
evalId: r.id,
|
|
90
|
+
experimentId: spec.experimentId,
|
|
91
|
+
result: r,
|
|
92
|
+
ref: { snapshot: `exp/snap-${runSeq}`, attempt: `${r.id}/a${r.attempt}` },
|
|
93
|
+
snapshot,
|
|
94
|
+
events: async () => null,
|
|
95
|
+
trace: async () => null,
|
|
96
|
+
o11y: async () => null,
|
|
97
|
+
agentSetup: async () => null,
|
|
98
|
+
diff: async () => null,
|
|
99
|
+
sources: async () => null,
|
|
100
|
+
}));
|
|
101
|
+
const evals = new Map<string, AttemptHandle[]>();
|
|
102
|
+
for (const attempt of attempts) evals.set(attempt.evalId, [...(evals.get(attempt.evalId) ?? []), attempt]);
|
|
103
|
+
snapshot.evals = [...evals.entries()].map(([id, list]) => ({ id, attempts: list }));
|
|
104
|
+
snapshot.attempts = attempts;
|
|
105
|
+
return snapshot;
|
|
106
|
+
}
|
|
207
107
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
const t = text(<GroupSummary data={missingCost} />);
|
|
212
|
-
for (const face of [h, t]) {
|
|
213
|
-
expect(face).toContain("no data");
|
|
214
|
-
expect(face).not.toContain("$0");
|
|
215
|
-
}
|
|
216
|
-
});
|
|
108
|
+
function scopeOf(snapshots: Snapshot[], warnings: ScopeWarning[] = []): Scope {
|
|
109
|
+
return makeScope("current-evals", snapshots, warnings);
|
|
110
|
+
}
|
|
217
111
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
112
|
+
function resultsOf(snapshots: Snapshot[]): Results {
|
|
113
|
+
const byId = new Map<string, Snapshot[]>();
|
|
114
|
+
for (const s of snapshots) byId.set(s.experimentId, [...(byId.get(s.experimentId) ?? []), s]);
|
|
115
|
+
const experiments = [...byId.entries()].map(([id, snaps]) => {
|
|
116
|
+
const sorted = [...snaps].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
|
117
|
+
return {
|
|
118
|
+
id,
|
|
119
|
+
snapshots: sorted,
|
|
120
|
+
latest: sorted[0]!,
|
|
121
|
+
evalIds: [...new Set(sorted.flatMap((s) => s.evals.map((e) => e.id)))].sort(),
|
|
223
122
|
};
|
|
224
|
-
const h = renderToStaticMarkup(<GroupSummary data={allPassed} />);
|
|
225
|
-
const t = text(<GroupSummary data={allPassed} />);
|
|
226
|
-
expect(h).not.toContain("nre-verdict-errored");
|
|
227
|
-
expect(h).not.toContain(">errored<");
|
|
228
|
-
expect(t).not.toContain("errored");
|
|
229
|
-
// failed 计数依旧渲染,哪怕是 0 —— 数据字段本身从不省略,只是 errored 的展示片段在 0 时省略
|
|
230
|
-
expect(h).toContain(">failed<");
|
|
231
|
-
expect(t).toContain("failed 0");
|
|
232
|
-
});
|
|
233
|
-
|
|
234
|
-
it("存在错误:两面都显示 errored 计数(与 groupSummaryData 基础 fixture 的 errored:1 一致)", () => {
|
|
235
|
-
expect(html).toContain("nre-verdict-errored");
|
|
236
|
-
expect(html).toContain(">errored<");
|
|
237
|
-
expect(term).toContain("errored 1");
|
|
238
|
-
});
|
|
239
|
-
|
|
240
|
-
it("zh-CN locale:web 面走中文字典(通过率/失败/错误/总成本/实验数),text 面同理;display 数字不本地化", () => {
|
|
241
|
-
const zhHtml = renderToStaticMarkup(<GroupSummary data={groupSummaryData} locale="zh-CN" />);
|
|
242
|
-
expect(zhHtml).toContain("通过率");
|
|
243
|
-
expect(zhHtml).toContain("失败");
|
|
244
|
-
expect(zhHtml).toContain("错误");
|
|
245
|
-
expect(zhHtml).toContain("总成本");
|
|
246
|
-
expect(zhHtml).toContain("实验数");
|
|
247
|
-
expect(zhHtml).toContain("60%"); // display 本身不随 locale 变
|
|
248
|
-
|
|
249
|
-
const zhCtx = createTextContext({ width: 80, locale: "zh-CN" });
|
|
250
|
-
const zhTerm = renderNodeToText(<GroupSummary data={groupSummaryData} />, zhCtx);
|
|
251
|
-
expect(zhTerm).toContain("通过率");
|
|
252
|
-
expect(zhTerm).toContain("失败 1");
|
|
253
|
-
expect(zhTerm).toContain("错误 1");
|
|
254
|
-
expect(zhTerm).toContain("60%");
|
|
255
|
-
});
|
|
256
|
-
});
|
|
257
|
-
|
|
258
|
-
describe("MetricTable 双面", () => {
|
|
259
|
-
const html = renderToStaticMarkup(<MetricTable data={tableData} />);
|
|
260
|
-
const term = text(<MetricTable data={tableData} />);
|
|
261
|
-
|
|
262
|
-
it("text 面形态:对齐列 + 覆盖率角标 + 缺数据 —", () => {
|
|
263
|
-
expect(term).toMatchInlineSnapshot(`
|
|
264
|
-
"agent pass rate code lines
|
|
265
|
-
codex 50% —
|
|
266
|
-
bub 87% 120 lines 5/6"
|
|
267
|
-
`);
|
|
268
|
-
});
|
|
269
|
-
|
|
270
|
-
it("两面同口径:行序一致(预排即终排)、角标一致、缺数据不补 0", () => {
|
|
271
|
-
// 行序:codex 在 bub 前(数据侧顺序,两面都不重排)
|
|
272
|
-
expect(html.indexOf(">codex<")).toBeLessThan(html.indexOf(">bub<"));
|
|
273
|
-
expect(term.indexOf("codex")).toBeLessThan(term.indexOf("bub"));
|
|
274
|
-
// 覆盖率角标 5/6 两面都在
|
|
275
|
-
expect(html).toContain("5/6");
|
|
276
|
-
expect(term).toContain("5/6");
|
|
277
|
-
// 全 null 格子:web 是 no data 文案,text 是 —,都绝不画 0
|
|
278
|
-
expect(html).toContain("no data");
|
|
279
|
-
expect(term).toContain("—");
|
|
280
|
-
});
|
|
281
|
-
});
|
|
282
|
-
|
|
283
|
-
describe("MetricMatrix 双面", () => {
|
|
284
|
-
const html = renderToStaticMarkup(<MetricMatrix data={matrixData} />);
|
|
285
|
-
const term = text(<MetricMatrix data={matrixData} />);
|
|
286
|
-
|
|
287
|
-
it("text 面形态:稀疏格子 — + 下钻命令指向最值得看的一行", () => {
|
|
288
|
-
expect(term).toMatchInlineSnapshot(`
|
|
289
|
-
"eval bub codex
|
|
290
|
-
algebra/quadratic 100% 0%
|
|
291
|
-
geometry/angles 50% —
|
|
292
|
-
|
|
293
|
-
next: niceeval show geometry/angles"
|
|
294
|
-
`);
|
|
295
123
|
});
|
|
124
|
+
return {
|
|
125
|
+
experiments,
|
|
126
|
+
skipped: [],
|
|
127
|
+
latest: () => makeScope("latest-snapshots", experiments.map((e) => e.latest), []),
|
|
128
|
+
current: () => makeScope("current-evals", experiments.map((e) => e.latest), []),
|
|
129
|
+
} as unknown as Results;
|
|
130
|
+
}
|
|
296
131
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
132
|
+
/** 管线便捷入口:resolve + validate + text 渲染,报告声明用最小 meta。 */
|
|
133
|
+
async function renderTreeText(node: ReportNode, scope: Scope, width = 100): Promise<string> {
|
|
134
|
+
const definition = defineReport(node);
|
|
135
|
+
const resolved = await resolveReportTree(pickReportPage(definition).content, {
|
|
136
|
+
scope,
|
|
137
|
+
results: resultsOf(scope.snapshots),
|
|
138
|
+
report: buildReportMeta(definition, scope, "report"),
|
|
139
|
+
memo: new ResolveMemo(),
|
|
140
|
+
});
|
|
141
|
+
validateReportTree(resolved);
|
|
142
|
+
return renderNodeToText(resolved, createTextContext({ width }));
|
|
143
|
+
}
|
|
305
144
|
|
|
306
|
-
|
|
307
|
-
const html = renderToStaticMarkup(<MetricBars data={matrixData} />);
|
|
308
|
-
const term = text(<MetricBars data={matrixData} />);
|
|
309
|
-
|
|
310
|
-
it("text 面形态:一组条 = 一个 row 键,条长即刻度,缺数据 —", () => {
|
|
311
|
-
expect(term).toMatchInlineSnapshot(`
|
|
312
|
-
"algebra/quadratic
|
|
313
|
-
bub ████████████████████ 100%
|
|
314
|
-
codex ░░░░░░░░░░░░░░░░░░░░ 0%
|
|
315
|
-
geometry/angles
|
|
316
|
-
bub ██████████░░░░░░░░░░ 50%
|
|
317
|
-
codex —"
|
|
318
|
-
`);
|
|
319
|
-
});
|
|
145
|
+
// ───────────────────────── spec / data 双形态 ─────────────────────────
|
|
320
146
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
147
|
+
describe("spec 形态与 data 形态", () => {
|
|
148
|
+
const scatterScope = () =>
|
|
149
|
+
scopeOf([
|
|
150
|
+
snap({
|
|
151
|
+
experimentId: "cmp/a",
|
|
152
|
+
agent: "bub",
|
|
153
|
+
results: [res("q", "passed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.2 } })],
|
|
154
|
+
}),
|
|
155
|
+
snap({
|
|
156
|
+
experimentId: "cmp/b",
|
|
157
|
+
agent: "codex",
|
|
158
|
+
results: [res("q", "failed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.1 } })],
|
|
159
|
+
}),
|
|
160
|
+
]);
|
|
161
|
+
|
|
162
|
+
it("spec 形态与「先手工调 *Data 再传 data」严格等价:两棵树渲染深等", async () => {
|
|
163
|
+
const scope = scatterScope();
|
|
164
|
+
const options = { points: "experiment", series: "agent", x: costUSD, y: endToEndPassRate } as const;
|
|
165
|
+
const specText = await renderTreeText(
|
|
166
|
+
<MetricScatter points="experiment" series="agent" x={costUSD} y={endToEndPassRate} />,
|
|
167
|
+
scope,
|
|
168
|
+
);
|
|
169
|
+
const data = await metricScatterData(scope, options);
|
|
170
|
+
const dataText = await renderTreeText(<MetricScatter data={data} />, scope);
|
|
171
|
+
expect(specText).toBe(dataText);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("同一组件同时给 data 与 spec 字段报完整用户反馈,不静默取一边", async () => {
|
|
175
|
+
const scope = scatterScope();
|
|
176
|
+
const data = await metricScatterData(scope, { points: "experiment", x: costUSD, y: endToEndPassRate });
|
|
177
|
+
await expect(
|
|
178
|
+
renderTreeText(
|
|
179
|
+
// @ts-expect-error data 与 spec 字段互斥,类型层已拒绝;这里模拟无类型 JS 输入
|
|
180
|
+
<MetricScatter data={data} points="experiment" x={costUSD} y={endToEndPassRate} />,
|
|
181
|
+
scope,
|
|
182
|
+
),
|
|
183
|
+
).rejects.toThrow(/both `data` and spec/);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("input 省略时取宿主注入的 Scope;显式 input 覆盖数据来源", async () => {
|
|
187
|
+
const a = snap({ experimentId: "in/a", results: [res("q", "passed")] });
|
|
188
|
+
const b = snap({ experimentId: "in/b", results: [res("q", "failed")] });
|
|
189
|
+
const scope = scopeOf([a, b]);
|
|
190
|
+
const all = await renderTreeText(<ScopeSummary />, scope);
|
|
191
|
+
expect(all).toContain("2 experiments");
|
|
192
|
+
const narrowed = await renderTreeText(
|
|
193
|
+
<ScopeSummary input={scope.filter((s) => s.experimentId === "in/a")} />,
|
|
194
|
+
scope,
|
|
195
|
+
);
|
|
196
|
+
expect(narrowed).toContain("1 experiment ·");
|
|
197
|
+
// input 也可以是手挑的 Snapshot[](按快照出行)
|
|
198
|
+
const snapshotsOnly = await renderTreeText(
|
|
199
|
+
<MetricTable input={[a]} rows="snapshot" columns={[endToEndPassRate]} />,
|
|
200
|
+
scope,
|
|
201
|
+
140,
|
|
202
|
+
);
|
|
203
|
+
expect(snapshotsOnly).toContain("in/a @");
|
|
204
|
+
expect(snapshotsOnly).not.toContain("in/b");
|
|
328
205
|
});
|
|
329
|
-
});
|
|
330
206
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
"agent total algebra geometry
|
|
338
|
-
bub 78.5/100 14/16 (1 missing) 3/4
|
|
339
|
-
codex 52/100 9/16 1.4/4 (2 missing)
|
|
340
|
-
weights: algebra/ ×2 · others ×1"
|
|
341
|
-
`);
|
|
342
|
-
});
|
|
207
|
+
it("data 结构校验:字段改名前的旧 JSON 报错且文案含版本漂移提示;round-trip 的同版本 JSON 照常渲染", async () => {
|
|
208
|
+
const scope = scatterScope();
|
|
209
|
+
const table = { dimension: "agent", columns: [], rows: [] }; // 旧形状:dimension 而非 rowDimension
|
|
210
|
+
await expect(renderTreeText(<MetricTable data={table as never} />, scope)).rejects.toThrow(
|
|
211
|
+
/does not match the current TableData shape[\s\S]*different niceeval version/,
|
|
212
|
+
);
|
|
343
213
|
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
expect(html).toContain("
|
|
348
|
-
expect(term).toContain("algebra/ ×2");
|
|
214
|
+
const fresh = await metricScatterData(scope, { points: "experiment", x: costUSD, y: endToEndPassRate });
|
|
215
|
+
const roundTrip = JSON.parse(JSON.stringify(fresh));
|
|
216
|
+
const html = renderToStaticMarkup(<MetricScatter data={roundTrip} />);
|
|
217
|
+
expect(html).toContain("nre-metric-scatter");
|
|
349
218
|
});
|
|
350
219
|
|
|
351
|
-
it("
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
220
|
+
it("text 面散点轴方向跟随 better:成本轴反向(便宜在右)、提示恒为右上;x 无 better 时整图无提示", async () => {
|
|
221
|
+
// cmp/a:cost $0.20、通过;cmp/b:cost $0.10、失败。cost better:"lower" → 轴反向,
|
|
222
|
+
// 便宜的 cmp/b(标记 B)落在贵的 cmp/a(标记 A)右侧;刻度仍显示真实值(右端是更小的 $0.10)。
|
|
223
|
+
const scope = scatterScope();
|
|
224
|
+
const text = await renderTreeText(
|
|
225
|
+
<MetricScatter points="experiment" x={costUSD} y={endToEndPassRate} />,
|
|
226
|
+
scope,
|
|
227
|
+
);
|
|
228
|
+
const markCol = (mark: string): number => {
|
|
229
|
+
const line = text.split("\n").find((l) => l.includes(mark));
|
|
230
|
+
expect(line, `plot row with mark ${mark}`).toBeTruthy();
|
|
231
|
+
return line!.indexOf(mark);
|
|
358
232
|
};
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
expect(
|
|
362
|
-
expect(
|
|
233
|
+
expect(markCol("B")).toBeGreaterThan(markCol("A"));
|
|
234
|
+
expect(text).toContain("better → upper right");
|
|
235
|
+
expect(text).toContain("$0.10");
|
|
236
|
+
expect(text).toContain("$0.20");
|
|
237
|
+
|
|
238
|
+
// x 未声明 better:轴正向、整图无方向提示(组件不猜「更好」朝哪边)。
|
|
239
|
+
const rawCost = defineMetric({
|
|
240
|
+
name: "rawCost",
|
|
241
|
+
unit: "$",
|
|
242
|
+
value: (attempt) => attempt.result.usage?.costUSD ?? null,
|
|
243
|
+
});
|
|
244
|
+
const noHint = await renderTreeText(
|
|
245
|
+
<MetricScatter points="experiment" x={rawCost} y={endToEndPassRate} />,
|
|
246
|
+
scope,
|
|
247
|
+
);
|
|
248
|
+
expect(noHint).not.toContain("better →");
|
|
249
|
+
// 正向轴:贵的 cmp/a(A)在右。
|
|
250
|
+
const noHintCol = (mark: string): number => {
|
|
251
|
+
const line = noHint.split("\n").find((l) => l.includes(mark));
|
|
252
|
+
expect(line, `plot row with mark ${mark}`).toBeTruthy();
|
|
253
|
+
return line!.indexOf(mark);
|
|
254
|
+
};
|
|
255
|
+
expect(noHintCol("A")).toBeGreaterThan(noHintCol("B"));
|
|
363
256
|
});
|
|
364
257
|
});
|
|
365
258
|
|
|
366
|
-
|
|
367
|
-
const html = renderToStaticMarkup(<MetricScatter data={scatterData} />);
|
|
368
|
-
const term = text(<MetricScatter data={scatterData} />);
|
|
369
|
-
|
|
370
|
-
it("text 面形态:字符坐标图 + 图例 + 缺数据注脚", () => {
|
|
371
|
-
expect(term).toMatchInlineSnapshot(`
|
|
372
|
-
" pass rate ↑
|
|
373
|
-
90%│B····
|
|
374
|
-
│ ·········
|
|
375
|
-
│ ··········
|
|
376
|
-
│ ·········
|
|
377
|
-
│ ·········
|
|
378
|
-
│ ·········
|
|
379
|
-
│ C ··········
|
|
380
|
-
│ ·········
|
|
381
|
-
50%│ ····A
|
|
382
|
-
└───────────────────────────────────────────────────────────────────────────→ cost (axis reversed: right = better)
|
|
383
|
-
$10.00 $5.00
|
|
384
|
-
|
|
385
|
-
better → upper right
|
|
386
|
-
A compare/bub-low B compare/bub-high C compare/codex-mid
|
|
387
|
-
1 point missing data"
|
|
388
|
-
`);
|
|
389
|
-
});
|
|
259
|
+
// ───────────────────────── resolve 记忆化 ─────────────────────────
|
|
390
260
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
const
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
261
|
+
describe("resolve 记忆化", () => {
|
|
262
|
+
it("Matrix 与 Bars 同 spec 时计算只发生一次;不同 spec 各自计算", async () => {
|
|
263
|
+
let calls = 0;
|
|
264
|
+
const counted = defineMetric({
|
|
265
|
+
name: "counted",
|
|
266
|
+
value: () => {
|
|
267
|
+
calls += 1;
|
|
268
|
+
return 1;
|
|
269
|
+
},
|
|
270
|
+
});
|
|
271
|
+
const scope = scopeOf([snap({ experimentId: "memo/a", results: [res("q", "passed")] })]);
|
|
272
|
+
await renderTreeText(
|
|
273
|
+
<Col>
|
|
274
|
+
<MetricMatrix rows="eval" columns="agent" cell={counted} />
|
|
275
|
+
<MetricBars rows="eval" columns="agent" cell={counted} />
|
|
276
|
+
</Col>,
|
|
277
|
+
scope,
|
|
278
|
+
);
|
|
279
|
+
expect(calls).toBe(1); // 一个 attempt,矩阵只算一遍
|
|
407
280
|
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
│ ··················B
|
|
418
|
-
│
|
|
419
|
-
│
|
|
420
|
-
│
|
|
421
|
-
│A··················
|
|
422
|
-
│ ·····································
|
|
423
|
-
30%│ ··················A
|
|
424
|
-
└───────────────────────────────────────────────────────────────────────────→ Simulated latency
|
|
425
|
-
100ms 300ms
|
|
426
|
-
|
|
427
|
-
A 1 agents B 16 agents
|
|
428
|
-
1 point missing data"
|
|
429
|
-
`);
|
|
281
|
+
calls = 0;
|
|
282
|
+
await renderTreeText(
|
|
283
|
+
<Col>
|
|
284
|
+
<MetricMatrix rows="eval" columns="agent" cell={counted} />
|
|
285
|
+
<MetricMatrix rows="eval" columns="model" cell={counted} />
|
|
286
|
+
</Col>,
|
|
287
|
+
scope,
|
|
288
|
+
);
|
|
289
|
+
expect(calls).toBe(2); // spec 不同(columns 维度不同):各自计算
|
|
430
290
|
});
|
|
431
291
|
|
|
432
|
-
it("
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
292
|
+
it("不同 input 各自计算;字段相同但实例不同的 Metric 各自计算、不报错", async () => {
|
|
293
|
+
let calls = 0;
|
|
294
|
+
const value = () => {
|
|
295
|
+
calls += 1;
|
|
296
|
+
return 1;
|
|
297
|
+
};
|
|
298
|
+
const m1 = defineMetric({ name: "twin", value });
|
|
299
|
+
const m2 = defineMetric({ name: "twin", value: (attempt) => value.call(null) }); // 引用不同的等价定义
|
|
300
|
+
const a = snap({ experimentId: "memo/in-a", results: [res("q", "passed")] });
|
|
301
|
+
const b = snap({ experimentId: "memo/in-b", results: [res("q", "passed")] });
|
|
302
|
+
const scope = scopeOf([a, b]);
|
|
303
|
+
await renderTreeText(
|
|
304
|
+
<Col>
|
|
305
|
+
<MetricMatrix input={[a]} rows="eval" columns="agent" cell={m1} />
|
|
306
|
+
<MetricMatrix input={[b]} rows="eval" columns="agent" cell={m1} />
|
|
307
|
+
<MetricMatrix input={[a]} rows="eval" columns="agent" cell={m2} />
|
|
308
|
+
</Col>,
|
|
309
|
+
scope,
|
|
310
|
+
);
|
|
311
|
+
expect(calls).toBe(3); // 两个 input 各一次 + 不同实例的 Metric 再一次
|
|
436
312
|
});
|
|
437
313
|
});
|
|
438
314
|
|
|
439
|
-
|
|
440
|
-
const html = renderToStaticMarkup(<DeltaTable data={deltaData} />);
|
|
441
|
-
const term = text(<DeltaTable data={deltaData} />);
|
|
315
|
+
// ───────────────────────── 组合组件与树形状 ─────────────────────────
|
|
442
316
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
317
|
+
describe("组合组件(函数形态)", () => {
|
|
318
|
+
it("resolve 阶段以 (props, ctx) 调用并递归展开;与手写等价树渲染相同;async 可用", async () => {
|
|
319
|
+
const scope = scopeOf([snap({ experimentId: "compose/a", results: [res("q", "passed")] })]);
|
|
320
|
+
const Composed = defineComponent(async (_props: Record<never, never>, ctx) => (
|
|
321
|
+
<Section title="wrapped">
|
|
322
|
+
<ScopeSummary input={ctx.scope} />
|
|
323
|
+
</Section>
|
|
324
|
+
));
|
|
325
|
+
const composed = await renderTreeText(<Composed />, scope);
|
|
326
|
+
const manual = await renderTreeText(
|
|
327
|
+
<Section title="wrapped">
|
|
328
|
+
<ScopeSummary />
|
|
329
|
+
</Section>,
|
|
330
|
+
scope,
|
|
331
|
+
);
|
|
332
|
+
expect(composed).toBe(manual);
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
it("ctx.results 可自行挑 Snapshot[] 喂 input;ctx.report 携带走完回退链的 title 与当前 pageId", async () => {
|
|
336
|
+
const named = snap({ experimentId: "hist/a", name: "Memory Evals", results: [res("q", "passed")] });
|
|
337
|
+
const scope = scopeOf([named]);
|
|
338
|
+
const Meta = defineComponent((_props: Record<never, never>, ctx) => {
|
|
339
|
+
const history = ctx.results.experiments[0]!.snapshots;
|
|
340
|
+
return (
|
|
341
|
+
<Col>
|
|
342
|
+
<Text>{`title=${typeof ctx.report.title === "string" ? ctx.report.title : "?"} page=${ctx.report.pageId}`}</Text>
|
|
343
|
+
<ScopeSummary input={history} />
|
|
344
|
+
</Col>
|
|
345
|
+
);
|
|
346
|
+
});
|
|
347
|
+
const definition = defineReport({ pages: [{ id: "meta", title: "Meta", content: <Meta /> }] });
|
|
348
|
+
const text = await renderReportToText(definition, { scope, results: resultsOf([named]) }, { pageId: "meta" });
|
|
349
|
+
// 声明没给 title → 唯一快照 name 回退
|
|
350
|
+
expect(text).toContain("title=Memory Evals page=meta");
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
it("同层 sibling 并行取数且输出保持声明顺序:慢 resolve 在前不换位", async () => {
|
|
354
|
+
const order: string[] = [];
|
|
355
|
+
const Slow = defineComponent<{ label: string }, { label: string }>({
|
|
356
|
+
resolve: async (props) => {
|
|
357
|
+
await new Promise((r) => setTimeout(r, 25));
|
|
358
|
+
order.push("slow-resolved");
|
|
359
|
+
return props;
|
|
360
|
+
},
|
|
361
|
+
web: ({ label }) => <p>{label}</p>,
|
|
362
|
+
text: ({ label }) => label,
|
|
363
|
+
});
|
|
364
|
+
const Fast = defineComponent<{ label: string }, { label: string }>({
|
|
365
|
+
resolve: (props) => {
|
|
366
|
+
order.push("fast-resolved");
|
|
367
|
+
return props;
|
|
368
|
+
},
|
|
369
|
+
web: ({ label }) => <p>{label}</p>,
|
|
370
|
+
text: ({ label }) => label,
|
|
371
|
+
});
|
|
372
|
+
const scope = scopeOf([]);
|
|
373
|
+
const text = await renderTreeText(
|
|
374
|
+
<Col>
|
|
375
|
+
<Slow label="first" />
|
|
376
|
+
<Fast label="second" />
|
|
377
|
+
</Col>,
|
|
378
|
+
scope,
|
|
379
|
+
);
|
|
380
|
+
expect(order[0]).toBe("fast-resolved"); // 真并行:快的先完成
|
|
381
|
+
expect(text.indexOf("first")).toBeLessThan(text.indexOf("second")); // 输出仍按声明序
|
|
458
382
|
});
|
|
459
383
|
});
|
|
460
384
|
|
|
461
|
-
describe("
|
|
462
|
-
const
|
|
463
|
-
const term = text(<AttemptList items={attemptListItems} />);
|
|
385
|
+
describe("ReportNode 形状与非法节点", () => {
|
|
386
|
+
const scope = () => scopeOf([snap({ experimentId: "node/a", results: [res("q", "passed")] })]);
|
|
464
387
|
|
|
465
|
-
it("
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
const longEvidence = "x".repeat(2_000);
|
|
486
|
-
const item = {
|
|
487
|
-
...attemptListItems[0],
|
|
488
|
-
assertions: [{ ...attemptListItems[0].assertions[0], evidence: longEvidence }],
|
|
489
|
-
};
|
|
490
|
-
const out = text(<AttemptList items={[item]} />);
|
|
491
|
-
expect(out).toContain(item.locator);
|
|
492
|
-
expect(out.replace(/\s+/g, " ")).toContain(`open ${item.locator} for full evidence`);
|
|
493
|
-
expect(out).not.toContain(longEvidence);
|
|
494
|
-
expect(out.length).toBeLessThan(1_000);
|
|
388
|
+
it("数组 / Fragment 展平保序;null / undefined / boolean 渲染为空", async () => {
|
|
389
|
+
const groups = ["g1", "g2"];
|
|
390
|
+
const text = await renderTreeText(
|
|
391
|
+
<Col>
|
|
392
|
+
{groups.map((g) => (
|
|
393
|
+
<Section key={g} title={g}>
|
|
394
|
+
<Text>{`body of ${g}`}</Text>
|
|
395
|
+
</Section>
|
|
396
|
+
))}
|
|
397
|
+
<>
|
|
398
|
+
<Text>in fragment</Text>
|
|
399
|
+
</>
|
|
400
|
+
{false && <Text>hidden</Text>}
|
|
401
|
+
{null}
|
|
402
|
+
</Col>,
|
|
403
|
+
scope(),
|
|
404
|
+
);
|
|
405
|
+
expect(text.indexOf("g1")).toBeLessThan(text.indexOf("g2"));
|
|
406
|
+
expect(text).toContain("in fragment");
|
|
407
|
+
expect(text).not.toContain("hidden");
|
|
495
408
|
});
|
|
496
409
|
|
|
497
|
-
it("
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
expect(lines.some((l) => /^ \S/.test(l))).toBe(true); // 断言行缩进两格
|
|
502
|
-
expect(lines.some((l) => /^ \S/.test(l))).toBe(true); // 断言 detail 再缩进两格,是卡片的层级,不是列
|
|
410
|
+
it("裸字符串在树校验时按完整用户反馈拒绝并指引包 <Text>", async () => {
|
|
411
|
+
await expect(renderTreeText(<Col>{"free text" as unknown as ReportNode}</Col>, scope())).rejects.toThrow(
|
|
412
|
+
/bare string[\s\S]*<Text>/,
|
|
413
|
+
);
|
|
503
414
|
});
|
|
504
|
-
});
|
|
505
415
|
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
expect(html).toContain(piece);
|
|
513
|
-
expect(term).toContain(piece);
|
|
514
|
-
}
|
|
416
|
+
it("React 组件 / 未包装函数与 HTML intrinsic 在展开遇到时拒绝", async () => {
|
|
417
|
+
const Plain = ({ label }: { label: string }) => <p>{label}</p>;
|
|
418
|
+
await expect(renderTreeText(<Plain label="x" />, scope())).rejects.toThrow(
|
|
419
|
+
/not a report component[\s\S]*defineComponent/,
|
|
420
|
+
);
|
|
421
|
+
await expect(renderTreeText(<div>x</div>, scope())).rejects.toThrow(/raw HTML <div>/);
|
|
515
422
|
});
|
|
516
423
|
|
|
517
|
-
it("
|
|
518
|
-
|
|
519
|
-
|
|
424
|
+
it("validateReportTree 拒绝缺任一渲染面的组件(无类型 JS 绕过 defineComponent 时)", () => {
|
|
425
|
+
const single = Object.assign(() => null, {
|
|
426
|
+
[Symbol.for("niceeval.report.faces")]: { web: () => null }, // 缺 text 面
|
|
427
|
+
});
|
|
428
|
+
expect(() => validateReportTree({ type: single, props: {} })).toThrow(/missing its text face/);
|
|
429
|
+
expect(() => validateReportTree({ type: "div", props: {} })).toThrow(/raw HTML <div>/);
|
|
520
430
|
});
|
|
521
431
|
|
|
522
|
-
it("
|
|
523
|
-
|
|
524
|
-
expect(blocks).toHaveLength(evalListItems.length);
|
|
525
|
-
expect(term.split("\n").some((l) => /^ \S/.test(l))).toBe(true); // 展开到 attempt 的行缩进两格
|
|
432
|
+
it("defineComponent 对象形态缺 text 或 web 在定义时报完整用户反馈", () => {
|
|
433
|
+
expect(() => defineComponent({ web: () => null } as never)).toThrow(/both faces/);
|
|
526
434
|
});
|
|
527
435
|
});
|
|
528
436
|
|
|
529
|
-
|
|
530
|
-
const html = renderToStaticMarkup(<ExperimentList items={experimentListItems} />);
|
|
531
|
-
const term = text(<ExperimentList items={experimentListItems} />);
|
|
437
|
+
// ───────────────────────── 双面同源 ─────────────────────────
|
|
532
438
|
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
439
|
+
describe("text/web 双面同源", () => {
|
|
440
|
+
it("双面显示同一份解析终值、覆盖率与 warning;宿主在报告树外统一显示 warning", async () => {
|
|
441
|
+
const s = snap({ experimentId: "dual/a", results: [res("q1", "passed"), res("q2", "skipped")] });
|
|
442
|
+
const warning: ScopeWarning = {
|
|
443
|
+
kind: "unfinished-snapshot",
|
|
444
|
+
experimentId: "dual/a",
|
|
445
|
+
startedAt: s.startedAt,
|
|
446
|
+
dir: s.dir,
|
|
447
|
+
message: "snapshot is incomplete",
|
|
448
|
+
command: "niceeval exp dual/a",
|
|
449
|
+
};
|
|
450
|
+
const scope = scopeOf([s], [warning]);
|
|
451
|
+
const definition = defineReport(<ScopeSummary />);
|
|
452
|
+
const ctx = { scope, results: resultsOf([s]) };
|
|
453
|
+
const text = await renderReportToText(definition, ctx);
|
|
454
|
+
const html = await renderReportToStaticHtml(definition, ctx);
|
|
455
|
+
for (const face of [text, html]) {
|
|
456
|
+
expect(face).toContain("100%"); // endToEndPassRate:skipped 不稀释
|
|
457
|
+
expect(face).toContain("1/2"); // samples/total 覆盖率
|
|
458
|
+
expect(face).toContain("snapshot is incomplete");
|
|
537
459
|
}
|
|
538
|
-
//
|
|
539
|
-
expect(html).toContain("
|
|
540
|
-
expect(
|
|
541
|
-
});
|
|
542
|
-
|
|
543
|
-
it("
|
|
544
|
-
const
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
460
|
+
// web 面把警告的 command 渲染为可复制命令
|
|
461
|
+
expect(html).toContain("niceeval exp dual/a");
|
|
462
|
+
expect(html).toContain("nre-warning-command");
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
it("web 排序/过滤只改变浏览状态:有无 filter prop 数值与行集合相同", async () => {
|
|
466
|
+
const scope = scopeOf([snap({ experimentId: "f/a", results: [res("q", "passed")] })]);
|
|
467
|
+
const definition = (filter: boolean) =>
|
|
468
|
+
defineReport(<MetricTable rows="experiment" columns={[endToEndPassRate]} filter={filter} />);
|
|
469
|
+
const ctx = { scope, results: resultsOf(scope.snapshots) };
|
|
470
|
+
const plain = await renderReportToStaticHtml(definition(false), ctx);
|
|
471
|
+
const filtered = await renderReportToStaticHtml(definition(true), ctx);
|
|
472
|
+
const values = (html: string) => html.match(/data-sort-value="[^"]*"/g);
|
|
473
|
+
expect(values(filtered)).toEqual(values(plain));
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
it("ExperimentList text 面保持实体层级:一题两 attempt 只出现一次 Eval 标题;失败摘要只在 Attempt 子行", async () => {
|
|
477
|
+
const s = snap({
|
|
478
|
+
experimentId: "grp/exp-a",
|
|
479
|
+
results: [
|
|
480
|
+
res("algebra/retry", "failed", {
|
|
481
|
+
attempt: 0,
|
|
482
|
+
assertions: [
|
|
483
|
+
{
|
|
484
|
+
name: "equals",
|
|
485
|
+
severity: "gate",
|
|
486
|
+
outcome: "failed",
|
|
487
|
+
score: 0,
|
|
488
|
+
detail: "equals(42)",
|
|
489
|
+
expected: "42",
|
|
490
|
+
received: "41",
|
|
491
|
+
},
|
|
492
|
+
],
|
|
493
|
+
}),
|
|
494
|
+
res("algebra/retry", "passed", { attempt: 1 }),
|
|
495
|
+
],
|
|
496
|
+
});
|
|
497
|
+
const text = await renderTreeText(<ExperimentList relativeTo="grp" />, scopeOf([s]), 160);
|
|
498
|
+
expect(text.match(/algebra\/retry/g)).toHaveLength(1); // Eval 父行只出现一次
|
|
499
|
+
expect(text).toContain("├─");
|
|
500
|
+
expect(text).toContain("└─");
|
|
501
|
+
// 失败摘要只在 Attempt 子行(父行不复述)
|
|
502
|
+
expect(text.match(/equals\(42\)/g)).toHaveLength(1);
|
|
503
|
+
// relativeTo:行标签只显示末段
|
|
504
|
+
expect(text).toContain("exp-a");
|
|
550
505
|
});
|
|
551
506
|
});
|
|
552
507
|
|
|
553
|
-
// ─────────────────────────
|
|
554
|
-
|
|
555
|
-
describe("
|
|
556
|
-
it("
|
|
557
|
-
const
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
const
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
const
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
);
|
|
581
|
-
|
|
582
|
-
expect(
|
|
583
|
-
expect(
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
const
|
|
589
|
-
|
|
590
|
-
<Text>left column body</Text>
|
|
591
|
-
<Text>right column body</Text>
|
|
592
|
-
</Row>
|
|
593
|
-
);
|
|
594
|
-
const narrow = renderNodeToText(node, createTextContext({ width: 30 }));
|
|
595
|
-
expect(narrow).not.toContain("│");
|
|
596
|
-
expect(narrow).toContain("left column body\n\nright column body");
|
|
508
|
+
// ───────────────────────── FailureList ─────────────────────────
|
|
509
|
+
|
|
510
|
+
describe("FailureList", () => {
|
|
511
|
+
it("与手写组合严格等价:failed/errored、开始时间降序、limit 截断且 total 报截断前总数", async () => {
|
|
512
|
+
const s = snap({
|
|
513
|
+
experimentId: "fail/a",
|
|
514
|
+
results: [
|
|
515
|
+
res("q1", "failed", { startedAt: "2026-07-01T01:00:00.000Z" }),
|
|
516
|
+
res("q2", "errored", {
|
|
517
|
+
startedAt: "2026-07-01T03:00:00.000Z",
|
|
518
|
+
error: { code: "x", message: "boom", phase: "eval.run" },
|
|
519
|
+
}),
|
|
520
|
+
res("q3", "failed", { startedAt: "2026-07-01T02:00:00.000Z" }),
|
|
521
|
+
res("q4", "passed", { startedAt: "2026-07-01T04:00:00.000Z" }),
|
|
522
|
+
],
|
|
523
|
+
});
|
|
524
|
+
const scope = scopeOf([s]);
|
|
525
|
+
const failureText = await renderTreeText(<FailureList limit={2} />, scope);
|
|
526
|
+
|
|
527
|
+
// 手写组合:attemptListData → 过滤 → 排序 → AttemptList data 形态
|
|
528
|
+
const all = await attemptListData(scope);
|
|
529
|
+
const startedAt = new Map(s.attempts.map((a) => [a.evalId, a.result.startedAt ?? ""]));
|
|
530
|
+
const failures = all
|
|
531
|
+
.filter((x) => x.verdict === "failed" || x.verdict === "errored")
|
|
532
|
+
.sort((a, b) => (startedAt.get(b.evalId) ?? "").localeCompare(startedAt.get(a.evalId) ?? ""));
|
|
533
|
+
const manualText = await renderTreeText(<AttemptList data={failures.slice(0, 2)} total={failures.length} />, scope);
|
|
534
|
+
expect(failureText).toBe(manualText);
|
|
535
|
+
expect(failureText).toContain("(1 more not shown)"); // 3 条失败,截断到 2
|
|
536
|
+
// 最近的失败在前
|
|
537
|
+
expect(failureText.indexOf("q2")).toBeLessThan(failureText.indexOf("q3"));
|
|
538
|
+
expect(failureText).not.toContain("q4"); // passed 不进失败清单
|
|
539
|
+
});
|
|
540
|
+
|
|
541
|
+
it("失败数少于 limit 时 total 等于 data 长度,不产出截断文案", async () => {
|
|
542
|
+
const s = snap({ experimentId: "fail/few", results: [res("q1", "failed")] });
|
|
543
|
+
const text = await renderTreeText(<FailureList />, scopeOf([s]));
|
|
544
|
+
expect(text).not.toContain("more not shown");
|
|
597
545
|
});
|
|
598
546
|
});
|
|
599
547
|
|
|
600
|
-
// ───────────────────────── Table
|
|
601
|
-
//
|
|
602
|
-
// 官方的 MetricTable / MetricMatrix / Scoreboard / DeltaTable 的 text 面就建在它上面
|
|
603
|
-
// (上面那几个 describe 的内联快照即它的输出);这里验收作者直接用它时的四条契约:
|
|
604
|
-
// 显示宽度对齐(中文不撕歪)、右对齐、缺数据 —、超宽折行。
|
|
605
|
-
|
|
606
|
-
/** 某个片段的右边缘落在第几显示列 —— 右对齐验收看的就是它。 */
|
|
607
|
-
function displayEndOf(line: string, piece: string): number {
|
|
608
|
-
const at = line.indexOf(piece);
|
|
609
|
-
expect(at).toBeGreaterThanOrEqual(0);
|
|
610
|
-
return stringWidth(line.slice(0, at + piece.length));
|
|
611
|
-
}
|
|
548
|
+
// ───────────────────────── Table 与排版原语 ─────────────────────────
|
|
612
549
|
|
|
613
|
-
describe("Table
|
|
614
|
-
const
|
|
615
|
-
<Table
|
|
616
|
-
columns={[
|
|
617
|
-
{ key: "eval", header: "题目" },
|
|
618
|
-
{ key: "pass", header: "通过率", align: "right" },
|
|
619
|
-
{ key: "cost", header: "成本", align: "right" },
|
|
620
|
-
]}
|
|
621
|
-
rows={[
|
|
622
|
-
{
|
|
623
|
-
key: "记忆/写缓存",
|
|
624
|
-
locator: "@160iuj3h" as AttemptLocator,
|
|
625
|
-
cells: { eval: "记忆/写缓存", pass: "87%", cost: "$0.09" },
|
|
626
|
-
},
|
|
627
|
-
{
|
|
628
|
-
key: "浏览/表单填写",
|
|
629
|
-
locator: "@1qrdcfq8" as AttemptLocator,
|
|
630
|
-
cells: { eval: "浏览/表单填写", pass: null, cost: null },
|
|
631
|
-
},
|
|
632
|
-
]}
|
|
633
|
-
/>
|
|
634
|
-
);
|
|
635
|
-
|
|
636
|
-
it("中文列宽:按显示宽度对齐(CJK 记 2 列),不是 .length —— String.padEnd 撕歪表的那一步", () => {
|
|
637
|
-
const term = text(cjkTable);
|
|
638
|
-
expect(term).toMatchInlineSnapshot(`
|
|
639
|
-
"题目 通过率 成本 attempt
|
|
640
|
-
记忆/写缓存 87% $0.09 @160iuj3h
|
|
641
|
-
浏览/表单填写 — — @1qrdcfq8"
|
|
642
|
-
`);
|
|
643
|
-
|
|
644
|
-
const [header, first, second] = term.split("\n");
|
|
645
|
-
// 每列的右边缘落在同一显示列:中文格子多一倍显示宽度,补齐必须按显示宽度算
|
|
646
|
-
expect(displayEndOf(first, "87%")).toBe(displayEndOf(header, "通过率"));
|
|
647
|
-
expect(displayEndOf(second, "—")).toBe(displayEndOf(header, "通过率"));
|
|
648
|
-
expect(displayEndOf(first, "$0.09")).toBe(displayEndOf(header, "成本"));
|
|
649
|
-
expect(displayEndOf(first, "@160iuj3h")).toBe(displayEndOf(second, "@1qrdcfq8"));
|
|
650
|
-
// 上面那几条不是巧合:UTF-16 码元数与显示宽度在中文上就是不同的两个数,
|
|
651
|
-
// 拿 .length 补齐(或 String.prototype.padEnd)必然歪
|
|
652
|
-
expect("记忆/写缓存".length).toBe(6);
|
|
653
|
-
expect(stringWidth("记忆/写缓存")).toBe(11);
|
|
654
|
-
});
|
|
550
|
+
describe("Table 与文本排版原语", () => {
|
|
551
|
+
const ctx = () => createTextContext({ width: 100 });
|
|
655
552
|
|
|
656
|
-
it("
|
|
657
|
-
const
|
|
553
|
+
it("null 单元格与 cells 缺键都渲染成 —,不补 0", () => {
|
|
554
|
+
const text = renderNodeToText(
|
|
658
555
|
<Table
|
|
659
556
|
columns={[
|
|
660
|
-
{ key: "
|
|
661
|
-
{ key: "
|
|
557
|
+
{ key: "a", header: "A" },
|
|
558
|
+
{ key: "b", header: "B", align: "right" },
|
|
662
559
|
]}
|
|
663
560
|
rows={[
|
|
664
|
-
{ key: "
|
|
665
|
-
{ key: "
|
|
561
|
+
{ key: "r1", cells: { a: "x", b: null } },
|
|
562
|
+
{ key: "r2", cells: { a: "y" } },
|
|
666
563
|
]}
|
|
667
564
|
/>,
|
|
565
|
+
ctx(),
|
|
668
566
|
);
|
|
669
|
-
expect(
|
|
670
|
-
|
|
671
|
-
bub $9.00
|
|
672
|
-
codex $123.45"
|
|
673
|
-
`);
|
|
674
|
-
const [, first, second] = term.split("\n");
|
|
675
|
-
expect(displayEndOf(first, "$9.00")).toBe(displayEndOf(second, "$123.45"));
|
|
676
|
-
// 左对齐列的左边缘对齐(不传 align 就是 left)
|
|
677
|
-
expect(first.startsWith("bub")).toBe(true);
|
|
678
|
-
expect(second.startsWith("codex")).toBe(true);
|
|
567
|
+
expect(text.match(/—/g)!.length).toBe(2);
|
|
568
|
+
expect(text).not.toMatch(/\b0\b/);
|
|
679
569
|
});
|
|
680
570
|
|
|
681
|
-
it("
|
|
682
|
-
|
|
571
|
+
it("cells 出现未声明的 key 以完整用户反馈报错;列 key 重复报错", () => {
|
|
572
|
+
expect(() =>
|
|
573
|
+
renderNodeToText(
|
|
574
|
+
<Table columns={[{ key: "a", header: "A" }]} rows={[{ key: "r", cells: { ghost: "x" } }]} />,
|
|
575
|
+
ctx(),
|
|
576
|
+
),
|
|
577
|
+
).toThrow(/no column declares/);
|
|
578
|
+
expect(() =>
|
|
579
|
+
renderNodeToText(
|
|
580
|
+
<Table
|
|
581
|
+
columns={
|
|
582
|
+
[
|
|
583
|
+
{ key: "a", header: "A" },
|
|
584
|
+
{ key: "a", header: "A2" },
|
|
585
|
+
] as never
|
|
586
|
+
}
|
|
587
|
+
rows={[]}
|
|
588
|
+
/>,
|
|
589
|
+
ctx(),
|
|
590
|
+
),
|
|
591
|
+
).toThrow(/declared twice/);
|
|
592
|
+
});
|
|
593
|
+
|
|
594
|
+
it("带 locator 的行多出 attempt 列;全部无 locator 时不出该列", () => {
|
|
595
|
+
const withLocator = renderNodeToText(
|
|
683
596
|
<Table
|
|
684
|
-
columns={[
|
|
685
|
-
{ key: "agent", header: "agent" },
|
|
686
|
-
{ key: "pass", header: "pass", align: "right" },
|
|
687
|
-
{ key: "cost", header: "cost", align: "right" },
|
|
688
|
-
]}
|
|
597
|
+
columns={[{ key: "a", header: "A" }]}
|
|
689
598
|
rows={[
|
|
690
|
-
{ key: "
|
|
691
|
-
{ key: "
|
|
599
|
+
{ key: "r1", cells: { a: "x" }, locator: "@1abc0def" as never },
|
|
600
|
+
{ key: "r2", cells: { a: "y" } },
|
|
692
601
|
]}
|
|
693
|
-
|
|
602
|
+
/>,
|
|
603
|
+
ctx(),
|
|
694
604
|
);
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
"
|
|
699
|
-
|
|
700
|
-
claude — —"
|
|
701
|
-
`);
|
|
702
|
-
for (const face of [term, html]) {
|
|
703
|
-
expect(face).toContain("—");
|
|
704
|
-
expect(face).not.toContain("$0.00");
|
|
705
|
-
expect(face).not.toContain("0%");
|
|
706
|
-
}
|
|
707
|
-
expect(html).toContain('<td class="nre-align-right"><span class="nre-missing">—</span></td>');
|
|
708
|
-
});
|
|
709
|
-
|
|
710
|
-
it("超宽:先折最宽的左对齐列(数字列不折),压到下限仍放不下就丢列并如实报数", () => {
|
|
711
|
-
const wide = (
|
|
712
|
-
<Table
|
|
713
|
-
columns={[
|
|
714
|
-
{ key: "eval", header: "eval" },
|
|
715
|
-
{ key: "reason", header: "reason" },
|
|
716
|
-
{ key: "cost", header: "cost", align: "right" },
|
|
717
|
-
]}
|
|
718
|
-
rows={[
|
|
719
|
-
{
|
|
720
|
-
key: "weather/brooklyn",
|
|
721
|
-
cells: {
|
|
722
|
-
eval: "weather/brooklyn",
|
|
723
|
-
reason: "gate calledTool(\"get_weather\") — the tool was never called by the agent",
|
|
724
|
-
cost: "$0.04",
|
|
725
|
-
},
|
|
726
|
-
},
|
|
727
|
-
]}
|
|
728
|
-
/>
|
|
605
|
+
expect(withLocator).toContain("@1abc0def");
|
|
606
|
+
expect(withLocator).toContain("attempt");
|
|
607
|
+
const without = renderNodeToText(
|
|
608
|
+
<Table columns={[{ key: "a", header: "A" }]} rows={[{ key: "r1", cells: { a: "x" } }]} />,
|
|
609
|
+
ctx(),
|
|
729
610
|
);
|
|
730
|
-
|
|
731
|
-
// 折行:一条逻辑行铺成多条物理行,没有一行溢出 48 列
|
|
732
|
-
const lines = narrow.split("\n");
|
|
733
|
-
expect(lines.length).toBeGreaterThan(2);
|
|
734
|
-
for (const line of lines) expect(stringWidth(line)).toBeLessThanOrEqual(48);
|
|
735
|
-
// 数字列没被折:$0.04 完整出现在某一行上
|
|
736
|
-
expect(narrow).toContain("$0.04");
|
|
737
|
-
expect(narrow).toContain("never called");
|
|
738
|
-
|
|
739
|
-
// 压到下限还是放不下 → 从右侧丢列,如实报剩余列数(不静默截断)
|
|
740
|
-
const tiny = renderNodeToText(wide, createTextContext({ width: 20 }));
|
|
741
|
-
expect(tiny).toContain("(1 more column not shown)");
|
|
742
|
-
expect(tiny).not.toContain("$0.04");
|
|
611
|
+
expect(without).not.toContain("attempt");
|
|
743
612
|
});
|
|
744
613
|
|
|
745
|
-
it("
|
|
746
|
-
const
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
});
|
|
752
|
-
});
|
|
753
|
-
|
|
754
|
-
// ───────────────────────── 树校验与自定义组件 ─────────────────────────
|
|
755
|
-
|
|
756
|
-
describe("渲染前树校验", () => {
|
|
757
|
-
it("字符串 intrinsic(<div>)报错,错误指名组件路径", () => {
|
|
758
|
-
const bad = (
|
|
759
|
-
<Col>
|
|
760
|
-
<Section title="x">
|
|
761
|
-
<div>raw</div>
|
|
762
|
-
</Section>
|
|
763
|
-
</Col>
|
|
614
|
+
it("Row text 面:宽度装得下按显示宽度并排,装不下整块纵向堆叠、内容完整", () => {
|
|
615
|
+
const tree = (
|
|
616
|
+
<Row>
|
|
617
|
+
<Text>left block</Text>
|
|
618
|
+
<Text>right block</Text>
|
|
619
|
+
</Row>
|
|
764
620
|
);
|
|
765
|
-
|
|
766
|
-
|
|
621
|
+
const wide = renderNodeToText(tree, createTextContext({ width: 80 }));
|
|
622
|
+
expect(wide.split("\n")[0]).toContain("left block");
|
|
623
|
+
expect(wide.split("\n")[0]).toContain("right block");
|
|
624
|
+
const narrow = renderNodeToText(tree, createTextContext({ width: 20 }));
|
|
625
|
+
expect(narrow).toContain("left block");
|
|
626
|
+
expect(narrow).toContain("right block");
|
|
627
|
+
expect(narrow.split("\n")[0]).not.toContain("right block"); // 纵向堆叠
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
it("Style 在 text 面零输出,web 面吐 <style>(页级全局)", async () => {
|
|
631
|
+
const scope = scopeOf([]);
|
|
632
|
+
const definition = defineReport(
|
|
633
|
+
<Col>
|
|
634
|
+
<Style>{`.nre .x { color: red; }`}</Style>
|
|
635
|
+
<Text>visible</Text>
|
|
636
|
+
</Col>,
|
|
767
637
|
);
|
|
638
|
+
const hostCtx = { scope, results: resultsOf([]) };
|
|
639
|
+
const text = await renderReportToText(definition, hostCtx);
|
|
640
|
+
expect(text).toBe("visible");
|
|
641
|
+
const html = await renderReportToStaticHtml(definition, hostCtx);
|
|
642
|
+
expect(html).toContain("<style>");
|
|
768
643
|
});
|
|
644
|
+
});
|
|
769
645
|
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
646
|
+
// ───────────────────────── Tabs ─────────────────────────
|
|
647
|
+
|
|
648
|
+
describe("Tabs", () => {
|
|
649
|
+
it("两面都输出全部 tab 完整内容:web 每 tab 一个 <details> 且仅首个 open;text 按声明序分节不省略", async () => {
|
|
650
|
+
const scope = scopeOf([]);
|
|
651
|
+
const definition = defineReport(
|
|
652
|
+
<Tabs>
|
|
653
|
+
<Tab title="First">
|
|
654
|
+
<Text>alpha body</Text>
|
|
655
|
+
</Tab>
|
|
656
|
+
<Tab title="Second">
|
|
657
|
+
<Text>beta body</Text>
|
|
658
|
+
</Tab>
|
|
659
|
+
</Tabs>,
|
|
775
660
|
);
|
|
776
|
-
|
|
777
|
-
const
|
|
778
|
-
expect(()
|
|
661
|
+
const hostCtx = { scope, results: resultsOf([]) };
|
|
662
|
+
const html = await renderReportToStaticHtml(definition, hostCtx);
|
|
663
|
+
expect(html.match(/<details/g)).toHaveLength(2);
|
|
664
|
+
expect(html.match(/<details[^>]* open/g)).toHaveLength(1);
|
|
665
|
+
expect(html).toContain("alpha body");
|
|
666
|
+
expect(html).toContain("beta body");
|
|
667
|
+
const text = await renderReportToText(definition, hostCtx);
|
|
668
|
+
expect(text.indexOf("First")).toBeLessThan(text.indexOf("Second"));
|
|
669
|
+
expect(text).toContain("alpha body");
|
|
670
|
+
expect(text).toContain("beta body"); // 不丢第二个 tab
|
|
671
|
+
});
|
|
672
|
+
|
|
673
|
+
it("空 Tabs、普通组件混作直接子节点、游离 Tab 都在树校验期给出完整用户反馈", () => {
|
|
674
|
+
expect(() => validateReportTree(<Tabs>{null}</Tabs>)).toThrow(/at least one <Tab>/);
|
|
675
|
+
expect(() =>
|
|
676
|
+
validateReportTree(
|
|
677
|
+
<Tabs>
|
|
678
|
+
<Text>stray</Text>
|
|
679
|
+
</Tabs>,
|
|
680
|
+
),
|
|
681
|
+
).toThrow(/only accepts <Tab>/);
|
|
682
|
+
expect(() =>
|
|
683
|
+
validateReportTree(
|
|
684
|
+
<Col>
|
|
685
|
+
<Tab title="loose">
|
|
686
|
+
<Text>x</Text>
|
|
687
|
+
</Tab>
|
|
688
|
+
</Col>,
|
|
689
|
+
),
|
|
690
|
+
).toThrow(/direct child of <Tabs>/);
|
|
779
691
|
});
|
|
780
692
|
});
|
|
781
693
|
|
|
782
|
-
|
|
783
|
-
it("faces 两键必填,少一个面在运行时也拦住(编译期本就不过)", () => {
|
|
784
|
-
// @ts-expect-error 缺 text 面编译不过;运行时同样报错
|
|
785
|
-
expect(() => defineComponent({ web: () => null })).toThrow(/requires both faces/);
|
|
786
|
-
});
|
|
694
|
+
// ───────────────────────── defineReport 装载规范化 ─────────────────────────
|
|
787
695
|
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
}
|
|
796
|
-
const PassBars = defineComponent<{ rows: BarRow[] }>({
|
|
797
|
-
web({ rows }) {
|
|
798
|
-
return (
|
|
799
|
-
<ul className="passbars">
|
|
800
|
-
{rows.map((r) => (
|
|
801
|
-
<li key={r.key}>
|
|
802
|
-
<span>{r.key}</span>
|
|
803
|
-
<b>{r.ratio === null ? "—" : r.display}</b>
|
|
804
|
-
</li>
|
|
805
|
-
))}
|
|
806
|
-
</ul>
|
|
807
|
-
);
|
|
808
|
-
},
|
|
809
|
-
text({ rows }, { width }) {
|
|
810
|
-
expect(width).toBe(80);
|
|
811
|
-
const label = Math.max(...rows.map((r) => stringWidth(r.key)));
|
|
812
|
-
const barWidth = Math.min(20, width - label - 8);
|
|
813
|
-
return rows
|
|
814
|
-
.map((r) => {
|
|
815
|
-
const chart = r.ratio === null ? padEnd("—", barWidth) : bar(r.ratio, barWidth);
|
|
816
|
-
return `${padEnd(r.key, label)} ${chart} ${r.display}`;
|
|
817
|
-
})
|
|
818
|
-
.join("\n");
|
|
819
|
-
},
|
|
696
|
+
describe("defineReport 装载规范化", () => {
|
|
697
|
+
it("三种写法装载出等价的规范化结果:树 ≡ {content} ≡ pages [{id: report}]", () => {
|
|
698
|
+
const tree = <ScopeSummary />;
|
|
699
|
+
const fromTree = defineReport(tree);
|
|
700
|
+
const fromContent = defineReport({ content: tree });
|
|
701
|
+
const fromPages = defineReport({
|
|
702
|
+
pages: [{ id: "report", title: { en: "Report", "zh-CN": "报告" }, content: tree }],
|
|
820
703
|
});
|
|
821
|
-
const
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
expect(
|
|
828
|
-
"bub █████████████████░░░ 87%
|
|
829
|
-
codex ████████████████░░░░ 80%
|
|
830
|
-
克劳德 — —"
|
|
831
|
-
`);
|
|
832
|
-
// 三行的条形都从同一显示列开始:"克劳德" 是 3 个码元、6 个显示列,
|
|
833
|
-
// 拿 .padEnd(label) 补齐会把这一行整体左移 3 列
|
|
834
|
-
const starts = term.split("\n").map((line, i) => stringWidth(line.slice(0, line.indexOf(["█", "█", "—"][i]))));
|
|
835
|
-
expect(new Set(starts).size).toBe(1);
|
|
836
|
-
|
|
837
|
-
const html = renderToStaticMarkup(<PassBars rows={rows} />);
|
|
838
|
-
expect(html).toContain("87%");
|
|
839
|
-
expect(html).toContain("—"); // 缺数据两面都是 —,不补 0
|
|
704
|
+
for (const definition of [fromTree, fromContent, fromPages]) {
|
|
705
|
+
expect(definition.kind).toBe("report");
|
|
706
|
+
expect(definition.pages).toHaveLength(1);
|
|
707
|
+
expect(definition.pages[0]!.id).toBe("report");
|
|
708
|
+
expect(definition.pages[0]!.content).toBe(tree);
|
|
709
|
+
}
|
|
710
|
+
expect(fromTree.pages[0]!.title).toEqual(fromContent.pages[0]!.title);
|
|
840
711
|
});
|
|
841
|
-
});
|
|
842
712
|
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
const snapshot: Snapshot = (() => {
|
|
847
|
-
const dir = "/results/compare_bub/snap-1";
|
|
848
|
-
const base = {
|
|
849
|
-
experimentId: "compare/bub",
|
|
850
|
-
startedAt: "2026-07-01T10:00:00Z",
|
|
851
|
-
completedAt: "2026-07-01T10:05:00Z",
|
|
852
|
-
agent: "bub",
|
|
853
|
-
schemaVersion: 1,
|
|
854
|
-
dir,
|
|
855
|
-
};
|
|
856
|
-
const mk = (id: string, verdict: "passed" | "failed", index: number) => ({
|
|
857
|
-
evalId: id,
|
|
858
|
-
experimentId: "compare/bub",
|
|
859
|
-
result: {
|
|
860
|
-
id,
|
|
861
|
-
agent: "bub",
|
|
862
|
-
verdict,
|
|
863
|
-
attempt: 0,
|
|
864
|
-
startedAt: `2026-07-01T10:0${index}:00Z`,
|
|
865
|
-
durationMs: 1000,
|
|
866
|
-
assertions: [],
|
|
867
|
-
},
|
|
868
|
-
ref: { snapshot: "compare_bub/snap-1", attempt: `${id}/a0` },
|
|
869
|
-
snapshot: base,
|
|
870
|
-
events: async () => null,
|
|
871
|
-
trace: async () => null,
|
|
872
|
-
o11y: async () => null,
|
|
873
|
-
diff: async () => null,
|
|
874
|
-
sources: async () => null,
|
|
875
|
-
});
|
|
876
|
-
const attempts = [mk("algebra/x", "passed", 0), mk("algebra/y", "failed", 1)];
|
|
877
|
-
return {
|
|
878
|
-
...base,
|
|
879
|
-
evals: attempts.map((a) => ({ id: a.evalId, attempts: [a] })),
|
|
880
|
-
attempts,
|
|
881
|
-
} as unknown as Snapshot;
|
|
882
|
-
})();
|
|
883
|
-
const selection: Selection = {
|
|
884
|
-
snapshots: [snapshot],
|
|
885
|
-
warnings: [],
|
|
886
|
-
filter: () => selection,
|
|
887
|
-
};
|
|
888
|
-
const results = {
|
|
889
|
-
experiments: [{ id: "compare/bub", snapshots: [snapshot], latest: snapshot, evalIds: ["algebra/x", "algebra/y"] }],
|
|
890
|
-
skipped: [],
|
|
891
|
-
latest: () => selection,
|
|
892
|
-
} as Results;
|
|
893
|
-
return { selection, results };
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
/**
|
|
897
|
-
* 三份快照、三个 experiment("compare/bub"、"other/codex"、"solo"),全部通过、无成本——
|
|
898
|
-
* 一个多实验 Selection 夹具:ExperimentList 出三项,MetricScatter 三个点都无成本
|
|
899
|
-
* (0 可画点,如实走空态)。`filter` 落实真实语义(不像 fakeContext 那样恒等返回自己)。
|
|
900
|
-
*/
|
|
901
|
-
function fakeMultiGroupContext(): { selection: Selection; results: Results } {
|
|
902
|
-
const mkSnapshot = (experimentId: string, agent: string, dirSuffix: string): Snapshot => {
|
|
903
|
-
const dir = `/results/${dirSuffix}`;
|
|
904
|
-
const base = {
|
|
905
|
-
experimentId,
|
|
906
|
-
startedAt: "2026-07-01T10:00:00Z",
|
|
907
|
-
completedAt: "2026-07-01T10:05:00Z",
|
|
908
|
-
agent,
|
|
909
|
-
schemaVersion: 1,
|
|
910
|
-
dir,
|
|
911
|
-
};
|
|
912
|
-
const attempt = {
|
|
913
|
-
evalId: "algebra/x",
|
|
914
|
-
experimentId,
|
|
915
|
-
result: {
|
|
916
|
-
id: "algebra/x",
|
|
917
|
-
agent,
|
|
918
|
-
verdict: "passed" as const,
|
|
919
|
-
attempt: 0,
|
|
920
|
-
startedAt: "2026-07-01T10:00:00Z",
|
|
921
|
-
durationMs: 1000,
|
|
922
|
-
assertions: [],
|
|
923
|
-
},
|
|
924
|
-
ref: { snapshot: dirSuffix, attempt: "algebra/x/a0" },
|
|
925
|
-
snapshot: base,
|
|
926
|
-
events: async () => null,
|
|
927
|
-
trace: async () => null,
|
|
928
|
-
o11y: async () => null,
|
|
929
|
-
diff: async () => null,
|
|
930
|
-
sources: async () => null,
|
|
931
|
-
};
|
|
932
|
-
return {
|
|
933
|
-
...base,
|
|
934
|
-
evals: [{ id: attempt.evalId, attempts: [attempt] }],
|
|
935
|
-
attempts: [attempt],
|
|
936
|
-
} as unknown as Snapshot;
|
|
937
|
-
};
|
|
938
|
-
|
|
939
|
-
const snapshots = [
|
|
940
|
-
mkSnapshot("compare/bub", "bub", "compare_bub/snap-1"),
|
|
941
|
-
mkSnapshot("other/codex", "codex", "other_codex/snap-1"),
|
|
942
|
-
mkSnapshot("solo", "bub", "solo/snap-1"),
|
|
943
|
-
];
|
|
944
|
-
|
|
945
|
-
const makeSelection = (snaps: Snapshot[]): Selection => ({
|
|
946
|
-
snapshots: snaps,
|
|
947
|
-
warnings: [],
|
|
948
|
-
filter: (predicate) => makeSelection(snaps.filter(predicate)),
|
|
713
|
+
it("content 与 pages 同时声明或都省略,装载报错且文案含 <ExperimentComparison /> 下一步", () => {
|
|
714
|
+
expect(() => defineReport({ content: <ScopeSummary />, pages: [] } as never)).toThrow(/both "content" and "pages"/);
|
|
715
|
+
expect(() => defineReport({ title: "X" } as never)).toThrow(/<ExperimentComparison \/>/);
|
|
949
716
|
});
|
|
950
|
-
const selection = makeSelection(snapshots);
|
|
951
|
-
const results = {
|
|
952
|
-
experiments: snapshots.map((s) => ({
|
|
953
|
-
id: s.experimentId,
|
|
954
|
-
snapshots: [s],
|
|
955
|
-
latest: s,
|
|
956
|
-
evalIds: ["algebra/x"],
|
|
957
|
-
})),
|
|
958
|
-
skipped: [],
|
|
959
|
-
latest: () => selection,
|
|
960
|
-
} as Results;
|
|
961
|
-
return { selection, results };
|
|
962
|
-
}
|
|
963
717
|
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
const report = defineReport(async ({ selection }) => {
|
|
969
|
-
const experiments = await ExperimentList.data(selection);
|
|
970
|
-
return (
|
|
971
|
-
<Col>
|
|
972
|
-
<RunOverview data={await RunOverview.data(selection)} />
|
|
973
|
-
<ExperimentList items={experiments} />
|
|
974
|
-
<Section title="考试成绩单">
|
|
975
|
-
<Scoreboard data={await Scoreboard.data(selection, { rows: "agent", subjects: "evalGroup" })} />
|
|
976
|
-
</Section>
|
|
977
|
-
</Col>
|
|
718
|
+
it("defineReport 产物不是 ReportNode:页里放产物装载报错;树校验同样拒绝", () => {
|
|
719
|
+
const inner = defineReport(<ScopeSummary />);
|
|
720
|
+
expect(() => defineReport({ pages: [{ id: "a", title: "A", content: inner as never }] })).toThrow(
|
|
721
|
+
/shell cannot nest/,
|
|
978
722
|
);
|
|
723
|
+
expect(() => defineReport(inner as never)).toThrow(/shell cannot nest/);
|
|
724
|
+
expect(() => validateReportTree([inner] as never)).toThrow(/not a report node/);
|
|
979
725
|
});
|
|
980
726
|
|
|
981
|
-
it("
|
|
982
|
-
expect(
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
expect(
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
expect(
|
|
996
|
-
expect(
|
|
997
|
-
|
|
998
|
-
expect(
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
it("renderReportToStaticHtml:同一棵树走 web 面,ExperimentList 自动接证据室深链", async () => {
|
|
1004
|
-
const html = await renderReportToStaticHtml(report, fakeContext());
|
|
1005
|
-
expect(html).toContain("nre-experiment-list");
|
|
1006
|
-
expect(html).toContain("nre-scoreboard");
|
|
1007
|
-
expect(html).toContain("考试成绩单");
|
|
1008
|
-
// 宿主注入的 attemptHref:ExperimentList 的 locator 徽标是普通 <a>,默认 view 路由(单段 locator)
|
|
1009
|
-
expect(html).toMatch(/href="#\/attempt\/@[0-9a-z]+"/);
|
|
1010
|
-
expect(html).not.toContain("<script");
|
|
727
|
+
it("重复或非法 page id 装载报错并点名冲突;LocalizedText 全空对象报错", () => {
|
|
728
|
+
expect(() =>
|
|
729
|
+
defineReport({
|
|
730
|
+
pages: [
|
|
731
|
+
{ id: "exam", title: "A", content: null },
|
|
732
|
+
{ id: "exam", title: "B", content: null },
|
|
733
|
+
],
|
|
734
|
+
}),
|
|
735
|
+
).toThrow(/"exam" is declared twice/);
|
|
736
|
+
expect(() => defineReport({ pages: [{ id: "Bad/Id", title: "A", content: null }] })).toThrow(/invalid/);
|
|
737
|
+
expect(() => defineReport({ title: {}, content: null })).toThrow(/no non-empty value/);
|
|
738
|
+
});
|
|
739
|
+
|
|
740
|
+
it("{src} 资产拒绝 .. 路径段、绝对路径与 ~;inline/src 互斥", () => {
|
|
741
|
+
expect(() => defineReport({ content: null, scripts: [{ src: "../x.js" }] })).toThrow(/not allowed/);
|
|
742
|
+
expect(() => defineReport({ content: null, scripts: [{ src: "/abs.js" }] })).toThrow(/not allowed/);
|
|
743
|
+
expect(() => defineReport({ content: null, styles: [{ src: "~/x.css" }] })).toThrow(/not allowed/);
|
|
744
|
+
expect(() => defineReport({ content: null, scripts: [{ src: "./a.js", inline: "x" } as never] })).toThrow(
|
|
745
|
+
/exactly one/,
|
|
746
|
+
);
|
|
747
|
+
expect(() => defineReport({ content: null, scripts: [{ src: "./assets/a.js" }] })).not.toThrow();
|
|
1011
748
|
});
|
|
1012
749
|
|
|
1013
|
-
it("
|
|
1014
|
-
const
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
startedAt: "2026-07-01T10:00:00Z",
|
|
1020
|
-
dir: "/results/compare_bub/snap-1",
|
|
1021
|
-
message,
|
|
750
|
+
it("--page 语义:pickReportPage 缺省第一页,未命中抛 ReportPageNotFoundError 列出可用页", () => {
|
|
751
|
+
const definition = defineReport({
|
|
752
|
+
pages: [
|
|
753
|
+
{ id: "overview", title: "Overview", content: null },
|
|
754
|
+
{ id: "exam", title: "Exam", content: null },
|
|
755
|
+
],
|
|
1022
756
|
});
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
));
|
|
1035
|
-
|
|
1036
|
-
|
|
757
|
+
expect(pickReportPage(definition).id).toBe("overview");
|
|
758
|
+
expect(pickReportPage(definition, "exam").id).toBe("exam");
|
|
759
|
+
try {
|
|
760
|
+
pickReportPage(definition, "typo");
|
|
761
|
+
expect.unreachable();
|
|
762
|
+
} catch (e) {
|
|
763
|
+
expect(e).toBeInstanceOf(ReportPageNotFoundError);
|
|
764
|
+
expect((e as ReportPageNotFoundError).available).toEqual(["overview", "exam"]);
|
|
765
|
+
}
|
|
766
|
+
// 树形态文件的唯一页 id 是缩写展开出的 report
|
|
767
|
+
const single = defineReport(<ScopeSummary />);
|
|
768
|
+
expect(pickReportPage(single, "report").id).toBe("report");
|
|
769
|
+
});
|
|
770
|
+
|
|
771
|
+
it("标题回退链:def.title → 唯一且相同的快照 name → 内置文案「Eval 运行结果 / Eval Results」;en 相同 zh 不同也落内置文案", () => {
|
|
772
|
+
const named = snap({ experimentId: "t/a", name: "Memory Evals", results: [res("q", "passed")] });
|
|
773
|
+
const definition = defineReport(<ScopeSummary />);
|
|
774
|
+
expect(resolveReportTitle(defineReport({ title: "Custom", content: null }), scopeOf([named]))).toBe("Custom");
|
|
775
|
+
expect(resolveReportTitle(definition, scopeOf([named]))).toBe("Memory Evals");
|
|
776
|
+
expect(resolveReportTitle(definition, scopeOf([snap({ experimentId: "t/b", results: [] })]))).toEqual(
|
|
777
|
+
FALLBACK_REPORT_TITLE,
|
|
778
|
+
);
|
|
779
|
+
const zhA = snap({ experimentId: "t/c", name: { en: "Same", "zh-CN": "一" }, results: [] });
|
|
780
|
+
const zhB = snap({ experimentId: "t/d", name: { en: "Same", "zh-CN": "二" }, results: [] });
|
|
781
|
+
expect(resolveReportTitle(definition, scopeOf([zhA, zhB]))).toEqual(FALLBACK_REPORT_TITLE);
|
|
782
|
+
expect(FALLBACK_REPORT_TITLE).toEqual({ en: "Eval Results", "zh-CN": "Eval 运行结果" });
|
|
1037
783
|
});
|
|
1038
784
|
|
|
1039
|
-
it("
|
|
1040
|
-
const
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
expect(()
|
|
1046
|
-
|
|
1047
|
-
|
|
785
|
+
it("ReportLink.icon 是 { svg: string }:defineReport 接受合法形状;无类型 JS 传其它形状定义时报完整用户反馈", () => {
|
|
786
|
+
const svg = '<svg viewBox="0 0 16 16"><path d="M0 0h16v16z"/></svg>';
|
|
787
|
+
const withIcon = defineReport({
|
|
788
|
+
content: null,
|
|
789
|
+
links: [{ label: "GitHub", href: "https://example.com", icon: { svg } }],
|
|
790
|
+
});
|
|
791
|
+
expect(withIcon.links[0]!.icon).toEqual({ svg });
|
|
792
|
+
|
|
793
|
+
const reactNode = { $$typeof: Symbol.for("react.transitional.element"), type: "svg", props: {} };
|
|
794
|
+
for (const icon of [reactNode, "<svg/>", { svg: 42 }, { svg: "" }]) {
|
|
795
|
+
expect(() =>
|
|
796
|
+
defineReport({
|
|
797
|
+
content: null,
|
|
798
|
+
links: [{ label: "GitHub", href: "https://example.com", icon: icon as never }],
|
|
799
|
+
}),
|
|
800
|
+
).toThrow(/"icon" must be \{ svg: string \}/);
|
|
801
|
+
}
|
|
1048
802
|
});
|
|
1049
803
|
});
|
|
1050
804
|
|
|
1051
|
-
// ─────────────────────────
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
it("是普通 ReportDefinition;text 面 = 成本×通过率散点 + 实验列表,别无它物", async () => {
|
|
1079
|
-
expect(isReportDefinition(ExperimentComparison)).toBe(true);
|
|
1080
|
-
const out = await renderReportToText(ExperimentComparison, fakeContext(), { width: 100 });
|
|
1081
|
-
// 散点:fakeContext 无成本数据 → 0 可画点,显式说明缺哪两个指标(而不是画一张空图)
|
|
1082
|
-
expect(out).toContain("No data to plot");
|
|
1083
|
-
expect(out).not.toContain("better → upper right");
|
|
1084
|
-
// 实验列表主行 + eval 级折叠计票 + 失败诊断(ExperimentList.data 在 build() 里直接 await)
|
|
1085
|
-
expect(out).toMatch(/compare\/bub\s+default\s+bub/);
|
|
1086
|
-
expect(out).toContain("1 passed / 1 failed");
|
|
1087
|
-
expect(out).toContain("50%");
|
|
1088
|
-
expect(out).toMatch(/✗ failed\s+algebra\/y[\s\S]*└─ @[0-9a-z]+/);
|
|
1089
|
-
// 只有两个直接业务组件:没有 RunOverview / GroupSummary / Section 分组
|
|
1090
|
-
expect(out).not.toContain("Current verdicts");
|
|
1091
|
-
});
|
|
1092
|
-
|
|
1093
|
-
it("web 面:散点空态 + 实验列表 <details> 展开区,无 <script>,无 Section 分组", async () => {
|
|
1094
|
-
const html = await renderReportToStaticHtml(ExperimentComparison, fakeContext());
|
|
1095
|
-
expect(html).toContain("nre-metric-scatter");
|
|
1096
|
-
expect(html).toContain("nre-scatter-empty"); // 0 可画点的空态
|
|
1097
|
-
expect(html).toContain('<details class="nre-experiment-entry">');
|
|
1098
|
-
expect(html).toContain("nre-experiment-head");
|
|
1099
|
-
expect(html).toContain('data-nre-experiment-filter=""');
|
|
1100
|
-
expect(html).toContain("nre-experiment-evals");
|
|
1101
|
-
expect(html).not.toContain("nre-section");
|
|
1102
|
-
expect(html).not.toContain("<script");
|
|
1103
|
-
});
|
|
1104
|
-
|
|
1105
|
-
it("locale 变体:en / zh-CN 都渲染(chrome 分语言),散点空态两面同一事实", async () => {
|
|
1106
|
-
const zhHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
|
|
1107
|
-
expect(zhHtml).toContain("成功率"); // passRate 的 zh-CN label(ExperimentList 主行)
|
|
1108
|
-
const enHtml = await renderReportToStaticHtml(ExperimentComparison, fakeContext(), { locale: "en" });
|
|
1109
|
-
expect(enHtml).toContain("Pass rate");
|
|
1110
|
-
const zhText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "zh-CN" });
|
|
1111
|
-
expect(zhText).toContain("没有可绘制的数据"); // 散点空态 zh
|
|
1112
|
-
const enText = await renderReportToText(ExperimentComparison, fakeContext(), { locale: "en" });
|
|
1113
|
-
expect(enText).toContain("No data to plot");
|
|
1114
|
-
});
|
|
805
|
+
// ───────────────────────── 内建报告 ─────────────────────────
|
|
806
|
+
|
|
807
|
+
describe("内建报告", () => {
|
|
808
|
+
it("niceeval/report/built-in 默认导出是普通 defineReport 产物,与用户写的 defineReport(<ExperimentComparison />) 等价", async () => {
|
|
809
|
+
expect(builtInReport.kind).toBe("report");
|
|
810
|
+
expect(builtInReport.pages).toHaveLength(1);
|
|
811
|
+
|
|
812
|
+
const s1 = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
|
|
813
|
+
const s2 = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
|
|
814
|
+
const scope = scopeOf([s1, s2]);
|
|
815
|
+
const ctx = { scope, results: resultsOf([s1, s2]) };
|
|
816
|
+
const builtin = await renderReportToText(builtInReport, ctx, { width: 120 });
|
|
817
|
+
const user = await renderReportToText(defineReport(<ExperimentComparison />), ctx, { width: 120 });
|
|
818
|
+
expect(builtin).toBe(user);
|
|
819
|
+
});
|
|
820
|
+
|
|
821
|
+
it("text 面多组时只输出组索引与单组查看命令;单组时输出散点与实验列表", async () => {
|
|
822
|
+
const g1 = snap({ experimentId: "compare/a", results: [res("q", "passed")] });
|
|
823
|
+
const g2 = snap({ experimentId: "dev/b", results: [res("q", "failed")] });
|
|
824
|
+
const multi = await renderReportToText(
|
|
825
|
+
builtInReport,
|
|
826
|
+
{ scope: scopeOf([g1, g2]), results: resultsOf([g1, g2]) },
|
|
827
|
+
{ width: 120 },
|
|
828
|
+
);
|
|
829
|
+
expect(multi).toContain("niceeval show --experiment compare");
|
|
830
|
+
expect(multi).toContain("niceeval show --experiment dev");
|
|
831
|
+
expect(multi).not.toContain("Eval / Attempt"); // 不倾倒组内 experiment 明细
|
|
1115
832
|
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
expect(
|
|
1126
|
-
|
|
1127
|
-
expect(
|
|
833
|
+
const priced = snap({
|
|
834
|
+
experimentId: "compare/priced",
|
|
835
|
+
results: [res("q", "passed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.2 } })],
|
|
836
|
+
});
|
|
837
|
+
const single = await renderReportToText(
|
|
838
|
+
builtInReport,
|
|
839
|
+
{ scope: scopeOf([priced]), results: resultsOf([priced]) },
|
|
840
|
+
{ width: 140 },
|
|
841
|
+
);
|
|
842
|
+
expect(single).toContain("Eval / Attempt"); // 单组直接进入组内详情
|
|
843
|
+
// 成本轴(better: lower)反向渲染,「更好」恒指向右上;两轴都声明 better → 提示在场
|
|
844
|
+
expect(single).toContain("better → upper right");
|
|
1128
845
|
});
|
|
1129
846
|
});
|