niceeval 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +69 -7
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +57 -3
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +32 -24
- package/dist/report/aggregate.js +158 -50
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +93 -160
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +87 -81
- package/dist/report/compute.js +597 -417
- package/dist/report/flag.d.ts +32 -6
- package/dist/report/flag.js +92 -4
- package/dist/report/format.d.ts +19 -11
- package/dist/report/format.js +30 -13
- package/dist/report/index.d.ts +16 -16
- package/dist/report/index.js +20 -21
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +57 -33
- package/dist/report/locale.js +122 -56
- package/dist/report/metrics.d.ts +23 -4
- package/dist/report/metrics.js +110 -25
- package/dist/report/primitives.d.ts +48 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +9 -7
- package/dist/report/react/AttemptList.js +17 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +4 -4
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +10 -0
- package/dist/report/react/ExperimentComparison.js +12 -0
- package/dist/report/react/ExperimentList.d.ts +4 -3
- package/dist/report/react/ExperimentList.js +17 -18
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +59 -28
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +110 -147
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -16
- package/dist/report/report.js +259 -28
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +253 -184
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +38 -7
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +247 -284
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +49 -14
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +4 -2
- package/dist/shared/aggregate.js +8 -7
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
- package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
- package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
- package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
- package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
- package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
- package/docs-site/zh/how-to/publish-report.mdx +105 -0
- package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
- package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
- package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
- package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
- package/docs-site/zh/index.mdx +24 -26
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +32 -5
- package/docs-site/zh/reference/capabilities.mdx +8 -8
- package/docs-site/zh/reference/cli.mdx +40 -12
- package/docs-site/zh/reference/define-agent.mdx +58 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
- package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
- package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
- package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +10 -2
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +13 -2
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +71 -8
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +446 -124
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +162 -30
- package/src/context/session.test.ts +2 -1
- package/src/context/session.ts +115 -7
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +81 -17
- package/src/i18n/zh-CN.ts +80 -17
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +178 -61
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -279
- package/src/report/compute.ts +723 -491
- package/src/report/dual-render.test.tsx +741 -1024
- package/src/report/flag.ts +104 -12
- package/src/report/format.ts +32 -12
- package/src/report/index.ts +119 -46
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +136 -65
- package/src/report/metrics.ts +108 -25
- package/src/report/primitives.tsx +196 -45
- package/src/report/react/AttemptList.tsx +30 -43
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +73 -0
- package/src/report/react/ExperimentList.tsx +50 -32
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +86 -34
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +89 -5
- package/src/report/react/fixtures.ts +114 -154
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +138 -158
- package/src/report/react/styles.css +243 -82
- package/src/report/report.test.ts +779 -841
- package/src/report/report.ts +423 -41
- package/src/report/text/faces.ts +290 -193
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +44 -7
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +261 -271
- package/src/report/web.ts +63 -20
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +13 -11
- package/src/results/attempt-evidence.ts +20 -13
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +150 -60
- package/src/results/host-equivalence.test.ts +34 -20
- package/src/results/index.ts +12 -4
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +15 -5
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +89 -54
- package/src/results/select.ts +104 -34
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +43 -14
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +243 -37
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +8 -6
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +127 -56
- package/src/show/render.ts +662 -131
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +320 -54
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +423 -11
- package/src/view/app/components/CodeView.tsx +41 -14
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +37 -17
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +10 -3
- package/src/view/data.ts +155 -49
- package/src/view/index.ts +56 -41
- package/src/view/server.ts +37 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +227 -0
- package/src/view/view-report.test.ts +167 -62
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
- package/dist/report/built-ins/experiment-comparison.js +0 -13
- package/dist/report/built-ins/index.d.ts +0 -1
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/docs-site/zh/guides/publish-report.mdx +0 -91
- package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
- package/src/report/built-in-user-parity.test.tsx +0 -640
- package/src/report/built-ins/experiment-comparison.tsx +0 -19
- package/src/report/built-ins/index.ts +0 -2
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
package/src/show/show.test.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
//
|
|
1
|
+
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
|
+
// niceeval show 终端宿主的测试(行为规范:docs-site/zh/how-to/viewing-results.mdx;
|
|
2
3
|
// 组合语义:docs/feature/reports/architecture.md「Selection 是计算入口」)。覆盖:
|
|
3
4
|
// - 榜单合成口径:每 experiment × eval 取最新判定,局部重跑从更早快照补齐,头部标注合成自几个快照;
|
|
4
5
|
// - 前缀过滤收窄 Selection,覆盖警告分母 = 已知并集 ∩ 范围;
|
|
5
6
|
// - --history 时间轴只列真实执行,resume 携带的复印件不占行;
|
|
6
7
|
// - --report 装载(合法 / 非法默认导出 / 文件缺失)、位置前缀收窄注入 Selection、attemptCommand 下钻;
|
|
7
8
|
// - 互斥:--history 与 --report;
|
|
8
|
-
// - 单 eval 详情、`@<locator>` 精确定位与 --
|
|
9
|
+
// - 单 eval 详情、`@<locator>` 精确定位与 --source / --execution / --diff 证据切面的输出形态。
|
|
9
10
|
//
|
|
10
11
|
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
11
12
|
// 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API(避免与并行重写的
|
|
@@ -17,9 +18,9 @@ import { tmpdir } from "node:os";
|
|
|
17
18
|
import { join } from "node:path";
|
|
18
19
|
import { createHash } from "node:crypto";
|
|
19
20
|
import { openResults } from "../results/index.ts";
|
|
20
|
-
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
|
|
21
|
+
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type TimingNode, type TraceSpan, type Verdict } from "../types.ts";
|
|
21
22
|
import { selectCurrentResults } from "../results/select.ts";
|
|
22
|
-
import {
|
|
23
|
+
import { attemptHistory } from "./compose.ts";
|
|
23
24
|
import { runShow, type ShowFlags } from "./index.ts";
|
|
24
25
|
import { stringWidth } from "../report/text/layout.ts";
|
|
25
26
|
|
|
@@ -52,7 +53,7 @@ type AttemptFixture = Pick<EvalResult, "id" | "verdict"> &
|
|
|
52
53
|
Partial<
|
|
53
54
|
Pick<
|
|
54
55
|
EvalResult,
|
|
55
|
-
"attempt" | "durationMs" | "assertions" | "estimatedCostUSD" | "usage" | "error" | "startedAt" | "artifactBase" | "hasEvents" | "hasTrace"
|
|
56
|
+
"attempt" | "durationMs" | "assertions" | "estimatedCostUSD" | "usage" | "error" | "diagnostics" | "startedAt" | "artifactBase" | "hasEvents" | "hasTrace" | "phases"
|
|
56
57
|
>
|
|
57
58
|
>;
|
|
58
59
|
|
|
@@ -112,7 +113,7 @@ interface Captured {
|
|
|
112
113
|
async function show(root: string, patterns: string[], flags: ShowFlags = {}, width = 100): Promise<Captured> {
|
|
113
114
|
let out = "";
|
|
114
115
|
let err = "";
|
|
115
|
-
const code = await runShow(root, patterns, {
|
|
116
|
+
const code = await runShow(root, patterns, { results: root, ...flags }, {
|
|
116
117
|
out: (s) => (out += s),
|
|
117
118
|
err: (s) => (err += s),
|
|
118
119
|
width,
|
|
@@ -135,7 +136,7 @@ async function seedComposedRoot(): Promise<string> {
|
|
|
135
136
|
name: 'fileChanged("src/components/Button.tsx")',
|
|
136
137
|
severity: "gate",
|
|
137
138
|
score: 0,
|
|
138
|
-
|
|
139
|
+
outcome: "failed" as const,
|
|
139
140
|
detail: "file was not modified",
|
|
140
141
|
},
|
|
141
142
|
],
|
|
@@ -151,12 +152,12 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
151
152
|
const root = await seedComposedRoot();
|
|
152
153
|
const { out, code } = await show(root, []);
|
|
153
154
|
expect(code).toBe(0);
|
|
154
|
-
expect(out).toContain("No data to plot Cost ×
|
|
155
|
+
expect(out).toContain("No data to plot Cost × End-to-end pass rate");
|
|
155
156
|
expect(out).toContain("1 point missing data");
|
|
156
157
|
expect(out).not.toContain("needed to compare");
|
|
157
158
|
expect(out).not.toContain("COMPARISON");
|
|
158
|
-
expect(out).toMatch(
|
|
159
|
-
expect(out).
|
|
159
|
+
expect(out).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
|
|
160
|
+
expect(out).toMatch(/1 passed[\s\S]*?\/ 1\s+failed/);
|
|
160
161
|
expect(out).toMatch(/✗ failed\s+fixtures\/button[\s\S]*└─ @1[0-9a-z]{7}[\s\S]*fileChanged/);
|
|
161
162
|
expect(out).toMatch(/✓ passed\s+weather\/brooklyn[\s\S]*└─ @1[0-9a-z]{7}/);
|
|
162
163
|
expect(out).not.toMatch(/\[[EXD⏱,]+\]/);
|
|
@@ -168,9 +169,9 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
168
169
|
try {
|
|
169
170
|
const { out, code } = await show(root, []);
|
|
170
171
|
expect(code).toBe(0);
|
|
171
|
-
expect(out).toContain("
|
|
172
|
+
expect(out).toContain("成本 × 端到端成功率 没有可绘制的数据");
|
|
172
173
|
expect(out).toContain("1 通过 / 1 失败");
|
|
173
|
-
expect(out).
|
|
174
|
+
expect(out).toMatch(/\bbub\s+默认\s+bub\s+1s\s+50%/);
|
|
174
175
|
} finally {
|
|
175
176
|
process.env.NICEEVAL_LANG = "en";
|
|
176
177
|
}
|
|
@@ -198,10 +199,11 @@ describe("默认报告:跨快照合成的现刻水位(ExperimentComparison text
|
|
|
198
199
|
]);
|
|
199
200
|
const { out, code } = await show(root, []);
|
|
200
201
|
expect(code).toBe(0);
|
|
202
|
+
// 成本轴(better: lower)反向,「更好」恒指向右上;两轴都声明 better → 提示在场
|
|
201
203
|
expect(out).toContain("better → upper right");
|
|
202
204
|
expect(out).toContain("A compare/a B compare/b");
|
|
203
|
-
expect(out).toMatch(
|
|
204
|
-
expect(out).toMatch(
|
|
205
|
+
expect(out).toMatch(/\bb\s+large\s+claude\s+1s\s+100%/);
|
|
206
|
+
expect(out).toMatch(/\ba\s+mini\s+codex\s+1s\s+50%/);
|
|
205
207
|
expect(out).toMatch(/✓ passed\s+q1[\s\S]*└─ @1[0-9a-z]{7}/);
|
|
206
208
|
expect(out).toMatch(/✗ failed\s+q2[\s\S]*└─ @1[0-9a-z]{7}/);
|
|
207
209
|
expect(out).not.toContain("needed to compare");
|
|
@@ -280,15 +282,15 @@ describe("单 eval 详情", () => {
|
|
|
280
282
|
name: 'calledTool("get_weather")',
|
|
281
283
|
severity: "gate",
|
|
282
284
|
score: 0,
|
|
283
|
-
|
|
285
|
+
outcome: "failed" as const,
|
|
284
286
|
detail: "tool was never called",
|
|
285
287
|
},
|
|
286
|
-
{ name: "succeeded()", severity: "gate", score: 1,
|
|
288
|
+
{ name: "succeeded()", severity: "gate", score: 1, outcome: "passed" as const },
|
|
287
289
|
{
|
|
288
290
|
name: 'judge("回答基于实时数据")',
|
|
289
291
|
severity: "soft",
|
|
290
292
|
score: 0.2,
|
|
291
|
-
|
|
293
|
+
outcome: "failed" as const,
|
|
292
294
|
detail: "reply invents a temperature without any tool call",
|
|
293
295
|
},
|
|
294
296
|
],
|
|
@@ -311,7 +313,7 @@ describe("单 eval 详情", () => {
|
|
|
311
313
|
// 每 experiment 一行都带紧凑索引(locator + 失败原因),详情块再补一条精确的 attempt locator
|
|
312
314
|
expect(out).toMatch(/✗ failed\s+2 attempts.*@1[0-9a-z]{7}.*gate calledTool/);
|
|
313
315
|
expect(out).toMatch(/attempt locator: @1[0-9a-z]{7}/);
|
|
314
|
-
expect(out).toMatch(/next: niceeval show @1[0-9a-z]{7} \[--
|
|
316
|
+
expect(out).toMatch(/next: niceeval show @1[0-9a-z]{7} \[--source\|--execution\|--diff\]/);
|
|
315
317
|
});
|
|
316
318
|
});
|
|
317
319
|
|
|
@@ -341,68 +343,94 @@ describe("--history 时间轴", () => {
|
|
|
341
343
|
res("weather/brooklyn", "failed", {
|
|
342
344
|
attempt: 1,
|
|
343
345
|
estimatedCostUSD: 0.04,
|
|
344
|
-
assertions: [{ name: 'calledTool("get_weather")', severity: "gate", score: 0,
|
|
346
|
+
assertions: [{ name: 'calledTool("get_weather")', severity: "gate", score: 0, outcome: "failed" as const }],
|
|
345
347
|
}),
|
|
346
348
|
],
|
|
347
349
|
);
|
|
348
350
|
return root;
|
|
349
351
|
}
|
|
350
352
|
|
|
351
|
-
it("
|
|
353
|
+
it("逐 attempt 分节:复印件按身份键去重不占行,startedAt 升序,行带摘要 / 成本 / locator", async () => {
|
|
352
354
|
const root = await seedHistoryRoot();
|
|
353
355
|
const results = await openResults(root);
|
|
354
356
|
const exp = results.experiments.find((e) => e.id === "compare/bub")!;
|
|
355
|
-
const rows =
|
|
356
|
-
// 快照2
|
|
357
|
+
const rows = attemptHistory(exp, "weather/brooklyn");
|
|
358
|
+
// 快照2 里复印件被识别(与快照1 的真实执行同身份键),历次 attempt = 快照1 的 passed +
|
|
359
|
+
// 快照2 的 failed(新 attempt);startedAt 升序,旧的在前。
|
|
357
360
|
expect(rows).toHaveLength(2);
|
|
358
|
-
expect(rows[0]).toMatchObject({ verdict: "
|
|
359
|
-
expect(rows[0].
|
|
360
|
-
expect(rows[1]).toMatchObject({ verdict: "
|
|
361
|
+
expect(rows[0]).toMatchObject({ verdict: "passed", costUSD: 0.03 });
|
|
362
|
+
expect(rows[0].summary).toBeUndefined();
|
|
363
|
+
expect(rows[1]).toMatchObject({ verdict: "failed", costUSD: 0.04 });
|
|
364
|
+
expect(rows[1].summary).toContain('calledTool("get_weather")');
|
|
365
|
+
expect(rows[1].locator).toMatch(/^@/);
|
|
361
366
|
|
|
362
367
|
const { out, code } = await show(root, ["weather/brooklyn"], { history: true });
|
|
363
368
|
expect(code).toBe(0);
|
|
364
|
-
|
|
365
|
-
expect(out).toContain("
|
|
366
|
-
|
|
367
|
-
// 复印件那份 passed 判定只出现一行(快照1),不在快照2 再占一行
|
|
369
|
+
// 分节头:experimentId + evalId 一起出现,计数按 attempt。
|
|
370
|
+
expect(out).toContain("weather/brooklyn · compare/bub · 2 attempts · passed 1/2");
|
|
371
|
+
// 复印件那份 passed 判定只出现一行,不在快照2 再占一行;升序:passed 行在 failed 行之前。
|
|
368
372
|
expect(out.match(/✓ passed/g)).toHaveLength(1);
|
|
373
|
+
expect(out.indexOf("✓ passed")).toBeLessThan(out.indexOf("✗ failed"));
|
|
374
|
+
// 每行携带可复制的 locator。
|
|
375
|
+
expect(out).toMatch(/@\w+/);
|
|
369
376
|
});
|
|
370
377
|
|
|
371
|
-
it("裸 --history
|
|
378
|
+
it("裸 --history:对 Scope 中每个 experimentId + evalId 分节,不再输出快照级序列", async () => {
|
|
372
379
|
const root = await seedHistoryRoot();
|
|
373
380
|
const { out, code } = await show(root, [], { history: true });
|
|
374
381
|
expect(code).toBe(0);
|
|
375
|
-
expect(out).toContain("compare/bub · 2
|
|
376
|
-
//
|
|
377
|
-
expect(out.
|
|
382
|
+
expect(out).toContain("weather/brooklyn · compare/bub · 2 attempts · passed 1/2");
|
|
383
|
+
// 不是「每个快照一行」的通过率序列。
|
|
384
|
+
expect(out).not.toContain("1/1 passed");
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
it("--history 与 --page 组合是用法矛盾:直说", async () => {
|
|
388
|
+
const root = await seedHistoryRoot();
|
|
389
|
+
const { err, code } = await show(root, [], { history: true, page: "report" });
|
|
390
|
+
expect(code).toBe(1);
|
|
391
|
+
expect(err).toContain("--page");
|
|
378
392
|
});
|
|
379
393
|
});
|
|
380
394
|
|
|
381
395
|
// ───────────────────────── --report 装载与组合语义 ─────────────────────────
|
|
382
396
|
|
|
383
397
|
describe("--report 装载", () => {
|
|
384
|
-
/**
|
|
398
|
+
/**
|
|
399
|
+
* 不经 niceeval 包也能造出合法报告:判别、组合组件与双面组件都锚在 Symbol.for 上
|
|
400
|
+
* (docs/feature/reports/library/shell.md「defineReport 产物」、layout.md「自定义组件」)。
|
|
401
|
+
* Custom 是双面组件,读 props 直接渲染;Wrapper 是组合组件,用 ComposeContext 的 scope
|
|
402
|
+
* 算出 evals / locator 再装配进 Custom —— 对应 defineReport(<Wrapper />) 的展开形态。
|
|
403
|
+
*/
|
|
385
404
|
async function writeReportFile(dir: string): Promise<string> {
|
|
386
405
|
const path = join(dir, "report.mjs");
|
|
387
406
|
await writeFile(
|
|
388
407
|
path,
|
|
389
408
|
[
|
|
390
409
|
'const FACES = Symbol.for("niceeval.report.faces");',
|
|
410
|
+
'const COMPOSE = Symbol.for("niceeval.report.compose");',
|
|
411
|
+
'const DEFINITION = Symbol.for("niceeval.report.definition");',
|
|
391
412
|
"const Custom = () => null;",
|
|
392
413
|
"Custom[FACES] = {",
|
|
393
414
|
" web: () => null,",
|
|
394
415
|
" text: (props, ctx) => `CUSTOM ${props.evals} · drill ${ctx.attemptCommand(props.locator)}`,",
|
|
395
416
|
"};",
|
|
396
|
-
"
|
|
397
|
-
|
|
398
|
-
"
|
|
399
|
-
"
|
|
400
|
-
"
|
|
401
|
-
"
|
|
402
|
-
"
|
|
403
|
-
"
|
|
404
|
-
"
|
|
417
|
+
"const Wrapper = () => null;",
|
|
418
|
+
"Wrapper[COMPOSE] = (props, ctx) => ({",
|
|
419
|
+
" type: Custom,",
|
|
420
|
+
" props: {",
|
|
421
|
+
" evals: ctx.scope.snapshots.flatMap((s) => s.evals.map((e) => e.id)).sort().join(\",\"),",
|
|
422
|
+
" locator: ctx.scope.snapshots[0].evals[0].attempts[0].locator,",
|
|
423
|
+
" },",
|
|
424
|
+
"});",
|
|
425
|
+
"const definition = {",
|
|
426
|
+
' kind: "report",',
|
|
427
|
+
" links: [],",
|
|
428
|
+
" scripts: [],",
|
|
429
|
+
" styles: [],",
|
|
430
|
+
' pages: [{ id: "report", title: "Report", content: { type: Wrapper, props: {} } }],',
|
|
405
431
|
"};",
|
|
432
|
+
"Object.defineProperty(definition, DEFINITION, { value: true });",
|
|
433
|
+
"export default definition;",
|
|
406
434
|
"",
|
|
407
435
|
].join("\n"),
|
|
408
436
|
"utf-8",
|
|
@@ -472,6 +500,29 @@ describe("--report 装载", () => {
|
|
|
472
500
|
expect(out).not.toContain("CUSTOM");
|
|
473
501
|
expect(out).toContain("no diff recorded"); // fixture 没有 diff artifact:如实说缺
|
|
474
502
|
});
|
|
503
|
+
|
|
504
|
+
it("--page 命中单页定义的唯一页 id `report`:与直接渲染等价", async () => {
|
|
505
|
+
const root = await seedComposedRoot();
|
|
506
|
+
const report = await writeReportFile(root);
|
|
507
|
+
const bare = await show(root, [], { report });
|
|
508
|
+
const paged = await show(root, [], { report, page: "report" });
|
|
509
|
+
expect(paged.code).toBe(0);
|
|
510
|
+
expect(paged.out).toBe(bare.out);
|
|
511
|
+
});
|
|
512
|
+
|
|
513
|
+
it("--page 未命中:按用法错误非零退出并列出可用页 id(内建报告同样成立)", async () => {
|
|
514
|
+
const root = await seedComposedRoot();
|
|
515
|
+
const report = await writeReportFile(root);
|
|
516
|
+
const miss = await show(root, [], { report, page: "typo" });
|
|
517
|
+
expect(miss.code).toBe(1);
|
|
518
|
+
expect(miss.err).toContain(`page "typo" not found in ${report}`);
|
|
519
|
+
expect(miss.err).toContain("Available pages: report");
|
|
520
|
+
|
|
521
|
+
const builtin = await show(root, [], { page: "typo" });
|
|
522
|
+
expect(builtin.code).toBe(1);
|
|
523
|
+
expect(builtin.err).toContain('page "typo" not found in the built-in report');
|
|
524
|
+
expect(builtin.err).toContain("Available pages: report");
|
|
525
|
+
});
|
|
475
526
|
});
|
|
476
527
|
|
|
477
528
|
// ───────────────────────── 证据切面:--execution ─────────────────────────
|
|
@@ -540,18 +591,18 @@ describe("--execution", () => {
|
|
|
540
591
|
});
|
|
541
592
|
});
|
|
542
593
|
|
|
543
|
-
// ───────────────────────── 证据切面:--
|
|
594
|
+
// ───────────────────────── 证据切面:--source ─────────────────────────
|
|
544
595
|
|
|
545
|
-
describe("--
|
|
596
|
+
describe("--source", () => {
|
|
546
597
|
it("evalSource === null 时如实说源码未捕获", async () => {
|
|
547
598
|
const root = await makeRoot();
|
|
548
599
|
await writeSnapshot(
|
|
549
600
|
root,
|
|
550
601
|
"2026-07-08T10-00-00-000Z",
|
|
551
602
|
{ experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" },
|
|
552
|
-
[res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
603
|
+
[res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const }] })],
|
|
553
604
|
);
|
|
554
|
-
const { out, code } = await show(root, ["weather/brooklyn"], {
|
|
605
|
+
const { out, code } = await show(root, ["weather/brooklyn"], { source: true });
|
|
555
606
|
expect(code).toBe(0);
|
|
556
607
|
expect(out).toContain("eval source unavailable for this attempt");
|
|
557
608
|
});
|
|
@@ -569,11 +620,11 @@ describe("--eval", () => {
|
|
|
569
620
|
name: 'calledTool("get_weather")',
|
|
570
621
|
severity: "gate",
|
|
571
622
|
score: 0,
|
|
572
|
-
|
|
623
|
+
outcome: "failed" as const,
|
|
573
624
|
detail: "tool was never called",
|
|
574
625
|
loc: { file: "evals/weather/brooklyn.eval.ts", line: 2 },
|
|
575
626
|
},
|
|
576
|
-
{ name: "unlocated()", severity: "soft", score: 0.5,
|
|
627
|
+
{ name: "unlocated()", severity: "soft", score: 0.5, outcome: "failed" as const, detail: "no loc on this one" },
|
|
577
628
|
],
|
|
578
629
|
}),
|
|
579
630
|
],
|
|
@@ -585,7 +636,7 @@ describe("--eval", () => {
|
|
|
585
636
|
await mkdir(join(dir, "sources"), { recursive: true });
|
|
586
637
|
await writeFile(join(dir, "sources", `${sha}.json`), JSON.stringify({ content }), "utf-8");
|
|
587
638
|
|
|
588
|
-
const { out, code } = await show(root, ["weather/brooklyn"], {
|
|
639
|
+
const { out, code } = await show(root, ["weather/brooklyn"], { source: true });
|
|
589
640
|
expect(code).toBe(0);
|
|
590
641
|
expect(out).toContain(`eval source: evals/weather/brooklyn.eval.ts · sha256:${sha.slice(0, 8)}`);
|
|
591
642
|
// 源码行的缩进是语义的一部分,渲染必须原样保留(不按词重排折叠掉这两个空格)。
|
|
@@ -599,7 +650,7 @@ describe("--eval", () => {
|
|
|
599
650
|
it("源码行标注保留 group 与值断言的 expected/received", async () => {
|
|
600
651
|
const root = await makeRoot();
|
|
601
652
|
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
602
|
-
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0,
|
|
653
|
+
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, outcome: "failed" as const, expected: "4", received: "1", groupPath: ["Issue 15193"], loc: { file: "evals/manager.eval.ts", line: 1 } }] }),
|
|
603
654
|
]);
|
|
604
655
|
const attemptDir = join(dir, "manager/a0");
|
|
605
656
|
const content = "t.check(actual, equals(expected));\n";
|
|
@@ -607,7 +658,7 @@ describe("--eval", () => {
|
|
|
607
658
|
await writeFile(join(attemptDir, "sources.json"), JSON.stringify([{ path: "evals/manager.eval.ts", sha256: sha }]), "utf-8");
|
|
608
659
|
await mkdir(join(dir, "sources"), { recursive: true });
|
|
609
660
|
await writeFile(join(dir, "sources", `${sha}.json`), JSON.stringify({ content }), "utf-8");
|
|
610
|
-
const { out } = await show(root, ["manager"], {
|
|
661
|
+
const { out } = await show(root, ["manager"], { source: true });
|
|
611
662
|
expect(out).toContain("gate · Issue 15193 · equals(4) · expected 4 · received 1");
|
|
612
663
|
});
|
|
613
664
|
});
|
|
@@ -618,7 +669,7 @@ describe("show @<locator>", () => {
|
|
|
618
669
|
it("默认页直接解释失败断言的 group、期望值、实际值和源码位置", async () => {
|
|
619
670
|
const root = await makeRoot();
|
|
620
671
|
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
621
|
-
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0,
|
|
672
|
+
res("manager", "failed", { assertions: [{ name: "equals(4)", severity: "gate", score: 0, outcome: "failed" as const, expected: "4", received: "1", groupPath: ["Issue 15193"], loc: { file: "evals/manager.eval.ts", line: 40, column: 11 } }] }),
|
|
622
673
|
]);
|
|
623
674
|
const results = await openResults(root);
|
|
624
675
|
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
@@ -649,6 +700,41 @@ describe("show @<locator>", () => {
|
|
|
649
700
|
expect(out).not.toContain("available:");
|
|
650
701
|
});
|
|
651
702
|
|
|
703
|
+
it("errored attempt 首页展开结构化 error(phase/code/message/cause)与 diagnostics", async () => {
|
|
704
|
+
const root = await makeRoot();
|
|
705
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/claude-e2b", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
706
|
+
res("agent-029", "errored", {
|
|
707
|
+
error: {
|
|
708
|
+
code: "sandbox-rate-limit",
|
|
709
|
+
message: "E2B sandbox allocation failed after 5 attempts",
|
|
710
|
+
phase: "sandbox.create",
|
|
711
|
+
cause: { name: "RateLimitError", message: "too many concurrent sandboxes" },
|
|
712
|
+
},
|
|
713
|
+
diagnostics: [
|
|
714
|
+
{ code: "teardown-failed", level: "warning", message: "container stop timed out", phase: "sandbox.teardown" },
|
|
715
|
+
],
|
|
716
|
+
}),
|
|
717
|
+
]);
|
|
718
|
+
const results = await openResults(root);
|
|
719
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
720
|
+
|
|
721
|
+
const { out, code } = await show(root, [locator]);
|
|
722
|
+
expect(code).toBe(0);
|
|
723
|
+
expect(out).toContain(`${locator} · agent-029 · compare/claude-e2b · errored`);
|
|
724
|
+
// 结构化 error 块:phase 直接展示闭集点分名,code/message/cause 各一行(见 docs/feature/reports/show.md)
|
|
725
|
+
expect(out).toContain("error:");
|
|
726
|
+
expect(out).toContain("phase: sandbox.create");
|
|
727
|
+
expect(out).toContain("code: sandbox-rate-limit");
|
|
728
|
+
expect(out).toContain("message: E2B sandbox allocation failed after 5 attempts");
|
|
729
|
+
expect(out).toContain("cause: RateLimitError · too many concurrent sandboxes");
|
|
730
|
+
// attempt 级 diagnostics 块(与 verdict 独立)
|
|
731
|
+
expect(out).toContain("diagnostics:");
|
|
732
|
+
expect(out).toContain("warning · sandbox.teardown · teardown-failed");
|
|
733
|
+
expect(out).toContain("container stop timed out");
|
|
734
|
+
// errored 且无断言:不打印空的 assertions 汇总行
|
|
735
|
+
expect(out).not.toContain("assertions:");
|
|
736
|
+
});
|
|
737
|
+
|
|
652
738
|
it("available 只逐行列出实际存在的证据命令,不打印能力字母或合并式 next", async () => {
|
|
653
739
|
const root = await makeRoot();
|
|
654
740
|
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
@@ -666,7 +752,7 @@ describe("show @<locator>", () => {
|
|
|
666
752
|
|
|
667
753
|
const { out, code } = await show(root, [locator]);
|
|
668
754
|
expect(code).toBe(0);
|
|
669
|
-
expect(out).toContain(`available:\n niceeval show ${locator} --
|
|
755
|
+
expect(out).toContain(`available:\n niceeval show ${locator} --source\n niceeval show ${locator} --execution`);
|
|
670
756
|
expect(out).not.toContain(`niceeval show ${locator} --diff`);
|
|
671
757
|
expect(out).not.toContain("evidence:");
|
|
672
758
|
expect(out).not.toContain("next:");
|
|
@@ -688,6 +774,186 @@ describe("show @<locator>", () => {
|
|
|
688
774
|
expect(out).not.toContain("available:");
|
|
689
775
|
});
|
|
690
776
|
|
|
777
|
+
it("单独 --timing 进入完整时间树,不回落到 attempt 首页", async () => {
|
|
778
|
+
const root = await makeRoot();
|
|
779
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
780
|
+
res("weather/brooklyn", "passed", {
|
|
781
|
+
durationMs: 50_000,
|
|
782
|
+
phases: [
|
|
783
|
+
{ name: "eval.run", durationMs: 42_000 },
|
|
784
|
+
{ name: "sandbox.stop", durationMs: 800 },
|
|
785
|
+
],
|
|
786
|
+
}),
|
|
787
|
+
]);
|
|
788
|
+
const results = await openResults(root);
|
|
789
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
790
|
+
|
|
791
|
+
const { out, code } = await show(root, [locator], { timing: true });
|
|
792
|
+
expect(code).toBe(0);
|
|
793
|
+
expect(out).toContain("total 50.0s");
|
|
794
|
+
expect(out).toContain("eval.run 42.0s");
|
|
795
|
+
expect(out).toContain("teardown (not counted in total):");
|
|
796
|
+
expect(out).toContain("sandbox.stop 800ms");
|
|
797
|
+
expect(out).not.toContain("assertions:");
|
|
798
|
+
expect(out).not.toContain("available:");
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
it("--timing 在 80 个 detail node 内与 full 等价,81 个开始原位省略", async () => {
|
|
802
|
+
const root = await makeRoot();
|
|
803
|
+
const children: TimingNode[] = Array.from({ length: 81 }, (_, index) => ({
|
|
804
|
+
id: `command-${String(index).padStart(3, "0")}`,
|
|
805
|
+
kind: "command",
|
|
806
|
+
label: "tool",
|
|
807
|
+
startOffsetMs: index * 10,
|
|
808
|
+
durationMs: 81 - index,
|
|
809
|
+
command: { display: `tool --item=${index}` },
|
|
810
|
+
}));
|
|
811
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
812
|
+
res("weather/brooklyn", "passed", {
|
|
813
|
+
durationMs: 50_000,
|
|
814
|
+
phases: [
|
|
815
|
+
{ name: "sandbox.create", durationMs: 1_000 },
|
|
816
|
+
{ name: "eval.run", durationMs: 42_000, children },
|
|
817
|
+
{ name: "sandbox.stop", durationMs: 800 },
|
|
818
|
+
],
|
|
819
|
+
}),
|
|
820
|
+
]);
|
|
821
|
+
const results = await openResults(root);
|
|
822
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
823
|
+
|
|
824
|
+
const bounded = await show(root, [locator], { timing: "summary" });
|
|
825
|
+
const full = await show(root, [locator], { timing: "full" });
|
|
826
|
+
expect(bounded.out).toContain("sandbox.create");
|
|
827
|
+
expect(bounded.out).toContain("eval.run");
|
|
828
|
+
expect(bounded.out).toContain("sandbox.stop");
|
|
829
|
+
expect(bounded.out).toMatch(/… 1 nodes omitted/);
|
|
830
|
+
expect(bounded.out).toContain(`niceeval show ${locator} --timing=full`);
|
|
831
|
+
expect(full.out).not.toContain("nodes omitted");
|
|
832
|
+
expect(full.out).toContain("tool --item=0");
|
|
833
|
+
expect(full.out).toContain("tool --item=80");
|
|
834
|
+
|
|
835
|
+
// 去掉第 81 个后,默认投影不再省略,detail 行与 full 完全一致。
|
|
836
|
+
children.pop();
|
|
837
|
+
const root80 = await makeRoot();
|
|
838
|
+
await writeSnapshot(root80, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
839
|
+
res("weather/brooklyn", "passed", {
|
|
840
|
+
durationMs: 50_000,
|
|
841
|
+
phases: [{ name: "eval.run", durationMs: 42_000, children }],
|
|
842
|
+
}),
|
|
843
|
+
]);
|
|
844
|
+
const results80 = await openResults(root80);
|
|
845
|
+
const locator80 = results80.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
846
|
+
const bounded80 = await show(root80, [locator80], { timing: "summary" });
|
|
847
|
+
const full80 = await show(root80, [locator80], { timing: "full" });
|
|
848
|
+
expect(bounded80.out).toBe(full80.out);
|
|
849
|
+
});
|
|
850
|
+
|
|
851
|
+
it("默认投影保留失败路径/慢点/首尾,omission 报失败数且不虚构 children 合计", async () => {
|
|
852
|
+
const root = await makeRoot();
|
|
853
|
+
const children: TimingNode[] = Array.from({ length: 120 }, (_, index) => ({
|
|
854
|
+
id: `node-${String(index).padStart(3, "0")}`,
|
|
855
|
+
kind: "provider",
|
|
856
|
+
label: `provider-step-${index}`,
|
|
857
|
+
startOffsetMs: index * 100,
|
|
858
|
+
durationMs: index === 60 ? 99_000 : index + 1,
|
|
859
|
+
...(index === 55 ? { failed: true as const } : {}),
|
|
860
|
+
}));
|
|
861
|
+
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
862
|
+
res("weather/brooklyn", "errored", {
|
|
863
|
+
durationMs: 120_000,
|
|
864
|
+
error: { code: "provider-failed", message: "boom", phase: "sandbox.create" },
|
|
865
|
+
phases: [{ name: "sandbox.create", durationMs: 120_000, failed: true, children }],
|
|
866
|
+
}),
|
|
867
|
+
]);
|
|
868
|
+
const results = await openResults(root);
|
|
869
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
870
|
+
const { out } = await show(root, [locator], { timing: "summary" });
|
|
871
|
+
|
|
872
|
+
expect(out).toContain("provider-step-0");
|
|
873
|
+
expect(out).toContain("provider-step-55");
|
|
874
|
+
expect(out).toContain("provider-step-60");
|
|
875
|
+
expect(out).toContain("provider-step-119");
|
|
876
|
+
expect(out).not.toContain("combined");
|
|
877
|
+
});
|
|
878
|
+
|
|
879
|
+
it("operation 使用 producer label;full 按 parentSpanId 展开唯一关联 OTel 树", async () => {
|
|
880
|
+
const root = await makeRoot();
|
|
881
|
+
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
882
|
+
res("weather/brooklyn", "passed", {
|
|
883
|
+
durationMs: 50_000,
|
|
884
|
+
hasTrace: true,
|
|
885
|
+
phases: [{
|
|
886
|
+
name: "workspace.diff",
|
|
887
|
+
durationMs: 5_000,
|
|
888
|
+
children: [{
|
|
889
|
+
id: "operation-1",
|
|
890
|
+
kind: "operation",
|
|
891
|
+
label: "export workspace diff · 1 window · 3,302 files",
|
|
892
|
+
startOffsetMs: 100,
|
|
893
|
+
durationMs: 4_000,
|
|
894
|
+
children: [{
|
|
895
|
+
id: "turn-1",
|
|
896
|
+
kind: "turn",
|
|
897
|
+
label: "s1/t1",
|
|
898
|
+
startOffsetMs: 200,
|
|
899
|
+
durationMs: 3_000,
|
|
900
|
+
traceId: "trace-1",
|
|
901
|
+
}],
|
|
902
|
+
}],
|
|
903
|
+
}],
|
|
904
|
+
}),
|
|
905
|
+
]);
|
|
906
|
+
const trace: TraceSpan[] = [
|
|
907
|
+
{ traceId: "trace-1", spanId: "child", parentSpanId: "root", name: "chat", kind: "model", startMs: 1_100, endMs: 2_000 },
|
|
908
|
+
{ traceId: "trace-1", spanId: "root", name: "codex.exec", kind: "agent", startMs: 1_000, endMs: 3_000 },
|
|
909
|
+
];
|
|
910
|
+
await writeFile(join(dir, "weather/brooklyn/a0/trace.json"), JSON.stringify(trace), "utf-8");
|
|
911
|
+
const results = await openResults(root);
|
|
912
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
913
|
+
const { out } = await show(root, [locator], { timing: "full" });
|
|
914
|
+
|
|
915
|
+
expect(out).toContain("operation · export workspace diff · 1 window · 3,302 files");
|
|
916
|
+
expect(out).toMatch(/agent · codex\.exec[\s\S]*model · chat/);
|
|
917
|
+
expect(out).not.toContain("git show ×");
|
|
918
|
+
});
|
|
919
|
+
|
|
920
|
+
it("attempt 首页 timing 只列大头,但保留变慢的 telemetry 阶段", async () => {
|
|
921
|
+
const root = await makeRoot();
|
|
922
|
+
const dir = await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
|
923
|
+
res("weather/brooklyn", "passed", {
|
|
924
|
+
durationMs: 100_000,
|
|
925
|
+
hasEvents: true,
|
|
926
|
+
phases: [
|
|
927
|
+
{ name: "sandbox.queue", durationMs: 0 },
|
|
928
|
+
{ name: "sandbox.create", durationMs: 3_000 },
|
|
929
|
+
{ name: "workspace.baseline", durationMs: 310 },
|
|
930
|
+
{ name: "telemetry.configure", durationMs: 358 },
|
|
931
|
+
{ name: "eval.run", durationMs: 45_000 },
|
|
932
|
+
{ name: "telemetry.collect", durationMs: 35_300 },
|
|
933
|
+
{ name: "sandbox.stop", durationMs: 8_500 },
|
|
934
|
+
],
|
|
935
|
+
}),
|
|
936
|
+
]);
|
|
937
|
+
await writeFile(
|
|
938
|
+
join(dir, "weather/brooklyn/a0/events.json"),
|
|
939
|
+
JSON.stringify([{ type: "message", role: "assistant", text: "done" }]),
|
|
940
|
+
"utf-8",
|
|
941
|
+
);
|
|
942
|
+
const results = await openResults(root);
|
|
943
|
+
const locator = results.experiments[0]!.latest.evals[0]!.attempts[0]!.locator!;
|
|
944
|
+
|
|
945
|
+
const { out } = await show(root, [locator]);
|
|
946
|
+
const timingLine = out.split("\n").find((line) => line.startsWith("timing:"));
|
|
947
|
+
expect(out).toContain("1 AI message");
|
|
948
|
+
expect(timingLine).toContain("sandbox.queue 0ms");
|
|
949
|
+
expect(timingLine).toContain("sandbox.create 3.0s");
|
|
950
|
+
expect(timingLine).toContain("eval.run 45.0s");
|
|
951
|
+
expect(timingLine).toContain("telemetry.collect 35.3s");
|
|
952
|
+
expect(timingLine).toContain("teardown +8.5s");
|
|
953
|
+
expect(timingLine).not.toContain("workspace.baseline");
|
|
954
|
+
expect(timingLine).not.toContain("telemetry.configure");
|
|
955
|
+
});
|
|
956
|
+
|
|
691
957
|
it("语法不对的 locator 报「not a valid attempt locator」,退出码 1,不崩", async () => {
|
|
692
958
|
const root = await makeRoot();
|
|
693
959
|
await writeSnapshot(root, "2026-07-08T10-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-08T10:00:00.000Z" }, [
|
package/src/tty-line.ts
CHANGED
|
@@ -1,32 +1,14 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 独立诊断行的 bootstrap 终端出口:只服务 run 启动前的 argv/config 错误,以及崩溃兜底/
|
|
2
|
+
// 尚无活跃 coordinator 时的兜底(见 src/runner/feedback/sink.ts 的 no-active-coordinator 分支)。
|
|
2
3
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
// memory/live-raw-stderr-write-desyncs-redraw.md)。
|
|
10
|
-
//
|
|
11
|
-
// 所以任何不经过 Reporter.progress() / onEvalComplete() 的独立诊断行,写之前都要先广播
|
|
12
|
-
// beforeExternalTerminalWrite(),live.ts 借此机会把已画内容清掉、drawnLines 归零,下一次
|
|
13
|
-
// draw() 就会把这行诊断消息之下当成全新起点重画,不会累积。
|
|
14
|
-
|
|
15
|
-
const listeners = new Set<() => void>();
|
|
16
|
-
|
|
17
|
-
/** live.ts 用来订阅"即将有一行独立诊断消息落地"。返回取消订阅函数。 */
|
|
18
|
-
export function onBeforeExternalTerminalWrite(fn: () => void): () => void {
|
|
19
|
-
listeners.add(fn);
|
|
20
|
-
return () => listeners.delete(fn);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/** 任何要绕开 Reporter 直接往终端打一行独立诊断消息的地方,写之前都先调用这个。 */
|
|
24
|
-
export function beforeExternalTerminalWrite(): void {
|
|
25
|
-
for (const fn of listeners) fn();
|
|
26
|
-
}
|
|
4
|
+
// run 激活之后的全部反馈(dashboard、durable event、heartbeat)由
|
|
5
|
+
// src/runner/feedback/coordinator.ts 统一排序;它通过 sink.ts 的模块级 coordinator 栈直接接管
|
|
6
|
+
// reportXxx() 调用的目的地(显式路由,不是广播-订阅),所以这里不再需要"写之前先广播、订阅方
|
|
7
|
+
// 自己清屏"的机制——那套机制是为已删除的 live.ts 的 ANSI 回跳重画设计的(见
|
|
8
|
+
// memory/live-raw-stderr-write-desyncs-redraw.md),新 coordinator 的 clear → append → redraw
|
|
9
|
+
// 顺序由 sink.ts 的显式路由保证,不依赖本文件。
|
|
27
10
|
|
|
28
11
|
/** process.stderr.write 的替代:文本需自带换行(沿用现有 i18n 字符串的约定)。 */
|
|
29
12
|
export function writeStderrLine(text: string): void {
|
|
30
|
-
beforeExternalTerminalWrite();
|
|
31
13
|
process.stderr.write(text);
|
|
32
14
|
}
|
package/src/util.test.ts
CHANGED