niceeval 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +69 -7
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +57 -3
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +32 -24
- package/dist/report/aggregate.js +158 -50
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +93 -160
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +87 -81
- package/dist/report/compute.js +597 -417
- package/dist/report/flag.d.ts +32 -6
- package/dist/report/flag.js +92 -4
- package/dist/report/format.d.ts +19 -11
- package/dist/report/format.js +30 -13
- package/dist/report/index.d.ts +16 -16
- package/dist/report/index.js +20 -21
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +57 -33
- package/dist/report/locale.js +122 -56
- package/dist/report/metrics.d.ts +23 -4
- package/dist/report/metrics.js +110 -25
- package/dist/report/primitives.d.ts +48 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +9 -7
- package/dist/report/react/AttemptList.js +17 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +4 -4
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +10 -0
- package/dist/report/react/ExperimentComparison.js +12 -0
- package/dist/report/react/ExperimentList.d.ts +4 -3
- package/dist/report/react/ExperimentList.js +17 -18
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +59 -28
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +110 -147
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -16
- package/dist/report/report.js +259 -28
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +253 -184
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +38 -7
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +247 -284
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +49 -14
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +4 -2
- package/dist/shared/aggregate.js +8 -7
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
- package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
- package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
- package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
- package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
- package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
- package/docs-site/zh/how-to/publish-report.mdx +105 -0
- package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
- package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
- package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
- package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
- package/docs-site/zh/index.mdx +24 -26
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +32 -5
- package/docs-site/zh/reference/capabilities.mdx +8 -8
- package/docs-site/zh/reference/cli.mdx +40 -12
- package/docs-site/zh/reference/define-agent.mdx +58 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
- package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
- package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
- package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +10 -2
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +13 -2
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +71 -8
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +446 -124
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +162 -30
- package/src/context/session.test.ts +2 -1
- package/src/context/session.ts +115 -7
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +81 -17
- package/src/i18n/zh-CN.ts +80 -17
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +178 -61
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -279
- package/src/report/compute.ts +723 -491
- package/src/report/dual-render.test.tsx +741 -1024
- package/src/report/flag.ts +104 -12
- package/src/report/format.ts +32 -12
- package/src/report/index.ts +119 -46
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +136 -65
- package/src/report/metrics.ts +108 -25
- package/src/report/primitives.tsx +196 -45
- package/src/report/react/AttemptList.tsx +30 -43
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +73 -0
- package/src/report/react/ExperimentList.tsx +50 -32
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +86 -34
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +89 -5
- package/src/report/react/fixtures.ts +114 -154
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +138 -158
- package/src/report/react/styles.css +243 -82
- package/src/report/report.test.ts +779 -841
- package/src/report/report.ts +423 -41
- package/src/report/text/faces.ts +290 -193
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +44 -7
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +261 -271
- package/src/report/web.ts +63 -20
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +13 -11
- package/src/results/attempt-evidence.ts +20 -13
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +150 -60
- package/src/results/host-equivalence.test.ts +34 -20
- package/src/results/index.ts +12 -4
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +15 -5
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +89 -54
- package/src/results/select.ts +104 -34
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +43 -14
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +243 -37
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +8 -6
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +127 -56
- package/src/show/render.ts +662 -131
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +320 -54
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +423 -11
- package/src/view/app/components/CodeView.tsx +41 -14
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +37 -17
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +10 -3
- package/src/view/data.ts +155 -49
- package/src/view/index.ts +56 -41
- package/src/view/server.ts +37 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +227 -0
- package/src/view/view-report.test.ts +167 -62
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
- package/dist/report/built-ins/experiment-comparison.js +0 -13
- package/dist/report/built-ins/index.d.ts +0 -1
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/docs-site/zh/guides/publish-report.mdx +0 -91
- package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
- package/src/report/built-in-user-parity.test.tsx +0 -640
- package/src/report/built-ins/experiment-comparison.tsx +0 -19
- package/src/report/built-ins/index.ts +0 -2
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
package/src/show/index.ts
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
|
-
// niceeval show —— 终端宿主(行为规范:docs
|
|
2
|
-
// 宿主组合语义:docs/feature/reports/architecture.md「
|
|
1
|
+
// niceeval show —— 终端宿主(行为规范:docs/feature/reports/show.md 与分篇;
|
|
2
|
+
// 宿主组合语义:docs/feature/reports/architecture.md「Scope 是计算入口」)。
|
|
3
3
|
//
|
|
4
4
|
// 位置参数 = eval id 前缀,或 `@<locator>`(精确指名单个 attempt,见 results/locator.ts):
|
|
5
|
-
// 裸跑 / 多 eval 前缀
|
|
5
|
+
// 裸跑 / 多 eval 前缀 内建报告(niceeval/report/built-in 默认导出)的 text 面(单 eval 前缀仍进入详情)
|
|
6
6
|
// 恰好一个 eval 单 eval 详情(attempt / 断言明细,宿主本体)
|
|
7
7
|
// @<locator> 精确 attempt:无证据 flag → 紧凑全景;带 flag → 对应证据切面
|
|
8
|
-
// --
|
|
9
|
-
// --history
|
|
10
|
-
// --report <文件> 整槽换成用户报告;位置前缀 / --
|
|
11
|
-
// --
|
|
8
|
+
// --source / --execution / --diff[=路径] 证据切面(宿主本体):出现即走证据室,不渲染报告槽
|
|
9
|
+
// --history 执行时间轴(逐 experimentId + evalId 分节),与 --report 互斥
|
|
10
|
+
// --report <文件> 整槽换成用户报告;位置前缀 / --results / --experiment 先收窄 Scope 再注入
|
|
11
|
+
// --page <id> 多页报告选页;未命中列出可用页 id 按用法错误退出
|
|
12
|
+
// --results <目录> 结果根换成该目录;--experiment 让 Scope 只留该实验
|
|
12
13
|
//
|
|
13
|
-
// 数据全部走 niceeval/results 的读取面(openResults + 合成
|
|
14
|
-
//
|
|
14
|
+
// 数据全部走 niceeval/results 的读取面(openResults + 合成 Scope + loadAttemptEvidence),
|
|
15
|
+
// 不自己爬目录;证据可用性只由 loadAttemptEvidence 在单 Attempt 页面计算。
|
|
15
16
|
|
|
16
17
|
import { existsSync } from "node:fs";
|
|
17
18
|
import { join, resolve } from "node:path";
|
|
@@ -23,51 +24,62 @@ import {
|
|
|
23
24
|
LocatorNotFoundError,
|
|
24
25
|
MalformedLocatorError,
|
|
25
26
|
} from "../results/index.ts";
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
|
|
32
|
-
// when niceeval is linked in — see tsconfig.report-build.json); show pulls all of it from
|
|
33
|
-
// there, not just the .tsx-touching pieces.
|
|
34
|
-
import { renderReportToText } from "../../dist/report/report.js";
|
|
35
|
-
import { ExperimentComparison } from "../../dist/report/built-ins/index.js";
|
|
36
|
-
import { ReportLoadError, loadReportFile } from "../../dist/report/load.js";
|
|
27
|
+
// ReportLoadError must come from the SAME module instance the report runtime is built
|
|
28
|
+
// against — `instanceof` is keyed by declaration site, so a raw src copy and the compiled
|
|
29
|
+
// dist copy of "the same" class are two different types. The package-owned report runtime
|
|
30
|
+
// ships as precompiled ESM (dist/report/**, built by `pnpm run build:report`); all report
|
|
31
|
+
// loading/rendering goes through ./report-host.ts (the single contact surface with it).
|
|
32
|
+
import { ReportLoadError } from "../../dist/report/load.js";
|
|
37
33
|
import { detectLocale, t } from "../i18n/index.ts";
|
|
38
34
|
import { foldEvalVerdict } from "../shared/verdict.ts";
|
|
39
35
|
import { selectCurrentResults, filterExperiments } from "../results/select.ts";
|
|
40
|
-
import {
|
|
36
|
+
import { evalPrefixPredicate } from "../shared/aggregate.ts";
|
|
37
|
+
import { attemptHistory } from "./compose.ts";
|
|
38
|
+
import {
|
|
39
|
+
HostReportError,
|
|
40
|
+
loadHostReport,
|
|
41
|
+
localizeText,
|
|
42
|
+
reportMetaFor,
|
|
43
|
+
renderHostPageText,
|
|
44
|
+
resolveReportTitle,
|
|
45
|
+
type HostCommandContext,
|
|
46
|
+
} from "./report-host.ts";
|
|
41
47
|
import {
|
|
42
48
|
attemptArtifactsPath,
|
|
43
49
|
attemptEvidenceHeader,
|
|
50
|
+
attemptHistoryText,
|
|
44
51
|
attemptIndexLine,
|
|
45
52
|
attemptOverviewText,
|
|
46
53
|
attemptsOfEval,
|
|
47
54
|
diffText,
|
|
48
55
|
evalDetailText,
|
|
49
|
-
evalHistoryText,
|
|
50
56
|
evalSourceText,
|
|
51
57
|
executionText,
|
|
52
|
-
|
|
58
|
+
pageIndexText,
|
|
59
|
+
timingText,
|
|
53
60
|
pickDetailAttempt,
|
|
54
61
|
skippedRunsText,
|
|
55
62
|
verdictReasonLine,
|
|
56
63
|
} from "./render.ts";
|
|
57
64
|
|
|
58
65
|
export interface ShowFlags {
|
|
59
|
-
/**
|
|
60
|
-
|
|
66
|
+
/** --source:该 attempt 运行时保存的 Eval 源码,断言标回源码行(证据切面)。 */
|
|
67
|
+
source?: boolean;
|
|
61
68
|
/** 该 attempt 的标准执行事件流 + OTel enrichment(证据切面)。 */
|
|
62
69
|
execution?: boolean;
|
|
70
|
+
/** --timing:默认有界诊断投影;full 逐节点展开。boolean 仅供库调用兼容,等价 summary。 */
|
|
71
|
+
timing?: boolean | "summary" | "full";
|
|
63
72
|
/** --diff(文件级摘要)。 */
|
|
64
73
|
diff?: boolean;
|
|
65
74
|
/** --diff=<路径>(单个文件的完整改动;路径必须 = 连写,位置参数永远留给 eval id 前缀)。 */
|
|
66
75
|
diffPath?: string;
|
|
67
76
|
history?: boolean;
|
|
68
77
|
experiment?: string;
|
|
69
|
-
|
|
78
|
+
/** --results:结果根目录(某次快照根或 `copySnapshots` 产物)。 */
|
|
79
|
+
results?: string;
|
|
70
80
|
report?: string;
|
|
81
|
+
/** --page:多页报告选页;未命中按用法错误退出并列出可用页 id。 */
|
|
82
|
+
page?: string;
|
|
71
83
|
}
|
|
72
84
|
|
|
73
85
|
/** 注入 IO 供测试;默认写 stdout/stderr、宽度取终端列数。 */
|
|
@@ -86,8 +98,8 @@ function clampWidth(columns: number | undefined): number {
|
|
|
86
98
|
return Math.max(40, Math.min(columns as number, 160));
|
|
87
99
|
}
|
|
88
100
|
|
|
89
|
-
// --report
|
|
90
|
-
export {
|
|
101
|
+
// --report 的装载与规范化住在 ./report-host.ts(两个宿主共用的唯一联系面)。
|
|
102
|
+
export { loadHostReport, normalizeHostReport, resolveReportTitle, localizeText } from "./report-host.ts";
|
|
91
103
|
|
|
92
104
|
export async function runShow(
|
|
93
105
|
cwd: string,
|
|
@@ -106,7 +118,7 @@ export async function runShow(
|
|
|
106
118
|
});
|
|
107
119
|
return 0;
|
|
108
120
|
} catch (e) {
|
|
109
|
-
if (e instanceof ShowError || e instanceof ReportLoadError) {
|
|
121
|
+
if (e instanceof ShowError || e instanceof ReportLoadError || e instanceof HostReportError) {
|
|
110
122
|
err(e.message.endsWith("\n") ? e.message : `${e.message}\n`);
|
|
111
123
|
return 1;
|
|
112
124
|
}
|
|
@@ -120,15 +132,27 @@ async function show(
|
|
|
120
132
|
flags: ShowFlags,
|
|
121
133
|
io: { out: (s: string) => void; err: (s: string) => void; width: number; now: number },
|
|
122
134
|
): Promise<void> {
|
|
123
|
-
const evidence =
|
|
135
|
+
const evidence =
|
|
136
|
+
flags.source === true ||
|
|
137
|
+
flags.execution === true ||
|
|
138
|
+
(flags.timing !== undefined && flags.timing !== false) ||
|
|
139
|
+
flags.diff === true ||
|
|
140
|
+
flags.diffPath !== undefined;
|
|
124
141
|
|
|
125
142
|
// 组合语义矩阵(docs/feature/reports/show.md「选择结果范围」):--history 与 --report 互斥,先于任何 IO 报出来。
|
|
126
143
|
if (flags.history && flags.report !== undefined) {
|
|
127
144
|
throw new ShowError(t("cli.show.historyReportConflict"));
|
|
128
145
|
}
|
|
129
146
|
|
|
130
|
-
|
|
131
|
-
if (flags.
|
|
147
|
+
// --page 只在报告槽里有意义:证据切面 / 时间轴与它组合是用法矛盾,先于任何 IO 报出来。
|
|
148
|
+
if (flags.page !== undefined && (evidence || flags.history)) {
|
|
149
|
+
throw new ShowError(
|
|
150
|
+
`--page selects a report page and cannot be combined with ${flags.history ? "--history" : "evidence flags"}.\n`,
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const root = flags.results !== undefined ? resolve(cwd, flags.results) : join(cwd, ".niceeval");
|
|
155
|
+
if (flags.results !== undefined && !existsSync(root)) {
|
|
132
156
|
throw new ShowError(t("cli.show.runDirMissing", { dir: root }));
|
|
133
157
|
}
|
|
134
158
|
|
|
@@ -142,7 +166,7 @@ async function show(
|
|
|
142
166
|
// (`@` 打头对 eval id 天然无歧义,见 locator.ts),必须在下面的前缀匹配逻辑之前分流掉,
|
|
143
167
|
// 不然 "@1x7f3q" 会被当成一个谁都匹配不到的 eval id 前缀,报「no eval match」这种文不对题的
|
|
144
168
|
// 错误。这一步只解析并渲染出「当前 show 对单个已解析 attempt 能渲染的东西」(单 eval 详情 /
|
|
145
|
-
// 三个证据切面)——真正的 `--
|
|
169
|
+
// 三个证据切面)——真正的 `--source`/`--execution`/`--diff` 统一 attempt 全景是后续阶段。
|
|
146
170
|
const locatorArg = patterns.find((p) => p.startsWith(ATTEMPT_LOCATOR_PREFIX));
|
|
147
171
|
if (locatorArg !== undefined) {
|
|
148
172
|
if (patterns.length !== 1) {
|
|
@@ -163,8 +187,14 @@ async function show(
|
|
|
163
187
|
const artifactPath = attemptArtifactsPath(attempt, cwd);
|
|
164
188
|
if (evidence) {
|
|
165
189
|
const blocks: string[] = [];
|
|
166
|
-
if (flags.
|
|
190
|
+
if (flags.source) blocks.push(evalSourceText(attemptEvidence, { header, artifactPath, width: io.width }));
|
|
167
191
|
if (flags.execution) blocks.push(executionText(attemptEvidence, { header, artifactPath, width: io.width }));
|
|
192
|
+
if (flags.timing) blocks.push(timingText(attemptEvidence, {
|
|
193
|
+
header,
|
|
194
|
+
artifactPath,
|
|
195
|
+
width: io.width,
|
|
196
|
+
mode: flags.timing === "full" ? "full" : "summary",
|
|
197
|
+
}));
|
|
168
198
|
if (flags.diff || flags.diffPath !== undefined) {
|
|
169
199
|
blocks.push(diffText({ header, diff: attemptEvidence.diff, artifactPath, file: flags.diffPath }));
|
|
170
200
|
}
|
|
@@ -226,8 +256,14 @@ async function show(
|
|
|
226
256
|
const header = attemptEvidenceHeader(attemptEvidence);
|
|
227
257
|
const artifactPath = attemptArtifactsPath(picked, cwd);
|
|
228
258
|
const blocks: string[] = [];
|
|
229
|
-
if (flags.
|
|
259
|
+
if (flags.source) blocks.push(evalSourceText(attemptEvidence, { header, artifactPath, width: io.width }));
|
|
230
260
|
if (flags.execution) blocks.push(executionText(attemptEvidence, { header, artifactPath, width: io.width }));
|
|
261
|
+
if (flags.timing) blocks.push(timingText(attemptEvidence, {
|
|
262
|
+
header,
|
|
263
|
+
artifactPath,
|
|
264
|
+
width: io.width,
|
|
265
|
+
mode: flags.timing === "full" ? "full" : "summary",
|
|
266
|
+
}));
|
|
231
267
|
if (flags.diff || flags.diffPath !== undefined) {
|
|
232
268
|
blocks.push(diffText({ header, diff: attemptEvidence.diff, artifactPath, file: flags.diffPath }));
|
|
233
269
|
}
|
|
@@ -235,30 +271,30 @@ async function show(
|
|
|
235
271
|
return;
|
|
236
272
|
}
|
|
237
273
|
|
|
238
|
-
// --history
|
|
274
|
+
// --history:执行时间轴(docs/feature/reports/show.md「--history:一个 eval 的执行时间轴」)。
|
|
275
|
+
// 对 Scope 中匹配的每个 experimentId + evalId 分节,逐 attempt 而非逐快照;时间轴只列
|
|
276
|
+
// 真实执行 —— resume 携带的复印件按 attempt 身份键去重后不占行。
|
|
239
277
|
if (flags.history) {
|
|
240
278
|
const experiments = filterExperiments(results.experiments, flags.experiment);
|
|
279
|
+
// eval 位置参数与 Scope 选择用同一个前缀谓词(单点在 shared/aggregate.ts),不另立口径。
|
|
280
|
+
const matchesPattern = patterns.length > 0 ? evalPrefixPredicate(patterns) : () => true;
|
|
241
281
|
const blocks: string[] = [];
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
const rows = evalHistory(exp, evalId);
|
|
249
|
-
if (rows.length === 0) continue;
|
|
250
|
-
blocks.push(evalHistoryText({ experimentId: exp.id, ...(multi ? { evalId } : {}), rows }));
|
|
251
|
-
}
|
|
282
|
+
for (const exp of experiments) {
|
|
283
|
+
const evalIds = [...exp.evalIds].filter(matchesPattern).sort();
|
|
284
|
+
for (const evalId of evalIds) {
|
|
285
|
+
const rows = attemptHistory(exp, evalId);
|
|
286
|
+
if (rows.length === 0) continue;
|
|
287
|
+
blocks.push(attemptHistoryText({ experimentId: exp.id, evalId, rows }));
|
|
252
288
|
}
|
|
253
289
|
}
|
|
254
290
|
io.out(blocks.join("\n\n") + "\n");
|
|
255
291
|
return;
|
|
256
292
|
}
|
|
257
293
|
|
|
258
|
-
// 单 eval 详情(宿主本体);--report 在场时报告槽优先,前缀只用来收窄
|
|
294
|
+
// 单 eval 详情(宿主本体);--report / --page 在场时报告槽优先,前缀只用来收窄 Scope。
|
|
259
295
|
// 挑哪个 attempt 展开明细不再收数字 --attempt——pickDetailAttempt 的默认启发式
|
|
260
296
|
// (最新一次失败,没有失败挑最新一次)是唯一路径;精确选某一次走 `@<locator>`。
|
|
261
|
-
if (flags.report === undefined && patterns.length > 0 && matchedEvalIds.length === 1) {
|
|
297
|
+
if (flags.report === undefined && flags.page === undefined && patterns.length > 0 && matchedEvalIds.length === 1) {
|
|
262
298
|
const evalId = matchedEvalIds[0];
|
|
263
299
|
const attempts = attemptsOfEval(selection.snapshots, evalId);
|
|
264
300
|
const detail = pickDetailAttempt(attempts);
|
|
@@ -275,13 +311,48 @@ async function show(
|
|
|
275
311
|
return;
|
|
276
312
|
}
|
|
277
313
|
|
|
278
|
-
//
|
|
279
|
-
//
|
|
280
|
-
//
|
|
281
|
-
|
|
282
|
-
const
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
314
|
+
// 报告槽:裸 show 装载 `niceeval/report/built-in` 的默认导出,--report 整槽替换——同一条
|
|
315
|
+
// 「装载 → 规范化(外壳 + 非空页列表)→ 逐页渲染」管线(docs/feature/reports/library/shell.md)。
|
|
316
|
+
// locale = CLI 界面语言(NICEEVAL_LANG / LC_* / LANG 检测):报告 chrome 文案跟随终端语言。
|
|
317
|
+
const report = await loadHostReport(cwd, flags.report);
|
|
318
|
+
const locale = detectLocale();
|
|
319
|
+
const commandContext: HostCommandContext = {
|
|
320
|
+
patterns,
|
|
321
|
+
...(flags.results !== undefined ? { results: flags.results } : {}),
|
|
322
|
+
...(flags.report !== undefined ? { report: flags.report } : {}),
|
|
323
|
+
...(flags.experiment !== undefined ? { experiment: flags.experiment } : {}),
|
|
324
|
+
};
|
|
325
|
+
const sourceLabel = flags.report ?? "the built-in report";
|
|
326
|
+
|
|
327
|
+
let page = report.pages[0];
|
|
328
|
+
if (flags.page !== undefined) {
|
|
329
|
+
const hit = report.pages.find((p) => p.id === flags.page);
|
|
330
|
+
if (!hit) {
|
|
331
|
+
// 用法错误:列出可用页 id(docs/feature/reports/show/reports.md Case 1/2 的报错样例)。
|
|
332
|
+
throw new ShowError(
|
|
333
|
+
`error: page "${flags.page}" not found in ${sourceLabel}. Available pages: ${report.pages.map((p) => p.id).join(", ")}\n`,
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
page = hit;
|
|
337
|
+
} else if (report.pages.length > 1) {
|
|
338
|
+
// 多页未选页:只输出页索引与可复制的单页命令,不倾倒页内容(与可比组索引同一模式)。
|
|
339
|
+
// 标题行走标题回退链(终点是内置文案「Eval 运行结果 / Eval Results」,恒有值)。
|
|
340
|
+
const title = localizeText(resolveReportTitle(report.title, selection.snapshots), locale) ?? "Eval Results";
|
|
341
|
+
io.out(pageIndexText({ report, title, command: commandContext, locale }) + "\n");
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// attemptCommand 留给渲染管线的默认值:AttemptLocator 已经是可直接 `niceeval show @<locator>`
|
|
346
|
+
// 的真实 CLI 语法,不需要再反查 eval id 拼一条近似命令。
|
|
347
|
+
const meta = reportMetaFor(report, page.id, selection.snapshots);
|
|
348
|
+
const text = await renderHostPageText(
|
|
349
|
+
page,
|
|
350
|
+
{ scope: selection, results, report: meta },
|
|
351
|
+
{
|
|
352
|
+
width: io.width,
|
|
353
|
+
locale,
|
|
354
|
+
commandContext: { ...commandContext, ...(flags.page !== undefined ? { page: flags.page } : {}) },
|
|
355
|
+
},
|
|
356
|
+
);
|
|
286
357
|
io.out(text + "\n");
|
|
287
358
|
}
|