niceeval 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +69 -7
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +57 -3
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +32 -24
- package/dist/report/aggregate.js +158 -50
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +93 -160
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +87 -81
- package/dist/report/compute.js +597 -417
- package/dist/report/flag.d.ts +32 -6
- package/dist/report/flag.js +92 -4
- package/dist/report/format.d.ts +19 -11
- package/dist/report/format.js +30 -13
- package/dist/report/index.d.ts +16 -16
- package/dist/report/index.js +20 -21
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +57 -33
- package/dist/report/locale.js +122 -56
- package/dist/report/metrics.d.ts +23 -4
- package/dist/report/metrics.js +110 -25
- package/dist/report/primitives.d.ts +48 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +9 -7
- package/dist/report/react/AttemptList.js +17 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +4 -4
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +10 -0
- package/dist/report/react/ExperimentComparison.js +12 -0
- package/dist/report/react/ExperimentList.d.ts +4 -3
- package/dist/report/react/ExperimentList.js +17 -18
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +59 -28
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +110 -147
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -16
- package/dist/report/report.js +259 -28
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +253 -184
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +38 -7
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +247 -284
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +49 -14
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +4 -2
- package/dist/shared/aggregate.js +8 -7
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
- package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
- package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
- package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
- package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
- package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
- package/docs-site/zh/how-to/publish-report.mdx +105 -0
- package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
- package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
- package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
- package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
- package/docs-site/zh/index.mdx +24 -26
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +32 -5
- package/docs-site/zh/reference/capabilities.mdx +8 -8
- package/docs-site/zh/reference/cli.mdx +40 -12
- package/docs-site/zh/reference/define-agent.mdx +58 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
- package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
- package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
- package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +10 -2
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +13 -2
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +71 -8
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +446 -124
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +162 -30
- package/src/context/session.test.ts +2 -1
- package/src/context/session.ts +115 -7
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +81 -17
- package/src/i18n/zh-CN.ts +80 -17
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +178 -61
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -279
- package/src/report/compute.ts +723 -491
- package/src/report/dual-render.test.tsx +741 -1024
- package/src/report/flag.ts +104 -12
- package/src/report/format.ts +32 -12
- package/src/report/index.ts +119 -46
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +136 -65
- package/src/report/metrics.ts +108 -25
- package/src/report/primitives.tsx +196 -45
- package/src/report/react/AttemptList.tsx +30 -43
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +73 -0
- package/src/report/react/ExperimentList.tsx +50 -32
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +86 -34
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +89 -5
- package/src/report/react/fixtures.ts +114 -154
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +138 -158
- package/src/report/react/styles.css +243 -82
- package/src/report/report.test.ts +779 -841
- package/src/report/report.ts +423 -41
- package/src/report/text/faces.ts +290 -193
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +44 -7
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +261 -271
- package/src/report/web.ts +63 -20
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +13 -11
- package/src/results/attempt-evidence.ts +20 -13
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +150 -60
- package/src/results/host-equivalence.test.ts +34 -20
- package/src/results/index.ts +12 -4
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +15 -5
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +89 -54
- package/src/results/select.ts +104 -34
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +43 -14
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +243 -37
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +8 -6
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +127 -56
- package/src/show/render.ts +662 -131
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +320 -54
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +423 -11
- package/src/view/app/components/CodeView.tsx +41 -14
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +37 -17
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +10 -3
- package/src/view/data.ts +155 -49
- package/src/view/index.ts +56 -41
- package/src/view/server.ts +37 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +227 -0
- package/src/view/view-report.test.ts +167 -62
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
- package/dist/report/built-ins/experiment-comparison.js +0 -13
- package/dist/report/built-ins/index.d.ts +0 -1
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/docs-site/zh/guides/publish-report.mdx +0 -91
- package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
- package/src/report/built-in-user-parity.test.tsx +0 -640
- package/src/report/built-ins/experiment-comparison.tsx +0 -19
- package/src/report/built-ins/index.ts +0 -2
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
package/src/show/render.ts
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
// show 的文本渲染:单 eval 详情、--history 时间轴、三个证据切面(transcript / trace / diff)。
|
|
2
|
-
// 输出形态照 docs-site/zh/
|
|
2
|
+
// 输出形态照 docs-site/zh/how-to/viewing-results.mdx 的示例块;长内容一律截断,
|
|
3
3
|
// 但截断永远如实标注剩余数量和原始 artifact 路径 —— 输出对上下文窗口友好,事实源留在盘上。
|
|
4
4
|
// 全部纯函数(时间经 now 显式传入),证据数据由调用方 await 好了递进来。
|
|
5
5
|
|
|
6
6
|
import { join, relative } from "node:path";
|
|
7
|
-
import type { AssertionResult, DiffData, EvalResult, Verdict } from "../types.ts";
|
|
7
|
+
import type { AssertionResult, DiffData, EvalResult, TimingNode, TraceSpan, Verdict } from "../types.ts";
|
|
8
8
|
import type { AttemptEvidence, AttemptHandle, Snapshot } from "../results/index.ts";
|
|
9
|
-
import type { AnnotatedSourceLine } from "../results/index.ts";
|
|
9
|
+
import type { AnnotatedSourceLine, SendAnnotation } from "../results/index.ts";
|
|
10
10
|
import { groupIncompatibleVersionSkips } from "../results/index.ts";
|
|
11
11
|
import type { SkippedDir } from "../results/index.ts";
|
|
12
12
|
import type { ExecutionNode } from "../o11y/execution-tree.ts";
|
|
13
13
|
import { foldEvalVerdict } from "../shared/verdict.ts";
|
|
14
|
+
import { summaryText } from "../scoring/display.ts";
|
|
14
15
|
import { attemptCostUSD } from "../report/metrics.ts";
|
|
15
16
|
import { formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD } from "../report/format.ts";
|
|
16
17
|
import { indentBlock, padDisplay, renderAlignedRows, wrapDisplay } from "../report/text/layout.ts";
|
|
17
|
-
import type {
|
|
18
|
+
import type { AttemptHistoryRow } from "./compose.ts";
|
|
19
|
+
import { localizeText, showCommand, type HostCommandContext, type HostReport } from "./report-host.ts";
|
|
18
20
|
|
|
19
21
|
const MISSING = "—";
|
|
20
22
|
|
|
@@ -71,8 +73,8 @@ export function attemptArtifactsPath(attempt: AttemptHandle, cwd: string): strin
|
|
|
71
73
|
|
|
72
74
|
/**
|
|
73
75
|
* 裸 `show` 零可读结果时,skipped 目录的展示文案。niceeval 自己写的、schemaVersion 不兼容的
|
|
74
|
-
* 落盘按 producer 版本分组,一条 `npx niceeval@<version> show --
|
|
75
|
-
* 快照——show 没有 view 的单快照直读模式,`--
|
|
76
|
+
* 落盘按 producer 版本分组,一条 `npx niceeval@<version> show --results <结果根>` 覆盖同版本全部
|
|
77
|
+
* 快照——show 没有 view 的单快照直读模式,`--results` 认的是结果根(其下可以有多个 experiment),
|
|
76
78
|
* 不是单个快照目录,所以不对每个目录重复拼一条各自的命令(那条命令用同一个 root 跑起来
|
|
77
79
|
* 结果完全一样,重复只会刷屏)。其余(第三方 harness、版本信息缺失、malformed、incomplete)
|
|
78
80
|
* 没有可执行的统一建议,原样逐条列出。
|
|
@@ -85,7 +87,7 @@ export function skippedRunsText(skipped: readonly SkippedDir[], root: string, cw
|
|
|
85
87
|
const count = g.dirs.length;
|
|
86
88
|
const version = g.producer.version;
|
|
87
89
|
const schema = g.schemaVersion !== undefined ? ` (schemaVersion ${g.schemaVersion})` : "";
|
|
88
|
-
const cmd = `npx niceeval@${version} show --
|
|
90
|
+
const cmd = `npx niceeval@${version} show --results ${rootDisplay}`;
|
|
89
91
|
lines.push(
|
|
90
92
|
` ${count} snapshot${count === 1 ? "" : "s"} written by niceeval ${version}${schema} — run \`${cmd}\` to view`,
|
|
91
93
|
);
|
|
@@ -128,43 +130,97 @@ function scoreText(score: number): string {
|
|
|
128
130
|
return formatPlainNumber(Math.round(score * 100) / 100);
|
|
129
131
|
}
|
|
130
132
|
|
|
131
|
-
/**
|
|
133
|
+
/** 分组路径标题(嵌套用 " > " 拼接);无分组返回 undefined。 */
|
|
134
|
+
function groupTitle(a: AssertionResult): string | undefined {
|
|
135
|
+
return a.groupPath && a.groupPath.length > 0 ? a.groupPath.join(" > ") : undefined;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** 断言行:✓/✗/◌ + severity + 标题;unavailable 带 reason,soft 恒带 score/1(失败再补 detail)。 */
|
|
132
139
|
export function assertionLine(a: AssertionResult): string {
|
|
133
|
-
const
|
|
134
|
-
const
|
|
140
|
+
const group = groupTitle(a);
|
|
141
|
+
const scope = group ? `${group} · ` : "";
|
|
142
|
+
const optional = a.optional ? "optional · " : "";
|
|
143
|
+
if (a.outcome === "unavailable") {
|
|
144
|
+
return `◌ ${a.severity} · ${optional}${scope}${a.name} — unavailable: ${a.reason}`;
|
|
145
|
+
}
|
|
146
|
+
const head = `${a.outcome === "passed" ? "✓" : "✗"} ${a.severity} ${optional}${scope}${a.name}`;
|
|
135
147
|
if (a.severity === "soft") {
|
|
136
|
-
const detail =
|
|
137
|
-
|
|
148
|
+
const detail = a.outcome === "failed" && a.detail ? `: ${a.detail}` : "";
|
|
149
|
+
const threshold = a.threshold !== undefined ? ` / ${scoreText(a.threshold)}` : "/1";
|
|
150
|
+
return `${head} — ${scoreText(a.score)}${threshold}${detail}`;
|
|
138
151
|
}
|
|
139
|
-
if (
|
|
152
|
+
if (a.outcome === "failed") {
|
|
140
153
|
const reason = a.detail ?? `score ${scoreText(a.score)}`;
|
|
141
|
-
|
|
154
|
+
const received = a.received ?? a.evidence;
|
|
155
|
+
// 单行面只放摘要收口后的预览;多行值(如 output tail)的完整版在 attempt 首页展开。
|
|
156
|
+
return received !== undefined ? `${head} — ${reason} · received: ${summaryText(received)}` : `${head} — ${reason}`;
|
|
142
157
|
}
|
|
143
158
|
return head;
|
|
144
159
|
}
|
|
145
160
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
161
|
+
/** 字段行:首行 `<label>: <值首行>`,值的其余行(如 CommandResult 的 output tail)缩进原样展开。 */
|
|
162
|
+
function fieldLines(label: string, value: string): string[] {
|
|
163
|
+
const [first, ...rest] = value.split("\n");
|
|
164
|
+
return [` ${label}: ${first ?? ""}`, ...rest.map((line) => ` ${line}`)];
|
|
149
165
|
}
|
|
150
166
|
|
|
151
|
-
/**
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
167
|
+
/** 每条的通用行组(见 docs/feature/scoring/library/display.md「通用渲染规则」):
|
|
168
|
+
* 首行 `severity · <标题>`(有分组时标题是分组路径,随后 assertion: 行给检查方式),
|
|
169
|
+
* 之后按有则显示的顺序列 expected / received / score / reason / source。 */
|
|
170
|
+
function assertionRecordLines(a: AssertionResult): string[] {
|
|
171
|
+
const group = groupTitle(a);
|
|
172
|
+
const optional = a.optional ? "optional · " : "";
|
|
173
|
+
const title = group ?? a.name;
|
|
174
|
+
const lines: string[] = [];
|
|
175
|
+
const scoreSuffix =
|
|
176
|
+
a.outcome !== "unavailable" && (a.severity === "soft" || a.threshold !== undefined)
|
|
177
|
+
? ` ${scoreText(a.score)}${a.threshold !== undefined ? ` / ${scoreText(a.threshold)}` : ""}`
|
|
178
|
+
: "";
|
|
179
|
+
lines.push(` ${a.severity} · ${optional}${title}${scoreSuffix}`);
|
|
180
|
+
const detailLine = a.detail ?? (group ? a.name : undefined);
|
|
181
|
+
if (detailLine !== undefined && detailLine !== title) lines.push(` assertion: ${detailLine}`);
|
|
182
|
+
if (a.outcome === "unavailable") {
|
|
183
|
+
lines.push(` reason: ${a.reason}`);
|
|
184
|
+
} else {
|
|
185
|
+
if (a.expected !== undefined) lines.push(...fieldLines("expected", a.expected));
|
|
186
|
+
if (a.received !== undefined) lines.push(...fieldLines("received", a.received));
|
|
187
|
+
if (a.evidence !== undefined && a.evidence !== a.received) lines.push(...fieldLines("evidence", a.evidence));
|
|
166
188
|
}
|
|
189
|
+
if (a.loc) lines.push(` source: ${a.loc.file}:${a.loc.line}${a.loc.column ? `:${a.loc.column}` : ""}`);
|
|
190
|
+
return lines;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* 默认 Attempt 页的首要诊断:按结果分节,只逐条列出需要看的(见 display.md)——
|
|
195
|
+
* `failures:`(gate 失败,含 --strict 下改判的 soft)、`soft below threshold:`、
|
|
196
|
+
* `scores:`(无阈值 judge 的纯打分)、`unavailable:`(证据评不了的,带 reason)。
|
|
197
|
+
* 全部通过时这些节整体省略(返回 undefined),只留计数行。
|
|
198
|
+
*/
|
|
199
|
+
export function failureDiagnostics(assertions: AssertionResult[], width: number, strict?: boolean): string | undefined {
|
|
200
|
+
const failures = assertions.filter(
|
|
201
|
+
(a) => a.outcome === "failed" && (a.severity === "gate" || strict),
|
|
202
|
+
);
|
|
203
|
+
const softBelow = assertions.filter(
|
|
204
|
+
(a) => a.outcome === "failed" && a.severity === "soft" && !strict,
|
|
205
|
+
);
|
|
206
|
+
const scores = assertions.filter(
|
|
207
|
+
(a) => a.outcome === "passed" && a.severity === "soft" && a.threshold === undefined && a.score < 1,
|
|
208
|
+
);
|
|
209
|
+
const unavailableOnes = assertions.filter((a) => a.outcome === "unavailable");
|
|
210
|
+
const lines: string[] = [];
|
|
211
|
+
const section = (title: string, items: AssertionResult[]) => {
|
|
212
|
+
if (items.length === 0) return;
|
|
213
|
+
if (lines.length > 0) lines.push("");
|
|
214
|
+
lines.push(title);
|
|
215
|
+
for (const a of items) lines.push(...assertionRecordLines(a));
|
|
216
|
+
};
|
|
217
|
+
section("failures:", failures);
|
|
218
|
+
section("soft below threshold:", softBelow);
|
|
219
|
+
section("scores:", scores);
|
|
220
|
+
section("unavailable:", unavailableOnes);
|
|
221
|
+
if (lines.length === 0) return undefined;
|
|
167
222
|
return lines.flatMap((line) => {
|
|
223
|
+
if (line === "") return [line];
|
|
168
224
|
const indent = line.length - line.trimStart().length;
|
|
169
225
|
return wrapDisplay(line.trimStart(), Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`);
|
|
170
226
|
}).join("\n");
|
|
@@ -205,10 +261,13 @@ export function attemptEvidenceHeader(evidence: AttemptEvidence): string {
|
|
|
205
261
|
* 依赖它的内部实现细节。
|
|
206
262
|
*/
|
|
207
263
|
export function verdictReasonLine(result: EvalResult): string | undefined {
|
|
208
|
-
if (result.error !== undefined) return result.error;
|
|
264
|
+
if (result.error !== undefined) return result.error.message;
|
|
209
265
|
if (result.skipReason !== undefined) return result.skipReason;
|
|
210
|
-
const gates = result.assertions.filter((a) =>
|
|
211
|
-
if (gates.length === 0)
|
|
266
|
+
const gates = result.assertions.filter((a) => a.outcome === "failed" && a.severity === "gate");
|
|
267
|
+
if (gates.length === 0) {
|
|
268
|
+
const gap = result.assertions.find((a) => a.outcome === "unavailable" && !a.optional);
|
|
269
|
+
return gap && gap.outcome === "unavailable" ? `unavailable ${gap.name} (${gap.reason})` : undefined;
|
|
270
|
+
}
|
|
212
271
|
return gates.map((a) => `gate ${a.name}`).join(", ");
|
|
213
272
|
}
|
|
214
273
|
|
|
@@ -233,13 +292,15 @@ export function attemptIndexLine(opts: {
|
|
|
233
292
|
* (未捕获源码),读 `result.assertions` 让这条摘要不因缺源码而跟着消失。
|
|
234
293
|
*/
|
|
235
294
|
export function assertionSummaryLine(assertions: AssertionResult[]): string {
|
|
236
|
-
const passed = assertions.filter((a) => a.passed).length;
|
|
237
|
-
const gateFailed = assertions.filter((a) =>
|
|
238
|
-
const softBelow = assertions.filter((a) =>
|
|
295
|
+
const passed = assertions.filter((a) => a.outcome === "passed").length;
|
|
296
|
+
const gateFailed = assertions.filter((a) => a.outcome === "failed" && a.severity === "gate").length;
|
|
297
|
+
const softBelow = assertions.filter((a) => a.outcome === "failed" && a.severity === "soft").length;
|
|
298
|
+
const unavailableCount = assertions.filter((a) => a.outcome === "unavailable").length;
|
|
239
299
|
const parts: string[] = [];
|
|
240
300
|
if (passed > 0) parts.push(`${passed} passed`);
|
|
241
301
|
if (gateFailed > 0) parts.push(`${gateFailed} gate failed`);
|
|
242
302
|
if (softBelow > 0) parts.push(`${softBelow} soft below target`);
|
|
303
|
+
if (unavailableCount > 0) parts.push(`${unavailableCount} unavailable`);
|
|
243
304
|
return `assertions: ${parts.length > 0 ? parts.join(" · ") : "(none)"}`;
|
|
244
305
|
}
|
|
245
306
|
|
|
@@ -301,7 +362,7 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
301
362
|
lines.push(indentBlock(wrapDisplay(assertionLine(a), width - 2).join("\n"), " "));
|
|
302
363
|
}
|
|
303
364
|
if (detail.result.error !== undefined) {
|
|
304
|
-
lines.push(indentBlock(wrapDisplay(`error: ${detail.result.error}`, width - 2).join("\n"), " "));
|
|
365
|
+
lines.push(indentBlock(wrapDisplay(`error: ${detail.result.error.message}`, width - 2).join("\n"), " "));
|
|
305
366
|
}
|
|
306
367
|
if (detail.result.skipReason !== undefined) {
|
|
307
368
|
lines.push(indentBlock(wrapDisplay(`skipped: ${detail.result.skipReason}`, width - 2).join("\n"), " "));
|
|
@@ -314,8 +375,8 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
314
375
|
if (detail.locator) tail.push(`attempt locator: ${detail.locator}`);
|
|
315
376
|
tail.push(
|
|
316
377
|
detail.locator
|
|
317
|
-
? `next: niceeval show ${detail.locator} [--
|
|
318
|
-
: `next: niceeval show ${evalId} [--
|
|
378
|
+
? `next: niceeval show ${detail.locator} [--source|--execution|--diff]`
|
|
379
|
+
: `next: niceeval show ${evalId} [--source|--execution|--diff]`,
|
|
319
380
|
);
|
|
320
381
|
blocks.push(tail.join("\n"));
|
|
321
382
|
}
|
|
@@ -325,43 +386,58 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
325
386
|
|
|
326
387
|
// ───────────────────────── --history ─────────────────────────
|
|
327
388
|
|
|
328
|
-
/**
|
|
329
|
-
|
|
389
|
+
/**
|
|
390
|
+
* 一个 experimentId + evalId 的执行时间轴分节(docs/feature/reports/show.md「--history」):
|
|
391
|
+
* 节头 `<evalId> · <experimentId> · N attempts · passed x/N`,节内每行
|
|
392
|
+
* 时间 / verdict / 单行摘要 / 耗时 / 成本 / locator,startedAt 升序(compose 已排好)。
|
|
393
|
+
*/
|
|
394
|
+
export function attemptHistoryText(opts: {
|
|
330
395
|
experimentId: string;
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
rows: EvalHistoryRow[];
|
|
396
|
+
evalId: string;
|
|
397
|
+
rows: AttemptHistoryRow[];
|
|
334
398
|
}): string {
|
|
335
399
|
const { experimentId, evalId, rows } = opts;
|
|
336
400
|
const passed = rows.filter((r) => r.verdict === "passed").length;
|
|
337
401
|
const head = [
|
|
338
|
-
|
|
402
|
+
evalId,
|
|
339
403
|
experimentId,
|
|
340
|
-
|
|
404
|
+
attemptsLabel(rows.length),
|
|
341
405
|
`passed ${passed}/${rows.length}`,
|
|
342
406
|
].join(" · ");
|
|
343
407
|
if (rows.length === 0) return head;
|
|
344
408
|
const table = renderAlignedRows(
|
|
345
409
|
rows.map((r) => [
|
|
346
|
-
timelineStamp(r.startedAt),
|
|
410
|
+
r.startedAt !== undefined ? timelineStamp(r.startedAt) : MISSING,
|
|
347
411
|
`${verdictMark(r.verdict)} ${r.verdict}`,
|
|
348
|
-
|
|
412
|
+
r.summary ?? MISSING,
|
|
413
|
+
formatDurationMs(r.durationMs),
|
|
349
414
|
r.costUSD === null ? MISSING : formatUSD(r.costUSD),
|
|
350
|
-
r.
|
|
415
|
+
r.locator ?? MISSING,
|
|
351
416
|
]),
|
|
352
417
|
);
|
|
353
418
|
return `${head}\n\n${indentBlock(table, " ")}`;
|
|
354
419
|
}
|
|
355
420
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
421
|
+
// ───────────────────────── --report 页索引 ─────────────────────────
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* 多页报告的页索引(docs/feature/reports/show/reports.md Case 2):标题行 + 每页一行
|
|
425
|
+
* (id / 本 locale 页名 / 可复制的 `--page` 命令)。索引命令携带完整上下文
|
|
426
|
+
* (--results / --report / 位置参数),复制即可精确复现下一层视图。
|
|
427
|
+
*/
|
|
428
|
+
export function pageIndexText(opts: {
|
|
429
|
+
report: HostReport;
|
|
430
|
+
title: string;
|
|
431
|
+
command: HostCommandContext;
|
|
432
|
+
locale: string;
|
|
433
|
+
}): string {
|
|
434
|
+
const { report, title, command, locale } = opts;
|
|
435
|
+
const head = `${title} · ${locale === "zh-CN" ? `${report.pages.length} 页` : `${report.pages.length} pages`}`;
|
|
360
436
|
const table = renderAlignedRows(
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
437
|
+
report.pages.map((page) => [
|
|
438
|
+
page.id,
|
|
439
|
+
localizeText(page.title, locale) ?? page.id,
|
|
440
|
+
showCommand({ ...command, page: page.id }),
|
|
365
441
|
]),
|
|
366
442
|
);
|
|
367
443
|
return `${head}\n\n${indentBlock(table, " ")}`;
|
|
@@ -403,28 +479,51 @@ function indentedText(text: string, width: number, indent = 4, maxLines = 18): s
|
|
|
403
479
|
|
|
404
480
|
const MAX_SOURCE_LINES = 400;
|
|
405
481
|
|
|
406
|
-
/** gate 失败 / soft
|
|
482
|
+
/** gate 失败 / soft 恒带分 / unavailable 带 reason——与 assertionLine 的口径一致,但不带断言
|
|
483
|
+
* name(源码行本身就是名字)。 */
|
|
407
484
|
function evalAssertionDetailLine(a: AssertionResult): string | undefined {
|
|
485
|
+
if (a.outcome === "unavailable") {
|
|
486
|
+
return `${a.severity} · unavailable · ${a.reason}`;
|
|
487
|
+
}
|
|
408
488
|
if (a.severity === "soft") {
|
|
409
|
-
const detail =
|
|
410
|
-
|
|
489
|
+
const detail = a.outcome === "failed" && a.detail ? ` · ${a.detail}` : "";
|
|
490
|
+
const threshold = a.threshold !== undefined ? ` / ${scoreText(a.threshold)}` : "/1";
|
|
491
|
+
return `soft · ${scoreText(a.score)}${threshold}${detail}`;
|
|
411
492
|
}
|
|
412
|
-
if (
|
|
493
|
+
if (a.outcome === "failed") {
|
|
494
|
+
// 标注行是源码页里的一行事实,不是证据面:expected / received 过摘要收口
|
|
495
|
+
// (折单行 + 上限),完整值在 attempt 首页与 events.json / diff.json。
|
|
413
496
|
const parts = ["gate"];
|
|
414
|
-
|
|
497
|
+
const group = groupTitle(a);
|
|
498
|
+
if (group) parts.push(group);
|
|
415
499
|
parts.push(a.name);
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
if (
|
|
419
|
-
if (a.detail) parts.push(a.detail);
|
|
500
|
+
if (a.expected !== undefined) parts.push(`expected ${summaryText(a.expected)}`);
|
|
501
|
+
const received = a.received ?? a.evidence;
|
|
502
|
+
if (received !== undefined) parts.push(`received ${summaryText(received)}`);
|
|
503
|
+
if (a.detail) parts.push(summaryText(a.detail));
|
|
420
504
|
return parts.join(" · ");
|
|
421
505
|
}
|
|
422
506
|
return undefined;
|
|
423
507
|
}
|
|
424
508
|
|
|
509
|
+
/** send 行标注:轮身份 · status · 墙钟(有记录才出现),契约见 show.md「--eval」。 */
|
|
510
|
+
function sendAnnotationLine(send: SendAnnotation): string {
|
|
511
|
+
const parts = [send.label, send.status];
|
|
512
|
+
if (send.durationMs !== undefined) parts.push(formatDurationMs(send.durationMs));
|
|
513
|
+
return parts.join(" · ");
|
|
514
|
+
}
|
|
515
|
+
|
|
425
516
|
function evalSourceLineText(line: AnnotatedSourceLine, gutterWidth: number, width: number): string[] {
|
|
426
|
-
const anyFailed = line.assertions.some((a) =>
|
|
427
|
-
|
|
517
|
+
const anyFailed = line.assertions.some((a) => a.outcome === "failed") ||
|
|
518
|
+
line.sends.some((send) => send.status === "failed");
|
|
519
|
+
const anyUnavailable = line.assertions.some((a) => a.outcome === "unavailable");
|
|
520
|
+
const glyph = line.assertions.length === 0 && line.sends.length === 0
|
|
521
|
+
? " "
|
|
522
|
+
: anyFailed
|
|
523
|
+
? "✗"
|
|
524
|
+
: anyUnavailable
|
|
525
|
+
? "◌"
|
|
526
|
+
: "✓";
|
|
428
527
|
const marginWidth = gutterWidth + 2; // 行号列 + glyph + 分隔空格
|
|
429
528
|
const prefix = `${padDisplay(String(line.line), gutterWidth)}${glyph} `;
|
|
430
529
|
// 源码行的空白(尤其是缩进)是语义的一部分:wrapDisplay 按单词重排会把连续空格
|
|
@@ -433,6 +532,11 @@ function evalSourceLineText(line: AnnotatedSourceLine, gutterWidth: number, widt
|
|
|
433
532
|
// 折行只对续行加统一 margin,救不回已经被吃掉的原始缩进,不如老实截断。
|
|
434
533
|
const out = [prefix + clip(line.text, Math.max(20, width - marginWidth))];
|
|
435
534
|
const margin = " ".repeat(marginWidth);
|
|
535
|
+
for (const send of line.sends) {
|
|
536
|
+
for (const wrapped2 of wrapDisplay(sendAnnotationLine(send), Math.max(20, width - marginWidth))) {
|
|
537
|
+
out.push(margin + wrapped2);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
436
540
|
for (const a of line.assertions) {
|
|
437
541
|
const detail = evalAssertionDetailLine(a);
|
|
438
542
|
if (detail === undefined) continue;
|
|
@@ -473,8 +577,14 @@ export function evalSourceText(
|
|
|
473
577
|
);
|
|
474
578
|
}
|
|
475
579
|
|
|
476
|
-
|
|
477
|
-
|
|
580
|
+
const tail = [assertionSummaryLine(evidence.result.assertions)];
|
|
581
|
+
// 标注行的值是收口预览;有未通过断言时给「更进一步」——attempt 首页展开完整 expected /
|
|
582
|
+
// received(含 output tail),再往下是 result.json / events.json。
|
|
583
|
+
if (evidence.result.assertions.some((a) => a.outcome !== "passed")) {
|
|
584
|
+
tail.push(`full failure detail: niceeval show ${evidence.locator}`);
|
|
585
|
+
}
|
|
586
|
+
if (artifact) tail.push(`full eval source: ${artifact}`);
|
|
587
|
+
blocks.push(tail.join("\n"));
|
|
478
588
|
return blocks.join("\n\n");
|
|
479
589
|
}
|
|
480
590
|
|
|
@@ -509,7 +619,7 @@ function execBody(
|
|
|
509
619
|
/**
|
|
510
620
|
* action / subagent 节点渲染成两行(call + result)——节点模型把 call+result 合并成一个
|
|
511
621
|
* ExecutionNode(按 callId),但分两行读更符合「先看调用、再看结果」的阅读顺序,
|
|
512
|
-
* 与 docs-site/zh/
|
|
622
|
+
* 与 docs-site/zh/how-to/agent-feedback-loop.mdx 的示例一致。
|
|
513
623
|
*/
|
|
514
624
|
function executionNodeLines(node: ExecutionNode, originMs: number, timingAvailable: boolean, width: number): string[] {
|
|
515
625
|
const time = node.kind !== "telemetry" && node.span ? relSeconds(node.span.startMs, originMs) : undefined;
|
|
@@ -589,10 +699,30 @@ export function executionText(
|
|
|
589
699
|
const agentNodes = tree.nodes.filter((node) => node.kind !== "telemetry");
|
|
590
700
|
const telemetryCount = tree.nodes.length - agentNodes.length;
|
|
591
701
|
const shown = agentNodes.slice(0, MAX_EVENTS);
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
702
|
+
|
|
703
|
+
// 按轮分段(见 docs/feature/reports/show.md「--execution」):每轮以 TURN 头行开始——身份
|
|
704
|
+
// s<session>/t<turn>(与 --timing 的 turn 节点、diff 的 windows 同一套标签,来自
|
|
705
|
+
// result.json.phases 的 turn 时间树)、该轮墙钟;边界按用户消息切(t.send 恒以用户消息开轮)。
|
|
706
|
+
const turnNodes = (evidence.result.phases ?? [])
|
|
707
|
+
.flatMap((p) => p.children ?? [])
|
|
708
|
+
.filter((n) => n.kind === "turn");
|
|
709
|
+
const lines: string[] = [];
|
|
710
|
+
let turnIndex = -1;
|
|
711
|
+
shown.forEach((node, index) => {
|
|
712
|
+
const isTurnStart = node.kind === "message" && node.role === "user";
|
|
713
|
+
if (isTurnStart) {
|
|
714
|
+
turnIndex += 1;
|
|
715
|
+
const turn = turnNodes[turnIndex];
|
|
716
|
+
const label = turn?.label ?? `t${turnIndex + 1}`;
|
|
717
|
+
const durationPart = turn ? ` · ${formatDurationMs(turn.durationMs)}` : "";
|
|
718
|
+
const failedPart = turn?.failed ? " · failed" : "";
|
|
719
|
+
if (index > 0) lines.push("");
|
|
720
|
+
lines.push(`TURN ${label}${failedPart || " · completed"}${durationPart}`);
|
|
721
|
+
} else if (index > 0) {
|
|
722
|
+
lines.push("");
|
|
723
|
+
}
|
|
724
|
+
lines.push(...executionNodeLines(node, originMs, timingAvailable, width).map((l) => (turnIndex >= 0 ? ` ${l}` : l)));
|
|
725
|
+
});
|
|
596
726
|
|
|
597
727
|
const tail: string[] = [];
|
|
598
728
|
if (timingAvailable) {
|
|
@@ -627,14 +757,38 @@ export function executionText(
|
|
|
627
757
|
|
|
628
758
|
const MAX_OVERVIEW_DIFF_NAMES = 5;
|
|
629
759
|
|
|
760
|
+
/** net 效果的单字母标记(A/M/D;none = 动过但净无变化,标 ±)。 */
|
|
761
|
+
function netLetter(net: string): string {
|
|
762
|
+
switch (net) {
|
|
763
|
+
case "added":
|
|
764
|
+
return "A";
|
|
765
|
+
case "deleted":
|
|
766
|
+
return "D";
|
|
767
|
+
case "none":
|
|
768
|
+
return "±";
|
|
769
|
+
default:
|
|
770
|
+
return "M";
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
630
774
|
function overviewDiffLine(diff: DiffData): string {
|
|
631
|
-
const names =
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
];
|
|
775
|
+
const names = Object.entries(diff.files)
|
|
776
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
777
|
+
.map(([p, f]) => `${netLetter(f.net)} ${p}`);
|
|
635
778
|
const shown = names.slice(0, MAX_OVERVIEW_DIFF_NAMES);
|
|
636
779
|
const more = names.length > shown.length ? ` · +${names.length - shown.length} more` : "";
|
|
637
|
-
return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed · ${shown.join("
|
|
780
|
+
return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed by agent · ${shown.join(" · ")}${more}`;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/** 有界行 diff(公共前后缀修剪):对单区域编辑精确,复杂编辑给出上界近似。 */
|
|
784
|
+
function lineDelta(before: string | undefined, after: string | undefined): { adds: number; dels: number } {
|
|
785
|
+
const a = before === undefined ? [] : before.split("\n");
|
|
786
|
+
const b = after === undefined ? [] : after.split("\n");
|
|
787
|
+
let prefix = 0;
|
|
788
|
+
while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
|
|
789
|
+
let suffix = 0;
|
|
790
|
+
while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix]) suffix++;
|
|
791
|
+
return { adds: b.length - prefix - suffix, dels: a.length - prefix - suffix };
|
|
638
792
|
}
|
|
639
793
|
|
|
640
794
|
/**
|
|
@@ -642,6 +796,48 @@ function overviewDiffLine(diff: DiffData): string {
|
|
|
642
796
|
* 执行事件计数、可选 OTel 时间指示、工作区 diff 摘要,不复现 `--eval` 的完整源码、
|
|
643
797
|
* `--execution` 的完整事件流或 `--diff` 的完整文件列表(那些内容各自的证据 flag 才给)。
|
|
644
798
|
*/
|
|
799
|
+
/** lifecycle operation → 首页可读标签:点换空格,与 docs/feature/reports/show.md 的
|
|
800
|
+
* `phase: sandbox provision` 字面一致(不引入第二套本地化标签,show 首页用英文原样呈现)。 */
|
|
801
|
+
function operationWords(operation: string): string {
|
|
802
|
+
return operation.replace(/\./g, " ");
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* errored attempt 的结构化 `error:` 块(见 docs/feature/reports/show.md「errored attempt 的首页」)。
|
|
807
|
+
* 先展开 phase(= operation 的可读形态)/ code / message / cause,stack 放在块后、保持原始换行。
|
|
808
|
+
* 字段来自结构化 `AttemptError`;非 errored(`r.error === undefined`)返回 undefined。
|
|
809
|
+
*/
|
|
810
|
+
function renderErrorBlock(r: EvalResult): string | undefined {
|
|
811
|
+
const err = r.error;
|
|
812
|
+
if (err === undefined) return undefined;
|
|
813
|
+
const lines = ["error:", ` phase: ${err.phase}`, ` code: ${err.code}`, ` message: ${err.message}`];
|
|
814
|
+
if (err.cause) {
|
|
815
|
+
const c = err.cause;
|
|
816
|
+
const causeText = c.name ? `${c.name} · ${c.message}` : c.message;
|
|
817
|
+
lines.push(` cause: ${causeText}`);
|
|
818
|
+
}
|
|
819
|
+
// stack 放在 error 块之后、保持原始换行(见 docs);没有 stack(如 timeout / turn-failed)就不加空块。
|
|
820
|
+
if (err.stack && err.stack.trim() !== "") return `${lines.join("\n")}\n\n${err.stack.replace(/\n+$/, "")}`;
|
|
821
|
+
return lines.join("\n");
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/**
|
|
825
|
+
* attempt 级诊断块(见 docs/feature/reports/show.md「diagnostics」)。每条一行标头
|
|
826
|
+
* `<level> · <operation> · <code>`,message 缩进在下一行;`count > 1` 时补 `(N occurrences)`。
|
|
827
|
+
* 诊断的 level 与 verdict 无关 —— passed / failed / errored 都可能带一条 cleanup / teardown
|
|
828
|
+
* warning。没有诊断返回 undefined。
|
|
829
|
+
*/
|
|
830
|
+
function renderAttemptDiagnostics(r: EvalResult): string | undefined {
|
|
831
|
+
if (!r.diagnostics || r.diagnostics.length === 0) return undefined;
|
|
832
|
+
const lines = ["diagnostics:"];
|
|
833
|
+
for (const d of r.diagnostics) {
|
|
834
|
+
lines.push(` ${d.level} · ${d.phase} · ${d.code}`);
|
|
835
|
+
const occ = d.count && d.count > 1 ? ` (${d.count} occurrences)` : "";
|
|
836
|
+
lines.push(` ${d.message}${occ}`);
|
|
837
|
+
}
|
|
838
|
+
return lines.join("\n");
|
|
839
|
+
}
|
|
840
|
+
|
|
645
841
|
export function attemptOverviewText(
|
|
646
842
|
evidence: AttemptEvidence,
|
|
647
843
|
opts: { header: string; artifactPath?: string; width: number },
|
|
@@ -656,27 +852,43 @@ export function attemptOverviewText(
|
|
|
656
852
|
|
|
657
853
|
const blocks: string[] = [[header, metaParts.join(" · ")].join("\n")];
|
|
658
854
|
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
)
|
|
855
|
+
// errored attempt 的首页不靠 trace 就要能解释基础设施错误(见 docs/feature/reports/show.md
|
|
856
|
+
// 「errored attempt 的首页」):先展开结构化 error(phase/operation/code/message/cause + stack),
|
|
857
|
+
// 断言块对没有断言的 errored attempt 省略(它在评分之前就挂了)。
|
|
858
|
+
const errorBlock = renderErrorBlock(r);
|
|
859
|
+
if (errorBlock) blocks.push(errorBlock);
|
|
860
|
+
|
|
861
|
+
// errored 且没有任何断言时不打印空的 "assertions: 0 passed" —— 主因已经在 error 块里说清楚了。
|
|
862
|
+
if (r.assertions.length > 0 || r.error === undefined) {
|
|
863
|
+
blocks.push(
|
|
864
|
+
[
|
|
865
|
+
assertionSummaryLine(r.assertions),
|
|
866
|
+
evidence.evalSource
|
|
867
|
+
? `eval source: ${evidence.evalSource.sourcePath} · sha256:${evidence.evalSource.sourceSha256.slice(0, 8)}…`
|
|
868
|
+
: "eval source: unavailable (not captured for this attempt)",
|
|
869
|
+
].join("\n"),
|
|
870
|
+
);
|
|
871
|
+
}
|
|
667
872
|
|
|
668
873
|
const diagnostics = failureDiagnostics(r.assertions, opts.width);
|
|
669
874
|
if (diagnostics) blocks.push(diagnostics);
|
|
670
875
|
|
|
876
|
+
// attempt 级诊断(teardown/cleanup 等,与 verdict 独立;passed/failed/errored 都可能有)。
|
|
877
|
+
const attemptDiag = renderAttemptDiagnostics(r);
|
|
878
|
+
if (attemptDiag) blocks.push(attemptDiag);
|
|
879
|
+
|
|
671
880
|
if (evidence.execution) {
|
|
672
881
|
const nodes = evidence.execution.nodes.filter((node) => node.kind !== "telemetry");
|
|
673
882
|
const skillLoads = nodes.filter((n) => n.kind === "skill.loaded").length;
|
|
674
883
|
const toolCalls = nodes.filter((n) => n.kind === "action").length;
|
|
675
884
|
const aiMessages = nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
|
|
676
|
-
const
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
885
|
+
const counted = (count: number, singular: string, plural = `${singular}s`) =>
|
|
886
|
+
`${count} ${count === 1 ? singular : plural}`;
|
|
887
|
+
const execLines = [
|
|
888
|
+
`execution: ${counted(nodes.length, "event")} · ${counted(skillLoads, "skill load")} · ${counted(toolCalls, "tool call")} · ${counted(aiMessages, "AI message")}`,
|
|
889
|
+
];
|
|
890
|
+
const timingLine = overviewTimingLine(r);
|
|
891
|
+
if (timingLine) execLines.push(timingLine);
|
|
680
892
|
blocks.push(execLines.join("\n"));
|
|
681
893
|
} else {
|
|
682
894
|
blocks.push("execution: unavailable (no events recorded for this attempt)");
|
|
@@ -693,8 +905,9 @@ export function attemptOverviewText(
|
|
|
693
905
|
const tail: string[] = [];
|
|
694
906
|
if (artifactPath) tail.push(`artifacts: ${artifactPath}/`);
|
|
695
907
|
const available = [
|
|
696
|
-
evidence.capabilities.
|
|
908
|
+
evidence.capabilities.source ? `niceeval show ${evidence.locator} --source` : undefined,
|
|
697
909
|
evidence.capabilities.execution ? `niceeval show ${evidence.locator} --execution` : undefined,
|
|
910
|
+
evidence.capabilities.timing ? `niceeval show ${evidence.locator} --timing` : undefined,
|
|
698
911
|
evidence.capabilities.diff ? `niceeval show ${evidence.locator} --diff` : undefined,
|
|
699
912
|
].filter((command): command is string => command !== undefined);
|
|
700
913
|
if (available.length > 0) tail.push(`available:\n${available.map((command) => ` ${command}`).join("\n")}`);
|
|
@@ -715,45 +928,363 @@ export function diffText(opts: {
|
|
|
715
928
|
file?: string;
|
|
716
929
|
}): string {
|
|
717
930
|
const { header, diff, artifactPath, file } = opts;
|
|
718
|
-
const source = artifactPath ? join(
|
|
931
|
+
const source = artifactPath ? join(artifactPath, "diff.json") : undefined;
|
|
719
932
|
if (!diff) {
|
|
720
|
-
return `${header}\n\
|
|
933
|
+
return `${header}\n\ndiff unavailable (no diff recorded for this attempt: remote agent, or diff artifact not published${source ? `; expected: ${source}` : ""})`;
|
|
721
934
|
}
|
|
722
935
|
|
|
723
936
|
if (file !== undefined) {
|
|
724
|
-
const
|
|
725
|
-
if (
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
return `${header}\n\nFile "${file}" is not in this attempt's diff. Files: ${known.join(", ") || "(none)"}`;
|
|
937
|
+
const summary = diff.files[file];
|
|
938
|
+
if (summary === undefined) {
|
|
939
|
+
const known = Object.keys(diff.files).sort();
|
|
940
|
+
return `${header}\n\nFile "${file}" is not in this attempt's agent diff. Files: ${known.join(", ") || "(none)"}`;
|
|
729
941
|
}
|
|
730
|
-
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
942
|
+
// 单文件 patch 按窗口逐段渲染(diff.json 存的就是逐窗口 delta,不产出跨窗口合成 patch)。
|
|
943
|
+
const head = `${netLetter(summary.net)} ${file} · touched in ${summary.windows.join(", ")}`;
|
|
944
|
+
if (summary.binary) {
|
|
945
|
+
const sections = diff.windows
|
|
946
|
+
.filter((w) => w.changes[file] !== undefined)
|
|
947
|
+
.map((w) => {
|
|
948
|
+
const c = w.changes[file]!;
|
|
949
|
+
const b = c.binary ?? {};
|
|
950
|
+
return `── window ${w.window}\nbinary · ${b.beforeBytes ?? 0} → ${b.afterBytes ?? 0} bytes`;
|
|
951
|
+
});
|
|
952
|
+
return `${header}\n\n${head}\n\n${sections.join("\n\n")}`;
|
|
953
|
+
}
|
|
954
|
+
const sections: string[] = [];
|
|
955
|
+
for (const w of diff.windows) {
|
|
956
|
+
const c = w.changes[file];
|
|
957
|
+
if (c === undefined) continue;
|
|
958
|
+
sections.push(`── window ${w.window}\n${windowHunk(c)}`);
|
|
959
|
+
}
|
|
960
|
+
return `${header}\n\n${head}\n\n${sections.join("\n\n")}${source ? `\n\n(full diff: ${source})` : ""}`;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
const entries = Object.entries(diff.files).sort(([a], [b]) => a.localeCompare(b));
|
|
964
|
+
if (entries.length === 0) {
|
|
965
|
+
return `${header}\n\n(no file changes by the agent in any send window${source ? ` · full diff: ${source}` : ""})`;
|
|
966
|
+
}
|
|
967
|
+
const rows = entries.map(([path, summary]) => {
|
|
968
|
+
if (summary.binary) {
|
|
969
|
+
return [netLetter(summary.net), path, "binary", summary.windows.join(", ")];
|
|
970
|
+
}
|
|
971
|
+
let adds = 0;
|
|
972
|
+
let dels = 0;
|
|
973
|
+
for (const w of diff.windows) {
|
|
974
|
+
const c = w.changes[path];
|
|
975
|
+
if (!c) continue;
|
|
976
|
+
const d = lineDelta(c.before, c.after);
|
|
977
|
+
adds += Math.max(0, d.adds);
|
|
978
|
+
dels += Math.max(0, d.dels);
|
|
979
|
+
}
|
|
980
|
+
const delta = [adds > 0 ? `+${adds}` : "", dels > 0 ? `-${dels}` : ""].filter(Boolean).join(" ") || "±0";
|
|
981
|
+
return [netLetter(summary.net), path, delta, summary.windows.join(", ")];
|
|
982
|
+
});
|
|
983
|
+
const headLine = `${entries.length} ${entries.length === 1 ? "file" : "files"} changed by agent`;
|
|
984
|
+
const single = entries[0] ? `\n\nsingle file: niceeval show @… --diff=${entries[0][0]}` : "";
|
|
985
|
+
return `${header}\n\n${headLine}\n${renderAlignedRows(rows).split("\n").map((l) => ` ${l}`).join("\n")}${single}`;
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/** 一个窗口内单文件的最小 unified hunk:公共前后缀修剪出的编辑区,一段 @@ 展示。 */
|
|
989
|
+
function windowHunk(c: { status: string; before?: string; after?: string }): string {
|
|
990
|
+
const a = c.before === undefined ? [] : c.before.replace(/\n$/, "").split("\n");
|
|
991
|
+
const b = c.after === undefined ? [] : c.after.replace(/\n$/, "").split("\n");
|
|
992
|
+
let prefix = 0;
|
|
993
|
+
while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
|
|
994
|
+
let suffix = 0;
|
|
995
|
+
while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix]) suffix++;
|
|
996
|
+
const removed = a.slice(prefix, a.length - suffix);
|
|
997
|
+
const added = b.slice(prefix, b.length - suffix);
|
|
998
|
+
const ctxBefore = a.slice(Math.max(0, prefix - 2), prefix);
|
|
999
|
+
const lines: string[] = [];
|
|
1000
|
+
lines.push(`@@ -${Math.max(1, prefix - ctxBefore.length + 1)},${removed.length + ctxBefore.length} +${Math.max(1, prefix - ctxBefore.length + 1)},${added.length + ctxBefore.length} @@`);
|
|
1001
|
+
for (const l of ctxBefore) lines.push(` ${l}`);
|
|
1002
|
+
const MAX_HUNK_LINES = 200;
|
|
1003
|
+
const shownRemoved = removed.slice(0, MAX_HUNK_LINES);
|
|
1004
|
+
const shownAdded = added.slice(0, MAX_HUNK_LINES);
|
|
1005
|
+
for (const l of shownRemoved) lines.push(`-${l}`);
|
|
1006
|
+
if (removed.length > shownRemoved.length) lines.push(`… (${removed.length - shownRemoved.length} more removed lines)`);
|
|
1007
|
+
for (const l of shownAdded) lines.push(`+${l}`);
|
|
1008
|
+
if (added.length > shownAdded.length) lines.push(`… (${added.length - shownAdded.length} more added lines)`);
|
|
1009
|
+
return lines.join("\n");
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
// ───────────────────────── 证据切面:--timing(统一时间树) ─────────────────────────
|
|
1013
|
+
|
|
1014
|
+
const CLOSING_PHASE_NAMES = new Set(["eval.teardown", "agent.teardown", "sandbox.teardown", "sandbox.suspend", "sandbox.stop"]);
|
|
1015
|
+
|
|
1016
|
+
// 首页只保留能回答「大头在哪」的阶段。分类账锚点与 telemetry bookkeeping 在很短时
|
|
1017
|
+
// 留给 --timing 完整树;一旦自身变成慢点或失败,仍必须抬到首页,不能被静默藏掉。
|
|
1018
|
+
const OVERVIEW_DETAIL_PHASE_NAMES = new Set(["workspace.baseline", "telemetry.configure", "telemetry.collect"]);
|
|
1019
|
+
const OVERVIEW_DETAIL_THRESHOLD_MS = 1_000;
|
|
1020
|
+
|
|
1021
|
+
/** phases 主链摘要行(首页 `timing:`):各主链阶段耗时点隔,收尾段合计以 `teardown +N` 单列。 */
|
|
1022
|
+
export function overviewTimingLine(r: EvalResult): string | undefined {
|
|
1023
|
+
if (!r.phases || r.phases.length === 0) return undefined;
|
|
1024
|
+
const main = r.phases.filter(
|
|
1025
|
+
(p) =>
|
|
1026
|
+
!CLOSING_PHASE_NAMES.has(p.name) &&
|
|
1027
|
+
(p.failed === true || !OVERVIEW_DETAIL_PHASE_NAMES.has(p.name) || p.durationMs >= OVERVIEW_DETAIL_THRESHOLD_MS),
|
|
1028
|
+
);
|
|
1029
|
+
const closing = r.phases.filter((p) => CLOSING_PHASE_NAMES.has(p.name));
|
|
1030
|
+
const parts = main.map((p) => `${p.name} ${formatDurationMs(p.durationMs)}${p.failed ? " ✗" : ""}`);
|
|
1031
|
+
const closingMs = closing.reduce((sum, p) => sum + p.durationMs, 0);
|
|
1032
|
+
if (closingMs > 0) parts.push(`teardown +${formatDurationMs(closingMs)}`);
|
|
1033
|
+
return `timing: ${parts.join(" · ")}`;
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
const TIMING_DETAIL_NODE_BUDGET = 80;
|
|
1037
|
+
|
|
1038
|
+
interface DiagnosticTimingNode {
|
|
1039
|
+
id: string;
|
|
1040
|
+
label: string;
|
|
1041
|
+
startOffsetMs: number;
|
|
1042
|
+
durationMs: number;
|
|
1043
|
+
failed: boolean;
|
|
1044
|
+
otel: boolean;
|
|
1045
|
+
children: DiagnosticTimingNode[];
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
function timingNodeLabel(node: TimingNode): string {
|
|
1049
|
+
if (node.kind === "command" && node.command) return `shell · ${node.command.display}`;
|
|
1050
|
+
if (node.kind === "turn") return `turn ${node.label}`;
|
|
1051
|
+
if (node.kind === "operation" || node.kind === "provider") return `${node.kind} · ${node.label}`;
|
|
1052
|
+
return node.label;
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
function traceReferenceCounts(nodes: readonly TimingNode[], counts = new Map<string, number>()): Map<string, number> {
|
|
1056
|
+
for (const node of nodes) {
|
|
1057
|
+
if (node.kind === "turn" && node.traceId) counts.set(node.traceId, (counts.get(node.traceId) ?? 0) + 1);
|
|
1058
|
+
traceReferenceCounts(node.children ?? [], counts);
|
|
1059
|
+
}
|
|
1060
|
+
return counts;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/** OTel 只按唯一 traceId 归属和 span parent 关系挂接;不按绝对墙钟猜跨进程顺序。 */
|
|
1064
|
+
function otelForest(traceId: string, spans: readonly TraceSpan[], turnStartOffsetMs: number): DiagnosticTimingNode[] {
|
|
1065
|
+
const unique = new Map<string, TraceSpan>();
|
|
1066
|
+
for (const span of spans) if (span.traceId === traceId && !unique.has(span.spanId)) unique.set(span.spanId, span);
|
|
1067
|
+
const selected = [...unique.values()];
|
|
1068
|
+
if (selected.length === 0) return [];
|
|
1069
|
+
const traceOrigin = Math.min(...selected.map((span) => span.startMs));
|
|
1070
|
+
const nodes = new Map<string, DiagnosticTimingNode>();
|
|
1071
|
+
for (const span of selected) {
|
|
1072
|
+
nodes.set(span.spanId, {
|
|
1073
|
+
id: `otel:${span.traceId}:${span.spanId}`,
|
|
1074
|
+
label: `${span.kind ?? "span"} · ${span.name}`,
|
|
1075
|
+
startOffsetMs: turnStartOffsetMs + Math.max(0, span.startMs - traceOrigin),
|
|
1076
|
+
durationMs: Math.max(0, span.endMs - span.startMs),
|
|
1077
|
+
failed: span.status === "error",
|
|
1078
|
+
otel: true,
|
|
1079
|
+
children: [],
|
|
1080
|
+
});
|
|
1081
|
+
}
|
|
1082
|
+
const roots: DiagnosticTimingNode[] = [];
|
|
1083
|
+
for (const span of selected) {
|
|
1084
|
+
const node = nodes.get(span.spanId)!;
|
|
1085
|
+
const parent = span.parentSpanId && span.parentSpanId !== span.spanId ? nodes.get(span.parentSpanId) : undefined;
|
|
1086
|
+
if (parent) parent.children.push(node);
|
|
1087
|
+
else roots.push(node);
|
|
1088
|
+
}
|
|
1089
|
+
const sort = (items: DiagnosticTimingNode[]) => {
|
|
1090
|
+
items.sort((a, b) => a.startOffsetMs - b.startOffsetMs || a.id.localeCompare(b.id));
|
|
1091
|
+
for (const item of items) sort(item.children);
|
|
1092
|
+
};
|
|
1093
|
+
sort(roots);
|
|
1094
|
+
return roots;
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
function diagnosticTimingNode(
|
|
1098
|
+
node: TimingNode,
|
|
1099
|
+
spans: readonly TraceSpan[],
|
|
1100
|
+
uniqueTraceIds: ReadonlySet<string>,
|
|
1101
|
+
): DiagnosticTimingNode {
|
|
1102
|
+
const children = (node.children ?? []).map((child) => diagnosticTimingNode(child, spans, uniqueTraceIds));
|
|
1103
|
+
if (node.kind === "turn" && node.traceId && uniqueTraceIds.has(node.traceId)) {
|
|
1104
|
+
children.push(...otelForest(node.traceId, spans, node.startOffsetMs));
|
|
1105
|
+
}
|
|
1106
|
+
return {
|
|
1107
|
+
id: `runner:${node.id}`,
|
|
1108
|
+
label: timingNodeLabel(node),
|
|
1109
|
+
startOffsetMs: node.startOffsetMs,
|
|
1110
|
+
durationMs: node.durationMs,
|
|
1111
|
+
failed: node.failed === true,
|
|
1112
|
+
otel: false,
|
|
1113
|
+
children,
|
|
1114
|
+
};
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
interface FlatTimingNode {
|
|
1118
|
+
node: DiagnosticTimingNode;
|
|
1119
|
+
path: readonly DiagnosticTimingNode[];
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
function flattenTimingForest(roots: readonly DiagnosticTimingNode[]): FlatTimingNode[] {
|
|
1123
|
+
const flat: FlatTimingNode[] = [];
|
|
1124
|
+
const visit = (node: DiagnosticTimingNode, ancestors: readonly DiagnosticTimingNode[]) => {
|
|
1125
|
+
const path = [...ancestors, node];
|
|
1126
|
+
flat.push({ node, path });
|
|
1127
|
+
for (const child of node.children) visit(child, path);
|
|
1128
|
+
};
|
|
1129
|
+
for (const root of roots) visit(root, []);
|
|
1130
|
+
return flat;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
/**
|
|
1134
|
+
* 默认 80-node 投影。四个稳定池先各用自己的配额,再把空余额按失败→最慢→首尾重分配。
|
|
1135
|
+
* 选中深层节点必须连同祖先整条加入;四池合并后按 id 去重。
|
|
1136
|
+
*/
|
|
1137
|
+
function selectTimingNodes(roots: readonly DiagnosticTimingNode[]): ReadonlySet<string> {
|
|
1138
|
+
const flat = flattenTimingForest(roots);
|
|
1139
|
+
if (flat.length <= TIMING_DETAIL_NODE_BUDGET) return new Set(flat.map(({ node }) => node.id));
|
|
1140
|
+
|
|
1141
|
+
const selected = new Set<string>();
|
|
1142
|
+
const byStart = [...flat].sort(
|
|
1143
|
+
(a, b) => a.node.startOffsetMs - b.node.startOffsetMs || a.node.id.localeCompare(b.node.id),
|
|
1144
|
+
);
|
|
1145
|
+
const failed = byStart.filter(({ node }) => node.failed);
|
|
1146
|
+
const slow = [...flat].sort(
|
|
1147
|
+
(a, b) => b.node.durationMs - a.node.durationMs || a.node.startOffsetMs - b.node.startOffsetMs || a.node.id.localeCompare(b.node.id),
|
|
1148
|
+
);
|
|
1149
|
+
const latest = [...flat].sort(
|
|
1150
|
+
(a, b) => b.node.startOffsetMs - a.node.startOffsetMs || a.node.id.localeCompare(b.node.id),
|
|
1151
|
+
);
|
|
1152
|
+
|
|
1153
|
+
const add = (candidate: FlatTimingNode, allowance: number): number => {
|
|
1154
|
+
const missing = candidate.path.filter((node) => !selected.has(node.id));
|
|
1155
|
+
if (missing.length === 0 || missing.length > allowance || selected.size + missing.length > TIMING_DETAIL_NODE_BUDGET) {
|
|
1156
|
+
return 0;
|
|
1157
|
+
}
|
|
1158
|
+
for (const node of missing) selected.add(node.id);
|
|
1159
|
+
return missing.length;
|
|
1160
|
+
};
|
|
1161
|
+
const pool = (candidates: readonly FlatTimingNode[], cap: number) => {
|
|
1162
|
+
let remaining = cap;
|
|
1163
|
+
for (const candidate of candidates) {
|
|
1164
|
+
if (remaining === 0) break;
|
|
1165
|
+
remaining -= add(candidate, remaining);
|
|
1166
|
+
}
|
|
1167
|
+
};
|
|
1168
|
+
|
|
1169
|
+
pool(failed, 40);
|
|
1170
|
+
pool(slow, 20);
|
|
1171
|
+
pool(byStart, 10);
|
|
1172
|
+
pool(latest, 10);
|
|
1173
|
+
|
|
1174
|
+
// 固定池未用满时,剩余额度按契约优先级继续分配;候选稳定、无法容纳的整条路径跳过。
|
|
1175
|
+
for (const candidate of [...failed, ...slow, ...byStart, ...latest]) {
|
|
1176
|
+
const remaining = TIMING_DETAIL_NODE_BUDGET - selected.size;
|
|
1177
|
+
if (remaining === 0) break;
|
|
1178
|
+
add(candidate, remaining);
|
|
1179
|
+
}
|
|
1180
|
+
return selected;
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
function subtreeStats(node: DiagnosticTimingNode): { nodes: number; failed: number } {
|
|
1184
|
+
let nodes = 1;
|
|
1185
|
+
let failed = node.failed ? 1 : 0;
|
|
1186
|
+
for (const child of node.children) {
|
|
1187
|
+
const stats = subtreeStats(child);
|
|
1188
|
+
nodes += stats.nodes;
|
|
1189
|
+
failed += stats.failed;
|
|
1190
|
+
}
|
|
1191
|
+
return { nodes, failed };
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
type VisibleTimingEntry =
|
|
1195
|
+
| { kind: "node"; node: DiagnosticTimingNode }
|
|
1196
|
+
| { kind: "omitted"; nodes: number; failed: number };
|
|
1197
|
+
|
|
1198
|
+
function visibleTimingEntries(
|
|
1199
|
+
nodes: readonly DiagnosticTimingNode[],
|
|
1200
|
+
selected: ReadonlySet<string> | undefined,
|
|
1201
|
+
): VisibleTimingEntry[] {
|
|
1202
|
+
if (!selected) return nodes.map((node) => ({ kind: "node", node }));
|
|
1203
|
+
const entries: VisibleTimingEntry[] = [];
|
|
1204
|
+
for (let i = 0; i < nodes.length;) {
|
|
1205
|
+
const node = nodes[i]!;
|
|
1206
|
+
if (selected.has(node.id)) {
|
|
1207
|
+
entries.push({ kind: "node", node });
|
|
1208
|
+
i += 1;
|
|
1209
|
+
continue;
|
|
1210
|
+
}
|
|
1211
|
+
let omittedNodes = 0;
|
|
1212
|
+
let omittedFailed = 0;
|
|
1213
|
+
while (i < nodes.length && !selected.has(nodes[i]!.id)) {
|
|
1214
|
+
const stats = subtreeStats(nodes[i]!);
|
|
1215
|
+
omittedNodes += stats.nodes;
|
|
1216
|
+
omittedFailed += stats.failed;
|
|
1217
|
+
i += 1;
|
|
1218
|
+
}
|
|
1219
|
+
entries.push({ kind: "omitted", nodes: omittedNodes, failed: omittedFailed });
|
|
1220
|
+
}
|
|
1221
|
+
return entries;
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
function diagnosticTimingLines(
|
|
1225
|
+
nodes: readonly DiagnosticTimingNode[],
|
|
1226
|
+
prefix: string,
|
|
1227
|
+
selected: ReadonlySet<string> | undefined,
|
|
1228
|
+
locator: string,
|
|
1229
|
+
): string[] {
|
|
1230
|
+
const entries = visibleTimingEntries(nodes, selected);
|
|
1231
|
+
const lines: string[] = [];
|
|
1232
|
+
entries.forEach((entry, index) => {
|
|
1233
|
+
const last = index === entries.length - 1;
|
|
1234
|
+
const branch = `${prefix}${last ? "└─ " : "├─ "}`;
|
|
1235
|
+
const childPrefix = `${prefix}${last ? " " : "│ "}`;
|
|
1236
|
+
if (entry.kind === "omitted") {
|
|
1237
|
+
const failed = entry.failed > 0 ? ` · ${entry.failed} failed` : "";
|
|
1238
|
+
lines.push(`${branch}… ${entry.nodes} nodes omitted${failed} · full: niceeval show ${locator} --timing=full`);
|
|
1239
|
+
return;
|
|
1240
|
+
}
|
|
1241
|
+
const node = entry.node;
|
|
1242
|
+
lines.push(
|
|
1243
|
+
`${branch}${node.label} ${formatDurationMs(node.durationMs)}${node.failed ? " ✗" : ""}${node.otel ? " OTel" : ""}`,
|
|
1244
|
+
);
|
|
1245
|
+
lines.push(...diagnosticTimingLines(node.children, childPrefix, selected, locator));
|
|
1246
|
+
});
|
|
1247
|
+
return lines;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
/**
|
|
1251
|
+
* `--timing`:整个 attempt 的统一时间树(见 docs/feature/reports/show.md)。先按
|
|
1252
|
+
* `result.json.phases` 输出 runner 生命周期,再展开 hook / 命令 / turn;turn 带 traceId 时
|
|
1253
|
+
* 从 trace.json 挂接 agent/model/tool spans。缩进表达包含关系,子项不能求和后与父项比较。
|
|
1254
|
+
*/
|
|
1255
|
+
export function timingText(
|
|
1256
|
+
evidence: AttemptEvidence,
|
|
1257
|
+
opts: { header: string; artifactPath?: string; width: number; mode?: "summary" | "full" },
|
|
1258
|
+
): string {
|
|
1259
|
+
const r = evidence.result;
|
|
1260
|
+
if (!r.phases || r.phases.length === 0) {
|
|
1261
|
+
return `${opts.header}\n\nphase timing unavailable (this result was not produced by a runner with phase timing)`;
|
|
1262
|
+
}
|
|
1263
|
+
const spans = evidence.trace;
|
|
1264
|
+
const main = r.phases.filter((p) => !CLOSING_PHASE_NAMES.has(p.name));
|
|
1265
|
+
const closing = r.phases.filter((p) => CLOSING_PHASE_NAMES.has(p.name));
|
|
1266
|
+
const traceCounts = new Map<string, number>();
|
|
1267
|
+
for (const phase of r.phases) traceReferenceCounts(phase.children ?? [], traceCounts);
|
|
1268
|
+
const uniqueTraceIds = new Set([...traceCounts].filter(([, count]) => count === 1).map(([traceId]) => traceId));
|
|
1269
|
+
const phaseForests = new Map(
|
|
1270
|
+
r.phases.map((phase) => [
|
|
1271
|
+
phase,
|
|
1272
|
+
(phase.children ?? []).map((node) => diagnosticTimingNode(node, spans ?? [], uniqueTraceIds)),
|
|
1273
|
+
]),
|
|
1274
|
+
);
|
|
1275
|
+
const allRoots = [...phaseForests.values()].flat();
|
|
1276
|
+
const selected = opts.mode === "full" ? undefined : selectTimingNodes(allRoots);
|
|
1277
|
+
|
|
1278
|
+
const lines: string[] = [`total ${formatDurationMs(r.durationMs)}`, ""];
|
|
1279
|
+
const renderPhase = (p: NonNullable<EvalResult["phases"]>[number]) => {
|
|
1280
|
+
const failedNote = p.failed ? ` ✗ failed here${r.error ? ` (${r.error.code})` : ""}` : "";
|
|
1281
|
+
lines.push(`${p.name.padEnd(22)}${formatDurationMs(p.durationMs)}${failedNote}`);
|
|
1282
|
+
lines.push(...diagnosticTimingLines(phaseForests.get(p) ?? [], " ", selected, evidence.locator));
|
|
1283
|
+
};
|
|
1284
|
+
for (const p of main) renderPhase(p);
|
|
1285
|
+
if (closing.length > 0) {
|
|
1286
|
+
lines.push("", "teardown (not counted in total):");
|
|
1287
|
+
for (const p of closing) renderPhase(p);
|
|
1288
|
+
}
|
|
1289
|
+
return `${opts.header}\n\n${lines.join("\n")}`;
|
|
759
1290
|
}
|