niceeval 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +72 -6
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +55 -1
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +5 -3
- package/dist/report/aggregate.js +32 -5
- package/dist/report/built-ins/experiment-comparison.d.ts +39 -0
- package/dist/report/built-ins/experiment-comparison.js +119 -0
- package/dist/report/built-ins/index.d.ts +2 -1
- package/dist/report/built-ins/index.js +2 -2
- package/dist/report/components.d.ts +10 -2
- package/dist/report/components.js +3 -3
- package/dist/report/compute.d.ts +11 -18
- package/dist/report/compute.js +68 -66
- package/dist/report/flag.d.ts +16 -1
- package/dist/report/flag.js +19 -1
- package/dist/report/format.d.ts +16 -14
- package/dist/report/format.js +28 -30
- package/dist/report/index.d.ts +5 -4
- package/dist/report/index.js +6 -5
- package/dist/report/locale.d.ts +23 -3
- package/dist/report/locale.js +47 -6
- package/dist/report/metrics.d.ts +13 -1
- package/dist/report/metrics.js +66 -15
- package/dist/report/primitives.d.ts +6 -0
- package/dist/report/react/AttemptList.d.ts +4 -4
- package/dist/report/react/AttemptList.js +8 -10
- package/dist/report/react/EvalList.d.ts +1 -1
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +8 -0
- package/dist/report/react/ExperimentComparison.js +11 -0
- package/dist/report/react/ExperimentList.d.ts +4 -2
- package/dist/report/react/ExperimentList.js +57 -7
- package/dist/report/react/MetricScatter.js +6 -16
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +3 -3
- package/dist/report/react/fixtures.js +30 -18
- package/dist/report/react/format.d.ts +1 -1
- package/dist/report/react/format.js +1 -1
- package/dist/report/react/index.d.ts +1 -1
- package/dist/report/report.d.ts +5 -1
- package/dist/report/report.js +6 -2
- package/dist/report/text/faces.d.ts +1 -1
- package/dist/report/text/faces.js +100 -61
- package/dist/report/text/table.js +36 -5
- package/dist/report/types.d.ts +40 -34
- package/dist/results/types.d.ts +11 -0
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +1 -0
- package/dist/shared/aggregate.js +3 -3
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/concepts/adapter.mdx +24 -6
- package/docs-site/zh/concepts/assert.mdx +11 -10
- package/docs-site/zh/concepts/evals.mdx +7 -6
- package/docs-site/zh/concepts/experiment.mdx +1 -1
- package/docs-site/zh/concepts/overview.mdx +7 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +35 -31
- package/docs-site/zh/guides/authoring.mdx +33 -0
- package/docs-site/zh/guides/ci-integration.mdx +23 -12
- package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
- package/docs-site/zh/guides/custom-reports.mdx +29 -34
- package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
- package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
- package/docs-site/zh/guides/debugging.mdx +210 -0
- package/docs-site/zh/guides/experiments.mdx +10 -3
- package/docs-site/zh/guides/fixtures.mdx +3 -1
- package/docs-site/zh/guides/official-adapters.mdx +27 -3
- package/docs-site/zh/guides/publish-report.mdx +30 -16
- package/docs-site/zh/guides/report-components.mdx +49 -37
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +42 -8
- package/docs-site/zh/guides/runner.mdx +17 -7
- package/docs-site/zh/guides/sandbox-agent.mdx +57 -7
- package/docs-site/zh/guides/sandbox-providers.mdx +258 -10
- package/docs-site/zh/guides/scoring-guide.mdx +4 -4
- package/docs-site/zh/guides/viewing-results.mdx +85 -41
- package/docs-site/zh/guides/write-experiment.mdx +5 -3
- package/docs-site/zh/guides/write-send.mdx +19 -2
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +27 -0
- package/docs-site/zh/reference/capabilities.mdx +2 -2
- package/docs-site/zh/reference/cli.mdx +35 -9
- package/docs-site/zh/reference/define-agent.mdx +60 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/events.mdx +2 -2
- package/docs-site/zh/reference/expect.mdx +36 -6
- package/package.json +5 -1
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +11 -0
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/types.ts +73 -6
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +411 -108
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +161 -29
- package/src/context/session.test.ts +1 -0
- package/src/context/session.ts +114 -6
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +65 -4
- package/src/i18n/zh-CN.ts +66 -4
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +34 -5
- package/src/report/built-in-user-parity.test.tsx +127 -173
- package/src/report/built-ins/experiment-comparison.tsx +179 -0
- package/src/report/built-ins/index.ts +7 -2
- package/src/report/components.tsx +11 -3
- package/src/report/compute.ts +80 -74
- package/src/report/dual-render.test.tsx +222 -91
- package/src/report/flag.ts +30 -2
- package/src/report/format.ts +36 -27
- package/src/report/index.ts +23 -6
- package/src/report/locale.ts +49 -6
- package/src/report/metrics.ts +68 -15
- package/src/report/primitives.tsx +6 -0
- package/src/report/react/AttemptList.tsx +9 -36
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +68 -0
- package/src/report/react/ExperimentList.tsx +173 -55
- package/src/report/react/MetricScatter.tsx +13 -25
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +72 -1
- package/src/report/react/fixtures.ts +34 -21
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +0 -1
- package/src/report/react/render.test.tsx +30 -69
- package/src/report/react/styles.css +112 -14
- package/src/report/report.test.ts +308 -105
- package/src/report/report.ts +6 -2
- package/src/report/text/faces.ts +111 -67
- package/src/report/text/table.ts +42 -5
- package/src/report/types.ts +42 -34
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +9 -7
- package/src/results/attempt-evidence.ts +15 -8
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +145 -55
- package/src/results/host-equivalence.test.ts +11 -9
- package/src/results/index.ts +2 -0
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +7 -1
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +85 -51
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +7 -0
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +242 -36
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +3 -2
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +2 -2
- package/src/show/index.ts +29 -16
- package/src/show/render.ts +626 -308
- package/src/show/show.test.ts +251 -36
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/components/AttemptModal.tsx +153 -2
- package/src/view/app/components/CodeView.tsx +32 -11
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +6 -0
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +2 -1
- package/src/view/data.ts +17 -7
- package/src/view/index.ts +12 -1
- package/src/view/server.ts +2 -0
- package/src/view/shared/types.ts +1 -1
- package/src/view/styles.css +3 -0
- package/src/view/view-report.test.ts +11 -10
- package/dist/o11y/execution-tree.d.ts +0 -103
- package/dist/o11y/otlp/select.d.ts +0 -22
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +0 -1
- package/dist/report/built-ins/cost-pass-rate-comparison.js +0 -17
- package/dist/results/annotated-source.d.ts +0 -61
- package/dist/results/attempt-evidence.d.ts +0 -69
- package/dist/results/attempt-source.d.ts +0 -15
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +0 -23
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
package/src/show/render.ts
CHANGED
|
@@ -4,16 +4,17 @@
|
|
|
4
4
|
// 全部纯函数(时间经 now 显式传入),证据数据由调用方 await 好了递进来。
|
|
5
5
|
|
|
6
6
|
import { join, relative } from "node:path";
|
|
7
|
-
import type { AssertionResult, DiffData, EvalResult, Verdict } from "../types.ts";
|
|
8
|
-
import type { AttemptEvidence,
|
|
9
|
-
import type { AnnotatedSourceLine } from "../results/index.ts";
|
|
7
|
+
import type { AssertionResult, DiffData, EvalResult, TimingNode, TraceSpan, Verdict } from "../types.ts";
|
|
8
|
+
import type { AttemptEvidence, AttemptHandle, Snapshot } from "../results/index.ts";
|
|
9
|
+
import type { AnnotatedSourceLine, SendAnnotation } from "../results/index.ts";
|
|
10
10
|
import { groupIncompatibleVersionSkips } from "../results/index.ts";
|
|
11
11
|
import type { SkippedDir } from "../results/index.ts";
|
|
12
12
|
import type { ExecutionNode } from "../o11y/execution-tree.ts";
|
|
13
13
|
import { foldEvalVerdict } from "../shared/verdict.ts";
|
|
14
|
+
import { summaryText } from "../scoring/display.ts";
|
|
14
15
|
import { attemptCostUSD } from "../report/metrics.ts";
|
|
15
16
|
import { formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD } from "../report/format.ts";
|
|
16
|
-
import { indentBlock, padDisplay, renderAlignedRows,
|
|
17
|
+
import { indentBlock, padDisplay, renderAlignedRows, wrapDisplay } from "../report/text/layout.ts";
|
|
17
18
|
import type { EvalHistoryRow, ExperimentHistoryRow } from "./compose.ts";
|
|
18
19
|
|
|
19
20
|
const MISSING = "—";
|
|
@@ -45,172 +46,6 @@ function verdictMark(verdict: Verdict): string {
|
|
|
45
46
|
return "✗";
|
|
46
47
|
}
|
|
47
48
|
|
|
48
|
-
function showVerdictLabel(verdict: Verdict): string {
|
|
49
|
-
switch (verdict) {
|
|
50
|
-
case "passed":
|
|
51
|
-
return "✓ passed";
|
|
52
|
-
case "failed":
|
|
53
|
-
return "✗ failed";
|
|
54
|
-
case "errored":
|
|
55
|
-
return "! errored";
|
|
56
|
-
case "skipped":
|
|
57
|
-
return "○ skipped";
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/** 按终端显示宽度截断,末尾保留一个省略号;不从 CJK 字符中间切开。 */
|
|
62
|
-
function clipDisplay(text: string, width: number): string {
|
|
63
|
-
if (stringWidth(text) <= width) return text;
|
|
64
|
-
if (width <= 1) return "…";
|
|
65
|
-
let out = "";
|
|
66
|
-
let used = 0;
|
|
67
|
-
for (const ch of text) {
|
|
68
|
-
const next = stringWidth(ch);
|
|
69
|
-
if (used + next > width - 1) break;
|
|
70
|
-
out += ch;
|
|
71
|
-
used += next;
|
|
72
|
-
}
|
|
73
|
-
return out + "…";
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function commandExitCode(evidence: string | undefined): string | undefined {
|
|
77
|
-
if (!evidence) return undefined;
|
|
78
|
-
try {
|
|
79
|
-
const parsed: unknown = JSON.parse(evidence);
|
|
80
|
-
if (typeof parsed === "object" && parsed !== null && "exitCode" in parsed) {
|
|
81
|
-
const exitCode = parsed.exitCode;
|
|
82
|
-
if (typeof exitCode === "number") return String(exitCode);
|
|
83
|
-
}
|
|
84
|
-
} catch {
|
|
85
|
-
// Assertion evidence is allowed to be arbitrary text; fall through to textual hints.
|
|
86
|
-
}
|
|
87
|
-
return /(?:returncode|retcode|exit(?:ed| code)?)\D{0,8}(-?\d+)/i.exec(evidence)?.[1];
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/** 默认索引 RESULT 列:只给能决定下一步的短原因,完整 evidence 留在 attempt 首页。 */
|
|
91
|
-
export function showResultReason(result: EvalResult): string {
|
|
92
|
-
if (result.verdict === "passed") return MISSING;
|
|
93
|
-
if (result.error !== undefined) return result.error;
|
|
94
|
-
if (result.skipReason !== undefined) return result.skipReason;
|
|
95
|
-
const gate = result.assertions.find((assertion) => !assertion.passed && assertion.severity === "gate");
|
|
96
|
-
if (!gate) return MISSING;
|
|
97
|
-
const expected = equalsExpected(gate.name);
|
|
98
|
-
if (expected !== undefined && gate.evidence !== undefined) {
|
|
99
|
-
return `expected ${expected}, received ${gate.evidence} · ${gate.name}`;
|
|
100
|
-
}
|
|
101
|
-
if (gate.name === "commandSucceeded()") {
|
|
102
|
-
const exitCode = commandExitCode(gate.evidence);
|
|
103
|
-
return `${exitCode === undefined ? "command failed" : `command exited ${exitCode}`} · ${gate.name}`;
|
|
104
|
-
}
|
|
105
|
-
return gate.detail ? `${gate.detail} · ${gate.name}` : gate.name;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
function average(values: number[]): number {
|
|
109
|
-
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
function snapshotSummary(snapshot: Snapshot): string {
|
|
113
|
-
const counts: Record<Verdict, number> = { passed: 0, failed: 0, errored: 0, skipped: 0 };
|
|
114
|
-
for (const attempt of snapshot.attempts) counts[attempt.result.verdict] += 1;
|
|
115
|
-
const costs = snapshot.attempts.map((attempt) => attemptCostUSD(attempt.result)).filter((cost): cost is number => cost !== null);
|
|
116
|
-
const duration = average(snapshot.attempts.map((attempt) => attempt.result.durationMs));
|
|
117
|
-
return [
|
|
118
|
-
`${counts.passed} passed`,
|
|
119
|
-
`${counts.failed} failed`,
|
|
120
|
-
`${counts.errored} errored`,
|
|
121
|
-
`${counts.skipped} skipped`,
|
|
122
|
-
attemptsLabel(snapshot.attempts.length),
|
|
123
|
-
formatDurationMs(duration),
|
|
124
|
-
costs.length === 0 ? MISSING : formatUSD(average(costs)),
|
|
125
|
-
].join(" · ");
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
function warningText(selection: Selection, width: number): string {
|
|
129
|
-
return selection.warnings
|
|
130
|
-
.flatMap((warning) => {
|
|
131
|
-
const lines = wrapDisplay(warning.message, Math.max(20, width - 9));
|
|
132
|
-
return lines.map((line, index) => `${index === 0 ? "WARNING " : " "}${line}`);
|
|
133
|
-
})
|
|
134
|
-
.join("\n");
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
function labeledBlock(label: string, value: string, width: number): string {
|
|
138
|
-
const prefix = padDisplay(label, 12);
|
|
139
|
-
return wrapDisplay(value, Math.max(8, width - stringWidth(prefix)))
|
|
140
|
-
.map((line, index) => `${index === 0 ? prefix : " ".repeat(stringWidth(prefix))}${line}`)
|
|
141
|
-
.join("\n");
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
function experimentAttemptTable(snapshot: Snapshot, width: number): string {
|
|
145
|
-
const separator = width >= 100 ? " " : width >= 70 ? " " : " ";
|
|
146
|
-
const fixedWidth = 9 + 9 + 8 + 5 + stringWidth(separator) * 5;
|
|
147
|
-
const flexible = Math.max(10, width - fixedWidth);
|
|
148
|
-
const naturalEval = Math.max(4, ...snapshot.attempts.map((attempt) => stringWidth(attempt.evalId)));
|
|
149
|
-
const evalWidth = Math.min(naturalEval, Math.max(4, Math.floor(flexible * 0.52)));
|
|
150
|
-
const resultWidth = Math.max(6, flexible - evalWidth);
|
|
151
|
-
const rows: string[][] = [["STATUS", "EVAL", "ATTEMPT", "RESULT", "DURATION", "COST"]];
|
|
152
|
-
for (const ev of snapshot.evals) {
|
|
153
|
-
for (const attempt of ev.attempts) {
|
|
154
|
-
const result = attempt.result;
|
|
155
|
-
const cost = attemptCostUSD(result);
|
|
156
|
-
rows.push([
|
|
157
|
-
showVerdictLabel(result.verdict),
|
|
158
|
-
clipDisplay(ev.id, evalWidth),
|
|
159
|
-
attempt.locator ?? MISSING,
|
|
160
|
-
clipDisplay(showResultReason(result), resultWidth),
|
|
161
|
-
result.verdict === "skipped" && result.durationMs === 0 ? MISSING : formatDurationMs(result.durationMs),
|
|
162
|
-
cost === null ? MISSING : formatUSD(cost),
|
|
163
|
-
]);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
return renderAlignedRows(rows, ["left", "left", "left", "left", "right", "right"], separator);
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
function comparisonTable(snapshots: Snapshot[], width: number): string {
|
|
170
|
-
const rows: string[][] = [["EXPERIMENT", "AGENT", "MODEL", "PASS", "FAILED", "ERROR", "SKIP", "DURATION", "COST"]];
|
|
171
|
-
for (const snapshot of snapshots) {
|
|
172
|
-
const verdicts = snapshot.evals.map((ev) => foldEvalVerdict(ev.attempts.map((attempt) => attempt.result)));
|
|
173
|
-
const passed = verdicts.filter((verdict) => verdict === "passed").length;
|
|
174
|
-
const failed = verdicts.filter((verdict) => verdict === "failed").length;
|
|
175
|
-
const errored = verdicts.filter((verdict) => verdict === "errored").length;
|
|
176
|
-
const skipped = verdicts.filter((verdict) => verdict === "skipped").length;
|
|
177
|
-
const ran = verdicts.length - skipped;
|
|
178
|
-
const costs = snapshot.attempts.map((attempt) => attemptCostUSD(attempt.result)).filter((cost): cost is number => cost !== null);
|
|
179
|
-
rows.push([
|
|
180
|
-
clipDisplay(snapshot.experimentId, Math.max(12, Math.floor(width * 0.24))),
|
|
181
|
-
snapshot.agent,
|
|
182
|
-
snapshot.model ?? MISSING,
|
|
183
|
-
ran === 0 ? MISSING : `${Math.round((passed / ran) * 1000) / 10}% ${passed}/${ran}`,
|
|
184
|
-
String(failed),
|
|
185
|
-
String(errored),
|
|
186
|
-
String(skipped),
|
|
187
|
-
formatDurationMs(average(snapshot.attempts.map((attempt) => attempt.result.durationMs))),
|
|
188
|
-
costs.length === 0 ? MISSING : formatUSD(average(costs)),
|
|
189
|
-
]);
|
|
190
|
-
}
|
|
191
|
-
return renderAlignedRows(rows, ["left", "left", "left", "right", "right", "right", "right", "right", "right"]);
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
/** docs/feature/reports/show.md 的默认结果索引;只服务裸 show,不是可替换报告组件。 */
|
|
195
|
-
export function showIndexText(selection: Selection, width: number): string {
|
|
196
|
-
const blocks: string[] = [];
|
|
197
|
-
const warnings = warningText(selection, width);
|
|
198
|
-
if (warnings) blocks.push(warnings);
|
|
199
|
-
if (selection.snapshots.length >= 2) blocks.push(`COMPARISON\n${comparisonTable(selection.snapshots, width)}`);
|
|
200
|
-
for (const snapshot of selection.snapshots) {
|
|
201
|
-
blocks.push(
|
|
202
|
-
[
|
|
203
|
-
labeledBlock("EXPERIMENT", `${snapshot.experimentId} · ${snapshot.agent}${snapshot.model ? ` · ${snapshot.model}` : ""}`, width),
|
|
204
|
-
labeledBlock("SUMMARY", snapshotSummary(snapshot), width),
|
|
205
|
-
"",
|
|
206
|
-
experimentAttemptTable(snapshot, width),
|
|
207
|
-
].join("\n"),
|
|
208
|
-
);
|
|
209
|
-
}
|
|
210
|
-
blocks.push(labeledBlock("DRILL DOWN", "niceeval show @<attempt> [--eval | --execution | --diff]", width));
|
|
211
|
-
return blocks.join("\n\n");
|
|
212
|
-
}
|
|
213
|
-
|
|
214
49
|
function attemptsLabel(n: number): string {
|
|
215
50
|
return `${n} ${n === 1 ? "attempt" : "attempts"}`;
|
|
216
51
|
}
|
|
@@ -294,43 +129,97 @@ function scoreText(score: number): string {
|
|
|
294
129
|
return formatPlainNumber(Math.round(score * 100) / 100);
|
|
295
130
|
}
|
|
296
131
|
|
|
297
|
-
/**
|
|
132
|
+
/** 分组路径标题(嵌套用 " > " 拼接);无分组返回 undefined。 */
|
|
133
|
+
function groupTitle(a: AssertionResult): string | undefined {
|
|
134
|
+
return a.groupPath && a.groupPath.length > 0 ? a.groupPath.join(" > ") : undefined;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** 断言行:✓/✗/◌ + severity + 标题;unavailable 带 reason,soft 恒带 score/1(失败再补 detail)。 */
|
|
298
138
|
export function assertionLine(a: AssertionResult): string {
|
|
299
|
-
const
|
|
300
|
-
const
|
|
139
|
+
const group = groupTitle(a);
|
|
140
|
+
const scope = group ? `${group} · ` : "";
|
|
141
|
+
const optional = a.optional ? "optional · " : "";
|
|
142
|
+
if (a.outcome === "unavailable") {
|
|
143
|
+
return `◌ ${a.severity} · ${optional}${scope}${a.name} — unavailable: ${a.reason}`;
|
|
144
|
+
}
|
|
145
|
+
const head = `${a.outcome === "passed" ? "✓" : "✗"} ${a.severity} ${optional}${scope}${a.name}`;
|
|
301
146
|
if (a.severity === "soft") {
|
|
302
|
-
const detail =
|
|
303
|
-
|
|
147
|
+
const detail = a.outcome === "failed" && a.detail ? `: ${a.detail}` : "";
|
|
148
|
+
const threshold = a.threshold !== undefined ? ` / ${scoreText(a.threshold)}` : "/1";
|
|
149
|
+
return `${head} — ${scoreText(a.score)}${threshold}${detail}`;
|
|
304
150
|
}
|
|
305
|
-
if (
|
|
151
|
+
if (a.outcome === "failed") {
|
|
306
152
|
const reason = a.detail ?? `score ${scoreText(a.score)}`;
|
|
307
|
-
|
|
153
|
+
const received = a.received ?? a.evidence;
|
|
154
|
+
// 单行面只放摘要收口后的预览;多行值(如 output tail)的完整版在 attempt 首页展开。
|
|
155
|
+
return received !== undefined ? `${head} — ${reason} · received: ${summaryText(received)}` : `${head} — ${reason}`;
|
|
308
156
|
}
|
|
309
157
|
return head;
|
|
310
158
|
}
|
|
311
159
|
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
160
|
+
/** 字段行:首行 `<label>: <值首行>`,值的其余行(如 CommandResult 的 output tail)缩进原样展开。 */
|
|
161
|
+
function fieldLines(label: string, value: string): string[] {
|
|
162
|
+
const [first, ...rest] = value.split("\n");
|
|
163
|
+
return [` ${label}: ${first ?? ""}`, ...rest.map((line) => ` ${line}`)];
|
|
315
164
|
}
|
|
316
165
|
|
|
317
|
-
/**
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
const
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
166
|
+
/** 每条的通用行组(见 docs/feature/scoring/library/display.md「通用渲染规则」):
|
|
167
|
+
* 首行 `severity · <标题>`(有分组时标题是分组路径,随后 assertion: 行给检查方式),
|
|
168
|
+
* 之后按有则显示的顺序列 expected / received / score / reason / source。 */
|
|
169
|
+
function assertionRecordLines(a: AssertionResult): string[] {
|
|
170
|
+
const group = groupTitle(a);
|
|
171
|
+
const optional = a.optional ? "optional · " : "";
|
|
172
|
+
const title = group ?? a.name;
|
|
173
|
+
const lines: string[] = [];
|
|
174
|
+
const scoreSuffix =
|
|
175
|
+
a.outcome !== "unavailable" && (a.severity === "soft" || a.threshold !== undefined)
|
|
176
|
+
? ` ${scoreText(a.score)}${a.threshold !== undefined ? ` / ${scoreText(a.threshold)}` : ""}`
|
|
177
|
+
: "";
|
|
178
|
+
lines.push(` ${a.severity} · ${optional}${title}${scoreSuffix}`);
|
|
179
|
+
const detailLine = a.detail ?? (group ? a.name : undefined);
|
|
180
|
+
if (detailLine !== undefined && detailLine !== title) lines.push(` assertion: ${detailLine}`);
|
|
181
|
+
if (a.outcome === "unavailable") {
|
|
182
|
+
lines.push(` reason: ${a.reason}`);
|
|
183
|
+
} else {
|
|
184
|
+
if (a.expected !== undefined) lines.push(...fieldLines("expected", a.expected));
|
|
185
|
+
if (a.received !== undefined) lines.push(...fieldLines("received", a.received));
|
|
186
|
+
if (a.evidence !== undefined && a.evidence !== a.received) lines.push(...fieldLines("evidence", a.evidence));
|
|
332
187
|
}
|
|
188
|
+
if (a.loc) lines.push(` source: ${a.loc.file}:${a.loc.line}${a.loc.column ? `:${a.loc.column}` : ""}`);
|
|
189
|
+
return lines;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* 默认 Attempt 页的首要诊断:按结果分节,只逐条列出需要看的(见 display.md)——
|
|
194
|
+
* `failures:`(gate 失败,含 --strict 下改判的 soft)、`soft below threshold:`、
|
|
195
|
+
* `scores:`(无阈值 judge 的纯打分)、`unavailable:`(证据评不了的,带 reason)。
|
|
196
|
+
* 全部通过时这些节整体省略(返回 undefined),只留计数行。
|
|
197
|
+
*/
|
|
198
|
+
export function failureDiagnostics(assertions: AssertionResult[], width: number, strict?: boolean): string | undefined {
|
|
199
|
+
const failures = assertions.filter(
|
|
200
|
+
(a) => a.outcome === "failed" && (a.severity === "gate" || strict),
|
|
201
|
+
);
|
|
202
|
+
const softBelow = assertions.filter(
|
|
203
|
+
(a) => a.outcome === "failed" && a.severity === "soft" && !strict,
|
|
204
|
+
);
|
|
205
|
+
const scores = assertions.filter(
|
|
206
|
+
(a) => a.outcome === "passed" && a.severity === "soft" && a.threshold === undefined && a.score < 1,
|
|
207
|
+
);
|
|
208
|
+
const unavailableOnes = assertions.filter((a) => a.outcome === "unavailable");
|
|
209
|
+
const lines: string[] = [];
|
|
210
|
+
const section = (title: string, items: AssertionResult[]) => {
|
|
211
|
+
if (items.length === 0) return;
|
|
212
|
+
if (lines.length > 0) lines.push("");
|
|
213
|
+
lines.push(title);
|
|
214
|
+
for (const a of items) lines.push(...assertionRecordLines(a));
|
|
215
|
+
};
|
|
216
|
+
section("failures:", failures);
|
|
217
|
+
section("soft below threshold:", softBelow);
|
|
218
|
+
section("scores:", scores);
|
|
219
|
+
section("unavailable:", unavailableOnes);
|
|
220
|
+
if (lines.length === 0) return undefined;
|
|
333
221
|
return lines.flatMap((line) => {
|
|
222
|
+
if (line === "") return [line];
|
|
334
223
|
const indent = line.length - line.trimStart().length;
|
|
335
224
|
return wrapDisplay(line.trimStart(), Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`);
|
|
336
225
|
}).join("\n");
|
|
@@ -353,7 +242,7 @@ export function attemptHeader(attempt: AttemptHandle): string {
|
|
|
353
242
|
|
|
354
243
|
// ───────────────────────── AttemptEvidence 共用 ─────────────────────────
|
|
355
244
|
// evalSourceText / executionText / attemptOverviewText(--eval / --execution / 默认全景)
|
|
356
|
-
// 与 evalDetailText 的紧凑索引列共用的小件:locator
|
|
245
|
+
// 与 evalDetailText 的紧凑索引列共用的小件:locator 头、失败原因、
|
|
357
246
|
// 断言计票摘要。三个证据 renderer 与全景面都只消费同一份 AttemptEvidence,不各自读
|
|
358
247
|
// artifact 或重新判定 capability(loadAttemptEvidence 已经算好)。
|
|
359
248
|
|
|
@@ -362,36 +251,6 @@ export function attemptEvidenceHeader(evidence: AttemptEvidence): string {
|
|
|
362
251
|
return [evidence.locator, evidence.identity.evalId, evidence.identity.experimentId, evidence.result.verdict].join(" · ");
|
|
363
252
|
}
|
|
364
253
|
|
|
365
|
-
/** capability 位 → 展示字母,只列为 true 的位;全部为 false 时返回空串(不打空 `[]`)。 */
|
|
366
|
-
export function capabilityLetters(capabilities: AttemptEvidenceCapabilities): string {
|
|
367
|
-
const letters: string[] = [];
|
|
368
|
-
if (capabilities.eval) letters.push("E");
|
|
369
|
-
if (capabilities.execution) letters.push("X");
|
|
370
|
-
if (capabilities.timing) letters.push("⏱");
|
|
371
|
-
if (capabilities.diff) letters.push("D");
|
|
372
|
-
return letters.length > 0 ? `[${letters.join(",")}]` : "";
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
/**
|
|
376
|
-
* 紧凑多 attempt 索引(裸 `niceeval show`、单 eval 多 experiment 行、evidence flag 撞多个
|
|
377
|
-
* eval 时的消歧提示)用的廉价 capability 估算:只读瘦身 `EvalResult` 字段(`hasSources` /
|
|
378
|
-
* `hasEvents` / `hasTrace`),不为了点亮一个字母去读 artifact —— 与
|
|
379
|
-
* `report/compute.ts::attemptRow` 的 `hasEvidence` 同一个「只认瘦身字段」的口径(它同样只用
|
|
380
|
-
* `hasEvents` / `hasTrace`,同样不读 diff.json)。`diff` 位没有对应的瘦身字段(不像
|
|
381
|
-
* events/trace/sources,写入面从不给 diff 算一个 has* 布尔),而 diff.json 可达上百 MB,
|
|
382
|
-
* 不该为了这一个字母在渲染路径上打开它(见 docs/feature/reports/architecture.md「计算与
|
|
383
|
-
* 渲染分离」)——因此这里恒报 `false`;只有 `loadAttemptEvidence()` real 出的
|
|
384
|
-
* capabilities(单 attempt 路径,如 `@<locator>` 全景)才知道真实答案。
|
|
385
|
-
*/
|
|
386
|
-
export function cheapCapabilities(result: EvalResult): AttemptEvidenceCapabilities {
|
|
387
|
-
return {
|
|
388
|
-
eval: result.hasSources === true,
|
|
389
|
-
execution: result.hasEvents === true,
|
|
390
|
-
timing: result.hasEvents === true && result.hasTrace === true,
|
|
391
|
-
diff: false,
|
|
392
|
-
};
|
|
393
|
-
}
|
|
394
|
-
|
|
395
254
|
/**
|
|
396
255
|
* 一次 attempt 未通过的判定原因,单行、不含 detail——供紧凑索引行使用。precedence 与
|
|
397
256
|
* `report/compute.ts::reasonFor` 同一条规则(error → skipReason → 未通过的 gate 断言,
|
|
@@ -401,22 +260,24 @@ export function cheapCapabilities(result: EvalResult): AttemptEvidenceCapabiliti
|
|
|
401
260
|
* 依赖它的内部实现细节。
|
|
402
261
|
*/
|
|
403
262
|
export function verdictReasonLine(result: EvalResult): string | undefined {
|
|
404
|
-
if (result.error !== undefined) return result.error;
|
|
263
|
+
if (result.error !== undefined) return result.error.message;
|
|
405
264
|
if (result.skipReason !== undefined) return result.skipReason;
|
|
406
|
-
const gates = result.assertions.filter((a) =>
|
|
407
|
-
if (gates.length === 0)
|
|
265
|
+
const gates = result.assertions.filter((a) => a.outcome === "failed" && a.severity === "gate");
|
|
266
|
+
if (gates.length === 0) {
|
|
267
|
+
const gap = result.assertions.find((a) => a.outcome === "unavailable" && !a.optional);
|
|
268
|
+
return gap && gap.outcome === "unavailable" ? `unavailable ${gap.name} (${gap.reason})` : undefined;
|
|
269
|
+
}
|
|
408
270
|
return gates.map((a) => `gate ${a.name}`).join(", ");
|
|
409
271
|
}
|
|
410
272
|
|
|
411
|
-
/** 紧凑多 attempt 索引的一行:`✗ weather/brooklyn @7K2M9Q
|
|
273
|
+
/** 紧凑多 attempt 索引的一行:`✗ weather/brooklyn @7K2M9Q gate calledTool(...)`。 */
|
|
412
274
|
export function attemptIndexLine(opts: {
|
|
413
275
|
evalId: string;
|
|
414
276
|
verdict: Verdict;
|
|
415
277
|
locator: string | undefined;
|
|
416
|
-
capabilities: AttemptEvidenceCapabilities;
|
|
417
278
|
reason?: string;
|
|
418
279
|
}): string {
|
|
419
|
-
const loc = opts.locator
|
|
280
|
+
const loc = opts.locator ?? MISSING;
|
|
420
281
|
const parts = [`${verdictMark(opts.verdict)} ${opts.evalId}`, loc];
|
|
421
282
|
if (opts.reason) parts.push(opts.reason);
|
|
422
283
|
return parts.join(" ");
|
|
@@ -430,13 +291,15 @@ export function attemptIndexLine(opts: {
|
|
|
430
291
|
* (未捕获源码),读 `result.assertions` 让这条摘要不因缺源码而跟着消失。
|
|
431
292
|
*/
|
|
432
293
|
export function assertionSummaryLine(assertions: AssertionResult[]): string {
|
|
433
|
-
const passed = assertions.filter((a) => a.passed).length;
|
|
434
|
-
const gateFailed = assertions.filter((a) =>
|
|
435
|
-
const softBelow = assertions.filter((a) =>
|
|
294
|
+
const passed = assertions.filter((a) => a.outcome === "passed").length;
|
|
295
|
+
const gateFailed = assertions.filter((a) => a.outcome === "failed" && a.severity === "gate").length;
|
|
296
|
+
const softBelow = assertions.filter((a) => a.outcome === "failed" && a.severity === "soft").length;
|
|
297
|
+
const unavailableCount = assertions.filter((a) => a.outcome === "unavailable").length;
|
|
436
298
|
const parts: string[] = [];
|
|
437
299
|
if (passed > 0) parts.push(`${passed} passed`);
|
|
438
300
|
if (gateFailed > 0) parts.push(`${gateFailed} gate failed`);
|
|
439
301
|
if (softBelow > 0) parts.push(`${softBelow} soft below target`);
|
|
302
|
+
if (unavailableCount > 0) parts.push(`${unavailableCount} unavailable`);
|
|
440
303
|
return `assertions: ${parts.length > 0 ? parts.join(" · ") : "(none)"}`;
|
|
441
304
|
}
|
|
442
305
|
|
|
@@ -463,7 +326,7 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
463
326
|
blocks.push(description ? `${evalId} — ${description}` : evalId);
|
|
464
327
|
|
|
465
328
|
// 每 experiment 一行:折叠判定、attempt 数、最新 attempt 的耗时、总成本、判定时间、
|
|
466
|
-
// 代表 attempt 的紧凑索引(locator +
|
|
329
|
+
// 代表 attempt 的紧凑索引(locator + 失败原因)——agent 从这张榜单
|
|
467
330
|
// 就能直接摘到一个 `@<locator>` 下钻,不必先跑一遍 `--eval`/`--execution` 才知道选谁。
|
|
468
331
|
const rows: string[][] = [];
|
|
469
332
|
for (const snapshot of snapshots) {
|
|
@@ -477,7 +340,7 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
477
340
|
if (c !== null) cost = (cost ?? 0) + c;
|
|
478
341
|
}
|
|
479
342
|
const rep = pickDetailAttempt(ev.attempts);
|
|
480
|
-
const locatorCell = rep?.locator
|
|
343
|
+
const locatorCell = rep?.locator ?? MISSING;
|
|
481
344
|
const reasonCell = rep ? (verdictReasonLine(rep.result) ?? "") : "";
|
|
482
345
|
rows.push([
|
|
483
346
|
snapshot.experimentId,
|
|
@@ -498,7 +361,7 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
498
361
|
lines.push(indentBlock(wrapDisplay(assertionLine(a), width - 2).join("\n"), " "));
|
|
499
362
|
}
|
|
500
363
|
if (detail.result.error !== undefined) {
|
|
501
|
-
lines.push(indentBlock(wrapDisplay(`error: ${detail.result.error}`, width - 2).join("\n"), " "));
|
|
364
|
+
lines.push(indentBlock(wrapDisplay(`error: ${detail.result.error.message}`, width - 2).join("\n"), " "));
|
|
502
365
|
}
|
|
503
366
|
if (detail.result.skipReason !== undefined) {
|
|
504
367
|
lines.push(indentBlock(wrapDisplay(`skipped: ${detail.result.skipReason}`, width - 2).join("\n"), " "));
|
|
@@ -600,28 +463,51 @@ function indentedText(text: string, width: number, indent = 4, maxLines = 18): s
|
|
|
600
463
|
|
|
601
464
|
const MAX_SOURCE_LINES = 400;
|
|
602
465
|
|
|
603
|
-
/** gate 失败 / soft
|
|
466
|
+
/** gate 失败 / soft 恒带分 / unavailable 带 reason——与 assertionLine 的口径一致,但不带断言
|
|
467
|
+
* name(源码行本身就是名字)。 */
|
|
604
468
|
function evalAssertionDetailLine(a: AssertionResult): string | undefined {
|
|
469
|
+
if (a.outcome === "unavailable") {
|
|
470
|
+
return `${a.severity} · unavailable · ${a.reason}`;
|
|
471
|
+
}
|
|
605
472
|
if (a.severity === "soft") {
|
|
606
|
-
const detail =
|
|
607
|
-
|
|
473
|
+
const detail = a.outcome === "failed" && a.detail ? ` · ${a.detail}` : "";
|
|
474
|
+
const threshold = a.threshold !== undefined ? ` / ${scoreText(a.threshold)}` : "/1";
|
|
475
|
+
return `soft · ${scoreText(a.score)}${threshold}${detail}`;
|
|
608
476
|
}
|
|
609
|
-
if (
|
|
477
|
+
if (a.outcome === "failed") {
|
|
478
|
+
// 标注行是源码页里的一行事实,不是证据面:expected / received 过摘要收口
|
|
479
|
+
// (折单行 + 上限),完整值在 attempt 首页与 events.json / diff.json。
|
|
610
480
|
const parts = ["gate"];
|
|
611
|
-
|
|
481
|
+
const group = groupTitle(a);
|
|
482
|
+
if (group) parts.push(group);
|
|
612
483
|
parts.push(a.name);
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
if (
|
|
616
|
-
if (a.detail) parts.push(a.detail);
|
|
484
|
+
if (a.expected !== undefined) parts.push(`expected ${summaryText(a.expected)}`);
|
|
485
|
+
const received = a.received ?? a.evidence;
|
|
486
|
+
if (received !== undefined) parts.push(`received ${summaryText(received)}`);
|
|
487
|
+
if (a.detail) parts.push(summaryText(a.detail));
|
|
617
488
|
return parts.join(" · ");
|
|
618
489
|
}
|
|
619
490
|
return undefined;
|
|
620
491
|
}
|
|
621
492
|
|
|
493
|
+
/** send 行标注:轮身份 · status · 墙钟(有记录才出现),契约见 show.md「--eval」。 */
|
|
494
|
+
function sendAnnotationLine(send: SendAnnotation): string {
|
|
495
|
+
const parts = [send.label, send.status];
|
|
496
|
+
if (send.durationMs !== undefined) parts.push(formatDurationMs(send.durationMs));
|
|
497
|
+
return parts.join(" · ");
|
|
498
|
+
}
|
|
499
|
+
|
|
622
500
|
function evalSourceLineText(line: AnnotatedSourceLine, gutterWidth: number, width: number): string[] {
|
|
623
|
-
const anyFailed = line.assertions.some((a) =>
|
|
624
|
-
|
|
501
|
+
const anyFailed = line.assertions.some((a) => a.outcome === "failed") ||
|
|
502
|
+
line.sends.some((send) => send.status === "failed");
|
|
503
|
+
const anyUnavailable = line.assertions.some((a) => a.outcome === "unavailable");
|
|
504
|
+
const glyph = line.assertions.length === 0 && line.sends.length === 0
|
|
505
|
+
? " "
|
|
506
|
+
: anyFailed
|
|
507
|
+
? "✗"
|
|
508
|
+
: anyUnavailable
|
|
509
|
+
? "◌"
|
|
510
|
+
: "✓";
|
|
625
511
|
const marginWidth = gutterWidth + 2; // 行号列 + glyph + 分隔空格
|
|
626
512
|
const prefix = `${padDisplay(String(line.line), gutterWidth)}${glyph} `;
|
|
627
513
|
// 源码行的空白(尤其是缩进)是语义的一部分:wrapDisplay 按单词重排会把连续空格
|
|
@@ -630,6 +516,11 @@ function evalSourceLineText(line: AnnotatedSourceLine, gutterWidth: number, widt
|
|
|
630
516
|
// 折行只对续行加统一 margin,救不回已经被吃掉的原始缩进,不如老实截断。
|
|
631
517
|
const out = [prefix + clip(line.text, Math.max(20, width - marginWidth))];
|
|
632
518
|
const margin = " ".repeat(marginWidth);
|
|
519
|
+
for (const send of line.sends) {
|
|
520
|
+
for (const wrapped2 of wrapDisplay(sendAnnotationLine(send), Math.max(20, width - marginWidth))) {
|
|
521
|
+
out.push(margin + wrapped2);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
633
524
|
for (const a of line.assertions) {
|
|
634
525
|
const detail = evalAssertionDetailLine(a);
|
|
635
526
|
if (detail === undefined) continue;
|
|
@@ -670,8 +561,14 @@ export function evalSourceText(
|
|
|
670
561
|
);
|
|
671
562
|
}
|
|
672
563
|
|
|
673
|
-
|
|
674
|
-
|
|
564
|
+
const tail = [assertionSummaryLine(evidence.result.assertions)];
|
|
565
|
+
// 标注行的值是收口预览;有未通过断言时给「更进一步」——attempt 首页展开完整 expected /
|
|
566
|
+
// received(含 output tail),再往下是 result.json / events.json。
|
|
567
|
+
if (evidence.result.assertions.some((a) => a.outcome !== "passed")) {
|
|
568
|
+
tail.push(`full failure detail: niceeval show ${evidence.locator}`);
|
|
569
|
+
}
|
|
570
|
+
if (artifact) tail.push(`full eval source: ${artifact}`);
|
|
571
|
+
blocks.push(tail.join("\n"));
|
|
675
572
|
return blocks.join("\n\n");
|
|
676
573
|
}
|
|
677
574
|
|
|
@@ -786,10 +683,30 @@ export function executionText(
|
|
|
786
683
|
const agentNodes = tree.nodes.filter((node) => node.kind !== "telemetry");
|
|
787
684
|
const telemetryCount = tree.nodes.length - agentNodes.length;
|
|
788
685
|
const shown = agentNodes.slice(0, MAX_EVENTS);
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
686
|
+
|
|
687
|
+
// 按轮分段(见 docs/feature/reports/show.md「--execution」):每轮以 TURN 头行开始——身份
|
|
688
|
+
// s<session>/t<turn>(与 --timing 的 turn 节点、diff 的 windows 同一套标签,来自
|
|
689
|
+
// result.json.phases 的 turn 时间树)、该轮墙钟;边界按用户消息切(t.send 恒以用户消息开轮)。
|
|
690
|
+
const turnNodes = (evidence.result.phases ?? [])
|
|
691
|
+
.flatMap((p) => p.children ?? [])
|
|
692
|
+
.filter((n) => n.kind === "turn");
|
|
693
|
+
const lines: string[] = [];
|
|
694
|
+
let turnIndex = -1;
|
|
695
|
+
shown.forEach((node, index) => {
|
|
696
|
+
const isTurnStart = node.kind === "message" && node.role === "user";
|
|
697
|
+
if (isTurnStart) {
|
|
698
|
+
turnIndex += 1;
|
|
699
|
+
const turn = turnNodes[turnIndex];
|
|
700
|
+
const label = turn?.label ?? `t${turnIndex + 1}`;
|
|
701
|
+
const durationPart = turn ? ` · ${formatDurationMs(turn.durationMs)}` : "";
|
|
702
|
+
const failedPart = turn?.failed ? " · failed" : "";
|
|
703
|
+
if (index > 0) lines.push("");
|
|
704
|
+
lines.push(`TURN ${label}${failedPart || " · completed"}${durationPart}`);
|
|
705
|
+
} else if (index > 0) {
|
|
706
|
+
lines.push("");
|
|
707
|
+
}
|
|
708
|
+
lines.push(...executionNodeLines(node, originMs, timingAvailable, width).map((l) => (turnIndex >= 0 ? ` ${l}` : l)));
|
|
709
|
+
});
|
|
793
710
|
|
|
794
711
|
const tail: string[] = [];
|
|
795
712
|
if (timingAvailable) {
|
|
@@ -824,14 +741,38 @@ export function executionText(
|
|
|
824
741
|
|
|
825
742
|
const MAX_OVERVIEW_DIFF_NAMES = 5;
|
|
826
743
|
|
|
744
|
+
/** net 效果的单字母标记(A/M/D;none = 动过但净无变化,标 ±)。 */
|
|
745
|
+
function netLetter(net: string): string {
|
|
746
|
+
switch (net) {
|
|
747
|
+
case "added":
|
|
748
|
+
return "A";
|
|
749
|
+
case "deleted":
|
|
750
|
+
return "D";
|
|
751
|
+
case "none":
|
|
752
|
+
return "±";
|
|
753
|
+
default:
|
|
754
|
+
return "M";
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
|
|
827
758
|
function overviewDiffLine(diff: DiffData): string {
|
|
828
|
-
const names =
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
];
|
|
759
|
+
const names = Object.entries(diff.files)
|
|
760
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
761
|
+
.map(([p, f]) => `${netLetter(f.net)} ${p}`);
|
|
832
762
|
const shown = names.slice(0, MAX_OVERVIEW_DIFF_NAMES);
|
|
833
763
|
const more = names.length > shown.length ? ` · +${names.length - shown.length} more` : "";
|
|
834
|
-
return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed · ${shown.join("
|
|
764
|
+
return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed by agent · ${shown.join(" · ")}${more}`;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/** 有界行 diff(公共前后缀修剪):对单区域编辑精确,复杂编辑给出上界近似。 */
|
|
768
|
+
function lineDelta(before: string | undefined, after: string | undefined): { adds: number; dels: number } {
|
|
769
|
+
const a = before === undefined ? [] : before.split("\n");
|
|
770
|
+
const b = after === undefined ? [] : after.split("\n");
|
|
771
|
+
let prefix = 0;
|
|
772
|
+
while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
|
|
773
|
+
let suffix = 0;
|
|
774
|
+
while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix]) suffix++;
|
|
775
|
+
return { adds: b.length - prefix - suffix, dels: a.length - prefix - suffix };
|
|
835
776
|
}
|
|
836
777
|
|
|
837
778
|
/**
|
|
@@ -839,6 +780,48 @@ function overviewDiffLine(diff: DiffData): string {
|
|
|
839
780
|
* 执行事件计数、可选 OTel 时间指示、工作区 diff 摘要,不复现 `--eval` 的完整源码、
|
|
840
781
|
* `--execution` 的完整事件流或 `--diff` 的完整文件列表(那些内容各自的证据 flag 才给)。
|
|
841
782
|
*/
|
|
783
|
+
/** lifecycle operation → 首页可读标签:点换空格,与 docs/feature/reports/show.md 的
|
|
784
|
+
* `phase: sandbox provision` 字面一致(不引入第二套本地化标签,show 首页用英文原样呈现)。 */
|
|
785
|
+
function operationWords(operation: string): string {
|
|
786
|
+
return operation.replace(/\./g, " ");
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* errored attempt 的结构化 `error:` 块(见 docs/feature/reports/show.md「errored attempt 的首页」)。
|
|
791
|
+
* 先展开 phase(= operation 的可读形态)/ code / message / cause,stack 放在块后、保持原始换行。
|
|
792
|
+
* 字段来自结构化 `AttemptError`;非 errored(`r.error === undefined`)返回 undefined。
|
|
793
|
+
*/
|
|
794
|
+
function renderErrorBlock(r: EvalResult): string | undefined {
|
|
795
|
+
const err = r.error;
|
|
796
|
+
if (err === undefined) return undefined;
|
|
797
|
+
const lines = ["error:", ` phase: ${err.phase}`, ` code: ${err.code}`, ` message: ${err.message}`];
|
|
798
|
+
if (err.cause) {
|
|
799
|
+
const c = err.cause;
|
|
800
|
+
const causeText = c.name ? `${c.name} · ${c.message}` : c.message;
|
|
801
|
+
lines.push(` cause: ${causeText}`);
|
|
802
|
+
}
|
|
803
|
+
// stack 放在 error 块之后、保持原始换行(见 docs);没有 stack(如 timeout / turn-failed)就不加空块。
|
|
804
|
+
if (err.stack && err.stack.trim() !== "") return `${lines.join("\n")}\n\n${err.stack.replace(/\n+$/, "")}`;
|
|
805
|
+
return lines.join("\n");
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
/**
|
|
809
|
+
* attempt 级诊断块(见 docs/feature/reports/show.md「diagnostics」)。每条一行标头
|
|
810
|
+
* `<level> · <operation> · <code>`,message 缩进在下一行;`count > 1` 时补 `(N occurrences)`。
|
|
811
|
+
* 诊断的 level 与 verdict 无关 —— passed / failed / errored 都可能带一条 cleanup / teardown
|
|
812
|
+
* warning。没有诊断返回 undefined。
|
|
813
|
+
*/
|
|
814
|
+
function renderAttemptDiagnostics(r: EvalResult): string | undefined {
|
|
815
|
+
if (!r.diagnostics || r.diagnostics.length === 0) return undefined;
|
|
816
|
+
const lines = ["diagnostics:"];
|
|
817
|
+
for (const d of r.diagnostics) {
|
|
818
|
+
lines.push(` ${d.level} · ${d.phase} · ${d.code}`);
|
|
819
|
+
const occ = d.count && d.count > 1 ? ` (${d.count} occurrences)` : "";
|
|
820
|
+
lines.push(` ${d.message}${occ}`);
|
|
821
|
+
}
|
|
822
|
+
return lines.join("\n");
|
|
823
|
+
}
|
|
824
|
+
|
|
842
825
|
export function attemptOverviewText(
|
|
843
826
|
evidence: AttemptEvidence,
|
|
844
827
|
opts: { header: string; artifactPath?: string; width: number },
|
|
@@ -853,27 +836,43 @@ export function attemptOverviewText(
|
|
|
853
836
|
|
|
854
837
|
const blocks: string[] = [[header, metaParts.join(" · ")].join("\n")];
|
|
855
838
|
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
)
|
|
839
|
+
// errored attempt 的首页不靠 trace 就要能解释基础设施错误(见 docs/feature/reports/show.md
|
|
840
|
+
// 「errored attempt 的首页」):先展开结构化 error(phase/operation/code/message/cause + stack),
|
|
841
|
+
// 断言块对没有断言的 errored attempt 省略(它在评分之前就挂了)。
|
|
842
|
+
const errorBlock = renderErrorBlock(r);
|
|
843
|
+
if (errorBlock) blocks.push(errorBlock);
|
|
844
|
+
|
|
845
|
+
// errored 且没有任何断言时不打印空的 "assertions: 0 passed" —— 主因已经在 error 块里说清楚了。
|
|
846
|
+
if (r.assertions.length > 0 || r.error === undefined) {
|
|
847
|
+
blocks.push(
|
|
848
|
+
[
|
|
849
|
+
assertionSummaryLine(r.assertions),
|
|
850
|
+
evidence.evalSource
|
|
851
|
+
? `eval source: ${evidence.evalSource.sourcePath} · sha256:${evidence.evalSource.sourceSha256.slice(0, 8)}…`
|
|
852
|
+
: "eval source: unavailable (not captured for this attempt)",
|
|
853
|
+
].join("\n"),
|
|
854
|
+
);
|
|
855
|
+
}
|
|
864
856
|
|
|
865
857
|
const diagnostics = failureDiagnostics(r.assertions, opts.width);
|
|
866
858
|
if (diagnostics) blocks.push(diagnostics);
|
|
867
859
|
|
|
860
|
+
// attempt 级诊断(teardown/cleanup 等,与 verdict 独立;passed/failed/errored 都可能有)。
|
|
861
|
+
const attemptDiag = renderAttemptDiagnostics(r);
|
|
862
|
+
if (attemptDiag) blocks.push(attemptDiag);
|
|
863
|
+
|
|
868
864
|
if (evidence.execution) {
|
|
869
865
|
const nodes = evidence.execution.nodes.filter((node) => node.kind !== "telemetry");
|
|
870
866
|
const skillLoads = nodes.filter((n) => n.kind === "skill.loaded").length;
|
|
871
867
|
const toolCalls = nodes.filter((n) => n.kind === "action").length;
|
|
872
868
|
const aiMessages = nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
|
|
873
|
-
const
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
869
|
+
const counted = (count: number, singular: string, plural = `${singular}s`) =>
|
|
870
|
+
`${count} ${count === 1 ? singular : plural}`;
|
|
871
|
+
const execLines = [
|
|
872
|
+
`execution: ${counted(nodes.length, "event")} · ${counted(skillLoads, "skill load")} · ${counted(toolCalls, "tool call")} · ${counted(aiMessages, "AI message")}`,
|
|
873
|
+
];
|
|
874
|
+
const timingLine = overviewTimingLine(r);
|
|
875
|
+
if (timingLine) execLines.push(timingLine);
|
|
877
876
|
blocks.push(execLines.join("\n"));
|
|
878
877
|
} else {
|
|
879
878
|
blocks.push("execution: unavailable (no events recorded for this attempt)");
|
|
@@ -892,6 +891,7 @@ export function attemptOverviewText(
|
|
|
892
891
|
const available = [
|
|
893
892
|
evidence.capabilities.eval ? `niceeval show ${evidence.locator} --eval` : undefined,
|
|
894
893
|
evidence.capabilities.execution ? `niceeval show ${evidence.locator} --execution` : undefined,
|
|
894
|
+
evidence.capabilities.timing ? `niceeval show ${evidence.locator} --timing` : undefined,
|
|
895
895
|
evidence.capabilities.diff ? `niceeval show ${evidence.locator} --diff` : undefined,
|
|
896
896
|
].filter((command): command is string => command !== undefined);
|
|
897
897
|
if (available.length > 0) tail.push(`available:\n${available.map((command) => ` ${command}`).join("\n")}`);
|
|
@@ -912,45 +912,363 @@ export function diffText(opts: {
|
|
|
912
912
|
file?: string;
|
|
913
913
|
}): string {
|
|
914
914
|
const { header, diff, artifactPath, file } = opts;
|
|
915
|
-
const source = artifactPath ? join(
|
|
915
|
+
const source = artifactPath ? join(artifactPath, "diff.json") : undefined;
|
|
916
916
|
if (!diff) {
|
|
917
|
-
return `${header}\n\
|
|
917
|
+
return `${header}\n\ndiff unavailable (no diff recorded for this attempt: remote agent, or diff artifact not published${source ? `; expected: ${source}` : ""})`;
|
|
918
918
|
}
|
|
919
919
|
|
|
920
920
|
if (file !== undefined) {
|
|
921
|
-
const
|
|
922
|
-
if (
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
return `${header}\n\nFile "${file}" is not in this attempt's diff. Files: ${known.join(", ") || "(none)"}`;
|
|
921
|
+
const summary = diff.files[file];
|
|
922
|
+
if (summary === undefined) {
|
|
923
|
+
const known = Object.keys(diff.files).sort();
|
|
924
|
+
return `${header}\n\nFile "${file}" is not in this attempt's agent diff. Files: ${known.join(", ") || "(none)"}`;
|
|
926
925
|
}
|
|
927
|
-
|
|
928
|
-
const
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
926
|
+
// 单文件 patch 按窗口逐段渲染(diff.json 存的就是逐窗口 delta,不产出跨窗口合成 patch)。
|
|
927
|
+
const head = `${netLetter(summary.net)} ${file} · touched in ${summary.windows.join(", ")}`;
|
|
928
|
+
if (summary.binary) {
|
|
929
|
+
const sections = diff.windows
|
|
930
|
+
.filter((w) => w.changes[file] !== undefined)
|
|
931
|
+
.map((w) => {
|
|
932
|
+
const c = w.changes[file]!;
|
|
933
|
+
const b = c.binary ?? {};
|
|
934
|
+
return `── window ${w.window}\nbinary · ${b.beforeBytes ?? 0} → ${b.afterBytes ?? 0} bytes`;
|
|
935
|
+
});
|
|
936
|
+
return `${header}\n\n${head}\n\n${sections.join("\n\n")}`;
|
|
937
|
+
}
|
|
938
|
+
const sections: string[] = [];
|
|
939
|
+
for (const w of diff.windows) {
|
|
940
|
+
const c = w.changes[file];
|
|
941
|
+
if (c === undefined) continue;
|
|
942
|
+
sections.push(`── window ${w.window}\n${windowHunk(c)}`);
|
|
943
|
+
}
|
|
944
|
+
return `${header}\n\n${head}\n\n${sections.join("\n\n")}${source ? `\n\n(full diff: ${source})` : ""}`;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
const entries = Object.entries(diff.files).sort(([a], [b]) => a.localeCompare(b));
|
|
948
|
+
if (entries.length === 0) {
|
|
949
|
+
return `${header}\n\n(no file changes by the agent in any send window${source ? ` · full diff: ${source}` : ""})`;
|
|
950
|
+
}
|
|
951
|
+
const rows = entries.map(([path, summary]) => {
|
|
952
|
+
if (summary.binary) {
|
|
953
|
+
return [netLetter(summary.net), path, "binary", summary.windows.join(", ")];
|
|
954
|
+
}
|
|
955
|
+
let adds = 0;
|
|
956
|
+
let dels = 0;
|
|
957
|
+
for (const w of diff.windows) {
|
|
958
|
+
const c = w.changes[path];
|
|
959
|
+
if (!c) continue;
|
|
960
|
+
const d = lineDelta(c.before, c.after);
|
|
961
|
+
adds += Math.max(0, d.adds);
|
|
962
|
+
dels += Math.max(0, d.dels);
|
|
963
|
+
}
|
|
964
|
+
const delta = [adds > 0 ? `+${adds}` : "", dels > 0 ? `-${dels}` : ""].filter(Boolean).join(" ") || "±0";
|
|
965
|
+
return [netLetter(summary.net), path, delta, summary.windows.join(", ")];
|
|
966
|
+
});
|
|
967
|
+
const headLine = `${entries.length} ${entries.length === 1 ? "file" : "files"} changed by agent`;
|
|
968
|
+
const single = entries[0] ? `\n\nsingle file: niceeval show @… --diff=${entries[0][0]}` : "";
|
|
969
|
+
return `${header}\n\n${headLine}\n${renderAlignedRows(rows).split("\n").map((l) => ` ${l}`).join("\n")}${single}`;
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
/** 一个窗口内单文件的最小 unified hunk:公共前后缀修剪出的编辑区,一段 @@ 展示。 */
|
|
973
|
+
function windowHunk(c: { status: string; before?: string; after?: string }): string {
|
|
974
|
+
const a = c.before === undefined ? [] : c.before.replace(/\n$/, "").split("\n");
|
|
975
|
+
const b = c.after === undefined ? [] : c.after.replace(/\n$/, "").split("\n");
|
|
976
|
+
let prefix = 0;
|
|
977
|
+
while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
|
|
978
|
+
let suffix = 0;
|
|
979
|
+
while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix]) suffix++;
|
|
980
|
+
const removed = a.slice(prefix, a.length - suffix);
|
|
981
|
+
const added = b.slice(prefix, b.length - suffix);
|
|
982
|
+
const ctxBefore = a.slice(Math.max(0, prefix - 2), prefix);
|
|
983
|
+
const lines: string[] = [];
|
|
984
|
+
lines.push(`@@ -${Math.max(1, prefix - ctxBefore.length + 1)},${removed.length + ctxBefore.length} +${Math.max(1, prefix - ctxBefore.length + 1)},${added.length + ctxBefore.length} @@`);
|
|
985
|
+
for (const l of ctxBefore) lines.push(` ${l}`);
|
|
986
|
+
const MAX_HUNK_LINES = 200;
|
|
987
|
+
const shownRemoved = removed.slice(0, MAX_HUNK_LINES);
|
|
988
|
+
const shownAdded = added.slice(0, MAX_HUNK_LINES);
|
|
989
|
+
for (const l of shownRemoved) lines.push(`-${l}`);
|
|
990
|
+
if (removed.length > shownRemoved.length) lines.push(`… (${removed.length - shownRemoved.length} more removed lines)`);
|
|
991
|
+
for (const l of shownAdded) lines.push(`+${l}`);
|
|
992
|
+
if (added.length > shownAdded.length) lines.push(`… (${added.length - shownAdded.length} more added lines)`);
|
|
993
|
+
return lines.join("\n");
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
// ───────────────────────── 证据切面:--timing(统一时间树) ─────────────────────────
|
|
997
|
+
|
|
998
|
+
const CLOSING_PHASE_NAMES = new Set(["eval.teardown", "agent.teardown", "sandbox.teardown", "sandbox.suspend", "sandbox.stop"]);
|
|
999
|
+
|
|
1000
|
+
// 首页只保留能回答「大头在哪」的阶段。分类账锚点与 telemetry bookkeeping 在很短时
|
|
1001
|
+
// 留给 --timing 完整树;一旦自身变成慢点或失败,仍必须抬到首页,不能被静默藏掉。
|
|
1002
|
+
const OVERVIEW_DETAIL_PHASE_NAMES = new Set(["workspace.baseline", "telemetry.configure", "telemetry.collect"]);
|
|
1003
|
+
const OVERVIEW_DETAIL_THRESHOLD_MS = 1_000;
|
|
1004
|
+
|
|
1005
|
+
/** phases 主链摘要行(首页 `timing:`):各主链阶段耗时点隔,收尾段合计以 `teardown +N` 单列。 */
|
|
1006
|
+
export function overviewTimingLine(r: EvalResult): string | undefined {
|
|
1007
|
+
if (!r.phases || r.phases.length === 0) return undefined;
|
|
1008
|
+
const main = r.phases.filter(
|
|
1009
|
+
(p) =>
|
|
1010
|
+
!CLOSING_PHASE_NAMES.has(p.name) &&
|
|
1011
|
+
(p.failed === true || !OVERVIEW_DETAIL_PHASE_NAMES.has(p.name) || p.durationMs >= OVERVIEW_DETAIL_THRESHOLD_MS),
|
|
1012
|
+
);
|
|
1013
|
+
const closing = r.phases.filter((p) => CLOSING_PHASE_NAMES.has(p.name));
|
|
1014
|
+
const parts = main.map((p) => `${p.name} ${formatDurationMs(p.durationMs)}${p.failed ? " ✗" : ""}`);
|
|
1015
|
+
const closingMs = closing.reduce((sum, p) => sum + p.durationMs, 0);
|
|
1016
|
+
if (closingMs > 0) parts.push(`teardown +${formatDurationMs(closingMs)}`);
|
|
1017
|
+
return `timing: ${parts.join(" · ")}`;
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
const TIMING_DETAIL_NODE_BUDGET = 80;
|
|
1021
|
+
|
|
1022
|
+
interface DiagnosticTimingNode {
|
|
1023
|
+
id: string;
|
|
1024
|
+
label: string;
|
|
1025
|
+
startOffsetMs: number;
|
|
1026
|
+
durationMs: number;
|
|
1027
|
+
failed: boolean;
|
|
1028
|
+
otel: boolean;
|
|
1029
|
+
children: DiagnosticTimingNode[];
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
function timingNodeLabel(node: TimingNode): string {
|
|
1033
|
+
if (node.kind === "command" && node.command) return `shell · ${node.command.display}`;
|
|
1034
|
+
if (node.kind === "turn") return `turn ${node.label}`;
|
|
1035
|
+
if (node.kind === "operation" || node.kind === "provider") return `${node.kind} · ${node.label}`;
|
|
1036
|
+
return node.label;
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
function traceReferenceCounts(nodes: readonly TimingNode[], counts = new Map<string, number>()): Map<string, number> {
|
|
1040
|
+
for (const node of nodes) {
|
|
1041
|
+
if (node.kind === "turn" && node.traceId) counts.set(node.traceId, (counts.get(node.traceId) ?? 0) + 1);
|
|
1042
|
+
traceReferenceCounts(node.children ?? [], counts);
|
|
1043
|
+
}
|
|
1044
|
+
return counts;
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
/** OTel 只按唯一 traceId 归属和 span parent 关系挂接;不按绝对墙钟猜跨进程顺序。 */
|
|
1048
|
+
function otelForest(traceId: string, spans: readonly TraceSpan[], turnStartOffsetMs: number): DiagnosticTimingNode[] {
|
|
1049
|
+
const unique = new Map<string, TraceSpan>();
|
|
1050
|
+
for (const span of spans) if (span.traceId === traceId && !unique.has(span.spanId)) unique.set(span.spanId, span);
|
|
1051
|
+
const selected = [...unique.values()];
|
|
1052
|
+
if (selected.length === 0) return [];
|
|
1053
|
+
const traceOrigin = Math.min(...selected.map((span) => span.startMs));
|
|
1054
|
+
const nodes = new Map<string, DiagnosticTimingNode>();
|
|
1055
|
+
for (const span of selected) {
|
|
1056
|
+
nodes.set(span.spanId, {
|
|
1057
|
+
id: `otel:${span.traceId}:${span.spanId}`,
|
|
1058
|
+
label: `${span.kind ?? "span"} · ${span.name}`,
|
|
1059
|
+
startOffsetMs: turnStartOffsetMs + Math.max(0, span.startMs - traceOrigin),
|
|
1060
|
+
durationMs: Math.max(0, span.endMs - span.startMs),
|
|
1061
|
+
failed: span.status === "error",
|
|
1062
|
+
otel: true,
|
|
1063
|
+
children: [],
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
const roots: DiagnosticTimingNode[] = [];
|
|
1067
|
+
for (const span of selected) {
|
|
1068
|
+
const node = nodes.get(span.spanId)!;
|
|
1069
|
+
const parent = span.parentSpanId && span.parentSpanId !== span.spanId ? nodes.get(span.parentSpanId) : undefined;
|
|
1070
|
+
if (parent) parent.children.push(node);
|
|
1071
|
+
else roots.push(node);
|
|
1072
|
+
}
|
|
1073
|
+
const sort = (items: DiagnosticTimingNode[]) => {
|
|
1074
|
+
items.sort((a, b) => a.startOffsetMs - b.startOffsetMs || a.id.localeCompare(b.id));
|
|
1075
|
+
for (const item of items) sort(item.children);
|
|
1076
|
+
};
|
|
1077
|
+
sort(roots);
|
|
1078
|
+
return roots;
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
function diagnosticTimingNode(
|
|
1082
|
+
node: TimingNode,
|
|
1083
|
+
spans: readonly TraceSpan[],
|
|
1084
|
+
uniqueTraceIds: ReadonlySet<string>,
|
|
1085
|
+
): DiagnosticTimingNode {
|
|
1086
|
+
const children = (node.children ?? []).map((child) => diagnosticTimingNode(child, spans, uniqueTraceIds));
|
|
1087
|
+
if (node.kind === "turn" && node.traceId && uniqueTraceIds.has(node.traceId)) {
|
|
1088
|
+
children.push(...otelForest(node.traceId, spans, node.startOffsetMs));
|
|
1089
|
+
}
|
|
1090
|
+
return {
|
|
1091
|
+
id: `runner:${node.id}`,
|
|
1092
|
+
label: timingNodeLabel(node),
|
|
1093
|
+
startOffsetMs: node.startOffsetMs,
|
|
1094
|
+
durationMs: node.durationMs,
|
|
1095
|
+
failed: node.failed === true,
|
|
1096
|
+
otel: false,
|
|
1097
|
+
children,
|
|
1098
|
+
};
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
interface FlatTimingNode {
|
|
1102
|
+
node: DiagnosticTimingNode;
|
|
1103
|
+
path: readonly DiagnosticTimingNode[];
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
function flattenTimingForest(roots: readonly DiagnosticTimingNode[]): FlatTimingNode[] {
|
|
1107
|
+
const flat: FlatTimingNode[] = [];
|
|
1108
|
+
const visit = (node: DiagnosticTimingNode, ancestors: readonly DiagnosticTimingNode[]) => {
|
|
1109
|
+
const path = [...ancestors, node];
|
|
1110
|
+
flat.push({ node, path });
|
|
1111
|
+
for (const child of node.children) visit(child, path);
|
|
1112
|
+
};
|
|
1113
|
+
for (const root of roots) visit(root, []);
|
|
1114
|
+
return flat;
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* 默认 80-node 投影。四个稳定池先各用自己的配额,再把空余额按失败→最慢→首尾重分配。
|
|
1119
|
+
* 选中深层节点必须连同祖先整条加入;四池合并后按 id 去重。
|
|
1120
|
+
*/
|
|
1121
|
+
function selectTimingNodes(roots: readonly DiagnosticTimingNode[]): ReadonlySet<string> {
|
|
1122
|
+
const flat = flattenTimingForest(roots);
|
|
1123
|
+
if (flat.length <= TIMING_DETAIL_NODE_BUDGET) return new Set(flat.map(({ node }) => node.id));
|
|
1124
|
+
|
|
1125
|
+
const selected = new Set<string>();
|
|
1126
|
+
const byStart = [...flat].sort(
|
|
1127
|
+
(a, b) => a.node.startOffsetMs - b.node.startOffsetMs || a.node.id.localeCompare(b.node.id),
|
|
1128
|
+
);
|
|
1129
|
+
const failed = byStart.filter(({ node }) => node.failed);
|
|
1130
|
+
const slow = [...flat].sort(
|
|
1131
|
+
(a, b) => b.node.durationMs - a.node.durationMs || a.node.startOffsetMs - b.node.startOffsetMs || a.node.id.localeCompare(b.node.id),
|
|
1132
|
+
);
|
|
1133
|
+
const latest = [...flat].sort(
|
|
1134
|
+
(a, b) => b.node.startOffsetMs - a.node.startOffsetMs || a.node.id.localeCompare(b.node.id),
|
|
1135
|
+
);
|
|
1136
|
+
|
|
1137
|
+
const add = (candidate: FlatTimingNode, allowance: number): number => {
|
|
1138
|
+
const missing = candidate.path.filter((node) => !selected.has(node.id));
|
|
1139
|
+
if (missing.length === 0 || missing.length > allowance || selected.size + missing.length > TIMING_DETAIL_NODE_BUDGET) {
|
|
1140
|
+
return 0;
|
|
1141
|
+
}
|
|
1142
|
+
for (const node of missing) selected.add(node.id);
|
|
1143
|
+
return missing.length;
|
|
1144
|
+
};
|
|
1145
|
+
const pool = (candidates: readonly FlatTimingNode[], cap: number) => {
|
|
1146
|
+
let remaining = cap;
|
|
1147
|
+
for (const candidate of candidates) {
|
|
1148
|
+
if (remaining === 0) break;
|
|
1149
|
+
remaining -= add(candidate, remaining);
|
|
1150
|
+
}
|
|
1151
|
+
};
|
|
1152
|
+
|
|
1153
|
+
pool(failed, 40);
|
|
1154
|
+
pool(slow, 20);
|
|
1155
|
+
pool(byStart, 10);
|
|
1156
|
+
pool(latest, 10);
|
|
1157
|
+
|
|
1158
|
+
// 固定池未用满时,剩余额度按契约优先级继续分配;候选稳定、无法容纳的整条路径跳过。
|
|
1159
|
+
for (const candidate of [...failed, ...slow, ...byStart, ...latest]) {
|
|
1160
|
+
const remaining = TIMING_DETAIL_NODE_BUDGET - selected.size;
|
|
1161
|
+
if (remaining === 0) break;
|
|
1162
|
+
add(candidate, remaining);
|
|
1163
|
+
}
|
|
1164
|
+
return selected;
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
function subtreeStats(node: DiagnosticTimingNode): { nodes: number; failed: number } {
|
|
1168
|
+
let nodes = 1;
|
|
1169
|
+
let failed = node.failed ? 1 : 0;
|
|
1170
|
+
for (const child of node.children) {
|
|
1171
|
+
const stats = subtreeStats(child);
|
|
1172
|
+
nodes += stats.nodes;
|
|
1173
|
+
failed += stats.failed;
|
|
1174
|
+
}
|
|
1175
|
+
return { nodes, failed };
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
type VisibleTimingEntry =
|
|
1179
|
+
| { kind: "node"; node: DiagnosticTimingNode }
|
|
1180
|
+
| { kind: "omitted"; nodes: number; failed: number };
|
|
1181
|
+
|
|
1182
|
+
function visibleTimingEntries(
|
|
1183
|
+
nodes: readonly DiagnosticTimingNode[],
|
|
1184
|
+
selected: ReadonlySet<string> | undefined,
|
|
1185
|
+
): VisibleTimingEntry[] {
|
|
1186
|
+
if (!selected) return nodes.map((node) => ({ kind: "node", node }));
|
|
1187
|
+
const entries: VisibleTimingEntry[] = [];
|
|
1188
|
+
for (let i = 0; i < nodes.length;) {
|
|
1189
|
+
const node = nodes[i]!;
|
|
1190
|
+
if (selected.has(node.id)) {
|
|
1191
|
+
entries.push({ kind: "node", node });
|
|
1192
|
+
i += 1;
|
|
1193
|
+
continue;
|
|
1194
|
+
}
|
|
1195
|
+
let omittedNodes = 0;
|
|
1196
|
+
let omittedFailed = 0;
|
|
1197
|
+
while (i < nodes.length && !selected.has(nodes[i]!.id)) {
|
|
1198
|
+
const stats = subtreeStats(nodes[i]!);
|
|
1199
|
+
omittedNodes += stats.nodes;
|
|
1200
|
+
omittedFailed += stats.failed;
|
|
1201
|
+
i += 1;
|
|
1202
|
+
}
|
|
1203
|
+
entries.push({ kind: "omitted", nodes: omittedNodes, failed: omittedFailed });
|
|
1204
|
+
}
|
|
1205
|
+
return entries;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
function diagnosticTimingLines(
|
|
1209
|
+
nodes: readonly DiagnosticTimingNode[],
|
|
1210
|
+
prefix: string,
|
|
1211
|
+
selected: ReadonlySet<string> | undefined,
|
|
1212
|
+
locator: string,
|
|
1213
|
+
): string[] {
|
|
1214
|
+
const entries = visibleTimingEntries(nodes, selected);
|
|
1215
|
+
const lines: string[] = [];
|
|
1216
|
+
entries.forEach((entry, index) => {
|
|
1217
|
+
const last = index === entries.length - 1;
|
|
1218
|
+
const branch = `${prefix}${last ? "└─ " : "├─ "}`;
|
|
1219
|
+
const childPrefix = `${prefix}${last ? " " : "│ "}`;
|
|
1220
|
+
if (entry.kind === "omitted") {
|
|
1221
|
+
const failed = entry.failed > 0 ? ` · ${entry.failed} failed` : "";
|
|
1222
|
+
lines.push(`${branch}… ${entry.nodes} nodes omitted${failed} · full: niceeval show ${locator} --timing=full`);
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
const node = entry.node;
|
|
1226
|
+
lines.push(
|
|
1227
|
+
`${branch}${node.label} ${formatDurationMs(node.durationMs)}${node.failed ? " ✗" : ""}${node.otel ? " OTel" : ""}`,
|
|
1228
|
+
);
|
|
1229
|
+
lines.push(...diagnosticTimingLines(node.children, childPrefix, selected, locator));
|
|
1230
|
+
});
|
|
1231
|
+
return lines;
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
/**
|
|
1235
|
+
* `--timing`:整个 attempt 的统一时间树(见 docs/feature/reports/show.md)。先按
|
|
1236
|
+
* `result.json.phases` 输出 runner 生命周期,再展开 hook / 命令 / turn;turn 带 traceId 时
|
|
1237
|
+
* 从 trace.json 挂接 agent/model/tool spans。缩进表达包含关系,子项不能求和后与父项比较。
|
|
1238
|
+
*/
|
|
1239
|
+
export function timingText(
|
|
1240
|
+
evidence: AttemptEvidence,
|
|
1241
|
+
opts: { header: string; artifactPath?: string; width: number; mode?: "summary" | "full" },
|
|
1242
|
+
): string {
|
|
1243
|
+
const r = evidence.result;
|
|
1244
|
+
if (!r.phases || r.phases.length === 0) {
|
|
1245
|
+
return `${opts.header}\n\nphase timing unavailable (this result was not produced by a runner with phase timing)`;
|
|
1246
|
+
}
|
|
1247
|
+
const spans = evidence.trace;
|
|
1248
|
+
const main = r.phases.filter((p) => !CLOSING_PHASE_NAMES.has(p.name));
|
|
1249
|
+
const closing = r.phases.filter((p) => CLOSING_PHASE_NAMES.has(p.name));
|
|
1250
|
+
const traceCounts = new Map<string, number>();
|
|
1251
|
+
for (const phase of r.phases) traceReferenceCounts(phase.children ?? [], traceCounts);
|
|
1252
|
+
const uniqueTraceIds = new Set([...traceCounts].filter(([, count]) => count === 1).map(([traceId]) => traceId));
|
|
1253
|
+
const phaseForests = new Map(
|
|
1254
|
+
r.phases.map((phase) => [
|
|
1255
|
+
phase,
|
|
1256
|
+
(phase.children ?? []).map((node) => diagnosticTimingNode(node, spans ?? [], uniqueTraceIds)),
|
|
1257
|
+
]),
|
|
1258
|
+
);
|
|
1259
|
+
const allRoots = [...phaseForests.values()].flat();
|
|
1260
|
+
const selected = opts.mode === "full" ? undefined : selectTimingNodes(allRoots);
|
|
1261
|
+
|
|
1262
|
+
const lines: string[] = [`total ${formatDurationMs(r.durationMs)}`, ""];
|
|
1263
|
+
const renderPhase = (p: NonNullable<EvalResult["phases"]>[number]) => {
|
|
1264
|
+
const failedNote = p.failed ? ` ✗ failed here${r.error ? ` (${r.error.code})` : ""}` : "";
|
|
1265
|
+
lines.push(`${p.name.padEnd(22)}${formatDurationMs(p.durationMs)}${failedNote}`);
|
|
1266
|
+
lines.push(...diagnosticTimingLines(phaseForests.get(p) ?? [], " ", selected, evidence.locator));
|
|
1267
|
+
};
|
|
1268
|
+
for (const p of main) renderPhase(p);
|
|
1269
|
+
if (closing.length > 0) {
|
|
1270
|
+
lines.push("", "teardown (not counted in total):");
|
|
1271
|
+
for (const p of closing) renderPhase(p);
|
|
1272
|
+
}
|
|
1273
|
+
return `${opts.header}\n\n${lines.join("\n")}`;
|
|
956
1274
|
}
|