niceeval 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +317 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +178 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +618 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +32 -0
- package/dist/report/format.js +126 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +83 -0
- package/dist/report/locale.js +160 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +20 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +10 -0
- package/dist/report/react/ExperimentList.js +15 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +100 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +393 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +503 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +420 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/annotated-source.d.ts +61 -0
- package/dist/results/attempt-evidence.d.ts +69 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +9 -4
- package/docs-site/zh/concepts/assert.mdx +8 -8
- package/docs-site/zh/concepts/evals.mdx +5 -5
- package/docs-site/zh/concepts/experiment.mdx +3 -1
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +2 -2
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -0
- package/docs-site/zh/guides/fixtures.mdx +1 -1
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +4 -4
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +35 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +10 -5
- package/docs-site/zh/guides/sandbox-providers.mdx +102 -0
- package/docs-site/zh/guides/viewing-results.mdx +199 -79
- package/docs-site/zh/guides/write-experiment.mdx +23 -1
- package/docs-site/zh/guides/write-send.mdx +2 -2
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +24 -12
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +7 -1
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/package.json +10 -6
- package/src/agents/bub.ts +134 -43
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +11 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +62 -2
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/context.ts +3 -0
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +26 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +82 -0
- package/src/define.ts +72 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +36 -15
- package/src/i18n/zh-CN.ts +39 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +643 -0
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +223 -50
- package/src/report/compute.ts +358 -53
- package/src/report/dual-render.test.tsx +1125 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +50 -2
- package/src/report/index.ts +48 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +206 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +115 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +108 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +136 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +129 -0
- package/src/report/react/fixtures.ts +163 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +22 -6
- package/src/report/react/render.test.tsx +164 -31
- package/src/report/react/styles.css +405 -213
- package/src/report/report.test.ts +403 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +303 -95
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +196 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +97 -15
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +69 -28
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +125 -94
- package/src/show/render.ts +660 -160
- package/src/show/show.test.ts +472 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/docs-site/zh/guides/sandbox-backends.mdx +0 -75
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/show/render.ts
CHANGED
|
@@ -1,23 +1,26 @@
|
|
|
1
1
|
// show 的文本渲染:单 eval 详情、--history 时间轴、三个证据切面(transcript / trace / diff)。
|
|
2
2
|
// 输出形态照 docs-site/zh/guides/viewing-results.mdx 的示例块;长内容一律截断,
|
|
3
|
-
//
|
|
3
|
+
// 但截断永远如实标注剩余数量和原始 artifact 路径 —— 输出对上下文窗口友好,事实源留在盘上。
|
|
4
4
|
// 全部纯函数(时间经 now 显式传入),证据数据由调用方 await 好了递进来。
|
|
5
5
|
|
|
6
6
|
import { join, relative } from "node:path";
|
|
7
|
-
import type { AssertionResult,
|
|
8
|
-
import type {
|
|
9
|
-
import type {
|
|
10
|
-
import {
|
|
7
|
+
import type { AssertionResult, DiffData, EvalResult, Verdict } from "../types.ts";
|
|
8
|
+
import type { AttemptEvidence, AttemptEvidenceCapabilities, AttemptHandle, Selection, Snapshot } from "../results/index.ts";
|
|
9
|
+
import type { AnnotatedSourceLine } from "../results/index.ts";
|
|
10
|
+
import { groupIncompatibleVersionSkips } from "../results/index.ts";
|
|
11
|
+
import type { SkippedDir } from "../results/index.ts";
|
|
12
|
+
import type { ExecutionNode } from "../o11y/execution-tree.ts";
|
|
13
|
+
import { foldEvalVerdict } from "../shared/verdict.ts";
|
|
11
14
|
import { attemptCostUSD } from "../report/metrics.ts";
|
|
12
15
|
import { formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD } from "../report/format.ts";
|
|
13
|
-
import { indentBlock, padDisplay, renderAlignedRows, wrapDisplay } from "../report/text/layout.ts";
|
|
16
|
+
import { indentBlock, padDisplay, renderAlignedRows, stringWidth, wrapDisplay } from "../report/text/layout.ts";
|
|
14
17
|
import type { EvalHistoryRow, ExperimentHistoryRow } from "./compose.ts";
|
|
15
18
|
|
|
16
19
|
const MISSING = "—";
|
|
17
20
|
|
|
18
21
|
// ───────────────────────── 时间与小件 ─────────────────────────
|
|
19
22
|
|
|
20
|
-
/**
|
|
23
|
+
/** 判定时间的相对标注("just now" / "41s ago" / "2h ago");未来时刻按 just now 兜底。 */
|
|
21
24
|
export function relativeAgo(iso: string | undefined, now: number): string {
|
|
22
25
|
if (!iso) return "";
|
|
23
26
|
const ms = now - Date.parse(iso);
|
|
@@ -36,35 +39,232 @@ export function timelineStamp(iso: string): string {
|
|
|
36
39
|
return iso.slice(0, 16).replace(/:/g, "-");
|
|
37
40
|
}
|
|
38
41
|
|
|
39
|
-
function
|
|
40
|
-
if (
|
|
41
|
-
if (
|
|
42
|
+
function verdictMark(verdict: Verdict): string {
|
|
43
|
+
if (verdict === "passed") return "✓";
|
|
44
|
+
if (verdict === "skipped") return "-";
|
|
42
45
|
return "✗";
|
|
43
46
|
}
|
|
44
47
|
|
|
48
|
+
function showVerdictLabel(verdict: Verdict): string {
|
|
49
|
+
switch (verdict) {
|
|
50
|
+
case "passed":
|
|
51
|
+
return "✓ passed";
|
|
52
|
+
case "failed":
|
|
53
|
+
return "✗ failed";
|
|
54
|
+
case "errored":
|
|
55
|
+
return "! errored";
|
|
56
|
+
case "skipped":
|
|
57
|
+
return "○ skipped";
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** 按终端显示宽度截断,末尾保留一个省略号;不从 CJK 字符中间切开。 */
|
|
62
|
+
function clipDisplay(text: string, width: number): string {
|
|
63
|
+
if (stringWidth(text) <= width) return text;
|
|
64
|
+
if (width <= 1) return "…";
|
|
65
|
+
let out = "";
|
|
66
|
+
let used = 0;
|
|
67
|
+
for (const ch of text) {
|
|
68
|
+
const next = stringWidth(ch);
|
|
69
|
+
if (used + next > width - 1) break;
|
|
70
|
+
out += ch;
|
|
71
|
+
used += next;
|
|
72
|
+
}
|
|
73
|
+
return out + "…";
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function commandExitCode(evidence: string | undefined): string | undefined {
|
|
77
|
+
if (!evidence) return undefined;
|
|
78
|
+
try {
|
|
79
|
+
const parsed: unknown = JSON.parse(evidence);
|
|
80
|
+
if (typeof parsed === "object" && parsed !== null && "exitCode" in parsed) {
|
|
81
|
+
const exitCode = parsed.exitCode;
|
|
82
|
+
if (typeof exitCode === "number") return String(exitCode);
|
|
83
|
+
}
|
|
84
|
+
} catch {
|
|
85
|
+
// Assertion evidence is allowed to be arbitrary text; fall through to textual hints.
|
|
86
|
+
}
|
|
87
|
+
return /(?:returncode|retcode|exit(?:ed| code)?)\D{0,8}(-?\d+)/i.exec(evidence)?.[1];
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** 默认索引 RESULT 列:只给能决定下一步的短原因,完整 evidence 留在 attempt 首页。 */
|
|
91
|
+
export function showResultReason(result: EvalResult): string {
|
|
92
|
+
if (result.verdict === "passed") return MISSING;
|
|
93
|
+
if (result.error !== undefined) return result.error;
|
|
94
|
+
if (result.skipReason !== undefined) return result.skipReason;
|
|
95
|
+
const gate = result.assertions.find((assertion) => !assertion.passed && assertion.severity === "gate");
|
|
96
|
+
if (!gate) return MISSING;
|
|
97
|
+
const expected = equalsExpected(gate.name);
|
|
98
|
+
if (expected !== undefined && gate.evidence !== undefined) {
|
|
99
|
+
return `expected ${expected}, received ${gate.evidence} · ${gate.name}`;
|
|
100
|
+
}
|
|
101
|
+
if (gate.name === "commandSucceeded()") {
|
|
102
|
+
const exitCode = commandExitCode(gate.evidence);
|
|
103
|
+
return `${exitCode === undefined ? "command failed" : `command exited ${exitCode}`} · ${gate.name}`;
|
|
104
|
+
}
|
|
105
|
+
return gate.detail ? `${gate.detail} · ${gate.name}` : gate.name;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function average(values: number[]): number {
|
|
109
|
+
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function snapshotSummary(snapshot: Snapshot): string {
|
|
113
|
+
const counts: Record<Verdict, number> = { passed: 0, failed: 0, errored: 0, skipped: 0 };
|
|
114
|
+
for (const attempt of snapshot.attempts) counts[attempt.result.verdict] += 1;
|
|
115
|
+
const costs = snapshot.attempts.map((attempt) => attemptCostUSD(attempt.result)).filter((cost): cost is number => cost !== null);
|
|
116
|
+
const duration = average(snapshot.attempts.map((attempt) => attempt.result.durationMs));
|
|
117
|
+
return [
|
|
118
|
+
`${counts.passed} passed`,
|
|
119
|
+
`${counts.failed} failed`,
|
|
120
|
+
`${counts.errored} errored`,
|
|
121
|
+
`${counts.skipped} skipped`,
|
|
122
|
+
attemptsLabel(snapshot.attempts.length),
|
|
123
|
+
formatDurationMs(duration),
|
|
124
|
+
costs.length === 0 ? MISSING : formatUSD(average(costs)),
|
|
125
|
+
].join(" · ");
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function warningText(selection: Selection, width: number): string {
|
|
129
|
+
return selection.warnings
|
|
130
|
+
.flatMap((warning) => {
|
|
131
|
+
const lines = wrapDisplay(warning.message, Math.max(20, width - 9));
|
|
132
|
+
return lines.map((line, index) => `${index === 0 ? "WARNING " : " "}${line}`);
|
|
133
|
+
})
|
|
134
|
+
.join("\n");
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function labeledBlock(label: string, value: string, width: number): string {
|
|
138
|
+
const prefix = padDisplay(label, 12);
|
|
139
|
+
return wrapDisplay(value, Math.max(8, width - stringWidth(prefix)))
|
|
140
|
+
.map((line, index) => `${index === 0 ? prefix : " ".repeat(stringWidth(prefix))}${line}`)
|
|
141
|
+
.join("\n");
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function experimentAttemptTable(snapshot: Snapshot, width: number): string {
|
|
145
|
+
const separator = width >= 100 ? " " : width >= 70 ? " " : " ";
|
|
146
|
+
const fixedWidth = 9 + 9 + 8 + 5 + stringWidth(separator) * 5;
|
|
147
|
+
const flexible = Math.max(10, width - fixedWidth);
|
|
148
|
+
const naturalEval = Math.max(4, ...snapshot.attempts.map((attempt) => stringWidth(attempt.evalId)));
|
|
149
|
+
const evalWidth = Math.min(naturalEval, Math.max(4, Math.floor(flexible * 0.52)));
|
|
150
|
+
const resultWidth = Math.max(6, flexible - evalWidth);
|
|
151
|
+
const rows: string[][] = [["STATUS", "EVAL", "ATTEMPT", "RESULT", "DURATION", "COST"]];
|
|
152
|
+
for (const ev of snapshot.evals) {
|
|
153
|
+
for (const attempt of ev.attempts) {
|
|
154
|
+
const result = attempt.result;
|
|
155
|
+
const cost = attemptCostUSD(result);
|
|
156
|
+
rows.push([
|
|
157
|
+
showVerdictLabel(result.verdict),
|
|
158
|
+
clipDisplay(ev.id, evalWidth),
|
|
159
|
+
attempt.locator ?? MISSING,
|
|
160
|
+
clipDisplay(showResultReason(result), resultWidth),
|
|
161
|
+
result.verdict === "skipped" && result.durationMs === 0 ? MISSING : formatDurationMs(result.durationMs),
|
|
162
|
+
cost === null ? MISSING : formatUSD(cost),
|
|
163
|
+
]);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return renderAlignedRows(rows, ["left", "left", "left", "left", "right", "right"], separator);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function comparisonTable(snapshots: Snapshot[], width: number): string {
|
|
170
|
+
const rows: string[][] = [["EXPERIMENT", "AGENT", "MODEL", "PASS", "FAILED", "ERROR", "SKIP", "DURATION", "COST"]];
|
|
171
|
+
for (const snapshot of snapshots) {
|
|
172
|
+
const verdicts = snapshot.evals.map((ev) => foldEvalVerdict(ev.attempts.map((attempt) => attempt.result)));
|
|
173
|
+
const passed = verdicts.filter((verdict) => verdict === "passed").length;
|
|
174
|
+
const failed = verdicts.filter((verdict) => verdict === "failed").length;
|
|
175
|
+
const errored = verdicts.filter((verdict) => verdict === "errored").length;
|
|
176
|
+
const skipped = verdicts.filter((verdict) => verdict === "skipped").length;
|
|
177
|
+
const ran = verdicts.length - skipped;
|
|
178
|
+
const costs = snapshot.attempts.map((attempt) => attemptCostUSD(attempt.result)).filter((cost): cost is number => cost !== null);
|
|
179
|
+
rows.push([
|
|
180
|
+
clipDisplay(snapshot.experimentId, Math.max(12, Math.floor(width * 0.24))),
|
|
181
|
+
snapshot.agent,
|
|
182
|
+
snapshot.model ?? MISSING,
|
|
183
|
+
ran === 0 ? MISSING : `${Math.round((passed / ran) * 1000) / 10}% ${passed}/${ran}`,
|
|
184
|
+
String(failed),
|
|
185
|
+
String(errored),
|
|
186
|
+
String(skipped),
|
|
187
|
+
formatDurationMs(average(snapshot.attempts.map((attempt) => attempt.result.durationMs))),
|
|
188
|
+
costs.length === 0 ? MISSING : formatUSD(average(costs)),
|
|
189
|
+
]);
|
|
190
|
+
}
|
|
191
|
+
return renderAlignedRows(rows, ["left", "left", "left", "right", "right", "right", "right", "right", "right"]);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** docs/feature/reports/show.md 的默认结果索引;只服务裸 show,不是可替换报告组件。 */
|
|
195
|
+
export function showIndexText(selection: Selection, width: number): string {
|
|
196
|
+
const blocks: string[] = [];
|
|
197
|
+
const warnings = warningText(selection, width);
|
|
198
|
+
if (warnings) blocks.push(warnings);
|
|
199
|
+
if (selection.snapshots.length >= 2) blocks.push(`COMPARISON\n${comparisonTable(selection.snapshots, width)}`);
|
|
200
|
+
for (const snapshot of selection.snapshots) {
|
|
201
|
+
blocks.push(
|
|
202
|
+
[
|
|
203
|
+
labeledBlock("EXPERIMENT", `${snapshot.experimentId} · ${snapshot.agent}${snapshot.model ? ` · ${snapshot.model}` : ""}`, width),
|
|
204
|
+
labeledBlock("SUMMARY", snapshotSummary(snapshot), width),
|
|
205
|
+
"",
|
|
206
|
+
experimentAttemptTable(snapshot, width),
|
|
207
|
+
].join("\n"),
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
blocks.push(labeledBlock("DRILL DOWN", "niceeval show @<attempt> [--eval | --execution | --diff]", width));
|
|
211
|
+
return blocks.join("\n\n");
|
|
212
|
+
}
|
|
213
|
+
|
|
45
214
|
function attemptsLabel(n: number): string {
|
|
46
215
|
return `${n} ${n === 1 ? "attempt" : "attempts"}`;
|
|
47
216
|
}
|
|
48
217
|
|
|
49
|
-
/** attempt 的展示序号:落盘 attempt 从 0 计,人看 1 计(
|
|
218
|
+
/** attempt 的展示序号:落盘 attempt 从 0 计,人看 1 计( artifact 目录后缀 __2 = attempt 3)。 */
|
|
50
219
|
export function displayAttemptNumber(attempt: AttemptHandle): number {
|
|
51
220
|
return attempt.result.attempt + 1;
|
|
52
221
|
}
|
|
53
222
|
|
|
54
|
-
/**
|
|
55
|
-
|
|
223
|
+
/**
|
|
224
|
+
* attempt artifact 目录的展示路径:尽量给相对 cwd 的短路径,出了 cwd 给绝对路径。
|
|
225
|
+
* 本快照跑出的条目:artifact 与 result.json 同目录,= `<snapshot.dir>/<ref.attempt>`。
|
|
226
|
+
* 携带条目(--resume 合入):落盘的 artifactBase 相对结果根(= snapshot.dir 的上两级,
|
|
227
|
+
* 即 `<experiment-dir>/<snapshot-dir>` 的上一层),指向原快照的 attempt 目录。
|
|
228
|
+
*/
|
|
229
|
+
export function attemptArtifactsPath(attempt: AttemptHandle, cwd: string): string {
|
|
56
230
|
const r = attempt.result;
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
if (!abs) return undefined;
|
|
231
|
+
const abs = r.artifactBase
|
|
232
|
+
? join(attempt.snapshot.dir, "..", "..", r.artifactBase)
|
|
233
|
+
: join(attempt.snapshot.dir, attempt.ref.attempt);
|
|
61
234
|
const rel = relative(cwd, abs);
|
|
62
235
|
return rel.startsWith("..") ? abs : rel;
|
|
63
236
|
}
|
|
64
237
|
|
|
238
|
+
/**
|
|
239
|
+
* 裸 `show` 零可读结果时,skipped 目录的展示文案。niceeval 自己写的、schemaVersion 不兼容的
|
|
240
|
+
* 落盘按 producer 版本分组,一条 `npx niceeval@<version> show --run <结果根>` 覆盖同版本全部
|
|
241
|
+
* 快照——show 没有 view 的单快照直读模式,`--run` 认的是结果根(其下可以有多个 experiment),
|
|
242
|
+
* 不是单个快照目录,所以不对每个目录重复拼一条各自的命令(那条命令用同一个 root 跑起来
|
|
243
|
+
* 结果完全一样,重复只会刷屏)。其余(第三方 harness、版本信息缺失、malformed、incomplete)
|
|
244
|
+
* 没有可执行的统一建议,原样逐条列出。
|
|
245
|
+
*/
|
|
246
|
+
export function skippedRunsText(skipped: readonly SkippedDir[], root: string, cwd: string): string {
|
|
247
|
+
const { groups, rest } = groupIncompatibleVersionSkips(skipped);
|
|
248
|
+
const rootDisplay = relative(cwd, root) || ".";
|
|
249
|
+
const lines: string[] = [];
|
|
250
|
+
for (const g of groups) {
|
|
251
|
+
const count = g.dirs.length;
|
|
252
|
+
const version = g.producer.version;
|
|
253
|
+
const schema = g.schemaVersion !== undefined ? ` (schemaVersion ${g.schemaVersion})` : "";
|
|
254
|
+
const cmd = `npx niceeval@${version} show --run ${rootDisplay}`;
|
|
255
|
+
lines.push(
|
|
256
|
+
` ${count} snapshot${count === 1 ? "" : "s"} written by niceeval ${version}${schema} — run \`${cmd}\` to view`,
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
for (const s of rest) {
|
|
260
|
+
lines.push(` skipped ${s.dir} (${s.reason})`);
|
|
261
|
+
}
|
|
262
|
+
return lines.join("\n");
|
|
263
|
+
}
|
|
264
|
+
|
|
65
265
|
// ───────────────────────── attempt 挑选 ─────────────────────────
|
|
66
266
|
|
|
67
|
-
/**
|
|
267
|
+
/** Selection 内某道题的全部 attempt(合成 Selection 里每实验只剩最新判定)。 */
|
|
68
268
|
export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHandle[] {
|
|
69
269
|
const out: AttemptHandle[] = [];
|
|
70
270
|
for (const snapshot of snapshots) {
|
|
@@ -75,22 +275,16 @@ export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHa
|
|
|
75
275
|
}
|
|
76
276
|
|
|
77
277
|
/**
|
|
78
|
-
*
|
|
79
|
-
*
|
|
278
|
+
* 详情块 / eval-id 前缀证据切面默认挑最新一次失败的 attempt;没有失败挑最新一次。
|
|
279
|
+
* 精确选某一次 attempt 走 `@<locator>`(`resolveLocator`),不再有数字 `--attempt`——
|
|
280
|
+
* --experiment 已在 Selection 合成时收窄。
|
|
80
281
|
*/
|
|
81
|
-
export function pickDetailAttempt(
|
|
82
|
-
attempts: AttemptHandle[],
|
|
83
|
-
attemptNumber?: number,
|
|
84
|
-
): AttemptHandle | undefined {
|
|
282
|
+
export function pickDetailAttempt(attempts: AttemptHandle[]): AttemptHandle | undefined {
|
|
85
283
|
if (attempts.length === 0) return undefined;
|
|
86
284
|
const byTime = [...attempts].sort((a, b) =>
|
|
87
285
|
(a.result.startedAt ?? "").localeCompare(b.result.startedAt ?? "") || a.result.attempt - b.result.attempt,
|
|
88
286
|
);
|
|
89
|
-
|
|
90
|
-
const wanted = byTime.filter((a) => a.result.attempt === attemptNumber - 1);
|
|
91
|
-
return wanted.at(-1);
|
|
92
|
-
}
|
|
93
|
-
const failing = byTime.filter((a) => a.result.outcome === "failed" || a.result.outcome === "errored");
|
|
287
|
+
const failing = byTime.filter((a) => a.result.verdict === "failed" || a.result.verdict === "errored");
|
|
94
288
|
return failing.at(-1) ?? byTime.at(-1);
|
|
95
289
|
}
|
|
96
290
|
|
|
@@ -102,22 +296,53 @@ function scoreText(score: number): string {
|
|
|
102
296
|
|
|
103
297
|
/** 断言行:✓/✗ + severity + name;gate 失败带 detail,soft 恒带 score/1(失败再补 detail)。 */
|
|
104
298
|
export function assertionLine(a: AssertionResult): string {
|
|
105
|
-
const
|
|
299
|
+
const scope = a.group ? `${a.group} · ` : "";
|
|
300
|
+
const head = `${a.passed ? "✓" : "✗"} ${a.severity} ${scope}${a.name}`;
|
|
106
301
|
if (a.severity === "soft") {
|
|
107
302
|
const detail = !a.passed && a.detail ? `: ${a.detail}` : "";
|
|
108
303
|
return `${head} — ${scoreText(a.score)}/1${detail}`;
|
|
109
304
|
}
|
|
110
|
-
if (!a.passed)
|
|
305
|
+
if (!a.passed) {
|
|
306
|
+
const reason = a.detail ?? `score ${scoreText(a.score)}`;
|
|
307
|
+
return a.evidence !== undefined ? `${head} — ${reason} · actual: ${a.evidence}` : `${head} — ${reason}`;
|
|
308
|
+
}
|
|
111
309
|
return head;
|
|
112
310
|
}
|
|
113
311
|
|
|
312
|
+
function equalsExpected(name: string): string | undefined {
|
|
313
|
+
const match = /^equals\((.*)\)$/.exec(name);
|
|
314
|
+
return match?.[1];
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** 默认 Attempt 页的首要诊断:不要求用户再猜一次 evidence flag 才知道为何失败。 */
|
|
318
|
+
export function failureDiagnostics(assertions: AssertionResult[], width: number): string | undefined {
|
|
319
|
+
const failed = assertions.filter((a) => !a.passed);
|
|
320
|
+
if (failed.length === 0) return undefined;
|
|
321
|
+
const lines = ["failures:"];
|
|
322
|
+
for (const a of failed) {
|
|
323
|
+
const label = a.group ?? a.name;
|
|
324
|
+
lines.push(` ${a.severity} · ${label}`);
|
|
325
|
+
if (a.group) lines.push(` assertion: ${a.name}`);
|
|
326
|
+
const expected = equalsExpected(a.name);
|
|
327
|
+
if (expected !== undefined) lines.push(` expected: ${expected}`);
|
|
328
|
+
if (a.evidence !== undefined) lines.push(` received: ${a.evidence}`);
|
|
329
|
+
if (a.detail) lines.push(` reason: ${a.detail}`);
|
|
330
|
+
if (a.loc) lines.push(` source: ${a.loc.file}:${a.loc.line}${a.loc.column ? `:${a.loc.column}` : ""}`);
|
|
331
|
+
if (a.severity === "soft") lines.push(` score: ${scoreText(a.score)}/1`);
|
|
332
|
+
}
|
|
333
|
+
return lines.flatMap((line) => {
|
|
334
|
+
const indent = line.length - line.trimStart().length;
|
|
335
|
+
return wrapDisplay(line.trimStart(), Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`);
|
|
336
|
+
}).join("\n");
|
|
337
|
+
}
|
|
338
|
+
|
|
114
339
|
/** 详情块头部:`attempt 3 · compare/codex-gpt-5.4 · failed · 41s · 12.3k tokens · $0.04`。 */
|
|
115
340
|
export function attemptHeader(attempt: AttemptHandle): string {
|
|
116
341
|
const r = attempt.result;
|
|
117
342
|
const parts = [
|
|
118
343
|
`attempt ${displayAttemptNumber(attempt)}`,
|
|
119
344
|
attempt.experimentId,
|
|
120
|
-
r.
|
|
345
|
+
r.verdict,
|
|
121
346
|
formatDurationMs(r.durationMs),
|
|
122
347
|
];
|
|
123
348
|
if (r.usage) parts.push(`${formatMetricValue(r.usage.inputTokens + r.usage.outputTokens)} tokens`);
|
|
@@ -126,6 +351,95 @@ export function attemptHeader(attempt: AttemptHandle): string {
|
|
|
126
351
|
return parts.join(" · ");
|
|
127
352
|
}
|
|
128
353
|
|
|
354
|
+
// ───────────────────────── AttemptEvidence 共用 ─────────────────────────
|
|
355
|
+
// evalSourceText / executionText / attemptOverviewText(--eval / --execution / 默认全景)
|
|
356
|
+
// 与 evalDetailText 的紧凑索引列共用的小件:locator 头、capability 字母、失败原因、
|
|
357
|
+
// 断言计票摘要。三个证据 renderer 与全景面都只消费同一份 AttemptEvidence,不各自读
|
|
358
|
+
// artifact 或重新判定 capability(loadAttemptEvidence 已经算好)。
|
|
359
|
+
|
|
360
|
+
/** `--eval` / `--execution` / 全景块的头行:`@<locator> · <evalId> · <experimentId> · <verdict>`。 */
|
|
361
|
+
export function attemptEvidenceHeader(evidence: AttemptEvidence): string {
|
|
362
|
+
return [evidence.locator, evidence.identity.evalId, evidence.identity.experimentId, evidence.result.verdict].join(" · ");
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** capability 位 → 展示字母,只列为 true 的位;全部为 false 时返回空串(不打空 `[]`)。 */
|
|
366
|
+
export function capabilityLetters(capabilities: AttemptEvidenceCapabilities): string {
|
|
367
|
+
const letters: string[] = [];
|
|
368
|
+
if (capabilities.eval) letters.push("E");
|
|
369
|
+
if (capabilities.execution) letters.push("X");
|
|
370
|
+
if (capabilities.timing) letters.push("⏱");
|
|
371
|
+
if (capabilities.diff) letters.push("D");
|
|
372
|
+
return letters.length > 0 ? `[${letters.join(",")}]` : "";
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* 紧凑多 attempt 索引(裸 `niceeval show`、单 eval 多 experiment 行、evidence flag 撞多个
|
|
377
|
+
* eval 时的消歧提示)用的廉价 capability 估算:只读瘦身 `EvalResult` 字段(`hasSources` /
|
|
378
|
+
* `hasEvents` / `hasTrace`),不为了点亮一个字母去读 artifact —— 与
|
|
379
|
+
* `report/compute.ts::attemptRow` 的 `hasEvidence` 同一个「只认瘦身字段」的口径(它同样只用
|
|
380
|
+
* `hasEvents` / `hasTrace`,同样不读 diff.json)。`diff` 位没有对应的瘦身字段(不像
|
|
381
|
+
* events/trace/sources,写入面从不给 diff 算一个 has* 布尔),而 diff.json 可达上百 MB,
|
|
382
|
+
* 不该为了这一个字母在渲染路径上打开它(见 docs/feature/reports/architecture.md「计算与
|
|
383
|
+
* 渲染分离」)——因此这里恒报 `false`;只有 `loadAttemptEvidence()` real 出的
|
|
384
|
+
* capabilities(单 attempt 路径,如 `@<locator>` 全景)才知道真实答案。
|
|
385
|
+
*/
|
|
386
|
+
export function cheapCapabilities(result: EvalResult): AttemptEvidenceCapabilities {
|
|
387
|
+
return {
|
|
388
|
+
eval: result.hasSources === true,
|
|
389
|
+
execution: result.hasEvents === true,
|
|
390
|
+
timing: result.hasEvents === true && result.hasTrace === true,
|
|
391
|
+
diff: false,
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* 一次 attempt 未通过的判定原因,单行、不含 detail——供紧凑索引行使用。precedence 与
|
|
397
|
+
* `report/compute.ts::reasonFor` 同一条规则(error → skipReason → 未通过的 gate 断言,
|
|
398
|
+
* soft 永不进入),但格式更短(`gate <name>`,不带 `: <detail>`——detail 留给
|
|
399
|
+
* `--eval`/单 eval 详情块的完整断言明细),这里独立实现而不是导入 report/ 的函数:
|
|
400
|
+
* report 包正被并行重写(见 plan/attempt-evidence-feedback-loop.md),show 的紧凑索引不应
|
|
401
|
+
* 依赖它的内部实现细节。
|
|
402
|
+
*/
|
|
403
|
+
export function verdictReasonLine(result: EvalResult): string | undefined {
|
|
404
|
+
if (result.error !== undefined) return result.error;
|
|
405
|
+
if (result.skipReason !== undefined) return result.skipReason;
|
|
406
|
+
const gates = result.assertions.filter((a) => !a.passed && a.severity === "gate");
|
|
407
|
+
if (gates.length === 0) return undefined;
|
|
408
|
+
return gates.map((a) => `gate ${a.name}`).join(", ");
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** 紧凑多 attempt 索引的一行:`✗ weather/brooklyn @7K2M9Q[E,X,⏱] gate calledTool(...)`。 */
|
|
412
|
+
export function attemptIndexLine(opts: {
|
|
413
|
+
evalId: string;
|
|
414
|
+
verdict: Verdict;
|
|
415
|
+
locator: string | undefined;
|
|
416
|
+
capabilities: AttemptEvidenceCapabilities;
|
|
417
|
+
reason?: string;
|
|
418
|
+
}): string {
|
|
419
|
+
const loc = opts.locator ? `${opts.locator}${capabilityLetters(opts.capabilities)}` : MISSING;
|
|
420
|
+
const parts = [`${verdictMark(opts.verdict)} ${opts.evalId}`, loc];
|
|
421
|
+
if (opts.reason) parts.push(opts.reason);
|
|
422
|
+
return parts.join(" ");
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* 断言计票摘要,`--eval` 与全景面共用:`assertions: 1 passed · 1 gate failed · 1 soft below
|
|
427
|
+
* target`。直接读 `EvalResult.assertions`(恒可用的瘦身字段)而不是
|
|
428
|
+
* `AnnotatedEvalSource.summary`——两者对同一批断言算出的计票恒等(`AnnotatedEvalSource` 正是
|
|
429
|
+
* 用这同一个数组喂 `buildAnnotatedEvalSource` 的),但 `evalSource` 可能是 `null`
|
|
430
|
+
* (未捕获源码),读 `result.assertions` 让这条摘要不因缺源码而跟着消失。
|
|
431
|
+
*/
|
|
432
|
+
export function assertionSummaryLine(assertions: AssertionResult[]): string {
|
|
433
|
+
const passed = assertions.filter((a) => a.passed).length;
|
|
434
|
+
const gateFailed = assertions.filter((a) => !a.passed && a.severity === "gate").length;
|
|
435
|
+
const softBelow = assertions.filter((a) => !a.passed && a.severity === "soft").length;
|
|
436
|
+
const parts: string[] = [];
|
|
437
|
+
if (passed > 0) parts.push(`${passed} passed`);
|
|
438
|
+
if (gateFailed > 0) parts.push(`${gateFailed} gate failed`);
|
|
439
|
+
if (softBelow > 0) parts.push(`${softBelow} soft below target`);
|
|
440
|
+
return `assertions: ${parts.length > 0 ? parts.join(" · ") : "(none)"}`;
|
|
441
|
+
}
|
|
442
|
+
|
|
129
443
|
export interface EvalDetailOptions {
|
|
130
444
|
evalId: string;
|
|
131
445
|
snapshots: Snapshot[];
|
|
@@ -136,7 +450,7 @@ export interface EvalDetailOptions {
|
|
|
136
450
|
width: number;
|
|
137
451
|
}
|
|
138
452
|
|
|
139
|
-
/** `niceeval show <eval id>`:各 experiment
|
|
453
|
+
/** `niceeval show <eval id>`:各 experiment 的判定行 + 默认 attempt 的断言明细。 */
|
|
140
454
|
export function evalDetailText(opts: EvalDetailOptions): string {
|
|
141
455
|
const { evalId, snapshots, detail, cwd, now, width } = opts;
|
|
142
456
|
const blocks: string[] = [];
|
|
@@ -148,25 +462,32 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
148
462
|
.find((d) => d !== undefined);
|
|
149
463
|
blocks.push(description ? `${evalId} — ${description}` : evalId);
|
|
150
464
|
|
|
151
|
-
// 每 experiment
|
|
465
|
+
// 每 experiment 一行:折叠判定、attempt 数、最新 attempt 的耗时、总成本、判定时间、
|
|
466
|
+
// 代表 attempt 的紧凑索引(locator + capability 字母 + 失败原因)——agent 从这张榜单
|
|
467
|
+
// 就能直接摘到一个 `@<locator>` 下钻,不必先跑一遍 `--eval`/`--execution` 才知道选谁。
|
|
152
468
|
const rows: string[][] = [];
|
|
153
469
|
for (const snapshot of snapshots) {
|
|
154
470
|
const ev = snapshot.evals.find((e) => e.id === evalId);
|
|
155
471
|
if (!ev || ev.attempts.length === 0) continue;
|
|
156
|
-
const
|
|
472
|
+
const verdict = foldEvalVerdict(ev.attempts.map((a) => a.result));
|
|
157
473
|
const latest = ev.attempts.reduce((a, b) => (b.result.attempt >= a.result.attempt ? b : a));
|
|
158
474
|
let cost: number | null = null;
|
|
159
475
|
for (const attempt of ev.attempts) {
|
|
160
476
|
const c = attemptCostUSD(attempt.result);
|
|
161
477
|
if (c !== null) cost = (cost ?? 0) + c;
|
|
162
478
|
}
|
|
479
|
+
const rep = pickDetailAttempt(ev.attempts);
|
|
480
|
+
const locatorCell = rep?.locator ? `${rep.locator}${capabilityLetters(cheapCapabilities(rep.result))}` : MISSING;
|
|
481
|
+
const reasonCell = rep ? (verdictReasonLine(rep.result) ?? "") : "";
|
|
163
482
|
rows.push([
|
|
164
483
|
snapshot.experimentId,
|
|
165
|
-
`${
|
|
484
|
+
`${verdictMark(verdict)} ${verdict}`,
|
|
166
485
|
attemptsLabel(ev.attempts.length),
|
|
167
486
|
formatDurationMs(latest.result.durationMs),
|
|
168
487
|
cost === null ? MISSING : formatUSD(cost),
|
|
169
488
|
`(${relativeAgo(latest.result.startedAt ?? snapshot.startedAt, now)})`,
|
|
489
|
+
locatorCell,
|
|
490
|
+
reasonCell,
|
|
170
491
|
]);
|
|
171
492
|
}
|
|
172
493
|
if (rows.length > 0) blocks.push(renderAlignedRows(rows));
|
|
@@ -186,8 +507,13 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
186
507
|
|
|
187
508
|
const tail: string[] = [];
|
|
188
509
|
const artifacts = attemptArtifactsPath(detail, cwd);
|
|
189
|
-
if (artifacts) tail.push(`artifacts: ${artifacts}/`);
|
|
190
|
-
tail.push(`
|
|
510
|
+
if ( artifacts) tail.push(`artifacts: ${artifacts}/`);
|
|
511
|
+
if (detail.locator) tail.push(`attempt locator: ${detail.locator}`);
|
|
512
|
+
tail.push(
|
|
513
|
+
detail.locator
|
|
514
|
+
? `next: niceeval show ${detail.locator} [--eval|--execution|--diff]`
|
|
515
|
+
: `next: niceeval show ${evalId} [--eval|--execution|--diff]`,
|
|
516
|
+
);
|
|
191
517
|
blocks.push(tail.join("\n"));
|
|
192
518
|
}
|
|
193
519
|
|
|
@@ -204,7 +530,7 @@ export function evalHistoryText(opts: {
|
|
|
204
530
|
rows: EvalHistoryRow[];
|
|
205
531
|
}): string {
|
|
206
532
|
const { experimentId, evalId, rows } = opts;
|
|
207
|
-
const passed = rows.filter((r) => r.
|
|
533
|
+
const passed = rows.filter((r) => r.verdict === "passed").length;
|
|
208
534
|
const head = [
|
|
209
535
|
...(evalId ? [evalId] : []),
|
|
210
536
|
experimentId,
|
|
@@ -215,7 +541,7 @@ export function evalHistoryText(opts: {
|
|
|
215
541
|
const table = renderAlignedRows(
|
|
216
542
|
rows.map((r) => [
|
|
217
543
|
timelineStamp(r.startedAt),
|
|
218
|
-
`${
|
|
544
|
+
`${verdictMark(r.verdict)} ${r.verdict}`,
|
|
219
545
|
attemptsLabel(r.attempts),
|
|
220
546
|
r.costUSD === null ? MISSING : formatUSD(r.costUSD),
|
|
221
547
|
r.failedAssertion ?? (r.error ? `error: ${r.error}` : ""),
|
|
@@ -238,7 +564,7 @@ export function experimentHistoryText(experimentId: string, rows: ExperimentHist
|
|
|
238
564
|
return `${head}\n\n${indentBlock(table, " ")}`;
|
|
239
565
|
}
|
|
240
566
|
|
|
241
|
-
// ─────────────────────────
|
|
567
|
+
// ───────────────────────── 截断预算(--eval / --execution / 全景共用) ─────────────────────────
|
|
242
568
|
|
|
243
569
|
const MAX_EVENTS = 80;
|
|
244
570
|
const MAX_TEXT = 600;
|
|
@@ -258,146 +584,320 @@ function jsonPreview(value: unknown, max = 200): string {
|
|
|
258
584
|
return clip(text, max);
|
|
259
585
|
}
|
|
260
586
|
|
|
261
|
-
|
|
587
|
+
function recordOf(value: unknown): Record<string, unknown> | undefined {
|
|
588
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
function indentedText(text: string, width: number, indent = 4, maxLines = 18): string[] {
|
|
592
|
+
const source = clip(text).split("\n");
|
|
593
|
+
const shown = source.slice(0, maxLines);
|
|
594
|
+
const lines = shown.flatMap((line) => wrapDisplay(line || " ", Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`));
|
|
595
|
+
if (source.length > shown.length) lines.push(`${" ".repeat(indent)}… (+${source.length - shown.length} more lines)`);
|
|
596
|
+
return lines;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// ───────────────────────── 证据切面:--eval(Eval 源码标注) ─────────────────────────
|
|
262
600
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
601
|
+
const MAX_SOURCE_LINES = 400;
|
|
602
|
+
|
|
603
|
+
/** gate 失败 / soft 恒带分——与 assertionLine 的严重度口径一致,但不带断言 name(源码行本身就是名字)。 */
|
|
604
|
+
function evalAssertionDetailLine(a: AssertionResult): string | undefined {
|
|
605
|
+
if (a.severity === "soft") {
|
|
606
|
+
const detail = !a.passed && a.detail ? ` · ${a.detail}` : "";
|
|
607
|
+
return `soft · ${scoreText(a.score)}/1${detail}`;
|
|
608
|
+
}
|
|
609
|
+
if (!a.passed) {
|
|
610
|
+
const parts = ["gate"];
|
|
611
|
+
if (a.group) parts.push(a.group);
|
|
612
|
+
parts.push(a.name);
|
|
613
|
+
const expected = equalsExpected(a.name);
|
|
614
|
+
if (expected !== undefined) parts.push(`expected ${expected}`);
|
|
615
|
+
if (a.evidence !== undefined) parts.push(`received ${a.evidence}`);
|
|
616
|
+
if (a.detail) parts.push(a.detail);
|
|
617
|
+
return parts.join(" · ");
|
|
618
|
+
}
|
|
619
|
+
return undefined;
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
function evalSourceLineText(line: AnnotatedSourceLine, gutterWidth: number, width: number): string[] {
|
|
623
|
+
const anyFailed = line.assertions.some((a) => !a.passed);
|
|
624
|
+
const glyph = line.assertions.length === 0 ? " " : anyFailed ? "✗" : "✓";
|
|
625
|
+
const marginWidth = gutterWidth + 2; // 行号列 + glyph + 分隔空格
|
|
626
|
+
const prefix = `${padDisplay(String(line.line), gutterWidth)}${glyph} `;
|
|
627
|
+
// 源码行的空白(尤其是缩进)是语义的一部分:wrapDisplay 按单词重排会把连续空格
|
|
628
|
+
// 吃成一个、把缩进整体丢掉(agent-feedback-loop.mdx 的示例明确保留 2/4/6 格嵌套缩进)。
|
|
629
|
+
// 过长的源码行只裁一刀(clip,与其它证据切面的截断口径一致),不按词折成多行——
|
|
630
|
+
// 折行只对续行加统一 margin,救不回已经被吃掉的原始缩进,不如老实截断。
|
|
631
|
+
const out = [prefix + clip(line.text, Math.max(20, width - marginWidth))];
|
|
632
|
+
const margin = " ".repeat(marginWidth);
|
|
633
|
+
for (const a of line.assertions) {
|
|
634
|
+
const detail = evalAssertionDetailLine(a);
|
|
635
|
+
if (detail === undefined) continue;
|
|
636
|
+
for (const wrapped2 of wrapDisplay(detail, Math.max(20, width - marginWidth))) out.push(margin + wrapped2);
|
|
637
|
+
}
|
|
638
|
+
return out;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* `--eval`:运行时保存的 Eval 源码,gate/soft 断言标回源码行,外加 unmapped 断言(永不丢弃)
|
|
643
|
+
* 与断言计票摘要。`evidence.evalSource === null` 时如实说明源码未捕获,不伪造空文档。
|
|
644
|
+
*/
|
|
645
|
+
export function evalSourceText(
|
|
646
|
+
evidence: AttemptEvidence,
|
|
647
|
+
opts: { header: string; artifactPath?: string; width: number },
|
|
648
|
+
): string {
|
|
649
|
+
const { header, artifactPath, width } = opts;
|
|
650
|
+
const source = evidence.evalSource;
|
|
651
|
+
const artifact = artifactPath ? join(artifactPath, "sources.json") : undefined;
|
|
652
|
+
if (!source) {
|
|
653
|
+
return `${header}\n\n(eval source unavailable for this attempt${artifact ? ` · expected: ${artifact}` : ""})`;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
const blocks: string[] = [header, `eval source: ${source.sourcePath} · sha256:${source.sourceSha256.slice(0, 8)}…`];
|
|
657
|
+
|
|
658
|
+
const gutterWidth = String(source.lines.length).length;
|
|
659
|
+
const shownLines = source.lines.slice(0, MAX_SOURCE_LINES);
|
|
660
|
+
const lineLines = shownLines.flatMap((line) => evalSourceLineText(line, gutterWidth, width));
|
|
661
|
+
if (source.lines.length > shownLines.length) {
|
|
662
|
+
lineLines.push(`(${source.lines.length - shownLines.length} more lines not shown)`);
|
|
663
|
+
}
|
|
664
|
+
blocks.push(lineLines.join("\n"));
|
|
665
|
+
|
|
666
|
+
if (source.unmapped.length > 0) {
|
|
667
|
+
const unmappedLines = source.unmapped.map((a) => indentBlock(wrapDisplay(assertionLine(a), width - 2).join("\n"), " "));
|
|
668
|
+
blocks.push(
|
|
669
|
+
[`unmapped assertions (${source.unmapped.length}, no source location):`, ...unmappedLines].join("\n"),
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
blocks.push(assertionSummaryLine(evidence.result.assertions));
|
|
674
|
+
if (artifact) blocks.push(`full eval source: ${artifact}`);
|
|
675
|
+
return blocks.join("\n\n");
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
// ───────────────────────── 证据切面:--execution(标准事件流 + OTel enrichment) ─────────────────────────
|
|
679
|
+
|
|
680
|
+
const EXEC_LABEL_PAD = 12;
|
|
681
|
+
const EXEC_TIME_PAD = 6;
|
|
682
|
+
|
|
683
|
+
function relSeconds(ms: number, originMs: number): string {
|
|
684
|
+
return `${((ms - originMs) / 1000).toFixed(1)}s`;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/** 一个 ExecutionNode 的一行(或折行后多行);时间列只在整棵树 timingAvailable 时出现。 */
|
|
688
|
+
function execBody(
|
|
689
|
+
time: string | undefined,
|
|
690
|
+
label: string,
|
|
691
|
+
text: string,
|
|
692
|
+
duration: string | undefined,
|
|
693
|
+
width: number,
|
|
694
|
+
timingAvailable: boolean,
|
|
695
|
+
): string[] {
|
|
696
|
+
const timeCol = timingAvailable ? padDisplay(time ?? "", EXEC_TIME_PAD) + " " : "";
|
|
697
|
+
const marginWidth = timeCol.length + EXEC_LABEL_PAD;
|
|
698
|
+
const wrapped = wrapDisplay(clip(text), Math.max(20, width - marginWidth));
|
|
699
|
+
return wrapped.map((line, i) => {
|
|
700
|
+
if (i !== 0) return " ".repeat(marginWidth) + line;
|
|
701
|
+
const head = timeCol + padDisplay(label, EXEC_LABEL_PAD) + line;
|
|
702
|
+
return duration ? `${head} ${duration}` : head;
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* action / subagent 节点渲染成两行(call + result)——节点模型把 call+result 合并成一个
|
|
708
|
+
* ExecutionNode(按 callId),但分两行读更符合「先看调用、再看结果」的阅读顺序,
|
|
709
|
+
* 与 docs-site/zh/guides/agent-feedback-loop.mdx 的示例一致。
|
|
710
|
+
*/
|
|
711
|
+
function executionNodeLines(node: ExecutionNode, originMs: number, timingAvailable: boolean, width: number): string[] {
|
|
712
|
+
const time = node.kind !== "telemetry" && node.span ? relSeconds(node.span.startMs, originMs) : undefined;
|
|
713
|
+
const duration = node.kind !== "telemetry" && node.span ? formatDurationMs(node.span.endMs - node.span.startMs) : undefined;
|
|
714
|
+
const meta = [time, duration].filter(Boolean).join(" · ");
|
|
715
|
+
const block = (title: string, body: string, extra: string[] = []): string[] => {
|
|
716
|
+
const heading = meta ? `${title} ${meta}` : title;
|
|
717
|
+
const bodyLines = wrapDisplay(clip(body), Math.max(20, width - 2)).map((line) => ` ${line}`);
|
|
718
|
+
return [heading, ...bodyLines, ...extra.flatMap((line) => wrapDisplay(line, Math.max(20, width - 2)).map((part) => ` ${part}`))];
|
|
267
719
|
};
|
|
268
|
-
|
|
720
|
+
|
|
721
|
+
switch (node.kind) {
|
|
269
722
|
case "message":
|
|
270
|
-
return
|
|
723
|
+
return block(node.role.toUpperCase(), node.text);
|
|
271
724
|
case "thinking":
|
|
272
|
-
return
|
|
273
|
-
case "
|
|
274
|
-
return
|
|
275
|
-
case "action
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
725
|
+
return block("THINKING", clip(node.text, 200));
|
|
726
|
+
case "skill.loaded":
|
|
727
|
+
return block(`SKILL · ${node.skill}`, "loaded");
|
|
728
|
+
case "action": {
|
|
729
|
+
const input = recordOf(node.input);
|
|
730
|
+
const output = recordOf(node.output);
|
|
731
|
+
const inputText = typeof input?.command === "string" ? input.command : jsonPreview(node.input);
|
|
732
|
+
const outputText = typeof output?.output === "string" ? output.output : node.output !== undefined ? jsonPreview(node.output) : "(no result)";
|
|
733
|
+
const exit = typeof output?.exit_code === "number" ? ` · exit ${output.exit_code}` : "";
|
|
734
|
+
return [
|
|
735
|
+
meta ? `TOOL · ${node.name} ${meta}` : `TOOL · ${node.name}`,
|
|
736
|
+
" input",
|
|
737
|
+
...indentedText(inputText, width),
|
|
738
|
+
` result · ${node.status}${exit}`,
|
|
739
|
+
...indentedText(outputText, width),
|
|
740
|
+
];
|
|
741
|
+
}
|
|
742
|
+
case "subagent": {
|
|
743
|
+
return block(`SUBAGENT · ${node.name}`, node.output === undefined ? node.status : `result · ${node.status}: ${jsonPreview(node.output)}`);
|
|
744
|
+
}
|
|
281
745
|
case "input.requested":
|
|
282
|
-
return
|
|
746
|
+
return block("INPUT REQUESTED", node.request.prompt ?? node.request.display ?? "(input requested)");
|
|
283
747
|
case "compaction":
|
|
284
|
-
return
|
|
748
|
+
return block("COMPACTION", node.reason ?? "context compacted");
|
|
285
749
|
case "error":
|
|
286
|
-
return
|
|
750
|
+
return block("ERROR", node.message);
|
|
751
|
+
case "telemetry":
|
|
752
|
+
return execBody(
|
|
753
|
+
relSeconds(node.span.startMs, originMs),
|
|
754
|
+
"telemetry",
|
|
755
|
+
node.span.name || node.span.kind || "span",
|
|
756
|
+
formatDurationMs(node.span.endMs - node.span.startMs),
|
|
757
|
+
width,
|
|
758
|
+
true,
|
|
759
|
+
);
|
|
287
760
|
}
|
|
288
761
|
}
|
|
289
762
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
763
|
+
/**
|
|
764
|
+
* `--execution`:标准事件流骨架(message / thinking / skill load / tool call+result / subagent /
|
|
765
|
+
* input.requested / compaction / error),有 OTel 时同一节点补相对时间与耗时;没有 OTel 时
|
|
766
|
+
* 节点、顺序与内容不变,只去掉时间列,并在结尾如实标 timing unavailable
|
|
767
|
+
* (ExecutionTree 的契约:骨架不因时间有无而变形,见 o11y/execution-tree.ts 头注)。
|
|
768
|
+
*/
|
|
769
|
+
export function executionText(
|
|
770
|
+
evidence: AttemptEvidence,
|
|
771
|
+
opts: { header: string; artifactPath?: string; width: number },
|
|
772
|
+
): string {
|
|
773
|
+
const { header, artifactPath, width } = opts;
|
|
774
|
+
const tree = evidence.execution;
|
|
775
|
+
const eventsSource = artifactPath ? join(artifactPath, "events.json") : undefined;
|
|
776
|
+
if (!tree) {
|
|
777
|
+
return `${header}\n\n(no events recorded for this attempt${eventsSource ? ` · expected: ${eventsSource}` : ""})`;
|
|
300
778
|
}
|
|
301
|
-
|
|
302
|
-
const
|
|
303
|
-
const
|
|
304
|
-
const
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
779
|
+
|
|
780
|
+
const timingAvailable = tree.timingAvailable;
|
|
781
|
+
const spanStarts = tree.nodes.flatMap((n) => (n.span ? [n.span.startMs] : []));
|
|
782
|
+
const originMs = spanStarts.length > 0 ? Math.min(...spanStarts) : 0;
|
|
783
|
+
|
|
784
|
+
// `--execution` 回答 Agent 做了什么。未关联到标准事件的原始 spans 属于 trace 证据,
|
|
785
|
+
// 逐条混入 transcript 会把几十条 SDK 内部 span 盖过消息和工具调用。
|
|
786
|
+
const agentNodes = tree.nodes.filter((node) => node.kind !== "telemetry");
|
|
787
|
+
const telemetryCount = tree.nodes.length - agentNodes.length;
|
|
788
|
+
const shown = agentNodes.slice(0, MAX_EVENTS);
|
|
789
|
+
const lines = shown.flatMap((node, index) => [
|
|
790
|
+
...(index === 0 ? [] : [""]),
|
|
791
|
+
...executionNodeLines(node, originMs, timingAvailable, width),
|
|
792
|
+
]);
|
|
793
|
+
|
|
794
|
+
const tail: string[] = [];
|
|
795
|
+
if (timingAvailable) {
|
|
796
|
+
const skillLoads = tree.nodes.filter((n) => n.kind === "skill.loaded").length;
|
|
797
|
+
const toolCalls = tree.nodes.filter((n) => n.kind === "action").length;
|
|
798
|
+
const aiMessages = tree.nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
|
|
799
|
+
tail.push(
|
|
800
|
+
[
|
|
801
|
+
`total ${formatDurationMs(evidence.result.durationMs)}`,
|
|
802
|
+
`${skillLoads} skill ${skillLoads === 1 ? "load" : "loads"}`,
|
|
803
|
+
`${toolCalls} tool ${toolCalls === 1 ? "call" : "calls"}`,
|
|
804
|
+
`${aiMessages} AI ${aiMessages === 1 ? "message" : "messages"}`,
|
|
805
|
+
...(agentNodes.length > shown.length ? [`${agentNodes.length - shown.length} more events not shown`] : []),
|
|
806
|
+
].join(" · "),
|
|
807
|
+
);
|
|
808
|
+
} else {
|
|
809
|
+
tail.push(
|
|
810
|
+
[
|
|
811
|
+
"timing unavailable · OTel trace was not collected",
|
|
812
|
+
...(agentNodes.length > shown.length ? [`${agentNodes.length - shown.length} more events not shown`] : []),
|
|
813
|
+
].join(" · "),
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
if (eventsSource) tail.push(`full events: ${eventsSource}`);
|
|
817
|
+
if (telemetryCount > 0) tail.push(`${telemetryCount} unlinked telemetry spans omitted; inspect the OTel trace for framework timing.`);
|
|
818
|
+
if (timingAvailable && artifactPath) tail.push(`full OTel trace: ${join(artifactPath, "trace.json")}`);
|
|
819
|
+
|
|
820
|
+
return `${header}\n\n${lines.join("\n")}\n\n${tail.join("\n")}`;
|
|
311
821
|
}
|
|
312
822
|
|
|
313
|
-
// ─────────────────────────
|
|
823
|
+
// ───────────────────────── 默认全景:AttemptEvidence 紧凑总览 ─────────────────────────
|
|
314
824
|
|
|
315
|
-
const
|
|
316
|
-
const TRACE_BAR_WIDTH = 20;
|
|
825
|
+
const MAX_OVERVIEW_DIFF_NAMES = 5;
|
|
317
826
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
const tool = span.attributes?.["tool_name"];
|
|
327
|
-
return typeof tool === "string" ? `execute_tool ${tool}` : "execute_tool";
|
|
328
|
-
}
|
|
329
|
-
case "agent":
|
|
330
|
-
return "invoke_agent";
|
|
331
|
-
default:
|
|
332
|
-
return span.name || "other";
|
|
333
|
-
}
|
|
827
|
+
function overviewDiffLine(diff: DiffData): string {
|
|
828
|
+
const names = [
|
|
829
|
+
...Object.keys(diff.generatedFiles).sort().map((p) => `M ${p}`),
|
|
830
|
+
...[...diff.deletedFiles].sort().map((p) => `D ${p}`),
|
|
831
|
+
];
|
|
832
|
+
const shown = names.slice(0, MAX_OVERVIEW_DIFF_NAMES);
|
|
833
|
+
const more = names.length > shown.length ? ` · +${names.length - shown.length} more` : "";
|
|
834
|
+
return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed · ${shown.join(", ")}${more}`;
|
|
334
835
|
}
|
|
335
836
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
837
|
+
/**
|
|
838
|
+
* `niceeval show @<locator>` 不带证据 flag 时的默认面:紧凑全景,只给摘要——Eval 断言计票、
|
|
839
|
+
* 执行事件计数、可选 OTel 时间指示、工作区 diff 摘要,不复现 `--eval` 的完整源码、
|
|
840
|
+
* `--execution` 的完整事件流或 `--diff` 的完整文件列表(那些内容各自的证据 flag 才给)。
|
|
841
|
+
*/
|
|
842
|
+
export function attemptOverviewText(
|
|
843
|
+
evidence: AttemptEvidence,
|
|
844
|
+
opts: { header: string; artifactPath?: string; width: number },
|
|
845
|
+
): string {
|
|
846
|
+
const { header, artifactPath } = opts;
|
|
847
|
+
const r = evidence.result;
|
|
848
|
+
|
|
849
|
+
const metaParts = [`snapshot ${evidence.identity.snapshotStartedAt}`, `attempt ${evidence.identity.attempt + 1}`, formatDurationMs(r.durationMs)];
|
|
850
|
+
if (r.usage) metaParts.push(`${formatMetricValue(r.usage.inputTokens + r.usage.outputTokens)} tokens`);
|
|
851
|
+
const cost = attemptCostUSD(r);
|
|
852
|
+
if (cost !== null) metaParts.push(formatUSD(cost));
|
|
342
853
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
}
|
|
854
|
+
const blocks: string[] = [[header, metaParts.join(" · ")].join("\n")];
|
|
855
|
+
|
|
856
|
+
blocks.push(
|
|
857
|
+
[
|
|
858
|
+
assertionSummaryLine(r.assertions),
|
|
859
|
+
evidence.evalSource
|
|
860
|
+
? `eval source: ${evidence.evalSource.sourcePath} · sha256:${evidence.evalSource.sourceSha256.slice(0, 8)}…`
|
|
861
|
+
: "eval source: unavailable (not captured for this attempt)",
|
|
862
|
+
].join("\n"),
|
|
863
|
+
);
|
|
354
864
|
|
|
355
|
-
const
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
roots.push(span);
|
|
865
|
+
const diagnostics = failureDiagnostics(r.assertions, opts.width);
|
|
866
|
+
if (diagnostics) blocks.push(diagnostics);
|
|
867
|
+
|
|
868
|
+
if (evidence.execution) {
|
|
869
|
+
const nodes = evidence.execution.nodes.filter((node) => node.kind !== "telemetry");
|
|
870
|
+
const skillLoads = nodes.filter((n) => n.kind === "skill.loaded").length;
|
|
871
|
+
const toolCalls = nodes.filter((n) => n.kind === "action").length;
|
|
872
|
+
const aiMessages = nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
|
|
873
|
+
const execLines = [`execution: ${nodes.length} events · ${skillLoads} skill loads · ${toolCalls} tool calls · ${aiMessages} AI messages`];
|
|
874
|
+
if (evidence.capabilities.timing) {
|
|
875
|
+
execLines.push("timing: OTel spans recorded for this attempt — see --execution for per-step timing.");
|
|
367
876
|
}
|
|
877
|
+
blocks.push(execLines.join("\n"));
|
|
878
|
+
} else {
|
|
879
|
+
blocks.push("execution: unavailable (no events recorded for this attempt)");
|
|
368
880
|
}
|
|
369
|
-
const byStart = (a: TraceSpan, b: TraceSpan) => a.startMs - b.startMs;
|
|
370
|
-
roots.sort(byStart);
|
|
371
|
-
for (const list of children.values()) list.sort(byStart);
|
|
372
|
-
|
|
373
|
-
const rows: { label: string; span: TraceSpan }[] = [];
|
|
374
|
-
const walk = (span: TraceSpan, prefix: string, childPrefix: string) => {
|
|
375
|
-
rows.push({ label: prefix + spanLabel(span), span });
|
|
376
|
-
const kids = children.get(span.spanId) ?? [];
|
|
377
|
-
kids.forEach((kid, i) => {
|
|
378
|
-
const last = i === kids.length - 1;
|
|
379
|
-
walk(kid, childPrefix + (last ? "└─ " : "├─ "), childPrefix + (last ? " " : "│ "));
|
|
380
|
-
});
|
|
381
|
-
};
|
|
382
|
-
for (const root of roots) walk(root, "", "");
|
|
383
|
-
|
|
384
|
-
const shown = rows.slice(0, MAX_SPANS);
|
|
385
|
-
const labelWidth = Math.max(...shown.map((r) => r.label.length));
|
|
386
|
-
const lines = shown.map(
|
|
387
|
-
(r) =>
|
|
388
|
-
`${padDisplay(r.label, labelWidth)} ${traceBar(r.span.startMs, r.span.endMs, windowStart, windowEnd)} ${formatDurationMs(
|
|
389
|
-
r.span.endMs - r.span.startMs,
|
|
390
|
-
)}${r.span.status === "error" ? " ✗" : ""}`,
|
|
391
|
-
);
|
|
392
881
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
882
|
+
if (evidence.diff && evidence.capabilities.diff) {
|
|
883
|
+
blocks.push(overviewDiffLine(evidence.diff));
|
|
884
|
+
} else if (evidence.diff) {
|
|
885
|
+
blocks.push("changes: diff unavailable · this attempt did not produce workspace file changes");
|
|
886
|
+
} else {
|
|
887
|
+
blocks.push("changes: diff unavailable · no workspace diff was recorded for this attempt");
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
const tail: string[] = [];
|
|
891
|
+
if (artifactPath) tail.push(`artifacts: ${artifactPath}/`);
|
|
892
|
+
const available = [
|
|
893
|
+
evidence.capabilities.eval ? `niceeval show ${evidence.locator} --eval` : undefined,
|
|
894
|
+
evidence.capabilities.execution ? `niceeval show ${evidence.locator} --execution` : undefined,
|
|
895
|
+
evidence.capabilities.diff ? `niceeval show ${evidence.locator} --diff` : undefined,
|
|
896
|
+
].filter((command): command is string => command !== undefined);
|
|
897
|
+
if (available.length > 0) tail.push(`available:\n${available.map((command) => ` ${command}`).join("\n")}`);
|
|
898
|
+
if (tail.length > 0) blocks.push(tail.join("\n"));
|
|
899
|
+
|
|
900
|
+
return blocks.join("\n\n");
|
|
401
901
|
}
|
|
402
902
|
|
|
403
903
|
// ───────────────────────── 证据切面:diff ─────────────────────────
|
|
@@ -412,7 +912,7 @@ export function diffText(opts: {
|
|
|
412
912
|
file?: string;
|
|
413
913
|
}): string {
|
|
414
914
|
const { header, diff, artifactPath, file } = opts;
|
|
415
|
-
const source = artifactPath ? join(artifactPath, "diff.json") : undefined;
|
|
915
|
+
const source = artifactPath ? join( artifactPath, "diff.json") : undefined;
|
|
416
916
|
if (!diff) {
|
|
417
917
|
return `${header}\n\n(no diff recorded for this attempt${source ? ` · expected: ${source}` : ""})`;
|
|
418
918
|
}
|