niceeval 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +321 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
- package/dist/report/built-ins/experiment-comparison.js +13 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +180 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +600 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +26 -0
- package/dist/report/format.js +109 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +94 -0
- package/dist/report/locale.js +183 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +19 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +11 -0
- package/dist/report/react/ExperimentList.js +67 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +96 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +398 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +541 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +408 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +7 -7
- package/docs-site/zh/concepts/assert.mdx +19 -18
- package/docs-site/zh/concepts/evals.mdx +12 -11
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +3 -3
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +4 -2
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +4 -4
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +61 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
- package/docs-site/zh/guides/viewing-results.mdx +200 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +4 -3
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +15 -13
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +9 -3
- package/docs-site/zh/reference/expect.mdx +13 -8
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +59 -3
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +33 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +640 -0
- package/src/report/built-ins/experiment-comparison.tsx +19 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +225 -50
- package/src/report/compute.ts +335 -54
- package/src/report/dual-render.test.tsx +1129 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +35 -2
- package/src/report/index.ts +47 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +229 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +113 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +220 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +126 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +168 -0
- package/src/report/react/fixtures.ts +169 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +21 -6
- package/src/report/react/render.test.tsx +180 -31
- package/src/report/react/styles.css +432 -213
- package/src/report/report.test.ts +398 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +344 -96
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +184 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +119 -95
- package/src/show/render.ts +462 -159
- package/src/show/show.test.ts +471 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/show/render.ts
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
// show 的文本渲染:单 eval 详情、--history 时间轴、三个证据切面(transcript / trace / diff)。
|
|
2
2
|
// 输出形态照 docs-site/zh/guides/viewing-results.mdx 的示例块;长内容一律截断,
|
|
3
|
-
//
|
|
3
|
+
// 但截断永远如实标注剩余数量和原始 artifact 路径 —— 输出对上下文窗口友好,事实源留在盘上。
|
|
4
4
|
// 全部纯函数(时间经 now 显式传入),证据数据由调用方 await 好了递进来。
|
|
5
5
|
|
|
6
6
|
import { join, relative } from "node:path";
|
|
7
|
-
import type { AssertionResult,
|
|
8
|
-
import type {
|
|
9
|
-
import type {
|
|
10
|
-
import {
|
|
7
|
+
import type { AssertionResult, DiffData, EvalResult, Verdict } from "../types.ts";
|
|
8
|
+
import type { AttemptEvidence, AttemptHandle, Snapshot } from "../results/index.ts";
|
|
9
|
+
import type { AnnotatedSourceLine } from "../results/index.ts";
|
|
10
|
+
import { groupIncompatibleVersionSkips } from "../results/index.ts";
|
|
11
|
+
import type { SkippedDir } from "../results/index.ts";
|
|
12
|
+
import type { ExecutionNode } from "../o11y/execution-tree.ts";
|
|
13
|
+
import { foldEvalVerdict } from "../shared/verdict.ts";
|
|
11
14
|
import { attemptCostUSD } from "../report/metrics.ts";
|
|
12
15
|
import { formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD } from "../report/format.ts";
|
|
13
16
|
import { indentBlock, padDisplay, renderAlignedRows, wrapDisplay } from "../report/text/layout.ts";
|
|
@@ -17,7 +20,7 @@ const MISSING = "—";
|
|
|
17
20
|
|
|
18
21
|
// ───────────────────────── 时间与小件 ─────────────────────────
|
|
19
22
|
|
|
20
|
-
/**
|
|
23
|
+
/** 判定时间的相对标注("just now" / "41s ago" / "2h ago");未来时刻按 just now 兜底。 */
|
|
21
24
|
export function relativeAgo(iso: string | undefined, now: number): string {
|
|
22
25
|
if (!iso) return "";
|
|
23
26
|
const ms = now - Date.parse(iso);
|
|
@@ -36,9 +39,9 @@ export function timelineStamp(iso: string): string {
|
|
|
36
39
|
return iso.slice(0, 16).replace(/:/g, "-");
|
|
37
40
|
}
|
|
38
41
|
|
|
39
|
-
function
|
|
40
|
-
if (
|
|
41
|
-
if (
|
|
42
|
+
function verdictMark(verdict: Verdict): string {
|
|
43
|
+
if (verdict === "passed") return "✓";
|
|
44
|
+
if (verdict === "skipped") return "-";
|
|
42
45
|
return "✗";
|
|
43
46
|
}
|
|
44
47
|
|
|
@@ -46,25 +49,56 @@ function attemptsLabel(n: number): string {
|
|
|
46
49
|
return `${n} ${n === 1 ? "attempt" : "attempts"}`;
|
|
47
50
|
}
|
|
48
51
|
|
|
49
|
-
/** attempt 的展示序号:落盘 attempt 从 0 计,人看 1 计(
|
|
52
|
+
/** attempt 的展示序号:落盘 attempt 从 0 计,人看 1 计( artifact 目录后缀 __2 = attempt 3)。 */
|
|
50
53
|
export function displayAttemptNumber(attempt: AttemptHandle): number {
|
|
51
54
|
return attempt.result.attempt + 1;
|
|
52
55
|
}
|
|
53
56
|
|
|
54
|
-
/**
|
|
55
|
-
|
|
57
|
+
/**
|
|
58
|
+
* attempt artifact 目录的展示路径:尽量给相对 cwd 的短路径,出了 cwd 给绝对路径。
|
|
59
|
+
* 本快照跑出的条目:artifact 与 result.json 同目录,= `<snapshot.dir>/<ref.attempt>`。
|
|
60
|
+
* 携带条目(--resume 合入):落盘的 artifactBase 相对结果根(= snapshot.dir 的上两级,
|
|
61
|
+
* 即 `<experiment-dir>/<snapshot-dir>` 的上一层),指向原快照的 attempt 目录。
|
|
62
|
+
*/
|
|
63
|
+
export function attemptArtifactsPath(attempt: AttemptHandle, cwd: string): string {
|
|
56
64
|
const r = attempt.result;
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
if (!abs) return undefined;
|
|
65
|
+
const abs = r.artifactBase
|
|
66
|
+
? join(attempt.snapshot.dir, "..", "..", r.artifactBase)
|
|
67
|
+
: join(attempt.snapshot.dir, attempt.ref.attempt);
|
|
61
68
|
const rel = relative(cwd, abs);
|
|
62
69
|
return rel.startsWith("..") ? abs : rel;
|
|
63
70
|
}
|
|
64
71
|
|
|
72
|
+
/**
|
|
73
|
+
* 裸 `show` 零可读结果时,skipped 目录的展示文案。niceeval 自己写的、schemaVersion 不兼容的
|
|
74
|
+
* 落盘按 producer 版本分组,一条 `npx niceeval@<version> show --run <结果根>` 覆盖同版本全部
|
|
75
|
+
* 快照——show 没有 view 的单快照直读模式,`--run` 认的是结果根(其下可以有多个 experiment),
|
|
76
|
+
* 不是单个快照目录,所以不对每个目录重复拼一条各自的命令(那条命令用同一个 root 跑起来
|
|
77
|
+
* 结果完全一样,重复只会刷屏)。其余(第三方 harness、版本信息缺失、malformed、incomplete)
|
|
78
|
+
* 没有可执行的统一建议,原样逐条列出。
|
|
79
|
+
*/
|
|
80
|
+
export function skippedRunsText(skipped: readonly SkippedDir[], root: string, cwd: string): string {
|
|
81
|
+
const { groups, rest } = groupIncompatibleVersionSkips(skipped);
|
|
82
|
+
const rootDisplay = relative(cwd, root) || ".";
|
|
83
|
+
const lines: string[] = [];
|
|
84
|
+
for (const g of groups) {
|
|
85
|
+
const count = g.dirs.length;
|
|
86
|
+
const version = g.producer.version;
|
|
87
|
+
const schema = g.schemaVersion !== undefined ? ` (schemaVersion ${g.schemaVersion})` : "";
|
|
88
|
+
const cmd = `npx niceeval@${version} show --run ${rootDisplay}`;
|
|
89
|
+
lines.push(
|
|
90
|
+
` ${count} snapshot${count === 1 ? "" : "s"} written by niceeval ${version}${schema} — run \`${cmd}\` to view`,
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
for (const s of rest) {
|
|
94
|
+
lines.push(` skipped ${s.dir} (${s.reason})`);
|
|
95
|
+
}
|
|
96
|
+
return lines.join("\n");
|
|
97
|
+
}
|
|
98
|
+
|
|
65
99
|
// ───────────────────────── attempt 挑选 ─────────────────────────
|
|
66
100
|
|
|
67
|
-
/**
|
|
101
|
+
/** Selection 内某道题的全部 attempt(合成 Selection 里每实验只剩最新判定)。 */
|
|
68
102
|
export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHandle[] {
|
|
69
103
|
const out: AttemptHandle[] = [];
|
|
70
104
|
for (const snapshot of snapshots) {
|
|
@@ -75,22 +109,16 @@ export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHa
|
|
|
75
109
|
}
|
|
76
110
|
|
|
77
111
|
/**
|
|
78
|
-
*
|
|
79
|
-
*
|
|
112
|
+
* 详情块 / eval-id 前缀证据切面默认挑最新一次失败的 attempt;没有失败挑最新一次。
|
|
113
|
+
* 精确选某一次 attempt 走 `@<locator>`(`resolveLocator`),不再有数字 `--attempt`——
|
|
114
|
+
* --experiment 已在 Selection 合成时收窄。
|
|
80
115
|
*/
|
|
81
|
-
export function pickDetailAttempt(
|
|
82
|
-
attempts: AttemptHandle[],
|
|
83
|
-
attemptNumber?: number,
|
|
84
|
-
): AttemptHandle | undefined {
|
|
116
|
+
export function pickDetailAttempt(attempts: AttemptHandle[]): AttemptHandle | undefined {
|
|
85
117
|
if (attempts.length === 0) return undefined;
|
|
86
118
|
const byTime = [...attempts].sort((a, b) =>
|
|
87
119
|
(a.result.startedAt ?? "").localeCompare(b.result.startedAt ?? "") || a.result.attempt - b.result.attempt,
|
|
88
120
|
);
|
|
89
|
-
|
|
90
|
-
const wanted = byTime.filter((a) => a.result.attempt === attemptNumber - 1);
|
|
91
|
-
return wanted.at(-1);
|
|
92
|
-
}
|
|
93
|
-
const failing = byTime.filter((a) => a.result.outcome === "failed" || a.result.outcome === "errored");
|
|
121
|
+
const failing = byTime.filter((a) => a.result.verdict === "failed" || a.result.verdict === "errored");
|
|
94
122
|
return failing.at(-1) ?? byTime.at(-1);
|
|
95
123
|
}
|
|
96
124
|
|
|
@@ -102,22 +130,53 @@ function scoreText(score: number): string {
|
|
|
102
130
|
|
|
103
131
|
/** 断言行:✓/✗ + severity + name;gate 失败带 detail,soft 恒带 score/1(失败再补 detail)。 */
|
|
104
132
|
export function assertionLine(a: AssertionResult): string {
|
|
105
|
-
const
|
|
133
|
+
const scope = a.group ? `${a.group} · ` : "";
|
|
134
|
+
const head = `${a.passed ? "✓" : "✗"} ${a.severity} ${scope}${a.name}`;
|
|
106
135
|
if (a.severity === "soft") {
|
|
107
136
|
const detail = !a.passed && a.detail ? `: ${a.detail}` : "";
|
|
108
137
|
return `${head} — ${scoreText(a.score)}/1${detail}`;
|
|
109
138
|
}
|
|
110
|
-
if (!a.passed)
|
|
139
|
+
if (!a.passed) {
|
|
140
|
+
const reason = a.detail ?? `score ${scoreText(a.score)}`;
|
|
141
|
+
return a.evidence !== undefined ? `${head} — ${reason} · actual: ${a.evidence}` : `${head} — ${reason}`;
|
|
142
|
+
}
|
|
111
143
|
return head;
|
|
112
144
|
}
|
|
113
145
|
|
|
146
|
+
function equalsExpected(name: string): string | undefined {
|
|
147
|
+
const match = /^equals\((.*)\)$/.exec(name);
|
|
148
|
+
return match?.[1];
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** 默认 Attempt 页的首要诊断:不要求用户再猜一次 evidence flag 才知道为何失败。 */
|
|
152
|
+
export function failureDiagnostics(assertions: AssertionResult[], width: number): string | undefined {
|
|
153
|
+
const failed = assertions.filter((a) => !a.passed);
|
|
154
|
+
if (failed.length === 0) return undefined;
|
|
155
|
+
const lines = ["failures:"];
|
|
156
|
+
for (const a of failed) {
|
|
157
|
+
const label = a.group ?? a.name;
|
|
158
|
+
lines.push(` ${a.severity} · ${label}`);
|
|
159
|
+
if (a.group) lines.push(` assertion: ${a.name}`);
|
|
160
|
+
const expected = equalsExpected(a.name);
|
|
161
|
+
if (expected !== undefined) lines.push(` expected: ${expected}`);
|
|
162
|
+
if (a.evidence !== undefined) lines.push(` received: ${a.evidence}`);
|
|
163
|
+
if (a.detail) lines.push(` reason: ${a.detail}`);
|
|
164
|
+
if (a.loc) lines.push(` source: ${a.loc.file}:${a.loc.line}${a.loc.column ? `:${a.loc.column}` : ""}`);
|
|
165
|
+
if (a.severity === "soft") lines.push(` score: ${scoreText(a.score)}/1`);
|
|
166
|
+
}
|
|
167
|
+
return lines.flatMap((line) => {
|
|
168
|
+
const indent = line.length - line.trimStart().length;
|
|
169
|
+
return wrapDisplay(line.trimStart(), Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`);
|
|
170
|
+
}).join("\n");
|
|
171
|
+
}
|
|
172
|
+
|
|
114
173
|
/** 详情块头部:`attempt 3 · compare/codex-gpt-5.4 · failed · 41s · 12.3k tokens · $0.04`。 */
|
|
115
174
|
export function attemptHeader(attempt: AttemptHandle): string {
|
|
116
175
|
const r = attempt.result;
|
|
117
176
|
const parts = [
|
|
118
177
|
`attempt ${displayAttemptNumber(attempt)}`,
|
|
119
178
|
attempt.experimentId,
|
|
120
|
-
r.
|
|
179
|
+
r.verdict,
|
|
121
180
|
formatDurationMs(r.durationMs),
|
|
122
181
|
];
|
|
123
182
|
if (r.usage) parts.push(`${formatMetricValue(r.usage.inputTokens + r.usage.outputTokens)} tokens`);
|
|
@@ -126,6 +185,64 @@ export function attemptHeader(attempt: AttemptHandle): string {
|
|
|
126
185
|
return parts.join(" · ");
|
|
127
186
|
}
|
|
128
187
|
|
|
188
|
+
// ───────────────────────── AttemptEvidence 共用 ─────────────────────────
|
|
189
|
+
// evalSourceText / executionText / attemptOverviewText(--eval / --execution / 默认全景)
|
|
190
|
+
// 与 evalDetailText 的紧凑索引列共用的小件:locator 头、失败原因、
|
|
191
|
+
// 断言计票摘要。三个证据 renderer 与全景面都只消费同一份 AttemptEvidence,不各自读
|
|
192
|
+
// artifact 或重新判定 capability(loadAttemptEvidence 已经算好)。
|
|
193
|
+
|
|
194
|
+
/** `--eval` / `--execution` / 全景块的头行:`@<locator> · <evalId> · <experimentId> · <verdict>`。 */
|
|
195
|
+
export function attemptEvidenceHeader(evidence: AttemptEvidence): string {
|
|
196
|
+
return [evidence.locator, evidence.identity.evalId, evidence.identity.experimentId, evidence.result.verdict].join(" · ");
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* 一次 attempt 未通过的判定原因,单行、不含 detail——供紧凑索引行使用。precedence 与
|
|
201
|
+
* `report/compute.ts::reasonFor` 同一条规则(error → skipReason → 未通过的 gate 断言,
|
|
202
|
+
* soft 永不进入),但格式更短(`gate <name>`,不带 `: <detail>`——detail 留给
|
|
203
|
+
* `--eval`/单 eval 详情块的完整断言明细),这里独立实现而不是导入 report/ 的函数:
|
|
204
|
+
* report 包正被并行重写(见 plan/attempt-evidence-feedback-loop.md),show 的紧凑索引不应
|
|
205
|
+
* 依赖它的内部实现细节。
|
|
206
|
+
*/
|
|
207
|
+
export function verdictReasonLine(result: EvalResult): string | undefined {
|
|
208
|
+
if (result.error !== undefined) return result.error;
|
|
209
|
+
if (result.skipReason !== undefined) return result.skipReason;
|
|
210
|
+
const gates = result.assertions.filter((a) => !a.passed && a.severity === "gate");
|
|
211
|
+
if (gates.length === 0) return undefined;
|
|
212
|
+
return gates.map((a) => `gate ${a.name}`).join(", ");
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** 紧凑多 attempt 索引的一行:`✗ weather/brooklyn @7K2M9Q gate calledTool(...)`。 */
|
|
216
|
+
export function attemptIndexLine(opts: {
|
|
217
|
+
evalId: string;
|
|
218
|
+
verdict: Verdict;
|
|
219
|
+
locator: string | undefined;
|
|
220
|
+
reason?: string;
|
|
221
|
+
}): string {
|
|
222
|
+
const loc = opts.locator ?? MISSING;
|
|
223
|
+
const parts = [`${verdictMark(opts.verdict)} ${opts.evalId}`, loc];
|
|
224
|
+
if (opts.reason) parts.push(opts.reason);
|
|
225
|
+
return parts.join(" ");
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* 断言计票摘要,`--eval` 与全景面共用:`assertions: 1 passed · 1 gate failed · 1 soft below
|
|
230
|
+
* target`。直接读 `EvalResult.assertions`(恒可用的瘦身字段)而不是
|
|
231
|
+
* `AnnotatedEvalSource.summary`——两者对同一批断言算出的计票恒等(`AnnotatedEvalSource` 正是
|
|
232
|
+
* 用这同一个数组喂 `buildAnnotatedEvalSource` 的),但 `evalSource` 可能是 `null`
|
|
233
|
+
* (未捕获源码),读 `result.assertions` 让这条摘要不因缺源码而跟着消失。
|
|
234
|
+
*/
|
|
235
|
+
export function assertionSummaryLine(assertions: AssertionResult[]): string {
|
|
236
|
+
const passed = assertions.filter((a) => a.passed).length;
|
|
237
|
+
const gateFailed = assertions.filter((a) => !a.passed && a.severity === "gate").length;
|
|
238
|
+
const softBelow = assertions.filter((a) => !a.passed && a.severity === "soft").length;
|
|
239
|
+
const parts: string[] = [];
|
|
240
|
+
if (passed > 0) parts.push(`${passed} passed`);
|
|
241
|
+
if (gateFailed > 0) parts.push(`${gateFailed} gate failed`);
|
|
242
|
+
if (softBelow > 0) parts.push(`${softBelow} soft below target`);
|
|
243
|
+
return `assertions: ${parts.length > 0 ? parts.join(" · ") : "(none)"}`;
|
|
244
|
+
}
|
|
245
|
+
|
|
129
246
|
export interface EvalDetailOptions {
|
|
130
247
|
evalId: string;
|
|
131
248
|
snapshots: Snapshot[];
|
|
@@ -136,7 +253,7 @@ export interface EvalDetailOptions {
|
|
|
136
253
|
width: number;
|
|
137
254
|
}
|
|
138
255
|
|
|
139
|
-
/** `niceeval show <eval id>`:各 experiment
|
|
256
|
+
/** `niceeval show <eval id>`:各 experiment 的判定行 + 默认 attempt 的断言明细。 */
|
|
140
257
|
export function evalDetailText(opts: EvalDetailOptions): string {
|
|
141
258
|
const { evalId, snapshots, detail, cwd, now, width } = opts;
|
|
142
259
|
const blocks: string[] = [];
|
|
@@ -148,25 +265,32 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
148
265
|
.find((d) => d !== undefined);
|
|
149
266
|
blocks.push(description ? `${evalId} — ${description}` : evalId);
|
|
150
267
|
|
|
151
|
-
// 每 experiment
|
|
268
|
+
// 每 experiment 一行:折叠判定、attempt 数、最新 attempt 的耗时、总成本、判定时间、
|
|
269
|
+
// 代表 attempt 的紧凑索引(locator + 失败原因)——agent 从这张榜单
|
|
270
|
+
// 就能直接摘到一个 `@<locator>` 下钻,不必先跑一遍 `--eval`/`--execution` 才知道选谁。
|
|
152
271
|
const rows: string[][] = [];
|
|
153
272
|
for (const snapshot of snapshots) {
|
|
154
273
|
const ev = snapshot.evals.find((e) => e.id === evalId);
|
|
155
274
|
if (!ev || ev.attempts.length === 0) continue;
|
|
156
|
-
const
|
|
275
|
+
const verdict = foldEvalVerdict(ev.attempts.map((a) => a.result));
|
|
157
276
|
const latest = ev.attempts.reduce((a, b) => (b.result.attempt >= a.result.attempt ? b : a));
|
|
158
277
|
let cost: number | null = null;
|
|
159
278
|
for (const attempt of ev.attempts) {
|
|
160
279
|
const c = attemptCostUSD(attempt.result);
|
|
161
280
|
if (c !== null) cost = (cost ?? 0) + c;
|
|
162
281
|
}
|
|
282
|
+
const rep = pickDetailAttempt(ev.attempts);
|
|
283
|
+
const locatorCell = rep?.locator ?? MISSING;
|
|
284
|
+
const reasonCell = rep ? (verdictReasonLine(rep.result) ?? "") : "";
|
|
163
285
|
rows.push([
|
|
164
286
|
snapshot.experimentId,
|
|
165
|
-
`${
|
|
287
|
+
`${verdictMark(verdict)} ${verdict}`,
|
|
166
288
|
attemptsLabel(ev.attempts.length),
|
|
167
289
|
formatDurationMs(latest.result.durationMs),
|
|
168
290
|
cost === null ? MISSING : formatUSD(cost),
|
|
169
291
|
`(${relativeAgo(latest.result.startedAt ?? snapshot.startedAt, now)})`,
|
|
292
|
+
locatorCell,
|
|
293
|
+
reasonCell,
|
|
170
294
|
]);
|
|
171
295
|
}
|
|
172
296
|
if (rows.length > 0) blocks.push(renderAlignedRows(rows));
|
|
@@ -186,8 +310,13 @@ export function evalDetailText(opts: EvalDetailOptions): string {
|
|
|
186
310
|
|
|
187
311
|
const tail: string[] = [];
|
|
188
312
|
const artifacts = attemptArtifactsPath(detail, cwd);
|
|
189
|
-
if (artifacts) tail.push(`artifacts: ${artifacts}/`);
|
|
190
|
-
tail.push(`
|
|
313
|
+
if ( artifacts) tail.push(`artifacts: ${artifacts}/`);
|
|
314
|
+
if (detail.locator) tail.push(`attempt locator: ${detail.locator}`);
|
|
315
|
+
tail.push(
|
|
316
|
+
detail.locator
|
|
317
|
+
? `next: niceeval show ${detail.locator} [--eval|--execution|--diff]`
|
|
318
|
+
: `next: niceeval show ${evalId} [--eval|--execution|--diff]`,
|
|
319
|
+
);
|
|
191
320
|
blocks.push(tail.join("\n"));
|
|
192
321
|
}
|
|
193
322
|
|
|
@@ -204,7 +333,7 @@ export function evalHistoryText(opts: {
|
|
|
204
333
|
rows: EvalHistoryRow[];
|
|
205
334
|
}): string {
|
|
206
335
|
const { experimentId, evalId, rows } = opts;
|
|
207
|
-
const passed = rows.filter((r) => r.
|
|
336
|
+
const passed = rows.filter((r) => r.verdict === "passed").length;
|
|
208
337
|
const head = [
|
|
209
338
|
...(evalId ? [evalId] : []),
|
|
210
339
|
experimentId,
|
|
@@ -215,7 +344,7 @@ export function evalHistoryText(opts: {
|
|
|
215
344
|
const table = renderAlignedRows(
|
|
216
345
|
rows.map((r) => [
|
|
217
346
|
timelineStamp(r.startedAt),
|
|
218
|
-
`${
|
|
347
|
+
`${verdictMark(r.verdict)} ${r.verdict}`,
|
|
219
348
|
attemptsLabel(r.attempts),
|
|
220
349
|
r.costUSD === null ? MISSING : formatUSD(r.costUSD),
|
|
221
350
|
r.failedAssertion ?? (r.error ? `error: ${r.error}` : ""),
|
|
@@ -238,7 +367,7 @@ export function experimentHistoryText(experimentId: string, rows: ExperimentHist
|
|
|
238
367
|
return `${head}\n\n${indentBlock(table, " ")}`;
|
|
239
368
|
}
|
|
240
369
|
|
|
241
|
-
// ─────────────────────────
|
|
370
|
+
// ───────────────────────── 截断预算(--eval / --execution / 全景共用) ─────────────────────────
|
|
242
371
|
|
|
243
372
|
const MAX_EVENTS = 80;
|
|
244
373
|
const MAX_TEXT = 600;
|
|
@@ -258,146 +387,320 @@ function jsonPreview(value: unknown, max = 200): string {
|
|
|
258
387
|
return clip(text, max);
|
|
259
388
|
}
|
|
260
389
|
|
|
261
|
-
|
|
390
|
+
function recordOf(value: unknown): Record<string, unknown> | undefined {
|
|
391
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function indentedText(text: string, width: number, indent = 4, maxLines = 18): string[] {
|
|
395
|
+
const source = clip(text).split("\n");
|
|
396
|
+
const shown = source.slice(0, maxLines);
|
|
397
|
+
const lines = shown.flatMap((line) => wrapDisplay(line || " ", Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`));
|
|
398
|
+
if (source.length > shown.length) lines.push(`${" ".repeat(indent)}… (+${source.length - shown.length} more lines)`);
|
|
399
|
+
return lines;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
// ───────────────────────── 证据切面:--eval(Eval 源码标注) ─────────────────────────
|
|
403
|
+
|
|
404
|
+
const MAX_SOURCE_LINES = 400;
|
|
405
|
+
|
|
406
|
+
/** gate 失败 / soft 恒带分——与 assertionLine 的严重度口径一致,但不带断言 name(源码行本身就是名字)。 */
|
|
407
|
+
function evalAssertionDetailLine(a: AssertionResult): string | undefined {
|
|
408
|
+
if (a.severity === "soft") {
|
|
409
|
+
const detail = !a.passed && a.detail ? ` · ${a.detail}` : "";
|
|
410
|
+
return `soft · ${scoreText(a.score)}/1${detail}`;
|
|
411
|
+
}
|
|
412
|
+
if (!a.passed) {
|
|
413
|
+
const parts = ["gate"];
|
|
414
|
+
if (a.group) parts.push(a.group);
|
|
415
|
+
parts.push(a.name);
|
|
416
|
+
const expected = equalsExpected(a.name);
|
|
417
|
+
if (expected !== undefined) parts.push(`expected ${expected}`);
|
|
418
|
+
if (a.evidence !== undefined) parts.push(`received ${a.evidence}`);
|
|
419
|
+
if (a.detail) parts.push(a.detail);
|
|
420
|
+
return parts.join(" · ");
|
|
421
|
+
}
|
|
422
|
+
return undefined;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
function evalSourceLineText(line: AnnotatedSourceLine, gutterWidth: number, width: number): string[] {
|
|
426
|
+
const anyFailed = line.assertions.some((a) => !a.passed);
|
|
427
|
+
const glyph = line.assertions.length === 0 ? " " : anyFailed ? "✗" : "✓";
|
|
428
|
+
const marginWidth = gutterWidth + 2; // 行号列 + glyph + 分隔空格
|
|
429
|
+
const prefix = `${padDisplay(String(line.line), gutterWidth)}${glyph} `;
|
|
430
|
+
// 源码行的空白(尤其是缩进)是语义的一部分:wrapDisplay 按单词重排会把连续空格
|
|
431
|
+
// 吃成一个、把缩进整体丢掉(agent-feedback-loop.mdx 的示例明确保留 2/4/6 格嵌套缩进)。
|
|
432
|
+
// 过长的源码行只裁一刀(clip,与其它证据切面的截断口径一致),不按词折成多行——
|
|
433
|
+
// 折行只对续行加统一 margin,救不回已经被吃掉的原始缩进,不如老实截断。
|
|
434
|
+
const out = [prefix + clip(line.text, Math.max(20, width - marginWidth))];
|
|
435
|
+
const margin = " ".repeat(marginWidth);
|
|
436
|
+
for (const a of line.assertions) {
|
|
437
|
+
const detail = evalAssertionDetailLine(a);
|
|
438
|
+
if (detail === undefined) continue;
|
|
439
|
+
for (const wrapped2 of wrapDisplay(detail, Math.max(20, width - marginWidth))) out.push(margin + wrapped2);
|
|
440
|
+
}
|
|
441
|
+
return out;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* `--eval`:运行时保存的 Eval 源码,gate/soft 断言标回源码行,外加 unmapped 断言(永不丢弃)
|
|
446
|
+
* 与断言计票摘要。`evidence.evalSource === null` 时如实说明源码未捕获,不伪造空文档。
|
|
447
|
+
*/
|
|
448
|
+
export function evalSourceText(
|
|
449
|
+
evidence: AttemptEvidence,
|
|
450
|
+
opts: { header: string; artifactPath?: string; width: number },
|
|
451
|
+
): string {
|
|
452
|
+
const { header, artifactPath, width } = opts;
|
|
453
|
+
const source = evidence.evalSource;
|
|
454
|
+
const artifact = artifactPath ? join(artifactPath, "sources.json") : undefined;
|
|
455
|
+
if (!source) {
|
|
456
|
+
return `${header}\n\n(eval source unavailable for this attempt${artifact ? ` · expected: ${artifact}` : ""})`;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
const blocks: string[] = [header, `eval source: ${source.sourcePath} · sha256:${source.sourceSha256.slice(0, 8)}…`];
|
|
460
|
+
|
|
461
|
+
const gutterWidth = String(source.lines.length).length;
|
|
462
|
+
const shownLines = source.lines.slice(0, MAX_SOURCE_LINES);
|
|
463
|
+
const lineLines = shownLines.flatMap((line) => evalSourceLineText(line, gutterWidth, width));
|
|
464
|
+
if (source.lines.length > shownLines.length) {
|
|
465
|
+
lineLines.push(`(${source.lines.length - shownLines.length} more lines not shown)`);
|
|
466
|
+
}
|
|
467
|
+
blocks.push(lineLines.join("\n"));
|
|
468
|
+
|
|
469
|
+
if (source.unmapped.length > 0) {
|
|
470
|
+
const unmappedLines = source.unmapped.map((a) => indentBlock(wrapDisplay(assertionLine(a), width - 2).join("\n"), " "));
|
|
471
|
+
blocks.push(
|
|
472
|
+
[`unmapped assertions (${source.unmapped.length}, no source location):`, ...unmappedLines].join("\n"),
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
blocks.push(assertionSummaryLine(evidence.result.assertions));
|
|
477
|
+
if (artifact) blocks.push(`full eval source: ${artifact}`);
|
|
478
|
+
return blocks.join("\n\n");
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// ───────────────────────── 证据切面:--execution(标准事件流 + OTel enrichment) ─────────────────────────
|
|
262
482
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
483
|
+
const EXEC_LABEL_PAD = 12;
|
|
484
|
+
const EXEC_TIME_PAD = 6;
|
|
485
|
+
|
|
486
|
+
function relSeconds(ms: number, originMs: number): string {
|
|
487
|
+
return `${((ms - originMs) / 1000).toFixed(1)}s`;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/** 一个 ExecutionNode 的一行(或折行后多行);时间列只在整棵树 timingAvailable 时出现。 */
|
|
491
|
+
function execBody(
|
|
492
|
+
time: string | undefined,
|
|
493
|
+
label: string,
|
|
494
|
+
text: string,
|
|
495
|
+
duration: string | undefined,
|
|
496
|
+
width: number,
|
|
497
|
+
timingAvailable: boolean,
|
|
498
|
+
): string[] {
|
|
499
|
+
const timeCol = timingAvailable ? padDisplay(time ?? "", EXEC_TIME_PAD) + " " : "";
|
|
500
|
+
const marginWidth = timeCol.length + EXEC_LABEL_PAD;
|
|
501
|
+
const wrapped = wrapDisplay(clip(text), Math.max(20, width - marginWidth));
|
|
502
|
+
return wrapped.map((line, i) => {
|
|
503
|
+
if (i !== 0) return " ".repeat(marginWidth) + line;
|
|
504
|
+
const head = timeCol + padDisplay(label, EXEC_LABEL_PAD) + line;
|
|
505
|
+
return duration ? `${head} ${duration}` : head;
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* action / subagent 节点渲染成两行(call + result)——节点模型把 call+result 合并成一个
|
|
511
|
+
* ExecutionNode(按 callId),但分两行读更符合「先看调用、再看结果」的阅读顺序,
|
|
512
|
+
* 与 docs-site/zh/guides/agent-feedback-loop.mdx 的示例一致。
|
|
513
|
+
*/
|
|
514
|
+
function executionNodeLines(node: ExecutionNode, originMs: number, timingAvailable: boolean, width: number): string[] {
|
|
515
|
+
const time = node.kind !== "telemetry" && node.span ? relSeconds(node.span.startMs, originMs) : undefined;
|
|
516
|
+
const duration = node.kind !== "telemetry" && node.span ? formatDurationMs(node.span.endMs - node.span.startMs) : undefined;
|
|
517
|
+
const meta = [time, duration].filter(Boolean).join(" · ");
|
|
518
|
+
const block = (title: string, body: string, extra: string[] = []): string[] => {
|
|
519
|
+
const heading = meta ? `${title} ${meta}` : title;
|
|
520
|
+
const bodyLines = wrapDisplay(clip(body), Math.max(20, width - 2)).map((line) => ` ${line}`);
|
|
521
|
+
return [heading, ...bodyLines, ...extra.flatMap((line) => wrapDisplay(line, Math.max(20, width - 2)).map((part) => ` ${part}`))];
|
|
267
522
|
};
|
|
268
|
-
|
|
523
|
+
|
|
524
|
+
switch (node.kind) {
|
|
269
525
|
case "message":
|
|
270
|
-
return
|
|
526
|
+
return block(node.role.toUpperCase(), node.text);
|
|
271
527
|
case "thinking":
|
|
272
|
-
return
|
|
273
|
-
case "
|
|
274
|
-
return
|
|
275
|
-
case "action
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
528
|
+
return block("THINKING", clip(node.text, 200));
|
|
529
|
+
case "skill.loaded":
|
|
530
|
+
return block(`SKILL · ${node.skill}`, "loaded");
|
|
531
|
+
case "action": {
|
|
532
|
+
const input = recordOf(node.input);
|
|
533
|
+
const output = recordOf(node.output);
|
|
534
|
+
const inputText = typeof input?.command === "string" ? input.command : jsonPreview(node.input);
|
|
535
|
+
const outputText = typeof output?.output === "string" ? output.output : node.output !== undefined ? jsonPreview(node.output) : "(no result)";
|
|
536
|
+
const exit = typeof output?.exit_code === "number" ? ` · exit ${output.exit_code}` : "";
|
|
537
|
+
return [
|
|
538
|
+
meta ? `TOOL · ${node.name} ${meta}` : `TOOL · ${node.name}`,
|
|
539
|
+
" input",
|
|
540
|
+
...indentedText(inputText, width),
|
|
541
|
+
` result · ${node.status}${exit}`,
|
|
542
|
+
...indentedText(outputText, width),
|
|
543
|
+
];
|
|
544
|
+
}
|
|
545
|
+
case "subagent": {
|
|
546
|
+
return block(`SUBAGENT · ${node.name}`, node.output === undefined ? node.status : `result · ${node.status}: ${jsonPreview(node.output)}`);
|
|
547
|
+
}
|
|
281
548
|
case "input.requested":
|
|
282
|
-
return
|
|
549
|
+
return block("INPUT REQUESTED", node.request.prompt ?? node.request.display ?? "(input requested)");
|
|
283
550
|
case "compaction":
|
|
284
|
-
return
|
|
551
|
+
return block("COMPACTION", node.reason ?? "context compacted");
|
|
285
552
|
case "error":
|
|
286
|
-
return
|
|
553
|
+
return block("ERROR", node.message);
|
|
554
|
+
case "telemetry":
|
|
555
|
+
return execBody(
|
|
556
|
+
relSeconds(node.span.startMs, originMs),
|
|
557
|
+
"telemetry",
|
|
558
|
+
node.span.name || node.span.kind || "span",
|
|
559
|
+
formatDurationMs(node.span.endMs - node.span.startMs),
|
|
560
|
+
width,
|
|
561
|
+
true,
|
|
562
|
+
);
|
|
287
563
|
}
|
|
288
564
|
}
|
|
289
565
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
566
|
+
/**
|
|
567
|
+
* `--execution`:标准事件流骨架(message / thinking / skill load / tool call+result / subagent /
|
|
568
|
+
* input.requested / compaction / error),有 OTel 时同一节点补相对时间与耗时;没有 OTel 时
|
|
569
|
+
* 节点、顺序与内容不变,只去掉时间列,并在结尾如实标 timing unavailable
|
|
570
|
+
* (ExecutionTree 的契约:骨架不因时间有无而变形,见 o11y/execution-tree.ts 头注)。
|
|
571
|
+
*/
|
|
572
|
+
export function executionText(
|
|
573
|
+
evidence: AttemptEvidence,
|
|
574
|
+
opts: { header: string; artifactPath?: string; width: number },
|
|
575
|
+
): string {
|
|
576
|
+
const { header, artifactPath, width } = opts;
|
|
577
|
+
const tree = evidence.execution;
|
|
578
|
+
const eventsSource = artifactPath ? join(artifactPath, "events.json") : undefined;
|
|
579
|
+
if (!tree) {
|
|
580
|
+
return `${header}\n\n(no events recorded for this attempt${eventsSource ? ` · expected: ${eventsSource}` : ""})`;
|
|
300
581
|
}
|
|
301
|
-
|
|
302
|
-
const
|
|
303
|
-
const
|
|
304
|
-
const
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
582
|
+
|
|
583
|
+
const timingAvailable = tree.timingAvailable;
|
|
584
|
+
const spanStarts = tree.nodes.flatMap((n) => (n.span ? [n.span.startMs] : []));
|
|
585
|
+
const originMs = spanStarts.length > 0 ? Math.min(...spanStarts) : 0;
|
|
586
|
+
|
|
587
|
+
// `--execution` 回答 Agent 做了什么。未关联到标准事件的原始 spans 属于 trace 证据,
|
|
588
|
+
// 逐条混入 transcript 会把几十条 SDK 内部 span 盖过消息和工具调用。
|
|
589
|
+
const agentNodes = tree.nodes.filter((node) => node.kind !== "telemetry");
|
|
590
|
+
const telemetryCount = tree.nodes.length - agentNodes.length;
|
|
591
|
+
const shown = agentNodes.slice(0, MAX_EVENTS);
|
|
592
|
+
const lines = shown.flatMap((node, index) => [
|
|
593
|
+
...(index === 0 ? [] : [""]),
|
|
594
|
+
...executionNodeLines(node, originMs, timingAvailable, width),
|
|
595
|
+
]);
|
|
596
|
+
|
|
597
|
+
const tail: string[] = [];
|
|
598
|
+
if (timingAvailable) {
|
|
599
|
+
const skillLoads = tree.nodes.filter((n) => n.kind === "skill.loaded").length;
|
|
600
|
+
const toolCalls = tree.nodes.filter((n) => n.kind === "action").length;
|
|
601
|
+
const aiMessages = tree.nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
|
|
602
|
+
tail.push(
|
|
603
|
+
[
|
|
604
|
+
`total ${formatDurationMs(evidence.result.durationMs)}`,
|
|
605
|
+
`${skillLoads} skill ${skillLoads === 1 ? "load" : "loads"}`,
|
|
606
|
+
`${toolCalls} tool ${toolCalls === 1 ? "call" : "calls"}`,
|
|
607
|
+
`${aiMessages} AI ${aiMessages === 1 ? "message" : "messages"}`,
|
|
608
|
+
...(agentNodes.length > shown.length ? [`${agentNodes.length - shown.length} more events not shown`] : []),
|
|
609
|
+
].join(" · "),
|
|
610
|
+
);
|
|
611
|
+
} else {
|
|
612
|
+
tail.push(
|
|
613
|
+
[
|
|
614
|
+
"timing unavailable · OTel trace was not collected",
|
|
615
|
+
...(agentNodes.length > shown.length ? [`${agentNodes.length - shown.length} more events not shown`] : []),
|
|
616
|
+
].join(" · "),
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
if (eventsSource) tail.push(`full events: ${eventsSource}`);
|
|
620
|
+
if (telemetryCount > 0) tail.push(`${telemetryCount} unlinked telemetry spans omitted; inspect the OTel trace for framework timing.`);
|
|
621
|
+
if (timingAvailable && artifactPath) tail.push(`full OTel trace: ${join(artifactPath, "trace.json")}`);
|
|
622
|
+
|
|
623
|
+
return `${header}\n\n${lines.join("\n")}\n\n${tail.join("\n")}`;
|
|
311
624
|
}
|
|
312
625
|
|
|
313
|
-
// ─────────────────────────
|
|
626
|
+
// ───────────────────────── 默认全景:AttemptEvidence 紧凑总览 ─────────────────────────
|
|
314
627
|
|
|
315
|
-
const
|
|
316
|
-
const TRACE_BAR_WIDTH = 20;
|
|
628
|
+
const MAX_OVERVIEW_DIFF_NAMES = 5;
|
|
317
629
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
const tool = span.attributes?.["tool_name"];
|
|
327
|
-
return typeof tool === "string" ? `execute_tool ${tool}` : "execute_tool";
|
|
328
|
-
}
|
|
329
|
-
case "agent":
|
|
330
|
-
return "invoke_agent";
|
|
331
|
-
default:
|
|
332
|
-
return span.name || "other";
|
|
333
|
-
}
|
|
630
|
+
function overviewDiffLine(diff: DiffData): string {
|
|
631
|
+
const names = [
|
|
632
|
+
...Object.keys(diff.generatedFiles).sort().map((p) => `M ${p}`),
|
|
633
|
+
...[...diff.deletedFiles].sort().map((p) => `D ${p}`),
|
|
634
|
+
];
|
|
635
|
+
const shown = names.slice(0, MAX_OVERVIEW_DIFF_NAMES);
|
|
636
|
+
const more = names.length > shown.length ? ` · +${names.length - shown.length} more` : "";
|
|
637
|
+
return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed · ${shown.join(", ")}${more}`;
|
|
334
638
|
}
|
|
335
639
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
640
|
+
/**
|
|
641
|
+
* `niceeval show @<locator>` 不带证据 flag 时的默认面:紧凑全景,只给摘要——Eval 断言计票、
|
|
642
|
+
* 执行事件计数、可选 OTel 时间指示、工作区 diff 摘要,不复现 `--eval` 的完整源码、
|
|
643
|
+
* `--execution` 的完整事件流或 `--diff` 的完整文件列表(那些内容各自的证据 flag 才给)。
|
|
644
|
+
*/
|
|
645
|
+
export function attemptOverviewText(
|
|
646
|
+
evidence: AttemptEvidence,
|
|
647
|
+
opts: { header: string; artifactPath?: string; width: number },
|
|
648
|
+
): string {
|
|
649
|
+
const { header, artifactPath } = opts;
|
|
650
|
+
const r = evidence.result;
|
|
651
|
+
|
|
652
|
+
const metaParts = [`snapshot ${evidence.identity.snapshotStartedAt}`, `attempt ${evidence.identity.attempt + 1}`, formatDurationMs(r.durationMs)];
|
|
653
|
+
if (r.usage) metaParts.push(`${formatMetricValue(r.usage.inputTokens + r.usage.outputTokens)} tokens`);
|
|
654
|
+
const cost = attemptCostUSD(r);
|
|
655
|
+
if (cost !== null) metaParts.push(formatUSD(cost));
|
|
342
656
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
}
|
|
657
|
+
const blocks: string[] = [[header, metaParts.join(" · ")].join("\n")];
|
|
658
|
+
|
|
659
|
+
blocks.push(
|
|
660
|
+
[
|
|
661
|
+
assertionSummaryLine(r.assertions),
|
|
662
|
+
evidence.evalSource
|
|
663
|
+
? `eval source: ${evidence.evalSource.sourcePath} · sha256:${evidence.evalSource.sourceSha256.slice(0, 8)}…`
|
|
664
|
+
: "eval source: unavailable (not captured for this attempt)",
|
|
665
|
+
].join("\n"),
|
|
666
|
+
);
|
|
354
667
|
|
|
355
|
-
const
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
roots.push(span);
|
|
668
|
+
const diagnostics = failureDiagnostics(r.assertions, opts.width);
|
|
669
|
+
if (diagnostics) blocks.push(diagnostics);
|
|
670
|
+
|
|
671
|
+
if (evidence.execution) {
|
|
672
|
+
const nodes = evidence.execution.nodes.filter((node) => node.kind !== "telemetry");
|
|
673
|
+
const skillLoads = nodes.filter((n) => n.kind === "skill.loaded").length;
|
|
674
|
+
const toolCalls = nodes.filter((n) => n.kind === "action").length;
|
|
675
|
+
const aiMessages = nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
|
|
676
|
+
const execLines = [`execution: ${nodes.length} events · ${skillLoads} skill loads · ${toolCalls} tool calls · ${aiMessages} AI messages`];
|
|
677
|
+
if (evidence.capabilities.timing) {
|
|
678
|
+
execLines.push("timing: OTel spans recorded for this attempt — see --execution for per-step timing.");
|
|
367
679
|
}
|
|
680
|
+
blocks.push(execLines.join("\n"));
|
|
681
|
+
} else {
|
|
682
|
+
blocks.push("execution: unavailable (no events recorded for this attempt)");
|
|
368
683
|
}
|
|
369
|
-
const byStart = (a: TraceSpan, b: TraceSpan) => a.startMs - b.startMs;
|
|
370
|
-
roots.sort(byStart);
|
|
371
|
-
for (const list of children.values()) list.sort(byStart);
|
|
372
|
-
|
|
373
|
-
const rows: { label: string; span: TraceSpan }[] = [];
|
|
374
|
-
const walk = (span: TraceSpan, prefix: string, childPrefix: string) => {
|
|
375
|
-
rows.push({ label: prefix + spanLabel(span), span });
|
|
376
|
-
const kids = children.get(span.spanId) ?? [];
|
|
377
|
-
kids.forEach((kid, i) => {
|
|
378
|
-
const last = i === kids.length - 1;
|
|
379
|
-
walk(kid, childPrefix + (last ? "└─ " : "├─ "), childPrefix + (last ? " " : "│ "));
|
|
380
|
-
});
|
|
381
|
-
};
|
|
382
|
-
for (const root of roots) walk(root, "", "");
|
|
383
|
-
|
|
384
|
-
const shown = rows.slice(0, MAX_SPANS);
|
|
385
|
-
const labelWidth = Math.max(...shown.map((r) => r.label.length));
|
|
386
|
-
const lines = shown.map(
|
|
387
|
-
(r) =>
|
|
388
|
-
`${padDisplay(r.label, labelWidth)} ${traceBar(r.span.startMs, r.span.endMs, windowStart, windowEnd)} ${formatDurationMs(
|
|
389
|
-
r.span.endMs - r.span.startMs,
|
|
390
|
-
)}${r.span.status === "error" ? " ✗" : ""}`,
|
|
391
|
-
);
|
|
392
684
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
685
|
+
if (evidence.diff && evidence.capabilities.diff) {
|
|
686
|
+
blocks.push(overviewDiffLine(evidence.diff));
|
|
687
|
+
} else if (evidence.diff) {
|
|
688
|
+
blocks.push("changes: diff unavailable · this attempt did not produce workspace file changes");
|
|
689
|
+
} else {
|
|
690
|
+
blocks.push("changes: diff unavailable · no workspace diff was recorded for this attempt");
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
const tail: string[] = [];
|
|
694
|
+
if (artifactPath) tail.push(`artifacts: ${artifactPath}/`);
|
|
695
|
+
const available = [
|
|
696
|
+
evidence.capabilities.eval ? `niceeval show ${evidence.locator} --eval` : undefined,
|
|
697
|
+
evidence.capabilities.execution ? `niceeval show ${evidence.locator} --execution` : undefined,
|
|
698
|
+
evidence.capabilities.diff ? `niceeval show ${evidence.locator} --diff` : undefined,
|
|
699
|
+
].filter((command): command is string => command !== undefined);
|
|
700
|
+
if (available.length > 0) tail.push(`available:\n${available.map((command) => ` ${command}`).join("\n")}`);
|
|
701
|
+
if (tail.length > 0) blocks.push(tail.join("\n"));
|
|
702
|
+
|
|
703
|
+
return blocks.join("\n\n");
|
|
401
704
|
}
|
|
402
705
|
|
|
403
706
|
// ───────────────────────── 证据切面:diff ─────────────────────────
|
|
@@ -412,7 +715,7 @@ export function diffText(opts: {
|
|
|
412
715
|
file?: string;
|
|
413
716
|
}): string {
|
|
414
717
|
const { header, diff, artifactPath, file } = opts;
|
|
415
|
-
const source = artifactPath ? join(artifactPath, "diff.json") : undefined;
|
|
718
|
+
const source = artifactPath ? join( artifactPath, "diff.json") : undefined;
|
|
416
719
|
if (!diff) {
|
|
417
720
|
return `${header}\n\n(no diff recorded for this attempt${source ? ` · expected: ${source}` : ""})`;
|
|
418
721
|
}
|