niceeval 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (337) hide show
  1. package/INDEX.md +55 -0
  2. package/dist/agents/types.d.ts +321 -0
  3. package/dist/context/types.d.ts +335 -0
  4. package/dist/i18n/core.d.ts +11 -0
  5. package/dist/i18n/en.d.ts +181 -0
  6. package/dist/i18n/index.d.ts +5 -0
  7. package/dist/i18n/zh-CN.d.ts +183 -0
  8. package/dist/o11y/otlp/parse.d.ts +3 -0
  9. package/dist/o11y/otlp/receiver.d.ts +21 -0
  10. package/dist/o11y/otlp/turn-otel.d.ts +43 -0
  11. package/dist/o11y/types.d.ts +190 -0
  12. package/dist/report/aggregate.d.ts +57 -0
  13. package/dist/report/aggregate.js +212 -0
  14. package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
  15. package/dist/report/built-ins/experiment-comparison.js +13 -0
  16. package/dist/report/built-ins/index.d.ts +1 -0
  17. package/dist/report/built-ins/index.js +2 -0
  18. package/dist/report/components.d.ts +180 -0
  19. package/dist/report/components.js +132 -0
  20. package/dist/report/compute.d.ts +106 -0
  21. package/dist/report/compute.js +600 -0
  22. package/dist/report/flag.d.ts +6 -0
  23. package/dist/report/flag.js +8 -0
  24. package/dist/report/format.d.ts +26 -0
  25. package/dist/report/format.js +109 -0
  26. package/dist/report/index.d.ts +19 -0
  27. package/dist/report/index.js +30 -0
  28. package/dist/report/load.d.ts +12 -0
  29. package/dist/report/load.js +56 -0
  30. package/dist/report/locale.d.ts +94 -0
  31. package/dist/report/locale.js +183 -0
  32. package/dist/report/metrics.d.ts +21 -0
  33. package/dist/report/metrics.js +115 -0
  34. package/dist/report/primitives.d.ts +64 -0
  35. package/dist/report/primitives.js +113 -0
  36. package/dist/report/react/AttemptList.d.ts +23 -0
  37. package/dist/report/react/AttemptList.js +19 -0
  38. package/dist/report/react/DeltaTable.d.ts +8 -0
  39. package/dist/report/react/DeltaTable.js +24 -0
  40. package/dist/report/react/EvalList.d.ts +10 -0
  41. package/dist/report/react/EvalList.js +0 -0
  42. package/dist/report/react/ExperimentList.d.ts +11 -0
  43. package/dist/report/react/ExperimentList.js +67 -0
  44. package/dist/report/react/GroupSummary.d.ts +8 -0
  45. package/dist/report/react/GroupSummary.js +8 -0
  46. package/dist/report/react/MetricBars.d.ts +10 -0
  47. package/dist/report/react/MetricBars.js +48 -0
  48. package/dist/report/react/MetricLine.d.ts +9 -0
  49. package/dist/report/react/MetricLine.js +68 -0
  50. package/dist/report/react/MetricMatrix.d.ts +10 -0
  51. package/dist/report/react/MetricMatrix.js +23 -0
  52. package/dist/report/react/MetricScatter.d.ts +9 -0
  53. package/dist/report/react/MetricScatter.js +96 -0
  54. package/dist/report/react/MetricTable.d.ts +11 -0
  55. package/dist/report/react/MetricTable.js +26 -0
  56. package/dist/report/react/RunOverview.d.ts +8 -0
  57. package/dist/report/react/RunOverview.js +12 -0
  58. package/dist/report/react/Scoreboard.d.ts +8 -0
  59. package/dist/report/react/Scoreboard.js +26 -0
  60. package/dist/report/react/cell.d.ts +9 -0
  61. package/dist/report/react/cell.js +9 -0
  62. package/dist/report/react/chart-math.d.ts +14 -0
  63. package/dist/report/react/chart-math.js +68 -0
  64. package/dist/report/react/colors.d.ts +14 -0
  65. package/dist/report/react/colors.js +45 -0
  66. package/dist/report/react/fixtures.d.ts +19 -0
  67. package/dist/report/react/fixtures.js +398 -0
  68. package/dist/report/react/format.d.ts +3 -0
  69. package/dist/report/react/format.js +7 -0
  70. package/dist/report/react/index.d.ts +6 -0
  71. package/dist/report/react/index.js +14 -0
  72. package/dist/report/report.d.ts +24 -0
  73. package/dist/report/report.js +49 -0
  74. package/dist/report/text/faces.d.ts +29 -0
  75. package/dist/report/text/faces.js +541 -0
  76. package/dist/report/text/layout.d.ts +75 -0
  77. package/dist/report/text/layout.js +173 -0
  78. package/dist/report/text/plot.d.ts +37 -0
  79. package/dist/report/text/plot.js +81 -0
  80. package/dist/report/text/table.d.ts +8 -0
  81. package/dist/report/text/table.js +86 -0
  82. package/dist/report/tree.d.ts +99 -0
  83. package/dist/report/tree.js +228 -0
  84. package/dist/report/types.d.ts +408 -0
  85. package/dist/report/types.js +4 -0
  86. package/dist/report/web.d.ts +15 -0
  87. package/dist/report/web.js +37 -0
  88. package/dist/results/locator.d.ts +75 -0
  89. package/dist/results/locator.js +0 -0
  90. package/dist/results/select.d.ts +45 -0
  91. package/dist/results/select.js +230 -0
  92. package/dist/results/source-hash.d.ts +9 -0
  93. package/dist/results/types.d.ts +204 -0
  94. package/dist/runner/eval-source.d.ts +22 -0
  95. package/dist/runner/fingerprint.d.ts +23 -0
  96. package/dist/runner/types.d.ts +332 -0
  97. package/dist/sandbox/docker-stream.d.ts +24 -0
  98. package/dist/sandbox/docker.d.ts +84 -0
  99. package/dist/sandbox/e2b.d.ts +31 -0
  100. package/dist/sandbox/errors.d.ts +4 -0
  101. package/dist/sandbox/local-files.d.ts +6 -0
  102. package/dist/sandbox/paths.d.ts +4 -0
  103. package/dist/sandbox/registry.d.ts +13 -0
  104. package/dist/sandbox/resolve.d.ts +40 -0
  105. package/dist/sandbox/retry.d.ts +10 -0
  106. package/dist/sandbox/shell.d.ts +11 -0
  107. package/dist/sandbox/source-files.d.ts +13 -0
  108. package/dist/sandbox/types.d.ts +203 -0
  109. package/dist/sandbox/vercel.d.ts +37 -0
  110. package/dist/scoring/types.d.ts +70 -0
  111. package/dist/shared/aggregate.d.ts +29 -0
  112. package/dist/shared/aggregate.js +55 -0
  113. package/dist/shared/types.d.ts +27 -0
  114. package/dist/shared/verdict.d.ts +27 -0
  115. package/dist/shared/verdict.js +34 -0
  116. package/dist/tty-line.d.ts +6 -0
  117. package/dist/types.d.ts +7 -0
  118. package/dist/util.d.ts +22 -0
  119. package/docs-site/zh/concepts/adapter.mdx +7 -7
  120. package/docs-site/zh/concepts/assert.mdx +19 -18
  121. package/docs-site/zh/concepts/evals.mdx +12 -11
  122. package/docs-site/zh/concepts/experiment.mdx +2 -2
  123. package/docs-site/zh/concepts/judge.mdx +9 -9
  124. package/docs-site/zh/concepts/overview.mdx +3 -3
  125. package/docs-site/zh/concepts/tier.mdx +6 -6
  126. package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
  127. package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
  128. package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
  129. package/docs-site/zh/guides/connect-otel.mdx +3 -3
  130. package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
  131. package/docs-site/zh/guides/custom-reports.mdx +196 -36
  132. package/docs-site/zh/guides/experiments.mdx +7 -1
  133. package/docs-site/zh/guides/fixtures.mdx +4 -2
  134. package/docs-site/zh/guides/official-adapters.mdx +31 -9
  135. package/docs-site/zh/guides/publish-report.mdx +4 -4
  136. package/docs-site/zh/guides/report-components.mdx +194 -55
  137. package/docs-site/zh/guides/reporters.mdx +2 -2
  138. package/docs-site/zh/guides/results-data.mdx +61 -34
  139. package/docs-site/zh/guides/runner.mdx +2 -2
  140. package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
  141. package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
  142. package/docs-site/zh/guides/viewing-results.mdx +200 -79
  143. package/docs-site/zh/guides/write-experiment.mdx +9 -1
  144. package/docs-site/zh/guides/write-send.mdx +4 -3
  145. package/docs-site/zh/index.mdx +4 -3
  146. package/docs-site/zh/introduction.mdx +2 -1
  147. package/docs-site/zh/quickstart.mdx +5 -3
  148. package/docs-site/zh/reference/builtin-agents.mdx +38 -13
  149. package/docs-site/zh/reference/capabilities.mdx +1 -1
  150. package/docs-site/zh/reference/cli.mdx +36 -13
  151. package/docs-site/zh/reference/define-agent.mdx +15 -13
  152. package/docs-site/zh/reference/define-config.mdx +1 -1
  153. package/docs-site/zh/reference/define-eval.mdx +3 -3
  154. package/docs-site/zh/reference/events.mdx +9 -3
  155. package/docs-site/zh/reference/expect.mdx +13 -8
  156. package/package.json +10 -6
  157. package/src/agents/bub.ts +129 -42
  158. package/src/agents/claude-code.test.ts +152 -0
  159. package/src/agents/claude-code.ts +127 -13
  160. package/src/agents/codex.test.ts +192 -0
  161. package/src/agents/codex.ts +151 -12
  162. package/src/agents/index.ts +9 -3
  163. package/src/agents/manifest.ts +41 -0
  164. package/src/agents/plugin-config.test.ts +94 -0
  165. package/src/agents/shared.ts +33 -2
  166. package/src/agents/skills.test.ts +214 -0
  167. package/src/agents/skills.ts +304 -0
  168. package/src/agents/streaming.ts +5 -5
  169. package/src/agents/types.ts +59 -3
  170. package/src/cli.ts +74 -47
  171. package/src/context/context.test.ts +20 -2
  172. package/src/context/control-flow.ts +1 -1
  173. package/src/context/session.test.ts +92 -1
  174. package/src/context/session.ts +23 -1
  175. package/src/context/types.ts +4 -4
  176. package/src/define.test.ts +8 -8
  177. package/src/define.ts +11 -11
  178. package/src/expect/index.ts +5 -5
  179. package/src/i18n/en.ts +33 -15
  180. package/src/i18n/zh-CN.ts +37 -21
  181. package/src/index.ts +1 -1
  182. package/src/o11y/execution-tree.test.ts +452 -0
  183. package/src/o11y/execution-tree.ts +367 -0
  184. package/src/o11y/otlp/receiver.ts +1 -1
  185. package/src/o11y/otlp/select.ts +7 -3
  186. package/src/o11y/parsers/claude-code.test.ts +175 -0
  187. package/src/o11y/parsers/claude-code.ts +32 -0
  188. package/src/o11y/types.ts +6 -1
  189. package/src/report/aggregate.ts +26 -20
  190. package/src/report/built-in-user-parity.test.tsx +640 -0
  191. package/src/report/built-ins/experiment-comparison.tsx +19 -0
  192. package/src/report/built-ins/index.ts +2 -0
  193. package/src/report/components.tsx +225 -50
  194. package/src/report/compute.ts +335 -54
  195. package/src/report/dual-render.test.tsx +1129 -0
  196. package/src/report/flag.ts +1 -1
  197. package/src/report/format.ts +35 -2
  198. package/src/report/index.ts +47 -14
  199. package/src/report/load.ts +1 -1
  200. package/src/report/locale.ts +229 -0
  201. package/src/report/metrics.ts +39 -15
  202. package/src/report/primitives.tsx +100 -2
  203. package/src/report/react/AttemptList.tsx +113 -0
  204. package/src/report/react/DeltaTable.tsx +9 -6
  205. package/src/report/react/EvalList.tsx +0 -0
  206. package/src/report/react/ExperimentList.tsx +220 -0
  207. package/src/report/react/GroupSummary.tsx +66 -0
  208. package/src/report/react/MetricBars.tsx +24 -13
  209. package/src/report/react/MetricLine.tsx +21 -21
  210. package/src/report/react/MetricMatrix.tsx +8 -4
  211. package/src/report/react/MetricScatter.tsx +126 -96
  212. package/src/report/react/MetricTable.tsx +115 -15
  213. package/src/report/react/RunOverview.tsx +48 -26
  214. package/src/report/react/Scoreboard.tsx +15 -7
  215. package/src/report/react/cell.tsx +18 -9
  216. package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
  217. package/src/report/react/colors.ts +24 -15
  218. package/src/report/react/enhance.js +168 -0
  219. package/src/report/react/fixtures.ts +169 -34
  220. package/src/report/react/format.ts +1 -1
  221. package/src/report/react/index.tsx +21 -6
  222. package/src/report/react/render.test.tsx +180 -31
  223. package/src/report/react/styles.css +432 -213
  224. package/src/report/report.test.ts +398 -94
  225. package/src/report/report.ts +29 -11
  226. package/src/report/text/faces.ts +344 -96
  227. package/src/report/text/layout.ts +76 -11
  228. package/src/report/text/table.ts +98 -0
  229. package/src/report/tree.ts +103 -20
  230. package/src/report/types.ts +184 -25
  231. package/src/report/web.ts +46 -19
  232. package/src/results/annotated-source.test.ts +146 -0
  233. package/src/results/annotated-source.ts +128 -0
  234. package/src/results/attempt-evidence.test.ts +158 -0
  235. package/src/results/attempt-evidence.ts +158 -0
  236. package/src/results/attempt-source.ts +48 -0
  237. package/src/results/copy.ts +154 -139
  238. package/src/results/format.ts +54 -47
  239. package/src/results/host-equivalence.test.ts +527 -0
  240. package/src/results/index.ts +56 -14
  241. package/src/results/locator.test.ts +198 -0
  242. package/src/results/locator.ts +0 -0
  243. package/src/results/open.ts +392 -175
  244. package/src/results/results.test.ts +943 -494
  245. package/src/results/select.ts +129 -23
  246. package/src/results/skipped-notice.ts +0 -0
  247. package/src/results/source-hash.ts +28 -0
  248. package/src/results/types.ts +98 -62
  249. package/src/results/writer.ts +304 -176
  250. package/src/runner/attempt.test.ts +170 -0
  251. package/src/runner/attempt.ts +48 -13
  252. package/src/runner/discover.test.ts +65 -0
  253. package/src/runner/discover.ts +6 -2
  254. package/src/runner/eval-source.test.ts +121 -0
  255. package/src/runner/eval-source.ts +45 -0
  256. package/src/runner/fingerprint.ts +54 -1
  257. package/src/runner/remote-sandbox.ts +1 -1
  258. package/src/runner/report.test.ts +6 -8
  259. package/src/runner/report.ts +5 -4
  260. package/src/runner/reporters/artifacts.ts +27 -41
  261. package/src/runner/reporters/braintrust.test.ts +3 -3
  262. package/src/runner/reporters/braintrust.ts +1 -1
  263. package/src/runner/reporters/console.ts +6 -6
  264. package/src/runner/reporters/json.ts +4 -4
  265. package/src/runner/reporters/live.test.ts +56 -0
  266. package/src/runner/reporters/live.ts +36 -10
  267. package/src/runner/reporters/quiet.test.ts +66 -0
  268. package/src/runner/reporters/quiet.ts +49 -0
  269. package/src/runner/reporters/shared.ts +5 -5
  270. package/src/runner/reporters/table.ts +17 -17
  271. package/src/runner/run.ts +61 -102
  272. package/src/runner/types.ts +43 -32
  273. package/src/sandbox/checkpoint.ts +1 -1
  274. package/src/sandbox/docker.ts +15 -2
  275. package/src/sandbox/e2b.ts +11 -5
  276. package/src/sandbox/errors.ts +13 -0
  277. package/src/sandbox/index.ts +3 -3
  278. package/src/sandbox/local-files.ts +1 -1
  279. package/src/sandbox/registry.ts +6 -5
  280. package/src/sandbox/resolve.ts +38 -23
  281. package/src/sandbox/retry.test.ts +68 -0
  282. package/src/sandbox/retry.ts +44 -0
  283. package/src/sandbox/types.ts +24 -24
  284. package/src/sandbox/vercel.ts +17 -3
  285. package/src/scoring/collector.ts +3 -3
  286. package/src/scoring/judge.ts +2 -2
  287. package/src/scoring/scoped.ts +13 -1
  288. package/src/scoring/types.ts +7 -7
  289. package/src/scoring/verdict.ts +4 -4
  290. package/src/shared/aggregate.ts +23 -3
  291. package/src/shared/types.ts +2 -2
  292. package/src/shared/{outcome.ts → verdict.ts} +15 -15
  293. package/src/show/compose.ts +12 -132
  294. package/src/show/index.ts +119 -95
  295. package/src/show/render.ts +462 -159
  296. package/src/show/show.test.ts +471 -253
  297. package/src/source-loc.ts +1 -1
  298. package/src/tty-line.ts +32 -0
  299. package/src/types.ts +2 -2
  300. package/src/util.test.ts +21 -1
  301. package/src/util.ts +5 -1
  302. package/src/view/app/App.tsx +33 -198
  303. package/src/view/app/components/AttemptModal.tsx +6 -6
  304. package/src/view/app/components/CodeView.tsx +2 -2
  305. package/src/view/app/components/CopyControls.tsx +32 -36
  306. package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
  307. package/src/view/app/components/ui/badge.tsx +1 -1
  308. package/src/view/app/components/ui/dialog.tsx +2 -2
  309. package/src/view/app/i18n.ts +40 -31
  310. package/src/view/app/lib/artifact-url.ts +1 -1
  311. package/src/view/app/lib/attempt-route.test.ts +37 -47
  312. package/src/view/app/lib/attempt-route.ts +26 -26
  313. package/src/view/app/lib/rows.ts +9 -112
  314. package/src/view/app/lib/verdict.ts +25 -0
  315. package/src/view/app/main.tsx +9 -12
  316. package/src/view/app/pages/RunsPage.tsx +4 -4
  317. package/src/view/app/pages/TracesPage.tsx +3 -3
  318. package/src/view/app/shared.ts +0 -1
  319. package/src/view/app/types.ts +6 -51
  320. package/src/view/artifact-serving.test.ts +136 -0
  321. package/src/view/client-dist/app.css +1 -1
  322. package/src/view/client-dist/app.js +21 -21
  323. package/src/view/data.test.ts +214 -143
  324. package/src/view/data.ts +166 -120
  325. package/src/view/index.ts +23 -11
  326. package/src/view/server.ts +52 -17
  327. package/src/view/shared/types.ts +22 -31
  328. package/src/view/styles.css +117 -327
  329. package/src/view/view-report.test.ts +146 -98
  330. package/src/report/default-report.tsx +0 -222
  331. package/src/report/dual-face.test.tsx +0 -527
  332. package/src/report/react/CaseList.tsx +0 -70
  333. package/src/view/app/components/CostScoreChart.tsx +0 -147
  334. package/src/view/app/components/ExperimentTable.tsx +0 -284
  335. package/src/view/app/components/GroupSelector.tsx +0 -61
  336. package/src/view/app/components/primitives.tsx +0 -43
  337. package/src/view/app/lib/outcome.ts +0 -82
@@ -1,13 +1,16 @@
1
1
  // show 的文本渲染:单 eval 详情、--history 时间轴、三个证据切面(transcript / trace / diff)。
2
2
  // 输出形态照 docs-site/zh/guides/viewing-results.mdx 的示例块;长内容一律截断,
3
- // 但截断永远如实标注剩余数量和原始工件路径 —— 输出对上下文窗口友好,事实源留在盘上。
3
+ // 但截断永远如实标注剩余数量和原始 artifact 路径 —— 输出对上下文窗口友好,事实源留在盘上。
4
4
  // 全部纯函数(时间经 now 显式传入),证据数据由调用方 await 好了递进来。
5
5
 
6
6
  import { join, relative } from "node:path";
7
- import type { AssertionResult, ResultOutcome, StreamEvent, TraceSpan } from "../types.ts";
8
- import type { DiffData } from "../types.ts";
9
- import type { AttemptHandle, Selection, Snapshot } from "../results/index.ts";
10
- import { foldEvalOutcome } from "../shared/outcome.ts";
7
+ import type { AssertionResult, DiffData, EvalResult, Verdict } from "../types.ts";
8
+ import type { AttemptEvidence, AttemptHandle, Snapshot } from "../results/index.ts";
9
+ import type { AnnotatedSourceLine } from "../results/index.ts";
10
+ import { groupIncompatibleVersionSkips } from "../results/index.ts";
11
+ import type { SkippedDir } from "../results/index.ts";
12
+ import type { ExecutionNode } from "../o11y/execution-tree.ts";
13
+ import { foldEvalVerdict } from "../shared/verdict.ts";
11
14
  import { attemptCostUSD } from "../report/metrics.ts";
12
15
  import { formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD } from "../report/format.ts";
13
16
  import { indentBlock, padDisplay, renderAlignedRows, wrapDisplay } from "../report/text/layout.ts";
@@ -17,7 +20,7 @@ const MISSING = "—";
17
20
 
18
21
  // ───────────────────────── 时间与小件 ─────────────────────────
19
22
 
20
- /** 判决时间的相对标注("just now" / "41s ago" / "2h ago");未来时刻按 just now 兜底。 */
23
+ /** 判定时间的相对标注("just now" / "41s ago" / "2h ago");未来时刻按 just now 兜底。 */
21
24
  export function relativeAgo(iso: string | undefined, now: number): string {
22
25
  if (!iso) return "";
23
26
  const ms = now - Date.parse(iso);
@@ -36,9 +39,9 @@ export function timelineStamp(iso: string): string {
36
39
  return iso.slice(0, 16).replace(/:/g, "-");
37
40
  }
38
41
 
39
- function outcomeMark(outcome: ResultOutcome): string {
40
- if (outcome === "passed") return "✓";
41
- if (outcome === "skipped") return "-";
42
+ function verdictMark(verdict: Verdict): string {
43
+ if (verdict === "passed") return "✓";
44
+ if (verdict === "skipped") return "-";
42
45
  return "✗";
43
46
  }
44
47
 
@@ -46,25 +49,56 @@ function attemptsLabel(n: number): string {
46
49
  return `${n} ${n === 1 ? "attempt" : "attempts"}`;
47
50
  }
48
51
 
49
- /** attempt 的展示序号:落盘 attempt 从 0 计,人看 1 计(工件目录后缀 __2 = attempt 3)。 */
52
+ /** attempt 的展示序号:落盘 attempt 从 0 计,人看 1 计( artifact 目录后缀 __2 = attempt 3)。 */
50
53
  export function displayAttemptNumber(attempt: AttemptHandle): number {
51
54
  return attempt.result.attempt + 1;
52
55
  }
53
56
 
54
- /** attempt 工件目录的展示路径:尽量给相对 cwd 的短路径,出了 cwd 给绝对路径。 */
55
- export function attemptArtifactsPath(attempt: AttemptHandle, cwd: string): string | undefined {
57
+ /**
58
+ * attempt artifact 目录的展示路径:尽量给相对 cwd 的短路径,出了 cwd 给绝对路径。
59
+ * 本快照跑出的条目:artifact 与 result.json 同目录,= `<snapshot.dir>/<ref.attempt>`。
60
+ * 携带条目(--resume 合入):落盘的 artifactBase 相对结果根(= snapshot.dir 的上两级,
61
+ * 即 `<experiment-dir>/<snapshot-dir>` 的上一层),指向原快照的 attempt 目录。
62
+ */
63
+ export function attemptArtifactsPath(attempt: AttemptHandle, cwd: string): string {
56
64
  const r = attempt.result;
57
- let abs: string | undefined;
58
- if (r.artifactsDir) abs = join(attempt.runDir.dir, r.artifactsDir);
59
- else if (r.artifactBase) abs = join(attempt.runDir.dir, "..", r.artifactBase);
60
- if (!abs) return undefined;
65
+ const abs = r.artifactBase
66
+ ? join(attempt.snapshot.dir, "..", "..", r.artifactBase)
67
+ : join(attempt.snapshot.dir, attempt.ref.attempt);
61
68
  const rel = relative(cwd, abs);
62
69
  return rel.startsWith("..") ? abs : rel;
63
70
  }
64
71
 
72
+ /**
73
+ * 裸 `show` 零可读结果时,skipped 目录的展示文案。niceeval 自己写的、schemaVersion 不兼容的
74
+ * 落盘按 producer 版本分组,一条 `npx niceeval@<version> show --run <结果根>` 覆盖同版本全部
75
+ * 快照——show 没有 view 的单快照直读模式,`--run` 认的是结果根(其下可以有多个 experiment),
76
+ * 不是单个快照目录,所以不对每个目录重复拼一条各自的命令(那条命令用同一个 root 跑起来
77
+ * 结果完全一样,重复只会刷屏)。其余(第三方 harness、版本信息缺失、malformed、incomplete)
78
+ * 没有可执行的统一建议,原样逐条列出。
79
+ */
80
+ export function skippedRunsText(skipped: readonly SkippedDir[], root: string, cwd: string): string {
81
+ const { groups, rest } = groupIncompatibleVersionSkips(skipped);
82
+ const rootDisplay = relative(cwd, root) || ".";
83
+ const lines: string[] = [];
84
+ for (const g of groups) {
85
+ const count = g.dirs.length;
86
+ const version = g.producer.version;
87
+ const schema = g.schemaVersion !== undefined ? ` (schemaVersion ${g.schemaVersion})` : "";
88
+ const cmd = `npx niceeval@${version} show --run ${rootDisplay}`;
89
+ lines.push(
90
+ ` ${count} snapshot${count === 1 ? "" : "s"} written by niceeval ${version}${schema} — run \`${cmd}\` to view`,
91
+ );
92
+ }
93
+ for (const s of rest) {
94
+ lines.push(` skipped ${s.dir} (${s.reason})`);
95
+ }
96
+ return lines.join("\n");
97
+ }
98
+
65
99
  // ───────────────────────── attempt 挑选 ─────────────────────────
66
100
 
67
- /** 选集内某道题的全部 attempt(合成选集里每实验只剩最新判决)。 */
101
+ /** Selection 内某道题的全部 attempt(合成 Selection 里每实验只剩最新判定)。 */
68
102
  export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHandle[] {
69
103
  const out: AttemptHandle[] = [];
70
104
  for (const snapshot of snapshots) {
@@ -75,22 +109,16 @@ export function attemptsOfEval(snapshots: Snapshot[], evalId: string): AttemptHa
75
109
  }
76
110
 
77
111
  /**
78
- * 证据切面与详情块默认挑最新一次失败的 attempt;没有失败挑最新一次。
79
- * --attempt 收人看的 1 计序号;--experiment 已在选集合成时收窄。
112
+ * 详情块 / eval-id 前缀证据切面默认挑最新一次失败的 attempt;没有失败挑最新一次。
113
+ * 精确选某一次 attempt `@<locator>`(`resolveLocator`),不再有数字 `--attempt`——
114
+ * --experiment 已在 Selection 合成时收窄。
80
115
  */
81
- export function pickDetailAttempt(
82
- attempts: AttemptHandle[],
83
- attemptNumber?: number,
84
- ): AttemptHandle | undefined {
116
+ export function pickDetailAttempt(attempts: AttemptHandle[]): AttemptHandle | undefined {
85
117
  if (attempts.length === 0) return undefined;
86
118
  const byTime = [...attempts].sort((a, b) =>
87
119
  (a.result.startedAt ?? "").localeCompare(b.result.startedAt ?? "") || a.result.attempt - b.result.attempt,
88
120
  );
89
- if (attemptNumber !== undefined) {
90
- const wanted = byTime.filter((a) => a.result.attempt === attemptNumber - 1);
91
- return wanted.at(-1);
92
- }
93
- const failing = byTime.filter((a) => a.result.outcome === "failed" || a.result.outcome === "errored");
121
+ const failing = byTime.filter((a) => a.result.verdict === "failed" || a.result.verdict === "errored");
94
122
  return failing.at(-1) ?? byTime.at(-1);
95
123
  }
96
124
 
@@ -102,22 +130,53 @@ function scoreText(score: number): string {
102
130
 
103
131
  /** 断言行:✓/✗ + severity + name;gate 失败带 detail,soft 恒带 score/1(失败再补 detail)。 */
104
132
  export function assertionLine(a: AssertionResult): string {
105
- const head = `${a.passed ? "✓" : ""} ${a.severity} ${a.name}`;
133
+ const scope = a.group ? `${a.group} · ` : "";
134
+ const head = `${a.passed ? "✓" : "✗"} ${a.severity} ${scope}${a.name}`;
106
135
  if (a.severity === "soft") {
107
136
  const detail = !a.passed && a.detail ? `: ${a.detail}` : "";
108
137
  return `${head} — ${scoreText(a.score)}/1${detail}`;
109
138
  }
110
- if (!a.passed) return a.detail ? `${head} — ${a.detail}` : `${head} — score ${scoreText(a.score)}`;
139
+ if (!a.passed) {
140
+ const reason = a.detail ?? `score ${scoreText(a.score)}`;
141
+ return a.evidence !== undefined ? `${head} — ${reason} · actual: ${a.evidence}` : `${head} — ${reason}`;
142
+ }
111
143
  return head;
112
144
  }
113
145
 
146
+ function equalsExpected(name: string): string | undefined {
147
+ const match = /^equals\((.*)\)$/.exec(name);
148
+ return match?.[1];
149
+ }
150
+
151
+ /** 默认 Attempt 页的首要诊断:不要求用户再猜一次 evidence flag 才知道为何失败。 */
152
+ export function failureDiagnostics(assertions: AssertionResult[], width: number): string | undefined {
153
+ const failed = assertions.filter((a) => !a.passed);
154
+ if (failed.length === 0) return undefined;
155
+ const lines = ["failures:"];
156
+ for (const a of failed) {
157
+ const label = a.group ?? a.name;
158
+ lines.push(` ${a.severity} · ${label}`);
159
+ if (a.group) lines.push(` assertion: ${a.name}`);
160
+ const expected = equalsExpected(a.name);
161
+ if (expected !== undefined) lines.push(` expected: ${expected}`);
162
+ if (a.evidence !== undefined) lines.push(` received: ${a.evidence}`);
163
+ if (a.detail) lines.push(` reason: ${a.detail}`);
164
+ if (a.loc) lines.push(` source: ${a.loc.file}:${a.loc.line}${a.loc.column ? `:${a.loc.column}` : ""}`);
165
+ if (a.severity === "soft") lines.push(` score: ${scoreText(a.score)}/1`);
166
+ }
167
+ return lines.flatMap((line) => {
168
+ const indent = line.length - line.trimStart().length;
169
+ return wrapDisplay(line.trimStart(), Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`);
170
+ }).join("\n");
171
+ }
172
+
114
173
  /** 详情块头部:`attempt 3 · compare/codex-gpt-5.4 · failed · 41s · 12.3k tokens · $0.04`。 */
115
174
  export function attemptHeader(attempt: AttemptHandle): string {
116
175
  const r = attempt.result;
117
176
  const parts = [
118
177
  `attempt ${displayAttemptNumber(attempt)}`,
119
178
  attempt.experimentId,
120
- r.outcome,
179
+ r.verdict,
121
180
  formatDurationMs(r.durationMs),
122
181
  ];
123
182
  if (r.usage) parts.push(`${formatMetricValue(r.usage.inputTokens + r.usage.outputTokens)} tokens`);
@@ -126,6 +185,64 @@ export function attemptHeader(attempt: AttemptHandle): string {
126
185
  return parts.join(" · ");
127
186
  }
128
187
 
188
+ // ───────────────────────── AttemptEvidence 共用 ─────────────────────────
189
+ // evalSourceText / executionText / attemptOverviewText(--eval / --execution / 默认全景)
190
+ // 与 evalDetailText 的紧凑索引列共用的小件:locator 头、失败原因、
191
+ // 断言计票摘要。三个证据 renderer 与全景面都只消费同一份 AttemptEvidence,不各自读
192
+ // artifact 或重新判定 capability(loadAttemptEvidence 已经算好)。
193
+
194
+ /** `--eval` / `--execution` / 全景块的头行:`@<locator> · <evalId> · <experimentId> · <verdict>`。 */
195
+ export function attemptEvidenceHeader(evidence: AttemptEvidence): string {
196
+ return [evidence.locator, evidence.identity.evalId, evidence.identity.experimentId, evidence.result.verdict].join(" · ");
197
+ }
198
+
199
+ /**
200
+ * 一次 attempt 未通过的判定原因,单行、不含 detail——供紧凑索引行使用。precedence 与
201
+ * `report/compute.ts::reasonFor` 同一条规则(error → skipReason → 未通过的 gate 断言,
202
+ * soft 永不进入),但格式更短(`gate <name>`,不带 `: <detail>`——detail 留给
203
+ * `--eval`/单 eval 详情块的完整断言明细),这里独立实现而不是导入 report/ 的函数:
204
+ * report 包正被并行重写(见 plan/attempt-evidence-feedback-loop.md),show 的紧凑索引不应
205
+ * 依赖它的内部实现细节。
206
+ */
207
+ export function verdictReasonLine(result: EvalResult): string | undefined {
208
+ if (result.error !== undefined) return result.error;
209
+ if (result.skipReason !== undefined) return result.skipReason;
210
+ const gates = result.assertions.filter((a) => !a.passed && a.severity === "gate");
211
+ if (gates.length === 0) return undefined;
212
+ return gates.map((a) => `gate ${a.name}`).join(", ");
213
+ }
214
+
215
+ /** 紧凑多 attempt 索引的一行:`✗ weather/brooklyn @7K2M9Q gate calledTool(...)`。 */
216
+ export function attemptIndexLine(opts: {
217
+ evalId: string;
218
+ verdict: Verdict;
219
+ locator: string | undefined;
220
+ reason?: string;
221
+ }): string {
222
+ const loc = opts.locator ?? MISSING;
223
+ const parts = [`${verdictMark(opts.verdict)} ${opts.evalId}`, loc];
224
+ if (opts.reason) parts.push(opts.reason);
225
+ return parts.join(" ");
226
+ }
227
+
228
+ /**
229
+ * 断言计票摘要,`--eval` 与全景面共用:`assertions: 1 passed · 1 gate failed · 1 soft below
230
+ * target`。直接读 `EvalResult.assertions`(恒可用的瘦身字段)而不是
231
+ * `AnnotatedEvalSource.summary`——两者对同一批断言算出的计票恒等(`AnnotatedEvalSource` 正是
232
+ * 用这同一个数组喂 `buildAnnotatedEvalSource` 的),但 `evalSource` 可能是 `null`
233
+ * (未捕获源码),读 `result.assertions` 让这条摘要不因缺源码而跟着消失。
234
+ */
235
+ export function assertionSummaryLine(assertions: AssertionResult[]): string {
236
+ const passed = assertions.filter((a) => a.passed).length;
237
+ const gateFailed = assertions.filter((a) => !a.passed && a.severity === "gate").length;
238
+ const softBelow = assertions.filter((a) => !a.passed && a.severity === "soft").length;
239
+ const parts: string[] = [];
240
+ if (passed > 0) parts.push(`${passed} passed`);
241
+ if (gateFailed > 0) parts.push(`${gateFailed} gate failed`);
242
+ if (softBelow > 0) parts.push(`${softBelow} soft below target`);
243
+ return `assertions: ${parts.length > 0 ? parts.join(" · ") : "(none)"}`;
244
+ }
245
+
129
246
  export interface EvalDetailOptions {
130
247
  evalId: string;
131
248
  snapshots: Snapshot[];
@@ -136,7 +253,7 @@ export interface EvalDetailOptions {
136
253
  width: number;
137
254
  }
138
255
 
139
- /** `niceeval show <eval id>`:各 experiment 的判决行 + 默认 attempt 的断言明细。 */
256
+ /** `niceeval show <eval id>`:各 experiment 的判定行 + 默认 attempt 的断言明细。 */
140
257
  export function evalDetailText(opts: EvalDetailOptions): string {
141
258
  const { evalId, snapshots, detail, cwd, now, width } = opts;
142
259
  const blocks: string[] = [];
@@ -148,25 +265,32 @@ export function evalDetailText(opts: EvalDetailOptions): string {
148
265
  .find((d) => d !== undefined);
149
266
  blocks.push(description ? `${evalId} — ${description}` : evalId);
150
267
 
151
- // 每 experiment 一行:折叠判决、attempt 数、最新 attempt 的耗时、总成本、判决时间
268
+ // 每 experiment 一行:折叠判定、attempt 数、最新 attempt 的耗时、总成本、判定时间、
269
+ // 代表 attempt 的紧凑索引(locator + 失败原因)——agent 从这张榜单
270
+ // 就能直接摘到一个 `@<locator>` 下钻,不必先跑一遍 `--eval`/`--execution` 才知道选谁。
152
271
  const rows: string[][] = [];
153
272
  for (const snapshot of snapshots) {
154
273
  const ev = snapshot.evals.find((e) => e.id === evalId);
155
274
  if (!ev || ev.attempts.length === 0) continue;
156
- const outcome = foldEvalOutcome(ev.attempts.map((a) => a.result));
275
+ const verdict = foldEvalVerdict(ev.attempts.map((a) => a.result));
157
276
  const latest = ev.attempts.reduce((a, b) => (b.result.attempt >= a.result.attempt ? b : a));
158
277
  let cost: number | null = null;
159
278
  for (const attempt of ev.attempts) {
160
279
  const c = attemptCostUSD(attempt.result);
161
280
  if (c !== null) cost = (cost ?? 0) + c;
162
281
  }
282
+ const rep = pickDetailAttempt(ev.attempts);
283
+ const locatorCell = rep?.locator ?? MISSING;
284
+ const reasonCell = rep ? (verdictReasonLine(rep.result) ?? "") : "";
163
285
  rows.push([
164
286
  snapshot.experimentId,
165
- `${outcomeMark(outcome)} ${outcome}`,
287
+ `${verdictMark(verdict)} ${verdict}`,
166
288
  attemptsLabel(ev.attempts.length),
167
289
  formatDurationMs(latest.result.durationMs),
168
290
  cost === null ? MISSING : formatUSD(cost),
169
291
  `(${relativeAgo(latest.result.startedAt ?? snapshot.startedAt, now)})`,
292
+ locatorCell,
293
+ reasonCell,
170
294
  ]);
171
295
  }
172
296
  if (rows.length > 0) blocks.push(renderAlignedRows(rows));
@@ -186,8 +310,13 @@ export function evalDetailText(opts: EvalDetailOptions): string {
186
310
 
187
311
  const tail: string[] = [];
188
312
  const artifacts = attemptArtifactsPath(detail, cwd);
189
- if (artifacts) tail.push(`artifacts: ${artifacts}/`);
190
- tail.push(`next: niceeval show ${evalId} --transcript | --trace | --diff`);
313
+ if ( artifacts) tail.push(`artifacts: ${artifacts}/`);
314
+ if (detail.locator) tail.push(`attempt locator: ${detail.locator}`);
315
+ tail.push(
316
+ detail.locator
317
+ ? `next: niceeval show ${detail.locator} [--eval|--execution|--diff]`
318
+ : `next: niceeval show ${evalId} [--eval|--execution|--diff]`,
319
+ );
191
320
  blocks.push(tail.join("\n"));
192
321
  }
193
322
 
@@ -204,7 +333,7 @@ export function evalHistoryText(opts: {
204
333
  rows: EvalHistoryRow[];
205
334
  }): string {
206
335
  const { experimentId, evalId, rows } = opts;
207
- const passed = rows.filter((r) => r.outcome === "passed").length;
336
+ const passed = rows.filter((r) => r.verdict === "passed").length;
208
337
  const head = [
209
338
  ...(evalId ? [evalId] : []),
210
339
  experimentId,
@@ -215,7 +344,7 @@ export function evalHistoryText(opts: {
215
344
  const table = renderAlignedRows(
216
345
  rows.map((r) => [
217
346
  timelineStamp(r.startedAt),
218
- `${outcomeMark(r.outcome)} ${r.outcome}`,
347
+ `${verdictMark(r.verdict)} ${r.verdict}`,
219
348
  attemptsLabel(r.attempts),
220
349
  r.costUSD === null ? MISSING : formatUSD(r.costUSD),
221
350
  r.failedAssertion ?? (r.error ? `error: ${r.error}` : ""),
@@ -238,7 +367,7 @@ export function experimentHistoryText(experimentId: string, rows: ExperimentHist
238
367
  return `${head}\n\n${indentBlock(table, " ")}`;
239
368
  }
240
369
 
241
- // ───────────────────────── 证据切面:transcript ─────────────────────────
370
+ // ───────────────────────── 截断预算(--eval / --execution / 全景共用) ─────────────────────────
242
371
 
243
372
  const MAX_EVENTS = 80;
244
373
  const MAX_TEXT = 600;
@@ -258,146 +387,320 @@ function jsonPreview(value: unknown, max = 200): string {
258
387
  return clip(text, max);
259
388
  }
260
389
 
261
- const ROLE_PAD = 12;
390
+ function recordOf(value: unknown): Record<string, unknown> | undefined {
391
+ return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
392
+ }
393
+
394
+ function indentedText(text: string, width: number, indent = 4, maxLines = 18): string[] {
395
+ const source = clip(text).split("\n");
396
+ const shown = source.slice(0, maxLines);
397
+ const lines = shown.flatMap((line) => wrapDisplay(line || " ", Math.max(20, width - indent)).map((part) => `${" ".repeat(indent)}${part}`));
398
+ if (source.length > shown.length) lines.push(`${" ".repeat(indent)}… (+${source.length - shown.length} more lines)`);
399
+ return lines;
400
+ }
401
+
402
+ // ───────────────────────── 证据切面:--eval(Eval 源码标注) ─────────────────────────
403
+
404
+ const MAX_SOURCE_LINES = 400;
405
+
406
+ /** gate 失败 / soft 恒带分——与 assertionLine 的严重度口径一致,但不带断言 name(源码行本身就是名字)。 */
407
+ function evalAssertionDetailLine(a: AssertionResult): string | undefined {
408
+ if (a.severity === "soft") {
409
+ const detail = !a.passed && a.detail ? ` · ${a.detail}` : "";
410
+ return `soft · ${scoreText(a.score)}/1${detail}`;
411
+ }
412
+ if (!a.passed) {
413
+ const parts = ["gate"];
414
+ if (a.group) parts.push(a.group);
415
+ parts.push(a.name);
416
+ const expected = equalsExpected(a.name);
417
+ if (expected !== undefined) parts.push(`expected ${expected}`);
418
+ if (a.evidence !== undefined) parts.push(`received ${a.evidence}`);
419
+ if (a.detail) parts.push(a.detail);
420
+ return parts.join(" · ");
421
+ }
422
+ return undefined;
423
+ }
424
+
425
+ function evalSourceLineText(line: AnnotatedSourceLine, gutterWidth: number, width: number): string[] {
426
+ const anyFailed = line.assertions.some((a) => !a.passed);
427
+ const glyph = line.assertions.length === 0 ? " " : anyFailed ? "✗" : "✓";
428
+ const marginWidth = gutterWidth + 2; // 行号列 + glyph + 分隔空格
429
+ const prefix = `${padDisplay(String(line.line), gutterWidth)}${glyph} `;
430
+ // 源码行的空白(尤其是缩进)是语义的一部分:wrapDisplay 按单词重排会把连续空格
431
+ // 吃成一个、把缩进整体丢掉(agent-feedback-loop.mdx 的示例明确保留 2/4/6 格嵌套缩进)。
432
+ // 过长的源码行只裁一刀(clip,与其它证据切面的截断口径一致),不按词折成多行——
433
+ // 折行只对续行加统一 margin,救不回已经被吃掉的原始缩进,不如老实截断。
434
+ const out = [prefix + clip(line.text, Math.max(20, width - marginWidth))];
435
+ const margin = " ".repeat(marginWidth);
436
+ for (const a of line.assertions) {
437
+ const detail = evalAssertionDetailLine(a);
438
+ if (detail === undefined) continue;
439
+ for (const wrapped2 of wrapDisplay(detail, Math.max(20, width - marginWidth))) out.push(margin + wrapped2);
440
+ }
441
+ return out;
442
+ }
443
+
444
+ /**
445
+ * `--eval`:运行时保存的 Eval 源码,gate/soft 断言标回源码行,外加 unmapped 断言(永不丢弃)
446
+ * 与断言计票摘要。`evidence.evalSource === null` 时如实说明源码未捕获,不伪造空文档。
447
+ */
448
+ export function evalSourceText(
449
+ evidence: AttemptEvidence,
450
+ opts: { header: string; artifactPath?: string; width: number },
451
+ ): string {
452
+ const { header, artifactPath, width } = opts;
453
+ const source = evidence.evalSource;
454
+ const artifact = artifactPath ? join(artifactPath, "sources.json") : undefined;
455
+ if (!source) {
456
+ return `${header}\n\n(eval source unavailable for this attempt${artifact ? ` · expected: ${artifact}` : ""})`;
457
+ }
458
+
459
+ const blocks: string[] = [header, `eval source: ${source.sourcePath} · sha256:${source.sourceSha256.slice(0, 8)}…`];
460
+
461
+ const gutterWidth = String(source.lines.length).length;
462
+ const shownLines = source.lines.slice(0, MAX_SOURCE_LINES);
463
+ const lineLines = shownLines.flatMap((line) => evalSourceLineText(line, gutterWidth, width));
464
+ if (source.lines.length > shownLines.length) {
465
+ lineLines.push(`(${source.lines.length - shownLines.length} more lines not shown)`);
466
+ }
467
+ blocks.push(lineLines.join("\n"));
468
+
469
+ if (source.unmapped.length > 0) {
470
+ const unmappedLines = source.unmapped.map((a) => indentBlock(wrapDisplay(assertionLine(a), width - 2).join("\n"), " "));
471
+ blocks.push(
472
+ [`unmapped assertions (${source.unmapped.length}, no source location):`, ...unmappedLines].join("\n"),
473
+ );
474
+ }
475
+
476
+ blocks.push(assertionSummaryLine(evidence.result.assertions));
477
+ if (artifact) blocks.push(`full eval source: ${artifact}`);
478
+ return blocks.join("\n\n");
479
+ }
480
+
481
+ // ───────────────────────── 证据切面:--execution(标准事件流 + OTel enrichment) ─────────────────────────
262
482
 
263
- function eventLines(event: StreamEvent, width: number): string[] {
264
- const body = (tag: string, text: string): string[] => {
265
- const wrapped = wrapDisplay(clip(text), Math.max(20, width - ROLE_PAD));
266
- return wrapped.map((line, i) => (i === 0 ? padDisplay(tag, ROLE_PAD) + line : " ".repeat(ROLE_PAD) + line));
483
+ const EXEC_LABEL_PAD = 12;
484
+ const EXEC_TIME_PAD = 6;
485
+
486
+ function relSeconds(ms: number, originMs: number): string {
487
+ return `${((ms - originMs) / 1000).toFixed(1)}s`;
488
+ }
489
+
490
+ /** 一个 ExecutionNode 的一行(或折行后多行);时间列只在整棵树 timingAvailable 时出现。 */
491
+ function execBody(
492
+ time: string | undefined,
493
+ label: string,
494
+ text: string,
495
+ duration: string | undefined,
496
+ width: number,
497
+ timingAvailable: boolean,
498
+ ): string[] {
499
+ const timeCol = timingAvailable ? padDisplay(time ?? "", EXEC_TIME_PAD) + " " : "";
500
+ const marginWidth = timeCol.length + EXEC_LABEL_PAD;
501
+ const wrapped = wrapDisplay(clip(text), Math.max(20, width - marginWidth));
502
+ return wrapped.map((line, i) => {
503
+ if (i !== 0) return " ".repeat(marginWidth) + line;
504
+ const head = timeCol + padDisplay(label, EXEC_LABEL_PAD) + line;
505
+ return duration ? `${head} ${duration}` : head;
506
+ });
507
+ }
508
+
509
+ /**
510
+ * action / subagent 节点渲染成两行(call + result)——节点模型把 call+result 合并成一个
511
+ * ExecutionNode(按 callId),但分两行读更符合「先看调用、再看结果」的阅读顺序,
512
+ * 与 docs-site/zh/guides/agent-feedback-loop.mdx 的示例一致。
513
+ */
514
+ function executionNodeLines(node: ExecutionNode, originMs: number, timingAvailable: boolean, width: number): string[] {
515
+ const time = node.kind !== "telemetry" && node.span ? relSeconds(node.span.startMs, originMs) : undefined;
516
+ const duration = node.kind !== "telemetry" && node.span ? formatDurationMs(node.span.endMs - node.span.startMs) : undefined;
517
+ const meta = [time, duration].filter(Boolean).join(" · ");
518
+ const block = (title: string, body: string, extra: string[] = []): string[] => {
519
+ const heading = meta ? `${title} ${meta}` : title;
520
+ const bodyLines = wrapDisplay(clip(body), Math.max(20, width - 2)).map((line) => ` ${line}`);
521
+ return [heading, ...bodyLines, ...extra.flatMap((line) => wrapDisplay(line, Math.max(20, width - 2)).map((part) => ` ${part}`))];
267
522
  };
268
- switch (event.type) {
523
+
524
+ switch (node.kind) {
269
525
  case "message":
270
- return body(`[${event.role}]`, event.text);
526
+ return block(node.role.toUpperCase(), node.text);
271
527
  case "thinking":
272
- return body("[thinking]", clip(event.text, 200));
273
- case "action.called":
274
- return body("[tool]", `${event.name}(${jsonPreview(event.input)})`);
275
- case "action.result":
276
- return body("", `→ ${event.status}${event.output !== undefined ? `: ${jsonPreview(event.output)}` : ""}`);
277
- case "subagent.called":
278
- return body("[subagent]", event.name);
279
- case "subagent.completed":
280
- return body("", `→ ${event.status}`);
528
+ return block("THINKING", clip(node.text, 200));
529
+ case "skill.loaded":
530
+ return block(`SKILL · ${node.skill}`, "loaded");
531
+ case "action": {
532
+ const input = recordOf(node.input);
533
+ const output = recordOf(node.output);
534
+ const inputText = typeof input?.command === "string" ? input.command : jsonPreview(node.input);
535
+ const outputText = typeof output?.output === "string" ? output.output : node.output !== undefined ? jsonPreview(node.output) : "(no result)";
536
+ const exit = typeof output?.exit_code === "number" ? ` · exit ${output.exit_code}` : "";
537
+ return [
538
+ meta ? `TOOL · ${node.name} ${meta}` : `TOOL · ${node.name}`,
539
+ " input",
540
+ ...indentedText(inputText, width),
541
+ ` result · ${node.status}${exit}`,
542
+ ...indentedText(outputText, width),
543
+ ];
544
+ }
545
+ case "subagent": {
546
+ return block(`SUBAGENT · ${node.name}`, node.output === undefined ? node.status : `result · ${node.status}: ${jsonPreview(node.output)}`);
547
+ }
281
548
  case "input.requested":
282
- return body("[input]", event.request.prompt ?? event.request.display ?? "(input requested)");
549
+ return block("INPUT REQUESTED", node.request.prompt ?? node.request.display ?? "(input requested)");
283
550
  case "compaction":
284
- return body("[compaction]", event.reason ?? "context compacted");
551
+ return block("COMPACTION", node.reason ?? "context compacted");
285
552
  case "error":
286
- return body("[error]", event.message);
553
+ return block("ERROR", node.message);
554
+ case "telemetry":
555
+ return execBody(
556
+ relSeconds(node.span.startMs, originMs),
557
+ "telemetry",
558
+ node.span.name || node.span.kind || "span",
559
+ formatDurationMs(node.span.endMs - node.span.startMs),
560
+ width,
561
+ true,
562
+ );
287
563
  }
288
564
  }
289
565
 
290
- export function transcriptText(opts: {
291
- header: string;
292
- events: StreamEvent[] | null;
293
- artifactPath?: string;
294
- width: number;
295
- }): string {
296
- const { header, events, artifactPath, width } = opts;
297
- const source = artifactPath ? join(artifactPath, "events.json") : undefined;
298
- if (!events || events.length === 0) {
299
- return `${header}\n\n(no events recorded for this attempt${source ? ` · expected: ${source}` : ""})`;
566
+ /**
567
+ * `--execution`:标准事件流骨架(message / thinking / skill load / tool call+result / subagent /
568
+ * input.requested / compaction / error),有 OTel 时同一节点补相对时间与耗时;没有 OTel 时
569
+ * 节点、顺序与内容不变,只去掉时间列,并在结尾如实标 timing unavailable
570
+ * (ExecutionTree 的契约:骨架不因时间有无而变形,见 o11y/execution-tree.ts 头注)。
571
+ */
572
+ export function executionText(
573
+ evidence: AttemptEvidence,
574
+ opts: { header: string; artifactPath?: string; width: number },
575
+ ): string {
576
+ const { header, artifactPath, width } = opts;
577
+ const tree = evidence.execution;
578
+ const eventsSource = artifactPath ? join(artifactPath, "events.json") : undefined;
579
+ if (!tree) {
580
+ return `${header}\n\n(no events recorded for this attempt${eventsSource ? ` · expected: ${eventsSource}` : ""})`;
300
581
  }
301
- const shown = events.slice(0, MAX_EVENTS);
302
- const lines = shown.flatMap((event) => eventLines(event, width));
303
- const toolCalls = events.filter((e) => e.type === "action.called").length;
304
- const footer = [
305
- `${events.length} ${events.length === 1 ? "event" : "events"}`,
306
- toolCalls === 0 ? "no tool calls" : `${toolCalls} tool calls`,
307
- ...(events.length > shown.length ? [`${events.length - shown.length} more events not shown`] : []),
308
- ...(source ? [`full stream: ${source}`] : []),
309
- ].join(" · ");
310
- return `${header}\n\n${lines.join("\n")}\n\n(${footer})`;
582
+
583
+ const timingAvailable = tree.timingAvailable;
584
+ const spanStarts = tree.nodes.flatMap((n) => (n.span ? [n.span.startMs] : []));
585
+ const originMs = spanStarts.length > 0 ? Math.min(...spanStarts) : 0;
586
+
587
+ // `--execution` 回答 Agent 做了什么。未关联到标准事件的原始 spans 属于 trace 证据,
588
+ // 逐条混入 transcript 会把几十条 SDK 内部 span 盖过消息和工具调用。
589
+ const agentNodes = tree.nodes.filter((node) => node.kind !== "telemetry");
590
+ const telemetryCount = tree.nodes.length - agentNodes.length;
591
+ const shown = agentNodes.slice(0, MAX_EVENTS);
592
+ const lines = shown.flatMap((node, index) => [
593
+ ...(index === 0 ? [] : [""]),
594
+ ...executionNodeLines(node, originMs, timingAvailable, width),
595
+ ]);
596
+
597
+ const tail: string[] = [];
598
+ if (timingAvailable) {
599
+ const skillLoads = tree.nodes.filter((n) => n.kind === "skill.loaded").length;
600
+ const toolCalls = tree.nodes.filter((n) => n.kind === "action").length;
601
+ const aiMessages = tree.nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
602
+ tail.push(
603
+ [
604
+ `total ${formatDurationMs(evidence.result.durationMs)}`,
605
+ `${skillLoads} skill ${skillLoads === 1 ? "load" : "loads"}`,
606
+ `${toolCalls} tool ${toolCalls === 1 ? "call" : "calls"}`,
607
+ `${aiMessages} AI ${aiMessages === 1 ? "message" : "messages"}`,
608
+ ...(agentNodes.length > shown.length ? [`${agentNodes.length - shown.length} more events not shown`] : []),
609
+ ].join(" · "),
610
+ );
611
+ } else {
612
+ tail.push(
613
+ [
614
+ "timing unavailable · OTel trace was not collected",
615
+ ...(agentNodes.length > shown.length ? [`${agentNodes.length - shown.length} more events not shown`] : []),
616
+ ].join(" · "),
617
+ );
618
+ }
619
+ if (eventsSource) tail.push(`full events: ${eventsSource}`);
620
+ if (telemetryCount > 0) tail.push(`${telemetryCount} unlinked telemetry spans omitted; inspect the OTel trace for framework timing.`);
621
+ if (timingAvailable && artifactPath) tail.push(`full OTel trace: ${join(artifactPath, "trace.json")}`);
622
+
623
+ return `${header}\n\n${lines.join("\n")}\n\n${tail.join("\n")}`;
311
624
  }
312
625
 
313
- // ───────────────────────── 证据切面:trace ─────────────────────────
626
+ // ───────────────────────── 默认全景:AttemptEvidence 紧凑总览 ─────────────────────────
314
627
 
315
- const MAX_SPANS = 40;
316
- const TRACE_BAR_WIDTH = 20;
628
+ const MAX_OVERVIEW_DIFF_NAMES = 5;
317
629
 
318
- /** span 类目与 web 版同一套标准化口径(SpanKind),不读原生 span 名;other 保留原名如实展示。 */
319
- function spanLabel(span: TraceSpan): string {
320
- switch (span.kind) {
321
- case "turn":
322
- return "agent run";
323
- case "model":
324
- return "inference";
325
- case "tool": {
326
- const tool = span.attributes?.["tool_name"];
327
- return typeof tool === "string" ? `execute_tool ${tool}` : "execute_tool";
328
- }
329
- case "agent":
330
- return "invoke_agent";
331
- default:
332
- return span.name || "other";
333
- }
630
+ function overviewDiffLine(diff: DiffData): string {
631
+ const names = [
632
+ ...Object.keys(diff.generatedFiles).sort().map((p) => `M ${p}`),
633
+ ...[...diff.deletedFiles].sort().map((p) => `D ${p}`),
634
+ ];
635
+ const shown = names.slice(0, MAX_OVERVIEW_DIFF_NAMES);
636
+ const more = names.length > shown.length ? ` · +${names.length - shown.length} more` : "";
637
+ return `changes: ${names.length} ${names.length === 1 ? "file" : "files"} changed · ${shown.join(", ")}${more}`;
334
638
  }
335
639
 
336
- function traceBar(startMs: number, endMs: number, windowStart: number, windowEnd: number): string {
337
- const total = Math.max(1, windowEnd - windowStart);
338
- const from = Math.max(0, Math.min(TRACE_BAR_WIDTH - 1, Math.floor(((startMs - windowStart) / total) * TRACE_BAR_WIDTH)));
339
- const to = Math.max(from + 1, Math.min(TRACE_BAR_WIDTH, Math.ceil(((endMs - windowStart) / total) * TRACE_BAR_WIDTH)));
340
- return `▕${"░".repeat(from)}${"█".repeat(to - from)}${"░".repeat(TRACE_BAR_WIDTH - to)}▏`;
341
- }
640
+ /**
641
+ * `niceeval show @<locator>` 不带证据 flag 时的默认面:紧凑全景,只给摘要——Eval 断言计票、
642
+ * 执行事件计数、可选 OTel 时间指示、工作区 diff 摘要,不复现 `--eval` 的完整源码、
643
+ * `--execution` 的完整事件流或 `--diff` 的完整文件列表(那些内容各自的证据 flag 才给)
644
+ */
645
+ export function attemptOverviewText(
646
+ evidence: AttemptEvidence,
647
+ opts: { header: string; artifactPath?: string; width: number },
648
+ ): string {
649
+ const { header, artifactPath } = opts;
650
+ const r = evidence.result;
651
+
652
+ const metaParts = [`snapshot ${evidence.identity.snapshotStartedAt}`, `attempt ${evidence.identity.attempt + 1}`, formatDurationMs(r.durationMs)];
653
+ if (r.usage) metaParts.push(`${formatMetricValue(r.usage.inputTokens + r.usage.outputTokens)} tokens`);
654
+ const cost = attemptCostUSD(r);
655
+ if (cost !== null) metaParts.push(formatUSD(cost));
342
656
 
343
- export function traceText(opts: {
344
- header: string;
345
- spans: TraceSpan[] | null;
346
- artifactPath?: string;
347
- width: number;
348
- }): string {
349
- const { header, spans, artifactPath } = opts;
350
- const source = artifactPath ? join(artifactPath, "trace.json") : undefined;
351
- if (!spans || spans.length === 0) {
352
- return `${header}\n\n(no trace recorded for this attempt${source ? ` · expected: ${source}` : ""})`;
353
- }
657
+ const blocks: string[] = [[header, metaParts.join(" · ")].join("\n")];
658
+
659
+ blocks.push(
660
+ [
661
+ assertionSummaryLine(r.assertions),
662
+ evidence.evalSource
663
+ ? `eval source: ${evidence.evalSource.sourcePath} · sha256:${evidence.evalSource.sourceSha256.slice(0, 8)}…`
664
+ : "eval source: unavailable (not captured for this attempt)",
665
+ ].join("\n"),
666
+ );
354
667
 
355
- const windowStart = Math.min(...spans.map((s) => s.startMs));
356
- const windowEnd = Math.max(...spans.map((s) => s.endMs));
357
- const ids = new Set(spans.map((s) => s.spanId));
358
- const children = new Map<string, TraceSpan[]>();
359
- const roots: TraceSpan[] = [];
360
- for (const span of spans) {
361
- if (span.parentSpanId && ids.has(span.parentSpanId)) {
362
- const list = children.get(span.parentSpanId) ?? [];
363
- list.push(span);
364
- children.set(span.parentSpanId, list);
365
- } else {
366
- roots.push(span);
668
+ const diagnostics = failureDiagnostics(r.assertions, opts.width);
669
+ if (diagnostics) blocks.push(diagnostics);
670
+
671
+ if (evidence.execution) {
672
+ const nodes = evidence.execution.nodes.filter((node) => node.kind !== "telemetry");
673
+ const skillLoads = nodes.filter((n) => n.kind === "skill.loaded").length;
674
+ const toolCalls = nodes.filter((n) => n.kind === "action").length;
675
+ const aiMessages = nodes.filter((n) => n.kind === "message" && n.role === "assistant").length;
676
+ const execLines = [`execution: ${nodes.length} events · ${skillLoads} skill loads · ${toolCalls} tool calls · ${aiMessages} AI messages`];
677
+ if (evidence.capabilities.timing) {
678
+ execLines.push("timing: OTel spans recorded for this attempt — see --execution for per-step timing.");
367
679
  }
680
+ blocks.push(execLines.join("\n"));
681
+ } else {
682
+ blocks.push("execution: unavailable (no events recorded for this attempt)");
368
683
  }
369
- const byStart = (a: TraceSpan, b: TraceSpan) => a.startMs - b.startMs;
370
- roots.sort(byStart);
371
- for (const list of children.values()) list.sort(byStart);
372
-
373
- const rows: { label: string; span: TraceSpan }[] = [];
374
- const walk = (span: TraceSpan, prefix: string, childPrefix: string) => {
375
- rows.push({ label: prefix + spanLabel(span), span });
376
- const kids = children.get(span.spanId) ?? [];
377
- kids.forEach((kid, i) => {
378
- const last = i === kids.length - 1;
379
- walk(kid, childPrefix + (last ? "└─ " : "├─ "), childPrefix + (last ? " " : "│ "));
380
- });
381
- };
382
- for (const root of roots) walk(root, "", "");
383
-
384
- const shown = rows.slice(0, MAX_SPANS);
385
- const labelWidth = Math.max(...shown.map((r) => r.label.length));
386
- const lines = shown.map(
387
- (r) =>
388
- `${padDisplay(r.label, labelWidth)} ${traceBar(r.span.startMs, r.span.endMs, windowStart, windowEnd)} ${formatDurationMs(
389
- r.span.endMs - r.span.startMs,
390
- )}${r.span.status === "error" ? " ✗" : ""}`,
391
- );
392
684
 
393
- const toolSpans = spans.filter((s) => s.kind === "tool").length;
394
- const footer = [
395
- `${spans.length} ${spans.length === 1 ? "span" : "spans"}`,
396
- toolSpans === 0 ? "no execute_tool spans" : `${toolSpans} execute_tool spans`,
397
- ...(rows.length > shown.length ? [`${rows.length - shown.length} more spans not shown`] : []),
398
- ...(source ? [`full trace: ${source}`] : []),
399
- ].join(" · ");
400
- return `${header}\n\n${lines.join("\n")}\n\n(${footer})`;
685
+ if (evidence.diff && evidence.capabilities.diff) {
686
+ blocks.push(overviewDiffLine(evidence.diff));
687
+ } else if (evidence.diff) {
688
+ blocks.push("changes: diff unavailable · this attempt did not produce workspace file changes");
689
+ } else {
690
+ blocks.push("changes: diff unavailable · no workspace diff was recorded for this attempt");
691
+ }
692
+
693
+ const tail: string[] = [];
694
+ if (artifactPath) tail.push(`artifacts: ${artifactPath}/`);
695
+ const available = [
696
+ evidence.capabilities.eval ? `niceeval show ${evidence.locator} --eval` : undefined,
697
+ evidence.capabilities.execution ? `niceeval show ${evidence.locator} --execution` : undefined,
698
+ evidence.capabilities.diff ? `niceeval show ${evidence.locator} --diff` : undefined,
699
+ ].filter((command): command is string => command !== undefined);
700
+ if (available.length > 0) tail.push(`available:\n${available.map((command) => ` ${command}`).join("\n")}`);
701
+ if (tail.length > 0) blocks.push(tail.join("\n"));
702
+
703
+ return blocks.join("\n\n");
401
704
  }
402
705
 
403
706
  // ───────────────────────── 证据切面:diff ─────────────────────────
@@ -412,7 +715,7 @@ export function diffText(opts: {
412
715
  file?: string;
413
716
  }): string {
414
717
  const { header, diff, artifactPath, file } = opts;
415
- const source = artifactPath ? join(artifactPath, "diff.json") : undefined;
718
+ const source = artifactPath ? join( artifactPath, "diff.json") : undefined;
416
719
  if (!diff) {
417
720
  return `${header}\n\n(no diff recorded for this attempt${source ? ` · expected: ${source}` : ""})`;
418
721
  }