niceeval 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. package/dist/agents/types.d.ts +72 -6
  2. package/dist/context/types.d.ts +32 -12
  3. package/dist/i18n/en.d.ts +54 -0
  4. package/dist/i18n/zh-CN.d.ts +55 -1
  5. package/dist/o11y/types.d.ts +16 -2
  6. package/dist/report/aggregate.d.ts +5 -3
  7. package/dist/report/aggregate.js +32 -5
  8. package/dist/report/built-ins/experiment-comparison.d.ts +39 -0
  9. package/dist/report/built-ins/experiment-comparison.js +119 -0
  10. package/dist/report/built-ins/index.d.ts +2 -1
  11. package/dist/report/built-ins/index.js +2 -2
  12. package/dist/report/components.d.ts +10 -2
  13. package/dist/report/components.js +3 -3
  14. package/dist/report/compute.d.ts +11 -18
  15. package/dist/report/compute.js +68 -66
  16. package/dist/report/flag.d.ts +16 -1
  17. package/dist/report/flag.js +19 -1
  18. package/dist/report/format.d.ts +16 -14
  19. package/dist/report/format.js +28 -30
  20. package/dist/report/index.d.ts +5 -4
  21. package/dist/report/index.js +6 -5
  22. package/dist/report/locale.d.ts +23 -3
  23. package/dist/report/locale.js +47 -6
  24. package/dist/report/metrics.d.ts +13 -1
  25. package/dist/report/metrics.js +66 -15
  26. package/dist/report/primitives.d.ts +6 -0
  27. package/dist/report/react/AttemptList.d.ts +4 -4
  28. package/dist/report/react/AttemptList.js +8 -10
  29. package/dist/report/react/EvalList.d.ts +1 -1
  30. package/dist/report/react/EvalList.js +0 -0
  31. package/dist/report/react/ExperimentComparison.d.ts +8 -0
  32. package/dist/report/react/ExperimentComparison.js +11 -0
  33. package/dist/report/react/ExperimentList.d.ts +4 -2
  34. package/dist/report/react/ExperimentList.js +57 -7
  35. package/dist/report/react/MetricScatter.js +6 -16
  36. package/dist/report/react/chart-math.d.ts +23 -6
  37. package/dist/report/react/chart-math.js +71 -19
  38. package/dist/report/react/fixtures.d.ts +3 -3
  39. package/dist/report/react/fixtures.js +30 -18
  40. package/dist/report/react/format.d.ts +1 -1
  41. package/dist/report/react/format.js +1 -1
  42. package/dist/report/react/index.d.ts +1 -1
  43. package/dist/report/report.d.ts +5 -1
  44. package/dist/report/report.js +6 -2
  45. package/dist/report/text/faces.d.ts +1 -1
  46. package/dist/report/text/faces.js +100 -61
  47. package/dist/report/text/table.js +36 -5
  48. package/dist/report/types.d.ts +40 -34
  49. package/dist/results/types.d.ts +11 -0
  50. package/dist/runner/feedback/sink.d.ts +110 -0
  51. package/dist/runner/types.d.ts +513 -22
  52. package/dist/sandbox/docker.d.ts +23 -2
  53. package/dist/sandbox/e2b.d.ts +15 -1
  54. package/dist/sandbox/errors.d.ts +30 -3
  55. package/dist/sandbox/io-retry.d.ts +17 -0
  56. package/dist/sandbox/registry.d.ts +2 -0
  57. package/dist/sandbox/resolve.d.ts +18 -5
  58. package/dist/sandbox/retry.d.ts +11 -1
  59. package/dist/sandbox/types.d.ts +39 -5
  60. package/dist/sandbox/vercel.d.ts +7 -1
  61. package/dist/scoring/coverage.d.ts +30 -0
  62. package/dist/scoring/display.d.ts +21 -0
  63. package/dist/scoring/display.js +120 -0
  64. package/dist/scoring/types.d.ts +103 -20
  65. package/dist/shared/aggregate.d.ts +1 -0
  66. package/dist/shared/aggregate.js +3 -3
  67. package/dist/shared/types.d.ts +28 -0
  68. package/dist/tty-line.d.ts +0 -4
  69. package/dist/util.d.ts +23 -0
  70. package/docs-site/zh/concepts/adapter.mdx +24 -6
  71. package/docs-site/zh/concepts/assert.mdx +11 -10
  72. package/docs-site/zh/concepts/evals.mdx +7 -6
  73. package/docs-site/zh/concepts/experiment.mdx +1 -1
  74. package/docs-site/zh/concepts/overview.mdx +7 -7
  75. package/docs-site/zh/guides/agent-feedback-loop.mdx +35 -31
  76. package/docs-site/zh/guides/authoring.mdx +33 -0
  77. package/docs-site/zh/guides/ci-integration.mdx +23 -12
  78. package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
  79. package/docs-site/zh/guides/custom-reports.mdx +29 -34
  80. package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
  81. package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
  82. package/docs-site/zh/guides/debugging.mdx +210 -0
  83. package/docs-site/zh/guides/experiments.mdx +10 -3
  84. package/docs-site/zh/guides/fixtures.mdx +3 -1
  85. package/docs-site/zh/guides/official-adapters.mdx +27 -3
  86. package/docs-site/zh/guides/publish-report.mdx +30 -16
  87. package/docs-site/zh/guides/report-components.mdx +49 -37
  88. package/docs-site/zh/guides/reporters.mdx +2 -2
  89. package/docs-site/zh/guides/results-data.mdx +42 -8
  90. package/docs-site/zh/guides/runner.mdx +17 -7
  91. package/docs-site/zh/guides/sandbox-agent.mdx +57 -7
  92. package/docs-site/zh/guides/sandbox-providers.mdx +258 -10
  93. package/docs-site/zh/guides/scoring-guide.mdx +4 -4
  94. package/docs-site/zh/guides/viewing-results.mdx +85 -41
  95. package/docs-site/zh/guides/write-experiment.mdx +5 -3
  96. package/docs-site/zh/guides/write-send.mdx +19 -2
  97. package/docs-site/zh/index.mdx +1 -1
  98. package/docs-site/zh/reference/builtin-agents.mdx +27 -0
  99. package/docs-site/zh/reference/capabilities.mdx +2 -2
  100. package/docs-site/zh/reference/cli.mdx +35 -9
  101. package/docs-site/zh/reference/define-agent.mdx +60 -5
  102. package/docs-site/zh/reference/define-config.mdx +1 -1
  103. package/docs-site/zh/reference/define-eval.mdx +42 -9
  104. package/docs-site/zh/reference/events.mdx +2 -2
  105. package/docs-site/zh/reference/expect.mdx +36 -6
  106. package/package.json +5 -1
  107. package/src/agents/ai-sdk-otel.test.ts +1 -0
  108. package/src/agents/ai-sdk.test.ts +3 -0
  109. package/src/agents/ai-sdk.ts +3 -0
  110. package/src/agents/bub-install-spec.test.ts +34 -0
  111. package/src/agents/bub-install-spec.ts +32 -0
  112. package/src/agents/bub.ts +31 -32
  113. package/src/agents/claude-code.test.ts +130 -9
  114. package/src/agents/claude-code.ts +76 -4
  115. package/src/agents/codex.test.ts +189 -40
  116. package/src/agents/codex.ts +155 -14
  117. package/src/agents/coding-cli-versions.test.ts +15 -0
  118. package/src/agents/coding-cli-versions.ts +3 -0
  119. package/src/agents/index.ts +11 -0
  120. package/src/agents/langgraph.test.ts +204 -0
  121. package/src/agents/langgraph.ts +495 -0
  122. package/src/agents/marketplace.ts +85 -0
  123. package/src/agents/native-config.test.ts +179 -0
  124. package/src/agents/native-config.ts +267 -0
  125. package/src/agents/openai-compat.test.ts +1 -0
  126. package/src/agents/openclaw.test.ts +31 -0
  127. package/src/agents/openclaw.ts +171 -0
  128. package/src/agents/plugin-config.test.ts +1 -0
  129. package/src/agents/sdk-streams.test.ts +79 -0
  130. package/src/agents/sdk-streams.ts +55 -10
  131. package/src/agents/skills.test.ts +1 -0
  132. package/src/agents/streaming.test.ts +3 -9
  133. package/src/agents/types.ts +73 -6
  134. package/src/agents/ui-message-stream.test.ts +3 -0
  135. package/src/cli.ts +411 -108
  136. package/src/context/context.test.ts +51 -12
  137. package/src/context/context.ts +161 -29
  138. package/src/context/session.test.ts +1 -0
  139. package/src/context/session.ts +114 -6
  140. package/src/context/types.ts +30 -12
  141. package/src/define.test.ts +13 -8
  142. package/src/define.ts +25 -4
  143. package/src/expect/index.ts +53 -23
  144. package/src/i18n/en.ts +65 -4
  145. package/src/i18n/zh-CN.ts +66 -4
  146. package/src/o11y/cost.test.ts +1 -0
  147. package/src/o11y/execution-tree.test.ts +1 -20
  148. package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
  149. package/src/o11y/otlp/parse.test.ts +1 -0
  150. package/src/o11y/otlp/turn-otel.test.ts +1 -0
  151. package/src/o11y/parsers/bub.test.ts +1 -0
  152. package/src/o11y/parsers/claude-code.test.ts +1 -34
  153. package/src/o11y/parsers/openclaw.test.ts +154 -0
  154. package/src/o11y/parsers/openclaw.ts +310 -0
  155. package/src/o11y/prices.json +746 -311
  156. package/src/o11y/tool-names.test.ts +1 -0
  157. package/src/o11y/types.ts +16 -2
  158. package/src/report/aggregate.ts +34 -5
  159. package/src/report/built-in-user-parity.test.tsx +127 -173
  160. package/src/report/built-ins/experiment-comparison.tsx +179 -0
  161. package/src/report/built-ins/index.ts +7 -2
  162. package/src/report/components.tsx +11 -3
  163. package/src/report/compute.ts +80 -74
  164. package/src/report/dual-render.test.tsx +222 -91
  165. package/src/report/flag.ts +30 -2
  166. package/src/report/format.ts +36 -27
  167. package/src/report/index.ts +23 -6
  168. package/src/report/locale.ts +49 -6
  169. package/src/report/metrics.ts +68 -15
  170. package/src/report/primitives.tsx +6 -0
  171. package/src/report/react/AttemptList.tsx +9 -36
  172. package/src/report/react/EvalList.tsx +0 -0
  173. package/src/report/react/ExperimentComparison.tsx +68 -0
  174. package/src/report/react/ExperimentList.tsx +173 -55
  175. package/src/report/react/MetricScatter.tsx +13 -25
  176. package/src/report/react/chart-math.test.ts +85 -0
  177. package/src/report/react/chart-math.ts +101 -22
  178. package/src/report/react/enhance.js +72 -1
  179. package/src/report/react/fixtures.ts +34 -21
  180. package/src/report/react/format.ts +1 -1
  181. package/src/report/react/index.tsx +0 -1
  182. package/src/report/react/render.test.tsx +30 -69
  183. package/src/report/react/styles.css +112 -14
  184. package/src/report/report.test.ts +308 -105
  185. package/src/report/report.ts +6 -2
  186. package/src/report/text/faces.ts +111 -67
  187. package/src/report/text/table.ts +42 -5
  188. package/src/report/types.ts +42 -34
  189. package/src/results/annotated-source.test.ts +62 -9
  190. package/src/results/annotated-source.ts +64 -6
  191. package/src/results/attempt-evidence.test.ts +9 -7
  192. package/src/results/attempt-evidence.ts +15 -8
  193. package/src/results/attempt-source.ts +6 -3
  194. package/src/results/copy.ts +145 -55
  195. package/src/results/host-equivalence.test.ts +11 -9
  196. package/src/results/index.ts +2 -0
  197. package/src/results/locator.test.ts +1 -22
  198. package/src/results/open.ts +7 -1
  199. package/src/results/publish.ts +149 -0
  200. package/src/results/results.test.ts +85 -51
  201. package/src/results/truncate.ts +90 -0
  202. package/src/results/types.ts +7 -0
  203. package/src/results/writer.ts +31 -13
  204. package/src/runner/attempt.test.ts +138 -7
  205. package/src/runner/attempt.ts +603 -104
  206. package/src/runner/discover.test.ts +47 -0
  207. package/src/runner/discover.ts +36 -2
  208. package/src/runner/eval-source.test.ts +1 -27
  209. package/src/runner/feedback/agent.test.ts +504 -0
  210. package/src/runner/feedback/agent.ts +409 -0
  211. package/src/runner/feedback/ci.test.ts +562 -0
  212. package/src/runner/feedback/ci.ts +401 -0
  213. package/src/runner/feedback/coordinator.test.ts +317 -0
  214. package/src/runner/feedback/coordinator.ts +397 -0
  215. package/src/runner/feedback/failure.ts +40 -0
  216. package/src/runner/feedback/human.test.ts +616 -0
  217. package/src/runner/feedback/human.ts +535 -0
  218. package/src/runner/feedback/index.ts +66 -0
  219. package/src/runner/feedback/io.ts +78 -0
  220. package/src/runner/feedback/profile.test.ts +50 -0
  221. package/src/runner/feedback/profile.ts +58 -0
  222. package/src/runner/feedback/reducer.test.ts +395 -0
  223. package/src/runner/feedback/reducer.ts +260 -0
  224. package/src/runner/feedback/renderer.ts +82 -0
  225. package/src/runner/feedback/sink.ts +203 -0
  226. package/src/runner/feedback/testing.ts +106 -0
  227. package/src/runner/ledger.test.ts +230 -0
  228. package/src/runner/ledger.ts +329 -0
  229. package/src/runner/report.test.ts +128 -3
  230. package/src/runner/report.ts +33 -9
  231. package/src/runner/reporters/artifacts.ts +8 -2
  232. package/src/runner/reporters/braintrust.test.ts +8 -7
  233. package/src/runner/reporters/braintrust.ts +9 -2
  234. package/src/runner/reporters/index.ts +2 -2
  235. package/src/runner/reporters/json.test.ts +162 -0
  236. package/src/runner/reporters/json.ts +35 -8
  237. package/src/runner/reporters/shared.ts +1 -5
  238. package/src/runner/run.test.ts +760 -3
  239. package/src/runner/run.ts +242 -36
  240. package/src/runner/sandbox-prep.ts +3 -42
  241. package/src/runner/timing.ts +158 -0
  242. package/src/runner/types.ts +518 -22
  243. package/src/sandbox/checkpoint.test.ts +55 -0
  244. package/src/sandbox/checkpoint.ts +29 -8
  245. package/src/sandbox/cli-commands.ts +407 -0
  246. package/src/sandbox/docker.ts +115 -16
  247. package/src/sandbox/e2b-agent-template.test.ts +56 -0
  248. package/src/sandbox/e2b-agent-template.ts +94 -0
  249. package/src/sandbox/e2b.ts +74 -9
  250. package/src/sandbox/errors.ts +111 -4
  251. package/src/sandbox/index.ts +2 -0
  252. package/src/sandbox/io-retry.test.ts +58 -0
  253. package/src/sandbox/io-retry.ts +45 -0
  254. package/src/sandbox/keep-registry.test.ts +86 -0
  255. package/src/sandbox/keep-registry.ts +142 -0
  256. package/src/sandbox/keep.ts +178 -0
  257. package/src/sandbox/paths.test.ts +1 -0
  258. package/src/sandbox/paths.ts +19 -8
  259. package/src/sandbox/registry.ts +20 -3
  260. package/src/sandbox/resolve.ts +76 -11
  261. package/src/sandbox/retry.test.ts +70 -0
  262. package/src/sandbox/retry.ts +46 -4
  263. package/src/sandbox/types.ts +44 -6
  264. package/src/sandbox/vercel.ts +43 -20
  265. package/src/scoring/collector.ts +60 -17
  266. package/src/scoring/coverage.ts +95 -0
  267. package/src/scoring/diff.ts +81 -0
  268. package/src/scoring/display.test.ts +121 -0
  269. package/src/scoring/display.ts +133 -0
  270. package/src/scoring/evidence.test.ts +189 -0
  271. package/src/scoring/judge.test.ts +142 -0
  272. package/src/scoring/judge.ts +15 -18
  273. package/src/scoring/scoped.ts +217 -50
  274. package/src/scoring/types.ts +117 -20
  275. package/src/scoring/verdict.ts +16 -4
  276. package/src/shared/aggregate.ts +3 -2
  277. package/src/shared/types.ts +31 -0
  278. package/src/show/compose.ts +2 -2
  279. package/src/show/index.ts +29 -16
  280. package/src/show/render.ts +626 -308
  281. package/src/show/show.test.ts +251 -36
  282. package/src/tty-line.ts +8 -26
  283. package/src/util.test.ts +1 -0
  284. package/src/util.ts +41 -0
  285. package/src/view/app/components/AttemptModal.tsx +153 -2
  286. package/src/view/app/components/CodeView.tsx +32 -11
  287. package/src/view/app/components/CopyControls.tsx +2 -2
  288. package/src/view/app/i18n.ts +6 -0
  289. package/src/view/app/lib/attempt-route.test.ts +1 -0
  290. package/src/view/app/lib/verdict.ts +7 -9
  291. package/src/view/artifact-serving.test.ts +2 -1
  292. package/src/view/client-dist/app.css +1 -1
  293. package/src/view/client-dist/app.js +17 -17
  294. package/src/view/data.test.ts +2 -1
  295. package/src/view/data.ts +17 -7
  296. package/src/view/index.ts +12 -1
  297. package/src/view/server.ts +2 -0
  298. package/src/view/shared/types.ts +1 -1
  299. package/src/view/styles.css +3 -0
  300. package/src/view/view-report.test.ts +11 -10
  301. package/dist/o11y/execution-tree.d.ts +0 -103
  302. package/dist/o11y/otlp/select.d.ts +0 -22
  303. package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +0 -1
  304. package/dist/report/built-ins/cost-pass-rate-comparison.js +0 -17
  305. package/dist/results/annotated-source.d.ts +0 -61
  306. package/dist/results/attempt-evidence.d.ts +0 -69
  307. package/dist/results/attempt-source.d.ts +0 -15
  308. package/src/report/built-ins/cost-pass-rate-comparison.tsx +0 -23
  309. package/src/runner/reporters/console.ts +0 -70
  310. package/src/runner/reporters/live.test.ts +0 -56
  311. package/src/runner/reporters/live.ts +0 -247
  312. package/src/runner/reporters/quiet.test.ts +0 -66
  313. package/src/runner/reporters/quiet.ts +0 -49
  314. package/src/runner/reporters/table.ts +0 -277
@@ -4,9 +4,9 @@
4
4
  // 零 react、零 IO、纯同步 —— 这是 text 宿主不需要 react-dom 的那一半。
5
5
  // chrome 文案(注脚、verdict 词、截断提示)经 ctx.locale 查 locale 字典,
6
6
  // 默认 en 与历史输出逐字一致;数据(display、键、warnings message)不本地化。
7
- import { attemptItemReason, capabilityBadge, formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD, verdictMark, } from "../format.js";
7
+ import { attemptItemReason, experimentDisplayName, formatDurationMs, formatMetricValue, formatPlainNumber, formatUSD, verdictMark, } from "../format.js";
8
8
  import { countText, localeText, resolveMetricLabel } from "../locale.js";
9
- import { indentBlock, padDisplay, textBar, wrapDisplay } from "./layout.js";
9
+ import { padDisplay, stringWidth, textBar, wrapDisplay } from "./layout.js";
10
10
  import { renderTableText } from "./table.js";
11
11
  import { renderCharPlot, renderCoordinateTable } from "./plot.js";
12
12
  const MISSING_MARK = "—";
@@ -290,10 +290,6 @@ export function scatterText(data, ctx) {
290
290
  if (drawable.length === 0) {
291
291
  return [localeText(locale, "scatter.noData", axes), ...footnotes].join("\n");
292
292
  }
293
- // 恰好 1 个可画点:成本 × 通过率的比较至少要两个实验,单点不成图。
294
- if (drawable.length === 1) {
295
- return [localeText(locale, "scatter.needTwo", axes), ...footnotes].join("\n");
296
- }
297
293
  // 点太密排不下时降级为坐标表,不硬挤
298
294
  if (drawable.length > POINT_MARKS.length || ctx.width < 44) {
299
295
  const table = renderCoordinateTable(drawable.map((r) => ({ key: r.key, x: r.x.display, y: r.y.display })), { key: data.points, x: axisLabel(data.x, locale), y: axisLabel(data.y, locale) });
@@ -400,45 +396,101 @@ export function deltaText(data, ctx) {
400
396
  }
401
397
  // ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
402
398
  //
403
- // 三面共用的紧凑标记:`locator✓[E,X,⏱]`(判定符紧跟 locator,证据能力方括号紧跟判定符,
404
- // 中间不留空格)——docs-site/zh/guides/report-components.mdx「终端输出形成反馈闭环」定的形态。
399
+ // 三面共用的紧凑标记:`locator✓`(判定符紧跟 locator,中间不留空格)。
405
400
  // ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的原因/耗时摘要,不重复整段
406
401
  // niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼 `niceeval show <locator>`——
407
- // 命令模板只在 AttemptList(叶子层)展示完整断言明细时才值得,不在中间层重复。
402
+ // 比较列表都不内联完整断言或命令模板。
408
403
  function locatorBadge(item) {
409
- return `${item.locator}${verdictMark(item.verdict)}${capabilityBadge(item.capabilities)}`;
404
+ return `${item.locator}${verdictMark(item.verdict)}`;
410
405
  }
411
406
  // ── ExperimentList ──
412
- function experimentListEvalLine(row) {
413
- const badges = row.attempts.map(locatorBadge).join(" ");
414
- const trailer = row.verdict === "passed"
415
- ? [formatDurationMs(row.duration.value ?? 0), row.cost.value === null ? undefined : formatUSD(row.cost.value)]
416
- .filter((s) => s !== undefined)
417
- .join(" · ")
418
- : (row.reason ?? "");
419
- return ` ${verdictMark(row.verdict)} ${row.evalId} ${badges} ${trailer}`;
407
+ function experimentSummaryTable(items, ctx, relativeTo) {
408
+ const locale = ctx.locale;
409
+ const compact = ctx.width < 100;
410
+ const columns = [
411
+ { key: "experiment", header: compact && locale === "en" ? "Exp." : localeText(locale, "experimentList.experiment") },
412
+ { key: "model", header: localeText(locale, "table.model") },
413
+ { key: "agent", header: localeText(locale, "table.agent") },
414
+ { key: "duration", header: compact && locale === "en" ? "Avg" : localeText(locale, "experimentList.avgDuration"), align: "right" },
415
+ { key: "passRate", header: compact && locale === "en" ? "Pass" : localeText(locale, "experimentList.passRate"), align: "right" },
416
+ { key: "result", header: localeText(locale, "experimentList.result") },
417
+ { key: "tokens", header: localeText(locale, "experimentList.tokens"), align: "right" },
418
+ { key: "cost", header: compact && locale === "en" ? "Cost" : localeText(locale, "experimentList.estimatedCost"), align: "right" },
419
+ ];
420
+ const rows = items.map((item) => ({
421
+ key: item.experimentId,
422
+ cells: {
423
+ experiment: experimentDisplayName(item.experimentId, relativeTo),
424
+ model: item.model ?? localeText(locale, "experimentList.defaultModel"),
425
+ agent: item.agent,
426
+ duration: cellText(item.duration),
427
+ passRate: cellText(item.passRate),
428
+ result: verdictTallyText(item.verdicts, locale),
429
+ tokens: cellText(item.tokens),
430
+ cost: cellText(item.cost),
431
+ },
432
+ }));
433
+ const metadata = items.flatMap((item) => wrapDisplay(`${experimentDisplayName(item.experimentId, relativeTo)}: ${localeText(locale, "overview.evalsCount", { n: item.evals })} · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })} · ${item.lastRunAt}`, Math.max(8, ctx.width - 2)).map((line) => ` ${line}`));
434
+ return [renderTableText({ columns, rows, locale }, ctx), metadata.join("\n")].join("\n");
420
435
  }
421
- export function experimentListText(items, ctx) {
436
+ function experimentDetailTable(item, ctx, relativeTo) {
422
437
  const locale = ctx.locale;
423
- if (items.length === 0)
424
- return localeText(locale, "attemptList.empty");
425
- const blocks = items.map((item) => {
426
- const identity = item.model ? `${item.experimentId} · ${item.agent} · ${item.model}` : `${item.experimentId} · ${item.agent}`;
427
- const summary = [
428
- `${localeText(locale, "overview.passRate")} ${cellText(item.passRate)}`,
429
- verdictTallyText(item.verdicts, locale),
430
- localeText(locale, "overview.attemptsCount", { n: item.attempts }),
431
- formatDurationMs(item.duration.value ?? 0),
432
- item.cost.value === null ? missingText(locale) : formatUSD(item.cost.value),
433
- ].join(" · ");
434
- const evalLines = item.evalRows.map(experimentListEvalLine);
435
- return [identity, ` ${summary}`, ...evalLines].join("\n");
438
+ const columns = [
439
+ { key: "status", header: localeText(locale, "experimentList.status") },
440
+ { key: "entity", header: localeText(locale, "experimentList.evalAttempt") },
441
+ // Result 是可扫读的失败预览,不是证据面:两行放不下的以 收口,完整值走 locator 下钻。
442
+ { key: "result", header: localeText(locale, "experimentList.result"), maxLines: 2 },
443
+ { key: "duration", header: localeText(locale, "experimentList.duration"), align: "right" },
444
+ { key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
445
+ ];
446
+ // Result 的字符预算 ≈ 两行 × 它能分到的列宽(总宽减其它列的自然宽与列距)。这里只做
447
+ // 优先级让位(标题先截、received 最后截)的粗预算;精确的按宽度收口由列的 maxLines 兜底。
448
+ const statusWidth = Math.max(stringWidth(localeText(locale, "experimentList.status")), ...item.evalRows.map((row) => stringWidth(`${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`)));
449
+ const entityWidth = Math.max(stringWidth(localeText(locale, "experimentList.evalAttempt")), ...item.evalRows.flatMap((row) => [stringWidth(row.evalId), ...row.attempts.map((a) => stringWidth(a.locator) + 3)]));
450
+ const fixedWidth = statusWidth + entityWidth + 8 /* duration */ + 6 /* cost */ + 3 * 4; /* 4 段列距 */
451
+ const resultBudget = Math.max(24, (ctx.width - fixedWidth) * 2);
452
+ const rows = item.evalRows.flatMap((row) => {
453
+ const parent = {
454
+ key: row.evalId,
455
+ cells: {
456
+ status: `${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`,
457
+ entity: row.evalId,
458
+ result: "",
459
+ duration: localeText(locale, "entityList.average", { value: cellText(row.duration) }),
460
+ cost: localeText(locale, "entityList.average", { value: cellText(row.cost) }),
461
+ },
462
+ };
463
+ const attempts = row.attempts.map((attempt, index) => ({
464
+ key: attempt.locator,
465
+ cells: {
466
+ status: ` ${verdictMark(attempt.verdict)}`,
467
+ entity: `${index === row.attempts.length - 1 ? "└─" : "├─"} ${attempt.locator}`,
468
+ result: attemptItemReason(attempt, resultBudget) ?? MISSING_MARK,
469
+ duration: attempt.verdict === "skipped" && attempt.durationMs === 0 ? null : formatDurationMs(attempt.durationMs),
470
+ cost: attempt.costUSD === undefined ? null : formatUSD(attempt.costUSD),
471
+ },
472
+ }));
473
+ return [parent, ...attempts];
436
474
  });
437
- return blocks.join("\n\n");
475
+ const flags = item.flags && Object.keys(item.flags).length > 0
476
+ ? `${localeText(locale, "experimentList.flags")} ${Object.entries(item.flags).map(([key, value]) => `${key}=${String(value)}`).join(" · ")}`
477
+ : undefined;
478
+ return [experimentDisplayName(item.experimentId, relativeTo), flags, renderTableText({ columns, rows, locale }, ctx)]
479
+ .filter(Boolean)
480
+ .join("\n");
481
+ }
482
+ export function experimentListText(items, ctx, relativeTo) {
483
+ if (items.length === 0)
484
+ return localeText(ctx.locale, "attemptList.empty");
485
+ return [
486
+ experimentSummaryTable(items, ctx, relativeTo),
487
+ ...items.map((item) => experimentDetailTable(item, ctx, relativeTo)),
488
+ ].join("\n\n");
438
489
  }
439
490
  // ── EvalList ──
440
- function evalListAttemptLine(item) {
441
- const reason = attemptItemReason(item);
491
+ function evalListAttemptLine(item, ctx) {
492
+ // 行式列表同守「Result 最多两行」:预算 = 两行终端宽,超出由 fit 投影按优先级让位。
493
+ const reason = attemptItemReason(item, Math.max(24, ctx.width * 2 - stringWidth(locatorBadge(item)) - 6));
442
494
  return ` ${locatorBadge(item)}${reason ? ` · ${reason}` : ""}`;
443
495
  }
444
496
  export function evalListText(items, ctx) {
@@ -450,52 +502,39 @@ export function evalListText(items, ctx) {
450
502
  const summary = [
451
503
  localeText(locale, "attemptList.score", { score: cellText(item.score) }),
452
504
  localeText(locale, "overview.attemptsCount", { n: item.attempts.length }),
453
- `${formatDurationMs(item.duration.value ?? 0)} avg`,
454
- item.cost.value === null ? `${missingText(locale)} avg` : `${formatUSD(item.cost.value)} avg`,
505
+ localeText(locale, "entityList.average", {
506
+ value: item.duration.value === null ? missingText(locale) : formatDurationMs(item.duration.value),
507
+ }),
508
+ localeText(locale, "entityList.average", {
509
+ value: item.cost.value === null ? missingText(locale) : formatUSD(item.cost.value),
510
+ }),
455
511
  ].join(" · ");
456
- const attemptLines = item.attempts.map(evalListAttemptLine);
512
+ const attemptLines = item.attempts.map((attempt) => evalListAttemptLine(attempt, ctx));
457
513
  return [identity, ` ${summary}`, ...attemptLines].join("\n");
458
514
  });
459
515
  return blocks.join("\n\n");
460
516
  }
461
517
  // ── AttemptList ──
462
- /** 一个 AttemptListItem 的完整 text 卡片:判定符 + locator + 身份 + 耗时/成本 + 证据能力,
463
- * 然后逐条断言(gate 与 soft 都列,与 web 面的 AttemptRow 同一份材料) */
464
- function attemptListItemText(item, ctx, locale) {
518
+ /** Attempt 比较卡片:只显示一条主失败摘要(至多两行终端宽);完整 assertions locator 下钻。 */
519
+ function attemptListItemText(item, ctx) {
465
520
  const head = [
466
521
  `${verdictMark(item.verdict)} ${item.locator}`,
467
522
  item.evalId,
468
523
  item.experimentId,
469
524
  formatDurationMs(item.durationMs),
470
525
  ...(item.costUSD !== undefined ? [formatUSD(item.costUSD)] : []),
471
- ...(capabilityBadge(item.capabilities) ? [capabilityBadge(item.capabilities)] : []),
472
526
  ].join(" · ");
473
527
  const lines = [head];
474
- if (item.error) {
475
- lines.push(indentBlock(wrapDisplay(item.error, ctx.width - 4).join("\n"), " "));
476
- }
477
- for (const assertion of item.assertions) {
478
- const scoreText = assertion.threshold !== undefined
479
- ? `${formatPlainNumber(assertion.score)}/${formatPlainNumber(assertion.threshold)}`
480
- : formatPlainNumber(assertion.score);
481
- lines.push(` ${assertion.severity} ${assertion.name} · ${localeText(locale, `verdict.${assertion.passed ? "passed" : "failed"}`)}${assertion.severity === "soft" ? ` ${scoreText}` : ""}`);
482
- if (assertion.detail)
483
- lines.push(indentBlock(wrapDisplay(assertion.detail, ctx.width - 4).join("\n"), " "));
484
- if (assertion.evidence) {
485
- const limit = Math.max(240, ctx.width * 5);
486
- const evidence = assertion.evidence.length <= limit
487
- ? assertion.evidence
488
- : `${assertion.evidence.slice(0, limit)}… (${assertion.evidence.length - limit} more chars; open ${item.locator} for full evidence)`;
489
- lines.push(indentBlock(wrapDisplay(evidence, ctx.width - 6).join("\n"), " "));
490
- }
491
- }
528
+ const reason = attemptItemReason(item, Math.max(24, ctx.width * 2 - 4));
529
+ if (reason)
530
+ lines.push(` ${reason}`);
492
531
  return lines.join("\n");
493
532
  }
494
533
  export function attemptListText(items, total, ctx) {
495
534
  const locale = ctx.locale;
496
535
  if (items.length === 0)
497
536
  return localeText(locale, "attemptList.empty");
498
- const blocks = items.map((item) => attemptListItemText(item, ctx, locale));
537
+ const blocks = items.map((item) => attemptListItemText(item, ctx));
499
538
  const remaining = (total ?? items.length) - items.length;
500
539
  if (remaining > 0)
501
540
  blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
@@ -6,7 +6,7 @@
6
6
  // 列宽按显示宽度算(CJK 记 2 列);null 渲染 —,不补 0;超宽先折最宽的左对齐列,
7
7
  // 压到下限仍放不下就从右侧丢列并如实报数(「截断报剩余」是既有契约,不在这里破例)。
8
8
  import { countText, localeText } from "../locale.js";
9
- import { renderAlignedRows, stringWidth, wrapDisplay } from "./layout.js";
9
+ import { charDisplayWidth, renderAlignedRows, stringWidth, wrapDisplay } from "./layout.js";
10
10
  const MISSING_MARK = "—";
11
11
  /** 列间距,与 renderAlignedRows 的 join(" ") 一致。 */
12
12
  const COLUMN_GAP = 3;
@@ -47,12 +47,36 @@ function fitWidths(natural, align, available) {
47
47
  }
48
48
  return { widths, hidden };
49
49
  }
50
- /** 逻辑行 物理行:每格折到自己的列宽,行高取最高的那格,矮格补空串。 */
51
- function toPhysicalRows(cells, widths) {
50
+ /** 按显示宽度截到 width(不够放 `…` 时原样返回)。 */
51
+ function truncateDisplay(text, width) {
52
+ if (stringWidth(text) <= width)
53
+ return text;
54
+ let out = "";
55
+ let used = 0;
56
+ for (const ch of text) {
57
+ const w = charDisplayWidth(ch.codePointAt(0));
58
+ if (used + w > width)
59
+ break;
60
+ out += ch;
61
+ used += w;
62
+ }
63
+ return out;
64
+ }
65
+ /** 列的 maxLines 收口:超出的行丢弃,最后一行以 `…` 如实标注被收口。 */
66
+ function clampCellLines(lines, maxLines, width) {
67
+ if (maxLines === undefined || lines.length <= maxLines)
68
+ return lines;
69
+ const kept = lines.slice(0, Math.max(1, maxLines));
70
+ const last = kept[kept.length - 1];
71
+ kept[kept.length - 1] = `${truncateDisplay(last, Math.max(1, width - 1))}…`;
72
+ return kept;
73
+ }
74
+ /** 逻辑行 → 物理行:每格折到自己的列宽,列带 maxLines 就收口,行高取最高的那格,矮格补空串。 */
75
+ function toPhysicalRows(cells, widths, maxLines) {
52
76
  const wrapped = cells.map((cell, c) =>
53
77
  // 放得下就原样保留:wrapDisplay 会把连续空格折成一个,而 DeltaTable 的
54
78
  // "50% → 62% +12pp" 这类格子内含固定间距 —— 不折行时一个字节都不能动。
55
- !cell.includes("\n") && stringWidth(cell) <= widths[c] ? [cell] : wrapDisplay(cell, widths[c]));
79
+ clampCellLines(!cell.includes("\n") && stringWidth(cell) <= widths[c] ? [cell] : wrapDisplay(cell, widths[c]), maxLines[c], widths[c]));
56
80
  const height = Math.max(...wrapped.map((lines) => lines.length), 1);
57
81
  const out = [];
58
82
  for (let i = 0; i < height; i++)
@@ -80,7 +104,14 @@ export function renderTableText(props, ctx) {
80
104
  const matrix = [header, ...body];
81
105
  const natural = header.map((_, c) => Math.max(...matrix.map((row) => stringWidth(row[c] ?? ""))));
82
106
  const { widths, hidden } = fitWidths(natural, align, ctx.width);
83
- const physical = matrix.flatMap((row) => toPhysicalRows(row.slice(0, widths.length), widths));
107
+ const maxLines = props.columns.map((column) => column.maxLines);
108
+ if (hasLocator)
109
+ maxLines.push(undefined);
110
+ // 表头不参与 maxLines 收口:表头是自己写的短词,收口只服务数据格。
111
+ const physical = [
112
+ ...toPhysicalRows(header.slice(0, widths.length), widths, widths.map(() => undefined)),
113
+ ...body.flatMap((row) => toPhysicalRows(row.slice(0, widths.length), widths, maxLines)),
114
+ ];
84
115
  const table = renderAlignedRows(physical, align);
85
116
  return hidden > 0 ? `${table}\n${countText(locale, "table.columnsHidden", hidden)}` : table;
86
117
  }
@@ -1,10 +1,9 @@
1
1
  import type { AttemptHandle, SelectionWarning } from "../results/types.ts";
2
2
  import type { AttemptLocator } from "../results/locator.ts";
3
- import type { AttemptEvidenceCapabilities } from "../results/attempt-evidence.ts";
4
- import type { AssertionResult, Verdict } from "../types.ts";
3
+ import type { AssertionResult, AttemptError, DiagnosticRecord, Verdict } from "../types.ts";
5
4
  import type { LocalizedLabel, ReportLocale } from "./locale.ts";
6
5
  export type { SelectionWarning };
7
- export type { AttemptLocator, AttemptEvidenceCapabilities };
6
+ export type { AttemptLocator };
8
7
  export type { LocalizedLabel, ReportLocale };
9
8
  /** 两级聚合里单级的折叠方式。 */
10
9
  export type Aggregator = "mean" | "sum" | "min" | "max" | ((values: number[]) => number);
@@ -22,7 +21,7 @@ export interface MetricAggregate {
22
21
  * 指标:纯函数,吃一个 AttemptHandle 吐一个值(null = 此 attempt 测不了这个指标,
23
22
  * 不进聚合;0 = 测了结果是零,照常进),外加名字、两级聚合方式和渲染提示。
24
23
  * 内置指标与自定义指标是同一个类型,没有特权。name 走字面量泛型:列键锚在指标
25
- * 对象上(`row.cells[passRate.name]`),拼错列名编译不过。
24
+ * 对象上(`row.cells[taskPassRate.name]`),拼错列名编译不过。
26
25
  */
27
26
  export interface Metric<Name extends string = string> {
28
27
  /** MetricColumn.key 与列头的来源;同一次计算里重名是错误。 */
@@ -68,8 +67,22 @@ export interface FlagRef {
68
67
  readonly label?: string | ((value: string | number | boolean) => string);
69
68
  readonly unit?: string;
70
69
  }
71
- /** 维度槽的输入:内置/自定义维度,或 experiment 声明的 flag。 */
72
- export type DimensionInput = Dimension | FlagRef;
70
+ /**
71
+ * config() 的产物:把顶层运行配置(快照 `ExperimentRunInfo` 投影的字段全集,外加桥接到
72
+ * 快照顶层权威字段的 `model` / `agent` 两个键)当维度或轴,槽位用法与 {@link FlagRef} 一致。
73
+ * 未投影的值不猜:分组如实归「(unset)」,作轴不画点、注脚报数。
74
+ */
75
+ export interface ConfigRef {
76
+ readonly kind: "config";
77
+ readonly name: string;
78
+ /** 组标签 / 轴标签;函数形态把投影值折成组名。 */
79
+ readonly label?: string | ((value: string | number | boolean) => string);
80
+ readonly unit?: string;
81
+ }
82
+ /** MetricLine 的 x 轴输入:experiment 声明的 flag,或顶层运行配置(config())。 */
83
+ export type AxisInput = FlagRef | ConfigRef;
84
+ /** 维度槽的输入:内置/自定义维度、experiment 声明的 flag,或顶层运行配置(config())。 */
85
+ export type DimensionInput = Dimension | FlagRef | ConfigRef;
73
86
  export interface MetricColumn {
74
87
  /** = metric.name,与 cells 的键对应。 */
75
88
  key: string;
@@ -259,7 +272,7 @@ export interface GroupSummaryData {
259
272
  /**
260
273
  * 组的通过率:eval 级折叠计票的 `passed / (passed + failed + errored)`(`skipped` 不进
261
274
  * 分母)——这是旧 `GroupSelector` 卡片的口径,不是 `OverviewData.totals.passRate` 那种
262
- * `computeCell` 两级聚合(两者服务不同问题:「这组题多少算过」vs「整体质量几分」)。
275
+ * `computeCell` 两级聚合(两者服务不同问题:「这组题多少算过」vs「每次运行成功多少」)。
263
276
  * 分母为 0(组内没有任何已跑的 eval)时 `value` 为 `null`,不编 0%。
264
277
  */
265
278
  passRate: MetricCell;
@@ -287,10 +300,11 @@ export interface OverviewData {
287
300
  errored: number;
288
301
  skipped: number;
289
302
  /**
290
- * 通过率的唯一官方口径:`computeCell(passRate, items)`,与 `MetricTable.data(...,
291
- * columns: [passRate])` 同一台两级聚合引擎(题内折叠 perEval、跨题折叠 across,默认都是
303
+ * 默认成功率的唯一官方口径:`computeCell(endToEndPassRate, items)`,与 `MetricTable.data(...,
304
+ * columns: [endToEndPassRate])` 同一台两级聚合引擎(题内折叠 perEval、跨题折叠 across,默认都是
292
305
  * mean)——一道题内多个 attempt 部分通过,贡献的是小数份额而不是二元票。`samples`/`total`
293
- * 是两级聚合口径下的 attempt 计数(`total` 含 skipped,`samples` 不含),不等于上面四个
306
+ * 是两级聚合口径下的 attempt 计数(`total` 含 skipped 与 errored——endToEndPassRate 对
307
+ * errored 记 0、只对 skipped 记 null,`samples` 因此只不含 skipped),不等于上面四个
294
308
  * verdict 计票的任何一个之和。
295
309
  */
296
310
  passRate: MetricCell;
@@ -325,10 +339,11 @@ export interface DeltaData<K extends string = string> {
325
339
  }[];
326
340
  }
327
341
  /**
328
- * `AttemptList` 一项 = 一个 Attempt:身份、判定、断言、error、耗时、成本、locator,外加
329
- * 证据能力标记(有没有保存的 Eval 源码 / 执行事件 / OTel 计时 / diff——与 `AttemptEvidence.capabilities`
330
- * 同一个类型,不重新发明四个布尔位的含义)。`ExperimentList` / `EvalList` 的下钻数组复用同一个
331
- * 类型,不是各自的精简版。
342
+ * `AttemptList` 一项 = 一个 Attempt:身份、判定、断言、结构化 error、diagnostics、耗时、
343
+ * 成本和 locator。`ExperimentList` / `EvalList` 的下钻数组复用同一个类型,不是各自的精简版。
344
+ * 渲染面只显示一条主失败断言摘要或 error 的一层摘要;完整 assertions、cause / stack
345
+ * diagnostics 属于 locator 下钻详情,不塞进比较列表,但随数据携带 —— `AttemptList.data` 的 `redact`
346
+ * 钩子覆盖它们的自由文本(见 docs/feature/reports/library.md「AttemptList」)。
332
347
  */
333
348
  export interface AttemptListItem {
334
349
  evalId: string;
@@ -336,32 +351,24 @@ export interface AttemptListItem {
336
351
  attempt: number;
337
352
  agent: string;
338
353
  verdict: Verdict;
339
- error?: string;
354
+ /** 结构化执行错误(与 `EvalResult.error` 同构):列表只显示 `message` 一层摘要。 */
355
+ error?: AttemptError;
356
+ /** 本 attempt 的有界诊断(teardown / cleanup 失败等,与 verdict 独立);属于下钻详情。 */
357
+ diagnostics?: DiagnosticRecord[];
340
358
  assertions: AssertionResult[];
341
359
  durationMs: number;
342
360
  costUSD?: number;
343
361
  locator: AttemptLocator;
344
- /**
345
- * 证据能力标记:`eval`(运行时 Eval 源码已保存)、`execution`(执行事件非空)、
346
- * `timing`(执行事件 + 这次运行接入过 OTel)、`diff`(工作区有文件改动)。逐位定义与
347
- * `AttemptEvidence.capabilities` 完全一致——这里复用同一个类型,不重复声明四个布尔位的门槛。
348
- * 计算函数从 `EvalResult` 上已有的 `hasEvents` / `hasSources` / `hasTrace` 摘要位与一次
349
- * `attempt.diff()` 懒加载算出,不对每个 item 调用完整的 `loadAttemptEvidence`
350
- * (那还会额外装配 Eval 源码标注与 ExecutionTree,这里只要四个布尔位)。
351
- */
352
- capabilities: AttemptEvidenceCapabilities;
353
362
  }
354
363
  /**
355
- * `ExperimentList` 一项里,一个 Eval 的展开行:折叠判定(`foldEvalVerdict`)、失败原因摘要
356
- * (`error` `skipReason` → 未通过的 gate 断言,`reasonFor` 的口径,soft 断言永不进入)
357
- * 该 Eval 内 attempt 的平均耗时/成本(两级聚合引擎在单一 eval 上退化成组内均值),以及这道题
358
- * 的全部 Attempt(升序,供进一步展开到 `AttemptList`)。
364
+ * `ExperimentList` 一项里,一个 Eval 的展开行:折叠判定(`foldEvalVerdict`)、该 Eval 内
365
+ * attempt 的平均耗时/成本(两级聚合引擎在单一 eval 上退化成组内均值),以及这道题的全部
366
+ * Attempt(升序,供进一步展开到 `AttemptList`)。失败原因属于各 Attempt,不在父行挑一轮重复。
359
367
  */
360
368
  export interface ExperimentListEvalRow {
361
369
  evalId: string;
362
370
  /** 折叠判定(任一 attempt 通过则通过,否则取最严重的)。 */
363
371
  verdict: Verdict;
364
- reason?: string;
365
372
  /** 这道题内 attempt 的平均耗时(`computeCell(durationMs, …)`,单一 eval 分组下即均值)。 */
366
373
  duration: MetricCell;
367
374
  /** 这道题内 attempt 的平均成本。 */
@@ -372,7 +379,7 @@ export interface ExperimentListEvalRow {
372
379
  /**
373
380
  * `ExperimentList.data(selection)` 的一项 = 一个 experiment:身份(experimentId/agent/model)、
374
381
  * 声明的 flags、Eval 判定构成(`foldEvalVerdict` 计票,与 view 榜单同一口径)、官方两级聚合
375
- * 汇总指标(passRate/cost/duration/tokens,直接来自 `computeCell`,不现场重算),以及展开到
382
+ * 汇总指标(endToEndPassRate/cost/duration/tokens,直接来自 `computeCell`,不现场重算),以及展开到
376
383
  * 这个 experiment 每道 Eval 的 `evalRows`(按 eval id 升序)。
377
384
  */
378
385
  export interface ExperimentListItem {
@@ -387,7 +394,7 @@ export interface ExperimentListItem {
387
394
  errored: number;
388
395
  skipped: number;
389
396
  };
390
- /** 官方两级聚合口径,与 `MetricTable.data(..., columns: [passRate])` 同一台引擎。 */
397
+ /** 官方两级聚合口径(endToEndPassRate),与 `MetricTable.data(..., columns: [endToEndPassRate])` 同一台引擎。 */
391
398
  passRate: MetricCell;
392
399
  cost: MetricCell;
393
400
  duration: MetricCell;
@@ -404,14 +411,13 @@ export interface ExperimentListItem {
404
411
  /**
405
412
  * `EvalList.data(selection)` 的一项 = 一个 `experimentId + evalId`(同一个 Eval 跑在两个
406
413
  * experiment 上是两条不同结果,不合并)。判定、分数(examScore 的两级聚合)、这道题内 attempt
407
- * 的平均耗时/成本,失败原因摘要(与 `ExperimentListEvalRow.reason` 同一口径),外加展开到这道题
408
- * 全部 Attempt`attempts`(按 attempt 序号升序)。
414
+ * 的平均耗时/成本,外加展开到这道题全部 Attempt 的 `attempts`(按 attempt 序号升序)。失败
415
+ * 原因只存在于各 `AttemptListItem`error / assertions,不会在 Eval 父项重复一份。
409
416
  */
410
417
  export interface EvalListItem {
411
418
  evalId: string;
412
419
  experimentId: string;
413
420
  verdict: Verdict;
414
- reason?: string;
415
421
  /** examScore 的两级聚合;单一 eval 分组下即这道题的题级分数。 */
416
422
  score: MetricCell;
417
423
  duration: MetricCell;
@@ -33,6 +33,13 @@ export interface SnapshotMeta {
33
33
  completedAt?: string;
34
34
  /** 写入时刻该实验已知的 eval 并集 —— 残缺检测的分母随数据走(copySnapshots 自动补记,writer 可声明)。 */
35
35
  knownEvalIds?: string[];
36
+ /**
37
+ * 发布拷贝的自描述标记:copySnapshots 补记,消毒函数 → "applied"、redact: false → "none";
38
+ * 本地事实根没有此字段。只声明流程,不证明无秘密;view --out 据此分级防呆。
39
+ */
40
+ publish?: {
41
+ redaction: "applied" | "none";
42
+ };
36
43
  /** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
37
44
  name?: LocalizedText;
38
45
  }
@@ -110,6 +117,10 @@ export interface Snapshot {
110
117
  dir: string;
111
118
  /** 写入时刻该实验已知的 eval 并集(可选);copySnapshots 自动补记,writer.snapshot() 也可声明。 */
112
119
  knownEvalIds?: string[];
120
+ /** 发布拷贝的自描述标记(见 SnapshotMeta.publish);本地事实根没有此字段。 */
121
+ publish?: {
122
+ redaction: "applied" | "none";
123
+ };
113
124
  }
114
125
  /** 一个实验的全部历史:同一 experiment id 的历次快照归在一起。 */
115
126
  export interface Experiment {
@@ -0,0 +1,110 @@
1
+ import type { AttemptLifecycleEvent, AttemptRef, FailureDetail } from "../types.ts";
2
+ import type { Verdict } from "../../scoring/types.ts";
3
+ import type { JsonValue } from "../../shared/types.ts";
4
+ import type { AttemptLocator } from "../../results/locator.ts";
5
+ /** `sink.diagnostic()` 的输入 —— 与 `DurableFeedbackEvent` 的 "diagnostic" 变体字段一致,
6
+ * 只是省略 `type`/`at`(由 coordinator 补上)。 */
7
+ export interface DiagnosticInput {
8
+ /** 稳定去重 key —— 同一种 warning/error 用同一个 key(见 cli.md「同一 dedupeKey 并发出现时
9
+ * 只留一条并显示次数」),不要把可变的实例细节(如具体 sandbox id)编进 key 本身,
10
+ * 那些细节放 `data`。 */
11
+ key: string;
12
+ severity: "warning" | "error";
13
+ /** 一句话人类可读摘要;renderer 的 appendDurable 直接展示,不需要再解析。 */
14
+ message: string;
15
+ identity?: AttemptRef;
16
+ data?: Readonly<Record<string, JsonValue>>;
17
+ }
18
+ /** `sink.failure()` 的输入 —— 与 `DurableFeedbackEvent` 的 "failure" 变体字段一致,只省略
19
+ * `type`/`at`(由 coordinator 补上)。`locator` 只有在 attempt 挂靠 experiment 时才存在
20
+ * (见 `results/locator.ts` 的 `encodeAttemptLocator`);调用方(run.ts)只在拿到 locator 之后
21
+ * 才应该调用这个函数——没有 locator 的裸 run 不产出这类永久失败通知。 */
22
+ export type FailureInput = FailureDetail;
23
+ /** `sink.budgetExhausted()` 的输入 —— 与 `DurableFeedbackEvent` 的 "budget-exhausted" 变体字段
24
+ * 一致,只省略 `type`/`at`。调用方(run.ts)对每一个因预算到顶而不派发的 attempt 各调一次
25
+ * (与 `AttemptLifecycleEvent` 的 "attempt:early-exit" 同构);`unstarted` 是调用方自己维护的、
26
+ * 发出这条时的累计未派发数,不是 reducer 能推导的值(见 reducer.ts 对应事件的注释)。 */
27
+ export interface BudgetExhaustedInput {
28
+ experimentId: string;
29
+ spent: number;
30
+ unstarted: number;
31
+ }
32
+ /** `sink.kept()` 的输入 —— 与 `DurableFeedbackEvent` 的 "kept" 变体字段一致,省略 type/at。 */
33
+ export interface KeptInput {
34
+ locator: AttemptLocator;
35
+ identity: AttemptRef;
36
+ who: string;
37
+ verdict: Verdict;
38
+ provider: string;
39
+ sandboxId: string;
40
+ enter?: string;
41
+ }
42
+ /** `sink.ts` façade 函数实际转发到的最小接口 —— `FeedbackCoordinator`(coordinator.ts)实现它。
43
+ * 定义在这里(而不是从 coordinator.ts 导入)是为了让 sink.ts 不必在运行时依赖 coordinator.ts,
44
+ * 避免两个模块互相 import 造成的循环依赖 —— coordinator.ts 反过来 `import type` 这个接口。 */
45
+ export interface FeedbackSink {
46
+ activity(text: string): void;
47
+ diagnostic(input: DiagnosticInput): void;
48
+ interrupted(): void;
49
+ reporterError(input: {
50
+ reporter: string;
51
+ required: boolean;
52
+ message: string;
53
+ }): void;
54
+ /** 一次失败/errored attempt 的永久通知(见 `FailureInput`)。 */
55
+ failure(input: FailureInput): void;
56
+ /** 一个因预算到顶而未被派发的 attempt(见 `BudgetExhaustedInput`)。 */
57
+ budgetExhausted(input: BudgetExhaustedInput): void;
58
+ /** 一次留存授予(--keep-sandbox);见 `KeptInput` 与 docs/feature/sandbox/cli.md。 */
59
+ kept(input: KeptInput): void;
60
+ /** attempt 生命周期事件(queued/start/phase/progress/complete/early-exit),见
61
+ * `AttemptLifecycleEvent`。只驱动 human dashboard 的 active slot,不落 RunSummary/结果文件,
62
+ * 所以没有活跃 coordinator 时(见 `reportAttemptLifecycle`)静默丢弃是安全的 —— 这类信息
63
+ * 本身就只服务「正在画着的 dashboard」,不是必须留痕的诊断。 */
64
+ lifecycle(event: AttemptLifecycleEvent): void;
65
+ }
66
+ /**
67
+ * coordinator 进入「活跃」阶段时调用一次(见 coordinator.ts 的 start()),把自己注册为当前
68
+ * reportXxx() 调用的目的地。返回的函数用于退出活跃状态(见 coordinator.ts 的 finish())——
69
+ * 调用一次即失效,重复调用是安全的 no-op。
70
+ */
71
+ export declare function activateFeedbackSink(sink: FeedbackSink): () => void;
72
+ /** 仅供测试:断言没有遗留未退出的活跃 coordinator(避免一个测试忘记 finish() 污染下一个)。 */
73
+ export declare function activeFeedbackSinkCount(): number;
74
+ /** 不需要去重、不进入 RunFeedbackState 的瞬时活动文本(docker 镜像拉取进度、vercel session
75
+ * rotate 成功通知……)。没有活跃 coordinator 时退回一行 stderr,与迁移前的裸写视觉效果一致。 */
76
+ export declare function reportActivity(text: string): void;
77
+ /** 需要去重、要出现在三种 profile 永久事件流里的一条 warning/error。 */
78
+ export declare function reportDiagnostic(input: DiagnosticInput): void;
79
+ /** 一次失败/errored attempt 的永久通知(见 `FailureInput`)。与 `reportDiagnostic` 同级别的
80
+ * 「必须留痕」信号 —— 没有活跃 coordinator 时退回一行 stderr,不像 `reportAttemptLifecycle`
81
+ * 那样静默丢弃(定位一次真实失败不该因为没接 coordinator 就彻底没有出口)。 */
82
+ export declare function reportFailure(input: FailureInput): void;
83
+ /** 一个因预算到顶而未被派发的 attempt(见 `BudgetExhaustedInput`)。没有活跃 coordinator 时的
84
+ * 兜底文案与 coordinator.ts 的 `fallbackTextFor` 对 "budget-exhausted" 事件的格式化保持一致。 */
85
+ /** 一次留存授予的永久通知(--keep-sandbox);没有活跃 coordinator 时退回一行 stderr。 */
86
+ export declare function reportKept(input: KeptInput): void;
87
+ export declare function reportBudgetExhausted(input: BudgetExhaustedInput): void;
88
+ /** 用户中断(Ctrl+C)。没有活跃 coordinator 时的兜底文案与迁移前的 `runner.interrupted` 完全
89
+ * 相同 —— 调用方(run.ts)不再需要自己持有这段 i18n 文案。 */
90
+ export declare function reportInterrupted(): void;
91
+ /**
92
+ * runner 自己驱动的 attempt 生命周期投影(见 docs/feature/experiments/cli.md「Attempt 阶段」)——
93
+ * `src/runner/attempt.ts`(phase/progress)与 `src/runner/run.ts`(start/complete/early-exit)是
94
+ * 目前仅有的两个调用方。没有活跃 coordinator 时(今天:CLI 还没接入 --output)静默 no-op ——
95
+ * 与其它 reportXxx() 不同,这里不退回 `writeStderrLine` 兜底:这类事件只服务尚不存在的
96
+ * dashboard,不是需要留痕的诊断,现有可观察行为(直到 CLI 接入真正的 coordinator 之前)因此
97
+ * 完全不变。
98
+ */
99
+ export declare function reportAttemptLifecycle(event: AttemptLifecycleEvent): void;
100
+ /** 一个 reporter 的某次回调抛错(见 `runner/report.ts` 的 `runReporter`)。`reporter`/`required`
101
+ * 来自调用方注册这个 reporter 时的 `ReporterRegistration`(见其字段注释)——默认 artifacts、
102
+ * 显式 `--json`/`--junit` 传 `required: true`,用户 `config.reporters`/`EvalDef.reporters`
103
+ * 传 `required: false`。coordinator 把它折进 `RunFeedbackState.diagnostics`(reducer 按
104
+ * `reporter-error:<reporter>` 去重),调用方(`cli.ts` 的 `assembleRunCompletion`)据此把
105
+ * `required` 为真的失败折进 `RunCompletion.reporterErrors`,让 completion/CI 退出码判红。 */
106
+ export declare function reportReporterError(input: {
107
+ reporter: string;
108
+ required: boolean;
109
+ message: string;
110
+ }): void;