niceeval 0.6.1 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (296) hide show
  1. package/dist/agents/types.d.ts +67 -5
  2. package/dist/context/types.d.ts +32 -12
  3. package/dist/i18n/en.d.ts +54 -0
  4. package/dist/i18n/zh-CN.d.ts +55 -1
  5. package/dist/o11y/types.d.ts +16 -2
  6. package/dist/report/aggregate.d.ts +5 -3
  7. package/dist/report/aggregate.js +32 -5
  8. package/dist/report/built-ins/experiment-comparison.d.ts +39 -1
  9. package/dist/report/built-ins/experiment-comparison.js +116 -10
  10. package/dist/report/built-ins/index.d.ts +1 -0
  11. package/dist/report/built-ins/index.js +1 -1
  12. package/dist/report/components.d.ts +8 -2
  13. package/dist/report/components.js +3 -3
  14. package/dist/report/compute.d.ts +11 -18
  15. package/dist/report/compute.js +54 -34
  16. package/dist/report/flag.d.ts +16 -1
  17. package/dist/report/flag.js +19 -1
  18. package/dist/report/format.d.ts +16 -8
  19. package/dist/report/format.js +27 -12
  20. package/dist/report/index.d.ts +4 -3
  21. package/dist/report/index.js +5 -4
  22. package/dist/report/locale.d.ts +11 -2
  23. package/dist/report/locale.js +23 -5
  24. package/dist/report/metrics.d.ts +13 -1
  25. package/dist/report/metrics.js +65 -14
  26. package/dist/report/primitives.d.ts +6 -0
  27. package/dist/report/react/AttemptList.d.ts +2 -2
  28. package/dist/report/react/AttemptList.js +5 -6
  29. package/dist/report/react/EvalList.d.ts +1 -1
  30. package/dist/report/react/EvalList.js +0 -0
  31. package/dist/report/react/ExperimentComparison.d.ts +8 -0
  32. package/dist/report/react/ExperimentComparison.js +11 -0
  33. package/dist/report/react/ExperimentList.d.ts +2 -1
  34. package/dist/report/react/ExperimentList.js +8 -10
  35. package/dist/report/react/MetricScatter.js +5 -11
  36. package/dist/report/react/chart-math.d.ts +23 -6
  37. package/dist/report/react/chart-math.js +71 -19
  38. package/dist/report/react/fixtures.d.ts +3 -3
  39. package/dist/report/react/fixtures.js +21 -14
  40. package/dist/report/report.d.ts +5 -1
  41. package/dist/report/report.js +6 -2
  42. package/dist/report/text/faces.d.ts +1 -1
  43. package/dist/report/text/faces.js +42 -41
  44. package/dist/report/text/table.js +36 -5
  45. package/dist/report/types.d.ts +39 -21
  46. package/dist/results/types.d.ts +11 -0
  47. package/dist/runner/feedback/sink.d.ts +110 -0
  48. package/dist/runner/types.d.ts +513 -22
  49. package/dist/sandbox/docker.d.ts +23 -2
  50. package/dist/sandbox/e2b.d.ts +15 -1
  51. package/dist/sandbox/errors.d.ts +30 -3
  52. package/dist/sandbox/io-retry.d.ts +17 -0
  53. package/dist/sandbox/registry.d.ts +2 -0
  54. package/dist/sandbox/resolve.d.ts +18 -5
  55. package/dist/sandbox/retry.d.ts +11 -1
  56. package/dist/sandbox/types.d.ts +39 -5
  57. package/dist/sandbox/vercel.d.ts +7 -1
  58. package/dist/scoring/coverage.d.ts +30 -0
  59. package/dist/scoring/display.d.ts +21 -0
  60. package/dist/scoring/display.js +120 -0
  61. package/dist/scoring/types.d.ts +103 -20
  62. package/dist/shared/aggregate.d.ts +1 -0
  63. package/dist/shared/aggregate.js +3 -3
  64. package/dist/shared/types.d.ts +28 -0
  65. package/dist/tty-line.d.ts +0 -4
  66. package/dist/util.d.ts +23 -0
  67. package/docs-site/zh/concepts/adapter.mdx +22 -4
  68. package/docs-site/zh/concepts/experiment.mdx +1 -1
  69. package/docs-site/zh/concepts/overview.mdx +6 -6
  70. package/docs-site/zh/guides/agent-feedback-loop.mdx +28 -26
  71. package/docs-site/zh/guides/authoring.mdx +33 -0
  72. package/docs-site/zh/guides/ci-integration.mdx +23 -12
  73. package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
  74. package/docs-site/zh/guides/custom-reports.mdx +29 -34
  75. package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
  76. package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
  77. package/docs-site/zh/guides/debugging.mdx +210 -0
  78. package/docs-site/zh/guides/experiments.mdx +10 -3
  79. package/docs-site/zh/guides/official-adapters.mdx +26 -2
  80. package/docs-site/zh/guides/publish-report.mdx +30 -16
  81. package/docs-site/zh/guides/report-components.mdx +42 -30
  82. package/docs-site/zh/guides/reporters.mdx +2 -2
  83. package/docs-site/zh/guides/results-data.mdx +17 -9
  84. package/docs-site/zh/guides/runner.mdx +17 -7
  85. package/docs-site/zh/guides/sandbox-agent.mdx +56 -7
  86. package/docs-site/zh/guides/sandbox-providers.mdx +257 -9
  87. package/docs-site/zh/guides/scoring-guide.mdx +4 -4
  88. package/docs-site/zh/guides/viewing-results.mdx +79 -36
  89. package/docs-site/zh/guides/write-experiment.mdx +5 -3
  90. package/docs-site/zh/guides/write-send.mdx +17 -1
  91. package/docs-site/zh/index.mdx +1 -1
  92. package/docs-site/zh/reference/builtin-agents.mdx +27 -0
  93. package/docs-site/zh/reference/capabilities.mdx +2 -2
  94. package/docs-site/zh/reference/cli.mdx +33 -7
  95. package/docs-site/zh/reference/define-agent.mdx +57 -4
  96. package/docs-site/zh/reference/define-config.mdx +1 -1
  97. package/docs-site/zh/reference/define-eval.mdx +42 -9
  98. package/docs-site/zh/reference/expect.mdx +26 -1
  99. package/package.json +5 -1
  100. package/src/agents/ai-sdk-otel.test.ts +1 -0
  101. package/src/agents/ai-sdk.test.ts +3 -0
  102. package/src/agents/ai-sdk.ts +3 -0
  103. package/src/agents/bub-install-spec.test.ts +34 -0
  104. package/src/agents/bub-install-spec.ts +32 -0
  105. package/src/agents/bub.ts +31 -32
  106. package/src/agents/claude-code.test.ts +130 -9
  107. package/src/agents/claude-code.ts +76 -4
  108. package/src/agents/codex.test.ts +189 -40
  109. package/src/agents/codex.ts +155 -14
  110. package/src/agents/coding-cli-versions.test.ts +15 -0
  111. package/src/agents/coding-cli-versions.ts +3 -0
  112. package/src/agents/index.ts +11 -0
  113. package/src/agents/langgraph.test.ts +204 -0
  114. package/src/agents/langgraph.ts +495 -0
  115. package/src/agents/marketplace.ts +85 -0
  116. package/src/agents/native-config.test.ts +179 -0
  117. package/src/agents/native-config.ts +267 -0
  118. package/src/agents/openai-compat.test.ts +1 -0
  119. package/src/agents/openclaw.test.ts +31 -0
  120. package/src/agents/openclaw.ts +171 -0
  121. package/src/agents/plugin-config.test.ts +1 -0
  122. package/src/agents/sdk-streams.test.ts +79 -0
  123. package/src/agents/sdk-streams.ts +55 -10
  124. package/src/agents/skills.test.ts +1 -0
  125. package/src/agents/streaming.test.ts +3 -9
  126. package/src/agents/types.ts +68 -5
  127. package/src/agents/ui-message-stream.test.ts +3 -0
  128. package/src/cli.ts +411 -108
  129. package/src/context/context.test.ts +51 -12
  130. package/src/context/context.ts +161 -29
  131. package/src/context/session.test.ts +1 -0
  132. package/src/context/session.ts +114 -6
  133. package/src/context/types.ts +30 -12
  134. package/src/define.test.ts +13 -8
  135. package/src/define.ts +25 -4
  136. package/src/expect/index.ts +53 -23
  137. package/src/i18n/en.ts +64 -2
  138. package/src/i18n/zh-CN.ts +65 -3
  139. package/src/o11y/cost.test.ts +1 -0
  140. package/src/o11y/execution-tree.test.ts +1 -20
  141. package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
  142. package/src/o11y/otlp/parse.test.ts +1 -0
  143. package/src/o11y/otlp/turn-otel.test.ts +1 -0
  144. package/src/o11y/parsers/bub.test.ts +1 -0
  145. package/src/o11y/parsers/claude-code.test.ts +1 -34
  146. package/src/o11y/parsers/openclaw.test.ts +154 -0
  147. package/src/o11y/parsers/openclaw.ts +310 -0
  148. package/src/o11y/prices.json +746 -311
  149. package/src/o11y/tool-names.test.ts +1 -0
  150. package/src/o11y/types.ts +16 -2
  151. package/src/report/aggregate.ts +34 -5
  152. package/src/report/built-in-user-parity.test.tsx +110 -153
  153. package/src/report/built-ins/experiment-comparison.tsx +173 -13
  154. package/src/report/built-ins/index.ts +6 -1
  155. package/src/report/components.tsx +9 -3
  156. package/src/report/compute.ts +70 -40
  157. package/src/report/dual-render.test.tsx +194 -67
  158. package/src/report/flag.ts +30 -2
  159. package/src/report/format.ts +35 -11
  160. package/src/report/index.ts +22 -4
  161. package/src/report/locale.ts +25 -5
  162. package/src/report/metrics.ts +67 -14
  163. package/src/report/primitives.tsx +6 -0
  164. package/src/report/react/AttemptList.tsx +6 -31
  165. package/src/report/react/EvalList.tsx +0 -0
  166. package/src/report/react/ExperimentComparison.tsx +68 -0
  167. package/src/report/react/ExperimentList.tsx +15 -9
  168. package/src/report/react/MetricScatter.tsx +12 -14
  169. package/src/report/react/chart-math.test.ts +85 -0
  170. package/src/report/react/chart-math.ts +101 -22
  171. package/src/report/react/enhance.js +33 -1
  172. package/src/report/react/fixtures.ts +24 -17
  173. package/src/report/react/render.test.tsx +9 -64
  174. package/src/report/react/styles.css +73 -2
  175. package/src/report/report.test.ts +306 -98
  176. package/src/report/report.ts +6 -2
  177. package/src/report/text/faces.ts +47 -43
  178. package/src/report/text/table.ts +42 -5
  179. package/src/report/types.ts +41 -21
  180. package/src/results/annotated-source.test.ts +62 -9
  181. package/src/results/annotated-source.ts +64 -6
  182. package/src/results/attempt-evidence.test.ts +9 -7
  183. package/src/results/attempt-evidence.ts +15 -8
  184. package/src/results/attempt-source.ts +6 -3
  185. package/src/results/copy.ts +145 -55
  186. package/src/results/host-equivalence.test.ts +8 -6
  187. package/src/results/index.ts +2 -0
  188. package/src/results/locator.test.ts +1 -22
  189. package/src/results/open.ts +7 -1
  190. package/src/results/publish.ts +149 -0
  191. package/src/results/results.test.ts +85 -51
  192. package/src/results/truncate.ts +90 -0
  193. package/src/results/types.ts +7 -0
  194. package/src/results/writer.ts +31 -13
  195. package/src/runner/attempt.test.ts +138 -7
  196. package/src/runner/attempt.ts +603 -104
  197. package/src/runner/discover.test.ts +47 -0
  198. package/src/runner/discover.ts +36 -2
  199. package/src/runner/eval-source.test.ts +1 -27
  200. package/src/runner/feedback/agent.test.ts +504 -0
  201. package/src/runner/feedback/agent.ts +409 -0
  202. package/src/runner/feedback/ci.test.ts +562 -0
  203. package/src/runner/feedback/ci.ts +401 -0
  204. package/src/runner/feedback/coordinator.test.ts +317 -0
  205. package/src/runner/feedback/coordinator.ts +397 -0
  206. package/src/runner/feedback/failure.ts +40 -0
  207. package/src/runner/feedback/human.test.ts +616 -0
  208. package/src/runner/feedback/human.ts +535 -0
  209. package/src/runner/feedback/index.ts +66 -0
  210. package/src/runner/feedback/io.ts +78 -0
  211. package/src/runner/feedback/profile.test.ts +50 -0
  212. package/src/runner/feedback/profile.ts +58 -0
  213. package/src/runner/feedback/reducer.test.ts +395 -0
  214. package/src/runner/feedback/reducer.ts +260 -0
  215. package/src/runner/feedback/renderer.ts +82 -0
  216. package/src/runner/feedback/sink.ts +203 -0
  217. package/src/runner/feedback/testing.ts +106 -0
  218. package/src/runner/ledger.test.ts +230 -0
  219. package/src/runner/ledger.ts +329 -0
  220. package/src/runner/report.test.ts +128 -3
  221. package/src/runner/report.ts +33 -9
  222. package/src/runner/reporters/artifacts.ts +8 -2
  223. package/src/runner/reporters/braintrust.test.ts +8 -7
  224. package/src/runner/reporters/braintrust.ts +9 -2
  225. package/src/runner/reporters/index.ts +2 -2
  226. package/src/runner/reporters/json.test.ts +162 -0
  227. package/src/runner/reporters/json.ts +35 -8
  228. package/src/runner/reporters/shared.ts +1 -5
  229. package/src/runner/run.test.ts +760 -3
  230. package/src/runner/run.ts +242 -36
  231. package/src/runner/sandbox-prep.ts +3 -42
  232. package/src/runner/timing.ts +158 -0
  233. package/src/runner/types.ts +518 -22
  234. package/src/sandbox/checkpoint.test.ts +55 -0
  235. package/src/sandbox/checkpoint.ts +29 -8
  236. package/src/sandbox/cli-commands.ts +407 -0
  237. package/src/sandbox/docker.ts +115 -16
  238. package/src/sandbox/e2b-agent-template.test.ts +56 -0
  239. package/src/sandbox/e2b-agent-template.ts +94 -0
  240. package/src/sandbox/e2b.ts +74 -9
  241. package/src/sandbox/errors.ts +111 -4
  242. package/src/sandbox/index.ts +2 -0
  243. package/src/sandbox/io-retry.test.ts +58 -0
  244. package/src/sandbox/io-retry.ts +45 -0
  245. package/src/sandbox/keep-registry.test.ts +86 -0
  246. package/src/sandbox/keep-registry.ts +142 -0
  247. package/src/sandbox/keep.ts +178 -0
  248. package/src/sandbox/paths.test.ts +1 -0
  249. package/src/sandbox/paths.ts +19 -8
  250. package/src/sandbox/registry.ts +20 -3
  251. package/src/sandbox/resolve.ts +76 -11
  252. package/src/sandbox/retry.test.ts +70 -0
  253. package/src/sandbox/retry.ts +46 -4
  254. package/src/sandbox/types.ts +44 -6
  255. package/src/sandbox/vercel.ts +43 -20
  256. package/src/scoring/collector.ts +60 -17
  257. package/src/scoring/coverage.ts +95 -0
  258. package/src/scoring/diff.ts +81 -0
  259. package/src/scoring/display.test.ts +121 -0
  260. package/src/scoring/display.ts +133 -0
  261. package/src/scoring/evidence.test.ts +189 -0
  262. package/src/scoring/judge.test.ts +142 -0
  263. package/src/scoring/judge.ts +15 -18
  264. package/src/scoring/scoped.ts +217 -50
  265. package/src/scoring/types.ts +117 -20
  266. package/src/scoring/verdict.ts +16 -4
  267. package/src/shared/aggregate.ts +3 -2
  268. package/src/shared/types.ts +31 -0
  269. package/src/show/compose.ts +2 -2
  270. package/src/show/index.ts +21 -1
  271. package/src/show/render.ts +619 -104
  272. package/src/show/show.test.ts +235 -19
  273. package/src/tty-line.ts +8 -26
  274. package/src/util.test.ts +1 -0
  275. package/src/util.ts +41 -0
  276. package/src/view/app/components/AttemptModal.tsx +153 -2
  277. package/src/view/app/components/CodeView.tsx +32 -11
  278. package/src/view/app/components/CopyControls.tsx +2 -2
  279. package/src/view/app/i18n.ts +6 -0
  280. package/src/view/app/lib/attempt-route.test.ts +1 -0
  281. package/src/view/app/lib/verdict.ts +7 -9
  282. package/src/view/artifact-serving.test.ts +2 -1
  283. package/src/view/client-dist/app.css +1 -1
  284. package/src/view/client-dist/app.js +17 -17
  285. package/src/view/data.test.ts +1 -0
  286. package/src/view/data.ts +11 -1
  287. package/src/view/index.ts +11 -0
  288. package/src/view/server.ts +2 -0
  289. package/src/view/styles.css +3 -0
  290. package/src/view/view-report.test.ts +6 -5
  291. package/src/runner/reporters/console.ts +0 -70
  292. package/src/runner/reporters/live.test.ts +0 -56
  293. package/src/runner/reporters/live.ts +0 -247
  294. package/src/runner/reporters/quiet.test.ts +0 -66
  295. package/src/runner/reporters/quiet.ts +0 -49
  296. package/src/runner/reporters/table.ts +0 -277
@@ -6,16 +6,33 @@
6
6
  //
7
7
  // 泛化自 src/view/app/lib/transcript-data.tsx 的 indexAsserts()(Map<string, Assertion[]>
8
8
  // + noloc 兜底桶的先例)——同一套"按 loc 分桶、没有 loc 进兜底"的思路,换成源码行数组
9
- // (不是 Map)存放,外加 sourceSha256 / summary 计数。indexTurns() 那部分(events → 轮次)
10
- // ExecutionTree 的地盘,不在这个模型里。
9
+ // (不是 Map)存放,外加 sourceSha256 / summary 计数。轮次的完整展开(events → ExecutionTree)
10
+ // 仍是 ExecutionTree 的地盘;这个模型只把每轮的头行事实标回 send 调用行(见 SendAnnotation),
11
+ // 作为源码页指向 --execution 的跨面指针。
11
12
  //
12
13
  // 本模块不 import react/jsx,纯数据 + 纯函数,可以在任何 Node 语境(CLI 的 show、
13
14
  // view 的 server 端数据准备)调用。
14
15
 
15
- import type { AssertionResult } from "../types.ts";
16
+ import type { AssertionResult, PhaseTiming, SourceLoc, StreamEvent } from "../types.ts";
16
17
  import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
17
18
 
18
- /** 一行源码 + 映射到这一行的全部断言(保持原始顺序;可以是空数组)。 */
19
+ /**
20
+ * 标回 `t.send(...)` 调用行的一轮 turn 头行事实(契约见 docs/feature/reports/show.md
21
+ * 「--eval:把断言放回源码」)。身份标签与 --execution / --timing / diff windows 同一套;
22
+ * 回复全文与轮内卡片不进这个模型——源码页只回答「这行代码对应哪一轮、这一轮成了没成」。
23
+ */
24
+ export interface SendAnnotation {
25
+ /** `s<session>/t<turn>`。 */
26
+ label: string;
27
+ /** 轮的终态;时间树只记 failed 位,waiting 需要事件流佐证时由派生方给。 */
28
+ status: "completed" | "failed" | "waiting";
29
+ /** 该轮墙钟;时间树缺这一轮的节点时省略。 */
30
+ durationMs?: number;
31
+ /** send 调用位置(用户消息事件的 loc)。 */
32
+ loc: SourceLoc;
33
+ }
34
+
35
+ /** 一行源码 + 映射到这一行的全部断言与 send 标注(保持原始顺序;可以是空数组)。 */
19
36
  export interface AnnotatedSourceLine {
20
37
  /** 1-indexed 行号,与 SourceLoc.line 同一坐标系。 */
21
38
  line: number;
@@ -28,6 +45,11 @@ export interface AnnotatedSourceLine {
28
45
  * 领域 model 的职责(与 indexAsserts() 的先例一致,它同样只分桶、不折叠)。
29
46
  */
30
47
  assertions: AssertionResult[];
48
+ /**
49
+ * 映射到这一行的 send 标注,按轮次顺序;循环里的 send 一行多轮。与断言的 never-drop
50
+ * 契约不同,定位不到行的轮不进任何兜底桶——轮次的全量面是 --execution,这里只是指针。
51
+ */
52
+ sends: SendAnnotation[];
31
53
  }
32
54
 
33
55
  export interface AnnotatedEvalSourceSummary {
@@ -67,6 +89,34 @@ export interface AnnotatedEvalSource {
67
89
  summary: AnnotatedEvalSourceSummary;
68
90
  }
69
91
 
92
+ /**
93
+ * 从标准事件流与阶段时间树派生 send 标注:第 i 条用户消息开第 i 轮(与 --execution 的
94
+ * 分轮边界同一条规则,见 show/render.ts::executionText),头行事实取 `eval.run` 下第 i 个
95
+ * turn 节点;用户消息没有 loc 的轮不产出标注。纯函数,无 IO。
96
+ */
97
+ export function deriveSendAnnotations(
98
+ events: readonly StreamEvent[] | null,
99
+ phases: readonly PhaseTiming[] | undefined,
100
+ ): SendAnnotation[] {
101
+ if (!events || events.length === 0) return [];
102
+ const turnNodes = (phases ?? []).flatMap((p) => p.children ?? []).filter((n) => n.kind === "turn");
103
+ const out: SendAnnotation[] = [];
104
+ let turnIndex = -1;
105
+ for (const event of events) {
106
+ if (event.type !== "message" || event.role !== "user") continue;
107
+ turnIndex += 1;
108
+ if (!event.loc) continue;
109
+ const turn = turnNodes[turnIndex];
110
+ out.push({
111
+ label: turn?.label ?? `t${turnIndex + 1}`,
112
+ status: turn?.failed ? "failed" : "completed",
113
+ ...(turn !== undefined ? { durationMs: turn.durationMs } : {}),
114
+ loc: event.loc,
115
+ });
116
+ }
117
+ return out;
118
+ }
119
+
70
120
  /**
71
121
  * 纯函数:给定一份源码文本(未归一化也可以,内部会 normalizeEvalSource)与一批断言,
72
122
  * 产出标注好的 AnnotatedEvalSource。幂等 / 无 IO —— 可以在渲染路径上安全调用。
@@ -77,13 +127,21 @@ export interface AnnotatedEvalSource {
77
127
  export function buildAnnotatedEvalSource(
78
128
  source: { path: string; content: string },
79
129
  assertions: readonly AssertionResult[],
130
+ sends: readonly SendAnnotation[] = [],
80
131
  ): AnnotatedEvalSource {
81
132
  const normalized = normalizeEvalSource(source.content);
82
133
  // 末尾的单个换行符不产出幻影空行(源码文件几乎总以换行符收尾);中间的空行原样保留为一行。
83
134
  const body = normalized.endsWith("\n") ? normalized.slice(0, -1) : normalized;
84
135
  const rawLines = body.length === 0 && normalized.length === 0 ? [""] : body.split("\n");
85
136
 
86
- const lines: AnnotatedSourceLine[] = rawLines.map((text, i) => ({ line: i + 1, text, assertions: [] }));
137
+ const lines: AnnotatedSourceLine[] = rawLines.map((text, i) => ({ line: i + 1, text, assertions: [], sends: [] }));
138
+
139
+ for (const send of sends) {
140
+ // 与断言同一条映射规则;不满足的轮直接丢(全量面在 --execution,见 SendAnnotation)。
141
+ if (send.loc.file === source.path && send.loc.line >= 1 && send.loc.line <= lines.length) {
142
+ lines[send.loc.line - 1]!.sends.push(send);
143
+ }
144
+ }
87
145
 
88
146
  const unmapped: AssertionResult[] = [];
89
147
  let mappedAssertions = 0;
@@ -93,7 +151,7 @@ export function buildAnnotatedEvalSource(
93
151
  let soft = 0;
94
152
 
95
153
  for (const assertion of assertions) {
96
- if (assertion.passed) passed++;
154
+ if (assertion.outcome === "passed") passed++;
97
155
  else failed++;
98
156
  if (assertion.severity === "gate") gate++;
99
157
  else soft++;
@@ -1,3 +1,4 @@
1
+ // cases: docs/engineering/unit-tests/results/cases.md
1
2
  // loadAttemptEvidence 单测(定稿契约见 plan/attempt-evidence-feedback-loop.md「中性数据准备」、
2
3
  // src/results/attempt-evidence.ts 的头注)。用真实 createResultsWriter → openResults 的读写链路
3
4
  // 落一份最小 fixture(不手写 JSON 文件,理由同 loadAnnotatedEvalSource 的既有端到端测试:
@@ -7,6 +8,7 @@ import { afterEach, describe, expect, it } from "vitest";
7
8
  import { mkdtemp, rm } from "node:fs/promises";
8
9
  import { tmpdir } from "node:os";
9
10
  import { join } from "node:path";
11
+ import type { DiffArtifact } from "../types.ts";
10
12
  import {
11
13
  createResultsWriter,
12
14
  loadAttemptEvidence,
@@ -34,7 +36,7 @@ const SOURCE_PATH = "evals/a.eval.ts";
34
36
  const SOURCE_CONTENT = 'import { defineEval } from "niceeval";\nexport default defineEval({\n test() {},\n});\n';
35
37
 
36
38
  const ASSERTIONS: EvalResult["assertions"] = [
37
- { name: "check-1", passed: true, severity: "gate", score: 1, loc: { file: SOURCE_PATH, line: 3 } },
39
+ { name: "check-1", outcome: "passed" as const, severity: "gate", score: 1, loc: { file: SOURCE_PATH, line: 3 } },
38
40
  ];
39
41
 
40
42
  const EVENTS: StreamEvent[] = [
@@ -47,8 +49,8 @@ const TRACE: TraceSpan[] = [
47
49
  { traceId: "t1", spanId: "s1", name: "tool.get_weather", startMs: 0, endMs: 10, attributes: { call_id: "c1" } },
48
50
  ];
49
51
 
50
- const NONEMPTY_DIFF: DiffData = { generatedFiles: { "a.txt": "hello" }, deletedFiles: [] };
51
- const EMPTY_DIFF: DiffData = { generatedFiles: {}, deletedFiles: [] };
52
+ const NONEMPTY_DIFF: DiffArtifact = [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "hello" } } }];
53
+ const EMPTY_DIFF: DiffArtifact = [{ window: "s1/t1", changes: {} }];
52
54
 
53
55
  /** 起一个 writer,写一条 attempt,finish,再从头 openResults 读回它的 AttemptHandle。 */
54
56
  async function seedAttempt(
@@ -72,7 +74,7 @@ describe("loadAttemptEvidence", () => {
72
74
  const root = await makeRoot();
73
75
  const attempt = await seedAttempt(
74
76
  root,
75
- { id: "weather/brooklyn", assertions: ASSERTIONS },
77
+ { id: "weather/brooklyn", assertions: ASSERTIONS, phases: [{ name: "eval.run", durationMs: 900 }] },
76
78
  { events: EVENTS, trace: TRACE, diff: NONEMPTY_DIFF, sources: [{ path: SOURCE_PATH, content: SOURCE_CONTENT }] },
77
79
  );
78
80
 
@@ -86,7 +88,7 @@ describe("loadAttemptEvidence", () => {
86
88
  const actionNode = evidence.execution!.nodes.find((n) => n.kind === "action");
87
89
  expect(actionNode).toBeDefined();
88
90
  expect((actionNode as { span?: TraceSpan }).span).toBeDefined();
89
- expect(evidence.diff).toEqual(NONEMPTY_DIFF);
91
+ expect(evidence.diff?.windows).toEqual(NONEMPTY_DIFF);
90
92
 
91
93
  expect(evidence.capabilities).toEqual({ eval: true, execution: true, timing: true, diff: true });
92
94
  });
@@ -103,7 +105,7 @@ describe("loadAttemptEvidence", () => {
103
105
  expect(evidence.capabilities).toEqual({ eval: false, execution: false, timing: false, diff: false });
104
106
  });
105
107
 
106
- it("有 events 没有 OTel spans:execution 为真、timing 为假", async () => {
108
+ it("有 events 没有 phases(旧 runner 产出):execution 为真、timing 为假", async () => {
107
109
  const root = await makeRoot();
108
110
  const attempt = await seedAttempt(root, { id: "weather/albany" }, { events: EVENTS });
109
111
 
@@ -122,7 +124,7 @@ describe("loadAttemptEvidence", () => {
122
124
 
123
125
  const evidence = await loadAttemptEvidence(attempt);
124
126
 
125
- expect(evidence.diff).toEqual(EMPTY_DIFF);
127
+ expect(evidence.diff?.windows).toEqual(EMPTY_DIFF);
126
128
  expect(evidence.capabilities.diff).toBe(false);
127
129
  });
128
130
 
@@ -31,7 +31,7 @@ import { join } from "node:path";
31
31
  import type { AttemptHandle } from "./types.ts";
32
32
  import { type AttemptIdentity, type AttemptLocator, encodeAttemptLocator } from "./locator.ts";
33
33
  import { loadAnnotatedEvalSource } from "./attempt-source.ts";
34
- import type { AnnotatedEvalSource } from "./annotated-source.ts";
34
+ import { deriveSendAnnotations, type AnnotatedEvalSource } from "./annotated-source.ts";
35
35
  import { buildExecutionTree, type ExecutionTree } from "../o11y/execution-tree.ts";
36
36
  import type { DiffData, EvalResult } from "../types.ts";
37
37
 
@@ -63,7 +63,7 @@ export interface AttemptEvidenceCapabilities {
63
63
  eval: boolean;
64
64
  /** events 非空数组——有骨架可看。 */
65
65
  execution: boolean;
66
- /** execution 成立且这次运行接入过 OTel(ExecutionTree.timingAvailable) */
66
+ /** result.json phases(阶段计时)——`--timing` 时间树可渲染。 */
67
67
  timing: boolean;
68
68
  /** diff 非空且至少改动/删除了一个文件。 */
69
69
  diff: boolean;
@@ -86,6 +86,8 @@ export interface AttemptEvidence {
86
86
  execution: ExecutionTree | null;
87
87
  /** 被测 Agent 对 Sandbox 工作区的文件变化(`--diff` 切面);原样透传 attempt.diff()。 */
88
88
  diff: DiffData | null;
89
+ /** OTel spans(`--timing` 把 turn 节点下的 agent/model/tool spans 挂回时间树);没有 trace 就是 null。 */
90
+ trace: import("../types.ts").TraceSpan[] | null;
89
91
  artifactPaths: EvidencePaths;
90
92
  capabilities: AttemptEvidenceCapabilities;
91
93
  }
@@ -110,12 +112,17 @@ export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<Attem
110
112
  };
111
113
  const locator = attempt.locator ?? encodeAttemptLocator(identity);
112
114
 
113
- const [evalSource, events, trace, diff] = await Promise.all([
114
- loadAnnotatedEvalSource(attempt),
115
+ const [events, trace, diff] = await Promise.all([
115
116
  attempt.events(),
116
117
  attempt.trace(),
117
118
  attempt.diff(),
118
119
  ]);
120
+ // send 标注要拿事件流里用户消息的 loc 与阶段时间树里的 turn 节点配对,所以在 events
121
+ // 就绪后再装配 evalSource;派生与分桶都是纯函数(annotated-source.ts),这里只接线。
122
+ const evalSource = await loadAnnotatedEvalSource(
123
+ attempt,
124
+ deriveSendAnnotations(events, attempt.result.phases),
125
+ );
119
126
 
120
127
  // execution 的 null 边界:没有非空事件骨架就是 null,即使 trace 里有 span(那些 span 在
121
128
  // buildExecutionTree 里只会全部落进 telemetry-only 桶,产出一棵没有骨架、只有遥测的树——
@@ -124,9 +131,8 @@ export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<Attem
124
131
  const execution = hasEvents ? buildExecutionTree(events, trace ?? []) : null;
125
132
 
126
133
  const evalCapable = evalSource !== null && (evalSource.lines.length > 0 || evalSource.summary.totalAssertions > 0);
127
- // 与 show/render.ts::diffText 的既有空判定同一口径:generatedFiles 与 deletedFiles 都空
128
- // 才算"没有变化"。
129
- const diffCapable = diff !== null && (Object.keys(diff.generatedFiles).length > 0 || diff.deletedFiles.length > 0);
134
+ // 与 show/render.ts::diffText 同一口径:任一窗口触及过任一文件才算"有变化"。
135
+ const diffCapable = diff !== null && Object.keys(diff.files).length > 0;
130
136
 
131
137
  return {
132
138
  locator,
@@ -135,11 +141,12 @@ export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<Attem
135
141
  evalSource,
136
142
  execution,
137
143
  diff,
144
+ trace,
138
145
  artifactPaths: { dir: attemptArtifactDir(attempt) },
139
146
  capabilities: {
140
147
  eval: evalCapable,
141
148
  execution: execution !== null,
142
- timing: execution !== null && execution.timingAvailable,
149
+ timing: (attempt.result.phases?.length ?? 0) > 0,
143
150
  diff: diffCapable,
144
151
  },
145
152
  };
@@ -8,7 +8,7 @@
8
8
  // 或参照。
9
9
 
10
10
  import type { AttemptHandle } from "./types.ts";
11
- import { buildAnnotatedEvalSource, type AnnotatedEvalSource } from "./annotated-source.ts";
11
+ import { buildAnnotatedEvalSource, type AnnotatedEvalSource, type SendAnnotation } from "./annotated-source.ts";
12
12
 
13
13
  /**
14
14
  * 给定一个 attempt,取回它的 eval 源码(经 sources() 解引用,可能来自本快照或
@@ -22,7 +22,10 @@ import { buildAnnotatedEvalSource, type AnnotatedEvalSource } from "./annotated-
22
22
  * 只服务这条打通链路的验证目的,不是最终选择策略(后续阶段如需要为每份引用到的文件
23
23
  * 都出一份 AnnotatedEvalSource,在这基础上很容易扩成返回数组)。
24
24
  */
25
- export async function loadAnnotatedEvalSource(attempt: AttemptHandle): Promise<AnnotatedEvalSource | null> {
25
+ export async function loadAnnotatedEvalSource(
26
+ attempt: AttemptHandle,
27
+ sends: readonly SendAnnotation[] = [],
28
+ ): Promise<AnnotatedEvalSource | null> {
26
29
  const sources = await attempt.sources();
27
30
  if (!sources || sources.length === 0) return null;
28
31
 
@@ -44,5 +47,5 @@ export async function loadAnnotatedEvalSource(attempt: AttemptHandle): Promise<A
44
47
  }
45
48
  }
46
49
 
47
- return buildAnnotatedEvalSource(primary, assertions);
50
+ return buildAnnotatedEvalSource(primary, assertions, sends);
48
51
  }
@@ -7,20 +7,38 @@
7
7
  // knownEvalIds(复制时刻该实验已知的 eval 并集),发布目录上重新 openResults().latest(),
8
8
  // 残缺警告被同一套机制重新算出来,不靠发布者转述。
9
9
 
10
- import { copyFile, mkdir, readdir, stat, writeFile } from "node:fs/promises";
10
+ import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
11
11
  import { basename, dirname, join, resolve } from "node:path";
12
- import type { EvalResult } from "../types.ts";
12
+ import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
13
13
  import { RESULTS_FORMAT } from "../types.ts";
14
14
  import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, experimentDirOf } from "./format.ts";
15
15
  import { experimentOfSnapshot } from "./open.ts";
16
16
  import { isNewerSnapshot } from "./select.ts";
17
17
  import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
18
+ import {
19
+ PUBLISH_FILE_MAX_BYTES,
20
+ redactEvents,
21
+ redactExperimentInfo,
22
+ redactJsonValue,
23
+ redactResultRecord,
24
+ redactSpans,
25
+ type Redactor,
26
+ } from "./publish.ts";
18
27
  import type { ArtifactKind, AttemptHandle, Selection, Snapshot, SnapshotMeta } from "./types.ts";
19
- import { ARTIFACT_KINDS } from "./types.ts";
28
+
29
+ /** 缺省携带的 artifact:events / trace / o11y / agentSetup / sources;diff 不截断、可达百 MB,缺省不带。 */
30
+ const DEFAULT_PUBLISH_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "sources"];
31
+ const VALID_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "diff", "sources"];
20
32
 
21
33
  export interface CopySnapshotsOptions {
22
- /** 要带上的 artifact 种类;省略 = 全部六类。diff 可达百 MB,发布时常见地不带;o11y 只有几 KB,报告用 turns 这类 artifact 档指标(见 docs/feature/reports/library.md「内置指标」)时记得带上。 */
34
+ /** 要带上的 artifact 种类;缺省带 events / trace / o11y / agentSetup / sources,不带 diff。 */
23
35
  artifacts?: ArtifactKind[];
36
+ /**
37
+ * 发布消毒(必填,没有隐式默认):传函数逐值消毒自由文本字段,或显式传 `false` 声明
38
+ * 「这批数据可以原文发布」——「不消毒」必须是写在代码里的选择,不是忘了传参数的副作用。
39
+ * 结构字段(格式、判定、身份、路径、哈希)永不经过 redactor。
40
+ */
41
+ redact: Redactor | false;
24
42
  }
25
43
 
26
44
  export interface CopySnapshotsResult {
@@ -38,7 +56,7 @@ export interface CopySnapshotsResult {
38
56
  export async function copySnapshots(
39
57
  selection: Selection | Snapshot[],
40
58
  destDir: string,
41
- opts?: CopySnapshotsOptions,
59
+ opts: CopySnapshotsOptions,
42
60
  ): Promise<CopySnapshotsResult> {
43
61
  const selected = Array.isArray(selection) ? selection : selection.snapshots;
44
62
  if (selected.length === 0) {
@@ -46,11 +64,17 @@ export async function copySnapshots(
46
64
  "copySnapshots got no snapshots to copy. Check the experiments filter, or pass snapshots from openResults().latest().",
47
65
  );
48
66
  }
49
- const kinds = opts?.artifacts ?? [...ARTIFACT_KINDS];
67
+ if (opts?.redact === undefined) {
68
+ throw new Error(
69
+ 'copySnapshots requires an explicit "redact" option: pass a (text) => string sanitizer, or the literal false to declare this data safe to publish verbatim.',
70
+ );
71
+ }
72
+ const redactor: Redactor | undefined = opts.redact === false ? undefined : opts.redact;
73
+ const kinds = opts.artifacts ?? [...DEFAULT_PUBLISH_ARTIFACTS];
50
74
  for (const kind of kinds) {
51
- if (!ARTIFACT_KINDS.includes(kind)) {
75
+ if (!VALID_ARTIFACTS.includes(kind)) {
52
76
  throw new Error(
53
- `Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${ARTIFACT_KINDS.join(", ")}.`,
77
+ `Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${VALID_ARTIFACTS.join(", ")}.`,
54
78
  );
55
79
  }
56
80
  }
@@ -73,96 +97,162 @@ export async function copySnapshots(
73
97
  if (isNewerSnapshot(snapshot, existing)) byExperiment.set(snapshot.experimentId, snapshot);
74
98
  }
75
99
 
76
- await mkdir(dest, { recursive: true });
100
+ // 发布前整文件预检:先规划并序列化全部目标文件(消毒发生在序列化后、写盘前),任一文件
101
+ // 超过 PUBLISH_FILE_MAX_BYTES 就整体失败,不留半成品目标目录。
102
+ const planned: PlannedFile[] = [];
77
103
  for (const snapshot of byExperiment.values()) {
78
- await copyOneSnapshot(snapshot, selected, dest, kinds);
104
+ planned.push(...(await planOneSnapshot(snapshot, selected, dest, kinds, redactor)));
105
+ }
106
+ const oversized = planned.filter((f) => f.bytes.byteLength > PUBLISH_FILE_MAX_BYTES);
107
+ if (oversized.length > 0) {
108
+ const lines = oversized.map(
109
+ (f) =>
110
+ ` ${f.source ?? f.path}: ${f.bytes.byteLength} bytes (limit ${PUBLISH_FILE_MAX_BYTES}). Exclude that artifact kind from "artifacts", or regenerate the history with the current writer.`,
111
+ );
112
+ throw new Error(`copySnapshots publish precheck failed; ${oversized.length} file(s) exceed the 50 MiB publish budget:\n${lines.join("\n")}`);
113
+ }
114
+
115
+ await mkdir(dest, { recursive: true });
116
+ for (const file of planned) {
117
+ await mkdir(dirname(file.path), { recursive: true });
118
+ await writeFile(file.path, file.bytes);
79
119
  }
80
120
 
81
121
  return { dir: dest, warnings };
82
122
  }
83
123
 
84
- async function copyOneSnapshot(snapshot: Snapshot, selected: Snapshot[], destRoot: string, kinds: ArtifactKind[]): Promise<void> {
124
+ interface PlannedFile {
125
+ /** 目标绝对路径。 */
126
+ path: string;
127
+ /** 序列化(且已消毒)后的字节。 */
128
+ bytes: Buffer;
129
+ /** 源文件路径(预检报错定位用);重建的 json 没有单一源。 */
130
+ source?: string;
131
+ }
132
+
133
+ async function planOneSnapshot(
134
+ snapshot: Snapshot,
135
+ selected: Snapshot[],
136
+ destRoot: string,
137
+ kinds: ArtifactKind[],
138
+ redactor: Redactor | undefined,
139
+ ): Promise<PlannedFile[]> {
85
140
  const destSnapDir = join(destRoot, experimentDirOf(snapshot.experimentId), basename(snapshot.dir));
86
- await mkdir(destSnapDir, { recursive: true });
141
+ const planned: PlannedFile[] = [];
87
142
 
88
143
  // sources 的去重仓库(sources/<sha256>.json)是快照级的:同一份源码被多个 attempt 引用时,
89
- // 复制到目的地也只应该有一份——这个 Set 记录本快照已经落过盘的 hash,整快照的 attempt 共享。
90
- const copiedSourceHashes = new Set<string>();
144
+ // 复制到目的地也只应该有一份——这个 Set 记录本快照已经规划过的 hash,整快照的 attempt 共享。
145
+ const plannedSourceHashes = new Set<string>();
91
146
  for (const attempt of snapshot.attempts) {
92
- await copyOneAttempt(attempt, destSnapDir, kinds, copiedSourceHashes);
147
+ planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes, redactor)));
93
148
  }
94
149
 
95
150
  const knownEvalIds = experimentOfSnapshot(snapshot)?.evalIds ?? fallbackUnion(selected, snapshot.experimentId);
151
+ const experiment =
152
+ snapshot.experiment !== undefined && redactor
153
+ ? (redactExperimentInfo(snapshot.experiment as unknown as Record<string, unknown>, redactor) as unknown as SnapshotMeta["experiment"])
154
+ : snapshot.experiment;
96
155
  const meta: SnapshotMeta = {
97
156
  format: RESULTS_FORMAT,
98
157
  schemaVersion: snapshot.schemaVersion,
99
158
  producer: snapshot.producer,
100
159
  experimentId: snapshot.experimentId,
101
- ...(snapshot.experiment !== undefined ? { experiment: snapshot.experiment } : {}),
160
+ ...(experiment !== undefined ? { experiment } : {}),
102
161
  agent: snapshot.agent,
103
162
  ...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
104
163
  startedAt: snapshot.startedAt,
105
164
  ...(snapshot.completedAt !== undefined ? { completedAt: snapshot.completedAt } : {}),
106
165
  ...(knownEvalIds.length ? { knownEvalIds } : {}),
107
- ...(snapshot.name !== undefined ? { name: snapshot.name } : {}),
166
+ ...(snapshot.name !== undefined
167
+ ? { name: redactor && typeof snapshot.name === "string" ? redactor(snapshot.name) : snapshot.name }
168
+ : {}),
169
+ // 发布根标记只声明流程,不证明结果:applied = 消毒函数对全部自由文本字段跑过;
170
+ // none = 作者显式的原文发布声明。view --out 的防呆据此分级。
171
+ publish: { redaction: redactor ? "applied" : "none" },
108
172
  };
109
- await writeFile(join(destSnapDir, SNAPSHOT_FILE), JSON.stringify(meta, null, 2), "utf-8");
173
+ planned.push({ path: join(destSnapDir, SNAPSHOT_FILE), bytes: Buffer.from(JSON.stringify(meta, null, 2), "utf-8") });
174
+ return planned;
110
175
  }
111
176
 
112
- async function copyOneAttempt(
177
+ async function planOneAttempt(
113
178
  attempt: AttemptHandle,
114
179
  destSnapDir: string,
115
180
  kinds: ArtifactKind[],
116
- copiedSourceHashes: Set<string>,
117
- ): Promise<void> {
181
+ plannedSourceHashes: Set<string>,
182
+ redactor: Redactor | undefined,
183
+ ): Promise<PlannedFile[]> {
118
184
  const destAttemptDir = join(destSnapDir, attempt.ref.attempt);
119
- await mkdir(destAttemptDir, { recursive: true });
185
+ const planned: PlannedFile[] = [];
120
186
 
121
187
  // sources 是唯一「两层」的 artifact(attempt 级引用 + 快照级去重仓库),不能像其它四类那样
122
- // 单文件 copyFile 原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的
123
- // attempt.sources() 拿到完整内容,再在目的地按内容哈希重新去重落盘,天然吸收 artifactBase
124
- // 回退链(携带条目复制后,原快照可能不在目的地里,必须此刻把内容落到自己脚下)。
188
+ // 单文件原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的 attempt.sources()
189
+ // 拿到完整内容,消毒后按(消毒后)内容哈希重新去重落盘——发布根里引用与内容永远一致。
125
190
  const genericKinds = kinds.filter((k) => k !== "sources");
126
191
  const files = await findArtifactFiles(attempt, genericKinds);
127
- await Promise.all(files.map(({ kind, source }) => copyFile(source, join(destAttemptDir, artifactFileOf(kind)))));
128
192
  const copied = new Set(files.map((f) => f.kind));
193
+ for (const { kind, source } of files) {
194
+ const raw = await readFile(source);
195
+ const bytes = redactor ? redactArtifactBytes(kind, raw, redactor) : raw;
196
+ planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes, source });
197
+ }
129
198
 
130
199
  if (kinds.includes("sources")) {
131
- const wroteSources = await copySources(attempt, destSnapDir, destAttemptDir, copiedSourceHashes);
132
- if (wroteSources) copied.add("sources");
200
+ const sources = await attempt.sources();
201
+ if (sources && sources.length > 0) {
202
+ copied.add("sources");
203
+ const destStoreDir = join(destSnapDir, "sources");
204
+ const refs: { path: string; sha256: string }[] = [];
205
+ for (const src of sources) {
206
+ const content = redactor ? redactor(src.content) : src.content;
207
+ const sha256 = hashEvalSource(normalizeEvalSource(content));
208
+ refs.push({ path: src.path, sha256 });
209
+ if (!plannedSourceHashes.has(sha256)) {
210
+ planned.push({
211
+ path: join(destStoreDir, `${sha256}.json`),
212
+ bytes: Buffer.from(JSON.stringify({ content }), "utf-8"),
213
+ });
214
+ plannedSourceHashes.add(sha256);
215
+ }
216
+ }
217
+ planned.push({
218
+ path: join(destAttemptDir, artifactFileOf("sources")),
219
+ bytes: Buffer.from(JSON.stringify(refs), "utf-8"),
220
+ });
221
+ }
133
222
  }
134
223
 
135
- const record = slimForCopy(attempt.result, copied);
136
- await writeFile(join(destAttemptDir, RESULT_FILE), JSON.stringify(record, null, 2), "utf-8");
224
+ let record = slimForCopy(attempt.result, copied);
225
+ if (redactor) record = redactResultRecord(record, redactor);
226
+ planned.push({ path: join(destAttemptDir, RESULT_FILE), bytes: Buffer.from(JSON.stringify(record, null, 2), "utf-8") });
227
+ return planned;
137
228
  }
138
229
 
139
- /**
140
- * sources 的复制:经 attempt.sources() 拿到已解引用的完整内容(null / 空数组 = 没有,不写任何
141
- * 文件,与其它 artifact「空数据不落文件」的约定一致),按内容 sha256 写进目的快照的
142
- * `sources/<sha256>.json`(同一快照内已经写过的 hash 不重写),attempt 目录下落一份引用。
143
- */
144
- async function copySources(
145
- attempt: AttemptHandle,
146
- destSnapDir: string,
147
- destAttemptDir: string,
148
- copiedSourceHashes: Set<string>,
149
- ): Promise<boolean> {
150
- const sources = await attempt.sources();
151
- if (!sources || sources.length === 0) return false;
152
-
153
- const destStoreDir = join(destSnapDir, "sources");
154
- const refs: { path: string; sha256: string }[] = [];
155
- for (const src of sources) {
156
- const sha256 = hashEvalSource(normalizeEvalSource(src.content));
157
- refs.push({ path: src.path, sha256 });
158
- if (!copiedSourceHashes.has(sha256)) {
159
- await mkdir(destStoreDir, { recursive: true });
160
- await writeFile(join(destStoreDir, `${sha256}.json`), JSON.stringify({ content: src.content }), "utf-8");
161
- copiedSourceHashes.add(sha256);
230
+ /** 单个 artifact 文件的消毒:按种类解析 JSON、走各自的自由文本标注,重新序列化。 */
231
+ function redactArtifactBytes(kind: ArtifactKind, raw: Buffer, redactor: Redactor): Buffer {
232
+ try {
233
+ const parsed = JSON.parse(raw.toString("utf-8")) as unknown;
234
+ let next: unknown;
235
+ switch (kind) {
236
+ case "events":
237
+ next = redactEvents(parsed as StreamEvent[], redactor);
238
+ break;
239
+ case "trace":
240
+ next = redactSpans(parsed as TraceSpan[], redactor);
241
+ break;
242
+ // diff before/after/内容与 o11y 的命令 / 错误 / URL 都是自由文本;路径键为结构字段。
243
+ case "diff":
244
+ case "o11y":
245
+ case "agentSetup":
246
+ next = redactJsonValue(parsed, redactor);
247
+ break;
248
+ default:
249
+ next = parsed;
162
250
  }
251
+ return Buffer.from(JSON.stringify(next), "utf-8");
252
+ } catch {
253
+ // 解析不了的按原字节透传(malformed 数据不该在发布路径上被静默改写)。
254
+ return raw;
163
255
  }
164
- await writeFile(join(destAttemptDir, artifactFileOf("sources")), JSON.stringify(refs), "utf-8");
165
- return true;
166
256
  }
167
257
 
168
258
  /** 目标目录非空即报错:盘上不该出现「我没写的东西被动过」的惊讶;发布脚本要幂等就自己先清目录。 */
@@ -1,3 +1,4 @@
1
+ // cases: docs/engineering/unit-tests/results/cases.md
1
2
  // show / view 宿主等价契约(设计契约:plan/show-view-equivalence.md 完成定义;
2
3
  // docs/feature/reports/architecture.md「Selection 是计算入口」;docs/feature/results/library.md「选择快照」)。
3
4
  //
@@ -339,7 +340,7 @@ describe("selectCurrentResults · 现刻水位结构化身份", () => {
339
340
  const rootA = await makeRoot();
340
341
  const rootB = await makeRoot();
341
342
  // 失败 eval 让 id 现于两面(通过 eval 只进折叠子行,不便断言);qa / qb 各根独有。
342
- const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0, passed: false, detail: "boom" }] };
343
+ const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0, outcome: "failed" as const, detail: "boom" }] };
343
344
  await writeSnapshot(rootA, "2026-07-01T08-00-00-000Z", { experimentId: "onlyA/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [res("qa", "failed", boom)]);
344
345
  await writeSnapshot(rootB, "2026-07-02T08-00-00-000Z", { experimentId: "onlyB/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [res("qb", "failed", boom)]);
345
346
  const normB = normalizeSelection(selectCurrentResults(await openResults(rootB)));
@@ -389,7 +390,7 @@ async function seedPartialRerun(): Promise<string> {
389
390
  const root = await makeRoot();
390
391
  await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
391
392
  res("q1", "passed"),
392
- res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0, passed: false, detail: "file was not modified" }] }),
393
+ res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0, outcome: "failed" as const, detail: "file was not modified" }] }),
393
394
  ]);
394
395
  await writeSnapshot(root, "2026-07-02T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
395
396
  res("q1", "passed"),
@@ -404,7 +405,7 @@ async function seedPartialCoverage(): Promise<string> {
404
405
  root,
405
406
  "2026-07-01T08-00-00-000Z",
406
407
  { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q1", "q2"] },
407
- [res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, passed: false, detail: "boom" }] })],
408
+ [res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] })],
408
409
  );
409
410
  return root;
410
411
  }
@@ -415,9 +416,10 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
415
416
  const text = await showText(root, []);
416
417
  const html = await viewHtml(root);
417
418
  for (const face of [text, html]) {
418
- expect(face).toContain("compare/bub");
419
419
  expect(face).toContain("q2"); // 周一补齐的失败 eval,若宿主回退 results.latest() 就会消失
420
420
  }
421
+ expect(text).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
422
+ expect(html).toContain('data-sort-value="compare/bub"');
421
423
  expect(text).toContain("1 passed / 1 failed");
422
424
  expect(html).toContain("50%");
423
425
  // 现刻水位覆盖齐全 → 两面都不出 partial-coverage(text: "verdicts cover"; html: data-kind)。
@@ -431,7 +433,7 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
431
433
  it("位置前缀收窄:两扇门都只见范围内的 eval,范围外一致排除", async () => {
432
434
  const root = await makeRoot();
433
435
  await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
434
- res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, passed: false, detail: "boom" }] }),
436
+ res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
435
437
  res("weather/queens", "passed"),
436
438
  res("algebra/quadratic", "passed"),
437
439
  ]);
@@ -497,7 +499,7 @@ function hasPartialCoverageHtml(html: string): boolean {
497
499
  it("通过计数一致:同一 fixture 两面反映 3 过 1 败", async () => {
498
500
  const root = await makeRoot();
499
501
  await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
500
- res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0, passed: false, detail: "boom" }] }),
502
+ res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
501
503
  res("q2", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
502
504
  res("q3", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
503
505
  res("q4", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
@@ -29,9 +29,11 @@ export {
29
29
  } from "./locator.ts";
30
30
  export {
31
31
  buildAnnotatedEvalSource,
32
+ deriveSendAnnotations,
32
33
  type AnnotatedEvalSource,
33
34
  type AnnotatedEvalSourceSummary,
34
35
  type AnnotatedSourceLine,
36
+ type SendAnnotation,
35
37
  } from "./annotated-source.ts";
36
38
  export { loadAnnotatedEvalSource } from "./attempt-source.ts";
37
39
  export {