niceeval 0.13.1 → 0.13.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (548) hide show
  1. package/INDEX.md +25 -2
  2. package/README.md +9 -10
  3. package/README.zh.md +9 -10
  4. package/dist/agents/ai-sdk.cjs.map +1 -1
  5. package/dist/agents/ai-sdk.d.cts +1 -1
  6. package/dist/agents/ai-sdk.d.mts +1 -1
  7. package/dist/agents/ai-sdk.d.ts +1 -1
  8. package/dist/agents/bub-install-spec.cjs +19 -5
  9. package/dist/agents/bub-install-spec.cjs.map +1 -1
  10. package/dist/agents/bub-install-spec.d.cts +9 -2
  11. package/dist/agents/bub-install-spec.d.mts +9 -2
  12. package/dist/agents/bub-install-spec.d.ts +9 -2
  13. package/dist/agents/bub-install-spec.js +14 -12
  14. package/dist/agents/bub.cjs +10 -6
  15. package/dist/agents/bub.cjs.map +1 -1
  16. package/dist/agents/bub.d.cts +2 -2
  17. package/dist/agents/bub.d.mts +2 -2
  18. package/dist/agents/bub.d.ts +2 -2
  19. package/dist/agents/claude-code.cjs.map +1 -1
  20. package/dist/agents/claude-code.d.cts +2 -2
  21. package/dist/agents/claude-code.d.mts +2 -2
  22. package/dist/agents/claude-code.d.ts +2 -2
  23. package/dist/agents/codex.cjs +4 -4
  24. package/dist/agents/codex.cjs.map +1 -1
  25. package/dist/agents/codex.d.cts +3 -3
  26. package/dist/agents/codex.d.mts +3 -3
  27. package/dist/agents/codex.d.ts +3 -3
  28. package/dist/agents/coding-cli-versions.cjs +10 -1
  29. package/dist/agents/coding-cli-versions.cjs.map +1 -1
  30. package/dist/agents/coding-cli-versions.d.cts +8 -3
  31. package/dist/agents/coding-cli-versions.d.mts +8 -3
  32. package/dist/agents/coding-cli-versions.d.ts +8 -3
  33. package/dist/agents/coding-cli-versions.js +18 -12
  34. package/dist/agents/deepseek-harness.cjs +151 -0
  35. package/dist/agents/deepseek-harness.cjs.map +1 -0
  36. package/dist/agents/deepseek-harness.d.cts +14 -0
  37. package/dist/agents/deepseek-harness.d.mts +14 -0
  38. package/dist/agents/deepseek-harness.d.ts +14 -0
  39. package/dist/agents/deepseek-harness.js +7 -0
  40. package/dist/agents/deepseek-harness.js.map +1 -0
  41. package/dist/agents/exact-npm-plugins.cjs +28 -0
  42. package/dist/agents/exact-npm-plugins.cjs.map +1 -0
  43. package/dist/agents/exact-npm-plugins.d.cts +10 -0
  44. package/dist/agents/exact-npm-plugins.d.mts +10 -0
  45. package/dist/agents/exact-npm-plugins.d.ts +10 -0
  46. package/dist/agents/exact-npm-plugins.js +8 -0
  47. package/dist/agents/exact-npm-plugins.js.map +1 -0
  48. package/dist/agents/index.cjs +5 -1
  49. package/dist/agents/index.cjs.map +1 -1
  50. package/dist/agents/index.d.cts +4 -0
  51. package/dist/agents/index.d.mts +4 -0
  52. package/dist/agents/index.d.ts +4 -0
  53. package/dist/agents/index.js +54 -50
  54. package/dist/agents/index.mjs +54 -50
  55. package/dist/agents/omp.cjs +170 -0
  56. package/dist/agents/omp.cjs.map +1 -0
  57. package/dist/agents/omp.d.cts +14 -0
  58. package/dist/agents/omp.d.mts +14 -0
  59. package/dist/agents/omp.d.ts +14 -0
  60. package/dist/agents/omp.js +7 -0
  61. package/dist/agents/omp.js.map +1 -0
  62. package/dist/agents/openclaw.cjs +51 -2
  63. package/dist/agents/openclaw.cjs.map +1 -1
  64. package/dist/agents/openclaw.d.cts +2 -0
  65. package/dist/agents/openclaw.d.mts +2 -0
  66. package/dist/agents/openclaw.d.ts +2 -0
  67. package/dist/agents/types.cjs.map +1 -1
  68. package/dist/agents/types.d.cts +4 -4
  69. package/dist/agents/types.d.mts +4 -4
  70. package/dist/agents/types.d.ts +4 -4
  71. package/dist/agents/ui-message-stream.cjs +24 -7
  72. package/dist/agents/ui-message-stream.cjs.map +1 -1
  73. package/dist/analysis/api.cjs +28 -2
  74. package/dist/analysis/api.cjs.map +1 -1
  75. package/dist/analysis/api.d.cts +22 -1
  76. package/dist/analysis/api.d.mts +22 -1
  77. package/dist/analysis/api.d.ts +22 -1
  78. package/dist/analysis/api.js +8 -6
  79. package/dist/analysis/bindings.cjs +11 -18
  80. package/dist/analysis/bindings.cjs.map +1 -1
  81. package/dist/analysis/bindings.d.cts +177 -49
  82. package/dist/analysis/bindings.d.mts +177 -49
  83. package/dist/analysis/bindings.d.ts +177 -49
  84. package/dist/analysis/contracts.cjs +1 -0
  85. package/dist/analysis/contracts.cjs.map +1 -1
  86. package/dist/analysis/contracts.d.cts +43 -3
  87. package/dist/analysis/contracts.d.mts +43 -3
  88. package/dist/analysis/contracts.d.ts +43 -3
  89. package/dist/analysis/cost-projection.cjs +2 -1
  90. package/dist/analysis/cost-projection.cjs.map +1 -1
  91. package/dist/analysis/cost-projection.d.cts +1 -1
  92. package/dist/analysis/cost-projection.d.mts +1 -1
  93. package/dist/analysis/cost-projection.d.ts +1 -1
  94. package/dist/analysis/cost.cjs +13 -3
  95. package/dist/analysis/cost.cjs.map +1 -1
  96. package/dist/analysis/cost.d.cts +10 -3
  97. package/dist/analysis/cost.d.mts +10 -3
  98. package/dist/analysis/cost.d.ts +10 -3
  99. package/dist/analysis/experiment-groups.cjs +96 -0
  100. package/dist/analysis/experiment-groups.cjs.map +1 -0
  101. package/dist/analysis/experiment-groups.d.cts +9 -0
  102. package/dist/analysis/experiment-groups.d.mts +9 -0
  103. package/dist/analysis/experiment-groups.d.ts +9 -0
  104. package/dist/analysis/experiment-groups.js +10 -0
  105. package/dist/analysis/experiment-groups.js.map +1 -0
  106. package/dist/analysis/index.cjs +4 -1
  107. package/dist/analysis/index.cjs.map +1 -1
  108. package/dist/analysis/index.d.cts +4 -2
  109. package/dist/analysis/index.d.mts +4 -2
  110. package/dist/analysis/index.d.ts +4 -2
  111. package/dist/analysis/index.js +40 -36
  112. package/dist/analysis/index.mjs +40 -36
  113. package/dist/assertions/limits.cjs +4 -0
  114. package/dist/assertions/limits.cjs.map +1 -1
  115. package/dist/assertions/limits.d.cts +4 -0
  116. package/dist/assertions/limits.d.mts +4 -0
  117. package/dist/assertions/limits.d.ts +4 -0
  118. package/dist/assertions/match.cjs +63 -25
  119. package/dist/assertions/match.cjs.map +1 -1
  120. package/dist/assertions/match.d.cts +7 -0
  121. package/dist/assertions/match.d.mts +7 -0
  122. package/dist/assertions/match.d.ts +7 -0
  123. package/dist/assertions/record/attachment.cjs +32 -4
  124. package/dist/assertions/record/attachment.cjs.map +1 -1
  125. package/dist/assertions/record/diff.cjs +10 -4
  126. package/dist/assertions/record/diff.cjs.map +1 -1
  127. package/dist/assertions/record/producer.cjs +101 -4
  128. package/dist/assertions/record/producer.cjs.map +1 -1
  129. package/dist/assertions/record/producer.d.cts +4 -1
  130. package/dist/assertions/record/producer.d.mts +4 -1
  131. package/dist/assertions/record/producer.d.ts +4 -1
  132. package/dist/assertions/runtime.cjs +45 -5
  133. package/dist/assertions/runtime.cjs.map +1 -1
  134. package/dist/cli/record.cjs +113 -8
  135. package/dist/cli/record.cjs.map +1 -1
  136. package/dist/cli/record.d.cts +20 -1
  137. package/dist/cli/record.d.mts +20 -1
  138. package/dist/cli/record.d.ts +20 -1
  139. package/dist/cli/record.js +4 -2
  140. package/dist/cli.cjs +386 -31
  141. package/dist/cli.cjs.map +1 -1
  142. package/dist/cli.d.cts +10 -2
  143. package/dist/cli.d.mts +10 -2
  144. package/dist/cli.d.ts +10 -2
  145. package/dist/context/send-retry.cjs +4 -4
  146. package/dist/context/send-retry.cjs.map +1 -1
  147. package/dist/coordination/host/runtime.cjs +8 -30
  148. package/dist/coordination/host/runtime.cjs.map +1 -1
  149. package/dist/coordination/host/runtime.d.cts +3 -3
  150. package/dist/coordination/host/runtime.d.mts +3 -3
  151. package/dist/coordination/host/runtime.d.ts +3 -3
  152. package/dist/coordination/host/types.cjs.map +1 -1
  153. package/dist/coordination/host/types.d.cts +0 -5
  154. package/dist/coordination/host/types.d.mts +0 -5
  155. package/dist/coordination/host/types.d.ts +0 -5
  156. package/dist/coordination/platform/node.cjs +90 -9
  157. package/dist/coordination/platform/node.cjs.map +1 -1
  158. package/dist/define.cjs +25 -1
  159. package/dist/define.cjs.map +1 -1
  160. package/dist/experiment/host/index.cjs +1 -0
  161. package/dist/experiment/host/index.cjs.map +1 -1
  162. package/dist/i18n/en.cjs +26 -8
  163. package/dist/i18n/en.cjs.map +1 -1
  164. package/dist/i18n/en.d.cts +19 -6
  165. package/dist/i18n/en.d.mts +19 -6
  166. package/dist/i18n/en.d.ts +19 -6
  167. package/dist/index.cjs.map +1 -1
  168. package/dist/index.d.cts +1 -1
  169. package/dist/index.d.mts +1 -1
  170. package/dist/index.d.ts +1 -1
  171. package/dist/o11y/parsers/hermes.cjs +1 -1
  172. package/dist/o11y/parsers/hermes.cjs.map +1 -1
  173. package/dist/o11y/prices.json +1 -1
  174. package/dist/o11y/record/families.cjs +12 -3
  175. package/dist/o11y/record/families.cjs.map +1 -1
  176. package/dist/o11y/record/families.d.cts +8 -8
  177. package/dist/o11y/record/families.d.mts +8 -8
  178. package/dist/o11y/record/families.d.ts +8 -8
  179. package/dist/o11y/record/family-projectors.d.cts +66 -18
  180. package/dist/o11y/record/family-projectors.d.mts +66 -18
  181. package/dist/o11y/record/family-projectors.d.ts +66 -18
  182. package/dist/o11y/record/family-writers.d.cts +11 -3
  183. package/dist/o11y/record/family-writers.d.mts +11 -3
  184. package/dist/o11y/record/family-writers.d.ts +11 -3
  185. package/dist/o11y/record/runner-producer.cjs +14 -4
  186. package/dist/o11y/record/runner-producer.cjs.map +1 -1
  187. package/dist/record/codec/core.d.cts +2 -2
  188. package/dist/record/codec/core.d.mts +2 -2
  189. package/dist/record/codec/core.d.ts +2 -2
  190. package/dist/record/family/catalog.d.cts +44 -12
  191. package/dist/record/family/catalog.d.mts +44 -12
  192. package/dist/record/family/catalog.d.ts +44 -12
  193. package/dist/record/family/index.js +128 -124
  194. package/dist/record/family/observability-v1.cjs +35 -0
  195. package/dist/record/family/observability-v1.cjs.map +1 -0
  196. package/dist/record/family/observability-v1.d.cts +3 -0
  197. package/dist/record/family/observability-v1.d.mts +3 -0
  198. package/dist/record/family/observability-v1.d.ts +3 -0
  199. package/dist/record/family/observability-v1.js +6 -0
  200. package/dist/record/family/observability-v1.js.map +1 -0
  201. package/dist/record/family/observability.cjs +58 -9
  202. package/dist/record/family/observability.cjs.map +1 -1
  203. package/dist/record/family/observability.d.cts +1014 -15
  204. package/dist/record/family/observability.d.mts +1014 -15
  205. package/dist/record/family/observability.d.ts +1014 -15
  206. package/dist/record/family/observability.js +22 -18
  207. package/dist/record/host/runtime.cjs +600 -31
  208. package/dist/record/host/runtime.cjs.map +1 -1
  209. package/dist/record/host/types.cjs.map +1 -1
  210. package/dist/record/host/types.d.cts +33 -9
  211. package/dist/record/host/types.d.mts +33 -9
  212. package/dist/record/host/types.d.ts +33 -9
  213. package/dist/record/maintenance/runtime.cjs +1 -2
  214. package/dist/record/maintenance/runtime.cjs.map +1 -1
  215. package/dist/record/model/core.cjs.map +1 -1
  216. package/dist/record/model/core.d.cts +1 -1
  217. package/dist/record/model/core.d.mts +1 -1
  218. package/dist/record/model/core.d.ts +1 -1
  219. package/dist/record/model/definition.d.cts +6 -6
  220. package/dist/record/model/definition.d.mts +6 -6
  221. package/dist/record/model/definition.d.ts +6 -6
  222. package/dist/record/model/identifiers.cjs +1 -1
  223. package/dist/record/model/identifiers.cjs.map +1 -1
  224. package/dist/record/model/identifiers.d.cts +1 -1
  225. package/dist/record/model/identifiers.d.mts +1 -1
  226. package/dist/record/model/identifiers.d.ts +1 -1
  227. package/dist/record/platform/node.cjs +159 -23
  228. package/dist/record/platform/node.cjs.map +1 -1
  229. package/dist/record/platform/services.cjs.map +1 -1
  230. package/dist/record/platform/services.d.cts +15 -2
  231. package/dist/record/platform/services.d.mts +15 -2
  232. package/dist/record/platform/services.d.ts +15 -2
  233. package/dist/record/reader/errors.cjs +37 -1
  234. package/dist/record/reader/errors.cjs.map +1 -1
  235. package/dist/record/reader/errors.d.cts +49 -1
  236. package/dist/record/reader/errors.d.mts +49 -1
  237. package/dist/record/reader/errors.d.ts +49 -1
  238. package/dist/record/reader/errors.js +22 -12
  239. package/dist/record/reader/format.cjs +76 -17
  240. package/dist/record/reader/format.cjs.map +1 -1
  241. package/dist/record/reader/format.d.cts +11 -4
  242. package/dist/record/reader/format.d.mts +11 -4
  243. package/dist/record/reader/format.d.ts +11 -4
  244. package/dist/record/reader/format.js +2 -0
  245. package/dist/record/reader/index.js +32 -18
  246. package/dist/record/reader/runtime.cjs +64 -0
  247. package/dist/record/reader/runtime.cjs.map +1 -1
  248. package/dist/record/reader/runtime.d.cts +15 -0
  249. package/dist/record/reader/runtime.d.mts +15 -0
  250. package/dist/record/reader/runtime.d.ts +15 -0
  251. package/dist/record/reader/runtime.js +2 -0
  252. package/dist/report/assets/styles.css +53 -29
  253. package/dist/report/built-in/analysis-values.cjs +21 -13
  254. package/dist/report/built-in/analysis-values.cjs.map +1 -1
  255. package/dist/report/built-in/analysis-values.d.cts +11 -22
  256. package/dist/report/built-in/analysis-values.d.mts +11 -22
  257. package/dist/report/built-in/analysis-values.d.ts +11 -22
  258. package/dist/report/built-in/analysis-values.js +4 -6
  259. package/dist/report/built-in/index.cjs +1 -3
  260. package/dist/report/built-in/index.cjs.map +1 -1
  261. package/dist/report/built-in/index.d.cts +1 -1
  262. package/dist/report/built-in/index.d.mts +1 -1
  263. package/dist/report/built-in/index.d.ts +1 -1
  264. package/dist/report/built-in/index.js +4 -8
  265. package/dist/report/built-in/index.mjs +4 -8
  266. package/dist/report/built-in/machine.cjs +0 -2
  267. package/dist/report/built-in/machine.cjs.map +1 -1
  268. package/dist/report/built-in/machine.d.cts +0 -2
  269. package/dist/report/built-in/machine.d.mts +0 -2
  270. package/dist/report/built-in/machine.d.ts +0 -2
  271. package/dist/report/built-in/result-components.cjs +49 -51
  272. package/dist/report/built-in/result-components.cjs.map +1 -1
  273. package/dist/report/built-in/result-components.d.cts +16 -17
  274. package/dist/report/built-in/result-components.d.mts +16 -17
  275. package/dist/report/built-in/result-components.d.ts +16 -17
  276. package/dist/report/built-in/result-components.js +0 -4
  277. package/dist/report/built-in/run-membership-overview.cjs +126 -17
  278. package/dist/report/built-in/run-membership-overview.cjs.map +1 -1
  279. package/dist/report/built-in/run-membership-overview.d.cts +14 -0
  280. package/dist/report/built-in/run-membership-overview.d.mts +14 -0
  281. package/dist/report/built-in/run-membership-overview.d.ts +14 -0
  282. package/dist/report/built-in/run-membership-overview.js +4 -2
  283. package/dist/report/built-in/standard.cjs +80 -45
  284. package/dist/report/built-in/standard.cjs.map +1 -1
  285. package/dist/report/built-in/standard.d.cts +19 -41
  286. package/dist/report/built-in/standard.d.mts +19 -41
  287. package/dist/report/built-in/standard.d.ts +19 -41
  288. package/dist/report/built-in/standard.js +6 -6
  289. package/dist/report/components/attempt-detail/compute.cjs +78 -2
  290. package/dist/report/components/attempt-detail/compute.cjs.map +1 -1
  291. package/dist/report/components/attempt-detail/compute.d.cts +1 -1
  292. package/dist/report/components/attempt-detail/compute.d.mts +1 -1
  293. package/dist/report/components/attempt-detail/compute.d.ts +1 -1
  294. package/dist/report/components/attempt-detail/content.cjs +1 -1
  295. package/dist/report/components/attempt-detail/content.cjs.map +1 -1
  296. package/dist/report/components/entity-lists/compute.cjs +42 -5
  297. package/dist/report/components/entity-lists/compute.cjs.map +1 -1
  298. package/dist/report/components/entity-lists/compute.d.cts +10 -8
  299. package/dist/report/components/entity-lists/compute.d.mts +10 -8
  300. package/dist/report/components/entity-lists/compute.d.ts +10 -8
  301. package/dist/report/components/entity-lists/content.cjs +31 -6
  302. package/dist/report/components/entity-lists/content.cjs.map +1 -1
  303. package/dist/report/components/entity-lists/index.cjs +9 -4
  304. package/dist/report/components/entity-lists/index.cjs.map +1 -1
  305. package/dist/report/components/entity-lists/index.d.cts +4 -7
  306. package/dist/report/components/entity-lists/index.d.mts +4 -7
  307. package/dist/report/components/entity-lists/index.d.ts +4 -7
  308. package/dist/report/components/summaries/compute.cjs +8 -1
  309. package/dist/report/components/summaries/compute.cjs.map +1 -1
  310. package/dist/report/components/summaries/compute.d.cts +9 -1
  311. package/dist/report/components/summaries/compute.d.mts +9 -1
  312. package/dist/report/components/summaries/compute.d.ts +9 -1
  313. package/dist/report/components/summaries/index.cjs +48 -4
  314. package/dist/report/components/summaries/index.cjs.map +1 -1
  315. package/dist/report/components/summaries/index.d.cts +7 -4
  316. package/dist/report/components/summaries/index.d.mts +7 -4
  317. package/dist/report/components/summaries/index.d.ts +7 -4
  318. package/dist/report/definition/cell.cjs +3 -0
  319. package/dist/report/definition/cell.cjs.map +1 -1
  320. package/dist/report/definition/cell.d.cts +2 -0
  321. package/dist/report/definition/cell.d.mts +2 -0
  322. package/dist/report/definition/cell.d.ts +2 -0
  323. package/dist/report/definition/primitives/chart.cjs +8 -5
  324. package/dist/report/definition/primitives/chart.cjs.map +1 -1
  325. package/dist/report/definition/primitives/points-dataset.cjs +1 -0
  326. package/dist/report/definition/primitives/points-dataset.cjs.map +1 -1
  327. package/dist/report/definition/primitives.cjs +15 -2
  328. package/dist/report/definition/primitives.cjs.map +1 -1
  329. package/dist/report/definition/report.cjs +27 -3
  330. package/dist/report/definition/report.cjs.map +1 -1
  331. package/dist/report/definition/report.d.cts +8 -0
  332. package/dist/report/definition/report.d.mts +8 -0
  333. package/dist/report/definition/report.d.ts +8 -0
  334. package/dist/report/execution/machine.cjs +30 -14
  335. package/dist/report/execution/machine.cjs.map +1 -1
  336. package/dist/report/execution/machine.d.cts +20 -10
  337. package/dist/report/execution/machine.d.mts +20 -10
  338. package/dist/report/execution/machine.d.ts +20 -10
  339. package/dist/report/execution/machine.js +6 -6
  340. package/dist/report/host/execute.cjs +25 -16
  341. package/dist/report/host/execute.cjs.map +1 -1
  342. package/dist/report/host/execute.d.cts +4 -0
  343. package/dist/report/host/execute.d.mts +4 -0
  344. package/dist/report/host/execute.d.ts +4 -0
  345. package/dist/report/host/from-record.cjs +22 -5
  346. package/dist/report/host/from-record.cjs.map +1 -1
  347. package/dist/report/host/machine.cjs +53 -11
  348. package/dist/report/host/machine.cjs.map +1 -1
  349. package/dist/report/host/site-runtime.cjs +9 -8
  350. package/dist/report/host/site-runtime.cjs.map +1 -1
  351. package/dist/report/host/site-runtime.d.cts +1 -1
  352. package/dist/report/host/site-runtime.d.mts +1 -1
  353. package/dist/report/host/site-runtime.d.ts +1 -1
  354. package/dist/report/host/static.cjs +56 -9
  355. package/dist/report/host/static.cjs.map +1 -1
  356. package/dist/report/index.cjs.map +1 -1
  357. package/dist/report/index.d.cts +1 -1
  358. package/dist/report/index.d.mts +1 -1
  359. package/dist/report/index.d.ts +1 -1
  360. package/dist/report/model/dataset.cjs +7 -2
  361. package/dist/report/model/dataset.cjs.map +1 -1
  362. package/dist/report/model/locale.cjs +2 -0
  363. package/dist/report/model/locale.cjs.map +1 -1
  364. package/dist/report/model/locale.d.cts +1 -0
  365. package/dist/report/model/locale.d.mts +1 -0
  366. package/dist/report/model/locale.d.ts +1 -0
  367. package/dist/report/model/metrics.cjs +3 -1
  368. package/dist/report/model/metrics.cjs.map +1 -1
  369. package/dist/report/model/pricing.cjs +16 -11
  370. package/dist/report/model/pricing.cjs.map +1 -1
  371. package/dist/report/react/styles.css +53 -29
  372. package/dist/runner/adoption.cjs +1 -0
  373. package/dist/runner/adoption.cjs.map +1 -1
  374. package/dist/runner/attempt.cjs +44 -5
  375. package/dist/runner/attempt.cjs.map +1 -1
  376. package/dist/runner/attempt.d.cts +13 -1
  377. package/dist/runner/attempt.d.mts +13 -1
  378. package/dist/runner/attempt.d.ts +13 -1
  379. package/dist/runner/config-identity.cjs +10 -2
  380. package/dist/runner/config-identity.cjs.map +1 -1
  381. package/dist/runner/config-identity.d.cts +4 -2
  382. package/dist/runner/config-identity.d.mts +4 -2
  383. package/dist/runner/config-identity.d.ts +4 -2
  384. package/dist/runner/feedback/human.cjs +298 -59
  385. package/dist/runner/feedback/human.cjs.map +1 -1
  386. package/dist/runner/feedback/json.cjs +14 -6
  387. package/dist/runner/feedback/json.cjs.map +1 -1
  388. package/dist/runner/feedback/json.d.cts +3 -2
  389. package/dist/runner/feedback/json.d.mts +3 -2
  390. package/dist/runner/feedback/json.d.ts +3 -2
  391. package/dist/runner/feedback/reducer.cjs +2 -0
  392. package/dist/runner/feedback/reducer.cjs.map +1 -1
  393. package/dist/runner/judge-config.cjs +1 -1
  394. package/dist/runner/judge-config.cjs.map +1 -1
  395. package/dist/runner/ledger.cjs +124 -12
  396. package/dist/runner/ledger.cjs.map +1 -1
  397. package/dist/runner/project-current.cjs +13 -5
  398. package/dist/runner/project-current.cjs.map +1 -1
  399. package/dist/runner/run.cjs +564 -256
  400. package/dist/runner/run.cjs.map +1 -1
  401. package/dist/runner/sandbox-pool.cjs +161 -30
  402. package/dist/runner/sandbox-pool.cjs.map +1 -1
  403. package/dist/runner/sandbox-pool.d.cts +34 -2
  404. package/dist/runner/sandbox-pool.d.mts +34 -2
  405. package/dist/runner/sandbox-pool.d.ts +34 -2
  406. package/dist/runner/shared-state-lease.cjs +803 -0
  407. package/dist/runner/shared-state-lease.cjs.map +1 -0
  408. package/dist/runner/shared-state-lease.d.cts +121 -0
  409. package/dist/runner/shared-state-lease.d.mts +121 -0
  410. package/dist/runner/shared-state-lease.d.ts +121 -0
  411. package/dist/runner/shared-state-lease.js +24 -0
  412. package/dist/runner/shared-state-lease.js.map +1 -0
  413. package/dist/runner/teardown-registry.cjs +97 -34
  414. package/dist/runner/teardown-registry.cjs.map +1 -1
  415. package/dist/runner/teardown-registry.d.cts +27 -1
  416. package/dist/runner/teardown-registry.d.mts +27 -1
  417. package/dist/runner/teardown-registry.d.ts +27 -1
  418. package/dist/runner/teardown-registry.js +20 -16
  419. package/dist/runner/types.cjs.map +1 -1
  420. package/dist/runner/types.d.cts +31 -5
  421. package/dist/runner/types.d.mts +31 -5
  422. package/dist/runner/types.d.ts +31 -5
  423. package/dist/sample/analysis.cjs +249 -459
  424. package/dist/sample/analysis.cjs.map +1 -1
  425. package/dist/sample/capability.cjs +138 -7
  426. package/dist/sample/capability.cjs.map +1 -1
  427. package/dist/sample/capability.d.cts +3 -1
  428. package/dist/sample/capability.d.mts +3 -1
  429. package/dist/sample/capability.d.ts +3 -1
  430. package/dist/sample/capability.js +8 -6
  431. package/dist/sandbox/docker-agent-image.cjs +5 -1
  432. package/dist/sandbox/docker-agent-image.cjs.map +1 -1
  433. package/dist/sandbox/docker-agent-image.d.cts +2 -0
  434. package/dist/sandbox/docker-agent-image.d.mts +2 -0
  435. package/dist/sandbox/docker-agent-image.d.ts +2 -0
  436. package/dist/sandbox/docker-agent-image.js +14 -10
  437. package/dist/sandbox/docker-profile/schema.cjs +283 -534
  438. package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
  439. package/dist/sandbox/docker-profile/schema.d.cts +10 -27
  440. package/dist/sandbox/docker-profile/schema.d.mts +10 -27
  441. package/dist/sandbox/docker-profile/schema.d.ts +10 -27
  442. package/dist/sandbox/e2b-agent-template.cjs +5 -2
  443. package/dist/sandbox/e2b-agent-template.cjs.map +1 -1
  444. package/dist/sandbox/index.cjs +4 -2
  445. package/dist/sandbox/index.cjs.map +1 -1
  446. package/dist/sandbox/index.d.cts +1 -1
  447. package/dist/sandbox/index.d.mts +1 -1
  448. package/dist/sandbox/index.d.ts +1 -1
  449. package/dist/sandbox/index.js +120 -116
  450. package/dist/sandbox/index.mjs +120 -116
  451. package/dist/sandbox/keep-registry.cjs +28 -66
  452. package/dist/sandbox/keep-registry.cjs.map +1 -1
  453. package/dist/sandbox/types.cjs.map +1 -1
  454. package/dist/sandbox/types.d.cts +1 -1
  455. package/dist/sandbox/types.d.mts +1 -1
  456. package/dist/sandbox/types.d.ts +1 -1
  457. package/dist/shared/aggregate.cjs +16 -6
  458. package/dist/shared/aggregate.cjs.map +1 -1
  459. package/dist/shared/aggregate.d.cts +12 -4
  460. package/dist/shared/aggregate.d.mts +12 -4
  461. package/dist/shared/aggregate.d.ts +12 -4
  462. package/dist/shared/turn-label.cjs +20 -2
  463. package/dist/shared/turn-label.cjs.map +1 -1
  464. package/dist/shared/turn-label.d.cts +2 -0
  465. package/dist/shared/turn-label.d.mts +2 -0
  466. package/dist/shared/turn-label.d.ts +2 -0
  467. package/dist/shared/turn-label.js +4 -2
  468. package/dist/vendor/remark/remark-gfm.cjs +7 -7
  469. package/dist/vendor/remark/remark-gfm.cjs.map +1 -1
  470. package/dist/vendor/remark/remark-parse.cjs +12 -12
  471. package/dist/vendor/remark/remark-parse.cjs.map +1 -1
  472. package/dist/vendor/remark/remark.cjs +13 -13
  473. package/dist/vendor/remark/remark.cjs.map +1 -1
  474. package/dist/view/server.cjs +6 -0
  475. package/dist/view/server.cjs.map +1 -1
  476. package/docs-site/images/hitl-handshake-zh.svg +3 -3
  477. package/docs-site/zh/examples/ai-agent-application.mdx +1 -1
  478. package/docs-site/zh/examples/coding-agent-extensions.mdx +1 -1
  479. package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +4 -3
  480. package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
  481. package/docs-site/zh/examples/integrations/codex-sdk.mdx +5 -5
  482. package/docs-site/zh/examples/integrations/langgraph.mdx +4 -4
  483. package/docs-site/zh/examples/integrations/pi-sdk.mdx +5 -5
  484. package/docs-site/zh/explanation/adapter.mdx +5 -5
  485. package/docs-site/zh/explanation/assert.mdx +7 -7
  486. package/docs-site/zh/explanation/drive.mdx +4 -2
  487. package/docs-site/zh/explanation/evals.mdx +10 -8
  488. package/docs-site/zh/explanation/experiment.mdx +2 -2
  489. package/docs-site/zh/explanation/hitl.mdx +9 -9
  490. package/docs-site/zh/explanation/judge.mdx +5 -5
  491. package/docs-site/zh/explanation/overview.mdx +3 -3
  492. package/docs-site/zh/explanation/runner.mdx +1 -1
  493. package/docs-site/zh/index.mdx +3 -3
  494. package/docs-site/zh/reference/builtin-agents.mdx +13 -13
  495. package/docs-site/zh/reference/capabilities.mdx +1 -1
  496. package/docs-site/zh/reference/cli.mdx +18 -12
  497. package/docs-site/zh/reference/define-agent.mdx +130 -5
  498. package/docs-site/zh/reference/define-config.mdx +2 -2
  499. package/docs-site/zh/reference/define-eval.mdx +4 -4
  500. package/docs-site/zh/reference/events.mdx +11 -11
  501. package/docs-site/zh/reference/official-adapters.mdx +6 -6
  502. package/docs-site/zh/reference/results-data.mdx +6 -0
  503. package/docs-site/zh/troubleshooting/debugging.mdx +1 -1
  504. package/docs-site/zh/troubleshooting/recover-after-kill.mdx +37 -5
  505. package/docs-site/zh/tutorials/accept-results.mdx +72 -0
  506. package/docs-site/zh/tutorials/agent-onboarding.mdx +6 -6
  507. package/docs-site/zh/tutorials/authoring.mdx +9 -13
  508. package/docs-site/zh/tutorials/ci-integration.mdx +6 -6
  509. package/docs-site/zh/tutorials/compare-runs.mdx +61 -0
  510. package/docs-site/zh/tutorials/concurrency.mdx +13 -12
  511. package/docs-site/zh/tutorials/configuration.mdx +4 -6
  512. package/docs-site/zh/tutorials/connect-your-agent.mdx +1 -1
  513. package/docs-site/zh/tutorials/control-run-cost.mdx +58 -0
  514. package/docs-site/zh/tutorials/criteria-files.mdx +2 -2
  515. package/docs-site/zh/tutorials/custom-reports.mdx +2 -2
  516. package/docs-site/zh/tutorials/debug-lifecycle-plan.mdx +46 -0
  517. package/docs-site/zh/tutorials/docker-in-docker.mdx +52 -43
  518. package/docs-site/zh/tutorials/eval-groups.mdx +6 -6
  519. package/docs-site/zh/tutorials/evaluation-kinds.mdx +3 -3
  520. package/docs-site/zh/tutorials/experiments.mdx +2 -3
  521. package/docs-site/zh/tutorials/handle-execution-failures.mdx +18 -10
  522. package/docs-site/zh/tutorials/install-custom-sandbox-agent.mdx +16 -8
  523. package/docs-site/zh/tutorials/nixos-managed-dind.mdx +160 -0
  524. package/docs-site/zh/tutorials/plugins.mdx +14 -0
  525. package/docs-site/zh/tutorials/rerun-and-cache.mdx +4 -2
  526. package/docs-site/zh/tutorials/sandbox-agent.mdx +17 -11
  527. package/docs-site/zh/tutorials/sandbox-providers.mdx +17 -13
  528. package/docs-site/zh/tutorials/sandbox-reuse.mdx +12 -19
  529. package/docs-site/zh/tutorials/select-and-preview.mdx +65 -0
  530. package/docs-site/zh/tutorials/verify-judge.mdx +78 -0
  531. package/docs-site/zh/tutorials/viewing-results.mdx +38 -1
  532. package/docs-site/zh/tutorials/write-experiment.mdx +7 -5
  533. package/docs-site/zh/tutorials/write-send.mdx +9 -8
  534. package/package.json +9 -5
  535. package/dist/report/built-in/overview.cjs +0 -44
  536. package/dist/report/built-in/overview.cjs.map +0 -1
  537. package/dist/report/built-in/overview.d.cts +0 -6
  538. package/dist/report/built-in/overview.d.mts +0 -6
  539. package/dist/report/built-in/overview.d.ts +0 -6
  540. package/dist/report/built-in/overview.js +0 -9
  541. package/dist/report/built-in/overview.js.map +0 -1
  542. package/dist/runner/gate-lease.cjs +0 -283
  543. package/dist/runner/gate-lease.cjs.map +0 -1
  544. package/dist/runner/gate-lease.d.cts +0 -62
  545. package/dist/runner/gate-lease.d.mts +0 -62
  546. package/dist/runner/gate-lease.d.ts +0 -62
  547. package/dist/runner/gate-lease.js +0 -22
  548. package/dist/runner/gate-lease.js.map +0 -1
@@ -41,6 +41,7 @@ exports.commandMatch = commandMatch;
41
41
  exports.eventMatch = eventMatch;
42
42
  const json_match_ts_1 = require("../shared/json-match.cjs");
43
43
  const util_ts_1 = require("../util.cjs");
44
+ const limits_ts_1 = require("./limits.cjs");
44
45
  // 这些 symbol 不导出。外部作者无法构造看似可消费的 Match;输入 variance 与 refinement
45
46
  // variance 也保持分离,ScoreMatch 不会伪造一个没有含义的输出 refinement。
46
47
  const matchInputBrand = Symbol("niceeval.match.input");
@@ -67,6 +68,7 @@ function diagnostic(code, message, options = {}) {
67
68
  ...(options.reason === undefined ? {} : { reason: options.reason }),
68
69
  ...(options.locator === undefined ? {} : { locator: options.locator }),
69
70
  ...(options.children === undefined ? {} : { children: Object.freeze([...options.children]) }),
71
+ ...(options.truncation === undefined ? {} : { truncation: Object.freeze({ ...options.truncation }) }),
70
72
  });
71
73
  }
72
74
  function matched(value, detail) {
@@ -1081,30 +1083,66 @@ function assertManagedEventMatch(value, label = "match") {
1081
1083
  * cannot manufacture a positive count or an absence proof.
1082
1084
  */
1083
1085
  async function evaluateToolMatchCollection(match, occurrences, options) {
1084
- const candidates = [];
1085
- for (const occurrence of occurrences) {
1086
- candidates.push(Object.freeze({
1086
+ const sampledCandidates = [];
1087
+ let matchCount = 0;
1088
+ let firstMatch;
1089
+ let firstUnavailable;
1090
+ let exactOverage;
1091
+ for (const [index, occurrence] of occurrences.entries()) {
1092
+ const candidate = Object.freeze({
1093
+ index,
1087
1094
  candidate: occurrence,
1088
1095
  result: await evaluateBooleanMatch(match, occurrence),
1089
- }));
1096
+ });
1097
+ if (sampledCandidates.length < limits_ts_1.assertionRuntimeLimits.collectionDiagnosticCandidates) {
1098
+ sampledCandidates.push(candidate);
1099
+ }
1100
+ if (candidate.result.state === "matched") {
1101
+ matchCount += 1;
1102
+ firstMatch ??= candidate;
1103
+ if (options.quantifier.kind === "exact" && matchCount === options.quantifier.count + 1) {
1104
+ exactOverage = candidate;
1105
+ }
1106
+ }
1107
+ else if (candidate.result.state === "unavailable") {
1108
+ firstUnavailable ??= candidate;
1109
+ }
1090
1110
  }
1091
- const children = candidates.map((candidate, index) => resultChild(candidate.result, index, `candidate ${candidate.candidate.id}`));
1092
- const matches = candidates.filter((candidate) => candidate.result.state === "matched");
1093
- const unavailableCandidate = candidates.find((candidate) => candidate.result.state === "unavailable");
1094
- const unavailableReason = unavailableCandidate?.result.state === "unavailable"
1095
- ? unavailableCandidate.result.reason
1111
+ const unavailableReason = firstUnavailable?.result.state === "unavailable"
1112
+ ? firstUnavailable.result.reason
1096
1113
  : options.coverageReason;
1097
1114
  const incomplete = unavailableReason !== undefined;
1098
- const received = `${matches.length} definite match${matches.length === 1 ? "" : "es"}`;
1115
+ const received = `${matchCount} definite match${matchCount === 1 ? "" : "es"}`;
1099
1116
  const candidateLocator = (candidate) => candidate === undefined ? undefined : { kind: "tool-occurrence", id: candidate.candidate.id };
1117
+ const diagnosticSample = (...decisive) => {
1118
+ const byIndex = new Map();
1119
+ for (const candidate of [...sampledCandidates, ...decisive]) {
1120
+ if (candidate !== undefined)
1121
+ byIndex.set(candidate.index, candidate);
1122
+ }
1123
+ const captured = [...byIndex.values()].sort((left, right) => left.index - right.index);
1124
+ return Object.freeze({
1125
+ children: Object.freeze(captured.map((candidate) => resultChild(candidate.result, candidate.index, `candidate ${candidate.candidate.id}`))),
1126
+ ...(captured.length === occurrences.length
1127
+ ? {}
1128
+ : {
1129
+ truncation: Object.freeze({
1130
+ code: "diagnostic-truncated",
1131
+ reason: "sampled",
1132
+ captured: captured.length,
1133
+ knownTotal: occurrences.length,
1134
+ }),
1135
+ }),
1136
+ });
1137
+ };
1100
1138
  if (options.quantifier.kind === "absent") {
1101
- const first = matches[0];
1139
+ const first = firstMatch;
1102
1140
  if (first !== undefined) {
1103
1141
  return mismatched(diagnostic("tool-absence-mismatch", `notCalledTool(${match.name}) observed a matching tool`, {
1104
1142
  expected: `no ${match.name}`,
1105
1143
  received,
1106
1144
  locator: candidateLocator(first),
1107
- children,
1145
+ ...diagnosticSample(first),
1108
1146
  }));
1109
1147
  }
1110
1148
  if (incomplete) {
@@ -1112,38 +1150,38 @@ async function evaluateToolMatchCollection(match, occurrences, options) {
1112
1150
  expected: `no ${match.name}`,
1113
1151
  received,
1114
1152
  reason: unavailableReason,
1115
- children,
1153
+ ...diagnosticSample(firstUnavailable),
1116
1154
  }));
1117
1155
  }
1118
1156
  return matched(undefined, diagnostic("tool-absence-match", `notCalledTool(${match.name}) observed no matching tool`, {
1119
1157
  expected: `no ${match.name}`,
1120
1158
  received,
1121
- children,
1159
+ ...diagnosticSample(),
1122
1160
  }));
1123
1161
  }
1124
1162
  const { kind, count } = options.quantifier;
1125
1163
  const expected = kind === "exact" ? `exactly ${count} × ${match.name}` : `at least ${count} × ${match.name}`;
1126
- if (kind === "exact" && matches.length > count) {
1127
- const overage = matches[count];
1164
+ if (kind === "exact" && matchCount > count) {
1165
+ const overage = exactOverage;
1128
1166
  return mismatched(diagnostic("tool-count-exceeded", `calledTool(${match.name}) exceeded its exact count`, {
1129
1167
  expected,
1130
1168
  received,
1131
1169
  locator: candidateLocator(overage),
1132
- children,
1170
+ ...diagnosticSample(overage),
1133
1171
  }));
1134
1172
  }
1135
- if (kind === "at-least" && matches.length >= count) {
1136
- return matched(matches[0]?.candidate, diagnostic("tool-count-match", `calledTool(${match.name}) satisfied its lower bound`, {
1173
+ if (kind === "at-least" && matchCount >= count) {
1174
+ return matched(firstMatch?.candidate, diagnostic("tool-count-match", `calledTool(${match.name}) satisfied its lower bound`, {
1137
1175
  expected,
1138
1176
  received,
1139
- children,
1177
+ ...diagnosticSample(firstMatch),
1140
1178
  }));
1141
1179
  }
1142
- if (kind === "exact" && matches.length === count && !incomplete) {
1143
- return matched(matches[0]?.candidate, diagnostic("tool-count-match", `calledTool(${match.name}) satisfied its exact count`, {
1180
+ if (kind === "exact" && matchCount === count && !incomplete) {
1181
+ return matched(firstMatch?.candidate, diagnostic("tool-count-match", `calledTool(${match.name}) satisfied its exact count`, {
1144
1182
  expected,
1145
1183
  received,
1146
- children,
1184
+ ...diagnosticSample(firstMatch),
1147
1185
  }));
1148
1186
  }
1149
1187
  if (incomplete) {
@@ -1151,13 +1189,13 @@ async function evaluateToolMatchCollection(match, occurrences, options) {
1151
1189
  expected,
1152
1190
  received,
1153
1191
  reason: unavailableReason,
1154
- children,
1192
+ ...diagnosticSample(firstUnavailable),
1155
1193
  }));
1156
1194
  }
1157
1195
  return mismatched(diagnostic("tool-count-mismatch", `calledTool(${match.name}) did not satisfy its count`, {
1158
1196
  expected,
1159
1197
  received,
1160
- children,
1198
+ ...diagnosticSample(),
1161
1199
  }));
1162
1200
  }
1163
1201
  function commandResult(occurrence, executable, argsStart, excludes) {