@tangle-network/agent-eval 0.129.0 → 0.130.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (428) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +2 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +81 -2872
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -360
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1188
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1709
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -891
  34. package/dist/benchmarks/index.js +2 -60
  35. package/dist/benchmarks-BJgDGkAD.js +754 -0
  36. package/dist/benchmarks-BJgDGkAD.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6381
  44. package/dist/campaign/index.js +3 -213
  45. package/dist/campaign-aKJt6emI.js +3886 -0
  46. package/dist/campaign-aKJt6emI.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -175
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5565
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1938
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -33
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -618
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CD_WZ_Xr.d.ts +2250 -0
  116. package/dist/index-CD_WZ_Xr.d.ts.map +1 -0
  117. package/dist/index-DSC51roc.d.ts +102 -0
  118. package/dist/index-DSC51roc.d.ts.map +1 -0
  119. package/dist/index-Em67JBjs.d.ts +335 -0
  120. package/dist/index-Em67JBjs.d.ts.map +1 -0
  121. package/dist/index.d.ts +3755 -15555
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11182 -11216
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -480
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1312
  196. package/dist/reporting.js +6 -51
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +760 -4010
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2325 -1958
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -2087
  211. package/dist/rollout/index.js +8 -168
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-CUmHkGbI.js +7718 -0
  253. package/dist/skillopt-optimization-method-CUmHkGbI.js.map +1 -0
  254. package/dist/skillopt-optimization-method-CWKVTnks.d.ts +1740 -0
  255. package/dist/skillopt-optimization-method-CWKVTnks.d.ts.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -959
  273. package/dist/supervisor-run/index.js +2 -65
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -252
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1173
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/docs/campaign-proposers.md +1 -0
  301. package/package.json +17 -9
  302. package/dist/benchmarks/index.js.map +0 -1
  303. package/dist/campaign/index.js.map +0 -1
  304. package/dist/chunk-2QU3YOPR.js +0 -7374
  305. package/dist/chunk-2QU3YOPR.js.map +0 -1
  306. package/dist/chunk-3OCR4R5I.js +0 -728
  307. package/dist/chunk-3OCR4R5I.js.map +0 -1
  308. package/dist/chunk-3RF76KTD.js +0 -84
  309. package/dist/chunk-3RF76KTD.js.map +0 -1
  310. package/dist/chunk-56TAVBOK.js +0 -698
  311. package/dist/chunk-5DTSBUL2.js +0 -159
  312. package/dist/chunk-5DTSBUL2.js.map +0 -1
  313. package/dist/chunk-7FO3TNPI.js +0 -232
  314. package/dist/chunk-7FO3TNPI.js.map +0 -1
  315. package/dist/chunk-7ZZMD7UK.js +0 -386
  316. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  317. package/dist/chunk-BOD4O7OF.js +0 -40
  318. package/dist/chunk-BOD4O7OF.js.map +0 -1
  319. package/dist/chunk-BSO5JDQH.js +0 -2335
  320. package/dist/chunk-BSO5JDQH.js.map +0 -1
  321. package/dist/chunk-C6LXANRU.js +0 -1550
  322. package/dist/chunk-C6LXANRU.js.map +0 -1
  323. package/dist/chunk-DODXQREJ.js +0 -752
  324. package/dist/chunk-DODXQREJ.js.map +0 -1
  325. package/dist/chunk-DRYIUNWY.js +0 -622
  326. package/dist/chunk-DRYIUNWY.js.map +0 -1
  327. package/dist/chunk-E7QXT7SX.js +0 -183
  328. package/dist/chunk-E7QXT7SX.js.map +0 -1
  329. package/dist/chunk-EG66UGL4.js +0 -341
  330. package/dist/chunk-EG66UGL4.js.map +0 -1
  331. package/dist/chunk-FXTVJPYD.js +0 -576
  332. package/dist/chunk-FXTVJPYD.js.map +0 -1
  333. package/dist/chunk-G7MGMCZD.js +0 -153
  334. package/dist/chunk-G7MGMCZD.js.map +0 -1
  335. package/dist/chunk-GGE4NNQT.js +0 -65
  336. package/dist/chunk-GGE4NNQT.js.map +0 -1
  337. package/dist/chunk-H23X7XKK.js +0 -181
  338. package/dist/chunk-H23X7XKK.js.map +0 -1
  339. package/dist/chunk-HHWE3POT.js +0 -94
  340. package/dist/chunk-HHWE3POT.js.map +0 -1
  341. package/dist/chunk-HPWUNB47.js +0 -289
  342. package/dist/chunk-HPWUNB47.js.map +0 -1
  343. package/dist/chunk-IYCLP2N2.js +0 -766
  344. package/dist/chunk-IYCLP2N2.js.map +0 -1
  345. package/dist/chunk-JHCHEVET.js +0 -274
  346. package/dist/chunk-JHCHEVET.js.map +0 -1
  347. package/dist/chunk-JQSF5DQT.js +0 -701
  348. package/dist/chunk-JQSF5DQT.js.map +0 -1
  349. package/dist/chunk-K4DBDHLK.js +0 -158
  350. package/dist/chunk-K4DBDHLK.js.map +0 -1
  351. package/dist/chunk-K6N6XJJX.js +0 -306
  352. package/dist/chunk-K6N6XJJX.js.map +0 -1
  353. package/dist/chunk-M4YBQKIJ.js +0 -1040
  354. package/dist/chunk-M4YBQKIJ.js.map +0 -1
  355. package/dist/chunk-MA6HLL3S.js +0 -65
  356. package/dist/chunk-MA6HLL3S.js.map +0 -1
  357. package/dist/chunk-MAZ26DC7.js +0 -99
  358. package/dist/chunk-MAZ26DC7.js.map +0 -1
  359. package/dist/chunk-NPCTHQIO.js +0 -91
  360. package/dist/chunk-NPCTHQIO.js.map +0 -1
  361. package/dist/chunk-NY44NC4A.js +0 -1056
  362. package/dist/chunk-NY44NC4A.js.map +0 -1
  363. package/dist/chunk-OIUOT4QD.js +0 -44
  364. package/dist/chunk-OIUOT4QD.js.map +0 -1
  365. package/dist/chunk-ONWEPEDO.js +0 -57
  366. package/dist/chunk-ONWEPEDO.js.map +0 -1
  367. package/dist/chunk-OWN5NPMC.js +0 -152
  368. package/dist/chunk-OWN5NPMC.js.map +0 -1
  369. package/dist/chunk-P6FYH6K4.js +0 -1161
  370. package/dist/chunk-P6FYH6K4.js.map +0 -1
  371. package/dist/chunk-PC4UYEBM.js +0 -166
  372. package/dist/chunk-PC4UYEBM.js.map +0 -1
  373. package/dist/chunk-PC5DOSM7.js +0 -579
  374. package/dist/chunk-PC5DOSM7.js.map +0 -1
  375. package/dist/chunk-PXE2VKMX.js +0 -140
  376. package/dist/chunk-PXE2VKMX.js.map +0 -1
  377. package/dist/chunk-PZ5AY32C.js +0 -10
  378. package/dist/chunk-PZ5AY32C.js.map +0 -1
  379. package/dist/chunk-QB6BDBP2.js +0 -4464
  380. package/dist/chunk-QB6BDBP2.js.map +0 -1
  381. package/dist/chunk-RXHCETDZ.js +0 -536
  382. package/dist/chunk-RXHCETDZ.js.map +0 -1
  383. package/dist/chunk-RZTMDUO7.js +0 -49
  384. package/dist/chunk-RZTMDUO7.js.map +0 -1
  385. package/dist/chunk-SFLLL76A.js +0 -669
  386. package/dist/chunk-SFLLL76A.js.map +0 -1
  387. package/dist/chunk-SZLVEKMJ.js +0 -1446
  388. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  389. package/dist/chunk-T4SQEITX.js +0 -95
  390. package/dist/chunk-T4SQEITX.js.map +0 -1
  391. package/dist/chunk-T6RLYGAD.js +0 -158
  392. package/dist/chunk-T6RLYGAD.js.map +0 -1
  393. package/dist/chunk-TJVT4QFF.js +0 -911
  394. package/dist/chunk-TJVT4QFF.js.map +0 -1
  395. package/dist/chunk-TQ7LNKZ3.js +0 -136
  396. package/dist/chunk-TQ7LNKZ3.js.map +0 -1
  397. package/dist/chunk-U4L7JRPZ.js +0 -1706
  398. package/dist/chunk-U4L7JRPZ.js.map +0 -1
  399. package/dist/chunk-U4PHLT2N.js +0 -419
  400. package/dist/chunk-U4PHLT2N.js.map +0 -1
  401. package/dist/chunk-VCZ5FQYW.js +0 -928
  402. package/dist/chunk-VCZ5FQYW.js.map +0 -1
  403. package/dist/chunk-VI2UW6B6.js +0 -162
  404. package/dist/chunk-VI2UW6B6.js.map +0 -1
  405. package/dist/chunk-VQMK5FMP.js +0 -247
  406. package/dist/chunk-VQMK5FMP.js.map +0 -1
  407. package/dist/chunk-WGXIEX7P.js +0 -116
  408. package/dist/chunk-WGXIEX7P.js.map +0 -1
  409. package/dist/chunk-WVATSFCP.js +0 -1553
  410. package/dist/chunk-WVATSFCP.js.map +0 -1
  411. package/dist/chunk-X4YIBDER.js +0 -1662
  412. package/dist/chunk-X4YIBDER.js.map +0 -1
  413. package/dist/chunk-YQN4ICPP.js +0 -355
  414. package/dist/chunk-YQN4ICPP.js.map +0 -1
  415. package/dist/chunk-ZET2UAYW.js +0 -89
  416. package/dist/chunk-ZET2UAYW.js.map +0 -1
  417. package/dist/chunk-ZHTZ4EYI.js +0 -1212
  418. package/dist/chunk-ZHTZ4EYI.js.map +0 -1
  419. package/dist/control.js.map +0 -1
  420. package/dist/hosted/index.js.map +0 -1
  421. package/dist/matrix/index.js.map +0 -1
  422. package/dist/reporting.js.map +0 -1
  423. package/dist/rollout/index.js.map +0 -1
  424. package/dist/run-campaign-OJJ7CZF4.js +0 -18
  425. package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
  426. package/dist/supervisor-run/index.js.map +0 -1
  427. package/dist/traces.js.map +0 -1
  428. package/dist/wire/index.js.map +0 -1
package/dist/fuzz.js CHANGED
@@ -1,199 +1,207 @@
1
- import {
2
- varianceBasedCurriculum
3
- } from "./chunk-G7MGMCZD.js";
4
- import {
5
- CostLedger,
6
- CostReceiptCaptureError
7
- } from "./chunk-VCZ5FQYW.js";
8
- import "./chunk-VI2UW6B6.js";
9
- import "./chunk-OIUOT4QD.js";
10
- import {
11
- ValidationError
12
- } from "./chunk-ONWEPEDO.js";
13
- import "./chunk-PZ5AY32C.js";
14
-
15
- // src/fuzz/cube.ts
1
+ import { s as ValidationError } from "./errors-8YnH8WlF.js";
2
+ import { i as CostLedger, o as CostReceiptCaptureError } from "./cost-ledger-DIgQUFZZ.js";
3
+ import { r as varianceBasedCurriculum } from "./active-curriculum-C4mk67HP.js";
4
+ //#region src/fuzz/cube.ts
5
+ /** Enumerate every input cell (cartesian product of the axes), in stable order. */
16
6
  function enumerateCells(space) {
17
- if (space.axes.length === 0) return [];
18
- let partials = [{}];
19
- for (const axis of space.axes) {
20
- const next = [];
21
- for (const partial of partials) {
22
- for (const value of axis.values) next.push({ ...partial, [axis.name]: value });
23
- }
24
- partials = next;
25
- }
26
- return partials.map((coords) => ({ id: cellId(space, coords), coords }));
7
+ if (space.axes.length === 0) return [];
8
+ let partials = [{}];
9
+ for (const axis of space.axes) {
10
+ const next = [];
11
+ for (const partial of partials) for (const value of axis.values) next.push({
12
+ ...partial,
13
+ [axis.name]: value
14
+ });
15
+ partials = next;
16
+ }
17
+ return partials.map((coords) => ({
18
+ id: cellId(space, coords),
19
+ coords
20
+ }));
27
21
  }
22
+ /** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */
28
23
  function cellId(space, coords) {
29
- return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
24
+ return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
30
25
  }
26
+ /** Nearest-rank percentile on a pre-sorted ascending sample. */
31
27
  function percentile(sorted, p) {
32
- const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
33
- return sorted[idx];
28
+ return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
34
29
  }
30
+ /** Summarize a sample. Throws on an empty sample — callers represent "no data"
31
+ * as `null`, never as a zeroed distribution. */
35
32
  function distribution(values) {
36
- if (values.length === 0)
37
- throw new Error("distribution: empty sample \u2014 represent missing data as null, not zeros");
38
- const sorted = [...values].sort((a, b) => a - b);
39
- const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
40
- return {
41
- mean,
42
- median: percentile(sorted, 0.5),
43
- p90: percentile(sorted, 0.9),
44
- min: sorted[0],
45
- max: sorted[sorted.length - 1],
46
- n: sorted.length
47
- };
33
+ if (values.length === 0) throw new Error("distribution: empty sample — represent missing data as null, not zeros");
34
+ const sorted = [...values].sort((a, b) => a - b);
35
+ return {
36
+ mean: sorted.reduce((a, b) => a + b, 0) / sorted.length,
37
+ median: percentile(sorted, .5),
38
+ p90: percentile(sorted, .9),
39
+ min: sorted[0],
40
+ max: sorted[sorted.length - 1],
41
+ n: sorted.length
42
+ };
48
43
  }
44
+ /**
45
+ * Project the evaluation log into the per-input-cell coverage map. A cell with
46
+ * no evaluations reports `score: null` (honestly uncovered), never zeros.
47
+ */
49
48
  function buildCoverage(cells, log, threshold) {
50
- const byCell = /* @__PURE__ */ new Map();
51
- for (const r of log) {
52
- const arr = byCell.get(r.cell.id) ?? [];
53
- arr.push(r);
54
- byCell.set(r.cell.id, arr);
55
- }
56
- return cells.map((cell) => {
57
- const recs = byCell.get(cell.id) ?? [];
58
- const runs = recs.length;
59
- if (runs === 0)
60
- return { cell, runs: 0, score: null, findingRate: 0, dimensions: {}, latencyMs: null };
61
- const score = distribution(recs.map((r) => r.ev.score));
62
- const latencyMs = distribution(recs.map((r) => r.latencyMs));
63
- const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
64
- const dimSamples = {};
65
- for (const r of recs) {
66
- for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
67
- const samples = dimSamples[k] ?? [];
68
- samples.push(v);
69
- dimSamples[k] = samples;
70
- }
71
- }
72
- const dimensions = {};
73
- for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
74
- const tracked = recs.filter((r) => r.costUsd !== void 0);
75
- const known = tracked.filter((r) => r.costUsd !== null);
76
- const cost = tracked.length > 0 ? {
77
- costUsd: known.reduce((a, r) => a + r.costUsd, 0),
78
- ...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
79
- } : {};
80
- return { cell, runs, score, findingRate, dimensions, latencyMs, ...cost };
81
- });
49
+ const byCell = /* @__PURE__ */ new Map();
50
+ for (const r of log) {
51
+ const arr = byCell.get(r.cell.id) ?? [];
52
+ arr.push(r);
53
+ byCell.set(r.cell.id, arr);
54
+ }
55
+ return cells.map((cell) => {
56
+ const recs = byCell.get(cell.id) ?? [];
57
+ const runs = recs.length;
58
+ if (runs === 0) return {
59
+ cell,
60
+ runs: 0,
61
+ score: null,
62
+ findingRate: 0,
63
+ dimensions: {},
64
+ latencyMs: null
65
+ };
66
+ const score = distribution(recs.map((r) => r.ev.score));
67
+ const latencyMs = distribution(recs.map((r) => r.latencyMs));
68
+ const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
69
+ const dimSamples = {};
70
+ for (const r of recs) for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
71
+ const samples = dimSamples[k] ?? [];
72
+ samples.push(v);
73
+ dimSamples[k] = samples;
74
+ }
75
+ const dimensions = {};
76
+ for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
77
+ const tracked = recs.filter((r) => r.costUsd !== void 0);
78
+ const known = tracked.filter((r) => r.costUsd !== null);
79
+ return {
80
+ cell,
81
+ runs,
82
+ score,
83
+ findingRate,
84
+ dimensions,
85
+ latencyMs,
86
+ ...tracked.length > 0 ? {
87
+ costUsd: known.reduce((a, r) => a + r.costUsd, 0),
88
+ ...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
89
+ } : {}
90
+ };
91
+ });
82
92
  }
83
-
84
- // src/fuzz/capsule.ts
93
+ //#endregion
94
+ //#region src/fuzz/capsule.ts
85
95
  function buildCapsule(input) {
86
- const coverage = buildCoverage(input.cells, input.log, input.threshold);
87
- const covered = coverage.filter((c) => c.runs > 0);
88
- const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
89
- const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
90
- const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
91
- return {
92
- target: input.target,
93
- objective: input.objective,
94
- coverage,
95
- findings: [...input.findings].sort((a, b) => b.interest - a.interest),
96
- archive: [...input.archive].sort((a, b) => b.interest - a.interest),
97
- stats: {
98
- totalRuns: input.runsUsed,
99
- cellsTotal: input.cells.length,
100
- cellsCovered: covered.length,
101
- behaviorBinsObserved,
102
- candidateFindings: input.candidateFindings,
103
- verifiedFindings: input.findings.length,
104
- robustness,
105
- latencyMs,
106
- ...input.cost ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns } : {},
107
- evalErrors: input.evalErrors,
108
- ...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
109
- }
110
- };
96
+ const coverage = buildCoverage(input.cells, input.log, input.threshold);
97
+ const covered = coverage.filter((c) => c.runs > 0);
98
+ const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
99
+ const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
100
+ const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
101
+ return {
102
+ target: input.target,
103
+ objective: input.objective,
104
+ coverage,
105
+ findings: [...input.findings].sort((a, b) => b.interest - a.interest),
106
+ archive: [...input.archive].sort((a, b) => b.interest - a.interest),
107
+ stats: {
108
+ totalRuns: input.runsUsed,
109
+ cellsTotal: input.cells.length,
110
+ cellsCovered: covered.length,
111
+ behaviorBinsObserved,
112
+ candidateFindings: input.candidateFindings,
113
+ verifiedFindings: input.findings.length,
114
+ robustness,
115
+ latencyMs,
116
+ ...input.cost ? {
117
+ costUsd: input.cost.costUsd,
118
+ costUnknownRuns: input.cost.costUnknownRuns
119
+ } : {},
120
+ evalErrors: input.evalErrors,
121
+ ...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
122
+ }
123
+ };
111
124
  }
112
125
  function esc(s) {
113
- return s.replace(
114
- /[&<>"]/g,
115
- (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]
116
- );
126
+ return s.replace(/[&<>"]/g, (c) => ({
127
+ "&": "&amp;",
128
+ "<": "&lt;",
129
+ ">": "&gt;",
130
+ "\"": "&quot;"
131
+ })[c]);
117
132
  }
133
+ /** red (0) → amber (.5) → green (1); uncovered cells render gray. */
118
134
  function robustnessColor(r) {
119
- if (r == null) return "#2a2a2e";
120
- return `hsl(${Math.round(r * 120)} 70% 42%)`;
135
+ if (r == null) return "#2a2a2e";
136
+ return `hsl(${Math.round(r * 120)} 70% 42%)`;
121
137
  }
122
138
  function pct(x) {
123
- return `${Math.round(x * 100)}%`;
139
+ return `${Math.round(x * 100)}%`;
124
140
  }
125
141
  function deriveAxes(coverage) {
126
- const order = [];
127
- const seen = /* @__PURE__ */ new Map();
128
- for (const c of coverage) {
129
- for (const [k, v] of Object.entries(c.cell.coords)) {
130
- if (!seen.has(k)) {
131
- seen.set(k, /* @__PURE__ */ new Set());
132
- order.push(k);
133
- }
134
- seen.get(k)?.add(v);
135
- }
136
- }
137
- return order.map((name) => ({ name, values: [...seen.get(name) ?? []] }));
142
+ const order = [];
143
+ const seen = /* @__PURE__ */ new Map();
144
+ for (const c of coverage) for (const [k, v] of Object.entries(c.cell.coords)) {
145
+ if (!seen.has(k)) {
146
+ seen.set(k, /* @__PURE__ */ new Set());
147
+ order.push(k);
148
+ }
149
+ seen.get(k)?.add(v);
150
+ }
151
+ return order.map((name) => ({
152
+ name,
153
+ values: [...seen.get(name) ?? []]
154
+ }));
138
155
  }
156
+ /** The weakest dimension chip for a cell, e.g. `safety 32%` — shown when scores exist. */
139
157
  function weakestDim(c) {
140
- const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
141
- if (entries.length === 0) return "";
142
- const sorted = entries.sort((a, b) => a[1] - b[1]);
143
- const w = sorted[0];
144
- if (!w) return "";
145
- return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
158
+ const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
159
+ if (entries.length === 0) return "";
160
+ const w = entries.sort((a, b) => a[1] - b[1])[0];
161
+ if (!w) return "";
162
+ return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
146
163
  }
147
164
  function heatmapHtml(coverage) {
148
- const axes = deriveAxes(coverage);
149
- const byId = new Map(coverage.map((c) => [c.cell.id, c]));
150
- const tile = (c, label) => {
151
- const r = c?.score?.mean ?? null;
152
- const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) \xB7 ${c.runs} runs \xB7 ${pct(c.findingRate)} flagged${c.latencyMs ? ` \xB7 ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` \xB7 $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
153
- return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : "\u2014"}</span>${c ? weakestDim(c) : ""}</div>`;
154
- };
155
- const rowAxis = axes[0];
156
- const colAxis = axes[1];
157
- if (axes.length === 2 && rowAxis && colAxis) {
158
- const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join("")}</tr>`;
159
- const rows = rowAxis.values.map((rv) => {
160
- const cells = colAxis.values.map((cv) => {
161
- const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`;
162
- return `<td>${tile(byId.get(id), "")}</td>`;
163
- }).join("");
164
- return `<tr><th class="rh">${esc(rv)}</th>${cells}</tr>`;
165
- }).join("");
166
- return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> \xB7 cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
167
- }
168
- const sorted = [...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2));
169
- return `<div class="grid">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(" \xB7 "))).join("")}</div>`;
165
+ const axes = deriveAxes(coverage);
166
+ const byId = new Map(coverage.map((c) => [c.cell.id, c]));
167
+ const tile = (c, label) => {
168
+ const r = c?.score?.mean ?? null;
169
+ const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) · ${c.runs} runs · ${pct(c.findingRate)} flagged${c.latencyMs ? ` · ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` · $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
170
+ return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : ""}</span>${c ? weakestDim(c) : ""}</div>`;
171
+ };
172
+ const rowAxis = axes[0];
173
+ const colAxis = axes[1];
174
+ if (axes.length === 2 && rowAxis && colAxis) {
175
+ const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join("")}</tr>`;
176
+ const rows = rowAxis.values.map((rv) => {
177
+ const cells = colAxis.values.map((cv) => {
178
+ const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`;
179
+ return `<td>${tile(byId.get(id), "")}</td>`;
180
+ }).join("");
181
+ return `<tr><th class="rh">${esc(rv)}</th>${cells}</tr>`;
182
+ }).join("");
183
+ return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> · cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
184
+ }
185
+ return `<div class="grid">${[...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2)).map((c) => tile(c, Object.values(c.cell.coords).join(" · "))).join("")}</div>`;
170
186
  }
171
187
  function findingsHtml(findings, limit) {
172
- if (findings.length === 0)
173
- return `<p class="none">No verified findings \u2014 the target held across every covered cell.</p>`;
174
- return findings.slice(0, limit).map((f) => {
175
- const coords = Object.entries(f.cell.coords).map(([k, v]) => `${k}:${v}`).join(" \xB7 ");
176
- const labels = (f.evaluation.labels ?? []).map((l) => `<span class="fclass">${esc(l)}</span>`).join("");
177
- const dims = Object.entries(f.evaluation.scores ?? {}).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([k, v]) => `<span class="dim">${esc(k)} ${pct(v)}</span>`).join("");
178
- return `<div class="fail"><div class="fmeta"><span class="fcell">${esc(coords)}</span>${labels}<span class="fsev">interest ${pct(f.interest)}</span></div><div class="sevbar"><div class="sevfill" style="width:${pct(f.interest)}"></div></div><div class="ftext">${esc(f.text ?? "(scenario text not captured)")}</div>${dims ? `<div class="fdims">${dims}</div>` : ""}</div>`;
179
- }).join("");
188
+ if (findings.length === 0) return `<p class="none">No verified findings — the target held across every covered cell.</p>`;
189
+ return findings.slice(0, limit).map((f) => {
190
+ const coords = Object.entries(f.cell.coords).map(([k, v]) => `${k}:${v}`).join(" · ");
191
+ const labels = (f.evaluation.labels ?? []).map((l) => `<span class="fclass">${esc(l)}</span>`).join("");
192
+ const dims = Object.entries(f.evaluation.scores ?? {}).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([k, v]) => `<span class="dim">${esc(k)} ${pct(v)}</span>`).join("");
193
+ return `<div class="fail"><div class="fmeta"><span class="fcell">${esc(coords)}</span>${labels}<span class="fsev">interest ${pct(f.interest)}</span></div><div class="sevbar"><div class="sevfill" style="width:${pct(f.interest)}"></div></div><div class="ftext">${esc(f.text ?? "(scenario text not captured)")}</div>${dims ? `<div class="fdims">${dims}</div>` : ""}</div>`;
194
+ }).join("");
180
195
  }
196
+ /** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
181
197
  function renderCapsuleHtml(capsule, opts = {}) {
182
- const s = capsule.stats;
183
- const kpi = (label, value, accent = "#e6e6e6") => `<div class="kpi"><div class="kv" style="color:${accent}">${esc(value)}</div><div class="kl">${esc(label)}</div></div>`;
184
- const lift = capsule.lift ? kpi(
185
- "hardening lift",
186
- `${capsule.lift.before.toFixed(2)} \u2192 ${capsule.lift.after.toFixed(2)}`,
187
- "#5ad17a"
188
- ) : "";
189
- const cost = s.costUsd !== void 0 ? kpi(
190
- s.costUnknownRuns ? `cost \xB7 ${s.costUnknownRuns} runs unpriced` : "cost",
191
- `$${s.costUsd.toFixed(2)}`,
192
- s.costUnknownRuns ? "#e5c07b" : "#e6e6e6"
193
- ) : "";
194
- const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
195
- return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
196
- <title>${esc(capsule.objective)} capsule \u2014 ${esc(capsule.target)}</title>
198
+ const s = capsule.stats;
199
+ const kpi = (label, value, accent = "#e6e6e6") => `<div class="kpi"><div class="kv" style="color:${accent}">${esc(value)}</div><div class="kl">${esc(label)}</div></div>`;
200
+ const lift = capsule.lift ? kpi("hardening lift", `${capsule.lift.before.toFixed(2)} → ${capsule.lift.after.toFixed(2)}`, "#5ad17a") : "";
201
+ const cost = s.costUsd !== void 0 ? kpi(s.costUnknownRuns ? `cost · ${s.costUnknownRuns} runs unpriced` : "cost", `$${s.costUsd.toFixed(2)}`, s.costUnknownRuns ? "#e5c07b" : "#e6e6e6") : "";
202
+ const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
203
+ return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
204
+ <title>${esc(capsule.objective)} capsule — ${esc(capsule.target)}</title>
197
205
  <style>
198
206
  :root{color-scheme:dark}
199
207
  *{box-sizing:border-box}
@@ -229,11 +237,11 @@ table.heat th.rh{text-align:right}
229
237
  .none{color:#5ad17a}
230
238
  .foot{color:#5a5a63;font-size:11px;margin-top:36px}
231
239
  </style></head><body><div class="wrap">
232
- <h1>${esc(capsule.objective)} exploration \xB7 ${esc(capsule.target)}</h1>
233
- <div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` \xB7 ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` \xB7 ${esc(stamp)}` : ""}</div>
240
+ <h1>${esc(capsule.objective)} exploration · ${esc(capsule.target)}</h1>
241
+ <div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` · ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` · ${esc(stamp)}` : ""}</div>
234
242
  <div class="kpis">
235
- ${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < 0.6 ? "#e58a96" : "#5ad17a") : ""}
236
- ${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}\u2013${pct(s.robustness.max)}`) : ""}
243
+ ${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < .6 ? "#e58a96" : "#5ad17a") : ""}
244
+ ${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}–${pct(s.robustness.max)}`) : ""}
237
245
  ${s.latencyMs ? kpi("median latency", `${(s.latencyMs.median / 1e3).toFixed(1)}s`, s.latencyMs.p90 > 4 * s.latencyMs.median ? "#e5b566" : "#e6e6e6") : ""}
238
246
  ${kpi("verified findings", String(s.verifiedFindings), s.verifiedFindings > 0 ? "#e58a96" : "#5ad17a")}
239
247
  ${kpi("cells covered", `${s.cellsCovered}/${s.cellsTotal}`)}
@@ -242,475 +250,519 @@ ${cost}
242
250
  ${s.evalErrors > 0 ? kpi("eval errors", String(s.evalErrors), "#e5b566") : ""}
243
251
  ${lift}
244
252
  </div>
245
- ${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} \u2014 coverage below reflects the completed portion only</div>` : ""}
253
+ ${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} coverage below reflects the completed portion only</div>` : ""}
246
254
  <h2>Coverage map</h2>
247
255
  ${heatmapHtml(capsule.coverage)}
248
- <h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` \xB7 ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
256
+ <h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` · ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
249
257
  ${findingsHtml(capsule.findings, opts.maxFindings ?? 8)}
250
258
  <div class="foot">Findings are gate-verified: each is a fair, answerable task that reproduces under a meaning-preserving rephrase, minimized to the smallest trigger.</div>
251
259
  </div></body></html>`;
252
260
  }
253
-
254
- // src/fuzz/policies.ts
255
- var clamp01 = (x) => Math.max(0, Math.min(1, x));
261
+ //#endregion
262
+ //#region src/fuzz/policies.ts
263
+ const clamp01 = (x) => Math.max(0, Math.min(1, x));
264
+ /**
265
+ * Perturbation-based search: apply the cell's mutation operators to the current
266
+ * elites + seeds, deduping by id. Elites first — mutating the most interesting
267
+ * scenario found so far is what makes the search deepen across rounds.
268
+ */
256
269
  function mutationProposer(opts) {
257
- return async (ctx) => {
258
- const mutations = opts.mutationsFor(ctx.cell);
259
- const parents = [...ctx.elites, ...ctx.seeds];
260
- const seen = new Set(parents.map(opts.scenarioId));
261
- const out = [];
262
- for (const parent of parents) {
263
- if (out.length >= ctx.count) break;
264
- for (const m of mutations) {
265
- const children = await m.mutate(parent, ctx.rng);
266
- for (const child of children) {
267
- const id = opts.scenarioId(child);
268
- if (seen.has(id)) continue;
269
- seen.add(id);
270
- out.push(child);
271
- if (out.length >= ctx.count) break;
272
- }
273
- if (out.length >= ctx.count) break;
274
- }
275
- }
276
- return out;
277
- };
270
+ return async (ctx) => {
271
+ const mutations = opts.mutationsFor(ctx.cell);
272
+ const parents = [...ctx.elites, ...ctx.seeds];
273
+ const seen = new Set(parents.map(opts.scenarioId));
274
+ const out = [];
275
+ for (const parent of parents) {
276
+ if (out.length >= ctx.count) break;
277
+ for (const m of mutations) {
278
+ const children = await m.mutate(parent, ctx.rng);
279
+ for (const child of children) {
280
+ const id = opts.scenarioId(child);
281
+ if (seen.has(id)) continue;
282
+ seen.add(id);
283
+ out.push(child);
284
+ if (out.length >= ctx.count) break;
285
+ }
286
+ if (out.length >= ctx.count) break;
287
+ }
288
+ }
289
+ return out;
290
+ };
278
291
  }
279
- function adversarialObjective(threshold = 0.5) {
280
- return { kind: "adversarial", threshold, interest: (ev) => clamp01(1 - ev.score) };
292
+ /** Adversarial: a low headline score is interesting — find where the agent fails. */
293
+ function adversarialObjective(threshold = .5) {
294
+ return {
295
+ kind: "adversarial",
296
+ threshold,
297
+ interest: (ev) => clamp01(1 - ev.score)
298
+ };
281
299
  }
282
300
  function hamming(a, b) {
283
- if (!a || !b) return 1;
284
- const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
285
- if (keys.size === 0) return 0;
286
- let diff = 0;
287
- for (const k of keys) if (a[k] !== b[k]) diff++;
288
- return diff / keys.size;
301
+ if (!a || !b) return 1;
302
+ const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
303
+ if (keys.size === 0) return 0;
304
+ let diff = 0;
305
+ for (const k of keys) if (a[k] !== b[k]) diff++;
306
+ return diff / keys.size;
289
307
  }
290
- function noveltyObjective(threshold = 0.3) {
291
- return {
292
- kind: "novelty",
293
- threshold,
294
- interest: (ev, ctx) => {
295
- const scoreNovelty = ctx.archiveScores.length === 0 ? 1 : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)));
296
- const descNovelty = ctx.archiveDescriptors.length === 0 ? 1 : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)));
297
- return clamp01(0.5 * scoreNovelty + 0.5 * descNovelty);
298
- }
299
- };
308
+ /**
309
+ * Novelty: interesting when far from the archive in score AND measured behavior
310
+ * descriptor — quality-diversity's diversity pressure; drives corpus growth
311
+ * rather than re-finding the same hole.
312
+ */
313
+ function noveltyObjective(threshold = .3) {
314
+ return {
315
+ kind: "novelty",
316
+ threshold,
317
+ interest: (ev, ctx) => {
318
+ const scoreNovelty = ctx.archiveScores.length === 0 ? 1 : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)));
319
+ const descNovelty = ctx.archiveDescriptors.length === 0 ? 1 : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)));
320
+ return clamp01(.5 * scoreNovelty + .5 * descNovelty);
321
+ }
322
+ };
300
323
  }
301
-
302
- // src/fuzz/explorer.ts
324
+ //#endregion
325
+ //#region src/fuzz/explorer.ts
326
+ /**
327
+ * The exploration engine — a stateful session over a behavior space.
328
+ *
329
+ * Each `step()`: allocate budget across INPUT cells (floor first, then variance
330
+ * steering toward the least-certain cells), propose candidates (the proposer
331
+ * reads current elites + findings, so the search deepens generationally),
332
+ * evaluate with bounded concurrency, archive the most interesting scenario per
333
+ * input×measured bin, and admit notable candidates that pass the validity gates.
334
+ * `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
335
+ * the surface `makeExploreTools` exposes so an agent can drive the session.
336
+ *
337
+ * One evaluation log (`EvalRecord[]`) is the source of truth; allocation
338
+ * observations and coverage are projections of it.
339
+ */
303
340
  async function pMap(items, fn, concurrency, signal) {
304
- let i = 0;
305
- const workers = Array.from(
306
- { length: Math.max(1, Math.min(concurrency, items.length)) },
307
- async () => {
308
- while (i < items.length) {
309
- if (signal?.aborted) return;
310
- const item = items[i++];
311
- if (item === void 0) return;
312
- await fn(item);
313
- }
314
- }
315
- );
316
- await Promise.all(workers);
341
+ let i = 0;
342
+ const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
343
+ while (i < items.length) {
344
+ if (signal?.aborted) return;
345
+ const item = items[i++];
346
+ if (item === void 0) return;
347
+ await fn(item);
348
+ }
349
+ });
350
+ await Promise.all(workers);
317
351
  }
318
352
  var BehaviorExplorer = class {
319
- constructor(opts) {
320
- this.opts = opts;
321
- this.cells = enumerateCells(opts.space);
322
- if (this.cells.length === 0)
323
- throw new Error("BehaviorExplorer: space has no cells \u2014 every axis needs \u22651 value");
324
- if (opts.costBudgetUsd !== void 0) {
325
- if (typeof opts.costBudgetUsd !== "number" || !Number.isFinite(opts.costBudgetUsd) || opts.costBudgetUsd < 0) {
326
- throw new RangeError(
327
- `BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`
328
- );
329
- }
330
- }
331
- if (!opts.costOf && (opts.costBudgetUsd !== void 0 || opts.ledger || opts.onCost)) {
332
- throw new ValidationError(
333
- "BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf \u2014 the explorer cannot know run cost without it; supply costOf or drop the cost options"
334
- );
335
- }
336
- if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) {
337
- throw new ValidationError(
338
- "BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation"
339
- );
340
- }
341
- if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) {
342
- throw new ValidationError(
343
- "BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling"
344
- );
345
- }
346
- if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
347
- this.cellById = new Map(this.cells.map((c) => [c.id, c]));
348
- this.objective = opts.objective ?? adversarialObjective(0.5);
349
- this.threshold = this.objective.threshold ?? 0.5;
350
- this.floorPerCell = opts.floorPerCell ?? 2;
351
- this.perRoundBudget = Math.max(
352
- this.cells.length * this.floorPerCell,
353
- Math.ceil(opts.budget / 4)
354
- );
355
- this.rngState = (opts.seed ?? 1) >>> 0;
356
- }
357
- opts;
358
- cells;
359
- cellById;
360
- objective;
361
- threshold;
362
- floorPerCell;
363
- perRoundBudget;
364
- /** The single evaluation log — coverage + allocation are projections of it. */
365
- log = [];
366
- /** binId (input × measured coords) → the most interesting entry seen. */
367
- archiveByBin = /* @__PURE__ */ new Map();
368
- _findings = [];
369
- runsUsed = 0;
370
- candidateFindings = 0;
371
- evalErrors = 0;
372
- consecutiveEvalErrors = 0;
373
- stoppedEarly;
374
- rngState;
375
- costLedger;
376
- costPhase = "fuzz.explore";
377
- // mulberry32 deterministic per session.
378
- rng = () => {
379
- this.rngState = this.rngState + 1831565813 | 0;
380
- let t = this.rngState;
381
- t = Math.imul(t ^ t >>> 15, t | 1);
382
- t ^= t + Math.imul(t ^ t >>> 7, t | 61);
383
- return ((t ^ t >>> 14) >>> 0) / 4294967296;
384
- };
385
- binId(cell, descriptor) {
386
- if (!descriptor || Object.keys(descriptor).length === 0) return cell.id;
387
- const measured = Object.keys(descriptor).sort().map((k) => `${k}=${descriptor[k]}`).join("|");
388
- return `${cell.id}|${measured}`;
389
- }
390
- allocate(budget) {
391
- if ((this.opts.allocation ?? "variance") === "uniform") {
392
- const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length));
393
- return this.cells.map((c) => ({ cellId: c.id, count: per }));
394
- }
395
- return varianceBasedCurriculum(
396
- this.log.map((r) => ({
397
- variantId: r.cell.id,
398
- scenarioId: r.scenarioId,
399
- score: r.ev.score,
400
- pass: r.ev.valid && r.ev.score >= 0.5
401
- })),
402
- this.cells.map((c) => ({ variantId: c.id, scenarioId: "*" })),
403
- { budget, floorPerCell: this.floorPerCell }
404
- ).map((a) => ({ cellId: a.variantId, count: a.count }));
405
- }
406
- objectiveContext() {
407
- const entries = [...this.archiveByBin.values()];
408
- return {
409
- archiveScores: entries.map((e) => e.evaluation.score),
410
- archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
411
- };
412
- }
413
- /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
414
- elitesFor(cellId2) {
415
- const out = [];
416
- for (const e of this.archiveByBin.values()) if (e.cell.id === cellId2) out.push(e.scenario);
417
- return out;
418
- }
419
- /** One allocate → propose → evaluate → gate → archive round. */
420
- async step() {
421
- const remaining = this.opts.budget - this.runsUsed;
422
- if (remaining <= 0 || this.opts.signal?.aborted) return { runs: 0, findings: [] };
423
- const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
424
- const newFindings = [];
425
- let runsThisStep = 0;
426
- for (const alloc of allocations) {
427
- if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
428
- break;
429
- const cell = this.cellById.get(alloc.cellId);
430
- if (!cell) continue;
431
- const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed);
432
- if (cap <= 0) continue;
433
- this.opts.onProgress?.({ type: "cell-allocated", cell, count: cap });
434
- const seeds = await this.opts.seedsFor(cell);
435
- const elites = this.elitesFor(cell.id);
436
- const proposed = await this.opts.proposer({
437
- cell,
438
- seeds,
439
- elites,
440
- findings: this._findings,
441
- count: cap,
442
- rng: this.rng
443
- });
444
- const cold = this.log.every((r) => r.cell.id !== cell.id);
445
- const toEval = [...cold ? seeds : [], ...proposed].slice(0, cap);
446
- await pMap(
447
- toEval,
448
- async (scenario) => {
449
- if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
450
- return;
451
- try {
452
- const startedAt = performance.now();
453
- const paid = this.costLedger ? await this.costLedger.runPaidCall({
454
- channel: "agent",
455
- phase: this.costPhase,
456
- actor: "fuzz.evaluate",
457
- tags: { target: this.opts.target, cell: cell.id },
458
- signal: this.opts.signal,
459
- maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
460
- execute: () => this.opts.evaluate(scenario, cell),
461
- receipt: (evaluation) => {
462
- const cost = this.opts.costOf(scenario, cell, evaluation);
463
- if (cost === null) {
464
- return {
465
- model: "unattributed",
466
- inputTokens: 0,
467
- outputTokens: 0,
468
- costUnknown: true
469
- };
470
- }
471
- if (!Number.isFinite(cost.usd) || cost.usd < 0) {
472
- throw new RangeError(
473
- `BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) \u2014 return null when cost is unknown, never a fabricated number`
474
- );
475
- }
476
- return {
477
- model: cost.model ?? "unattributed",
478
- inputTokens: 0,
479
- outputTokens: 0,
480
- actualCostUsd: cost.usd
481
- };
482
- }
483
- }) : void 0;
484
- if (paid && !paid.succeeded) {
485
- if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) {
486
- throw paid.error.receiptError;
487
- }
488
- throw paid.error;
489
- }
490
- const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
491
- const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
492
- this.runsUsed++;
493
- runsThisStep++;
494
- this.consecutiveEvalErrors = 0;
495
- const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
496
- if (paid?.receipt.actualCostUsd !== void 0) {
497
- this.opts.onCost?.({ usd: paid.receipt.actualCostUsd, channel: "agent" });
498
- }
499
- const interest = this.objective.interest(ev, this.objectiveContext());
500
- this.log.push({
501
- cell,
502
- ev,
503
- interest,
504
- latencyMs,
505
- ...costUsd !== void 0 ? { costUsd } : {},
506
- scenarioId: this.opts.scenarioId(scenario)
507
- });
508
- this.opts.onProgress?.({ type: "evaluated", cell, scenario, evaluation: ev });
509
- const bin = this.binId(cell, ev.descriptor);
510
- const cur = this.archiveByBin.get(bin);
511
- if (!cur || interest > cur.interest)
512
- this.archiveByBin.set(bin, { binId: bin, cell, scenario, evaluation: ev, interest });
513
- if (interest < this.threshold) return;
514
- this.candidateFindings++;
515
- if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell))
516
- return;
517
- if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell))
518
- return;
519
- const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
520
- const finding = {
521
- id: this.opts.scenarioId(scenario),
522
- cell,
523
- scenario,
524
- minimized,
525
- text: this.opts.scenarioText?.(minimized),
526
- evaluation: ev,
527
- interest,
528
- objective: this.objective.kind
529
- };
530
- this._findings.push(finding);
531
- newFindings.push(finding);
532
- this.opts.onProgress?.({ type: "finding", finding });
533
- } catch (err) {
534
- if (err instanceof RangeError) throw err;
535
- this.evalErrors++;
536
- this.consecutiveEvalErrors++;
537
- const message = err instanceof Error ? err.message : String(err);
538
- this.opts.onProgress?.({
539
- type: "eval-error",
540
- cell,
541
- scenarioId: this.opts.scenarioId(scenario),
542
- message
543
- });
544
- const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
545
- if (this.consecutiveEvalErrors >= limit) {
546
- this.stoppedEarly = {
547
- reason: "eval-errors",
548
- detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
549
- };
550
- }
551
- }
552
- },
553
- this.opts.concurrency ?? 1,
554
- this.opts.signal
555
- );
556
- }
557
- this.opts.onProgress?.({ type: "round", runsUsed: this.runsUsed, budget: this.opts.budget });
558
- return { runs: runsThisStep, findings: newFindings };
559
- }
560
- /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
561
- * or no progress is made. */
562
- async run() {
563
- while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
564
- const { runs } = await this.step();
565
- if (runs === 0 && this.stoppedEarly === void 0) break;
566
- }
567
- return this.capsule();
568
- }
569
- coverage() {
570
- return this.capsule().coverage;
571
- }
572
- findings() {
573
- return [...this._findings].sort((a, b) => b.interest - a.interest);
574
- }
575
- capsule() {
576
- const cost = this.costLedger?.summary({
577
- phase: this.costPhase,
578
- tags: { target: this.opts.target }
579
- });
580
- return buildCapsule({
581
- target: this.opts.target,
582
- objective: this.objective.kind,
583
- cells: this.cells,
584
- log: this.log,
585
- threshold: this.threshold,
586
- archive: [...this.archiveByBin.values()],
587
- findings: this._findings,
588
- candidateFindings: this.candidateFindings,
589
- runsUsed: this.runsUsed,
590
- cost: cost ? {
591
- costUsd: cost.totalCostUsd,
592
- costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
593
- } : void 0,
594
- evalErrors: this.evalErrors,
595
- stoppedEarly: this.stoppedEarly
596
- });
597
- }
353
+ opts;
354
+ cells;
355
+ cellById;
356
+ objective;
357
+ threshold;
358
+ floorPerCell;
359
+ perRoundBudget;
360
+ /** The single evaluation log — coverage + allocation are projections of it. */
361
+ log = [];
362
+ /** binId (input × measured coords) → the most interesting entry seen. */
363
+ archiveByBin = /* @__PURE__ */ new Map();
364
+ _findings = [];
365
+ runsUsed = 0;
366
+ candidateFindings = 0;
367
+ evalErrors = 0;
368
+ consecutiveEvalErrors = 0;
369
+ stoppedEarly;
370
+ rngState;
371
+ costLedger;
372
+ costPhase = "fuzz.explore";
373
+ constructor(opts) {
374
+ this.opts = opts;
375
+ this.cells = enumerateCells(opts.space);
376
+ if (this.cells.length === 0) throw new Error("BehaviorExplorer: space has no cells — every axis needs ≥1 value");
377
+ if (opts.costBudgetUsd !== void 0) {
378
+ if (typeof opts.costBudgetUsd !== "number" || !Number.isFinite(opts.costBudgetUsd) || opts.costBudgetUsd < 0) throw new RangeError(`BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`);
379
+ }
380
+ if (!opts.costOf && (opts.costBudgetUsd !== void 0 || opts.ledger || opts.onCost)) throw new ValidationError("BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf — the explorer cannot know run cost without it; supply costOf or drop the cost options");
381
+ if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) throw new ValidationError("BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation");
382
+ if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) throw new ValidationError("BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling");
383
+ if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
384
+ this.cellById = new Map(this.cells.map((c) => [c.id, c]));
385
+ this.objective = opts.objective ?? adversarialObjective(.5);
386
+ this.threshold = this.objective.threshold ?? .5;
387
+ this.floorPerCell = opts.floorPerCell ?? 2;
388
+ this.perRoundBudget = Math.max(this.cells.length * this.floorPerCell, Math.ceil(opts.budget / 4));
389
+ this.rngState = (opts.seed ?? 1) >>> 0;
390
+ }
391
+ rng = () => {
392
+ this.rngState = this.rngState + 1831565813 | 0;
393
+ let t = this.rngState;
394
+ t = Math.imul(t ^ t >>> 15, t | 1);
395
+ t ^= t + Math.imul(t ^ t >>> 7, t | 61);
396
+ return ((t ^ t >>> 14) >>> 0) / 4294967296;
397
+ };
398
+ binId(cell, descriptor) {
399
+ if (!descriptor || Object.keys(descriptor).length === 0) return cell.id;
400
+ const measured = Object.keys(descriptor).sort().map((k) => `${k}=${descriptor[k]}`).join("|");
401
+ return `${cell.id}|${measured}`;
402
+ }
403
+ allocate(budget) {
404
+ if ((this.opts.allocation ?? "variance") === "uniform") {
405
+ const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length));
406
+ return this.cells.map((c) => ({
407
+ cellId: c.id,
408
+ count: per
409
+ }));
410
+ }
411
+ return varianceBasedCurriculum(this.log.map((r) => ({
412
+ variantId: r.cell.id,
413
+ scenarioId: r.scenarioId,
414
+ score: r.ev.score,
415
+ pass: r.ev.valid && r.ev.score >= .5
416
+ })), this.cells.map((c) => ({
417
+ variantId: c.id,
418
+ scenarioId: "*"
419
+ })), {
420
+ budget,
421
+ floorPerCell: this.floorPerCell
422
+ }).map((a) => ({
423
+ cellId: a.variantId,
424
+ count: a.count
425
+ }));
426
+ }
427
+ objectiveContext() {
428
+ const entries = [...this.archiveByBin.values()];
429
+ return {
430
+ archiveScores: entries.map((e) => e.evaluation.score),
431
+ archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
432
+ };
433
+ }
434
+ /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
435
+ elitesFor(cellId) {
436
+ const out = [];
437
+ for (const e of this.archiveByBin.values()) if (e.cell.id === cellId) out.push(e.scenario);
438
+ return out;
439
+ }
440
+ /** One allocate → propose → evaluate → gate → archive round. */
441
+ async step() {
442
+ const remaining = this.opts.budget - this.runsUsed;
443
+ if (remaining <= 0 || this.opts.signal?.aborted) return {
444
+ runs: 0,
445
+ findings: []
446
+ };
447
+ const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
448
+ const newFindings = [];
449
+ let runsThisStep = 0;
450
+ for (const alloc of allocations) {
451
+ if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) break;
452
+ const cell = this.cellById.get(alloc.cellId);
453
+ if (!cell) continue;
454
+ const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed);
455
+ if (cap <= 0) continue;
456
+ this.opts.onProgress?.({
457
+ type: "cell-allocated",
458
+ cell,
459
+ count: cap
460
+ });
461
+ const seeds = await this.opts.seedsFor(cell);
462
+ const elites = this.elitesFor(cell.id);
463
+ const proposed = await this.opts.proposer({
464
+ cell,
465
+ seeds,
466
+ elites,
467
+ findings: this._findings,
468
+ count: cap,
469
+ rng: this.rng
470
+ });
471
+ await pMap([...this.log.every((r) => r.cell.id !== cell.id) ? seeds : [], ...proposed].slice(0, cap), async (scenario) => {
472
+ if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) return;
473
+ try {
474
+ const startedAt = performance.now();
475
+ const paid = this.costLedger ? await this.costLedger.runPaidCall({
476
+ channel: "agent",
477
+ phase: this.costPhase,
478
+ actor: "fuzz.evaluate",
479
+ tags: {
480
+ target: this.opts.target,
481
+ cell: cell.id
482
+ },
483
+ signal: this.opts.signal,
484
+ maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
485
+ execute: () => this.opts.evaluate(scenario, cell),
486
+ receipt: (evaluation) => {
487
+ const cost = this.opts.costOf(scenario, cell, evaluation);
488
+ if (cost === null) return {
489
+ model: "unattributed",
490
+ inputTokens: 0,
491
+ outputTokens: 0,
492
+ costUnknown: true
493
+ };
494
+ if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new RangeError(`BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) — return null when cost is unknown, never a fabricated number`);
495
+ return {
496
+ model: cost.model ?? "unattributed",
497
+ inputTokens: 0,
498
+ outputTokens: 0,
499
+ actualCostUsd: cost.usd
500
+ };
501
+ }
502
+ }) : void 0;
503
+ if (paid && !paid.succeeded) {
504
+ if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) throw paid.error.receiptError;
505
+ throw paid.error;
506
+ }
507
+ const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
508
+ const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
509
+ this.runsUsed++;
510
+ runsThisStep++;
511
+ this.consecutiveEvalErrors = 0;
512
+ const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
513
+ if (paid?.receipt.actualCostUsd !== void 0) this.opts.onCost?.({
514
+ usd: paid.receipt.actualCostUsd,
515
+ channel: "agent"
516
+ });
517
+ const interest = this.objective.interest(ev, this.objectiveContext());
518
+ this.log.push({
519
+ cell,
520
+ ev,
521
+ interest,
522
+ latencyMs,
523
+ ...costUsd !== void 0 ? { costUsd } : {},
524
+ scenarioId: this.opts.scenarioId(scenario)
525
+ });
526
+ this.opts.onProgress?.({
527
+ type: "evaluated",
528
+ cell,
529
+ scenario,
530
+ evaluation: ev
531
+ });
532
+ const bin = this.binId(cell, ev.descriptor);
533
+ const cur = this.archiveByBin.get(bin);
534
+ if (!cur || interest > cur.interest) this.archiveByBin.set(bin, {
535
+ binId: bin,
536
+ cell,
537
+ scenario,
538
+ evaluation: ev,
539
+ interest
540
+ });
541
+ if (interest < this.threshold) return;
542
+ this.candidateFindings++;
543
+ if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell)) return;
544
+ if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell)) return;
545
+ const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
546
+ const finding = {
547
+ id: this.opts.scenarioId(scenario),
548
+ cell,
549
+ scenario,
550
+ minimized,
551
+ text: this.opts.scenarioText?.(minimized),
552
+ evaluation: ev,
553
+ interest,
554
+ objective: this.objective.kind
555
+ };
556
+ this._findings.push(finding);
557
+ newFindings.push(finding);
558
+ this.opts.onProgress?.({
559
+ type: "finding",
560
+ finding
561
+ });
562
+ } catch (err) {
563
+ if (err instanceof RangeError) throw err;
564
+ this.evalErrors++;
565
+ this.consecutiveEvalErrors++;
566
+ const message = err instanceof Error ? err.message : String(err);
567
+ this.opts.onProgress?.({
568
+ type: "eval-error",
569
+ cell,
570
+ scenarioId: this.opts.scenarioId(scenario),
571
+ message
572
+ });
573
+ const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
574
+ if (this.consecutiveEvalErrors >= limit) this.stoppedEarly = {
575
+ reason: "eval-errors",
576
+ detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
577
+ };
578
+ }
579
+ }, this.opts.concurrency ?? 1, this.opts.signal);
580
+ }
581
+ this.opts.onProgress?.({
582
+ type: "round",
583
+ runsUsed: this.runsUsed,
584
+ budget: this.opts.budget
585
+ });
586
+ return {
587
+ runs: runsThisStep,
588
+ findings: newFindings
589
+ };
590
+ }
591
+ /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
592
+ * or no progress is made. */
593
+ async run() {
594
+ while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
595
+ const { runs } = await this.step();
596
+ if (runs === 0 && this.stoppedEarly === void 0) break;
597
+ }
598
+ return this.capsule();
599
+ }
600
+ coverage() {
601
+ return this.capsule().coverage;
602
+ }
603
+ findings() {
604
+ return [...this._findings].sort((a, b) => b.interest - a.interest);
605
+ }
606
+ capsule() {
607
+ const cost = this.costLedger?.summary({
608
+ phase: this.costPhase,
609
+ tags: { target: this.opts.target }
610
+ });
611
+ return buildCapsule({
612
+ target: this.opts.target,
613
+ objective: this.objective.kind,
614
+ cells: this.cells,
615
+ log: this.log,
616
+ threshold: this.threshold,
617
+ archive: [...this.archiveByBin.values()],
618
+ findings: this._findings,
619
+ candidateFindings: this.candidateFindings,
620
+ runsUsed: this.runsUsed,
621
+ cost: cost ? {
622
+ costUsd: cost.totalCostUsd,
623
+ costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
624
+ } : void 0,
625
+ evalErrors: this.evalErrors,
626
+ stoppedEarly: this.stoppedEarly
627
+ });
628
+ }
598
629
  };
599
-
600
- // src/fuzz/fuzz-agent.ts
630
+ //#endregion
631
+ //#region src/fuzz/fuzz-agent.ts
632
+ /**
633
+ * `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
634
+ *
635
+ * One call: explore the space to budget with the adversarial objective and
636
+ * return the capsule. For agent-driven, incremental, or multi-objective use,
637
+ * construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
638
+ */
601
639
  async function fuzzAgent(opts) {
602
- const { failureThreshold, ...rest } = opts;
603
- const explorer = new BehaviorExplorer({
604
- ...rest,
605
- objective: adversarialObjective(failureThreshold ?? 0.5)
606
- });
607
- return { capsule: await explorer.run() };
640
+ const { failureThreshold, ...rest } = opts;
641
+ return { capsule: await new BehaviorExplorer({
642
+ ...rest,
643
+ objective: adversarialObjective(failureThreshold ?? .5)
644
+ }).run() };
608
645
  }
609
-
610
- // src/fuzz/gates.ts
646
+ //#endregion
647
+ //#region src/fuzz/gates.ts
648
+ /** Combine gate sets; a candidate must pass every gate in every set. */
611
649
  function composeGates(...sets) {
612
- const present = sets.filter((s) => s != null);
613
- return {
614
- isValid: async (scenario, ev, cell) => {
615
- for (const g of present) {
616
- if (g.isValid && !await g.isValid(scenario, ev, cell)) return false;
617
- }
618
- return true;
619
- },
620
- isUncontaminated: async (scenario, ev, cell) => {
621
- for (const g of present) {
622
- if (g.isUncontaminated && !await g.isUncontaminated(scenario, ev, cell)) return false;
623
- }
624
- return true;
625
- }
626
- };
650
+ const present = sets.filter((s) => s != null);
651
+ return {
652
+ isValid: async (scenario, ev, cell) => {
653
+ for (const g of present) if (g.isValid && !await g.isValid(scenario, ev, cell)) return false;
654
+ return true;
655
+ },
656
+ isUncontaminated: async (scenario, ev, cell) => {
657
+ for (const g of present) if (g.isUncontaminated && !await g.isUncontaminated(scenario, ev, cell)) return false;
658
+ return true;
659
+ }
660
+ };
627
661
  }
662
+ /**
663
+ * Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the
664
+ * flagged scenario; keep the finding only when the rephrase ALSO scores below the
665
+ * threshold. A finding that flips under a cosmetic rewrite was keyed to surface
666
+ * form, not the task — a false signal we must not report. Costs one extra
667
+ * evaluation per candidate (candidates are rare, so cheap).
668
+ */
628
669
  function perturbationStabilityGate(opts) {
629
- const threshold = opts.failureThreshold ?? 0.5;
630
- return {
631
- isUncontaminated: async (scenario, _ev, cell) => {
632
- const rephrased = opts.perturb(scenario);
633
- if (rephrased == null) return true;
634
- const re = await opts.evaluate(rephrased, cell);
635
- return re.score < threshold;
636
- }
637
- };
670
+ const threshold = opts.failureThreshold ?? .5;
671
+ return { isUncontaminated: async (scenario, _ev, cell) => {
672
+ const rephrased = opts.perturb(scenario);
673
+ if (rephrased == null) return true;
674
+ return (await opts.evaluate(rephrased, cell)).score < threshold;
675
+ } };
638
676
  }
677
+ /**
678
+ * Severity-floor gate. Reject borderline candidates whose score sits in a band
679
+ * just under the threshold — judge noise, not a real defect.
680
+ */
639
681
  function severityFloorGate(opts) {
640
- const threshold = opts.failureThreshold ?? 0.5;
641
- const margin = opts.margin ?? 0.1;
642
- return {
643
- isValid: (_scenario, ev) => ev.score <= threshold - margin
644
- };
682
+ const threshold = opts.failureThreshold ?? .5;
683
+ const margin = opts.margin ?? .1;
684
+ return { isValid: (_scenario, ev) => ev.score <= threshold - margin };
645
685
  }
646
-
647
- // src/fuzz/tools.ts
648
- var NO_ARGS = { type: "object", properties: {}, additionalProperties: false };
686
+ //#endregion
687
+ //#region src/fuzz/tools.ts
688
+ /**
689
+ * Agent-drivable surface over a live exploration session.
690
+ *
691
+ * Framework-neutral tool defs ({name, description, parameters: JSON Schema,
692
+ * handler}) so the on-demand agent — not a batch script — drives the search:
693
+ * step it, read coverage, inspect findings, render the capsule. Transport
694
+ * encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
695
+ */
696
+ const NO_ARGS = {
697
+ type: "object",
698
+ properties: {},
699
+ additionalProperties: false
700
+ };
649
701
  function makeExploreTools(explorer) {
650
- return [
651
- {
652
- name: "explore_step",
653
- description: "Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.",
654
- parameters: NO_ARGS,
655
- handler: async () => {
656
- const { runs, findings } = await explorer.step();
657
- return { runs, newFindings: findings.length, findings };
658
- }
659
- },
660
- {
661
- name: "explore_coverage",
662
- description: "Read the live coverage map: per planned cell \u2014 runs, robustness (null = uncovered), finding rate, per-dimension means.",
663
- parameters: NO_ARGS,
664
- handler: async () => explorer.coverage()
665
- },
666
- {
667
- name: "explore_findings",
668
- description: "List gate-verified findings so far, sorted by descending interest.",
669
- parameters: {
670
- type: "object",
671
- properties: { limit: { type: "number", description: "max findings to return" } },
672
- additionalProperties: false
673
- },
674
- handler: async (args) => {
675
- const limit = args?.limit;
676
- const all = explorer.findings();
677
- return typeof limit === "number" ? all.slice(0, limit) : all;
678
- }
679
- },
680
- {
681
- name: "explore_capsule",
682
- description: 'Build the capsule artifact from the current session state. format "data" returns the structured CapsuleData; "html" returns the standalone page.',
683
- parameters: {
684
- type: "object",
685
- properties: {
686
- format: { type: "string", enum: ["data", "html"] },
687
- generatedAt: { type: "string", description: "ISO timestamp to stamp into the page" }
688
- },
689
- additionalProperties: false
690
- },
691
- handler: async (args) => {
692
- const a = args ?? {};
693
- const capsule = explorer.capsule();
694
- if (a.format === "html") return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt });
695
- return capsule;
696
- }
697
- }
698
- ];
702
+ return [
703
+ {
704
+ name: "explore_step",
705
+ description: "Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.",
706
+ parameters: NO_ARGS,
707
+ handler: async () => {
708
+ const { runs, findings } = await explorer.step();
709
+ return {
710
+ runs,
711
+ newFindings: findings.length,
712
+ findings
713
+ };
714
+ }
715
+ },
716
+ {
717
+ name: "explore_coverage",
718
+ description: "Read the live coverage map: per planned cell — runs, robustness (null = uncovered), finding rate, per-dimension means.",
719
+ parameters: NO_ARGS,
720
+ handler: async () => explorer.coverage()
721
+ },
722
+ {
723
+ name: "explore_findings",
724
+ description: "List gate-verified findings so far, sorted by descending interest.",
725
+ parameters: {
726
+ type: "object",
727
+ properties: { limit: {
728
+ type: "number",
729
+ description: "max findings to return"
730
+ } },
731
+ additionalProperties: false
732
+ },
733
+ handler: async (args) => {
734
+ const limit = args?.limit;
735
+ const all = explorer.findings();
736
+ return typeof limit === "number" ? all.slice(0, limit) : all;
737
+ }
738
+ },
739
+ {
740
+ name: "explore_capsule",
741
+ description: "Build the capsule artifact from the current session state. format \"data\" returns the structured CapsuleData; \"html\" returns the standalone page.",
742
+ parameters: {
743
+ type: "object",
744
+ properties: {
745
+ format: {
746
+ type: "string",
747
+ enum: ["data", "html"]
748
+ },
749
+ generatedAt: {
750
+ type: "string",
751
+ description: "ISO timestamp to stamp into the page"
752
+ }
753
+ },
754
+ additionalProperties: false
755
+ },
756
+ handler: async (args) => {
757
+ const a = args ?? {};
758
+ const capsule = explorer.capsule();
759
+ if (a.format === "html") return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt });
760
+ return capsule;
761
+ }
762
+ }
763
+ ];
699
764
  }
700
- export {
701
- BehaviorExplorer,
702
- adversarialObjective,
703
- buildCapsule,
704
- buildCoverage,
705
- cellId,
706
- composeGates,
707
- enumerateCells,
708
- fuzzAgent,
709
- makeExploreTools,
710
- mutationProposer,
711
- noveltyObjective,
712
- perturbationStabilityGate,
713
- renderCapsuleHtml,
714
- severityFloorGate
715
- };
765
+ //#endregion
766
+ export { BehaviorExplorer, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };
767
+
716
768
  //# sourceMappingURL=fuzz.js.map