@tangle-network/agent-eval 0.128.2 → 0.130.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (424) hide show
  1. package/CHANGELOG.md +279 -0
  2. package/README.md +19 -0
  3. package/dist/active-curriculum-C4mk67HP.js +214 -0
  4. package/dist/active-curriculum-C4mk67HP.js.map +1 -0
  5. package/dist/adversarial-smnADNFS.d.ts +21 -0
  6. package/dist/adversarial-smnADNFS.d.ts.map +1 -0
  7. package/dist/analyst/index.d.ts +83 -2932
  8. package/dist/analyst/index.d.ts.map +1 -0
  9. package/dist/analyst/index.js +319 -364
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/analyst-BkTS3C58.d.ts +89 -0
  12. package/dist/analyst-BkTS3C58.d.ts.map +1 -0
  13. package/dist/analyst-LsnNpSkm.js +152 -0
  14. package/dist/analyst-LsnNpSkm.js.map +1 -0
  15. package/dist/analyze-runs-C1CavBMk.js +1067 -0
  16. package/dist/analyze-runs-C1CavBMk.js.map +1 -0
  17. package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
  18. package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
  19. package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
  20. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
  21. package/dist/authenticity/index.d.ts +81 -79
  22. package/dist/authenticity/index.d.ts.map +1 -0
  23. package/dist/authenticity/index.js +209 -193
  24. package/dist/authenticity/index.js.map +1 -1
  25. package/dist/baseline-HsBvw_dk.js +550 -0
  26. package/dist/baseline-HsBvw_dk.js.map +1 -0
  27. package/dist/baseline-hG3K85h4.d.ts +125 -0
  28. package/dist/baseline-hG3K85h4.d.ts.map +1 -0
  29. package/dist/belief-state/index.d.ts +448 -1205
  30. package/dist/belief-state/index.d.ts.map +1 -0
  31. package/dist/belief-state/index.js +1617 -1710
  32. package/dist/belief-state/index.js.map +1 -1
  33. package/dist/benchmarks/index.d.ts +2 -894
  34. package/dist/benchmarks/index.js +2 -59
  35. package/dist/benchmarks-DviOvUNr.js +754 -0
  36. package/dist/benchmarks-DviOvUNr.js.map +1 -0
  37. package/dist/builder-eval/index.d.ts +150 -662
  38. package/dist/builder-eval/index.d.ts.map +1 -0
  39. package/dist/builder-eval/index.js +356 -345
  40. package/dist/builder-eval/index.js.map +1 -1
  41. package/dist/calibration-CNWWA6K8.js +94 -0
  42. package/dist/calibration-CNWWA6K8.js.map +1 -0
  43. package/dist/campaign/index.d.ts +5 -6390
  44. package/dist/campaign/index.js +3 -212
  45. package/dist/campaign-CBKZvQ1H.js +3885 -0
  46. package/dist/campaign-CBKZvQ1H.js.map +1 -0
  47. package/dist/cli.d.ts +1 -1
  48. package/dist/cli.js +164 -174
  49. package/dist/cli.js.map +1 -1
  50. package/dist/client-C97NMzqi.d.ts +581 -0
  51. package/dist/client-C97NMzqi.d.ts.map +1 -0
  52. package/dist/client-CYzbdJOZ.js +637 -0
  53. package/dist/client-CYzbdJOZ.js.map +1 -0
  54. package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
  55. package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
  56. package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
  57. package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
  58. package/dist/concurrency-DIxRZF_J.js +85 -0
  59. package/dist/concurrency-DIxRZF_J.js.map +1 -0
  60. package/dist/contract/index.d.ts +645 -5605
  61. package/dist/contract/index.d.ts.map +1 -0
  62. package/dist/contract/index.js +1730 -1937
  63. package/dist/contract/index.js.map +1 -1
  64. package/dist/control.d.ts +3 -1030
  65. package/dist/control.js +2 -32
  66. package/dist/cost-ledger-DIgQUFZZ.js +801 -0
  67. package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
  68. package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
  69. package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
  70. package/dist/dataset-BvtnC8Dc.d.ts +115 -0
  71. package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
  72. package/dist/default-registry-C-vFCSEc.js +2579 -0
  73. package/dist/default-registry-C-vFCSEc.js.map +1 -0
  74. package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
  75. package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
  76. package/dist/emitter-CPBAhxum.js +266 -0
  77. package/dist/emitter-CPBAhxum.js.map +1 -0
  78. package/dist/emitter-DGQGoLyj.d.ts +113 -0
  79. package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
  80. package/dist/errors-8YnH8WlF.js +64 -0
  81. package/dist/errors-8YnH8WlF.js.map +1 -0
  82. package/dist/errors-CEk209JS.d.ts +76 -0
  83. package/dist/errors-CEk209JS.d.ts.map +1 -0
  84. package/dist/eval-campaign-DEm6c8ru.js +349 -0
  85. package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
  86. package/dist/execution-tracks-CpgFPpS5.js +93 -0
  87. package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
  88. package/dist/exporters-q9iL-2Jf.js +148 -0
  89. package/dist/exporters-q9iL-2Jf.js.map +1 -0
  90. package/dist/extract-usage-BrQ8mCLX.js +155 -0
  91. package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
  92. package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
  93. package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
  94. package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
  95. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
  96. package/dist/fuzz.d.ts +320 -646
  97. package/dist/fuzz.d.ts.map +1 -0
  98. package/dist/fuzz.js +670 -617
  99. package/dist/fuzz.js.map +1 -1
  100. package/dist/hf-dataset-DBJXXoY1.js +763 -0
  101. package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
  102. package/dist/hosted/index.d.ts +11 -831
  103. package/dist/hosted/index.d.ts.map +1 -0
  104. package/dist/hosted/index.js +2 -37
  105. package/dist/index-2JJSA6-r2.d.ts +926 -0
  106. package/dist/index-2JJSA6-r2.d.ts.map +1 -0
  107. package/dist/index-6N0aYmpW.d.ts +217 -0
  108. package/dist/index-6N0aYmpW.d.ts.map +1 -0
  109. package/dist/index-BAvgST_9.d.ts +131 -0
  110. package/dist/index-BAvgST_9.d.ts.map +1 -0
  111. package/dist/index-BvnJuTGD.d.ts +68 -0
  112. package/dist/index-BvnJuTGD.d.ts.map +1 -0
  113. package/dist/index-C61Wi7yg.d.ts +547 -0
  114. package/dist/index-C61Wi7yg.d.ts.map +1 -0
  115. package/dist/index-CAPUUKaM.d.ts +335 -0
  116. package/dist/index-CAPUUKaM.d.ts.map +1 -0
  117. package/dist/index-DE5fb3EC.d.ts +2244 -0
  118. package/dist/index-DE5fb3EC.d.ts.map +1 -0
  119. package/dist/index-DSC51roc.d.ts +102 -0
  120. package/dist/index-DSC51roc.d.ts.map +1 -0
  121. package/dist/index.d.ts +3776 -15120
  122. package/dist/index.d.ts.map +1 -0
  123. package/dist/index.js +11185 -11191
  124. package/dist/index.js.map +1 -1
  125. package/dist/integrity-BzRbCHzi.js +424 -0
  126. package/dist/integrity-BzRbCHzi.js.map +1 -0
  127. package/dist/integrity-rmVhXWA7.d.ts +61 -0
  128. package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
  129. package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
  130. package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
  131. package/dist/ledger-core/index.d.ts +2 -0
  132. package/dist/ledger-core/index.js +2 -0
  133. package/dist/ledger-core-DtZz1RG0.js +388 -0
  134. package/dist/ledger-core-DtZz1RG0.js.map +1 -0
  135. package/dist/llm-client--GR4JbZE.js +687 -0
  136. package/dist/llm-client--GR4JbZE.js.map +1 -0
  137. package/dist/llm-client-B_nIBlYo.d.ts +290 -0
  138. package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
  139. package/dist/matrix/index.d.ts +3 -155
  140. package/dist/matrix/index.js +2 -8
  141. package/dist/matrix-BzQnu2S6.js +270 -0
  142. package/dist/matrix-BzQnu2S6.js.map +1 -0
  143. package/dist/meta-eval/index.d.ts +4 -1027
  144. package/dist/meta-eval/index.js +393 -390
  145. package/dist/meta-eval/index.js.map +1 -1
  146. package/dist/metrics-C9YY1OcL.js +239 -0
  147. package/dist/metrics-C9YY1OcL.js.map +1 -0
  148. package/dist/mint-yN2M2eh0.js +201 -0
  149. package/dist/mint-yN2M2eh0.js.map +1 -0
  150. package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
  151. package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
  152. package/dist/multishot/index.d.ts +273 -481
  153. package/dist/multishot/index.d.ts.map +1 -0
  154. package/dist/multishot/index.js +595 -548
  155. package/dist/multishot/index.js.map +1 -1
  156. package/dist/off-policy-DvgzvtIx.js +220 -0
  157. package/dist/off-policy-DvgzvtIx.js.map +1 -0
  158. package/dist/off-policy-mskQw8Mb.d.ts +153 -0
  159. package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
  160. package/dist/openapi.json +1 -1
  161. package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
  162. package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
  163. package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
  164. package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
  165. package/dist/outcome-store-ChBKlTd_.js +75 -0
  166. package/dist/outcome-store-ChBKlTd_.js.map +1 -0
  167. package/dist/paired-arms-D9D0wXj2.js +260 -0
  168. package/dist/paired-arms-D9D0wXj2.js.map +1 -0
  169. package/dist/pipelines/index.d.ts +95 -532
  170. package/dist/pipelines/index.d.ts.map +1 -0
  171. package/dist/pipelines/index.js +497 -478
  172. package/dist/pipelines/index.js.map +1 -1
  173. package/dist/pre-registration-DakwTRXk.js +96 -0
  174. package/dist/pre-registration-DakwTRXk.js.map +1 -0
  175. package/dist/propose-review-control-LhIWGzHl.js +1458 -0
  176. package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
  177. package/dist/query-CJ_DX8vl.d.ts +28 -0
  178. package/dist/query-CJ_DX8vl.d.ts.map +1 -0
  179. package/dist/query-Di7eEQ79.js +83 -0
  180. package/dist/query-Di7eEQ79.js.map +1 -0
  181. package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
  182. package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
  183. package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
  184. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
  185. package/dist/redact-7Aq1ukl-.js +107 -0
  186. package/dist/redact-7Aq1ukl-.js.map +1 -0
  187. package/dist/release-report-Cz9NKH39.js +603 -0
  188. package/dist/release-report-Cz9NKH39.js.map +1 -0
  189. package/dist/release-report-mvB2G4_J.d.ts +244 -0
  190. package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
  191. package/dist/replay-GnyotH0J.js +1741 -0
  192. package/dist/replay-GnyotH0J.js.map +1 -0
  193. package/dist/replay-OoidtG1E.d.ts +749 -0
  194. package/dist/replay-OoidtG1E.d.ts.map +1 -0
  195. package/dist/reporting.d.ts +6 -1298
  196. package/dist/reporting.js +6 -50
  197. package/dist/researcher-CwTdwXG1.d.ts +314 -0
  198. package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
  199. package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
  200. package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
  201. package/dist/reward-hacking-qipEpKvY.js +596 -0
  202. package/dist/reward-hacking-qipEpKvY.js.map +1 -0
  203. package/dist/reward-nw2xZGZG.js +137 -0
  204. package/dist/reward-nw2xZGZG.js.map +1 -0
  205. package/dist/rl.d.ts +916 -3596
  206. package/dist/rl.d.ts.map +1 -0
  207. package/dist/rl.js +2362 -1751
  208. package/dist/rl.js.map +1 -1
  209. package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
  210. package/dist/rollout/index.d.ts +3 -1048
  211. package/dist/rollout/index.js +8 -110
  212. package/dist/rollout-BOYjemfR.js +624 -0
  213. package/dist/rollout-BOYjemfR.js.map +1 -0
  214. package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
  215. package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
  216. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
  217. package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
  218. package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
  219. package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
  220. package/dist/run-record-BuoE80Dq.js +467 -0
  221. package/dist/run-record-BuoE80Dq.js.map +1 -0
  222. package/dist/run-record-CnZu_gjl.d.ts +357 -0
  223. package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
  224. package/dist/run-score-iEEAWiBY.js +41 -0
  225. package/dist/run-score-iEEAWiBY.js.map +1 -0
  226. package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
  227. package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
  228. package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
  229. package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
  230. package/dist/schema-BtVldJ3T.d.ts +206 -0
  231. package/dist/schema-BtVldJ3T.d.ts.map +1 -0
  232. package/dist/schema-C6DW4ZHR.js +821 -0
  233. package/dist/schema-C6DW4ZHR.js.map +1 -0
  234. package/dist/schema-CRhEY1SO.js +69 -0
  235. package/dist/schema-CRhEY1SO.js.map +1 -0
  236. package/dist/schema-Cef2cFmb.d.ts +408 -0
  237. package/dist/schema-Cef2cFmb.d.ts.map +1 -0
  238. package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
  239. package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
  240. package/dist/sequential-Br0mAPHA.js +148 -0
  241. package/dist/sequential-Br0mAPHA.js.map +1 -0
  242. package/dist/sequential-CYwq6Ff_.d.ts +141 -0
  243. package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
  244. package/dist/series-convergence-CjO2QdRW.js +43 -0
  245. package/dist/series-convergence-CjO2QdRW.js.map +1 -0
  246. package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
  247. package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
  248. package/dist/server-m5D9cvnG.js +1040 -0
  249. package/dist/server-m5D9cvnG.js.map +1 -0
  250. package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
  251. package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
  252. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
  253. package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
  254. package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
  255. package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
  256. package/dist/statistics-Cmj6nynr.d.ts +514 -0
  257. package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
  258. package/dist/statistics-CnnxdpOg.js +1437 -0
  259. package/dist/statistics-CnnxdpOg.js.map +1 -0
  260. package/dist/store-CT9YIIve.d.ts +117 -0
  261. package/dist/store-CT9YIIve.d.ts.map +1 -0
  262. package/dist/store-CxJry_cs.d.ts +229 -0
  263. package/dist/store-CxJry_cs.d.ts.map +1 -0
  264. package/dist/storyboard/index.d.ts +84 -203
  265. package/dist/storyboard/index.d.ts.map +1 -0
  266. package/dist/storyboard/index.js +609 -542
  267. package/dist/storyboard/index.js.map +1 -1
  268. package/dist/summary-report-BNs5nmXI.js +862 -0
  269. package/dist/summary-report-BNs5nmXI.js.map +1 -0
  270. package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
  271. package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
  272. package/dist/supervisor-run/index.d.ts +2 -849
  273. package/dist/supervisor-run/index.js +2 -64
  274. package/dist/supervisor-run-_lnTLM3z.js +1679 -0
  275. package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
  276. package/dist/task-failure-attributes-CQZlB3et.js +311 -0
  277. package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
  278. package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
  279. package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
  280. package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
  281. package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
  282. package/dist/tools-BmuN627J.js +1085 -0
  283. package/dist/tools-BmuN627J.js.map +1 -0
  284. package/dist/trace-attributes.d.ts +2 -52
  285. package/dist/trace-attributes.js +131 -61
  286. package/dist/trace-attributes.js.map +1 -1
  287. package/dist/traces.d.ts +12 -2365
  288. package/dist/traces.js +12 -251
  289. package/dist/trajectory-D_7rLrvE.js +56 -0
  290. package/dist/trajectory-D_7rLrvE.js.map +1 -0
  291. package/dist/types-DGsxbAEd.d.ts +387 -0
  292. package/dist/types-DGsxbAEd.d.ts.map +1 -0
  293. package/dist/types-k9tZGKUg.d.ts +640 -0
  294. package/dist/types-k9tZGKUg.d.ts.map +1 -0
  295. package/dist/verdict-Dps8_okt.d.ts +37 -0
  296. package/dist/verdict-Dps8_okt.d.ts.map +1 -0
  297. package/dist/wire/index.d.ts +702 -1174
  298. package/dist/wire/index.d.ts.map +1 -0
  299. package/dist/wire/index.js +2 -81
  300. package/docs/feature-guide.md +1 -1
  301. package/docs/rollout.md +116 -2
  302. package/package.json +18 -10
  303. package/dist/benchmarks/index.js.map +0 -1
  304. package/dist/campaign/index.js.map +0 -1
  305. package/dist/chunk-2JX3CFMB.js +0 -695
  306. package/dist/chunk-2JX3CFMB.js.map +0 -1
  307. package/dist/chunk-2MKQIFS4.js +0 -183
  308. package/dist/chunk-2MKQIFS4.js.map +0 -1
  309. package/dist/chunk-3RF76KTD.js +0 -84
  310. package/dist/chunk-3RF76KTD.js.map +0 -1
  311. package/dist/chunk-5DTSBUL2.js +0 -159
  312. package/dist/chunk-5DTSBUL2.js.map +0 -1
  313. package/dist/chunk-7ZZMD7UK.js +0 -386
  314. package/dist/chunk-7ZZMD7UK.js.map +0 -1
  315. package/dist/chunk-BOD4O7OF.js +0 -40
  316. package/dist/chunk-BOD4O7OF.js.map +0 -1
  317. package/dist/chunk-BYT7ELPS.js +0 -1553
  318. package/dist/chunk-BYT7ELPS.js.map +0 -1
  319. package/dist/chunk-DJKY2TSY.js +0 -2428
  320. package/dist/chunk-DJKY2TSY.js.map +0 -1
  321. package/dist/chunk-DPUHNQLN.js +0 -232
  322. package/dist/chunk-DPUHNQLN.js.map +0 -1
  323. package/dist/chunk-DRYIUNWY.js +0 -622
  324. package/dist/chunk-DRYIUNWY.js.map +0 -1
  325. package/dist/chunk-EJGRPCO3.js +0 -617
  326. package/dist/chunk-EJGRPCO3.js.map +0 -1
  327. package/dist/chunk-EOSZT7PL.js +0 -2001
  328. package/dist/chunk-EOSZT7PL.js.map +0 -1
  329. package/dist/chunk-EZJEIH2R.js +0 -1559
  330. package/dist/chunk-EZJEIH2R.js.map +0 -1
  331. package/dist/chunk-GGE4NNQT.js +0 -65
  332. package/dist/chunk-GGE4NNQT.js.map +0 -1
  333. package/dist/chunk-HHWE3POT.js +0 -94
  334. package/dist/chunk-HHWE3POT.js.map +0 -1
  335. package/dist/chunk-IHQDPH7D.js +0 -171
  336. package/dist/chunk-IHQDPH7D.js.map +0 -1
  337. package/dist/chunk-JHCHEVET.js +0 -274
  338. package/dist/chunk-JHCHEVET.js.map +0 -1
  339. package/dist/chunk-K4DBDHLK.js +0 -158
  340. package/dist/chunk-K4DBDHLK.js.map +0 -1
  341. package/dist/chunk-K6N6XJJX.js +0 -306
  342. package/dist/chunk-K6N6XJJX.js.map +0 -1
  343. package/dist/chunk-MA6HLL3S.js +0 -65
  344. package/dist/chunk-MA6HLL3S.js.map +0 -1
  345. package/dist/chunk-MAZ26DC7.js +0 -99
  346. package/dist/chunk-MAZ26DC7.js.map +0 -1
  347. package/dist/chunk-MHELPNRP.js +0 -1212
  348. package/dist/chunk-MHELPNRP.js.map +0 -1
  349. package/dist/chunk-NACAGYSY.js +0 -1040
  350. package/dist/chunk-NACAGYSY.js.map +0 -1
  351. package/dist/chunk-NKAGIDE2.js +0 -7633
  352. package/dist/chunk-NKAGIDE2.js.map +0 -1
  353. package/dist/chunk-NPCTHQIO.js +0 -91
  354. package/dist/chunk-NPCTHQIO.js.map +0 -1
  355. package/dist/chunk-NYLOYM6N.js +0 -332
  356. package/dist/chunk-NYLOYM6N.js.map +0 -1
  357. package/dist/chunk-ONWEPEDO.js +0 -57
  358. package/dist/chunk-ONWEPEDO.js.map +0 -1
  359. package/dist/chunk-P5W7RQKK.js +0 -576
  360. package/dist/chunk-P5W7RQKK.js.map +0 -1
  361. package/dist/chunk-P6FYH6K4.js +0 -1161
  362. package/dist/chunk-P6FYH6K4.js.map +0 -1
  363. package/dist/chunk-PBE2LOSS.js +0 -669
  364. package/dist/chunk-PBE2LOSS.js.map +0 -1
  365. package/dist/chunk-PC4UYEBM.js +0 -166
  366. package/dist/chunk-PC4UYEBM.js.map +0 -1
  367. package/dist/chunk-PXE2VKMX.js +0 -140
  368. package/dist/chunk-PXE2VKMX.js.map +0 -1
  369. package/dist/chunk-PZ5AY32C.js +0 -10
  370. package/dist/chunk-PZ5AY32C.js.map +0 -1
  371. package/dist/chunk-RZTMDUO7.js +0 -49
  372. package/dist/chunk-RZTMDUO7.js.map +0 -1
  373. package/dist/chunk-S5YLIBFX.js +0 -136
  374. package/dist/chunk-S5YLIBFX.js.map +0 -1
  375. package/dist/chunk-SZLVEKMJ.js +0 -1446
  376. package/dist/chunk-SZLVEKMJ.js.map +0 -1
  377. package/dist/chunk-T4SQEITX.js +0 -95
  378. package/dist/chunk-T4SQEITX.js.map +0 -1
  379. package/dist/chunk-TBL77AUT.js +0 -355
  380. package/dist/chunk-TBL77AUT.js.map +0 -1
  381. package/dist/chunk-TSN7JT6D.js +0 -1646
  382. package/dist/chunk-TSN7JT6D.js.map +0 -1
  383. package/dist/chunk-TT4KNT67.js +0 -124
  384. package/dist/chunk-TT4KNT67.js.map +0 -1
  385. package/dist/chunk-UB2LOJ6Q.js +0 -4461
  386. package/dist/chunk-UB2LOJ6Q.js.map +0 -1
  387. package/dist/chunk-UWZZKKU7.js +0 -237
  388. package/dist/chunk-UWZZKKU7.js.map +0 -1
  389. package/dist/chunk-VBQ3CRKH.js +0 -291
  390. package/dist/chunk-VBQ3CRKH.js.map +0 -1
  391. package/dist/chunk-VGRCHJON.js +0 -163
  392. package/dist/chunk-VGRCHJON.js.map +0 -1
  393. package/dist/chunk-VI2UW6B6.js +0 -162
  394. package/dist/chunk-VI2UW6B6.js.map +0 -1
  395. package/dist/chunk-VLOATJQ2.js +0 -908
  396. package/dist/chunk-VLOATJQ2.js.map +0 -1
  397. package/dist/chunk-VQMK5FMP.js +0 -247
  398. package/dist/chunk-VQMK5FMP.js.map +0 -1
  399. package/dist/chunk-VZSRQ272.js +0 -149
  400. package/dist/chunk-VZSRQ272.js.map +0 -1
  401. package/dist/chunk-WGXIEX7P.js +0 -116
  402. package/dist/chunk-WGXIEX7P.js.map +0 -1
  403. package/dist/chunk-WS3NZZQQ.js +0 -929
  404. package/dist/chunk-WS3NZZQQ.js.map +0 -1
  405. package/dist/chunk-XDWDC2MP.js +0 -695
  406. package/dist/chunk-XDWDC2MP.js.map +0 -1
  407. package/dist/chunk-XPRT64IE.js +0 -766
  408. package/dist/chunk-XPRT64IE.js.map +0 -1
  409. package/dist/chunk-YJBNWCAA.js +0 -1056
  410. package/dist/chunk-YJBNWCAA.js.map +0 -1
  411. package/dist/chunk-ZET2UAYW.js +0 -89
  412. package/dist/chunk-ZET2UAYW.js.map +0 -1
  413. package/dist/chunk-ZUUWPZCV.js +0 -752
  414. package/dist/chunk-ZUUWPZCV.js.map +0 -1
  415. package/dist/control.js.map +0 -1
  416. package/dist/hosted/index.js.map +0 -1
  417. package/dist/matrix/index.js.map +0 -1
  418. package/dist/reporting.js.map +0 -1
  419. package/dist/rollout/index.js.map +0 -1
  420. package/dist/run-campaign-ISHFZ7FJ.js +0 -17
  421. package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
  422. package/dist/supervisor-run/index.js.map +0 -1
  423. package/dist/traces.js.map +0 -1
  424. package/dist/wire/index.js.map +0 -1
package/dist/fuzz.js CHANGED
@@ -1,198 +1,207 @@
1
- import {
2
- varianceBasedCurriculum
3
- } from "./chunk-VZSRQ272.js";
4
- import {
5
- CostLedger,
6
- CostReceiptCaptureError
7
- } from "./chunk-WS3NZZQQ.js";
8
- import "./chunk-VI2UW6B6.js";
9
- import {
10
- ValidationError
11
- } from "./chunk-ONWEPEDO.js";
12
- import "./chunk-PZ5AY32C.js";
13
-
14
- // src/fuzz/cube.ts
1
+ import { s as ValidationError } from "./errors-8YnH8WlF.js";
2
+ import { i as CostLedger, o as CostReceiptCaptureError } from "./cost-ledger-DIgQUFZZ.js";
3
+ import { r as varianceBasedCurriculum } from "./active-curriculum-C4mk67HP.js";
4
+ //#region src/fuzz/cube.ts
5
+ /** Enumerate every input cell (cartesian product of the axes), in stable order. */
15
6
  function enumerateCells(space) {
16
- if (space.axes.length === 0) return [];
17
- let partials = [{}];
18
- for (const axis of space.axes) {
19
- const next = [];
20
- for (const partial of partials) {
21
- for (const value of axis.values) next.push({ ...partial, [axis.name]: value });
22
- }
23
- partials = next;
24
- }
25
- return partials.map((coords) => ({ id: cellId(space, coords), coords }));
7
+ if (space.axes.length === 0) return [];
8
+ let partials = [{}];
9
+ for (const axis of space.axes) {
10
+ const next = [];
11
+ for (const partial of partials) for (const value of axis.values) next.push({
12
+ ...partial,
13
+ [axis.name]: value
14
+ });
15
+ partials = next;
16
+ }
17
+ return partials.map((coords) => ({
18
+ id: cellId(space, coords),
19
+ coords
20
+ }));
26
21
  }
22
+ /** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */
27
23
  function cellId(space, coords) {
28
- return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
24
+ return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
29
25
  }
26
+ /** Nearest-rank percentile on a pre-sorted ascending sample. */
30
27
  function percentile(sorted, p) {
31
- const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
32
- return sorted[idx];
28
+ return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
33
29
  }
30
+ /** Summarize a sample. Throws on an empty sample — callers represent "no data"
31
+ * as `null`, never as a zeroed distribution. */
34
32
  function distribution(values) {
35
- if (values.length === 0)
36
- throw new Error("distribution: empty sample \u2014 represent missing data as null, not zeros");
37
- const sorted = [...values].sort((a, b) => a - b);
38
- const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
39
- return {
40
- mean,
41
- median: percentile(sorted, 0.5),
42
- p90: percentile(sorted, 0.9),
43
- min: sorted[0],
44
- max: sorted[sorted.length - 1],
45
- n: sorted.length
46
- };
33
+ if (values.length === 0) throw new Error("distribution: empty sample — represent missing data as null, not zeros");
34
+ const sorted = [...values].sort((a, b) => a - b);
35
+ return {
36
+ mean: sorted.reduce((a, b) => a + b, 0) / sorted.length,
37
+ median: percentile(sorted, .5),
38
+ p90: percentile(sorted, .9),
39
+ min: sorted[0],
40
+ max: sorted[sorted.length - 1],
41
+ n: sorted.length
42
+ };
47
43
  }
44
+ /**
45
+ * Project the evaluation log into the per-input-cell coverage map. A cell with
46
+ * no evaluations reports `score: null` (honestly uncovered), never zeros.
47
+ */
48
48
  function buildCoverage(cells, log, threshold) {
49
- const byCell = /* @__PURE__ */ new Map();
50
- for (const r of log) {
51
- const arr = byCell.get(r.cell.id) ?? [];
52
- arr.push(r);
53
- byCell.set(r.cell.id, arr);
54
- }
55
- return cells.map((cell) => {
56
- const recs = byCell.get(cell.id) ?? [];
57
- const runs = recs.length;
58
- if (runs === 0)
59
- return { cell, runs: 0, score: null, findingRate: 0, dimensions: {}, latencyMs: null };
60
- const score = distribution(recs.map((r) => r.ev.score));
61
- const latencyMs = distribution(recs.map((r) => r.latencyMs));
62
- const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
63
- const dimSamples = {};
64
- for (const r of recs) {
65
- for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
66
- const samples = dimSamples[k] ?? [];
67
- samples.push(v);
68
- dimSamples[k] = samples;
69
- }
70
- }
71
- const dimensions = {};
72
- for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
73
- const tracked = recs.filter((r) => r.costUsd !== void 0);
74
- const known = tracked.filter((r) => r.costUsd !== null);
75
- const cost = tracked.length > 0 ? {
76
- costUsd: known.reduce((a, r) => a + r.costUsd, 0),
77
- ...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
78
- } : {};
79
- return { cell, runs, score, findingRate, dimensions, latencyMs, ...cost };
80
- });
49
+ const byCell = /* @__PURE__ */ new Map();
50
+ for (const r of log) {
51
+ const arr = byCell.get(r.cell.id) ?? [];
52
+ arr.push(r);
53
+ byCell.set(r.cell.id, arr);
54
+ }
55
+ return cells.map((cell) => {
56
+ const recs = byCell.get(cell.id) ?? [];
57
+ const runs = recs.length;
58
+ if (runs === 0) return {
59
+ cell,
60
+ runs: 0,
61
+ score: null,
62
+ findingRate: 0,
63
+ dimensions: {},
64
+ latencyMs: null
65
+ };
66
+ const score = distribution(recs.map((r) => r.ev.score));
67
+ const latencyMs = distribution(recs.map((r) => r.latencyMs));
68
+ const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
69
+ const dimSamples = {};
70
+ for (const r of recs) for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
71
+ const samples = dimSamples[k] ?? [];
72
+ samples.push(v);
73
+ dimSamples[k] = samples;
74
+ }
75
+ const dimensions = {};
76
+ for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
77
+ const tracked = recs.filter((r) => r.costUsd !== void 0);
78
+ const known = tracked.filter((r) => r.costUsd !== null);
79
+ return {
80
+ cell,
81
+ runs,
82
+ score,
83
+ findingRate,
84
+ dimensions,
85
+ latencyMs,
86
+ ...tracked.length > 0 ? {
87
+ costUsd: known.reduce((a, r) => a + r.costUsd, 0),
88
+ ...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
89
+ } : {}
90
+ };
91
+ });
81
92
  }
82
-
83
- // src/fuzz/capsule.ts
93
+ //#endregion
94
+ //#region src/fuzz/capsule.ts
84
95
  function buildCapsule(input) {
85
- const coverage = buildCoverage(input.cells, input.log, input.threshold);
86
- const covered = coverage.filter((c) => c.runs > 0);
87
- const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
88
- const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
89
- const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
90
- return {
91
- target: input.target,
92
- objective: input.objective,
93
- coverage,
94
- findings: [...input.findings].sort((a, b) => b.interest - a.interest),
95
- archive: [...input.archive].sort((a, b) => b.interest - a.interest),
96
- stats: {
97
- totalRuns: input.runsUsed,
98
- cellsTotal: input.cells.length,
99
- cellsCovered: covered.length,
100
- behaviorBinsObserved,
101
- candidateFindings: input.candidateFindings,
102
- verifiedFindings: input.findings.length,
103
- robustness,
104
- latencyMs,
105
- ...input.cost ? { costUsd: input.cost.costUsd, costUnknownRuns: input.cost.costUnknownRuns } : {},
106
- evalErrors: input.evalErrors,
107
- ...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
108
- }
109
- };
96
+ const coverage = buildCoverage(input.cells, input.log, input.threshold);
97
+ const covered = coverage.filter((c) => c.runs > 0);
98
+ const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
99
+ const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
100
+ const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
101
+ return {
102
+ target: input.target,
103
+ objective: input.objective,
104
+ coverage,
105
+ findings: [...input.findings].sort((a, b) => b.interest - a.interest),
106
+ archive: [...input.archive].sort((a, b) => b.interest - a.interest),
107
+ stats: {
108
+ totalRuns: input.runsUsed,
109
+ cellsTotal: input.cells.length,
110
+ cellsCovered: covered.length,
111
+ behaviorBinsObserved,
112
+ candidateFindings: input.candidateFindings,
113
+ verifiedFindings: input.findings.length,
114
+ robustness,
115
+ latencyMs,
116
+ ...input.cost ? {
117
+ costUsd: input.cost.costUsd,
118
+ costUnknownRuns: input.cost.costUnknownRuns
119
+ } : {},
120
+ evalErrors: input.evalErrors,
121
+ ...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
122
+ }
123
+ };
110
124
  }
111
125
  function esc(s) {
112
- return s.replace(
113
- /[&<>"]/g,
114
- (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]
115
- );
126
+ return s.replace(/[&<>"]/g, (c) => ({
127
+ "&": "&amp;",
128
+ "<": "&lt;",
129
+ ">": "&gt;",
130
+ "\"": "&quot;"
131
+ })[c]);
116
132
  }
133
+ /** red (0) → amber (.5) → green (1); uncovered cells render gray. */
117
134
  function robustnessColor(r) {
118
- if (r == null) return "#2a2a2e";
119
- return `hsl(${Math.round(r * 120)} 70% 42%)`;
135
+ if (r == null) return "#2a2a2e";
136
+ return `hsl(${Math.round(r * 120)} 70% 42%)`;
120
137
  }
121
138
  function pct(x) {
122
- return `${Math.round(x * 100)}%`;
139
+ return `${Math.round(x * 100)}%`;
123
140
  }
124
141
  function deriveAxes(coverage) {
125
- const order = [];
126
- const seen = /* @__PURE__ */ new Map();
127
- for (const c of coverage) {
128
- for (const [k, v] of Object.entries(c.cell.coords)) {
129
- if (!seen.has(k)) {
130
- seen.set(k, /* @__PURE__ */ new Set());
131
- order.push(k);
132
- }
133
- seen.get(k)?.add(v);
134
- }
135
- }
136
- return order.map((name) => ({ name, values: [...seen.get(name) ?? []] }));
142
+ const order = [];
143
+ const seen = /* @__PURE__ */ new Map();
144
+ for (const c of coverage) for (const [k, v] of Object.entries(c.cell.coords)) {
145
+ if (!seen.has(k)) {
146
+ seen.set(k, /* @__PURE__ */ new Set());
147
+ order.push(k);
148
+ }
149
+ seen.get(k)?.add(v);
150
+ }
151
+ return order.map((name) => ({
152
+ name,
153
+ values: [...seen.get(name) ?? []]
154
+ }));
137
155
  }
156
+ /** The weakest dimension chip for a cell, e.g. `safety 32%` — shown when scores exist. */
138
157
  function weakestDim(c) {
139
- const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
140
- if (entries.length === 0) return "";
141
- const sorted = entries.sort((a, b) => a[1] - b[1]);
142
- const w = sorted[0];
143
- if (!w) return "";
144
- return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
158
+ const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
159
+ if (entries.length === 0) return "";
160
+ const w = entries.sort((a, b) => a[1] - b[1])[0];
161
+ if (!w) return "";
162
+ return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
145
163
  }
146
164
  function heatmapHtml(coverage) {
147
- const axes = deriveAxes(coverage);
148
- const byId = new Map(coverage.map((c) => [c.cell.id, c]));
149
- const tile = (c, label) => {
150
- const r = c?.score?.mean ?? null;
151
- const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) \xB7 ${c.runs} runs \xB7 ${pct(c.findingRate)} flagged${c.latencyMs ? ` \xB7 ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` \xB7 $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
152
- return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : "\u2014"}</span>${c ? weakestDim(c) : ""}</div>`;
153
- };
154
- const rowAxis = axes[0];
155
- const colAxis = axes[1];
156
- if (axes.length === 2 && rowAxis && colAxis) {
157
- const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join("")}</tr>`;
158
- const rows = rowAxis.values.map((rv) => {
159
- const cells = colAxis.values.map((cv) => {
160
- const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`;
161
- return `<td>${tile(byId.get(id), "")}</td>`;
162
- }).join("");
163
- return `<tr><th class="rh">${esc(rv)}</th>${cells}</tr>`;
164
- }).join("");
165
- return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> \xB7 cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
166
- }
167
- const sorted = [...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2));
168
- return `<div class="grid">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(" \xB7 "))).join("")}</div>`;
165
+ const axes = deriveAxes(coverage);
166
+ const byId = new Map(coverage.map((c) => [c.cell.id, c]));
167
+ const tile = (c, label) => {
168
+ const r = c?.score?.mean ?? null;
169
+ const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) · ${c.runs} runs · ${pct(c.findingRate)} flagged${c.latencyMs ? ` · ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` · $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
170
+ return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : ""}</span>${c ? weakestDim(c) : ""}</div>`;
171
+ };
172
+ const rowAxis = axes[0];
173
+ const colAxis = axes[1];
174
+ if (axes.length === 2 && rowAxis && colAxis) {
175
+ const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join("")}</tr>`;
176
+ const rows = rowAxis.values.map((rv) => {
177
+ const cells = colAxis.values.map((cv) => {
178
+ const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`;
179
+ return `<td>${tile(byId.get(id), "")}</td>`;
180
+ }).join("");
181
+ return `<tr><th class="rh">${esc(rv)}</th>${cells}</tr>`;
182
+ }).join("");
183
+ return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> · cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
184
+ }
185
+ return `<div class="grid">${[...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2)).map((c) => tile(c, Object.values(c.cell.coords).join(" · "))).join("")}</div>`;
169
186
  }
170
187
  function findingsHtml(findings, limit) {
171
- if (findings.length === 0)
172
- return `<p class="none">No verified findings \u2014 the target held across every covered cell.</p>`;
173
- return findings.slice(0, limit).map((f) => {
174
- const coords = Object.entries(f.cell.coords).map(([k, v]) => `${k}:${v}`).join(" \xB7 ");
175
- const labels = (f.evaluation.labels ?? []).map((l) => `<span class="fclass">${esc(l)}</span>`).join("");
176
- const dims = Object.entries(f.evaluation.scores ?? {}).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([k, v]) => `<span class="dim">${esc(k)} ${pct(v)}</span>`).join("");
177
- return `<div class="fail"><div class="fmeta"><span class="fcell">${esc(coords)}</span>${labels}<span class="fsev">interest ${pct(f.interest)}</span></div><div class="sevbar"><div class="sevfill" style="width:${pct(f.interest)}"></div></div><div class="ftext">${esc(f.text ?? "(scenario text not captured)")}</div>${dims ? `<div class="fdims">${dims}</div>` : ""}</div>`;
178
- }).join("");
188
+ if (findings.length === 0) return `<p class="none">No verified findings — the target held across every covered cell.</p>`;
189
+ return findings.slice(0, limit).map((f) => {
190
+ const coords = Object.entries(f.cell.coords).map(([k, v]) => `${k}:${v}`).join(" · ");
191
+ const labels = (f.evaluation.labels ?? []).map((l) => `<span class="fclass">${esc(l)}</span>`).join("");
192
+ const dims = Object.entries(f.evaluation.scores ?? {}).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([k, v]) => `<span class="dim">${esc(k)} ${pct(v)}</span>`).join("");
193
+ return `<div class="fail"><div class="fmeta"><span class="fcell">${esc(coords)}</span>${labels}<span class="fsev">interest ${pct(f.interest)}</span></div><div class="sevbar"><div class="sevfill" style="width:${pct(f.interest)}"></div></div><div class="ftext">${esc(f.text ?? "(scenario text not captured)")}</div>${dims ? `<div class="fdims">${dims}</div>` : ""}</div>`;
194
+ }).join("");
179
195
  }
196
+ /** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
180
197
  function renderCapsuleHtml(capsule, opts = {}) {
181
- const s = capsule.stats;
182
- const kpi = (label, value, accent = "#e6e6e6") => `<div class="kpi"><div class="kv" style="color:${accent}">${esc(value)}</div><div class="kl">${esc(label)}</div></div>`;
183
- const lift = capsule.lift ? kpi(
184
- "hardening lift",
185
- `${capsule.lift.before.toFixed(2)} \u2192 ${capsule.lift.after.toFixed(2)}`,
186
- "#5ad17a"
187
- ) : "";
188
- const cost = s.costUsd !== void 0 ? kpi(
189
- s.costUnknownRuns ? `cost \xB7 ${s.costUnknownRuns} runs unpriced` : "cost",
190
- `$${s.costUsd.toFixed(2)}`,
191
- s.costUnknownRuns ? "#e5c07b" : "#e6e6e6"
192
- ) : "";
193
- const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
194
- return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
195
- <title>${esc(capsule.objective)} capsule \u2014 ${esc(capsule.target)}</title>
198
+ const s = capsule.stats;
199
+ const kpi = (label, value, accent = "#e6e6e6") => `<div class="kpi"><div class="kv" style="color:${accent}">${esc(value)}</div><div class="kl">${esc(label)}</div></div>`;
200
+ const lift = capsule.lift ? kpi("hardening lift", `${capsule.lift.before.toFixed(2)} → ${capsule.lift.after.toFixed(2)}`, "#5ad17a") : "";
201
+ const cost = s.costUsd !== void 0 ? kpi(s.costUnknownRuns ? `cost · ${s.costUnknownRuns} runs unpriced` : "cost", `$${s.costUsd.toFixed(2)}`, s.costUnknownRuns ? "#e5c07b" : "#e6e6e6") : "";
202
+ const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
203
+ return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
204
+ <title>${esc(capsule.objective)} capsule — ${esc(capsule.target)}</title>
196
205
  <style>
197
206
  :root{color-scheme:dark}
198
207
  *{box-sizing:border-box}
@@ -228,11 +237,11 @@ table.heat th.rh{text-align:right}
228
237
  .none{color:#5ad17a}
229
238
  .foot{color:#5a5a63;font-size:11px;margin-top:36px}
230
239
  </style></head><body><div class="wrap">
231
- <h1>${esc(capsule.objective)} exploration \xB7 ${esc(capsule.target)}</h1>
232
- <div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` \xB7 ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` \xB7 ${esc(stamp)}` : ""}</div>
240
+ <h1>${esc(capsule.objective)} exploration · ${esc(capsule.target)}</h1>
241
+ <div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` · ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` · ${esc(stamp)}` : ""}</div>
233
242
  <div class="kpis">
234
- ${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < 0.6 ? "#e58a96" : "#5ad17a") : ""}
235
- ${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}\u2013${pct(s.robustness.max)}`) : ""}
243
+ ${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < .6 ? "#e58a96" : "#5ad17a") : ""}
244
+ ${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}–${pct(s.robustness.max)}`) : ""}
236
245
  ${s.latencyMs ? kpi("median latency", `${(s.latencyMs.median / 1e3).toFixed(1)}s`, s.latencyMs.p90 > 4 * s.latencyMs.median ? "#e5b566" : "#e6e6e6") : ""}
237
246
  ${kpi("verified findings", String(s.verifiedFindings), s.verifiedFindings > 0 ? "#e58a96" : "#5ad17a")}
238
247
  ${kpi("cells covered", `${s.cellsCovered}/${s.cellsTotal}`)}
@@ -241,475 +250,519 @@ ${cost}
241
250
  ${s.evalErrors > 0 ? kpi("eval errors", String(s.evalErrors), "#e5b566") : ""}
242
251
  ${lift}
243
252
  </div>
244
- ${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} \u2014 coverage below reflects the completed portion only</div>` : ""}
253
+ ${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} coverage below reflects the completed portion only</div>` : ""}
245
254
  <h2>Coverage map</h2>
246
255
  ${heatmapHtml(capsule.coverage)}
247
- <h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` \xB7 ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
256
+ <h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` · ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
248
257
  ${findingsHtml(capsule.findings, opts.maxFindings ?? 8)}
249
258
  <div class="foot">Findings are gate-verified: each is a fair, answerable task that reproduces under a meaning-preserving rephrase, minimized to the smallest trigger.</div>
250
259
  </div></body></html>`;
251
260
  }
252
-
253
- // src/fuzz/policies.ts
254
- var clamp01 = (x) => Math.max(0, Math.min(1, x));
261
+ //#endregion
262
+ //#region src/fuzz/policies.ts
263
+ const clamp01 = (x) => Math.max(0, Math.min(1, x));
264
+ /**
265
+ * Perturbation-based search: apply the cell's mutation operators to the current
266
+ * elites + seeds, deduping by id. Elites first — mutating the most interesting
267
+ * scenario found so far is what makes the search deepen across rounds.
268
+ */
255
269
  function mutationProposer(opts) {
256
- return async (ctx) => {
257
- const mutations = opts.mutationsFor(ctx.cell);
258
- const parents = [...ctx.elites, ...ctx.seeds];
259
- const seen = new Set(parents.map(opts.scenarioId));
260
- const out = [];
261
- for (const parent of parents) {
262
- if (out.length >= ctx.count) break;
263
- for (const m of mutations) {
264
- const children = await m.mutate(parent, ctx.rng);
265
- for (const child of children) {
266
- const id = opts.scenarioId(child);
267
- if (seen.has(id)) continue;
268
- seen.add(id);
269
- out.push(child);
270
- if (out.length >= ctx.count) break;
271
- }
272
- if (out.length >= ctx.count) break;
273
- }
274
- }
275
- return out;
276
- };
270
+ return async (ctx) => {
271
+ const mutations = opts.mutationsFor(ctx.cell);
272
+ const parents = [...ctx.elites, ...ctx.seeds];
273
+ const seen = new Set(parents.map(opts.scenarioId));
274
+ const out = [];
275
+ for (const parent of parents) {
276
+ if (out.length >= ctx.count) break;
277
+ for (const m of mutations) {
278
+ const children = await m.mutate(parent, ctx.rng);
279
+ for (const child of children) {
280
+ const id = opts.scenarioId(child);
281
+ if (seen.has(id)) continue;
282
+ seen.add(id);
283
+ out.push(child);
284
+ if (out.length >= ctx.count) break;
285
+ }
286
+ if (out.length >= ctx.count) break;
287
+ }
288
+ }
289
+ return out;
290
+ };
277
291
  }
278
- function adversarialObjective(threshold = 0.5) {
279
- return { kind: "adversarial", threshold, interest: (ev) => clamp01(1 - ev.score) };
292
+ /** Adversarial: a low headline score is interesting — find where the agent fails. */
293
+ function adversarialObjective(threshold = .5) {
294
+ return {
295
+ kind: "adversarial",
296
+ threshold,
297
+ interest: (ev) => clamp01(1 - ev.score)
298
+ };
280
299
  }
281
300
  function hamming(a, b) {
282
- if (!a || !b) return 1;
283
- const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
284
- if (keys.size === 0) return 0;
285
- let diff = 0;
286
- for (const k of keys) if (a[k] !== b[k]) diff++;
287
- return diff / keys.size;
301
+ if (!a || !b) return 1;
302
+ const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
303
+ if (keys.size === 0) return 0;
304
+ let diff = 0;
305
+ for (const k of keys) if (a[k] !== b[k]) diff++;
306
+ return diff / keys.size;
288
307
  }
289
- function noveltyObjective(threshold = 0.3) {
290
- return {
291
- kind: "novelty",
292
- threshold,
293
- interest: (ev, ctx) => {
294
- const scoreNovelty = ctx.archiveScores.length === 0 ? 1 : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)));
295
- const descNovelty = ctx.archiveDescriptors.length === 0 ? 1 : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)));
296
- return clamp01(0.5 * scoreNovelty + 0.5 * descNovelty);
297
- }
298
- };
308
+ /**
309
+ * Novelty: interesting when far from the archive in score AND measured behavior
310
+ * descriptor — quality-diversity's diversity pressure; drives corpus growth
311
+ * rather than re-finding the same hole.
312
+ */
313
+ function noveltyObjective(threshold = .3) {
314
+ return {
315
+ kind: "novelty",
316
+ threshold,
317
+ interest: (ev, ctx) => {
318
+ const scoreNovelty = ctx.archiveScores.length === 0 ? 1 : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)));
319
+ const descNovelty = ctx.archiveDescriptors.length === 0 ? 1 : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)));
320
+ return clamp01(.5 * scoreNovelty + .5 * descNovelty);
321
+ }
322
+ };
299
323
  }
300
-
301
- // src/fuzz/explorer.ts
324
+ //#endregion
325
+ //#region src/fuzz/explorer.ts
326
+ /**
327
+ * The exploration engine — a stateful session over a behavior space.
328
+ *
329
+ * Each `step()`: allocate budget across INPUT cells (floor first, then variance
330
+ * steering toward the least-certain cells), propose candidates (the proposer
331
+ * reads current elites + findings, so the search deepens generationally),
332
+ * evaluate with bounded concurrency, archive the most interesting scenario per
333
+ * input×measured bin, and admit notable candidates that pass the validity gates.
334
+ * `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
335
+ * the surface `makeExploreTools` exposes so an agent can drive the session.
336
+ *
337
+ * One evaluation log (`EvalRecord[]`) is the source of truth; allocation
338
+ * observations and coverage are projections of it.
339
+ */
302
340
  async function pMap(items, fn, concurrency, signal) {
303
- let i = 0;
304
- const workers = Array.from(
305
- { length: Math.max(1, Math.min(concurrency, items.length)) },
306
- async () => {
307
- while (i < items.length) {
308
- if (signal?.aborted) return;
309
- const item = items[i++];
310
- if (item === void 0) return;
311
- await fn(item);
312
- }
313
- }
314
- );
315
- await Promise.all(workers);
341
+ let i = 0;
342
+ const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
343
+ while (i < items.length) {
344
+ if (signal?.aborted) return;
345
+ const item = items[i++];
346
+ if (item === void 0) return;
347
+ await fn(item);
348
+ }
349
+ });
350
+ await Promise.all(workers);
316
351
  }
317
352
  var BehaviorExplorer = class {
318
- constructor(opts) {
319
- this.opts = opts;
320
- this.cells = enumerateCells(opts.space);
321
- if (this.cells.length === 0)
322
- throw new Error("BehaviorExplorer: space has no cells \u2014 every axis needs \u22651 value");
323
- if (opts.costBudgetUsd !== void 0) {
324
- if (typeof opts.costBudgetUsd !== "number" || !Number.isFinite(opts.costBudgetUsd) || opts.costBudgetUsd < 0) {
325
- throw new RangeError(
326
- `BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`
327
- );
328
- }
329
- }
330
- if (!opts.costOf && (opts.costBudgetUsd !== void 0 || opts.ledger || opts.onCost)) {
331
- throw new ValidationError(
332
- "BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf \u2014 the explorer cannot know run cost without it; supply costOf or drop the cost options"
333
- );
334
- }
335
- if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) {
336
- throw new ValidationError(
337
- "BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation"
338
- );
339
- }
340
- if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) {
341
- throw new ValidationError(
342
- "BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling"
343
- );
344
- }
345
- if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
346
- this.cellById = new Map(this.cells.map((c) => [c.id, c]));
347
- this.objective = opts.objective ?? adversarialObjective(0.5);
348
- this.threshold = this.objective.threshold ?? 0.5;
349
- this.floorPerCell = opts.floorPerCell ?? 2;
350
- this.perRoundBudget = Math.max(
351
- this.cells.length * this.floorPerCell,
352
- Math.ceil(opts.budget / 4)
353
- );
354
- this.rngState = (opts.seed ?? 1) >>> 0;
355
- }
356
- opts;
357
- cells;
358
- cellById;
359
- objective;
360
- threshold;
361
- floorPerCell;
362
- perRoundBudget;
363
- /** The single evaluation log — coverage + allocation are projections of it. */
364
- log = [];
365
- /** binId (input × measured coords) → the most interesting entry seen. */
366
- archiveByBin = /* @__PURE__ */ new Map();
367
- _findings = [];
368
- runsUsed = 0;
369
- candidateFindings = 0;
370
- evalErrors = 0;
371
- consecutiveEvalErrors = 0;
372
- stoppedEarly;
373
- rngState;
374
- costLedger;
375
- costPhase = "fuzz.explore";
376
- // mulberry32 deterministic per session.
377
- rng = () => {
378
- this.rngState = this.rngState + 1831565813 | 0;
379
- let t = this.rngState;
380
- t = Math.imul(t ^ t >>> 15, t | 1);
381
- t ^= t + Math.imul(t ^ t >>> 7, t | 61);
382
- return ((t ^ t >>> 14) >>> 0) / 4294967296;
383
- };
384
- binId(cell, descriptor) {
385
- if (!descriptor || Object.keys(descriptor).length === 0) return cell.id;
386
- const measured = Object.keys(descriptor).sort().map((k) => `${k}=${descriptor[k]}`).join("|");
387
- return `${cell.id}|${measured}`;
388
- }
389
- allocate(budget) {
390
- if ((this.opts.allocation ?? "variance") === "uniform") {
391
- const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length));
392
- return this.cells.map((c) => ({ cellId: c.id, count: per }));
393
- }
394
- return varianceBasedCurriculum(
395
- this.log.map((r) => ({
396
- variantId: r.cell.id,
397
- scenarioId: r.scenarioId,
398
- score: r.ev.score,
399
- pass: r.ev.valid && r.ev.score >= 0.5
400
- })),
401
- this.cells.map((c) => ({ variantId: c.id, scenarioId: "*" })),
402
- { budget, floorPerCell: this.floorPerCell }
403
- ).map((a) => ({ cellId: a.variantId, count: a.count }));
404
- }
405
- objectiveContext() {
406
- const entries = [...this.archiveByBin.values()];
407
- return {
408
- archiveScores: entries.map((e) => e.evaluation.score),
409
- archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
410
- };
411
- }
412
- /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
413
- elitesFor(cellId2) {
414
- const out = [];
415
- for (const e of this.archiveByBin.values()) if (e.cell.id === cellId2) out.push(e.scenario);
416
- return out;
417
- }
418
- /** One allocate → propose → evaluate → gate → archive round. */
419
- async step() {
420
- const remaining = this.opts.budget - this.runsUsed;
421
- if (remaining <= 0 || this.opts.signal?.aborted) return { runs: 0, findings: [] };
422
- const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
423
- const newFindings = [];
424
- let runsThisStep = 0;
425
- for (const alloc of allocations) {
426
- if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
427
- break;
428
- const cell = this.cellById.get(alloc.cellId);
429
- if (!cell) continue;
430
- const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed);
431
- if (cap <= 0) continue;
432
- this.opts.onProgress?.({ type: "cell-allocated", cell, count: cap });
433
- const seeds = await this.opts.seedsFor(cell);
434
- const elites = this.elitesFor(cell.id);
435
- const proposed = await this.opts.proposer({
436
- cell,
437
- seeds,
438
- elites,
439
- findings: this._findings,
440
- count: cap,
441
- rng: this.rng
442
- });
443
- const cold = this.log.every((r) => r.cell.id !== cell.id);
444
- const toEval = [...cold ? seeds : [], ...proposed].slice(0, cap);
445
- await pMap(
446
- toEval,
447
- async (scenario) => {
448
- if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted)
449
- return;
450
- try {
451
- const startedAt = performance.now();
452
- const paid = this.costLedger ? await this.costLedger.runPaidCall({
453
- channel: "agent",
454
- phase: this.costPhase,
455
- actor: "fuzz.evaluate",
456
- tags: { target: this.opts.target, cell: cell.id },
457
- signal: this.opts.signal,
458
- maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
459
- execute: () => this.opts.evaluate(scenario, cell),
460
- receipt: (evaluation) => {
461
- const cost = this.opts.costOf(scenario, cell, evaluation);
462
- if (cost === null) {
463
- return {
464
- model: "unattributed",
465
- inputTokens: 0,
466
- outputTokens: 0,
467
- costUnknown: true
468
- };
469
- }
470
- if (!Number.isFinite(cost.usd) || cost.usd < 0) {
471
- throw new RangeError(
472
- `BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) \u2014 return null when cost is unknown, never a fabricated number`
473
- );
474
- }
475
- return {
476
- model: cost.model ?? "unattributed",
477
- inputTokens: 0,
478
- outputTokens: 0,
479
- actualCostUsd: cost.usd
480
- };
481
- }
482
- }) : void 0;
483
- if (paid && !paid.succeeded) {
484
- if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) {
485
- throw paid.error.receiptError;
486
- }
487
- throw paid.error;
488
- }
489
- const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
490
- const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
491
- this.runsUsed++;
492
- runsThisStep++;
493
- this.consecutiveEvalErrors = 0;
494
- const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
495
- if (paid?.receipt.actualCostUsd !== void 0) {
496
- this.opts.onCost?.({ usd: paid.receipt.actualCostUsd, channel: "agent" });
497
- }
498
- const interest = this.objective.interest(ev, this.objectiveContext());
499
- this.log.push({
500
- cell,
501
- ev,
502
- interest,
503
- latencyMs,
504
- ...costUsd !== void 0 ? { costUsd } : {},
505
- scenarioId: this.opts.scenarioId(scenario)
506
- });
507
- this.opts.onProgress?.({ type: "evaluated", cell, scenario, evaluation: ev });
508
- const bin = this.binId(cell, ev.descriptor);
509
- const cur = this.archiveByBin.get(bin);
510
- if (!cur || interest > cur.interest)
511
- this.archiveByBin.set(bin, { binId: bin, cell, scenario, evaluation: ev, interest });
512
- if (interest < this.threshold) return;
513
- this.candidateFindings++;
514
- if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell))
515
- return;
516
- if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell))
517
- return;
518
- const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
519
- const finding = {
520
- id: this.opts.scenarioId(scenario),
521
- cell,
522
- scenario,
523
- minimized,
524
- text: this.opts.scenarioText?.(minimized),
525
- evaluation: ev,
526
- interest,
527
- objective: this.objective.kind
528
- };
529
- this._findings.push(finding);
530
- newFindings.push(finding);
531
- this.opts.onProgress?.({ type: "finding", finding });
532
- } catch (err) {
533
- if (err instanceof RangeError) throw err;
534
- this.evalErrors++;
535
- this.consecutiveEvalErrors++;
536
- const message = err instanceof Error ? err.message : String(err);
537
- this.opts.onProgress?.({
538
- type: "eval-error",
539
- cell,
540
- scenarioId: this.opts.scenarioId(scenario),
541
- message
542
- });
543
- const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
544
- if (this.consecutiveEvalErrors >= limit) {
545
- this.stoppedEarly = {
546
- reason: "eval-errors",
547
- detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
548
- };
549
- }
550
- }
551
- },
552
- this.opts.concurrency ?? 1,
553
- this.opts.signal
554
- );
555
- }
556
- this.opts.onProgress?.({ type: "round", runsUsed: this.runsUsed, budget: this.opts.budget });
557
- return { runs: runsThisStep, findings: newFindings };
558
- }
559
- /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
560
- * or no progress is made. */
561
- async run() {
562
- while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
563
- const { runs } = await this.step();
564
- if (runs === 0 && this.stoppedEarly === void 0) break;
565
- }
566
- return this.capsule();
567
- }
568
- coverage() {
569
- return this.capsule().coverage;
570
- }
571
- findings() {
572
- return [...this._findings].sort((a, b) => b.interest - a.interest);
573
- }
574
- capsule() {
575
- const cost = this.costLedger?.summary({
576
- phase: this.costPhase,
577
- tags: { target: this.opts.target }
578
- });
579
- return buildCapsule({
580
- target: this.opts.target,
581
- objective: this.objective.kind,
582
- cells: this.cells,
583
- log: this.log,
584
- threshold: this.threshold,
585
- archive: [...this.archiveByBin.values()],
586
- findings: this._findings,
587
- candidateFindings: this.candidateFindings,
588
- runsUsed: this.runsUsed,
589
- cost: cost ? {
590
- costUsd: cost.totalCostUsd,
591
- costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
592
- } : void 0,
593
- evalErrors: this.evalErrors,
594
- stoppedEarly: this.stoppedEarly
595
- });
596
- }
353
+ opts;
354
+ cells;
355
+ cellById;
356
+ objective;
357
+ threshold;
358
+ floorPerCell;
359
+ perRoundBudget;
360
+ /** The single evaluation log — coverage + allocation are projections of it. */
361
+ log = [];
362
+ /** binId (input × measured coords) → the most interesting entry seen. */
363
+ archiveByBin = /* @__PURE__ */ new Map();
364
+ _findings = [];
365
+ runsUsed = 0;
366
+ candidateFindings = 0;
367
+ evalErrors = 0;
368
+ consecutiveEvalErrors = 0;
369
+ stoppedEarly;
370
+ rngState;
371
+ costLedger;
372
+ costPhase = "fuzz.explore";
373
+ constructor(opts) {
374
+ this.opts = opts;
375
+ this.cells = enumerateCells(opts.space);
376
+ if (this.cells.length === 0) throw new Error("BehaviorExplorer: space has no cells — every axis needs ≥1 value");
377
+ if (opts.costBudgetUsd !== void 0) {
378
+ if (typeof opts.costBudgetUsd !== "number" || !Number.isFinite(opts.costBudgetUsd) || opts.costBudgetUsd < 0) throw new RangeError(`BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`);
379
+ }
380
+ if (!opts.costOf && (opts.costBudgetUsd !== void 0 || opts.ledger || opts.onCost)) throw new ValidationError("BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf — the explorer cannot know run cost without it; supply costOf or drop the cost options");
381
+ if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) throw new ValidationError("BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation");
382
+ if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) throw new ValidationError("BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling");
383
+ if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
384
+ this.cellById = new Map(this.cells.map((c) => [c.id, c]));
385
+ this.objective = opts.objective ?? adversarialObjective(.5);
386
+ this.threshold = this.objective.threshold ?? .5;
387
+ this.floorPerCell = opts.floorPerCell ?? 2;
388
+ this.perRoundBudget = Math.max(this.cells.length * this.floorPerCell, Math.ceil(opts.budget / 4));
389
+ this.rngState = (opts.seed ?? 1) >>> 0;
390
+ }
391
+ rng = () => {
392
+ this.rngState = this.rngState + 1831565813 | 0;
393
+ let t = this.rngState;
394
+ t = Math.imul(t ^ t >>> 15, t | 1);
395
+ t ^= t + Math.imul(t ^ t >>> 7, t | 61);
396
+ return ((t ^ t >>> 14) >>> 0) / 4294967296;
397
+ };
398
+ binId(cell, descriptor) {
399
+ if (!descriptor || Object.keys(descriptor).length === 0) return cell.id;
400
+ const measured = Object.keys(descriptor).sort().map((k) => `${k}=${descriptor[k]}`).join("|");
401
+ return `${cell.id}|${measured}`;
402
+ }
403
+ allocate(budget) {
404
+ if ((this.opts.allocation ?? "variance") === "uniform") {
405
+ const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length));
406
+ return this.cells.map((c) => ({
407
+ cellId: c.id,
408
+ count: per
409
+ }));
410
+ }
411
+ return varianceBasedCurriculum(this.log.map((r) => ({
412
+ variantId: r.cell.id,
413
+ scenarioId: r.scenarioId,
414
+ score: r.ev.score,
415
+ pass: r.ev.valid && r.ev.score >= .5
416
+ })), this.cells.map((c) => ({
417
+ variantId: c.id,
418
+ scenarioId: "*"
419
+ })), {
420
+ budget,
421
+ floorPerCell: this.floorPerCell
422
+ }).map((a) => ({
423
+ cellId: a.variantId,
424
+ count: a.count
425
+ }));
426
+ }
427
+ objectiveContext() {
428
+ const entries = [...this.archiveByBin.values()];
429
+ return {
430
+ archiveScores: entries.map((e) => e.evaluation.score),
431
+ archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
432
+ };
433
+ }
434
+ /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
435
+ elitesFor(cellId) {
436
+ const out = [];
437
+ for (const e of this.archiveByBin.values()) if (e.cell.id === cellId) out.push(e.scenario);
438
+ return out;
439
+ }
440
+ /** One allocate → propose → evaluate → gate → archive round. */
441
+ async step() {
442
+ const remaining = this.opts.budget - this.runsUsed;
443
+ if (remaining <= 0 || this.opts.signal?.aborted) return {
444
+ runs: 0,
445
+ findings: []
446
+ };
447
+ const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
448
+ const newFindings = [];
449
+ let runsThisStep = 0;
450
+ for (const alloc of allocations) {
451
+ if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) break;
452
+ const cell = this.cellById.get(alloc.cellId);
453
+ if (!cell) continue;
454
+ const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed);
455
+ if (cap <= 0) continue;
456
+ this.opts.onProgress?.({
457
+ type: "cell-allocated",
458
+ cell,
459
+ count: cap
460
+ });
461
+ const seeds = await this.opts.seedsFor(cell);
462
+ const elites = this.elitesFor(cell.id);
463
+ const proposed = await this.opts.proposer({
464
+ cell,
465
+ seeds,
466
+ elites,
467
+ findings: this._findings,
468
+ count: cap,
469
+ rng: this.rng
470
+ });
471
+ await pMap([...this.log.every((r) => r.cell.id !== cell.id) ? seeds : [], ...proposed].slice(0, cap), async (scenario) => {
472
+ if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) return;
473
+ try {
474
+ const startedAt = performance.now();
475
+ const paid = this.costLedger ? await this.costLedger.runPaidCall({
476
+ channel: "agent",
477
+ phase: this.costPhase,
478
+ actor: "fuzz.evaluate",
479
+ tags: {
480
+ target: this.opts.target,
481
+ cell: cell.id
482
+ },
483
+ signal: this.opts.signal,
484
+ maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
485
+ execute: () => this.opts.evaluate(scenario, cell),
486
+ receipt: (evaluation) => {
487
+ const cost = this.opts.costOf(scenario, cell, evaluation);
488
+ if (cost === null) return {
489
+ model: "unattributed",
490
+ inputTokens: 0,
491
+ outputTokens: 0,
492
+ costUnknown: true
493
+ };
494
+ if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new RangeError(`BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) — return null when cost is unknown, never a fabricated number`);
495
+ return {
496
+ model: cost.model ?? "unattributed",
497
+ inputTokens: 0,
498
+ outputTokens: 0,
499
+ actualCostUsd: cost.usd
500
+ };
501
+ }
502
+ }) : void 0;
503
+ if (paid && !paid.succeeded) {
504
+ if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) throw paid.error.receiptError;
505
+ throw paid.error;
506
+ }
507
+ const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
508
+ const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
509
+ this.runsUsed++;
510
+ runsThisStep++;
511
+ this.consecutiveEvalErrors = 0;
512
+ const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
513
+ if (paid?.receipt.actualCostUsd !== void 0) this.opts.onCost?.({
514
+ usd: paid.receipt.actualCostUsd,
515
+ channel: "agent"
516
+ });
517
+ const interest = this.objective.interest(ev, this.objectiveContext());
518
+ this.log.push({
519
+ cell,
520
+ ev,
521
+ interest,
522
+ latencyMs,
523
+ ...costUsd !== void 0 ? { costUsd } : {},
524
+ scenarioId: this.opts.scenarioId(scenario)
525
+ });
526
+ this.opts.onProgress?.({
527
+ type: "evaluated",
528
+ cell,
529
+ scenario,
530
+ evaluation: ev
531
+ });
532
+ const bin = this.binId(cell, ev.descriptor);
533
+ const cur = this.archiveByBin.get(bin);
534
+ if (!cur || interest > cur.interest) this.archiveByBin.set(bin, {
535
+ binId: bin,
536
+ cell,
537
+ scenario,
538
+ evaluation: ev,
539
+ interest
540
+ });
541
+ if (interest < this.threshold) return;
542
+ this.candidateFindings++;
543
+ if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell)) return;
544
+ if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell)) return;
545
+ const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
546
+ const finding = {
547
+ id: this.opts.scenarioId(scenario),
548
+ cell,
549
+ scenario,
550
+ minimized,
551
+ text: this.opts.scenarioText?.(minimized),
552
+ evaluation: ev,
553
+ interest,
554
+ objective: this.objective.kind
555
+ };
556
+ this._findings.push(finding);
557
+ newFindings.push(finding);
558
+ this.opts.onProgress?.({
559
+ type: "finding",
560
+ finding
561
+ });
562
+ } catch (err) {
563
+ if (err instanceof RangeError) throw err;
564
+ this.evalErrors++;
565
+ this.consecutiveEvalErrors++;
566
+ const message = err instanceof Error ? err.message : String(err);
567
+ this.opts.onProgress?.({
568
+ type: "eval-error",
569
+ cell,
570
+ scenarioId: this.opts.scenarioId(scenario),
571
+ message
572
+ });
573
+ const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
574
+ if (this.consecutiveEvalErrors >= limit) this.stoppedEarly = {
575
+ reason: "eval-errors",
576
+ detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
577
+ };
578
+ }
579
+ }, this.opts.concurrency ?? 1, this.opts.signal);
580
+ }
581
+ this.opts.onProgress?.({
582
+ type: "round",
583
+ runsUsed: this.runsUsed,
584
+ budget: this.opts.budget
585
+ });
586
+ return {
587
+ runs: runsThisStep,
588
+ findings: newFindings
589
+ };
590
+ }
591
+ /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
592
+ * or no progress is made. */
593
+ async run() {
594
+ while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
595
+ const { runs } = await this.step();
596
+ if (runs === 0 && this.stoppedEarly === void 0) break;
597
+ }
598
+ return this.capsule();
599
+ }
600
+ coverage() {
601
+ return this.capsule().coverage;
602
+ }
603
+ findings() {
604
+ return [...this._findings].sort((a, b) => b.interest - a.interest);
605
+ }
606
+ capsule() {
607
+ const cost = this.costLedger?.summary({
608
+ phase: this.costPhase,
609
+ tags: { target: this.opts.target }
610
+ });
611
+ return buildCapsule({
612
+ target: this.opts.target,
613
+ objective: this.objective.kind,
614
+ cells: this.cells,
615
+ log: this.log,
616
+ threshold: this.threshold,
617
+ archive: [...this.archiveByBin.values()],
618
+ findings: this._findings,
619
+ candidateFindings: this.candidateFindings,
620
+ runsUsed: this.runsUsed,
621
+ cost: cost ? {
622
+ costUsd: cost.totalCostUsd,
623
+ costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
624
+ } : void 0,
625
+ evalErrors: this.evalErrors,
626
+ stoppedEarly: this.stoppedEarly
627
+ });
628
+ }
597
629
  };
598
-
599
- // src/fuzz/fuzz-agent.ts
630
+ //#endregion
631
+ //#region src/fuzz/fuzz-agent.ts
632
+ /**
633
+ * `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
634
+ *
635
+ * One call: explore the space to budget with the adversarial objective and
636
+ * return the capsule. For agent-driven, incremental, or multi-objective use,
637
+ * construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
638
+ */
600
639
  async function fuzzAgent(opts) {
601
- const { failureThreshold, ...rest } = opts;
602
- const explorer = new BehaviorExplorer({
603
- ...rest,
604
- objective: adversarialObjective(failureThreshold ?? 0.5)
605
- });
606
- return { capsule: await explorer.run() };
640
+ const { failureThreshold, ...rest } = opts;
641
+ return { capsule: await new BehaviorExplorer({
642
+ ...rest,
643
+ objective: adversarialObjective(failureThreshold ?? .5)
644
+ }).run() };
607
645
  }
608
-
609
- // src/fuzz/gates.ts
646
+ //#endregion
647
+ //#region src/fuzz/gates.ts
648
+ /** Combine gate sets; a candidate must pass every gate in every set. */
610
649
  function composeGates(...sets) {
611
- const present = sets.filter((s) => s != null);
612
- return {
613
- isValid: async (scenario, ev, cell) => {
614
- for (const g of present) {
615
- if (g.isValid && !await g.isValid(scenario, ev, cell)) return false;
616
- }
617
- return true;
618
- },
619
- isUncontaminated: async (scenario, ev, cell) => {
620
- for (const g of present) {
621
- if (g.isUncontaminated && !await g.isUncontaminated(scenario, ev, cell)) return false;
622
- }
623
- return true;
624
- }
625
- };
650
+ const present = sets.filter((s) => s != null);
651
+ return {
652
+ isValid: async (scenario, ev, cell) => {
653
+ for (const g of present) if (g.isValid && !await g.isValid(scenario, ev, cell)) return false;
654
+ return true;
655
+ },
656
+ isUncontaminated: async (scenario, ev, cell) => {
657
+ for (const g of present) if (g.isUncontaminated && !await g.isUncontaminated(scenario, ev, cell)) return false;
658
+ return true;
659
+ }
660
+ };
626
661
  }
662
+ /**
663
+ * Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the
664
+ * flagged scenario; keep the finding only when the rephrase ALSO scores below the
665
+ * threshold. A finding that flips under a cosmetic rewrite was keyed to surface
666
+ * form, not the task — a false signal we must not report. Costs one extra
667
+ * evaluation per candidate (candidates are rare, so cheap).
668
+ */
627
669
  function perturbationStabilityGate(opts) {
628
- const threshold = opts.failureThreshold ?? 0.5;
629
- return {
630
- isUncontaminated: async (scenario, _ev, cell) => {
631
- const rephrased = opts.perturb(scenario);
632
- if (rephrased == null) return true;
633
- const re = await opts.evaluate(rephrased, cell);
634
- return re.score < threshold;
635
- }
636
- };
670
+ const threshold = opts.failureThreshold ?? .5;
671
+ return { isUncontaminated: async (scenario, _ev, cell) => {
672
+ const rephrased = opts.perturb(scenario);
673
+ if (rephrased == null) return true;
674
+ return (await opts.evaluate(rephrased, cell)).score < threshold;
675
+ } };
637
676
  }
677
+ /**
678
+ * Severity-floor gate. Reject borderline candidates whose score sits in a band
679
+ * just under the threshold — judge noise, not a real defect.
680
+ */
638
681
  function severityFloorGate(opts) {
639
- const threshold = opts.failureThreshold ?? 0.5;
640
- const margin = opts.margin ?? 0.1;
641
- return {
642
- isValid: (_scenario, ev) => ev.score <= threshold - margin
643
- };
682
+ const threshold = opts.failureThreshold ?? .5;
683
+ const margin = opts.margin ?? .1;
684
+ return { isValid: (_scenario, ev) => ev.score <= threshold - margin };
644
685
  }
645
-
646
- // src/fuzz/tools.ts
647
- var NO_ARGS = { type: "object", properties: {}, additionalProperties: false };
686
+ //#endregion
687
+ //#region src/fuzz/tools.ts
688
+ /**
689
+ * Agent-drivable surface over a live exploration session.
690
+ *
691
+ * Framework-neutral tool defs ({name, description, parameters: JSON Schema,
692
+ * handler}) so the on-demand agent — not a batch script — drives the search:
693
+ * step it, read coverage, inspect findings, render the capsule. Transport
694
+ * encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
695
+ */
696
+ const NO_ARGS = {
697
+ type: "object",
698
+ properties: {},
699
+ additionalProperties: false
700
+ };
648
701
  function makeExploreTools(explorer) {
649
- return [
650
- {
651
- name: "explore_step",
652
- description: "Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.",
653
- parameters: NO_ARGS,
654
- handler: async () => {
655
- const { runs, findings } = await explorer.step();
656
- return { runs, newFindings: findings.length, findings };
657
- }
658
- },
659
- {
660
- name: "explore_coverage",
661
- description: "Read the live coverage map: per planned cell \u2014 runs, robustness (null = uncovered), finding rate, per-dimension means.",
662
- parameters: NO_ARGS,
663
- handler: async () => explorer.coverage()
664
- },
665
- {
666
- name: "explore_findings",
667
- description: "List gate-verified findings so far, sorted by descending interest.",
668
- parameters: {
669
- type: "object",
670
- properties: { limit: { type: "number", description: "max findings to return" } },
671
- additionalProperties: false
672
- },
673
- handler: async (args) => {
674
- const limit = args?.limit;
675
- const all = explorer.findings();
676
- return typeof limit === "number" ? all.slice(0, limit) : all;
677
- }
678
- },
679
- {
680
- name: "explore_capsule",
681
- description: 'Build the capsule artifact from the current session state. format "data" returns the structured CapsuleData; "html" returns the standalone page.',
682
- parameters: {
683
- type: "object",
684
- properties: {
685
- format: { type: "string", enum: ["data", "html"] },
686
- generatedAt: { type: "string", description: "ISO timestamp to stamp into the page" }
687
- },
688
- additionalProperties: false
689
- },
690
- handler: async (args) => {
691
- const a = args ?? {};
692
- const capsule = explorer.capsule();
693
- if (a.format === "html") return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt });
694
- return capsule;
695
- }
696
- }
697
- ];
702
+ return [
703
+ {
704
+ name: "explore_step",
705
+ description: "Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.",
706
+ parameters: NO_ARGS,
707
+ handler: async () => {
708
+ const { runs, findings } = await explorer.step();
709
+ return {
710
+ runs,
711
+ newFindings: findings.length,
712
+ findings
713
+ };
714
+ }
715
+ },
716
+ {
717
+ name: "explore_coverage",
718
+ description: "Read the live coverage map: per planned cell — runs, robustness (null = uncovered), finding rate, per-dimension means.",
719
+ parameters: NO_ARGS,
720
+ handler: async () => explorer.coverage()
721
+ },
722
+ {
723
+ name: "explore_findings",
724
+ description: "List gate-verified findings so far, sorted by descending interest.",
725
+ parameters: {
726
+ type: "object",
727
+ properties: { limit: {
728
+ type: "number",
729
+ description: "max findings to return"
730
+ } },
731
+ additionalProperties: false
732
+ },
733
+ handler: async (args) => {
734
+ const limit = args?.limit;
735
+ const all = explorer.findings();
736
+ return typeof limit === "number" ? all.slice(0, limit) : all;
737
+ }
738
+ },
739
+ {
740
+ name: "explore_capsule",
741
+ description: "Build the capsule artifact from the current session state. format \"data\" returns the structured CapsuleData; \"html\" returns the standalone page.",
742
+ parameters: {
743
+ type: "object",
744
+ properties: {
745
+ format: {
746
+ type: "string",
747
+ enum: ["data", "html"]
748
+ },
749
+ generatedAt: {
750
+ type: "string",
751
+ description: "ISO timestamp to stamp into the page"
752
+ }
753
+ },
754
+ additionalProperties: false
755
+ },
756
+ handler: async (args) => {
757
+ const a = args ?? {};
758
+ const capsule = explorer.capsule();
759
+ if (a.format === "html") return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt });
760
+ return capsule;
761
+ }
762
+ }
763
+ ];
698
764
  }
699
- export {
700
- BehaviorExplorer,
701
- adversarialObjective,
702
- buildCapsule,
703
- buildCoverage,
704
- cellId,
705
- composeGates,
706
- enumerateCells,
707
- fuzzAgent,
708
- makeExploreTools,
709
- mutationProposer,
710
- noveltyObjective,
711
- perturbationStabilityGate,
712
- renderCapsuleHtml,
713
- severityFloorGate
714
- };
765
+ //#endregion
766
+ export { BehaviorExplorer, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };
767
+
715
768
  //# sourceMappingURL=fuzz.js.map