@tangle-network/agent-eval 0.129.0 → 0.130.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +2 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +81 -2872
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -360
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1188
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1709
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -891
- package/dist/benchmarks/index.js +2 -60
- package/dist/benchmarks-BJgDGkAD.js +754 -0
- package/dist/benchmarks-BJgDGkAD.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6381
- package/dist/campaign/index.js +3 -213
- package/dist/campaign-aKJt6emI.js +3886 -0
- package/dist/campaign-aKJt6emI.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -175
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5565
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1938
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -33
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -618
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CD_WZ_Xr.d.ts +2250 -0
- package/dist/index-CD_WZ_Xr.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index-Em67JBjs.d.ts +335 -0
- package/dist/index-Em67JBjs.d.ts.map +1 -0
- package/dist/index.d.ts +3755 -15555
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11182 -11216
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -480
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1312
- package/dist/reporting.js +6 -51
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +760 -4010
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2325 -1958
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -2087
- package/dist/rollout/index.js +8 -168
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-CUmHkGbI.js +7718 -0
- package/dist/skillopt-optimization-method-CUmHkGbI.js.map +1 -0
- package/dist/skillopt-optimization-method-CWKVTnks.d.ts +1740 -0
- package/dist/skillopt-optimization-method-CWKVTnks.d.ts.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -959
- package/dist/supervisor-run/index.js +2 -65
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -252
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1173
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/campaign-proposers.md +1 -0
- package/package.json +17 -9
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2QU3YOPR.js +0 -7374
- package/dist/chunk-2QU3YOPR.js.map +0 -1
- package/dist/chunk-3OCR4R5I.js +0 -728
- package/dist/chunk-3OCR4R5I.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-56TAVBOK.js +0 -698
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7FO3TNPI.js +0 -232
- package/dist/chunk-7FO3TNPI.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BSO5JDQH.js +0 -2335
- package/dist/chunk-BSO5JDQH.js.map +0 -1
- package/dist/chunk-C6LXANRU.js +0 -1550
- package/dist/chunk-C6LXANRU.js.map +0 -1
- package/dist/chunk-DODXQREJ.js +0 -752
- package/dist/chunk-DODXQREJ.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-E7QXT7SX.js +0 -183
- package/dist/chunk-E7QXT7SX.js.map +0 -1
- package/dist/chunk-EG66UGL4.js +0 -341
- package/dist/chunk-EG66UGL4.js.map +0 -1
- package/dist/chunk-FXTVJPYD.js +0 -576
- package/dist/chunk-FXTVJPYD.js.map +0 -1
- package/dist/chunk-G7MGMCZD.js +0 -153
- package/dist/chunk-G7MGMCZD.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-H23X7XKK.js +0 -181
- package/dist/chunk-H23X7XKK.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-HPWUNB47.js +0 -289
- package/dist/chunk-HPWUNB47.js.map +0 -1
- package/dist/chunk-IYCLP2N2.js +0 -766
- package/dist/chunk-IYCLP2N2.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-JQSF5DQT.js +0 -701
- package/dist/chunk-JQSF5DQT.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-M4YBQKIJ.js +0 -1040
- package/dist/chunk-M4YBQKIJ.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NY44NC4A.js +0 -1056
- package/dist/chunk-NY44NC4A.js.map +0 -1
- package/dist/chunk-OIUOT4QD.js +0 -44
- package/dist/chunk-OIUOT4QD.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-OWN5NPMC.js +0 -152
- package/dist/chunk-OWN5NPMC.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PC5DOSM7.js +0 -579
- package/dist/chunk-PC5DOSM7.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-QB6BDBP2.js +0 -4464
- package/dist/chunk-QB6BDBP2.js.map +0 -1
- package/dist/chunk-RXHCETDZ.js +0 -536
- package/dist/chunk-RXHCETDZ.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-SFLLL76A.js +0 -669
- package/dist/chunk-SFLLL76A.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-T6RLYGAD.js +0 -158
- package/dist/chunk-T6RLYGAD.js.map +0 -1
- package/dist/chunk-TJVT4QFF.js +0 -911
- package/dist/chunk-TJVT4QFF.js.map +0 -1
- package/dist/chunk-TQ7LNKZ3.js +0 -136
- package/dist/chunk-TQ7LNKZ3.js.map +0 -1
- package/dist/chunk-U4L7JRPZ.js +0 -1706
- package/dist/chunk-U4L7JRPZ.js.map +0 -1
- package/dist/chunk-U4PHLT2N.js +0 -419
- package/dist/chunk-U4PHLT2N.js.map +0 -1
- package/dist/chunk-VCZ5FQYW.js +0 -928
- package/dist/chunk-VCZ5FQYW.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WVATSFCP.js +0 -1553
- package/dist/chunk-WVATSFCP.js.map +0 -1
- package/dist/chunk-X4YIBDER.js +0 -1662
- package/dist/chunk-X4YIBDER.js.map +0 -1
- package/dist/chunk-YQN4ICPP.js +0 -355
- package/dist/chunk-YQN4ICPP.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZHTZ4EYI.js +0 -1212
- package/dist/chunk-ZHTZ4EYI.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-OJJ7CZF4.js +0 -18
- package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/fuzz.js
CHANGED
|
@@ -1,199 +1,207 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
} from "./
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
CostReceiptCaptureError
|
|
7
|
-
} from "./chunk-VCZ5FQYW.js";
|
|
8
|
-
import "./chunk-VI2UW6B6.js";
|
|
9
|
-
import "./chunk-OIUOT4QD.js";
|
|
10
|
-
import {
|
|
11
|
-
ValidationError
|
|
12
|
-
} from "./chunk-ONWEPEDO.js";
|
|
13
|
-
import "./chunk-PZ5AY32C.js";
|
|
14
|
-
|
|
15
|
-
// src/fuzz/cube.ts
|
|
1
|
+
import { s as ValidationError } from "./errors-8YnH8WlF.js";
|
|
2
|
+
import { i as CostLedger, o as CostReceiptCaptureError } from "./cost-ledger-DIgQUFZZ.js";
|
|
3
|
+
import { r as varianceBasedCurriculum } from "./active-curriculum-C4mk67HP.js";
|
|
4
|
+
//#region src/fuzz/cube.ts
|
|
5
|
+
/** Enumerate every input cell (cartesian product of the axes), in stable order. */
|
|
16
6
|
function enumerateCells(space) {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
7
|
+
if (space.axes.length === 0) return [];
|
|
8
|
+
let partials = [{}];
|
|
9
|
+
for (const axis of space.axes) {
|
|
10
|
+
const next = [];
|
|
11
|
+
for (const partial of partials) for (const value of axis.values) next.push({
|
|
12
|
+
...partial,
|
|
13
|
+
[axis.name]: value
|
|
14
|
+
});
|
|
15
|
+
partials = next;
|
|
16
|
+
}
|
|
17
|
+
return partials.map((coords) => ({
|
|
18
|
+
id: cellId(space, coords),
|
|
19
|
+
coords
|
|
20
|
+
}));
|
|
27
21
|
}
|
|
22
|
+
/** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */
|
|
28
23
|
function cellId(space, coords) {
|
|
29
|
-
|
|
24
|
+
return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
|
|
30
25
|
}
|
|
26
|
+
/** Nearest-rank percentile on a pre-sorted ascending sample. */
|
|
31
27
|
function percentile(sorted, p) {
|
|
32
|
-
|
|
33
|
-
return sorted[idx];
|
|
28
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
|
|
34
29
|
}
|
|
30
|
+
/** Summarize a sample. Throws on an empty sample — callers represent "no data"
|
|
31
|
+
* as `null`, never as a zeroed distribution. */
|
|
35
32
|
function distribution(values) {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
n: sorted.length
|
|
47
|
-
};
|
|
33
|
+
if (values.length === 0) throw new Error("distribution: empty sample — represent missing data as null, not zeros");
|
|
34
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
35
|
+
return {
|
|
36
|
+
mean: sorted.reduce((a, b) => a + b, 0) / sorted.length,
|
|
37
|
+
median: percentile(sorted, .5),
|
|
38
|
+
p90: percentile(sorted, .9),
|
|
39
|
+
min: sorted[0],
|
|
40
|
+
max: sorted[sorted.length - 1],
|
|
41
|
+
n: sorted.length
|
|
42
|
+
};
|
|
48
43
|
}
|
|
44
|
+
/**
|
|
45
|
+
* Project the evaluation log into the per-input-cell coverage map. A cell with
|
|
46
|
+
* no evaluations reports `score: null` (honestly uncovered), never zeros.
|
|
47
|
+
*/
|
|
49
48
|
function buildCoverage(cells, log, threshold) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
49
|
+
const byCell = /* @__PURE__ */ new Map();
|
|
50
|
+
for (const r of log) {
|
|
51
|
+
const arr = byCell.get(r.cell.id) ?? [];
|
|
52
|
+
arr.push(r);
|
|
53
|
+
byCell.set(r.cell.id, arr);
|
|
54
|
+
}
|
|
55
|
+
return cells.map((cell) => {
|
|
56
|
+
const recs = byCell.get(cell.id) ?? [];
|
|
57
|
+
const runs = recs.length;
|
|
58
|
+
if (runs === 0) return {
|
|
59
|
+
cell,
|
|
60
|
+
runs: 0,
|
|
61
|
+
score: null,
|
|
62
|
+
findingRate: 0,
|
|
63
|
+
dimensions: {},
|
|
64
|
+
latencyMs: null
|
|
65
|
+
};
|
|
66
|
+
const score = distribution(recs.map((r) => r.ev.score));
|
|
67
|
+
const latencyMs = distribution(recs.map((r) => r.latencyMs));
|
|
68
|
+
const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
|
|
69
|
+
const dimSamples = {};
|
|
70
|
+
for (const r of recs) for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
|
|
71
|
+
const samples = dimSamples[k] ?? [];
|
|
72
|
+
samples.push(v);
|
|
73
|
+
dimSamples[k] = samples;
|
|
74
|
+
}
|
|
75
|
+
const dimensions = {};
|
|
76
|
+
for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
|
|
77
|
+
const tracked = recs.filter((r) => r.costUsd !== void 0);
|
|
78
|
+
const known = tracked.filter((r) => r.costUsd !== null);
|
|
79
|
+
return {
|
|
80
|
+
cell,
|
|
81
|
+
runs,
|
|
82
|
+
score,
|
|
83
|
+
findingRate,
|
|
84
|
+
dimensions,
|
|
85
|
+
latencyMs,
|
|
86
|
+
...tracked.length > 0 ? {
|
|
87
|
+
costUsd: known.reduce((a, r) => a + r.costUsd, 0),
|
|
88
|
+
...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
|
|
89
|
+
} : {}
|
|
90
|
+
};
|
|
91
|
+
});
|
|
82
92
|
}
|
|
83
|
-
|
|
84
|
-
|
|
93
|
+
//#endregion
|
|
94
|
+
//#region src/fuzz/capsule.ts
|
|
85
95
|
function buildCapsule(input) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
96
|
+
const coverage = buildCoverage(input.cells, input.log, input.threshold);
|
|
97
|
+
const covered = coverage.filter((c) => c.runs > 0);
|
|
98
|
+
const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
|
|
99
|
+
const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
|
|
100
|
+
const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
|
|
101
|
+
return {
|
|
102
|
+
target: input.target,
|
|
103
|
+
objective: input.objective,
|
|
104
|
+
coverage,
|
|
105
|
+
findings: [...input.findings].sort((a, b) => b.interest - a.interest),
|
|
106
|
+
archive: [...input.archive].sort((a, b) => b.interest - a.interest),
|
|
107
|
+
stats: {
|
|
108
|
+
totalRuns: input.runsUsed,
|
|
109
|
+
cellsTotal: input.cells.length,
|
|
110
|
+
cellsCovered: covered.length,
|
|
111
|
+
behaviorBinsObserved,
|
|
112
|
+
candidateFindings: input.candidateFindings,
|
|
113
|
+
verifiedFindings: input.findings.length,
|
|
114
|
+
robustness,
|
|
115
|
+
latencyMs,
|
|
116
|
+
...input.cost ? {
|
|
117
|
+
costUsd: input.cost.costUsd,
|
|
118
|
+
costUnknownRuns: input.cost.costUnknownRuns
|
|
119
|
+
} : {},
|
|
120
|
+
evalErrors: input.evalErrors,
|
|
121
|
+
...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
|
|
122
|
+
}
|
|
123
|
+
};
|
|
111
124
|
}
|
|
112
125
|
function esc(s) {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
126
|
+
return s.replace(/[&<>"]/g, (c) => ({
|
|
127
|
+
"&": "&",
|
|
128
|
+
"<": "<",
|
|
129
|
+
">": ">",
|
|
130
|
+
"\"": """
|
|
131
|
+
})[c]);
|
|
117
132
|
}
|
|
133
|
+
/** red (0) → amber (.5) → green (1); uncovered cells render gray. */
|
|
118
134
|
function robustnessColor(r) {
|
|
119
|
-
|
|
120
|
-
|
|
135
|
+
if (r == null) return "#2a2a2e";
|
|
136
|
+
return `hsl(${Math.round(r * 120)} 70% 42%)`;
|
|
121
137
|
}
|
|
122
138
|
function pct(x) {
|
|
123
|
-
|
|
139
|
+
return `${Math.round(x * 100)}%`;
|
|
124
140
|
}
|
|
125
141
|
function deriveAxes(coverage) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
142
|
+
const order = [];
|
|
143
|
+
const seen = /* @__PURE__ */ new Map();
|
|
144
|
+
for (const c of coverage) for (const [k, v] of Object.entries(c.cell.coords)) {
|
|
145
|
+
if (!seen.has(k)) {
|
|
146
|
+
seen.set(k, /* @__PURE__ */ new Set());
|
|
147
|
+
order.push(k);
|
|
148
|
+
}
|
|
149
|
+
seen.get(k)?.add(v);
|
|
150
|
+
}
|
|
151
|
+
return order.map((name) => ({
|
|
152
|
+
name,
|
|
153
|
+
values: [...seen.get(name) ?? []]
|
|
154
|
+
}));
|
|
138
155
|
}
|
|
156
|
+
/** The weakest dimension chip for a cell, e.g. `safety 32%` — shown when scores exist. */
|
|
139
157
|
function weakestDim(c) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
|
|
158
|
+
const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
|
|
159
|
+
if (entries.length === 0) return "";
|
|
160
|
+
const w = entries.sort((a, b) => a[1] - b[1])[0];
|
|
161
|
+
if (!w) return "";
|
|
162
|
+
return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
|
|
146
163
|
}
|
|
147
164
|
function heatmapHtml(coverage) {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
return `<div class="grid">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(" \xB7 "))).join("")}</div>`;
|
|
165
|
+
const axes = deriveAxes(coverage);
|
|
166
|
+
const byId = new Map(coverage.map((c) => [c.cell.id, c]));
|
|
167
|
+
const tile = (c, label) => {
|
|
168
|
+
const r = c?.score?.mean ?? null;
|
|
169
|
+
const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) · ${c.runs} runs · ${pct(c.findingRate)} flagged${c.latencyMs ? ` · ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` · $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
|
|
170
|
+
return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : "—"}</span>${c ? weakestDim(c) : ""}</div>`;
|
|
171
|
+
};
|
|
172
|
+
const rowAxis = axes[0];
|
|
173
|
+
const colAxis = axes[1];
|
|
174
|
+
if (axes.length === 2 && rowAxis && colAxis) {
|
|
175
|
+
const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join("")}</tr>`;
|
|
176
|
+
const rows = rowAxis.values.map((rv) => {
|
|
177
|
+
const cells = colAxis.values.map((cv) => {
|
|
178
|
+
const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`;
|
|
179
|
+
return `<td>${tile(byId.get(id), "")}</td>`;
|
|
180
|
+
}).join("");
|
|
181
|
+
return `<tr><th class="rh">${esc(rv)}</th>${cells}</tr>`;
|
|
182
|
+
}).join("");
|
|
183
|
+
return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> · cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
|
|
184
|
+
}
|
|
185
|
+
return `<div class="grid">${[...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2)).map((c) => tile(c, Object.values(c.cell.coords).join(" · "))).join("")}</div>`;
|
|
170
186
|
}
|
|
171
187
|
function findingsHtml(findings, limit) {
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
}).join("");
|
|
188
|
+
if (findings.length === 0) return `<p class="none">No verified findings — the target held across every covered cell.</p>`;
|
|
189
|
+
return findings.slice(0, limit).map((f) => {
|
|
190
|
+
const coords = Object.entries(f.cell.coords).map(([k, v]) => `${k}:${v}`).join(" · ");
|
|
191
|
+
const labels = (f.evaluation.labels ?? []).map((l) => `<span class="fclass">${esc(l)}</span>`).join("");
|
|
192
|
+
const dims = Object.entries(f.evaluation.scores ?? {}).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([k, v]) => `<span class="dim">${esc(k)} ${pct(v)}</span>`).join("");
|
|
193
|
+
return `<div class="fail"><div class="fmeta"><span class="fcell">${esc(coords)}</span>${labels}<span class="fsev">interest ${pct(f.interest)}</span></div><div class="sevbar"><div class="sevfill" style="width:${pct(f.interest)}"></div></div><div class="ftext">${esc(f.text ?? "(scenario text not captured)")}</div>${dims ? `<div class="fdims">${dims}</div>` : ""}</div>`;
|
|
194
|
+
}).join("");
|
|
180
195
|
}
|
|
196
|
+
/** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
|
|
181
197
|
function renderCapsuleHtml(capsule, opts = {}) {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
const cost = s.costUsd !== void 0 ? kpi(
|
|
190
|
-
s.costUnknownRuns ? `cost \xB7 ${s.costUnknownRuns} runs unpriced` : "cost",
|
|
191
|
-
`$${s.costUsd.toFixed(2)}`,
|
|
192
|
-
s.costUnknownRuns ? "#e5c07b" : "#e6e6e6"
|
|
193
|
-
) : "";
|
|
194
|
-
const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
|
|
195
|
-
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
196
|
-
<title>${esc(capsule.objective)} capsule \u2014 ${esc(capsule.target)}</title>
|
|
198
|
+
const s = capsule.stats;
|
|
199
|
+
const kpi = (label, value, accent = "#e6e6e6") => `<div class="kpi"><div class="kv" style="color:${accent}">${esc(value)}</div><div class="kl">${esc(label)}</div></div>`;
|
|
200
|
+
const lift = capsule.lift ? kpi("hardening lift", `${capsule.lift.before.toFixed(2)} → ${capsule.lift.after.toFixed(2)}`, "#5ad17a") : "";
|
|
201
|
+
const cost = s.costUsd !== void 0 ? kpi(s.costUnknownRuns ? `cost · ${s.costUnknownRuns} runs unpriced` : "cost", `$${s.costUsd.toFixed(2)}`, s.costUnknownRuns ? "#e5c07b" : "#e6e6e6") : "";
|
|
202
|
+
const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
|
|
203
|
+
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
204
|
+
<title>${esc(capsule.objective)} capsule — ${esc(capsule.target)}</title>
|
|
197
205
|
<style>
|
|
198
206
|
:root{color-scheme:dark}
|
|
199
207
|
*{box-sizing:border-box}
|
|
@@ -229,11 +237,11 @@ table.heat th.rh{text-align:right}
|
|
|
229
237
|
.none{color:#5ad17a}
|
|
230
238
|
.foot{color:#5a5a63;font-size:11px;margin-top:36px}
|
|
231
239
|
</style></head><body><div class="wrap">
|
|
232
|
-
<h1>${esc(capsule.objective)} exploration
|
|
233
|
-
<div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? `
|
|
240
|
+
<h1>${esc(capsule.objective)} exploration · ${esc(capsule.target)}</h1>
|
|
241
|
+
<div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` · ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` · ${esc(stamp)}` : ""}</div>
|
|
234
242
|
<div class="kpis">
|
|
235
|
-
${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean <
|
|
236
|
-
${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}
|
|
243
|
+
${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < .6 ? "#e58a96" : "#5ad17a") : ""}
|
|
244
|
+
${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}–${pct(s.robustness.max)}`) : ""}
|
|
237
245
|
${s.latencyMs ? kpi("median latency", `${(s.latencyMs.median / 1e3).toFixed(1)}s`, s.latencyMs.p90 > 4 * s.latencyMs.median ? "#e5b566" : "#e6e6e6") : ""}
|
|
238
246
|
${kpi("verified findings", String(s.verifiedFindings), s.verifiedFindings > 0 ? "#e58a96" : "#5ad17a")}
|
|
239
247
|
${kpi("cells covered", `${s.cellsCovered}/${s.cellsTotal}`)}
|
|
@@ -242,475 +250,519 @@ ${cost}
|
|
|
242
250
|
${s.evalErrors > 0 ? kpi("eval errors", String(s.evalErrors), "#e5b566") : ""}
|
|
243
251
|
${lift}
|
|
244
252
|
</div>
|
|
245
|
-
${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)}
|
|
253
|
+
${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} — coverage below reflects the completed portion only</div>` : ""}
|
|
246
254
|
<h2>Coverage map</h2>
|
|
247
255
|
${heatmapHtml(capsule.coverage)}
|
|
248
|
-
<h2>Verified findings${s.candidateFindings > s.verifiedFindings ? `
|
|
256
|
+
<h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` · ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
|
|
249
257
|
${findingsHtml(capsule.findings, opts.maxFindings ?? 8)}
|
|
250
258
|
<div class="foot">Findings are gate-verified: each is a fair, answerable task that reproduces under a meaning-preserving rephrase, minimized to the smallest trigger.</div>
|
|
251
259
|
</div></body></html>`;
|
|
252
260
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
261
|
+
//#endregion
|
|
262
|
+
//#region src/fuzz/policies.ts
|
|
263
|
+
const clamp01 = (x) => Math.max(0, Math.min(1, x));
|
|
264
|
+
/**
|
|
265
|
+
* Perturbation-based search: apply the cell's mutation operators to the current
|
|
266
|
+
* elites + seeds, deduping by id. Elites first — mutating the most interesting
|
|
267
|
+
* scenario found so far is what makes the search deepen across rounds.
|
|
268
|
+
*/
|
|
256
269
|
function mutationProposer(opts) {
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
270
|
+
return async (ctx) => {
|
|
271
|
+
const mutations = opts.mutationsFor(ctx.cell);
|
|
272
|
+
const parents = [...ctx.elites, ...ctx.seeds];
|
|
273
|
+
const seen = new Set(parents.map(opts.scenarioId));
|
|
274
|
+
const out = [];
|
|
275
|
+
for (const parent of parents) {
|
|
276
|
+
if (out.length >= ctx.count) break;
|
|
277
|
+
for (const m of mutations) {
|
|
278
|
+
const children = await m.mutate(parent, ctx.rng);
|
|
279
|
+
for (const child of children) {
|
|
280
|
+
const id = opts.scenarioId(child);
|
|
281
|
+
if (seen.has(id)) continue;
|
|
282
|
+
seen.add(id);
|
|
283
|
+
out.push(child);
|
|
284
|
+
if (out.length >= ctx.count) break;
|
|
285
|
+
}
|
|
286
|
+
if (out.length >= ctx.count) break;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return out;
|
|
290
|
+
};
|
|
278
291
|
}
|
|
279
|
-
|
|
280
|
-
|
|
292
|
+
/** Adversarial: a low headline score is interesting — find where the agent fails. */
|
|
293
|
+
function adversarialObjective(threshold = .5) {
|
|
294
|
+
return {
|
|
295
|
+
kind: "adversarial",
|
|
296
|
+
threshold,
|
|
297
|
+
interest: (ev) => clamp01(1 - ev.score)
|
|
298
|
+
};
|
|
281
299
|
}
|
|
282
300
|
function hamming(a, b) {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
301
|
+
if (!a || !b) return 1;
|
|
302
|
+
const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
|
|
303
|
+
if (keys.size === 0) return 0;
|
|
304
|
+
let diff = 0;
|
|
305
|
+
for (const k of keys) if (a[k] !== b[k]) diff++;
|
|
306
|
+
return diff / keys.size;
|
|
289
307
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
308
|
+
/**
|
|
309
|
+
* Novelty: interesting when far from the archive in score AND measured behavior
|
|
310
|
+
* descriptor — quality-diversity's diversity pressure; drives corpus growth
|
|
311
|
+
* rather than re-finding the same hole.
|
|
312
|
+
*/
|
|
313
|
+
function noveltyObjective(threshold = .3) {
|
|
314
|
+
return {
|
|
315
|
+
kind: "novelty",
|
|
316
|
+
threshold,
|
|
317
|
+
interest: (ev, ctx) => {
|
|
318
|
+
const scoreNovelty = ctx.archiveScores.length === 0 ? 1 : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)));
|
|
319
|
+
const descNovelty = ctx.archiveDescriptors.length === 0 ? 1 : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)));
|
|
320
|
+
return clamp01(.5 * scoreNovelty + .5 * descNovelty);
|
|
321
|
+
}
|
|
322
|
+
};
|
|
300
323
|
}
|
|
301
|
-
|
|
302
|
-
|
|
324
|
+
//#endregion
|
|
325
|
+
//#region src/fuzz/explorer.ts
|
|
326
|
+
/**
|
|
327
|
+
* The exploration engine — a stateful session over a behavior space.
|
|
328
|
+
*
|
|
329
|
+
* Each `step()`: allocate budget across INPUT cells (floor first, then variance
|
|
330
|
+
* steering toward the least-certain cells), propose candidates (the proposer
|
|
331
|
+
* reads current elites + findings, so the search deepens generationally),
|
|
332
|
+
* evaluate with bounded concurrency, archive the most interesting scenario per
|
|
333
|
+
* input×measured bin, and admit notable candidates that pass the validity gates.
|
|
334
|
+
* `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
|
|
335
|
+
* the surface `makeExploreTools` exposes so an agent can drive the session.
|
|
336
|
+
*
|
|
337
|
+
* One evaluation log (`EvalRecord[]`) is the source of truth; allocation
|
|
338
|
+
* observations and coverage are projections of it.
|
|
339
|
+
*/
|
|
303
340
|
async function pMap(items, fn, concurrency, signal) {
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
}
|
|
315
|
-
);
|
|
316
|
-
await Promise.all(workers);
|
|
341
|
+
let i = 0;
|
|
342
|
+
const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
|
|
343
|
+
while (i < items.length) {
|
|
344
|
+
if (signal?.aborted) return;
|
|
345
|
+
const item = items[i++];
|
|
346
|
+
if (item === void 0) return;
|
|
347
|
+
await fn(item);
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
await Promise.all(workers);
|
|
317
351
|
}
|
|
318
352
|
var BehaviorExplorer = class {
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
stoppedEarly: this.stoppedEarly
|
|
596
|
-
});
|
|
597
|
-
}
|
|
353
|
+
opts;
|
|
354
|
+
cells;
|
|
355
|
+
cellById;
|
|
356
|
+
objective;
|
|
357
|
+
threshold;
|
|
358
|
+
floorPerCell;
|
|
359
|
+
perRoundBudget;
|
|
360
|
+
/** The single evaluation log — coverage + allocation are projections of it. */
|
|
361
|
+
log = [];
|
|
362
|
+
/** binId (input × measured coords) → the most interesting entry seen. */
|
|
363
|
+
archiveByBin = /* @__PURE__ */ new Map();
|
|
364
|
+
_findings = [];
|
|
365
|
+
runsUsed = 0;
|
|
366
|
+
candidateFindings = 0;
|
|
367
|
+
evalErrors = 0;
|
|
368
|
+
consecutiveEvalErrors = 0;
|
|
369
|
+
stoppedEarly;
|
|
370
|
+
rngState;
|
|
371
|
+
costLedger;
|
|
372
|
+
costPhase = "fuzz.explore";
|
|
373
|
+
constructor(opts) {
|
|
374
|
+
this.opts = opts;
|
|
375
|
+
this.cells = enumerateCells(opts.space);
|
|
376
|
+
if (this.cells.length === 0) throw new Error("BehaviorExplorer: space has no cells — every axis needs ≥1 value");
|
|
377
|
+
if (opts.costBudgetUsd !== void 0) {
|
|
378
|
+
if (typeof opts.costBudgetUsd !== "number" || !Number.isFinite(opts.costBudgetUsd) || opts.costBudgetUsd < 0) throw new RangeError(`BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`);
|
|
379
|
+
}
|
|
380
|
+
if (!opts.costOf && (opts.costBudgetUsd !== void 0 || opts.ledger || opts.onCost)) throw new ValidationError("BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf — the explorer cannot know run cost without it; supply costOf or drop the cost options");
|
|
381
|
+
if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) throw new ValidationError("BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation");
|
|
382
|
+
if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) throw new ValidationError("BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling");
|
|
383
|
+
if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
|
|
384
|
+
this.cellById = new Map(this.cells.map((c) => [c.id, c]));
|
|
385
|
+
this.objective = opts.objective ?? adversarialObjective(.5);
|
|
386
|
+
this.threshold = this.objective.threshold ?? .5;
|
|
387
|
+
this.floorPerCell = opts.floorPerCell ?? 2;
|
|
388
|
+
this.perRoundBudget = Math.max(this.cells.length * this.floorPerCell, Math.ceil(opts.budget / 4));
|
|
389
|
+
this.rngState = (opts.seed ?? 1) >>> 0;
|
|
390
|
+
}
|
|
391
|
+
rng = () => {
|
|
392
|
+
this.rngState = this.rngState + 1831565813 | 0;
|
|
393
|
+
let t = this.rngState;
|
|
394
|
+
t = Math.imul(t ^ t >>> 15, t | 1);
|
|
395
|
+
t ^= t + Math.imul(t ^ t >>> 7, t | 61);
|
|
396
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
397
|
+
};
|
|
398
|
+
binId(cell, descriptor) {
|
|
399
|
+
if (!descriptor || Object.keys(descriptor).length === 0) return cell.id;
|
|
400
|
+
const measured = Object.keys(descriptor).sort().map((k) => `${k}=${descriptor[k]}`).join("|");
|
|
401
|
+
return `${cell.id}|${measured}`;
|
|
402
|
+
}
|
|
403
|
+
allocate(budget) {
|
|
404
|
+
if ((this.opts.allocation ?? "variance") === "uniform") {
|
|
405
|
+
const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length));
|
|
406
|
+
return this.cells.map((c) => ({
|
|
407
|
+
cellId: c.id,
|
|
408
|
+
count: per
|
|
409
|
+
}));
|
|
410
|
+
}
|
|
411
|
+
return varianceBasedCurriculum(this.log.map((r) => ({
|
|
412
|
+
variantId: r.cell.id,
|
|
413
|
+
scenarioId: r.scenarioId,
|
|
414
|
+
score: r.ev.score,
|
|
415
|
+
pass: r.ev.valid && r.ev.score >= .5
|
|
416
|
+
})), this.cells.map((c) => ({
|
|
417
|
+
variantId: c.id,
|
|
418
|
+
scenarioId: "*"
|
|
419
|
+
})), {
|
|
420
|
+
budget,
|
|
421
|
+
floorPerCell: this.floorPerCell
|
|
422
|
+
}).map((a) => ({
|
|
423
|
+
cellId: a.variantId,
|
|
424
|
+
count: a.count
|
|
425
|
+
}));
|
|
426
|
+
}
|
|
427
|
+
objectiveContext() {
|
|
428
|
+
const entries = [...this.archiveByBin.values()];
|
|
429
|
+
return {
|
|
430
|
+
archiveScores: entries.map((e) => e.evaluation.score),
|
|
431
|
+
archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
/** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
|
|
435
|
+
elitesFor(cellId) {
|
|
436
|
+
const out = [];
|
|
437
|
+
for (const e of this.archiveByBin.values()) if (e.cell.id === cellId) out.push(e.scenario);
|
|
438
|
+
return out;
|
|
439
|
+
}
|
|
440
|
+
/** One allocate → propose → evaluate → gate → archive round. */
|
|
441
|
+
async step() {
|
|
442
|
+
const remaining = this.opts.budget - this.runsUsed;
|
|
443
|
+
if (remaining <= 0 || this.opts.signal?.aborted) return {
|
|
444
|
+
runs: 0,
|
|
445
|
+
findings: []
|
|
446
|
+
};
|
|
447
|
+
const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
|
|
448
|
+
const newFindings = [];
|
|
449
|
+
let runsThisStep = 0;
|
|
450
|
+
for (const alloc of allocations) {
|
|
451
|
+
if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) break;
|
|
452
|
+
const cell = this.cellById.get(alloc.cellId);
|
|
453
|
+
if (!cell) continue;
|
|
454
|
+
const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed);
|
|
455
|
+
if (cap <= 0) continue;
|
|
456
|
+
this.opts.onProgress?.({
|
|
457
|
+
type: "cell-allocated",
|
|
458
|
+
cell,
|
|
459
|
+
count: cap
|
|
460
|
+
});
|
|
461
|
+
const seeds = await this.opts.seedsFor(cell);
|
|
462
|
+
const elites = this.elitesFor(cell.id);
|
|
463
|
+
const proposed = await this.opts.proposer({
|
|
464
|
+
cell,
|
|
465
|
+
seeds,
|
|
466
|
+
elites,
|
|
467
|
+
findings: this._findings,
|
|
468
|
+
count: cap,
|
|
469
|
+
rng: this.rng
|
|
470
|
+
});
|
|
471
|
+
await pMap([...this.log.every((r) => r.cell.id !== cell.id) ? seeds : [], ...proposed].slice(0, cap), async (scenario) => {
|
|
472
|
+
if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) return;
|
|
473
|
+
try {
|
|
474
|
+
const startedAt = performance.now();
|
|
475
|
+
const paid = this.costLedger ? await this.costLedger.runPaidCall({
|
|
476
|
+
channel: "agent",
|
|
477
|
+
phase: this.costPhase,
|
|
478
|
+
actor: "fuzz.evaluate",
|
|
479
|
+
tags: {
|
|
480
|
+
target: this.opts.target,
|
|
481
|
+
cell: cell.id
|
|
482
|
+
},
|
|
483
|
+
signal: this.opts.signal,
|
|
484
|
+
maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
|
|
485
|
+
execute: () => this.opts.evaluate(scenario, cell),
|
|
486
|
+
receipt: (evaluation) => {
|
|
487
|
+
const cost = this.opts.costOf(scenario, cell, evaluation);
|
|
488
|
+
if (cost === null) return {
|
|
489
|
+
model: "unattributed",
|
|
490
|
+
inputTokens: 0,
|
|
491
|
+
outputTokens: 0,
|
|
492
|
+
costUnknown: true
|
|
493
|
+
};
|
|
494
|
+
if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new RangeError(`BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) — return null when cost is unknown, never a fabricated number`);
|
|
495
|
+
return {
|
|
496
|
+
model: cost.model ?? "unattributed",
|
|
497
|
+
inputTokens: 0,
|
|
498
|
+
outputTokens: 0,
|
|
499
|
+
actualCostUsd: cost.usd
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
}) : void 0;
|
|
503
|
+
if (paid && !paid.succeeded) {
|
|
504
|
+
if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) throw paid.error.receiptError;
|
|
505
|
+
throw paid.error;
|
|
506
|
+
}
|
|
507
|
+
const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
|
|
508
|
+
const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
|
|
509
|
+
this.runsUsed++;
|
|
510
|
+
runsThisStep++;
|
|
511
|
+
this.consecutiveEvalErrors = 0;
|
|
512
|
+
const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
|
|
513
|
+
if (paid?.receipt.actualCostUsd !== void 0) this.opts.onCost?.({
|
|
514
|
+
usd: paid.receipt.actualCostUsd,
|
|
515
|
+
channel: "agent"
|
|
516
|
+
});
|
|
517
|
+
const interest = this.objective.interest(ev, this.objectiveContext());
|
|
518
|
+
this.log.push({
|
|
519
|
+
cell,
|
|
520
|
+
ev,
|
|
521
|
+
interest,
|
|
522
|
+
latencyMs,
|
|
523
|
+
...costUsd !== void 0 ? { costUsd } : {},
|
|
524
|
+
scenarioId: this.opts.scenarioId(scenario)
|
|
525
|
+
});
|
|
526
|
+
this.opts.onProgress?.({
|
|
527
|
+
type: "evaluated",
|
|
528
|
+
cell,
|
|
529
|
+
scenario,
|
|
530
|
+
evaluation: ev
|
|
531
|
+
});
|
|
532
|
+
const bin = this.binId(cell, ev.descriptor);
|
|
533
|
+
const cur = this.archiveByBin.get(bin);
|
|
534
|
+
if (!cur || interest > cur.interest) this.archiveByBin.set(bin, {
|
|
535
|
+
binId: bin,
|
|
536
|
+
cell,
|
|
537
|
+
scenario,
|
|
538
|
+
evaluation: ev,
|
|
539
|
+
interest
|
|
540
|
+
});
|
|
541
|
+
if (interest < this.threshold) return;
|
|
542
|
+
this.candidateFindings++;
|
|
543
|
+
if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell)) return;
|
|
544
|
+
if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell)) return;
|
|
545
|
+
const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
|
|
546
|
+
const finding = {
|
|
547
|
+
id: this.opts.scenarioId(scenario),
|
|
548
|
+
cell,
|
|
549
|
+
scenario,
|
|
550
|
+
minimized,
|
|
551
|
+
text: this.opts.scenarioText?.(minimized),
|
|
552
|
+
evaluation: ev,
|
|
553
|
+
interest,
|
|
554
|
+
objective: this.objective.kind
|
|
555
|
+
};
|
|
556
|
+
this._findings.push(finding);
|
|
557
|
+
newFindings.push(finding);
|
|
558
|
+
this.opts.onProgress?.({
|
|
559
|
+
type: "finding",
|
|
560
|
+
finding
|
|
561
|
+
});
|
|
562
|
+
} catch (err) {
|
|
563
|
+
if (err instanceof RangeError) throw err;
|
|
564
|
+
this.evalErrors++;
|
|
565
|
+
this.consecutiveEvalErrors++;
|
|
566
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
567
|
+
this.opts.onProgress?.({
|
|
568
|
+
type: "eval-error",
|
|
569
|
+
cell,
|
|
570
|
+
scenarioId: this.opts.scenarioId(scenario),
|
|
571
|
+
message
|
|
572
|
+
});
|
|
573
|
+
const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
|
|
574
|
+
if (this.consecutiveEvalErrors >= limit) this.stoppedEarly = {
|
|
575
|
+
reason: "eval-errors",
|
|
576
|
+
detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
}, this.opts.concurrency ?? 1, this.opts.signal);
|
|
580
|
+
}
|
|
581
|
+
this.opts.onProgress?.({
|
|
582
|
+
type: "round",
|
|
583
|
+
runsUsed: this.runsUsed,
|
|
584
|
+
budget: this.opts.budget
|
|
585
|
+
});
|
|
586
|
+
return {
|
|
587
|
+
runs: runsThisStep,
|
|
588
|
+
findings: newFindings
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
/** Loop `step()` until the run or dollar budget is spent, the signal aborts,
|
|
592
|
+
* or no progress is made. */
|
|
593
|
+
async run() {
|
|
594
|
+
while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
|
|
595
|
+
const { runs } = await this.step();
|
|
596
|
+
if (runs === 0 && this.stoppedEarly === void 0) break;
|
|
597
|
+
}
|
|
598
|
+
return this.capsule();
|
|
599
|
+
}
|
|
600
|
+
coverage() {
|
|
601
|
+
return this.capsule().coverage;
|
|
602
|
+
}
|
|
603
|
+
findings() {
|
|
604
|
+
return [...this._findings].sort((a, b) => b.interest - a.interest);
|
|
605
|
+
}
|
|
606
|
+
capsule() {
|
|
607
|
+
const cost = this.costLedger?.summary({
|
|
608
|
+
phase: this.costPhase,
|
|
609
|
+
tags: { target: this.opts.target }
|
|
610
|
+
});
|
|
611
|
+
return buildCapsule({
|
|
612
|
+
target: this.opts.target,
|
|
613
|
+
objective: this.objective.kind,
|
|
614
|
+
cells: this.cells,
|
|
615
|
+
log: this.log,
|
|
616
|
+
threshold: this.threshold,
|
|
617
|
+
archive: [...this.archiveByBin.values()],
|
|
618
|
+
findings: this._findings,
|
|
619
|
+
candidateFindings: this.candidateFindings,
|
|
620
|
+
runsUsed: this.runsUsed,
|
|
621
|
+
cost: cost ? {
|
|
622
|
+
costUsd: cost.totalCostUsd,
|
|
623
|
+
costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
|
|
624
|
+
} : void 0,
|
|
625
|
+
evalErrors: this.evalErrors,
|
|
626
|
+
stoppedEarly: this.stoppedEarly
|
|
627
|
+
});
|
|
628
|
+
}
|
|
598
629
|
};
|
|
599
|
-
|
|
600
|
-
|
|
630
|
+
//#endregion
|
|
631
|
+
//#region src/fuzz/fuzz-agent.ts
|
|
632
|
+
/**
|
|
633
|
+
* `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
|
|
634
|
+
*
|
|
635
|
+
* One call: explore the space to budget with the adversarial objective and
|
|
636
|
+
* return the capsule. For agent-driven, incremental, or multi-objective use,
|
|
637
|
+
* construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
|
|
638
|
+
*/
|
|
601
639
|
async function fuzzAgent(opts) {
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
return { capsule: await explorer.run() };
|
|
640
|
+
const { failureThreshold, ...rest } = opts;
|
|
641
|
+
return { capsule: await new BehaviorExplorer({
|
|
642
|
+
...rest,
|
|
643
|
+
objective: adversarialObjective(failureThreshold ?? .5)
|
|
644
|
+
}).run() };
|
|
608
645
|
}
|
|
609
|
-
|
|
610
|
-
|
|
646
|
+
//#endregion
|
|
647
|
+
//#region src/fuzz/gates.ts
|
|
648
|
+
/** Combine gate sets; a candidate must pass every gate in every set. */
|
|
611
649
|
function composeGates(...sets) {
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
}
|
|
624
|
-
return true;
|
|
625
|
-
}
|
|
626
|
-
};
|
|
650
|
+
const present = sets.filter((s) => s != null);
|
|
651
|
+
return {
|
|
652
|
+
isValid: async (scenario, ev, cell) => {
|
|
653
|
+
for (const g of present) if (g.isValid && !await g.isValid(scenario, ev, cell)) return false;
|
|
654
|
+
return true;
|
|
655
|
+
},
|
|
656
|
+
isUncontaminated: async (scenario, ev, cell) => {
|
|
657
|
+
for (const g of present) if (g.isUncontaminated && !await g.isUncontaminated(scenario, ev, cell)) return false;
|
|
658
|
+
return true;
|
|
659
|
+
}
|
|
660
|
+
};
|
|
627
661
|
}
|
|
662
|
+
/**
|
|
663
|
+
* Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the
|
|
664
|
+
* flagged scenario; keep the finding only when the rephrase ALSO scores below the
|
|
665
|
+
* threshold. A finding that flips under a cosmetic rewrite was keyed to surface
|
|
666
|
+
* form, not the task — a false signal we must not report. Costs one extra
|
|
667
|
+
* evaluation per candidate (candidates are rare, so cheap).
|
|
668
|
+
*/
|
|
628
669
|
function perturbationStabilityGate(opts) {
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
return re.score < threshold;
|
|
636
|
-
}
|
|
637
|
-
};
|
|
670
|
+
const threshold = opts.failureThreshold ?? .5;
|
|
671
|
+
return { isUncontaminated: async (scenario, _ev, cell) => {
|
|
672
|
+
const rephrased = opts.perturb(scenario);
|
|
673
|
+
if (rephrased == null) return true;
|
|
674
|
+
return (await opts.evaluate(rephrased, cell)).score < threshold;
|
|
675
|
+
} };
|
|
638
676
|
}
|
|
677
|
+
/**
|
|
678
|
+
* Severity-floor gate. Reject borderline candidates whose score sits in a band
|
|
679
|
+
* just under the threshold — judge noise, not a real defect.
|
|
680
|
+
*/
|
|
639
681
|
function severityFloorGate(opts) {
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
isValid: (_scenario, ev) => ev.score <= threshold - margin
|
|
644
|
-
};
|
|
682
|
+
const threshold = opts.failureThreshold ?? .5;
|
|
683
|
+
const margin = opts.margin ?? .1;
|
|
684
|
+
return { isValid: (_scenario, ev) => ev.score <= threshold - margin };
|
|
645
685
|
}
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
686
|
+
//#endregion
|
|
687
|
+
//#region src/fuzz/tools.ts
|
|
688
|
+
/**
|
|
689
|
+
* Agent-drivable surface over a live exploration session.
|
|
690
|
+
*
|
|
691
|
+
* Framework-neutral tool defs ({name, description, parameters: JSON Schema,
|
|
692
|
+
* handler}) so the on-demand agent — not a batch script — drives the search:
|
|
693
|
+
* step it, read coverage, inspect findings, render the capsule. Transport
|
|
694
|
+
* encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
|
|
695
|
+
*/
|
|
696
|
+
const NO_ARGS = {
|
|
697
|
+
type: "object",
|
|
698
|
+
properties: {},
|
|
699
|
+
additionalProperties: false
|
|
700
|
+
};
|
|
649
701
|
function makeExploreTools(explorer) {
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
702
|
+
return [
|
|
703
|
+
{
|
|
704
|
+
name: "explore_step",
|
|
705
|
+
description: "Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.",
|
|
706
|
+
parameters: NO_ARGS,
|
|
707
|
+
handler: async () => {
|
|
708
|
+
const { runs, findings } = await explorer.step();
|
|
709
|
+
return {
|
|
710
|
+
runs,
|
|
711
|
+
newFindings: findings.length,
|
|
712
|
+
findings
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
},
|
|
716
|
+
{
|
|
717
|
+
name: "explore_coverage",
|
|
718
|
+
description: "Read the live coverage map: per planned cell — runs, robustness (null = uncovered), finding rate, per-dimension means.",
|
|
719
|
+
parameters: NO_ARGS,
|
|
720
|
+
handler: async () => explorer.coverage()
|
|
721
|
+
},
|
|
722
|
+
{
|
|
723
|
+
name: "explore_findings",
|
|
724
|
+
description: "List gate-verified findings so far, sorted by descending interest.",
|
|
725
|
+
parameters: {
|
|
726
|
+
type: "object",
|
|
727
|
+
properties: { limit: {
|
|
728
|
+
type: "number",
|
|
729
|
+
description: "max findings to return"
|
|
730
|
+
} },
|
|
731
|
+
additionalProperties: false
|
|
732
|
+
},
|
|
733
|
+
handler: async (args) => {
|
|
734
|
+
const limit = args?.limit;
|
|
735
|
+
const all = explorer.findings();
|
|
736
|
+
return typeof limit === "number" ? all.slice(0, limit) : all;
|
|
737
|
+
}
|
|
738
|
+
},
|
|
739
|
+
{
|
|
740
|
+
name: "explore_capsule",
|
|
741
|
+
description: "Build the capsule artifact from the current session state. format \"data\" returns the structured CapsuleData; \"html\" returns the standalone page.",
|
|
742
|
+
parameters: {
|
|
743
|
+
type: "object",
|
|
744
|
+
properties: {
|
|
745
|
+
format: {
|
|
746
|
+
type: "string",
|
|
747
|
+
enum: ["data", "html"]
|
|
748
|
+
},
|
|
749
|
+
generatedAt: {
|
|
750
|
+
type: "string",
|
|
751
|
+
description: "ISO timestamp to stamp into the page"
|
|
752
|
+
}
|
|
753
|
+
},
|
|
754
|
+
additionalProperties: false
|
|
755
|
+
},
|
|
756
|
+
handler: async (args) => {
|
|
757
|
+
const a = args ?? {};
|
|
758
|
+
const capsule = explorer.capsule();
|
|
759
|
+
if (a.format === "html") return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt });
|
|
760
|
+
return capsule;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
];
|
|
699
764
|
}
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
buildCapsule,
|
|
704
|
-
buildCoverage,
|
|
705
|
-
cellId,
|
|
706
|
-
composeGates,
|
|
707
|
-
enumerateCells,
|
|
708
|
-
fuzzAgent,
|
|
709
|
-
makeExploreTools,
|
|
710
|
-
mutationProposer,
|
|
711
|
-
noveltyObjective,
|
|
712
|
-
perturbationStabilityGate,
|
|
713
|
-
renderCapsuleHtml,
|
|
714
|
-
severityFloorGate
|
|
715
|
-
};
|
|
765
|
+
//#endregion
|
|
766
|
+
export { BehaviorExplorer, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };
|
|
767
|
+
|
|
716
768
|
//# sourceMappingURL=fuzz.js.map
|