@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/fuzz.js
CHANGED
|
@@ -1,198 +1,207 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
} from "./
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
CostReceiptCaptureError
|
|
7
|
-
} from "./chunk-WS3NZZQQ.js";
|
|
8
|
-
import "./chunk-VI2UW6B6.js";
|
|
9
|
-
import {
|
|
10
|
-
ValidationError
|
|
11
|
-
} from "./chunk-ONWEPEDO.js";
|
|
12
|
-
import "./chunk-PZ5AY32C.js";
|
|
13
|
-
|
|
14
|
-
// src/fuzz/cube.ts
|
|
1
|
+
import { s as ValidationError } from "./errors-8YnH8WlF.js";
|
|
2
|
+
import { i as CostLedger, o as CostReceiptCaptureError } from "./cost-ledger-DIgQUFZZ.js";
|
|
3
|
+
import { r as varianceBasedCurriculum } from "./active-curriculum-C4mk67HP.js";
|
|
4
|
+
//#region src/fuzz/cube.ts
|
|
5
|
+
/** Enumerate every input cell (cartesian product of the axes), in stable order. */
|
|
15
6
|
function enumerateCells(space) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
7
|
+
if (space.axes.length === 0) return [];
|
|
8
|
+
let partials = [{}];
|
|
9
|
+
for (const axis of space.axes) {
|
|
10
|
+
const next = [];
|
|
11
|
+
for (const partial of partials) for (const value of axis.values) next.push({
|
|
12
|
+
...partial,
|
|
13
|
+
[axis.name]: value
|
|
14
|
+
});
|
|
15
|
+
partials = next;
|
|
16
|
+
}
|
|
17
|
+
return partials.map((coords) => ({
|
|
18
|
+
id: cellId(space, coords),
|
|
19
|
+
coords
|
|
20
|
+
}));
|
|
26
21
|
}
|
|
22
|
+
/** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */
|
|
27
23
|
function cellId(space, coords) {
|
|
28
|
-
|
|
24
|
+
return space.axes.map((a) => `${a.name}=${coords[a.name]}`).join("|");
|
|
29
25
|
}
|
|
26
|
+
/** Nearest-rank percentile on a pre-sorted ascending sample. */
|
|
30
27
|
function percentile(sorted, p) {
|
|
31
|
-
|
|
32
|
-
return sorted[idx];
|
|
28
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
|
|
33
29
|
}
|
|
30
|
+
/** Summarize a sample. Throws on an empty sample — callers represent "no data"
|
|
31
|
+
* as `null`, never as a zeroed distribution. */
|
|
34
32
|
function distribution(values) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
n: sorted.length
|
|
46
|
-
};
|
|
33
|
+
if (values.length === 0) throw new Error("distribution: empty sample — represent missing data as null, not zeros");
|
|
34
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
35
|
+
return {
|
|
36
|
+
mean: sorted.reduce((a, b) => a + b, 0) / sorted.length,
|
|
37
|
+
median: percentile(sorted, .5),
|
|
38
|
+
p90: percentile(sorted, .9),
|
|
39
|
+
min: sorted[0],
|
|
40
|
+
max: sorted[sorted.length - 1],
|
|
41
|
+
n: sorted.length
|
|
42
|
+
};
|
|
47
43
|
}
|
|
44
|
+
/**
|
|
45
|
+
* Project the evaluation log into the per-input-cell coverage map. A cell with
|
|
46
|
+
* no evaluations reports `score: null` (honestly uncovered), never zeros.
|
|
47
|
+
*/
|
|
48
48
|
function buildCoverage(cells, log, threshold) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
49
|
+
const byCell = /* @__PURE__ */ new Map();
|
|
50
|
+
for (const r of log) {
|
|
51
|
+
const arr = byCell.get(r.cell.id) ?? [];
|
|
52
|
+
arr.push(r);
|
|
53
|
+
byCell.set(r.cell.id, arr);
|
|
54
|
+
}
|
|
55
|
+
return cells.map((cell) => {
|
|
56
|
+
const recs = byCell.get(cell.id) ?? [];
|
|
57
|
+
const runs = recs.length;
|
|
58
|
+
if (runs === 0) return {
|
|
59
|
+
cell,
|
|
60
|
+
runs: 0,
|
|
61
|
+
score: null,
|
|
62
|
+
findingRate: 0,
|
|
63
|
+
dimensions: {},
|
|
64
|
+
latencyMs: null
|
|
65
|
+
};
|
|
66
|
+
const score = distribution(recs.map((r) => r.ev.score));
|
|
67
|
+
const latencyMs = distribution(recs.map((r) => r.latencyMs));
|
|
68
|
+
const findingRate = recs.filter((r) => r.interest >= threshold).length / runs;
|
|
69
|
+
const dimSamples = {};
|
|
70
|
+
for (const r of recs) for (const [k, v] of Object.entries(r.ev.scores ?? {})) {
|
|
71
|
+
const samples = dimSamples[k] ?? [];
|
|
72
|
+
samples.push(v);
|
|
73
|
+
dimSamples[k] = samples;
|
|
74
|
+
}
|
|
75
|
+
const dimensions = {};
|
|
76
|
+
for (const [k, xs] of Object.entries(dimSamples)) dimensions[k] = distribution(xs);
|
|
77
|
+
const tracked = recs.filter((r) => r.costUsd !== void 0);
|
|
78
|
+
const known = tracked.filter((r) => r.costUsd !== null);
|
|
79
|
+
return {
|
|
80
|
+
cell,
|
|
81
|
+
runs,
|
|
82
|
+
score,
|
|
83
|
+
findingRate,
|
|
84
|
+
dimensions,
|
|
85
|
+
latencyMs,
|
|
86
|
+
...tracked.length > 0 ? {
|
|
87
|
+
costUsd: known.reduce((a, r) => a + r.costUsd, 0),
|
|
88
|
+
...tracked.length > known.length ? { costUnknownRuns: tracked.length - known.length } : {}
|
|
89
|
+
} : {}
|
|
90
|
+
};
|
|
91
|
+
});
|
|
81
92
|
}
|
|
82
|
-
|
|
83
|
-
|
|
93
|
+
//#endregion
|
|
94
|
+
//#region src/fuzz/capsule.ts
|
|
84
95
|
function buildCapsule(input) {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
96
|
+
const coverage = buildCoverage(input.cells, input.log, input.threshold);
|
|
97
|
+
const covered = coverage.filter((c) => c.runs > 0);
|
|
98
|
+
const robustness = covered.length === 0 ? null : distribution(covered.map((c) => c.score.mean));
|
|
99
|
+
const latencyMs = input.log.length === 0 ? null : distribution(input.log.map((r) => r.latencyMs));
|
|
100
|
+
const behaviorBinsObserved = input.archive.filter((e) => e.binId !== e.cell.id).length;
|
|
101
|
+
return {
|
|
102
|
+
target: input.target,
|
|
103
|
+
objective: input.objective,
|
|
104
|
+
coverage,
|
|
105
|
+
findings: [...input.findings].sort((a, b) => b.interest - a.interest),
|
|
106
|
+
archive: [...input.archive].sort((a, b) => b.interest - a.interest),
|
|
107
|
+
stats: {
|
|
108
|
+
totalRuns: input.runsUsed,
|
|
109
|
+
cellsTotal: input.cells.length,
|
|
110
|
+
cellsCovered: covered.length,
|
|
111
|
+
behaviorBinsObserved,
|
|
112
|
+
candidateFindings: input.candidateFindings,
|
|
113
|
+
verifiedFindings: input.findings.length,
|
|
114
|
+
robustness,
|
|
115
|
+
latencyMs,
|
|
116
|
+
...input.cost ? {
|
|
117
|
+
costUsd: input.cost.costUsd,
|
|
118
|
+
costUnknownRuns: input.cost.costUnknownRuns
|
|
119
|
+
} : {},
|
|
120
|
+
evalErrors: input.evalErrors,
|
|
121
|
+
...input.stoppedEarly ? { stoppedEarly: input.stoppedEarly } : {}
|
|
122
|
+
}
|
|
123
|
+
};
|
|
110
124
|
}
|
|
111
125
|
function esc(s) {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
126
|
+
return s.replace(/[&<>"]/g, (c) => ({
|
|
127
|
+
"&": "&",
|
|
128
|
+
"<": "<",
|
|
129
|
+
">": ">",
|
|
130
|
+
"\"": """
|
|
131
|
+
})[c]);
|
|
116
132
|
}
|
|
133
|
+
/** red (0) → amber (.5) → green (1); uncovered cells render gray. */
|
|
117
134
|
function robustnessColor(r) {
|
|
118
|
-
|
|
119
|
-
|
|
135
|
+
if (r == null) return "#2a2a2e";
|
|
136
|
+
return `hsl(${Math.round(r * 120)} 70% 42%)`;
|
|
120
137
|
}
|
|
121
138
|
function pct(x) {
|
|
122
|
-
|
|
139
|
+
return `${Math.round(x * 100)}%`;
|
|
123
140
|
}
|
|
124
141
|
function deriveAxes(coverage) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
142
|
+
const order = [];
|
|
143
|
+
const seen = /* @__PURE__ */ new Map();
|
|
144
|
+
for (const c of coverage) for (const [k, v] of Object.entries(c.cell.coords)) {
|
|
145
|
+
if (!seen.has(k)) {
|
|
146
|
+
seen.set(k, /* @__PURE__ */ new Set());
|
|
147
|
+
order.push(k);
|
|
148
|
+
}
|
|
149
|
+
seen.get(k)?.add(v);
|
|
150
|
+
}
|
|
151
|
+
return order.map((name) => ({
|
|
152
|
+
name,
|
|
153
|
+
values: [...seen.get(name) ?? []]
|
|
154
|
+
}));
|
|
137
155
|
}
|
|
156
|
+
/** The weakest dimension chip for a cell, e.g. `safety 32%` — shown when scores exist. */
|
|
138
157
|
function weakestDim(c) {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
|
|
158
|
+
const entries = Object.entries(c.dimensions).map(([k, d]) => [k, d.mean]);
|
|
159
|
+
if (entries.length === 0) return "";
|
|
160
|
+
const w = entries.sort((a, b) => a[1] - b[1])[0];
|
|
161
|
+
if (!w) return "";
|
|
162
|
+
return `<span class="dim">${esc(w[0])} ${pct(w[1])}</span>`;
|
|
145
163
|
}
|
|
146
164
|
function heatmapHtml(coverage) {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
return `<div class="grid">${sorted.map((c) => tile(c, Object.values(c.cell.coords).join(" \xB7 "))).join("")}</div>`;
|
|
165
|
+
const axes = deriveAxes(coverage);
|
|
166
|
+
const byId = new Map(coverage.map((c) => [c.cell.id, c]));
|
|
167
|
+
const tile = (c, label) => {
|
|
168
|
+
const r = c?.score?.mean ?? null;
|
|
169
|
+
const title = c?.score ? `${pct(c.score.mean)} robust (median ${pct(c.score.median)}, min ${pct(c.score.min)}) · ${c.runs} runs · ${pct(c.findingRate)} flagged${c.latencyMs ? ` · ${(c.latencyMs.median / 1e3).toFixed(1)}s median` : ""}${c.costUsd !== void 0 ? ` · $${c.costUsd.toFixed(2)}` : ""}` : "not covered";
|
|
170
|
+
return `<div class="tile" style="background:${robustnessColor(r)}" title="${esc(title)}">${label ? `<span class="tl">${esc(label)}</span>` : ""}<span class="tv">${c && r != null ? pct(r) : "—"}</span>${c ? weakestDim(c) : ""}</div>`;
|
|
171
|
+
};
|
|
172
|
+
const rowAxis = axes[0];
|
|
173
|
+
const colAxis = axes[1];
|
|
174
|
+
if (axes.length === 2 && rowAxis && colAxis) {
|
|
175
|
+
const head = `<tr><th></th>${colAxis.values.map((v) => `<th>${esc(v)}</th>`).join("")}</tr>`;
|
|
176
|
+
const rows = rowAxis.values.map((rv) => {
|
|
177
|
+
const cells = colAxis.values.map((cv) => {
|
|
178
|
+
const id = `${rowAxis.name}=${rv}|${colAxis.name}=${cv}`;
|
|
179
|
+
return `<td>${tile(byId.get(id), "")}</td>`;
|
|
180
|
+
}).join("");
|
|
181
|
+
return `<tr><th class="rh">${esc(rv)}</th>${cells}</tr>`;
|
|
182
|
+
}).join("");
|
|
183
|
+
return `<div class="axis-label">rows: <b>${esc(rowAxis.name)}</b> · cols: <b>${esc(colAxis.name)}</b></div><table class="heat">${head}${rows}</table>`;
|
|
184
|
+
}
|
|
185
|
+
return `<div class="grid">${[...coverage].sort((a, b) => (a.score?.mean ?? 2) - (b.score?.mean ?? 2)).map((c) => tile(c, Object.values(c.cell.coords).join(" · "))).join("")}</div>`;
|
|
169
186
|
}
|
|
170
187
|
function findingsHtml(findings, limit) {
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
}).join("");
|
|
188
|
+
if (findings.length === 0) return `<p class="none">No verified findings — the target held across every covered cell.</p>`;
|
|
189
|
+
return findings.slice(0, limit).map((f) => {
|
|
190
|
+
const coords = Object.entries(f.cell.coords).map(([k, v]) => `${k}:${v}`).join(" · ");
|
|
191
|
+
const labels = (f.evaluation.labels ?? []).map((l) => `<span class="fclass">${esc(l)}</span>`).join("");
|
|
192
|
+
const dims = Object.entries(f.evaluation.scores ?? {}).sort((a, b) => a[1] - b[1]).slice(0, 3).map(([k, v]) => `<span class="dim">${esc(k)} ${pct(v)}</span>`).join("");
|
|
193
|
+
return `<div class="fail"><div class="fmeta"><span class="fcell">${esc(coords)}</span>${labels}<span class="fsev">interest ${pct(f.interest)}</span></div><div class="sevbar"><div class="sevfill" style="width:${pct(f.interest)}"></div></div><div class="ftext">${esc(f.text ?? "(scenario text not captured)")}</div>${dims ? `<div class="fdims">${dims}</div>` : ""}</div>`;
|
|
194
|
+
}).join("");
|
|
179
195
|
}
|
|
196
|
+
/** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
|
|
180
197
|
function renderCapsuleHtml(capsule, opts = {}) {
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
const cost = s.costUsd !== void 0 ? kpi(
|
|
189
|
-
s.costUnknownRuns ? `cost \xB7 ${s.costUnknownRuns} runs unpriced` : "cost",
|
|
190
|
-
`$${s.costUsd.toFixed(2)}`,
|
|
191
|
-
s.costUnknownRuns ? "#e5c07b" : "#e6e6e6"
|
|
192
|
-
) : "";
|
|
193
|
-
const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
|
|
194
|
-
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
195
|
-
<title>${esc(capsule.objective)} capsule \u2014 ${esc(capsule.target)}</title>
|
|
198
|
+
const s = capsule.stats;
|
|
199
|
+
const kpi = (label, value, accent = "#e6e6e6") => `<div class="kpi"><div class="kv" style="color:${accent}">${esc(value)}</div><div class="kl">${esc(label)}</div></div>`;
|
|
200
|
+
const lift = capsule.lift ? kpi("hardening lift", `${capsule.lift.before.toFixed(2)} → ${capsule.lift.after.toFixed(2)}`, "#5ad17a") : "";
|
|
201
|
+
const cost = s.costUsd !== void 0 ? kpi(s.costUnknownRuns ? `cost · ${s.costUnknownRuns} runs unpriced` : "cost", `$${s.costUsd.toFixed(2)}`, s.costUnknownRuns ? "#e5c07b" : "#e6e6e6") : "";
|
|
202
|
+
const stamp = opts.generatedAt ?? capsule.generatedAt ?? "";
|
|
203
|
+
return `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
204
|
+
<title>${esc(capsule.objective)} capsule — ${esc(capsule.target)}</title>
|
|
196
205
|
<style>
|
|
197
206
|
:root{color-scheme:dark}
|
|
198
207
|
*{box-sizing:border-box}
|
|
@@ -228,11 +237,11 @@ table.heat th.rh{text-align:right}
|
|
|
228
237
|
.none{color:#5ad17a}
|
|
229
238
|
.foot{color:#5a5a63;font-size:11px;margin-top:36px}
|
|
230
239
|
</style></head><body><div class="wrap">
|
|
231
|
-
<h1>${esc(capsule.objective)} exploration
|
|
232
|
-
<div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? `
|
|
240
|
+
<h1>${esc(capsule.objective)} exploration · ${esc(capsule.target)}</h1>
|
|
241
|
+
<div class="sub">${s.totalRuns} scenarios across ${s.cellsCovered}/${s.cellsTotal} planned cells${s.behaviorBinsObserved > 0 ? ` · ${s.behaviorBinsObserved} measured behavior bins` : ""}${stamp ? ` · ${esc(stamp)}` : ""}</div>
|
|
233
242
|
<div class="kpis">
|
|
234
|
-
${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean <
|
|
235
|
-
${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}
|
|
243
|
+
${s.robustness ? kpi("robustness", `${pct(s.robustness.mean)}`, s.robustness.mean < .6 ? "#e58a96" : "#5ad17a") : ""}
|
|
244
|
+
${s.robustness ? kpi("cell spread", `${pct(s.robustness.min)}–${pct(s.robustness.max)}`) : ""}
|
|
236
245
|
${s.latencyMs ? kpi("median latency", `${(s.latencyMs.median / 1e3).toFixed(1)}s`, s.latencyMs.p90 > 4 * s.latencyMs.median ? "#e5b566" : "#e6e6e6") : ""}
|
|
237
246
|
${kpi("verified findings", String(s.verifiedFindings), s.verifiedFindings > 0 ? "#e58a96" : "#5ad17a")}
|
|
238
247
|
${kpi("cells covered", `${s.cellsCovered}/${s.cellsTotal}`)}
|
|
@@ -241,475 +250,519 @@ ${cost}
|
|
|
241
250
|
${s.evalErrors > 0 ? kpi("eval errors", String(s.evalErrors), "#e5b566") : ""}
|
|
242
251
|
${lift}
|
|
243
252
|
</div>
|
|
244
|
-
${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)}
|
|
253
|
+
${s.stoppedEarly ? `<div class="sub" style="color:#e5b566">stopped early: ${esc(s.stoppedEarly.detail)} — coverage below reflects the completed portion only</div>` : ""}
|
|
245
254
|
<h2>Coverage map</h2>
|
|
246
255
|
${heatmapHtml(capsule.coverage)}
|
|
247
|
-
<h2>Verified findings${s.candidateFindings > s.verifiedFindings ? `
|
|
256
|
+
<h2>Verified findings${s.candidateFindings > s.verifiedFindings ? ` · ${s.verifiedFindings} of ${s.candidateFindings} candidates passed the validity gates` : ""}</h2>
|
|
248
257
|
${findingsHtml(capsule.findings, opts.maxFindings ?? 8)}
|
|
249
258
|
<div class="foot">Findings are gate-verified: each is a fair, answerable task that reproduces under a meaning-preserving rephrase, minimized to the smallest trigger.</div>
|
|
250
259
|
</div></body></html>`;
|
|
251
260
|
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
261
|
+
//#endregion
|
|
262
|
+
//#region src/fuzz/policies.ts
|
|
263
|
+
const clamp01 = (x) => Math.max(0, Math.min(1, x));
|
|
264
|
+
/**
|
|
265
|
+
* Perturbation-based search: apply the cell's mutation operators to the current
|
|
266
|
+
* elites + seeds, deduping by id. Elites first — mutating the most interesting
|
|
267
|
+
* scenario found so far is what makes the search deepen across rounds.
|
|
268
|
+
*/
|
|
255
269
|
function mutationProposer(opts) {
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
270
|
+
return async (ctx) => {
|
|
271
|
+
const mutations = opts.mutationsFor(ctx.cell);
|
|
272
|
+
const parents = [...ctx.elites, ...ctx.seeds];
|
|
273
|
+
const seen = new Set(parents.map(opts.scenarioId));
|
|
274
|
+
const out = [];
|
|
275
|
+
for (const parent of parents) {
|
|
276
|
+
if (out.length >= ctx.count) break;
|
|
277
|
+
for (const m of mutations) {
|
|
278
|
+
const children = await m.mutate(parent, ctx.rng);
|
|
279
|
+
for (const child of children) {
|
|
280
|
+
const id = opts.scenarioId(child);
|
|
281
|
+
if (seen.has(id)) continue;
|
|
282
|
+
seen.add(id);
|
|
283
|
+
out.push(child);
|
|
284
|
+
if (out.length >= ctx.count) break;
|
|
285
|
+
}
|
|
286
|
+
if (out.length >= ctx.count) break;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return out;
|
|
290
|
+
};
|
|
277
291
|
}
|
|
278
|
-
|
|
279
|
-
|
|
292
|
+
/** Adversarial: a low headline score is interesting — find where the agent fails. */
|
|
293
|
+
function adversarialObjective(threshold = .5) {
|
|
294
|
+
return {
|
|
295
|
+
kind: "adversarial",
|
|
296
|
+
threshold,
|
|
297
|
+
interest: (ev) => clamp01(1 - ev.score)
|
|
298
|
+
};
|
|
280
299
|
}
|
|
281
300
|
function hamming(a, b) {
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
301
|
+
if (!a || !b) return 1;
|
|
302
|
+
const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
|
|
303
|
+
if (keys.size === 0) return 0;
|
|
304
|
+
let diff = 0;
|
|
305
|
+
for (const k of keys) if (a[k] !== b[k]) diff++;
|
|
306
|
+
return diff / keys.size;
|
|
288
307
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
308
|
+
/**
|
|
309
|
+
* Novelty: interesting when far from the archive in score AND measured behavior
|
|
310
|
+
* descriptor — quality-diversity's diversity pressure; drives corpus growth
|
|
311
|
+
* rather than re-finding the same hole.
|
|
312
|
+
*/
|
|
313
|
+
function noveltyObjective(threshold = .3) {
|
|
314
|
+
return {
|
|
315
|
+
kind: "novelty",
|
|
316
|
+
threshold,
|
|
317
|
+
interest: (ev, ctx) => {
|
|
318
|
+
const scoreNovelty = ctx.archiveScores.length === 0 ? 1 : Math.min(...ctx.archiveScores.map((s) => Math.abs(s - ev.score)));
|
|
319
|
+
const descNovelty = ctx.archiveDescriptors.length === 0 ? 1 : Math.min(...ctx.archiveDescriptors.map((d) => hamming(d, ev.descriptor)));
|
|
320
|
+
return clamp01(.5 * scoreNovelty + .5 * descNovelty);
|
|
321
|
+
}
|
|
322
|
+
};
|
|
299
323
|
}
|
|
300
|
-
|
|
301
|
-
|
|
324
|
+
//#endregion
|
|
325
|
+
//#region src/fuzz/explorer.ts
|
|
326
|
+
/**
|
|
327
|
+
* The exploration engine — a stateful session over a behavior space.
|
|
328
|
+
*
|
|
329
|
+
* Each `step()`: allocate budget across INPUT cells (floor first, then variance
|
|
330
|
+
* steering toward the least-certain cells), propose candidates (the proposer
|
|
331
|
+
* reads current elites + findings, so the search deepens generationally),
|
|
332
|
+
* evaluate with bounded concurrency, archive the most interesting scenario per
|
|
333
|
+
* input×measured bin, and admit notable candidates that pass the validity gates.
|
|
334
|
+
* `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
|
|
335
|
+
* the surface `makeExploreTools` exposes so an agent can drive the session.
|
|
336
|
+
*
|
|
337
|
+
* One evaluation log (`EvalRecord[]`) is the source of truth; allocation
|
|
338
|
+
* observations and coverage are projections of it.
|
|
339
|
+
*/
|
|
302
340
|
async function pMap(items, fn, concurrency, signal) {
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
}
|
|
314
|
-
);
|
|
315
|
-
await Promise.all(workers);
|
|
341
|
+
let i = 0;
|
|
342
|
+
const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
|
|
343
|
+
while (i < items.length) {
|
|
344
|
+
if (signal?.aborted) return;
|
|
345
|
+
const item = items[i++];
|
|
346
|
+
if (item === void 0) return;
|
|
347
|
+
await fn(item);
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
await Promise.all(workers);
|
|
316
351
|
}
|
|
317
352
|
var BehaviorExplorer = class {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
stoppedEarly: this.stoppedEarly
|
|
595
|
-
});
|
|
596
|
-
}
|
|
353
|
+
opts;
|
|
354
|
+
cells;
|
|
355
|
+
cellById;
|
|
356
|
+
objective;
|
|
357
|
+
threshold;
|
|
358
|
+
floorPerCell;
|
|
359
|
+
perRoundBudget;
|
|
360
|
+
/** The single evaluation log — coverage + allocation are projections of it. */
|
|
361
|
+
log = [];
|
|
362
|
+
/** binId (input × measured coords) → the most interesting entry seen. */
|
|
363
|
+
archiveByBin = /* @__PURE__ */ new Map();
|
|
364
|
+
_findings = [];
|
|
365
|
+
runsUsed = 0;
|
|
366
|
+
candidateFindings = 0;
|
|
367
|
+
evalErrors = 0;
|
|
368
|
+
consecutiveEvalErrors = 0;
|
|
369
|
+
stoppedEarly;
|
|
370
|
+
rngState;
|
|
371
|
+
costLedger;
|
|
372
|
+
costPhase = "fuzz.explore";
|
|
373
|
+
constructor(opts) {
|
|
374
|
+
this.opts = opts;
|
|
375
|
+
this.cells = enumerateCells(opts.space);
|
|
376
|
+
if (this.cells.length === 0) throw new Error("BehaviorExplorer: space has no cells — every axis needs ≥1 value");
|
|
377
|
+
if (opts.costBudgetUsd !== void 0) {
|
|
378
|
+
if (typeof opts.costBudgetUsd !== "number" || !Number.isFinite(opts.costBudgetUsd) || opts.costBudgetUsd < 0) throw new RangeError(`BehaviorExplorer: costBudgetUsd must be a nonnegative finite number, got ${String(opts.costBudgetUsd)}`);
|
|
379
|
+
}
|
|
380
|
+
if (!opts.costOf && (opts.costBudgetUsd !== void 0 || opts.ledger || opts.onCost)) throw new ValidationError("BehaviorExplorer: costBudgetUsd/ledger/onCost require costOf — the explorer cannot know run cost without it; supply costOf or drop the cost options");
|
|
381
|
+
if ((opts.costBudgetUsd !== void 0 || opts.ledger?.costCeilingUsd !== void 0) && !opts.maximumChargeOf) throw new ValidationError("BehaviorExplorer: capped cost tracking requires maximumChargeOf before evaluation");
|
|
382
|
+
if (opts.ledger && opts.costBudgetUsd !== void 0 && opts.ledger.costCeilingUsd !== opts.costBudgetUsd) throw new ValidationError("BehaviorExplorer: costBudgetUsd must match the supplied CostLedger ceiling");
|
|
383
|
+
if (opts.costOf) this.costLedger = opts.ledger ?? new CostLedger(opts.costBudgetUsd);
|
|
384
|
+
this.cellById = new Map(this.cells.map((c) => [c.id, c]));
|
|
385
|
+
this.objective = opts.objective ?? adversarialObjective(.5);
|
|
386
|
+
this.threshold = this.objective.threshold ?? .5;
|
|
387
|
+
this.floorPerCell = opts.floorPerCell ?? 2;
|
|
388
|
+
this.perRoundBudget = Math.max(this.cells.length * this.floorPerCell, Math.ceil(opts.budget / 4));
|
|
389
|
+
this.rngState = (opts.seed ?? 1) >>> 0;
|
|
390
|
+
}
|
|
391
|
+
rng = () => {
|
|
392
|
+
this.rngState = this.rngState + 1831565813 | 0;
|
|
393
|
+
let t = this.rngState;
|
|
394
|
+
t = Math.imul(t ^ t >>> 15, t | 1);
|
|
395
|
+
t ^= t + Math.imul(t ^ t >>> 7, t | 61);
|
|
396
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
397
|
+
};
|
|
398
|
+
binId(cell, descriptor) {
|
|
399
|
+
if (!descriptor || Object.keys(descriptor).length === 0) return cell.id;
|
|
400
|
+
const measured = Object.keys(descriptor).sort().map((k) => `${k}=${descriptor[k]}`).join("|");
|
|
401
|
+
return `${cell.id}|${measured}`;
|
|
402
|
+
}
|
|
403
|
+
allocate(budget) {
|
|
404
|
+
if ((this.opts.allocation ?? "variance") === "uniform") {
|
|
405
|
+
const per = Math.max(this.floorPerCell, Math.floor(budget / this.cells.length));
|
|
406
|
+
return this.cells.map((c) => ({
|
|
407
|
+
cellId: c.id,
|
|
408
|
+
count: per
|
|
409
|
+
}));
|
|
410
|
+
}
|
|
411
|
+
return varianceBasedCurriculum(this.log.map((r) => ({
|
|
412
|
+
variantId: r.cell.id,
|
|
413
|
+
scenarioId: r.scenarioId,
|
|
414
|
+
score: r.ev.score,
|
|
415
|
+
pass: r.ev.valid && r.ev.score >= .5
|
|
416
|
+
})), this.cells.map((c) => ({
|
|
417
|
+
variantId: c.id,
|
|
418
|
+
scenarioId: "*"
|
|
419
|
+
})), {
|
|
420
|
+
budget,
|
|
421
|
+
floorPerCell: this.floorPerCell
|
|
422
|
+
}).map((a) => ({
|
|
423
|
+
cellId: a.variantId,
|
|
424
|
+
count: a.count
|
|
425
|
+
}));
|
|
426
|
+
}
|
|
427
|
+
objectiveContext() {
|
|
428
|
+
const entries = [...this.archiveByBin.values()];
|
|
429
|
+
return {
|
|
430
|
+
archiveScores: entries.map((e) => e.evaluation.score),
|
|
431
|
+
archiveDescriptors: entries.map((e) => e.evaluation.descriptor)
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
/** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
|
|
435
|
+
elitesFor(cellId) {
|
|
436
|
+
const out = [];
|
|
437
|
+
for (const e of this.archiveByBin.values()) if (e.cell.id === cellId) out.push(e.scenario);
|
|
438
|
+
return out;
|
|
439
|
+
}
|
|
440
|
+
/** One allocate → propose → evaluate → gate → archive round. */
|
|
441
|
+
async step() {
|
|
442
|
+
const remaining = this.opts.budget - this.runsUsed;
|
|
443
|
+
if (remaining <= 0 || this.opts.signal?.aborted) return {
|
|
444
|
+
runs: 0,
|
|
445
|
+
findings: []
|
|
446
|
+
};
|
|
447
|
+
const allocations = this.allocate(Math.min(this.perRoundBudget, remaining));
|
|
448
|
+
const newFindings = [];
|
|
449
|
+
let runsThisStep = 0;
|
|
450
|
+
for (const alloc of allocations) {
|
|
451
|
+
if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) break;
|
|
452
|
+
const cell = this.cellById.get(alloc.cellId);
|
|
453
|
+
if (!cell) continue;
|
|
454
|
+
const cap = Math.min(alloc.count, this.opts.budget - this.runsUsed);
|
|
455
|
+
if (cap <= 0) continue;
|
|
456
|
+
this.opts.onProgress?.({
|
|
457
|
+
type: "cell-allocated",
|
|
458
|
+
cell,
|
|
459
|
+
count: cap
|
|
460
|
+
});
|
|
461
|
+
const seeds = await this.opts.seedsFor(cell);
|
|
462
|
+
const elites = this.elitesFor(cell.id);
|
|
463
|
+
const proposed = await this.opts.proposer({
|
|
464
|
+
cell,
|
|
465
|
+
seeds,
|
|
466
|
+
elites,
|
|
467
|
+
findings: this._findings,
|
|
468
|
+
count: cap,
|
|
469
|
+
rng: this.rng
|
|
470
|
+
});
|
|
471
|
+
await pMap([...this.log.every((r) => r.cell.id !== cell.id) ? seeds : [], ...proposed].slice(0, cap), async (scenario) => {
|
|
472
|
+
if (this.runsUsed >= this.opts.budget || this.stoppedEarly !== void 0 || this.opts.signal?.aborted) return;
|
|
473
|
+
try {
|
|
474
|
+
const startedAt = performance.now();
|
|
475
|
+
const paid = this.costLedger ? await this.costLedger.runPaidCall({
|
|
476
|
+
channel: "agent",
|
|
477
|
+
phase: this.costPhase,
|
|
478
|
+
actor: "fuzz.evaluate",
|
|
479
|
+
tags: {
|
|
480
|
+
target: this.opts.target,
|
|
481
|
+
cell: cell.id
|
|
482
|
+
},
|
|
483
|
+
signal: this.opts.signal,
|
|
484
|
+
maximumCharge: this.opts.maximumChargeOf?.(scenario, cell),
|
|
485
|
+
execute: () => this.opts.evaluate(scenario, cell),
|
|
486
|
+
receipt: (evaluation) => {
|
|
487
|
+
const cost = this.opts.costOf(scenario, cell, evaluation);
|
|
488
|
+
if (cost === null) return {
|
|
489
|
+
model: "unattributed",
|
|
490
|
+
inputTokens: 0,
|
|
491
|
+
outputTokens: 0,
|
|
492
|
+
costUnknown: true
|
|
493
|
+
};
|
|
494
|
+
if (!Number.isFinite(cost.usd) || cost.usd < 0) throw new RangeError(`BehaviorExplorer: costOf returned an invalid usd (${String(cost.usd)}) — return null when cost is unknown, never a fabricated number`);
|
|
495
|
+
return {
|
|
496
|
+
model: cost.model ?? "unattributed",
|
|
497
|
+
inputTokens: 0,
|
|
498
|
+
outputTokens: 0,
|
|
499
|
+
actualCostUsd: cost.usd
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
}) : void 0;
|
|
503
|
+
if (paid && !paid.succeeded) {
|
|
504
|
+
if (paid.error instanceof CostReceiptCaptureError && paid.error.receiptError instanceof RangeError) throw paid.error.receiptError;
|
|
505
|
+
throw paid.error;
|
|
506
|
+
}
|
|
507
|
+
const ev = paid ? paid.value : await this.opts.evaluate(scenario, cell);
|
|
508
|
+
const latencyMs = ev.latencyMs ?? performance.now() - startedAt;
|
|
509
|
+
this.runsUsed++;
|
|
510
|
+
runsThisStep++;
|
|
511
|
+
this.consecutiveEvalErrors = 0;
|
|
512
|
+
const costUsd = paid ? paid.receipt.costUnknown ? null : paid.receipt.costUsd : void 0;
|
|
513
|
+
if (paid?.receipt.actualCostUsd !== void 0) this.opts.onCost?.({
|
|
514
|
+
usd: paid.receipt.actualCostUsd,
|
|
515
|
+
channel: "agent"
|
|
516
|
+
});
|
|
517
|
+
const interest = this.objective.interest(ev, this.objectiveContext());
|
|
518
|
+
this.log.push({
|
|
519
|
+
cell,
|
|
520
|
+
ev,
|
|
521
|
+
interest,
|
|
522
|
+
latencyMs,
|
|
523
|
+
...costUsd !== void 0 ? { costUsd } : {},
|
|
524
|
+
scenarioId: this.opts.scenarioId(scenario)
|
|
525
|
+
});
|
|
526
|
+
this.opts.onProgress?.({
|
|
527
|
+
type: "evaluated",
|
|
528
|
+
cell,
|
|
529
|
+
scenario,
|
|
530
|
+
evaluation: ev
|
|
531
|
+
});
|
|
532
|
+
const bin = this.binId(cell, ev.descriptor);
|
|
533
|
+
const cur = this.archiveByBin.get(bin);
|
|
534
|
+
if (!cur || interest > cur.interest) this.archiveByBin.set(bin, {
|
|
535
|
+
binId: bin,
|
|
536
|
+
cell,
|
|
537
|
+
scenario,
|
|
538
|
+
evaluation: ev,
|
|
539
|
+
interest
|
|
540
|
+
});
|
|
541
|
+
if (interest < this.threshold) return;
|
|
542
|
+
this.candidateFindings++;
|
|
543
|
+
if (this.opts.gates?.isValid && !await this.opts.gates.isValid(scenario, ev, cell)) return;
|
|
544
|
+
if (this.opts.gates?.isUncontaminated && !await this.opts.gates.isUncontaminated(scenario, ev, cell)) return;
|
|
545
|
+
const minimized = this.opts.minimize ? await this.opts.minimize(scenario, this.opts.evaluate, cell) : scenario;
|
|
546
|
+
const finding = {
|
|
547
|
+
id: this.opts.scenarioId(scenario),
|
|
548
|
+
cell,
|
|
549
|
+
scenario,
|
|
550
|
+
minimized,
|
|
551
|
+
text: this.opts.scenarioText?.(minimized),
|
|
552
|
+
evaluation: ev,
|
|
553
|
+
interest,
|
|
554
|
+
objective: this.objective.kind
|
|
555
|
+
};
|
|
556
|
+
this._findings.push(finding);
|
|
557
|
+
newFindings.push(finding);
|
|
558
|
+
this.opts.onProgress?.({
|
|
559
|
+
type: "finding",
|
|
560
|
+
finding
|
|
561
|
+
});
|
|
562
|
+
} catch (err) {
|
|
563
|
+
if (err instanceof RangeError) throw err;
|
|
564
|
+
this.evalErrors++;
|
|
565
|
+
this.consecutiveEvalErrors++;
|
|
566
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
567
|
+
this.opts.onProgress?.({
|
|
568
|
+
type: "eval-error",
|
|
569
|
+
cell,
|
|
570
|
+
scenarioId: this.opts.scenarioId(scenario),
|
|
571
|
+
message
|
|
572
|
+
});
|
|
573
|
+
const limit = this.opts.maxConsecutiveEvalErrors ?? 5;
|
|
574
|
+
if (this.consecutiveEvalErrors >= limit) this.stoppedEarly = {
|
|
575
|
+
reason: "eval-errors",
|
|
576
|
+
detail: `${this.consecutiveEvalErrors} consecutive eval errors (last: ${message})`
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
}, this.opts.concurrency ?? 1, this.opts.signal);
|
|
580
|
+
}
|
|
581
|
+
this.opts.onProgress?.({
|
|
582
|
+
type: "round",
|
|
583
|
+
runsUsed: this.runsUsed,
|
|
584
|
+
budget: this.opts.budget
|
|
585
|
+
});
|
|
586
|
+
return {
|
|
587
|
+
runs: runsThisStep,
|
|
588
|
+
findings: newFindings
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
/** Loop `step()` until the run or dollar budget is spent, the signal aborts,
|
|
592
|
+
* or no progress is made. */
|
|
593
|
+
async run() {
|
|
594
|
+
while (this.runsUsed < this.opts.budget && this.stoppedEarly === void 0 && !this.opts.signal?.aborted) {
|
|
595
|
+
const { runs } = await this.step();
|
|
596
|
+
if (runs === 0 && this.stoppedEarly === void 0) break;
|
|
597
|
+
}
|
|
598
|
+
return this.capsule();
|
|
599
|
+
}
|
|
600
|
+
coverage() {
|
|
601
|
+
return this.capsule().coverage;
|
|
602
|
+
}
|
|
603
|
+
findings() {
|
|
604
|
+
return [...this._findings].sort((a, b) => b.interest - a.interest);
|
|
605
|
+
}
|
|
606
|
+
capsule() {
|
|
607
|
+
const cost = this.costLedger?.summary({
|
|
608
|
+
phase: this.costPhase,
|
|
609
|
+
tags: { target: this.opts.target }
|
|
610
|
+
});
|
|
611
|
+
return buildCapsule({
|
|
612
|
+
target: this.opts.target,
|
|
613
|
+
objective: this.objective.kind,
|
|
614
|
+
cells: this.cells,
|
|
615
|
+
log: this.log,
|
|
616
|
+
threshold: this.threshold,
|
|
617
|
+
archive: [...this.archiveByBin.values()],
|
|
618
|
+
findings: this._findings,
|
|
619
|
+
candidateFindings: this.candidateFindings,
|
|
620
|
+
runsUsed: this.runsUsed,
|
|
621
|
+
cost: cost ? {
|
|
622
|
+
costUsd: cost.totalCostUsd,
|
|
623
|
+
costUnknownRuns: cost.byChannel.reduce((sum, row) => sum + row.unpricedCalls, 0)
|
|
624
|
+
} : void 0,
|
|
625
|
+
evalErrors: this.evalErrors,
|
|
626
|
+
stoppedEarly: this.stoppedEarly
|
|
627
|
+
});
|
|
628
|
+
}
|
|
597
629
|
};
|
|
598
|
-
|
|
599
|
-
|
|
630
|
+
//#endregion
|
|
631
|
+
//#region src/fuzz/fuzz-agent.ts
|
|
632
|
+
/**
|
|
633
|
+
* `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
|
|
634
|
+
*
|
|
635
|
+
* One call: explore the space to budget with the adversarial objective and
|
|
636
|
+
* return the capsule. For agent-driven, incremental, or multi-objective use,
|
|
637
|
+
* construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
|
|
638
|
+
*/
|
|
600
639
|
async function fuzzAgent(opts) {
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
return { capsule: await explorer.run() };
|
|
640
|
+
const { failureThreshold, ...rest } = opts;
|
|
641
|
+
return { capsule: await new BehaviorExplorer({
|
|
642
|
+
...rest,
|
|
643
|
+
objective: adversarialObjective(failureThreshold ?? .5)
|
|
644
|
+
}).run() };
|
|
607
645
|
}
|
|
608
|
-
|
|
609
|
-
|
|
646
|
+
//#endregion
|
|
647
|
+
//#region src/fuzz/gates.ts
|
|
648
|
+
/** Combine gate sets; a candidate must pass every gate in every set. */
|
|
610
649
|
function composeGates(...sets) {
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
}
|
|
623
|
-
return true;
|
|
624
|
-
}
|
|
625
|
-
};
|
|
650
|
+
const present = sets.filter((s) => s != null);
|
|
651
|
+
return {
|
|
652
|
+
isValid: async (scenario, ev, cell) => {
|
|
653
|
+
for (const g of present) if (g.isValid && !await g.isValid(scenario, ev, cell)) return false;
|
|
654
|
+
return true;
|
|
655
|
+
},
|
|
656
|
+
isUncontaminated: async (scenario, ev, cell) => {
|
|
657
|
+
for (const g of present) if (g.isUncontaminated && !await g.isUncontaminated(scenario, ev, cell)) return false;
|
|
658
|
+
return true;
|
|
659
|
+
}
|
|
660
|
+
};
|
|
626
661
|
}
|
|
662
|
+
/**
|
|
663
|
+
* Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the
|
|
664
|
+
* flagged scenario; keep the finding only when the rephrase ALSO scores below the
|
|
665
|
+
* threshold. A finding that flips under a cosmetic rewrite was keyed to surface
|
|
666
|
+
* form, not the task — a false signal we must not report. Costs one extra
|
|
667
|
+
* evaluation per candidate (candidates are rare, so cheap).
|
|
668
|
+
*/
|
|
627
669
|
function perturbationStabilityGate(opts) {
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
return re.score < threshold;
|
|
635
|
-
}
|
|
636
|
-
};
|
|
670
|
+
const threshold = opts.failureThreshold ?? .5;
|
|
671
|
+
return { isUncontaminated: async (scenario, _ev, cell) => {
|
|
672
|
+
const rephrased = opts.perturb(scenario);
|
|
673
|
+
if (rephrased == null) return true;
|
|
674
|
+
return (await opts.evaluate(rephrased, cell)).score < threshold;
|
|
675
|
+
} };
|
|
637
676
|
}
|
|
677
|
+
/**
|
|
678
|
+
* Severity-floor gate. Reject borderline candidates whose score sits in a band
|
|
679
|
+
* just under the threshold — judge noise, not a real defect.
|
|
680
|
+
*/
|
|
638
681
|
function severityFloorGate(opts) {
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
isValid: (_scenario, ev) => ev.score <= threshold - margin
|
|
643
|
-
};
|
|
682
|
+
const threshold = opts.failureThreshold ?? .5;
|
|
683
|
+
const margin = opts.margin ?? .1;
|
|
684
|
+
return { isValid: (_scenario, ev) => ev.score <= threshold - margin };
|
|
644
685
|
}
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
686
|
+
//#endregion
|
|
687
|
+
//#region src/fuzz/tools.ts
|
|
688
|
+
/**
|
|
689
|
+
* Agent-drivable surface over a live exploration session.
|
|
690
|
+
*
|
|
691
|
+
* Framework-neutral tool defs ({name, description, parameters: JSON Schema,
|
|
692
|
+
* handler}) so the on-demand agent — not a batch script — drives the search:
|
|
693
|
+
* step it, read coverage, inspect findings, render the capsule. Transport
|
|
694
|
+
* encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
|
|
695
|
+
*/
|
|
696
|
+
const NO_ARGS = {
|
|
697
|
+
type: "object",
|
|
698
|
+
properties: {},
|
|
699
|
+
additionalProperties: false
|
|
700
|
+
};
|
|
648
701
|
function makeExploreTools(explorer) {
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
702
|
+
return [
|
|
703
|
+
{
|
|
704
|
+
name: "explore_step",
|
|
705
|
+
description: "Run one exploration round: allocate budget across cells, propose + evaluate scenarios, archive elites, admit gate-verified findings. Returns runs spent and new findings.",
|
|
706
|
+
parameters: NO_ARGS,
|
|
707
|
+
handler: async () => {
|
|
708
|
+
const { runs, findings } = await explorer.step();
|
|
709
|
+
return {
|
|
710
|
+
runs,
|
|
711
|
+
newFindings: findings.length,
|
|
712
|
+
findings
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
},
|
|
716
|
+
{
|
|
717
|
+
name: "explore_coverage",
|
|
718
|
+
description: "Read the live coverage map: per planned cell — runs, robustness (null = uncovered), finding rate, per-dimension means.",
|
|
719
|
+
parameters: NO_ARGS,
|
|
720
|
+
handler: async () => explorer.coverage()
|
|
721
|
+
},
|
|
722
|
+
{
|
|
723
|
+
name: "explore_findings",
|
|
724
|
+
description: "List gate-verified findings so far, sorted by descending interest.",
|
|
725
|
+
parameters: {
|
|
726
|
+
type: "object",
|
|
727
|
+
properties: { limit: {
|
|
728
|
+
type: "number",
|
|
729
|
+
description: "max findings to return"
|
|
730
|
+
} },
|
|
731
|
+
additionalProperties: false
|
|
732
|
+
},
|
|
733
|
+
handler: async (args) => {
|
|
734
|
+
const limit = args?.limit;
|
|
735
|
+
const all = explorer.findings();
|
|
736
|
+
return typeof limit === "number" ? all.slice(0, limit) : all;
|
|
737
|
+
}
|
|
738
|
+
},
|
|
739
|
+
{
|
|
740
|
+
name: "explore_capsule",
|
|
741
|
+
description: "Build the capsule artifact from the current session state. format \"data\" returns the structured CapsuleData; \"html\" returns the standalone page.",
|
|
742
|
+
parameters: {
|
|
743
|
+
type: "object",
|
|
744
|
+
properties: {
|
|
745
|
+
format: {
|
|
746
|
+
type: "string",
|
|
747
|
+
enum: ["data", "html"]
|
|
748
|
+
},
|
|
749
|
+
generatedAt: {
|
|
750
|
+
type: "string",
|
|
751
|
+
description: "ISO timestamp to stamp into the page"
|
|
752
|
+
}
|
|
753
|
+
},
|
|
754
|
+
additionalProperties: false
|
|
755
|
+
},
|
|
756
|
+
handler: async (args) => {
|
|
757
|
+
const a = args ?? {};
|
|
758
|
+
const capsule = explorer.capsule();
|
|
759
|
+
if (a.format === "html") return renderCapsuleHtml(capsule, { generatedAt: a.generatedAt });
|
|
760
|
+
return capsule;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
];
|
|
698
764
|
}
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
buildCapsule,
|
|
703
|
-
buildCoverage,
|
|
704
|
-
cellId,
|
|
705
|
-
composeGates,
|
|
706
|
-
enumerateCells,
|
|
707
|
-
fuzzAgent,
|
|
708
|
-
makeExploreTools,
|
|
709
|
-
mutationProposer,
|
|
710
|
-
noveltyObjective,
|
|
711
|
-
perturbationStabilityGate,
|
|
712
|
-
renderCapsuleHtml,
|
|
713
|
-
severityFloorGate
|
|
714
|
-
};
|
|
765
|
+
//#endregion
|
|
766
|
+
export { BehaviorExplorer, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };
|
|
767
|
+
|
|
715
768
|
//# sourceMappingURL=fuzz.js.map
|