@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/fuzz.d.ts
CHANGED
|
@@ -1,285 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
outputTokens: number;
|
|
6
|
-
/** Reasoning-token subset of outputTokens, when reported. */
|
|
7
|
-
reasoningTokens?: number;
|
|
8
|
-
/** Prompt tokens served from a provider cache. */
|
|
9
|
-
cachedTokens?: number;
|
|
10
|
-
/** Prompt tokens written into a provider cache. */
|
|
11
|
-
cacheWriteTokens?: number;
|
|
12
|
-
}
|
|
13
|
-
interface CostCallBase {
|
|
14
|
-
callId: string;
|
|
15
|
-
channel: CostChannel;
|
|
16
|
-
phase: string;
|
|
17
|
-
actor: string;
|
|
18
|
-
model: string;
|
|
19
|
-
maximumCostUsd?: number;
|
|
20
|
-
tags?: Record<string, string>;
|
|
21
|
-
timestamp: number;
|
|
22
|
-
}
|
|
23
|
-
interface PendingCostCall extends CostCallBase {
|
|
24
|
-
status: 'pending';
|
|
25
|
-
}
|
|
26
|
-
interface PendingCostCallView extends PendingCostCall {
|
|
27
|
-
state: 'active' | 'late' | 'interrupted';
|
|
28
|
-
}
|
|
29
|
-
interface CostReceipt extends CostCallBase, CostUsage {
|
|
30
|
-
status: 'settled';
|
|
31
|
-
costUsd: number;
|
|
32
|
-
costUnknown: boolean;
|
|
33
|
-
usageUnknown?: boolean;
|
|
34
|
-
/** Rates used to estimate cost locally. Absent when cost is provider-reported or unknown. */
|
|
35
|
-
pricing?: {
|
|
36
|
-
inputUsdPerThousand: number;
|
|
37
|
-
cachedInputUsdPerThousand?: number;
|
|
38
|
-
cacheWriteUsdPerThousand?: number;
|
|
39
|
-
outputUsdPerThousand: number;
|
|
40
|
-
};
|
|
41
|
-
/** Cost reported by the provider, not a local token-price calculation. */
|
|
42
|
-
actualCostUsd?: number;
|
|
43
|
-
error?: string;
|
|
44
|
-
}
|
|
45
|
-
interface CostReceiptInput extends CostUsage {
|
|
46
|
-
model: string;
|
|
47
|
-
/** Caller-supplied rates for a local estimate when the provider does not report billed cost. */
|
|
48
|
-
customTokenPricing?: CustomTokenPricing;
|
|
49
|
-
actualCostUsd?: number;
|
|
50
|
-
costUnknown?: boolean;
|
|
51
|
-
usageUnknown?: boolean;
|
|
52
|
-
}
|
|
53
|
-
/** Per-million token rates for a model or endpoint not covered by package pricing. */
|
|
54
|
-
interface CustomTokenPricing {
|
|
55
|
-
/** Non-cached input tokens. */
|
|
56
|
-
inputUsdPerMillion: number;
|
|
57
|
-
/** Cache-read tokens. Falls back to the normal input rate when omitted. */
|
|
58
|
-
cachedInputUsdPerMillion?: number;
|
|
59
|
-
/** Cache-creation or cache-write tokens. Falls back to the normal input rate when omitted. */
|
|
60
|
-
cacheWriteUsdPerMillion?: number;
|
|
61
|
-
outputUsdPerMillion: number;
|
|
62
|
-
}
|
|
63
|
-
type MaximumCharge = {
|
|
64
|
-
externallyEnforcedMaximumUsd: number;
|
|
65
|
-
} | ({
|
|
66
|
-
customTokenPricing: CustomTokenPricing;
|
|
67
|
-
} & Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>) | ({
|
|
68
|
-
model: string;
|
|
69
|
-
} & CostUsage);
|
|
70
|
-
interface RunPaidCallInput<T> {
|
|
71
|
-
callId?: string;
|
|
72
|
-
channel: CostChannel;
|
|
73
|
-
phase: string;
|
|
74
|
-
actor: string;
|
|
75
|
-
/** Used before a provider receipt exists and on failures without one. */
|
|
76
|
-
model?: string;
|
|
77
|
-
tags?: Record<string, string>;
|
|
78
|
-
signal?: AbortSignal;
|
|
79
|
-
/** Provider-enforced dollar maximum, or maximum token usage with known pricing. Required when capped. */
|
|
80
|
-
maximumCharge?: MaximumCharge;
|
|
81
|
-
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
82
|
-
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
83
|
-
receipt(value: T): CostReceiptInput;
|
|
84
|
-
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
85
|
-
}
|
|
86
|
-
type PaidCallResult<T> = {
|
|
87
|
-
succeeded: true;
|
|
88
|
-
callId: string;
|
|
89
|
-
value: T;
|
|
90
|
-
receipt: CostReceipt;
|
|
91
|
-
} | {
|
|
92
|
-
succeeded: false;
|
|
93
|
-
callId?: string;
|
|
94
|
-
error: Error;
|
|
95
|
-
receipt?: CostReceipt;
|
|
96
|
-
};
|
|
97
|
-
interface ChannelRollup {
|
|
98
|
-
channel: CostChannel;
|
|
99
|
-
calls: number;
|
|
100
|
-
inputTokens: number;
|
|
101
|
-
outputTokens: number;
|
|
102
|
-
reasoningTokens?: number;
|
|
103
|
-
cachedTokens: number;
|
|
104
|
-
cacheWriteTokens?: number;
|
|
105
|
-
costUsd: number;
|
|
106
|
-
unpricedCalls: number;
|
|
107
|
-
unknownUsageCalls: number;
|
|
108
|
-
}
|
|
109
|
-
interface CostLedgerSummary {
|
|
110
|
-
totalCalls: number;
|
|
111
|
-
pendingCalls: number;
|
|
112
|
-
unresolvedCalls: number;
|
|
113
|
-
reservedCostUsd: number;
|
|
114
|
-
inputTokens: number;
|
|
115
|
-
outputTokens: number;
|
|
116
|
-
reasoningTokens?: number;
|
|
117
|
-
cachedTokens: number;
|
|
118
|
-
cacheWriteTokens?: number;
|
|
119
|
-
totalCostUsd: number;
|
|
120
|
-
byChannel: ChannelRollup[];
|
|
121
|
-
unpricedModels: string[];
|
|
122
|
-
fullyPriced: boolean;
|
|
123
|
-
usageComplete: boolean;
|
|
124
|
-
accountingComplete: boolean;
|
|
125
|
-
incompleteReasons: string[];
|
|
126
|
-
}
|
|
127
|
-
interface CostLedgerFilter {
|
|
128
|
-
channel?: CostChannel;
|
|
129
|
-
phase?: string;
|
|
130
|
-
tags?: Record<string, string>;
|
|
131
|
-
}
|
|
132
|
-
interface CostLedgerWaitOptions {
|
|
133
|
-
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
134
|
-
timeoutMs?: number;
|
|
135
|
-
/** Wait only for calls matching this attribution filter. */
|
|
136
|
-
filter?: CostLedgerFilter;
|
|
137
|
-
}
|
|
138
|
-
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
139
|
-
interface CostLedgerPersistence {
|
|
140
|
-
read(): {
|
|
141
|
-
revision: string;
|
|
142
|
-
events: string;
|
|
143
|
-
};
|
|
144
|
-
append(expectedRevision: string, event: string): string | undefined;
|
|
145
|
-
}
|
|
146
|
-
interface CostLedgerOptions {
|
|
147
|
-
costCeilingUsd?: number;
|
|
148
|
-
persistence?: CostLedgerPersistence;
|
|
149
|
-
/** Import already-settled receipts without admitting new paid work. */
|
|
150
|
-
receipts?: readonly CostReceipt[];
|
|
151
|
-
}
|
|
152
|
-
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
153
|
-
declare class CostLedger {
|
|
154
|
-
private readonly records;
|
|
155
|
-
private readonly activeCallIds;
|
|
156
|
-
private readonly lateCallIds;
|
|
157
|
-
private readonly idleWaiters;
|
|
158
|
-
private completedTasks;
|
|
159
|
-
private revision;
|
|
160
|
-
private costLimitPersisted;
|
|
161
|
-
readonly costCeilingUsd?: number;
|
|
162
|
-
private readonly persistence?;
|
|
163
|
-
constructor(input?: number | CostLedgerOptions);
|
|
164
|
-
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
165
|
-
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
166
|
-
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
167
|
-
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
168
|
-
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
169
|
-
error?: string;
|
|
170
|
-
}): CostReceipt;
|
|
171
|
-
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
172
|
-
/** Read pending calls without exposing mutable ledger state. */
|
|
173
|
-
listPending(filter?: CostLedgerFilter): PendingCostCallView[];
|
|
174
|
-
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
175
|
-
markCompleted(count?: number): void;
|
|
176
|
-
costPerCompletedTask(): number | null;
|
|
177
|
-
private execute;
|
|
178
|
-
private captureLateOutcome;
|
|
179
|
-
private releaseActiveCall;
|
|
180
|
-
private commitOutcome;
|
|
181
|
-
private captureFailure;
|
|
182
|
-
private commitReceipt;
|
|
183
|
-
private resolveMaximum;
|
|
184
|
-
private hasIncompleteSettledCall;
|
|
185
|
-
private appendRecord;
|
|
186
|
-
private ensureCostLimitPersisted;
|
|
187
|
-
private appendEvent;
|
|
188
|
-
}
|
|
189
|
-
/** Public callback surface for a shared cost ledger.
|
|
190
|
-
*
|
|
191
|
-
* Declaration bundles may expose this type through multiple package subpaths.
|
|
192
|
-
* Keeping callback contracts structural lets those subpaths compose while the
|
|
193
|
-
* concrete {@link CostLedger} retains its private durable state.
|
|
194
|
-
*/
|
|
195
|
-
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'listPending' | 'waitForIdle'>> & Partial<Pick<CostLedger, 'listPending' | 'waitForIdle'>>;
|
|
196
|
-
|
|
197
|
-
/**
|
|
198
|
-
* Adversarial mutation contract.
|
|
199
|
-
*
|
|
200
|
-
* `AdversarialMutation<S>` is the scenario-mutation strategy the fuzz harness
|
|
201
|
-
* (`fuzzAgent`, src/fuzz) drives: paraphrase, edge-case substitution, or
|
|
202
|
-
* compositional combination of a scenario the policy currently passes, looking
|
|
203
|
-
* for the tail inputs that break it. The harness supplies the loop; consumers
|
|
204
|
-
* supply the mutations and the failure detector.
|
|
205
|
-
*/
|
|
206
|
-
interface AdversarialMutation<S> {
|
|
207
|
-
id: string;
|
|
208
|
-
/**
|
|
209
|
-
* Mutate one scenario. Return null to skip; return one or more new
|
|
210
|
-
* scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
|
|
211
|
-
*/
|
|
212
|
-
mutate(parent: S, rng: () => number): Promise<S[]> | S[];
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
/**
|
|
216
|
-
* Validator-output verdict — substrate primitive for "did this output pass,
|
|
217
|
-
* and how well?"
|
|
218
|
-
*
|
|
219
|
-
* Used by:
|
|
220
|
-
* - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
|
|
221
|
-
* - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
|
|
222
|
-
* Runtime keeps `Validator` because it's coupled to runtime-shaped
|
|
223
|
-
* `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
|
|
224
|
-
* itself is a substrate concept and lives here.
|
|
225
|
-
*
|
|
226
|
-
* Repo layering: agent-eval is the substrate (no upward deps). Both
|
|
227
|
-
* agent-runtime and agent-knowledge consume this type FROM agent-eval —
|
|
228
|
-
* never the other way around. See CLAUDE.md "Repo layering" for the rule.
|
|
229
|
-
*/
|
|
230
|
-
/**
|
|
231
|
-
* Minimal verdict shape — `valid` + `score` are required; `scores` +
|
|
232
|
-
* `notes` are optional surface. Validators that need richer shapes
|
|
233
|
-
* parameterise `Validator<Output, MyVerdict>` with their own type.
|
|
234
|
-
*
|
|
235
|
-
* Need structured extras? Extend DefaultVerdict with typed fields — never
|
|
236
|
-
* serialize extras into `notes`.
|
|
237
|
-
*/
|
|
238
|
-
interface DefaultVerdict {
|
|
239
|
-
/** Whether the output meets the validator's pass criteria. */
|
|
240
|
-
valid: boolean;
|
|
241
|
-
/** Aggregate score in [0, 1]. Drivers use this for winner selection. */
|
|
242
|
-
score: number;
|
|
243
|
-
/** Per-dimension scores. Free-form; weighted into `score` by the validator. */
|
|
244
|
-
scores?: Record<string, number>;
|
|
245
|
-
/** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
|
|
246
|
-
notes?: string;
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
/**
|
|
250
|
-
* Behavior-space exploration — types.
|
|
251
|
-
*
|
|
252
|
-
* One engine searches a space of inputs against a target, scores each run with a
|
|
253
|
-
* multi-objective verdict, keeps a quality-diversity archive, and admits only
|
|
254
|
-
* findings that pass the validity gates. Adversarial fuzzing is the headline
|
|
255
|
-
* preset (`fuzzAgent`); swapping the `Objective` re-points the same engine at
|
|
256
|
-
* novelty search, curriculum growth, or user-simulation.
|
|
257
|
-
*
|
|
258
|
-
* Two kinds of coordinates, deliberately distinct:
|
|
259
|
-
* - INPUT axes (`space.axes`) are the stratification plan — enumerable up front,
|
|
260
|
-
* so allocation and the coverage denominator (planned vs covered) are honest.
|
|
261
|
-
* - MEASURED descriptors (`descriptor(scenario, ev)`) are read off the rollout —
|
|
262
|
-
* they bin the archive by what the agent DID, and never inflate the coverage
|
|
263
|
-
* denominator (a behavior you haven't seen yet is not a planned cell).
|
|
264
|
-
*
|
|
265
|
-
* An `Evaluation` IS a `DefaultVerdict` — same spine as judges and verifiers,
|
|
266
|
-
* never a parallel score shape.
|
|
267
|
-
*/
|
|
268
|
-
|
|
1
|
+
import { t as DefaultVerdict } from "./verdict-Dps8_okt.js";
|
|
2
|
+
import { a as CostChannel, b as MaximumCharge, c as CostLedgerHandle } from "./cost-ledger-Dye6jCgg.js";
|
|
3
|
+
import { t as AdversarialMutation } from "./adversarial-smnADNFS.js";
|
|
4
|
+
//#region src/fuzz/types.d.ts
|
|
269
5
|
/** One input axis of the stratification plan (e.g. matterType, difficulty, personaRigor). */
|
|
270
6
|
interface SpaceAxis {
|
|
271
|
-
|
|
272
|
-
|
|
7
|
+
name: string;
|
|
8
|
+
values: string[];
|
|
273
9
|
}
|
|
274
10
|
/** The input space to stratify. Cells are the cartesian product of the axes. */
|
|
275
11
|
interface BehaviorSpace {
|
|
276
|
-
|
|
12
|
+
axes: SpaceAxis[];
|
|
277
13
|
}
|
|
278
14
|
/** One input cell: a coordinate in the stratification plan. */
|
|
279
15
|
interface Cell {
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
16
|
+
/** Stable id, e.g. `matterType=nda|difficulty=hard`. */
|
|
17
|
+
id: string;
|
|
18
|
+
coords: Record<string, string>;
|
|
283
19
|
}
|
|
284
20
|
/**
|
|
285
21
|
* The outcome of running the target against one scenario: a `DefaultVerdict`
|
|
@@ -288,31 +24,31 @@ interface Cell {
|
|
|
288
24
|
* WHICH dimension is weak only when evaluations carry it.
|
|
289
25
|
*/
|
|
290
26
|
interface Evaluation extends DefaultVerdict {
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
27
|
+
/** Measured behavior coordinates (e.g. `{ outcome: 'refused' }`). Bins the archive. */
|
|
28
|
+
descriptor?: Record<string, string>;
|
|
29
|
+
/** Surfaced output — drives exemplars + minimization. */
|
|
30
|
+
output?: string;
|
|
31
|
+
/** RunRecord id when the target persisted a trace. */
|
|
32
|
+
runId?: string;
|
|
33
|
+
/** Structured labels, e.g. failure classes (`hallucination`, `refusal`). */
|
|
34
|
+
labels?: string[];
|
|
35
|
+
/** Wall-clock for the evaluation, when the consumer measures it more precisely
|
|
36
|
+
* than the engine can (e.g. excluding judge time). Engine-measured otherwise. */
|
|
37
|
+
latencyMs?: number;
|
|
302
38
|
}
|
|
303
39
|
/** Run the target against one scenario in a cell. */
|
|
304
40
|
type Evaluator<S> = (scenario: S, cell: Cell) => Promise<Evaluation>;
|
|
305
41
|
/** Context a proposer sees — prior elites + findings let a skill-backed proposer steer. */
|
|
306
42
|
interface ProposeContext<S> {
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
43
|
+
cell: Cell;
|
|
44
|
+
seeds: S[];
|
|
45
|
+
/** Current archive elites whose input cell matches. */
|
|
46
|
+
elites: S[];
|
|
47
|
+
/** Verified findings so far (read-only) — probe new gaps, not re-found ones. */
|
|
48
|
+
findings: ReadonlyArray<Finding<S>>;
|
|
49
|
+
/** How many candidates to propose. */
|
|
50
|
+
count: number;
|
|
51
|
+
rng: () => number;
|
|
316
52
|
}
|
|
317
53
|
/**
|
|
318
54
|
* Produces candidate scenarios for a cell. A plain function — `mutationProposer`
|
|
@@ -331,14 +67,14 @@ type MutationProposer<S> = (ctx: ProposeContext<S>) => Promise<S[]> | S[];
|
|
|
331
67
|
* score is interesting) and `noveltyObjective` (far from the archive) ship.
|
|
332
68
|
*/
|
|
333
69
|
interface Objective {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
70
|
+
kind: string;
|
|
71
|
+
interest(ev: Evaluation, ctx: ObjectiveContext): number;
|
|
72
|
+
/** Default 0.5. */
|
|
73
|
+
threshold?: number;
|
|
338
74
|
}
|
|
339
75
|
interface ObjectiveContext {
|
|
340
|
-
|
|
341
|
-
|
|
76
|
+
archiveScores: number[];
|
|
77
|
+
archiveDescriptors: Array<Record<string, string> | undefined>;
|
|
342
78
|
}
|
|
343
79
|
/**
|
|
344
80
|
* Validity gates — the moat. A notable candidate is admitted ONLY when it is a
|
|
@@ -347,113 +83,113 @@ interface ObjectiveContext {
|
|
|
347
83
|
* LLM); live wiring supplies real gates.
|
|
348
84
|
*/
|
|
349
85
|
interface ValidityGates<S> {
|
|
350
|
-
|
|
351
|
-
|
|
86
|
+
isValid?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
|
|
87
|
+
isUncontaminated?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
|
|
352
88
|
}
|
|
353
89
|
/** A gate-verified, minimized finding — the unit the capsule reports. */
|
|
354
90
|
interface Finding<S> {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
91
|
+
id: string;
|
|
92
|
+
cell: Cell;
|
|
93
|
+
scenario: S;
|
|
94
|
+
/** The minimized trigger (== `scenario` when no minimizer is supplied). */
|
|
95
|
+
minimized: S;
|
|
96
|
+
/** Legible text of the minimized trigger, when `scenarioText` is supplied. */
|
|
97
|
+
text?: string;
|
|
98
|
+
/** The full multi-objective verdict. */
|
|
99
|
+
evaluation: Evaluation;
|
|
100
|
+
/** The objective's interest score that flagged it. */
|
|
101
|
+
interest: number;
|
|
102
|
+
/** Which objective flagged it. */
|
|
103
|
+
objective: string;
|
|
368
104
|
}
|
|
369
105
|
/** An archive elite — the most interesting scenario seen for one bin. */
|
|
370
106
|
interface ArchiveEntry<S> {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
107
|
+
/** Input cell + measured descriptor coords combined, e.g. `difficulty=hard|outcome=refused`. */
|
|
108
|
+
binId: string;
|
|
109
|
+
cell: Cell;
|
|
110
|
+
scenario: S;
|
|
111
|
+
evaluation: Evaluation;
|
|
112
|
+
interest: number;
|
|
377
113
|
}
|
|
378
114
|
/** Summary of a sample — every aggregate carries its spread, never a bare mean. */
|
|
379
115
|
interface Distribution {
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
116
|
+
mean: number;
|
|
117
|
+
median: number;
|
|
118
|
+
p90: number;
|
|
119
|
+
min: number;
|
|
120
|
+
max: number;
|
|
121
|
+
n: number;
|
|
386
122
|
}
|
|
387
123
|
/** Per-INPUT-cell coverage — the planned-vs-covered map. */
|
|
388
124
|
interface CoverageCell {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
125
|
+
cell: Cell;
|
|
126
|
+
runs: number;
|
|
127
|
+
/** Headline score distribution in [0,1]; `null` when the cell was never run
|
|
128
|
+
* (honestly uncovered — never a fabricated zero). */
|
|
129
|
+
score: Distribution | null;
|
|
130
|
+
/** Fraction of runs the objective flagged as notable. */
|
|
131
|
+
findingRate: number;
|
|
132
|
+
/** Per-dimension score distributions — surfaces WHICH dimension is weak and
|
|
133
|
+
* how consistently. */
|
|
134
|
+
dimensions: Record<string, Distribution>;
|
|
135
|
+
/** Evaluation wall-clock per run; engine-measured unless the evaluation
|
|
136
|
+
* carried its own `latencyMs`. `null` when the cell was never run. */
|
|
137
|
+
latencyMs: Distribution | null;
|
|
138
|
+
/** Known dollars spent in this cell — present only when cost tracking was
|
|
139
|
+
* wired; runs with unknown cost are counted apart, never folded in as $0. */
|
|
140
|
+
costUsd?: number;
|
|
141
|
+
costUnknownRuns?: number;
|
|
406
142
|
}
|
|
407
143
|
/** The artifact every exploration produces. */
|
|
408
144
|
interface CapsuleData<S> {
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
};
|
|
145
|
+
target: string;
|
|
146
|
+
objective: string;
|
|
147
|
+
/** Stamped by the caller — the engine stays clock-free and deterministic. */
|
|
148
|
+
generatedAt?: string;
|
|
149
|
+
coverage: CoverageCell[];
|
|
150
|
+
/** Verified findings, sorted by descending interest. */
|
|
151
|
+
findings: Finding<S>[];
|
|
152
|
+
/** QD archive elites (binned by input × measured coords). */
|
|
153
|
+
archive: ArchiveEntry<S>[];
|
|
154
|
+
/** Post-harden lift, filled by a second pass after an improvement. */
|
|
155
|
+
lift?: {
|
|
156
|
+
before: number;
|
|
157
|
+
after: number;
|
|
158
|
+
verdict: string;
|
|
159
|
+
};
|
|
160
|
+
stats: {
|
|
161
|
+
totalRuns: number;
|
|
162
|
+
/** Input-cell denominator — the stratification plan. */
|
|
163
|
+
cellsTotal: number;
|
|
164
|
+
cellsCovered: number;
|
|
165
|
+
/** Distinct measured-descriptor bins observed (never part of the denominator). */
|
|
166
|
+
behaviorBinsObserved: number;
|
|
167
|
+
candidateFindings: number;
|
|
168
|
+
verifiedFindings: number;
|
|
169
|
+
/** Distribution of per-cell mean scores across covered cells (cells weigh
|
|
170
|
+
* equally — variance steering sends more runs to weak cells, so a
|
|
171
|
+
* run-weighted average would bias low). `null` when nothing ran. */
|
|
172
|
+
robustness: Distribution | null;
|
|
173
|
+
/** Evaluation wall-clock across all runs. `null` when nothing ran. */
|
|
174
|
+
latencyMs: Distribution | null;
|
|
175
|
+
/** Known dollars spent on this exploration's runs. Present only when cost
|
|
176
|
+
* tracking was wired (`costOf`) — absent means "not tracked", never $0. */
|
|
177
|
+
costUsd?: number;
|
|
178
|
+
/** Runs whose cost was unknown (`costOf` returned null) — counted apart,
|
|
179
|
+
* never folded into `costUsd` as a fabricated $0. */
|
|
180
|
+
costUnknownRuns?: number;
|
|
181
|
+
/** Evaluations that threw (transport/backend failures). They consumed no
|
|
182
|
+
* run budget and scored nothing — an infra axis, never folded into
|
|
183
|
+
* robustness or reported as findings. */
|
|
184
|
+
evalErrors: number;
|
|
185
|
+
/** Present when the run stopped before its budget because consecutive
|
|
186
|
+
* eval errors tripped the circuit breaker (a dead backend must not burn
|
|
187
|
+
* the remaining budget). The capsule-so-far is complete and honest. */
|
|
188
|
+
stoppedEarly?: {
|
|
189
|
+
reason: 'eval-errors';
|
|
190
|
+
detail: string;
|
|
456
191
|
};
|
|
192
|
+
};
|
|
457
193
|
}
|
|
458
194
|
/**
|
|
459
195
|
* Known cost of one evaluated run. `model` attributes the spend in the ledger's
|
|
@@ -461,121 +197,111 @@ interface CapsuleData<S> {
|
|
|
461
197
|
* real either way — recorded as `actualCostUsd`, never an estimate).
|
|
462
198
|
*/
|
|
463
199
|
interface RunCost {
|
|
464
|
-
|
|
465
|
-
|
|
200
|
+
usd: number;
|
|
201
|
+
model?: string;
|
|
466
202
|
}
|
|
467
203
|
type ExploreEvent<S> = {
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
204
|
+
type: 'cell-allocated';
|
|
205
|
+
cell: Cell;
|
|
206
|
+
count: number;
|
|
471
207
|
} | {
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
208
|
+
type: 'evaluated';
|
|
209
|
+
cell: Cell;
|
|
210
|
+
scenario: S;
|
|
211
|
+
evaluation: Evaluation;
|
|
476
212
|
} | {
|
|
477
|
-
|
|
478
|
-
|
|
213
|
+
type: 'finding';
|
|
214
|
+
finding: Finding<S>;
|
|
479
215
|
} | {
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
216
|
+
type: 'eval-error';
|
|
217
|
+
cell: Cell;
|
|
218
|
+
scenarioId: string;
|
|
219
|
+
message: string;
|
|
484
220
|
} | {
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
221
|
+
type: 'round';
|
|
222
|
+
runsUsed: number;
|
|
223
|
+
budget: number;
|
|
488
224
|
};
|
|
489
225
|
interface ExploreOptions<S> {
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
226
|
+
/** Name of the target under exploration — labels the capsule. */
|
|
227
|
+
target: string;
|
|
228
|
+
/** The input stratification plan. */
|
|
229
|
+
space: BehaviorSpace;
|
|
230
|
+
/** Candidate generator. */
|
|
231
|
+
proposer: MutationProposer<S>;
|
|
232
|
+
/** Runs the target → multi-objective `Evaluation`. */
|
|
233
|
+
evaluate: Evaluator<S>;
|
|
234
|
+
/** Seed corpus per cell. */
|
|
235
|
+
seedsFor: (cell: Cell) => S[] | Promise<S[]>;
|
|
236
|
+
/** Stable id for a scenario (dedup + lineage). */
|
|
237
|
+
scenarioId: (scenario: S) => string;
|
|
238
|
+
/** Human-legible text — drives capsule exemplars. */
|
|
239
|
+
scenarioText?: (scenario: S) => string;
|
|
240
|
+
/** Measured behavior coords appended to the archive bin. Default: input cell only. */
|
|
241
|
+
descriptor?: (scenario: S, ev: Evaluation) => Record<string, string>;
|
|
242
|
+
/** What "interesting" means. Default: `adversarialObjective(0.5)`. */
|
|
243
|
+
objective?: Objective;
|
|
244
|
+
/** Validity gates. Default pass-through. */
|
|
245
|
+
gates?: ValidityGates<S>;
|
|
246
|
+
/** Budget steering across input cells. `variance` chases uncertainty; `uniform` is the unsteered ablation baseline. Default `variance`. */
|
|
247
|
+
allocation?: 'variance' | 'uniform';
|
|
248
|
+
/** Total target evaluations. */
|
|
249
|
+
budget: number;
|
|
250
|
+
/** Minimum evaluations per input cell before steering. Default 2. */
|
|
251
|
+
floorPerCell?: number;
|
|
252
|
+
/** Shrink a notable scenario to its minimal trigger. Default: identity. */
|
|
253
|
+
minimize?: (scenario: S, evaluate: Evaluator<S>, cell: Cell) => Promise<S> | S;
|
|
254
|
+
/** Max concurrent `evaluate` calls. Default 1. */
|
|
255
|
+
concurrency?: number;
|
|
256
|
+
/** Stop the run after this many CONSECUTIVE eval errors (a dead backend must
|
|
257
|
+
* not burn the remaining budget). Successes reset the streak. Default 5. */
|
|
258
|
+
maxConsecutiveEvalErrors?: number;
|
|
259
|
+
/** Cooperative cancellation. */
|
|
260
|
+
signal?: AbortSignal;
|
|
261
|
+
/** Progress stream. */
|
|
262
|
+
onProgress?: (event: ExploreEvent<S>) => void;
|
|
263
|
+
/** Deterministic seed. Default 1. */
|
|
264
|
+
seed?: number;
|
|
265
|
+
/**
|
|
266
|
+
* Cost of one evaluated run — consumer-supplied; the explorer cannot know
|
|
267
|
+
* token usage. Return null when the cost is unknown: the run is COUNTED in
|
|
268
|
+
* `stats.costUnknownRuns`, never folded into the total as $0. Required by
|
|
269
|
+
* every other cost option (`costBudgetUsd` / `ledger` / `onCost`).
|
|
270
|
+
*/
|
|
271
|
+
costOf?: (scenario: S, cell: Cell, ev: Evaluation) => RunCost | null;
|
|
272
|
+
/** Pre-call hard maximum. Required whenever the explorer uses a capped ledger. */
|
|
273
|
+
maximumChargeOf?: (scenario: S, cell: Cell) => MaximumCharge;
|
|
274
|
+
/**
|
|
275
|
+
* Hard dollar cap. Each evaluation reserves `maximumChargeOf` before it starts;
|
|
276
|
+
* calls that do not fit are rejected. Unknown totals stop further paid work.
|
|
277
|
+
*/
|
|
278
|
+
costBudgetUsd?: number;
|
|
279
|
+
/**
|
|
280
|
+
* Sink for per-run cost entries — each known `costOf` result is recorded
|
|
281
|
+
* with channel 'agent' and `actualCostUsd` (token axes are zero: the
|
|
282
|
+
* explorer only sees dollars). Pass the program's shared `CostLedger` so
|
|
283
|
+
* `costReport` stamps fuzz spend alongside judge/analyst spend.
|
|
284
|
+
*/
|
|
285
|
+
ledger?: CostLedgerHandle;
|
|
286
|
+
/** Observer fired for every known-cost run recorded. */
|
|
287
|
+
onCost?: (entry: {
|
|
288
|
+
usd: number;
|
|
289
|
+
channel: CostChannel;
|
|
290
|
+
}) => void;
|
|
555
291
|
}
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
* Input-space tiling + coverage projection.
|
|
559
|
-
*
|
|
560
|
-
* Cells are the cartesian product of the input axes — the stratification plan,
|
|
561
|
-
* enumerable up front so the planned-vs-covered denominator is honest. Coverage
|
|
562
|
-
* is projected from the evaluation log: per cell, the full DISTRIBUTION of the
|
|
563
|
-
* headline score, of each scored dimension, and of evaluation latency — a bare
|
|
564
|
-
* mean hides outliers, so every aggregate carries its spread. Per-cell cost is
|
|
565
|
-
* split known-dollars vs unknown-runs, never folded into a fabricated $0.
|
|
566
|
-
*/
|
|
567
|
-
|
|
292
|
+
//#endregion
|
|
293
|
+
//#region src/fuzz/cube.d.ts
|
|
568
294
|
/** One recorded evaluation — the unit coverage and the capsule are built from. */
|
|
569
295
|
interface EvalRecord {
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
296
|
+
cell: Cell;
|
|
297
|
+
ev: Evaluation;
|
|
298
|
+
/** The objective's interest score for this evaluation. */
|
|
299
|
+
interest: number;
|
|
300
|
+
/** Evaluation wall-clock — engine-measured unless `ev.latencyMs` overrode it. */
|
|
301
|
+
latencyMs: number;
|
|
302
|
+
/** Known dollars for this run. `null` = cost tracking was wired but this
|
|
303
|
+
* run's cost was unknowable (counted apart). Absent = not tracked at all. */
|
|
304
|
+
costUsd?: number | null;
|
|
579
305
|
}
|
|
580
306
|
/** Enumerate every input cell (cartesian product of the axes), in stable order. */
|
|
581
307
|
declare function enumerateCells(space: BehaviorSpace): Cell[];
|
|
@@ -586,132 +312,95 @@ declare function cellId(space: BehaviorSpace, coords: Record<string, string>): s
|
|
|
586
312
|
* no evaluations reports `score: null` (honestly uncovered), never zeros.
|
|
587
313
|
*/
|
|
588
314
|
declare function buildCoverage(cells: Cell[], log: EvalRecord[], threshold: number): CoverageCell[];
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
* The capsule — the artifact every exploration produces.
|
|
592
|
-
*
|
|
593
|
-
* `buildCapsule` assembles coverage + verified findings + the QD archive into a
|
|
594
|
-
* pure `CapsuleData` (no clock, no I/O — deterministic and snapshot-testable).
|
|
595
|
-
* `renderCapsuleHtml` turns it into a standalone page: the input-cell heat-map
|
|
596
|
-
* (planned vs covered), per-dimension weakness chips, and the minimized finding
|
|
597
|
-
* exemplars. One artifact — the hardening map and the shareable proof object.
|
|
598
|
-
*/
|
|
599
|
-
|
|
315
|
+
//#endregion
|
|
316
|
+
//#region src/fuzz/capsule.d.ts
|
|
600
317
|
interface BuildCapsuleInput<S> {
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
318
|
+
target: string;
|
|
319
|
+
objective: string;
|
|
320
|
+
cells: Cell[];
|
|
321
|
+
log: EvalRecord[];
|
|
322
|
+
/** The objective's notable threshold — drives findingRate. */
|
|
323
|
+
threshold: number;
|
|
324
|
+
archive: ArchiveEntry<S>[];
|
|
325
|
+
findings: Finding<S>[];
|
|
326
|
+
candidateFindings: number;
|
|
327
|
+
runsUsed: number;
|
|
328
|
+
/** Known-dollar / unknown-run split — present only when cost tracking was
|
|
329
|
+
* wired; the capsule never fabricates a $0 total. */
|
|
330
|
+
cost?: {
|
|
331
|
+
costUsd: number;
|
|
332
|
+
costUnknownRuns: number;
|
|
333
|
+
};
|
|
334
|
+
/** Evaluations that threw — infra outcomes, never folded into robustness. */
|
|
335
|
+
evalErrors: number;
|
|
336
|
+
/** Set when the consecutive-error circuit breaker stopped the run early. */
|
|
337
|
+
stoppedEarly?: {
|
|
338
|
+
reason: 'eval-errors';
|
|
339
|
+
detail: string;
|
|
340
|
+
};
|
|
624
341
|
}
|
|
625
342
|
declare function buildCapsule<S>(input: BuildCapsuleInput<S>): CapsuleData<S>;
|
|
626
343
|
interface RenderCapsuleOptions {
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
344
|
+
/** Max finding exemplars to show. Default 8. */
|
|
345
|
+
maxFindings?: number;
|
|
346
|
+
/** ISO timestamp to stamp into the page (keeps the pure capsule clock-free). */
|
|
347
|
+
generatedAt?: string;
|
|
631
348
|
}
|
|
632
349
|
/** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
|
|
633
350
|
declare function renderCapsuleHtml<S>(capsule: CapsuleData<S>, opts?: RenderCapsuleOptions): string;
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
* The exploration engine — a stateful session over a behavior space.
|
|
637
|
-
*
|
|
638
|
-
* Each `step()`: allocate budget across INPUT cells (floor first, then variance
|
|
639
|
-
* steering toward the least-certain cells), propose candidates (the proposer
|
|
640
|
-
* reads current elites + findings, so the search deepens generationally),
|
|
641
|
-
* evaluate with bounded concurrency, archive the most interesting scenario per
|
|
642
|
-
* input×measured bin, and admit notable candidates that pass the validity gates.
|
|
643
|
-
* `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
|
|
644
|
-
* the surface `makeExploreTools` exposes so an agent can drive the session.
|
|
645
|
-
*
|
|
646
|
-
* One evaluation log (`EvalRecord[]`) is the source of truth; allocation
|
|
647
|
-
* observations and coverage are projections of it.
|
|
648
|
-
*/
|
|
649
|
-
|
|
351
|
+
//#endregion
|
|
352
|
+
//#region src/fuzz/explorer.d.ts
|
|
650
353
|
declare class BehaviorExplorer<S> {
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
* `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
|
|
693
|
-
*
|
|
694
|
-
* One call: explore the space to budget with the adversarial objective and
|
|
695
|
-
* return the capsule. For agent-driven, incremental, or multi-objective use,
|
|
696
|
-
* construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
|
|
697
|
-
*/
|
|
698
|
-
|
|
354
|
+
private readonly opts;
|
|
355
|
+
private readonly cells;
|
|
356
|
+
private readonly cellById;
|
|
357
|
+
private readonly objective;
|
|
358
|
+
private readonly threshold;
|
|
359
|
+
private readonly floorPerCell;
|
|
360
|
+
private readonly perRoundBudget;
|
|
361
|
+
/** The single evaluation log — coverage + allocation are projections of it. */
|
|
362
|
+
private readonly log;
|
|
363
|
+
/** binId (input × measured coords) → the most interesting entry seen. */
|
|
364
|
+
private readonly archiveByBin;
|
|
365
|
+
private readonly _findings;
|
|
366
|
+
private runsUsed;
|
|
367
|
+
private candidateFindings;
|
|
368
|
+
private evalErrors;
|
|
369
|
+
private consecutiveEvalErrors;
|
|
370
|
+
private stoppedEarly;
|
|
371
|
+
private rngState;
|
|
372
|
+
private readonly costLedger?;
|
|
373
|
+
private readonly costPhase;
|
|
374
|
+
constructor(opts: ExploreOptions<S>);
|
|
375
|
+
private rng;
|
|
376
|
+
private binId;
|
|
377
|
+
private allocate;
|
|
378
|
+
private objectiveContext;
|
|
379
|
+
/** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
|
|
380
|
+
private elitesFor;
|
|
381
|
+
/** One allocate → propose → evaluate → gate → archive round. */
|
|
382
|
+
step(): Promise<{
|
|
383
|
+
runs: number;
|
|
384
|
+
findings: Finding<S>[];
|
|
385
|
+
}>;
|
|
386
|
+
/** Loop `step()` until the run or dollar budget is spent, the signal aborts,
|
|
387
|
+
* or no progress is made. */
|
|
388
|
+
run(): Promise<CapsuleData<S>>;
|
|
389
|
+
coverage(): CoverageCell[];
|
|
390
|
+
findings(): Finding<S>[];
|
|
391
|
+
capsule(): CapsuleData<S>;
|
|
392
|
+
}
|
|
393
|
+
//#endregion
|
|
394
|
+
//#region src/fuzz/fuzz-agent.d.ts
|
|
699
395
|
type FuzzAgentOptions<S> = Omit<ExploreOptions<S>, 'objective'> & {
|
|
700
|
-
|
|
701
|
-
|
|
396
|
+
/** Score strictly below this is a candidate failure. Default 0.5. */
|
|
397
|
+
failureThreshold?: number;
|
|
702
398
|
};
|
|
703
399
|
declare function fuzzAgent<S>(opts: FuzzAgentOptions<S>): Promise<{
|
|
704
|
-
|
|
400
|
+
capsule: CapsuleData<S>;
|
|
705
401
|
}>;
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
* Validity gates — what separates a fuzzer from a slop generator.
|
|
709
|
-
*
|
|
710
|
-
* A notable candidate is admitted only when it is fair and reproducible. None of
|
|
711
|
-
* these are on by default: the live wiring opts in, so reported findings carry
|
|
712
|
-
* their proof.
|
|
713
|
-
*/
|
|
714
|
-
|
|
402
|
+
//#endregion
|
|
403
|
+
//#region src/fuzz/gates.d.ts
|
|
715
404
|
/** Combine gate sets; a candidate must pass every gate in every set. */
|
|
716
405
|
declare function composeGates<S>(...sets: Array<ValidityGates<S> | undefined>): ValidityGates<S>;
|
|
717
406
|
/**
|
|
@@ -722,38 +411,30 @@ declare function composeGates<S>(...sets: Array<ValidityGates<S> | undefined>):
|
|
|
722
411
|
* evaluation per candidate (candidates are rare, so cheap).
|
|
723
412
|
*/
|
|
724
413
|
declare function perturbationStabilityGate<S>(opts: {
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
414
|
+
evaluate: Evaluator<S>;
|
|
415
|
+
/** Produce a semantic-preserving rephrase. Return null to skip (treated as pass). */
|
|
416
|
+
perturb: (scenario: S) => S | null;
|
|
417
|
+
/** Score strictly below this still counts as failing. Default 0.5. */
|
|
418
|
+
failureThreshold?: number;
|
|
730
419
|
}): ValidityGates<S>;
|
|
731
420
|
/**
|
|
732
421
|
* Severity-floor gate. Reject borderline candidates whose score sits in a band
|
|
733
422
|
* just under the threshold — judge noise, not a real defect.
|
|
734
423
|
*/
|
|
735
424
|
declare function severityFloorGate<S>(opts: {
|
|
736
|
-
|
|
737
|
-
|
|
425
|
+
failureThreshold?: number;
|
|
426
|
+
margin?: number;
|
|
738
427
|
}): ValidityGates<S>;
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
* Shipped policies for the exploration engine.
|
|
742
|
-
*
|
|
743
|
-
* `MutationProposer` is a plain function type — an agent running a generator skill
|
|
744
|
-
* IS a proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`
|
|
745
|
-
* builds the deterministic, LLM-free one from mutation operators. Objectives are
|
|
746
|
-
* interfaces because the engine reads `kind` + `threshold` off them.
|
|
747
|
-
*/
|
|
748
|
-
|
|
428
|
+
//#endregion
|
|
429
|
+
//#region src/fuzz/policies.d.ts
|
|
749
430
|
/**
|
|
750
431
|
* Perturbation-based search: apply the cell's mutation operators to the current
|
|
751
432
|
* elites + seeds, deduping by id. Elites first — mutating the most interesting
|
|
752
433
|
* scenario found so far is what makes the search deepen across rounds.
|
|
753
434
|
*/
|
|
754
435
|
declare function mutationProposer<S>(opts: {
|
|
755
|
-
|
|
756
|
-
|
|
436
|
+
mutationsFor: (cell: Cell) => AdversarialMutation<S>[];
|
|
437
|
+
scenarioId: (s: S) => string;
|
|
757
438
|
}): MutationProposer<S>;
|
|
758
439
|
/** Adversarial: a low headline score is interesting — find where the agent fails. */
|
|
759
440
|
declare function adversarialObjective(threshold?: number): Objective;
|
|
@@ -763,25 +444,18 @@ declare function adversarialObjective(threshold?: number): Objective;
|
|
|
763
444
|
* rather than re-finding the same hole.
|
|
764
445
|
*/
|
|
765
446
|
declare function noveltyObjective(threshold?: number): Objective;
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
* Agent-drivable surface over a live exploration session.
|
|
769
|
-
*
|
|
770
|
-
* Framework-neutral tool defs ({name, description, parameters: JSON Schema,
|
|
771
|
-
* handler}) so the on-demand agent — not a batch script — drives the search:
|
|
772
|
-
* step it, read coverage, inspect findings, render the capsule. Transport
|
|
773
|
-
* encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
|
|
774
|
-
*/
|
|
775
|
-
|
|
447
|
+
//#endregion
|
|
448
|
+
//#region src/fuzz/tools.d.ts
|
|
776
449
|
interface ExploreToolDef {
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
450
|
+
name: string;
|
|
451
|
+
description: string;
|
|
452
|
+
/** JSON Schema (draft-07+) for the arguments. */
|
|
453
|
+
parameters: Record<string, unknown>;
|
|
454
|
+
handler: (args: unknown, ctx?: {
|
|
455
|
+
signal?: AbortSignal;
|
|
456
|
+
}) => Promise<unknown>;
|
|
784
457
|
}
|
|
785
458
|
declare function makeExploreTools<S>(explorer: BehaviorExplorer<S>): ExploreToolDef[];
|
|
786
|
-
|
|
459
|
+
//#endregion
|
|
787
460
|
export { type ArchiveEntry, BehaviorExplorer, type BehaviorSpace, type BuildCapsuleInput, type CapsuleData, type Cell, type CoverageCell, type EvalRecord, type Evaluation, type Evaluator, type ExploreEvent, type ExploreOptions, type ExploreToolDef, type Finding, type FuzzAgentOptions, type MutationProposer, type Objective, type ObjectiveContext, type ProposeContext, type RenderCapsuleOptions, type RunCost, type SpaceAxis, type ValidityGates, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };
|
|
461
|
+
//# sourceMappingURL=fuzz.d.ts.map
|