@tangle-network/agent-eval 0.129.0 → 0.130.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +2 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +81 -2872
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -360
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1188
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1709
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -891
- package/dist/benchmarks/index.js +2 -60
- package/dist/benchmarks-BJgDGkAD.js +754 -0
- package/dist/benchmarks-BJgDGkAD.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6381
- package/dist/campaign/index.js +3 -213
- package/dist/campaign-aKJt6emI.js +3886 -0
- package/dist/campaign-aKJt6emI.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -175
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5565
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1938
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -33
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -618
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CD_WZ_Xr.d.ts +2250 -0
- package/dist/index-CD_WZ_Xr.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index-Em67JBjs.d.ts +335 -0
- package/dist/index-Em67JBjs.d.ts.map +1 -0
- package/dist/index.d.ts +3755 -15555
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11182 -11216
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -480
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1312
- package/dist/reporting.js +6 -51
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +760 -4010
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2325 -1958
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -2087
- package/dist/rollout/index.js +8 -168
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-CUmHkGbI.js +7718 -0
- package/dist/skillopt-optimization-method-CUmHkGbI.js.map +1 -0
- package/dist/skillopt-optimization-method-CWKVTnks.d.ts +1740 -0
- package/dist/skillopt-optimization-method-CWKVTnks.d.ts.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -959
- package/dist/supervisor-run/index.js +2 -65
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -252
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1173
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/campaign-proposers.md +1 -0
- package/package.json +17 -9
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2QU3YOPR.js +0 -7374
- package/dist/chunk-2QU3YOPR.js.map +0 -1
- package/dist/chunk-3OCR4R5I.js +0 -728
- package/dist/chunk-3OCR4R5I.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-56TAVBOK.js +0 -698
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7FO3TNPI.js +0 -232
- package/dist/chunk-7FO3TNPI.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BSO5JDQH.js +0 -2335
- package/dist/chunk-BSO5JDQH.js.map +0 -1
- package/dist/chunk-C6LXANRU.js +0 -1550
- package/dist/chunk-C6LXANRU.js.map +0 -1
- package/dist/chunk-DODXQREJ.js +0 -752
- package/dist/chunk-DODXQREJ.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-E7QXT7SX.js +0 -183
- package/dist/chunk-E7QXT7SX.js.map +0 -1
- package/dist/chunk-EG66UGL4.js +0 -341
- package/dist/chunk-EG66UGL4.js.map +0 -1
- package/dist/chunk-FXTVJPYD.js +0 -576
- package/dist/chunk-FXTVJPYD.js.map +0 -1
- package/dist/chunk-G7MGMCZD.js +0 -153
- package/dist/chunk-G7MGMCZD.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-H23X7XKK.js +0 -181
- package/dist/chunk-H23X7XKK.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-HPWUNB47.js +0 -289
- package/dist/chunk-HPWUNB47.js.map +0 -1
- package/dist/chunk-IYCLP2N2.js +0 -766
- package/dist/chunk-IYCLP2N2.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-JQSF5DQT.js +0 -701
- package/dist/chunk-JQSF5DQT.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-M4YBQKIJ.js +0 -1040
- package/dist/chunk-M4YBQKIJ.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NY44NC4A.js +0 -1056
- package/dist/chunk-NY44NC4A.js.map +0 -1
- package/dist/chunk-OIUOT4QD.js +0 -44
- package/dist/chunk-OIUOT4QD.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-OWN5NPMC.js +0 -152
- package/dist/chunk-OWN5NPMC.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PC5DOSM7.js +0 -579
- package/dist/chunk-PC5DOSM7.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-QB6BDBP2.js +0 -4464
- package/dist/chunk-QB6BDBP2.js.map +0 -1
- package/dist/chunk-RXHCETDZ.js +0 -536
- package/dist/chunk-RXHCETDZ.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-SFLLL76A.js +0 -669
- package/dist/chunk-SFLLL76A.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-T6RLYGAD.js +0 -158
- package/dist/chunk-T6RLYGAD.js.map +0 -1
- package/dist/chunk-TJVT4QFF.js +0 -911
- package/dist/chunk-TJVT4QFF.js.map +0 -1
- package/dist/chunk-TQ7LNKZ3.js +0 -136
- package/dist/chunk-TQ7LNKZ3.js.map +0 -1
- package/dist/chunk-U4L7JRPZ.js +0 -1706
- package/dist/chunk-U4L7JRPZ.js.map +0 -1
- package/dist/chunk-U4PHLT2N.js +0 -419
- package/dist/chunk-U4PHLT2N.js.map +0 -1
- package/dist/chunk-VCZ5FQYW.js +0 -928
- package/dist/chunk-VCZ5FQYW.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WVATSFCP.js +0 -1553
- package/dist/chunk-WVATSFCP.js.map +0 -1
- package/dist/chunk-X4YIBDER.js +0 -1662
- package/dist/chunk-X4YIBDER.js.map +0 -1
- package/dist/chunk-YQN4ICPP.js +0 -355
- package/dist/chunk-YQN4ICPP.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZHTZ4EYI.js +0 -1212
- package/dist/chunk-ZHTZ4EYI.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-OJJ7CZF4.js +0 -18
- package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-CD_WZ_Xr.d.ts","names":[],"sources":["../src/campaign/analyst-surface.ts","../src/paired-arms.ts","../src/campaign/cross-surface-types.ts","../src/campaign/cross-surface-interaction.ts","../src/campaign/fixtures.ts","../src/campaign/gates/neutralization-gate.ts","../src/pre-registration.ts","../src/campaign/gates/sequential.ts","../src/campaign/gates/statistical-heldout.ts","../src/campaign/grounded-reflection.ts","../src/campaign/labeled-store/fs-adapter.ts","../src/campaign/neutralize.ts","../src/artifact-validator.ts","../src/completion-verifier.ts","../src/produced-state.ts","../src/agent-profile.ts","../src/integrity/backend-integrity.ts","../src/campaign/presets/run-profile-matrix.ts","../src/campaign/presets/playback.ts","../src/campaign/run-dir.ts","../src/campaign/scenario-selection.ts","../src/campaign/score-utils.ts","../src/campaign/search-ledger-errors.ts","../src/campaign/search-ledger.ts","../src/campaign/single-run-lock.ts","../src/campaign/surface-identity.ts","../src/campaign/transient-failure.ts","../src/campaign/worktree/index.ts"],"mappings":";;;;;;;;;;;;;;;;;;;UAoCiB,wBAAwB;EACvC;;EAEA,QAAQ;;;EAGR;;;;;;;EAOA,sBAAsB;IAAQ;IAAY;;;;;;;EAM1C;;;UAIe;EACf;EACA;;;EAGA;;UAGe;;;;;;EAMf,gBAAgB,KAAK;;EAErB,WACE,OAAO,oBACP,SAAS,yBACN,QAAQ;;;;;;;iBAkBC,4BACd,MAAM,sCAEN,SAAS,gBACT,UAAU,iBACV,KAAK,oBACF,QAAQ;UAgBI;;;EAGf;;;;;;;;;iBAUc,uBACd,OAAM,gCACL,YAAY,iBAAiB;;;;;UC7Ff;;;EAGf;;;;;EAKA;;EAEA;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;;EAEA;;;UAIe;EACf;;;;EAIA;EACA,UAAU;EACV,WAAW;;UAGI;;EAEf,OAAO;;;EAGP,kBAAkB;;EAElB,mBAAmB;;;;;;;;;;;;;;;;;;;;iBAqBL,SAAS,eAAe,gBAAgB,MAAM,kBAAkB;;UA8G/D;;EAEf;;EAEA;;EAEA,SAAS;;EAET,gBAAgB;;;UAID;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,aAAa;;EAEb;IAAY;IAAW;;;UAGR,iCAAiC;;;;EAIhD;;EAEA,YAAY;;UAGG;EACf;EACA;EACA;;;EAGA,aAAa;EACb,cAAc;;;;;;;;;;;;;;;iBAgBA,kBACd,eAAe,gBACf,MAAM,2BACL;UAmEc;EACf;EACA;EACA,UAAU;EACV,WAAW;;UAGI;EACf,OAAO;EACP,kBAAkB;EAClB,mBAAmB;;;;;;;;;iBAeL,eACd,uBAAuB,aACvB,wBAAwB,cACvB;;;;KCjXS;;UAGK;EACf;EACA;;EAEA;;;UAIe;EACf;EACA;EACA;EACA;;;UAIe;EACf;;EAEA;;EAEA;;;;;;;UAQe;EACf;EACA;;EAEA;EACA,cAAc;EACd;EACA;;;;;;EAMA,MAAM;EACN,mBAAmB;;EAEnB;;UAGe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;;EAEA,kCAAkC;;EAElC;;UAGe,qCACf,aAAa,sBAAsB;EAEnC,qBAAqB;EACrB,qBAAqB;EACrB,eAAe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;EACA,WAAW;EACX,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf,aAAa;EACb;EACA;EACA;EACA;;KAGU;UAWK;EACf;EACA,SAAS;;UAGM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,WAAW;EACX,SAAS;EACT,OAAO;EACP,OAAO,eAAe;EACtB,QAAQ;EACR,QAAQ;;EAER,sBAAsB;;EAEtB,aAAa;;UAGE;EACf;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA,QAAQ;EACR,cAAc,eAAe;;UAGd;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;;EAEA;EACA;;UAGe;EACf,SAAS;EACT;EACA;EACA;EACA;EACA,eAAe;EACf,gBAAgB;;KAGN;UAYK;EACf;EACA,SAAS;EACT;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,4BAA4B,iCAAiC;EAC7D,wBAAwB,eAAe;EACvC,QAAQ;EACR,QAAQ;EACR,aAAa;EACb,eAAe;;UAGA;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA,SAAS;;UAGM;;EAEf;EACA;;KAGU;UAaK;EACf;EACA;EACA;EACA;EACA;EACA,uBAAuB;EACvB;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,YAAY;EACZ;;;UAIe;EACf;EACA;EACA;EACA;EACA,OAAO;;UAGQ;;EAEf;;EAEA;EACA;;EAEA;EACA;;EAEA,gBAAgB;;EAEhB,OAAO;;UAGQ;EACf,YAAY;EACZ,YAAY;EACZ,kBAAkB;;UAGH,8BACf,aAAa,sBAAsB;EAEnC;EACA;EACA;EACA;;EAEA,MAAM;EACN,iBAAiB;EACjB,iBAAiB;EACjB,YAAY;EACZ,UAAU;EACV,YAAY;;;;;;;;;iBCnRE,gCAAgC,aAAa,qBAC3D,OAAO,qCAAqC,QAC3C,8BAA8B;;;KC7CrB;UAEK;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;EACP;;UAGe,4BAA4B;EAC3C;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf,aAAa;;EAEb;;UAGe,wCAAwC;EACvD;;UAGe,0BAA0B,6BACjC,KACN,uBAAuB,qBAAqB;EAG9C;EACA,aAAa;EACb;EACA;EACA,UAAU;;KAGA,qBAAqB;EAC/B,UAAU,MAAM,KAAK;;;iBAIP,qBAAqB;;;;;iBA4BrB,gBACd,kBACA,cACA,UAAS,yBACR;;iBA4Ca,yBACd,kBACA,UAAS,kCACR;;;;;iBAuBa,mBAAmB,qBACjC,SAAS,0BAA0B,aAClC;;;UChJc,0BAA0B,kBAAkB,WAAW;EACtE,WAAW;;;;;EAKX;;;;;;iBAuBc,mBAAmB,WAAW,kBAAkB,UAC9D,SAAS,0BAA0B,aAClC,KAAK,WAAW;;;;;;;;;;;;;UC9CF;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;;;;;;;;;;KAWU;UAEK,uBAAuB;;EAEtC;;;;;;;;;EASA,OAAO;;UAGQ;EACf,UAAU;EACV;EACA;EACA;;;EAGA;;EAEA,kBAAkB;EAGlB;;;;;;;;;;;iBAYc,aAAa;;;;;;;;;;;;;;;;;;;;;;iBA8BP,SAAS,GAAG,KAAK,IAAI;;;;;;;;;;iBAkBrB,aAAa,GAAG,qBAAqB,QAAQ;;;;;;;iBAW7C,eAAe,GAAG,iBAAiB;;;;;iBAWnC,mBACpB,UAAU,gBACV;EAAY;EAAW;EAAgB;IACtC,QAAQ;;;KCpHC;UAEK;EACf,UAAU;;EAEV;;EAEA;;;EAGA;;UAGe;;;EAGf;;;;EAIA;;;EAGA;;EAEA;;;;EAIA;;;EAGA;;;EAGA,kBAAkB;;EAElB;;UAGe,qBAAqB,qBAAqB,kBAAkB,WAAW,kBAC9E,KAAK,WAAW;;;;;;;;EAQxB,QAAQ,gBAAgB;;EAExB,SAAS;IAAkB,UAAU;;;;;;;;;;;;;;;;;iBA0MvB,qBAAqB,qBAAqB,kBAAkB,WAAW,UACrF,SAAS,8BACR,qBAAqB,WAAW;UAsFlB;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;GACd;IAAQ,SAAS;;IAAyB;IAAe;;;EAE1D,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;iBA0BK,iBAAiB,UAAS,0BAA+B;;;UCjYxD;;EAEf;;EAEA;;EAEA;;;;;;;;;;;;;iBAcc,YACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,SAAS,GAAG,oCACX;UAsDc;EACf,QAAQ;;;EAGR,WAAW;;;;EAIX,iBAAiB;;;;EAIjB;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;;EAEA;EACA;;;;;;;;iBASc,oBACd,QAAQ,eACR,OAAM,6BACL;UA+Cc;EACf;EACA,WAAW;;;EAGX;EACA;EACA;;;;;iBAMc,YAAY;;;;;;;iBAUZ,qBACd,WAAW,YAAY,eAAe,cACtC,UAAU,YAAY,eAAe,cACrC,aAAa,aACb,8BACA;EAAQ;EAAoB;EAAqB;EAAoB;IACpE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UC3Mc;WACN;WACA,MAAM,SAAS;;;UAIT;;WAEN;;WAEA;WACA,gBAAgB;;UAGV;;WAEN;;WAEA;;UAGM;;WAEN;;WAEA,eAAe;;WAEf,eAAe;;;;;;;;iBASV,oBACd,mBAAmB,iBACnB,OAAM,6BACL;UAsCc;;WAEN;;WAEA;;;;;;;;;iBAUK,2BACd,cACA,MAAM,KAAK,0DACV;;;UCjFc;;EAEf;;;EAGA;;EAEA;;;cAIW,kCAAkC;WAE3B;EADlB,YACkB,cAChB;;;;;;cAiBS,kCAAkC;mBAIhB;mBAHZ;mBACA;EAEjB,YAA6B,SAAS;EAKhC,QAAQ,OAAO,uBAAuB;EAYtC,OAAO,MAAM,4BAA4B,QAAQ;EA8CjD,QAAQ;IACZ;IACA;IACA,UAAU;IACV,SAAS,OAAO;;UAkCV;UAiCA;UAmBA;UAkBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBCtNM,eAAe;;;;;;;;;;;;;;;;UClBd;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,WAAW;;UAGI;EACf;EACA;;EAEA,iBAAiB;;EAEjB,QAAQ;;UAGO;EACf;EACA;;EAEA;;UAGe;EACf;;EAEA;EACA,QAAQ;;EAER,WAAW;;UAGI;;EAEf;;EAEA;;EAEA,SAAS,UAAU,UAAU,SAAS,oBAAoB,QAAQ;;;;;;iBAWpD,kBACd,YAAY,qBACZ;EAAY;EAAe;IAC1B;;iBAgCa,WAAW,cAAc,SAAS,SAAS;;iBAkB3C,YAAY,cAAc,0BAA0B;;iBAuCpD,gBAAgB,cAAc,aAAa,cAAc;;iBAoBzD,YACd,cACA,oBACA;EAAY;IACX;;;;KCnJS;UAEK;;EAEf;;EAEA;;EAEA;;EAEA,cAAc;;UAGC;EACf;EACA,cAAc;;UAGC;EACf;EACA;EACA;;EAEA;;;UAIe;;EAEf,WAAW;;EAEX,WAAW;;EAEX;;UAGe;EACf;EACA;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;EASA;;EAEA;;EAEA;;;;;UAMe,0BAA0B;EACzC;EACA,cAAc;;EAEd;;EAEA;;EAEA;;;;;;;;;iBAUc,kBAAkB;EAChC;EACA,cAAc;IACZ;;;;;;KAoCQ,sBACV,aAAa,uBACb,oBACG;EAAU;EAAkB;;;;;;;;;iBAqLX,iBACpB,MAAM,UACN,OAAO,eACP,kBAAkB,qBACjB,QAAQ;UAoFM;EACf;;EAEA,aAAa;EACb;EACA,WAAW;EACX,SAAS;;EAET;;;;;;EAMA;;;;;;EAMA,UAAU;;;;;;;;;iBAUI,yBAAyB;EAAgB;EAAkB;;;;;;;;iBAiC3D,4BACd,MAAM,YACN,OAAM,4BACL;;;;;;;;;;;;;;;iBA6Ia,yBACd;EAAQ;EAAoB;IAC3B;;;;UC/mBc;EACf;EACA;;;;;;;UAQe;EACf;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EAIA;;;;;;;;KASU,mBACR,oBACA,oBACA;EACE;;;;;;;;;;;;iBAmBU,qBAAqB,iBAAiB,qBAAqB;;;;;;;;;;cCjE9D,2BAA2B;UAOvB;;;EAGf,MAAM;;EAEN,qBAAqB;;;EAGrB;;;;;;EAMA;;;;;;;cAQW;;;;;;;;;;;;;;;;;;iBAmBG,kBAAkB,MAAM,kBAAkB;;;;;;;;iBAuD1C,cACd,SAAS,KAAK;EACX,SAAS;EAAa;;;;;;;;;iBAiBX,eAAe,SAAS;;;;;iBASxB,oBAAoB,SAAS;;;;;;;;;;;iBA0C7B,iBAAiB,SAAS;;;UChKzB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;cAQW,8BAA8B;WAGvB,QAAQ;EAF1B,YACE,iBACgB,QAAQ;;;;;;;iBAWZ,0BACd,SAAS,cAAc,aACtB;;iBAWa,+BACd,UAAU,cAAc,eACvB;;;;;;;;iBA0Ga,kBACd,SAAS,cAAc,YACvB;EAAQ;IACP;;iBAMa,wBACd,UAAU,cAAc,cACxB;EAAQ;IACP;;;;;;cC/HU,2BAA2B;EACtC,YAAY;;;;;KAQF,kBAAkB,kBAAkB,UAAU,cACxD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;UAEI,wBAAwB,kBAAkB,UAAU;;EAEnE,UAAU;;EAEV,WAAW;;EAEX,UAAU,kBAAkB,WAAW;;EAEvC,SAAS,YAAY,WAAW;;;EAGhC;;;EAGA;;;;EAIA;;EAEA,WAAW;;EAEX;;EAEA;;;;;;;;EAQA;;;EAGA;;;EAGA;;EAEA;;EAEA,eAAe;EACf,gBAAgB;;;EAGhB,UAAU;;EAEV,YAAY;;;EAGZ,aAAa,UAAU;;;;EAIvB;;;;;;;EAOA,cACE,UAAU,WACV,UAAU;IACL;IAAgB;;;UAGR;EACf;EACA;EACA;;EAEA;;EAEA;EACA;;;EAGA,WAAW;;UAGI;EACf;EACA;;UAGe,uBAAuB,WAAW,kBAAkB;EACnE;EACA;;;;EAIA,SAAS;EACT,WAAW,eAAe;EAC1B,YAAY,eAAe;;EAE3B,YAAY,eAAe;;EAE3B,WAAW;;EAEX,WAAW,eAAe,eAAe,WAAW;;;;;iBA0GhC,iBAAiB,kBAAkB,UAAU,WACjE,MAAM,wBAAwB,WAAW,aACxC,QAAQ,uBAAuB,WAAW;;;;;UC/Q5B;;EAEf;;EAEA,UAAU;;;;;;;UAQK,kBAAkB;;EAEjC;;EAEA,OAAO;;EAEP,cAAc;;;UAIC,wBAAwB;EACvC,SAAS;;;;;;;;;;;UAYM,eAAe,eAAe,YAAY;EACzD,IAAI,OAAO,QAAQ,KAAK,kBAAkB,iBAAiB;;;;;;;iBAQ7C,qBAAqB,eAAe,WAClD,QAAQ,eAAe,UACtB,kBAAkB,QAAQ;;UAQZ,yBAAyB;EACxC;;;;;;;;iBASoB,eACpB,OAAO,WACP,OAAO,eACP,kBAAkB,qBACjB,QAAQ;;UAUM;EACf;EACA;EACA;EACA;EACA;EACA;;;;;;;iBAQc,oBAAoB,mBAAmB,qBAAqB;;UAkB3D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;iBAIc,kBAAkB,eAAe,kBAAkB;UAwBlD;;EAEf;;EAEA,OAAO;;EAEP;;;;;;;;;iBAkBc,yBACd,eAAe,iBACf,OAAM;;;;;;;;iBCnMQ;;;;;iBAQA,cAAc,gBAAgB;;;;;;;;;;;;;;;;;UCD7B;EACf;;EAEA;;UAGe;EACf;;EAEA;;EAEA;EACA;;EAEA;;;;;;;;;;iBA4Cc,oBACd,SAAS,kBACT;EAAS;IACR;;;;;;;;;;;iBA0Ba,qBACd,SAAS,kBACT,WACA;EAAS;;;;;;;iBC3FK,sBAAsB,WAAW,kBAAkB,UACjE,UAAU,eAAe,WAAW;;;;iBA0CtB,gBAAgB,sBAAsB;UAWrC;;EAEf,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;;iBAK5D,kBAAkB,WAAW,kBAAkB,UAC7D,UAAU,eAAe,WAAW,aACnC;;;;cC/EU,0BAA0B;;cAG1B,mCAAmC;;cAGnC,kCAAkC;;;cC8BlC;KAED,mBAAmB;KAEnB;;;UAYK;EACf;EACA;EACA,QAAQ;EACR;;;;UAKe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA,MAAM;EACN,UAAU;;UAGK;;EAEf;EACA;EACA;EACA;EACA,gBAAgB;;KAGN;UAOK;EACf;EACA,QAAQ;EACR,WAAW;;;EAGX;;UAGe;EACf;EACA,MAAM;;UAGS;EACf;;;EAGA;;UAGe;;EAEf,gBAAgB;;EAEhB,OAAO;;EAEP,YAAY;;KAGF;EAEN;EACA;EACA;EACA;;EAGA;EACA;;KAGM;EAEN;EACA;EACA;;EAGA;;EAEA;EACA;;UAGW;EACf,QAAQ;EACR,MAAM;;UAGS;EACf;EACA;;KAGU;EAEN;EACA;EACA,SAAS;;EAGT;EACA;EACA,SAAS;EACT,SAAS;;EAGT;EACA,SAAS;EACT,OAAO;IAAwB;;;KAGzB;EAEN;EACA;EACA;EACA;EACA;;EAGA;EACA;;;;UAKW;EACf;EACA;EACA;EACA,QAAQ;EACR,UAAU;;UAGF;EACR;EACA;EACA,WAAW;;UAGI,2BAA2B;EAC1C;EACA,MAAM;;UAGS,uCAAuC;EACtD;EACA;EACA;EACA;EACA,SAAS;EACT,UAAU;;UAGK,uCAAuC;EACtD;EACA;EACA;EACA,QAAQ;;UAGO,iCAAiC;EAChD;EACA;EACA;EACA;EACA;IACE;IACA,QAAQ;;EAEV;IACE,OAAO;IACP,OAAO;IACP,WAAW;;EAEb,SAAS;EACT,YAAY;EACZ,iBAAiB;;UAGF,qCAAqC;EACpD;EACA;EACA,eAAe;EACf;IAEM;IACA,OAAO;IACP,QAAQ;;IAGR;IACA,QAAQ;;EAEd;IACM;;IACA;IAAmB,SAAS;;IAC5B;IAAkB,SAAS;;EACjC,YAAY;;UAGG,oCAAoC;EACnD;EACA;EACA;IACM;;IAEA;IACA,QAAQ;;;UAIC,6BAA6B;EAC5C;EACA;IAEM;IACA;;IAGA;IACA,QAAQ;;;KAIJ,oBACR,qBACA,iCACA,iCACA,2BACA,+BACA,8BACA;UAEa;EACf,eAAe;EACf;EACA;EACA,cAAc;EACd,OAAO;EACP,WAAW;;KAGD;EAEN;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGW;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IAAY;IAAgB;IAAgB;;EAC5C;IAAqB;IAAmB;IAAiB;;EACzD;IAAa;IAAkB;IAAkB;;EACjD;IACE;IACA;IACA;IACA;IACA;IACA;;EAEF;EACA;EACA,YAAY;EACZ,UAAU;;UAGK;EACf,SAAS;EACT,MAAM;EACN,YAAY;EACZ,sBAAsB;EACtB,UAAU;EACV,YAAY;EACZ,WAAW;EACX,YAAY;EACZ,OAAO;;UAGQ;EACf,OAAO;;EAEP;EACA,QAAQ;;;;iBA+ZM,0BAA0B,iBAAiB;UAQ1C;EACf;EACA;;UAGe;WACN;WACA;EACT,OAAO,OAAO,oBAAoB,QAAQ;EAC1C,UAAU,QAAQ;;;;iBAKJ,iBAAiB,SAAS,0BAA0B;;cA8BvD,4BAA4B;WAC9B;WACA;mBACQ;EAMjB,YAAY,cAAc;EAWpB,UAAU,QAAQ;EAIlB,OAAO,OAAO,oBAAoB,QAAQ;;;;;;;;;;;;;;;;;;;UCt0BjC;;WAEN;;WAEA;;WAEA;;WAEA;;UAGM;;EAEf;;;;;;;iBAwBc,qBAAqB,MAAM,uBAAuB;;;;iBCpDlD,0BAA0B,2BAA2B,WAAW;;iBAmChE,uBAAuB,2BAA2B,WAAW;;iBA8B7D,iCAAiC,SAAS;;;;iBAa1C,4BAA4B,SAAS;;iBAgBrC,mBAAmB,SAAS;;iBAW5B,YAAY,SAAS;;iBAKrB,kBACd,eAAe,gBACf,iBAAiB;;;;;;;;;;;;;;;;;;;;;UCpGF;;;;;;;WAON;;WAEA,yBAAyB;;;;;;iBAYpB,4BACd,oCACA,OAAM;;;KCNH,qBAAqB;KACrB,iBAAiB,SAAS;KAC1B,aAAa,gBAAgB,aAAa,MAAM,mBAAmB;UAcvD;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;UAGM;;EAEf,OAAO;IAAQ;IAAiB;MAAkB,QAAQ;;;EAG1D,SAAS,UAAU,UAAU,kBAAkB,QAAQ;;EAEvD,QAAQ,UAAU,WAAW;;;cAIlB,6BAA6B;WAG7B;EAFX,YACE,iBACS;;UAOI;;EAEf;;EAEA;;EAEA;;;;EAIA,MAAM;;UAobS;;EAEf;;EAEA;;EAEA;;;EAGA,YAAY;;;;;iBAoJE,mBAAmB,MAAM,4BAA4B;;;;iBA0FrD,kBACd,SAAS,aACT,uBACC;;;iBAOa,oBAAoB,SAAS,aAAa"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { t as DefaultVerdict } from "./verdict-Dps8_okt.js";
|
|
2
|
+
//#region src/matrix/types.d.ts
|
|
3
|
+
/** One axis = one dimension to iterate. `V` is the value type — pass any
|
|
4
|
+
* substrate type (AgentProfile, Driver, Validator, rubric record). */
|
|
5
|
+
interface MatrixAxis<V> {
|
|
6
|
+
/** Axis name. Becomes the key in `MatrixResult.byAxis`. */
|
|
7
|
+
name: string;
|
|
8
|
+
/** Stable id per value. Used as the bucket key in aggregation. */
|
|
9
|
+
values: Array<{
|
|
10
|
+
id: string;
|
|
11
|
+
value: V;
|
|
12
|
+
}>;
|
|
13
|
+
/** Optional bucket label override. Receives the same `(value, id)` the
|
|
14
|
+
* runner stored on the cell; default label is `id`. */
|
|
15
|
+
label?: (value: V, id: string) => string;
|
|
16
|
+
}
|
|
17
|
+
/** A cell carries one picked value from each axis, keyed by axis name. */
|
|
18
|
+
interface MatrixCell {
|
|
19
|
+
axes: Record<string, {
|
|
20
|
+
id: string;
|
|
21
|
+
value: unknown;
|
|
22
|
+
}>;
|
|
23
|
+
/** 0-based replicate index within the same axis combination. */
|
|
24
|
+
rep: number;
|
|
25
|
+
/** Stable sort key — preserves cartesian order across concurrent execution. */
|
|
26
|
+
ordinal: number;
|
|
27
|
+
}
|
|
28
|
+
interface CellResult<Output> {
|
|
29
|
+
output: Output;
|
|
30
|
+
verdict: DefaultVerdict;
|
|
31
|
+
costUsd: number;
|
|
32
|
+
durationMs: number;
|
|
33
|
+
runId?: string;
|
|
34
|
+
/** Populated when `runCell` threw. The cell contributes 0 to passRate AND
|
|
35
|
+
* meanScore regardless of `verdict`. */
|
|
36
|
+
error?: {
|
|
37
|
+
message: string;
|
|
38
|
+
kind: string;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
interface AxisSummary {
|
|
42
|
+
axisName: string;
|
|
43
|
+
axisValue: string;
|
|
44
|
+
cells: number;
|
|
45
|
+
passRate: number;
|
|
46
|
+
meanScore: number;
|
|
47
|
+
p50Score: number;
|
|
48
|
+
p90Score: number;
|
|
49
|
+
totalCostUsd: number;
|
|
50
|
+
meanDurationMs: number;
|
|
51
|
+
}
|
|
52
|
+
interface MatrixResult<Output> {
|
|
53
|
+
cells: Array<{
|
|
54
|
+
cell: MatrixCell;
|
|
55
|
+
runs: CellResult<Output>[];
|
|
56
|
+
}>;
|
|
57
|
+
/** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
|
|
58
|
+
* named in `aggregateBy` (default = every axis in `axes`). */
|
|
59
|
+
byAxis: Record<string, Record<string, AxisSummary>>;
|
|
60
|
+
summary: {
|
|
61
|
+
totalCells: number;
|
|
62
|
+
runsExecuted: number;
|
|
63
|
+
/** Cells removed by `filter` plus cells unscheduled after the cost
|
|
64
|
+
* ceiling or abort signal tripped. */
|
|
65
|
+
cellsSkipped: number;
|
|
66
|
+
overallPassRate: number;
|
|
67
|
+
overallMeanScore: number;
|
|
68
|
+
totalCostUsd: number;
|
|
69
|
+
durationMs: number;
|
|
70
|
+
};
|
|
71
|
+
/** Stable id-like string generated at the end of the run. */
|
|
72
|
+
matrixId: string;
|
|
73
|
+
}
|
|
74
|
+
interface RunAgentMatrixOptions<Output> {
|
|
75
|
+
axes: MatrixAxis<unknown>[];
|
|
76
|
+
/** User-supplied cell executor. May throw; the matrix captures throws as
|
|
77
|
+
* `CellResult.error` and continues. */
|
|
78
|
+
runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
|
|
79
|
+
/** Replicates per cell. Default 1. */
|
|
80
|
+
reps?: number;
|
|
81
|
+
/** Prune cells from the cartesian BEFORE rep expansion. */
|
|
82
|
+
filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
|
|
83
|
+
/** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
|
|
84
|
+
aggregateBy?: string[];
|
|
85
|
+
/** Max concurrent in-flight `runCell` invocations. Default 4. */
|
|
86
|
+
maxConcurrency?: number;
|
|
87
|
+
/** Cumulative-cost abort threshold (USD). When the running sum of
|
|
88
|
+
* `result.costUsd` crosses this value, no new cells are scheduled.
|
|
89
|
+
* In-flight cells finish. Default `Infinity`. */
|
|
90
|
+
costCeiling?: number;
|
|
91
|
+
/** Fires once per executed cell, after its promise settles. */
|
|
92
|
+
onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
|
|
93
|
+
/** External cancellation. Aborts in-flight cells via a forwarded signal
|
|
94
|
+
* and suppresses scheduling of new ones. */
|
|
95
|
+
signal?: AbortSignal;
|
|
96
|
+
}
|
|
97
|
+
//#endregion
|
|
98
|
+
//#region src/matrix/runner.d.ts
|
|
99
|
+
declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
|
|
100
|
+
//#endregion
|
|
101
|
+
export { MatrixCell as a, MatrixAxis as i, AxisSummary as n, MatrixResult as o, CellResult as r, RunAgentMatrixOptions as s, runAgentMatrix as t };
|
|
102
|
+
//# sourceMappingURL=index-DSC51roc.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-DSC51roc.d.ts","names":[],"sources":["../src/matrix/types.ts","../src/matrix/runner.ts"],"mappings":";;;;UAgBiB,WAAW;;EAE1B;;EAEA,QAAQ;IAAQ;IAAY,OAAO;;;;EAGnC,SAAS,OAAO,GAAG;;;UAIJ;EACf,MAAM;IAAiB;IAAY;;;EAEnC;;EAEA;;UAGe,WAAW;EAC1B,QAAQ;EACR,SAAS;EACT;EACA;EACA;;;EAGA;IAAU;IAAiB;;;UAGZ;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe,aAAa;EAC5B,OAAO;IAAQ,MAAM;IAAY,MAAM,WAAW;;;;EAGlD,QAAQ,eAAe,eAAe;EACtC;IACE;IACA;;;IAGA;IACA;IACA;IACA;IACA;;;EAGF;;UAGe,sBAAsB;EACrC,MAAM;;;EAGN,UAAU,MAAM,eAAe,QAAQ,WAAW;;EAElD;;EAEA,UAAU,MAAM,KAAK;;EAErB;;EAEA;;;;EAIA;;EAEA,kBAAkB,MAAM,YAAY,QAAQ,WAAW;;;EAGvD,SAAS;;;;iBCjBW,eAAe,QACnC,MAAM,sBAAsB,UAC3B,QAAQ,aAAa"}
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
import { s as RunSplitTag } from "./run-record-CnZu_gjl.js";
|
|
2
|
+
import { R as Scenario, a as CampaignResult, d as DispatchContext } from "./types-k9tZGKUg.js";
|
|
3
|
+
import { sn as CampaignStorage } from "./skillopt-optimization-method-CWKVTnks.js";
|
|
4
|
+
import "./index-CD_WZ_Xr.js";
|
|
5
|
+
//#region src/benchmarks/types.d.ts
|
|
6
|
+
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
7
|
+
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
8
|
+
interface BenchmarkDatasetItem<TPayload = unknown> {
|
|
9
|
+
/** Stable dataset-local item id (used for split assignment + paper
|
|
10
|
+
* references). Unique within a benchmark. */
|
|
11
|
+
id: string;
|
|
12
|
+
/** Free-form payload. Each benchmark defines its own shape. */
|
|
13
|
+
payload: TPayload;
|
|
14
|
+
/** Optional precomputed split. When absent, adapters use assignSplit(id). */
|
|
15
|
+
split?: RunSplitTag;
|
|
16
|
+
/** Benchmark family this row came from, e.g. `beir` or `crag`. */
|
|
17
|
+
family?: BenchmarkFamily | string;
|
|
18
|
+
/** Benchmark-local task kind, e.g. retrieval vs answer quality. */
|
|
19
|
+
taskKind?: BenchmarkTaskKind | string;
|
|
20
|
+
/** Slice labels such as language, domain, freshness, multihop, long-tail. */
|
|
21
|
+
tags?: string[];
|
|
22
|
+
/** Dataset provenance, version, URL, or license notes. */
|
|
23
|
+
source?: BenchmarkSource;
|
|
24
|
+
metadata?: Record<string, unknown>;
|
|
25
|
+
}
|
|
26
|
+
interface BenchmarkEvaluation {
|
|
27
|
+
/** [0, 1] score for the response on this item. Exact-match
|
|
28
|
+
* benchmarks use 0/1; partial-credit benchmarks may return
|
|
29
|
+
* fractional values. */
|
|
30
|
+
score: number;
|
|
31
|
+
/** Optional pass/fail projection. Defaults to `score > 0` when absent. */
|
|
32
|
+
passed?: boolean;
|
|
33
|
+
/** Numeric sub-metrics. These become report dimensions. */
|
|
34
|
+
dimensions?: Record<string, number>;
|
|
35
|
+
/** Optional bag of raw scoring signals — e.g. parsed numeric
|
|
36
|
+
* answer, regex match, judge sub-scores. */
|
|
37
|
+
raw: Record<string, unknown>;
|
|
38
|
+
notes?: string;
|
|
39
|
+
}
|
|
40
|
+
interface BenchmarkSource {
|
|
41
|
+
name?: string;
|
|
42
|
+
url?: string;
|
|
43
|
+
version?: string;
|
|
44
|
+
license?: string;
|
|
45
|
+
citation?: string;
|
|
46
|
+
}
|
|
47
|
+
/** Common signature implemented by every adapter under `src/benchmarks/*`. */
|
|
48
|
+
interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {
|
|
49
|
+
/** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. */
|
|
50
|
+
id?: string;
|
|
51
|
+
family?: BenchmarkFamily | string;
|
|
52
|
+
taskKind?: BenchmarkTaskKind | string;
|
|
53
|
+
description?: string;
|
|
54
|
+
source?: BenchmarkSource;
|
|
55
|
+
defaultMetric?: string;
|
|
56
|
+
/** Load the dataset for the given split. May hit the network on
|
|
57
|
+
* first call but should be cache-friendly. Adapters that don't
|
|
58
|
+
* ship the dataset itself MUST throw a clearly-marked error
|
|
59
|
+
* pointing the caller at the loader script. */
|
|
60
|
+
loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
|
|
61
|
+
/** Score a single response. Pure with respect to the inputs. */
|
|
62
|
+
evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>;
|
|
63
|
+
/** Deterministic split assignment via item id hashing. The
|
|
64
|
+
* fraction of items in each split is implementation-defined but
|
|
65
|
+
* MUST be stable across processes and platforms. */
|
|
66
|
+
assignSplit(itemId: string): RunSplitTag;
|
|
67
|
+
}
|
|
68
|
+
interface BenchmarkScenario<TPayload = unknown> extends Scenario {
|
|
69
|
+
kind: 'benchmark';
|
|
70
|
+
benchmarkId: string;
|
|
71
|
+
family: BenchmarkFamily | string;
|
|
72
|
+
taskKind: BenchmarkTaskKind | string;
|
|
73
|
+
splitTag: RunSplitTag;
|
|
74
|
+
item: BenchmarkDatasetItem<TPayload>;
|
|
75
|
+
}
|
|
76
|
+
type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {
|
|
77
|
+
scenario: BenchmarkScenario<TPayload>;
|
|
78
|
+
item: BenchmarkDatasetItem<TPayload>;
|
|
79
|
+
context: DispatchContext;
|
|
80
|
+
}) => Promise<TArtifact> | TArtifact;
|
|
81
|
+
/** Split-assignment seed shared across all benchmarks. Bumping this
|
|
82
|
+
* value reshuffles every split — do NOT do that lightly. */
|
|
83
|
+
declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
|
|
84
|
+
/**
|
|
85
|
+
* Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
|
|
86
|
+
* stable 32-bit hash of `${seed}::${id}`. Default proportions:
|
|
87
|
+
*
|
|
88
|
+
* search: 60% (optimization-readable)
|
|
89
|
+
* dev: 20% (held-out for tuning, leak-on-purpose during dev)
|
|
90
|
+
* holdout:20% (paper-grade held-out, gated reads)
|
|
91
|
+
*/
|
|
92
|
+
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
|
|
93
|
+
//#endregion
|
|
94
|
+
//#region src/benchmarks/calibration.d.ts
|
|
95
|
+
interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
|
|
96
|
+
adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
|
|
97
|
+
item: BenchmarkDatasetItem<TPayload>;
|
|
98
|
+
weakArtifact: TArtifact;
|
|
99
|
+
strongArtifact: TArtifact;
|
|
100
|
+
maxWeakScore?: number;
|
|
101
|
+
minStrongScore?: number;
|
|
102
|
+
minGap?: number;
|
|
103
|
+
}
|
|
104
|
+
interface BenchmarkMetricCalibrationResult {
|
|
105
|
+
passed: boolean;
|
|
106
|
+
weak: BenchmarkEvaluation;
|
|
107
|
+
strong: BenchmarkEvaluation;
|
|
108
|
+
weakScore: number;
|
|
109
|
+
strongScore: number;
|
|
110
|
+
gap: number;
|
|
111
|
+
reasons: string[];
|
|
112
|
+
}
|
|
113
|
+
declare function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>): Promise<BenchmarkMetricCalibrationResult>;
|
|
114
|
+
//#endregion
|
|
115
|
+
//#region src/benchmarks/routing/dataset.d.ts
|
|
116
|
+
/**
|
|
117
|
+
* Synthetic routing dataset. 16 tasks across 4 categories. Used as a
|
|
118
|
+
* deterministic, dependency-free benchmark for any router that maps a
|
|
119
|
+
* natural-language request to one of a fixed set of route labels.
|
|
120
|
+
*
|
|
121
|
+
* Format (see `routing/README.md` for prose):
|
|
122
|
+
*
|
|
123
|
+
* {
|
|
124
|
+
* id: stable per-task ID (matches across processes).
|
|
125
|
+
* category: one of the four route labels.
|
|
126
|
+
* prompt: the user-facing request the router must classify.
|
|
127
|
+
* route: the ground-truth route the router should pick.
|
|
128
|
+
* synonyms: other strings that count as a correct answer.
|
|
129
|
+
* hardNegatives:close-but-wrong route labels — used to detect the
|
|
130
|
+
* "always picks the popular route" failure mode.
|
|
131
|
+
* }
|
|
132
|
+
*
|
|
133
|
+
* The four categories are intentionally cross-domain (file ops,
|
|
134
|
+
* math, search, conversation) so a router that collapses to one
|
|
135
|
+
* category is easy to spot.
|
|
136
|
+
*/
|
|
137
|
+
interface RoutingItem {
|
|
138
|
+
id: string;
|
|
139
|
+
category: 'file' | 'math' | 'search' | 'chat';
|
|
140
|
+
prompt: string;
|
|
141
|
+
/** Canonical correct route label. */
|
|
142
|
+
route: string;
|
|
143
|
+
/** Alternate route labels that also count as correct. */
|
|
144
|
+
synonyms: string[];
|
|
145
|
+
/** Wrong-but-tempting route labels (for analysis, not grading). */
|
|
146
|
+
hardNegatives: string[];
|
|
147
|
+
}
|
|
148
|
+
declare const ROUTING_DATASET: RoutingItem[];
|
|
149
|
+
declare namespace index_d_exports$1 {
|
|
150
|
+
export { ROUTING_DATASET, RoutingAdapter, RoutingDatasetItem, RoutingItem, RoutingPayload, assignSplit, evaluate, extractRouteTokens, loadDataset };
|
|
151
|
+
}
|
|
152
|
+
type RoutingPayload = RoutingItem;
|
|
153
|
+
type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
|
|
154
|
+
declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
|
|
155
|
+
readonly id = "first-party/routing";
|
|
156
|
+
readonly family = "first-party";
|
|
157
|
+
readonly taskKind = "routing";
|
|
158
|
+
readonly description = "Synthetic fixed-route classification smoke benchmark";
|
|
159
|
+
readonly defaultMetric = "route_exact_match";
|
|
160
|
+
loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
|
|
161
|
+
evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
|
|
162
|
+
assignSplit(itemId: string): RunSplitTag;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Pull route-shaped tokens out of a model response. Routes look like
|
|
166
|
+
* `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
|
|
167
|
+
* are not routes, but `category.action` patterns are robust to most
|
|
168
|
+
* model wrappers (JSON output, prose explanations, code fences).
|
|
169
|
+
*/
|
|
170
|
+
declare function extractRouteTokens(response: string): string[];
|
|
171
|
+
declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
|
|
172
|
+
declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
|
|
173
|
+
declare const assignSplit: (itemId: string) => RunSplitTag;
|
|
174
|
+
//#endregion
|
|
175
|
+
//#region src/benchmarks/runner.d.ts
|
|
176
|
+
interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {
|
|
177
|
+
adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
|
|
178
|
+
respond: BenchmarkResponder<TPayload, TArtifact>;
|
|
179
|
+
splits?: readonly RunSplitTag[];
|
|
180
|
+
runDir: string;
|
|
181
|
+
repo?: string;
|
|
182
|
+
seed?: number;
|
|
183
|
+
reps?: number;
|
|
184
|
+
resumable?: boolean;
|
|
185
|
+
costCeiling?: number;
|
|
186
|
+
maxConcurrency?: number;
|
|
187
|
+
dispatchTimeoutMs?: number;
|
|
188
|
+
expectUsage?: 'assert' | 'warn' | 'off';
|
|
189
|
+
storage?: CampaignStorage;
|
|
190
|
+
now?: () => Date;
|
|
191
|
+
}
|
|
192
|
+
interface BenchmarkReport {
|
|
193
|
+
benchmarkId: string;
|
|
194
|
+
family: BenchmarkFamily | string;
|
|
195
|
+
taskKind: BenchmarkTaskKind | string;
|
|
196
|
+
source?: BenchmarkAdapter['source'];
|
|
197
|
+
runDir: string;
|
|
198
|
+
manifestHash: string;
|
|
199
|
+
seed: number;
|
|
200
|
+
startedAt: string;
|
|
201
|
+
endedAt: string;
|
|
202
|
+
durationMs: number;
|
|
203
|
+
totalItems: number;
|
|
204
|
+
totalCells: number;
|
|
205
|
+
cellsFailed: number;
|
|
206
|
+
cellsCached: number;
|
|
207
|
+
totalCostUsd: number;
|
|
208
|
+
splits: Record<string, BenchmarkSliceSummary>;
|
|
209
|
+
tags: Record<string, BenchmarkSliceSummary>;
|
|
210
|
+
dimensions: Record<string, BenchmarkDistribution>;
|
|
211
|
+
score: BenchmarkDistribution;
|
|
212
|
+
costUsd: BenchmarkDistribution;
|
|
213
|
+
latencyMs: BenchmarkDistribution;
|
|
214
|
+
}
|
|
215
|
+
interface BenchmarkSliceSummary {
|
|
216
|
+
n: number;
|
|
217
|
+
meanScore: number;
|
|
218
|
+
passRate: number;
|
|
219
|
+
score: BenchmarkDistribution;
|
|
220
|
+
costUsd: BenchmarkDistribution;
|
|
221
|
+
latencyMs: BenchmarkDistribution;
|
|
222
|
+
}
|
|
223
|
+
interface BenchmarkDistribution {
|
|
224
|
+
n: number;
|
|
225
|
+
min: number;
|
|
226
|
+
mean: number;
|
|
227
|
+
median: number;
|
|
228
|
+
p90: number;
|
|
229
|
+
max: number;
|
|
230
|
+
}
|
|
231
|
+
interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
|
|
232
|
+
scenarios: Array<BenchmarkScenario<TPayload>>;
|
|
233
|
+
campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
|
|
234
|
+
report: BenchmarkReport;
|
|
235
|
+
reportJsonPath: string;
|
|
236
|
+
reportMarkdownPath: string;
|
|
237
|
+
}
|
|
238
|
+
declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
|
|
239
|
+
declare function summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {
|
|
240
|
+
adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
|
|
241
|
+
scenarios: Array<BenchmarkScenario<TPayload>>;
|
|
242
|
+
campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
|
|
243
|
+
}): BenchmarkReport;
|
|
244
|
+
declare function renderBenchmarkReportMarkdown(report: BenchmarkReport): string;
|
|
245
|
+
//#endregion
|
|
246
|
+
//#region src/benchmarks/standard-formats.d.ts
|
|
247
|
+
interface StandardRetrievalDocument {
|
|
248
|
+
id: string;
|
|
249
|
+
title?: string;
|
|
250
|
+
text: string;
|
|
251
|
+
metadata?: Record<string, unknown>;
|
|
252
|
+
}
|
|
253
|
+
interface StandardRetrievalQuery {
|
|
254
|
+
id: string;
|
|
255
|
+
text: string;
|
|
256
|
+
metadata?: Record<string, unknown>;
|
|
257
|
+
}
|
|
258
|
+
interface StandardRetrievalQrel {
|
|
259
|
+
queryId: string;
|
|
260
|
+
documentId: string;
|
|
261
|
+
score: number;
|
|
262
|
+
}
|
|
263
|
+
interface StandardRetrievalPayload {
|
|
264
|
+
queryId: string;
|
|
265
|
+
query: string;
|
|
266
|
+
expectedDocumentIds: string[];
|
|
267
|
+
expectedScores: Record<string, number>;
|
|
268
|
+
corpus?: Record<string, StandardRetrievalDocument>;
|
|
269
|
+
metadata?: Record<string, unknown>;
|
|
270
|
+
}
|
|
271
|
+
interface BuildStandardRetrievalItemsOptions {
|
|
272
|
+
benchmarkId: string;
|
|
273
|
+
family: BenchmarkFamily | string;
|
|
274
|
+
queries: readonly StandardRetrievalQuery[];
|
|
275
|
+
qrels: readonly StandardRetrievalQrel[];
|
|
276
|
+
corpus?: readonly StandardRetrievalDocument[];
|
|
277
|
+
includeCorpusInPayload?: boolean;
|
|
278
|
+
source?: BenchmarkSource;
|
|
279
|
+
tags?: readonly string[];
|
|
280
|
+
splitOf?: (queryId: string) => RunSplitTag;
|
|
281
|
+
}
|
|
282
|
+
interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {
|
|
283
|
+
responseIdPattern?: RegExp;
|
|
284
|
+
cutoffs?: readonly number[];
|
|
285
|
+
primaryMetric?: string;
|
|
286
|
+
passMetric?: string;
|
|
287
|
+
passThreshold?: number;
|
|
288
|
+
}
|
|
289
|
+
interface StandardRetrievalResult {
|
|
290
|
+
id?: string;
|
|
291
|
+
documentId?: string;
|
|
292
|
+
docId?: string;
|
|
293
|
+
score?: number;
|
|
294
|
+
}
|
|
295
|
+
type StandardRetrievalArtifact = string | readonly string[] | readonly StandardRetrievalResult[] | {
|
|
296
|
+
ids?: readonly string[];
|
|
297
|
+
documentIds?: readonly string[];
|
|
298
|
+
results?: readonly StandardRetrievalResult[];
|
|
299
|
+
};
|
|
300
|
+
interface StandardRetrievalEvaluationOptions {
|
|
301
|
+
responseIdPattern?: RegExp;
|
|
302
|
+
cutoffs?: readonly number[];
|
|
303
|
+
primaryMetric?: string;
|
|
304
|
+
passMetric?: string;
|
|
305
|
+
passThreshold?: number;
|
|
306
|
+
}
|
|
307
|
+
declare function parseJsonlRows<T = unknown>(text: string): T[];
|
|
308
|
+
declare function parseTsvRows(text: string): string[][];
|
|
309
|
+
declare function parseQrels(text: string): StandardRetrievalQrel[];
|
|
310
|
+
declare function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[];
|
|
311
|
+
declare function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[];
|
|
312
|
+
declare function buildStandardRetrievalItems(options: BuildStandardRetrievalItemsOptions): Array<BenchmarkDatasetItem<StandardRetrievalPayload>>;
|
|
313
|
+
declare function createRetrievalIdBenchmarkAdapter(options: RetrievalIdAdapterOptions): BenchmarkAdapter<BenchmarkDatasetItem<StandardRetrievalPayload>, StandardRetrievalPayload, StandardRetrievalArtifact>;
|
|
314
|
+
declare function evaluateStandardRetrieval(payload: StandardRetrievalPayload, artifact: StandardRetrievalArtifact, options?: StandardRetrievalEvaluationOptions): {
|
|
315
|
+
score: number;
|
|
316
|
+
passed: boolean;
|
|
317
|
+
dimensions: Record<string, number>;
|
|
318
|
+
raw: {
|
|
319
|
+
rankedDocumentIds: string[];
|
|
320
|
+
expectedDocumentIds: string[];
|
|
321
|
+
expectedScores: Record<string, number>;
|
|
322
|
+
};
|
|
323
|
+
};
|
|
324
|
+
declare function normalizeRetrievedDocumentIds(artifact: StandardRetrievalArtifact, responseIdPattern?: RegExp): string[];
|
|
325
|
+
declare function retrievalMetricsAtCutoff(input: {
|
|
326
|
+
rankedDocumentIds: readonly string[];
|
|
327
|
+
expectedScores: Record<string, number>;
|
|
328
|
+
cutoff: number;
|
|
329
|
+
}): Record<string, number>;
|
|
330
|
+
declare namespace index_d_exports {
|
|
331
|
+
export { BENCHMARK_SPLIT_SEED, BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkDistribution, BenchmarkEvaluation, BenchmarkFamily, BenchmarkMetricCalibrationOptions, BenchmarkMetricCalibrationResult, BenchmarkReport, BenchmarkResponder, BenchmarkRunOptions, BenchmarkRunResult, BenchmarkScenario, BenchmarkSliceSummary, BenchmarkSource, BenchmarkTaskKind, BuildStandardRetrievalItemsOptions, RetrievalIdAdapterOptions, StandardRetrievalArtifact, StandardRetrievalDocument, StandardRetrievalEvaluationOptions, StandardRetrievalPayload, StandardRetrievalQrel, StandardRetrievalQuery, StandardRetrievalResult, buildStandardRetrievalItems, calibrateBenchmarkMetric, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, normalizeRetrievedDocumentIds, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseJsonlRows, parseQrels, parseTsvRows, renderBenchmarkReportMarkdown, retrievalMetricsAtCutoff, index_d_exports$1 as routing, runBenchmarkAdapter, summarizeBenchmarkCampaign };
|
|
332
|
+
}
|
|
333
|
+
//#endregion
|
|
334
|
+
export { BenchmarkMetricCalibrationOptions as A, BenchmarkSource as B, BenchmarkRunOptions as C, runBenchmarkAdapter as D, renderBenchmarkReportMarkdown as E, BenchmarkDatasetItem as F, deterministicSplit as H, BenchmarkEvaluation as I, BenchmarkFamily as L, calibrateBenchmarkMetric as M, BENCHMARK_SPLIT_SEED as N, summarizeBenchmarkCampaign as O, BenchmarkAdapter as P, BenchmarkResponder as R, BenchmarkReport as S, BenchmarkSliceSummary as T, BenchmarkTaskKind as V, parseJsonlRows as _, StandardRetrievalDocument as a, retrievalMetricsAtCutoff as b, StandardRetrievalQrel as c, buildStandardRetrievalItems as d, createRetrievalIdBenchmarkAdapter as f, parseBeirQueriesJsonl as g, parseBeirCorpusJsonl as h, StandardRetrievalArtifact as i, BenchmarkMetricCalibrationResult as j, index_d_exports$1 as k, StandardRetrievalQuery as l, normalizeRetrievedDocumentIds as m, BuildStandardRetrievalItemsOptions as n, StandardRetrievalEvaluationOptions as o, evaluateStandardRetrieval as p, RetrievalIdAdapterOptions as r, StandardRetrievalPayload as s, index_d_exports as t, StandardRetrievalResult as u, parseQrels as v, BenchmarkRunResult as w, BenchmarkDistribution as x, parseTsvRows as y, BenchmarkScenario as z };
|
|
335
|
+
//# sourceMappingURL=index-Em67JBjs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-Em67JBjs.d.ts","names":[],"sources":["../src/benchmarks/types.ts","../src/benchmarks/calibration.ts","../src/benchmarks/routing/dataset.ts","../src/benchmarks/routing/index.ts","../src/benchmarks/runner.ts","../src/benchmarks/standard-formats.ts","../src/benchmarks/index.ts"],"mappings":";;;;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC;;;UCjJc,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;iBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;UCFM;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;;;UClCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;iBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;iBAmGxB,2BAA2B,UAAU,WAAW;EAC9D,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;IACpD;iBA6CY,8BAA8B,QAAQ;;;UCpOrC;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;iBAGc,eAAe,aAAa,eAAe;iBAc3C,aAAa;iBASb,WAAW,eAAe;iBAgB1B,qBAAqB,eAAe;iBAgBpC,sBAAsB,eAAe;iBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;iBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;iBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS;;;;;;;;;;iBAsCK,8BACd,UAAU,2BACV,oBAAmB;iBA0BL,yBAAyB;EACvC;EACA,gBAAgB;EAChB;IACE"}
|