@tangle-network/agent-eval 0.129.0 → 0.130.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +2 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +81 -2872
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -360
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1188
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1709
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -891
- package/dist/benchmarks/index.js +2 -60
- package/dist/benchmarks-BJgDGkAD.js +754 -0
- package/dist/benchmarks-BJgDGkAD.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6381
- package/dist/campaign/index.js +3 -213
- package/dist/campaign-aKJt6emI.js +3886 -0
- package/dist/campaign-aKJt6emI.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -175
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5565
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1938
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -33
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -618
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CD_WZ_Xr.d.ts +2250 -0
- package/dist/index-CD_WZ_Xr.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index-Em67JBjs.d.ts +335 -0
- package/dist/index-Em67JBjs.d.ts.map +1 -0
- package/dist/index.d.ts +3755 -15555
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11182 -11216
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -480
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1312
- package/dist/reporting.js +6 -51
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +760 -4010
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2325 -1958
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -2087
- package/dist/rollout/index.js +8 -168
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-CUmHkGbI.js +7718 -0
- package/dist/skillopt-optimization-method-CUmHkGbI.js.map +1 -0
- package/dist/skillopt-optimization-method-CWKVTnks.d.ts +1740 -0
- package/dist/skillopt-optimization-method-CWKVTnks.d.ts.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -959
- package/dist/supervisor-run/index.js +2 -65
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -252
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1173
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/campaign-proposers.md +1 -0
- package/package.json +17 -9
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2QU3YOPR.js +0 -7374
- package/dist/chunk-2QU3YOPR.js.map +0 -1
- package/dist/chunk-3OCR4R5I.js +0 -728
- package/dist/chunk-3OCR4R5I.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-56TAVBOK.js +0 -698
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7FO3TNPI.js +0 -232
- package/dist/chunk-7FO3TNPI.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BSO5JDQH.js +0 -2335
- package/dist/chunk-BSO5JDQH.js.map +0 -1
- package/dist/chunk-C6LXANRU.js +0 -1550
- package/dist/chunk-C6LXANRU.js.map +0 -1
- package/dist/chunk-DODXQREJ.js +0 -752
- package/dist/chunk-DODXQREJ.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-E7QXT7SX.js +0 -183
- package/dist/chunk-E7QXT7SX.js.map +0 -1
- package/dist/chunk-EG66UGL4.js +0 -341
- package/dist/chunk-EG66UGL4.js.map +0 -1
- package/dist/chunk-FXTVJPYD.js +0 -576
- package/dist/chunk-FXTVJPYD.js.map +0 -1
- package/dist/chunk-G7MGMCZD.js +0 -153
- package/dist/chunk-G7MGMCZD.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-H23X7XKK.js +0 -181
- package/dist/chunk-H23X7XKK.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-HPWUNB47.js +0 -289
- package/dist/chunk-HPWUNB47.js.map +0 -1
- package/dist/chunk-IYCLP2N2.js +0 -766
- package/dist/chunk-IYCLP2N2.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-JQSF5DQT.js +0 -701
- package/dist/chunk-JQSF5DQT.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-M4YBQKIJ.js +0 -1040
- package/dist/chunk-M4YBQKIJ.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NY44NC4A.js +0 -1056
- package/dist/chunk-NY44NC4A.js.map +0 -1
- package/dist/chunk-OIUOT4QD.js +0 -44
- package/dist/chunk-OIUOT4QD.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-OWN5NPMC.js +0 -152
- package/dist/chunk-OWN5NPMC.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PC5DOSM7.js +0 -579
- package/dist/chunk-PC5DOSM7.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-QB6BDBP2.js +0 -4464
- package/dist/chunk-QB6BDBP2.js.map +0 -1
- package/dist/chunk-RXHCETDZ.js +0 -536
- package/dist/chunk-RXHCETDZ.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-SFLLL76A.js +0 -669
- package/dist/chunk-SFLLL76A.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-T6RLYGAD.js +0 -158
- package/dist/chunk-T6RLYGAD.js.map +0 -1
- package/dist/chunk-TJVT4QFF.js +0 -911
- package/dist/chunk-TJVT4QFF.js.map +0 -1
- package/dist/chunk-TQ7LNKZ3.js +0 -136
- package/dist/chunk-TQ7LNKZ3.js.map +0 -1
- package/dist/chunk-U4L7JRPZ.js +0 -1706
- package/dist/chunk-U4L7JRPZ.js.map +0 -1
- package/dist/chunk-U4PHLT2N.js +0 -419
- package/dist/chunk-U4PHLT2N.js.map +0 -1
- package/dist/chunk-VCZ5FQYW.js +0 -928
- package/dist/chunk-VCZ5FQYW.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WVATSFCP.js +0 -1553
- package/dist/chunk-WVATSFCP.js.map +0 -1
- package/dist/chunk-X4YIBDER.js +0 -1662
- package/dist/chunk-X4YIBDER.js.map +0 -1
- package/dist/chunk-YQN4ICPP.js +0 -355
- package/dist/chunk-YQN4ICPP.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZHTZ4EYI.js +0 -1212
- package/dist/chunk-ZHTZ4EYI.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-OJJ7CZF4.js +0 -18
- package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/chunk-C6LXANRU.js
DELETED
|
@@ -1,1550 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
projectCampaignCellQuality
|
|
3
|
-
} from "./chunk-E7QXT7SX.js";
|
|
4
|
-
import {
|
|
5
|
-
confidenceInterval
|
|
6
|
-
} from "./chunk-ZHTZ4EYI.js";
|
|
7
|
-
import {
|
|
8
|
-
CostAccountingIncompleteError,
|
|
9
|
-
CostLedger
|
|
10
|
-
} from "./chunk-VCZ5FQYW.js";
|
|
11
|
-
import {
|
|
12
|
-
AgentEvalError,
|
|
13
|
-
ValidationError
|
|
14
|
-
} from "./chunk-ONWEPEDO.js";
|
|
15
|
-
|
|
16
|
-
// src/campaign/run-campaign.ts
|
|
17
|
-
import { join as join3 } from "path";
|
|
18
|
-
|
|
19
|
-
// src/integrity/backend-integrity.ts
|
|
20
|
-
var BackendIntegrityError = class extends AgentEvalError {
|
|
21
|
-
constructor(message, report) {
|
|
22
|
-
super("backend_integrity", message);
|
|
23
|
-
this.report = report;
|
|
24
|
-
}
|
|
25
|
-
report;
|
|
26
|
-
};
|
|
27
|
-
function summarizeBackendIntegrity(records) {
|
|
28
|
-
return summarizeBackendUsage(
|
|
29
|
-
records.map((record) => ({
|
|
30
|
-
inputTokens: record.tokenUsage.input,
|
|
31
|
-
outputTokens: record.tokenUsage.output,
|
|
32
|
-
costUsd: record.costUsd
|
|
33
|
-
}))
|
|
34
|
-
);
|
|
35
|
-
}
|
|
36
|
-
function summarizeAgentReceiptIntegrity(receipts) {
|
|
37
|
-
return summarizeBackendUsage(
|
|
38
|
-
receipts.filter((receipt) => receipt.channel === "agent").map((receipt) => ({
|
|
39
|
-
inputTokens: receipt.inputTokens,
|
|
40
|
-
outputTokens: receipt.outputTokens,
|
|
41
|
-
costUsd: receipt.costUsd
|
|
42
|
-
}))
|
|
43
|
-
);
|
|
44
|
-
}
|
|
45
|
-
function summarizeBackendUsage(records) {
|
|
46
|
-
const totalRecords = records.length;
|
|
47
|
-
let stubRecords = 0;
|
|
48
|
-
let realRecords = 0;
|
|
49
|
-
let uncostedRecords = 0;
|
|
50
|
-
let totalInputTokens = 0;
|
|
51
|
-
let totalOutputTokens = 0;
|
|
52
|
-
let totalCostUsd = 0;
|
|
53
|
-
for (const rec of records) {
|
|
54
|
-
totalInputTokens += rec.inputTokens;
|
|
55
|
-
totalOutputTokens += rec.outputTokens;
|
|
56
|
-
totalCostUsd += rec.costUsd ?? 0;
|
|
57
|
-
if (rec.inputTokens === 0 && rec.outputTokens === 0) stubRecords++;
|
|
58
|
-
else realRecords++;
|
|
59
|
-
if (rec.outputTokens > 0 && (rec.costUsd === null || rec.costUsd === 0)) uncostedRecords++;
|
|
60
|
-
}
|
|
61
|
-
const verdict = totalRecords === 0 ? "stub" : stubRecords === totalRecords ? "stub" : stubRecords === 0 ? "real" : "mixed";
|
|
62
|
-
const diagnosis = buildDiagnosis({
|
|
63
|
-
totalRecords,
|
|
64
|
-
stubRecords,
|
|
65
|
-
realRecords,
|
|
66
|
-
uncostedRecords,
|
|
67
|
-
totalInputTokens,
|
|
68
|
-
totalOutputTokens,
|
|
69
|
-
totalCostUsd,
|
|
70
|
-
verdict
|
|
71
|
-
});
|
|
72
|
-
return {
|
|
73
|
-
totalRecords,
|
|
74
|
-
stubRecords,
|
|
75
|
-
realRecords,
|
|
76
|
-
uncostedRecords,
|
|
77
|
-
totalInputTokens,
|
|
78
|
-
totalOutputTokens,
|
|
79
|
-
totalCostUsd,
|
|
80
|
-
verdict,
|
|
81
|
-
diagnosis
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
function buildDiagnosis(r) {
|
|
85
|
-
if (r.totalRecords === 0) {
|
|
86
|
-
return "no records \u2014 eval produced zero runs; backend likely failed before first turn";
|
|
87
|
-
}
|
|
88
|
-
if (r.verdict === "stub") {
|
|
89
|
-
return [
|
|
90
|
-
`all ${r.totalRecords} records have zero token usage \u2014 the LLM backend was never called.`,
|
|
91
|
-
"common causes: --backend sandbox without a sandbox bridge running; stub model returning hard-coded strings;",
|
|
92
|
-
"auth misconfigured so requests were silently dropped before the LLM. Re-run with --backend tcloud and TANGLE_API_KEY set,",
|
|
93
|
-
"or boot the cli-bridge / sandbox before invoking the eval."
|
|
94
|
-
].join(" ");
|
|
95
|
-
}
|
|
96
|
-
if (r.verdict === "mixed") {
|
|
97
|
-
const pct = (r.stubRecords / r.totalRecords * 100).toFixed(0);
|
|
98
|
-
return [
|
|
99
|
-
`${r.stubRecords}/${r.totalRecords} records (${pct}%) have zero token usage \u2014 the backend partially failed.`,
|
|
100
|
-
"common causes: rate-limit cascade (429s after the first N personas);",
|
|
101
|
-
"transient auth expiry mid-run; provider outage. Treat the affected records as missing data, not agent failures."
|
|
102
|
-
].join(" ");
|
|
103
|
-
}
|
|
104
|
-
if (r.uncostedRecords > 0) {
|
|
105
|
-
const pct = (r.uncostedRecords / r.totalRecords * 100).toFixed(0);
|
|
106
|
-
return [
|
|
107
|
-
`${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens).`,
|
|
108
|
-
`${r.uncostedRecords} (${pct}%) have output tokens but costUsd=0. Two distinct roots:`,
|
|
109
|
-
"(a) cost ledger mis-wired \u2014 no usage propagation from the runtime stream into RunRecord; or",
|
|
110
|
-
"(b) the model is unpriced at the source (sandbox/router returned $0 despite real tokens).",
|
|
111
|
-
"For (b), price the measured tokens against the substrate table (estimateCost) instead of leaving $0."
|
|
112
|
-
].join(" ");
|
|
113
|
-
}
|
|
114
|
-
return `${r.totalRecords} records with real LLM activity (in=${r.totalInputTokens}, out=${r.totalOutputTokens} tokens, $${r.totalCostUsd.toFixed(4)}).`;
|
|
115
|
-
}
|
|
116
|
-
function assertRealBackend(records, opts = {}) {
|
|
117
|
-
const report = summarizeBackendIntegrity(records);
|
|
118
|
-
return assertBackendReport(report, opts);
|
|
119
|
-
}
|
|
120
|
-
function assertRealAgentReceipts(receipts, opts = {}) {
|
|
121
|
-
const report = summarizeAgentReceiptIntegrity(receipts);
|
|
122
|
-
return assertBackendReport(report, opts);
|
|
123
|
-
}
|
|
124
|
-
function assertBackendReport(report, opts) {
|
|
125
|
-
const allowMixed = opts.allowMixed ?? true;
|
|
126
|
-
if (report.verdict === "stub") {
|
|
127
|
-
throw new BackendIntegrityError(
|
|
128
|
-
`backend-integrity: ran against a stub or unconfigured backend \u2014 ${report.diagnosis}`,
|
|
129
|
-
report
|
|
130
|
-
);
|
|
131
|
-
}
|
|
132
|
-
if (!allowMixed && report.verdict === "mixed") {
|
|
133
|
-
throw new BackendIntegrityError(
|
|
134
|
-
`backend-integrity: partial backend failure rejected \u2014 ${report.diagnosis}`,
|
|
135
|
-
report
|
|
136
|
-
);
|
|
137
|
-
}
|
|
138
|
-
return report;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
// src/verdict-cache.ts
|
|
142
|
-
import { createHash } from "crypto";
|
|
143
|
-
import { appendFileSync, existsSync, readFileSync } from "fs";
|
|
144
|
-
function canonicalizeAt(value, path) {
|
|
145
|
-
if (value === null) return "null";
|
|
146
|
-
switch (typeof value) {
|
|
147
|
-
case "boolean":
|
|
148
|
-
return value ? "true" : "false";
|
|
149
|
-
case "number":
|
|
150
|
-
if (!Number.isFinite(value)) {
|
|
151
|
-
throw new Error(
|
|
152
|
-
`canonicalJson: non-finite number (${value}) at ${path} \u2014 ambiguity is an error, not a coercion`
|
|
153
|
-
);
|
|
154
|
-
}
|
|
155
|
-
return JSON.stringify(value);
|
|
156
|
-
case "string":
|
|
157
|
-
return JSON.stringify(value);
|
|
158
|
-
case "undefined":
|
|
159
|
-
case "function":
|
|
160
|
-
case "symbol":
|
|
161
|
-
throw new Error(
|
|
162
|
-
`canonicalJson: ${typeof value} at ${path} \u2014 ambiguity is an error, not a coercion`
|
|
163
|
-
);
|
|
164
|
-
case "bigint":
|
|
165
|
-
throw new Error(`canonicalJson: bigint at ${path} \u2014 not representable in JSON`);
|
|
166
|
-
case "object":
|
|
167
|
-
break;
|
|
168
|
-
}
|
|
169
|
-
const obj = value;
|
|
170
|
-
if (typeof obj.toJSON === "function") {
|
|
171
|
-
return canonicalizeAt(obj.toJSON(), path);
|
|
172
|
-
}
|
|
173
|
-
if (Array.isArray(obj)) {
|
|
174
|
-
return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(",")}]`;
|
|
175
|
-
}
|
|
176
|
-
if (obj instanceof Map || obj instanceof Set) {
|
|
177
|
-
throw new Error(
|
|
178
|
-
`canonicalJson: ${obj instanceof Map ? "Map" : "Set"} at ${path} \u2014 would serialize as '{}'; convert to a plain object/array first`
|
|
179
|
-
);
|
|
180
|
-
}
|
|
181
|
-
const keys = Object.keys(obj).sort();
|
|
182
|
-
const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`);
|
|
183
|
-
return `{${parts.join(",")}}`;
|
|
184
|
-
}
|
|
185
|
-
function canonicalJson(value) {
|
|
186
|
-
return canonicalizeAt(value, "$");
|
|
187
|
-
}
|
|
188
|
-
function contentHash(value) {
|
|
189
|
-
return createHash("sha256").update(canonicalJson(value)).digest("hex");
|
|
190
|
-
}
|
|
191
|
-
function inMemoryVerdictCache() {
|
|
192
|
-
const entries = /* @__PURE__ */ new Map();
|
|
193
|
-
return {
|
|
194
|
-
get: (key) => entries.get(key),
|
|
195
|
-
set: (key, score) => {
|
|
196
|
-
entries.set(key, score);
|
|
197
|
-
}
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
function parseCacheLine(line, path, lineNo) {
|
|
201
|
-
let parsed;
|
|
202
|
-
try {
|
|
203
|
-
parsed = JSON.parse(line);
|
|
204
|
-
} catch (err) {
|
|
205
|
-
throw new Error(
|
|
206
|
-
`fileVerdictCache: corrupt JSONL at ${path}:${lineNo} \u2014 ${err instanceof Error ? err.message : String(err)}`
|
|
207
|
-
);
|
|
208
|
-
}
|
|
209
|
-
const rec = parsed;
|
|
210
|
-
if (typeof rec !== "object" || rec === null || typeof rec.key !== "string" || typeof rec.score !== "object" || rec.score === null || typeof rec.score.composite !== "number" || typeof rec.score.dimensions !== "object") {
|
|
211
|
-
throw new Error(
|
|
212
|
-
`fileVerdictCache: invalid record shape at ${path}:${lineNo} \u2014 expected {key, score:{dimensions, composite, notes}}`
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
return rec;
|
|
216
|
-
}
|
|
217
|
-
function fileVerdictCache(path) {
|
|
218
|
-
const entries = /* @__PURE__ */ new Map();
|
|
219
|
-
if (existsSync(path)) {
|
|
220
|
-
const lines = readFileSync(path, "utf8").split("\n");
|
|
221
|
-
for (let i = 0; i < lines.length; i++) {
|
|
222
|
-
const line = lines[i];
|
|
223
|
-
if (line === void 0 || line.trim() === "") continue;
|
|
224
|
-
const rec = parseCacheLine(line, path, i + 1);
|
|
225
|
-
entries.set(rec.key, rec.score);
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
return {
|
|
229
|
-
get: (key) => entries.get(key),
|
|
230
|
-
set: (key, score) => {
|
|
231
|
-
appendFileSync(path, `${JSON.stringify({ key, score })}
|
|
232
|
-
`, "utf8");
|
|
233
|
-
entries.set(key, score);
|
|
234
|
-
}
|
|
235
|
-
};
|
|
236
|
-
}
|
|
237
|
-
function cachedJudge(judge, store, options) {
|
|
238
|
-
if (typeof options.judgeVersion !== "string" || options.judgeVersion.trim() === "") {
|
|
239
|
-
throw new Error("cachedJudge: judgeVersion is required and must be a non-empty string");
|
|
240
|
-
}
|
|
241
|
-
const stats = { hits: 0, misses: 0 };
|
|
242
|
-
const wrapped = {
|
|
243
|
-
name: judge.name,
|
|
244
|
-
dimensions: judge.dimensions,
|
|
245
|
-
judgeVersion: options.judgeVersion,
|
|
246
|
-
async score(input) {
|
|
247
|
-
const key = contentHash({
|
|
248
|
-
artifact: canonicalJson(input.artifact),
|
|
249
|
-
scenarioId: input.scenario.id,
|
|
250
|
-
judgeName: judge.name,
|
|
251
|
-
dimensions: judge.dimensions,
|
|
252
|
-
judgeVersion: options.judgeVersion
|
|
253
|
-
});
|
|
254
|
-
const cached = await store.get(key);
|
|
255
|
-
if (cached !== void 0) {
|
|
256
|
-
stats.hits += 1;
|
|
257
|
-
return cached;
|
|
258
|
-
}
|
|
259
|
-
const score = await judge.score(input);
|
|
260
|
-
await store.set(key, score);
|
|
261
|
-
stats.misses += 1;
|
|
262
|
-
return score;
|
|
263
|
-
},
|
|
264
|
-
stats: () => ({ ...stats })
|
|
265
|
-
};
|
|
266
|
-
if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo;
|
|
267
|
-
return wrapped;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
// src/campaign/coverage.ts
|
|
271
|
-
function assertCampaignDesign(scenarios, reps) {
|
|
272
|
-
if (!Number.isSafeInteger(reps) || reps < 1) {
|
|
273
|
-
throw new Error("campaign design requires reps to be a positive safe integer");
|
|
274
|
-
}
|
|
275
|
-
const scenarioIds = /* @__PURE__ */ new Set();
|
|
276
|
-
for (const scenario of scenarios) {
|
|
277
|
-
if (typeof scenario.id !== "string" || scenario.id.trim().length === 0) {
|
|
278
|
-
throw new Error("campaign design requires every scenario to have a non-empty id");
|
|
279
|
-
}
|
|
280
|
-
if (scenarioIds.has(scenario.id)) {
|
|
281
|
-
throw new Error(`campaign design contains duplicate scenario id '${scenario.id}'`);
|
|
282
|
-
}
|
|
283
|
-
if (typeof scenario.kind !== "string" || scenario.kind.trim().length === 0) {
|
|
284
|
-
throw new Error("campaign design requires every scenario to have a non-empty kind");
|
|
285
|
-
}
|
|
286
|
-
if (scenario.seedGroup !== void 0 && (typeof scenario.seedGroup !== "string" || scenario.seedGroup.trim().length === 0)) {
|
|
287
|
-
throw new Error("campaign design requires seedGroup to be a non-empty string when set");
|
|
288
|
-
}
|
|
289
|
-
scenarioIds.add(scenario.id);
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
function campaignScenarioIdentity(scenario) {
|
|
293
|
-
assertCampaignDesign([scenario], 1);
|
|
294
|
-
return {
|
|
295
|
-
id: scenario.id,
|
|
296
|
-
kind: scenario.kind,
|
|
297
|
-
scenarioDigest: `sha256:${contentHash(scenario)}`
|
|
298
|
-
};
|
|
299
|
-
}
|
|
300
|
-
function campaignSplitDigestFromIdentities(scenarios, reps) {
|
|
301
|
-
assertCampaignDesign(scenarios, reps);
|
|
302
|
-
for (const scenario of scenarios) {
|
|
303
|
-
if (!/^sha256:[a-f0-9]{64}$/.test(scenario.scenarioDigest)) {
|
|
304
|
-
throw new Error(`campaign scenario '${scenario.id}' has an invalid digest`);
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
return `sha256:${contentHash({
|
|
308
|
-
schema: "tangle.campaign-split",
|
|
309
|
-
scenarios: scenarios.map(({ id, kind, scenarioDigest }) => ({ id, kind, scenarioDigest })),
|
|
310
|
-
reps
|
|
311
|
-
})}`;
|
|
312
|
-
}
|
|
313
|
-
function campaignSplitDigest(scenarios, reps) {
|
|
314
|
-
assertCampaignDesign(scenarios, reps);
|
|
315
|
-
return campaignSplitDigestFromIdentities(scenarios.map(campaignScenarioIdentity), reps);
|
|
316
|
-
}
|
|
317
|
-
function assertCampaignSplitIdentity(scenarios, reps, splitDigest) {
|
|
318
|
-
if (campaignSplitDigestFromIdentities(scenarios, reps) !== splitDigest) {
|
|
319
|
-
throw new Error("campaign split digest does not match its retained scenario identities");
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
function campaignCoverage(cells, scenarios, reps, requireJudgeScore) {
|
|
323
|
-
assertCampaignDesign(scenarios, reps);
|
|
324
|
-
const expectedCellIds = designedCellIds(scenarios, reps);
|
|
325
|
-
const cellsById = /* @__PURE__ */ new Map();
|
|
326
|
-
for (const cell of cells) {
|
|
327
|
-
const matches = cellsById.get(cell.cellId) ?? [];
|
|
328
|
-
matches.push(cell);
|
|
329
|
-
cellsById.set(cell.cellId, matches);
|
|
330
|
-
}
|
|
331
|
-
const scorableCellIds = [];
|
|
332
|
-
const unscorableCells = [];
|
|
333
|
-
for (const cellId of expectedCellIds) {
|
|
334
|
-
const matches = cellsById.get(cellId) ?? [];
|
|
335
|
-
if (matches.length === 0) {
|
|
336
|
-
unscorableCells.push({ cellId, reason: "missing campaign cell" });
|
|
337
|
-
continue;
|
|
338
|
-
}
|
|
339
|
-
if (matches.length > 1) {
|
|
340
|
-
unscorableCells.push({ cellId, reason: `duplicate campaign cell (${matches.length})` });
|
|
341
|
-
continue;
|
|
342
|
-
}
|
|
343
|
-
const cell = matches[0];
|
|
344
|
-
const scoreEntries = Object.entries(cell.judgeScores);
|
|
345
|
-
const successfulScores = scoreEntries.map(([, score]) => score).filter((score) => score.failed !== true && Number.isFinite(score.composite));
|
|
346
|
-
const nonFiniteScores = scoreEntries.filter(
|
|
347
|
-
([, score]) => score.failed !== true && (!Number.isFinite(score.composite) || Object.values(score.dimensions).some((value) => !Number.isFinite(value)))
|
|
348
|
-
);
|
|
349
|
-
const reasons = [];
|
|
350
|
-
if (cell.error) reasons.push(cell.error);
|
|
351
|
-
if (cell.artifact === null || cell.artifact === void 0) reasons.push("missing artifact");
|
|
352
|
-
if (!cell.error && requireJudgeScore && successfulScores.length === 0) {
|
|
353
|
-
reasons.push("no successful finite judge score");
|
|
354
|
-
}
|
|
355
|
-
if (scoreEntries.some(([, score]) => score.failed === true)) {
|
|
356
|
-
reasons.push("judge score marked failed");
|
|
357
|
-
}
|
|
358
|
-
const failedPanelJudges = [
|
|
359
|
-
...new Set(scoreEntries.flatMap(([, score]) => score.failedJudges ?? []))
|
|
360
|
-
].sort();
|
|
361
|
-
if (failedPanelJudges.length > 0) {
|
|
362
|
-
reasons.push(`judge panel incomplete: ${failedPanelJudges.join(", ")}`);
|
|
363
|
-
}
|
|
364
|
-
if (nonFiniteScores.length > 0) {
|
|
365
|
-
reasons.push(
|
|
366
|
-
`non-finite judge score: ${nonFiniteScores.map(([name]) => name).sort().join(", ")}`
|
|
367
|
-
);
|
|
368
|
-
}
|
|
369
|
-
if (reasons.length > 0) {
|
|
370
|
-
unscorableCells.push({ cellId, reason: reasons.join("; ") });
|
|
371
|
-
} else {
|
|
372
|
-
scorableCellIds.push(cellId);
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
const expected = new Set(expectedCellIds);
|
|
376
|
-
for (const cell of cells) {
|
|
377
|
-
if (cell.cellId !== `${cell.scenarioId}:${cell.rep}`) {
|
|
378
|
-
unscorableCells.push({
|
|
379
|
-
cellId: cell.cellId,
|
|
380
|
-
reason: "campaign cell id does not match scenario id and rep"
|
|
381
|
-
});
|
|
382
|
-
continue;
|
|
383
|
-
}
|
|
384
|
-
if (!expected.has(cell.cellId)) {
|
|
385
|
-
unscorableCells.push({ cellId: cell.cellId, reason: "unexpected campaign cell" });
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
return {
|
|
389
|
-
complete: unscorableCells.length === 0 && scorableCellIds.length === expectedCellIds.length,
|
|
390
|
-
expectedCellIds,
|
|
391
|
-
scorableCellIds,
|
|
392
|
-
unscorableCells
|
|
393
|
-
};
|
|
394
|
-
}
|
|
395
|
-
function formatCoverageFailures(coverage) {
|
|
396
|
-
const shown = coverage.unscorableCells.slice(0, 3).map((cell) => `${cell.cellId}: ${cell.reason}`).join("; ");
|
|
397
|
-
const remainder = coverage.unscorableCells.length - Math.min(3, coverage.unscorableCells.length);
|
|
398
|
-
return remainder > 0 ? `${shown}; +${remainder} more` : shown || "unknown coverage failure";
|
|
399
|
-
}
|
|
400
|
-
function designedCellIds(scenarios, reps) {
|
|
401
|
-
const ids = [];
|
|
402
|
-
for (const scenario of scenarios) {
|
|
403
|
-
for (let rep = 0; rep < reps; rep++) ids.push(`${scenario.id}:${rep}`);
|
|
404
|
-
}
|
|
405
|
-
return ids;
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
// src/campaign/run-dir.ts
|
|
409
|
-
import { homedir } from "os";
|
|
410
|
-
import { basename, isAbsolute, join } from "path";
|
|
411
|
-
function tangleTracesRoot() {
|
|
412
|
-
return join(homedir(), ".tangle", "traces");
|
|
413
|
-
}
|
|
414
|
-
function resolveRunDir(runDir, repo) {
|
|
415
|
-
if (isAbsolute(runDir) || runDir.startsWith("mem://")) return runDir;
|
|
416
|
-
const r = repo && repo.trim().length > 0 ? repo : basename(process.cwd());
|
|
417
|
-
return join(tangleTracesRoot(), r, "runs", runDir);
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
// src/campaign/storage.ts
|
|
421
|
-
import { createRequire } from "module";
|
|
422
|
-
import { join as join2 } from "path";
|
|
423
|
-
|
|
424
|
-
// src/campaign/search-ledger-file.ts
|
|
425
|
-
import { closeSync as closeSync2, constants as constants2, fsyncSync as fsyncSync2, mkdirSync, openSync as openSync2, writeSync } from "fs";
|
|
426
|
-
import { dirname } from "path";
|
|
427
|
-
|
|
428
|
-
// src/campaign/atomic-file-lock.ts
|
|
429
|
-
import { randomUUID } from "crypto";
|
|
430
|
-
import {
|
|
431
|
-
closeSync,
|
|
432
|
-
constants,
|
|
433
|
-
existsSync as existsSync2,
|
|
434
|
-
fsyncSync,
|
|
435
|
-
linkSync,
|
|
436
|
-
openSync,
|
|
437
|
-
readFileSync as readFileSync2,
|
|
438
|
-
renameSync,
|
|
439
|
-
unlinkSync,
|
|
440
|
-
writeFileSync
|
|
441
|
-
} from "fs";
|
|
442
|
-
import { hostname } from "os";
|
|
443
|
-
var AtomicFileLockError = class extends Error {
|
|
444
|
-
name = "AtomicFileLockError";
|
|
445
|
-
};
|
|
446
|
-
function probeAtomicFileLock(options) {
|
|
447
|
-
if (existsSync2(recoveryPath(options.lockPath))) return { acquired: false, reason: "recovery" };
|
|
448
|
-
const state = ownerState(options.lockPath);
|
|
449
|
-
if (state.state === "held") {
|
|
450
|
-
return { acquired: false, reason: "held", holder: state.owner };
|
|
451
|
-
}
|
|
452
|
-
return null;
|
|
453
|
-
}
|
|
454
|
-
function tryAcquireAtomicFileLock(options) {
|
|
455
|
-
const pid = options.pid ?? process.pid;
|
|
456
|
-
if (!Number.isSafeInteger(pid) || pid <= 0) {
|
|
457
|
-
throw new AtomicFileLockError("atomic file lock pid must be a positive integer");
|
|
458
|
-
}
|
|
459
|
-
const owner = { pid, host: hostname(), nonce: randomUUID() };
|
|
460
|
-
for (let attempt = 0; attempt < 8; attempt += 1) {
|
|
461
|
-
if (existsSync2(recoveryPath(options.lockPath))) {
|
|
462
|
-
return { acquired: false, reason: "recovery" };
|
|
463
|
-
}
|
|
464
|
-
if (tryLinkOwner(options.lockPath, owner, `acquire.${attempt}`)) {
|
|
465
|
-
return acquired(options.lockPath, owner);
|
|
466
|
-
}
|
|
467
|
-
const holder = ownerState(options.lockPath);
|
|
468
|
-
if (holder.state === "missing") continue;
|
|
469
|
-
if (holder.state === "held") {
|
|
470
|
-
return { acquired: false, reason: "held", holder: holder.owner };
|
|
471
|
-
}
|
|
472
|
-
const reclaimPath = recoveryPath(options.lockPath);
|
|
473
|
-
if (!tryLinkOwner(reclaimPath, owner, `reclaim.${attempt}`)) {
|
|
474
|
-
return { acquired: false, reason: "recovery" };
|
|
475
|
-
}
|
|
476
|
-
try {
|
|
477
|
-
const current = ownerState(options.lockPath);
|
|
478
|
-
if (current.state === "held") {
|
|
479
|
-
return { acquired: false, reason: "held", holder: current.owner };
|
|
480
|
-
}
|
|
481
|
-
if (current.state === "stale") {
|
|
482
|
-
const tombstone = `${options.lockPath}.stale.${owner.nonce}.${attempt}`;
|
|
483
|
-
try {
|
|
484
|
-
renameSync(options.lockPath, tombstone);
|
|
485
|
-
unlinkSync(tombstone);
|
|
486
|
-
} catch (error) {
|
|
487
|
-
if (!isMissing(error)) throw error;
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
if (tryLinkOwner(options.lockPath, owner, `recovered.${attempt}`)) {
|
|
491
|
-
return acquired(options.lockPath, owner);
|
|
492
|
-
}
|
|
493
|
-
} finally {
|
|
494
|
-
releaseOwnedPath(reclaimPath, owner);
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
throw new AtomicFileLockError(`could not acquire atomic file lock ${options.lockPath}`);
|
|
498
|
-
}
|
|
499
|
-
function acquired(lockPath, owner) {
|
|
500
|
-
return {
|
|
501
|
-
acquired: true,
|
|
502
|
-
lock: {
|
|
503
|
-
owner,
|
|
504
|
-
release: () => releaseOwnedPath(lockPath, owner)
|
|
505
|
-
}
|
|
506
|
-
};
|
|
507
|
-
}
|
|
508
|
-
function tryLinkOwner(lockPath, owner, suffix) {
|
|
509
|
-
const ownerPath = `${lockPath}.${owner.pid}.${owner.nonce}.${suffix}.owner`;
|
|
510
|
-
const descriptor = openSync(
|
|
511
|
-
ownerPath,
|
|
512
|
-
constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
|
|
513
|
-
384
|
|
514
|
-
);
|
|
515
|
-
try {
|
|
516
|
-
writeFileSync(descriptor, `${canonicalOwner(owner)}
|
|
517
|
-
`, "utf8");
|
|
518
|
-
fsyncSync(descriptor);
|
|
519
|
-
} catch (error) {
|
|
520
|
-
closeSync(descriptor);
|
|
521
|
-
unlinkIfExists(ownerPath);
|
|
522
|
-
throw error;
|
|
523
|
-
}
|
|
524
|
-
closeSync(descriptor);
|
|
525
|
-
try {
|
|
526
|
-
linkSync(ownerPath, lockPath);
|
|
527
|
-
return true;
|
|
528
|
-
} catch (error) {
|
|
529
|
-
if (!isAlreadyExists(error)) throw error;
|
|
530
|
-
return false;
|
|
531
|
-
} finally {
|
|
532
|
-
unlinkIfExists(ownerPath);
|
|
533
|
-
}
|
|
534
|
-
}
|
|
535
|
-
function ownerState(lockPath) {
|
|
536
|
-
const owner = readOwner(lockPath);
|
|
537
|
-
if (!owner) return { state: "missing" };
|
|
538
|
-
if (owner.host !== hostname()) return { state: "held", owner };
|
|
539
|
-
try {
|
|
540
|
-
process.kill(owner.pid, 0);
|
|
541
|
-
return { state: "held", owner };
|
|
542
|
-
} catch (error) {
|
|
543
|
-
if (isNoSuchProcess(error)) return { state: "stale", owner };
|
|
544
|
-
return { state: "held", owner };
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
function readOwner(lockPath) {
|
|
548
|
-
let contents;
|
|
549
|
-
try {
|
|
550
|
-
contents = readFileSync2(lockPath, "utf8").trim();
|
|
551
|
-
} catch (error) {
|
|
552
|
-
if (isMissing(error)) return void 0;
|
|
553
|
-
throw error;
|
|
554
|
-
}
|
|
555
|
-
let value;
|
|
556
|
-
try {
|
|
557
|
-
value = JSON.parse(contents);
|
|
558
|
-
} catch (error) {
|
|
559
|
-
throw invalidOwner(lockPath, error);
|
|
560
|
-
}
|
|
561
|
-
if (!isOwner(value)) throw invalidOwner(lockPath);
|
|
562
|
-
return value;
|
|
563
|
-
}
|
|
564
|
-
function isOwner(value) {
|
|
565
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
566
|
-
const record = value;
|
|
567
|
-
return Object.keys(record).length === 3 && Number.isSafeInteger(record.pid) && Number(record.pid) > 0 && typeof record.host === "string" && record.host.trim().length > 0 && typeof record.nonce === "string" && record.nonce.trim().length > 0;
|
|
568
|
-
}
|
|
569
|
-
function invalidOwner(lockPath, cause) {
|
|
570
|
-
const error = new AtomicFileLockError(
|
|
571
|
-
`atomic file lock has an invalid owner (${lockPath}); refusing unsafe recovery`
|
|
572
|
-
);
|
|
573
|
-
if (cause !== void 0) Object.defineProperty(error, "cause", { value: cause });
|
|
574
|
-
return error;
|
|
575
|
-
}
|
|
576
|
-
function releaseOwnedPath(lockPath, owner) {
|
|
577
|
-
const current = readOwner(lockPath);
|
|
578
|
-
if (!current) return;
|
|
579
|
-
if (canonicalOwner(current) !== canonicalOwner(owner)) {
|
|
580
|
-
throw new AtomicFileLockError(`atomic file lock owner changed before release (${lockPath})`);
|
|
581
|
-
}
|
|
582
|
-
try {
|
|
583
|
-
unlinkSync(lockPath);
|
|
584
|
-
} catch (error) {
|
|
585
|
-
if (!isMissing(error)) throw error;
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
function canonicalOwner(owner) {
|
|
589
|
-
return JSON.stringify({ host: owner.host, nonce: owner.nonce, pid: owner.pid });
|
|
590
|
-
}
|
|
591
|
-
function recoveryPath(lockPath) {
|
|
592
|
-
return `${lockPath}.reclaim`;
|
|
593
|
-
}
|
|
594
|
-
function isAlreadyExists(error) {
|
|
595
|
-
return error instanceof Error && "code" in error && error.code === "EEXIST";
|
|
596
|
-
}
|
|
597
|
-
function isMissing(error) {
|
|
598
|
-
return error instanceof Error && "code" in error && error.code === "ENOENT";
|
|
599
|
-
}
|
|
600
|
-
function isNoSuchProcess(error) {
|
|
601
|
-
return error instanceof Error && "code" in error && error.code === "ESRCH";
|
|
602
|
-
}
|
|
603
|
-
function unlinkIfExists(path) {
|
|
604
|
-
try {
|
|
605
|
-
unlinkSync(path);
|
|
606
|
-
} catch (error) {
|
|
607
|
-
if (!isMissing(error)) throw error;
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
// src/campaign/search-ledger-errors.ts
|
|
612
|
-
var SearchLedgerError = class extends ValidationError {
|
|
613
|
-
};
|
|
614
|
-
var SearchLedgerIntegrityError = class extends SearchLedgerError {
|
|
615
|
-
};
|
|
616
|
-
var SearchLedgerConflictError = class extends SearchLedgerError {
|
|
617
|
-
};
|
|
618
|
-
|
|
619
|
-
// src/campaign/search-ledger-file.ts
|
|
620
|
-
function appendSearchLedgerLine(path, line) {
|
|
621
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
622
|
-
const fd = openSync2(path, constants2.O_CREAT | constants2.O_WRONLY | constants2.O_APPEND, 384);
|
|
623
|
-
try {
|
|
624
|
-
writeAll(fd, Buffer.from(line, "utf8"));
|
|
625
|
-
fsyncSync2(fd);
|
|
626
|
-
} finally {
|
|
627
|
-
closeSync2(fd);
|
|
628
|
-
}
|
|
629
|
-
fsyncDirectory(dirname(path));
|
|
630
|
-
}
|
|
631
|
-
function withSearchLedgerFileLock(ledgerPath, run) {
|
|
632
|
-
const result = tryWithSearchLedgerFileLock(ledgerPath, run);
|
|
633
|
-
if (!result.acquired) {
|
|
634
|
-
throw new SearchLedgerIntegrityError(`search ledger lock is held (${ledgerPath})`);
|
|
635
|
-
}
|
|
636
|
-
return result.value;
|
|
637
|
-
}
|
|
638
|
-
function tryWithSearchLedgerFileLock(ledgerPath, run) {
|
|
639
|
-
mkdirSync(dirname(ledgerPath), { recursive: true });
|
|
640
|
-
const lockPath = `${ledgerPath}.lock`;
|
|
641
|
-
try {
|
|
642
|
-
const acquisition = tryAcquireAtomicFileLock({ lockPath });
|
|
643
|
-
if (!acquisition.acquired) return { acquired: false };
|
|
644
|
-
try {
|
|
645
|
-
return { acquired: true, value: run() };
|
|
646
|
-
} finally {
|
|
647
|
-
acquisition.lock.release();
|
|
648
|
-
}
|
|
649
|
-
} catch (error) {
|
|
650
|
-
if (error instanceof AtomicFileLockError) {
|
|
651
|
-
throw new SearchLedgerIntegrityError(error.message, { cause: error });
|
|
652
|
-
}
|
|
653
|
-
throw error;
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
function writeAll(fd, bytes) {
|
|
657
|
-
let offset = 0;
|
|
658
|
-
while (offset < bytes.byteLength) {
|
|
659
|
-
const written = writeSync(fd, bytes, offset, bytes.byteLength - offset);
|
|
660
|
-
if (written <= 0) throw new SearchLedgerIntegrityError("filesystem wrote zero bytes");
|
|
661
|
-
offset += written;
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
function fsyncDirectory(path) {
|
|
665
|
-
const fd = openSync2(path, constants2.O_RDONLY);
|
|
666
|
-
try {
|
|
667
|
-
fsyncSync2(fd);
|
|
668
|
-
} finally {
|
|
669
|
-
closeSync2(fd);
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
// src/campaign/storage.ts
|
|
674
|
-
function fsCampaignStorage() {
|
|
675
|
-
const nodeRequire = createRequire(import.meta.url);
|
|
676
|
-
const { existsSync: existsSync3, mkdirSync: mkdirSync2, readFileSync: readFileSync3, statSync, writeFileSync: writeFileSync2 } = nodeRequire(
|
|
677
|
-
"node:fs"
|
|
678
|
-
);
|
|
679
|
-
return {
|
|
680
|
-
ensureDir(dir) {
|
|
681
|
-
if (!existsSync3(dir)) mkdirSync2(dir, { recursive: true });
|
|
682
|
-
},
|
|
683
|
-
exists(path) {
|
|
684
|
-
return existsSync3(path);
|
|
685
|
-
},
|
|
686
|
-
read(path) {
|
|
687
|
-
try {
|
|
688
|
-
return readFileSync3(path, "utf8");
|
|
689
|
-
} catch {
|
|
690
|
-
return void 0;
|
|
691
|
-
}
|
|
692
|
-
},
|
|
693
|
-
write(path, content) {
|
|
694
|
-
writeFileSync2(path, content);
|
|
695
|
-
},
|
|
696
|
-
append(path, content, expectedBytes) {
|
|
697
|
-
const result = tryWithSearchLedgerFileLock(path, () => {
|
|
698
|
-
let actualBytes = 0;
|
|
699
|
-
try {
|
|
700
|
-
actualBytes = statSync(path).size;
|
|
701
|
-
} catch (error) {
|
|
702
|
-
if (error.code !== "ENOENT") throw error;
|
|
703
|
-
}
|
|
704
|
-
if (actualBytes !== expectedBytes) return void 0;
|
|
705
|
-
appendSearchLedgerLine(path, content);
|
|
706
|
-
return expectedBytes + Buffer.byteLength(content);
|
|
707
|
-
});
|
|
708
|
-
return result.acquired ? result.value : void 0;
|
|
709
|
-
}
|
|
710
|
-
};
|
|
711
|
-
}
|
|
712
|
-
function inMemoryCampaignStorage() {
|
|
713
|
-
const files = /* @__PURE__ */ new Map();
|
|
714
|
-
const dirs = /* @__PURE__ */ new Set();
|
|
715
|
-
return {
|
|
716
|
-
ensureDir(dir) {
|
|
717
|
-
dirs.add(dir);
|
|
718
|
-
},
|
|
719
|
-
exists(path) {
|
|
720
|
-
return files.has(path) || dirs.has(path);
|
|
721
|
-
},
|
|
722
|
-
read(path) {
|
|
723
|
-
const value = files.get(path);
|
|
724
|
-
if (value === void 0) return void 0;
|
|
725
|
-
return typeof value === "string" ? value : new TextDecoder().decode(value);
|
|
726
|
-
},
|
|
727
|
-
write(path, content) {
|
|
728
|
-
files.set(path, content);
|
|
729
|
-
},
|
|
730
|
-
append(path, content, expectedBytes) {
|
|
731
|
-
const current = files.get(path);
|
|
732
|
-
const currentText = current === void 0 ? "" : typeof current === "string" ? current : new TextDecoder().decode(current);
|
|
733
|
-
const currentBytes = new TextEncoder().encode(currentText).byteLength;
|
|
734
|
-
if (currentBytes !== expectedBytes) return void 0;
|
|
735
|
-
files.set(path, `${currentText}${content}`);
|
|
736
|
-
return currentBytes + new TextEncoder().encode(content).byteLength;
|
|
737
|
-
}
|
|
738
|
-
};
|
|
739
|
-
}
|
|
740
|
-
function createRunCostLedger(input) {
|
|
741
|
-
const path = join2(input.runDir, "cost-ledger.jsonl");
|
|
742
|
-
input.storage.ensureDir(input.runDir);
|
|
743
|
-
return new CostLedger({
|
|
744
|
-
costCeilingUsd: input.costCeilingUsd,
|
|
745
|
-
persistence: {
|
|
746
|
-
read: () => {
|
|
747
|
-
const stored = input.storage.read(path);
|
|
748
|
-
if (stored === void 0 && input.storage.exists(path)) {
|
|
749
|
-
throw new Error(`CostLedger: cannot read existing event log '${path}'`);
|
|
750
|
-
}
|
|
751
|
-
const events = stored ?? "";
|
|
752
|
-
return {
|
|
753
|
-
revision: String(new TextEncoder().encode(events).byteLength),
|
|
754
|
-
events
|
|
755
|
-
};
|
|
756
|
-
},
|
|
757
|
-
append: (expectedRevision, event) => {
|
|
758
|
-
const expectedBytes = Number(expectedRevision);
|
|
759
|
-
if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0) {
|
|
760
|
-
throw new Error(`CostLedger: invalid storage revision '${expectedRevision}'`);
|
|
761
|
-
}
|
|
762
|
-
const next = input.storage.append(path, event, expectedBytes);
|
|
763
|
-
return next === void 0 ? void 0 : String(next);
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
});
|
|
767
|
-
}
|
|
768
|
-
|
|
769
|
-
// src/campaign/run-campaign.ts
|
|
770
|
-
async function runCampaign(opts) {
|
|
771
|
-
const seed = opts.seed ?? 42;
|
|
772
|
-
const reps = opts.reps ?? 1;
|
|
773
|
-
const resumable = opts.resumable ?? true;
|
|
774
|
-
const now = opts.now ?? (() => /* @__PURE__ */ new Date());
|
|
775
|
-
const judges = opts.judges ?? [];
|
|
776
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
777
|
-
const costPhase = opts.costPhase ?? "campaign";
|
|
778
|
-
const dispatchShutdownTimeoutMs = opts.dispatchShutdownTimeoutMs ?? 5e3;
|
|
779
|
-
assertCampaignDesign(opts.scenarios, reps);
|
|
780
|
-
if (!Number.isSafeInteger(dispatchShutdownTimeoutMs) || dispatchShutdownTimeoutMs <= 0) {
|
|
781
|
-
throw new Error("runCampaign: dispatchShutdownTimeoutMs must be a positive safe integer");
|
|
782
|
-
}
|
|
783
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
|
|
784
|
-
throw new Error("runCampaign: runDir is required and must be a non-empty string");
|
|
785
|
-
}
|
|
786
|
-
opts.runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
787
|
-
storage.ensureDir(opts.runDir);
|
|
788
|
-
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
789
|
-
storage,
|
|
790
|
-
runDir: opts.runDir,
|
|
791
|
-
costCeilingUsd: opts.costCeiling
|
|
792
|
-
});
|
|
793
|
-
if (opts.costCeiling !== void 0 && costLedger.costCeilingUsd !== opts.costCeiling) {
|
|
794
|
-
throw new Error("runCampaign: costCeiling must match the shared CostLedger ceiling");
|
|
795
|
-
}
|
|
796
|
-
const maxConcurrency = opts.maxConcurrency ?? 2;
|
|
797
|
-
const manifestHash = computeManifestHash({
|
|
798
|
-
scenarios: opts.scenarios,
|
|
799
|
-
judges,
|
|
800
|
-
dispatchRef: dispatchRefFor(opts.dispatch, opts.dispatchRef),
|
|
801
|
-
seed,
|
|
802
|
-
reps
|
|
803
|
-
});
|
|
804
|
-
const splitDigest = campaignSplitDigest(opts.scenarios, reps);
|
|
805
|
-
const startedAt = now();
|
|
806
|
-
const runAttemptId = globalThis.crypto.randomUUID();
|
|
807
|
-
const cells = [];
|
|
808
|
-
const artifactsByPath = {};
|
|
809
|
-
const schedule = buildCellSchedule(opts.scenarios, seed, reps);
|
|
810
|
-
const campaignAbort = new AbortController();
|
|
811
|
-
const onOwnerAbort = () => campaignAbort.abort(opts.signal?.reason);
|
|
812
|
-
if (opts.signal?.aborted) campaignAbort.abort(opts.signal.reason);
|
|
813
|
-
else opts.signal?.addEventListener("abort", onOwnerAbort, { once: true });
|
|
814
|
-
const campaignSignal = campaignAbort.signal;
|
|
815
|
-
const lanes = [];
|
|
816
|
-
let nextIdx = 0;
|
|
817
|
-
const cellsRef = cells;
|
|
818
|
-
let firstLaneError;
|
|
819
|
-
let firstCellFailure;
|
|
820
|
-
for (let i = 0; i < maxConcurrency; i++) {
|
|
821
|
-
lanes.push(
|
|
822
|
-
(async () => {
|
|
823
|
-
try {
|
|
824
|
-
while (true) {
|
|
825
|
-
if (campaignSignal.aborted) return;
|
|
826
|
-
const myIdx = nextIdx++;
|
|
827
|
-
if (myIdx >= schedule.length) return;
|
|
828
|
-
const slot = schedule[myIdx];
|
|
829
|
-
const result = await executeCell({
|
|
830
|
-
slot,
|
|
831
|
-
opts,
|
|
832
|
-
manifestHash,
|
|
833
|
-
resumable,
|
|
834
|
-
now,
|
|
835
|
-
storage,
|
|
836
|
-
buildTraceWriter: opts.buildTraceWriter ?? defaultBuildTraceWriter(storage),
|
|
837
|
-
signal: campaignSignal,
|
|
838
|
-
dispatchTimeoutMs: opts.dispatchTimeoutMs,
|
|
839
|
-
dispatchShutdownTimeoutMs,
|
|
840
|
-
costLedger,
|
|
841
|
-
costPhase,
|
|
842
|
-
runAttemptId,
|
|
843
|
-
onFailure: opts.abortOnCellError ? (failure) => {
|
|
844
|
-
if (firstCellFailure === void 0) {
|
|
845
|
-
firstCellFailure = failure;
|
|
846
|
-
campaignAbort.abort(failure.cause);
|
|
847
|
-
}
|
|
848
|
-
} : void 0
|
|
849
|
-
});
|
|
850
|
-
cellsRef.push(result.cell);
|
|
851
|
-
Object.assign(artifactsByPath, result.artifactsByPath);
|
|
852
|
-
if (opts.labeledStore && opts.labeledStore !== "off" && !result.cell.error) {
|
|
853
|
-
await captureToStore({
|
|
854
|
-
store: opts.labeledStore,
|
|
855
|
-
cell: result.cell,
|
|
856
|
-
scenario: slot.scenario,
|
|
857
|
-
opts,
|
|
858
|
-
now
|
|
859
|
-
}).catch((err) => {
|
|
860
|
-
console.warn(
|
|
861
|
-
`[runCampaign] capture failed for ${result.cell.cellId}: ${err instanceof Error ? err.message : String(err)}`
|
|
862
|
-
);
|
|
863
|
-
});
|
|
864
|
-
}
|
|
865
|
-
if (opts.abortOnCellError && result.failure) {
|
|
866
|
-
return;
|
|
867
|
-
}
|
|
868
|
-
}
|
|
869
|
-
} catch (error) {
|
|
870
|
-
if (firstLaneError === void 0) {
|
|
871
|
-
firstLaneError = error;
|
|
872
|
-
campaignAbort.abort(error);
|
|
873
|
-
}
|
|
874
|
-
throw error;
|
|
875
|
-
}
|
|
876
|
-
})()
|
|
877
|
-
);
|
|
878
|
-
}
|
|
879
|
-
const laneResults = await Promise.allSettled(lanes);
|
|
880
|
-
opts.signal?.removeEventListener("abort", onOwnerAbort);
|
|
881
|
-
const failedLane = laneResults.find(
|
|
882
|
-
(result) => result.status === "rejected"
|
|
883
|
-
);
|
|
884
|
-
if (firstCellFailure) throw firstCellFailure.cause;
|
|
885
|
-
if (failedLane) throw firstLaneError ?? failedLane.reason;
|
|
886
|
-
const endedAt = now();
|
|
887
|
-
cellsRef.sort((a, b) => a.cellId.localeCompare(b.cellId));
|
|
888
|
-
const campaignCost = costLedger.summary({ tags: { runDir: opts.runDir } });
|
|
889
|
-
const aggregates = computeAggregates(
|
|
890
|
-
cellsRef,
|
|
891
|
-
judges,
|
|
892
|
-
seed,
|
|
893
|
-
campaignCost
|
|
894
|
-
);
|
|
895
|
-
return {
|
|
896
|
-
manifestHash,
|
|
897
|
-
splitDigest,
|
|
898
|
-
seed,
|
|
899
|
-
reps,
|
|
900
|
-
startedAt: startedAt.toISOString(),
|
|
901
|
-
endedAt: endedAt.toISOString(),
|
|
902
|
-
durationMs: endedAt.getTime() - startedAt.getTime(),
|
|
903
|
-
cells: cellsRef,
|
|
904
|
-
aggregates,
|
|
905
|
-
runDir: opts.runDir,
|
|
906
|
-
artifactsByPath,
|
|
907
|
-
scenarios: opts.scenarios.map(campaignScenarioIdentity)
|
|
908
|
-
};
|
|
909
|
-
}
|
|
910
|
-
async function executeCell(args) {
|
|
911
|
-
const storage = args.storage;
|
|
912
|
-
const cellDir = join3(args.opts.runDir, args.slot.cellId.replace(/[^a-zA-Z0-9_-]/g, "_"));
|
|
913
|
-
storage.ensureDir(cellDir);
|
|
914
|
-
const stableCostTags = {
|
|
915
|
-
...args.opts.costTags ?? {},
|
|
916
|
-
runDir: args.opts.runDir,
|
|
917
|
-
cellId: args.slot.cellId,
|
|
918
|
-
scenarioId: args.slot.scenario.id,
|
|
919
|
-
rep: String(args.slot.rep)
|
|
920
|
-
};
|
|
921
|
-
const costTags = { ...stableCostTags, runAttemptId: args.runAttemptId };
|
|
922
|
-
const cachePath = join3(cellDir, "cached-result.json");
|
|
923
|
-
if (args.resumable) {
|
|
924
|
-
const cached = readCachedCell({
|
|
925
|
-
storage,
|
|
926
|
-
cachePath,
|
|
927
|
-
cellId: args.slot.cellId,
|
|
928
|
-
manifestHash: args.manifestHash
|
|
929
|
-
});
|
|
930
|
-
if (cached.status === "hit") {
|
|
931
|
-
enforceDispatchUsage(cached.cell, args.opts.expectUsage ?? "warn");
|
|
932
|
-
const cachedHasUsage = cached.cell.costUsd > 0 || cached.cell.tokenUsage.input > 0 || cached.cell.tokenUsage.output > 0;
|
|
933
|
-
if (cached.cell.costCallIds === void 0) {
|
|
934
|
-
if (cachedHasUsage || Object.keys(cached.cell.judgeScores).length > 0) {
|
|
935
|
-
throw new CostAccountingIncompleteError(
|
|
936
|
-
`runCampaign: cached cell '${args.slot.cellId}' does not identify its ledger receipts`
|
|
937
|
-
);
|
|
938
|
-
}
|
|
939
|
-
} else if (!Array.isArray(cached.cell.costCallIds) || cached.cell.costCallIds.some(
|
|
940
|
-
(callId) => typeof callId !== "string" || callId.trim().length === 0
|
|
941
|
-
) || new Set(cached.cell.costCallIds).size !== cached.cell.costCallIds.length) {
|
|
942
|
-
throw new CostAccountingIncompleteError(
|
|
943
|
-
`runCampaign: cached cell '${args.slot.cellId}' has invalid ledger receipt IDs`
|
|
944
|
-
);
|
|
945
|
-
} else {
|
|
946
|
-
const restoredCallIds = new Set(
|
|
947
|
-
args.costLedger.list({ tags: stableCostTags }).map((receipt) => receipt.callId)
|
|
948
|
-
);
|
|
949
|
-
const missingCallIds = cached.cell.costCallIds.filter(
|
|
950
|
-
(callId) => !restoredCallIds.has(callId)
|
|
951
|
-
);
|
|
952
|
-
if (missingCallIds.length > 0) {
|
|
953
|
-
throw new CostAccountingIncompleteError(
|
|
954
|
-
`runCampaign: cached cell '${args.slot.cellId}' is missing ledger receipt(s): ${missingCallIds.join(", ")}`
|
|
955
|
-
);
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
return { cell: { ...cached.cell, cached: true }, artifactsByPath: {} };
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
const startMs = Date.now();
|
|
962
|
-
const trace = args.buildTraceWriter(args.slot.cellId, cellDir);
|
|
963
|
-
const artifactsByPath = {};
|
|
964
|
-
let paidCallStarted = false;
|
|
965
|
-
const artifacts = {
|
|
966
|
-
async write(path, content) {
|
|
967
|
-
const fullPath = join3(cellDir, path);
|
|
968
|
-
storage.ensureDir(join3(fullPath, ".."));
|
|
969
|
-
storage.write(fullPath, content);
|
|
970
|
-
artifactsByPath[`${args.slot.cellId}/${path}`] = fullPath;
|
|
971
|
-
return fullPath;
|
|
972
|
-
},
|
|
973
|
-
async writeJson(path, value) {
|
|
974
|
-
return artifacts.write(path, JSON.stringify(value, null, 2));
|
|
975
|
-
}
|
|
976
|
-
};
|
|
977
|
-
const cost = {
|
|
978
|
-
async runPaidCall(input) {
|
|
979
|
-
paidCallStarted = true;
|
|
980
|
-
const result = await args.costLedger.runPaidCall({
|
|
981
|
-
...input,
|
|
982
|
-
channel: input.channel ?? "agent",
|
|
983
|
-
phase: args.costPhase,
|
|
984
|
-
actor: input.actor,
|
|
985
|
-
tags: costTags,
|
|
986
|
-
signal: cellAbort.signal
|
|
987
|
-
});
|
|
988
|
-
if (result.receipt) {
|
|
989
|
-
trace.span(`cost.${result.receipt.actor}`, { amountUsd: result.receipt.costUsd }).end();
|
|
990
|
-
}
|
|
991
|
-
return result;
|
|
992
|
-
}
|
|
993
|
-
};
|
|
994
|
-
const placement = args.opts.cellPlacement?.({
|
|
995
|
-
scenario: args.slot.scenario,
|
|
996
|
-
rep: args.slot.rep
|
|
997
|
-
});
|
|
998
|
-
const cellAbort = new AbortController();
|
|
999
|
-
const onCampaignAbort = () => cellAbort.abort(args.signal.reason);
|
|
1000
|
-
if (args.signal.aborted) cellAbort.abort(args.signal.reason);
|
|
1001
|
-
else args.signal.addEventListener("abort", onCampaignAbort, { once: true });
|
|
1002
|
-
const ctx = {
|
|
1003
|
-
cellId: args.slot.cellId,
|
|
1004
|
-
rep: args.slot.rep,
|
|
1005
|
-
seed: args.slot.cellSeed,
|
|
1006
|
-
signal: cellAbort.signal,
|
|
1007
|
-
trace,
|
|
1008
|
-
artifacts,
|
|
1009
|
-
cost,
|
|
1010
|
-
placement
|
|
1011
|
-
};
|
|
1012
|
-
let artifact;
|
|
1013
|
-
let errorMessage;
|
|
1014
|
-
let failure;
|
|
1015
|
-
let fatalCellError;
|
|
1016
|
-
let dispatched;
|
|
1017
|
-
const timeoutMs = args.dispatchTimeoutMs;
|
|
1018
|
-
let timeoutTimer;
|
|
1019
|
-
let removeAbortListener = () => void 0;
|
|
1020
|
-
try {
|
|
1021
|
-
dispatched = Promise.resolve(args.opts.dispatch(args.slot.scenario, ctx));
|
|
1022
|
-
const aborted = new Promise((_resolve, reject) => {
|
|
1023
|
-
const rejectAbort = () => {
|
|
1024
|
-
const reason = cellAbort.signal.reason;
|
|
1025
|
-
reject(reason instanceof Error ? reason : new Error(String(reason ?? "dispatch aborted")));
|
|
1026
|
-
};
|
|
1027
|
-
if (cellAbort.signal.aborted) {
|
|
1028
|
-
rejectAbort();
|
|
1029
|
-
return;
|
|
1030
|
-
}
|
|
1031
|
-
cellAbort.signal.addEventListener("abort", rejectAbort, { once: true });
|
|
1032
|
-
removeAbortListener = () => cellAbort.signal.removeEventListener("abort", rejectAbort);
|
|
1033
|
-
});
|
|
1034
|
-
if (timeoutMs !== void 0 && timeoutMs > 0) {
|
|
1035
|
-
artifact = await Promise.race([
|
|
1036
|
-
dispatched,
|
|
1037
|
-
aborted,
|
|
1038
|
-
new Promise((_, reject) => {
|
|
1039
|
-
timeoutTimer = setTimeout(() => {
|
|
1040
|
-
const timeoutError = new Error(
|
|
1041
|
-
`dispatch exceeded ${timeoutMs}ms for cell '${args.slot.cellId}' \u2014 aborted and failed loud (no silent hang)`
|
|
1042
|
-
);
|
|
1043
|
-
reject(timeoutError);
|
|
1044
|
-
cellAbort.abort(timeoutError);
|
|
1045
|
-
}, timeoutMs);
|
|
1046
|
-
if (typeof timeoutTimer.unref === "function")
|
|
1047
|
-
timeoutTimer.unref();
|
|
1048
|
-
})
|
|
1049
|
-
]);
|
|
1050
|
-
} else {
|
|
1051
|
-
artifact = await Promise.race([dispatched, aborted]);
|
|
1052
|
-
}
|
|
1053
|
-
} catch (err) {
|
|
1054
|
-
errorMessage = err instanceof Error ? err.message : String(err);
|
|
1055
|
-
failure = { stage: "dispatch", cause: err };
|
|
1056
|
-
} finally {
|
|
1057
|
-
if (timeoutTimer) clearTimeout(timeoutTimer);
|
|
1058
|
-
removeAbortListener();
|
|
1059
|
-
args.signal.removeEventListener("abort", onCampaignAbort);
|
|
1060
|
-
}
|
|
1061
|
-
if (dispatched) {
|
|
1062
|
-
const dispatchSettled = await settlesWithin(dispatched, args.dispatchShutdownTimeoutMs);
|
|
1063
|
-
if (!dispatchSettled) {
|
|
1064
|
-
await trace.flush();
|
|
1065
|
-
throw new CostAccountingIncompleteError(
|
|
1066
|
-
`dispatch for cell '${args.slot.cellId}' ignored cancellation and did not stop within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`
|
|
1067
|
-
);
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
if (paidCallStarted) {
|
|
1071
|
-
if (typeof args.costLedger.waitForIdle !== "function") {
|
|
1072
|
-
await trace.flush();
|
|
1073
|
-
throw new CostAccountingIncompleteError(
|
|
1074
|
-
`cost ledger for cell '${args.slot.cellId}' cannot prove that paid calls stopped`
|
|
1075
|
-
);
|
|
1076
|
-
}
|
|
1077
|
-
const paidCallsSettled = await args.costLedger.waitForIdle({
|
|
1078
|
-
timeoutMs: args.dispatchShutdownTimeoutMs,
|
|
1079
|
-
filter: { channel: "agent", phase: args.costPhase, tags: costTags }
|
|
1080
|
-
});
|
|
1081
|
-
if (!paidCallsSettled) {
|
|
1082
|
-
await trace.flush();
|
|
1083
|
-
throw new CostAccountingIncompleteError(
|
|
1084
|
-
`paid calls for cell '${args.slot.cellId}' did not settle within ${args.dispatchShutdownTimeoutMs}ms; no campaign result was produced`
|
|
1085
|
-
);
|
|
1086
|
-
}
|
|
1087
|
-
}
|
|
1088
|
-
const agentReceipts = args.costLedger.list({ channel: "agent", tags: costTags });
|
|
1089
|
-
const agentCost = args.costLedger.summary({ channel: "agent", tags: costTags });
|
|
1090
|
-
const tokenUsage = {
|
|
1091
|
-
input: agentCost.inputTokens,
|
|
1092
|
-
output: agentCost.outputTokens,
|
|
1093
|
-
...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {}
|
|
1094
|
-
};
|
|
1095
|
-
const resolvedModel = agentReceipts.at(-1)?.model;
|
|
1096
|
-
const dispatchResult = {
|
|
1097
|
-
cellId: args.slot.cellId,
|
|
1098
|
-
artifact,
|
|
1099
|
-
error: errorMessage,
|
|
1100
|
-
costUsd: agentCost.totalCostUsd,
|
|
1101
|
-
tokenUsage
|
|
1102
|
-
};
|
|
1103
|
-
try {
|
|
1104
|
-
enforceDispatchUsage(dispatchResult, args.opts.expectUsage ?? "warn");
|
|
1105
|
-
} catch (error) {
|
|
1106
|
-
await trace.flush();
|
|
1107
|
-
throw error;
|
|
1108
|
-
}
|
|
1109
|
-
const judgeScores = {};
|
|
1110
|
-
if (artifact !== void 0) {
|
|
1111
|
-
for (const judge of args.opts.judges ?? []) {
|
|
1112
|
-
if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue;
|
|
1113
|
-
try {
|
|
1114
|
-
const score = await runJudgeCell(judge, {
|
|
1115
|
-
artifact,
|
|
1116
|
-
scenario: args.slot.scenario,
|
|
1117
|
-
signal: args.signal,
|
|
1118
|
-
costLedger: args.costLedger,
|
|
1119
|
-
costPhase: args.costPhase,
|
|
1120
|
-
costTags
|
|
1121
|
-
});
|
|
1122
|
-
judgeScores[judge.name] = score;
|
|
1123
|
-
} catch (err) {
|
|
1124
|
-
errorMessage = `judge '${judge.name}' failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
1125
|
-
failure = { stage: "judge", judge: judge.name, cause: err };
|
|
1126
|
-
if (err instanceof CostAccountingIncompleteError) fatalCellError = err;
|
|
1127
|
-
break;
|
|
1128
|
-
}
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1131
|
-
if (failure) {
|
|
1132
|
-
await waitForFailedCellCostSettlement({
|
|
1133
|
-
costLedger: args.costLedger,
|
|
1134
|
-
costPhase: args.costPhase,
|
|
1135
|
-
costTags,
|
|
1136
|
-
cellId: args.slot.cellId,
|
|
1137
|
-
timeoutMs: args.dispatchShutdownTimeoutMs
|
|
1138
|
-
});
|
|
1139
|
-
}
|
|
1140
|
-
const costCallIds = args.costLedger.list({ tags: costTags }).map((receipt) => receipt.callId).sort();
|
|
1141
|
-
const cell = {
|
|
1142
|
-
manifestHash: args.manifestHash,
|
|
1143
|
-
cellId: args.slot.cellId,
|
|
1144
|
-
scenarioId: args.slot.scenario.id,
|
|
1145
|
-
rep: args.slot.rep,
|
|
1146
|
-
artifact: artifact ?? null,
|
|
1147
|
-
judgeScores,
|
|
1148
|
-
costUsd: agentCost.totalCostUsd,
|
|
1149
|
-
costEstimated: agentReceipts.some(
|
|
1150
|
-
(receipt) => receipt.actualCostUsd === void 0 && !receipt.costUnknown
|
|
1151
|
-
),
|
|
1152
|
-
costCallIds,
|
|
1153
|
-
tokenUsage,
|
|
1154
|
-
...resolvedModel ? { resolvedModel } : {},
|
|
1155
|
-
durationMs: Date.now() - startMs,
|
|
1156
|
-
seed: args.slot.cellSeed,
|
|
1157
|
-
cached: false,
|
|
1158
|
-
...failure ? { errorStage: failure.stage } : {},
|
|
1159
|
-
...failure?.judge ? { errorJudge: failure.judge } : {},
|
|
1160
|
-
error: errorMessage
|
|
1161
|
-
};
|
|
1162
|
-
if (failure) {
|
|
1163
|
-
const failurePath = join3(cellDir, "failure-receipt.json");
|
|
1164
|
-
const receipt = {
|
|
1165
|
-
schemaVersion: 1,
|
|
1166
|
-
runAttemptId: args.runAttemptId,
|
|
1167
|
-
recordedAt: args.now().toISOString(),
|
|
1168
|
-
failure: {
|
|
1169
|
-
stage: failure.stage,
|
|
1170
|
-
...failure.judge ? { judge: failure.judge } : {},
|
|
1171
|
-
error: serializeCellError(failure.cause)
|
|
1172
|
-
},
|
|
1173
|
-
cell,
|
|
1174
|
-
cost: args.costLedger.summary({ phase: args.costPhase, tags: costTags })
|
|
1175
|
-
};
|
|
1176
|
-
storage.write(failurePath, JSON.stringify(receipt, null, 2));
|
|
1177
|
-
artifactsByPath[`${args.slot.cellId}/failure-receipt.json`] = failurePath;
|
|
1178
|
-
args.onFailure?.(failure);
|
|
1179
|
-
}
|
|
1180
|
-
await trace.flush();
|
|
1181
|
-
if (!errorMessage && args.resumable) {
|
|
1182
|
-
storage.write(cachePath, JSON.stringify(cell));
|
|
1183
|
-
}
|
|
1184
|
-
if (fatalCellError !== void 0) throw fatalCellError;
|
|
1185
|
-
return { cell, artifactsByPath, ...failure ? { failure } : {} };
|
|
1186
|
-
}
|
|
1187
|
-
async function waitForFailedCellCostSettlement(input) {
|
|
1188
|
-
const filter = { phase: input.costPhase, tags: input.costTags };
|
|
1189
|
-
if (input.costLedger.summary(filter).pendingCalls === 0) return;
|
|
1190
|
-
if (typeof input.costLedger.waitForIdle !== "function") {
|
|
1191
|
-
throw new CostAccountingIncompleteError(
|
|
1192
|
-
`cost ledger for failed cell '${input.cellId}' cannot prove that paid calls stopped`
|
|
1193
|
-
);
|
|
1194
|
-
}
|
|
1195
|
-
const settled = await input.costLedger.waitForIdle({
|
|
1196
|
-
timeoutMs: input.timeoutMs,
|
|
1197
|
-
filter
|
|
1198
|
-
});
|
|
1199
|
-
if (!settled || input.costLedger.summary(filter).pendingCalls > 0) {
|
|
1200
|
-
throw new CostAccountingIncompleteError(
|
|
1201
|
-
`paid calls for failed cell '${input.cellId}' did not settle within ${input.timeoutMs}ms; no complete failure receipt was produced`
|
|
1202
|
-
);
|
|
1203
|
-
}
|
|
1204
|
-
}
|
|
1205
|
-
function serializeCellError(error) {
|
|
1206
|
-
if (error instanceof Error) {
|
|
1207
|
-
return {
|
|
1208
|
-
name: error.name,
|
|
1209
|
-
message: error.message,
|
|
1210
|
-
...error.stack ? { stack: error.stack } : {}
|
|
1211
|
-
};
|
|
1212
|
-
}
|
|
1213
|
-
return { name: "NonErrorThrown", message: String(error) };
|
|
1214
|
-
}
|
|
1215
|
-
function settlesWithin(promise, timeoutMs) {
|
|
1216
|
-
return new Promise((resolve) => {
|
|
1217
|
-
let completed = false;
|
|
1218
|
-
let timer;
|
|
1219
|
-
const finish = (value) => {
|
|
1220
|
-
if (completed) return;
|
|
1221
|
-
completed = true;
|
|
1222
|
-
if (timer) clearTimeout(timer);
|
|
1223
|
-
resolve(value);
|
|
1224
|
-
};
|
|
1225
|
-
timer = setTimeout(() => finish(false), timeoutMs);
|
|
1226
|
-
promise.then(
|
|
1227
|
-
() => finish(true),
|
|
1228
|
-
() => finish(true)
|
|
1229
|
-
);
|
|
1230
|
-
});
|
|
1231
|
-
}
|
|
1232
|
-
function planCampaignRun(opts) {
|
|
1233
|
-
const seed = opts.seed ?? 42;
|
|
1234
|
-
const reps = opts.reps ?? 1;
|
|
1235
|
-
const resumable = opts.resumable ?? true;
|
|
1236
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
1237
|
-
assertCampaignDesign(opts.scenarios, reps);
|
|
1238
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
|
|
1239
|
-
throw new Error("planCampaignRun: runDir is required and must be a non-empty string");
|
|
1240
|
-
}
|
|
1241
|
-
opts.runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
1242
|
-
const manifestHash = computeManifestHash({
|
|
1243
|
-
scenarios: opts.scenarios,
|
|
1244
|
-
judges: opts.judges ?? [],
|
|
1245
|
-
dispatchRef: dispatchRefFor(opts.dispatch, opts.dispatchRef),
|
|
1246
|
-
seed,
|
|
1247
|
-
reps
|
|
1248
|
-
});
|
|
1249
|
-
const splitDigest = campaignSplitDigest(opts.scenarios, reps);
|
|
1250
|
-
const cells = buildCellSchedule(opts.scenarios, seed, reps).map((slot) => {
|
|
1251
|
-
const cachePath = join3(
|
|
1252
|
-
opts.runDir,
|
|
1253
|
-
slot.cellId.replace(/[^a-zA-Z0-9_-]/g, "_"),
|
|
1254
|
-
"cached-result.json"
|
|
1255
|
-
);
|
|
1256
|
-
if (!resumable) {
|
|
1257
|
-
return {
|
|
1258
|
-
cellId: slot.cellId,
|
|
1259
|
-
scenarioId: slot.scenario.id,
|
|
1260
|
-
rep: slot.rep,
|
|
1261
|
-
seed: slot.cellSeed,
|
|
1262
|
-
cachePath,
|
|
1263
|
-
status: "run",
|
|
1264
|
-
reason: "resumable-off"
|
|
1265
|
-
};
|
|
1266
|
-
}
|
|
1267
|
-
const cached = readCachedCell({
|
|
1268
|
-
storage,
|
|
1269
|
-
cachePath,
|
|
1270
|
-
cellId: slot.cellId,
|
|
1271
|
-
manifestHash
|
|
1272
|
-
});
|
|
1273
|
-
if (cached.status === "hit") {
|
|
1274
|
-
return {
|
|
1275
|
-
cellId: slot.cellId,
|
|
1276
|
-
scenarioId: slot.scenario.id,
|
|
1277
|
-
rep: slot.rep,
|
|
1278
|
-
seed: slot.cellSeed,
|
|
1279
|
-
cachePath,
|
|
1280
|
-
status: "cached"
|
|
1281
|
-
};
|
|
1282
|
-
}
|
|
1283
|
-
return {
|
|
1284
|
-
cellId: slot.cellId,
|
|
1285
|
-
scenarioId: slot.scenario.id,
|
|
1286
|
-
rep: slot.rep,
|
|
1287
|
-
seed: slot.cellSeed,
|
|
1288
|
-
cachePath,
|
|
1289
|
-
status: "run",
|
|
1290
|
-
reason: cached.reason
|
|
1291
|
-
};
|
|
1292
|
-
});
|
|
1293
|
-
const cellsCached = cells.filter((cell) => cell.status === "cached").length;
|
|
1294
|
-
return {
|
|
1295
|
-
manifestHash,
|
|
1296
|
-
splitDigest,
|
|
1297
|
-
totalCells: cells.length,
|
|
1298
|
-
cellsCached,
|
|
1299
|
-
cellsToRun: cells.length - cellsCached,
|
|
1300
|
-
cells
|
|
1301
|
-
};
|
|
1302
|
-
}
|
|
1303
|
-
function enforceDispatchUsage(cell, mode) {
|
|
1304
|
-
if (mode === "off") return;
|
|
1305
|
-
if (cell.artifact === null || cell.artifact === void 0) return;
|
|
1306
|
-
const zeroTokens = cell.tokenUsage.input === 0 && cell.tokenUsage.output === 0;
|
|
1307
|
-
if (cell.costUsd !== 0 || !zeroTokens) return;
|
|
1308
|
-
const msg = `cell '${cell.cellId}' produced an artifact but reported zero cost and zero tokens \u2014 the dispatch made no paid call through ctx.cost.runPaidCall (a stub cell)`;
|
|
1309
|
-
if (mode === "assert") {
|
|
1310
|
-
const report = {
|
|
1311
|
-
totalRecords: 1,
|
|
1312
|
-
stubRecords: 1,
|
|
1313
|
-
realRecords: 0,
|
|
1314
|
-
uncostedRecords: 0,
|
|
1315
|
-
totalInputTokens: 0,
|
|
1316
|
-
totalOutputTokens: 0,
|
|
1317
|
-
totalCostUsd: 0,
|
|
1318
|
-
verdict: "stub",
|
|
1319
|
-
diagnosis: msg
|
|
1320
|
-
};
|
|
1321
|
-
throw new BackendIntegrityError(`expectUsage: ${msg}`, report);
|
|
1322
|
-
}
|
|
1323
|
-
console.warn(`[runCampaign] expectUsage: ${msg}`);
|
|
1324
|
-
}
|
|
1325
|
-
async function runJudgeCell(judge, input) {
|
|
1326
|
-
const previousJudgeCalls = new Set(
|
|
1327
|
-
input.costLedger?.list({ channel: "judge", tags: input.costTags }).map((receipt) => receipt.callId) ?? []
|
|
1328
|
-
);
|
|
1329
|
-
try {
|
|
1330
|
-
const score = await judge.score(input);
|
|
1331
|
-
assertReportedJudgeCallRecorded(judge.name, score, input, previousJudgeCalls);
|
|
1332
|
-
return score;
|
|
1333
|
-
} catch (error) {
|
|
1334
|
-
assertReportedJudgeCallRecorded(judge.name, error, input, previousJudgeCalls, error);
|
|
1335
|
-
throw error;
|
|
1336
|
-
}
|
|
1337
|
-
}
|
|
1338
|
-
function assertReportedJudgeCallRecorded(judgeName, value, input, previousCallIds, cause) {
|
|
1339
|
-
if (!hasLlmCall(value)) return;
|
|
1340
|
-
const recorded = input.costLedger?.list({ channel: "judge", tags: input.costTags }).some((receipt) => !previousCallIds.has(receipt.callId));
|
|
1341
|
-
if (recorded) return;
|
|
1342
|
-
throw new CostAccountingIncompleteError(
|
|
1343
|
-
`runCampaign: judge '${judgeName}' reported a paid LLM call without a CostLedger receipt`,
|
|
1344
|
-
cause === void 0 ? void 0 : { cause }
|
|
1345
|
-
);
|
|
1346
|
-
}
|
|
1347
|
-
function hasLlmCall(value) {
|
|
1348
|
-
return typeof value === "object" && value !== null && "llmCall" in value && value.llmCall !== void 0;
|
|
1349
|
-
}
|
|
1350
|
-
function defaultBuildTraceWriter(storage) {
|
|
1351
|
-
return (cellId, dir) => {
|
|
1352
|
-
const spans = [];
|
|
1353
|
-
return {
|
|
1354
|
-
span(name, attributes) {
|
|
1355
|
-
const startMs = Date.now();
|
|
1356
|
-
const record = { name, cellId, startMs, ...attributes ?? {} };
|
|
1357
|
-
const finish = {
|
|
1358
|
-
end(endAttrs) {
|
|
1359
|
-
record.durationMs = Date.now() - startMs;
|
|
1360
|
-
if (endAttrs) Object.assign(record, endAttrs);
|
|
1361
|
-
spans.push(record);
|
|
1362
|
-
},
|
|
1363
|
-
setAttribute(key, value) {
|
|
1364
|
-
record[key] = value;
|
|
1365
|
-
}
|
|
1366
|
-
};
|
|
1367
|
-
return finish;
|
|
1368
|
-
},
|
|
1369
|
-
async flush() {
|
|
1370
|
-
storage.write(join3(dir, "spans.jsonl"), spans.map((s) => JSON.stringify(s)).join("\n"));
|
|
1371
|
-
}
|
|
1372
|
-
};
|
|
1373
|
-
};
|
|
1374
|
-
}
|
|
1375
|
-
function buildCellSchedule(scenarios, seed, reps) {
|
|
1376
|
-
const schedule = [];
|
|
1377
|
-
const groupIndexes = /* @__PURE__ */ new Map();
|
|
1378
|
-
let nextGroupIndex = 0;
|
|
1379
|
-
for (const scenario of scenarios) {
|
|
1380
|
-
let groupIndex;
|
|
1381
|
-
if (scenario.seedGroup === void 0) {
|
|
1382
|
-
groupIndex = nextGroupIndex;
|
|
1383
|
-
nextGroupIndex += 1;
|
|
1384
|
-
} else {
|
|
1385
|
-
const existing = groupIndexes.get(scenario.seedGroup);
|
|
1386
|
-
if (existing !== void 0) {
|
|
1387
|
-
groupIndex = existing;
|
|
1388
|
-
} else {
|
|
1389
|
-
groupIndex = nextGroupIndex;
|
|
1390
|
-
nextGroupIndex += 1;
|
|
1391
|
-
groupIndexes.set(scenario.seedGroup, groupIndex);
|
|
1392
|
-
}
|
|
1393
|
-
}
|
|
1394
|
-
for (let rep = 0; rep < reps; rep++) {
|
|
1395
|
-
const cellId = `${scenario.id}:${rep}`;
|
|
1396
|
-
const cellSeed = seed + groupIndex * reps + rep;
|
|
1397
|
-
schedule.push({ scenario, rep, cellId, cellSeed });
|
|
1398
|
-
}
|
|
1399
|
-
}
|
|
1400
|
-
return schedule;
|
|
1401
|
-
}
|
|
1402
|
-
function dispatchRefFor(dispatch, override) {
|
|
1403
|
-
const ref = override ?? dispatch?.name ?? "anonymous";
|
|
1404
|
-
if (typeof ref !== "string" || ref.trim().length === 0) {
|
|
1405
|
-
throw new Error("runCampaign: dispatchRef must be a non-empty string when provided");
|
|
1406
|
-
}
|
|
1407
|
-
return ref;
|
|
1408
|
-
}
|
|
1409
|
-
function readCachedCell(args) {
|
|
1410
|
-
const raw = args.storage.read(args.cachePath);
|
|
1411
|
-
if (raw === void 0) return { status: "miss", reason: "missing" };
|
|
1412
|
-
try {
|
|
1413
|
-
const cached = JSON.parse(raw);
|
|
1414
|
-
if (cached.cellId !== args.cellId) return { status: "miss", reason: "cell-mismatch" };
|
|
1415
|
-
if (cached.manifestHash !== args.manifestHash) {
|
|
1416
|
-
return { status: "miss", reason: "manifest-mismatch" };
|
|
1417
|
-
}
|
|
1418
|
-
return { status: "hit", cell: cached };
|
|
1419
|
-
} catch {
|
|
1420
|
-
return { status: "miss", reason: "corrupt" };
|
|
1421
|
-
}
|
|
1422
|
-
}
|
|
1423
|
-
async function captureToStore(args) {
|
|
1424
|
-
await args.store.observe({
|
|
1425
|
-
scenario: args.scenario,
|
|
1426
|
-
artifact: args.cell.artifact,
|
|
1427
|
-
judgeScores: args.cell.judgeScores,
|
|
1428
|
-
source: args.opts.captureSource ?? "eval-run",
|
|
1429
|
-
sourceVersionHash: args.opts.captureSourceVersionHash ?? "unknown",
|
|
1430
|
-
capturedAt: args.now().toISOString(),
|
|
1431
|
-
redactionStatus: "raw"
|
|
1432
|
-
});
|
|
1433
|
-
}
|
|
1434
|
-
function computeManifestHash(input) {
|
|
1435
|
-
return contentHash({
|
|
1436
|
-
scenarios: input.scenarios,
|
|
1437
|
-
judges: input.judges.map((judge) => ({
|
|
1438
|
-
name: judge.name,
|
|
1439
|
-
dims: judge.dimensions,
|
|
1440
|
-
version: judgeVersionFor(judge)
|
|
1441
|
-
})),
|
|
1442
|
-
dispatch: input.dispatchRef,
|
|
1443
|
-
seed: input.seed,
|
|
1444
|
-
reps: input.reps
|
|
1445
|
-
});
|
|
1446
|
-
}
|
|
1447
|
-
function judgeVersionFor(judge) {
|
|
1448
|
-
if (judge.judgeVersion !== void 0) {
|
|
1449
|
-
const version = judge.judgeVersion.trim();
|
|
1450
|
-
if (version.length === 0) {
|
|
1451
|
-
throw new Error(`runCampaign: judge '${judge.name}' has an empty judgeVersion`);
|
|
1452
|
-
}
|
|
1453
|
-
return version;
|
|
1454
|
-
}
|
|
1455
|
-
return contentHash({
|
|
1456
|
-
score: judge.score.toString(),
|
|
1457
|
-
appliesTo: judge.appliesTo?.toString() ?? null
|
|
1458
|
-
});
|
|
1459
|
-
}
|
|
1460
|
-
function computeAggregates(cells, judges, seed, cost) {
|
|
1461
|
-
const cellQuality = cells.map((cell) => ({
|
|
1462
|
-
cell,
|
|
1463
|
-
quality: projectCampaignCellQuality(cell)
|
|
1464
|
-
}));
|
|
1465
|
-
const byJudge = {};
|
|
1466
|
-
for (const judge of judges) {
|
|
1467
|
-
const scores = [];
|
|
1468
|
-
for (const { quality } of cellQuality) {
|
|
1469
|
-
const s = quality.successfulJudgeScores[judge.name];
|
|
1470
|
-
if (s !== void 0) scores.push(s.composite);
|
|
1471
|
-
}
|
|
1472
|
-
if (scores.length > 0) byJudge[judge.name] = aggregate(scores, seed);
|
|
1473
|
-
}
|
|
1474
|
-
const byScenario = {};
|
|
1475
|
-
const scenarioGroups = /* @__PURE__ */ new Map();
|
|
1476
|
-
for (const { cell, quality } of cellQuality) {
|
|
1477
|
-
const score = quality.score;
|
|
1478
|
-
if (score === void 0) continue;
|
|
1479
|
-
const arr = scenarioGroups.get(cell.scenarioId) ?? [];
|
|
1480
|
-
arr.push(score);
|
|
1481
|
-
scenarioGroups.set(cell.scenarioId, arr);
|
|
1482
|
-
}
|
|
1483
|
-
for (const [scenarioId, samples] of scenarioGroups) {
|
|
1484
|
-
const ag = aggregate(samples, seed);
|
|
1485
|
-
byScenario[scenarioId] = { meanComposite: ag.mean, ci95: ag.ci95, n: ag.n };
|
|
1486
|
-
}
|
|
1487
|
-
const dispatchFailures = cells.filter((cell) => cell.errorStage === "dispatch");
|
|
1488
|
-
const judgeFailures = cellQuality.filter(({ cell, quality }) => cell.errorStage === "judge" || quality.failedJudges.length > 0).map(({ cell }) => cell);
|
|
1489
|
-
const unclassifiedFailures = cells.filter(
|
|
1490
|
-
(cell) => Boolean(cell.error) && !cell.error?.startsWith("skipped:") && cell.errorStage === void 0
|
|
1491
|
-
);
|
|
1492
|
-
return {
|
|
1493
|
-
byJudge,
|
|
1494
|
-
byScenario,
|
|
1495
|
-
cost,
|
|
1496
|
-
cellsExecuted: cells.filter(
|
|
1497
|
-
(cell) => !cell.error?.startsWith("skipped:") && cell.errorStage !== "dispatch" && !(cell.error && cell.errorStage === void 0)
|
|
1498
|
-
).length,
|
|
1499
|
-
cellsSkipped: cells.filter((c) => c.error?.startsWith("skipped:")).length,
|
|
1500
|
-
cellsCached: cells.filter((c) => c.cached).length,
|
|
1501
|
-
cellsFailed: (/* @__PURE__ */ new Set([...dispatchFailures, ...judgeFailures, ...unclassifiedFailures])).size,
|
|
1502
|
-
cellsDispatchFailed: dispatchFailures.length,
|
|
1503
|
-
cellsJudgeFailed: judgeFailures.length,
|
|
1504
|
-
cellsUnclassifiedFailed: unclassifiedFailures.length
|
|
1505
|
-
};
|
|
1506
|
-
}
|
|
1507
|
-
function aggregate(samples, seed) {
|
|
1508
|
-
const n = samples.length;
|
|
1509
|
-
if (n === 0) throw new Error("aggregate requires at least one finite score");
|
|
1510
|
-
const mean = samples.reduce((a, b) => a + b, 0) / n;
|
|
1511
|
-
const variance = samples.reduce((a, b) => a + (b - mean) ** 2, 0) / Math.max(1, n - 1);
|
|
1512
|
-
const stdev = Math.sqrt(variance);
|
|
1513
|
-
const ci = confidenceInterval(samples, 0.95, { seed, resamples: 1e3 });
|
|
1514
|
-
return { mean, stdev, ci95: [ci.lower, ci.upper], n };
|
|
1515
|
-
}
|
|
1516
|
-
|
|
1517
|
-
export {
|
|
1518
|
-
BackendIntegrityError,
|
|
1519
|
-
summarizeBackendIntegrity,
|
|
1520
|
-
summarizeAgentReceiptIntegrity,
|
|
1521
|
-
assertRealBackend,
|
|
1522
|
-
assertRealAgentReceipts,
|
|
1523
|
-
canonicalJson,
|
|
1524
|
-
contentHash,
|
|
1525
|
-
inMemoryVerdictCache,
|
|
1526
|
-
fileVerdictCache,
|
|
1527
|
-
cachedJudge,
|
|
1528
|
-
assertCampaignDesign,
|
|
1529
|
-
campaignScenarioIdentity,
|
|
1530
|
-
campaignSplitDigestFromIdentities,
|
|
1531
|
-
campaignSplitDigest,
|
|
1532
|
-
assertCampaignSplitIdentity,
|
|
1533
|
-
campaignCoverage,
|
|
1534
|
-
formatCoverageFailures,
|
|
1535
|
-
tangleTracesRoot,
|
|
1536
|
-
resolveRunDir,
|
|
1537
|
-
probeAtomicFileLock,
|
|
1538
|
-
tryAcquireAtomicFileLock,
|
|
1539
|
-
SearchLedgerError,
|
|
1540
|
-
SearchLedgerIntegrityError,
|
|
1541
|
-
SearchLedgerConflictError,
|
|
1542
|
-
appendSearchLedgerLine,
|
|
1543
|
-
withSearchLedgerFileLock,
|
|
1544
|
-
fsCampaignStorage,
|
|
1545
|
-
inMemoryCampaignStorage,
|
|
1546
|
-
createRunCostLedger,
|
|
1547
|
-
runCampaign,
|
|
1548
|
-
planCampaignRun
|
|
1549
|
-
};
|
|
1550
|
-
//# sourceMappingURL=chunk-C6LXANRU.js.map
|