@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/multishot/index.js
CHANGED
|
@@ -1,589 +1,636 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
} from "
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
// src/multishot/router.ts
|
|
1
|
+
import { t as runAgentMatrix } from "../matrix-BzQnu2S6.js";
|
|
2
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
//#region src/multishot/router.ts
|
|
7
5
|
async function routerCompletion(req) {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
6
|
+
const body = {
|
|
7
|
+
model: req.model,
|
|
8
|
+
messages: req.messages,
|
|
9
|
+
temperature: req.temperature ?? .7,
|
|
10
|
+
max_tokens: req.maxTokens ?? 2e3
|
|
11
|
+
};
|
|
12
|
+
if (req.tools?.length) body.tools = req.tools;
|
|
13
|
+
const url = `${req.baseUrl.replace(/\/+$/, "")}/chat/completions`;
|
|
14
|
+
const res = await fetch(url, {
|
|
15
|
+
method: "POST",
|
|
16
|
+
headers: {
|
|
17
|
+
Authorization: `Bearer ${req.apiKey}`,
|
|
18
|
+
"Content-Type": "application/json"
|
|
19
|
+
},
|
|
20
|
+
body: JSON.stringify(body),
|
|
21
|
+
signal: req.signal
|
|
22
|
+
});
|
|
23
|
+
if (!res.ok) {
|
|
24
|
+
const text = await res.text();
|
|
25
|
+
throw new Error(`router ${res.status}: ${text.slice(0, 300)}`);
|
|
26
|
+
}
|
|
27
|
+
const json = await res.json();
|
|
28
|
+
const choice = json.choices[0];
|
|
29
|
+
if (!choice) throw new Error(`router returned no choices: ${JSON.stringify(json).slice(0, 200)}`);
|
|
30
|
+
return {
|
|
31
|
+
message: choice.message,
|
|
32
|
+
usage: json.usage
|
|
33
|
+
};
|
|
30
34
|
}
|
|
31
35
|
function estimateRouterCost(model, usage) {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
36
|
+
if (!usage) return 0;
|
|
37
|
+
const inputTok = usage.prompt_tokens ?? 0;
|
|
38
|
+
const outputTok = usage.completion_tokens ?? 0;
|
|
39
|
+
let inPer1k = .003;
|
|
40
|
+
let outPer1k = .015;
|
|
41
|
+
if (model.includes("gpt-4o-mini")) {
|
|
42
|
+
inPer1k = 15e-5;
|
|
43
|
+
outPer1k = 6e-4;
|
|
44
|
+
} else if (model.includes("gpt-5.4") || model.includes("claude-sonnet")) {
|
|
45
|
+
inPer1k = .003;
|
|
46
|
+
outPer1k = .015;
|
|
47
|
+
} else if (model.includes("kimi") || model.includes("glm") || model.includes("deepseek")) {
|
|
48
|
+
inPer1k = 5e-4;
|
|
49
|
+
outPer1k = .002;
|
|
50
|
+
}
|
|
51
|
+
return (inputTok * inPer1k + outputTok * outPer1k) / 1e3;
|
|
48
52
|
}
|
|
49
53
|
function defaultRouterBaseUrl() {
|
|
50
|
-
|
|
51
|
-
/\/+$/,
|
|
52
|
-
""
|
|
53
|
-
);
|
|
54
|
+
return (process.env.TANGLE_ROUTER_BASE_URL ?? "https://router.tangle.tools/v1").replace(/\/+$/, "");
|
|
54
55
|
}
|
|
55
56
|
function requireRouterApiKey() {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
const key = process.env.TANGLE_API_KEY;
|
|
58
|
+
if (!key) throw new Error("multishot requires TANGLE_API_KEY (router-scoped sk-tan-* key)");
|
|
59
|
+
return key;
|
|
59
60
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
61
|
+
//#endregion
|
|
62
|
+
//#region src/multishot/default-tools.ts
|
|
63
|
+
const DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
|
|
64
|
+
const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
|
|
65
|
+
const GENERIC_RESEARCHER_SYSTEM = "You are a research specialist. Return a markdown brief with 3-5 findings. Each finding cites a specific source by name. Add a confidence level (high/medium/low) per finding. No fluff, no preamble.";
|
|
66
|
+
const GENERIC_CODER_SYSTEM = "You are an expert engineer. Output ONE fenced code block containing the complete solution. Inline-comment non-obvious decisions. No explanation outside the block.";
|
|
67
|
+
const DEFAULT_DELEGATE_RESEARCH_TOOL = {
|
|
68
|
+
type: "function",
|
|
69
|
+
function: {
|
|
70
|
+
name: "delegate_research",
|
|
71
|
+
description: "Research a topic deeply via specialist. Returns evidence-bearing items with citations. Use for audience research, competitive intel, regulatory landscape, market data, citation-grounded analysis.",
|
|
72
|
+
parameters: {
|
|
73
|
+
type: "object",
|
|
74
|
+
properties: {
|
|
75
|
+
question: {
|
|
76
|
+
type: "string",
|
|
77
|
+
description: "Specific question to research"
|
|
78
|
+
},
|
|
79
|
+
scope: {
|
|
80
|
+
type: "string",
|
|
81
|
+
description: "Optional scope: time window, geography, jurisdiction, segment"
|
|
82
|
+
}
|
|
83
|
+
},
|
|
84
|
+
required: ["question"]
|
|
85
|
+
}
|
|
86
|
+
}
|
|
83
87
|
};
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
88
|
+
const DEFAULT_DELEGATE_CODE_TOOL = {
|
|
89
|
+
type: "function",
|
|
90
|
+
function: {
|
|
91
|
+
name: "delegate_code",
|
|
92
|
+
description: "Generate a runnable script, template, pipeline, or tool via specialist. Returns complete working code or structured markdown. Use for content pipelines, calc snippets, dashboards, compliance checklists, deadline trackers.",
|
|
93
|
+
parameters: {
|
|
94
|
+
type: "object",
|
|
95
|
+
properties: {
|
|
96
|
+
goal: {
|
|
97
|
+
type: "string",
|
|
98
|
+
description: "What the code must accomplish"
|
|
99
|
+
},
|
|
100
|
+
language: {
|
|
101
|
+
type: "string",
|
|
102
|
+
description: "Optional language preference (default: TypeScript)"
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
required: ["goal"]
|
|
106
|
+
}
|
|
107
|
+
}
|
|
101
108
|
};
|
|
102
109
|
function createResearchExecutor(config = {}) {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
110
|
+
const systemPrompt = config.systemPrompt ?? GENERIC_RESEARCHER_SYSTEM;
|
|
111
|
+
const model = config.model ?? "openai/gpt-4o-mini";
|
|
112
|
+
return async (args, ctx) => {
|
|
113
|
+
const question = String(args.question ?? "");
|
|
114
|
+
const scope = args.scope ? String(args.scope) : void 0;
|
|
115
|
+
const { message, usage } = await routerCompletion({
|
|
116
|
+
apiKey: ctx.apiKey,
|
|
117
|
+
baseUrl: ctx.baseUrl,
|
|
118
|
+
model,
|
|
119
|
+
temperature: .3,
|
|
120
|
+
maxTokens: 1800,
|
|
121
|
+
messages: [{
|
|
122
|
+
role: "system",
|
|
123
|
+
content: systemPrompt
|
|
124
|
+
}, {
|
|
125
|
+
role: "user",
|
|
126
|
+
content: `Research: ${question}${scope ? `\nScope: ${scope}` : ""}`
|
|
127
|
+
}],
|
|
128
|
+
signal: ctx.signal
|
|
129
|
+
});
|
|
130
|
+
return {
|
|
131
|
+
content: message.content ?? "",
|
|
132
|
+
costUsd: estimateRouterCost(model, usage)
|
|
133
|
+
};
|
|
134
|
+
};
|
|
123
135
|
}
|
|
124
136
|
function createCodeExecutor(config = {}) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
137
|
+
const systemPrompt = config.systemPrompt ?? GENERIC_CODER_SYSTEM;
|
|
138
|
+
const model = config.model ?? "openai/gpt-4o-mini";
|
|
139
|
+
return async (args, ctx) => {
|
|
140
|
+
const goal = String(args.goal ?? "");
|
|
141
|
+
const language = args.language ? String(args.language) : "TypeScript";
|
|
142
|
+
const { message, usage } = await routerCompletion({
|
|
143
|
+
apiKey: ctx.apiKey,
|
|
144
|
+
baseUrl: ctx.baseUrl,
|
|
145
|
+
model,
|
|
146
|
+
temperature: .2,
|
|
147
|
+
maxTokens: 2e3,
|
|
148
|
+
messages: [{
|
|
149
|
+
role: "system",
|
|
150
|
+
content: `${systemPrompt}\n\nLanguage: ${language}`
|
|
151
|
+
}, {
|
|
152
|
+
role: "user",
|
|
153
|
+
content: `Produce: ${goal}`
|
|
154
|
+
}],
|
|
155
|
+
signal: ctx.signal
|
|
156
|
+
});
|
|
157
|
+
return {
|
|
158
|
+
content: message.content ?? "",
|
|
159
|
+
costUsd: estimateRouterCost(model, usage)
|
|
160
|
+
};
|
|
161
|
+
};
|
|
146
162
|
}
|
|
147
163
|
function defaultDelegationTools(config = {}) {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
164
|
+
return {
|
|
165
|
+
tools: [DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_DELEGATE_CODE_TOOL],
|
|
166
|
+
executors: {
|
|
167
|
+
delegate_research: createResearchExecutor(config.research),
|
|
168
|
+
delegate_code: createCodeExecutor(config.code)
|
|
169
|
+
},
|
|
170
|
+
artifactTypeFor: (name) => name === "delegate_research" ? "research" : name === "delegate_code" ? "code" : void 0
|
|
171
|
+
};
|
|
156
172
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
173
|
+
//#endregion
|
|
174
|
+
//#region src/multishot/judges.ts
|
|
175
|
+
const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
|
|
160
176
|
async function runJudge(judge, input) {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
177
|
+
const apiKey = judge.apiKey ?? requireRouterApiKey();
|
|
178
|
+
const baseUrl = judge.baseUrl ?? defaultRouterBaseUrl();
|
|
179
|
+
const model = judge.model ?? process.env.JUDGE_MODEL ?? "openai/gpt-4o-mini";
|
|
180
|
+
const prompt = judge.buildPrompt(input);
|
|
181
|
+
let raw = "";
|
|
182
|
+
try {
|
|
183
|
+
const { message } = await routerCompletion({
|
|
184
|
+
apiKey,
|
|
185
|
+
baseUrl,
|
|
186
|
+
model,
|
|
187
|
+
temperature: 0,
|
|
188
|
+
maxTokens: judge.maxTokens ?? 1500,
|
|
189
|
+
messages: [{
|
|
190
|
+
role: "system",
|
|
191
|
+
content: judge.systemPrompt
|
|
192
|
+
}, {
|
|
193
|
+
role: "user",
|
|
194
|
+
content: prompt
|
|
195
|
+
}]
|
|
196
|
+
});
|
|
197
|
+
raw = (message.content ?? "").trim();
|
|
198
|
+
} catch (err) {
|
|
199
|
+
return {
|
|
200
|
+
dimensions: {},
|
|
201
|
+
composite: 0,
|
|
202
|
+
failed: true,
|
|
203
|
+
notes: `judge ${judge.name} call failed: ${err instanceof Error ? err.message : String(err)}`
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
let parsed = null;
|
|
207
|
+
try {
|
|
208
|
+
const cleaned = raw.replace(/^```json\s*/i, "").replace(/```\s*$/, "").trim();
|
|
209
|
+
parsed = JSON.parse(cleaned);
|
|
210
|
+
} catch {
|
|
211
|
+
return {
|
|
212
|
+
dimensions: {},
|
|
213
|
+
composite: 0,
|
|
214
|
+
failed: true,
|
|
215
|
+
notes: `judge ${judge.name} returned non-JSON: ${raw.slice(0, 200)}`
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
const dimensions = {};
|
|
219
|
+
let sum = 0;
|
|
220
|
+
for (const dim of judge.dimensions) {
|
|
221
|
+
const v = Number(parsed[dim.key] ?? 0);
|
|
222
|
+
const clamped = Number.isFinite(v) ? Math.max(0, Math.min(10, v)) : 0;
|
|
223
|
+
dimensions[dim.key] = clamped;
|
|
224
|
+
sum += clamped;
|
|
225
|
+
}
|
|
226
|
+
return {
|
|
227
|
+
dimensions,
|
|
228
|
+
composite: judge.dimensions.length === 0 ? 0 : sum / judge.dimensions.length,
|
|
229
|
+
notes: typeof parsed.notes === "string" ? parsed.notes : ""
|
|
230
|
+
};
|
|
212
231
|
}
|
|
232
|
+
/** Convenience: stringified dimension list for inclusion in a judge prompt.
|
|
233
|
+
* Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
|
|
213
234
|
function renderDimensions(dims) {
|
|
214
|
-
|
|
235
|
+
return dims.map((d) => `- ${d.key}: ${d.description}`).join("\n");
|
|
215
236
|
}
|
|
237
|
+
/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
|
|
216
238
|
function renderJsonFooter(dims) {
|
|
217
|
-
|
|
218
|
-
return `Respond with ONLY this JSON (no markdown, no preamble):
|
|
219
|
-
{${fields},"notes":"1-2 sentence critique"}`;
|
|
239
|
+
return `Respond with ONLY this JSON (no markdown, no preamble):\n{${dims.map((d) => `"${d.key}":N`).join(",")},"notes":"1-2 sentence critique"}`;
|
|
220
240
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
import { mkdirSync, writeFileSync } from "fs";
|
|
224
|
-
import { join } from "path";
|
|
225
|
-
|
|
226
|
-
// src/multishot/types.ts
|
|
241
|
+
//#endregion
|
|
242
|
+
//#region src/multishot/types.ts
|
|
227
243
|
var MultishotDriverEmptyError = class extends Error {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
244
|
+
turn;
|
|
245
|
+
constructor(turn) {
|
|
246
|
+
super(`multishot: driver returned empty content twice at turn ${turn} — failing loud`);
|
|
247
|
+
this.turn = turn;
|
|
248
|
+
this.name = "MultishotDriverEmptyError";
|
|
249
|
+
}
|
|
234
250
|
};
|
|
235
251
|
var MultishotFatalToolError = class extends Error {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
252
|
+
constructor(message) {
|
|
253
|
+
super(message);
|
|
254
|
+
this.name = "MultishotFatalToolError";
|
|
255
|
+
}
|
|
240
256
|
};
|
|
241
|
-
|
|
242
|
-
|
|
257
|
+
//#endregion
|
|
258
|
+
//#region src/multishot/multishot.ts
|
|
243
259
|
async function runMultishot(opts) {
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
260
|
+
const apiKey = opts.apiKey ?? requireRouterApiKey();
|
|
261
|
+
const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl();
|
|
262
|
+
const maxTurns = opts.maxTurns ?? 10;
|
|
263
|
+
const maxToolDispatches = opts.maxToolDispatches ?? 4;
|
|
264
|
+
const agentModel = opts.agentModel ?? "openai/gpt-5.4";
|
|
265
|
+
const driverModels = [opts.driverModel ?? "openai/gpt-4o-mini", ...opts.driverFallbackModels ?? []];
|
|
266
|
+
const agentMaxTokens = opts.agentMaxTokens ?? 2500;
|
|
267
|
+
const toolFollowupMaxTokens = opts.toolFollowupMaxTokens ?? 2e3;
|
|
268
|
+
const driverMaxTokens = opts.driverMaxTokens ?? 600;
|
|
269
|
+
const bundle = opts.tools && opts.toolExecutors ? {
|
|
270
|
+
tools: opts.tools,
|
|
271
|
+
executors: opts.toolExecutors,
|
|
272
|
+
artifactTypeFor: opts.artifactTypeFor ?? (() => void 0)
|
|
273
|
+
} : defaultDelegationTools();
|
|
274
|
+
const tools = opts.tools ?? bundle.tools;
|
|
275
|
+
const executors = opts.toolExecutors ?? bundle.executors;
|
|
276
|
+
const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor;
|
|
277
|
+
const routerTransport = (req) => routerCompletion({
|
|
278
|
+
apiKey,
|
|
279
|
+
baseUrl,
|
|
280
|
+
...req
|
|
281
|
+
});
|
|
282
|
+
const agentTransport = opts.agentTransport ?? routerTransport;
|
|
283
|
+
const driverTransport = opts.driverTransport ?? routerTransport;
|
|
284
|
+
const start = Date.now();
|
|
285
|
+
const transcript = [];
|
|
286
|
+
const artifacts = [];
|
|
287
|
+
let toolCalls = 0;
|
|
288
|
+
let totalCostUsd = 0;
|
|
289
|
+
const opener = opts.shape.buildOpener(opts.persona);
|
|
290
|
+
transcript.push({
|
|
291
|
+
role: "user",
|
|
292
|
+
content: opener
|
|
293
|
+
});
|
|
294
|
+
const agentMessages = [{
|
|
295
|
+
role: "system",
|
|
296
|
+
content: opts.profile.prompt?.systemPrompt ?? ""
|
|
297
|
+
}, {
|
|
298
|
+
role: "user",
|
|
299
|
+
content: opener
|
|
300
|
+
}];
|
|
301
|
+
for (let turn = 0; turn < maxTurns; turn++) {
|
|
302
|
+
if (opts.signal?.aborted) throw new Error("multishot aborted");
|
|
303
|
+
let dispatchesThisTurn = 0;
|
|
304
|
+
while (true) {
|
|
305
|
+
const { message: agentMsg, usage: agentUsage, costUsd: agentCostUsd } = await agentTransport({
|
|
306
|
+
model: agentModel,
|
|
307
|
+
messages: agentMessages,
|
|
308
|
+
tools,
|
|
309
|
+
temperature: .7,
|
|
310
|
+
maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,
|
|
311
|
+
signal: opts.signal
|
|
312
|
+
});
|
|
313
|
+
totalCostUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage);
|
|
314
|
+
const agentText = (agentMsg.content ?? "").trim();
|
|
315
|
+
const agentToolCalls = (agentMsg.tool_calls ?? []).map((tc) => ({
|
|
316
|
+
id: tc.id,
|
|
317
|
+
name: tc.function.name,
|
|
318
|
+
args: (() => {
|
|
319
|
+
try {
|
|
320
|
+
return JSON.parse(tc.function.arguments);
|
|
321
|
+
} catch {
|
|
322
|
+
return {};
|
|
323
|
+
}
|
|
324
|
+
})()
|
|
325
|
+
}));
|
|
326
|
+
agentMessages.push({
|
|
327
|
+
role: "assistant",
|
|
328
|
+
content: agentText || null,
|
|
329
|
+
...agentMsg.tool_calls?.length ? { tool_calls: agentMsg.tool_calls } : {}
|
|
330
|
+
});
|
|
331
|
+
transcript.push({
|
|
332
|
+
role: "assistant",
|
|
333
|
+
content: agentText,
|
|
334
|
+
toolCalls: agentToolCalls.length > 0 ? agentToolCalls : void 0
|
|
335
|
+
});
|
|
336
|
+
if (agentToolCalls.length === 0) break;
|
|
337
|
+
dispatchesThisTurn += agentToolCalls.length;
|
|
338
|
+
if (dispatchesThisTurn > maxToolDispatches) throw new Error(`multishot: tool dispatch cap exceeded (${dispatchesThisTurn}/${maxToolDispatches}) on turn ${turn}`);
|
|
339
|
+
for (const tc of agentToolCalls) {
|
|
340
|
+
toolCalls++;
|
|
341
|
+
let toolResult = "";
|
|
342
|
+
try {
|
|
343
|
+
const executor = executors[tc.name];
|
|
344
|
+
if (!executor) toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` });
|
|
345
|
+
else {
|
|
346
|
+
const r = await executor(tc.args, {
|
|
347
|
+
apiKey,
|
|
348
|
+
baseUrl,
|
|
349
|
+
signal: opts.signal
|
|
350
|
+
});
|
|
351
|
+
toolResult = r.content;
|
|
352
|
+
totalCostUsd += r.costUsd;
|
|
353
|
+
const artifactType = artifactTypeFor(tc.name);
|
|
354
|
+
if (artifactType) artifacts.push({
|
|
355
|
+
type: artifactType,
|
|
356
|
+
turn,
|
|
357
|
+
invocation: {
|
|
358
|
+
name: tc.name,
|
|
359
|
+
args: tc.args
|
|
360
|
+
},
|
|
361
|
+
content: toolResult
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
} catch (err) {
|
|
365
|
+
if (err instanceof MultishotFatalToolError) throw err;
|
|
366
|
+
toolResult = JSON.stringify({ error: err instanceof Error ? err.message : String(err) });
|
|
367
|
+
}
|
|
368
|
+
agentMessages.push({
|
|
369
|
+
role: "tool",
|
|
370
|
+
tool_call_id: tc.id,
|
|
371
|
+
content: toolResult || "done"
|
|
372
|
+
});
|
|
373
|
+
transcript.push({
|
|
374
|
+
role: "tool",
|
|
375
|
+
content: toolResult || "done",
|
|
376
|
+
toolCallId: tc.id
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
if (turn < maxTurns - 1) {
|
|
381
|
+
const driver = await driverTurn({
|
|
382
|
+
transport: driverTransport,
|
|
383
|
+
persona: opts.persona,
|
|
384
|
+
shape: opts.shape,
|
|
385
|
+
transcript,
|
|
386
|
+
turn,
|
|
387
|
+
models: driverModels,
|
|
388
|
+
maxTokens: driverMaxTokens,
|
|
389
|
+
signal: opts.signal
|
|
390
|
+
});
|
|
391
|
+
totalCostUsd += driver.costUsd;
|
|
392
|
+
agentMessages.push({
|
|
393
|
+
role: "user",
|
|
394
|
+
content: driver.content
|
|
395
|
+
});
|
|
396
|
+
transcript.push({
|
|
397
|
+
role: "user",
|
|
398
|
+
content: driver.content
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return {
|
|
403
|
+
transcript,
|
|
404
|
+
artifacts,
|
|
405
|
+
toolCalls,
|
|
406
|
+
durationMs: Date.now() - start,
|
|
407
|
+
costUsd: totalCostUsd
|
|
408
|
+
};
|
|
369
409
|
}
|
|
370
410
|
async function driverTurn(opts) {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
411
|
+
const driverMessages = [{
|
|
412
|
+
role: "system",
|
|
413
|
+
content: opts.shape.buildDriverSystemPrompt(opts.persona)
|
|
414
|
+
}];
|
|
415
|
+
for (const msg of opts.transcript) {
|
|
416
|
+
if (msg.role === "tool") continue;
|
|
417
|
+
const content = driverVisibleContent(msg);
|
|
418
|
+
if (!content) continue;
|
|
419
|
+
if (msg.role === "assistant") driverMessages.push({
|
|
420
|
+
role: "user",
|
|
421
|
+
content
|
|
422
|
+
});
|
|
423
|
+
else if (msg.role === "user") driverMessages.push({
|
|
424
|
+
role: "assistant",
|
|
425
|
+
content
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
for (const model of opts.models) for (let attempt = 0; attempt < 2; attempt++) {
|
|
429
|
+
const { message, usage, costUsd } = await opts.transport({
|
|
430
|
+
model,
|
|
431
|
+
messages: driverMessages,
|
|
432
|
+
temperature: .9,
|
|
433
|
+
maxTokens: opts.maxTokens,
|
|
434
|
+
signal: opts.signal
|
|
435
|
+
});
|
|
436
|
+
const content = (message.content ?? "").trim();
|
|
437
|
+
if (content.length > 0) return {
|
|
438
|
+
content,
|
|
439
|
+
costUsd: costUsd ?? estimateRouterCost(model, usage)
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
throw new MultishotDriverEmptyError(opts.turn);
|
|
395
443
|
}
|
|
396
444
|
function driverVisibleContent(msg) {
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
445
|
+
const text = msg.content.trim();
|
|
446
|
+
if (text.length > 0) return text;
|
|
447
|
+
if (msg.role !== "assistant" || !msg.toolCalls?.length) return null;
|
|
448
|
+
const toolNames = msg.toolCalls.map((call) => call.name.trim()).filter(Boolean);
|
|
449
|
+
if (toolNames.length === 0) return "Agent called tools.";
|
|
450
|
+
return `Agent called ${toolNames.length === 1 ? "tool" : "tools"}: ${toolNames.join(", ")}.`;
|
|
403
451
|
}
|
|
404
|
-
|
|
405
|
-
|
|
452
|
+
//#endregion
|
|
453
|
+
//#region src/multishot/matrix.ts
|
|
454
|
+
/** Mean composite over non-failed scores. `0` when the list is empty (a
|
|
455
|
+
* configured judge with nothing to score contributes 0, matching the cell
|
|
456
|
+
* composite's long-standing semantics); `null` when scores exist but every
|
|
457
|
+
* one failed — no signal, so the slot must be EXCLUDED from the cell mean
|
|
458
|
+
* rather than dragging it to zero. */
|
|
406
459
|
function meanCompositeExcludingFailed(scores) {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
460
|
+
if (scores.length === 0) return 0;
|
|
461
|
+
const live = scores.filter((s) => !s.failed);
|
|
462
|
+
if (live.length === 0) return null;
|
|
463
|
+
return live.reduce((sum, s) => sum + s.composite, 0) / live.length;
|
|
411
464
|
}
|
|
465
|
+
/** Cell composite = mean over configured judge slots, excluding failed
|
|
466
|
+
* scores: a failed conversation judge or an all-failed artifact slot carries
|
|
467
|
+
* no signal and is dropped from the mean. `composite` is 0 only when EVERY
|
|
468
|
+
* configured slot failed (`allJudgesFailed` distinguishes that from a real
|
|
469
|
+
* zero). Pure — exported for deterministic testing. */
|
|
412
470
|
function computeCellComposite(input) {
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
471
|
+
const contributions = [];
|
|
472
|
+
if (!input.conversation.failed) contributions.push(input.conversation.composite);
|
|
473
|
+
const codeMean = input.codeReviews ? meanCompositeExcludingFailed(input.codeReviews) : void 0;
|
|
474
|
+
if (typeof codeMean === "number") contributions.push(codeMean);
|
|
475
|
+
const contentMean = input.contentReviews ? meanCompositeExcludingFailed(input.contentReviews) : void 0;
|
|
476
|
+
if (typeof contentMean === "number") contributions.push(contentMean);
|
|
477
|
+
return {
|
|
478
|
+
composite: contributions.length === 0 ? 0 : contributions.reduce((s, v) => s + v, 0) / contributions.length,
|
|
479
|
+
codeComposite: codeMean ?? 0,
|
|
480
|
+
contentComposite: contentMean ?? 0,
|
|
481
|
+
allJudgesFailed: contributions.length === 0
|
|
482
|
+
};
|
|
425
483
|
}
|
|
426
484
|
async function runMultishotMatrix(opts) {
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
485
|
+
const codeTypes = new Set(opts.judges.codeArtifactTypes ?? ["code"]);
|
|
486
|
+
const contentTypes = new Set(opts.judges.contentArtifactTypes ?? ["research"]);
|
|
487
|
+
mkdirSync(opts.runDir, { recursive: true });
|
|
488
|
+
const matrix = await runAgentMatrix({
|
|
489
|
+
axes: [{
|
|
490
|
+
name: "profile",
|
|
491
|
+
values: opts.profiles
|
|
492
|
+
}, {
|
|
493
|
+
name: "persona",
|
|
494
|
+
values: opts.personas.map((p) => ({
|
|
495
|
+
id: p.id,
|
|
496
|
+
value: p
|
|
497
|
+
}))
|
|
498
|
+
}],
|
|
499
|
+
reps: opts.reps ?? 1,
|
|
500
|
+
maxConcurrency: opts.maxConcurrency ?? 2,
|
|
501
|
+
costCeiling: opts.costCeiling,
|
|
502
|
+
async runCell(cell) {
|
|
503
|
+
const profile = cell.axes.profile?.value;
|
|
504
|
+
const persona = cell.axes.persona?.value;
|
|
505
|
+
const profileId = String(cell.axes.profile?.id ?? "unknown");
|
|
506
|
+
const personaId = String(cell.axes.persona?.id ?? "unknown");
|
|
507
|
+
const sim = await runMultishot({
|
|
508
|
+
profile,
|
|
509
|
+
persona,
|
|
510
|
+
shape: opts.shape,
|
|
511
|
+
tools: opts.tools,
|
|
512
|
+
toolExecutors: opts.toolExecutors,
|
|
513
|
+
artifactTypeFor: opts.artifactTypeFor,
|
|
514
|
+
maxTurns: opts.maxTurns,
|
|
515
|
+
maxToolDispatches: opts.maxToolDispatches,
|
|
516
|
+
agentModel: opts.agentModel,
|
|
517
|
+
driverModel: opts.driverModel,
|
|
518
|
+
driverFallbackModels: opts.driverFallbackModels,
|
|
519
|
+
agentMaxTokens: opts.agentMaxTokens,
|
|
520
|
+
toolFollowupMaxTokens: opts.toolFollowupMaxTokens,
|
|
521
|
+
driverMaxTokens: opts.driverMaxTokens,
|
|
522
|
+
agentTransport: opts.agentTransport,
|
|
523
|
+
driverTransport: opts.driverTransport,
|
|
524
|
+
apiKey: opts.apiKey,
|
|
525
|
+
baseUrl: opts.baseUrl
|
|
526
|
+
});
|
|
527
|
+
const codeArtifacts = sim.artifacts.filter((a) => codeTypes.has(a.type));
|
|
528
|
+
const contentArtifacts = sim.artifacts.filter((a) => contentTypes.has(a.type));
|
|
529
|
+
const [conversation, codeReviews, contentReviews] = await Promise.all([
|
|
530
|
+
runJudge(withJudgeMaxTokens(opts.judges.conversation, opts.judgeMaxTokens), {
|
|
531
|
+
transcript: sim.transcript,
|
|
532
|
+
persona
|
|
533
|
+
}),
|
|
534
|
+
opts.judges.codeReview ? Promise.all(codeArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.codeReview, opts.judgeMaxTokens), {
|
|
535
|
+
artifact,
|
|
536
|
+
persona
|
|
537
|
+
}).then((s) => ({
|
|
538
|
+
...s,
|
|
539
|
+
turn: artifact.turn,
|
|
540
|
+
type: artifact.type
|
|
541
|
+
})))) : Promise.resolve([]),
|
|
542
|
+
opts.judges.contentQuality ? Promise.all(contentArtifacts.map((artifact) => runJudge(withJudgeMaxTokens(opts.judges.contentQuality, opts.judgeMaxTokens), {
|
|
543
|
+
artifact,
|
|
544
|
+
persona
|
|
545
|
+
}).then((s) => ({
|
|
546
|
+
...s,
|
|
547
|
+
turn: artifact.turn,
|
|
548
|
+
type: artifact.type
|
|
549
|
+
})))) : Promise.resolve([])
|
|
550
|
+
]);
|
|
551
|
+
const { composite, codeComposite, contentComposite, allJudgesFailed } = computeCellComposite({
|
|
552
|
+
conversation,
|
|
553
|
+
codeReviews: opts.judges.codeReview ? codeReviews : void 0,
|
|
554
|
+
contentReviews: opts.judges.contentQuality ? contentReviews : void 0
|
|
555
|
+
});
|
|
556
|
+
const cellScore = {
|
|
557
|
+
composite,
|
|
558
|
+
conversation
|
|
559
|
+
};
|
|
560
|
+
if (opts.judges.codeReview) cellScore.codeReview = {
|
|
561
|
+
perArtifact: codeReviews,
|
|
562
|
+
composite: codeComposite
|
|
563
|
+
};
|
|
564
|
+
if (opts.judges.contentQuality) cellScore.contentQuality = {
|
|
565
|
+
perArtifact: contentReviews,
|
|
566
|
+
composite: contentComposite
|
|
567
|
+
};
|
|
568
|
+
const cellDir = join(opts.runDir, profileId, personaId, `rep-${cell.rep}`);
|
|
569
|
+
mkdirSync(cellDir, { recursive: true });
|
|
570
|
+
writeFileSync(join(cellDir, "transcript.json"), JSON.stringify(sim.transcript, null, 2));
|
|
571
|
+
writeFileSync(join(cellDir, "artifacts.json"), JSON.stringify(sim.artifacts, null, 2));
|
|
572
|
+
writeFileSync(join(cellDir, "scores.json"), JSON.stringify(cellScore, null, 2));
|
|
573
|
+
const notes = [`convo=${conversation.composite.toFixed(1)}`];
|
|
574
|
+
if (opts.judges.codeReview) notes.push(`code=${codeComposite.toFixed(1)}`);
|
|
575
|
+
if (opts.judges.contentQuality) notes.push(`content=${contentComposite.toFixed(1)}`);
|
|
576
|
+
if (allJudgesFailed) notes.push("all-judges-failed");
|
|
577
|
+
return {
|
|
578
|
+
output: {
|
|
579
|
+
turns: sim.transcript.length,
|
|
580
|
+
toolCalls: sim.toolCalls,
|
|
581
|
+
artifactCount: sim.artifacts.length
|
|
582
|
+
},
|
|
583
|
+
verdict: {
|
|
584
|
+
valid: composite >= 5,
|
|
585
|
+
score: composite,
|
|
586
|
+
notes: notes.join(" ")
|
|
587
|
+
},
|
|
588
|
+
costUsd: sim.costUsd,
|
|
589
|
+
durationMs: sim.durationMs
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
});
|
|
593
|
+
const summary = {
|
|
594
|
+
cells: matrix.summary.totalCells,
|
|
595
|
+
passRate: matrix.summary.overallPassRate,
|
|
596
|
+
meanScore: matrix.summary.overallMeanScore,
|
|
597
|
+
totalCostUsd: matrix.summary.totalCostUsd,
|
|
598
|
+
durationMs: matrix.summary.durationMs,
|
|
599
|
+
runsExecuted: matrix.summary.runsExecuted,
|
|
600
|
+
cellsSkipped: matrix.summary.cellsSkipped,
|
|
601
|
+
byProfile: matrix.byAxis.profile,
|
|
602
|
+
byPersona: matrix.byAxis.persona
|
|
603
|
+
};
|
|
604
|
+
writeFileSync(join(opts.runDir, "summary.json"), JSON.stringify(summary, null, 2));
|
|
605
|
+
const md = [
|
|
606
|
+
`# Multishot matrix`,
|
|
607
|
+
``,
|
|
608
|
+
`**Cells**: ${matrix.summary.totalCells} | **Pass rate**: ${(matrix.summary.overallPassRate * 100).toFixed(0)}% | **Mean**: ${matrix.summary.overallMeanScore.toFixed(2)} | **Cost**: $${matrix.summary.totalCostUsd.toFixed(2)} | **Duration**: ${(matrix.summary.durationMs / 1e3).toFixed(0)}s`,
|
|
609
|
+
``,
|
|
610
|
+
`## By profile`,
|
|
611
|
+
``,
|
|
612
|
+
"| profile | pass | mean | cost |",
|
|
613
|
+
"|---|---|---|---|",
|
|
614
|
+
...Object.entries(matrix.byAxis.profile ?? {}).map(([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`),
|
|
615
|
+
``,
|
|
616
|
+
`## By persona`,
|
|
617
|
+
``,
|
|
618
|
+
"| persona | pass | mean | cost |",
|
|
619
|
+
"|---|---|---|---|",
|
|
620
|
+
...Object.entries(matrix.byAxis.persona ?? {}).map(([id, s]) => `| ${id} | ${(s.passRate * 100).toFixed(0)}% | ${s.meanScore.toFixed(2)} | $${s.totalCostUsd.toFixed(2)} |`),
|
|
621
|
+
``
|
|
622
|
+
];
|
|
623
|
+
writeFileSync(join(opts.runDir, "summary.md"), md.join("\n"));
|
|
624
|
+
return { matrix };
|
|
562
625
|
}
|
|
563
626
|
function withJudgeMaxTokens(judge, maxTokens) {
|
|
564
|
-
|
|
565
|
-
|
|
627
|
+
if (maxTokens === void 0 || judge.maxTokens !== void 0) return judge;
|
|
628
|
+
return {
|
|
629
|
+
...judge,
|
|
630
|
+
maxTokens
|
|
631
|
+
};
|
|
566
632
|
}
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
DEFAULT_DELEGATE_RESEARCH_TOOL,
|
|
571
|
-
DEFAULT_JUDGE_MODEL,
|
|
572
|
-
DEFAULT_RESEARCHER_MODEL,
|
|
573
|
-
MultishotDriverEmptyError,
|
|
574
|
-
MultishotFatalToolError,
|
|
575
|
-
computeCellComposite,
|
|
576
|
-
createCodeExecutor,
|
|
577
|
-
createResearchExecutor,
|
|
578
|
-
defaultDelegationTools,
|
|
579
|
-
defaultRouterBaseUrl,
|
|
580
|
-
estimateRouterCost,
|
|
581
|
-
renderDimensions,
|
|
582
|
-
renderJsonFooter,
|
|
583
|
-
requireRouterApiKey,
|
|
584
|
-
routerCompletion,
|
|
585
|
-
runJudge,
|
|
586
|
-
runMultishot,
|
|
587
|
-
runMultishotMatrix
|
|
588
|
-
};
|
|
633
|
+
//#endregion
|
|
634
|
+
export { DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, MultishotDriverEmptyError, MultishotFatalToolError, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultRouterBaseUrl, estimateRouterCost, renderDimensions, renderJsonFooter, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };
|
|
635
|
+
|
|
589
636
|
//# sourceMappingURL=index.js.map
|