@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
|
@@ -0,0 +1,547 @@
|
|
|
1
|
+
import { b as RolloutSplit, h as RolloutLine } from "./schema-Cef2cFmb.js";
|
|
2
|
+
//#region src/supervisor-run/types.d.ts
|
|
3
|
+
/** A metric that could not be computed, with the reason its artifact was missing. */
|
|
4
|
+
interface Unavailable {
|
|
5
|
+
readonly unavailable: string;
|
|
6
|
+
}
|
|
7
|
+
/** A metric value, or the reason it is unknown. NEVER collapse `unavailable` to 0. */
|
|
8
|
+
type Measured<T> = T | Unavailable;
|
|
9
|
+
declare function unavailable(reason: string): Unavailable;
|
|
10
|
+
declare function isUnavailable(v: unknown): v is Unavailable;
|
|
11
|
+
/** Render a measured scalar for the markdown/headline: `0` and `unavailable` stay distinct. */
|
|
12
|
+
declare function showMeasured(v: Measured<number | string | boolean | null>): string;
|
|
13
|
+
/** One worker's logs, as read. `null` = the artifact did not exist. */
|
|
14
|
+
interface WorkerLogSource {
|
|
15
|
+
readonly label: string;
|
|
16
|
+
/** Worker event stream — started / progress / finished / message events (JSONL). */
|
|
17
|
+
readonly events: string | null;
|
|
18
|
+
/** The durable steer queue — one line per steer request (JSONL). */
|
|
19
|
+
readonly inbox: string | null;
|
|
20
|
+
/** Worker patch byte length, or null when absent. */
|
|
21
|
+
readonly patchBytes: number | null;
|
|
22
|
+
/** Where this worker's transcript lives, for the rollout row. Null = no such artifact. */
|
|
23
|
+
readonly transcriptRef?: string | null;
|
|
24
|
+
/** Where this worker's delivered patch lives. Null = the store keeps no patch per worker. */
|
|
25
|
+
readonly patchPath?: string | null;
|
|
26
|
+
/** This worker's own inference tokens, when the store records them per worker. */
|
|
27
|
+
readonly tokensIn?: number | null;
|
|
28
|
+
readonly tokensOut?: number | null;
|
|
29
|
+
readonly cacheRead?: number | null;
|
|
30
|
+
readonly cacheWrite?: number | null;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Facts a SOURCE structurally cannot express, each with the reason.
|
|
34
|
+
*
|
|
35
|
+
* The difference between "the artifact is missing" and "this store never
|
|
36
|
+
* records that fact" is the difference between a run that spent $0 and a
|
|
37
|
+
* harness that does not price inference — and the second harness is where a
|
|
38
|
+
* loops-shaped assumption becomes a fabricated zero. A reader declares its
|
|
39
|
+
* limits once; the analyzer reports `unavailable` for everything downstream.
|
|
40
|
+
*
|
|
41
|
+
* `null` on a field means the source DOES carry that fact.
|
|
42
|
+
*/
|
|
43
|
+
interface SourceLimits {
|
|
44
|
+
/** Reason inference spend has no price in this store (null = the store prices it). */
|
|
45
|
+
readonly spendUsd: string | null;
|
|
46
|
+
/** Reason workers carry no pass/fail verdict (null = verdicts are recorded). */
|
|
47
|
+
readonly workerVerdicts: string | null;
|
|
48
|
+
/** Reason no delivered artifact (patch/diff) is retained per worker (null = retained). */
|
|
49
|
+
readonly deliverables: string | null;
|
|
50
|
+
}
|
|
51
|
+
/** A source that carries every fact the analyzer can use. */
|
|
52
|
+
declare const NO_SOURCE_LIMITS: SourceLimits;
|
|
53
|
+
/**
|
|
54
|
+
* Everything the pure analyzer reads — already-read bytes, never paths. Each
|
|
55
|
+
* field is `null` when its artifact was absent, which is what turns the
|
|
56
|
+
* dependent metrics into `unavailable` rather than 0.
|
|
57
|
+
*
|
|
58
|
+
* This is the whole input contract. Any store that can produce these strings
|
|
59
|
+
* (an on-disk loops run, an object-store archive, a database, a test fixture)
|
|
60
|
+
* is a valid source; `loopsSupervisorRunReader` is ONE implementation.
|
|
61
|
+
*/
|
|
62
|
+
interface SupervisorRunSources {
|
|
63
|
+
/** Stable identity of the run being analyzed (a directory, a run id, a URL). */
|
|
64
|
+
readonly runRef: string;
|
|
65
|
+
readonly instanceId: string | null;
|
|
66
|
+
/** Which arm/variant of a comparison this run is, when the run belongs to one. */
|
|
67
|
+
readonly arm: string | null;
|
|
68
|
+
/** Identity of the supervision-tree store this was read from; null = none found. */
|
|
69
|
+
readonly supRunDir: string | null;
|
|
70
|
+
/** Supervision journal — spawned / settled / cancelled / metered events (JSONL). */
|
|
71
|
+
readonly journal: string | null;
|
|
72
|
+
/** Per-brain-call tap (JSONL): finish_reason, completion tokens, requested max tokens. */
|
|
73
|
+
readonly brainLog: string | null;
|
|
74
|
+
/** Supervisor state document (JSON). */
|
|
75
|
+
readonly state: string | null;
|
|
76
|
+
/** Supervisor progress stream (JSONL). */
|
|
77
|
+
readonly progress: string | null;
|
|
78
|
+
/** Per-worker logs; `null` = the worker log store itself was missing. */
|
|
79
|
+
readonly workers: readonly WorkerLogSource[] | null;
|
|
80
|
+
/** Why `workers` is null (only set when it is). */
|
|
81
|
+
readonly workersMissingReason: string | null;
|
|
82
|
+
/** Run result document (JSON). */
|
|
83
|
+
readonly result: string | null;
|
|
84
|
+
/**
|
|
85
|
+
* Judge verdict document (JSON), or the matching ledger row re-encoded as
|
|
86
|
+
* one. Runners that write the verdict straight to a ledger leave no judge
|
|
87
|
+
* document, so the ledger row is the same fact from the same run — not a
|
|
88
|
+
* substitute measurement.
|
|
89
|
+
*/
|
|
90
|
+
readonly judge: string | null;
|
|
91
|
+
/** Where `judge` came from, for the report's provenance line. */
|
|
92
|
+
readonly judgeSource: string | null;
|
|
93
|
+
/** Delivered unified-diff patch text. */
|
|
94
|
+
readonly patch: string | null;
|
|
95
|
+
/** Outer-driver log (used for the driver's steer verbs + deadline evidence). */
|
|
96
|
+
readonly driverLog: string | null;
|
|
97
|
+
/**
|
|
98
|
+
* Worker tokens recovered from a harness session store; null = store unavailable.
|
|
99
|
+
* `store` names the store in the report's provenance line (e.g. `opencode`).
|
|
100
|
+
*/
|
|
101
|
+
readonly harnessWorkerTokens: {
|
|
102
|
+
store: string;
|
|
103
|
+
sessions: number;
|
|
104
|
+
input: number;
|
|
105
|
+
output: number;
|
|
106
|
+
/** Cached prompt tokens, when the store counts them separately. */
|
|
107
|
+
cacheRead?: number;
|
|
108
|
+
cacheWrite?: number;
|
|
109
|
+
} | null;
|
|
110
|
+
readonly harnessMissingReason: string | null;
|
|
111
|
+
/** What this store structurally cannot record. See `SourceLimits`. */
|
|
112
|
+
readonly limits: SourceLimits;
|
|
113
|
+
/**
|
|
114
|
+
* Where the ROOT invocation's transcript lives. Undefined lets the rollout
|
|
115
|
+
* minter fall back to the loops layout (`<supRunDir>/journal.jsonl`); any
|
|
116
|
+
* other store must say, or the row points at a path that never existed.
|
|
117
|
+
*/
|
|
118
|
+
readonly rootTranscriptRef?: string | null;
|
|
119
|
+
/**
|
|
120
|
+
* The `traces` CLI command that covers this run's harness-session layer.
|
|
121
|
+
* Null falls back to the analyzer's default (an opencode worker fleet).
|
|
122
|
+
*/
|
|
123
|
+
readonly traceCommand: string | null;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* A source of supervisor-run bytes. Implementations own their storage layout;
|
|
127
|
+
* the analyzer only ever sees `SupervisorRunSources`.
|
|
128
|
+
*/
|
|
129
|
+
interface SupervisorRunReader {
|
|
130
|
+
/** Stable identity of what this reader points at (for logs and report labels). */
|
|
131
|
+
readonly runRef: string;
|
|
132
|
+
read(): Promise<SupervisorRunSources>;
|
|
133
|
+
}
|
|
134
|
+
declare const SUPERVISOR_RUN_SCHEMA = "tangle.supervisor-run@1";
|
|
135
|
+
declare const SUPERVISOR_RUN_ROLLUP_SCHEMA = "tangle.supervisor-run-rollup@1";
|
|
136
|
+
interface SteerBreakdown {
|
|
137
|
+
readonly worker: string;
|
|
138
|
+
/** Steer requests durably queued to this worker's inbox. */
|
|
139
|
+
readonly queued: number;
|
|
140
|
+
/** Steers the worker's executor actually accepted (control event `delivered:true`). */
|
|
141
|
+
readonly delivered: number;
|
|
142
|
+
}
|
|
143
|
+
interface OrchestrationMetrics {
|
|
144
|
+
readonly workersSpawned: Measured<number>;
|
|
145
|
+
readonly workersSettled: Measured<number>;
|
|
146
|
+
readonly workersCancelled: Measured<number>;
|
|
147
|
+
/** THE HEADLINE: mid-task steers the brain sent to live workers. 0 ≠ unavailable. */
|
|
148
|
+
readonly steers: Measured<number>;
|
|
149
|
+
readonly steersDelivered: Measured<number>;
|
|
150
|
+
readonly steersByWorker: Measured<readonly SteerBreakdown[]>;
|
|
151
|
+
/** Outer-driver `supervisor_steer` tool calls seen in the driver log (a second steer path). */
|
|
152
|
+
readonly driverSteerCalls: Measured<number>;
|
|
153
|
+
/**
|
|
154
|
+
* Spawn waves. A wave is a maximal run of worker spawns with no settle/cancel between
|
|
155
|
+
* them: wave N+1 begins at the first spawn issued after at least one worker from an
|
|
156
|
+
* earlier wave has settled. Structural, not a time threshold — no tunable constant.
|
|
157
|
+
*/
|
|
158
|
+
readonly waves: Measured<number>;
|
|
159
|
+
readonly waveSizes: Measured<readonly number[]>;
|
|
160
|
+
readonly maxConcurrency: Measured<number>;
|
|
161
|
+
/** Worker spawns issued after the first settlement — the retry/respawn tail. */
|
|
162
|
+
readonly respawns: Measured<number>;
|
|
163
|
+
/** Labels spawned more than once (a literal retry of the same subtask). */
|
|
164
|
+
readonly repeatedLabels: Measured<readonly string[]>;
|
|
165
|
+
/** Longest parent chain below the root, in worker hops. */
|
|
166
|
+
readonly delegationDepth: Measured<number>;
|
|
167
|
+
readonly timeToFirstSpawnMs: Measured<number>;
|
|
168
|
+
readonly supervisorWallMs: Measured<number>;
|
|
169
|
+
/** Wall time inside the supervisor run with ZERO live workers. */
|
|
170
|
+
readonly idleMs: Measured<number>;
|
|
171
|
+
readonly idlePct: Measured<number>;
|
|
172
|
+
/** sum(worker wall) / supervisor wall. >1 means real parallelism. */
|
|
173
|
+
readonly workerUtilization: Measured<number>;
|
|
174
|
+
}
|
|
175
|
+
interface DecisionMetrics {
|
|
176
|
+
readonly settledByStatus: Measured<Record<string, number>>;
|
|
177
|
+
readonly settledVerdicts: Measured<Record<string, number>>;
|
|
178
|
+
/** Worker verified its own work green AND produced a patch. */
|
|
179
|
+
readonly accepted: Measured<number>;
|
|
180
|
+
/** Worker settled with a failing verify. */
|
|
181
|
+
readonly rejected: Measured<number>;
|
|
182
|
+
/** Worker verified green but delivered no patch bytes — output with nothing to accept. */
|
|
183
|
+
readonly emptyPass: Measured<number>;
|
|
184
|
+
/** Settlements the brain observed before issuing its next spawn (evidence→respawn). */
|
|
185
|
+
readonly observeThenRespawn: Measured<number>;
|
|
186
|
+
/** Respawns with no settled evidence in front of them. */
|
|
187
|
+
readonly respawnWithoutEvidence: Measured<number>;
|
|
188
|
+
/** Steer + question traffic on the live down/up legs — the only "review while running" signal. */
|
|
189
|
+
readonly reviewActions: Measured<number>;
|
|
190
|
+
readonly workerEvidenceBytes: Measured<number>;
|
|
191
|
+
}
|
|
192
|
+
interface RoleSpend {
|
|
193
|
+
readonly tokensIn: Measured<number>;
|
|
194
|
+
readonly tokensOut: Measured<number>;
|
|
195
|
+
/**
|
|
196
|
+
* Cached prompt tokens read/written. On a harness that caches aggressively
|
|
197
|
+
* these dwarf `tokensIn`, so a report that omits them understates the context
|
|
198
|
+
* each invocation actually consumed. `unavailable` = the store has no such counter.
|
|
199
|
+
*/
|
|
200
|
+
readonly cacheRead: Measured<number>;
|
|
201
|
+
readonly cacheWrite: Measured<number>;
|
|
202
|
+
readonly usd: Measured<number>;
|
|
203
|
+
readonly source: string;
|
|
204
|
+
}
|
|
205
|
+
interface PerWorkerRow {
|
|
206
|
+
readonly worker: string;
|
|
207
|
+
readonly wallMs: number | null;
|
|
208
|
+
/** `null` = this store does not attribute tokens per worker (NOT "zero tokens"). */
|
|
209
|
+
readonly tokensIn: number | null;
|
|
210
|
+
readonly tokensOut: number | null;
|
|
211
|
+
readonly usd: number | null;
|
|
212
|
+
readonly patchBytes: number | null;
|
|
213
|
+
readonly passed: boolean | null;
|
|
214
|
+
}
|
|
215
|
+
interface WallDistribution {
|
|
216
|
+
readonly n: number;
|
|
217
|
+
readonly min: number;
|
|
218
|
+
readonly p50: number;
|
|
219
|
+
readonly p90: number;
|
|
220
|
+
readonly max: number;
|
|
221
|
+
readonly sum: number;
|
|
222
|
+
}
|
|
223
|
+
interface EconomicsMetrics {
|
|
224
|
+
/** Driver/brain inference — journal `metered` events. */
|
|
225
|
+
readonly brain: RoleSpend;
|
|
226
|
+
/**
|
|
227
|
+
* Brain completions that came back `finish_reason: "length"` — output TRUNCATED. Any value
|
|
228
|
+
* above 0 means the supervisor planned into a wall and then acted on the half-written plan,
|
|
229
|
+
* which is a defect and not a cost figure. The journal's `metered` rows carry token counts
|
|
230
|
+
* but no finish reason, so this reads the per-call brain tap; a run whose supervisor
|
|
231
|
+
* predates that tap reports `unavailable`, never 0.
|
|
232
|
+
*/
|
|
233
|
+
readonly brainTruncations: Measured<number>;
|
|
234
|
+
/** Worker inference — journal `settled` spend plus the harness session join. */
|
|
235
|
+
readonly workers: RoleSpend;
|
|
236
|
+
readonly totalUsd: Measured<number>;
|
|
237
|
+
/**
|
|
238
|
+
* Where `totalUsd` came from. CLI-backend workers never price their own inference into
|
|
239
|
+
* the journal, so on those arms the total is BRAIN-ONLY and the worker row's token
|
|
240
|
+
* counts (recovered from the harness store) are the honest worker-side figure.
|
|
241
|
+
*/
|
|
242
|
+
readonly totalUsdSource: string;
|
|
243
|
+
readonly costPerAcceptedPatchUsd: Measured<number>;
|
|
244
|
+
readonly workerWallMsDistribution: Measured<WallDistribution>;
|
|
245
|
+
readonly perWorker: Measured<readonly PerWorkerRow[]>;
|
|
246
|
+
}
|
|
247
|
+
interface PatchStats {
|
|
248
|
+
readonly files: number;
|
|
249
|
+
readonly linesAdded: number;
|
|
250
|
+
readonly linesRemoved: number;
|
|
251
|
+
readonly testFilesTouched: readonly string[];
|
|
252
|
+
}
|
|
253
|
+
interface OutcomeMetrics {
|
|
254
|
+
readonly supStatus: Measured<string>;
|
|
255
|
+
readonly supVerdict: Measured<string>;
|
|
256
|
+
readonly delivered: Measured<boolean>;
|
|
257
|
+
readonly judgeResolved: Measured<boolean | null>;
|
|
258
|
+
readonly judgeScore: Measured<number | null>;
|
|
259
|
+
readonly judgePassed: Measured<number | null>;
|
|
260
|
+
readonly judgeTotal: Measured<number | null>;
|
|
261
|
+
readonly verifyPass: Measured<boolean>;
|
|
262
|
+
readonly verifyRc: Measured<number>;
|
|
263
|
+
readonly patch: Measured<PatchStats>;
|
|
264
|
+
/** Which document the judge fields came from (a judge file, a ledger row, or nothing). */
|
|
265
|
+
readonly judgeSource: string | null;
|
|
266
|
+
}
|
|
267
|
+
interface SupervisorRunReport {
|
|
268
|
+
readonly schema: typeof SUPERVISOR_RUN_SCHEMA;
|
|
269
|
+
/** The `runRef` of the sources this report was computed from. */
|
|
270
|
+
readonly runRef: string;
|
|
271
|
+
readonly instanceId: string | null;
|
|
272
|
+
readonly arm: string | null;
|
|
273
|
+
readonly supervisorId: Measured<string>;
|
|
274
|
+
readonly generatedAt: string;
|
|
275
|
+
readonly orchestration: OrchestrationMetrics;
|
|
276
|
+
readonly decision: DecisionMetrics;
|
|
277
|
+
readonly economics: EconomicsMetrics;
|
|
278
|
+
readonly outcome: OutcomeMetrics;
|
|
279
|
+
/** Artifacts that were missing, in read order — the provenance of every `unavailable`. */
|
|
280
|
+
readonly gaps: readonly string[];
|
|
281
|
+
/** The `traces` CLI command that covers the harness-session layer for this run. */
|
|
282
|
+
readonly traceCommand: string;
|
|
283
|
+
}
|
|
284
|
+
interface RollupCellRow {
|
|
285
|
+
readonly instanceId: string | null;
|
|
286
|
+
readonly arm: string | null;
|
|
287
|
+
readonly steers: Measured<number>;
|
|
288
|
+
readonly waves: Measured<number>;
|
|
289
|
+
readonly utilization: Measured<number>;
|
|
290
|
+
readonly idlePct: Measured<number>;
|
|
291
|
+
readonly resolved: Measured<boolean | null>;
|
|
292
|
+
readonly usd: Measured<number>;
|
|
293
|
+
}
|
|
294
|
+
interface SupervisorRunRollup {
|
|
295
|
+
readonly schema: typeof SUPERVISOR_RUN_ROLLUP_SCHEMA;
|
|
296
|
+
readonly cells: number;
|
|
297
|
+
readonly steersTotal: Measured<number>;
|
|
298
|
+
readonly cellsWithSteers: Measured<number>;
|
|
299
|
+
readonly cellsWithUnavailableSteers: number;
|
|
300
|
+
readonly wavesMean: Measured<number>;
|
|
301
|
+
readonly maxConcurrencyMax: Measured<number>;
|
|
302
|
+
readonly utilizationMean: Measured<number>;
|
|
303
|
+
readonly idlePctMean: Measured<number>;
|
|
304
|
+
readonly workersSpawnedTotal: Measured<number>;
|
|
305
|
+
readonly acceptedTotal: Measured<number>;
|
|
306
|
+
readonly usdTotal: Measured<number>;
|
|
307
|
+
readonly resolvedCount: Measured<number>;
|
|
308
|
+
readonly perCell: readonly RollupCellRow[];
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* A supervision tree expressed in the canonical rollout row type: one
|
|
312
|
+
* `RolloutLine` per invocation, joined by `parent_rollout_id`. The root row
|
|
313
|
+
* carries `role: 'supervisor'`; every spawned worker carries `role: 'worker'`
|
|
314
|
+
* with the root as its parent.
|
|
315
|
+
*/
|
|
316
|
+
interface SupervisorRunTree {
|
|
317
|
+
readonly rootId: string | null;
|
|
318
|
+
readonly nodes: readonly RolloutLine[];
|
|
319
|
+
/** Why a node could not be recovered, in read order. */
|
|
320
|
+
readonly gaps: readonly string[];
|
|
321
|
+
}
|
|
322
|
+
//#endregion
|
|
323
|
+
//#region src/supervisor-run/analyze.d.ts
|
|
324
|
+
interface Tokens {
|
|
325
|
+
input: number;
|
|
326
|
+
output: number;
|
|
327
|
+
cacheRead: number;
|
|
328
|
+
cacheWrite: number;
|
|
329
|
+
/** False when the event carried no cache counters at all — not "zero cached". */
|
|
330
|
+
hasCache: boolean;
|
|
331
|
+
}
|
|
332
|
+
interface SpendLike {
|
|
333
|
+
tokens: Tokens;
|
|
334
|
+
usd: number;
|
|
335
|
+
}
|
|
336
|
+
interface SpawnRow {
|
|
337
|
+
id: string;
|
|
338
|
+
parent: string | null;
|
|
339
|
+
label: string;
|
|
340
|
+
at: number | null;
|
|
341
|
+
}
|
|
342
|
+
interface CloseRow {
|
|
343
|
+
id: string;
|
|
344
|
+
kind: 'settled' | 'cancelled';
|
|
345
|
+
status: string | null;
|
|
346
|
+
verdict: string | null;
|
|
347
|
+
at: number | null;
|
|
348
|
+
spend: SpendLike;
|
|
349
|
+
/** False when the close event carried no spend object — not "spent nothing". */
|
|
350
|
+
hasSpend: boolean;
|
|
351
|
+
}
|
|
352
|
+
interface WorkerLogFacts {
|
|
353
|
+
started: number | null;
|
|
354
|
+
/** True once a `finished` event was seen — independent of whether its `at` parsed. */
|
|
355
|
+
finished: boolean;
|
|
356
|
+
finishedAt: number | null;
|
|
357
|
+
passed: boolean | null;
|
|
358
|
+
/** `patchBytes` as reported by the finished event (not the patch file's size). */
|
|
359
|
+
finishedPatchBytes: number | null;
|
|
360
|
+
evidenceBytes: number;
|
|
361
|
+
steersQueued: number;
|
|
362
|
+
steersDelivered: number;
|
|
363
|
+
questions: number;
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* The tree + timeline the report is computed from, exposed because the rollout-row
|
|
367
|
+
* minter needs exactly the same parse (one parser, two consumers).
|
|
368
|
+
*/
|
|
369
|
+
interface SupervisorTreeFacts {
|
|
370
|
+
readonly rootId: string | null;
|
|
371
|
+
readonly spawns: readonly SpawnRow[];
|
|
372
|
+
readonly closes: readonly CloseRow[];
|
|
373
|
+
readonly workerSpawns: readonly SpawnRow[];
|
|
374
|
+
readonly workerCloses: readonly CloseRow[];
|
|
375
|
+
readonly brain: {
|
|
376
|
+
tokensIn: number;
|
|
377
|
+
tokensOut: number;
|
|
378
|
+
cacheRead: number;
|
|
379
|
+
cacheWrite: number;
|
|
380
|
+
/** False when no metered event carried cache counters — not "nothing cached". */
|
|
381
|
+
hasCache: boolean;
|
|
382
|
+
usd: number;
|
|
383
|
+
meteredCount: number;
|
|
384
|
+
};
|
|
385
|
+
readonly workerLogs: ReadonlyMap<string, WorkerLogFacts>;
|
|
386
|
+
readonly startedAt: number | null;
|
|
387
|
+
readonly completedAt: number | null;
|
|
388
|
+
}
|
|
389
|
+
declare function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFacts;
|
|
390
|
+
/**
|
|
391
|
+
* Analyze already-read supervisor-run bytes. Pure and synchronous: same bytes
|
|
392
|
+
* in, same report out (modulo `generatedAt`, which `now` pins in tests).
|
|
393
|
+
*/
|
|
394
|
+
declare function analyzeSupervisorRunSources(src: SupervisorRunSources, now?: () => number): SupervisorRunReport;
|
|
395
|
+
/** Unified-diff stats. Counts `+++ b/<path>` targets, body +/- lines, and test-file touches. */
|
|
396
|
+
declare function parsePatch(text: string): PatchStats;
|
|
397
|
+
/**
|
|
398
|
+
* Aggregate many supervisor-run reports. A metric no run could measure stays
|
|
399
|
+
* `unavailable` rather than becoming a 0-valued mean, and cells whose steer
|
|
400
|
+
* count was unavailable are counted separately from cells that measured zero.
|
|
401
|
+
*/
|
|
402
|
+
declare function rollupSupervisorRuns(reports: readonly SupervisorRunReport[]): SupervisorRunRollup;
|
|
403
|
+
//#endregion
|
|
404
|
+
//#region src/supervisor-run/claude-code-reader.d.ts
|
|
405
|
+
/** Tool names that spawn a child agent. `Task` is the older name for `Agent`. */
|
|
406
|
+
declare const DEFAULT_SPAWN_TOOLS: readonly ['Agent', 'Task'];
|
|
407
|
+
/** Tool names that deliver a message to an ALREADY-RUNNING child agent. */
|
|
408
|
+
declare const DEFAULT_STEER_TOOLS: readonly ['SendMessage'];
|
|
409
|
+
/** Tool names that stop a running child agent. */
|
|
410
|
+
declare const DEFAULT_CANCEL_TOOLS: readonly ['TaskStop', 'KillAgent'];
|
|
411
|
+
interface ClaudeCodeReaderOptions {
|
|
412
|
+
/** The main session transcript: `~/.claude/projects/<slug>/<sessionId>.jsonl`. */
|
|
413
|
+
readonly transcriptPath: string;
|
|
414
|
+
/**
|
|
415
|
+
* Directory of child transcripts. Defaults to `<transcript-dir>/<sessionId>/subagents`.
|
|
416
|
+
* `null` skips the join, and every per-worker token count becomes unavailable.
|
|
417
|
+
*/
|
|
418
|
+
readonly subagentsDir?: string | null;
|
|
419
|
+
readonly runRef?: string;
|
|
420
|
+
readonly instanceId?: string | null;
|
|
421
|
+
readonly arm?: string | null;
|
|
422
|
+
readonly spawnTools?: readonly string[];
|
|
423
|
+
readonly steerTools?: readonly string[];
|
|
424
|
+
readonly cancelTools?: readonly string[];
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Read a Claude Code session (plus its subagent transcripts) as supervision-tree
|
|
428
|
+
* source bytes. Never throws on a missing artifact.
|
|
429
|
+
*/
|
|
430
|
+
declare function readClaudeCodeSupervisorRun(opts: ClaudeCodeReaderOptions): Promise<SupervisorRunSources>;
|
|
431
|
+
/** A `SupervisorRunReader` over a Claude Code session — the same contract loops implements. */
|
|
432
|
+
declare function claudeCodeSupervisorRunReader(opts: ClaudeCodeReaderOptions): SupervisorRunReader;
|
|
433
|
+
//#endregion
|
|
434
|
+
//#region src/supervisor-run/loops-reader.d.ts
|
|
435
|
+
/** Locate the (single) supervisor run dir under `<ws>/.loops/supervisor`. */
|
|
436
|
+
declare function findSupervisorRunDirIn(ws: string): Promise<string | null>;
|
|
437
|
+
interface LoopsReaderOptions {
|
|
438
|
+
/** Override the workspace dir (default `<runDir>/ws`). */
|
|
439
|
+
readonly ws?: string;
|
|
440
|
+
/** Delivered patch path (default: `patchPath` from result.json). */
|
|
441
|
+
readonly patchPath?: string;
|
|
442
|
+
/** opencode sqlite store; set to `null` to skip the worker-token join entirely. */
|
|
443
|
+
readonly opencodeDb?: string | null;
|
|
444
|
+
/** Ledger to fall back to when the run has no `judge.json` (matched on iid + arm + runDir). */
|
|
445
|
+
readonly ledgerPath?: string;
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Read a loops supervisor run directory into source bytes. Never throws on a
|
|
449
|
+
* missing artifact — an absent file becomes a `null` field, which is what makes
|
|
450
|
+
* the dependent metric `unavailable` instead of 0.
|
|
451
|
+
*/
|
|
452
|
+
declare function readLoopsSupervisorRun(runDir: string, opts?: LoopsReaderOptions): Promise<SupervisorRunSources>;
|
|
453
|
+
/** The loops on-disk layout, as a `SupervisorRunReader`. */
|
|
454
|
+
declare function loopsSupervisorRunReader(runDir: string, opts?: LoopsReaderOptions): SupervisorRunReader;
|
|
455
|
+
/**
|
|
456
|
+
* Analyze a supervisor run. Accepts a run directory (read through the loops
|
|
457
|
+
* reader), any `SupervisorRunReader`, or already-read source bytes — so a
|
|
458
|
+
* caller with its own store never has to touch the filesystem layout.
|
|
459
|
+
*/
|
|
460
|
+
declare function analyzeSupervisorRun(input: string | SupervisorRunReader | SupervisorRunSources, opts?: LoopsReaderOptions): Promise<SupervisorRunReport>;
|
|
461
|
+
interface WriteSupervisorRunOptions extends LoopsReaderOptions {
|
|
462
|
+
/** Append the headline block here (the experiment's run log). */
|
|
463
|
+
readonly appendHeadlineTo?: string;
|
|
464
|
+
/** Also console.log the headline (default true). */
|
|
465
|
+
readonly echo?: boolean;
|
|
466
|
+
/**
|
|
467
|
+
* Write `run-report.{json,md}` here instead of into the run dir. Set when
|
|
468
|
+
* reporting over a run directory that must stay READ-ONLY (a live run, an
|
|
469
|
+
* archived generation).
|
|
470
|
+
*/
|
|
471
|
+
readonly reportDir?: string;
|
|
472
|
+
}
|
|
473
|
+
/**
|
|
474
|
+
* Read a completed run, write `run-report.json` + `run-report.md` beside its
|
|
475
|
+
* artifacts, and append the headline block to the run log. Never throws on a
|
|
476
|
+
* missing artifact — a run that produced nothing still yields a report whose
|
|
477
|
+
* every metric says why.
|
|
478
|
+
*/
|
|
479
|
+
declare function writeSupervisorRunReport(runDir: string, opts?: WriteSupervisorRunOptions): Promise<SupervisorRunReport>;
|
|
480
|
+
/**
|
|
481
|
+
* File stem for out-of-tree reports. Built from the run path's identifying
|
|
482
|
+
* segments — candidate tag (the segment under `arm-runs/`), rep, instance, arm
|
|
483
|
+
* — so two runs of the same instance from different candidates/reps never
|
|
484
|
+
* overwrite each other.
|
|
485
|
+
*/
|
|
486
|
+
declare function supervisorReportStem(runDir: string): string;
|
|
487
|
+
/**
|
|
488
|
+
* Best-effort wrapper for a hot path: a reporting failure must never kill a run
|
|
489
|
+
* that already produced real work. Returns null and logs the reason instead.
|
|
490
|
+
*/
|
|
491
|
+
declare function writeSupervisorRunReportSafe(runDir: string, opts?: WriteSupervisorRunOptions): Promise<SupervisorRunReport | null>;
|
|
492
|
+
/**
|
|
493
|
+
* Report every run under an experiment `outDir` (any depth of
|
|
494
|
+
* `runs/<iid>/<arm>`), write each run's report, and write the rollup at
|
|
495
|
+
* `<outDir>/run-report-round.{json,md}`.
|
|
496
|
+
*/
|
|
497
|
+
declare function reportSupervisorRound(outDir: string, opts?: WriteSupervisorRunOptions & {
|
|
498
|
+
title?: string;
|
|
499
|
+
}): Promise<SupervisorRunRollup>;
|
|
500
|
+
/** Every `<...>/runs/<iid>/<arm>` directory under `root`. */
|
|
501
|
+
declare function findSupervisorRunDirs(root: string): Promise<string[]>;
|
|
502
|
+
//#endregion
|
|
503
|
+
//#region src/supervisor-run/render.d.ts
|
|
504
|
+
/**
|
|
505
|
+
* The block appended to a run log after every run — the answers an operator asks
|
|
506
|
+
* for, in the log tail, with no extra command.
|
|
507
|
+
*/
|
|
508
|
+
declare function renderSupervisorRunHeadline(r: SupervisorRunReport): string;
|
|
509
|
+
declare function renderSupervisorRunMarkdown(r: SupervisorRunReport): string;
|
|
510
|
+
declare function renderSupervisorRollupMarkdown(rollup: SupervisorRunRollup, title?: string): string;
|
|
511
|
+
//#endregion
|
|
512
|
+
//#region src/supervisor-run/rollout-nodes.d.ts
|
|
513
|
+
interface SupervisorRolloutOptions {
|
|
514
|
+
/** Benchmark/suite id for `task.suite`. Defaults to `'supervisor-run'`. */
|
|
515
|
+
readonly suite?: string;
|
|
516
|
+
/** `task.split`. Defaults to `'search'` (the trainable pool). */
|
|
517
|
+
readonly split?: RolloutSplit;
|
|
518
|
+
/** Replicate index. Defaults to 0. */
|
|
519
|
+
readonly rep?: number;
|
|
520
|
+
/** Sampling seed the campaign pinned. Defaults to null (not recorded). */
|
|
521
|
+
readonly seed?: number | null;
|
|
522
|
+
/** `run_id` for every node. Defaults to the supervisor root id, else `runRef`. */
|
|
523
|
+
readonly runId?: string;
|
|
524
|
+
/** Harness that drove the supervisor. */
|
|
525
|
+
readonly supervisorHarness?: string | null;
|
|
526
|
+
/** Harness that drove the workers. */
|
|
527
|
+
readonly workerHarness?: string | null;
|
|
528
|
+
/** Model the supervisor ran on. */
|
|
529
|
+
readonly supervisorModel?: string | null;
|
|
530
|
+
/** Model the workers ran on. */
|
|
531
|
+
readonly workerModel?: string | null;
|
|
532
|
+
readonly experimentId?: string | null;
|
|
533
|
+
readonly candidateId?: string | null;
|
|
534
|
+
readonly generation?: number | null;
|
|
535
|
+
readonly candidateIndex?: number | null;
|
|
536
|
+
/** Pins `provenance.captured_at`; defaults to now. */
|
|
537
|
+
readonly capturedAt?: string;
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Mint the supervision tree as rollout rows. Returns the rows plus the gaps
|
|
541
|
+
* that made any of them incomplete — same unavailable-vs-zero discipline as
|
|
542
|
+
* the report: a row with no transcript says WHY, it never pretends to be empty.
|
|
543
|
+
*/
|
|
544
|
+
declare function supervisorRunRolloutLines(src: SupervisorRunSources, opts?: SupervisorRolloutOptions): SupervisorRunTree;
|
|
545
|
+
//#endregion
|
|
546
|
+
export { isUnavailable as $, rollupSupervisorRuns as A, RollupCellRow as B, CloseRow as C, analyzeSupervisorRunSources as D, WorkerLogFacts as E, OrchestrationMetrics as F, SupervisorRunReader as G, SUPERVISOR_RUN_SCHEMA as H, OutcomeMetrics as I, SupervisorRunSources as J, SupervisorRunReport as K, PatchStats as L, EconomicsMetrics as M, Measured as N, parsePatch as O, NO_SOURCE_LIMITS as P, WorkerLogSource as Q, PerWorkerRow as R, readClaudeCodeSupervisorRun as S, SupervisorTreeFacts as T, SourceLimits as U, SUPERVISOR_RUN_ROLLUP_SCHEMA as V, SteerBreakdown as W, Unavailable as X, SupervisorRunTree as Y, WallDistribution as Z, ClaudeCodeReaderOptions as _, renderSupervisorRunMarkdown as a, DEFAULT_STEER_TOOLS as b, analyzeSupervisorRun as c, loopsSupervisorRunReader as d, showMeasured as et, readLoopsSupervisorRun as f, writeSupervisorRunReportSafe as g, writeSupervisorRunReport as h, renderSupervisorRunHeadline as i, DecisionMetrics as j, parseSupervisorTree as k, findSupervisorRunDirIn as l, supervisorReportStem as m, supervisorRunRolloutLines as n, LoopsReaderOptions as o, reportSupervisorRound as p, SupervisorRunRollup as q, renderSupervisorRollupMarkdown as r, WriteSupervisorRunOptions as s, SupervisorRolloutOptions as t, unavailable as tt, findSupervisorRunDirs as u, DEFAULT_CANCEL_TOOLS as v, SpawnRow as w, claudeCodeSupervisorRunReader as x, DEFAULT_SPAWN_TOOLS as y, RoleSpend as z };
|
|
547
|
+
//# sourceMappingURL=index-C61Wi7yg.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-C61Wi7yg.d.ts","names":[],"sources":["../src/supervisor-run/types.ts","../src/supervisor-run/analyze.ts","../src/supervisor-run/claude-code-reader.ts","../src/supervisor-run/loops-reader.ts","../src/supervisor-run/render.ts","../src/supervisor-run/rollout-nodes.ts"],"mappings":";;;UAiCiB;WACN;;;KAIC,SAAS,KAAK,IAAI;iBAEd,YAAY,iBAAiB;iBAI7B,cAAc,aAAa,KAAK;;iBAKhC,aAAa,GAAG;;UAWf;WACN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA;WACA;WACA;;;;;;;;;;;;;UAcM;;WAEN;;WAEA;;WAEA;;;cAIE,kBAAkB;;;;;;;;;;UAed;;WAEN;WACA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,kBAAkB;;WAElB;;WAEA;;;;;;;WAOA;;WAEA;;WAEA;;WAEA;;;;;WAKA;IACP;IACA;IACA;IACA;;IAEA;IACA;;WAEO;;WAEA,QAAQ;;;;;;WAMR;;;;;WAKA;;;;;;UAOM;;WAEN;EACT,QAAQ,QAAQ;;cAOL;cACA;UAEI;WACN;;WAEA;;WAEA;;UAGM;WACN,gBAAgB;WAChB,gBAAgB;WAChB,kBAAkB;;WAElB,QAAQ;WACR,iBAAiB;WACjB,gBAAgB,kBAAkB;;WAElC,kBAAkB;;;;;;WAMlB,OAAO;WACP,WAAW;WACX,gBAAgB;;WAEhB,UAAU;;WAEV,gBAAgB;;WAEhB,iBAAiB;WACjB,oBAAoB;WACpB,kBAAkB;;WAElB,QAAQ;WACR,SAAS;;WAET,mBAAmB;;UAGb;WACN,iBAAiB,SAAS;WAC1B,iBAAiB,SAAS;;WAE1B,UAAU;;WAEV,UAAU;;WAEV,WAAW;;WAEX,oBAAoB;;WAEpB,wBAAwB;;WAExB,eAAe;WACf,qBAAqB;;UAGf;WACN,UAAU;WACV,WAAW;;;;;;WAMX,WAAW;WACX,YAAY;WACZ,KAAK;WACL;;UAGM;WACN;WACA;;WAEA;WACA;WACA;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA;WACA;WACA;;UAGM;;WAEN,OAAO;;;;;;;;WAQP,kBAAkB;;WAElB,SAAS;WACT,UAAU;;;;;;WAMV;WACA,yBAAyB;WACzB,0BAA0B,SAAS;WACnC,WAAW,kBAAkB;;UAGvB;WACN;WACA;WACA;WACA;;UAGM;WACN,WAAW;WACX,YAAY;WACZ,WAAW;WACX,eAAe;WACf,YAAY;WACZ,aAAa;WACb,YAAY;WACZ,YAAY;WACZ,UAAU;WACV,OAAO,SAAS;;WAEhB;;UAGM;WACN,eAAe;;WAEf;WACA;WACA;WACA,cAAc;WACd;WACA,eAAe;WACf,UAAU;WACV,WAAW;WACX,SAAS;;WAET;;WAEA;;UAGM;WACN;WACA;WACA,QAAQ;WACR,OAAO;WACP,aAAa;WACb,SAAS;WACT,UAAU;WACV,KAAK;;UAGC;WACN,eAAe;WACf;WACA,aAAa;WACb,iBAAiB;WACjB;WACA,WAAW;WACX,mBAAmB;WACnB,iBAAiB;WACjB,aAAa;WACb,qBAAqB;WACrB,eAAe;WACf,UAAU;WACV,eAAe;WACf,kBAAkB;;;;;;;;UASZ;WACN;WACA,gBAAgB;;WAEhB;;;;UC3VD;EACR;EACA;EACA;EACA;;EAEA;;UAGQ;EACR,QAAQ;EACR;;UAoEe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;;EAEP;;UAGe;EACf;;EAEA;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;;;;;UAOe;WACN;WACA,iBAAiB;WACjB,iBAAiB;WACjB,uBAAuB;WACvB,uBAAuB;WACvB;IACP;IACA;IACA;IACA;;IAEA;IACA;IACA;;WAEO,YAAY,oBAAoB;WAChC;WACA;;iBAGK,oBAAoB,KAAK,uBAAuB;;;;;iBAyIhD,4BACd,KAAK,sBACL,qBACC;;iBAgfa,WAAW,eAAe;;;;;;iBA+C1B,qBAAqB,kBAAkB,wBAAwB;;;;cC3xBlE;;cAEA;;cAEA;UASI;;WAEN;;;;;WAKA;WACA;WACA;WACA;WACA;WACA;WACA;;;;;;iBAmOW,4BACpB,MAAM,0BACL,QAAQ;;iBA8TK,8BAA8B,MAAM,0BAA0B;;;;iBCzlBxD,uBAAuB,aAAa;UAOzC;;WAEN;;WAEA;;WAEA;;WAEA;;;;;;;iBAQW,uBACpB,gBACA,OAAM,qBACL,QAAQ;;iBAiHK,yBACd,gBACA,OAAM,qBACL;;;;;;iBAkCmB,qBACpB,gBAAgB,sBAAsB,sBACtC,OAAM,qBACL,QAAQ;UAYM,kCAAkC;;WAExC;;WAEA;;;;;;WAMA;;;;;;;;iBASW,yBACpB,gBACA,OAAM,4BACL,QAAQ;;;;;;;iBAuBK,qBAAqB;;;;;iBAiBf,6BACpB,gBACA,OAAM,4BACL,QAAQ;;;;;;iBAgBW,sBACpB,gBACA,OAAM;EAA8B;IACnC,QAAQ;;iBA0BW,sBAAsB,eAAe;;;;;;;iBCnU3C,4BAA4B,GAAG;iBAgC/B,4BAA4B,GAAG;iBAkK/B,+BACd,QAAQ,qBACR;;;UCrLe;;WAEN;;WAEA,QAAQ;;WAER;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA;WACA;WACA;WACA;;WAEA;;;;;;;iBAkBK,0BACd,KAAK,sBACL,OAAM,2BACL"}
|