@tangle-network/agent-eval 0.129.0 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +1 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +81 -2872
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -360
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1188
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1709
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -891
- package/dist/benchmarks/index.js +2 -60
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6381
- package/dist/campaign/index.js +3 -213
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -175
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5565
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1938
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -33
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -618
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3755 -15555
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11182 -11216
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -480
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1312
- package/dist/reporting.js +6 -51
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +760 -4010
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2325 -1958
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -2087
- package/dist/rollout/index.js +8 -168
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/{chunk-56TAVBOK.js.map → run-record-BuoE80Dq.js.map} +1 -1
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -959
- package/dist/supervisor-run/index.js +2 -65
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -252
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1173
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/package.json +17 -9
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2QU3YOPR.js +0 -7374
- package/dist/chunk-2QU3YOPR.js.map +0 -1
- package/dist/chunk-3OCR4R5I.js +0 -728
- package/dist/chunk-3OCR4R5I.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-56TAVBOK.js +0 -698
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7FO3TNPI.js +0 -232
- package/dist/chunk-7FO3TNPI.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BSO5JDQH.js +0 -2335
- package/dist/chunk-BSO5JDQH.js.map +0 -1
- package/dist/chunk-C6LXANRU.js +0 -1550
- package/dist/chunk-C6LXANRU.js.map +0 -1
- package/dist/chunk-DODXQREJ.js +0 -752
- package/dist/chunk-DODXQREJ.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-E7QXT7SX.js +0 -183
- package/dist/chunk-E7QXT7SX.js.map +0 -1
- package/dist/chunk-EG66UGL4.js +0 -341
- package/dist/chunk-EG66UGL4.js.map +0 -1
- package/dist/chunk-FXTVJPYD.js +0 -576
- package/dist/chunk-FXTVJPYD.js.map +0 -1
- package/dist/chunk-G7MGMCZD.js +0 -153
- package/dist/chunk-G7MGMCZD.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-H23X7XKK.js +0 -181
- package/dist/chunk-H23X7XKK.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-HPWUNB47.js +0 -289
- package/dist/chunk-HPWUNB47.js.map +0 -1
- package/dist/chunk-IYCLP2N2.js +0 -766
- package/dist/chunk-IYCLP2N2.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-JQSF5DQT.js +0 -701
- package/dist/chunk-JQSF5DQT.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-M4YBQKIJ.js +0 -1040
- package/dist/chunk-M4YBQKIJ.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NY44NC4A.js +0 -1056
- package/dist/chunk-NY44NC4A.js.map +0 -1
- package/dist/chunk-OIUOT4QD.js +0 -44
- package/dist/chunk-OIUOT4QD.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-OWN5NPMC.js +0 -152
- package/dist/chunk-OWN5NPMC.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PC5DOSM7.js +0 -579
- package/dist/chunk-PC5DOSM7.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-QB6BDBP2.js +0 -4464
- package/dist/chunk-QB6BDBP2.js.map +0 -1
- package/dist/chunk-RXHCETDZ.js +0 -536
- package/dist/chunk-RXHCETDZ.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-SFLLL76A.js +0 -669
- package/dist/chunk-SFLLL76A.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-T6RLYGAD.js +0 -158
- package/dist/chunk-T6RLYGAD.js.map +0 -1
- package/dist/chunk-TJVT4QFF.js +0 -911
- package/dist/chunk-TJVT4QFF.js.map +0 -1
- package/dist/chunk-TQ7LNKZ3.js +0 -136
- package/dist/chunk-TQ7LNKZ3.js.map +0 -1
- package/dist/chunk-U4L7JRPZ.js +0 -1706
- package/dist/chunk-U4L7JRPZ.js.map +0 -1
- package/dist/chunk-U4PHLT2N.js +0 -419
- package/dist/chunk-U4PHLT2N.js.map +0 -1
- package/dist/chunk-VCZ5FQYW.js +0 -928
- package/dist/chunk-VCZ5FQYW.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WVATSFCP.js +0 -1553
- package/dist/chunk-WVATSFCP.js.map +0 -1
- package/dist/chunk-X4YIBDER.js +0 -1662
- package/dist/chunk-X4YIBDER.js.map +0 -1
- package/dist/chunk-YQN4ICPP.js +0 -355
- package/dist/chunk-YQN4ICPP.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZHTZ4EYI.js +0 -1212
- package/dist/chunk-ZHTZ4EYI.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-OJJ7CZF4.js +0 -18
- package/dist/run-campaign-OJJ7CZF4.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
import { t as FAILURE_CLASSES } from "./schema-CRhEY1SO.js";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
//#region src/hosted/types.ts
|
|
4
|
+
const HOSTED_WIRE_VERSION = "2026-07-24.v1";
|
|
5
|
+
//#endregion
|
|
6
|
+
//#region src/hosted/schemas.ts
|
|
7
|
+
const finiteNumber = z.number().finite();
|
|
8
|
+
const nonNegativeNumber = finiteNumber.nonnegative();
|
|
9
|
+
const nonNegativeInteger = z.number().int().nonnegative();
|
|
10
|
+
const nonEmptyString = z.string().min(1).refine((value) => value.trim().length > 0, "must not be blank");
|
|
11
|
+
const attributeValue = z.union([
|
|
12
|
+
z.string(),
|
|
13
|
+
finiteNumber,
|
|
14
|
+
z.boolean()
|
|
15
|
+
]);
|
|
16
|
+
const attributes = z.record(z.string(), attributeValue);
|
|
17
|
+
const FailureClassSchema = z.custom((value) => typeof value === "string" && FAILURE_CLASSES.includes(value), `expected one of ${FAILURE_CLASSES.join(", ")}`);
|
|
18
|
+
const UINT64_MAX = 18446744073709551615n;
|
|
19
|
+
const UnixNanoTimestampSchema = z.string().regex(/^(0|[1-9][0-9]*)$/, "expected an unsigned base-10 integer string").refine((value) => BigInt(value) <= UINT64_MAX, "must fit in an unsigned 64-bit integer");
|
|
20
|
+
const GainDistributionBinSchema = z.object({
|
|
21
|
+
lo: finiteNumber,
|
|
22
|
+
hi: finiteNumber,
|
|
23
|
+
count: nonNegativeInteger
|
|
24
|
+
}).strict();
|
|
25
|
+
const ScalarDistributionSchema = z.object({
|
|
26
|
+
n: nonNegativeInteger,
|
|
27
|
+
mean: finiteNumber.nullable(),
|
|
28
|
+
p50: finiteNumber.nullable(),
|
|
29
|
+
p95: finiteNumber.nullable(),
|
|
30
|
+
stddev: nonNegativeNumber.nullable(),
|
|
31
|
+
min: finiteNumber.nullable(),
|
|
32
|
+
max: finiteNumber.nullable(),
|
|
33
|
+
histogram: z.array(GainDistributionBinSchema),
|
|
34
|
+
tailRuns: z.array(z.object({
|
|
35
|
+
runId: nonEmptyString,
|
|
36
|
+
score: finiteNumber
|
|
37
|
+
}).strict()).optional()
|
|
38
|
+
}).strict().superRefine((distribution, ctx) => {
|
|
39
|
+
const values = [
|
|
40
|
+
distribution.mean,
|
|
41
|
+
distribution.p50,
|
|
42
|
+
distribution.p95,
|
|
43
|
+
distribution.stddev,
|
|
44
|
+
distribution.min,
|
|
45
|
+
distribution.max
|
|
46
|
+
];
|
|
47
|
+
if (distribution.n === 0 && values.some((value) => value !== null)) ctx.addIssue({
|
|
48
|
+
code: "custom",
|
|
49
|
+
message: "distribution values must be null when n is 0"
|
|
50
|
+
});
|
|
51
|
+
if (distribution.n > 0 && values.some((value) => value === null)) ctx.addIssue({
|
|
52
|
+
code: "custom",
|
|
53
|
+
message: "distribution values must be numbers when n is greater than 0"
|
|
54
|
+
});
|
|
55
|
+
if (distribution.min !== null && distribution.max !== null && distribution.min > distribution.max) ctx.addIssue({
|
|
56
|
+
code: "custom",
|
|
57
|
+
path: ["min"],
|
|
58
|
+
message: "min must not exceed max"
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
const TokenUsageInsightSchema = z.object({
|
|
62
|
+
input: ScalarDistributionSchema,
|
|
63
|
+
output: ScalarDistributionSchema,
|
|
64
|
+
reasoning: ScalarDistributionSchema,
|
|
65
|
+
cached: ScalarDistributionSchema,
|
|
66
|
+
cacheWrite: ScalarDistributionSchema,
|
|
67
|
+
totals: z.object({
|
|
68
|
+
input: nonNegativeNumber,
|
|
69
|
+
output: nonNegativeNumber,
|
|
70
|
+
reasoning: nonNegativeNumber,
|
|
71
|
+
cached: nonNegativeNumber,
|
|
72
|
+
cacheWrite: nonNegativeNumber
|
|
73
|
+
}).strict()
|
|
74
|
+
}).strict();
|
|
75
|
+
const ExecutionErrorOutcomeCellSchema = z.object({
|
|
76
|
+
withErrors: nonNegativeInteger,
|
|
77
|
+
withoutErrors: nonNegativeInteger,
|
|
78
|
+
unreported: nonNegativeInteger
|
|
79
|
+
}).strict();
|
|
80
|
+
const ExecutionInsightSchema = z.object({
|
|
81
|
+
durationMs: ScalarDistributionSchema,
|
|
82
|
+
queueMs: ScalarDistributionSchema,
|
|
83
|
+
tokenUsage: TokenUsageInsightSchema,
|
|
84
|
+
aggregateUsage: z.object({
|
|
85
|
+
runs: nonNegativeInteger,
|
|
86
|
+
tokenUsage: TokenUsageInsightSchema,
|
|
87
|
+
costUsd: ScalarDistributionSchema,
|
|
88
|
+
totalCostUsd: nonNegativeNumber
|
|
89
|
+
}).strict(),
|
|
90
|
+
models: z.array(z.object({
|
|
91
|
+
model: nonEmptyString,
|
|
92
|
+
runs: nonNegativeInteger
|
|
93
|
+
}).strict()),
|
|
94
|
+
modelCalls: z.object({
|
|
95
|
+
runs: nonNegativeInteger,
|
|
96
|
+
events: nonNegativeInteger,
|
|
97
|
+
reportingRuns: nonNegativeInteger
|
|
98
|
+
}).strict(),
|
|
99
|
+
executionErrors: z.object({
|
|
100
|
+
runs: nonNegativeInteger,
|
|
101
|
+
fraction: finiteNumber.min(0).max(1).nullable(),
|
|
102
|
+
events: nonNegativeInteger,
|
|
103
|
+
reportingRuns: nonNegativeInteger,
|
|
104
|
+
errorSpanEvents: nonNegativeInteger,
|
|
105
|
+
errorSpanReportingRuns: nonNegativeInteger,
|
|
106
|
+
byTerminalOutcome: z.object({
|
|
107
|
+
succeeded: ExecutionErrorOutcomeCellSchema,
|
|
108
|
+
failed: ExecutionErrorOutcomeCellSchema,
|
|
109
|
+
cancelled: ExecutionErrorOutcomeCellSchema,
|
|
110
|
+
incomplete: ExecutionErrorOutcomeCellSchema,
|
|
111
|
+
unknown: ExecutionErrorOutcomeCellSchema
|
|
112
|
+
}).strict()
|
|
113
|
+
}).strict(),
|
|
114
|
+
terminalOutcomes: z.object({
|
|
115
|
+
succeeded: nonNegativeInteger,
|
|
116
|
+
failed: nonNegativeInteger,
|
|
117
|
+
cancelled: nonNegativeInteger,
|
|
118
|
+
incomplete: nonNegativeInteger,
|
|
119
|
+
unknown: nonNegativeInteger
|
|
120
|
+
}).strict()
|
|
121
|
+
}).strict();
|
|
122
|
+
const CostProvenanceSummarySchema = z.object({
|
|
123
|
+
observed: z.object({
|
|
124
|
+
n: nonNegativeInteger,
|
|
125
|
+
totalUsd: nonNegativeNumber
|
|
126
|
+
}).strict(),
|
|
127
|
+
estimated: z.object({
|
|
128
|
+
n: nonNegativeInteger,
|
|
129
|
+
totalUsd: nonNegativeNumber
|
|
130
|
+
}).strict(),
|
|
131
|
+
uncaptured: z.object({ n: nonNegativeInteger }).strict(),
|
|
132
|
+
knownFraction: finiteNumber.min(0).max(1)
|
|
133
|
+
}).strict();
|
|
134
|
+
const ParetoFigureSpecSchema = z.object({
|
|
135
|
+
kind: z.literal("pareto-cost-quality"),
|
|
136
|
+
split: z.enum(["search", "holdout"]),
|
|
137
|
+
points: z.array(z.object({
|
|
138
|
+
candidateId: nonEmptyString,
|
|
139
|
+
cost: nonNegativeNumber,
|
|
140
|
+
quality: finiteNumber,
|
|
141
|
+
n: nonNegativeInteger,
|
|
142
|
+
onFrontier: z.boolean(),
|
|
143
|
+
gate: z.enum(["promote", "reject"]).optional()
|
|
144
|
+
}).strict()),
|
|
145
|
+
axes: z.object({
|
|
146
|
+
x: z.literal("costUsd"),
|
|
147
|
+
y: z.literal("score")
|
|
148
|
+
}).strict()
|
|
149
|
+
}).strict();
|
|
150
|
+
const ContinuousAgreementSchema = z.object({
|
|
151
|
+
weightedKappa: finiteNumber,
|
|
152
|
+
icc: finiteNumber,
|
|
153
|
+
pearson: finiteNumber,
|
|
154
|
+
spearman: finiteNumber,
|
|
155
|
+
ci: z.object({
|
|
156
|
+
icc: z.tuple([finiteNumber, finiteNumber]),
|
|
157
|
+
weightedKappa: z.tuple([finiteNumber, finiteNumber])
|
|
158
|
+
}).strict(),
|
|
159
|
+
n: nonNegativeInteger,
|
|
160
|
+
raters: nonNegativeInteger
|
|
161
|
+
}).strict();
|
|
162
|
+
const JudgeInsightSchema = z.object({
|
|
163
|
+
n: nonNegativeInteger,
|
|
164
|
+
meanScore: finiteNumber,
|
|
165
|
+
calibration: ContinuousAgreementSchema.optional(),
|
|
166
|
+
positionalBias: finiteNumber.optional(),
|
|
167
|
+
selfPreference: finiteNumber.optional(),
|
|
168
|
+
verbosityBias: finiteNumber.optional()
|
|
169
|
+
}).strict();
|
|
170
|
+
const InterRaterInsightSchema = z.object({
|
|
171
|
+
raters: nonNegativeInteger,
|
|
172
|
+
jointlyRated: nonNegativeInteger,
|
|
173
|
+
kappa: finiteNumber,
|
|
174
|
+
icc: finiteNumber,
|
|
175
|
+
pearson: finiteNumber,
|
|
176
|
+
spearman: finiteNumber,
|
|
177
|
+
perPair: z.record(z.string(), finiteNumber),
|
|
178
|
+
disagreementCases: z.array(z.object({
|
|
179
|
+
runId: nonEmptyString,
|
|
180
|
+
ratings: z.array(z.object({
|
|
181
|
+
rater: nonEmptyString,
|
|
182
|
+
score: finiteNumber
|
|
183
|
+
}).strict()),
|
|
184
|
+
range: nonNegativeNumber
|
|
185
|
+
}).strict())
|
|
186
|
+
}).strict();
|
|
187
|
+
const LiftInsightSchema = z.object({
|
|
188
|
+
baselineMean: finiteNumber,
|
|
189
|
+
candidateMean: finiteNumber,
|
|
190
|
+
delta: finiteNumber,
|
|
191
|
+
ci95: z.tuple([finiteNumber, finiteNumber]),
|
|
192
|
+
pValue: finiteNumber.min(0).max(1),
|
|
193
|
+
n: nonNegativeInteger,
|
|
194
|
+
unpairedBaseline: nonNegativeInteger,
|
|
195
|
+
unpairedCandidate: nonNegativeInteger,
|
|
196
|
+
cohensD: finiteNumber.nullable(),
|
|
197
|
+
mde: nonNegativeNumber,
|
|
198
|
+
requiredN: nonNegativeInteger.nullable()
|
|
199
|
+
}).strict();
|
|
200
|
+
const FailureClusterInsightSchema = z.object({
|
|
201
|
+
clusters: z.array(z.object({
|
|
202
|
+
id: nonEmptyString,
|
|
203
|
+
name: nonEmptyString,
|
|
204
|
+
share: finiteNumber.min(0).max(1),
|
|
205
|
+
exemplars: z.array(nonEmptyString).max(5),
|
|
206
|
+
suggestedFix: nonEmptyString.optional()
|
|
207
|
+
}).strict()),
|
|
208
|
+
totalFailures: nonNegativeInteger
|
|
209
|
+
}).strict();
|
|
210
|
+
const ContaminationInsightSchema = z.object({
|
|
211
|
+
leaks: nonNegativeInteger,
|
|
212
|
+
holdoutAuditPassed: z.boolean(),
|
|
213
|
+
details: z.array(z.object({
|
|
214
|
+
runId: nonEmptyString,
|
|
215
|
+
canary: nonEmptyString,
|
|
216
|
+
matched: nonEmptyString
|
|
217
|
+
}).strict()).optional()
|
|
218
|
+
}).strict();
|
|
219
|
+
const OutcomeCorrelationInsightSchema = z.object({
|
|
220
|
+
metric: nonEmptyString,
|
|
221
|
+
n: nonNegativeInteger,
|
|
222
|
+
pearson: finiteNumber,
|
|
223
|
+
spearman: finiteNumber,
|
|
224
|
+
rewardModel: z.object({
|
|
225
|
+
intercept: finiteNumber,
|
|
226
|
+
slope: finiteNumber,
|
|
227
|
+
r2: finiteNumber
|
|
228
|
+
}).strict().optional()
|
|
229
|
+
}).strict();
|
|
230
|
+
const ReleaseSummarySchema = z.object({
|
|
231
|
+
status: z.enum([
|
|
232
|
+
"pass",
|
|
233
|
+
"warn",
|
|
234
|
+
"fail"
|
|
235
|
+
]),
|
|
236
|
+
axes: z.array(z.object({
|
|
237
|
+
name: z.enum([
|
|
238
|
+
"quality-lift",
|
|
239
|
+
"contamination",
|
|
240
|
+
"composite-distribution"
|
|
241
|
+
]),
|
|
242
|
+
status: z.enum([
|
|
243
|
+
"pass",
|
|
244
|
+
"warn",
|
|
245
|
+
"fail",
|
|
246
|
+
"not_evaluated"
|
|
247
|
+
]),
|
|
248
|
+
detail: nonEmptyString
|
|
249
|
+
}).strict()),
|
|
250
|
+
issues: z.array(z.string())
|
|
251
|
+
}).strict();
|
|
252
|
+
const MetricDeltaSchema = z.object({
|
|
253
|
+
current: finiteNumber,
|
|
254
|
+
baseline: finiteNumber,
|
|
255
|
+
delta: finiteNumber,
|
|
256
|
+
ci95: z.tuple([finiteNumber, finiteNumber]),
|
|
257
|
+
pValue: finiteNumber.min(0).max(1),
|
|
258
|
+
cohensD: finiteNumber,
|
|
259
|
+
baselineN: nonNegativeInteger,
|
|
260
|
+
currentN: nonNegativeInteger,
|
|
261
|
+
significant: z.boolean()
|
|
262
|
+
}).strict();
|
|
263
|
+
const PriorPeriodComparisonSchema = z.object({
|
|
264
|
+
baselineN: nonNegativeInteger,
|
|
265
|
+
currentN: nonNegativeInteger,
|
|
266
|
+
windowLabel: nonEmptyString.optional(),
|
|
267
|
+
metrics: z.record(z.string(), MetricDeltaSchema),
|
|
268
|
+
regressedMetrics: z.array(nonEmptyString),
|
|
269
|
+
improvedMetrics: z.array(nonEmptyString)
|
|
270
|
+
}).strict();
|
|
271
|
+
const RecommendationSchema = z.object({
|
|
272
|
+
priority: z.enum([
|
|
273
|
+
"critical",
|
|
274
|
+
"high",
|
|
275
|
+
"medium",
|
|
276
|
+
"low"
|
|
277
|
+
]),
|
|
278
|
+
kind: z.enum([
|
|
279
|
+
"ship",
|
|
280
|
+
"hold",
|
|
281
|
+
"investigate",
|
|
282
|
+
"fix",
|
|
283
|
+
"recalibrate",
|
|
284
|
+
"expand-corpus"
|
|
285
|
+
]),
|
|
286
|
+
title: nonEmptyString,
|
|
287
|
+
detail: nonEmptyString,
|
|
288
|
+
evidencePath: nonEmptyString.optional()
|
|
289
|
+
}).strict();
|
|
290
|
+
const InsightReportSchema = z.object({
|
|
291
|
+
n: nonNegativeInteger,
|
|
292
|
+
execution: ExecutionInsightSchema,
|
|
293
|
+
composite: ScalarDistributionSchema,
|
|
294
|
+
perDimension: z.record(z.string(), ScalarDistributionSchema),
|
|
295
|
+
costQuality: z.object({
|
|
296
|
+
cost: ScalarDistributionSchema,
|
|
297
|
+
pareto: ParetoFigureSpecSchema,
|
|
298
|
+
provenance: CostProvenanceSummarySchema.optional(),
|
|
299
|
+
degraded: z.object({
|
|
300
|
+
cost: nonEmptyString.optional(),
|
|
301
|
+
pareto: nonEmptyString.optional()
|
|
302
|
+
}).strict().optional()
|
|
303
|
+
}).strict(),
|
|
304
|
+
judges: z.record(z.string(), JudgeInsightSchema),
|
|
305
|
+
interRater: InterRaterInsightSchema.optional(),
|
|
306
|
+
lift: LiftInsightSchema.optional(),
|
|
307
|
+
failureClusters: FailureClusterInsightSchema.optional(),
|
|
308
|
+
contamination: ContaminationInsightSchema.optional(),
|
|
309
|
+
outcomeCorrelation: OutcomeCorrelationInsightSchema.optional(),
|
|
310
|
+
release: ReleaseSummarySchema,
|
|
311
|
+
priorPeriodComparison: PriorPeriodComparisonSchema.optional(),
|
|
312
|
+
failureClasses: z.array(z.object({
|
|
313
|
+
failureClass: FailureClassSchema,
|
|
314
|
+
count: nonNegativeInteger,
|
|
315
|
+
share: finiteNumber.min(0).max(1)
|
|
316
|
+
}).strict()).optional(),
|
|
317
|
+
recommendations: z.array(RecommendationSchema)
|
|
318
|
+
}).strict();
|
|
319
|
+
const sha256Digest = z.custom((value) => typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value), "expected sha256:<64 lowercase hex characters>");
|
|
320
|
+
const MutableSurfaceSchema = z.union([
|
|
321
|
+
z.string(),
|
|
322
|
+
z.object({
|
|
323
|
+
kind: z.literal("components"),
|
|
324
|
+
components: z.record(z.string(), z.string())
|
|
325
|
+
}).strict(),
|
|
326
|
+
z.object({
|
|
327
|
+
kind: z.literal("code"),
|
|
328
|
+
worktreeRef: nonEmptyString,
|
|
329
|
+
baseRef: nonEmptyString,
|
|
330
|
+
baseCommit: nonEmptyString,
|
|
331
|
+
baseTree: nonEmptyString,
|
|
332
|
+
candidateCommit: nonEmptyString,
|
|
333
|
+
candidateTree: nonEmptyString,
|
|
334
|
+
patch: z.object({
|
|
335
|
+
format: z.literal("git-diff-binary"),
|
|
336
|
+
sha256: sha256Digest,
|
|
337
|
+
byteLength: nonNegativeInteger
|
|
338
|
+
}).strict(),
|
|
339
|
+
summary: z.string().optional()
|
|
340
|
+
}).strict()
|
|
341
|
+
]);
|
|
342
|
+
const RunTerminalOutcomeSchema = z.enum([
|
|
343
|
+
"succeeded",
|
|
344
|
+
"failed",
|
|
345
|
+
"cancelled",
|
|
346
|
+
"incomplete",
|
|
347
|
+
"unknown"
|
|
348
|
+
]);
|
|
349
|
+
const EvalRunCellScoreSchema = z.object({
|
|
350
|
+
scenarioId: nonEmptyString,
|
|
351
|
+
rep: nonNegativeInteger,
|
|
352
|
+
compositeMean: finiteNumber.nullable(),
|
|
353
|
+
dimensions: z.record(z.string(), z.record(z.string(), finiteNumber)),
|
|
354
|
+
terminalOutcome: RunTerminalOutcomeSchema,
|
|
355
|
+
executionErrorCount: nonNegativeInteger.nullable(),
|
|
356
|
+
errorMessage: z.string().optional()
|
|
357
|
+
}).strict();
|
|
358
|
+
const EvalRunGenerationSnapshotSchema = z.object({
|
|
359
|
+
index: nonNegativeInteger,
|
|
360
|
+
surfaceHash: nonEmptyString,
|
|
361
|
+
surface: MutableSurfaceSchema.optional(),
|
|
362
|
+
cells: z.array(EvalRunCellScoreSchema),
|
|
363
|
+
compositeMean: finiteNumber.nullable(),
|
|
364
|
+
costUsd: nonNegativeNumber,
|
|
365
|
+
durationMs: nonNegativeNumber
|
|
366
|
+
}).strict();
|
|
367
|
+
const EvalRunStatusSchema = z.enum([
|
|
368
|
+
"started",
|
|
369
|
+
"baseline-complete",
|
|
370
|
+
"generation-complete",
|
|
371
|
+
"gate-decided",
|
|
372
|
+
"finished",
|
|
373
|
+
"errored"
|
|
374
|
+
]);
|
|
375
|
+
const GateDecisionSchema = z.enum([
|
|
376
|
+
"ship",
|
|
377
|
+
"hold",
|
|
378
|
+
"need_more_work",
|
|
379
|
+
"model_ceiling",
|
|
380
|
+
"arch_ceiling"
|
|
381
|
+
]);
|
|
382
|
+
const EvalRunEventSchema = z.object({
|
|
383
|
+
runId: nonEmptyString,
|
|
384
|
+
runDir: nonEmptyString,
|
|
385
|
+
timestamp: z.string().datetime({ offset: true }),
|
|
386
|
+
status: EvalRunStatusSchema,
|
|
387
|
+
labels: z.record(z.string(), z.string()),
|
|
388
|
+
baseline: EvalRunGenerationSnapshotSchema.optional(),
|
|
389
|
+
generations: z.array(EvalRunGenerationSnapshotSchema),
|
|
390
|
+
gateDecision: GateDecisionSchema.optional(),
|
|
391
|
+
holdoutLift: finiteNumber.optional(),
|
|
392
|
+
totalCostUsd: nonNegativeNumber,
|
|
393
|
+
totalDurationMs: nonNegativeNumber,
|
|
394
|
+
errorMessage: z.string().optional(),
|
|
395
|
+
insightReport: InsightReportSchema.optional()
|
|
396
|
+
}).strict().superRefine((event, ctx) => {
|
|
397
|
+
if (event.baseline && event.baseline.index !== 0) ctx.addIssue({
|
|
398
|
+
code: "custom",
|
|
399
|
+
path: ["baseline", "index"],
|
|
400
|
+
message: "baseline index must be 0"
|
|
401
|
+
});
|
|
402
|
+
const seen = /* @__PURE__ */ new Set();
|
|
403
|
+
for (let i = 0; i < event.generations.length; i++) {
|
|
404
|
+
const index = event.generations[i].index;
|
|
405
|
+
if (seen.has(index)) ctx.addIssue({
|
|
406
|
+
code: "custom",
|
|
407
|
+
path: [
|
|
408
|
+
"generations",
|
|
409
|
+
i,
|
|
410
|
+
"index"
|
|
411
|
+
],
|
|
412
|
+
message: `duplicate generation index ${index}`
|
|
413
|
+
});
|
|
414
|
+
seen.add(index);
|
|
415
|
+
}
|
|
416
|
+
if (event.status === "errored" && !event.errorMessage?.trim()) ctx.addIssue({
|
|
417
|
+
code: "custom",
|
|
418
|
+
path: ["errorMessage"],
|
|
419
|
+
message: "errorMessage is required when status is errored"
|
|
420
|
+
});
|
|
421
|
+
});
|
|
422
|
+
const TraceSpanEventEntrySchema = z.object({
|
|
423
|
+
timeUnixNano: UnixNanoTimestampSchema,
|
|
424
|
+
name: nonEmptyString,
|
|
425
|
+
attributes: attributes.optional()
|
|
426
|
+
}).strict();
|
|
427
|
+
const TraceSpanEventSchema = z.object({
|
|
428
|
+
traceId: nonEmptyString,
|
|
429
|
+
spanId: nonEmptyString,
|
|
430
|
+
parentSpanId: nonEmptyString.optional(),
|
|
431
|
+
name: nonEmptyString,
|
|
432
|
+
startTimeUnixNano: UnixNanoTimestampSchema,
|
|
433
|
+
endTimeUnixNano: UnixNanoTimestampSchema,
|
|
434
|
+
attributes,
|
|
435
|
+
events: z.array(TraceSpanEventEntrySchema).optional(),
|
|
436
|
+
status: z.object({
|
|
437
|
+
code: z.enum([
|
|
438
|
+
"OK",
|
|
439
|
+
"ERROR",
|
|
440
|
+
"UNSET"
|
|
441
|
+
]),
|
|
442
|
+
message: z.string().optional()
|
|
443
|
+
}).strict().optional(),
|
|
444
|
+
"tangle.runId": nonEmptyString.optional(),
|
|
445
|
+
"tangle.generation": nonNegativeInteger.optional(),
|
|
446
|
+
"tangle.cellId": nonEmptyString.optional(),
|
|
447
|
+
"tangle.scenarioId": nonEmptyString.optional()
|
|
448
|
+
}).strict().refine((span) => BigInt(span.endTimeUnixNano) >= BigInt(span.startTimeUnixNano), {
|
|
449
|
+
path: ["endTimeUnixNano"],
|
|
450
|
+
message: "endTimeUnixNano must be greater than or equal to startTimeUnixNano"
|
|
451
|
+
});
|
|
452
|
+
z.object({
|
|
453
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
454
|
+
events: z.array(z.unknown())
|
|
455
|
+
}).strict();
|
|
456
|
+
z.object({
|
|
457
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
458
|
+
spans: z.array(z.unknown())
|
|
459
|
+
}).strict();
|
|
460
|
+
const IngestEvalRunsRequestSchema = z.object({
|
|
461
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
462
|
+
events: z.array(EvalRunEventSchema)
|
|
463
|
+
}).strict();
|
|
464
|
+
const IngestTracesRequestSchema = z.object({
|
|
465
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
466
|
+
spans: z.array(TraceSpanEventSchema)
|
|
467
|
+
}).strict();
|
|
468
|
+
const IngestResponseSchema = z.object({
|
|
469
|
+
accepted: nonNegativeInteger,
|
|
470
|
+
rejected: z.array(z.object({
|
|
471
|
+
index: nonNegativeInteger,
|
|
472
|
+
reason: nonEmptyString
|
|
473
|
+
}).strict())
|
|
474
|
+
}).strict();
|
|
475
|
+
//#endregion
|
|
476
|
+
//#region src/hosted/client.ts
|
|
477
|
+
/**
|
|
478
|
+
* # Hosted-tier ingest client.
|
|
479
|
+
*
|
|
480
|
+
* Ships eval-run events + trace spans to any orchestrator (ours, a
|
|
481
|
+
* partner's self-hosted one, or a future open implementation) that
|
|
482
|
+
* speaks the wire format in `./types.ts`.
|
|
483
|
+
*
|
|
484
|
+
* Three modes:
|
|
485
|
+
* - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —
|
|
486
|
+
* the client appends the versioned `/v1/ingest/...` path itself; a trailing
|
|
487
|
+
* `/v1` on the endpoint is tolerated and normalized away). We handle ingest
|
|
488
|
+
* + storage + dashboard.
|
|
489
|
+
* - **Self-hosted:** point at whatever URL runs the reference receiver
|
|
490
|
+
* from `examples/hosted-ingest-server/`.
|
|
491
|
+
* - **Off (default):** when `hostedTenant` is unset, nothing is sent.
|
|
492
|
+
* Everything stays local.
|
|
493
|
+
*/
|
|
494
|
+
const MAX_IDEMPOTENCY_KEY_LENGTH = 256;
|
|
495
|
+
function sleep(ms) {
|
|
496
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
497
|
+
}
|
|
498
|
+
function normalizeHostedBase(endpoint) {
|
|
499
|
+
return endpoint.trim().replace(/\/+$/, "").replace(/\/v1$/, "");
|
|
500
|
+
}
|
|
501
|
+
function resolveIdempotencyKey(key) {
|
|
502
|
+
const resolved = key ?? globalThis.crypto.randomUUID();
|
|
503
|
+
if (resolved.trim().length === 0) throw new Error("idempotency key must not be blank");
|
|
504
|
+
if (resolved.length > MAX_IDEMPOTENCY_KEY_LENGTH) throw new Error(`idempotency key must be at most ${MAX_IDEMPOTENCY_KEY_LENGTH} characters`);
|
|
505
|
+
return resolved;
|
|
506
|
+
}
|
|
507
|
+
function responseValidationReason(error) {
|
|
508
|
+
return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.map(String).join(".") : "value"}: ${issue.message}`).join("; ");
|
|
509
|
+
}
|
|
510
|
+
async function post(tenant, path, body, opts = {}) {
|
|
511
|
+
const timeoutMs = tenant.timeoutMs ?? 3e4;
|
|
512
|
+
const maxRetries = tenant.retries ?? 2;
|
|
513
|
+
const f = tenant.fetchImpl ?? ((...args) => fetch(...args));
|
|
514
|
+
const url = `${normalizeHostedBase(tenant.endpoint)}${path}`;
|
|
515
|
+
const idempotencyKey = resolveIdempotencyKey(opts.idempotencyKey);
|
|
516
|
+
const headers = {
|
|
517
|
+
"content-type": "application/json",
|
|
518
|
+
authorization: `Bearer ${tenant.apiKey}`,
|
|
519
|
+
"x-tangle-tenant-id": tenant.tenantId,
|
|
520
|
+
"x-tangle-wire-version": HOSTED_WIRE_VERSION,
|
|
521
|
+
"idempotency-key": idempotencyKey
|
|
522
|
+
};
|
|
523
|
+
let lastError;
|
|
524
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
525
|
+
const ourTimeout = AbortSignal.timeout(timeoutMs);
|
|
526
|
+
const combinedSignal = opts.signal ? AbortSignal.any([opts.signal, ourTimeout]) : ourTimeout;
|
|
527
|
+
let res;
|
|
528
|
+
try {
|
|
529
|
+
res = await f(url, {
|
|
530
|
+
method: "POST",
|
|
531
|
+
headers,
|
|
532
|
+
body: JSON.stringify(body),
|
|
533
|
+
signal: combinedSignal
|
|
534
|
+
});
|
|
535
|
+
} catch (err) {
|
|
536
|
+
if (opts.signal?.aborted) throw err;
|
|
537
|
+
lastError = err;
|
|
538
|
+
if (attempt === maxRetries) throw err;
|
|
539
|
+
await sleep(2 ** attempt * 200 + Math.random() * 200);
|
|
540
|
+
continue;
|
|
541
|
+
}
|
|
542
|
+
if (!res.ok) {
|
|
543
|
+
const text = await res.text().catch(() => "");
|
|
544
|
+
const error = /* @__PURE__ */ new Error(`hosted ingest ${url} failed (${res.status}): ${text.slice(0, 500)}`);
|
|
545
|
+
if (!(res.status >= 500 || res.status === 408 || res.status === 429) || attempt === maxRetries) throw error;
|
|
546
|
+
lastError = error;
|
|
547
|
+
await sleep(2 ** attempt * 200 + Math.random() * 200);
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
let rawResponse;
|
|
551
|
+
try {
|
|
552
|
+
rawResponse = await res.json();
|
|
553
|
+
} catch (error) {
|
|
554
|
+
throw new Error(`hosted ingest ${url} returned invalid JSON`, { cause: error });
|
|
555
|
+
}
|
|
556
|
+
const parsed = IngestResponseSchema.safeParse(rawResponse);
|
|
557
|
+
if (!parsed.success) throw new Error(`hosted ingest ${url} returned an invalid response: ${responseValidationReason(parsed.error)}`);
|
|
558
|
+
return parsed.data;
|
|
559
|
+
}
|
|
560
|
+
throw lastError ?? /* @__PURE__ */ new Error("hosted ingest exhausted retries");
|
|
561
|
+
}
|
|
562
|
+
function createHostedClient(tenant) {
|
|
563
|
+
if (normalizeHostedBase(tenant.endpoint).length === 0) throw new Error("endpoint is required");
|
|
564
|
+
if (tenant.apiKey.trim().length === 0) throw new Error("apiKey is required");
|
|
565
|
+
if (tenant.tenantId.trim().length === 0) throw new Error("tenantId is required");
|
|
566
|
+
if (tenant.timeoutMs !== void 0 && (!Number.isFinite(tenant.timeoutMs) || tenant.timeoutMs <= 0)) throw new Error("timeoutMs must be greater than 0");
|
|
567
|
+
if (tenant.retries !== void 0 && (!Number.isInteger(tenant.retries) || tenant.retries < 0)) throw new Error("retries must be a non-negative integer");
|
|
568
|
+
return {
|
|
569
|
+
tenant,
|
|
570
|
+
wireVersion: HOSTED_WIRE_VERSION,
|
|
571
|
+
async ingestEvalRun(event, idempotencyKey) {
|
|
572
|
+
return this.ingestEvalRuns([event], idempotencyKey);
|
|
573
|
+
},
|
|
574
|
+
async ingestEvalRuns(events, idempotencyKey) {
|
|
575
|
+
return post(tenant, "/v1/ingest/eval-runs", IngestEvalRunsRequestSchema.parse({
|
|
576
|
+
wireVersion: HOSTED_WIRE_VERSION,
|
|
577
|
+
events
|
|
578
|
+
}), { idempotencyKey });
|
|
579
|
+
},
|
|
580
|
+
async ingestTraces(spans, idempotencyKey) {
|
|
581
|
+
return post(tenant, "/v1/ingest/traces", IngestTracesRequestSchema.parse({
|
|
582
|
+
wireVersion: HOSTED_WIRE_VERSION,
|
|
583
|
+
spans
|
|
584
|
+
}), { idempotencyKey });
|
|
585
|
+
}
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
/**
|
|
589
|
+
* Build a `HostedClient` from environment, or `undefined` when ingest is not
|
|
590
|
+
* configured — the canonical, fail-soft wiring every product uses so eval-run +
|
|
591
|
+
* trace provenance lands in the Intelligence dashboard with ONE call:
|
|
592
|
+
*
|
|
593
|
+
* const hosted = hostedClientFromEnv()
|
|
594
|
+
* // ...run the loop...
|
|
595
|
+
* await emitLoopProvenance({ ..., hostedClient: hosted }) // no-op if undefined
|
|
596
|
+
*
|
|
597
|
+
* Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is
|
|
598
|
+
* missing — so a product wires the ship call unconditionally and it stays a
|
|
599
|
+
* no-op until the env is set. Env precedence:
|
|
600
|
+
* - endpoint: `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`
|
|
601
|
+
* - apiKey: `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`
|
|
602
|
+
* - tenantId: `TANGLE_TENANT_ID`
|
|
603
|
+
* A trailing slash on the endpoint is stripped. Pass `overrides` to supply any
|
|
604
|
+
* field directly (e.g. a fixed `tenantId` per product) — overrides win over env.
|
|
605
|
+
*/
|
|
606
|
+
/**
|
|
607
|
+
* Build a {@link HostedTenant} config from env — the input `selfImprove`'s
|
|
608
|
+
* `hostedTenant` and `emitLoopProvenance` take. Same env precedence + overrides
|
|
609
|
+
* as {@link hostedClientFromEnv}; returns `undefined` (not an error) when any of
|
|
610
|
+
* endpoint / apiKey / tenantId is missing, so a product wires
|
|
611
|
+
* `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally
|
|
612
|
+
* and it stays off until the env is set.
|
|
613
|
+
*/
|
|
614
|
+
function hostedTenantFromEnv(overrides = {}) {
|
|
615
|
+
const env = overrides.env ?? process.env;
|
|
616
|
+
const endpoint = (overrides.endpoint ?? env.TANGLE_INGEST_URL ?? env.TANGLE_ORCHESTRATOR_URL)?.trim();
|
|
617
|
+
const apiKey = (overrides.apiKey ?? env.TANGLE_INGEST_API_KEY ?? env.TANGLE_API_KEY)?.trim();
|
|
618
|
+
const tenantId = (overrides.tenantId ?? env.TANGLE_TENANT_ID)?.trim();
|
|
619
|
+
if (!endpoint || !apiKey || !tenantId) return void 0;
|
|
620
|
+
const tenant = {
|
|
621
|
+
endpoint: endpoint.replace(/\/+$/, ""),
|
|
622
|
+
apiKey,
|
|
623
|
+
tenantId
|
|
624
|
+
};
|
|
625
|
+
if (overrides.fetchImpl) tenant.fetchImpl = overrides.fetchImpl;
|
|
626
|
+
if (overrides.timeoutMs !== void 0) tenant.timeoutMs = overrides.timeoutMs;
|
|
627
|
+
if (overrides.retries !== void 0) tenant.retries = overrides.retries;
|
|
628
|
+
return tenant;
|
|
629
|
+
}
|
|
630
|
+
function hostedClientFromEnv(overrides = {}) {
|
|
631
|
+
const tenant = hostedTenantFromEnv(overrides);
|
|
632
|
+
return tenant ? createHostedClient(tenant) : void 0;
|
|
633
|
+
}
|
|
634
|
+
//#endregion
|
|
635
|
+
export { EvalRunEventSchema as a, IngestResponseSchema as c, MutableSurfaceSchema as d, RunTerminalOutcomeSchema as f, HOSTED_WIRE_VERSION as h, EvalRunCellScoreSchema as i, IngestTracesRequestSchema as l, UnixNanoTimestampSchema as m, hostedClientFromEnv as n, EvalRunGenerationSnapshotSchema as o, TraceSpanEventSchema as p, hostedTenantFromEnv as r, IngestEvalRunsRequestSchema as s, createHostedClient as t, InsightReportSchema as u };
|
|
636
|
+
|
|
637
|
+
//# sourceMappingURL=client-CYzbdJOZ.js.map
|