@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/wire/index.d.ts
CHANGED
|
@@ -1,859 +1,111 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
inputTokens: number;
|
|
10
|
-
/** Includes reasoning tokens when the provider bills them as output. */
|
|
11
|
-
outputTokens: number;
|
|
12
|
-
/** Reasoning-token subset of outputTokens, when reported. */
|
|
13
|
-
reasoningTokens?: number;
|
|
14
|
-
/** Prompt tokens served from a provider cache. */
|
|
15
|
-
cachedTokens?: number;
|
|
16
|
-
/** Prompt tokens written into a provider cache. */
|
|
17
|
-
cacheWriteTokens?: number;
|
|
18
|
-
}
|
|
19
|
-
interface CostCallBase {
|
|
20
|
-
callId: string;
|
|
21
|
-
channel: CostChannel;
|
|
22
|
-
phase: string;
|
|
23
|
-
actor: string;
|
|
24
|
-
model: string;
|
|
25
|
-
maximumCostUsd?: number;
|
|
26
|
-
tags?: Record<string, string>;
|
|
27
|
-
timestamp: number;
|
|
28
|
-
}
|
|
29
|
-
interface PendingCostCall extends CostCallBase {
|
|
30
|
-
status: 'pending';
|
|
31
|
-
}
|
|
32
|
-
interface PendingCostCallView extends PendingCostCall {
|
|
33
|
-
state: 'active' | 'late' | 'interrupted';
|
|
34
|
-
}
|
|
35
|
-
interface CostReceipt extends CostCallBase, CostUsage {
|
|
36
|
-
status: 'settled';
|
|
37
|
-
costUsd: number;
|
|
38
|
-
costUnknown: boolean;
|
|
39
|
-
usageUnknown?: boolean;
|
|
40
|
-
/** Rates used to estimate cost locally. Absent when cost is provider-reported or unknown. */
|
|
41
|
-
pricing?: {
|
|
42
|
-
inputUsdPerThousand: number;
|
|
43
|
-
cachedInputUsdPerThousand?: number;
|
|
44
|
-
cacheWriteUsdPerThousand?: number;
|
|
45
|
-
outputUsdPerThousand: number;
|
|
46
|
-
};
|
|
47
|
-
/** Cost reported by the provider, not a local token-price calculation. */
|
|
48
|
-
actualCostUsd?: number;
|
|
49
|
-
error?: string;
|
|
50
|
-
}
|
|
51
|
-
interface CostReceiptInput extends CostUsage {
|
|
52
|
-
model: string;
|
|
53
|
-
/** Caller-supplied rates for a local estimate when the provider does not report billed cost. */
|
|
54
|
-
customTokenPricing?: CustomTokenPricing;
|
|
55
|
-
actualCostUsd?: number;
|
|
56
|
-
costUnknown?: boolean;
|
|
57
|
-
usageUnknown?: boolean;
|
|
58
|
-
}
|
|
59
|
-
/** Per-million token rates for a model or endpoint not covered by package pricing. */
|
|
60
|
-
interface CustomTokenPricing {
|
|
61
|
-
/** Non-cached input tokens. */
|
|
62
|
-
inputUsdPerMillion: number;
|
|
63
|
-
/** Cache-read tokens. Falls back to the normal input rate when omitted. */
|
|
64
|
-
cachedInputUsdPerMillion?: number;
|
|
65
|
-
/** Cache-creation or cache-write tokens. Falls back to the normal input rate when omitted. */
|
|
66
|
-
cacheWriteUsdPerMillion?: number;
|
|
67
|
-
outputUsdPerMillion: number;
|
|
68
|
-
}
|
|
69
|
-
type MaximumCharge = {
|
|
70
|
-
externallyEnforcedMaximumUsd: number;
|
|
71
|
-
} | ({
|
|
72
|
-
customTokenPricing: CustomTokenPricing;
|
|
73
|
-
} & Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>) | ({
|
|
74
|
-
model: string;
|
|
75
|
-
} & CostUsage);
|
|
76
|
-
interface RunPaidCallInput<T> {
|
|
77
|
-
callId?: string;
|
|
78
|
-
channel: CostChannel;
|
|
79
|
-
phase: string;
|
|
80
|
-
actor: string;
|
|
81
|
-
/** Used before a provider receipt exists and on failures without one. */
|
|
82
|
-
model?: string;
|
|
83
|
-
tags?: Record<string, string>;
|
|
84
|
-
signal?: AbortSignal;
|
|
85
|
-
/** Provider-enforced dollar maximum, or maximum token usage with known pricing. Required when capped. */
|
|
86
|
-
maximumCharge?: MaximumCharge;
|
|
87
|
-
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
88
|
-
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
89
|
-
receipt(value: T): CostReceiptInput;
|
|
90
|
-
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
91
|
-
}
|
|
92
|
-
type PaidCallResult<T> = {
|
|
93
|
-
succeeded: true;
|
|
94
|
-
callId: string;
|
|
95
|
-
value: T;
|
|
96
|
-
receipt: CostReceipt;
|
|
97
|
-
} | {
|
|
98
|
-
succeeded: false;
|
|
99
|
-
callId?: string;
|
|
100
|
-
error: Error;
|
|
101
|
-
receipt?: CostReceipt;
|
|
102
|
-
};
|
|
103
|
-
interface ChannelRollup {
|
|
104
|
-
channel: CostChannel;
|
|
105
|
-
calls: number;
|
|
106
|
-
inputTokens: number;
|
|
107
|
-
outputTokens: number;
|
|
108
|
-
reasoningTokens?: number;
|
|
109
|
-
cachedTokens: number;
|
|
110
|
-
cacheWriteTokens?: number;
|
|
111
|
-
costUsd: number;
|
|
112
|
-
unpricedCalls: number;
|
|
113
|
-
unknownUsageCalls: number;
|
|
114
|
-
}
|
|
115
|
-
interface CostLedgerSummary {
|
|
116
|
-
totalCalls: number;
|
|
117
|
-
pendingCalls: number;
|
|
118
|
-
unresolvedCalls: number;
|
|
119
|
-
reservedCostUsd: number;
|
|
120
|
-
inputTokens: number;
|
|
121
|
-
outputTokens: number;
|
|
122
|
-
reasoningTokens?: number;
|
|
123
|
-
cachedTokens: number;
|
|
124
|
-
cacheWriteTokens?: number;
|
|
125
|
-
totalCostUsd: number;
|
|
126
|
-
byChannel: ChannelRollup[];
|
|
127
|
-
unpricedModels: string[];
|
|
128
|
-
fullyPriced: boolean;
|
|
129
|
-
usageComplete: boolean;
|
|
130
|
-
accountingComplete: boolean;
|
|
131
|
-
incompleteReasons: string[];
|
|
132
|
-
}
|
|
133
|
-
interface CostLedgerFilter {
|
|
134
|
-
channel?: CostChannel;
|
|
135
|
-
phase?: string;
|
|
136
|
-
tags?: Record<string, string>;
|
|
137
|
-
}
|
|
138
|
-
interface CostLedgerWaitOptions {
|
|
139
|
-
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
140
|
-
timeoutMs?: number;
|
|
141
|
-
/** Wait only for calls matching this attribution filter. */
|
|
142
|
-
filter?: CostLedgerFilter;
|
|
143
|
-
}
|
|
144
|
-
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
145
|
-
interface CostLedgerPersistence {
|
|
146
|
-
read(): {
|
|
147
|
-
revision: string;
|
|
148
|
-
events: string;
|
|
149
|
-
};
|
|
150
|
-
append(expectedRevision: string, event: string): string | undefined;
|
|
151
|
-
}
|
|
152
|
-
interface CostLedgerOptions {
|
|
153
|
-
costCeilingUsd?: number;
|
|
154
|
-
persistence?: CostLedgerPersistence;
|
|
155
|
-
/** Import already-settled receipts without admitting new paid work. */
|
|
156
|
-
receipts?: readonly CostReceipt[];
|
|
157
|
-
}
|
|
158
|
-
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
159
|
-
declare class CostLedger {
|
|
160
|
-
private readonly records;
|
|
161
|
-
private readonly activeCallIds;
|
|
162
|
-
private readonly lateCallIds;
|
|
163
|
-
private readonly idleWaiters;
|
|
164
|
-
private completedTasks;
|
|
165
|
-
private revision;
|
|
166
|
-
private costLimitPersisted;
|
|
167
|
-
readonly costCeilingUsd?: number;
|
|
168
|
-
private readonly persistence?;
|
|
169
|
-
constructor(input?: number | CostLedgerOptions);
|
|
170
|
-
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
171
|
-
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
172
|
-
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
173
|
-
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
174
|
-
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
175
|
-
error?: string;
|
|
176
|
-
}): CostReceipt;
|
|
177
|
-
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
178
|
-
/** Read pending calls without exposing mutable ledger state. */
|
|
179
|
-
listPending(filter?: CostLedgerFilter): PendingCostCallView[];
|
|
180
|
-
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
181
|
-
markCompleted(count?: number): void;
|
|
182
|
-
costPerCompletedTask(): number | null;
|
|
183
|
-
private execute;
|
|
184
|
-
private captureLateOutcome;
|
|
185
|
-
private releaseActiveCall;
|
|
186
|
-
private commitOutcome;
|
|
187
|
-
private captureFailure;
|
|
188
|
-
private commitReceipt;
|
|
189
|
-
private resolveMaximum;
|
|
190
|
-
private hasIncompleteSettledCall;
|
|
191
|
-
private appendRecord;
|
|
192
|
-
private ensureCostLimitPersisted;
|
|
193
|
-
private appendEvent;
|
|
194
|
-
}
|
|
195
|
-
/** Public callback surface for a shared cost ledger.
|
|
196
|
-
*
|
|
197
|
-
* Declaration bundles may expose this type through multiple package subpaths.
|
|
198
|
-
* Keeping callback contracts structural lets those subpaths compose while the
|
|
199
|
-
* concrete {@link CostLedger} retains its private durable state.
|
|
200
|
-
*/
|
|
201
|
-
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'listPending' | 'waitForIdle'>> & Partial<Pick<CostLedger, 'listPending' | 'waitForIdle'>>;
|
|
202
|
-
|
|
203
|
-
type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
|
|
204
|
-
interface BudgetSpec {
|
|
205
|
-
tokens?: number;
|
|
206
|
-
wallMs?: number;
|
|
207
|
-
calls?: number;
|
|
208
|
-
usd?: number;
|
|
209
|
-
}
|
|
210
|
-
interface RunOutcome {
|
|
211
|
-
score?: number;
|
|
212
|
-
pass?: boolean;
|
|
213
|
-
failureClass?: FailureClass;
|
|
214
|
-
notes?: string;
|
|
215
|
-
}
|
|
216
|
-
/**
|
|
217
|
-
* Layer — optional classification in a nested build workflow.
|
|
218
|
-
* `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
|
|
219
|
-
* `app-build`: sandbox harness that compiled + tested the generated scaffold.
|
|
220
|
-
* `app-runtime`: a run of the generated agent against a domain scenario.
|
|
221
|
-
* `meta`: any meta-eval (judge replay, correlation analysis).
|
|
222
|
-
*/
|
|
223
|
-
type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
|
|
224
|
-
interface Run {
|
|
225
|
-
runId: string;
|
|
226
|
-
/**
|
|
227
|
-
* Stable identifier of the scenario being executed.
|
|
228
|
-
*
|
|
229
|
-
* Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
|
|
230
|
-
* input WITHOUT this field, substituting a sensible default
|
|
231
|
-
* (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
|
|
232
|
-
* curated scenario to anchor to (runtime / operator / meta-eval runs). This
|
|
233
|
-
* keeps the persisted shape unambiguous for downstream filters + aggregations
|
|
234
|
-
* while removing the boilerplate of inventing placeholder ids at the call site.
|
|
235
|
-
*/
|
|
236
|
-
scenarioId: string;
|
|
237
|
-
variantId?: string;
|
|
238
|
-
datasetVersion?: string;
|
|
239
|
-
/** Git SHA of agent code at run time. */
|
|
240
|
-
codeSha?: string;
|
|
241
|
-
/** Hash of the prompt template + any system prompt. */
|
|
242
|
-
promptSha?: string;
|
|
243
|
-
/** Model id + date + system-prompt hash, concatenated. */
|
|
244
|
-
modelFingerprint?: string;
|
|
245
|
-
seed?: number;
|
|
246
|
-
/** Arbitrary environment markers (shell, docker version, tz). */
|
|
247
|
-
envFingerprint?: Record<string, string>;
|
|
248
|
-
/** Version of the redaction rules applied to this run. */
|
|
249
|
-
redactionVersion?: string;
|
|
250
|
-
/** Parent run in a nested build workflow. A builder run's children are
|
|
251
|
-
* app-build runs; those children are app-runtime runs. */
|
|
252
|
-
parentRunId?: string;
|
|
253
|
-
/** Stable project identifier — groups runs across chats + sessions. */
|
|
254
|
-
projectId?: string;
|
|
255
|
-
/** Chat/conversation identifier within a project. */
|
|
256
|
-
chatId?: string;
|
|
257
|
-
/** Layer classification — hint for aggregation; not enforced. */
|
|
258
|
-
layer?: RunLayer;
|
|
259
|
-
startedAt: number;
|
|
260
|
-
endedAt?: number;
|
|
261
|
-
status: RunStatus;
|
|
262
|
-
outcome?: RunOutcome;
|
|
263
|
-
budget?: BudgetSpec;
|
|
264
|
-
/** Free-form labels for downstream grouping. */
|
|
265
|
-
tags?: Record<string, string>;
|
|
266
|
-
}
|
|
267
|
-
type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
|
|
268
|
-
type SpanStatus = 'ok' | 'error';
|
|
269
|
-
interface SpanBase {
|
|
270
|
-
spanId: string;
|
|
271
|
-
parentSpanId?: string;
|
|
272
|
-
runId: string;
|
|
273
|
-
kind: SpanKind;
|
|
274
|
-
name: string;
|
|
275
|
-
startedAt: number;
|
|
276
|
-
endedAt?: number;
|
|
277
|
-
status?: SpanStatus;
|
|
278
|
-
error?: string;
|
|
279
|
-
/** Anything not covered by typed fields. Kept deliberately free-form. */
|
|
280
|
-
attributes?: Record<string, unknown>;
|
|
281
|
-
}
|
|
282
|
-
interface Message {
|
|
283
|
-
role: 'system' | 'user' | 'assistant' | 'tool';
|
|
284
|
-
content: string;
|
|
285
|
-
tokens?: number;
|
|
286
|
-
/** Multi-modal content descriptors; blobs themselves live in Artifacts. */
|
|
287
|
-
images?: Array<{
|
|
288
|
-
artifactId?: string;
|
|
289
|
-
url?: string;
|
|
290
|
-
mime?: string;
|
|
291
|
-
}>;
|
|
292
|
-
}
|
|
293
|
-
interface LlmSpan extends SpanBase {
|
|
294
|
-
kind: 'llm';
|
|
295
|
-
model: string;
|
|
296
|
-
messages: Message[];
|
|
297
|
-
output?: string;
|
|
298
|
-
inputTokens?: number;
|
|
299
|
-
/** All generated tokens, including the reasoning subset when present. */
|
|
300
|
-
outputTokens?: number;
|
|
301
|
-
cachedTokens?: number;
|
|
302
|
-
cacheWriteTokens?: number;
|
|
303
|
-
/** Reasoning-token subset of `outputTokens`. */
|
|
304
|
-
reasoningTokens?: number;
|
|
305
|
-
costUsd?: number;
|
|
306
|
-
finishReason?: string;
|
|
307
|
-
}
|
|
308
|
-
interface ToolSpan extends SpanBase {
|
|
309
|
-
kind: 'tool';
|
|
310
|
-
toolName: string;
|
|
311
|
-
args: unknown;
|
|
312
|
-
/** False when the source observed the call but did not capture its arguments. */
|
|
313
|
-
argsCaptured?: boolean;
|
|
314
|
-
result?: unknown;
|
|
315
|
-
latencyMs?: number;
|
|
316
|
-
}
|
|
317
|
-
interface RetrievalSpan extends SpanBase {
|
|
318
|
-
kind: 'retrieval';
|
|
319
|
-
query: string;
|
|
320
|
-
hits: Array<{
|
|
321
|
-
docId: string;
|
|
322
|
-
score: number;
|
|
323
|
-
content?: string;
|
|
324
|
-
}>;
|
|
325
|
-
}
|
|
326
|
-
interface JudgeSpan extends SpanBase {
|
|
327
|
-
kind: 'judge';
|
|
328
|
-
judgeId: string;
|
|
329
|
-
/** Span this judgment applies to. */
|
|
330
|
-
targetSpanId: string;
|
|
331
|
-
dimension: string;
|
|
332
|
-
/** Numeric score (free-range; interpretation up to the judge). */
|
|
333
|
-
score: number;
|
|
334
|
-
rationale?: string;
|
|
335
|
-
evidence?: string;
|
|
336
|
-
}
|
|
337
|
-
interface SandboxSpan extends SpanBase {
|
|
338
|
-
kind: 'sandbox';
|
|
339
|
-
image?: string;
|
|
340
|
-
command?: string;
|
|
341
|
-
exitCode?: number;
|
|
342
|
-
testsTotal?: number;
|
|
343
|
-
testsPassed?: number;
|
|
344
|
-
stdoutHash?: string;
|
|
345
|
-
stderrHash?: string;
|
|
346
|
-
/** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
|
|
347
|
-
wallMs?: number;
|
|
348
|
-
}
|
|
349
|
-
interface GenericSpan extends SpanBase {
|
|
350
|
-
kind: 'agent' | 'custom';
|
|
351
|
-
}
|
|
352
|
-
type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
|
|
353
|
-
type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
|
|
354
|
-
interface TraceEvent$1 {
|
|
355
|
-
eventId: string;
|
|
356
|
-
runId: string;
|
|
357
|
-
spanId?: string;
|
|
358
|
-
kind: EventKind;
|
|
359
|
-
timestamp: number;
|
|
360
|
-
payload: Record<string, unknown>;
|
|
361
|
-
}
|
|
362
|
-
interface BudgetLedgerEntry {
|
|
363
|
-
runId: string;
|
|
364
|
-
dimension: keyof BudgetSpec;
|
|
365
|
-
limit: number;
|
|
366
|
-
consumed: number;
|
|
367
|
-
remaining: number;
|
|
368
|
-
timestamp: number;
|
|
369
|
-
breached: boolean;
|
|
370
|
-
/** Span that triggered this entry, if any. */
|
|
371
|
-
spanId?: string;
|
|
372
|
-
}
|
|
373
|
-
interface Artifact {
|
|
374
|
-
artifactId: string;
|
|
375
|
-
runId: string;
|
|
376
|
-
spanId?: string;
|
|
377
|
-
contentType: string;
|
|
378
|
-
sizeBytes: number;
|
|
379
|
-
/** sha256 in hex. */
|
|
380
|
-
hash: string;
|
|
381
|
-
/** External storage URL (R2, S3, filesystem path). */
|
|
382
|
-
storageUrl?: string;
|
|
383
|
-
/** Inline content for small blobs — keep under ~64KB. */
|
|
384
|
-
inlineContent?: string;
|
|
385
|
-
}
|
|
386
|
-
type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
|
|
387
|
-
|
|
388
|
-
interface RunFilter {
|
|
389
|
-
scenarioId?: string;
|
|
390
|
-
variantId?: string;
|
|
391
|
-
status?: RunStatus;
|
|
392
|
-
since?: number;
|
|
393
|
-
until?: number;
|
|
394
|
-
tag?: {
|
|
395
|
-
key: string;
|
|
396
|
-
value: string;
|
|
397
|
-
};
|
|
398
|
-
parentRunId?: string;
|
|
399
|
-
projectId?: string;
|
|
400
|
-
chatId?: string;
|
|
401
|
-
layer?: RunLayer;
|
|
402
|
-
}
|
|
403
|
-
interface SpanFilter {
|
|
404
|
-
runId?: string;
|
|
405
|
-
parentSpanId?: string;
|
|
406
|
-
kind?: SpanKind;
|
|
407
|
-
name?: string;
|
|
408
|
-
toolName?: string;
|
|
409
|
-
judgeId?: string;
|
|
410
|
-
since?: number;
|
|
411
|
-
until?: number;
|
|
412
|
-
}
|
|
413
|
-
interface EventFilter {
|
|
414
|
-
runId?: string;
|
|
415
|
-
spanId?: string;
|
|
416
|
-
kind?: EventKind;
|
|
417
|
-
since?: number;
|
|
418
|
-
until?: number;
|
|
419
|
-
}
|
|
420
|
-
interface TraceStore {
|
|
421
|
-
appendRun(run: Run): Promise<void>;
|
|
422
|
-
updateRun(runId: string, patch: Partial<Run>): Promise<void>;
|
|
423
|
-
appendSpan(span: Span): Promise<void>;
|
|
424
|
-
updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
|
|
425
|
-
appendEvent(event: TraceEvent$1): Promise<void>;
|
|
426
|
-
appendArtifact(artifact: Artifact): Promise<void>;
|
|
427
|
-
appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
|
|
428
|
-
getRun(runId: string): Promise<Run | undefined>;
|
|
429
|
-
listRuns(filter?: RunFilter): Promise<Run[]>;
|
|
430
|
-
spans(filter?: SpanFilter): Promise<Span[]>;
|
|
431
|
-
events(filter?: EventFilter): Promise<TraceEvent$1[]>;
|
|
432
|
-
budget(runId: string): Promise<BudgetLedgerEntry[]>;
|
|
433
|
-
artifacts(runId: string): Promise<Artifact[]>;
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
/**
|
|
437
|
-
* Policy-based agent control runtime.
|
|
438
|
-
*
|
|
439
|
-
* This is the minimal reusable loop behind driver-agent patterns:
|
|
440
|
-
*
|
|
441
|
-
* observe state -> validate -> decide next action -> act -> observe -> ...
|
|
442
|
-
*
|
|
443
|
-
* It deliberately does not model named "topologies". Direct execution,
|
|
444
|
-
* critic/revise, driver intervention, specialist calls, and human escalation
|
|
445
|
-
* are all just actions chosen by the control policy.
|
|
446
|
-
*/
|
|
447
|
-
|
|
448
|
-
type ControlSeverity = 'info' | 'warning' | 'error' | 'critical';
|
|
449
|
-
interface ControlEvalResult {
|
|
450
|
-
/** Stable validator or judge id. */
|
|
451
|
-
id: string;
|
|
452
|
-
/** Whether this check passed. */
|
|
453
|
-
passed: boolean;
|
|
454
|
-
/** Optional normalized score. 1 = best, 0 = worst. */
|
|
455
|
-
score?: number;
|
|
456
|
-
/** Objective validators should usually be "error" or "critical" when failed. */
|
|
457
|
-
severity?: ControlSeverity;
|
|
458
|
-
/** Human-readable result. */
|
|
459
|
-
detail?: string;
|
|
460
|
-
/** Small evidence string or pointer. Avoid large payloads. */
|
|
461
|
-
evidence?: string;
|
|
462
|
-
/** True when the result came from deterministic state, not LLM judgment. */
|
|
463
|
-
objective?: boolean;
|
|
464
|
-
/** Structured details for downstream control policies and reports. */
|
|
465
|
-
metadata?: Record<string, unknown>;
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
/**
|
|
469
|
-
* Dataset — versioned, sliceable, content-hashed scenario collection.
|
|
470
|
-
*
|
|
471
|
-
* Scenarios stop being ephemeral arrays and become first-class
|
|
472
|
-
* artifacts. Every Dataset carries:
|
|
473
|
-
* - content hash (sha256 over canonicalized scenario array)
|
|
474
|
-
* - provenance (contributor, createdAt, sourceUrl)
|
|
475
|
-
* - split labels (train | dev | test | holdout)
|
|
476
|
-
* - difficulty tiers (easy | medium | hard | extreme)
|
|
477
|
-
* - tags (free-form, per-scenario)
|
|
478
|
-
*
|
|
479
|
-
* `Dataset.slice({ difficulty, split, holdout, seed })` returns a
|
|
480
|
-
* deterministic, reproducible subset. Holdout slices are locked: you
|
|
481
|
-
* can read them but `mutate` throws, which prevents "oh I'll just
|
|
482
|
-
* tweak that one scenario" contamination drift.
|
|
483
|
-
*/
|
|
484
|
-
type DatasetSplit = 'train' | 'dev' | 'test' | 'holdout';
|
|
485
|
-
|
|
486
|
-
type FeedbackArtifactType = 'text' | 'code' | 'plan' | 'research' | 'action' | 'ui' | 'decision' | 'data' | 'other';
|
|
487
|
-
type FeedbackLabelSource = 'user' | 'judge' | 'environment' | 'metric' | 'policy' | 'system';
|
|
488
|
-
type FeedbackLabelKind = 'approve' | 'reject' | 'select' | 'edit' | 'rank' | 'rate' | 'comment' | 'metric_outcome' | 'policy_block' | 'revision_request';
|
|
489
|
-
type FeedbackSeverity = 'info' | 'warning' | 'error' | 'critical';
|
|
490
|
-
interface FeedbackTask {
|
|
491
|
-
intent: string;
|
|
492
|
-
context?: unknown;
|
|
493
|
-
}
|
|
494
|
-
interface ProposedSideEffect {
|
|
495
|
-
type: string;
|
|
496
|
-
risk?: 'low' | 'medium' | 'high';
|
|
497
|
-
costUsd?: number;
|
|
498
|
-
externalSideEffect?: boolean;
|
|
499
|
-
requiresApproval?: boolean;
|
|
500
|
-
metadata?: Record<string, unknown>;
|
|
501
|
-
}
|
|
502
|
-
interface FeedbackLabel {
|
|
503
|
-
id?: string;
|
|
504
|
-
source: FeedbackLabelSource;
|
|
505
|
-
kind: FeedbackLabelKind;
|
|
506
|
-
value: unknown;
|
|
507
|
-
reason?: string;
|
|
508
|
-
severity?: FeedbackSeverity;
|
|
509
|
-
createdAt: string;
|
|
510
|
-
metadata?: Record<string, unknown>;
|
|
511
|
-
}
|
|
512
|
-
interface FeedbackAttempt {
|
|
513
|
-
id: string;
|
|
514
|
-
stepIndex: number;
|
|
515
|
-
artifactType: FeedbackArtifactType;
|
|
516
|
-
artifact: unknown;
|
|
517
|
-
options?: unknown[];
|
|
518
|
-
proposedAction?: ProposedSideEffect;
|
|
519
|
-
evals?: ControlEvalResult[];
|
|
520
|
-
feedback?: FeedbackLabel[];
|
|
521
|
-
createdAt: string;
|
|
522
|
-
metadata?: Record<string, unknown>;
|
|
523
|
-
}
|
|
524
|
-
interface FeedbackOutcome {
|
|
525
|
-
success?: boolean;
|
|
526
|
-
score?: number;
|
|
527
|
-
metrics?: Record<string, number>;
|
|
528
|
-
costUsd?: number;
|
|
529
|
-
detail?: string;
|
|
530
|
-
observedAt?: string;
|
|
531
|
-
metadata?: Record<string, unknown>;
|
|
532
|
-
}
|
|
533
|
-
interface FeedbackTrajectory$1 {
|
|
534
|
-
id: string;
|
|
535
|
-
projectId?: string;
|
|
536
|
-
scenarioId?: string;
|
|
537
|
-
task: FeedbackTask;
|
|
538
|
-
attempts: FeedbackAttempt[];
|
|
539
|
-
labels: FeedbackLabel[];
|
|
540
|
-
outcome?: FeedbackOutcome;
|
|
541
|
-
split?: DatasetSplit;
|
|
542
|
-
tags?: Record<string, string>;
|
|
543
|
-
createdAt: string;
|
|
544
|
-
updatedAt?: string;
|
|
545
|
-
metadata?: Record<string, unknown>;
|
|
546
|
-
}
|
|
547
|
-
interface FeedbackTrajectoryStore {
|
|
548
|
-
save(trajectory: FeedbackTrajectory$1): Promise<void>;
|
|
549
|
-
get(id: string): Promise<FeedbackTrajectory$1 | null>;
|
|
550
|
-
list(filter?: FeedbackTrajectoryFilter): Promise<FeedbackTrajectory$1[]>;
|
|
551
|
-
appendAttempt(id: string, attempt: FeedbackAttempt): Promise<FeedbackTrajectory$1>;
|
|
552
|
-
appendLabel(id: string, label: FeedbackLabel, attemptId?: string): Promise<FeedbackTrajectory$1>;
|
|
553
|
-
}
|
|
554
|
-
interface FeedbackTrajectoryFilter {
|
|
555
|
-
projectId?: string;
|
|
556
|
-
scenarioId?: string;
|
|
557
|
-
split?: DatasetSplit;
|
|
558
|
-
tag?: [string, string];
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
/**
|
|
562
|
-
* RawProviderSink — first-class persistence for the actual HTTP-level
|
|
563
|
-
* request/response bodies of every LLM provider call.
|
|
564
|
-
*
|
|
565
|
-
* Why this is a separate sink from the structured `LlmSpan`:
|
|
566
|
-
*
|
|
567
|
-
* - `LlmSpan` records the *intent* — model name, messages, output text,
|
|
568
|
-
* usage. It's what dashboards read; it's NOT enough for forensics.
|
|
569
|
-
* - When a downstream consumer reports "the verifier used the wrong route"
|
|
570
|
-
* or "tokens look right but reasoning was missing," the only way to
|
|
571
|
-
* answer is the raw HTTP body. Span fields can lie (a proxy can echo
|
|
572
|
-
* a different `model` value than what actually answered); the raw
|
|
573
|
-
* response is ground truth.
|
|
574
|
-
*
|
|
575
|
-
* Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
|
|
576
|
-
* matrix runner / BuilderSession sets it up automatically) and every
|
|
577
|
-
* request, response, and error is recorded — including retries, with the
|
|
578
|
-
* attempt index attached so a flaky call's full event chain is recoverable.
|
|
579
|
-
*
|
|
580
|
-
* Redaction is enforced at sink time. The default redactor strips
|
|
581
|
-
* `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
|
|
582
|
-
* payload field whose key matches `apiKey | api_key | bearer | password |
|
|
583
|
-
* secret | token` (case-insensitive). Override via the sink constructor or
|
|
584
|
-
* the per-call `redactor`. The `redactedFields` array on the persisted
|
|
585
|
-
* event lets a reviewer see what was stripped without exposing the values.
|
|
586
|
-
*/
|
|
587
|
-
type RawProviderDirection = 'request' | 'response' | 'error';
|
|
588
|
-
interface RawProviderEvent {
|
|
589
|
-
/** Stable id. Generated by the sink if omitted. */
|
|
590
|
-
eventId: string;
|
|
591
|
-
/** Trace context populated by `LlmClient` when the call is wrapped in a span. */
|
|
592
|
-
runId?: string;
|
|
593
|
-
spanId?: string;
|
|
594
|
-
/**
|
|
595
|
-
* Logical provider name. Free-form so callers can use whatever id matches
|
|
596
|
-
* their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
|
|
597
|
-
* omitted, derived from `baseUrl` in `LlmClientOptions`.
|
|
598
|
-
*/
|
|
599
|
-
provider: string;
|
|
600
|
-
model: string;
|
|
601
|
-
/** Endpoint path, e.g. `'/v1/chat/completions'`. */
|
|
602
|
-
endpoint: string;
|
|
603
|
-
/** Base URL used for the call (already-normalised — no trailing slash). */
|
|
604
|
-
baseUrl: string;
|
|
605
|
-
/** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
|
|
606
|
-
attemptIndex: number;
|
|
607
|
-
direction: RawProviderDirection;
|
|
608
|
-
/** Unix ms. */
|
|
609
|
-
timestamp: number;
|
|
610
|
-
/** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
|
|
611
|
-
durationMs?: number;
|
|
612
|
-
statusCode?: number;
|
|
613
|
-
requestHeaders?: Record<string, string>;
|
|
614
|
-
requestBody?: unknown;
|
|
615
|
-
responseHeaders?: Record<string, string>;
|
|
616
|
-
responseBody?: unknown;
|
|
617
|
-
/** Set on `direction: 'error'` events. */
|
|
618
|
-
errorMessage?: string;
|
|
619
|
-
/** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
|
|
620
|
-
redactedFields: string[];
|
|
621
|
-
}
|
|
622
|
-
interface RawProviderSinkFilter {
|
|
623
|
-
runId?: string;
|
|
624
|
-
spanId?: string;
|
|
625
|
-
direction?: RawProviderDirection;
|
|
626
|
-
attemptIndex?: number;
|
|
627
|
-
}
|
|
628
|
-
interface RawProviderSink {
|
|
629
|
-
record(event: RawProviderEvent): Promise<void>;
|
|
630
|
-
/** Optional listing — implementations that durably persist (file, db) should support this. */
|
|
631
|
-
list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
|
|
632
|
-
/** Optional teardown for backed implementations. */
|
|
633
|
-
close?(): Promise<void>;
|
|
634
|
-
}
|
|
635
|
-
type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
|
|
636
|
-
|
|
637
|
-
/**
|
|
638
|
-
* LLM client with graceful degrade.
|
|
639
|
-
*
|
|
640
|
-
* OpenAI-compatible `/v1/chat/completions` client with:
|
|
641
|
-
* - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
|
|
642
|
-
* - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
|
|
643
|
-
* - One retry at temperature 1 when a model explicitly requires it.
|
|
644
|
-
* - Graceful json_schema → json_object degrade on 400 with schema-reject body.
|
|
645
|
-
* - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
|
|
646
|
-
* - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
|
|
647
|
-
* directly, cli-bridge subscriptions, and any router that speaks the spec.
|
|
648
|
-
*
|
|
649
|
-
* Usage:
|
|
650
|
-
* const { value, result } = await callLlmJson<MyType>(
|
|
651
|
-
* { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
|
|
652
|
-
* { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
|
|
653
|
-
* )
|
|
654
|
-
*
|
|
655
|
-
* This is THE llm-calling seam for agent-eval primitives that need structured
|
|
656
|
-
* output (semantic concept judge, reviewer directives, critic scores). Primitives
|
|
657
|
-
* that need free-form text use `callLlm` and parse output themselves.
|
|
658
|
-
*/
|
|
659
|
-
|
|
660
|
-
type LlmThinkingMode = 'enabled' | 'disabled';
|
|
661
|
-
interface LlmClientOptions {
|
|
662
|
-
/** Base URL (without trailing slash). Must end at the `/v1` prefix. */
|
|
663
|
-
baseUrl?: string;
|
|
664
|
-
/** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
|
|
665
|
-
apiKey?: string;
|
|
666
|
-
bearer?: string;
|
|
667
|
-
/** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */
|
|
668
|
-
authHeader?: {
|
|
669
|
-
name: string;
|
|
670
|
-
value: string;
|
|
671
|
-
};
|
|
672
|
-
/** Stable provider idempotency key, reused across retries of this logical call. */
|
|
673
|
-
idempotencyKey?: string;
|
|
674
|
-
/** Default timeout in ms. Per-call can override. */
|
|
675
|
-
defaultTimeoutMs?: number;
|
|
676
|
-
/**
|
|
677
|
-
* Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
|
|
678
|
-
* each attempt's per-attempt timeout controller, so aborting it cancels
|
|
679
|
-
* the in-flight fetch. A caller abort is FATAL: it is not retried even
|
|
680
|
-
* though an AbortError otherwise matches the transient patterns.
|
|
681
|
-
*/
|
|
682
|
-
signal?: AbortSignal;
|
|
683
|
-
/**
|
|
684
|
-
* Cross-attempt wall-clock budget in ms, measured from the first attempt.
|
|
685
|
-
* Before launching each attempt the loop checks the remaining budget and
|
|
686
|
-
* stops retrying once it is exhausted, rather than waiting the full
|
|
687
|
-
* per-attempt timeout on every retry. Bounds total time independent of
|
|
688
|
-
* total attempts × `timeoutMs`.
|
|
689
|
-
*/
|
|
690
|
-
deadlineMs?: number;
|
|
691
|
-
/** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */
|
|
692
|
-
maxRetries?: number;
|
|
693
|
-
/** Token rates used when the provider omits cost or package pricing does not cover the model. */
|
|
694
|
-
customTokenPricing?: CustomTokenPricing;
|
|
695
|
-
/**
|
|
696
|
-
* Transport for requests that declare `jsonSchema`. `native` sends
|
|
697
|
-
* `response_format: json_schema`; `json-object` sends the broadly supported
|
|
698
|
-
* JSON mode and relies on the caller to include the schema in model-visible
|
|
699
|
-
* instructions. Default: `native`.
|
|
700
|
-
*/
|
|
701
|
-
jsonSchemaTransport?: 'native' | 'json-object';
|
|
702
|
-
/**
|
|
703
|
-
* JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.
|
|
704
|
-
* `exact` requires the complete response content to be one JSON value.
|
|
705
|
-
* Default: `extract`.
|
|
706
|
-
*/
|
|
707
|
-
jsonPayloadMode?: 'extract' | 'exact';
|
|
708
|
-
/** Default provider reasoning mode. A per-call request value takes precedence. */
|
|
709
|
-
thinking?: LlmThinkingMode;
|
|
710
|
-
/** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
|
|
711
|
-
fetch?: typeof fetch;
|
|
712
|
-
/**
|
|
713
|
-
* Optional raw HTTP capture sink. When provided, every request, response,
|
|
714
|
-
* and error (across all retry attempts) is recorded to the sink, with auth
|
|
715
|
-
* headers and credential-shaped body fields redacted by default. This is
|
|
716
|
-
* the layer-1 forensics primitive: structured `LlmSpan`s record intent,
|
|
717
|
-
* raw events record what actually crossed the wire.
|
|
718
|
-
*/
|
|
719
|
-
rawSink?: RawProviderSink;
|
|
720
|
-
/**
|
|
721
|
-
* Logical provider id attached to raw events. When omitted, derived from
|
|
722
|
-
* `baseUrl` via `providerFromBaseUrl`.
|
|
723
|
-
*/
|
|
724
|
-
provider?: string;
|
|
725
|
-
/** Trace context attached to raw events; populated by emitter-aware callers. */
|
|
726
|
-
traceContext?: {
|
|
727
|
-
runId?: string;
|
|
728
|
-
spanId?: string;
|
|
729
|
-
};
|
|
730
|
-
/** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */
|
|
731
|
-
redactor?: ProviderRedactor;
|
|
732
|
-
}
|
|
733
|
-
interface LlmRouteRequirements {
|
|
734
|
-
/**
|
|
735
|
-
* Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
|
|
736
|
-
* `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using
|
|
737
|
-
* the public/free-tier router is a defect — the launch reviewer needs to
|
|
738
|
-
* know exactly which provider answered.
|
|
739
|
-
*/
|
|
740
|
-
requireExplicitBaseUrl?: boolean;
|
|
741
|
-
/**
|
|
742
|
-
* Allowlist of acceptable base URLs. Strings match by prefix
|
|
743
|
-
* (case-insensitive); RegExps test against the full base URL.
|
|
744
|
-
*/
|
|
745
|
-
allowedBaseUrls?: Array<string | RegExp>;
|
|
746
|
-
/** Blocklist that takes precedence over `allowedBaseUrls`. */
|
|
747
|
-
blockedBaseUrls?: Array<string | RegExp>;
|
|
748
|
-
/** Throw if no auth header / api key is configured. */
|
|
749
|
-
requireAuth?: boolean;
|
|
750
|
-
/**
|
|
751
|
-
* Logical provider id the configured `baseUrl` is expected to match (via
|
|
752
|
-
* `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.
|
|
753
|
-
*/
|
|
754
|
-
expectedProvider?: string;
|
|
755
|
-
}
|
|
756
|
-
|
|
1
|
+
import { s as TraceStore } from "../store-CT9YIIve.js";
|
|
2
|
+
import { c as CostLedgerHandle } from "../cost-ledger-Dye6jCgg.js";
|
|
3
|
+
import { o as LlmClientOptions, u as LlmRouteRequirements } from "../llm-client-B_nIBlYo.js";
|
|
4
|
+
import { h as FeedbackTrajectoryStore } from "../feedback-trajectory-CVaeREXV.js";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
import { ServerType } from "@hono/node-server";
|
|
7
|
+
import { Hono } from "hono";
|
|
8
|
+
//#region src/wire/schemas.d.ts
|
|
757
9
|
declare const RubricDimensionSchema: z.ZodObject<{
|
|
10
|
+
id: z.ZodString;
|
|
11
|
+
description: z.ZodString;
|
|
12
|
+
weight: z.ZodDefault<z.ZodNumber>;
|
|
13
|
+
min: z.ZodDefault<z.ZodNumber>;
|
|
14
|
+
max: z.ZodDefault<z.ZodNumber>;
|
|
15
|
+
}, z.core.$strip>;
|
|
16
|
+
declare const FailureModeSchema: z.ZodObject<{
|
|
17
|
+
id: z.ZodString;
|
|
18
|
+
description: z.ZodString;
|
|
19
|
+
}, z.core.$strip>;
|
|
20
|
+
declare const RubricSchema: z.ZodObject<{
|
|
21
|
+
name: z.ZodString;
|
|
22
|
+
description: z.ZodString;
|
|
23
|
+
systemPrompt: z.ZodString;
|
|
24
|
+
dimensions: z.ZodArray<z.ZodObject<{
|
|
758
25
|
id: z.ZodString;
|
|
759
26
|
description: z.ZodString;
|
|
760
27
|
weight: z.ZodDefault<z.ZodNumber>;
|
|
761
28
|
min: z.ZodDefault<z.ZodNumber>;
|
|
762
29
|
max: z.ZodDefault<z.ZodNumber>;
|
|
763
|
-
}, z.core.$strip
|
|
764
|
-
|
|
30
|
+
}, z.core.$strip>>;
|
|
31
|
+
failureModes: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
32
|
+
id: z.ZodString;
|
|
33
|
+
description: z.ZodString;
|
|
34
|
+
}, z.core.$strip>>>;
|
|
35
|
+
wins: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
765
36
|
id: z.ZodString;
|
|
766
37
|
description: z.ZodString;
|
|
38
|
+
}, z.core.$strip>>>;
|
|
767
39
|
}, z.core.$strip>;
|
|
768
|
-
declare const
|
|
40
|
+
declare const JudgeRequestSchema: z.ZodObject<{
|
|
41
|
+
rubricName: z.ZodOptional<z.ZodString>;
|
|
42
|
+
rubric: z.ZodOptional<z.ZodObject<{
|
|
769
43
|
name: z.ZodString;
|
|
770
44
|
description: z.ZodString;
|
|
771
45
|
systemPrompt: z.ZodString;
|
|
772
46
|
dimensions: z.ZodArray<z.ZodObject<{
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
47
|
+
id: z.ZodString;
|
|
48
|
+
description: z.ZodString;
|
|
49
|
+
weight: z.ZodDefault<z.ZodNumber>;
|
|
50
|
+
min: z.ZodDefault<z.ZodNumber>;
|
|
51
|
+
max: z.ZodDefault<z.ZodNumber>;
|
|
778
52
|
}, z.core.$strip>>;
|
|
779
53
|
failureModes: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
780
|
-
|
|
781
|
-
|
|
54
|
+
id: z.ZodString;
|
|
55
|
+
description: z.ZodString;
|
|
782
56
|
}, z.core.$strip>>>;
|
|
783
57
|
wins: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
784
|
-
|
|
785
|
-
|
|
58
|
+
id: z.ZodString;
|
|
59
|
+
description: z.ZodString;
|
|
786
60
|
}, z.core.$strip>>>;
|
|
787
|
-
}, z.core.$strip
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
name: z.ZodString;
|
|
792
|
-
description: z.ZodString;
|
|
793
|
-
systemPrompt: z.ZodString;
|
|
794
|
-
dimensions: z.ZodArray<z.ZodObject<{
|
|
795
|
-
id: z.ZodString;
|
|
796
|
-
description: z.ZodString;
|
|
797
|
-
weight: z.ZodDefault<z.ZodNumber>;
|
|
798
|
-
min: z.ZodDefault<z.ZodNumber>;
|
|
799
|
-
max: z.ZodDefault<z.ZodNumber>;
|
|
800
|
-
}, z.core.$strip>>;
|
|
801
|
-
failureModes: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
802
|
-
id: z.ZodString;
|
|
803
|
-
description: z.ZodString;
|
|
804
|
-
}, z.core.$strip>>>;
|
|
805
|
-
wins: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
806
|
-
id: z.ZodString;
|
|
807
|
-
description: z.ZodString;
|
|
808
|
-
}, z.core.$strip>>>;
|
|
809
|
-
}, z.core.$strip>>;
|
|
810
|
-
content: z.ZodString;
|
|
811
|
-
context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
812
|
-
model: z.ZodOptional<z.ZodString>;
|
|
61
|
+
}, z.core.$strip>>;
|
|
62
|
+
content: z.ZodString;
|
|
63
|
+
context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
64
|
+
model: z.ZodOptional<z.ZodString>;
|
|
813
65
|
}, z.core.$strip>;
|
|
814
66
|
declare const JudgeResultSchema: z.ZodObject<{
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
67
|
+
composite: z.ZodNumber;
|
|
68
|
+
dimensions: z.ZodRecord<z.ZodString, z.ZodNumber>;
|
|
69
|
+
failureModes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
70
|
+
wins: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
71
|
+
rationale: z.ZodString;
|
|
72
|
+
rubricVersion: z.ZodString;
|
|
73
|
+
model: z.ZodString;
|
|
74
|
+
durationMs: z.ZodNumber;
|
|
823
75
|
}, z.core.$strip>;
|
|
824
76
|
declare const RubricInfoSchema: z.ZodObject<{
|
|
77
|
+
name: z.ZodString;
|
|
78
|
+
description: z.ZodString;
|
|
79
|
+
dimensions: z.ZodArray<z.ZodObject<{
|
|
80
|
+
id: z.ZodString;
|
|
81
|
+
description: z.ZodString;
|
|
82
|
+
weight: z.ZodNumber;
|
|
83
|
+
}, z.core.$strip>>;
|
|
84
|
+
failureModes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
85
|
+
rubricVersion: z.ZodString;
|
|
86
|
+
}, z.core.$strip>;
|
|
87
|
+
declare const ListRubricsResponseSchema: z.ZodObject<{
|
|
88
|
+
rubrics: z.ZodArray<z.ZodObject<{
|
|
825
89
|
name: z.ZodString;
|
|
826
90
|
description: z.ZodString;
|
|
827
91
|
dimensions: z.ZodArray<z.ZodObject<{
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
92
|
+
id: z.ZodString;
|
|
93
|
+
description: z.ZodString;
|
|
94
|
+
weight: z.ZodNumber;
|
|
831
95
|
}, z.core.$strip>>;
|
|
832
96
|
failureModes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
833
97
|
rubricVersion: z.ZodString;
|
|
834
|
-
}, z.core.$strip
|
|
835
|
-
declare const ListRubricsResponseSchema: z.ZodObject<{
|
|
836
|
-
rubrics: z.ZodArray<z.ZodObject<{
|
|
837
|
-
name: z.ZodString;
|
|
838
|
-
description: z.ZodString;
|
|
839
|
-
dimensions: z.ZodArray<z.ZodObject<{
|
|
840
|
-
id: z.ZodString;
|
|
841
|
-
description: z.ZodString;
|
|
842
|
-
weight: z.ZodNumber;
|
|
843
|
-
}, z.core.$strip>>;
|
|
844
|
-
failureModes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
845
|
-
rubricVersion: z.ZodString;
|
|
846
|
-
}, z.core.$strip>>;
|
|
98
|
+
}, z.core.$strip>>;
|
|
847
99
|
}, z.core.$strip>;
|
|
848
100
|
declare const VersionResponseSchema: z.ZodObject<{
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
101
|
+
package: z.ZodString;
|
|
102
|
+
version: z.ZodString;
|
|
103
|
+
wireVersion: z.ZodString;
|
|
104
|
+
apiSurface: z.ZodArray<z.ZodString>;
|
|
853
105
|
}, z.core.$strip>;
|
|
854
106
|
declare const HealthResponseSchema: z.ZodObject<{
|
|
855
|
-
|
|
856
|
-
|
|
107
|
+
status: z.ZodLiteral<"ok">;
|
|
108
|
+
uptimeSec: z.ZodNumber;
|
|
857
109
|
}, z.core.$strip>;
|
|
858
110
|
/**
|
|
859
111
|
* Minimal `TraceEvent` shape that the production runtime emits.
|
|
@@ -862,274 +114,274 @@ declare const HealthResponseSchema: z.ZodObject<{
|
|
|
862
114
|
* on internal types.
|
|
863
115
|
*/
|
|
864
116
|
declare const TraceEventSchema: z.ZodObject<{
|
|
117
|
+
eventId: z.ZodString;
|
|
118
|
+
runId: z.ZodString;
|
|
119
|
+
spanId: z.ZodOptional<z.ZodString>;
|
|
120
|
+
kind: z.ZodEnum<{
|
|
121
|
+
budget_breach: "budget_breach";
|
|
122
|
+
budget_decrement: "budget_decrement";
|
|
123
|
+
custom: "custom";
|
|
124
|
+
error: "error";
|
|
125
|
+
log: "log";
|
|
126
|
+
policy_violation: "policy_violation";
|
|
127
|
+
redaction_applied: "redaction_applied";
|
|
128
|
+
state_mutation: "state_mutation";
|
|
129
|
+
}>;
|
|
130
|
+
timestamp: z.ZodNumber;
|
|
131
|
+
payload: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
132
|
+
}, z.core.$strip>;
|
|
133
|
+
declare const TracesIngestRequestSchema: z.ZodObject<{
|
|
134
|
+
events: z.ZodArray<z.ZodObject<{
|
|
865
135
|
eventId: z.ZodString;
|
|
866
136
|
runId: z.ZodString;
|
|
867
137
|
spanId: z.ZodOptional<z.ZodString>;
|
|
868
138
|
kind: z.ZodEnum<{
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
139
|
+
budget_breach: "budget_breach";
|
|
140
|
+
budget_decrement: "budget_decrement";
|
|
141
|
+
custom: "custom";
|
|
142
|
+
error: "error";
|
|
143
|
+
log: "log";
|
|
144
|
+
policy_violation: "policy_violation";
|
|
145
|
+
redaction_applied: "redaction_applied";
|
|
146
|
+
state_mutation: "state_mutation";
|
|
877
147
|
}>;
|
|
878
148
|
timestamp: z.ZodNumber;
|
|
879
149
|
payload: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
880
|
-
}, z.core.$strip
|
|
881
|
-
declare const TracesIngestRequestSchema: z.ZodObject<{
|
|
882
|
-
events: z.ZodArray<z.ZodObject<{
|
|
883
|
-
eventId: z.ZodString;
|
|
884
|
-
runId: z.ZodString;
|
|
885
|
-
spanId: z.ZodOptional<z.ZodString>;
|
|
886
|
-
kind: z.ZodEnum<{
|
|
887
|
-
error: "error";
|
|
888
|
-
custom: "custom";
|
|
889
|
-
policy_violation: "policy_violation";
|
|
890
|
-
log: "log";
|
|
891
|
-
budget_decrement: "budget_decrement";
|
|
892
|
-
budget_breach: "budget_breach";
|
|
893
|
-
state_mutation: "state_mutation";
|
|
894
|
-
redaction_applied: "redaction_applied";
|
|
895
|
-
}>;
|
|
896
|
-
timestamp: z.ZodNumber;
|
|
897
|
-
payload: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
898
|
-
}, z.core.$strip>>;
|
|
150
|
+
}, z.core.$strip>>;
|
|
899
151
|
}, z.core.$strip>;
|
|
900
152
|
declare const TracesIngestResponseSchema: z.ZodObject<{
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
153
|
+
accepted: z.ZodNumber;
|
|
154
|
+
rejected: z.ZodNumber;
|
|
155
|
+
errors: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
156
|
+
eventId: z.ZodString;
|
|
157
|
+
message: z.ZodString;
|
|
158
|
+
}, z.core.$strip>>>;
|
|
907
159
|
}, z.core.$strip>;
|
|
908
160
|
declare const FeedbackLabelSchema: z.ZodObject<{
|
|
161
|
+
id: z.ZodOptional<z.ZodString>;
|
|
162
|
+
source: z.ZodEnum<{
|
|
163
|
+
environment: "environment";
|
|
164
|
+
judge: "judge";
|
|
165
|
+
metric: "metric";
|
|
166
|
+
policy: "policy";
|
|
167
|
+
system: "system";
|
|
168
|
+
user: "user";
|
|
169
|
+
}>;
|
|
170
|
+
kind: z.ZodEnum<{
|
|
171
|
+
approve: "approve";
|
|
172
|
+
comment: "comment";
|
|
173
|
+
edit: "edit";
|
|
174
|
+
metric_outcome: "metric_outcome";
|
|
175
|
+
policy_block: "policy_block";
|
|
176
|
+
rank: "rank";
|
|
177
|
+
rate: "rate";
|
|
178
|
+
reject: "reject";
|
|
179
|
+
revision_request: "revision_request";
|
|
180
|
+
select: "select";
|
|
181
|
+
}>;
|
|
182
|
+
value: z.ZodUnknown;
|
|
183
|
+
reason: z.ZodOptional<z.ZodString>;
|
|
184
|
+
severity: z.ZodOptional<z.ZodEnum<{
|
|
185
|
+
critical: "critical";
|
|
186
|
+
error: "error";
|
|
187
|
+
info: "info";
|
|
188
|
+
warning: "warning";
|
|
189
|
+
}>>;
|
|
190
|
+
createdAt: z.ZodString;
|
|
191
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
192
|
+
}, z.core.$strip>;
|
|
193
|
+
declare const FeedbackAttemptSchema: z.ZodObject<{
|
|
194
|
+
id: z.ZodString;
|
|
195
|
+
stepIndex: z.ZodNumber;
|
|
196
|
+
artifactType: z.ZodEnum<{
|
|
197
|
+
action: "action";
|
|
198
|
+
code: "code";
|
|
199
|
+
data: "data";
|
|
200
|
+
decision: "decision";
|
|
201
|
+
other: "other";
|
|
202
|
+
plan: "plan";
|
|
203
|
+
research: "research";
|
|
204
|
+
text: "text";
|
|
205
|
+
ui: "ui";
|
|
206
|
+
}>;
|
|
207
|
+
artifact: z.ZodUnknown;
|
|
208
|
+
options: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
|
|
209
|
+
proposedAction: z.ZodOptional<z.ZodObject<{
|
|
210
|
+
type: z.ZodString;
|
|
211
|
+
risk: z.ZodOptional<z.ZodEnum<{
|
|
212
|
+
high: "high";
|
|
213
|
+
low: "low";
|
|
214
|
+
medium: "medium";
|
|
215
|
+
}>>;
|
|
216
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
217
|
+
externalSideEffect: z.ZodOptional<z.ZodBoolean>;
|
|
218
|
+
requiresApproval: z.ZodOptional<z.ZodBoolean>;
|
|
219
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
220
|
+
}, z.core.$strip>>;
|
|
221
|
+
feedback: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
909
222
|
id: z.ZodOptional<z.ZodString>;
|
|
910
223
|
source: z.ZodEnum<{
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
224
|
+
environment: "environment";
|
|
225
|
+
judge: "judge";
|
|
226
|
+
metric: "metric";
|
|
227
|
+
policy: "policy";
|
|
228
|
+
system: "system";
|
|
229
|
+
user: "user";
|
|
917
230
|
}>;
|
|
918
231
|
kind: z.ZodEnum<{
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
232
|
+
approve: "approve";
|
|
233
|
+
comment: "comment";
|
|
234
|
+
edit: "edit";
|
|
235
|
+
metric_outcome: "metric_outcome";
|
|
236
|
+
policy_block: "policy_block";
|
|
237
|
+
rank: "rank";
|
|
238
|
+
rate: "rate";
|
|
239
|
+
reject: "reject";
|
|
240
|
+
revision_request: "revision_request";
|
|
241
|
+
select: "select";
|
|
929
242
|
}>;
|
|
930
243
|
value: z.ZodUnknown;
|
|
931
244
|
reason: z.ZodOptional<z.ZodString>;
|
|
932
245
|
severity: z.ZodOptional<z.ZodEnum<{
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
246
|
+
critical: "critical";
|
|
247
|
+
error: "error";
|
|
248
|
+
info: "info";
|
|
249
|
+
warning: "warning";
|
|
937
250
|
}>>;
|
|
938
251
|
createdAt: z.ZodString;
|
|
939
252
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
253
|
+
}, z.core.$strip>>>;
|
|
254
|
+
createdAt: z.ZodString;
|
|
255
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
940
256
|
}, z.core.$strip>;
|
|
941
|
-
declare const
|
|
257
|
+
declare const FeedbackTrajectorySchema: z.ZodObject<{
|
|
258
|
+
id: z.ZodString;
|
|
259
|
+
projectId: z.ZodOptional<z.ZodString>;
|
|
260
|
+
scenarioId: z.ZodOptional<z.ZodString>;
|
|
261
|
+
task: z.ZodObject<{
|
|
262
|
+
intent: z.ZodString;
|
|
263
|
+
context: z.ZodOptional<z.ZodUnknown>;
|
|
264
|
+
}, z.core.$strip>;
|
|
265
|
+
attempts: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
942
266
|
id: z.ZodString;
|
|
943
267
|
stepIndex: z.ZodNumber;
|
|
944
268
|
artifactType: z.ZodEnum<{
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
269
|
+
action: "action";
|
|
270
|
+
code: "code";
|
|
271
|
+
data: "data";
|
|
272
|
+
decision: "decision";
|
|
273
|
+
other: "other";
|
|
274
|
+
plan: "plan";
|
|
275
|
+
research: "research";
|
|
276
|
+
text: "text";
|
|
277
|
+
ui: "ui";
|
|
954
278
|
}>;
|
|
955
279
|
artifact: z.ZodUnknown;
|
|
956
280
|
options: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
|
|
957
281
|
proposedAction: z.ZodOptional<z.ZodObject<{
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
282
|
+
type: z.ZodString;
|
|
283
|
+
risk: z.ZodOptional<z.ZodEnum<{
|
|
284
|
+
high: "high";
|
|
285
|
+
low: "low";
|
|
286
|
+
medium: "medium";
|
|
287
|
+
}>>;
|
|
288
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
289
|
+
externalSideEffect: z.ZodOptional<z.ZodBoolean>;
|
|
290
|
+
requiresApproval: z.ZodOptional<z.ZodBoolean>;
|
|
291
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
968
292
|
}, z.core.$strip>>;
|
|
969
293
|
feedback: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
294
|
+
id: z.ZodOptional<z.ZodString>;
|
|
295
|
+
source: z.ZodEnum<{
|
|
296
|
+
environment: "environment";
|
|
297
|
+
judge: "judge";
|
|
298
|
+
metric: "metric";
|
|
299
|
+
policy: "policy";
|
|
300
|
+
system: "system";
|
|
301
|
+
user: "user";
|
|
302
|
+
}>;
|
|
303
|
+
kind: z.ZodEnum<{
|
|
304
|
+
approve: "approve";
|
|
305
|
+
comment: "comment";
|
|
306
|
+
edit: "edit";
|
|
307
|
+
metric_outcome: "metric_outcome";
|
|
308
|
+
policy_block: "policy_block";
|
|
309
|
+
rank: "rank";
|
|
310
|
+
rate: "rate";
|
|
311
|
+
reject: "reject";
|
|
312
|
+
revision_request: "revision_request";
|
|
313
|
+
select: "select";
|
|
314
|
+
}>;
|
|
315
|
+
value: z.ZodUnknown;
|
|
316
|
+
reason: z.ZodOptional<z.ZodString>;
|
|
317
|
+
severity: z.ZodOptional<z.ZodEnum<{
|
|
318
|
+
critical: "critical";
|
|
319
|
+
error: "error";
|
|
320
|
+
info: "info";
|
|
321
|
+
warning: "warning";
|
|
322
|
+
}>>;
|
|
323
|
+
createdAt: z.ZodString;
|
|
324
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1001
325
|
}, z.core.$strip>>>;
|
|
1002
326
|
createdAt: z.ZodString;
|
|
1003
327
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1004
|
-
}, z.core.$strip
|
|
1005
|
-
|
|
1006
|
-
id: z.ZodString
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
high: "high";
|
|
1035
|
-
}>>;
|
|
1036
|
-
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
1037
|
-
externalSideEffect: z.ZodOptional<z.ZodBoolean>;
|
|
1038
|
-
requiresApproval: z.ZodOptional<z.ZodBoolean>;
|
|
1039
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1040
|
-
}, z.core.$strip>>;
|
|
1041
|
-
feedback: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1042
|
-
id: z.ZodOptional<z.ZodString>;
|
|
1043
|
-
source: z.ZodEnum<{
|
|
1044
|
-
judge: "judge";
|
|
1045
|
-
user: "user";
|
|
1046
|
-
system: "system";
|
|
1047
|
-
policy: "policy";
|
|
1048
|
-
environment: "environment";
|
|
1049
|
-
metric: "metric";
|
|
1050
|
-
}>;
|
|
1051
|
-
kind: z.ZodEnum<{
|
|
1052
|
-
approve: "approve";
|
|
1053
|
-
reject: "reject";
|
|
1054
|
-
select: "select";
|
|
1055
|
-
edit: "edit";
|
|
1056
|
-
rank: "rank";
|
|
1057
|
-
rate: "rate";
|
|
1058
|
-
comment: "comment";
|
|
1059
|
-
metric_outcome: "metric_outcome";
|
|
1060
|
-
policy_block: "policy_block";
|
|
1061
|
-
revision_request: "revision_request";
|
|
1062
|
-
}>;
|
|
1063
|
-
value: z.ZodUnknown;
|
|
1064
|
-
reason: z.ZodOptional<z.ZodString>;
|
|
1065
|
-
severity: z.ZodOptional<z.ZodEnum<{
|
|
1066
|
-
error: "error";
|
|
1067
|
-
info: "info";
|
|
1068
|
-
critical: "critical";
|
|
1069
|
-
warning: "warning";
|
|
1070
|
-
}>>;
|
|
1071
|
-
createdAt: z.ZodString;
|
|
1072
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1073
|
-
}, z.core.$strip>>>;
|
|
1074
|
-
createdAt: z.ZodString;
|
|
1075
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1076
|
-
}, z.core.$strip>>>;
|
|
1077
|
-
labels: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
1078
|
-
id: z.ZodOptional<z.ZodString>;
|
|
1079
|
-
source: z.ZodEnum<{
|
|
1080
|
-
judge: "judge";
|
|
1081
|
-
user: "user";
|
|
1082
|
-
system: "system";
|
|
1083
|
-
policy: "policy";
|
|
1084
|
-
environment: "environment";
|
|
1085
|
-
metric: "metric";
|
|
1086
|
-
}>;
|
|
1087
|
-
kind: z.ZodEnum<{
|
|
1088
|
-
approve: "approve";
|
|
1089
|
-
reject: "reject";
|
|
1090
|
-
select: "select";
|
|
1091
|
-
edit: "edit";
|
|
1092
|
-
rank: "rank";
|
|
1093
|
-
rate: "rate";
|
|
1094
|
-
comment: "comment";
|
|
1095
|
-
metric_outcome: "metric_outcome";
|
|
1096
|
-
policy_block: "policy_block";
|
|
1097
|
-
revision_request: "revision_request";
|
|
1098
|
-
}>;
|
|
1099
|
-
value: z.ZodUnknown;
|
|
1100
|
-
reason: z.ZodOptional<z.ZodString>;
|
|
1101
|
-
severity: z.ZodOptional<z.ZodEnum<{
|
|
1102
|
-
error: "error";
|
|
1103
|
-
info: "info";
|
|
1104
|
-
critical: "critical";
|
|
1105
|
-
warning: "warning";
|
|
1106
|
-
}>>;
|
|
1107
|
-
createdAt: z.ZodString;
|
|
1108
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1109
|
-
}, z.core.$strip>>>;
|
|
1110
|
-
outcome: z.ZodOptional<z.ZodObject<{
|
|
1111
|
-
success: z.ZodOptional<z.ZodBoolean>;
|
|
1112
|
-
score: z.ZodOptional<z.ZodNumber>;
|
|
1113
|
-
metrics: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
|
|
1114
|
-
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
1115
|
-
detail: z.ZodOptional<z.ZodString>;
|
|
1116
|
-
observedAt: z.ZodOptional<z.ZodString>;
|
|
1117
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1118
|
-
}, z.core.$strip>>;
|
|
1119
|
-
split: z.ZodOptional<z.ZodEnum<{
|
|
1120
|
-
train: "train";
|
|
1121
|
-
dev: "dev";
|
|
1122
|
-
test: "test";
|
|
1123
|
-
holdout: "holdout";
|
|
328
|
+
}, z.core.$strip>>>;
|
|
329
|
+
labels: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
330
|
+
id: z.ZodOptional<z.ZodString>;
|
|
331
|
+
source: z.ZodEnum<{
|
|
332
|
+
environment: "environment";
|
|
333
|
+
judge: "judge";
|
|
334
|
+
metric: "metric";
|
|
335
|
+
policy: "policy";
|
|
336
|
+
system: "system";
|
|
337
|
+
user: "user";
|
|
338
|
+
}>;
|
|
339
|
+
kind: z.ZodEnum<{
|
|
340
|
+
approve: "approve";
|
|
341
|
+
comment: "comment";
|
|
342
|
+
edit: "edit";
|
|
343
|
+
metric_outcome: "metric_outcome";
|
|
344
|
+
policy_block: "policy_block";
|
|
345
|
+
rank: "rank";
|
|
346
|
+
rate: "rate";
|
|
347
|
+
reject: "reject";
|
|
348
|
+
revision_request: "revision_request";
|
|
349
|
+
select: "select";
|
|
350
|
+
}>;
|
|
351
|
+
value: z.ZodUnknown;
|
|
352
|
+
reason: z.ZodOptional<z.ZodString>;
|
|
353
|
+
severity: z.ZodOptional<z.ZodEnum<{
|
|
354
|
+
critical: "critical";
|
|
355
|
+
error: "error";
|
|
356
|
+
info: "info";
|
|
357
|
+
warning: "warning";
|
|
1124
358
|
}>>;
|
|
1125
|
-
tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
1126
359
|
createdAt: z.ZodString;
|
|
1127
|
-
updatedAt: z.ZodOptional<z.ZodString>;
|
|
1128
360
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
361
|
+
}, z.core.$strip>>>;
|
|
362
|
+
outcome: z.ZodOptional<z.ZodObject<{
|
|
363
|
+
success: z.ZodOptional<z.ZodBoolean>;
|
|
364
|
+
score: z.ZodOptional<z.ZodNumber>;
|
|
365
|
+
metrics: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
|
|
366
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
367
|
+
detail: z.ZodOptional<z.ZodString>;
|
|
368
|
+
observedAt: z.ZodOptional<z.ZodString>;
|
|
369
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
370
|
+
}, z.core.$strip>>;
|
|
371
|
+
split: z.ZodOptional<z.ZodEnum<{
|
|
372
|
+
dev: "dev";
|
|
373
|
+
holdout: "holdout";
|
|
374
|
+
test: "test";
|
|
375
|
+
train: "train";
|
|
376
|
+
}>>;
|
|
377
|
+
tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
378
|
+
createdAt: z.ZodString;
|
|
379
|
+
updatedAt: z.ZodOptional<z.ZodString>;
|
|
380
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
1129
381
|
}, z.core.$strip>;
|
|
1130
382
|
declare const FeedbackIngestResponseSchema: z.ZodObject<{
|
|
1131
|
-
|
|
1132
|
-
|
|
383
|
+
id: z.ZodString;
|
|
384
|
+
persisted: z.ZodBoolean;
|
|
1133
385
|
}, z.core.$strip>;
|
|
1134
386
|
type TraceEvent = z.infer<typeof TraceEventSchema>;
|
|
1135
387
|
type TracesIngestRequest = z.infer<typeof TracesIngestRequestSchema>;
|
|
@@ -1137,11 +389,11 @@ type TracesIngestResponse = z.infer<typeof TracesIngestResponseSchema>;
|
|
|
1137
389
|
type FeedbackTrajectory = z.infer<typeof FeedbackTrajectorySchema>;
|
|
1138
390
|
type FeedbackIngestResponse = z.infer<typeof FeedbackIngestResponseSchema>;
|
|
1139
391
|
declare const ErrorResponseSchema: z.ZodObject<{
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
392
|
+
error: z.ZodObject<{
|
|
393
|
+
code: z.ZodString;
|
|
394
|
+
message: z.ZodString;
|
|
395
|
+
details: z.ZodOptional<z.ZodUnknown>;
|
|
396
|
+
}, z.core.$strip>;
|
|
1145
397
|
}, z.core.$strip>;
|
|
1146
398
|
type RubricDimension = z.infer<typeof RubricDimensionSchema>;
|
|
1147
399
|
type FailureMode = z.infer<typeof FailureModeSchema>;
|
|
@@ -1162,33 +414,22 @@ declare const WIRE_VERSION = "1.0.0";
|
|
|
1162
414
|
* if the rubricVersion matches, the rubric was identical.
|
|
1163
415
|
*/
|
|
1164
416
|
declare function hashRubric(rubric: Rubric): string;
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
* Pure handler functions — the "business logic" behind every wire-protocol
|
|
1168
|
-
* method. The HTTP server (`server.ts`) and the stdio RPC (`rpc.ts`) both
|
|
1169
|
-
* call these. Tests call these directly without spinning a server.
|
|
1170
|
-
*
|
|
1171
|
-
* Each handler:
|
|
1172
|
-
* - Takes a parsed request (already Zod-validated by the transport).
|
|
1173
|
-
* - Returns a result that matches the response schema.
|
|
1174
|
-
* - Throws `WireError` for caller-fixable errors (404, 400, 422).
|
|
1175
|
-
* - Lets unexpected errors bubble — the transport maps them to 500.
|
|
1176
|
-
*/
|
|
1177
|
-
|
|
417
|
+
//#endregion
|
|
418
|
+
//#region src/wire/handlers.d.ts
|
|
1178
419
|
/** Caller-fixable error. The transport renders this to 4xx + ErrorResponse. */
|
|
1179
420
|
declare class WireError extends Error {
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
421
|
+
readonly code: string;
|
|
422
|
+
readonly status: number;
|
|
423
|
+
readonly details?: unknown;
|
|
424
|
+
constructor(code: string, message: string, status?: number, details?: unknown);
|
|
1184
425
|
}
|
|
1185
426
|
interface HandleJudgeOptions {
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
427
|
+
costLedger?: CostLedgerHandle;
|
|
428
|
+
costPhase?: string;
|
|
429
|
+
llm?: LlmClientOptions;
|
|
430
|
+
defaultModel?: string;
|
|
431
|
+
routeRequirements?: LlmRouteRequirements;
|
|
432
|
+
signal?: AbortSignal;
|
|
1192
433
|
}
|
|
1193
434
|
declare function handleJudge(req: JudgeRequest, options?: HandleJudgeOptions): Promise<JudgeResult>;
|
|
1194
435
|
declare function handleListRubrics(): ListRubricsResponse;
|
|
@@ -1202,8 +443,8 @@ declare function handleVersion(): VersionResponse;
|
|
|
1202
443
|
* stores.
|
|
1203
444
|
*/
|
|
1204
445
|
interface IngestionStores {
|
|
1205
|
-
|
|
1206
|
-
|
|
446
|
+
traceStore?: TraceStore;
|
|
447
|
+
feedbackStore?: FeedbackTrajectoryStore;
|
|
1207
448
|
}
|
|
1208
449
|
/**
|
|
1209
450
|
* `POST /v1/traces/ingest` — accept a batch of `TraceEvent`s from the
|
|
@@ -1222,122 +463,408 @@ declare function handleTracesIngest(req: TracesIngestRequest, stores: IngestionS
|
|
|
1222
463
|
* replaces the prior record.
|
|
1223
464
|
*/
|
|
1224
465
|
declare function handleFeedbackIngest(req: FeedbackTrajectory, stores: IngestionStores): Promise<FeedbackIngestResponse>;
|
|
1225
|
-
|
|
466
|
+
//#endregion
|
|
467
|
+
//#region node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/specification-extension.d.ts
|
|
468
|
+
type IExtensionName = `x-${string}`;
|
|
469
|
+
type IExtensionType = any;
|
|
470
|
+
type ISpecificationExtension = {
|
|
471
|
+
[extensionName: IExtensionName]: IExtensionType;
|
|
472
|
+
};
|
|
473
|
+
//#endregion
|
|
474
|
+
//#region node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/oas-common.d.ts
|
|
475
|
+
interface ServerObject extends ISpecificationExtension {
|
|
476
|
+
url: string;
|
|
477
|
+
description?: string;
|
|
478
|
+
variables?: {
|
|
479
|
+
[v: string]: ServerVariableObject;
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
interface ServerVariableObject extends ISpecificationExtension {
|
|
483
|
+
enum?: string[] | boolean[] | number[];
|
|
484
|
+
default: string | boolean | number;
|
|
485
|
+
description?: string;
|
|
486
|
+
}
|
|
487
|
+
//#endregion
|
|
488
|
+
//#region node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/openapi31.d.ts
|
|
489
|
+
interface OpenAPIObject extends ISpecificationExtension {
|
|
490
|
+
openapi: string;
|
|
491
|
+
info: InfoObject;
|
|
492
|
+
servers?: ServerObject[];
|
|
493
|
+
paths?: PathsObject;
|
|
494
|
+
components?: ComponentsObject;
|
|
495
|
+
security?: SecurityRequirementObject[];
|
|
496
|
+
tags?: TagObject[];
|
|
497
|
+
externalDocs?: ExternalDocumentationObject;
|
|
498
|
+
webhooks?: PathsObject;
|
|
499
|
+
}
|
|
500
|
+
interface InfoObject extends ISpecificationExtension {
|
|
501
|
+
title: string;
|
|
502
|
+
description?: string;
|
|
503
|
+
termsOfService?: string;
|
|
504
|
+
contact?: ContactObject;
|
|
505
|
+
license?: LicenseObject;
|
|
506
|
+
version: string;
|
|
507
|
+
}
|
|
508
|
+
interface ContactObject extends ISpecificationExtension {
|
|
509
|
+
name?: string;
|
|
510
|
+
url?: string;
|
|
511
|
+
email?: string;
|
|
512
|
+
}
|
|
513
|
+
interface LicenseObject extends ISpecificationExtension {
|
|
514
|
+
name: string;
|
|
515
|
+
identifier?: string;
|
|
516
|
+
url?: string;
|
|
517
|
+
}
|
|
518
|
+
interface ComponentsObject extends ISpecificationExtension {
|
|
519
|
+
schemas?: {
|
|
520
|
+
[schema: string]: SchemaObject | ReferenceObject;
|
|
521
|
+
};
|
|
522
|
+
responses?: {
|
|
523
|
+
[response: string]: ResponseObject | ReferenceObject;
|
|
524
|
+
};
|
|
525
|
+
parameters?: {
|
|
526
|
+
[parameter: string]: ParameterObject | ReferenceObject;
|
|
527
|
+
};
|
|
528
|
+
examples?: {
|
|
529
|
+
[example: string]: ExampleObject | ReferenceObject;
|
|
530
|
+
};
|
|
531
|
+
requestBodies?: {
|
|
532
|
+
[request: string]: RequestBodyObject | ReferenceObject;
|
|
533
|
+
};
|
|
534
|
+
headers?: {
|
|
535
|
+
[header: string]: HeaderObject | ReferenceObject;
|
|
536
|
+
};
|
|
537
|
+
securitySchemes?: {
|
|
538
|
+
[securityScheme: string]: SecuritySchemeObject | ReferenceObject;
|
|
539
|
+
};
|
|
540
|
+
links?: {
|
|
541
|
+
[link: string]: LinkObject | ReferenceObject;
|
|
542
|
+
};
|
|
543
|
+
callbacks?: {
|
|
544
|
+
[callback: string]: CallbackObject | ReferenceObject;
|
|
545
|
+
};
|
|
546
|
+
pathItems?: {
|
|
547
|
+
[pathItem: string]: PathItemObject | ReferenceObject;
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
interface PathsObject extends ISpecificationExtension {
|
|
551
|
+
[path: string]: PathItemObject;
|
|
552
|
+
}
|
|
553
|
+
interface PathItemObject extends ISpecificationExtension {
|
|
554
|
+
$ref?: string;
|
|
555
|
+
summary?: string;
|
|
556
|
+
description?: string;
|
|
557
|
+
get?: OperationObject;
|
|
558
|
+
put?: OperationObject;
|
|
559
|
+
post?: OperationObject;
|
|
560
|
+
delete?: OperationObject;
|
|
561
|
+
options?: OperationObject;
|
|
562
|
+
head?: OperationObject;
|
|
563
|
+
patch?: OperationObject;
|
|
564
|
+
trace?: OperationObject;
|
|
565
|
+
servers?: ServerObject[];
|
|
566
|
+
parameters?: (ParameterObject | ReferenceObject)[];
|
|
567
|
+
}
|
|
568
|
+
interface OperationObject extends ISpecificationExtension {
|
|
569
|
+
tags?: string[];
|
|
570
|
+
summary?: string;
|
|
571
|
+
description?: string;
|
|
572
|
+
externalDocs?: ExternalDocumentationObject;
|
|
573
|
+
operationId?: string;
|
|
574
|
+
parameters?: (ParameterObject | ReferenceObject)[];
|
|
575
|
+
requestBody?: RequestBodyObject | ReferenceObject;
|
|
576
|
+
responses?: ResponsesObject;
|
|
577
|
+
callbacks?: CallbacksObject;
|
|
578
|
+
deprecated?: boolean;
|
|
579
|
+
security?: SecurityRequirementObject[];
|
|
580
|
+
servers?: ServerObject[];
|
|
581
|
+
}
|
|
582
|
+
interface ExternalDocumentationObject extends ISpecificationExtension {
|
|
583
|
+
description?: string;
|
|
584
|
+
url: string;
|
|
585
|
+
}
|
|
586
|
+
type ParameterLocation = 'query' | 'header' | 'path' | 'cookie';
|
|
587
|
+
type ParameterStyle = 'matrix' | 'label' | 'form' | 'simple' | 'spaceDelimited' | 'pipeDelimited' | 'deepObject';
|
|
588
|
+
interface BaseParameterObject extends ISpecificationExtension {
|
|
589
|
+
description?: string;
|
|
590
|
+
required?: boolean;
|
|
591
|
+
deprecated?: boolean;
|
|
592
|
+
allowEmptyValue?: boolean;
|
|
593
|
+
style?: ParameterStyle;
|
|
594
|
+
explode?: boolean;
|
|
595
|
+
allowReserved?: boolean;
|
|
596
|
+
schema?: SchemaObject | ReferenceObject;
|
|
597
|
+
examples?: {
|
|
598
|
+
[param: string]: ExampleObject | ReferenceObject;
|
|
599
|
+
};
|
|
600
|
+
example?: any;
|
|
601
|
+
content?: ContentObject;
|
|
602
|
+
}
|
|
603
|
+
interface ParameterObject extends BaseParameterObject {
|
|
604
|
+
name: string;
|
|
605
|
+
in: ParameterLocation;
|
|
606
|
+
}
|
|
607
|
+
interface RequestBodyObject extends ISpecificationExtension {
|
|
608
|
+
description?: string;
|
|
609
|
+
content: ContentObject;
|
|
610
|
+
required?: boolean;
|
|
611
|
+
}
|
|
612
|
+
interface ContentObject {
|
|
613
|
+
[mediatype: string]: MediaTypeObject;
|
|
614
|
+
}
|
|
615
|
+
interface MediaTypeObject extends ISpecificationExtension {
|
|
616
|
+
schema?: SchemaObject | ReferenceObject;
|
|
617
|
+
examples?: ExamplesObject;
|
|
618
|
+
example?: any;
|
|
619
|
+
encoding?: EncodingObject;
|
|
620
|
+
}
|
|
621
|
+
interface EncodingObject extends ISpecificationExtension {
|
|
622
|
+
[property: string]: EncodingPropertyObject | any;
|
|
623
|
+
}
|
|
624
|
+
interface EncodingPropertyObject {
|
|
625
|
+
contentType?: string;
|
|
626
|
+
headers?: {
|
|
627
|
+
[key: string]: HeaderObject | ReferenceObject;
|
|
628
|
+
};
|
|
629
|
+
style?: string;
|
|
630
|
+
explode?: boolean;
|
|
631
|
+
allowReserved?: boolean;
|
|
632
|
+
[key: string]: any;
|
|
633
|
+
}
|
|
634
|
+
interface ResponsesObject extends ISpecificationExtension {
|
|
635
|
+
default?: ResponseObject | ReferenceObject;
|
|
636
|
+
[statuscode: string]: ResponseObject | ReferenceObject | any;
|
|
637
|
+
}
|
|
638
|
+
interface ResponseObject extends ISpecificationExtension {
|
|
639
|
+
description: string;
|
|
640
|
+
headers?: HeadersObject;
|
|
641
|
+
content?: ContentObject;
|
|
642
|
+
links?: LinksObject;
|
|
643
|
+
}
|
|
644
|
+
interface CallbacksObject extends ISpecificationExtension {
|
|
645
|
+
[name: string]: CallbackObject | ReferenceObject | any;
|
|
646
|
+
}
|
|
647
|
+
interface CallbackObject extends ISpecificationExtension {
|
|
648
|
+
[name: string]: PathItemObject | any;
|
|
649
|
+
}
|
|
650
|
+
interface HeadersObject {
|
|
651
|
+
[name: string]: HeaderObject | ReferenceObject;
|
|
652
|
+
}
|
|
653
|
+
interface ExampleObject {
|
|
654
|
+
summary?: string;
|
|
655
|
+
description?: string;
|
|
656
|
+
value?: any;
|
|
657
|
+
externalValue?: string;
|
|
658
|
+
[property: string]: any;
|
|
659
|
+
}
|
|
660
|
+
interface LinksObject {
|
|
661
|
+
[name: string]: LinkObject | ReferenceObject;
|
|
662
|
+
}
|
|
663
|
+
interface LinkObject extends ISpecificationExtension {
|
|
664
|
+
operationRef?: string;
|
|
665
|
+
operationId?: string;
|
|
666
|
+
parameters?: LinkParametersObject;
|
|
667
|
+
requestBody?: any | string;
|
|
668
|
+
description?: string;
|
|
669
|
+
server?: ServerObject;
|
|
670
|
+
[property: string]: any;
|
|
671
|
+
}
|
|
672
|
+
interface LinkParametersObject {
|
|
673
|
+
[name: string]: any | string;
|
|
674
|
+
}
|
|
675
|
+
interface HeaderObject extends BaseParameterObject {
|
|
676
|
+
$ref?: string;
|
|
677
|
+
}
|
|
678
|
+
interface TagObject extends ISpecificationExtension {
|
|
679
|
+
name: string;
|
|
680
|
+
description?: string;
|
|
681
|
+
externalDocs?: ExternalDocumentationObject;
|
|
682
|
+
[extension: string]: any;
|
|
683
|
+
}
|
|
684
|
+
interface ExamplesObject {
|
|
685
|
+
[name: string]: ExampleObject | ReferenceObject;
|
|
686
|
+
}
|
|
687
|
+
interface ReferenceObject {
|
|
688
|
+
$ref: string;
|
|
689
|
+
summary?: string;
|
|
690
|
+
description?: string;
|
|
691
|
+
}
|
|
692
|
+
type SchemaObjectType = 'integer' | 'number' | 'string' | 'boolean' | 'object' | 'null' | 'array';
|
|
693
|
+
interface SchemaObject extends ISpecificationExtension {
|
|
694
|
+
$ref?: string;
|
|
695
|
+
discriminator?: DiscriminatorObject;
|
|
696
|
+
readOnly?: boolean;
|
|
697
|
+
writeOnly?: boolean;
|
|
698
|
+
xml?: XmlObject;
|
|
699
|
+
externalDocs?: ExternalDocumentationObject;
|
|
700
|
+
example?: any;
|
|
701
|
+
examples?: any[];
|
|
702
|
+
deprecated?: boolean;
|
|
703
|
+
type?: SchemaObjectType | SchemaObjectType[];
|
|
704
|
+
format?: 'int32' | 'int64' | 'float' | 'double' | 'byte' | 'binary' | 'date' | 'date-time' | 'password' | string;
|
|
705
|
+
allOf?: (SchemaObject | ReferenceObject)[];
|
|
706
|
+
oneOf?: (SchemaObject | ReferenceObject)[];
|
|
707
|
+
anyOf?: (SchemaObject | ReferenceObject)[];
|
|
708
|
+
not?: SchemaObject | ReferenceObject;
|
|
709
|
+
items?: SchemaObject | ReferenceObject;
|
|
710
|
+
properties?: {
|
|
711
|
+
[propertyName: string]: SchemaObject | ReferenceObject;
|
|
712
|
+
};
|
|
713
|
+
additionalProperties?: SchemaObject | ReferenceObject | boolean;
|
|
714
|
+
propertyNames?: SchemaObject | ReferenceObject;
|
|
715
|
+
description?: string;
|
|
716
|
+
default?: any;
|
|
717
|
+
title?: string;
|
|
718
|
+
multipleOf?: number;
|
|
719
|
+
maximum?: number;
|
|
720
|
+
const?: any;
|
|
721
|
+
exclusiveMaximum?: number;
|
|
722
|
+
minimum?: number;
|
|
723
|
+
exclusiveMinimum?: number;
|
|
724
|
+
maxLength?: number;
|
|
725
|
+
minLength?: number;
|
|
726
|
+
pattern?: string;
|
|
727
|
+
maxItems?: number;
|
|
728
|
+
minItems?: number;
|
|
729
|
+
uniqueItems?: boolean;
|
|
730
|
+
maxProperties?: number;
|
|
731
|
+
minProperties?: number;
|
|
732
|
+
required?: string[];
|
|
733
|
+
enum?: any[];
|
|
734
|
+
prefixItems?: (SchemaObject | ReferenceObject)[];
|
|
735
|
+
contentMediaType?: string;
|
|
736
|
+
contentEncoding?: string;
|
|
737
|
+
}
|
|
738
|
+
interface DiscriminatorObject {
|
|
739
|
+
propertyName: string;
|
|
740
|
+
mapping?: {
|
|
741
|
+
[key: string]: string;
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
interface XmlObject extends ISpecificationExtension {
|
|
745
|
+
name?: string;
|
|
746
|
+
namespace?: string;
|
|
747
|
+
prefix?: string;
|
|
748
|
+
attribute?: boolean;
|
|
749
|
+
wrapped?: boolean;
|
|
750
|
+
}
|
|
751
|
+
type SecuritySchemeType = 'apiKey' | 'http' | 'oauth2' | 'openIdConnect';
|
|
752
|
+
interface SecuritySchemeObject extends ISpecificationExtension {
|
|
753
|
+
type: SecuritySchemeType;
|
|
754
|
+
description?: string;
|
|
755
|
+
name?: string;
|
|
756
|
+
in?: string;
|
|
757
|
+
scheme?: string;
|
|
758
|
+
bearerFormat?: string;
|
|
759
|
+
flows?: OAuthFlowsObject;
|
|
760
|
+
openIdConnectUrl?: string;
|
|
761
|
+
}
|
|
762
|
+
interface OAuthFlowsObject extends ISpecificationExtension {
|
|
763
|
+
implicit?: OAuthFlowObject;
|
|
764
|
+
password?: OAuthFlowObject;
|
|
765
|
+
clientCredentials?: OAuthFlowObject;
|
|
766
|
+
authorizationCode?: OAuthFlowObject;
|
|
767
|
+
}
|
|
768
|
+
interface OAuthFlowObject extends ISpecificationExtension {
|
|
769
|
+
authorizationUrl?: string;
|
|
770
|
+
tokenUrl?: string;
|
|
771
|
+
refreshUrl?: string;
|
|
772
|
+
scopes: ScopesObject;
|
|
773
|
+
}
|
|
774
|
+
interface ScopesObject extends ISpecificationExtension {
|
|
775
|
+
[scope: string]: any;
|
|
776
|
+
}
|
|
777
|
+
interface SecurityRequirementObject {
|
|
778
|
+
[name: string]: string[];
|
|
779
|
+
}
|
|
780
|
+
//#endregion
|
|
781
|
+
//#region src/wire/openapi.d.ts
|
|
1226
782
|
declare function buildOpenApi(packageVersion: string): OpenAPIObject;
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
* stdio RPC transport.
|
|
1230
|
-
*
|
|
1231
|
-
* For batch / cron use without a running server. The Python client falls
|
|
1232
|
-
* back to this when no server is reachable.
|
|
1233
|
-
*
|
|
1234
|
-
* Protocol (line-delimited JSON over stdin/stdout):
|
|
1235
|
-
* IN: one JSON object on stdin: {"method":"judge","params":{...}}
|
|
1236
|
-
* OUT: one JSON object on stdout: {"result":{...}} or {"error":{...}}
|
|
1237
|
-
*
|
|
1238
|
-
* One request per process invocation. To pipeline many calls, the client
|
|
1239
|
-
* writes JSONL to stdin and reads JSONL from stdout — see batch mode below.
|
|
1240
|
-
*/
|
|
1241
|
-
|
|
783
|
+
//#endregion
|
|
784
|
+
//#region src/wire/rpc.d.ts
|
|
1242
785
|
interface RpcRequest {
|
|
1243
|
-
|
|
1244
|
-
|
|
786
|
+
method: 'judge' | 'listRubrics' | 'version';
|
|
787
|
+
params?: unknown;
|
|
1245
788
|
}
|
|
1246
789
|
interface RpcSuccess {
|
|
1247
|
-
|
|
790
|
+
result: unknown;
|
|
1248
791
|
}
|
|
1249
792
|
interface RpcError {
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
793
|
+
error: {
|
|
794
|
+
code: string;
|
|
795
|
+
message: string;
|
|
796
|
+
details?: unknown;
|
|
797
|
+
};
|
|
1255
798
|
}
|
|
1256
799
|
interface RpcOptions {
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
800
|
+
llm?: LlmClientOptions;
|
|
801
|
+
judgeModel?: string;
|
|
802
|
+
llmRouteRequirements?: LlmRouteRequirements;
|
|
1260
803
|
}
|
|
1261
804
|
declare function dispatchRpc(req: RpcRequest, options?: RpcOptions): Promise<RpcSuccess | RpcError>;
|
|
1262
805
|
/** Read one JSON request from stdin, write one JSON response to stdout. */
|
|
1263
806
|
declare function runRpcOnce(method?: string, options?: RpcOptions): Promise<number>;
|
|
1264
807
|
/** Read JSONL requests from stdin, write JSONL responses to stdout. */
|
|
1265
808
|
declare function runRpcBatch(method?: string, options?: RpcOptions): Promise<number>;
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
* Built-in rubrics shipped with agent-eval.
|
|
1269
|
-
*
|
|
1270
|
-
* A rubric is a set of scoring axes plus a system prompt that tells the
|
|
1271
|
-
* judging LLM how to grade against those axes. Built-in rubrics are
|
|
1272
|
-
* curated for use cases that recur across Tangle projects — call them
|
|
1273
|
-
* by name from any client.
|
|
1274
|
-
*
|
|
1275
|
-
* Adding a rubric:
|
|
1276
|
-
* 1. Define the Rubric object below with a clear `description` and
|
|
1277
|
-
* named `dimensions`.
|
|
1278
|
-
* 2. Register it in `BUILTIN_RUBRICS` at the bottom.
|
|
1279
|
-
* 3. Add a test in `tests/wire/rubrics.test.ts`.
|
|
1280
|
-
*
|
|
1281
|
-
* Custom rubrics: callers pass `rubric` inline to /v1/judge instead of
|
|
1282
|
-
* `rubricName` — see schemas.ts.
|
|
1283
|
-
*/
|
|
1284
|
-
|
|
809
|
+
//#endregion
|
|
810
|
+
//#region src/wire/rubrics.d.ts
|
|
1285
811
|
declare const BUILTIN_RUBRICS: Record<string, Rubric>;
|
|
1286
812
|
/** Get a built-in rubric by name, or undefined. */
|
|
1287
813
|
declare function getBuiltinRubric(name: string): Rubric | undefined;
|
|
1288
814
|
/** List built-in rubrics with their stable versions. */
|
|
1289
815
|
declare function listBuiltinRubrics(): {
|
|
1290
|
-
|
|
816
|
+
name: string;
|
|
817
|
+
description: string;
|
|
818
|
+
dimensions: {
|
|
819
|
+
id: string;
|
|
1291
820
|
description: string;
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
}[];
|
|
1297
|
-
failureModes: string[];
|
|
1298
|
-
rubricVersion: string;
|
|
821
|
+
weight: number;
|
|
822
|
+
}[];
|
|
823
|
+
failureModes: string[];
|
|
824
|
+
rubricVersion: string;
|
|
1299
825
|
}[];
|
|
1300
|
-
|
|
826
|
+
//#endregion
|
|
827
|
+
//#region src/wire/server.d.ts
|
|
1301
828
|
interface CreateAppOptions {
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
}
|
|
1323
|
-
declare function createApp(opts?: CreateAppOptions): Hono<
|
|
829
|
+
/** Stores wired to the ingestion endpoints. */
|
|
830
|
+
stores?: IngestionStores;
|
|
831
|
+
/** Model provider used by `/v1/judge`. */
|
|
832
|
+
llm?: LlmClientOptions;
|
|
833
|
+
/** Default judge model when a request does not provide one. */
|
|
834
|
+
judgeModel?: string;
|
|
835
|
+
/** Model route checks applied before every judge call. */
|
|
836
|
+
llmRouteRequirements?: LlmRouteRequirements;
|
|
837
|
+
/**
|
|
838
|
+
* Bearer-token auth. When provided, every endpoint EXCEPT `/healthz`
|
|
839
|
+
* and `/v1/version` requires `Authorization: Bearer <token>`. The
|
|
840
|
+
* token may be a static string OR a function for time-bounded /
|
|
841
|
+
* rotating tokens.
|
|
842
|
+
*
|
|
843
|
+
* Recommended for any server that accepts ingestion writes from the
|
|
844
|
+
* public internet. Read-only deployments may omit it.
|
|
845
|
+
*/
|
|
846
|
+
auth?: {
|
|
847
|
+
bearer: string | ((token: string) => boolean | Promise<boolean>);
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
declare function createApp(opts?: CreateAppOptions): Hono<import("hono/types").BlankEnv, import("hono/types").BlankSchema, "/">;
|
|
1324
851
|
interface ServeOptions extends CreateAppOptions {
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
852
|
+
/** Default 5005. */
|
|
853
|
+
port?: number;
|
|
854
|
+
/** Default '127.0.0.1'. Set to '0.0.0.0' to listen on all interfaces. */
|
|
855
|
+
host?: string;
|
|
1329
856
|
}
|
|
1330
857
|
declare function startServer(opts?: ServeOptions): ServerType;
|
|
1331
858
|
interface StartedServer {
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
859
|
+
server: ServerType;
|
|
860
|
+
/** The OS-assigned port. When opts.port was 0, this is the actual port the
|
|
861
|
+
* kernel bound — callers that need to dial back (smoke tests, sidecars
|
|
862
|
+
* registering with a parent) read this rather than guessing a free port. */
|
|
863
|
+
port: number;
|
|
864
|
+
/** Resolved host the server bound to (defaults to 127.0.0.1). */
|
|
865
|
+
host: string;
|
|
866
|
+
/** Close the server. Resolves once active connections have drained. */
|
|
867
|
+
close(): Promise<void>;
|
|
1341
868
|
}
|
|
1342
869
|
/**
|
|
1343
870
|
* Promise-returning variant of `startServer` that resolves once the server is
|
|
@@ -1345,5 +872,6 @@ interface StartedServer {
|
|
|
1345
872
|
* (`startServerAsync({ port: 0 })`) and any caller that needs to dial back.
|
|
1346
873
|
*/
|
|
1347
874
|
declare function startServerAsync(opts?: ServeOptions): Promise<StartedServer>;
|
|
1348
|
-
|
|
1349
|
-
export { BUILTIN_RUBRICS,
|
|
875
|
+
//#endregion
|
|
876
|
+
export { BUILTIN_RUBRICS, ErrorResponse, ErrorResponseSchema, FailureMode, FailureModeSchema, FeedbackAttemptSchema, FeedbackIngestResponse, FeedbackIngestResponseSchema, FeedbackLabelSchema, FeedbackTrajectory, FeedbackTrajectorySchema, HandleJudgeOptions, HealthResponseSchema, IngestionStores, JudgeRequest, JudgeRequestSchema, JudgeResult, JudgeResultSchema, ListRubricsResponse, ListRubricsResponseSchema, type RpcOptions, Rubric, RubricDimension, RubricDimensionSchema, RubricInfo, RubricInfoSchema, RubricSchema, type ServeOptions, type StartedServer, TraceEvent, TraceEventSchema, TracesIngestRequest, TracesIngestRequestSchema, TracesIngestResponse, TracesIngestResponseSchema, VersionResponse, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
|
|
877
|
+
//# sourceMappingURL=index.d.ts.map
|