@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
//#region src/trace/emitter.ts
|
|
2
|
+
var TraceEmitter = class {
|
|
3
|
+
store;
|
|
4
|
+
stack = [];
|
|
5
|
+
_runId;
|
|
6
|
+
now;
|
|
7
|
+
id;
|
|
8
|
+
hooks;
|
|
9
|
+
hookErrors;
|
|
10
|
+
constructor(store, options = {}) {
|
|
11
|
+
this.store = store;
|
|
12
|
+
this.now = options.now ?? (() => Date.now());
|
|
13
|
+
this.id = options.id ?? (() => cryptoRandomId());
|
|
14
|
+
this._runId = options.runId ?? this.id();
|
|
15
|
+
this.hooks = options.onRunComplete ?? [];
|
|
16
|
+
this.hookErrors = options.hookErrors ?? "swallow";
|
|
17
|
+
}
|
|
18
|
+
get runId() {
|
|
19
|
+
return this._runId;
|
|
20
|
+
}
|
|
21
|
+
get traceStore() {
|
|
22
|
+
return this.store;
|
|
23
|
+
}
|
|
24
|
+
/** Append a hook after construction (e.g. attach the trace analyst). */
|
|
25
|
+
addRunCompleteHook(hook) {
|
|
26
|
+
this.hooks.push(hook);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Begin a Run.
|
|
30
|
+
*
|
|
31
|
+
* `scenarioId` is required on the persisted Run shape — every Run downstream
|
|
32
|
+
* gets a non-empty scenarioId so filters and aggregations stay simple — but
|
|
33
|
+
* the INPUT here accepts it as optional. When omitted, startRun substitutes
|
|
34
|
+
* a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so
|
|
35
|
+
* runtime / operator / meta-eval runs that have no curated-scenario corpus
|
|
36
|
+
* to anchor to don't have to invent placeholder strings at the call site.
|
|
37
|
+
*/
|
|
38
|
+
async startRun(run) {
|
|
39
|
+
const scenarioId = run.scenarioId ?? run.layer ?? run.tags?.kind ?? "runtime";
|
|
40
|
+
const full = {
|
|
41
|
+
...run,
|
|
42
|
+
scenarioId,
|
|
43
|
+
runId: this._runId,
|
|
44
|
+
startedAt: this.now(),
|
|
45
|
+
status: "running"
|
|
46
|
+
};
|
|
47
|
+
await this.store.appendRun(full);
|
|
48
|
+
return full;
|
|
49
|
+
}
|
|
50
|
+
async endRun(outcome) {
|
|
51
|
+
const status = outcome?.pass === false ? "failed" : "completed";
|
|
52
|
+
await this.store.updateRun(this._runId, {
|
|
53
|
+
endedAt: this.now(),
|
|
54
|
+
status,
|
|
55
|
+
outcome
|
|
56
|
+
});
|
|
57
|
+
await this.runHooks({
|
|
58
|
+
runId: this._runId,
|
|
59
|
+
emitter: this,
|
|
60
|
+
store: this.store,
|
|
61
|
+
outcome,
|
|
62
|
+
status
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
async abortRun(reason) {
|
|
66
|
+
const outcome = {
|
|
67
|
+
pass: false,
|
|
68
|
+
notes: reason
|
|
69
|
+
};
|
|
70
|
+
await this.store.updateRun(this._runId, {
|
|
71
|
+
endedAt: this.now(),
|
|
72
|
+
status: "aborted",
|
|
73
|
+
outcome
|
|
74
|
+
});
|
|
75
|
+
await this.runHooks({
|
|
76
|
+
runId: this._runId,
|
|
77
|
+
emitter: this,
|
|
78
|
+
store: this.store,
|
|
79
|
+
outcome,
|
|
80
|
+
status: "aborted"
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
async runHooks(ctx) {
|
|
84
|
+
for (const hook of this.hooks) try {
|
|
85
|
+
await hook(ctx);
|
|
86
|
+
} catch (err) {
|
|
87
|
+
if (this.hookErrors === "throw") throw err;
|
|
88
|
+
try {
|
|
89
|
+
await this.store.appendEvent({
|
|
90
|
+
eventId: this.id(),
|
|
91
|
+
runId: this._runId,
|
|
92
|
+
kind: "log",
|
|
93
|
+
timestamp: this.now(),
|
|
94
|
+
payload: {
|
|
95
|
+
source: "run_complete_hook",
|
|
96
|
+
error: err instanceof Error ? err.message : String(err)
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
} catch {}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
async span(init) {
|
|
103
|
+
const spanId = this.id();
|
|
104
|
+
const span = {
|
|
105
|
+
spanId,
|
|
106
|
+
parentSpanId: init.parentSpanId ?? this.stack[this.stack.length - 1],
|
|
107
|
+
runId: this._runId,
|
|
108
|
+
startedAt: this.now(),
|
|
109
|
+
...init
|
|
110
|
+
};
|
|
111
|
+
await this.store.appendSpan(span);
|
|
112
|
+
this.stack.push(spanId);
|
|
113
|
+
return this.handle(span);
|
|
114
|
+
}
|
|
115
|
+
handle(span) {
|
|
116
|
+
return {
|
|
117
|
+
span,
|
|
118
|
+
end: async (patch) => {
|
|
119
|
+
const endedAt = this.now();
|
|
120
|
+
await this.store.updateSpan(span.spanId, {
|
|
121
|
+
endedAt,
|
|
122
|
+
status: "ok",
|
|
123
|
+
...patch
|
|
124
|
+
});
|
|
125
|
+
this.pop(span.spanId);
|
|
126
|
+
},
|
|
127
|
+
fail: async (error, patch) => {
|
|
128
|
+
const endedAt = this.now();
|
|
129
|
+
const errStr = error instanceof Error ? error.message : error;
|
|
130
|
+
await this.store.updateSpan(span.spanId, {
|
|
131
|
+
endedAt,
|
|
132
|
+
status: "error",
|
|
133
|
+
error: errStr,
|
|
134
|
+
...patch
|
|
135
|
+
});
|
|
136
|
+
this.pop(span.spanId);
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
pop(spanId) {
|
|
141
|
+
const idx = this.stack.lastIndexOf(spanId);
|
|
142
|
+
if (idx >= 0) this.stack.splice(idx, 1);
|
|
143
|
+
}
|
|
144
|
+
llm(init) {
|
|
145
|
+
return this.span({
|
|
146
|
+
kind: "llm",
|
|
147
|
+
...init
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
tool(init) {
|
|
151
|
+
return this.span({
|
|
152
|
+
kind: "tool",
|
|
153
|
+
...init
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
retrieval(init) {
|
|
157
|
+
return this.span({
|
|
158
|
+
kind: "retrieval",
|
|
159
|
+
...init
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
async recordJudge(verdict) {
|
|
163
|
+
const spanId = this.id();
|
|
164
|
+
const now = this.now();
|
|
165
|
+
const full = {
|
|
166
|
+
spanId,
|
|
167
|
+
runId: this._runId,
|
|
168
|
+
kind: "judge",
|
|
169
|
+
startedAt: now,
|
|
170
|
+
endedAt: now,
|
|
171
|
+
status: "ok",
|
|
172
|
+
...verdict
|
|
173
|
+
};
|
|
174
|
+
await this.store.appendSpan(full);
|
|
175
|
+
return full;
|
|
176
|
+
}
|
|
177
|
+
sandbox(init) {
|
|
178
|
+
return this.span({
|
|
179
|
+
kind: "sandbox",
|
|
180
|
+
...init
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
async emit(event) {
|
|
184
|
+
const full = {
|
|
185
|
+
eventId: this.id(),
|
|
186
|
+
runId: this._runId,
|
|
187
|
+
spanId: event.spanId ?? this.stack[this.stack.length - 1],
|
|
188
|
+
kind: event.kind,
|
|
189
|
+
timestamp: this.now(),
|
|
190
|
+
payload: event.payload ?? {}
|
|
191
|
+
};
|
|
192
|
+
await this.store.appendEvent(full);
|
|
193
|
+
return full;
|
|
194
|
+
}
|
|
195
|
+
async recordBudget(entry) {
|
|
196
|
+
const full = {
|
|
197
|
+
runId: this._runId,
|
|
198
|
+
timestamp: entry.timestamp ?? this.now(),
|
|
199
|
+
dimension: entry.dimension,
|
|
200
|
+
limit: entry.limit,
|
|
201
|
+
consumed: entry.consumed,
|
|
202
|
+
remaining: entry.remaining,
|
|
203
|
+
breached: entry.breached,
|
|
204
|
+
spanId: entry.spanId ?? this.stack[this.stack.length - 1]
|
|
205
|
+
};
|
|
206
|
+
await this.store.appendBudgetEntry(full);
|
|
207
|
+
if (full.breached) await this.emit({
|
|
208
|
+
kind: "budget_breach",
|
|
209
|
+
spanId: full.spanId,
|
|
210
|
+
payload: {
|
|
211
|
+
dimension: full.dimension,
|
|
212
|
+
limit: full.limit,
|
|
213
|
+
consumed: full.consumed
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
return full;
|
|
217
|
+
}
|
|
218
|
+
async recordArtifact(artifact) {
|
|
219
|
+
const full = {
|
|
220
|
+
artifactId: this.id(),
|
|
221
|
+
runId: this._runId,
|
|
222
|
+
...artifact
|
|
223
|
+
};
|
|
224
|
+
await this.store.appendArtifact(full);
|
|
225
|
+
return full;
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
|
|
229
|
+
* Returns the fn's return value. Use this for the 95% case.
|
|
230
|
+
*/
|
|
231
|
+
async within(init, fn) {
|
|
232
|
+
const handle = await this.span(init);
|
|
233
|
+
try {
|
|
234
|
+
const result = await fn(handle);
|
|
235
|
+
await handle.end();
|
|
236
|
+
return result;
|
|
237
|
+
} catch (err) {
|
|
238
|
+
await handle.fail(err instanceof Error ? err : String(err));
|
|
239
|
+
throw err;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
};
|
|
243
|
+
function cryptoRandomId() {
|
|
244
|
+
if (typeof globalThis.crypto?.randomUUID === "function") return globalThis.crypto.randomUUID();
|
|
245
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
246
|
+
}
|
|
247
|
+
/** Helper to build an LLM span handle args object from a provider-shaped response. */
|
|
248
|
+
function llmSpanFromProvider(args) {
|
|
249
|
+
return {
|
|
250
|
+
name: args.name ?? args.model,
|
|
251
|
+
model: args.model,
|
|
252
|
+
messages: args.messages,
|
|
253
|
+
output: args.output,
|
|
254
|
+
inputTokens: args.usage?.inputTokens,
|
|
255
|
+
outputTokens: args.usage?.outputTokens,
|
|
256
|
+
cachedTokens: args.usage?.cachedTokens,
|
|
257
|
+
cacheWriteTokens: args.usage?.cacheWriteTokens,
|
|
258
|
+
reasoningTokens: args.usage?.reasoningTokens,
|
|
259
|
+
costUsd: args.costUsd,
|
|
260
|
+
finishReason: args.finishReason
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
//#endregion
|
|
264
|
+
export { llmSpanFromProvider as n, TraceEmitter as t };
|
|
265
|
+
|
|
266
|
+
//# sourceMappingURL=emitter-CPBAhxum.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"emitter-CPBAhxum.js","names":[],"sources":["../src/trace/emitter.ts"],"sourcesContent":["/**\n * TraceEmitter — hierarchical span builder that auto-parents using an\n * internal stack. One emitter per Run; emitters do NOT share state.\n *\n * Convenience methods (`llm`, `tool`, `retrieval`, `judge`, `sandbox`)\n * return a `SpanHandle` with `.end()` / `.fail()` so callers don't\n * have to thread spanIds manually. For async workflows that can't use\n * the stack (e.g. fan-out parallel calls), pass `parentSpanId`\n * explicitly.\n */\n\nimport type {\n Artifact,\n BudgetLedgerEntry,\n EventKind,\n JudgeSpan,\n LlmSpan,\n Message,\n RetrievalSpan,\n Run,\n RunOutcome,\n SandboxSpan,\n Span,\n SpanKind,\n ToolSpan,\n TraceEvent,\n} from './schema'\nimport type { TraceStore } from './store'\n\nexport interface SpanHandle<S extends Span = Span> {\n span: S\n end(patch?: Partial<S>): Promise<void>\n fail(error: string | Error, patch?: Partial<S>): Promise<void>\n}\n\nexport interface RunCompleteHookContext {\n runId: string\n emitter: TraceEmitter\n store: TraceStore\n /** Outcome the caller passed to `endRun` (undefined for `abortRun`). */\n outcome?: RunOutcome\n /** Final run status. */\n status: 'completed' | 'failed' | 'aborted'\n}\n\nexport type RunCompleteHook = (ctx: RunCompleteHookContext) => Promise<void> | void\n\nexport interface TraceEmitterOptions {\n runId?: string\n /** Inject a clock for deterministic tests. */\n now?: () => number\n /** Inject an id generator for deterministic tests. */\n id?: () => string\n /**\n * Hooks fired after `endRun` / `abortRun` writes the final run state.\n * Designed for trace-analyst auto-execution, integrity assertions, and\n * outbound notifications. Hooks run sequentially in the order supplied.\n *\n * By default a hook that throws is swallowed and logged as a `note` event\n * on the run — auto-orchestration must not crash the underlying flow.\n * Set `hookErrors: 'throw'` to propagate.\n */\n onRunComplete?: RunCompleteHook[]\n /** `'swallow'` (default) | `'throw'`. */\n hookErrors?: 'swallow' | 'throw'\n}\n\nexport class TraceEmitter {\n private store: TraceStore\n private stack: string[] = []\n private _runId: string\n private now: () => number\n private id: () => string\n private hooks: RunCompleteHook[]\n private hookErrors: 'swallow' | 'throw'\n\n constructor(store: TraceStore, options: TraceEmitterOptions = {}) {\n this.store = store\n this.now = options.now ?? (() => Date.now())\n this.id = options.id ?? (() => cryptoRandomId())\n this._runId = options.runId ?? this.id()\n this.hooks = options.onRunComplete ?? []\n this.hookErrors = options.hookErrors ?? 'swallow'\n }\n\n get runId(): string {\n return this._runId\n }\n\n get traceStore(): TraceStore {\n return this.store\n }\n\n /** Append a hook after construction (e.g. attach the trace analyst). */\n addRunCompleteHook(hook: RunCompleteHook): void {\n this.hooks.push(hook)\n }\n\n // ── Run lifecycle ──────────────────────────────────────────────────\n\n /**\n * Begin a Run.\n *\n * `scenarioId` is required on the persisted Run shape — every Run downstream\n * gets a non-empty scenarioId so filters and aggregations stay simple — but\n * the INPUT here accepts it as optional. When omitted, startRun substitutes\n * a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so\n * runtime / operator / meta-eval runs that have no curated-scenario corpus\n * to anchor to don't have to invent placeholder strings at the call site.\n */\n async startRun(\n run: Omit<Run, 'runId' | 'scenarioId' | 'startedAt' | 'status'> & { scenarioId?: string },\n ): Promise<Run> {\n const scenarioId = run.scenarioId ?? run.layer ?? run.tags?.kind ?? 'runtime'\n const full: Run = {\n ...run,\n scenarioId,\n runId: this._runId,\n startedAt: this.now(),\n status: 'running',\n }\n await this.store.appendRun(full)\n return full\n }\n\n async endRun(outcome?: RunOutcome): Promise<void> {\n const status: 'completed' | 'failed' = outcome?.pass === false ? 'failed' : 'completed'\n await this.store.updateRun(this._runId, { endedAt: this.now(), status, outcome })\n await this.runHooks({ runId: this._runId, emitter: this, store: this.store, outcome, status })\n }\n\n async abortRun(reason: string): Promise<void> {\n const outcome = { pass: false, notes: reason }\n await this.store.updateRun(this._runId, {\n endedAt: this.now(),\n status: 'aborted',\n outcome,\n })\n await this.runHooks({\n runId: this._runId,\n emitter: this,\n store: this.store,\n outcome,\n status: 'aborted',\n })\n }\n\n private async runHooks(ctx: RunCompleteHookContext): Promise<void> {\n for (const hook of this.hooks) {\n try {\n await hook(ctx)\n } catch (err) {\n if (this.hookErrors === 'throw') throw err\n try {\n await this.store.appendEvent({\n eventId: this.id(),\n runId: this._runId,\n kind: 'log',\n timestamp: this.now(),\n payload: {\n source: 'run_complete_hook',\n error: err instanceof Error ? err.message : String(err),\n },\n })\n } catch {\n // best-effort\n }\n }\n }\n }\n\n // ── Generic span ───────────────────────────────────────────────────\n\n async span<S extends Span = Span>(\n init: {\n kind: SpanKind\n name: string\n parentSpanId?: string\n attributes?: Record<string, unknown>\n } & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>,\n ): Promise<SpanHandle<S>> {\n const spanId = this.id()\n const parent = init.parentSpanId ?? this.stack[this.stack.length - 1]\n const span = {\n spanId,\n parentSpanId: parent,\n runId: this._runId,\n startedAt: this.now(),\n ...init,\n } as unknown as S\n await this.store.appendSpan(span)\n this.stack.push(spanId)\n return this.handle<S>(span)\n }\n\n private handle<S extends Span>(span: S): SpanHandle<S> {\n return {\n span,\n end: async (patch?: Partial<S>) => {\n const endedAt = this.now()\n await this.store.updateSpan(span.spanId, {\n endedAt,\n status: 'ok',\n ...patch,\n } as Partial<Span>)\n this.pop(span.spanId)\n },\n fail: async (error: string | Error, patch?: Partial<S>) => {\n const endedAt = this.now()\n const errStr = error instanceof Error ? error.message : error\n await this.store.updateSpan(span.spanId, {\n endedAt,\n status: 'error',\n error: errStr,\n ...patch,\n } as Partial<Span>)\n this.pop(span.spanId)\n },\n }\n }\n\n private pop(spanId: string): void {\n const idx = this.stack.lastIndexOf(spanId)\n if (idx >= 0) this.stack.splice(idx, 1)\n }\n\n // ── Typed span conveniences ────────────────────────────────────────\n\n llm(\n init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>,\n ): Promise<SpanHandle<LlmSpan>> {\n return this.span<LlmSpan>({ kind: 'llm', ...init })\n }\n\n tool(\n init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>,\n ): Promise<SpanHandle<ToolSpan>> {\n return this.span<ToolSpan>({ kind: 'tool', ...init })\n }\n\n retrieval(\n init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>,\n ): Promise<SpanHandle<RetrievalSpan>> {\n return this.span<RetrievalSpan>({ kind: 'retrieval', ...init })\n }\n\n async recordJudge(\n verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>,\n ): Promise<JudgeSpan> {\n const spanId = this.id()\n const now = this.now()\n const full: JudgeSpan = {\n spanId,\n runId: this._runId,\n kind: 'judge',\n startedAt: now,\n endedAt: now,\n status: 'ok',\n ...verdict,\n }\n await this.store.appendSpan(full)\n return full\n }\n\n sandbox(\n init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>,\n ): Promise<SpanHandle<SandboxSpan>> {\n return this.span<SandboxSpan>({ kind: 'sandbox', ...init })\n }\n\n // ── Events ─────────────────────────────────────────────────────────\n\n async emit(event: {\n kind: EventKind\n spanId?: string\n payload?: Record<string, unknown>\n }): Promise<TraceEvent> {\n const full: TraceEvent = {\n eventId: this.id(),\n runId: this._runId,\n spanId: event.spanId ?? this.stack[this.stack.length - 1],\n kind: event.kind,\n timestamp: this.now(),\n payload: event.payload ?? {},\n }\n await this.store.appendEvent(full)\n return full\n }\n\n // ── Budget ledger ──────────────────────────────────────────────────\n\n async recordBudget(\n entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & { timestamp?: number },\n ): Promise<BudgetLedgerEntry> {\n const full: BudgetLedgerEntry = {\n runId: this._runId,\n timestamp: entry.timestamp ?? this.now(),\n dimension: entry.dimension,\n limit: entry.limit,\n consumed: entry.consumed,\n remaining: entry.remaining,\n breached: entry.breached,\n spanId: entry.spanId ?? this.stack[this.stack.length - 1],\n }\n await this.store.appendBudgetEntry(full)\n if (full.breached) {\n await this.emit({\n kind: 'budget_breach',\n spanId: full.spanId,\n payload: { dimension: full.dimension, limit: full.limit, consumed: full.consumed },\n })\n }\n return full\n }\n\n // ── Artifacts ──────────────────────────────────────────────────────\n\n async recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact> {\n const full: Artifact = { artifactId: this.id(), runId: this._runId, ...artifact }\n await this.store.appendArtifact(full)\n return full\n }\n\n // ── Nested composition ─────────────────────────────────────────────\n\n /**\n * Runs `fn` inside a span; auto-ends on success, auto-fails on throw.\n * Returns the fn's return value. Use this for the 95% case.\n */\n async within<T>(\n init: Parameters<TraceEmitter['span']>[0],\n fn: (handle: SpanHandle) => Promise<T>,\n ): Promise<T> {\n const handle = await this.span(init)\n try {\n const result = await fn(handle)\n await handle.end()\n return result\n } catch (err) {\n await handle.fail(err instanceof Error ? err : String(err))\n throw err\n }\n }\n}\n\n// Helpers -------------------------------------------------------------\n\nfunction cryptoRandomId(): string {\n if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`\n}\n\n/** Helper to build an LLM span handle args object from a provider-shaped response. */\nexport function llmSpanFromProvider(args: {\n name?: string\n model: string\n messages: Message[]\n output: string\n usage?: {\n inputTokens?: number\n outputTokens?: number\n cachedTokens?: number\n cacheWriteTokens?: number\n reasoningTokens?: number\n }\n costUsd?: number\n finishReason?: string\n}): Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'> {\n return {\n name: args.name ?? args.model,\n model: args.model,\n messages: args.messages,\n output: args.output,\n inputTokens: args.usage?.inputTokens,\n outputTokens: args.usage?.outputTokens,\n cachedTokens: args.usage?.cachedTokens,\n cacheWriteTokens: args.usage?.cacheWriteTokens,\n reasoningTokens: args.usage?.reasoningTokens,\n costUsd: args.costUsd,\n finishReason: args.finishReason,\n }\n}\n"],"mappings":";AAmEA,IAAa,eAAb,MAA0B;CACxB;CACA,QAA0B,CAAC;CAC3B;CACA;CACA;CACA;CACA;CAEA,YAAY,OAAmB,UAA+B,CAAC,GAAG;EAChE,KAAK,QAAQ;EACb,KAAK,MAAM,QAAQ,cAAc,KAAK,IAAI;EAC1C,KAAK,KAAK,QAAQ,aAAa,eAAe;EAC9C,KAAK,SAAS,QAAQ,SAAS,KAAK,GAAG;EACvC,KAAK,QAAQ,QAAQ,iBAAiB,CAAC;EACvC,KAAK,aAAa,QAAQ,cAAc;CAC1C;CAEA,IAAI,QAAgB;EAClB,OAAO,KAAK;CACd;CAEA,IAAI,aAAyB;EAC3B,OAAO,KAAK;CACd;;CAGA,mBAAmB,MAA6B;EAC9C,KAAK,MAAM,KAAK,IAAI;CACtB;;;;;;;;;;;CAcA,MAAM,SACJ,KACc;EACd,MAAM,aAAa,IAAI,cAAc,IAAI,SAAS,IAAI,MAAM,QAAQ;EACpE,MAAM,OAAY;GAChB,GAAG;GACH;GACA,OAAO,KAAK;GACZ,WAAW,KAAK,IAAI;GACpB,QAAQ;EACV;EACA,MAAM,KAAK,MAAM,UAAU,IAAI;EAC/B,OAAO;CACT;CAEA,MAAM,OAAO,SAAqC;EAChD,MAAM,SAAiC,SAAS,SAAS,QAAQ,WAAW;EAC5E,MAAM,KAAK,MAAM,UAAU,KAAK,QAAQ;GAAE,SAAS,KAAK,IAAI;GAAG;GAAQ;EAAQ,CAAC;EAChF,MAAM,KAAK,SAAS;GAAE,OAAO,KAAK;GAAQ,SAAS;GAAM,OAAO,KAAK;GAAO;GAAS;EAAO,CAAC;CAC/F;CAEA,MAAM,SAAS,QAA+B;EAC5C,MAAM,UAAU;GAAE,MAAM;GAAO,OAAO;EAAO;EAC7C,MAAM,KAAK,MAAM,UAAU,KAAK,QAAQ;GACtC,SAAS,KAAK,IAAI;GAClB,QAAQ;GACR;EACF,CAAC;EACD,MAAM,KAAK,SAAS;GAClB,OAAO,KAAK;GACZ,SAAS;GACT,OAAO,KAAK;GACZ;GACA,QAAQ;EACV,CAAC;CACH;CAEA,MAAc,SAAS,KAA4C;EACjE,KAAK,MAAM,QAAQ,KAAK,OACtB,IAAI;GACF,MAAM,KAAK,GAAG;EAChB,SAAS,KAAK;GACZ,IAAI,KAAK,eAAe,SAAS,MAAM;GACvC,IAAI;IACF,MAAM,KAAK,MAAM,YAAY;KAC3B,SAAS,KAAK,GAAG;KACjB,OAAO,KAAK;KACZ,MAAM;KACN,WAAW,KAAK,IAAI;KACpB,SAAS;MACP,QAAQ;MACR,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;KACxD;IACF,CAAC;GACH,QAAQ,CAER;EACF;CAEJ;CAIA,MAAM,KACJ,MAMwB;EACxB,MAAM,SAAS,KAAK,GAAG;EAEvB,MAAM,OAAO;GACX;GACA,cAHa,KAAK,gBAAgB,KAAK,MAAM,KAAK,MAAM,SAAS;GAIjE,OAAO,KAAK;GACZ,WAAW,KAAK,IAAI;GACpB,GAAG;EACL;EACA,MAAM,KAAK,MAAM,WAAW,IAAI;EAChC,KAAK,MAAM,KAAK,MAAM;EACtB,OAAO,KAAK,OAAU,IAAI;CAC5B;CAEA,OAA+B,MAAwB;EACrD,OAAO;GACL;GACA,KAAK,OAAO,UAAuB;IACjC,MAAM,UAAU,KAAK,IAAI;IACzB,MAAM,KAAK,MAAM,WAAW,KAAK,QAAQ;KACvC;KACA,QAAQ;KACR,GAAG;IACL,CAAkB;IAClB,KAAK,IAAI,KAAK,MAAM;GACtB;GACA,MAAM,OAAO,OAAuB,UAAuB;IACzD,MAAM,UAAU,KAAK,IAAI;IACzB,MAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU;IACxD,MAAM,KAAK,MAAM,WAAW,KAAK,QAAQ;KACvC;KACA,QAAQ;KACR,OAAO;KACP,GAAG;IACL,CAAkB;IAClB,KAAK,IAAI,KAAK,MAAM;GACtB;EACF;CACF;CAEA,IAAY,QAAsB;EAChC,MAAM,MAAM,KAAK,MAAM,YAAY,MAAM;EACzC,IAAI,OAAO,GAAG,KAAK,MAAM,OAAO,KAAK,CAAC;CACxC;CAIA,IACE,MAC8B;EAC9B,OAAO,KAAK,KAAc;GAAE,MAAM;GAAO,GAAG;EAAK,CAAC;CACpD;CAEA,KACE,MAC+B;EAC/B,OAAO,KAAK,KAAe;GAAE,MAAM;GAAQ,GAAG;EAAK,CAAC;CACtD;CAEA,UACE,MACoC;EACpC,OAAO,KAAK,KAAoB;GAAE,MAAM;GAAa,GAAG;EAAK,CAAC;CAChE;CAEA,MAAM,YACJ,SACoB;EACpB,MAAM,SAAS,KAAK,GAAG;EACvB,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,OAAkB;GACtB;GACA,OAAO,KAAK;GACZ,MAAM;GACN,WAAW;GACX,SAAS;GACT,QAAQ;GACR,GAAG;EACL;EACA,MAAM,KAAK,MAAM,WAAW,IAAI;EAChC,OAAO;CACT;CAEA,QACE,MACkC;EAClC,OAAO,KAAK,KAAkB;GAAE,MAAM;GAAW,GAAG;EAAK,CAAC;CAC5D;CAIA,MAAM,KAAK,OAIa;EACtB,MAAM,OAAmB;GACvB,SAAS,KAAK,GAAG;GACjB,OAAO,KAAK;GACZ,QAAQ,MAAM,UAAU,KAAK,MAAM,KAAK,MAAM,SAAS;GACvD,MAAM,MAAM;GACZ,WAAW,KAAK,IAAI;GACpB,SAAS,MAAM,WAAW,CAAC;EAC7B;EACA,MAAM,KAAK,MAAM,YAAY,IAAI;EACjC,OAAO;CACT;CAIA,MAAM,aACJ,OAC4B;EAC5B,MAAM,OAA0B;GAC9B,OAAO,KAAK;GACZ,WAAW,MAAM,aAAa,KAAK,IAAI;GACvC,WAAW,MAAM;GACjB,OAAO,MAAM;GACb,UAAU,MAAM;GAChB,WAAW,MAAM;GACjB,UAAU,MAAM;GAChB,QAAQ,MAAM,UAAU,KAAK,MAAM,KAAK,MAAM,SAAS;EACzD;EACA,MAAM,KAAK,MAAM,kBAAkB,IAAI;EACvC,IAAI,KAAK,UACP,MAAM,KAAK,KAAK;GACd,MAAM;GACN,QAAQ,KAAK;GACb,SAAS;IAAE,WAAW,KAAK;IAAW,OAAO,KAAK;IAAO,UAAU,KAAK;GAAS;EACnF,CAAC;EAEH,OAAO;CACT;CAIA,MAAM,eAAe,UAAqE;EACxF,MAAM,OAAiB;GAAE,YAAY,KAAK,GAAG;GAAG,OAAO,KAAK;GAAQ,GAAG;EAAS;EAChF,MAAM,KAAK,MAAM,eAAe,IAAI;EACpC,OAAO;CACT;;;;;CAQA,MAAM,OACJ,MACA,IACY;EACZ,MAAM,SAAS,MAAM,KAAK,KAAK,IAAI;EACnC,IAAI;GACF,MAAM,SAAS,MAAM,GAAG,MAAM;GAC9B,MAAM,OAAO,IAAI;GACjB,OAAO;EACT,SAAS,KAAK;GACZ,MAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;GAC1D,MAAM;EACR;CACF;AACF;AAIA,SAAS,iBAAyB;CAChC,IAAI,OAAO,WAAW,QAAQ,eAAe,YAAY,OAAO,WAAW,OAAO,WAAW;CAC7F,OAAO,GAAG,KAAK,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;AAC7E;;AAGA,SAAgB,oBAAoB,MAcyB;CAC3D,OAAO;EACL,MAAM,KAAK,QAAQ,KAAK;EACxB,OAAO,KAAK;EACZ,UAAU,KAAK;EACf,QAAQ,KAAK;EACb,aAAa,KAAK,OAAO;EACzB,cAAc,KAAK,OAAO;EAC1B,cAAc,KAAK,OAAO;EAC1B,kBAAkB,KAAK,OAAO;EAC9B,iBAAiB,KAAK,OAAO;EAC7B,SAAS,KAAK;EACd,cAAc,KAAK;CACrB;AACF"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { C as TraceEvent, S as ToolSpan, _ as Span, c as JudgeSpan, d as RetrievalSpan, f as Run, g as SandboxSpan, i as EventKind, l as LlmSpan, m as RunOutcome, n as BudgetLedgerEntry, t as Artifact, u as Message, y as SpanKind } from "./schema-BtVldJ3T.js";
|
|
2
|
+
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
3
|
+
//#region src/trace/emitter.d.ts
|
|
4
|
+
interface SpanHandle<S extends Span = Span> {
|
|
5
|
+
span: S;
|
|
6
|
+
end(patch?: Partial<S>): Promise<void>;
|
|
7
|
+
fail(error: string | Error, patch?: Partial<S>): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
interface RunCompleteHookContext {
|
|
10
|
+
runId: string;
|
|
11
|
+
emitter: TraceEmitter;
|
|
12
|
+
store: TraceStore;
|
|
13
|
+
/** Outcome the caller passed to `endRun` (undefined for `abortRun`). */
|
|
14
|
+
outcome?: RunOutcome;
|
|
15
|
+
/** Final run status. */
|
|
16
|
+
status: 'completed' | 'failed' | 'aborted';
|
|
17
|
+
}
|
|
18
|
+
type RunCompleteHook = (ctx: RunCompleteHookContext) => Promise<void> | void;
|
|
19
|
+
interface TraceEmitterOptions {
|
|
20
|
+
runId?: string;
|
|
21
|
+
/** Inject a clock for deterministic tests. */
|
|
22
|
+
now?: () => number;
|
|
23
|
+
/** Inject an id generator for deterministic tests. */
|
|
24
|
+
id?: () => string;
|
|
25
|
+
/**
|
|
26
|
+
* Hooks fired after `endRun` / `abortRun` writes the final run state.
|
|
27
|
+
* Designed for trace-analyst auto-execution, integrity assertions, and
|
|
28
|
+
* outbound notifications. Hooks run sequentially in the order supplied.
|
|
29
|
+
*
|
|
30
|
+
* By default a hook that throws is swallowed and logged as a `note` event
|
|
31
|
+
* on the run — auto-orchestration must not crash the underlying flow.
|
|
32
|
+
* Set `hookErrors: 'throw'` to propagate.
|
|
33
|
+
*/
|
|
34
|
+
onRunComplete?: RunCompleteHook[];
|
|
35
|
+
/** `'swallow'` (default) | `'throw'`. */
|
|
36
|
+
hookErrors?: 'swallow' | 'throw';
|
|
37
|
+
}
|
|
38
|
+
declare class TraceEmitter {
|
|
39
|
+
private store;
|
|
40
|
+
private stack;
|
|
41
|
+
private _runId;
|
|
42
|
+
private now;
|
|
43
|
+
private id;
|
|
44
|
+
private hooks;
|
|
45
|
+
private hookErrors;
|
|
46
|
+
constructor(store: TraceStore, options?: TraceEmitterOptions);
|
|
47
|
+
get runId(): string;
|
|
48
|
+
get traceStore(): TraceStore;
|
|
49
|
+
/** Append a hook after construction (e.g. attach the trace analyst). */
|
|
50
|
+
addRunCompleteHook(hook: RunCompleteHook): void;
|
|
51
|
+
/**
|
|
52
|
+
* Begin a Run.
|
|
53
|
+
*
|
|
54
|
+
* `scenarioId` is required on the persisted Run shape — every Run downstream
|
|
55
|
+
* gets a non-empty scenarioId so filters and aggregations stay simple — but
|
|
56
|
+
* the INPUT here accepts it as optional. When omitted, startRun substitutes
|
|
57
|
+
* a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so
|
|
58
|
+
* runtime / operator / meta-eval runs that have no curated-scenario corpus
|
|
59
|
+
* to anchor to don't have to invent placeholder strings at the call site.
|
|
60
|
+
*/
|
|
61
|
+
startRun(run: Omit<Run, 'runId' | 'scenarioId' | 'startedAt' | 'status'> & {
|
|
62
|
+
scenarioId?: string;
|
|
63
|
+
}): Promise<Run>;
|
|
64
|
+
endRun(outcome?: RunOutcome): Promise<void>;
|
|
65
|
+
abortRun(reason: string): Promise<void>;
|
|
66
|
+
private runHooks;
|
|
67
|
+
span<S extends Span = Span>(init: {
|
|
68
|
+
kind: SpanKind;
|
|
69
|
+
name: string;
|
|
70
|
+
parentSpanId?: string;
|
|
71
|
+
attributes?: Record<string, unknown>;
|
|
72
|
+
} & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>): Promise<SpanHandle<S>>;
|
|
73
|
+
private handle;
|
|
74
|
+
private pop;
|
|
75
|
+
llm(init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<LlmSpan>>;
|
|
76
|
+
tool(init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<ToolSpan>>;
|
|
77
|
+
retrieval(init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<RetrievalSpan>>;
|
|
78
|
+
recordJudge(verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>): Promise<JudgeSpan>;
|
|
79
|
+
sandbox(init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<SandboxSpan>>;
|
|
80
|
+
emit(event: {
|
|
81
|
+
kind: EventKind;
|
|
82
|
+
spanId?: string;
|
|
83
|
+
payload?: Record<string, unknown>;
|
|
84
|
+
}): Promise<TraceEvent>;
|
|
85
|
+
recordBudget(entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & {
|
|
86
|
+
timestamp?: number;
|
|
87
|
+
}): Promise<BudgetLedgerEntry>;
|
|
88
|
+
recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact>;
|
|
89
|
+
/**
|
|
90
|
+
* Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
|
|
91
|
+
* Returns the fn's return value. Use this for the 95% case.
|
|
92
|
+
*/
|
|
93
|
+
within<T>(init: Parameters<TraceEmitter['span']>[0], fn: (handle: SpanHandle) => Promise<T>): Promise<T>;
|
|
94
|
+
}
|
|
95
|
+
/** Helper to build an LLM span handle args object from a provider-shaped response. */
|
|
96
|
+
declare function llmSpanFromProvider(args: {
|
|
97
|
+
name?: string;
|
|
98
|
+
model: string;
|
|
99
|
+
messages: Message[];
|
|
100
|
+
output: string;
|
|
101
|
+
usage?: {
|
|
102
|
+
inputTokens?: number;
|
|
103
|
+
outputTokens?: number;
|
|
104
|
+
cachedTokens?: number;
|
|
105
|
+
cacheWriteTokens?: number;
|
|
106
|
+
reasoningTokens?: number;
|
|
107
|
+
};
|
|
108
|
+
costUsd?: number;
|
|
109
|
+
finishReason?: string;
|
|
110
|
+
}): Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>;
|
|
111
|
+
//#endregion
|
|
112
|
+
export { TraceEmitterOptions as a, TraceEmitter as i, RunCompleteHookContext as n, llmSpanFromProvider as o, SpanHandle as r, RunCompleteHook as t };
|
|
113
|
+
//# sourceMappingURL=emitter-DGQGoLyj.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"emitter-DGQGoLyj.d.ts","names":[],"sources":["../src/trace/emitter.ts"],"mappings":";;;UA6BiB,WAAW,UAAU,OAAO;EAC3C,MAAM;EACN,IAAI,QAAQ,QAAQ,KAAK;EACzB,KAAK,gBAAgB,OAAO,QAAQ,QAAQ,KAAK;;UAGlC;EACf;EACA,SAAS;EACT,OAAO;;EAEP,UAAU;;EAEV;;KAGU,mBAAmB,KAAK,2BAA2B;UAE9C;EACf;;EAEA;;EAEA;;;;;;;;;;EAUA,gBAAgB;;EAEhB;;cAGW;UACH;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,OAAO,YAAY,UAAS;MASpC;MAIA,cAAc;;EAKlB,mBAAmB,MAAM;;;;;;;;;;;EAgBnB,SACJ,KAAK,KAAK;IAA0D;MACnE,QAAQ;EAaL,OAAO,UAAU,aAAa;EAM9B,SAAS,iBAAiB;UAgBlB;EA0BR,KAAK,UAAU,OAAO,MAC1B;IACE,MAAM;IACN;IACA;IACA,aAAa;MACX,QAAQ,KAAK,0DAChB,QAAQ,WAAW;UAed;UA0BA;EAOR,IACE,MAAM,KAAK,sDACV,QAAQ,WAAW;EAItB,KACE,MAAM,KAAK,uDACV,QAAQ,WAAW;EAItB,UACE,MAAM,KAAK,4DACV,QAAQ,WAAW;EAIhB,YACJ,SAAS,KAAK,oEACb,QAAQ;EAgBX,QACE,MAAM,KAAK,0DACV,QAAQ,WAAW;EAMhB,KAAK;IACT,MAAM;IACN;IACA,UAAU;MACR,QAAQ;EAeN,aACJ,OAAO,KAAK;IAA8C;MACzD,QAAQ;EAwBL,eAAe,UAAU,KAAK,oCAAoC,QAAQ;;;;;EAY1E,OAAO,GACX,MAAM,WAAW,0BACjB,KAAK,QAAQ,eAAe,QAAQ,KACnC,QAAQ;;;iBAqBG,oBAAoB;EAClC;EACA;EACA,UAAU;EACV;EACA;IACE;IACA;IACA;IACA;IACA;;EAEF;EACA;IACE,KAAK"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
//#region src/errors.ts
|
|
2
|
+
/**
|
|
3
|
+
* Base class for every contract error this package throws — carries the stable
|
|
4
|
+
* string `code` taxonomy so consumers can `instanceof`-match or switch on `code`.
|
|
5
|
+
*/
|
|
6
|
+
var AgentEvalError = class extends Error {
|
|
7
|
+
/** Stable string code. Survives minification; safe to switch on. */
|
|
8
|
+
code;
|
|
9
|
+
constructor(code, message, options) {
|
|
10
|
+
super(message, options);
|
|
11
|
+
this.name = this.constructor.name;
|
|
12
|
+
this.code = code;
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
/** Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). */
|
|
16
|
+
var ValidationError = class extends AgentEvalError {
|
|
17
|
+
constructor(message, options) {
|
|
18
|
+
super("validation", message, options);
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
/** A named resource (run, span, rubric, scenario, dataset row, route) does not exist. */
|
|
22
|
+
var NotFoundError = class extends AgentEvalError {
|
|
23
|
+
constructor(message, options) {
|
|
24
|
+
super("not_found", message, options);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
/** Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). */
|
|
28
|
+
var ConfigError = class extends AgentEvalError {
|
|
29
|
+
constructor(message, options) {
|
|
30
|
+
super("config", message, options);
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* A run is missing the artifacts a launch-grade check requires:
|
|
35
|
+
* raw HTTP capture absent, no LLM spans, route assertion failed, run-end
|
|
36
|
+
* assertion tripped. Block ship on this; do not catch and move on.
|
|
37
|
+
*/
|
|
38
|
+
var CaptureIntegrityError = class extends AgentEvalError {
|
|
39
|
+
constructor(message, options) {
|
|
40
|
+
super("capture_integrity", message, options);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
/** A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. */
|
|
44
|
+
var JudgeError = class extends AgentEvalError {
|
|
45
|
+
constructor(message, options) {
|
|
46
|
+
super("judge", message, options);
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
/** A verifier signalled a hard failure (compile, test, schema) — distinct from a low judge score. */
|
|
50
|
+
var VerificationError = class extends AgentEvalError {
|
|
51
|
+
constructor(message, options) {
|
|
52
|
+
super("verification", message, options);
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
/** Replay cache cannot satisfy a request: miss with no fallback, sink lacks list(), unsupported URL. */
|
|
56
|
+
var ReplayError = class extends AgentEvalError {
|
|
57
|
+
constructor(message, options) {
|
|
58
|
+
super("replay", message, options);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
//#endregion
|
|
62
|
+
export { NotFoundError as a, VerificationError as c, JudgeError as i, CaptureIntegrityError as n, ReplayError as o, ConfigError as r, ValidationError as s, AgentEvalError as t };
|
|
63
|
+
|
|
64
|
+
//# sourceMappingURL=errors-8YnH8WlF.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors-8YnH8WlF.js","names":[],"sources":["../src/errors.ts"],"sourcesContent":["/**\n * Error taxonomy for `@tangle-network/agent-eval`.\n *\n * Every error this package throws as part of its *public contract* extends\n * `AgentEvalError`. Consumers can pattern-match by `instanceof <Subclass>` or\n * by the stable string `code` carried on the base class.\n *\n * The codes are stable across minor versions; new codes can be added, but\n * existing codes never change meaning. New subclasses are non-breaking.\n *\n * Internal invariant guards (`throw new Error('this should never happen')`)\n * remain plain `Error`s on purpose — they're programmer-mistake assertions,\n * not consumer-catchable contract failures.\n */\n\nexport type AgentEvalErrorCode =\n | 'validation'\n | 'not_found'\n | 'config'\n | 'capture_integrity'\n | 'judge'\n | 'verification'\n | 'replay'\n | 'backend_integrity'\n | 'profile_matrix'\n\n/**\n * Base class for every contract error this package throws — carries the stable\n * string `code` taxonomy so consumers can `instanceof`-match or switch on `code`.\n */\nexport class AgentEvalError extends Error {\n /** Stable string code. Survives minification; safe to switch on. */\n readonly code: AgentEvalErrorCode\n\n constructor(code: AgentEvalErrorCode, message: string, options?: { cause?: unknown }) {\n super(message, options)\n this.name = this.constructor.name\n this.code = code\n }\n}\n\n/** Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). */\nexport class ValidationError extends AgentEvalError {\n constructor(message: string, options?: { cause?: unknown }) {\n super('validation', message, options)\n }\n}\n\n/** A named resource (run, span, rubric, scenario, dataset row, route) does not exist. */\nexport class NotFoundError extends AgentEvalError {\n constructor(message: string, options?: { cause?: unknown }) {\n super('not_found', message, options)\n }\n}\n\n/** Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). */\nexport class ConfigError extends AgentEvalError {\n constructor(message: string, options?: { cause?: unknown }) {\n super('config', message, options)\n }\n}\n\n/**\n * A run is missing the artifacts a launch-grade check requires:\n * raw HTTP capture absent, no LLM spans, route assertion failed, run-end\n * assertion tripped. Block ship on this; do not catch and move on.\n */\nexport class CaptureIntegrityError extends AgentEvalError {\n constructor(message: string, options?: { cause?: unknown }) {\n super('capture_integrity', message, options)\n }\n}\n\n/** A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. */\nexport class JudgeError extends AgentEvalError {\n constructor(message: string, options?: { cause?: unknown }) {\n super('judge', message, options)\n }\n}\n\n/** A verifier signalled a hard failure (compile, test, schema) — distinct from a low judge score. */\nexport class VerificationError extends AgentEvalError {\n constructor(message: string, options?: { cause?: unknown }) {\n super('verification', message, options)\n }\n}\n\n/** Replay cache cannot satisfy a request: miss with no fallback, sink lacks list(), unsupported URL. */\nexport class ReplayError extends AgentEvalError {\n constructor(message: string, options?: { cause?: unknown }) {\n super('replay', message, options)\n }\n}\n"],"mappings":";;;;;AA8BA,IAAa,iBAAb,cAAoC,MAAM;;CAExC;CAEA,YAAY,MAA0B,SAAiB,SAA+B;EACpF,MAAM,SAAS,OAAO;EACtB,KAAK,OAAO,KAAK,YAAY;EAC7B,KAAK,OAAO;CACd;AACF;;AAGA,IAAa,kBAAb,cAAqC,eAAe;CAClD,YAAY,SAAiB,SAA+B;EAC1D,MAAM,cAAc,SAAS,OAAO;CACtC;AACF;;AAGA,IAAa,gBAAb,cAAmC,eAAe;CAChD,YAAY,SAAiB,SAA+B;EAC1D,MAAM,aAAa,SAAS,OAAO;CACrC;AACF;;AAGA,IAAa,cAAb,cAAiC,eAAe;CAC9C,YAAY,SAAiB,SAA+B;EAC1D,MAAM,UAAU,SAAS,OAAO;CAClC;AACF;;;;;;AAOA,IAAa,wBAAb,cAA2C,eAAe;CACxD,YAAY,SAAiB,SAA+B;EAC1D,MAAM,qBAAqB,SAAS,OAAO;CAC7C;AACF;;AAGA,IAAa,aAAb,cAAgC,eAAe;CAC7C,YAAY,SAAiB,SAA+B;EAC1D,MAAM,SAAS,SAAS,OAAO;CACjC;AACF;;AAGA,IAAa,oBAAb,cAAuC,eAAe;CACpD,YAAY,SAAiB,SAA+B;EAC1D,MAAM,gBAAgB,SAAS,OAAO;CACxC;AACF;;AAGA,IAAa,cAAb,cAAiC,eAAe;CAC9C,YAAY,SAAiB,SAA+B;EAC1D,MAAM,UAAU,SAAS,OAAO;CAClC;AACF"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
//#region src/errors.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* Error taxonomy for `@tangle-network/agent-eval`.
|
|
4
|
+
*
|
|
5
|
+
* Every error this package throws as part of its *public contract* extends
|
|
6
|
+
* `AgentEvalError`. Consumers can pattern-match by `instanceof <Subclass>` or
|
|
7
|
+
* by the stable string `code` carried on the base class.
|
|
8
|
+
*
|
|
9
|
+
* The codes are stable across minor versions; new codes can be added, but
|
|
10
|
+
* existing codes never change meaning. New subclasses are non-breaking.
|
|
11
|
+
*
|
|
12
|
+
* Internal invariant guards (`throw new Error('this should never happen')`)
|
|
13
|
+
* remain plain `Error`s on purpose — they're programmer-mistake assertions,
|
|
14
|
+
* not consumer-catchable contract failures.
|
|
15
|
+
*/
|
|
16
|
+
type AgentEvalErrorCode = 'validation' | 'not_found' | 'config' | 'capture_integrity' | 'judge' | 'verification' | 'replay' | 'backend_integrity' | 'profile_matrix';
|
|
17
|
+
/**
|
|
18
|
+
* Base class for every contract error this package throws — carries the stable
|
|
19
|
+
* string `code` taxonomy so consumers can `instanceof`-match or switch on `code`.
|
|
20
|
+
*/
|
|
21
|
+
declare class AgentEvalError extends Error {
|
|
22
|
+
/** Stable string code. Survives minification; safe to switch on. */
|
|
23
|
+
readonly code: AgentEvalErrorCode;
|
|
24
|
+
constructor(code: AgentEvalErrorCode, message: string, options?: {
|
|
25
|
+
cause?: unknown;
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
/** Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). */
|
|
29
|
+
declare class ValidationError extends AgentEvalError {
|
|
30
|
+
constructor(message: string, options?: {
|
|
31
|
+
cause?: unknown;
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
/** A named resource (run, span, rubric, scenario, dataset row, route) does not exist. */
|
|
35
|
+
declare class NotFoundError extends AgentEvalError {
|
|
36
|
+
constructor(message: string, options?: {
|
|
37
|
+
cause?: unknown;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
/** Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). */
|
|
41
|
+
declare class ConfigError extends AgentEvalError {
|
|
42
|
+
constructor(message: string, options?: {
|
|
43
|
+
cause?: unknown;
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* A run is missing the artifacts a launch-grade check requires:
|
|
48
|
+
* raw HTTP capture absent, no LLM spans, route assertion failed, run-end
|
|
49
|
+
* assertion tripped. Block ship on this; do not catch and move on.
|
|
50
|
+
*/
|
|
51
|
+
declare class CaptureIntegrityError extends AgentEvalError {
|
|
52
|
+
constructor(message: string, options?: {
|
|
53
|
+
cause?: unknown;
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
/** A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. */
|
|
57
|
+
declare class JudgeError extends AgentEvalError {
|
|
58
|
+
constructor(message: string, options?: {
|
|
59
|
+
cause?: unknown;
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
/** A verifier signalled a hard failure (compile, test, schema) — distinct from a low judge score. */
|
|
63
|
+
declare class VerificationError extends AgentEvalError {
|
|
64
|
+
constructor(message: string, options?: {
|
|
65
|
+
cause?: unknown;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
/** Replay cache cannot satisfy a request: miss with no fallback, sink lacks list(), unsupported URL. */
|
|
69
|
+
declare class ReplayError extends AgentEvalError {
|
|
70
|
+
constructor(message: string, options?: {
|
|
71
|
+
cause?: unknown;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
//#endregion
|
|
75
|
+
export { JudgeError as a, ValidationError as c, ConfigError as i, VerificationError as l, AgentEvalErrorCode as n, NotFoundError as o, CaptureIntegrityError as r, ReplayError as s, AgentEvalError as t };
|
|
76
|
+
//# sourceMappingURL=errors-CEk209JS.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors-CEk209JS.d.ts","names":[],"sources":["../src/errors.ts"],"mappings":";;;;;;;;;;;;;;;KAeY;;;;;cAeC,uBAAuB;;WAEzB,MAAM;EAEf,YAAY,MAAM,oBAAoB,iBAAiB;IAAY;;;;cAQxD,wBAAwB;EACnC,YAAY,iBAAiB;IAAY;;;;cAM9B,sBAAsB;EACjC,YAAY,iBAAiB;IAAY;;;;cAM9B,oBAAoB;EAC/B,YAAY,iBAAiB;IAAY;;;;;;;;cAU9B,8BAA8B;EACzC,YAAY,iBAAiB;IAAY;;;;cAM9B,mBAAmB;EAC9B,YAAY,iBAAiB;IAAY;;;;cAM9B,0BAA0B;EACrC,YAAY,iBAAiB;IAAY;;;;cAM9B,oBAAoB;EAC/B,YAAY,iBAAiB;IAAY"}
|