@tangle-network/agent-eval 0.144.11 → 0.144.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/agent-profile-B9_GGsG8.d.ts +84 -0
- package/dist/agent-profile-B9_GGsG8.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts → agent-profile-cell-BkcRDikH.d.ts} +2 -2
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts.map → agent-profile-cell-BkcRDikH.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +134 -16
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +364 -10
- package/dist/analyst/index.js.map +1 -1
- package/dist/backend-integrity-CsVin_Wb.d.ts +280 -0
- package/dist/backend-integrity-CsVin_Wb.d.ts.map +1 -0
- package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js} +3 -2
- package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map} +1 -1
- package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js} +2 -2
- package/dist/{benchmark-CWeqGl7x.js.map → benchmark-BhT16ep9.js.map} +1 -1
- package/dist/{agent-profile-DPi7IZg7.d.ts → benchmark-Ceoan7vk.d.ts} +4 -91
- package/dist/benchmark-Ceoan7vk.d.ts.map +1 -0
- package/dist/{benchmark-command-BteMFN62.js → benchmark-command-CY6cbhOB.js} +36 -79
- package/dist/benchmark-command-CY6cbhOB.js.map +1 -0
- package/dist/benchmarks/index.d.ts +244 -2
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/index.js +733 -1
- package/dist/benchmarks/index.js.map +1 -0
- package/dist/builder-eval/index.d.ts +23 -2
- package/dist/builder-eval/index.d.ts.map +1 -1
- package/dist/builder-eval/index.js +227 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +10 -8
- package/dist/campaign/index.js +9 -6
- package/dist/{campaign-C2TTzQII.js → campaign-BYjBAypg.js} +21 -634
- package/dist/campaign-BYjBAypg.js.map +1 -0
- package/dist/{canonical-D011XM8r.js → canonical-D-XsTQ6_.js} +2 -2
- package/dist/{canonical-D011XM8r.js.map → canonical-D-XsTQ6_.js.map} +1 -1
- package/dist/{index-CQTZ-4XN.d.ts → capture-fetch-DDvpjVRU.d.ts} +3 -3
- package/dist/capture-fetch-DDvpjVRU.d.ts.map +1 -0
- package/dist/{default-registry-BmktKy8r.js → chat-client-Bp4Ebuuc.js} +1276 -1276
- package/dist/chat-client-Bp4Ebuuc.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-C9gzZE59.d.ts → client-KUbGulm_.d.ts} +4 -4
- package/dist/{client-C9gzZE59.d.ts.map → client-KUbGulm_.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +13 -390
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +18 -542
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-DMFxsLKr.js → cost-ledger-BSe92yAV.js} +3 -3
- package/dist/{cost-ledger-DMFxsLKr.js.map → cost-ledger-BSe92yAV.js.map} +1 -1
- package/dist/{cost-ledger-Bv_e8XHY.d.ts → cost-ledger-DbQdN3nO.d.ts} +2 -2
- package/dist/{cost-ledger-Bv_e8XHY.d.ts.map → cost-ledger-DbQdN3nO.d.ts.map} +1 -1
- package/dist/{counterfactual-CxmxAONP.d.ts → counterfactual--bpysZF0.d.ts} +2 -14
- package/dist/{counterfactual-CxmxAONP.d.ts.map → counterfactual--bpysZF0.d.ts.map} +1 -1
- package/dist/{counterfactual-CWPTrMH7.js → counterfactual-lDfCx0Uz.js} +3 -31
- package/dist/{counterfactual-CWPTrMH7.js.map → counterfactual-lDfCx0Uz.js.map} +1 -1
- package/dist/{dataset-C8xaLXdY.d.ts → dataset-CJjKqQfA.d.ts} +2 -8
- package/dist/dataset-CJjKqQfA.d.ts.map +1 -0
- package/dist/{default-registry-Bf8Woqmq.d.ts → default-registry-Di6HP6pG.d.ts} +7 -12
- package/dist/default-registry-Di6HP6pG.d.ts.map +1 -0
- package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js} +542 -130
- package/dist/define-agent-eval-iqjT--ZZ.js.map +1 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts +388 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts.map +1 -0
- package/dist/descriptive-B5MwKfbf.js +144 -0
- package/dist/descriptive-B5MwKfbf.js.map +1 -0
- package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js} +4 -4
- package/dist/{dspy-rlm-engine-DbTk4JdR.js.map → dspy-rlm-engine-B_qhSc21.js.map} +1 -1
- package/dist/effect-sizes-DiH8MGOH.js +82 -0
- package/dist/effect-sizes-DiH8MGOH.js.map +1 -0
- package/dist/{engine-3hL-XqwJ.d.ts → engine-otFpE2gF.d.ts} +10 -38
- package/dist/engine-otFpE2gF.d.ts.map +1 -0
- package/dist/{errors-CKPfb2aH.d.ts → errors-DEE6u6ot.d.ts} +2 -14
- package/dist/{errors-CKPfb2aH.d.ts.map → errors-DEE6u6ot.d.ts.map} +1 -1
- package/dist/{errors-D-LKuDhb.js → errors-Dngq5h35.js} +2 -8
- package/dist/{errors-D-LKuDhb.js.map → errors-Dngq5h35.js.map} +1 -1
- package/dist/{eval-campaign-B_7wcnav.js → eval-campaign-UB-usSQ2.js} +6 -6
- package/dist/{eval-campaign-B_7wcnav.js.map → eval-campaign-UB-usSQ2.js.map} +1 -1
- package/dist/{exact-types-DSFFpLLI.d.ts → exact-types-BH1twmAJ.d.ts} +2 -2
- package/dist/{exact-types-DSFFpLLI.d.ts.map → exact-types-BH1twmAJ.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +9 -6
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +11 -7
- package/dist/experiment/index.js.map +1 -1
- package/dist/experiment-tracker-C29gXM4B.js +269 -0
- package/dist/experiment-tracker-C29gXM4B.js.map +1 -0
- package/dist/{experiment-tracker-IMntXr6J.d.ts → experiment-tracker-DWHZBAYL.d.ts} +77 -121
- package/dist/experiment-tracker-DWHZBAYL.d.ts.map +1 -0
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts → external-optimizer-contracts-lixrOZdX.d.ts} +3 -3
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts.map → external-optimizer-contracts-lixrOZdX.d.ts.map} +1 -1
- package/dist/external-optimizer-process-BTiNB-RH.js +301 -0
- package/dist/external-optimizer-process-BTiNB-RH.js.map +1 -0
- package/dist/{single-run-lock-DFWHEB09.js → external-optimizer-subprocess-DrJ9hR8u.js} +144 -438
- package/dist/external-optimizer-subprocess-DrJ9hR8u.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/{failure-cluster-CqcvCcdR.d.ts → failure-cluster-BLURuWG4.d.ts} +2 -3
- package/dist/failure-cluster-BLURuWG4.d.ts.map +1 -0
- package/dist/{feedback-trajectory-CacHpxsp.d.ts → feedback-trajectory-juOozjAc.d.ts} +5 -36
- package/dist/feedback-trajectory-juOozjAc.d.ts.map +1 -0
- package/dist/fuzz.d.ts +2 -2
- package/dist/fuzz.js +2 -2
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-CvSN3IG1.d.ts → index-B8Ui1mr1.d.ts} +2 -2
- package/dist/{index-CvSN3IG1.d.ts.map → index-B8Ui1mr1.d.ts.map} +1 -1
- package/dist/{skill-usage-CO9OLRBx.d.ts → index-BWDrSVfw.d.ts} +11 -136
- package/dist/index-BWDrSVfw.d.ts.map +1 -0
- package/dist/index-Ba3YrbAL.d.ts +1 -0
- package/dist/{index-BnEuDAK2.d.ts → index-COQYtuRF.d.ts} +3 -3
- package/dist/{index-BnEuDAK2.d.ts.map → index-COQYtuRF.d.ts.map} +1 -1
- package/dist/{index-C3ssXVLv.d.ts → index-CvjYbU0D.d.ts} +2 -2
- package/dist/{index-C3ssXVLv.d.ts.map → index-CvjYbU0D.d.ts.map} +1 -1
- package/dist/{index-DPPGNJ_R.d.ts → index-DSmEylT9.d.ts} +17 -115
- package/dist/index-DSmEylT9.d.ts.map +1 -0
- package/dist/index.d.ts +2397 -5308
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5914 -10496
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-CgX_s0Ez.d.ts → insight-report-CRi-Ufrj.d.ts} +4 -4
- package/dist/{insight-report-CgX_s0Ez.d.ts.map → insight-report-CRi-Ufrj.d.ts.map} +1 -1
- package/dist/{integrity-MLzHOfV9.js → integrity-Cy9WHAtb.js} +2 -2
- package/dist/{integrity-MLzHOfV9.js.map → integrity-Cy9WHAtb.js.map} +1 -1
- package/dist/{integrity-DY6tIbl0.js → integrity-DysDBWDu.js} +2 -2
- package/dist/{integrity-DY6tIbl0.js.map → integrity-DysDBWDu.js.map} +1 -1
- package/dist/{integrity-BuqEKu-x.d.ts → integrity-OrcI9Nau.d.ts} +3 -3
- package/dist/{integrity-BuqEKu-x.d.ts.map → integrity-OrcI9Nau.d.ts.map} +1 -1
- package/dist/internal-BDHPCnjk.js +230 -0
- package/dist/internal-BDHPCnjk.js.map +1 -0
- package/dist/judge-calibration-C5CbMYce.d.ts +117 -0
- package/dist/judge-calibration-C5CbMYce.d.ts.map +1 -0
- package/dist/judge-calibration-DZkWrm5H.js +317 -0
- package/dist/judge-calibration-DZkWrm5H.js.map +1 -0
- package/dist/{kind-factory-B8-r8-y8.js → kind-factory-CPmSd58s.js} +208 -208
- package/dist/{kind-factory-B8-r8-y8.js.map → kind-factory-CPmSd58s.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/{ledger-core-DXZIqu17.js → ledger-core-BmZt19oQ.js} +110 -110
- package/dist/{ledger-core-DXZIqu17.js.map → ledger-core-BmZt19oQ.js.map} +1 -1
- package/dist/{llm-client-DzvMUsS_.js → llm-client-d0-2TT1g.js} +4 -64
- package/dist/{llm-client-DzvMUsS_.js.map → llm-client-d0-2TT1g.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CQdVeM8k.js → llm-judge-dZ8P6nGI.js} +3349 -5401
- package/dist/llm-judge-dZ8P6nGI.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +3 -3
- package/dist/{metrics-C9YY1OcL.js → metrics-Cl0L1KUy.js} +2 -108
- package/dist/{metrics-C9YY1OcL.js.map → metrics-Cl0L1KUy.js.map} +1 -1
- package/dist/{mint-B2O60ACG.js → mint-Dj9Ww_3I.js} +4 -4
- package/dist/{mint-B2O60ACG.js.map → mint-Dj9Ww_3I.js.map} +1 -1
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts → multi-layer-verifier-DIguZc8Z.d.ts} +3 -3
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts.map → multi-layer-verifier-DIguZc8Z.d.ts.map} +1 -1
- package/dist/multiplicity-DIWHvysC.d.ts +43 -0
- package/dist/multiplicity-DIWHvysC.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +3 -3
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-8r6WUfHc.js → opencode-sqlite-DJWAXLms.js} +2 -2
- package/dist/{opencode-sqlite-8r6WUfHc.js.map → opencode-sqlite-DJWAXLms.js.map} +1 -1
- package/dist/package-version-D7lQHt_-.js +34 -0
- package/dist/package-version-D7lQHt_-.js.map +1 -0
- package/dist/paired-arms-D-XRF_fy.js +1045 -0
- package/dist/paired-arms-D-XRF_fy.js.map +1 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts +251 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts.map +1 -0
- package/dist/paired-tests-BHIhYVdu.js +213 -0
- package/dist/paired-tests-BHIhYVdu.js.map +1 -0
- package/dist/pareto-BqNW3LJR.d.ts +117 -0
- package/dist/pareto-BqNW3LJR.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +3 -64
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -284
- package/dist/pipelines/index.js.map +1 -1
- package/dist/power-and-mde-CHIrXJll.js +195 -0
- package/dist/power-and-mde-CHIrXJll.js.map +1 -0
- package/dist/{promotion-policy-CrLrmys8.js → power-preflight-DEw-uC7q.js} +4 -184
- package/dist/power-preflight-DEw-uC7q.js.map +1 -0
- package/dist/pre-registration-CZwSFQS4.d.ts +577 -0
- package/dist/pre-registration-CZwSFQS4.d.ts.map +1 -0
- package/dist/{prime-protocol-BfSalTfR.js → prime-protocol-6tZTVsWm.js} +72 -21
- package/dist/prime-protocol-6tZTVsWm.js.map +1 -0
- package/dist/produced-state-DU79a81m.js +586 -0
- package/dist/produced-state-DU79a81m.js.map +1 -0
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/promotion-policy-D0nPhkSy.d.ts +134 -0
- package/dist/promotion-policy-D0nPhkSy.d.ts.map +1 -0
- package/dist/promotion-policy-xzA40Evo.js +186 -0
- package/dist/promotion-policy-xzA40Evo.js.map +1 -0
- package/dist/{skillopt-optimization-method-USDKhxSA.d.ts → provenance-CCdxgLDT.d.ts} +57 -478
- package/dist/provenance-CCdxgLDT.d.ts.map +1 -0
- package/dist/registry-oJeeI4-a.d.ts +178 -0
- package/dist/registry-oJeeI4-a.d.ts.map +1 -0
- package/dist/{release-report-DA2BCu5a.d.ts → release-confidence-BFRE5WSp.d.ts} +180 -112
- package/dist/release-confidence-BFRE5WSp.d.ts.map +1 -0
- package/dist/{release-report-BUYmoKo2.js → release-confidence-CxDuiAev.js} +125 -217
- package/dist/release-confidence-CxDuiAev.js.map +1 -0
- package/dist/reporting.d.ts +6 -5
- package/dist/reporting.js +7 -5
- package/dist/{researcher-BchpD55R.d.ts → researcher-DaN4GST-.d.ts} +7 -40
- package/dist/{researcher-BchpD55R.d.ts.map → researcher-DaN4GST-.d.ts.map} +1 -1
- package/dist/{reward-hacking-BDToousL.js → reward-hacking-DNgjilrV.js} +3 -3
- package/dist/reward-hacking-DNgjilrV.js.map +1 -0
- package/dist/{reward-hacking-BI0OMAlo.d.ts → reward-hacking-DSSTuI9r.d.ts} +5 -5
- package/dist/{reward-hacking-BI0OMAlo.d.ts.map → reward-hacking-DSSTuI9r.d.ts.map} +1 -1
- package/dist/rl.d.ts +7 -7
- package/dist/rl.js +11 -10
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +4 -4
- package/dist/{rollout-DRrksrcV.js → rollout-BWtw0I_6.js} +3 -3
- package/dist/{rollout-DRrksrcV.js.map → rollout-BWtw0I_6.js.map} +1 -1
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts → rubric-predictive-validity-BgxtKe4G.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts.map → rubric-predictive-validity-BgxtKe4G.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BRR632r1.js → rubric-predictive-validity-Cwwyd7ah.js} +2 -2
- package/dist/{rubric-predictive-validity-BRR632r1.js.map → rubric-predictive-validity-Cwwyd7ah.js.map} +1 -1
- package/dist/{run-record-BmSPWXJR.js → run-record-BvHPVS-i.js} +3 -3
- package/dist/{run-record-BmSPWXJR.js.map → run-record-BvHPVS-i.js.map} +1 -1
- package/dist/{run-record-CF4Dwpxr.d.ts → run-record-CKiihE6f.d.ts} +4 -4
- package/dist/{run-record-CF4Dwpxr.d.ts.map → run-record-CKiihE6f.d.ts.map} +1 -1
- package/dist/{proposal-findings-2GIUo1et.js → run-score-lDzV0X8j.js} +31 -31
- package/dist/run-score-lDzV0X8j.js.map +1 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts +70 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -0
- package/dist/{schema-Cef2cFmb2.d.ts → schema-Cef2cFmb.d.ts} +1 -1
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-BiJxScqe.js +406 -0
- package/dist/semantic-concept-judge-BiJxScqe.js.map +1 -0
- package/dist/{sequential-D-BLJBKU.js → sequential-C458DXNf.js} +4 -3
- package/dist/{sequential-D-BLJBKU.js.map → sequential-C458DXNf.js.map} +1 -1
- package/dist/sequential-eprocess-CbUt2htw.js +83 -0
- package/dist/sequential-eprocess-CbUt2htw.js.map +1 -0
- package/dist/series-convergence-D1cL1f-4.d.ts +129 -0
- package/dist/series-convergence-D1cL1f-4.d.ts.map +1 -0
- package/dist/{server-iu0ede49.js → server-ulsOdrTI.js} +5 -21
- package/dist/server-ulsOdrTI.js.map +1 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts +427 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js +1999 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js.map +1 -0
- package/dist/{statistical-heldout-TQ-4CYiN.d.ts → statistical-heldout-DoFF5KJX.d.ts} +5 -278
- package/dist/statistical-heldout-DoFF5KJX.d.ts.map +1 -0
- package/dist/{store-otlp-Dw8PPIlL.js → store-otlp-CsptLYpN.js} +3 -3
- package/dist/{store-otlp-Dw8PPIlL.js.map → store-otlp-CsptLYpN.js.map} +1 -1
- package/dist/store-tool-spans-Cq9mFd-q.js +667 -0
- package/dist/store-tool-spans-Cq9mFd-q.js.map +1 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts +342 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts.map +1 -0
- package/dist/student-t-CvBq2mve.js +38 -0
- package/dist/student-t-CvBq2mve.js.map +1 -0
- package/dist/{summary-report-Lf-5I7xh.js → summary-report-Blysd6Z2.js} +6 -3
- package/dist/{summary-report-Lf-5I7xh.js.map → summary-report-Blysd6Z2.js.map} +1 -1
- package/dist/{summary-report-BNR7DWTj.d.ts → summary-report-CaL-Hnxt.d.ts} +5 -5
- package/dist/{summary-report-BNR7DWTj.d.ts.map → summary-report-CaL-Hnxt.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +391 -3
- package/dist/supervisor-run/index.d.ts.map +1 -0
- package/dist/supervisor-run/index.js +1689 -2
- package/dist/{supervisor-run-D_sokXcO.js.map → supervisor-run/index.js.map} +1 -1
- package/dist/{extract-usage-CdZdoj1s.js → task-failure-attributes-CpQ4y5RD.js} +5 -157
- package/dist/task-failure-attributes-CpQ4y5RD.js.map +1 -0
- package/dist/{tool-groups-CZPGGlHf.d.ts → tool-groups-Cteb03Ps.d.ts} +3 -3
- package/dist/tool-groups-Cteb03Ps.d.ts.map +1 -0
- package/dist/tool-waste-BDdBZG1F.js +803 -0
- package/dist/tool-waste-BDdBZG1F.js.map +1 -0
- package/dist/tool-waste-DjRDEsuI.d.ts +128 -0
- package/dist/tool-waste-DjRDEsuI.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +14 -5
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +35 -7
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +406 -7
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +1011 -10
- package/dist/traces.js.map +1 -0
- package/dist/trajectory-replay/index.d.ts +16 -3
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +52 -5
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/types-BEPZc6eo.d.ts +93 -0
- package/dist/types-BEPZc6eo.d.ts.map +1 -0
- package/dist/{usage-receipt-t7vAzCRQ.js → types-BI4fT3HN.js} +67 -67
- package/dist/types-BI4fT3HN.js.map +1 -0
- package/dist/{types-BnjdJ70P.d.ts → types-BZ59Ahr8.d.ts} +5 -5
- package/dist/{types-BnjdJ70P.d.ts.map → types-BZ59Ahr8.d.ts.map} +1 -1
- package/dist/{types-D216SgwM.d.ts → types-Cx3YUh2r.d.ts} +4 -241
- package/dist/types-Cx3YUh2r.d.ts.map +1 -0
- package/dist/{types-BvDKaULh.d.ts → types-DLQx4mKU.d.ts} +4 -4
- package/dist/{types-BvDKaULh.d.ts.map → types-DLQx4mKU.d.ts.map} +1 -1
- package/dist/{types-D4mog56g.d.ts → types-yLK8gXE9.d.ts} +2 -2
- package/dist/{types-D4mog56g.d.ts.map → types-yLK8gXE9.d.ts.map} +1 -1
- package/dist/verdict-BndeTAh_.js +61 -0
- package/dist/verdict-BndeTAh_.js.map +1 -0
- package/dist/verdict-E4eRNf7-.d.ts +392 -0
- package/dist/verdict-E4eRNf7-.d.ts.map +1 -0
- package/dist/{verdict-cache-BCcOh0kF.js → verdict-cache-mZf5FEiY.js} +3 -55
- package/dist/{verdict-cache-BCcOh0kF.js.map → verdict-cache-mZf5FEiY.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/charter.md +3 -3
- package/docs/control-runtime.md +3 -42
- package/docs/experiment.md +0 -1
- package/docs/feature-guide.md +2 -2
- package/docs/trace-repair-grader.md +1 -0
- package/docs/trajectory-replay.md +1 -0
- package/docs/verdicts.md +43 -0
- package/docs/verification-strategies.md +3 -2
- package/package.json +6 -11
- package/dist/agent-profile-DPi7IZg7.d.ts.map +0 -1
- package/dist/analyze-runs-C30yljDJ.js.map +0 -1
- package/dist/baseline-CavEbRyH.d.ts +0 -136
- package/dist/baseline-CavEbRyH.d.ts.map +0 -1
- package/dist/benchmark-command-BteMFN62.js.map +0 -1
- package/dist/benchmarks-Dzs8CKb1.js +0 -755
- package/dist/benchmarks-Dzs8CKb1.js.map +0 -1
- package/dist/campaign-C2TTzQII.js.map +0 -1
- package/dist/completion-verifier-DJA5BhPb.d.ts +0 -414
- package/dist/completion-verifier-DJA5BhPb.d.ts.map +0 -1
- package/dist/control.d.ts +0 -3
- package/dist/control.js +0 -2
- package/dist/dataset-C8xaLXdY.d.ts.map +0 -1
- package/dist/default-registry-Bf8Woqmq.d.ts.map +0 -1
- package/dist/default-registry-BmktKy8r.js.map +0 -1
- package/dist/engine-3hL-XqwJ.d.ts.map +0 -1
- package/dist/experiment-tracker-CnRICnMl.js +0 -500
- package/dist/experiment-tracker-CnRICnMl.js.map +0 -1
- package/dist/experiment-tracker-IMntXr6J.d.ts.map +0 -1
- package/dist/extract-usage-CdZdoj1s.js.map +0 -1
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +0 -1
- package/dist/feedback-trajectory-CacHpxsp.d.ts.map +0 -1
- package/dist/index-BZ3-y4YL.d.ts +0 -391
- package/dist/index-BZ3-y4YL.d.ts.map +0 -1
- package/dist/index-CQTZ-4XN.d.ts.map +0 -1
- package/dist/index-DPPGNJ_R.d.ts.map +0 -1
- package/dist/index-YE4KdKbO2.d.ts +0 -335
- package/dist/index-YE4KdKbO2.d.ts.map +0 -1
- package/dist/paired-arms-iZ08VFMN.js +0 -260
- package/dist/paired-arms-iZ08VFMN.js.map +0 -1
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +0 -114
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +0 -1
- package/dist/prime-protocol-BfSalTfR.js.map +0 -1
- package/dist/promotion-policy-Ckjhzg_4.d.ts +0 -289
- package/dist/promotion-policy-Ckjhzg_4.d.ts.map +0 -1
- package/dist/promotion-policy-CrLrmys8.js.map +0 -1
- package/dist/proposal-findings-2GIUo1et.js.map +0 -1
- package/dist/propose-review-control-dSNPjFUH.js +0 -1458
- package/dist/propose-review-control-dSNPjFUH.js.map +0 -1
- package/dist/release-report-BUYmoKo2.js.map +0 -1
- package/dist/release-report-DA2BCu5a.d.ts.map +0 -1
- package/dist/replay-CohS93nE.js +0 -1859
- package/dist/replay-CohS93nE.js.map +0 -1
- package/dist/replay-DbhZ4Ked.d.ts +0 -834
- package/dist/replay-DbhZ4Ked.d.ts.map +0 -1
- package/dist/reward-hacking-BDToousL.js.map +0 -1
- package/dist/run-evidence-Dhi3C81V.d.ts +0 -225
- package/dist/run-evidence-Dhi3C81V.d.ts.map +0 -1
- package/dist/schema-Cef2cFmb2.d.ts.map +0 -1
- package/dist/semantic-concept-judge-D1z-KepS.js +0 -767
- package/dist/semantic-concept-judge-D1z-KepS.js.map +0 -1
- package/dist/series-convergence-ofsqPWhs.d.ts +0 -35
- package/dist/series-convergence-ofsqPWhs.d.ts.map +0 -1
- package/dist/server-iu0ede49.js.map +0 -1
- package/dist/single-run-lock-DFWHEB09.js.map +0 -1
- package/dist/skill-usage-CO9OLRBx.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CQdVeM8k.js.map +0 -1
- package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +0 -1
- package/dist/statistical-heldout-TQ-4CYiN.d.ts.map +0 -1
- package/dist/statistics-ByxzSiOM.js +0 -2212
- package/dist/statistics-ByxzSiOM.js.map +0 -1
- package/dist/statistics-D6Uebe_4.d.ts +0 -968
- package/dist/statistics-D6Uebe_4.d.ts.map +0 -1
- package/dist/supervisor-run-D_sokXcO.js +0 -1690
- package/dist/test-graded-scenario-D1TaI2va.d.ts +0 -141
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +0 -1
- package/dist/test-graded-scenario-JHcKQNpq.js +0 -318
- package/dist/test-graded-scenario-JHcKQNpq.js.map +0 -1
- package/dist/tool-groups-CZPGGlHf.d.ts.map +0 -1
- package/dist/tool-use-metrics-DEGMKycK.js +0 -370
- package/dist/tool-use-metrics-DEGMKycK.js.map +0 -1
- package/dist/types-D216SgwM.d.ts.map +0 -1
- package/dist/usage-receipt-t7vAzCRQ.js.map +0 -1
- package/dist/verdict-DExhxfgR.d.ts +0 -201
- package/dist/verdict-DExhxfgR.d.ts.map +0 -1
|
@@ -1,834 +0,0 @@
|
|
|
1
|
-
import { c as ReplayError, l as ValidationError, o as LimitExceededError, r as CaptureIntegrityError, s as NotFoundError, t as AgentEvalError } from "./errors-CKPfb2aH.js";
|
|
2
|
-
import { c as CostLedgerHandle } from "./cost-ledger-Bv_e8XHY.js";
|
|
3
|
-
import { S as ToolSpan, f as Run } from "./schema-BtVldJ3T.js";
|
|
4
|
-
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-CF4Dwpxr.js";
|
|
5
|
-
import { Ft as RawProviderSink, Pt as RawProviderEvent } from "./types-D216SgwM.js";
|
|
6
|
-
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
7
|
-
import { n as RunCompleteHookContext, t as RunCompleteHook } from "./emitter-DGQGoLyj.js";
|
|
8
|
-
import { A as SearchSpanResult, B as ViewSpansResult, D as DatasetOverview, F as TraceAnalystFilters, H as ViewTraceResult, L as TraceAnalystSpanKind, R as TraceAnalystSpanStatus, S as BoundedTraceAnalysisStoreOptions, T as TraceAnalysisStoreContext, f as AnalystUsageReceipt, i as AnalystFinding, j as SearchTraceResult, k as QueryTracesPage, w as TraceAnalysisStore } from "./types-BvDKaULh.js";
|
|
9
|
-
import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-3hL-XqwJ.js";
|
|
10
|
-
import { t as TraceToolGroupName } from "./tool-groups-CZPGGlHf.js";
|
|
11
|
-
//#region src/trace/otel.d.ts
|
|
12
|
-
declare const OTEL_AGENT_EVAL_SCOPE: {
|
|
13
|
-
name: string;
|
|
14
|
-
version: string;
|
|
15
|
-
};
|
|
16
|
-
interface OtlpSpan {
|
|
17
|
-
traceId: string;
|
|
18
|
-
spanId: string;
|
|
19
|
-
parentSpanId?: string;
|
|
20
|
-
name: string;
|
|
21
|
-
kind: number;
|
|
22
|
-
startTimeUnixNano: string;
|
|
23
|
-
endTimeUnixNano: string;
|
|
24
|
-
attributes: Array<{
|
|
25
|
-
key: string;
|
|
26
|
-
value: {
|
|
27
|
-
stringValue?: string;
|
|
28
|
-
intValue?: string;
|
|
29
|
-
doubleValue?: number;
|
|
30
|
-
boolValue?: boolean;
|
|
31
|
-
};
|
|
32
|
-
}>;
|
|
33
|
-
events?: Array<{
|
|
34
|
-
timeUnixNano: string;
|
|
35
|
-
name: string;
|
|
36
|
-
attributes?: OtlpSpan['attributes'];
|
|
37
|
-
}>;
|
|
38
|
-
status?: {
|
|
39
|
-
code: number;
|
|
40
|
-
message?: string;
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
interface OtlpResourceSpans {
|
|
44
|
-
resource: {
|
|
45
|
-
attributes: OtlpSpan['attributes'];
|
|
46
|
-
};
|
|
47
|
-
scopeSpans: Array<{
|
|
48
|
-
scope: typeof OTEL_AGENT_EVAL_SCOPE;
|
|
49
|
-
spans: OtlpSpan[];
|
|
50
|
-
}>;
|
|
51
|
-
}
|
|
52
|
-
interface OtlpExport {
|
|
53
|
-
resourceSpans: OtlpResourceSpans[];
|
|
54
|
-
}
|
|
55
|
-
/** Export a single run's spans + events in OTLP/JSON. */
|
|
56
|
-
declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
|
|
57
|
-
//#endregion
|
|
58
|
-
//#region src/trace/otlp-attributes.d.ts
|
|
59
|
-
type ToolSpanOtlpInput = Pick<ToolSpan, 'toolName' | 'args' | 'argsCaptured' | 'result' | 'latencyMs'>;
|
|
60
|
-
type OtlpSpanRole = 'AGENT' | 'CHAIN' | 'EVALUATOR' | 'GUARDRAIL' | 'LLM' | 'SPAN' | 'TOOL' | 'UNKNOWN';
|
|
61
|
-
interface OtlpSpanRoleInput {
|
|
62
|
-
name: string;
|
|
63
|
-
attributes: Record<string, unknown>;
|
|
64
|
-
kind?: string | null;
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Classify a span once for both measurement and error accounting.
|
|
68
|
-
* An explicit OpenInference kind wins; untyped spans use the same tool and
|
|
69
|
-
* model signals in online and offline intake.
|
|
70
|
-
*/
|
|
71
|
-
declare function classifyOtlpSpanRole(input: OtlpSpanRoleInput): OtlpSpanRole;
|
|
72
|
-
declare function isOtlpModelCall(input: OtlpSpanRoleInput): boolean;
|
|
73
|
-
declare function applyToolSpanOtlpAttributes(attributes: Record<string, unknown>, span: ToolSpanOtlpInput): void;
|
|
74
|
-
declare function traceSpanKindToOpenInferenceKind(kind: string): string;
|
|
75
|
-
//#endregion
|
|
76
|
-
//#region src/trace/otel-export.d.ts
|
|
77
|
-
interface OtelExportConfig {
|
|
78
|
-
/** OTLP endpoint. Reads OTEL_EXPORTER_OTLP_ENDPOINT env by default. */
|
|
79
|
-
endpoint?: string;
|
|
80
|
-
/** OTLP headers. Reads OTEL_EXPORTER_OTLP_HEADERS env by default. */
|
|
81
|
-
headers?: Record<string, string>;
|
|
82
|
-
/** Batch size before flush. Default 64. */
|
|
83
|
-
batchSize?: number;
|
|
84
|
-
/** Flush interval ms. Default 5000. */
|
|
85
|
-
flushIntervalMs?: number;
|
|
86
|
-
/** Resource attributes stamped on every export. */
|
|
87
|
-
resourceAttributes?: Record<string, string | number | boolean>;
|
|
88
|
-
/** Service name. Default 'agent-eval'. */
|
|
89
|
-
serviceName?: string;
|
|
90
|
-
}
|
|
91
|
-
interface OtelExporter {
|
|
92
|
-
/** Called by the TraceEmitter on every span close. */
|
|
93
|
-
exportSpan(span: ExportableSpan): void;
|
|
94
|
-
/** Force flush pending spans. */
|
|
95
|
-
flush(): Promise<void>;
|
|
96
|
-
/** Shutdown cleanly — flushes remaining spans and stops the timer. */
|
|
97
|
-
shutdown(): Promise<void>;
|
|
98
|
-
}
|
|
99
|
-
interface ExportableSpan {
|
|
100
|
-
traceId: string;
|
|
101
|
-
spanId: string;
|
|
102
|
-
parentSpanId?: string;
|
|
103
|
-
name: string;
|
|
104
|
-
kind: string;
|
|
105
|
-
startedAt: number;
|
|
106
|
-
endedAt?: number;
|
|
107
|
-
status?: string;
|
|
108
|
-
error?: string;
|
|
109
|
-
model?: string;
|
|
110
|
-
inputTokens?: number;
|
|
111
|
-
outputTokens?: number;
|
|
112
|
-
reasoningTokens?: number;
|
|
113
|
-
cachedTokens?: number;
|
|
114
|
-
cacheWriteTokens?: number;
|
|
115
|
-
costUsd?: number;
|
|
116
|
-
tool?: ToolSpanOtlpInput;
|
|
117
|
-
attributes?: Record<string, unknown>;
|
|
118
|
-
}
|
|
119
|
-
/**
|
|
120
|
-
* Create an OTEL exporter. Returns undefined when no endpoint is configured
|
|
121
|
-
* (neither via config nor env) — callers should check before attaching.
|
|
122
|
-
*/
|
|
123
|
-
declare function createOtelExporter(config?: OtelExportConfig): OtelExporter | undefined;
|
|
124
|
-
//#endregion
|
|
125
|
-
//#region src/trace/otel-bridge.d.ts
|
|
126
|
-
/**
|
|
127
|
-
* Create a RunCompleteHook that exports all spans from the completed run
|
|
128
|
-
* to the OTEL exporter, then flushes.
|
|
129
|
-
*/
|
|
130
|
-
declare function otelRunCompleteHook(exporter: OtelExporter): RunCompleteHook;
|
|
131
|
-
/**
|
|
132
|
-
* Create an auto-exporting TraceStore wrapper that intercepts updateSpan
|
|
133
|
-
* calls. When a span gets an endedAt, it's exported immediately. This
|
|
134
|
-
* gives real-time streaming instead of batch-at-end.
|
|
135
|
-
*
|
|
136
|
-
* This is the preferred integration path: wrap the store before
|
|
137
|
-
* constructing the TraceEmitter.
|
|
138
|
-
*/
|
|
139
|
-
declare function createOtelTracingStore(inner: TraceStore, exporter: OtelExporter, traceId: string): TraceStore;
|
|
140
|
-
//#endregion
|
|
141
|
-
//#region src/trace/redact.d.ts
|
|
142
|
-
/**
|
|
143
|
-
* Redaction — remove PII / secrets from trace payloads before persist.
|
|
144
|
-
*
|
|
145
|
-
* Pre-persistence rules mean raw traces in storage are already scrubbed.
|
|
146
|
-
* Unredacted variants (for debugging / post-mortems) live in a separate
|
|
147
|
-
* storage layer with stricter access controls; this module only covers
|
|
148
|
-
* the default scrub-then-persist path.
|
|
149
|
-
*
|
|
150
|
-
* Rules compose: pass an array of `RedactionRule`, each is applied in
|
|
151
|
-
* order. Strings that match get replaced with a tagged sentinel so the
|
|
152
|
-
* eval framework can count how many redactions happened per run
|
|
153
|
-
* (surfaced via `redaction_applied` events).
|
|
154
|
-
*/
|
|
155
|
-
interface RedactionRule {
|
|
156
|
-
id: string;
|
|
157
|
-
pattern: RegExp;
|
|
158
|
-
/** Replacement — e.g. '[PII:email]'. Defaults to `[redacted:{id}]`. */
|
|
159
|
-
replacement?: string;
|
|
160
|
-
}
|
|
161
|
-
interface RedactionReport {
|
|
162
|
-
redactionCount: number;
|
|
163
|
-
byRule: Record<string, number>;
|
|
164
|
-
}
|
|
165
|
-
/** OWASP / common-sense defaults — extend per-domain. */
|
|
166
|
-
declare const DEFAULT_REDACTION_RULES: RedactionRule[];
|
|
167
|
-
declare const REDACTION_VERSION = "1.0.0";
|
|
168
|
-
/**
|
|
169
|
-
* Redact a single string. Returns the new string and a per-rule count of
|
|
170
|
-
* how many substitutions fired.
|
|
171
|
-
*/
|
|
172
|
-
declare function redactString(input: string, rules?: RedactionRule[]): {
|
|
173
|
-
output: string;
|
|
174
|
-
report: RedactionReport;
|
|
175
|
-
};
|
|
176
|
-
/**
|
|
177
|
-
* Walk a JSON-ish value applying `redactString` to every string leaf.
|
|
178
|
-
* Arrays and plain objects are recursed; other types pass through
|
|
179
|
-
* untouched. Circular references throw — traces should be tree-shaped.
|
|
180
|
-
*/
|
|
181
|
-
declare function redactValue(value: unknown, rules?: RedactionRule[], report?: RedactionReport): {
|
|
182
|
-
value: unknown;
|
|
183
|
-
report: RedactionReport;
|
|
184
|
-
};
|
|
185
|
-
//#endregion
|
|
186
|
-
//#region src/trace/store-to-otlp.d.ts
|
|
187
|
-
interface TracesToOtlpResult {
|
|
188
|
-
/** Total spans emitted across every cell. */
|
|
189
|
-
spanCount: number;
|
|
190
|
-
/** Total run-anchor spans (one per Run) appended for analyst visibility. */
|
|
191
|
-
runCount: number;
|
|
192
|
-
/** Cells whose shards parsed cleanly. */
|
|
193
|
-
cellCount: number;
|
|
194
|
-
/** Cells that errored mid-conversion — surfaced so a partial conversion
|
|
195
|
-
* isn't silently masked. */
|
|
196
|
-
cellErrorCount: number;
|
|
197
|
-
}
|
|
198
|
-
/**
|
|
199
|
-
* A trace source the analyst should ingest. Two layouts are supported:
|
|
200
|
-
*
|
|
201
|
-
* - `celled` (default): the root holds one cell subdirectory per
|
|
202
|
-
* persona/variant, each a `FileSystemTraceStore` (`runs.ndjson`,
|
|
203
|
-
* `spans.ndjson`, `events.ndjson`).
|
|
204
|
-
* - `flat`: the root is itself a single `FileSystemTraceStore` (e.g. a
|
|
205
|
-
* production-ingestion sidecar that appends one ndjson set directly under
|
|
206
|
-
* the chosen directory).
|
|
207
|
-
*/
|
|
208
|
-
interface TraceStoreSource {
|
|
209
|
-
/** Absolute path to the trace store root. */
|
|
210
|
-
root: string;
|
|
211
|
-
/** Layout — `celled` (default) or `flat`. */
|
|
212
|
-
layout?: 'celled' | 'flat';
|
|
213
|
-
/** OTLP `service.name` for this source. Overrides the options default. */
|
|
214
|
-
serviceName?: string;
|
|
215
|
-
}
|
|
216
|
-
/** Domain hooks. The walker is generic; consumers inject the namespaced
|
|
217
|
-
* attributes that ride along on every run + span for analyst discovery. */
|
|
218
|
-
interface TraceStoreToOtlpOptions {
|
|
219
|
-
/** Default OTLP `service.name` when a source doesn't set its own.
|
|
220
|
-
* Default `agent-eval`. */
|
|
221
|
-
serviceName?: string;
|
|
222
|
-
/** Extra resource attributes per run, e.g.
|
|
223
|
-
* `(run) => ({ 'legal.persona_id': run.tags?.personaId ?? '' })`. */
|
|
224
|
-
resourceAttributes?: (run: Run) => Record<string, unknown>;
|
|
225
|
-
/** Extra attributes on the per-run anchor span, e.g. domain outcome fields. */
|
|
226
|
-
runAttributes?: (run: Run) => Record<string, unknown>;
|
|
227
|
-
}
|
|
228
|
-
/**
|
|
229
|
-
* Read every per-cell shard under each source root and write a flat OTLP-JSONL
|
|
230
|
-
* view of the corpus to `outPath`. Each cell directory is a
|
|
231
|
-
* `FileSystemTraceStore` — NDJSON append-only with size-based rotation;
|
|
232
|
-
* `updateRun`/`updateSpan` append `{ id, ...patch, _update: true }` rows
|
|
233
|
-
* rather than rewriting, so readers must merge those patches in (done here).
|
|
234
|
-
*
|
|
235
|
-
* A `string` source is treated as a celled root.
|
|
236
|
-
*/
|
|
237
|
-
declare function convertTraceStoresToOtlp(source: string | TraceStoreSource | readonly TraceStoreSource[], outPath: string, opts?: TraceStoreToOtlpOptions): TracesToOtlpResult;
|
|
238
|
-
//#endregion
|
|
239
|
-
//#region src/trace-analyst/store-boundary.d.ts
|
|
240
|
-
/** Apply the public validation, cancellation, not-found, and size rules to any adapter. */
|
|
241
|
-
declare function createBoundedTraceAnalysisStore(source: TraceAnalysisStore, options?: BoundedTraceAnalysisStoreOptions): TraceAnalysisStore;
|
|
242
|
-
//#endregion
|
|
243
|
-
//#region src/trace-analyst/analyst.d.ts
|
|
244
|
-
interface AnalyzeTracesInput {
|
|
245
|
-
question: string;
|
|
246
|
-
id?: string;
|
|
247
|
-
description?: string;
|
|
248
|
-
area?: string;
|
|
249
|
-
}
|
|
250
|
-
type AnalyzeTracesResult = TraceAnalysisEngineResult;
|
|
251
|
-
interface AnalyzeTracesOptions {
|
|
252
|
-
/** OTLP JSONL path or a caller-owned store. */
|
|
253
|
-
source: string | TraceAnalysisStore;
|
|
254
|
-
/** Recursive research engine. */
|
|
255
|
-
engine: TraceAnalysisEngine;
|
|
256
|
-
/** Override the general trace-research policy. */
|
|
257
|
-
instructions?: string;
|
|
258
|
-
/** Smallest tool set that can answer the question. Default: all. */
|
|
259
|
-
toolGroup?: TraceToolGroupName;
|
|
260
|
-
limits?: Partial<TraceAnalystLimits>;
|
|
261
|
-
runId?: string;
|
|
262
|
-
budgetUsd?: number;
|
|
263
|
-
costLedger?: CostLedgerHandle;
|
|
264
|
-
costPhase?: string;
|
|
265
|
-
priorFindings?: readonly AnalystFinding[];
|
|
266
|
-
upstreamFindings?: readonly AnalystFinding[];
|
|
267
|
-
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
268
|
-
tags?: Record<string, string>;
|
|
269
|
-
log?: (message: string, fields?: Record<string, unknown>) => void;
|
|
270
|
-
signal?: AbortSignal;
|
|
271
|
-
}
|
|
272
|
-
/**
|
|
273
|
-
* Answer one question by recursively inspecting a trace store.
|
|
274
|
-
*
|
|
275
|
-
* The returned answer, cited findings, engine steps, call counts, and runtime
|
|
276
|
-
* identity are one audit record. A direct one-shot model call is not used.
|
|
277
|
-
*/
|
|
278
|
-
declare function analyzeTraces(input: AnalyzeTracesInput, options: AnalyzeTracesOptions): Promise<AnalyzeTracesResult>;
|
|
279
|
-
//#endregion
|
|
280
|
-
//#region src/trace-analyst/errors.d.ts
|
|
281
|
-
/** Invalid trace-tool arguments, including malformed filters and regexes. */
|
|
282
|
-
declare class TraceAnalysisValidationError extends ValidationError {
|
|
283
|
-
readonly code: 'validation';
|
|
284
|
-
}
|
|
285
|
-
/** A trace read cannot fit within a documented count or byte limit. */
|
|
286
|
-
declare class TraceAnalysisLimitError extends LimitExceededError {
|
|
287
|
-
readonly code: 'limit_exceeded';
|
|
288
|
-
readonly operation: string;
|
|
289
|
-
readonly actual: number;
|
|
290
|
-
readonly limit: number;
|
|
291
|
-
constructor(operation: string, actual: number, limit: number, message?: string);
|
|
292
|
-
}
|
|
293
|
-
/** A supplied store returned a shape that violates the trace read contract. */
|
|
294
|
-
declare class TraceAnalysisStoreContractError extends AgentEvalError {
|
|
295
|
-
readonly code: 'backend_integrity';
|
|
296
|
-
readonly operation: string;
|
|
297
|
-
constructor(operation: string, message: string, options?: {
|
|
298
|
-
cause?: unknown;
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
declare class TraceFileMissingError extends NotFoundError {
|
|
302
|
-
readonly code: 'not_found';
|
|
303
|
-
readonly path: string;
|
|
304
|
-
constructor(path: string);
|
|
305
|
-
}
|
|
306
|
-
declare class TraceFileTooLargeError extends TraceAnalysisLimitError {
|
|
307
|
-
readonly path: string;
|
|
308
|
-
readonly size_bytes: number;
|
|
309
|
-
readonly max_bytes: number;
|
|
310
|
-
constructor(path: string, size_bytes: number, max_bytes: number);
|
|
311
|
-
}
|
|
312
|
-
declare class TraceFileMalformedError extends CaptureIntegrityError {
|
|
313
|
-
readonly path: string;
|
|
314
|
-
readonly line_number: number;
|
|
315
|
-
readonly byte_offset: number;
|
|
316
|
-
constructor(path: string, line_number: number, byte_offset: number, cause?: unknown);
|
|
317
|
-
}
|
|
318
|
-
declare class TraceNotFoundError extends NotFoundError {
|
|
319
|
-
readonly code: 'not_found';
|
|
320
|
-
readonly trace_id: string;
|
|
321
|
-
constructor(trace_id: string);
|
|
322
|
-
}
|
|
323
|
-
declare class SpanNotFoundError extends NotFoundError {
|
|
324
|
-
readonly code: 'not_found';
|
|
325
|
-
readonly trace_id: string;
|
|
326
|
-
readonly span_id: string;
|
|
327
|
-
constructor(trace_id: string, span_id: string);
|
|
328
|
-
}
|
|
329
|
-
//#endregion
|
|
330
|
-
//#region src/trace-analyst/hook.d.ts
|
|
331
|
-
interface TraceAnalystHookOptions {
|
|
332
|
-
/**
|
|
333
|
-
* Options forwarded to `analyzeTraces`. The hook supplies the question
|
|
334
|
-
* if you don't pass one — defaulting to a launch-grade prompt that asks
|
|
335
|
-
* for failure modes, surprising findings, and a recommendation.
|
|
336
|
-
*/
|
|
337
|
-
analyze: Omit<AnalyzeTracesOptions, 'source'> & {
|
|
338
|
-
source?: AnalyzeTracesOptions['source'];
|
|
339
|
-
};
|
|
340
|
-
/**
|
|
341
|
-
* Override the question. The default is intentionally generic:
|
|
342
|
-
* "Summarise what happened in this run, surface any failure modes,
|
|
343
|
-
* surprising findings, or evidence the verdict is wrong."
|
|
344
|
-
*/
|
|
345
|
-
question?: string;
|
|
346
|
-
/**
|
|
347
|
-
* Persist the result. The hook calls this with the analysis output and
|
|
348
|
-
* the run context. Common implementations write to a TraceAnalysisStore
|
|
349
|
-
* or append to a per-run JSONL.
|
|
350
|
-
*/
|
|
351
|
-
save?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => Promise<void>;
|
|
352
|
-
/**
|
|
353
|
-
* Predicate gating execution per run. Default: every completed run.
|
|
354
|
-
* Use to skip aborted runs, debug runs, or runs without LLM activity.
|
|
355
|
-
*/
|
|
356
|
-
shouldRun?: (ctx: RunCompleteHookContext) => boolean;
|
|
357
|
-
/**
|
|
358
|
-
* Optional gate: if set and returns false, the hook records the failure
|
|
359
|
-
* as a log event on the run instead of staying quiet. The caller can
|
|
360
|
-
* then trigger downstream alerts off `analyst_gate_failed` log events.
|
|
361
|
-
*/
|
|
362
|
-
gateOn?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => boolean;
|
|
363
|
-
}
|
|
364
|
-
declare function traceAnalystOnRunComplete(opts: TraceAnalystHookOptions): RunCompleteHook;
|
|
365
|
-
//#endregion
|
|
366
|
-
//#region src/trace-analyst/insights.d.ts
|
|
367
|
-
interface TraceInsightTask {
|
|
368
|
-
id: string;
|
|
369
|
-
name: string;
|
|
370
|
-
prompt?: string;
|
|
371
|
-
difficulty?: string;
|
|
372
|
-
tags?: string[];
|
|
373
|
-
outcome?: string;
|
|
374
|
-
score?: number;
|
|
375
|
-
gaps?: string[];
|
|
376
|
-
}
|
|
377
|
-
interface TraceInsightSuite {
|
|
378
|
-
name: string;
|
|
379
|
-
collectionId?: string;
|
|
380
|
-
tasks: TraceInsightTask[];
|
|
381
|
-
}
|
|
382
|
-
interface TraceInsightFinding {
|
|
383
|
-
kind: string;
|
|
384
|
-
severity?: string;
|
|
385
|
-
taskIds: string[];
|
|
386
|
-
evidence?: string;
|
|
387
|
-
proposedFixClass?: string;
|
|
388
|
-
}
|
|
389
|
-
interface TraceInsightQuestion {
|
|
390
|
-
id: string;
|
|
391
|
-
question: string;
|
|
392
|
-
why: string;
|
|
393
|
-
}
|
|
394
|
-
interface TraceInsightPanelRole {
|
|
395
|
-
id: string;
|
|
396
|
-
name: string;
|
|
397
|
-
responsibility: string;
|
|
398
|
-
}
|
|
399
|
-
interface TraceInsightPromptInput {
|
|
400
|
-
suite: TraceInsightSuite;
|
|
401
|
-
findings?: TraceInsightFinding[];
|
|
402
|
-
agent?: Record<string, unknown>;
|
|
403
|
-
totals?: Record<string, unknown>;
|
|
404
|
-
maxRepresentativeTraces?: number;
|
|
405
|
-
}
|
|
406
|
-
interface TraceInsightContext {
|
|
407
|
-
suite: TraceInsightSuite;
|
|
408
|
-
scope: string;
|
|
409
|
-
keywords: string[];
|
|
410
|
-
questions: TraceInsightQuestion[];
|
|
411
|
-
panel: TraceInsightPanelRole[];
|
|
412
|
-
findings: TraceInsightFinding[];
|
|
413
|
-
agent: Record<string, unknown> | null;
|
|
414
|
-
totals: Record<string, unknown> | null;
|
|
415
|
-
}
|
|
416
|
-
interface TraceInsightQualityGate {
|
|
417
|
-
id: string;
|
|
418
|
-
label: string;
|
|
419
|
-
passed: boolean;
|
|
420
|
-
severity: 'critical' | 'high' | 'medium' | 'low';
|
|
421
|
-
detail: string;
|
|
422
|
-
}
|
|
423
|
-
interface TraceInsightReadiness {
|
|
424
|
-
score: number;
|
|
425
|
-
grade: 'external-ready' | 'internal-review' | 'raw-analysis';
|
|
426
|
-
gates: TraceInsightQualityGate[];
|
|
427
|
-
}
|
|
428
|
-
declare function tokenizeDomainWords(value: string): string[];
|
|
429
|
-
declare function inferDomainKeywords(suite: TraceInsightSuite): string[];
|
|
430
|
-
declare function domainEvidencePattern(keywords: string[]): RegExp;
|
|
431
|
-
declare function describeTraceInsightScope(suite: TraceInsightSuite): string;
|
|
432
|
-
declare function planTraceInsightQuestions(input: TraceInsightPromptInput): TraceInsightQuestion[];
|
|
433
|
-
declare function buildTraceInsightContext(input: TraceInsightPromptInput): TraceInsightContext;
|
|
434
|
-
declare function scoreTraceInsightReadiness(context: TraceInsightContext): TraceInsightReadiness;
|
|
435
|
-
declare function defaultTraceInsightPanel(): TraceInsightPanelRole[];
|
|
436
|
-
declare function buildTraceInsightPrompt(input: TraceInsightPromptInput): string;
|
|
437
|
-
//#endregion
|
|
438
|
-
//#region src/trace/otlp-flat.d.ts
|
|
439
|
-
type OtlpStatusCode = 'STATUS_CODE_OK' | 'STATUS_CODE_ERROR' | 'STATUS_CODE_UNSET';
|
|
440
|
-
interface OtlpFlatLine {
|
|
441
|
-
trace_id: string;
|
|
442
|
-
span_id: string;
|
|
443
|
-
parent_span_id: string | null;
|
|
444
|
-
name: string;
|
|
445
|
-
kind: string;
|
|
446
|
-
start_time: string;
|
|
447
|
-
end_time: string;
|
|
448
|
-
status: {
|
|
449
|
-
code: OtlpStatusCode;
|
|
450
|
-
message?: string;
|
|
451
|
-
};
|
|
452
|
-
resource: {
|
|
453
|
-
attributes: Record<string, unknown>;
|
|
454
|
-
};
|
|
455
|
-
attributes: Record<string, unknown>;
|
|
456
|
-
events?: Array<{
|
|
457
|
-
name: string;
|
|
458
|
-
timeUnixNano?: string;
|
|
459
|
-
attributes?: Record<string, unknown>;
|
|
460
|
-
}>;
|
|
461
|
-
}
|
|
462
|
-
//#endregion
|
|
463
|
-
//#region src/trace-analyst/otlp-flatten.d.ts
|
|
464
|
-
interface FlattenOtlpOptions {
|
|
465
|
-
/** `'openinference'` (default) maps source per-span attributes into the
|
|
466
|
-
* canonical OpenInference vocabulary the analyst readers consume. `'none'`
|
|
467
|
-
* passes attributes through untouched. */
|
|
468
|
-
attributeVocabulary?: 'openinference' | 'none';
|
|
469
|
-
/** Override the numeric-kind → otlp-string mapping. */
|
|
470
|
-
kindMap?: Partial<Record<number, string>>;
|
|
471
|
-
}
|
|
472
|
-
declare function flattenOtlpExportToNdjson(otlpExport: OtlpExport, opts?: FlattenOtlpOptions): OtlpFlatLine[];
|
|
473
|
-
//#endregion
|
|
474
|
-
//#region src/trace-analyst/otlp-span.d.ts
|
|
475
|
-
/**
|
|
476
|
-
* The structural fields a flat OTLP-JSONL line projects to. `attributes`
|
|
477
|
-
* is the merged resource+span attribute map (span overrides resource);
|
|
478
|
-
* the named fields are the pivots every reader of a trace needs without
|
|
479
|
-
* paying the full attribute materialisation.
|
|
480
|
-
*/
|
|
481
|
-
interface ProjectedOtlpSpan {
|
|
482
|
-
trace_id: string;
|
|
483
|
-
span_id: string;
|
|
484
|
-
parent_span_id: string | null;
|
|
485
|
-
name: string;
|
|
486
|
-
kind: TraceAnalystSpanKind;
|
|
487
|
-
start_time: string;
|
|
488
|
-
end_time: string;
|
|
489
|
-
duration_ms: number;
|
|
490
|
-
status: TraceAnalystSpanStatus;
|
|
491
|
-
status_message: string | undefined;
|
|
492
|
-
service_name: string | null;
|
|
493
|
-
agent_name: string | null;
|
|
494
|
-
model_name: string | null;
|
|
495
|
-
tool_name: string | null;
|
|
496
|
-
/** Merged resource + span attributes, span winning on overlap. */
|
|
497
|
-
attributes: Record<string, unknown>;
|
|
498
|
-
}
|
|
499
|
-
/**
|
|
500
|
-
* Project one parsed OTLP-JSONL object to `ProjectedOtlpSpan`, or `null`
|
|
501
|
-
* when the line is missing the mandatory `trace_id` + `span_id`.
|
|
502
|
-
*/
|
|
503
|
-
declare function projectOtlpFlatLine(raw: Record<string, unknown>): ProjectedOtlpSpan | null;
|
|
504
|
-
declare function readOtlpStatus(raw: Record<string, unknown>): {
|
|
505
|
-
code: TraceAnalystSpanStatus;
|
|
506
|
-
message: string | undefined;
|
|
507
|
-
};
|
|
508
|
-
declare function inferOtlpKind(attrs: Record<string, unknown>): TraceAnalystSpanKind;
|
|
509
|
-
/**
|
|
510
|
-
* Flatten OTLP `attributes` + `resource.attributes` into a single
|
|
511
|
-
* dotted-key map. Span attributes override resource attributes when keys
|
|
512
|
-
* overlap. Nested objects/arrays are preserved as-is.
|
|
513
|
-
*/
|
|
514
|
-
declare function extractOtlpAttributes(raw: Record<string, unknown>): Record<string, unknown>;
|
|
515
|
-
declare function stringField(raw: Record<string, unknown>, key: string): string | undefined;
|
|
516
|
-
declare function asString(v: unknown): string | null;
|
|
517
|
-
/** First non-empty string value across a list of candidate attribute keys. */
|
|
518
|
-
declare function firstStringAttr(attrs: Record<string, unknown>, keys: readonly string[]): string | null;
|
|
519
|
-
//#endregion
|
|
520
|
-
//#region src/trace-analyst/otlp-to-run-records.d.ts
|
|
521
|
-
interface OtlpToRunRecordsOptions {
|
|
522
|
-
/** Logical experiment grouping for every produced record. */
|
|
523
|
-
experimentId: string;
|
|
524
|
-
/** Candidate (variant) id — the surface these traces exercised. The
|
|
525
|
-
* bench passes the proposer label here so `compareOptimizationMethods` can pair rows. */
|
|
526
|
-
candidateId: string;
|
|
527
|
-
/** Split assignment for every produced record. Default `'holdout'` —
|
|
528
|
-
* ingested traces are evidence, not the optimizer's training pool. */
|
|
529
|
-
splitTag?: RunSplitTag;
|
|
530
|
-
/** Git SHA the traces were produced from. Default `'unknown'`. */
|
|
531
|
-
commitSha?: string;
|
|
532
|
-
/** sha256 of the effective prompt surface. Default `'unknown'`. */
|
|
533
|
-
promptHash?: string;
|
|
534
|
-
/** sha256 of the effective config. Default `'unknown'`. */
|
|
535
|
-
configHash?: string;
|
|
536
|
-
/** RNG seed recorded on every row. Default 0. */
|
|
537
|
-
seed?: number;
|
|
538
|
-
/**
|
|
539
|
-
* Fallback model snapshot when the trace exposes no LLM model attribute
|
|
540
|
-
* OR exposes a bare alias `validateRunRecord` would reject. The trace's
|
|
541
|
-
* own model wins when it already carries a snapshot. Default
|
|
542
|
-
* `'unknown@otlp'` (opaque-snapshot form the validator accepts).
|
|
543
|
-
*/
|
|
544
|
-
fallbackModel?: string;
|
|
545
|
-
/**
|
|
546
|
-
* USD per total token (input+output) used to price a trace when no
|
|
547
|
-
* per-span cost attribute is present. When unset, an unpriced trace
|
|
548
|
-
* records `costUsd: null` and `raw.cost_unpriced = 1`.
|
|
549
|
-
*/
|
|
550
|
-
priceUsdPerToken?: number;
|
|
551
|
-
/**
|
|
552
|
-
* Map each OTLP `trace_id` to the logical run it belongs to. Use this when a
|
|
553
|
-
* provider emits several traces for one task attempt. All mapped traces are
|
|
554
|
-
* aggregated into one record; duplicate span ids remain isolated by their
|
|
555
|
-
* source trace. The callback must return a non-empty id for every trace.
|
|
556
|
-
*
|
|
557
|
-
* When omitted, every `trace_id` remains an independent run.
|
|
558
|
-
*/
|
|
559
|
-
logicalRunIdForTrace?: (traceId: string) => string;
|
|
560
|
-
/**
|
|
561
|
-
* Score for a produced run's outcome (AppWorld `world.evaluate()` →
|
|
562
|
-
* TGC/SGC, or
|
|
563
|
-
* any [0,1] task-success signal). Keyed by the logical run id when
|
|
564
|
-
* `logicalRunIdForTrace` is supplied, otherwise by `trace_id`. When the map
|
|
565
|
-
* has no entry or the function returns undefined, the record remains
|
|
566
|
-
* unlabeled.
|
|
567
|
-
*/
|
|
568
|
-
scoreForTrace?: (runId: string, span: TraceAggregate) => number | undefined;
|
|
569
|
-
/**
|
|
570
|
-
* Per-record judge metadata when an external judge produced the score.
|
|
571
|
-
* Keyed by the logical run id when supplied, otherwise by `trace_id`.
|
|
572
|
-
*/
|
|
573
|
-
judgeMetadataForTrace?: (runId: string) => RunRecord['judgeMetadata'] | undefined;
|
|
574
|
-
}
|
|
575
|
-
/** A `RunRecord` plus the verbatim prompt/completion text when the trace's
|
|
576
|
-
* LLM spans exposed it. The text is NOT on the validated `RunRecord`
|
|
577
|
-
* (`outcome.raw` is numeric-only) but consumers ingesting full traces want
|
|
578
|
-
* it — so it rides alongside. */
|
|
579
|
-
interface OtlpTraceRunRecord {
|
|
580
|
-
record: RunRecord;
|
|
581
|
-
/** Verbatim first-LLM-span `input.value`, when present. */
|
|
582
|
-
promptText?: string;
|
|
583
|
-
/** Verbatim last-LLM-span `output.value`, when present. */
|
|
584
|
-
completionText?: string;
|
|
585
|
-
}
|
|
586
|
-
/** Per-trace rollup the score callback can inspect. */
|
|
587
|
-
interface TraceAggregate {
|
|
588
|
-
traceId: string;
|
|
589
|
-
/** Number of source OTLP traces folded into this run. */
|
|
590
|
-
sourceTraceCount: number;
|
|
591
|
-
/** Source OTLP trace ids folded into this run, sorted for deterministic audit output. */
|
|
592
|
-
sourceTraceIds: readonly string[];
|
|
593
|
-
spanCount: number;
|
|
594
|
-
llmSpanCount: number;
|
|
595
|
-
toolSpanCount: number;
|
|
596
|
-
agentSpanCount: number;
|
|
597
|
-
errorSpanCount: number;
|
|
598
|
-
executionErrorCount: number;
|
|
599
|
-
processErrorCount: number;
|
|
600
|
-
guardrailErrorCount: number;
|
|
601
|
-
judgeErrorCount: number;
|
|
602
|
-
propagatedErrorCount: number;
|
|
603
|
-
unclassifiedErrorCount: number;
|
|
604
|
-
tokenUsage: RunTokenUsage;
|
|
605
|
-
/** First error span's normalized status message, if any. */
|
|
606
|
-
firstErrorMessage?: string;
|
|
607
|
-
model: string;
|
|
608
|
-
startTime: string;
|
|
609
|
-
endTime: string;
|
|
610
|
-
wallMs: number;
|
|
611
|
-
/** Root-span terminal result. Child span errors do not change this value. */
|
|
612
|
-
terminalOutcome: RunTerminalOutcome;
|
|
613
|
-
}
|
|
614
|
-
/**
|
|
615
|
-
* Parse + aggregate an OTLP traces.jsonl string into validated
|
|
616
|
-
* `RunRecord[]` (one per trace). Use {@link otlpToTraceRunRecords} when you
|
|
617
|
-
* also want the verbatim prompt/completion text alongside each record.
|
|
618
|
-
*/
|
|
619
|
-
declare function otlpToRunRecords(otlpJsonl: string, opts: OtlpToRunRecordsOptions): RunRecord[];
|
|
620
|
-
/**
|
|
621
|
-
* Aggregate already-parsed OTLP flat rows without serializing them back to
|
|
622
|
-
* JSONL. This is the in-memory counterpart to {@link otlpToRunRecords}; both
|
|
623
|
-
* paths share projection, reconciliation, validation, and ordering.
|
|
624
|
-
*/
|
|
625
|
-
declare function otlpRowsToRunRecords(rows: Iterable<object>, opts: OtlpToRunRecordsOptions): RunRecord[];
|
|
626
|
-
/** As {@link otlpToRunRecords} but returns the prompt/completion text too. */
|
|
627
|
-
declare function otlpToTraceRunRecords(otlpJsonl: string, opts: OtlpToRunRecordsOptions): OtlpTraceRunRecord[];
|
|
628
|
-
/** Parsed-row counterpart to {@link otlpToTraceRunRecords}. */
|
|
629
|
-
declare function otlpRowsToTraceRunRecords(rows: Iterable<object>, opts: OtlpToRunRecordsOptions): OtlpTraceRunRecord[];
|
|
630
|
-
//#endregion
|
|
631
|
-
//#region src/trace-analyst/prompts.d.ts
|
|
632
|
-
/** General policy for recursive, evidence-backed trace analysis. */
|
|
633
|
-
declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "Answer the question by inspecting the OTLP trace dataset with the available tools.\n\n1. Call getDatasetOverview first. Use its real trace ids and dataset size to plan the investigation.\n2. Narrow with queryTraces and countTraces before scanning large payloads.\n3. For a small trace, use viewTrace. For a large trace, use searchTrace and then viewSpans or searchSpan.\n4. Never invent a trace id, span id, tool result, error, frequency, or final outcome.\n5. When a search reports has_more, refine the query before drawing a conclusion.\n6. Use llm_query only over evidence already loaded. A recursive query cannot inspect traces itself.\n7. Cite exact evidence URIs returned by the tools. Include a short exact excerpt when it supports the claim.\n8. Return no finding when the available evidence cannot support one.\n\nThe prose answer must directly answer the question and state important uncertainty.\nThe findings array contains only actionable or decision-relevant claims supported by inspected evidence.";
|
|
634
|
-
declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-research-v1-2026-07-30";
|
|
635
|
-
//#endregion
|
|
636
|
-
//#region src/trace-analyst/store-otlp.d.ts
|
|
637
|
-
interface ToolSpansToTraceAnalysisStoreOptions {
|
|
638
|
-
/** Override the discovery (`viewTrace`) per-attribute byte cap. */
|
|
639
|
-
perAttributeViewBudget?: number;
|
|
640
|
-
/** Override the surgical (`viewSpans`) per-attribute byte cap. */
|
|
641
|
-
perAttributeSpanBudget?: number;
|
|
642
|
-
/** Override the per-call ceiling that triggers oversized summaries. */
|
|
643
|
-
perCallByteCeiling?: number;
|
|
644
|
-
/** Override the per-match text budget. */
|
|
645
|
-
perMatchTextBudget?: number;
|
|
646
|
-
}
|
|
647
|
-
type BufferedOtlpTraceStoreOptions = ToolSpansToTraceAnalysisStoreOptions;
|
|
648
|
-
interface OtlpFileTraceStoreOptions extends BufferedOtlpTraceStoreOptions {
|
|
649
|
-
/** Path to the OTLP-JSONL file. */
|
|
650
|
-
path: string;
|
|
651
|
-
/**
|
|
652
|
-
* Hard ceiling on the trace file size in bytes. The store reads the
|
|
653
|
-
* whole file into one Buffer and indexes it in memory, so an
|
|
654
|
-
* unbounded file OOMs the process. Above this size the store fails
|
|
655
|
-
* loud with `TraceFileTooLargeError` instead of degrading silently.
|
|
656
|
-
* Default 256 MiB.
|
|
657
|
-
*/
|
|
658
|
-
maxFileBytes?: number;
|
|
659
|
-
}
|
|
660
|
-
declare abstract class BufferedOtlpTraceStore implements TraceAnalysisStore {
|
|
661
|
-
private readonly perAttributeViewBudget;
|
|
662
|
-
private readonly perAttributeSpanBudget;
|
|
663
|
-
private readonly perCallByteCeiling;
|
|
664
|
-
private readonly perMatchTextBudget;
|
|
665
|
-
private indexValue?;
|
|
666
|
-
private indexTask?;
|
|
667
|
-
private bufferValue?;
|
|
668
|
-
private bufferTask?;
|
|
669
|
-
constructor(opts: BufferedOtlpTraceStoreOptions);
|
|
670
|
-
hasTrace(trace_id: string, context?: TraceAnalysisStoreContext): Promise<boolean>;
|
|
671
|
-
hasSpans(opts: {
|
|
672
|
-
trace_id: string;
|
|
673
|
-
span_ids: readonly string[];
|
|
674
|
-
}, context?: TraceAnalysisStoreContext): Promise<string[]>;
|
|
675
|
-
getOverview(filters?: TraceAnalystFilters, context?: TraceAnalysisStoreContext): Promise<DatasetOverview>;
|
|
676
|
-
queryTraces(opts: {
|
|
677
|
-
filters?: TraceAnalystFilters;
|
|
678
|
-
limit: number;
|
|
679
|
-
offset?: number;
|
|
680
|
-
}, context?: TraceAnalysisStoreContext): Promise<QueryTracesPage>;
|
|
681
|
-
countTraces(filters?: TraceAnalystFilters, context?: TraceAnalysisStoreContext): Promise<number>;
|
|
682
|
-
viewTrace(opts: {
|
|
683
|
-
trace_id: string;
|
|
684
|
-
per_attribute_byte_cap?: number;
|
|
685
|
-
}, context?: TraceAnalysisStoreContext): Promise<ViewTraceResult>;
|
|
686
|
-
viewSpans(opts: {
|
|
687
|
-
trace_id: string;
|
|
688
|
-
span_ids: readonly string[];
|
|
689
|
-
per_attribute_byte_cap?: number;
|
|
690
|
-
}, context?: TraceAnalysisStoreContext): Promise<ViewSpansResult>;
|
|
691
|
-
searchTrace(opts: {
|
|
692
|
-
trace_id: string;
|
|
693
|
-
regex_pattern: string;
|
|
694
|
-
max_matches?: number;
|
|
695
|
-
}, context?: TraceAnalysisStoreContext): Promise<SearchTraceResult>;
|
|
696
|
-
searchSpan(opts: {
|
|
697
|
-
trace_id: string;
|
|
698
|
-
span_id: string;
|
|
699
|
-
regex_pattern: string;
|
|
700
|
-
max_matches?: number;
|
|
701
|
-
}, context?: TraceAnalysisStoreContext): Promise<SearchSpanResult>;
|
|
702
|
-
/** Force the index to materialise. Useful to amortise startup cost
|
|
703
|
-
* before the first agent call. */
|
|
704
|
-
ensureIndexed(context?: TraceAnalysisStoreContext): Promise<void>;
|
|
705
|
-
private buffer;
|
|
706
|
-
protected abstract readBuffer(signal?: AbortSignal): Promise<Buffer>;
|
|
707
|
-
protected abstract sourcePath(): string;
|
|
708
|
-
private index;
|
|
709
|
-
private buildIndex;
|
|
710
|
-
private matchedTraces;
|
|
711
|
-
private toSummary;
|
|
712
|
-
private projectSpan;
|
|
713
|
-
private buildOversizedSummary;
|
|
714
|
-
private scanSpanForMatches;
|
|
715
|
-
}
|
|
716
|
-
declare class OtlpFileTraceStore extends BufferedOtlpTraceStore {
|
|
717
|
-
private readonly path;
|
|
718
|
-
private readonly maxFileBytes;
|
|
719
|
-
constructor(opts: OtlpFileTraceStoreOptions);
|
|
720
|
-
/** Stat-then-read so an oversized file fails loud before allocating the
|
|
721
|
-
* source buffer. Missing files remain distinct from malformed traces. */
|
|
722
|
-
protected readBuffer(signal?: AbortSignal): Promise<Buffer>;
|
|
723
|
-
protected sourcePath(): string;
|
|
724
|
-
}
|
|
725
|
-
/** Build the bounded trace store directly from OpenInference JSONL text. */
|
|
726
|
-
declare function otlpTextToTraceAnalysisStore(text: string, opts?: ToolSpansToTraceAnalysisStoreOptions): TraceAnalysisStore;
|
|
727
|
-
//#endregion
|
|
728
|
-
//#region src/trace-analyst/store-tool-spans.d.ts
|
|
729
|
-
/** Missing tool spans cannot distinguish a tool-free run from broken capture. */
|
|
730
|
-
declare class ToolTraceMissingError extends CaptureIntegrityError {
|
|
731
|
-
constructor();
|
|
732
|
-
}
|
|
733
|
-
/**
|
|
734
|
-
* Snapshot canonical tool spans into the read interface used by trace analysts.
|
|
735
|
-
* One run becomes one trace while arguments, results, errors, and timing remain searchable.
|
|
736
|
-
*/
|
|
737
|
-
declare function toolSpansToTraceAnalysisStore(spans: readonly ToolSpan[] | null | undefined, options?: ToolSpansToTraceAnalysisStoreOptions): TraceAnalysisStore;
|
|
738
|
-
//#endregion
|
|
739
|
-
//#region src/replay.d.ts
|
|
740
|
-
declare class ReplayCacheMissError extends ReplayError {
|
|
741
|
-
readonly url: string;
|
|
742
|
-
readonly requestKey: string;
|
|
743
|
-
constructor(url: string, requestKey: string, message?: string);
|
|
744
|
-
}
|
|
745
|
-
interface ReplayCacheEntry {
|
|
746
|
-
request: RawProviderEvent;
|
|
747
|
-
response: RawProviderEvent;
|
|
748
|
-
}
|
|
749
|
-
interface ReplayCacheStats {
|
|
750
|
-
total: number;
|
|
751
|
-
byProvider: Record<string, number>;
|
|
752
|
-
byModel: Record<string, number>;
|
|
753
|
-
/** Spans for which we have a request but no response (run aborted mid-call). */
|
|
754
|
-
orphanRequests: number;
|
|
755
|
-
}
|
|
756
|
-
/**
|
|
757
|
-
* In-memory deterministic cache of (request → response) keyed on a stable
|
|
758
|
-
* hash of the request body. Built from a `RawProviderSink` containing
|
|
759
|
-
* paired `request` and `response` events from a previous run.
|
|
760
|
-
*
|
|
761
|
-
* The cache is the source of truth for replay; `createReplayFetch` is a
|
|
762
|
-
* thin wrapper that reads from it.
|
|
763
|
-
*/
|
|
764
|
-
declare class ReplayCache {
|
|
765
|
-
private byKey;
|
|
766
|
-
private orphans;
|
|
767
|
-
private byProvider;
|
|
768
|
-
private byModel;
|
|
769
|
-
/**
|
|
770
|
-
* Build a cache from a sink's events. The sink must implement `list()`.
|
|
771
|
-
* Filter by `runId` / `spanId` to scope to a specific replay.
|
|
772
|
-
*/
|
|
773
|
-
static fromSink(sink: RawProviderSink, filter?: {
|
|
774
|
-
runId?: string;
|
|
775
|
-
spanId?: string;
|
|
776
|
-
}): Promise<ReplayCache>;
|
|
777
|
-
/** Build a cache from an in-memory event list. */
|
|
778
|
-
static fromEvents(events: RawProviderEvent[]): Promise<ReplayCache>;
|
|
779
|
-
/** Number of cacheable (request, response) pairs in the cache. */
|
|
780
|
-
size(): number;
|
|
781
|
-
stats(): ReplayCacheStats;
|
|
782
|
-
/** Iterate every cached `(request, response)` pair in insertion order. */
|
|
783
|
-
entries(): IterableIterator<ReplayCacheEntry>;
|
|
784
|
-
/**
|
|
785
|
-
* Look up a cached response by hashing the (model, messages, temperature,
|
|
786
|
-
* maxTokens, response_format) shape. Returns `undefined` on miss; the
|
|
787
|
-
* caller decides whether to throw, fall back to the network, or skip.
|
|
788
|
-
*/
|
|
789
|
-
lookup(requestBody: unknown): Promise<ReplayCacheEntry | undefined>;
|
|
790
|
-
}
|
|
791
|
-
interface ReplayFetchOptions {
|
|
792
|
-
/**
|
|
793
|
-
* Behaviour on cache miss. Default `'throw'`. `'fallback'` calls the
|
|
794
|
-
* `fallbackFetch` (typically `globalThis.fetch`) so a partial replay can
|
|
795
|
-
* still complete; `'fail-closed'` returns a synthetic 599 response so the
|
|
796
|
-
* call site sees a non-retriable failure.
|
|
797
|
-
*/
|
|
798
|
-
onMiss?: 'throw' | 'fallback' | 'fail-closed';
|
|
799
|
-
fallbackFetch?: typeof fetch;
|
|
800
|
-
/** Optional callback fired once per replayed call (for telemetry / counters). */
|
|
801
|
-
onHit?: (info: {
|
|
802
|
-
url: string;
|
|
803
|
-
provider: string;
|
|
804
|
-
model: string;
|
|
805
|
-
}) => void;
|
|
806
|
-
/** Optional callback fired on cache miss before the `onMiss` policy applies. */
|
|
807
|
-
onMissNotify?: (info: {
|
|
808
|
-
url: string;
|
|
809
|
-
requestBody: unknown;
|
|
810
|
-
}) => void;
|
|
811
|
-
}
|
|
812
|
-
/**
|
|
813
|
-
* Build a `fetch`-shaped function that serves cached responses out of a
|
|
814
|
-
* `ReplayCache` for any URL ending in `/chat/completions`. Pass through
|
|
815
|
-
* `LlmClientOptions.fetch` and `callLlm` becomes free.
|
|
816
|
-
*
|
|
817
|
-
* Non-`/chat/completions` URLs are passed straight to the fallback fetch
|
|
818
|
-
* (default: `globalThis.fetch`). This matters because non-LLM HTTP work
|
|
819
|
-
* (judge HTTP servers, sandbox callbacks) sometimes flows through the same
|
|
820
|
-
* `fetch` and shouldn't be intercepted.
|
|
821
|
-
*/
|
|
822
|
-
declare function createReplayFetch(cache: ReplayCache, opts?: ReplayFetchOptions): typeof fetch;
|
|
823
|
-
/**
|
|
824
|
-
* Convenience iterator over `(request, response)` pairs in a sink — for
|
|
825
|
-
* post-hoc scoring that doesn't need a `fetch` shim. The judge or scorer
|
|
826
|
-
* runs purely in-process over cached LLM outputs.
|
|
827
|
-
*/
|
|
828
|
-
declare function iterateRawCalls(sink: RawProviderSink, filter?: {
|
|
829
|
-
runId?: string;
|
|
830
|
-
spanId?: string;
|
|
831
|
-
}): AsyncGenerator<ReplayCacheEntry>;
|
|
832
|
-
//#endregion
|
|
833
|
-
export { traceAnalystOnRunComplete as $, stringField as A, OtlpSpanRoleInput as At, TraceInsightReadiness as B, exportRunAsOtlp as Bt, ProjectedOtlpSpan as C, createOtelTracingStore as Ct, inferOtlpKind as D, OtelExporter as Dt, firstStringAttr as E, OtelExportConfig as Et, TraceInsightFinding as F, traceSpanKindToOpenInferenceKind as Ft, defaultTraceInsightPanel as G, TraceInsightTask as H, TraceInsightPanelRole as I, OTEL_AGENT_EVAL_SCOPE as It, inferDomainKeywords as J, describeTraceInsightScope as K, TraceInsightPromptInput as L, OtlpExport as Lt, flattenOtlpExportToNdjson as M, applyToolSpanOtlpAttributes as Mt, OtlpFlatLine as N, classifyOtlpSpanRole as Nt, projectOtlpFlatLine as O, createOtelExporter as Ot, TraceInsightContext as P, isOtlpModelCall as Pt, TraceAnalystHookOptions as Q, TraceInsightQualityGate as R, OtlpResourceSpans as Rt, otlpToTraceRunRecords as S, redactValue as St, extractOtlpAttributes as T, ExportableSpan as Tt, buildTraceInsightContext as U, TraceInsightSuite as V, buildTraceInsightPrompt as W, scoreTraceInsightReadiness as X, planTraceInsightQuestions as Y, tokenizeDomainWords as Z, OtlpTraceRunRecord as _, DEFAULT_REDACTION_RULES as _t, ReplayFetchOptions as a, TraceFileMissingError as at, otlpRowsToTraceRunRecords as b, RedactionRule as bt, ToolTraceMissingError as c, AnalyzeTracesInput as ct, OtlpFileTraceStoreOptions as d, analyzeTraces as dt, SpanNotFoundError as et, ToolSpansToTraceAnalysisStoreOptions as f, createBoundedTraceAnalysisStore as ft, OtlpToRunRecordsOptions as g, convertTraceStoresToOtlp as gt, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION as h, TracesToOtlpResult as ht, ReplayCacheStats as i, TraceFileMalformedError as it, FlattenOtlpOptions as j, ToolSpanOtlpInput as jt, readOtlpStatus as k, OtlpSpanRole as kt, toolSpansToTraceAnalysisStore as l, AnalyzeTracesOptions as lt, TRACE_ANALYST_ACTOR_DESCRIPTION as m, TraceStoreToOtlpOptions as mt, ReplayCacheEntry as n, TraceAnalysisStoreContractError as nt, createReplayFetch as o, TraceFileTooLargeError as ot, otlpTextToTraceAnalysisStore as p, TraceStoreSource as pt, domainEvidencePattern as q, ReplayCacheMissError as r, TraceAnalysisValidationError as rt, iterateRawCalls as s, TraceNotFoundError as st, ReplayCache as t, TraceAnalysisLimitError as tt, OtlpFileTraceStore as u, AnalyzeTracesResult as ut, TraceAggregate as v, REDACTION_VERSION as vt, asString as w, otelRunCompleteHook as wt, otlpToRunRecords as x, redactString as xt, otlpRowsToRunRecords as y, RedactionReport as yt, TraceInsightQuestion as z, OtlpSpan as zt };
|
|
834
|
-
//# sourceMappingURL=replay-DbhZ4Ked.d.ts.map
|