@tangle-network/agent-eval 0.187.2 → 0.188.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-uAIQK3Nr.d.ts → agent-profile-CfbuZk8b.d.ts} +4 -4
- package/dist/{agent-profile-uAIQK3Nr.d.ts.map → agent-profile-CfbuZk8b.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +12 -12
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +6 -6
- package/dist/{analyze-runs-DapYZz02.d.ts → analyze-runs-CLz5O01x.d.ts} +5 -5
- package/dist/{analyze-runs-DapYZz02.d.ts.map → analyze-runs-CLz5O01x.d.ts.map} +1 -1
- package/dist/{benchmark-command-DGokT8zd.js → benchmark-command-CdUGku3W.js} +12 -12
- package/dist/benchmark-command-CdUGku3W.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-DlxdTOvw.js → campaign-DZuysBbS.js} +6 -6
- package/dist/{campaign-DlxdTOvw.js.map → campaign-DZuysBbS.js.map} +1 -1
- package/dist/{chat-client-DqPqWzVV.js → chat-client-C4dg7Ng_.js} +2 -2
- package/dist/{chat-client-DqPqWzVV.js.map → chat-client-C4dg7Ng_.js.map} +1 -1
- package/dist/{chat-json-call-C-DnDxEy.js → chat-json-call-JVGbfx95.js} +2 -2
- package/dist/{chat-json-call-C-DnDxEy.js.map → chat-json-call-JVGbfx95.js.map} +1 -1
- package/dist/cli.js +4 -4
- package/dist/{client-DCc7a8ZC.d.ts → client-CFj9dfcn.d.ts} +4 -4
- package/dist/{client-DCc7a8ZC.d.ts.map → client-CFj9dfcn.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +6 -6
- package/dist/{counterfactual-CLgrwhkY.d.ts → counterfactual-CPaWN99-.d.ts} +4 -4
- package/dist/{counterfactual-CLgrwhkY.d.ts.map → counterfactual-CPaWN99-.d.ts.map} +1 -1
- package/dist/{counterfactual-Bjq1mlUu.js → counterfactual-DtMWvn7D.js} +2 -2
- package/dist/{counterfactual-Bjq1mlUu.js.map → counterfactual-DtMWvn7D.js.map} +1 -1
- package/dist/{default-registry-DMxeb4GD.d.ts → default-registry-BLsGfnoO.d.ts} +6 -6
- package/dist/{default-registry-DMxeb4GD.d.ts.map → default-registry-BLsGfnoO.d.ts.map} +1 -1
- package/dist/{default-registry-Ca2IWH1X.js → default-registry-BtZ_t-9b.js} +2 -2
- package/dist/{default-registry-Ca2IWH1X.js.map → default-registry-BtZ_t-9b.js.map} +1 -1
- package/dist/{define-agent-eval-C0EqJ9Gz.js → define-agent-eval-ChtddasD.js} +3 -3
- package/dist/{define-agent-eval-C0EqJ9Gz.js.map → define-agent-eval-ChtddasD.js.map} +1 -1
- package/dist/{define-agent-eval-CGBwPPaq.d.ts → define-agent-eval-CsicmDxt.d.ts} +7 -7
- package/dist/{define-agent-eval-CGBwPPaq.d.ts.map → define-agent-eval-CsicmDxt.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +38 -96
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +83 -134
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-BE9lCzzL.js → dspy-rlm-engine-sPX7B5ac.js} +3 -3
- package/dist/{dspy-rlm-engine-BE9lCzzL.js.map → dspy-rlm-engine-sPX7B5ac.js.map} +1 -1
- package/dist/emitter-DbMXUIVP.js +334 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -0
- package/dist/{emitter-Cs0egaFd.d.ts → emitter-HcCmFQl_.d.ts} +37 -6
- package/dist/emitter-HcCmFQl_.d.ts.map +1 -0
- package/dist/{engine-DNlKhIm0.d.ts → engine-Gw2n1irt.d.ts} +15 -5
- package/dist/engine-Gw2n1irt.d.ts.map +1 -0
- package/dist/{eval-campaign-XFHEpN86.js → eval-campaign-CeotQ1ZR.js} +7 -4
- package/dist/{eval-campaign-XFHEpN86.js.map → eval-campaign-CeotQ1ZR.js.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts → exact-types-CrhPoDp8.d.ts} +2 -2
- package/dist/{exact-types-BEHN7tnq.d.ts.map → exact-types-CrhPoDp8.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +1 -2
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-BbhvXTFx.js → external-optimizer-process-D3BLpZyR.js} +2 -2
- package/dist/{external-optimizer-process-BbhvXTFx.js.map → external-optimizer-process-D3BLpZyR.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js → external-optimizer-subprocess-BrfsgL92.js} +2 -2
- package/dist/{external-optimizer-subprocess-DMgmqTB6.js.map → external-optimizer-subprocess-BrfsgL92.js.map} +1 -1
- package/dist/{failure-cluster-OldNRoAt.d.ts → failure-cluster-CbgEDcFJ.d.ts} +3 -3
- package/dist/{failure-cluster-OldNRoAt.d.ts.map → failure-cluster-CbgEDcFJ.d.ts.map} +1 -1
- package/dist/failure-taxonomy-CjabTATj.js +538 -0
- package/dist/failure-taxonomy-CjabTATj.js.map +1 -0
- package/dist/{feedback-trajectory-CytTR7u4.d.ts → feedback-trajectory-BJi4t3Vf.d.ts} +6 -6
- package/dist/{feedback-trajectory-CytTR7u4.d.ts.map → feedback-trajectory-BJi4t3Vf.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-CKZa_rnt.d.ts → index-B-s8qDRJ.d.ts} +2 -2
- package/dist/{index-CKZa_rnt.d.ts.map → index-B-s8qDRJ.d.ts.map} +1 -1
- package/dist/{index-YZrbnBSV.d.ts → index-BD1oSbHr.d.ts} +11 -11
- package/dist/{index-YZrbnBSV.d.ts.map → index-BD1oSbHr.d.ts.map} +1 -1
- package/dist/{index-vYNHCV2G.d.ts → index-Ca0_IjFu.d.ts} +7 -7
- package/dist/{index-vYNHCV2G.d.ts.map → index-Ca0_IjFu.d.ts.map} +1 -1
- package/dist/{index-BRsrfSHT.d.ts → index-eTTnPe3r.d.ts} +9 -9
- package/dist/{index-BRsrfSHT.d.ts.map → index-eTTnPe3r.d.ts.map} +1 -1
- package/dist/index.d.ts +383 -68
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1482 -207
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-DpOo-3iL.d.ts → insight-report-DeE1solA.d.ts} +4 -4
- package/dist/{insight-report-DpOo-3iL.d.ts.map → insight-report-DeE1solA.d.ts.map} +1 -1
- package/dist/{integrity-Cy9WHAtb.js → integrity-BFP4kjGj.js} +6 -1
- package/dist/integrity-BFP4kjGj.js.map +1 -0
- package/dist/{integrity-rGOfSUle.d.ts → integrity-CaeNeusS.d.ts} +4 -4
- package/dist/integrity-CaeNeusS.d.ts.map +1 -0
- package/dist/jev.d.ts +2 -2
- package/dist/{kind-factory-V79Hw0AU.js → kind-factory-g1ROw67A.js} +15 -4
- package/dist/{kind-factory-V79Hw0AU.js.map → kind-factory-g1ROw67A.js.map} +1 -1
- package/dist/{llm-client-DDhZ29Vm.js → llm-client-DF9bPNYA.js} +2 -2
- package/dist/{llm-client-DDhZ29Vm.js.map → llm-client-DF9bPNYA.js.map} +1 -1
- package/dist/{llm-judge-BVso5x-u.js → llm-judge-DF18J3CR.js} +9 -9
- package/dist/llm-judge-DF18J3CR.js.map +1 -0
- package/dist/{matrix-CPKsRY8Y.d.ts → matrix-BvA1WUpj.d.ts} +2 -2
- package/dist/{matrix-CPKsRY8Y.d.ts.map → matrix-BvA1WUpj.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +4 -4
- package/dist/{mint-BI14xehJ.d.ts → mint-CJFSYBIU.d.ts} +3 -3
- package/dist/{mint-BI14xehJ.d.ts.map → mint-CJFSYBIU.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +67 -12
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +178 -41
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{prime-protocol-CDBcL9NM.d.ts → prime-protocol-CF8Z43Fj.d.ts} +2 -2
- package/dist/{prime-protocol-CDBcL9NM.d.ts.map → prime-protocol-CF8Z43Fj.d.ts.map} +1 -1
- package/dist/{produced-state-Cn9ECr4v.js → produced-state-xsMMjYVc.js} +3 -3
- package/dist/{produced-state-Cn9ECr4v.js.map → produced-state-xsMMjYVc.js.map} +1 -1
- package/dist/{query-D6W6MaGx.d.ts → query-DBUQhehH.d.ts} +3 -3
- package/dist/{query-D6W6MaGx.d.ts.map → query-DBUQhehH.d.ts.map} +1 -1
- package/dist/random-DGEyr3zG.d.ts +83 -0
- package/dist/random-DGEyr3zG.d.ts.map +1 -0
- package/dist/{raw-provider-sink-BQd7mzyT.js → raw-provider-sink-C96CcqeG.js} +19 -56
- package/dist/raw-provider-sink-C96CcqeG.js.map +1 -0
- package/dist/redact-DPoH1-OW.d.ts +158 -0
- package/dist/redact-DPoH1-OW.d.ts.map +1 -0
- package/dist/redact-DxwlUAvh.js +824 -0
- package/dist/redact-DxwlUAvh.js.map +1 -0
- package/dist/{registry-D5Ly3GRV.d.ts → registry-BM2Kdk8T.d.ts} +4 -4
- package/dist/{registry-D5Ly3GRV.d.ts.map → registry-BM2Kdk8T.d.ts.map} +1 -1
- package/dist/{release-confidence-CSp3MsU4.d.ts → release-confidence-CG4Z9QLB.d.ts} +4 -4
- package/dist/{release-confidence-CSp3MsU4.d.ts.map → release-confidence-CG4Z9QLB.d.ts.map} +1 -1
- package/dist/{report-command-CSTjzFGY.js → report-command-DKJ_yAIj.js} +21 -12
- package/dist/{report-command-CSTjzFGY.js.map → report-command-DKJ_yAIj.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/{researcher-D7E92STT.d.ts → researcher-Bjcd_8T_.d.ts} +7 -7
- package/dist/{researcher-D7E92STT.d.ts.map → researcher-Bjcd_8T_.d.ts.map} +1 -1
- package/dist/{reward-hacking-BIQ2BWLG.d.ts → reward-hacking-DrKXJtgH.d.ts} +2 -2
- package/dist/{reward-hacking-BIQ2BWLG.d.ts.map → reward-hacking-DrKXJtgH.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts → rubric-predictive-validity-B14yBAmM.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-CTU8y8Cv.d.ts.map → rubric-predictive-validity-B14yBAmM.d.ts.map} +1 -1
- package/dist/{run-record-D9wS35fK.d.ts → run-record-xEULpgML.d.ts} +2 -2
- package/dist/{run-record-D9wS35fK.d.ts.map → run-record-xEULpgML.d.ts.map} +1 -1
- package/dist/{schema-CR5cpjQ3.d.ts → schema-2g7trUwH.d.ts} +17 -2
- package/dist/schema-2g7trUwH.d.ts.map +1 -0
- package/dist/schema-CSf6qWgZ.js.map +1 -1
- package/dist/{semantic-concept-judge-Cnt0TWp0.js → semantic-concept-judge-70LxrCSL.js} +5 -5
- package/dist/{semantic-concept-judge-Cnt0TWp0.js.map → semantic-concept-judge-70LxrCSL.js.map} +1 -1
- package/dist/{series-convergence-BnMs_uAr.d.ts → series-convergence-DY7ha0kx.d.ts} +2 -2
- package/dist/{series-convergence-BnMs_uAr.d.ts.map → series-convergence-DY7ha0kx.d.ts.map} +1 -1
- package/dist/{server-DmpcFiVK.js → server-C8XvqCg8.js} +2 -2
- package/dist/{server-DmpcFiVK.js.map → server-C8XvqCg8.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js → skillopt-optimization-method-C60h0Got.js} +5 -5
- package/dist/{skillopt-optimization-method-Cc9SVLqi.js.map → skillopt-optimization-method-C60h0Got.js.map} +1 -1
- package/dist/spans-W0o8pbEg.d.ts +60 -0
- package/dist/spans-W0o8pbEg.d.ts.map +1 -0
- package/dist/{statistical-heldout-DTmhTW8P.d.ts → statistical-heldout-CPv-y6sK.d.ts} +3 -3
- package/dist/{statistical-heldout-DTmhTW8P.d.ts.map → statistical-heldout-CPv-y6sK.d.ts.map} +1 -1
- package/dist/{store-BErPvYBr.d.ts → store-C4avFjIi.d.ts} +2 -2
- package/dist/{store-BErPvYBr.d.ts.map → store-C4avFjIi.d.ts.map} +1 -1
- package/dist/{store-otlp-2gYCy-v0.js → store-otlp-DPgzSQKS.js} +2 -2
- package/dist/{store-otlp-2gYCy-v0.js.map → store-otlp-DPgzSQKS.js.map} +1 -1
- package/dist/{store-tool-spans-B3bX9yqn.d.ts → store-tool-spans-C2lLwDzg.d.ts} +9 -54
- package/dist/store-tool-spans-C2lLwDzg.d.ts.map +1 -0
- package/dist/{store-tool-spans-B1e6Auc2.js → store-tool-spans-ClPxcvh2.js} +4 -4
- package/dist/{store-tool-spans-B1e6Auc2.js.map → store-tool-spans-ClPxcvh2.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-Iy6kXo8x.d.ts → summary-report-SNvJoAvD.d.ts} +3 -3
- package/dist/{summary-report-Iy6kXo8x.d.ts.map → summary-report-SNvJoAvD.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{tool-groups-veQYPiTy.d.ts → tool-groups-CCQqAa16.d.ts} +3 -3
- package/dist/tool-groups-CCQqAa16.d.ts.map +1 -0
- package/dist/{tool-waste-nbHPAxei.js → tool-waste-CJDfxH45.js} +3 -537
- package/dist/tool-waste-CJDfxH45.js.map +1 -0
- package/dist/{tool-waste-D23I0zWm.d.ts → tool-waste-CP5DO383.d.ts} +4 -4
- package/dist/{tool-waste-D23I0zWm.d.ts.map → tool-waste-CP5DO383.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/traces.d.ts +12 -11
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +8 -8
- package/dist/{trajectory-D7qrNvaN.d.ts → trajectory-Y1fmdlw7.d.ts} +3 -3
- package/dist/{trajectory-D7qrNvaN.d.ts.map → trajectory-Y1fmdlw7.d.ts.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -3
- package/dist/trajectory-replay/index.js +2 -2
- package/dist/{types--18oHwVF.d.ts → types-5ak2PqnJ.d.ts} +3 -3
- package/dist/{types--18oHwVF.d.ts.map → types-5ak2PqnJ.d.ts.map} +1 -1
- package/dist/{types-BTRcIqFr.d.ts → types-B1vDZ694.d.ts} +4 -4
- package/dist/{types-BTRcIqFr.d.ts.map → types-B1vDZ694.d.ts.map} +1 -1
- package/dist/{types-CBbLtr2J.d.ts → types-C0bWoSKC.d.ts} +10 -11
- package/dist/types-C0bWoSKC.d.ts.map +1 -0
- package/dist/{types-BjB2wZM9.d.ts → types-DDp2Mb8Y.d.ts} +3 -3
- package/dist/{types-BjB2wZM9.d.ts.map → types-DDp2Mb8Y.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/diagnosis.md +2 -3
- package/docs/eval-surface-map.md +1 -1
- package/docs/experiment.md +0 -1
- package/docs/redaction.md +59 -0
- package/docs/trace-contracts.md +102 -0
- package/docs/verdicts.md +1 -1
- package/package.json +3 -2
- package/dist/benchmark-command-DGokT8zd.js.map +0 -1
- package/dist/emitter-Cs0egaFd.d.ts.map +0 -1
- package/dist/emitter-DeQHiDMm.js +0 -257
- package/dist/emitter-DeQHiDMm.js.map +0 -1
- package/dist/engine-DNlKhIm0.d.ts.map +0 -1
- package/dist/experiment-tracker-BKEumQug.js +0 -280
- package/dist/experiment-tracker-BKEumQug.js.map +0 -1
- package/dist/experiment-tracker-URRGP9ek.d.ts +0 -237
- package/dist/experiment-tracker-URRGP9ek.d.ts.map +0 -1
- package/dist/integrity-Cy9WHAtb.js.map +0 -1
- package/dist/integrity-rGOfSUle.d.ts.map +0 -1
- package/dist/llm-judge-BVso5x-u.js.map +0 -1
- package/dist/raw-provider-sink-BQd7mzyT.js.map +0 -1
- package/dist/redact-7Aq1ukl-.js +0 -107
- package/dist/redact-7Aq1ukl-.js.map +0 -1
- package/dist/schema-CR5cpjQ3.d.ts.map +0 -1
- package/dist/store-tool-spans-B3bX9yqn.d.ts.map +0 -1
- package/dist/tool-groups-veQYPiTy.d.ts.map +0 -1
- package/dist/tool-waste-nbHPAxei.js.map +0 -1
- package/dist/types-CBbLtr2J.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-waste-
|
|
1
|
+
{"version":3,"file":"tool-waste-CP5DO383.d.ts","names":[],"sources":["../src/baseline.ts","../src/pipelines/budget-breach.ts","../src/pipelines/judge-agreement.ts","../src/tool-use-metrics.ts","../src/pipelines/tool-waste.ts"],"mappings":";;;;UA2BiB;EACf;EACA;EACA;EACA;;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EACA;;UAGe;EACf,SAAS;;EAET;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;;iBAiHc,IAAI;;;UCjKH;EACf;EACA;EACA;EACA,iBAAiB;EACjB;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV,aAAa;EACb,YAAY;EACZ,WAAW;EACX;EACA;;iBAGoB,iBACpB,OAAO,YACP;EAAW;EAAqB;IAC/B,QAAQ;;;UCnBM;EACf;EACA;EACA;;EAEA;EACA;EACA;;UAGe;EACf,OAAO;EACP;EACA;;iBAGoB,mBAAmB,OAAO,aAAa,QAAQ;;;UCjBpD;EACf;EACA;;EAEA;EACA,QAAQ,eAAe;EACvB;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;;EAEf,kBAAkB;;iBAGE,sBACpB,OAAO,YACP,eACA,UAAS,iBACR,QAAQ;;;UC5BM;EACf;EACA;EACA;EACA;;UAGe;EACf,OAAO;EACP;;UAGe;EACf;EACA,eAAe,MAAM,UAAU;IAAS,KAAK,QAAQ,kBAAkB;;;iBAGnD,cACpB,OAAO,YACP,UAAS,mBACR,QAAQ"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { r as CaptureIntegrityError } from "../errors-DEE6u6ot.js";
|
|
2
2
|
import { b as CustomTokenPricing, c as CostLedgerHandle, p as CostProvenance } from "../cost-ledger-DbQdN3nO.js";
|
|
3
|
-
import { u as RunTokenUsage } from "../run-record-
|
|
3
|
+
import { u as RunTokenUsage } from "../run-record-xEULpgML.js";
|
|
4
4
|
import { t as DefaultVerdict } from "../verdict-E4eRNf7-.js";
|
|
5
|
-
import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-
|
|
5
|
+
import { a as TraceAnalystLimits, n as TraceAnalysisEngine } from "../engine-Gw2n1irt.js";
|
|
6
6
|
import { t as PrimeBridgeTransport } from "../prime-bridge-transport-6feEglLf.js";
|
|
7
7
|
import { a as RecordedTrajectoryStep, h as isRecordedTimeout } from "../steps-Bg5-KVx1.js";
|
|
8
8
|
//#region src/trace-repair/mini-swe-scaffold.d.ts
|
package/dist/traces.d.ts
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { c as ValidationError, o as LimitExceededError, r as CaptureIntegrityError, s as NotFoundError, t as AgentEvalError } from "./errors-DEE6u6ot.js";
|
|
2
|
-
import { C as
|
|
3
|
-
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-
|
|
4
|
-
import { A as DatasetOverview, B as TraceAnalystSpanKind, C as ReadSpanSourceInput, D as TraceAnalysisStore, E as TRACE_ANALYSIS_LIMITS, F as SpanMatchRecord, G as ViewTraceResult, H as TraceAnalystTraceSummary, I as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, L as TraceAnalystByteBudgets, M as QueryTracesPage, N as SearchSpanResult, O as TraceAnalysisStoreContext, P as SearchTraceResult, R as TraceAnalystFilters, S as BoundedTraceAnalysisStoreOptions, T as SpanSourceReader, U as ViewSpansResult, V as TraceAnalystSpanStatus, W as ViewTraceOversized, j as ErrorCluster, k as DEFAULT_TRACE_ANALYST_BUDGETS, w as ReadSpanSourceResult, z as TraceAnalystSpan } from "./types-
|
|
5
|
-
import {
|
|
6
|
-
import { $ as NoopRawProviderSink, Q as InMemoryRawProviderSinkOptions, X as FileSystemRawProviderSinkOptions, Y as FileSystemRawProviderSink, Z as InMemoryRawProviderSink, at as defaultProviderRedactor, et as ProviderRedactor, it as RawProviderSinkFilter, nt as RawProviderEvent, ot as providerFromBaseUrl, rt as RawProviderSink, tt as RawProviderDirection } from "./types-
|
|
7
|
-
import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, r as FileSystemTraceStoreOptions, s as TraceStore, t as EventFilter } from "./store-
|
|
8
|
-
import { _ as
|
|
9
|
-
import { a as
|
|
2
|
+
import { C as ToolSpan, D as isToolSpan, E as isLlmSpan, S as TRACE_SCHEMA_VERSION, T as isJudgeSpan, _ as SandboxSpan, a as FAILURE_CLASSES, b as SpanKind, c as JudgeSpan, d as RetrievalSpan, f as Run, g as RunStatus, h as RunOutcome, i as EventKind, l as LlmSpan, m as RunLayer, n as BudgetLedgerEntry, o as FailureClass, p as RunCaptureReport, r as BudgetSpec, s as GenericSpan, t as Artifact, u as Message, v as Span, w as TraceEvent, x as SpanStatus, y as SpanBase } from "./schema-2g7trUwH.js";
|
|
3
|
+
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-xEULpgML.js";
|
|
4
|
+
import { A as DatasetOverview, B as TraceAnalystSpanKind, C as ReadSpanSourceInput, D as TraceAnalysisStore, E as TRACE_ANALYSIS_LIMITS, F as SpanMatchRecord, G as ViewTraceResult, H as TraceAnalystTraceSummary, I as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, L as TraceAnalystByteBudgets, M as QueryTracesPage, N as SearchSpanResult, O as TraceAnalysisStoreContext, P as SearchTraceResult, R as TraceAnalystFilters, S as BoundedTraceAnalysisStoreOptions, T as SpanSourceReader, U as ViewSpansResult, V as TraceAnalystSpanStatus, W as ViewTraceOversized, j as ErrorCluster, k as DEFAULT_TRACE_ANALYST_BUDGETS, w as ReadSpanSourceResult, z as TraceAnalystSpan } from "./types-DDp2Mb8Y.js";
|
|
5
|
+
import { A as AnalyzeTracesInput, B as CaptureFetchContext, C as inferDomainKeywords, D as planTraceInsightQuestions, E as isTraceInsightEvidence, F as OTEL_AGENT_EVAL_SCOPE, G as SseUsageMode, H as captureFetchToRawSink, I as OtlpExport, J as extractUsageFromSse, K as extractUsage, L as OtlpResourceSpans, M as AnalyzeTracesResult, N as analyzeTraces, O as scoreTraceInsightReadiness, P as OtlpFlatLine, R as OtlpSpan, S as domainEvidencePattern, T as isTraceInsightAttribution, U as ExtractUsageFromSseOptions, V as CaptureFetchOptions, W as ExtractedUsage, Y as createBoundedTraceAnalysisStore, _ as TraceInsightTask, a as ToolSpansToTraceAnalysisStoreOptions, b as defaultTraceInsightPanel, c as TraceInsightContext, d as TraceInsightPanelRole, f as TraceInsightPromptInput, g as TraceInsightSuite, h as TraceInsightReadiness, i as OtlpFileTraceStoreOptions, j as AnalyzeTracesOptions, k as tokenizeDomainWords, l as TraceInsightEvidence, m as TraceInsightQuestion, n as toolSpansToTraceAnalysisStore, o as otlpTextToTraceAnalysisStore, p as TraceInsightQualityGate, q as extractUsageFromResponse, r as OtlpFileTraceStore, s as TraceInsightAttribution, t as ToolTraceMissingError, u as TraceInsightFinding, v as buildTraceInsightContext, w as isTraceInsightArtifact, x as describeTraceInsightScope, y as buildTraceInsightPrompt, z as exportRunAsOtlp } from "./store-tool-spans-C2lLwDzg.js";
|
|
6
|
+
import { $ as NoopRawProviderSink, Q as InMemoryRawProviderSinkOptions, X as FileSystemRawProviderSinkOptions, Y as FileSystemRawProviderSink, Z as InMemoryRawProviderSink, at as defaultProviderRedactor, et as ProviderRedactor, it as RawProviderSinkFilter, nt as RawProviderEvent, ot as providerFromBaseUrl, rt as RawProviderSink, tt as RawProviderDirection } from "./types-C0bWoSKC.js";
|
|
7
|
+
import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, r as FileSystemTraceStoreOptions, s as TraceStore, t as EventFilter } from "./store-C4avFjIi.js";
|
|
8
|
+
import { _ as buildTraceAnalysisToolDescriptors, g as UNTRUSTED_TRACE_TEXT, h as TraceAnalysisToolDescriptor, m as TRACE_ANALYST_TOOL_NAMESPACE, p as BuildTraceAnalysisToolsOptions, v as traceAnalystFunctionGroup } from "./engine-Gw2n1irt.js";
|
|
9
|
+
import { a as SpanHandle, i as RunCompleteHookContext, n as CaptureWrite, o as TraceEmitter, r as RunCompleteHook, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-HcCmFQl_.js";
|
|
10
|
+
import { S as shareAllowed, _ as emptyRedactionReport, a as RedactionFinding, b as redactForShare, c as SafetyCategory, d as ShareSafetyVerdict, f as assessShareSafety, g as detectPersonalData, h as detectCredential, i as RedactOptions, l as ShareSafetyFinding, m as combineVerdicts, n as REDACTION_PROFILES, o as RedactionProfile, p as classifyKey, r as REDACTION_VERSION, s as RedactionReport, t as AssessOptions, u as ShareSafetyStatus, v as normalizeKey, x as redactText, y as redact } from "./redact-DPoH1-OW.js";
|
|
10
11
|
import { C as TOOL_LATENCY_MS, D as asNumber, E as applyLlmSpanOtlpAttributes, O as contextInputTokens, S as TOOL_ARGS_CAPTURED, T as TOOL_NAME_ATTR_KEYS, _ as LlmSpanOtlpInput, a as LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, b as RUN_COST_ATTR_KEYS, c as LLM_COST_USD, d as LLM_MODEL_ATTR_KEYS, f as LLM_MODEL_NAME, g as LLM_REASONING_TOKEN_ATTR_KEYS, h as LLM_REASONING_TOKENS, i as LLM_CACHE_WRITE_TOKENS, k as firstNumberAttr, l as LLM_INPUT_TOKENS, m as LLM_OUTPUT_TOKEN_ATTR_KEYS, n as LLM_CACHED_TOKENS, o as LLM_CONTEXT_TOKENS, p as LLM_OUTPUT_TOKENS, r as LLM_CACHED_TOKEN_ATTR_KEYS, s as LLM_COST_ATTR_KEYS, t as INPUT_VALUE, u as LLM_INPUT_TOKEN_ATTR_KEYS, v as OPENINFERENCE_SPAN_KIND, w as TOOL_NAME, x as SPAN_KIND_ATTR_KEYS, y as OUTPUT_VALUE } from "./attribute-vocabulary-DLJ6303h.js";
|
|
11
|
-
import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, s as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-
|
|
12
|
-
import { a as groupBy, c as judgeSpans, d as runMetricExtractor, f as runsForScenario, i as argHash, l as llmSpans, n as RunMetric, o as hasCapturedToolArgs, p as toolSpans, r as aggregateLlm, s as isRunMetric, t as RUN_METRICS, u as runFailureClass } from "./query-
|
|
12
|
+
import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, s as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-CaeNeusS.js";
|
|
13
|
+
import { a as groupBy, c as judgeSpans, d as runMetricExtractor, f as runsForScenario, i as argHash, l as llmSpans, n as RunMetric, o as hasCapturedToolArgs, p as toolSpans, r as aggregateLlm, s as isRunMetric, t as RUN_METRICS, u as runFailureClass } from "./query-DBUQhehH.js";
|
|
13
14
|
//#region src/trace-analyst/errors.d.ts
|
|
14
15
|
/** Invalid trace-tool arguments, including malformed filters and regexes. */
|
|
15
16
|
declare class TraceAnalysisValidationError extends ValidationError {
|
|
@@ -395,5 +396,5 @@ declare function otlpRowsToTraceRunRecords(rows: Iterable<object>, opts: OtlpToR
|
|
|
395
396
|
declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "Answer the question by inspecting the OTLP trace dataset with the available tools.\n\n1. Call getDatasetOverview first. Use its real trace ids and dataset size to plan the investigation.\n2. Narrow with queryTraces and countTraces before scanning large payloads.\n3. For a small trace, use viewTrace. For a large trace, use searchTrace and then viewSpans or searchSpan.\n4. Never invent a trace id, span id, tool result, error, frequency, or final outcome.\n5. When a search reports has_more, refine the query before drawing a conclusion.\n6. Use llm_query only over evidence already loaded. A recursive query cannot inspect traces itself.\n7. Cite exact evidence URIs returned by the tools. Include a short exact excerpt when it supports the claim.\n8. Return no finding when the available evidence cannot support one.\n\nThe prose answer must directly answer the question and state important uncertainty.\nThe findings array contains only actionable or decision-relevant claims supported by inspected evidence.";
|
|
396
397
|
declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-research-v1-2026-07-30";
|
|
397
398
|
//#endregion
|
|
398
|
-
export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, Artifact, type BoundedTraceAnalysisStoreOptions, BudgetLedgerEntry, BudgetSpec, type BuildTraceAnalysisToolsOptions, CaptureFetchContext, CaptureFetchOptions,
|
|
399
|
+
export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, Artifact, AssessOptions, type BoundedTraceAnalysisStoreOptions, BudgetLedgerEntry, BudgetSpec, type BuildTraceAnalysisToolsOptions, CaptureErrorHandler, CaptureFetchContext, CaptureFetchOptions, CaptureWrite, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, type ErrorCluster, EventFilter, EventKind, ExportableSpan, ExtractUsageFromSseOptions, ExtractedUsage, FAILURE_CLASSES, FailureClass, FileSystemRawProviderSink, FileSystemRawProviderSinkOptions, FileSystemTraceStore, FileSystemTraceStoreOptions, type FlattenOtlpOptions, GenericSpan, INPUT_VALUE, InMemoryRawProviderSink, InMemoryRawProviderSinkOptions, InMemoryTraceStore, JudgeSpan, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, LlmSpan, LlmSpanOtlpInput, Message, NoopRawProviderSink, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtelExportConfig, OtelExporter, OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, OtlpResourceSpans, OtlpSpan, OtlpSpanRole, OtlpSpanRoleInput, type OtlpToRunRecordsOptions, type OtlpTraceRunRecord, type ProjectedOtlpSpan, ProviderRedactor, type QueryTracesPage, REDACTION_PROFILES, REDACTION_VERSION, RUN_COST_ATTR_KEYS, RUN_METRICS, RawProviderDirection, RawProviderEvent, RawProviderSink, RawProviderSinkFilter, type ReadSpanSourceInput, type ReadSpanSourceResult, RedactOptions, RedactionFinding, RedactionProfile, RedactionReport, RetrievalSpan, Run, RunCaptureReport, RunCompleteHook, RunCompleteHookContext, RunFilter, RunIntegrityError, RunIntegrityExpectations, RunIntegrityIssue, RunIntegrityIssueCode, RunIntegrityReport, RunLayer, RunMetric, RunOutcome, RunStatus, SPAN_KIND_ATTR_KEYS, SafetyCategory, SandboxSpan, type SearchSpanResult, type SearchTraceResult, ShareSafetyFinding, ShareSafetyStatus, ShareSafetyVerdict, Span, SpanBase, SpanFilter, SpanHandle, SpanKind, type SpanMatchRecord, SpanNotFoundError, type SpanSourceReader, SpanStatus, SseUsageMode, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, ToolSpan, ToolSpanOtlpInput, type ToolSpansToTraceAnalysisStoreOptions, ToolTraceMissingError, type TraceAggregate, TraceAnalysisLimitError, type TraceAnalysisStore, type TraceAnalysisStoreContext, TraceAnalysisStoreContractError, type TraceAnalysisToolDescriptor, TraceAnalysisValidationError, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceEmitter, TraceEmitterOptions, TraceEvent, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, type TraceInsightAttribution, type TraceInsightContext, type TraceInsightEvidence, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, TraceStoreSource, TraceStoreToOtlpOptions, TracesToOtlpResult, UNTRUSTED_TRACE_TEXT, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, aggregateLlm, analyzeTraces, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertRunCaptured, assessShareSafety, buildTraceAnalysisToolDescriptors, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, classifyKey, classifyOtlpSpanRole, combineVerdicts, contextInputTokens, convertTraceStoresToOtlp, createBoundedTraceAnalysisStore, createOtelExporter, createOtelTracingStore, defaultProviderRedactor, defaultTraceInsightPanel, describeTraceInsightScope, detectCredential, detectPersonalData, domainEvidencePattern, emptyRedactionReport, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, groupBy, hasCapturedToolArgs, inferDomainKeywords, isJudgeSpan, isLlmSpan, isOtlpModelCall, isRunMetric, isToolSpan, isTraceInsightArtifact, isTraceInsightAttribution, isTraceInsightEvidence, judgeSpans, llmSpans, normalizeKey, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, providerFromBaseUrl, redact, redactForShare, redactText, runFailureClass, runMetricExtractor, runsForScenario, scoreTraceInsightReadiness, shareAllowed, stringField, throwIfRunIncomplete, tokenizeDomainWords, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind };
|
|
399
400
|
//# sourceMappingURL=traces.d.ts.map
|
package/dist/traces.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"traces.d.ts","names":[],"sources":["../src/trace-analyst/errors.ts","../src/trace/otlp-attributes.ts","../src/trace/otel-export.ts","../src/trace/otel-bridge.ts","../src/trace/store-to-otlp.ts","../src/trace-analyst/hook.ts","../src/trace-analyst/otlp-flatten.ts","../src/trace-analyst/otlp-span.ts","../src/trace-analyst/otlp-to-run-records.ts","../src/trace-analyst/prompts.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"traces.d.ts","names":[],"sources":["../src/trace-analyst/errors.ts","../src/trace/otlp-attributes.ts","../src/trace/otel-export.ts","../src/trace/otel-bridge.ts","../src/trace/store-to-otlp.ts","../src/trace-analyst/hook.ts","../src/trace-analyst/otlp-flatten.ts","../src/trace-analyst/otlp-span.ts","../src/trace-analyst/otlp-to-run-records.ts","../src/trace-analyst/prompts.ts"],"mappings":";;;;;;;;;;;;;;;cASa,qCAAqC;WAC/B;;;cAIN,gCAAgC;WAC1B;WACR;WACA;WACA;EAET,YAAY,mBAAmB,gBAAgB,eAAe;;;cASnD,wCAAwC;WAClC;WACR;EAET,YAAY,mBAAmB,iBAAiB;IAAY;;;cAMjD,8BAA8B;WACxB;WACR;EAET,YAAY;;cAMD,+BAA+B;WACjC;WACA;WACA;EAET,YAAY,cAAc,oBAAoB;;cAcnC,gCAAgC;WAClC;WACA;WACA;EAET,YAAY,cAAc,qBAAqB,qBAAqB;;cAQzD,2BAA2B;WACrB;WACR;EAET,YAAY;;cAMD,0BAA0B;WACpB;WACR;WACA;EAET,YAAY,kBAAkB;;;;KCxEpB,oBAAoB,KAC9B;KAIU;UAUK;EACf;EACA,YAAY;EACZ;;;;;;;iBAkBc,qBAAqB,OAAO,oBAAoB;iBA2BhD,gBAAgB,OAAO;iBAkBvB,4BACd,YAAY,yBACZ,MAAM;iBAMQ,iCAAiC;;;UC7FhC;;EAEf;;EAEA,UAAU;;EAEV;;EAEA;;EAEA,qBAAqB;;EAErB;;UAGe;;EAEf,WAAW,MAAM;;EAEjB,SAAS;;EAET,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,aAAa;;;;;;iBAOC,mBAAmB,SAAS,mBAAmB;;;;;;;;;;;iBC9C/C,uBACd,OAAO,YACP,UAAU,cACV,kBACC;;;UCUc;;EAEf;;EAEA;;EAEA;;;EAGA;;;;;;;;;;;;UAae;;EAEf;;EAEA;;EAEA;;;;UAKe;;;EAGf;;;EAGA,sBAAsB,KAAK,QAAQ;;EAEnC,iBAAiB,KAAK,QAAQ;;;;;;;;;;;iBAYhB,yBACd,iBAAiB,4BAA4B,oBAC7C,iBACA,OAAM,0BACL;;;UCxEc;;;;;;EAMf,SAAS,KAAK;IAAoC,SAAS;;;;;;;EAM3D;;;;;;EAMA,QAAQ,QAAQ,qBAAqB,KAAK,2BAA2B;;;;;EAKrE,aAAa,KAAK;;;;;;EAMlB,UAAU,QAAQ,qBAAqB,KAAK;;iBAM9B,0BAA0B,MAAM,0BAA0B;;;UCnCzD;;;;EAIf;;EAEA,UAAU,QAAQ;;iBA+DJ,0BACd,YAAY,YACZ,OAAM,qBACL;;;;;;;;;UC1Dc;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA,YAAY;;;;;;iBAOE,oBAAoB,KAAK,0BAA0B;;;;;;iBA8GnD,sBAAsB,KAAK,0BAA0B;iBAoBrD,YAAY,KAAK,yBAAyB;iBAK1C,SAAS;;iBAKT,gBACd,OAAO,yBACP;;;UCnIe;;EAEf;;;EAGA;;;EAGA,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;;;;;;EAOA;;;;;;EAMA;;;;;;;;;EASA,wBAAwB;;;;;;;;;EASxB,iBAAiB,eAAe,MAAM;;;;;EAKtC,yBAAyB,kBAAkB;;;;;;UAO5B;EACf,QAAQ;;EAER;;EAEA;;;UAIe;EACf;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;;EAEZ;EACA;EACA;EACA;EACA;;EAEA,iBAAiB;;;;;;;iBAeH,iBAAiB,mBAAmB,MAAM,0BAA0B;;;;;;iBASpE,qBACd,MAAM,kBACN,MAAM,0BACL;;iBAKa,sBACd,mBACA,MAAM,0BACL;;iBAQa,0BACd,MAAM,kBACN,MAAM,0BACL;;;;cC/MU;cAcA"}
|
package/dist/traces.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { a as isToolSpan, i as isLlmSpan, n as TRACE_SCHEMA_VERSION, r as isJudgeSpan, t as FAILURE_CLASSES } from "./schema-CSf6qWgZ.js";
|
|
2
2
|
import { a as hasCapturedToolArgs, c as llmSpans, d as runsForScenario, f as toolSpans, i as groupBy, l as runFailureClass, n as aggregateLlm, o as isRunMetric, r as argHash, s as judgeSpans, t as RUN_METRICS, u as runMetricExtractor } from "./query-DlV0pdx4.js";
|
|
3
3
|
import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-tg_5H2IK.js";
|
|
4
|
-
import { i as
|
|
5
|
-
import { t as TraceEmitter } from "./emitter-
|
|
6
|
-
import { A as
|
|
4
|
+
import { a as combineVerdicts, c as emptyRedactionReport, d as redactForShare, f as redactText, i as classifyKey, l as normalizeKey, n as REDACTION_VERSION, o as detectCredential, p as shareAllowed, r as assessShareSafety, s as detectPersonalData, t as REDACTION_PROFILES, u as redact } from "./redact-DxwlUAvh.js";
|
|
5
|
+
import { t as TraceEmitter } from "./emitter-DbMXUIVP.js";
|
|
6
|
+
import { A as spanEpochMillis, C as TraceFileTooLargeError, D as extractOtlpAttributes, E as compareSpanTime, O as firstStringAttr, S as TraceFileMissingError, T as asString, _ as SpanNotFoundError, a as TRACE_ANALYST_TOOL_NAMESPACE, b as TraceAnalysisValidationError, c as traceAnalystFunctionGroup, g as TRACE_ANALYSIS_LIMITS, h as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, j as stringField, k as projectOtlpFlatLine, m as DEFAULT_TRACE_ANALYST_BUDGETS, o as UNTRUSTED_TRACE_TEXT, s as buildTraceAnalysisToolDescriptors, u as createBoundedTraceAnalysisStore, v as TraceAnalysisLimitError, w as TraceNotFoundError, x as TraceFileMalformedError, y as TraceAnalysisStoreContractError } from "./kind-factory-g1ROw67A.js";
|
|
7
7
|
import { INPUT_VALUE, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OUTPUT_VALUE, RUN_COST_ATTR_KEYS, SPAN_KIND_ATTR_KEYS, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, applyLlmSpanOtlpAttributes, asNumber, contextInputTokens, firstNumberAttr } from "./trace-attributes.js";
|
|
8
8
|
import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-BrQ8mCLX.js";
|
|
9
|
-
import { a as providerFromBaseUrl, i as defaultProviderRedactor, n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-
|
|
10
|
-
import { C as spanIdForWire, D as captureFetchToRawSink, E as toOtlpAttributes, S as exportRunAsOtlp, T as msToUnixNano, _ as tokenizeDomainWords, a as spanStatusToOtlp, b as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, c as defaultTraceInsightPanel, d as inferDomainKeywords, f as isTraceInsightArtifact, g as scoreTraceInsightReadiness, h as planTraceInsightQuestions, i as epochMillisToIso, l as describeTraceInsightScope, m as isTraceInsightEvidence, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, p as isTraceInsightAttribution, r as createOtlpFlatLine, s as buildTraceInsightPrompt, t as ToolTraceMissingError, u as domainEvidencePattern, v as analyzeTraces, w as traceIdForWire, x as OTEL_AGENT_EVAL_SCOPE, y as TRACE_ANALYST_ACTOR_DESCRIPTION } from "./store-tool-spans-
|
|
11
|
-
import { n as assertRunCaptured, r as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-
|
|
9
|
+
import { a as providerFromBaseUrl, i as defaultProviderRedactor, n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-C96CcqeG.js";
|
|
10
|
+
import { C as spanIdForWire, D as captureFetchToRawSink, E as toOtlpAttributes, S as exportRunAsOtlp, T as msToUnixNano, _ as tokenizeDomainWords, a as spanStatusToOtlp, b as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, c as defaultTraceInsightPanel, d as inferDomainKeywords, f as isTraceInsightArtifact, g as scoreTraceInsightReadiness, h as planTraceInsightQuestions, i as epochMillisToIso, l as describeTraceInsightScope, m as isTraceInsightEvidence, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, p as isTraceInsightAttribution, r as createOtlpFlatLine, s as buildTraceInsightPrompt, t as ToolTraceMissingError, u as domainEvidencePattern, v as analyzeTraces, w as traceIdForWire, x as OTEL_AGENT_EVAL_SCOPE, y as TRACE_ANALYST_ACTOR_DESCRIPTION } from "./store-tool-spans-ClPxcvh2.js";
|
|
11
|
+
import { n as assertRunCaptured, r as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-BFP4kjGj.js";
|
|
12
12
|
import { i as traceSpanKindToOpenInferenceKind, n as classifyOtlpSpanRole, r as isOtlpModelCall, t as applyToolSpanOtlpAttributes } from "./otlp-attributes-BpC_blLF.js";
|
|
13
13
|
import { n as InMemoryTraceStore, t as FileSystemTraceStore } from "./store-DNe_Uv1Q.js";
|
|
14
|
-
import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-
|
|
14
|
+
import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-DPgzSQKS.js";
|
|
15
15
|
import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "./task-failure-attributes-B34TiMBe.js";
|
|
16
16
|
import { readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
17
17
|
import { join } from "node:path";
|
|
@@ -990,6 +990,6 @@ function ensureSnapshot(model, fallbackModel) {
|
|
|
990
990
|
return `${model}${fallbackModel.includes("@") ? fallbackModel.slice(fallbackModel.indexOf("@")) : "@otlp"}`;
|
|
991
991
|
}
|
|
992
992
|
//#endregion
|
|
993
|
-
export {
|
|
993
|
+
export { DEFAULT_TRACE_ANALYST_BUDGETS, FAILURE_CLASSES, FileSystemRawProviderSink, FileSystemTraceStore, INPUT_VALUE, InMemoryRawProviderSink, InMemoryTraceStore, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, NoopRawProviderSink, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpFileTraceStore, REDACTION_PROFILES, REDACTION_VERSION, RUN_COST_ATTR_KEYS, RUN_METRICS, RunIntegrityError, SPAN_KIND_ATTR_KEYS, SpanNotFoundError, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, ToolTraceMissingError, TraceAnalysisLimitError, TraceAnalysisStoreContractError, TraceAnalysisValidationError, TraceEmitter, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, TraceNotFoundError, UNTRUSTED_TRACE_TEXT, aggregateLlm, analyzeTraces, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertRunCaptured, assessShareSafety, buildTraceAnalysisToolDescriptors, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, classifyKey, classifyOtlpSpanRole, combineVerdicts, contextInputTokens, convertTraceStoresToOtlp, createBoundedTraceAnalysisStore, createOtelExporter, createOtelTracingStore, defaultProviderRedactor, defaultTraceInsightPanel, describeTraceInsightScope, detectCredential, detectPersonalData, domainEvidencePattern, emptyRedactionReport, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, groupBy, hasCapturedToolArgs, inferDomainKeywords, isJudgeSpan, isLlmSpan, isOtlpModelCall, isRunMetric, isToolSpan, isTraceInsightArtifact, isTraceInsightAttribution, isTraceInsightEvidence, judgeSpans, llmSpans, normalizeKey, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, providerFromBaseUrl, redact, redactForShare, redactText, runFailureClass, runMetricExtractor, runsForScenario, scoreTraceInsightReadiness, shareAllowed, stringField, throwIfRunIncomplete, tokenizeDomainWords, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind };
|
|
994
994
|
|
|
995
995
|
//# sourceMappingURL=traces.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { s as TraceStore } from "./store-
|
|
1
|
+
import { v as Span, w as TraceEvent } from "./schema-2g7trUwH.js";
|
|
2
|
+
import { s as TraceStore } from "./store-C4avFjIi.js";
|
|
3
3
|
//#region src/trajectory.d.ts
|
|
4
4
|
interface TrajectoryStep {
|
|
5
5
|
index: number;
|
|
@@ -21,4 +21,4 @@ interface Trajectory {
|
|
|
21
21
|
declare function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory>;
|
|
22
22
|
//#endregion
|
|
23
23
|
export { TrajectoryStep as n, buildTrajectory as r, Trajectory as t };
|
|
24
|
-
//# sourceMappingURL=trajectory-
|
|
24
|
+
//# sourceMappingURL=trajectory-Y1fmdlw7.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trajectory-
|
|
1
|
+
{"version":3,"file":"trajectory-Y1fmdlw7.d.ts","names":[],"sources":["../src/trajectory.ts"],"mappings":";;;UAaiB;EACf;EACA,MAAM;;EAEN;;EAEA,QAAQ;;UAGO;EACf;EACA,OAAO;EACP;EACA;EACA;EACA;EACA;;iBAGoB,gBAAgB,OAAO,YAAY,gBAAgB,QAAQ"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { s as TraceStore } from "../store-
|
|
1
|
+
import { s as TraceStore } from "../store-C4avFjIi.js";
|
|
2
2
|
import { t as DefaultVerdict } from "../verdict-E4eRNf7-.js";
|
|
3
|
-
import {
|
|
4
|
-
import { i as CounterfactualRunner, t as CounterfactualContext } from "../counterfactual-
|
|
3
|
+
import { o as TraceEmitter } from "../emitter-HcCmFQl_.js";
|
|
4
|
+
import { i as CounterfactualRunner, t as CounterfactualContext } from "../counterfactual-CPaWN99-.js";
|
|
5
5
|
import { _ as isSubmitOnlyAction, a as RecordedTrajectoryStep, b as unreadableExitCount, c as TIMEOUT_OBSERVATION_MARKER, d as decodeRecordedTurns, f as deriveFailureSignature, g as isSubmitAction, h as isRecordedTimeout, i as RecordedObservationKind, l as assertReplayableTrajectory, m as isElidedField, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as RecordedTrajectoryTurn, p as finalRecordedOutcome, r as RecordedFinalOutcome, s as SUBMIT_ACTION_SIGNATURE, t as DecodedTrajectory, u as classifyObservation, v as parseObservationOutput, y as parseRecordedReturncode } from "../steps-Bg5-KVx1.js";
|
|
6
6
|
//#region src/trajectory-replay/corpus.d.ts
|
|
7
7
|
interface CorpusSpec {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as mulberry32 } from "../random-Dn5fPWkt.js";
|
|
2
2
|
import { t as certificationEvidenceDigest } from "../verdict-DhPoXApk.js";
|
|
3
|
-
import { t as TraceEmitter } from "../emitter-
|
|
4
|
-
import { t as runCounterfactual } from "../counterfactual-
|
|
3
|
+
import { t as TraceEmitter } from "../emitter-DbMXUIVP.js";
|
|
4
|
+
import { t as runCounterfactual } from "../counterfactual-DtMWvn7D.js";
|
|
5
5
|
import { n as InMemoryTraceStore } from "../store-DNe_Uv1Q.js";
|
|
6
6
|
import { t as packageVersion } from "../package-version-D7lQHt_-.js";
|
|
7
7
|
import { a as assertReplayableTrajectory, c as deriveFailureSignature, d as isRecordedTimeout, f as isSubmitAction, g as unreadableExitCount, h as parseRecordedReturncode, i as TIMEOUT_OBSERVATION_MARKER, l as finalRecordedOutcome, m as parseObservationOutput, n as FORMAT_ERROR_OBSERVATION_PREFIX, o as classifyObservation, p as isSubmitOnlyAction, r as SUBMIT_ACTION_SIGNATURE, s as decodeRecordedTurns, t as wrapActionForExec, u as isElidedField } from "../exec-ChJ6d5if.js";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { s as RunSplitTag } from "./run-record-
|
|
2
|
-
import { R as Scenario, d as DispatchContext } from "./types-
|
|
1
|
+
import { s as RunSplitTag } from "./run-record-xEULpgML.js";
|
|
2
|
+
import { R as Scenario, d as DispatchContext } from "./types-B1vDZ694.js";
|
|
3
3
|
//#region src/benchmarks/types.d.ts
|
|
4
4
|
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
5
5
|
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
@@ -99,4 +99,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
|
|
|
99
99
|
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
|
|
100
100
|
//#endregion
|
|
101
101
|
export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
|
|
102
|
-
//# sourceMappingURL=types
|
|
102
|
+
//# sourceMappingURL=types-5ak2PqnJ.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types
|
|
1
|
+
{"version":3,"file":"types-5ak2PqnJ.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { S as PaidCallResult, T as RunPaidCallInput, a as CostChannel, c as CostLedgerHandle, f as CostLedgerSummary, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { u as RunTokenUsage } from "./run-record-
|
|
3
|
-
import { _ as ProposalFinding } from "./types-
|
|
4
|
-
import { M as LlmCallMetadata } from "./types-
|
|
2
|
+
import { u as RunTokenUsage } from "./run-record-xEULpgML.js";
|
|
3
|
+
import { _ as ProposalFinding } from "./types-DDp2Mb8Y.js";
|
|
4
|
+
import { M as LlmCallMetadata } from "./types-C0bWoSKC.js";
|
|
5
5
|
import { t as SeriesDistribution } from "./descriptive-B2iPaT9J.js";
|
|
6
6
|
//#region src/campaign/types.d.ts
|
|
7
7
|
/** Stable identifier + kind tag for any scenario. Consumers
|
|
@@ -670,4 +670,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
|
|
|
670
670
|
}
|
|
671
671
|
//#endregion
|
|
672
672
|
export { LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, GateDecision as _, CampaignResult as a, GenerationRecord as b, CampaignTraceWriter as c, DispatchContext as d, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, GateCheckStatus as m, CampaignArtifactWriter as n, CampaignScenarioIdentity as o, Gate as p, CampaignCellResult as r, CampaignTokenUsage as s, CampaignAggregates as t, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
673
|
-
//# sourceMappingURL=types-
|
|
673
|
+
//# sourceMappingURL=types-B1vDZ694.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-B1vDZ694.d.ts","names":[],"sources":["../src/campaign/types.ts"],"mappings":";;;;;;;;UAiCiB;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;;;;EAOV;;;;EAIA,eAAe,eAAe;IAAQ;IAAe;;;;;EAIrD;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;;;EAMA,cAAc,SAAS;;;;iBAKT,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;;;WAGjC,gBAAgB;WAChB,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB,gBAAgB;;;;WAIhB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;UAKX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;;;EAIA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;;;EAMA,cAAc;;UAGC;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;;;EAIA,cAAc,SAAS;;UAGR;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
|
|
@@ -175,12 +175,12 @@ declare function assertCrossFamilyServed(pairs: ReadonlyArray<{
|
|
|
175
175
|
* request, response, and error is recorded — including retries, with the
|
|
176
176
|
* attempt index attached so a flaky call's full event chain is recoverable.
|
|
177
177
|
*
|
|
178
|
-
* Redaction is enforced at sink time. The default redactor
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
*
|
|
178
|
+
* Redaction is enforced at sink time. The default redactor runs the redaction
|
|
179
|
+
* core (`./redact`, default profile) over headers, bodies and the error
|
|
180
|
+
* message: credential headers and fields are replaced by name, credential and
|
|
181
|
+
* personal-data values by shape. Override via the sink constructor or the
|
|
182
|
+
* per-call `redactor`. The `redactedFields` array on the persisted event lets
|
|
183
|
+
* a reviewer see what was replaced without exposing the values.
|
|
184
184
|
*/
|
|
185
185
|
type RawProviderDirection = 'request' | 'response' | 'error';
|
|
186
186
|
interface RawProviderEvent {
|
|
@@ -214,7 +214,7 @@ interface RawProviderEvent {
|
|
|
214
214
|
responseBody?: unknown;
|
|
215
215
|
/** Set on `direction: 'error'` events. */
|
|
216
216
|
errorMessage?: string;
|
|
217
|
-
/**
|
|
217
|
+
/** JSON Pointers of the fields the redactor replaced ('/requestHeaders/Authorization', '/requestBody/apiKey', …). */
|
|
218
218
|
redactedFields: string[];
|
|
219
219
|
}
|
|
220
220
|
interface RawProviderSinkFilter {
|
|
@@ -232,9 +232,8 @@ interface RawProviderSink {
|
|
|
232
232
|
}
|
|
233
233
|
type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
|
|
234
234
|
/**
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
* `event.redactedFields` so a downstream reviewer can see what was removed.
|
|
235
|
+
* The default redactor: the redaction core over the parts of an event that
|
|
236
|
+
* carry provider traffic. Every replaced path is added to `redactedFields`.
|
|
238
237
|
*/
|
|
239
238
|
declare function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent;
|
|
240
239
|
interface InMemoryRawProviderSinkOptions {
|
|
@@ -863,4 +862,4 @@ interface CheckResult {
|
|
|
863
862
|
}
|
|
864
863
|
//#endregion
|
|
865
864
|
export { NoopRawProviderSink as $, createChatClient as A, LlmToolCall as B, ChatResponse as C, judgeFamily as Ct, MockTransportOpts as D, CustomTransportOpts as E, LlmChargeBounds as F, costReceiptFromLlmError as G, LlmToolDefinition as H, LlmMessage as I, stripFencedJson as J, isTransientLlmError as K, LlmResponseError as L, LlmCallMetadata as M, LlmCallRequest as N, OpenAiCompatibleTransportOpts as O, LlmCallResult as P, InMemoryRawProviderSinkOptions as Q, LlmThinkingMode as R, ChatRequest as S, assertCrossFamily as St, CreateChatClientOpts as T, LlmUsage as U, LlmToolChoice as V, costReceiptFromLlm as W, FileSystemRawProviderSinkOptions as X, FileSystemRawProviderSink as Y, InMemoryRawProviderSink as Z, ScenarioResult as _, checkServedModel as _t, CompletionCriterion as a, defaultProviderRedactor as at, ChatCallOpts as b, CrossFamilyError as bt, JudgeInput as c, AssertServedModelOptions as ct, PersonaConfig as d, ServedModelCheck as dt, ProviderRedactor as et, ProductClientConfig as f, ServedModelPolicy as ft, ScenarioFile as g, assertServedModels as gt, Scenario as h, assertServedModel as ht, CollectedArtifacts as i, RawProviderSinkFilter as it, LlmCallError as j, SandboxSdkTransportOpts as k, JudgeRubric as l, ModelSubstitutionError as lt, RubricDimension as m, assertCrossFamilyServed as mt, ArtifactResult as n, RawProviderEvent as nt, DriverState as o, providerFromBaseUrl as ot, RouteMap as p, ServedModelVerdict as pt, maximumChargeForLlmRequest as q, CheckResult as r, RawProviderSink as rt, JudgeFn as s, AssertCrossFamilyServedOptions as st, ArtifactCheck as t, RawProviderDirection as tt, JudgeScore as u, ServedCrossFamilyError as ut, Turn as v, servedModelAcceptable as vt, ChatTransport as w, ChatClient as x, JudgeFamily as xt, TurnResult as y, AssertCrossFamilyOptions as yt, LlmTokenLogprob as z };
|
|
866
|
-
//# sourceMappingURL=types-
|
|
865
|
+
//# sourceMappingURL=types-C0bWoSKC.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-C0bWoSKC.d.ts","names":[],"sources":["../src/judge-families.ts","../src/integrity/served-model.ts","../src/trace/raw-provider-sink.ts","../src/llm-client.ts","../src/analyst/chat-client.ts","../src/types.ts"],"mappings":";;;;;;;;;;;;;KAWY;;;;;;;iBAsEI,YAAY,kBAAkB;UAc7B;;EAEf;;;;EAIA;;cAGW,yBAAyB;WAGlB,UAAU;WACV;EAHlB,YACE,iBACgB,UAAU,eACV;;;;;;;;;;;;;;;iBAoBJ,kBACd,kBACA,OAAM,2BACL;;;;KCvFS;;;;;;;;;;;UAYK;;EAEf;;EAEA;EACA,iBAAiB;;EAEjB,cAAc;EACd,SAAS;;EAET;;;;;;;;;;;;iBA4Cc,iBACd,mBACA,oCACC;cA4CU,+BAA+B;WAGxB,QAAQ,cAAc;EAFxC,YACE,iBACgB,QAAQ,cAAc;;;;;;;;;KAc9B;UAWK;;;;;;EAMf;;;;;EAKA;;EAEA;;;;;;;iBAQc,sBACd,OAAO,kBACP,OAAM;;;;;;iBA6BQ,kBACd,mBACA,mCACA,OAAM,2BACL;;;;;iBAea,mBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,2BACL;UAec,uCAAuC;;EAEtD;;EAEA;;cAGW,+BAA+B;WAGxB,UAAU;WACV,QAAQ,cAAc;EAHxC,YACE,iBACgB,UAAU,eACV,QAAQ,cAAc;;;;;;;;;;;iBAgB1B,wBACd,OAAO;EAAgB;EAAmB;IAC1C,OAAM,iCACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KCjRS;UAEK;;EAEf;;EAEA;EACA;;;;;;EAMA;EACA;;EAEA;;EAEA;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA,iBAAiB;EACjB;EACA,kBAAkB;EAClB;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;;UAGe;EACf,OAAO,OAAO,mBAAmB;;EAEjC,MAAM,SAAS,wBAAwB,QAAQ;;EAE/C,UAAU;;KAGA,oBAAoB,OAAO,qBAAqB;;;;;iBAM5C,wBAAwB,OAAO,mBAAmB;UAqBjD;EACf,WAAW;;cAGA,mCAAmC;UACtC;UACA;EAER,YAAY,OAAM;EAIZ,OAAO,OAAO,mBAAmB;EAIjC,KAAK,SAAQ,wBAA6B,QAAQ;EAUxD;;cAKW,+BAA+B;EACpC,UAAU;;;;;;;EASV,QAAQ,QAAQ;;UAOP;;EAEf;;EAEA;;EAEA;EACA,WAAW;;cAGA,qCAAqC;UACxC;UACA;UACA;UACA;UACA;UACA;UACA;EAER,YAAY,MAAM;UAOJ;UAON;EAKF,OAAO,OAAO,mBAAmB;EAYjC,KAAK,SAAQ,wBAA6B,QAAQ;;;;;;iBAkC1C,oBAAoB;;;UCpLnB;EACf;;;;;EAKA,kBAEI;IACM;IAAc;;IACd;IAAmB;MAAa;MAAa;;;;EAGvD,YAAY;;EAEZ;;KAGU;;UAGK;EACf;EACA;IACE;IACA;IACA,YAAY;;;;KAKJ;EAIN;EAAkB;IAAY;;;;UAGnB;EACf;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;;EAEV;;EAEA;IAAe;IAAc,QAAQ;;;EAErC,QAAQ;;EAER,aAAa;EACb;EACA;;;;;EAKA;IAAa;;;EAEb,WAAW;;EAEX;;;;;;UAOe;;EAEf;;EAEA;;EAEA,WAAW;;EAEX,qBAAqB;;iBAGP,2BACd,SAAS,KACP,0GAGF,UAAS,kBACR;UAgCc;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;;;UAKe;EACf;EACA;EACA,KAAK;IAAgB;IAAe;;;UAGrB;;EAEf;;EAEA,YAAY;EACZ,OAAO;;;;;EAKP;;;;;;;;;EASA;;;;;;;;;;;;EAYA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;;;;;;;;EAQA,WAAW,cAAc;;EAEzB,KAAK;;KAGK,kBAAkB,KAAK;;;;;;iBAOnB,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA6Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cAMU,qBAAqB;WAGd;WACA;WACA;EAJlB,YACE,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EAF1B,YACE,iBACgB,QAAQ,eACxB;IAAY;;;UAMC,yBAAyB;;EAExC;;EAEA;EACA;;EAEA;IAAe;IAAc;;;EAE7B;;EAEA;;;;;;;EAOA,SAAS;;;;;;;;EAQT;;;;;;EAMA;;EAEA,eAAe;;;;;;;;EAQf,UAAU;;;;;EAKV;;EAEA;IAAiB;IAAgB;;;EAEjC,WAAW;;;;;;;;;;EAUX,8BAA8B;;;;;;;;;;;;;iBAwEhB,oBAAoB;;;;;;iBA6PpB,gBAAgB;;;;;;UC3qBf;;WAEN,WAAW;;WAEX;;WAEA;;EAGT,KAAK,KAAK,aAAa,OAAO,eAAe,QAAQ;;KAG3C;UAMK,oBAAoB,KAAK;;EAExC;;KAGU,eAAe;UAEV;;EAEf,SAAS;;EAET;;EAEA;;EAEA;;KAKU,uBACR,0BACA,sBACA,gCACA;UAEM;EACR;;EAEA;;;;;;UAOe,gCAAgC;EAC/C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;UAI1C,4BAA4B;EAC3C;EACA,OAAO,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;;;;;;;;;;;;;;;UAqB1C,sCACP,mBACN,KACE;EAaJ;;EAEA;;EAEA;;EAEA;;EAEA;IAAe;IAAc;;;;;;;UAOd,0BAA0B;EACzC;EACA,UAAU,KAAK,aAAa,OAAO,iBAAiB,QAAQ;;;;;;iBAO9C,iBAAiB,MAAM,uBAAuB;;;UC/J7C;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,gBAAgB;EAChB;;UAGe;EACf;EACA;EACA;EACA;;UAKe;EACf;EAQA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;;UAKe;EACf;EACA;EACA,OAAO;EACP,iBAAiB;EACjB,aAAa;EACb;EACA;EACA;EACA,WAAW;;EAEX,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;IAAmB;IAAc;;EACjC;EACA;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;IAAc;IAAc;;EAC5B;IAAmB;IAAc,QAAQ;;EACzC;IAAc;IAAkB;;EAChC;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;GACC;;UAGc;EACf;EACA,QAAQ;;EAER;;UAKe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IACE;MACE;MACA;MACA;;;EAGJ,OAAO;EACP,gBAAgB;;UAKD;EACf;EACA,QAAQ,OAAO;EACf,YAAY,OAAO;;;;;;;;;;KAWT;UAEK;EACf;EACA;EACA;EACA,oBAAoB;EACpB;;EAEA,QAAQ;;;;;;;EAOR;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;EACP,WAAW;;EAEX,aAAa;EACb;EACA,WAAW;EACX,SAAS;;KAGC,WAAW,MAAM,YAAY,OAAO,eAAe,QAAQ;UAYtD;EACf;EACA;EACA;EACA"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-
|
|
3
|
-
import { c as JudgeInput, x as ChatClient } from "./types-
|
|
2
|
+
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-xEULpgML.js";
|
|
3
|
+
import { c as JudgeInput, x as ChatClient } from "./types-C0bWoSKC.js";
|
|
4
4
|
import { RE2JS } from "re2js";
|
|
5
5
|
//#region src/trace-analyst/types.d.ts
|
|
6
6
|
/**
|
|
@@ -635,4 +635,4 @@ type AnalystRunEvent = {
|
|
|
635
635
|
};
|
|
636
636
|
//#endregion
|
|
637
637
|
export { DatasetOverview as A, TraceAnalystSpanKind as B, ReadSpanSourceInput as C, TraceAnalysisStore as D, TRACE_ANALYSIS_LIMITS as E, SpanMatchRecord as F, ViewTraceResult as G, TraceAnalystTraceSummary as H, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as I, TraceAnalystByteBudgets as L, QueryTracesPage as M, SearchSpanResult as N, TraceAnalysisStoreContext as O, SearchTraceResult as P, TraceAnalystFilters as R, BoundedTraceAnalysisStoreOptions as S, SpanSourceReader as T, ViewSpansResult as U, TraceAnalystSpanStatus as V, ViewTraceOversized as W, ProposalFinding as _, AnalystInputKind as a, makeFinding as b, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, ExecutionProbeRequest as g, ExecutionProbeOutcome as h, AnalystFinding as i, ErrorCluster as j, DEFAULT_TRACE_ANALYST_BUDGETS as k, AnalystRunResult as l, ExecutionProbe as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, ProposalFindingOrigin as v, ReadSpanSourceResult as w, makeProposalFinding as x, computeFindingId as y, TraceAnalystSpan as z };
|
|
638
|
-
//# sourceMappingURL=types-
|
|
638
|
+
//# sourceMappingURL=types-DDp2Mb8Y.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-DDp2Mb8Y.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KAgBY;KAUA;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC7MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;UAIM;EACf;EACA;EACA;EACA;EACA;EACA;;KAGU;EAEN;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA;;;;KAKI,oBACV,OAAO,qBACP,UAAU,8BACP,QAAQ;;;;;;;;UASI;;EAEf,iBAAiB;EAEjB,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;;;;UC9GH;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;;;cAKA;KAED,0BAA0B;;KAG1B;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
|
package/dist/wire/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { b as CustomTokenPricing, c as CostLedgerHandle } from "../cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { x as ChatClient } from "../types-
|
|
3
|
-
import { s as TraceStore } from "../store-
|
|
4
|
-
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-
|
|
2
|
+
import { x as ChatClient } from "../types-C0bWoSKC.js";
|
|
3
|
+
import { s as TraceStore } from "../store-C4avFjIi.js";
|
|
4
|
+
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-BJi4t3Vf.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { ServerType } from "@hono/node-server";
|
|
7
7
|
import { Hono } from "hono";
|
package/dist/wire/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { A as WIRE_VERSION, C as RUBRIC_VERSION_SCHEME, D as TracesIngestRequestSchema, E as TraceEventSchema, O as TracesIngestResponseSchema, S as ListRubricsResponseSchema, T as RubricSchema, _ as FeedbackIngestResponseSchema, a as runRpcBatch, b as JudgeRequestSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as ErrorResponseSchema, h as listBuiltinRubrics, i as dispatchRpc, j as hashRubric, k as VersionResponseSchema, l as handleFeedbackIngest, m as getBuiltinRubric, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FeedbackTrajectorySchema, w as RubricDimensionSchema, x as JudgeResultSchema, y as HealthResponseSchema } from "../server-
|
|
1
|
+
import { A as WIRE_VERSION, C as RUBRIC_VERSION_SCHEME, D as TracesIngestRequestSchema, E as TraceEventSchema, O as TracesIngestResponseSchema, S as ListRubricsResponseSchema, T as RubricSchema, _ as FeedbackIngestResponseSchema, a as runRpcBatch, b as JudgeRequestSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as ErrorResponseSchema, h as listBuiltinRubrics, i as dispatchRpc, j as hashRubric, k as VersionResponseSchema, l as handleFeedbackIngest, m as getBuiltinRubric, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FeedbackTrajectorySchema, w as RubricDimensionSchema, x as JudgeResultSchema, y as HealthResponseSchema } from "../server-C8XvqCg8.js";
|
|
2
2
|
export { ErrorResponseSchema, FeedbackIngestResponseSchema, FeedbackTrajectorySchema, HealthResponseSchema, JudgeRequestSchema, JudgeResultSchema, ListRubricsResponseSchema, RUBRIC_VERSION_SCHEME, RubricDimensionSchema, RubricSchema, TraceEventSchema, TracesIngestRequestSchema, TracesIngestResponseSchema, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
|
package/docs/diagnosis.md
CHANGED
|
@@ -20,9 +20,8 @@ Only an internal context may carry a `topology` (a run graph); a customer contex
|
|
|
20
20
|
|
|
21
21
|
## Order of operations
|
|
22
22
|
|
|
23
|
-
1. The
|
|
24
|
-
|
|
25
|
-
A match becomes `[REDACTED:<rule>]`, and the document's `coverage.redaction` counts matches by rule.
|
|
23
|
+
1. The redaction core ([redaction.md](./redaction.md), default profile) runs on span attributes, names, status messages, the focus question, and the topology.
|
|
24
|
+
A string that holds a credential becomes `[REDACTED:<detector>]`, and the document's `coverage.redaction` counts replacements by detector.
|
|
26
25
|
2. The deterministic pass reports the trace contract's capability table with the trace's own reasons, execution facts with token and cost accounting, and `observed` findings.
|
|
27
26
|
Each observed finding carries a measure with its denominator and cites the spans it counted.
|
|
28
27
|
3. Model mode reads the runs with the most errors through the prime protocol (`runPrimeExchange`).
|
package/docs/eval-surface-map.md
CHANGED
|
@@ -74,7 +74,7 @@ Use the decision diagnostics and intervals to distinguish insufficient evidence
|
|
|
74
74
|
|
|
75
75
|
| Subpath | Use |
|
|
76
76
|
|---|---|
|
|
77
|
-
| `/traces`, `/trace-attributes` | Store trace evidence, [connect observability exporters](./adapters-observability.md), and use canonical measurement attribute names. |
|
|
77
|
+
| `/traces`, `/trace-attributes` | Store trace evidence, [redact it and check it is safe to share](./redaction.md), [connect observability exporters](./adapters-observability.md), and use canonical measurement attribute names. |
|
|
78
78
|
| `/analyst` | Execute declared analysts against recorded evidence. |
|
|
79
79
|
| `/reporting`, `/pipelines` | Compare runs, render [research reports](./research-report-methodology.md), and extract recorded failure patterns. |
|
|
80
80
|
| `/supervisor-run` | Read recursive run directories and their evidence coverage. |
|
package/docs/experiment.md
CHANGED
|
@@ -269,7 +269,6 @@ The statistical machinery underneath is re-exported from its existing homes; thi
|
|
|
269
269
|
| `paretoSignificanceGate`, `buildEvidenceVector` | `src/campaign/gates/promotion-policy.ts` |
|
|
270
270
|
| `pairArms`, `comparePairedArms`, `pairRunRecords` | `src/paired-arms.ts` |
|
|
271
271
|
| `hashJson`, `manifestContentDigest`, `signManifest`, `verifyManifest`, `HypothesisManifest` | `src/pre-registration.ts` |
|
|
272
|
-
| `ExperimentTracker` (run ledger with KEEP/ITERATE/NOISE/REGRESSION) | `src/experiment-tracker.ts` |
|
|
273
272
|
|
|
274
273
|
`HypothesisManifest` stays as the lightweight single-metric registration; `sealExperiment` is the full-design registration.
|
|
275
274
|
The trace-repair admission machinery (`buildDenominatorChain`, oracle determinism, control policy) keeps its repair vocabulary in `./trace-repair`; this module is the general form new experiments should register against.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Redaction and share safety
|
|
2
|
+
|
|
3
|
+
`@tangle-network/agent-eval/traces` owns the one redaction core for the Tangle agent stack.
|
|
4
|
+
agent-runtime, traces and run-capsule import it instead of keeping their own patterns.
|
|
5
|
+
The source is `src/trace/redact.ts`.
|
|
6
|
+
|
|
7
|
+
## Redact a value
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
import { redact, redactText } from '@tangle-network/agent-eval/traces'
|
|
11
|
+
|
|
12
|
+
const { value, report } = redact(spanAttributes, { profile: 'share' })
|
|
13
|
+
const message = redactText(error.message, { knownSecrets: [process.env.ROUTER_KEY ?? ''] })
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
`redact` returns a deep copy and a report.
|
|
17
|
+
It never mutates its input and never throws on the value's shape.
|
|
18
|
+
Cycles, binary data and nesting past 64 levels become markers.
|
|
19
|
+
The report lists the JSON Pointer, category and detector of each change, never the removed value.
|
|
20
|
+
|
|
21
|
+
| Profile | Removes | String cap |
|
|
22
|
+
|---|---|---|
|
|
23
|
+
| `default` | Credentials by field name and value shape; email, card, SSN and phone values | 1 MiB |
|
|
24
|
+
| `share` | Also phone, IP and postal-address fields; pseudonymizes user, account, tenant, session and request ids | 64 KiB |
|
|
25
|
+
| `strict` | Also prompt, completion, message, tool payload and other raw-content fields, and `data:` media | 4 KiB |
|
|
26
|
+
|
|
27
|
+
A string that holds a credential is replaced whole with `[REDACTED:<detector>]`.
|
|
28
|
+
Personal data inside prose is replaced in place.
|
|
29
|
+
Pass `knownSecrets` to remove exact values in any form: as written, base64, base64url or URL-encoded.
|
|
30
|
+
|
|
31
|
+
## Field names
|
|
32
|
+
|
|
33
|
+
`classifyKey` normalizes camelCase, kebab-case and dotted names to snake_case before it matches them.
|
|
34
|
+
`apiKey`, `x-api-key`, `client.secret`, `refreshToken` and `OPENAI_API_KEY` are credentials.
|
|
35
|
+
Token counts and model limits are not: `inputTokens`, `outputTokens`, `max_tokens`, `token_count`, `gen_ai.usage.input_tokens` and `next_page_token` survive.
|
|
36
|
+
`author`, `auth_type`, `secret_name` and `content_type` also survive, because they name or describe a value instead of holding it.
|
|
37
|
+
A number under a credential name survives, because it is a limit or a count.
|
|
38
|
+
|
|
39
|
+
## Share-safety verdict
|
|
40
|
+
|
|
41
|
+
`assessShareSafety(value, { profile })` reads a value without changing it.
|
|
42
|
+
`redactForShare(value, { profile })` redacts it and then assesses the redacted output, so a detector gap appears as UNSAFE instead of a leak.
|
|
43
|
+
|
|
44
|
+
| Status | Meaning | Share allowed |
|
|
45
|
+
|---|---|---|
|
|
46
|
+
| `SAFE` | No finding | Yes |
|
|
47
|
+
| `SAFE_WITH_WARNINGS` | Only warnings, such as raw content under `default` or identifiers under `share` | Yes |
|
|
48
|
+
| `UNSAFE` | A credential, or personal data under `share` or `strict`, remains | No |
|
|
49
|
+
| `UNKNOWN` | Part of the value could not be read: a cycle, binary data, or nesting past 64 levels | No |
|
|
50
|
+
|
|
51
|
+
`shareAllowed(verdict)` is the gate.
|
|
52
|
+
`traces verify-safe`, `traces upload` and run-capsule's opted-in upload refuse UNSAFE and UNKNOWN.
|
|
53
|
+
|
|
54
|
+
## Corpus
|
|
55
|
+
|
|
56
|
+
`scripts/redaction-corpus.json` holds the must-flag and must-not-flag cases.
|
|
57
|
+
`pnpm redaction:corpus` runs the core over every case and exits 1 on any miss.
|
|
58
|
+
Add a case whenever a real trace shows a false positive or a leak.
|
|
59
|
+
The corpus includes the agent-inspect safety corpus (MIT, commit 3c3cbeda) with its notice.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Trace contracts
|
|
2
|
+
|
|
3
|
+
A trace contract states which path an agent run must take, and checks the recorded spans against it.
|
|
4
|
+
Two runs can give the same answer while one of them calls a forbidden tool; an output check passes both, and a contract fails the wrong one.
|
|
5
|
+
|
|
6
|
+
The checker is deterministic and calls no model.
|
|
7
|
+
It reads any span array that has `name`, `kind` or span-kind attributes, timestamps, and attributes: agent-eval `TraceStore` spans, otel-bridge spans, or OTLP rows that `traces check` maps.
|
|
8
|
+
|
|
9
|
+
## Write a contract
|
|
10
|
+
|
|
11
|
+
Write the contract as JSON and compile it with `compileTraceContractSpec`:
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{
|
|
15
|
+
"name": "refund-desk",
|
|
16
|
+
"description": "Answer refund questions from the orders file without a shell",
|
|
17
|
+
"run": { "requireCompleted": true, "maxDurationMs": 120000 },
|
|
18
|
+
"tools": {
|
|
19
|
+
"required": ["Read"],
|
|
20
|
+
"forbidden": ["Bash"],
|
|
21
|
+
"allowed": ["Read", "Grep", "Glob"],
|
|
22
|
+
"maxCalls": 10
|
|
23
|
+
},
|
|
24
|
+
"llm": { "maxTotalTokens": 200000 }
|
|
25
|
+
}
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
| Key | Checks |
|
|
29
|
+
|---|---|
|
|
30
|
+
| `run.requireCompleted` | The run reached a terminal status and recorded an end time. |
|
|
31
|
+
| `run.allowedStatuses` | The run status is one of `running`, `completed`, `failed`, `aborted`. |
|
|
32
|
+
| `run.maxDurationMs` | End minus start. An unknown duration fails. |
|
|
33
|
+
| `tools.required` | Each tool is called at least once. |
|
|
34
|
+
| `tools.forbidden` | No listed tool is called. |
|
|
35
|
+
| `tools.allowed` | Every tool call names a listed tool. A call without a tool name fails. |
|
|
36
|
+
| `tools.maxCalls`, `tools.maxCallsPerTool` | Ceilings on tool calls, overall and per tool. |
|
|
37
|
+
| `tools.requiredOrder`, `tools.orderMode` | Each tool is called, and each comes before the next. |
|
|
38
|
+
| `tools.arguments` | An RFC 6901 JSON Pointer check (`exists`, `equals`, `oneOf`, `type`) on tool-call arguments. A call without captured arguments fails. |
|
|
39
|
+
| `llm.maxCalls`, `llm.maxTotalTokens` | Ceilings on model calls and on input plus output tokens. An unrecorded token count fails. |
|
|
40
|
+
| `llm.allowedModels` | Every model call names a listed model. A call without a model fails. |
|
|
41
|
+
| `scope.root` | Check only the one span that matches this predicate and its descendants: one sub-agent or one search-tree node. Zero or several matches make every rule an error. |
|
|
42
|
+
| `alternatives.anyOf` | Named paths, each with its own `run`, `tools`, `llm` and `rules`. At least one must pass completely, as well as the base checks. Use it for a legitimate shortcut such as a cache hit. |
|
|
43
|
+
| `rules` | Low-level rules for checks the keys above cannot state (see below). |
|
|
44
|
+
|
|
45
|
+
Tool keys match TOOL spans only, and model keys match LLM spans only.
|
|
46
|
+
An LLM span that has a tool's name never satisfies a tool rule.
|
|
47
|
+
|
|
48
|
+
Parsing is strict.
|
|
49
|
+
An unknown key or an unknown status is an error that names the closest known word, so a typo never removes a check.
|
|
50
|
+
`lintTraceContractSpec` reports contradictions (a tool that is both required and forbidden) and likely mistakes (an order that uses the default `start-order` mode, which passes overlapping calls).
|
|
51
|
+
`explainTraceContract` states each compiled rule in one line.
|
|
52
|
+
|
|
53
|
+
## Rule semantics
|
|
54
|
+
|
|
55
|
+
Every rule reports `pass`, `fail` or `error` in `ContractVerdict.ruleExecutions`.
|
|
56
|
+
`error` means the rule could not be evaluated: its predicate threw, or the scope selected no unique subtree.
|
|
57
|
+
A verdict with an errored rule has status `error` and is never valid.
|
|
58
|
+
A contract with no rules is rejected before evaluation.
|
|
59
|
+
|
|
60
|
+
Missing evidence fails the rule.
|
|
61
|
+
A required successor that never ran fails `precedes`, a span without the timestamp an order needs fails the order, and an unknown token count fails a token ceiling.
|
|
62
|
+
|
|
63
|
+
| Operator | Meaning |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `always(p)`, `never(p)`, `eventually(p)` | Every span, no span, or some span matches `p`. |
|
|
66
|
+
| `precedes(a, b, { order })` | Both `a` and `b` occur, and `a` comes first. |
|
|
67
|
+
| `neverUnless(p, prior, { order })` | Every `p` has an earlier `prior`. A run without `p` passes. |
|
|
68
|
+
| `atMost(p, max)`, `tokensAtMost(p, max)` | Call and token ceilings. |
|
|
69
|
+
| `run({ ... })` | The run status and duration. |
|
|
70
|
+
| `argument(p, { pointer, check, occurrence })` | A JSON Pointer check on tool arguments. |
|
|
71
|
+
|
|
72
|
+
Ordering has three modes.
|
|
73
|
+
`start-order` (the default) needs an `a` that started before each `b`.
|
|
74
|
+
`finish-before-start` needs an `a` that finished before each `b` started.
|
|
75
|
+
`all-occurrences` needs every `a` to finish before each `b` starts.
|
|
76
|
+
|
|
77
|
+
In TypeScript, the builder states the same rules:
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
import { checkTraceContracts, traceContract } from '@tangle-network/agent-eval'
|
|
81
|
+
|
|
82
|
+
const contract = traceContract('proposal-before-generation')
|
|
83
|
+
.neverUnless({ tool: /generate/ }, { tool: 'submit_proposal' }, 'generation-needs-proposal')
|
|
84
|
+
.build()
|
|
85
|
+
const result = checkTraceContracts(await store.spans({ runId }), [contract])
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Gate CI on a contract
|
|
89
|
+
|
|
90
|
+
`traces check` runs a contract over a recorded trace and exits with a code a CI step can read:
|
|
91
|
+
|
|
92
|
+
```sh
|
|
93
|
+
traces check spans.otlp.jsonl --contract refund-desk.contract.json --junit contract.xml
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
| Exit | Meaning |
|
|
97
|
+
|---|---|
|
|
98
|
+
| 0 | Every rule passed. |
|
|
99
|
+
| 1 | A rule failed. |
|
|
100
|
+
| 2 | The contract is malformed or contradicts itself, or a rule could not be evaluated. |
|
|
101
|
+
| 3 | The trace could not be read. |
|
|
102
|
+
| 4 | The trace reference is ambiguous. |
|