@tangle-network/agent-eval 0.208.2 → 0.210.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +1 -1
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-CEhxR_I3.d.ts → agent-profile-BGyt2z8U.d.ts} +3 -3
- package/dist/{agent-profile-CEhxR_I3.d.ts.map → agent-profile-BGyt2z8U.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-BHcB-mZX.d.ts → agent-profile-cell-BNp-Js9F.d.ts} +2 -3
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +1 -0
- package/dist/{agent-profile-cell-DcFW4vOc.js → agent-profile-cell-BYc0bxsL.js} +2 -2
- package/dist/agent-profile-cell-BYc0bxsL.js.map +1 -0
- package/dist/analyst/index.d.ts +12 -12
- package/dist/analyst/index.js +5 -5
- package/dist/{analyze-runs-GZ0VSwui.js → analyze-runs-3xpby3p2.js} +6 -6
- package/dist/analyze-runs-3xpby3p2.js.map +1 -0
- package/dist/{analyze-runs-CTJyUplR.d.ts → analyze-runs-BlslTkOJ.d.ts} +4 -4
- package/dist/{analyze-runs-CTJyUplR.d.ts.map → analyze-runs-BlslTkOJ.d.ts.map} +1 -1
- package/dist/{benchmark-command-BC265T1r.js → benchmark-command-S8tCtkxh.js} +6 -6
- package/dist/{benchmark-command-BC265T1r.js.map → benchmark-command-S8tCtkxh.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +5 -7
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +10 -10
- package/dist/{campaign-evidence-DW3Krxov.js → campaign-evidence-OcUciD-N.js} +4 -4
- package/dist/campaign-evidence-OcUciD-N.js.map +1 -0
- package/dist/{campaign-C5pjR-Qn.js → campaign-wvvJA1ps.js} +14 -14
- package/dist/{campaign-C5pjR-Qn.js.map → campaign-wvvJA1ps.js.map} +1 -1
- package/dist/claude-jsonl-dZir12m6.js +86 -0
- package/dist/claude-jsonl-dZir12m6.js.map +1 -0
- package/dist/cli.js +7 -7
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +92 -16
- package/dist/contract/index.js.map +1 -1
- package/dist/counterfactual-DtMWvn7D.js.map +1 -1
- package/dist/{default-registry-DZFWY2_E.d.ts → default-registry-Crk3QEy1.d.ts} +6 -6
- package/dist/{default-registry-DZFWY2_E.d.ts.map → default-registry-Crk3QEy1.d.ts.map} +1 -1
- package/dist/{default-registry-DX5R7_n9.js → default-registry-DF4Yp7HQ.js} +2 -2
- package/dist/{default-registry-DX5R7_n9.js.map → default-registry-DF4Yp7HQ.js.map} +1 -1
- package/dist/{define-agent-eval-C8zBSbsc.d.ts → define-agent-eval-B_zZY2AZ.d.ts} +7 -7
- package/dist/{define-agent-eval-C8zBSbsc.d.ts.map → define-agent-eval-B_zZY2AZ.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BjKXcDQp.js → define-agent-eval-CoXXDCMu.js} +7 -7
- package/dist/{define-agent-eval-BjKXcDQp.js.map → define-agent-eval-CoXXDCMu.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -5
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +2 -26
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-sHwdSxK3.js → dspy-rlm-engine-BpaKVKJA.js} +2 -2
- package/dist/{dspy-rlm-engine-sHwdSxK3.js.map → dspy-rlm-engine-BpaKVKJA.js.map} +1 -1
- package/dist/{engine-DProXJ7A.d.ts → engine-D-YZ32ZJ.d.ts} +5 -12
- package/dist/engine-D-YZ32ZJ.d.ts.map +1 -0
- package/dist/{eval-campaign-DU_p8qVt.js → eval-campaign-4byTtqrf.js} +4 -4
- package/dist/{eval-campaign-DU_p8qVt.js.map → eval-campaign-4byTtqrf.js.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-Cq6xXNVR.d.ts → exact-types-DjhfZglz.d.ts} +2 -2
- package/dist/{exact-types-Cq6xXNVR.d.ts.map → exact-types-DjhfZglz.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +8 -8
- package/dist/{feedback-trajectory-C6Z8PRoF.d.ts → feedback-trajectory-CX01700m.d.ts} +3 -3
- package/dist/{feedback-trajectory-C6Z8PRoF.d.ts.map → feedback-trajectory-CX01700m.d.ts.map} +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-CbBfavbT.d.ts → index-D0_l25We.d.ts} +21 -42
- package/dist/{index-CbBfavbT.d.ts.map → index-D0_l25We.d.ts.map} +1 -1
- package/dist/{index-nwNoS582.d.ts → index-DnZdBxCL.d.ts} +14 -11
- package/dist/{index-nwNoS582.d.ts.map → index-DnZdBxCL.d.ts.map} +1 -1
- package/dist/{index-DkucR0KJ.d.ts → index-UxNTvn-w.d.ts} +6 -6
- package/dist/{index-DkucR0KJ.d.ts.map → index-UxNTvn-w.d.ts.map} +1 -1
- package/dist/{index-Bzto04CY.d.ts → index-xnhRdqik.d.ts} +9 -20
- package/dist/{index-Bzto04CY.d.ts.map → index-xnhRdqik.d.ts.map} +1 -1
- package/dist/index.d.ts +28 -37
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -90
- package/dist/index.js.map +1 -1
- package/dist/judge-gate-C59znGmR.js +638 -0
- package/dist/judge-gate-C59znGmR.js.map +1 -0
- package/dist/judge-gate-D2hCppem.d.ts +439 -0
- package/dist/judge-gate-D2hCppem.d.ts.map +1 -0
- package/dist/{kind-factory-Cn1lbsAd.js → kind-factory-ivTx6koH.js} +2 -11
- package/dist/{kind-factory-Cn1lbsAd.js.map → kind-factory-ivTx6koH.js.map} +1 -1
- package/dist/{llm-judge-HLWASwUJ.js → llm-judge-Tr7MBUnf.js} +9 -9
- package/dist/llm-judge-Tr7MBUnf.js.map +1 -0
- package/dist/{matrix-Cqkk7eYI.d.ts → matrix-jHTsHEkR.d.ts} +2 -2
- package/dist/{matrix-Cqkk7eYI.d.ts.map → matrix-jHTsHEkR.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +3 -133
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +3 -276
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{mint-C1W8w5wo.d.ts → mint-CDy-brUR.d.ts} +2 -2
- package/dist/{mint-C1W8w5wo.d.ts.map → mint-CDy-brUR.d.ts.map} +1 -1
- package/dist/{mint-CWG-s2mI.js → mint-Csor1BLJ.js} +2 -2
- package/dist/{mint-CWG-s2mI.js.map → mint-Csor1BLJ.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-CgWT7sHy.js → paired-arms-DzKn0TfY.js} +3 -5
- package/dist/{paired-arms-CgWT7sHy.js.map → paired-arms-DzKn0TfY.js.map} +1 -1
- package/dist/{paired-binary-b2L3WKqr.js → paired-binary-Q36bhn2R.js} +13 -9
- package/dist/paired-binary-Q36bhn2R.js.map +1 -0
- package/dist/{paired-promotion-decision-2deJ7sDX.js → paired-promotion-decision-BN6IQTiu.js} +2 -2
- package/dist/{paired-promotion-decision-2deJ7sDX.js.map → paired-promotion-decision-BN6IQTiu.js.map} +1 -1
- package/dist/pipelines/index.d.ts +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/{prime-protocol-Csr9kcOM.d.ts → prime-protocol-B7et3tI7.d.ts} +2 -2
- package/dist/{prime-protocol-Csr9kcOM.d.ts.map → prime-protocol-B7et3tI7.d.ts.map} +1 -1
- package/dist/{produced-state-Bzlh09oj.js → produced-state-CKybuR9F.js} +3 -3
- package/dist/produced-state-CKybuR9F.js.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-policy-CCmyEH_c.js → promotion-policy-CWW35UWq.js} +3 -3
- package/dist/{promotion-policy-CCmyEH_c.js.map → promotion-policy-CWW35UWq.js.map} +1 -1
- package/dist/{query-DlV0pdx4.js → query-B1D1qp2Z.js} +2 -2
- package/dist/{query-DlV0pdx4.js.map → query-B1D1qp2Z.js.map} +1 -1
- package/dist/{query-DBUQhehH.d.ts → query-DK1nWcmp.d.ts} +2 -3
- package/dist/{query-DBUQhehH.d.ts.map → query-DK1nWcmp.d.ts.map} +1 -1
- package/dist/{registry-DMkmhBQ5.d.ts → registry-BL7H33tj.d.ts} +3 -3
- package/dist/{registry-DMkmhBQ5.d.ts.map → registry-BL7H33tj.d.ts.map} +1 -1
- package/dist/{release-confidence-CkoXFx9I.d.ts → release-confidence-C4kI7lQi.d.ts} +4 -12
- package/dist/release-confidence-C4kI7lQi.d.ts.map +1 -0
- package/dist/{release-confidence-CiFg1EPq.js → release-confidence-Dbe9AS_X.js} +2 -2
- package/dist/{release-confidence-CiFg1EPq.js.map → release-confidence-Dbe9AS_X.js.map} +1 -1
- package/dist/{report-command-pXtxg5YK.js → report-command-CtIIsQKP.js} +3 -28
- package/dist/{report-command-pXtxg5YK.js.map → report-command-CtIIsQKP.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +3 -3
- package/dist/{researcher-CZFZjj-u.d.ts → researcher-DuIJ9IaA.d.ts} +4 -4
- package/dist/{researcher-CZFZjj-u.d.ts.map → researcher-DuIJ9IaA.d.ts.map} +1 -1
- package/dist/{reward-hacking-Cyk-QHk9.d.ts → reward-hacking-C6Kp54Av.d.ts} +2 -2
- package/dist/{reward-hacking-Cyk-QHk9.d.ts.map → reward-hacking-C6Kp54Av.d.ts.map} +1 -1
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +10 -29
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +7 -7
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +3 -3
- package/dist/rollout/index.js +4 -4
- package/dist/{rollout-l9w3Gdvh.js → rollout-D9bKKBnZ.js} +30 -165
- package/dist/rollout-D9bKKBnZ.js.map +1 -0
- package/dist/{rubric-predictive-validity-DIUwJJ9Y.d.ts → rubric-predictive-validity-BiNNEjpG.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-DIUwJJ9Y.d.ts.map → rubric-predictive-validity-BiNNEjpG.d.ts.map} +1 -1
- package/dist/{run-record-BRlM99rX.js → run-record-DlTZ_lqe.js} +3 -13
- package/dist/run-record-DlTZ_lqe.js.map +1 -0
- package/dist/{run-record-BB290Y9A.js → run-record-ES07wfz9.js} +2 -2
- package/dist/{run-record-BB290Y9A.js.map → run-record-ES07wfz9.js.map} +1 -1
- package/dist/{run-record-DF-fuu31.d.ts → run-record-p2AYTrCn.d.ts} +3 -7
- package/dist/run-record-p2AYTrCn.d.ts.map +1 -0
- package/dist/search/index.d.ts +2 -24
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +3 -3
- package/dist/{search-kernel-CIie6xig.js → search-kernel-BIXWM7YV.js} +4 -4
- package/dist/search-kernel-BIXWM7YV.js.map +1 -0
- package/dist/{search-kernel-uKdPVRAe.d.ts → search-kernel-ah2GESe3.d.ts} +3 -3
- package/dist/{search-kernel-uKdPVRAe.d.ts.map → search-kernel-ah2GESe3.d.ts.map} +1 -1
- package/dist/{search-ledger-CXQPwdWz.js → search-ledger-aMTNQn_m.js} +2 -2
- package/dist/{search-ledger-CXQPwdWz.js.map → search-ledger-aMTNQn_m.js.map} +1 -1
- package/dist/{search-shipper-DhhofkDX.js → search-shipper-DKXWuLnp.js} +2 -2
- package/dist/{search-shipper-DhhofkDX.js.map → search-shipper-DKXWuLnp.js.map} +1 -1
- package/dist/{search-summary-BV5uehLd.js → search-summary-nwrtXxC_.js} +3 -3
- package/dist/{search-summary-BV5uehLd.js.map → search-summary-nwrtXxC_.js.map} +1 -1
- package/dist/{semantic-concept-judge-D1wH-YKc.js → semantic-concept-judge-BQxZyXdN.js} +3 -3
- package/dist/{semantic-concept-judge-D1wH-YKc.js.map → semantic-concept-judge-BQxZyXdN.js.map} +1 -1
- package/dist/{sequential-DMqr3sni.js → sequential-D-1PPCmM.js} +3 -3
- package/dist/{sequential-DMqr3sni.js.map → sequential-D-1PPCmM.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Dcnh5Pvr.js → skillopt-optimization-method-DujPOiwD.js} +5 -5
- package/dist/{skillopt-optimization-method-Dcnh5Pvr.js.map → skillopt-optimization-method-DujPOiwD.js.map} +1 -1
- package/dist/{statistical-heldout-BP6iEJvR.js → statistical-heldout-1Gbmex7v.js} +2 -2
- package/dist/{statistical-heldout-BP6iEJvR.js.map → statistical-heldout-1Gbmex7v.js.map} +1 -1
- package/dist/{statistical-heldout-DFKtp-Qm.d.ts → statistical-heldout-8RGPIeyo.d.ts} +17 -14
- package/dist/statistical-heldout-8RGPIeyo.d.ts.map +1 -0
- package/dist/{store-otlp-BA7G1Du-.js → store-otlp-__ebQ6Hn.js} +2 -2
- package/dist/{store-otlp-BA7G1Du-.js.map → store-otlp-__ebQ6Hn.js.map} +1 -1
- package/dist/{store-tool-spans-DKoVbE-w.js → store-tool-spans-TbRBuIM7.js} +3 -3
- package/dist/{store-tool-spans-DKoVbE-w.js.map → store-tool-spans-TbRBuIM7.js.map} +1 -1
- package/dist/{store-tool-spans-9YJNMP-z.d.ts → store-tool-spans-wQ9idsnn.d.ts} +5 -5
- package/dist/{store-tool-spans-9YJNMP-z.d.ts.map → store-tool-spans-wQ9idsnn.d.ts.map} +1 -1
- package/dist/{summary-report-DvvesgR_.d.ts → summary-report-CK0ueoJd.d.ts} +2 -2
- package/dist/{summary-report-DvvesgR_.d.ts.map → summary-report-CK0ueoJd.d.ts.map} +1 -1
- package/dist/{summary-report-fXurE2TR.js → summary-report-DtdRucM0.js} +3 -3
- package/dist/{summary-report-fXurE2TR.js.map → summary-report-DtdRucM0.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts +0 -8
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +36 -71
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-review-Bt4JcxMc.d.ts → systemone-review-BReh3mvm.d.ts} +2 -2
- package/dist/{systemone-review-Bt4JcxMc.d.ts.map → systemone-review-BReh3mvm.d.ts.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/{tool-groups-j19pmOfP.d.ts → tool-groups-v28A-ZAF.d.ts} +3 -3
- package/dist/tool-groups-v28A-ZAF.d.ts.map +1 -0
- package/dist/{tool-waste-x_E8h9Ih.js → tool-waste-BYt5rdt8.js} +2 -2
- package/dist/{tool-waste-x_E8h9Ih.js.map → tool-waste-BYt5rdt8.js.map} +1 -1
- package/dist/{tool-waste-DxlebQkx.d.ts → tool-waste-Wi4g6Ekb.d.ts} +2 -2
- package/dist/{tool-waste-DxlebQkx.d.ts.map → tool-waste-Wi4g6Ekb.d.ts.map} +1 -1
- package/dist/{trace-questions-dRvmsbTk.d.ts → trace-questions-C26UvI3D.d.ts} +3 -3
- package/dist/{trace-questions-dRvmsbTk.d.ts.map → trace-questions-C26UvI3D.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +2 -9
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +1 -1
- package/dist/traces.d.ts +6 -8
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +6 -6
- package/dist/trajectory-replay/index.d.ts +0 -2
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +1 -1
- package/dist/{tree-CJOXbVcv.js → tree-kc1d80yF.js} +5 -25
- package/dist/tree-kc1d80yF.js.map +1 -0
- package/dist/{types-CFCUZk0B.d.ts → types-Dv9sYwBB.d.ts} +4 -4
- package/dist/{types-CFCUZk0B.d.ts.map → types-Dv9sYwBB.d.ts.map} +1 -1
- package/dist/{types-DAPpAkd4.d.ts → types-KEkKkETv.d.ts} +2 -2
- package/dist/{types-DAPpAkd4.d.ts.map → types-KEkKkETv.d.ts.map} +1 -1
- package/dist/{types-BlSBUNRd.d.ts → types-tWFY6bON.d.ts} +3 -3
- package/dist/{types-BlSBUNRd.d.ts.map → types-tWFY6bON.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/docs/concepts.md +13 -0
- package/docs/experiment.md +10 -0
- package/docs/public-api.md +399 -424
- package/docs/trace-analysis.md +3 -0
- package/package.json +6 -5
- package/dist/agent-profile-cell-BHcB-mZX.d.ts.map +0 -1
- package/dist/agent-profile-cell-DcFW4vOc.js.map +0 -1
- package/dist/analyze-runs-GZ0VSwui.js.map +0 -1
- package/dist/campaign-evidence-DW3Krxov.js.map +0 -1
- package/dist/claude-jsonl-CG3JUbTJ.js +0 -211
- package/dist/claude-jsonl-CG3JUbTJ.js.map +0 -1
- package/dist/engine-DProXJ7A.d.ts.map +0 -1
- package/dist/llm-judge-HLWASwUJ.js.map +0 -1
- package/dist/paired-binary-b2L3WKqr.js.map +0 -1
- package/dist/produced-state-Bzlh09oj.js.map +0 -1
- package/dist/release-confidence-CkoXFx9I.d.ts.map +0 -1
- package/dist/rollout-l9w3Gdvh.js.map +0 -1
- package/dist/run-record-BRlM99rX.js.map +0 -1
- package/dist/run-record-DF-fuu31.d.ts.map +0 -1
- package/dist/search-kernel-CIie6xig.js.map +0 -1
- package/dist/series-convergence-CjO2QdRW.js +0 -43
- package/dist/series-convergence-CjO2QdRW.js.map +0 -1
- package/dist/series-convergence-hEfKdoi-.d.ts +0 -126
- package/dist/series-convergence-hEfKdoi-.d.ts.map +0 -1
- package/dist/statistical-heldout-DFKtp-Qm.d.ts.map +0 -1
- package/dist/tool-groups-j19pmOfP.d.ts.map +0 -1
- package/dist/tree-CJOXbVcv.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"counterfactual-DtMWvn7D.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n | { kind: 'swap-model'; at: number; newModel: string }\n | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n | { kind: 'truncate-after'; at: number }\n | { kind: 'inject-system-message'; at: number; content: string }\n | {\n kind: 'custom'\n at: number\n describe: string\n apply: (step: TrajectoryStep) => TrajectoryStep\n }\n\nexport interface CounterfactualContext {\n originalRunId: string\n originalTrajectory: Trajectory\n /** Steps up to (but not including) the mutation point — the prefix the\n * replayed agent inherits as its prior conversation/tool history. */\n prefix: TrajectoryStep[]\n mutation: CounterfactualMutation\n /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n * as the FIRST step the replayed agent emits (they decide whether to\n * re-emit it or continue from there). */\n mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n counterfactualRunId: string\n originalRunId: string\n mutation: CounterfactualMutation\n /** Structured delta summary — caller can extend via scoring. */\n delta: {\n originalOutcomeScore: number | null\n counterfactualOutcomeScore: number | null\n deltaScore: number | null\n }\n}\n\nexport interface CounterfactualRunner {\n /**\n * Execute the agent from `ctx.prefix` with the mutation applied.\n * MUST emit spans into the provided emitter so they become part of\n * the counterfactual run. MUST call emitter.endRun() with a verdict.\n */\n executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n store: TraceStore,\n originalRunId: string,\n mutation: CounterfactualMutation,\n runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n const originalRun = await store.getRun(originalRunId)\n if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n const trajectory = await buildTrajectory(store, originalRunId)\n if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n throw new ValidationError(\n `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n )\n }\n const targetStep = trajectory.steps[mutation.at]!\n const mutatedStep = applyMutation(targetStep, mutation)\n\n const cfEmitter = new TraceEmitter(store)\n await cfEmitter.startRun({\n scenarioId: originalRun.scenarioId,\n variantId: originalRun.variantId\n ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n : `cf:${mutation.kind}@${mutation.at}`,\n projectId: originalRun.projectId,\n parentRunId: originalRunId,\n layer: 'meta',\n tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n })\n\n await runner.executeFrom(\n {\n originalRunId,\n originalTrajectory: trajectory,\n prefix: trajectory.steps.slice(0, mutation.at),\n mutation,\n mutatedStep,\n },\n cfEmitter,\n )\n\n const counterfactual = await store.getRun(cfEmitter.runId)\n const delta = {\n originalOutcomeScore: originalRun.outcome?.score ?? null,\n counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n deltaScore:\n originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n ? counterfactual.outcome.score - originalRun.outcome.score\n : null,\n }\n return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return { ...step, span: { ...llm, model: mutation.newModel } }\n }\n if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n const tool = step.span as ToolSpan\n return { ...step, span: { ...tool, result: mutation.newResult } }\n }\n if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return {\n ...step,\n span: {\n ...llm,\n messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n },\n }\n }\n if (mutation.kind === 'custom') return mutation.apply(step)\n // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n return step\n}\n\n
|
|
1
|
+
{"version":3,"file":"counterfactual-DtMWvn7D.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n | { kind: 'swap-model'; at: number; newModel: string }\n | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n | { kind: 'truncate-after'; at: number }\n | { kind: 'inject-system-message'; at: number; content: string }\n | {\n kind: 'custom'\n at: number\n describe: string\n apply: (step: TrajectoryStep) => TrajectoryStep\n }\n\nexport interface CounterfactualContext {\n originalRunId: string\n originalTrajectory: Trajectory\n /** Steps up to (but not including) the mutation point — the prefix the\n * replayed agent inherits as its prior conversation/tool history. */\n prefix: TrajectoryStep[]\n mutation: CounterfactualMutation\n /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n * as the FIRST step the replayed agent emits (they decide whether to\n * re-emit it or continue from there). */\n mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n counterfactualRunId: string\n originalRunId: string\n mutation: CounterfactualMutation\n /** Structured delta summary — caller can extend via scoring. */\n delta: {\n originalOutcomeScore: number | null\n counterfactualOutcomeScore: number | null\n deltaScore: number | null\n }\n}\n\nexport interface CounterfactualRunner {\n /**\n * Execute the agent from `ctx.prefix` with the mutation applied.\n * MUST emit spans into the provided emitter so they become part of\n * the counterfactual run. MUST call emitter.endRun() with a verdict.\n */\n executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n store: TraceStore,\n originalRunId: string,\n mutation: CounterfactualMutation,\n runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n const originalRun = await store.getRun(originalRunId)\n if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n const trajectory = await buildTrajectory(store, originalRunId)\n if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n throw new ValidationError(\n `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n )\n }\n const targetStep = trajectory.steps[mutation.at]!\n const mutatedStep = applyMutation(targetStep, mutation)\n\n const cfEmitter = new TraceEmitter(store)\n await cfEmitter.startRun({\n scenarioId: originalRun.scenarioId,\n variantId: originalRun.variantId\n ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n : `cf:${mutation.kind}@${mutation.at}`,\n projectId: originalRun.projectId,\n parentRunId: originalRunId,\n layer: 'meta',\n tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n })\n\n await runner.executeFrom(\n {\n originalRunId,\n originalTrajectory: trajectory,\n prefix: trajectory.steps.slice(0, mutation.at),\n mutation,\n mutatedStep,\n },\n cfEmitter,\n )\n\n const counterfactual = await store.getRun(cfEmitter.runId)\n const delta = {\n originalOutcomeScore: originalRun.outcome?.score ?? null,\n counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n deltaScore:\n originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n ? counterfactual.outcome.score - originalRun.outcome.score\n : null,\n }\n return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return { ...step, span: { ...llm, model: mutation.newModel } }\n }\n if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n const tool = step.span as ToolSpan\n return { ...step, span: { ...tool, result: mutation.newResult } }\n }\n if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return {\n ...step,\n span: {\n ...llm,\n messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n },\n }\n }\n if (mutation.kind === 'custom') return mutation.apply(step)\n // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n return step\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { O as TraceAnalysisStore, i as AnalystFinding, n as AnalystContext } from "./types-
|
|
2
|
-
import { l as ExactCapableAnalyst } from "./exact-types-
|
|
3
|
-
import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-
|
|
4
|
-
import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine } from "./engine-
|
|
5
|
-
import { t as TraceToolGroupName } from "./tool-groups-
|
|
1
|
+
import { O as TraceAnalysisStore, i as AnalystFinding, n as AnalystContext } from "./types-KEkKkETv.js";
|
|
2
|
+
import { l as ExactCapableAnalyst } from "./exact-types-DjhfZglz.js";
|
|
3
|
+
import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-BL7H33tj.js";
|
|
4
|
+
import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine } from "./engine-D-YZ32ZJ.js";
|
|
5
|
+
import { t as TraceToolGroupName } from "./tool-groups-v28A-ZAF.js";
|
|
6
6
|
//#region src/trace-analyst/behavioral-metrics.d.ts
|
|
7
7
|
type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
|
|
8
8
|
interface SuboptimalSignal {
|
|
@@ -110,4 +110,4 @@ interface DefaultAnalystRegistryOptions {
|
|
|
110
110
|
declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
|
|
111
111
|
//#endregion
|
|
112
112
|
export { createTraceAnalyst as a, BehavioralAnalystOptions as c, TraceAnalystDefinition as i, behavioralAnalyst as l, buildDefaultAnalystRegistry as n, renderPriorFindings as o, CreateTraceAnalystOptions as r, runTraceAnalyst as s, DefaultAnalystRegistryOptions as t, deriveEfficiencyFindings as u };
|
|
113
|
-
//# sourceMappingURL=default-registry-
|
|
113
|
+
//# sourceMappingURL=default-registry-Crk3QEy1.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"default-registry-
|
|
1
|
+
{"version":3,"file":"default-registry-Crk3QEy1.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgHI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA2RP,oBAAoB,OAAO;;;UChc1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { a as hashCanonical, r as canonicalString } from "./canonical-BsZVgBhW.js";
|
|
2
2
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
3
3
|
import { C as combineAbortSignals } from "./llm-client-v968tWw8.js";
|
|
4
|
-
import {
|
|
4
|
+
import { A as spanEpochMillis, H as findingSubjectGrammarPromptFor, J as snapshotExactExecutionPlan, q as snapshotExactExecutionComponentIdentity, t as createTraceAnalyst } from "./kind-factory-ivTx6koH.js";
|
|
5
5
|
import { i as validateUsageSettlementTimeout, t as assertValidAnalystUsageReceipt } from "./usage-receipt-Cx3xNeyB.js";
|
|
6
6
|
import { LLM_CONTEXT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_OUTPUT_TOKEN_ATTR_KEYS, TOOL_NAME_ATTR_KEYS } from "./trace-attributes.js";
|
|
7
7
|
import "./otlp-attributes-D9ptWaIJ.js";
|
|
@@ -2366,4 +2366,4 @@ function buildDefaultAnalystRegistry(options = {}) {
|
|
|
2366
2366
|
//#endregion
|
|
2367
2367
|
export { completedAnalystReviewQuality as _, KNOWLEDGE_POISONING_KIND_SPEC as a, validateAnalystReviewDecisions as b, FAILURE_MODE_KIND_SPEC as c, emitControlIntegrityFindings as d, behavioralAnalyst as f, assertUniqueFindingIds as g, analystRunDigest as h, DEFAULT_TRACE_ANALYST_KINDS as i, CONTROL_INTEGRITY_ANALYST as l, analystFindingDigest as m, AnalystRegistry as n, KNOWLEDGE_GAP_KIND_SPEC as o, deriveEfficiencyFindings as p, ExactAnalystRunExecutionError as r, IMPROVEMENT_KIND_SPEC as s, buildDefaultAnalystRegistry as t, ControlIntegrityAnalyst as u, readAnalystReview as v, snapshotAnalystRun as y };
|
|
2368
2368
|
|
|
2369
|
-
//# sourceMappingURL=default-registry-
|
|
2369
|
+
//# sourceMappingURL=default-registry-DF4Yp7HQ.js.map
|