@tangle-network/agent-eval 0.150.2 → 0.161.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -1
- package/README.md +7 -3
- package/dist/{active-curriculum-C4mk67HP.js → active-curriculum-CD5TU2yW.js} +3 -13
- package/dist/active-curriculum-CD5TU2yW.js.map +1 -0
- package/dist/{agent-profile-cell-BkcRDikH.d.ts → agent-profile-cell-CTOZJUuE.d.ts} +4 -2
- package/dist/agent-profile-cell-CTOZJUuE.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +19 -36
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +8 -8
- package/dist/analyst/index.js.map +1 -1
- package/dist/{backend-integrity-DOCa_QrR.d.ts → backend-integrity-DxuQCu_A.d.ts} +4 -3
- package/dist/backend-integrity-DxuQCu_A.d.ts.map +1 -0
- package/dist/{benchmark-BtAWA8nT.d.ts → benchmark-CGPp-kDC.d.ts} +3 -3
- package/dist/{benchmark-BtAWA8nT.d.ts.map → benchmark-CGPp-kDC.d.ts.map} +1 -1
- package/dist/{benchmark-command-BU1Las59.js → benchmark-command-BDC3Gocz.js} +26 -31
- package/dist/benchmark-command-BDC3Gocz.js.map +1 -0
- package/dist/benchmarks/index.d.ts +6 -19
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +9 -9
- package/dist/campaign/index.js +7 -7
- package/dist/{campaign-la-gEYNz.js → campaign-BSmOwskD.js} +77 -795
- package/dist/campaign-BSmOwskD.js.map +1 -0
- package/dist/{canonical-D-XsTQ6_.js → canonical-IL-Bu-14.js} +26 -2
- package/dist/canonical-IL-Bu-14.js.map +1 -0
- package/dist/{capture-fetch-BBVFzhkk.d.ts → capture-fetch-CqwsJkkG.d.ts} +3 -3
- package/dist/{capture-fetch-BBVFzhkk.d.ts.map → capture-fetch-CqwsJkkG.d.ts.map} +1 -1
- package/dist/{chat-client-Bvmxedyv.js → chat-client-DlMlAeYI.js} +5 -55
- package/dist/{chat-client-Bvmxedyv.js.map → chat-client-DlMlAeYI.js.map} +1 -1
- package/dist/chat-json-call-6g5sJobJ.js +53 -0
- package/dist/chat-json-call-6g5sJobJ.js.map +1 -0
- package/dist/cli.js +54 -19
- package/dist/cli.js.map +1 -1
- package/dist/{client-LIuo-KPv.js → client-CX7KqIdB.js} +3 -3
- package/dist/client-CX7KqIdB.js.map +1 -0
- package/dist/{client-kPQYT_56.d.ts → client-L9VVPkim.d.ts} +4 -4
- package/dist/{client-kPQYT_56.d.ts.map → client-L9VVPkim.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +13 -27
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +14 -17
- package/dist/contract/index.js.map +1 -1
- package/dist/{counterfactual--bpysZF0.d.ts → counterfactual-BaFUWK3H.d.ts} +4 -4
- package/dist/{counterfactual--bpysZF0.d.ts.map → counterfactual-BaFUWK3H.d.ts.map} +1 -1
- package/dist/{counterfactual-lDfCx0Uz.js → counterfactual-D_VWavVm.js} +2 -2
- package/dist/{counterfactual-lDfCx0Uz.js.map → counterfactual-D_VWavVm.js.map} +1 -1
- package/dist/{dataset-CJjKqQfA.d.ts → dataset-DQqhOCPt.d.ts} +5 -4
- package/dist/{dataset-CJjKqQfA.d.ts.map → dataset-DQqhOCPt.d.ts.map} +1 -1
- package/dist/{default-registry-Cw0Ohdoj.d.ts → default-registry-G9CKMNkc.d.ts} +7 -8
- package/dist/{default-registry-Cw0Ohdoj.d.ts.map → default-registry-G9CKMNkc.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CEQWL9Hy.d.ts → define-agent-eval-Dx1JnPEa.d.ts} +26 -6
- package/dist/define-agent-eval-Dx1JnPEa.d.ts.map +1 -0
- package/dist/{define-agent-eval-C8V8sMqP.js → define-agent-eval-h-s-sI-v.js} +15 -9
- package/dist/define-agent-eval-h-s-sI-v.js.map +1 -0
- package/dist/{dspy-rlm-engine-CBYlPvNy.js → dspy-rlm-engine-DptEII26.js} +95 -15
- package/dist/dspy-rlm-engine-DptEII26.js.map +1 -0
- package/dist/{emitter-CPBAhxum.js → emitter-BpYFQPj4.js} +2 -18
- package/dist/emitter-BpYFQPj4.js.map +1 -0
- package/dist/{emitter-DGQGoLyj.d.ts → emitter-D_jYSGRd.d.ts} +4 -20
- package/dist/{emitter-DGQGoLyj.d.ts.map → emitter-D_jYSGRd.d.ts.map} +1 -1
- package/dist/{engine-BLzhNzoY.d.ts → engine-Cu5qD5Fc.d.ts} +9 -11
- package/dist/{engine-BLzhNzoY.d.ts.map → engine-Cu5qD5Fc.d.ts.map} +1 -1
- package/dist/{eval-campaign-CQuZrLR_.js → eval-campaign-BsXWL2-2.js} +17 -28
- package/dist/eval-campaign-BsXWL2-2.js.map +1 -0
- package/dist/{exact-types-ccQAyut1.d.ts → exact-types-qnexxJ1Z.d.ts} +2 -2
- package/dist/{exact-types-ccQAyut1.d.ts.map → exact-types-qnexxJ1Z.d.ts.map} +1 -1
- package/dist/{exec-y-DCLqK7.js → exec-D9WpA2p-.js} +2 -2
- package/dist/exec-D9WpA2p-.js.map +1 -0
- package/dist/experiment/index.d.ts +45 -11
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +30 -11
- package/dist/experiment/index.js.map +1 -1
- package/dist/{experiment-tracker-0MhuPArU.d.ts → experiment-tracker-DCO6Cz4s.d.ts} +2 -2
- package/dist/{experiment-tracker-0MhuPArU.d.ts.map → experiment-tracker-DCO6Cz4s.d.ts.map} +1 -1
- package/dist/{exporters-q9iL-2Jf.js → exporters-Df7TgHFv.js} +3 -3
- package/dist/exporters-Df7TgHFv.js.map +1 -0
- package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts → external-optimizer-contracts-szBJ_1vh.d.ts} +2 -2
- package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts.map → external-optimizer-contracts-szBJ_1vh.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-x9oEXKsU.js → external-optimizer-process-WosTBChy.js} +4 -4
- package/dist/{external-optimizer-process-x9oEXKsU.js.map → external-optimizer-process-WosTBChy.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-CKNb42oM.js → external-optimizer-subprocess-BIWbHpgD.js} +4 -4
- package/dist/{external-optimizer-subprocess-CKNb42oM.js.map → external-optimizer-subprocess-BIWbHpgD.js.map} +1 -1
- package/dist/{failure-cluster-BLURuWG4.d.ts → failure-cluster-CXL8NbEw.d.ts} +3 -3
- package/dist/{failure-cluster-BLURuWG4.d.ts.map → failure-cluster-CXL8NbEw.d.ts.map} +1 -1
- package/dist/{feedback-trajectory-DpTTjo0q.d.ts → feedback-trajectory-B3ZHaHV_.d.ts} +7 -7
- package/dist/{feedback-trajectory-DpTTjo0q.d.ts.map → feedback-trajectory-B3ZHaHV_.d.ts.map} +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-XggBupCr.js → hf-dataset-D8_RNIis.js} +4 -4
- package/dist/{hf-dataset-XggBupCr.js.map → hf-dataset-D8_RNIis.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -14
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +2 -2
- package/dist/{index-Aj3WO3_a.d.ts → index-CGtH1piv.d.ts} +48 -26
- package/dist/index-CGtH1piv.d.ts.map +1 -0
- package/dist/{index-BNPtkBPf.d.ts → index-D-IiQIBB.d.ts} +5 -10
- package/dist/index-D-IiQIBB.d.ts.map +1 -0
- package/dist/{index-IQccV3Ou.d.ts → index-D-V8gCs_.d.ts} +13 -90
- package/dist/index-D-V8gCs_.d.ts.map +1 -0
- package/dist/{index-B8Ui1mr1.d.ts → index-lfaSeKSD.d.ts} +18 -2
- package/dist/index-lfaSeKSD.d.ts.map +1 -0
- package/dist/index-vrJugRal.d.ts +1 -0
- package/dist/index.d.ts +68 -56
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +59 -110
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-BeT8KCgI.d.ts → insight-report-DRe8LB6d.d.ts} +4 -4
- package/dist/{insight-report-BeT8KCgI.d.ts.map → insight-report-DRe8LB6d.d.ts.map} +1 -1
- package/dist/{integrity-DL91tucI.js → integrity-CyWSSoQS.js} +2 -2
- package/dist/{integrity-DL91tucI.js.map → integrity-CyWSSoQS.js.map} +1 -1
- package/dist/{integrity-B0dZ96EO.d.ts → integrity-DUNX9Fao.d.ts} +3 -3
- package/dist/{integrity-B0dZ96EO.d.ts.map → integrity-DUNX9Fao.d.ts.map} +1 -1
- package/dist/{kind-factory-DmAa0h3K.js → kind-factory-DY8FdoXf.js} +3 -71
- package/dist/kind-factory-DY8FdoXf.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-DTae9rv_.js → ledger-core-BOzlRygb.js} +2 -2
- package/dist/{ledger-core-DTae9rv_.js.map → ledger-core-BOzlRygb.js.map} +1 -1
- package/dist/{llm-client-Bg32RW0j.js → llm-client-hgDieDNN.js} +53 -98
- package/dist/llm-client-hgDieDNN.js.map +1 -0
- package/dist/{llm-judge-BqqMS8t7.js → llm-judge-BhasIPFT.js} +1135 -77
- package/dist/llm-judge-BhasIPFT.js.map +1 -0
- package/dist/{matrix-DrVnRp4G.d.ts → matrix-eXKRMHnL.d.ts} +74 -72
- package/dist/matrix-eXKRMHnL.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +8 -6
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +8 -6
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{mint-BV6tLVWl.js → mint-DfODW1KW.js} +3 -3
- package/dist/{mint-BV6tLVWl.js.map → mint-DfODW1KW.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +2 -8
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +56 -86
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +11 -46
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +30 -83
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-DJWAXLms.js → opencode-sqlite-eK6HW6dr.js} +2 -6
- package/dist/{opencode-sqlite-DJWAXLms.js.map → opencode-sqlite-eK6HW6dr.js.map} +1 -1
- package/dist/pipelines/index.d.ts +5 -5
- package/dist/pipelines/index.js +3 -3
- package/dist/{pre-registration-zFSLEiFU.d.ts → pre-registration-CzFCcwYk.d.ts} +55 -40
- package/dist/pre-registration-CzFCcwYk.d.ts.map +1 -0
- package/dist/pre-registration-KN9jkh58.js +110 -0
- package/dist/pre-registration-KN9jkh58.js.map +1 -0
- package/dist/{produced-state-Bnq4FaDO.js → produced-state-DZ89riy5.js} +8 -8
- package/dist/produced-state-DZ89riy5.js.map +1 -0
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +31 -5
- package/dist/profile-cell.js.map +1 -1
- package/dist/{promotion-policy-DLOUkYhI.d.ts → promotion-policy-DtnOIZvk.d.ts} +2 -2
- package/dist/{promotion-policy-DLOUkYhI.d.ts.map → promotion-policy-DtnOIZvk.d.ts.map} +1 -1
- package/dist/{query-Di7eEQ79.js → query-CHmMP42p.js} +20 -11
- package/dist/query-CHmMP42p.js.map +1 -0
- package/dist/{query-CJ_DX8vl.d.ts → query-DxPYqpmT.d.ts} +10 -4
- package/dist/query-DxPYqpmT.d.ts.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/{registry-BQwrSYpC.d.ts → registry-8You7OK1.d.ts} +5 -7
- package/dist/{registry-BQwrSYpC.d.ts.map → registry-8You7OK1.d.ts.map} +1 -1
- package/dist/{release-confidence-BknrpBnO.js → release-confidence-DKfD2RYU.js} +28 -14
- package/dist/release-confidence-DKfD2RYU.js.map +1 -0
- package/dist/{release-confidence-4XrqlpFD.d.ts → release-confidence-Dqt0NFep.d.ts} +7 -6
- package/dist/release-confidence-Dqt0NFep.d.ts.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +3 -3
- package/dist/{researcher-DJnoUE8c.d.ts → researcher-Cz565b7D.d.ts} +34 -21
- package/dist/researcher-Cz565b7D.d.ts.map +1 -0
- package/dist/{reward-hacking-62tojkQd.d.ts → reward-hacking-MBf7qpSB.d.ts} +2 -2
- package/dist/{reward-hacking-62tojkQd.d.ts.map → reward-hacking-MBf7qpSB.d.ts.map} +1 -1
- package/dist/{reward-hacking-C0x0xihA.js → reward-hacking-t4lB1yt8.js} +2 -2
- package/dist/{reward-hacking-C0x0xihA.js.map → reward-hacking-t4lB1yt8.js.map} +1 -1
- package/dist/rl.d.ts +11 -42
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +41 -24
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +3 -3
- package/dist/rollout/index.js +7 -7
- package/dist/{rollout-ytVQ7WT8.js → rollout-Dm2tSdiQ.js} +6 -6
- package/dist/{rollout-ytVQ7WT8.js.map → rollout-Dm2tSdiQ.js.map} +1 -1
- package/dist/{rubric-predictive-validity-CzxLoZge.js → rubric-predictive-validity-CK8SCOg-.js} +5 -15
- package/dist/rubric-predictive-validity-CK8SCOg-.js.map +1 -0
- package/dist/{rubric-predictive-validity-C7LnNvF2.d.ts → rubric-predictive-validity-CxycqzX5.d.ts} +4 -3
- package/dist/rubric-predictive-validity-CxycqzX5.d.ts.map +1 -0
- package/dist/{run-record-D2lDdSAz.js → run-record-BC0ebuRP.js} +2 -2
- package/dist/{run-record-D2lDdSAz.js.map → run-record-BC0ebuRP.js.map} +1 -1
- package/dist/{run-record-DVV82Gwh.d.ts → run-record-VVy4T9OW.d.ts} +3 -3
- package/dist/{run-record-DVV82Gwh.d.ts.map → run-record-VVy4T9OW.d.ts.map} +1 -1
- package/dist/{schema-BtVldJ3T.d.ts → schema-Bjgdsn73.d.ts} +2 -4
- package/dist/{schema-BtVldJ3T.d.ts.map → schema-Bjgdsn73.d.ts.map} +1 -1
- package/dist/{schema-Cef2cFmb.d.ts → schema-BzWDXhOR.d.ts} +2 -5
- package/dist/schema-BzWDXhOR.d.ts.map +1 -0
- package/dist/{schema-C6DW4ZHR.js → schema-C1aaAxTf.js} +2 -2
- package/dist/schema-C1aaAxTf.js.map +1 -0
- package/dist/{schema-CRhEY1SO.js → schema-k6ZBftVv.js} +2 -8
- package/dist/{schema-CRhEY1SO.js.map → schema-k6ZBftVv.js.map} +1 -1
- package/dist/{semantic-concept-judge-laMCnTLn.js → semantic-concept-judge-BSkKKHeq.js} +14 -38
- package/dist/semantic-concept-judge-BSkKKHeq.js.map +1 -0
- package/dist/{sequential-eprocess-CbUt2htw.js → sequential-eprocess-D1jKoihe.js} +49 -2
- package/dist/sequential-eprocess-D1jKoihe.js.map +1 -0
- package/dist/{sequential-C458DXNf.js → sequential-rYW-Ophm.js} +41 -16
- package/dist/sequential-rYW-Ophm.js.map +1 -0
- package/dist/{series-convergence-BxKEgBwA.d.ts → series-convergence-D9WgpXGi.d.ts} +2 -2
- package/dist/{series-convergence-BxKEgBwA.d.ts.map → series-convergence-D9WgpXGi.d.ts.map} +1 -1
- package/dist/{server-dIWwF3j_.js → server-BtFd4uzB.js} +19 -42
- package/dist/server-BtFd4uzB.js.map +1 -0
- package/dist/{skillopt-optimization-method-CPBlTcj5.js → skillopt-optimization-method-DbaekMcn.js} +794 -8
- package/dist/skillopt-optimization-method-DbaekMcn.js.map +1 -0
- package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts → skillopt-optimization-method-x7TTF23P.d.ts} +20 -7
- package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts.map → skillopt-optimization-method-x7TTF23P.d.ts.map} +1 -1
- package/dist/{statistical-heldout-_woZ9q9j.d.ts → statistical-heldout-Cy3EhjlC.d.ts} +21 -8
- package/dist/statistical-heldout-Cy3EhjlC.d.ts.map +1 -0
- package/dist/{steps-AmkT-GIM.d.ts → steps-CiNVJry_.d.ts} +2 -17
- package/dist/steps-CiNVJry_.d.ts.map +1 -0
- package/dist/{store-CT9YIIve.d.ts → store-B06JdC56.d.ts} +2 -2
- package/dist/{store-CT9YIIve.d.ts.map → store-B06JdC56.d.ts.map} +1 -1
- package/dist/{store-otlp-CDYWW_8N.js → store-otlp-C_Rq5I4D.js} +2 -2
- package/dist/{store-otlp-CDYWW_8N.js.map → store-otlp-C_Rq5I4D.js.map} +1 -1
- package/dist/{store-tool-spans-Br2_IUhm.d.ts → store-tool-spans-DPUG7UUY.d.ts} +6 -6
- package/dist/{store-tool-spans-Br2_IUhm.d.ts.map → store-tool-spans-DPUG7UUY.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CykkbOlv.js → store-tool-spans-Dlh9vkFK.js} +3 -3
- package/dist/{store-tool-spans-CykkbOlv.js.map → store-tool-spans-Dlh9vkFK.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-DW2bEpdB.js → summary-report-BI5hUtvK.js} +6 -16
- package/dist/summary-report-BI5hUtvK.js.map +1 -0
- package/dist/{summary-report-B__Y5ub3.d.ts → summary-report-CC07PhEL.d.ts} +6 -5
- package/dist/summary-report-CC07PhEL.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +3 -16
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +3 -3
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{task-failure-attributes--ZTP3tYO.js → task-failure-attributes-DTl-7-Kw.js} +3 -3
- package/dist/{task-failure-attributes--ZTP3tYO.js.map → task-failure-attributes-DTl-7-Kw.js.map} +1 -1
- package/dist/{tool-groups-B4tqh8jB.d.ts → tool-groups-Ci8i9ErB.d.ts} +3 -3
- package/dist/tool-groups-Ci8i9ErB.d.ts.map +1 -0
- package/dist/{tool-waste-C-VHSRwF.js → tool-waste-BqzmVdJk.js} +2 -2
- package/dist/{tool-waste-C-VHSRwF.js.map → tool-waste-BqzmVdJk.js.map} +1 -1
- package/dist/{tool-waste-DjRDEsuI.d.ts → tool-waste-Dro0gJi3.d.ts} +4 -4
- package/dist/{tool-waste-DjRDEsuI.d.ts.map → tool-waste-Dro0gJi3.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +4 -77
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +5 -15
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +13 -23
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -20
- package/dist/traces.js.map +1 -1
- package/dist/{trajectory-YC15QDYQ.d.ts → trajectory-Bi157Gun.d.ts} +3 -3
- package/dist/{trajectory-YC15QDYQ.d.ts.map → trajectory-Bi157Gun.d.ts.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +5 -5
- package/dist/trajectory-replay/index.js +5 -5
- package/dist/{provenance-oA4-zUqm.d.ts → transient-failure-DKF5Mofa.d.ts} +468 -13
- package/dist/transient-failure-DKF5Mofa.d.ts.map +1 -0
- package/dist/types-B3jzCp0p.js.map +1 -1
- package/dist/{types-CLAwnY-L.d.ts → types-BPb2Kf_C2.d.ts} +3 -3
- package/dist/types-BPb2Kf_C2.d.ts.map +1 -0
- package/dist/types-Bfk0uxRj.d.ts +443 -0
- package/dist/types-Bfk0uxRj.d.ts.map +1 -0
- package/dist/{types-DdFNuyxQ.d.ts → types-D4s7Z6nq.d.ts} +30 -6
- package/dist/types-D4s7Z6nq.d.ts.map +1 -0
- package/dist/{types-B2NsbrNy.d.ts → types-D9ssmxKL.d.ts} +3 -3
- package/dist/{types-B2NsbrNy.d.ts.map → types-D9ssmxKL.d.ts.map} +1 -1
- package/dist/{types-I5WwVzQ7.d.ts → types-DeIUdzNd.d.ts} +2 -2
- package/dist/{types-I5WwVzQ7.d.ts.map → types-DeIUdzNd.d.ts.map} +1 -1
- package/dist/{verdict-BndeTAh_.js → verdict-B0xltqu6.js} +2 -2
- package/dist/{verdict-BndeTAh_.js.map → verdict-B0xltqu6.js.map} +1 -1
- package/dist/verdict-cache-CdVVTVmn.js +88 -0
- package/dist/verdict-cache-CdVVTVmn.js.map +1 -0
- package/dist/wire/index.d.ts +21 -111
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/building-doctrine.md +3 -3
- package/docs/campaign-proposers.md +41 -10
- package/docs/design/statistics-decisions.md +89 -1
- package/docs/eval-surface-map.md +14 -0
- package/docs/experiment.md +19 -2
- package/docs/feedback-trajectories.md +1 -1
- package/docs/multishot-golden-records.md +4 -4
- package/docs/public-api.md +1616 -0
- package/docs/research-report-methodology.md +1 -1
- package/docs/search-history-receipts.md +39 -1
- package/docs/trace-analysis.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/trace-repair-continuation.md +1 -1
- package/docs/verdicts.md +24 -0
- package/package.json +7 -3
- package/dist/active-curriculum-C4mk67HP.js.map +0 -1
- package/dist/agent-profile-cell-BkcRDikH.d.ts.map +0 -1
- package/dist/backend-integrity-DOCa_QrR.d.ts.map +0 -1
- package/dist/benchmark-command-BU1Las59.js.map +0 -1
- package/dist/campaign-la-gEYNz.js.map +0 -1
- package/dist/canonical-D-XsTQ6_.js.map +0 -1
- package/dist/client-LIuo-KPv.js.map +0 -1
- package/dist/define-agent-eval-C8V8sMqP.js.map +0 -1
- package/dist/define-agent-eval-CEQWL9Hy.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-CBYlPvNy.js.map +0 -1
- package/dist/emitter-CPBAhxum.js.map +0 -1
- package/dist/eval-campaign-CQuZrLR_.js.map +0 -1
- package/dist/exec-y-DCLqK7.js.map +0 -1
- package/dist/exporters-q9iL-2Jf.js.map +0 -1
- package/dist/index-Aj3WO3_a.d.ts.map +0 -1
- package/dist/index-B8Ui1mr1.d.ts.map +0 -1
- package/dist/index-BNPtkBPf.d.ts.map +0 -1
- package/dist/index-C1ravkGA.d.ts +0 -1
- package/dist/index-IQccV3Ou.d.ts.map +0 -1
- package/dist/kind-factory-DmAa0h3K.js.map +0 -1
- package/dist/llm-client-Bg32RW0j.js.map +0 -1
- package/dist/llm-judge-BqqMS8t7.js.map +0 -1
- package/dist/matrix-DrVnRp4G.d.ts.map +0 -1
- package/dist/pre-registration-DakwTRXk.js +0 -96
- package/dist/pre-registration-DakwTRXk.js.map +0 -1
- package/dist/pre-registration-zFSLEiFU.d.ts.map +0 -1
- package/dist/produced-state-Bnq4FaDO.js.map +0 -1
- package/dist/provenance-oA4-zUqm.d.ts.map +0 -1
- package/dist/query-CJ_DX8vl.d.ts.map +0 -1
- package/dist/query-Di7eEQ79.js.map +0 -1
- package/dist/release-confidence-4XrqlpFD.d.ts.map +0 -1
- package/dist/release-confidence-BknrpBnO.js.map +0 -1
- package/dist/researcher-DJnoUE8c.d.ts.map +0 -1
- package/dist/rubric-predictive-validity-C7LnNvF2.d.ts.map +0 -1
- package/dist/rubric-predictive-validity-CzxLoZge.js.map +0 -1
- package/dist/schema-C6DW4ZHR.js.map +0 -1
- package/dist/schema-Cef2cFmb.d.ts.map +0 -1
- package/dist/semantic-concept-judge-laMCnTLn.js.map +0 -1
- package/dist/sequential-C458DXNf.js.map +0 -1
- package/dist/sequential-eprocess-CbUt2htw.js.map +0 -1
- package/dist/server-dIWwF3j_.js.map +0 -1
- package/dist/skillopt-optimization-method-CPBlTcj5.js.map +0 -1
- package/dist/statistical-heldout-_woZ9q9j.d.ts.map +0 -1
- package/dist/steps-AmkT-GIM.d.ts.map +0 -1
- package/dist/summary-report-B__Y5ub3.d.ts.map +0 -1
- package/dist/summary-report-DW2bEpdB.js.map +0 -1
- package/dist/tool-groups-B4tqh8jB.d.ts.map +0 -1
- package/dist/types-CLAwnY-L.d.ts.map +0 -1
- package/dist/types-DdFNuyxQ.d.ts.map +0 -1
- package/dist/types-jUBXJ7Iz.d.ts +0 -884
- package/dist/types-jUBXJ7Iz.d.ts.map +0 -1
- package/dist/verdict-cache-mZf5FEiY.js +0 -107
- package/dist/verdict-cache-mZf5FEiY.js.map +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { s as TraceStore } from "./store-
|
|
2
|
-
import { i as TraceEmitter } from "./emitter-
|
|
3
|
-
import { n as TrajectoryStep, t as Trajectory } from "./trajectory-
|
|
1
|
+
import { s as TraceStore } from "./store-B06JdC56.js";
|
|
2
|
+
import { i as TraceEmitter } from "./emitter-D_jYSGRd.js";
|
|
3
|
+
import { n as TrajectoryStep, t as Trajectory } from "./trajectory-Bi157Gun.js";
|
|
4
4
|
//#region src/counterfactual.d.ts
|
|
5
5
|
type CounterfactualMutation = {
|
|
6
6
|
kind: 'swap-model';
|
|
@@ -57,4 +57,4 @@ interface CounterfactualRunner {
|
|
|
57
57
|
declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
|
|
58
58
|
//#endregion
|
|
59
59
|
export { runCounterfactual as a, CounterfactualRunner as i, CounterfactualMutation as n, CounterfactualResult as r, CounterfactualContext as t };
|
|
60
|
-
//# sourceMappingURL=counterfactual
|
|
60
|
+
//# sourceMappingURL=counterfactual-BaFUWK3H.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"counterfactual
|
|
1
|
+
{"version":3,"file":"counterfactual-BaFUWK3H.d.ts","names":[],"sources":["../src/counterfactual.ts"],"mappings":";;;;KAoBY;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;iBAGhD,kBACpB,OAAO,YACP,uBACA,UAAU,wBACV,QAAQ,uBACP,QAAQ"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { o as NotFoundError, s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
|
-
import { t as TraceEmitter } from "./emitter-
|
|
2
|
+
import { t as TraceEmitter } from "./emitter-BpYFQPj4.js";
|
|
3
3
|
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
4
4
|
//#region src/counterfactual.ts
|
|
5
5
|
/**
|
|
@@ -95,4 +95,4 @@ function applyMutation(step, mutation) {
|
|
|
95
95
|
//#endregion
|
|
96
96
|
export { runCounterfactual as t };
|
|
97
97
|
|
|
98
|
-
//# sourceMappingURL=counterfactual-
|
|
98
|
+
//# sourceMappingURL=counterfactual-D_VWavVm.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"counterfactual-
|
|
1
|
+
{"version":3,"file":"counterfactual-D_VWavVm.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n | { kind: 'swap-model'; at: number; newModel: string }\n | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n | { kind: 'truncate-after'; at: number }\n | { kind: 'inject-system-message'; at: number; content: string }\n | {\n kind: 'custom'\n at: number\n describe: string\n apply: (step: TrajectoryStep) => TrajectoryStep\n }\n\nexport interface CounterfactualContext {\n originalRunId: string\n originalTrajectory: Trajectory\n /** Steps up to (but not including) the mutation point — the prefix the\n * replayed agent inherits as its prior conversation/tool history. */\n prefix: TrajectoryStep[]\n mutation: CounterfactualMutation\n /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n * as the FIRST step the replayed agent emits (they decide whether to\n * re-emit it or continue from there). */\n mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n counterfactualRunId: string\n originalRunId: string\n mutation: CounterfactualMutation\n /** Structured delta summary — caller can extend via scoring. */\n delta: {\n originalOutcomeScore: number | null\n counterfactualOutcomeScore: number | null\n deltaScore: number | null\n }\n}\n\nexport interface CounterfactualRunner {\n /**\n * Execute the agent from `ctx.prefix` with the mutation applied.\n * MUST emit spans into the provided emitter so they become part of\n * the counterfactual run. MUST call emitter.endRun() with a verdict.\n */\n executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n store: TraceStore,\n originalRunId: string,\n mutation: CounterfactualMutation,\n runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n const originalRun = await store.getRun(originalRunId)\n if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n const trajectory = await buildTrajectory(store, originalRunId)\n if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n throw new ValidationError(\n `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n )\n }\n const targetStep = trajectory.steps[mutation.at]!\n const mutatedStep = applyMutation(targetStep, mutation)\n\n const cfEmitter = new TraceEmitter(store)\n await cfEmitter.startRun({\n scenarioId: originalRun.scenarioId,\n variantId: originalRun.variantId\n ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n : `cf:${mutation.kind}@${mutation.at}`,\n projectId: originalRun.projectId,\n parentRunId: originalRunId,\n layer: 'meta',\n tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n })\n\n await runner.executeFrom(\n {\n originalRunId,\n originalTrajectory: trajectory,\n prefix: trajectory.steps.slice(0, mutation.at),\n mutation,\n mutatedStep,\n },\n cfEmitter,\n )\n\n const counterfactual = await store.getRun(cfEmitter.runId)\n const delta = {\n originalOutcomeScore: originalRun.outcome?.score ?? null,\n counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n deltaScore:\n originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n ? counterfactual.outcome.score - originalRun.outcome.score\n : null,\n }\n return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return { ...step, span: { ...llm, model: mutation.newModel } }\n }\n if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n const tool = step.span as ToolSpan\n return { ...step, span: { ...tool, result: mutation.newResult } }\n }\n if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n const llm = step.span as LlmSpan\n return {\n ...step,\n span: {\n ...llm,\n messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n },\n }\n }\n if (mutation.kind === 'custom') return mutation.apply(step)\n // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n mutationKind: CounterfactualMutation['kind']\n n: number\n meanAbsDelta: number\n meanSignedDelta: number\n}> {\n const grouped = new Map<string, CounterfactualResult[]>()\n for (const r of results) {\n const arr = grouped.get(r.mutation.kind) ?? []\n arr.push(r)\n grouped.set(r.mutation.kind, arr)\n }\n const out: Array<{\n mutationKind: CounterfactualMutation['kind']\n n: number\n meanAbsDelta: number\n meanSignedDelta: number\n }> = []\n for (const [kind, items] of grouped) {\n const deltas = items\n .map((i) => i.delta.deltaScore)\n .filter((d): d is number => typeof d === 'number')\n if (deltas.length === 0) continue\n const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n out.push({\n mutationKind: kind as CounterfactualMutation['kind'],\n n: deltas.length,\n meanAbsDelta: meanAbs,\n meanSignedDelta: meanSigned,\n })\n }\n return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}
|
|
@@ -49,7 +49,7 @@ interface DatasetProvenance {
|
|
|
49
49
|
interface DatasetManifest {
|
|
50
50
|
name: string;
|
|
51
51
|
provenance: DatasetProvenance;
|
|
52
|
-
/** sha256 hex over
|
|
52
|
+
/** sha256 hex over the RFC 8785 canonical JSON of the id-sorted scenarios. */
|
|
53
53
|
contentHash: string;
|
|
54
54
|
scenarioCount: number;
|
|
55
55
|
splitCounts: Record<DatasetSplit, number>;
|
|
@@ -98,12 +98,13 @@ declare class Dataset {
|
|
|
98
98
|
add(scenario: DatasetScenario): void;
|
|
99
99
|
remove(scenarioId: string): void;
|
|
100
100
|
/**
|
|
101
|
-
* Stable JSON-Lines serialization —
|
|
102
|
-
* Write to disk for
|
|
101
|
+
* Stable JSON-Lines serialization — one RFC 8785 canonical row per scenario,
|
|
102
|
+
* sorted by id, deterministic byte-for-byte. Write to disk for
|
|
103
|
+
* contamination-verifiable archives.
|
|
103
104
|
*/
|
|
104
105
|
toJsonl(): string;
|
|
105
106
|
static fromJsonl(jsonl: string, manifest: Omit<DatasetManifest, 'contentHash' | 'scenarioCount' | 'splitCounts'>): Dataset;
|
|
106
107
|
}
|
|
107
108
|
//#endregion
|
|
108
109
|
export { DatasetSplit as i, DatasetManifest as n, DatasetScenario as r, Dataset as t };
|
|
109
|
-
//# sourceMappingURL=dataset-
|
|
110
|
+
//# sourceMappingURL=dataset-DQqhOCPt.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset-
|
|
1
|
+
{"version":3,"file":"dataset-DQqhOCPt.d.ts","names":[],"sources":["../src/dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;KAiBY;KACA;UAEK;EACf;;EAEA;EACA,QAAQ;EACR,aAAa;;EAEb;;;;;;;;;;EAUA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA,YAAY;;EAEZ;EACA;EACA,aAAa,OAAO;;UAGL;EACf,QAAQ;EACR,aAAa;;EAEb;EACA;;EAEA,UAAU,UAAU;;EAEpB;;cAeW;WACF;WACA,YAAY;UACb;UACA;EAER,YAAY;IACV;IACA,YAAY;IACZ,WAAW;IACX;;;EASF,gBAAgB;MAIZ;;;;;EAQJ,MAAM,UAAS,eAAoB;;;;;EAqB7B,YAAY,QAAQ;;EAgB1B,MAAM,YAAW;IAAU;IAAc;OAA0B;EAWnE;EAIA,IAAI,UAAU;EAQd,OAAO;;;;;;EAYP;SAQO,UACL,eACA,UAAU,KAAK,oEACd"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-
|
|
2
|
-
import { l as ExactCapableAnalyst } from "./exact-types-
|
|
3
|
-
import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-
|
|
4
|
-
import {
|
|
5
|
-
import { t as TraceToolGroupName } from "./tool-groups-
|
|
1
|
+
import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-D9ssmxKL.js";
|
|
2
|
+
import { l as ExactCapableAnalyst } from "./exact-types-qnexxJ1Z.js";
|
|
3
|
+
import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-8You7OK1.js";
|
|
4
|
+
import { c as RawAnalystFinding, i as TraceAnalystLimits, r as TraceAnalysisEngineResult, t as TraceAnalysisEngine } from "./engine-Cu5qD5Fc.js";
|
|
5
|
+
import { t as TraceToolGroupName } from "./tool-groups-Ci8i9ErB.js";
|
|
6
6
|
//#region src/trace-analyst/behavioral-metrics.d.ts
|
|
7
7
|
type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
|
|
8
8
|
interface SuboptimalSignal {
|
|
@@ -92,7 +92,6 @@ declare function runTraceAnalyst(args: {
|
|
|
92
92
|
/** Adapt a research definition to the common Analyst registry contract. */
|
|
93
93
|
declare function createTraceAnalyst(definition: TraceAnalystDefinition, options: CreateTraceAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
|
|
94
94
|
declare function renderPriorFindings(prior: AnalystContext['priorFindings']): string;
|
|
95
|
-
declare function renderUpstreamFindings(upstream: AnalystContext['upstreamFindings']): string;
|
|
96
95
|
//#endregion
|
|
97
96
|
//#region src/analyst/default-registry.d.ts
|
|
98
97
|
interface DefaultAnalystRegistryOptions {
|
|
@@ -110,5 +109,5 @@ interface DefaultAnalystRegistryOptions {
|
|
|
110
109
|
}
|
|
111
110
|
declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
|
|
112
111
|
//#endregion
|
|
113
|
-
export { createTraceAnalyst as a,
|
|
114
|
-
//# sourceMappingURL=default-registry-
|
|
112
|
+
export { createTraceAnalyst as a, BehavioralAnalystOptions as c, TraceAnalystDefinition as i, behavioralAnalyst as l, buildDefaultAnalystRegistry as n, renderPriorFindings as o, CreateTraceAnalystOptions as r, runTraceAnalyst as s, DefaultAnalystRegistryOptions as t, deriveEfficiencyFindings as u };
|
|
113
|
+
//# sourceMappingURL=default-registry-G9CKMNkc.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"default-registry-
|
|
1
|
+
{"version":3,"file":"default-registry-G9CKMNkc.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;;;UCpZ1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
2
|
+
import { An as CampaignCellRetryPolicy, C as RunImprovementLoopResult, E as RunOptimizationOptions, Ln as CampaignStorage, T as PremeasuredOptimizationBaseline, at as OptimizationMethod, f as LoopProvenanceRecord, k as RunEvalOptions, lt as OptimizationMethodProvenance, ut as OptimizationMethodResult, wt as SearchHistoryReceipt } from "./transient-failure-DKF5Mofa.js";
|
|
3
|
+
import { _ as ProposalFinding } from "./types-D9ssmxKL.js";
|
|
4
|
+
import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate } from "./types-D4s7Z6nq.js";
|
|
5
|
+
import { o as InsightReport } from "./insight-report-DRe8LB6d.js";
|
|
6
6
|
import { o as PowerPreflight } from "./pareto-BqNW3LJR.js";
|
|
7
|
-
import { n as HostedTenant } from "./client-
|
|
7
|
+
import { n as HostedTenant } from "./client-L9VVPkim.js";
|
|
8
8
|
//#region src/contract/self-improve.d.ts
|
|
9
9
|
interface SelfImproveBudget {
|
|
10
10
|
/** Hard spend cap across the full run. Each paid call reserves its enforced
|
|
@@ -166,6 +166,13 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
|
166
166
|
/** Per-cell agent dispatch deadline, applied to baseline, candidate, and
|
|
167
167
|
* held-out campaigns. Default 600_000 ms. Set 0 to disable. */
|
|
168
168
|
dispatchTimeoutMs?: number;
|
|
169
|
+
/** Bounded in-run retry of failed cells — same as
|
|
170
|
+
* `RunCampaignOptions.cellRetry`, applied to baseline, candidate, and
|
|
171
|
+
* held-out campaigns. Pair with `transientDispatchFailure()` so a
|
|
172
|
+
* transport hiccup (a router 503, a dropped stream) is re-dispatched in
|
|
173
|
+
* the same slot instead of leaving holdout coverage incomplete. Absent by
|
|
174
|
+
* default: a failed cell is final. */
|
|
175
|
+
cellRetry?: CampaignCellRetryPolicy;
|
|
169
176
|
/** Streaming hook — fires on baseline + each generation + gate decision.
|
|
170
177
|
* Consumer routes events wherever (UI, dashboard, logs). */
|
|
171
178
|
onProgress?: (event: SelfImproveProgressEvent) => void;
|
|
@@ -225,6 +232,16 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
|
225
232
|
* winner-selection and the ship-gate rank on the identical metric and cannot
|
|
226
233
|
* invert. See `RunOptimizationOptions.selectionRankKey`. */
|
|
227
234
|
selectionRankKey?: RunOptimizationOptions<TScenario, TArtifact>['selectionRankKey'];
|
|
235
|
+
/** Override which scored surface each generation MUTATES. Defaults to the
|
|
236
|
+
* global incumbent (hill-climb; the recorded lineage is a chain). Pass
|
|
237
|
+
* `crowdedFrontierParent({ seed })` or a custom policy to draw the parent
|
|
238
|
+
* from the Pareto frontier; promotion still compares against the incumbent.
|
|
239
|
+
* Proposer mode only. See `RunOptimizationOptions.selectParent`. */
|
|
240
|
+
selectParent?: RunOptimizationOptions<TScenario, TArtifact>['selectParent'];
|
|
241
|
+
/** Record this run's candidate search into a durable `SearchLedger` and
|
|
242
|
+
* return the bounded receipt on `searchHistory`. See
|
|
243
|
+
* `RunOptimizationOptions.searchLedger`. */
|
|
244
|
+
searchLedger?: RunOptimizationOptions<TScenario, TArtifact>['searchLedger'];
|
|
228
245
|
}
|
|
229
246
|
interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
|
|
230
247
|
/** Composite mean across all scenarios, baseline run. When
|
|
@@ -275,6 +292,9 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
|
|
|
275
292
|
/** Run-wide receipts across proposal, search, holdout, judging, analysis,
|
|
276
293
|
* and promotion work, with phase and actor attribution. */
|
|
277
294
|
receipts: CostReceipt[];
|
|
295
|
+
/** Bounded proof envelope over this run's canonical search ledger. Present
|
|
296
|
+
* only when `searchLedger` was supplied. */
|
|
297
|
+
searchHistory?: SearchHistoryReceipt;
|
|
278
298
|
/** Exact external method and source identity, when `method` was used. */
|
|
279
299
|
optimization?: {
|
|
280
300
|
name: string;
|
|
@@ -384,4 +404,4 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
|
|
|
384
404
|
declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
|
|
385
405
|
//#endregion
|
|
386
406
|
export { DefinedAgentEval as a, SelfImproveOptions as c, SelfImproveRunError as d, selfImprove as f, DefineAgentEvalOptions as i, SelfImproveProgressEvent as l, AgentEvalEvaluateOptions as n, defineAgentEval as o, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveResult as u };
|
|
387
|
-
//# sourceMappingURL=define-agent-eval-
|
|
407
|
+
//# sourceMappingURL=define-agent-eval-Dx1JnPEa.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"define-agent-eval-Dx1JnPEa.d.ts","names":[],"sources":["../src/contract/self-improve.ts","../src/contract/define-agent-eval.ts"],"mappings":";;;;;;;;UAiEiB;;;EAGf;;;;EAIA;;EAEA;;EAEA;;;EAGA;;;EAGA;;;;EAIA;;EAEA,mBAAmB;;;;;;;;;EASnB;;EAEA;;;;;EAKA;;KAGU;EACN;EAA0B;;EAC1B;EAA4B;EAAuB;;EACnD;EAA4B;EAAe;;EAC3C;EAA8B;EAAe;EAAuB;;EAGpE;EAAsB;EAAkB;;EACxC;EAAyB;EAAW;EAAY;EAAa;;UAElD,mBAAmB,kBAAkB,UAAU;;;;;;;;;;;;;;EAc9D,QAAQ,SAAS,gBAAgB,UAAU,WAAW,KAAK,oBAAoB,QAAQ;;;;;;;EAQvF;;;EAIA,WAAW;;EAGX,OAAO,YAAY,WAAW;;;EAI9B,iBAAiB;;EAGjB,SAAS;;;;;;;;;;;;EAaT,sBAAsB,gCAAgC,WAAW;;;;;EAMjE,WAAW,gBAAgB;;;;;;EAO3B,SAAS,mBAAmB,WAAW;;;EAIvC,qBAAqB;;;EAIrB,OAAO,KAAK,WAAW;;;;;;;;EASvB,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;;;EAKjF,UAAU;;;;EAKV;;;EAIA,gBAAgB,QAAQ;;;;;EAMxB,iBAAiB;IACf,UAAU;IACV;IACA;;;;EAKF;;;;;;;EAQA,YAAY;;;EAIZ,cAAc,OAAO;;;;EAKrB;EACA;EACA;;;;;;;;;;;;;;EAeA,eAAe;;;EAIf,eAAe;;;;EAKf,eAAe;;EAGf;;;;;;;;EASA;;;;;;;;;EAUA,oBAAoB,uBAAuB,WAAW;;;EAItD,WAAW;;;;;;;EAQX,mBAAmB,uBAAuB,WAAW;;;;;;EAOrD,eAAe,uBAAuB,WAAW;;;;EAKjD,eAAe,uBAAuB,WAAW;;UAGlC,kBAAkB,kBAAkB,UAAU;;;;EAI7D;IACE;IACA,aAAa;;;;;EAKf;IACE;IACA,aAAa;IACb,SAAS;;;IAGT;;;;IAIA;;;;;;EAMF;;;EAGA;;;;EAIA,YAAY;;EAEZ;;;EAGA;;EAEA;;EAEA;;EAEA,MAAM;;;EAGN,UAAU;;;EAGV,gBAAgB;;EAEhB;IACE;IACA,MAAM;IACN;IACA,aAAa;;;;;;;;EAQf,SAAS;;;;;EAKT,QAAQ;;;;;;EAMR,KAAK,yBAAyB,WAAW;;;cAI9B,4BAA4B;WAC9B,MAAM;WACN,UAAU;EAEnB,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgMhB,YAAY,kBAAkB,UAAU,WAC5D,MAAM,mBAAmB,WAAW,aACnC,QAAQ,kBAAkB,WAAW;;;KChkB5B,eAAe,kBAAkB,UAAU,cACrD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;KAED,uBAAuB,kBAAkB,UAAU,aAAa,mBAC1E,WACA;UAGe,yBAAyB,kBAAkB,UAAU,mBAC5D,KACN,eAAe,WAAW;;EAI5B,YAAY;;EAEZ,UAAU;;EAEV,QAAQ,eAAe,WAAW;;EAElC,QAAQ,YAAY,WAAW;;EAE/B,SAAS,YAAY,WAAW;;EAEhC;;KAGU,wBAAwB,kBAAkB,UAAU,aAAa,KAC3E,QAAQ,mBAAmB,WAAW;EAGtC,SAAS,QAAQ;EACjB,eAAe,QAAQ;;UAGR,iBAAiB,kBAAkB,UAAU;;WAEnD,oBAAoB;;WAEpB,iBAAiB;;;;;EAK1B,SACE,OAAO,yBAAyB,WAAW,aAC1C,QAAQ,eAAe,WAAW;;;;;;;EAOrC,QACE,OAAO,wBAAwB,WAAW,aACzC,QAAQ,kBAAkB,WAAW;;;;;;;;;iBAU1B,gBAAgB,kBAAkB,UAAU,WAC1D,UAAU,uBAAuB,WAAW,aAC3C,iBAAiB,WAAW"}
|
|
@@ -6,14 +6,15 @@ import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-C
|
|
|
6
6
|
import { r as pairRunRecords } from "./paired-arms-D-XRF_fy.js";
|
|
7
7
|
import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
|
|
8
8
|
import { r as welchsTTest } from "./baseline-BhPRQBVn.js";
|
|
9
|
+
import "./query-CHmMP42p.js";
|
|
9
10
|
import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
|
|
10
|
-
import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-
|
|
11
|
-
import { r as paretoChart } from "./summary-report-
|
|
12
|
-
import {
|
|
13
|
-
import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-
|
|
14
|
-
import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-
|
|
11
|
+
import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-BC0ebuRP.js";
|
|
12
|
+
import { r as paretoChart } from "./summary-report-BI5hUtvK.js";
|
|
13
|
+
import { N as surfaceContentHash, P as surfaceHash, Y as defaultProductionGate, c as emitLoopProvenance, it as resolveRunDir, l as loopProvenanceArgsFromResult, m as runImprovementLoop, tt as runEval, w as assertOptimizationResult } from "./llm-judge-BhasIPFT.js";
|
|
14
|
+
import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-t4lB1yt8.js";
|
|
15
|
+
import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-BIWbHpgD.js";
|
|
15
16
|
import { t as powerPreflight } from "./power-preflight-DEw-uC7q.js";
|
|
16
|
-
import { t as createHostedClient } from "./client-
|
|
17
|
+
import { t as createHostedClient } from "./client-CX7KqIdB.js";
|
|
17
18
|
//#region src/contamination-guard.ts
|
|
18
19
|
function checkCanaries(output, scenarios) {
|
|
19
20
|
const leaks = [];
|
|
@@ -987,7 +988,7 @@ function assertSelfImproveSearchMode(opts) {
|
|
|
987
988
|
const budget = opts.budget;
|
|
988
989
|
if (budget?.generations !== void 0 && budget.generations !== 1) throw new Error("selfImprove: method owns its rounds; budget.generations must be 1 when set");
|
|
989
990
|
if (budget?.populationSize !== void 0 && budget.populationSize !== 1) throw new Error("selfImprove: method owns its candidates; budget.populationSize must be 1 when set");
|
|
990
|
-
if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, and
|
|
991
|
+
if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0 || opts.selectParent !== void 0) throw new Error("selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, findings, and selectParent apply only to proposer mode");
|
|
991
992
|
}
|
|
992
993
|
function splitMethodPartitions(searchScenarios, explicitSelection, fraction) {
|
|
993
994
|
if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) throw new Error("selfImprove: budget.selectionFraction must be in (0, 1)");
|
|
@@ -1147,6 +1148,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
|
|
|
1147
1148
|
maxConcurrency,
|
|
1148
1149
|
reps: budget.reps,
|
|
1149
1150
|
dispatchTimeoutMs: opts.dispatchTimeoutMs,
|
|
1151
|
+
cellRetry: opts.cellRetry,
|
|
1150
1152
|
expectUsage,
|
|
1151
1153
|
costCeiling: budget.dollars
|
|
1152
1154
|
}),
|
|
@@ -1196,13 +1198,16 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
|
|
|
1196
1198
|
maxConcurrency,
|
|
1197
1199
|
cellPlacement: opts.cellPlacement,
|
|
1198
1200
|
dispatchTimeoutMs: opts.dispatchTimeoutMs,
|
|
1201
|
+
cellRetry: opts.cellRetry,
|
|
1199
1202
|
costLedger,
|
|
1200
1203
|
expectUsage,
|
|
1201
1204
|
labeledStore: opts.labeledStore,
|
|
1202
1205
|
captureSource: opts.captureSource,
|
|
1203
1206
|
analyzeGeneration: opts.analyzeGeneration,
|
|
1204
1207
|
findings: opts.findings,
|
|
1205
|
-
selectionRankKey: opts.selectionRankKey
|
|
1208
|
+
selectionRankKey: opts.selectionRankKey,
|
|
1209
|
+
selectParent: opts.selectParent,
|
|
1210
|
+
searchLedger: opts.searchLedger
|
|
1206
1211
|
});
|
|
1207
1212
|
const reportSplit = holdoutDeferred ? "search" : "holdout";
|
|
1208
1213
|
const reportBaselineCampaign = holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout;
|
|
@@ -1292,6 +1297,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
|
|
|
1292
1297
|
...optimizationResult.durationMs === void 0 ? {} : { durationMs: optimizationResult.durationMs },
|
|
1293
1298
|
...optimizationResult.provenance === void 0 ? {} : { provenance: structuredClone(optimizationResult.provenance) }
|
|
1294
1299
|
} } : {},
|
|
1300
|
+
...result.searchHistory ? { searchHistory: result.searchHistory } : {},
|
|
1295
1301
|
insight,
|
|
1296
1302
|
...power ? { power } : {},
|
|
1297
1303
|
raw: result
|
|
@@ -1491,4 +1497,4 @@ function requirePositiveInteger(value, field) {
|
|
|
1491
1497
|
//#endregion
|
|
1492
1498
|
export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
|
|
1493
1499
|
|
|
1494
|
-
//# sourceMappingURL=define-agent-eval-
|
|
1500
|
+
//# sourceMappingURL=define-agent-eval-h-s-sI-v.js.map
|