@tangle-network/agent-eval 0.150.2 → 0.161.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +149 -1
- package/README.md +7 -3
- package/dist/{active-curriculum-C4mk67HP.js → active-curriculum-CD5TU2yW.js} +3 -13
- package/dist/active-curriculum-CD5TU2yW.js.map +1 -0
- package/dist/{agent-profile-cell-BkcRDikH.d.ts → agent-profile-cell-CTOZJUuE.d.ts} +4 -2
- package/dist/agent-profile-cell-CTOZJUuE.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +19 -36
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +8 -8
- package/dist/analyst/index.js.map +1 -1
- package/dist/{backend-integrity-DOCa_QrR.d.ts → backend-integrity-DxuQCu_A.d.ts} +4 -3
- package/dist/backend-integrity-DxuQCu_A.d.ts.map +1 -0
- package/dist/{benchmark-BtAWA8nT.d.ts → benchmark-CGPp-kDC.d.ts} +3 -3
- package/dist/{benchmark-BtAWA8nT.d.ts.map → benchmark-CGPp-kDC.d.ts.map} +1 -1
- package/dist/{benchmark-command-BU1Las59.js → benchmark-command-BVtaq_ve.js} +26 -31
- package/dist/benchmark-command-BVtaq_ve.js.map +1 -0
- package/dist/benchmarks/index.d.ts +6 -19
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +9 -9
- package/dist/campaign/index.js +7 -7
- package/dist/{campaign-la-gEYNz.js → campaign-BSmOwskD.js} +77 -795
- package/dist/campaign-BSmOwskD.js.map +1 -0
- package/dist/{canonical-D-XsTQ6_.js → canonical-IL-Bu-14.js} +26 -2
- package/dist/canonical-IL-Bu-14.js.map +1 -0
- package/dist/{capture-fetch-BBVFzhkk.d.ts → capture-fetch-CqwsJkkG.d.ts} +3 -3
- package/dist/{capture-fetch-BBVFzhkk.d.ts.map → capture-fetch-CqwsJkkG.d.ts.map} +1 -1
- package/dist/{chat-client-Bvmxedyv.js → chat-client-DlMlAeYI.js} +5 -55
- package/dist/{chat-client-Bvmxedyv.js.map → chat-client-DlMlAeYI.js.map} +1 -1
- package/dist/chat-json-call-6g5sJobJ.js +53 -0
- package/dist/chat-json-call-6g5sJobJ.js.map +1 -0
- package/dist/cli.js +54 -19
- package/dist/cli.js.map +1 -1
- package/dist/{client-LIuo-KPv.js → client-CX7KqIdB.js} +3 -3
- package/dist/client-CX7KqIdB.js.map +1 -0
- package/dist/{client-kPQYT_56.d.ts → client-L9VVPkim.d.ts} +4 -4
- package/dist/{client-kPQYT_56.d.ts.map → client-L9VVPkim.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +13 -27
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +14 -17
- package/dist/contract/index.js.map +1 -1
- package/dist/{counterfactual--bpysZF0.d.ts → counterfactual-BaFUWK3H.d.ts} +4 -4
- package/dist/{counterfactual--bpysZF0.d.ts.map → counterfactual-BaFUWK3H.d.ts.map} +1 -1
- package/dist/{counterfactual-lDfCx0Uz.js → counterfactual-D_VWavVm.js} +2 -2
- package/dist/{counterfactual-lDfCx0Uz.js.map → counterfactual-D_VWavVm.js.map} +1 -1
- package/dist/{dataset-CJjKqQfA.d.ts → dataset-DQqhOCPt.d.ts} +5 -4
- package/dist/{dataset-CJjKqQfA.d.ts.map → dataset-DQqhOCPt.d.ts.map} +1 -1
- package/dist/{default-registry-Cw0Ohdoj.d.ts → default-registry-G9CKMNkc.d.ts} +7 -8
- package/dist/{default-registry-Cw0Ohdoj.d.ts.map → default-registry-G9CKMNkc.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CEQWL9Hy.d.ts → define-agent-eval-Dx1JnPEa.d.ts} +26 -6
- package/dist/define-agent-eval-Dx1JnPEa.d.ts.map +1 -0
- package/dist/{define-agent-eval-C8V8sMqP.js → define-agent-eval-h-s-sI-v.js} +15 -9
- package/dist/define-agent-eval-h-s-sI-v.js.map +1 -0
- package/dist/{dspy-rlm-engine-CBYlPvNy.js → dspy-rlm-engine-DptEII26.js} +95 -15
- package/dist/dspy-rlm-engine-DptEII26.js.map +1 -0
- package/dist/{emitter-CPBAhxum.js → emitter-BpYFQPj4.js} +2 -18
- package/dist/emitter-BpYFQPj4.js.map +1 -0
- package/dist/{emitter-DGQGoLyj.d.ts → emitter-D_jYSGRd.d.ts} +4 -20
- package/dist/{emitter-DGQGoLyj.d.ts.map → emitter-D_jYSGRd.d.ts.map} +1 -1
- package/dist/{engine-BLzhNzoY.d.ts → engine-Cu5qD5Fc.d.ts} +9 -11
- package/dist/{engine-BLzhNzoY.d.ts.map → engine-Cu5qD5Fc.d.ts.map} +1 -1
- package/dist/{eval-campaign-CQuZrLR_.js → eval-campaign-BsXWL2-2.js} +17 -28
- package/dist/eval-campaign-BsXWL2-2.js.map +1 -0
- package/dist/{exact-types-ccQAyut1.d.ts → exact-types-qnexxJ1Z.d.ts} +2 -2
- package/dist/{exact-types-ccQAyut1.d.ts.map → exact-types-qnexxJ1Z.d.ts.map} +1 -1
- package/dist/{exec-y-DCLqK7.js → exec-D9WpA2p-.js} +2 -2
- package/dist/exec-D9WpA2p-.js.map +1 -0
- package/dist/experiment/index.d.ts +45 -11
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +30 -11
- package/dist/experiment/index.js.map +1 -1
- package/dist/{experiment-tracker-0MhuPArU.d.ts → experiment-tracker-DCO6Cz4s.d.ts} +2 -2
- package/dist/{experiment-tracker-0MhuPArU.d.ts.map → experiment-tracker-DCO6Cz4s.d.ts.map} +1 -1
- package/dist/{exporters-q9iL-2Jf.js → exporters-Df7TgHFv.js} +3 -3
- package/dist/exporters-Df7TgHFv.js.map +1 -0
- package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts → external-optimizer-contracts-szBJ_1vh.d.ts} +2 -2
- package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts.map → external-optimizer-contracts-szBJ_1vh.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-x9oEXKsU.js → external-optimizer-process-WosTBChy.js} +4 -4
- package/dist/{external-optimizer-process-x9oEXKsU.js.map → external-optimizer-process-WosTBChy.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-CKNb42oM.js → external-optimizer-subprocess-BIWbHpgD.js} +4 -4
- package/dist/{external-optimizer-subprocess-CKNb42oM.js.map → external-optimizer-subprocess-BIWbHpgD.js.map} +1 -1
- package/dist/{failure-cluster-BLURuWG4.d.ts → failure-cluster-CXL8NbEw.d.ts} +3 -3
- package/dist/{failure-cluster-BLURuWG4.d.ts.map → failure-cluster-CXL8NbEw.d.ts.map} +1 -1
- package/dist/{feedback-trajectory-DpTTjo0q.d.ts → feedback-trajectory-B3ZHaHV_.d.ts} +7 -7
- package/dist/{feedback-trajectory-DpTTjo0q.d.ts.map → feedback-trajectory-B3ZHaHV_.d.ts.map} +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-XggBupCr.js → hf-dataset-D8_RNIis.js} +4 -4
- package/dist/{hf-dataset-XggBupCr.js.map → hf-dataset-D8_RNIis.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -14
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +2 -2
- package/dist/{index-Aj3WO3_a.d.ts → index-CGtH1piv.d.ts} +48 -26
- package/dist/index-CGtH1piv.d.ts.map +1 -0
- package/dist/{index-BNPtkBPf.d.ts → index-D-IiQIBB.d.ts} +5 -10
- package/dist/index-D-IiQIBB.d.ts.map +1 -0
- package/dist/{index-IQccV3Ou.d.ts → index-D-V8gCs_.d.ts} +13 -90
- package/dist/index-D-V8gCs_.d.ts.map +1 -0
- package/dist/{index-B8Ui1mr1.d.ts → index-lfaSeKSD.d.ts} +18 -2
- package/dist/index-lfaSeKSD.d.ts.map +1 -0
- package/dist/index-vrJugRal.d.ts +1 -0
- package/dist/index.d.ts +68 -56
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +59 -110
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-BeT8KCgI.d.ts → insight-report-DRe8LB6d.d.ts} +4 -4
- package/dist/{insight-report-BeT8KCgI.d.ts.map → insight-report-DRe8LB6d.d.ts.map} +1 -1
- package/dist/{integrity-DL91tucI.js → integrity-CyWSSoQS.js} +2 -2
- package/dist/{integrity-DL91tucI.js.map → integrity-CyWSSoQS.js.map} +1 -1
- package/dist/{integrity-B0dZ96EO.d.ts → integrity-DUNX9Fao.d.ts} +3 -3
- package/dist/{integrity-B0dZ96EO.d.ts.map → integrity-DUNX9Fao.d.ts.map} +1 -1
- package/dist/{kind-factory-DmAa0h3K.js → kind-factory-DY8FdoXf.js} +3 -71
- package/dist/kind-factory-DY8FdoXf.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-DTae9rv_.js → ledger-core-BOzlRygb.js} +2 -2
- package/dist/{ledger-core-DTae9rv_.js.map → ledger-core-BOzlRygb.js.map} +1 -1
- package/dist/{llm-client-Bg32RW0j.js → llm-client-hgDieDNN.js} +53 -98
- package/dist/llm-client-hgDieDNN.js.map +1 -0
- package/dist/{llm-judge-BqqMS8t7.js → llm-judge-BhasIPFT.js} +1135 -77
- package/dist/llm-judge-BhasIPFT.js.map +1 -0
- package/dist/{matrix-DrVnRp4G.d.ts → matrix-eXKRMHnL.d.ts} +74 -72
- package/dist/matrix-eXKRMHnL.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +8 -6
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +8 -6
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{mint-BV6tLVWl.js → mint-DfODW1KW.js} +3 -3
- package/dist/{mint-BV6tLVWl.js.map → mint-DfODW1KW.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +2 -8
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +56 -86
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +11 -46
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +30 -83
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-DJWAXLms.js → opencode-sqlite-eK6HW6dr.js} +2 -6
- package/dist/{opencode-sqlite-DJWAXLms.js.map → opencode-sqlite-eK6HW6dr.js.map} +1 -1
- package/dist/pipelines/index.d.ts +5 -5
- package/dist/pipelines/index.js +3 -3
- package/dist/{pre-registration-zFSLEiFU.d.ts → pre-registration-CzFCcwYk.d.ts} +55 -40
- package/dist/pre-registration-CzFCcwYk.d.ts.map +1 -0
- package/dist/pre-registration-KN9jkh58.js +110 -0
- package/dist/pre-registration-KN9jkh58.js.map +1 -0
- package/dist/{produced-state-Bnq4FaDO.js → produced-state-DZ89riy5.js} +8 -8
- package/dist/produced-state-DZ89riy5.js.map +1 -0
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +31 -5
- package/dist/profile-cell.js.map +1 -1
- package/dist/{promotion-policy-DLOUkYhI.d.ts → promotion-policy-DtnOIZvk.d.ts} +2 -2
- package/dist/{promotion-policy-DLOUkYhI.d.ts.map → promotion-policy-DtnOIZvk.d.ts.map} +1 -1
- package/dist/{query-Di7eEQ79.js → query-CHmMP42p.js} +20 -11
- package/dist/query-CHmMP42p.js.map +1 -0
- package/dist/{query-CJ_DX8vl.d.ts → query-DxPYqpmT.d.ts} +10 -4
- package/dist/query-DxPYqpmT.d.ts.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/{registry-BQwrSYpC.d.ts → registry-8You7OK1.d.ts} +5 -7
- package/dist/{registry-BQwrSYpC.d.ts.map → registry-8You7OK1.d.ts.map} +1 -1
- package/dist/{release-confidence-BknrpBnO.js → release-confidence-DKfD2RYU.js} +28 -14
- package/dist/release-confidence-DKfD2RYU.js.map +1 -0
- package/dist/{release-confidence-4XrqlpFD.d.ts → release-confidence-Dqt0NFep.d.ts} +7 -6
- package/dist/release-confidence-Dqt0NFep.d.ts.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +3 -3
- package/dist/{researcher-DJnoUE8c.d.ts → researcher-Cz565b7D.d.ts} +34 -21
- package/dist/researcher-Cz565b7D.d.ts.map +1 -0
- package/dist/{reward-hacking-62tojkQd.d.ts → reward-hacking-MBf7qpSB.d.ts} +2 -2
- package/dist/{reward-hacking-62tojkQd.d.ts.map → reward-hacking-MBf7qpSB.d.ts.map} +1 -1
- package/dist/{reward-hacking-C0x0xihA.js → reward-hacking-t4lB1yt8.js} +2 -2
- package/dist/{reward-hacking-C0x0xihA.js.map → reward-hacking-t4lB1yt8.js.map} +1 -1
- package/dist/rl.d.ts +11 -42
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +41 -24
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +3 -3
- package/dist/rollout/index.js +7 -7
- package/dist/{rollout-ytVQ7WT8.js → rollout-Dm2tSdiQ.js} +6 -6
- package/dist/{rollout-ytVQ7WT8.js.map → rollout-Dm2tSdiQ.js.map} +1 -1
- package/dist/{rubric-predictive-validity-CzxLoZge.js → rubric-predictive-validity-CK8SCOg-.js} +5 -15
- package/dist/rubric-predictive-validity-CK8SCOg-.js.map +1 -0
- package/dist/{rubric-predictive-validity-C7LnNvF2.d.ts → rubric-predictive-validity-CxycqzX5.d.ts} +4 -3
- package/dist/rubric-predictive-validity-CxycqzX5.d.ts.map +1 -0
- package/dist/{run-record-D2lDdSAz.js → run-record-BC0ebuRP.js} +2 -2
- package/dist/{run-record-D2lDdSAz.js.map → run-record-BC0ebuRP.js.map} +1 -1
- package/dist/{run-record-DVV82Gwh.d.ts → run-record-VVy4T9OW.d.ts} +3 -3
- package/dist/{run-record-DVV82Gwh.d.ts.map → run-record-VVy4T9OW.d.ts.map} +1 -1
- package/dist/{schema-BtVldJ3T.d.ts → schema-Bjgdsn73.d.ts} +2 -4
- package/dist/{schema-BtVldJ3T.d.ts.map → schema-Bjgdsn73.d.ts.map} +1 -1
- package/dist/{schema-Cef2cFmb.d.ts → schema-BzWDXhOR.d.ts} +2 -5
- package/dist/schema-BzWDXhOR.d.ts.map +1 -0
- package/dist/{schema-C6DW4ZHR.js → schema-C1aaAxTf.js} +2 -2
- package/dist/schema-C1aaAxTf.js.map +1 -0
- package/dist/{schema-CRhEY1SO.js → schema-k6ZBftVv.js} +2 -8
- package/dist/{schema-CRhEY1SO.js.map → schema-k6ZBftVv.js.map} +1 -1
- package/dist/{semantic-concept-judge-laMCnTLn.js → semantic-concept-judge-BSkKKHeq.js} +14 -38
- package/dist/semantic-concept-judge-BSkKKHeq.js.map +1 -0
- package/dist/{sequential-eprocess-CbUt2htw.js → sequential-eprocess-D1jKoihe.js} +49 -2
- package/dist/sequential-eprocess-D1jKoihe.js.map +1 -0
- package/dist/{sequential-C458DXNf.js → sequential-rYW-Ophm.js} +41 -16
- package/dist/sequential-rYW-Ophm.js.map +1 -0
- package/dist/{series-convergence-BxKEgBwA.d.ts → series-convergence-D9WgpXGi.d.ts} +2 -2
- package/dist/{series-convergence-BxKEgBwA.d.ts.map → series-convergence-D9WgpXGi.d.ts.map} +1 -1
- package/dist/{server-dIWwF3j_.js → server-BtFd4uzB.js} +19 -42
- package/dist/server-BtFd4uzB.js.map +1 -0
- package/dist/{skillopt-optimization-method-CPBlTcj5.js → skillopt-optimization-method-DbaekMcn.js} +794 -8
- package/dist/skillopt-optimization-method-DbaekMcn.js.map +1 -0
- package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts → skillopt-optimization-method-x7TTF23P.d.ts} +20 -7
- package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts.map → skillopt-optimization-method-x7TTF23P.d.ts.map} +1 -1
- package/dist/{statistical-heldout-_woZ9q9j.d.ts → statistical-heldout-Cy3EhjlC.d.ts} +21 -8
- package/dist/statistical-heldout-Cy3EhjlC.d.ts.map +1 -0
- package/dist/{steps-AmkT-GIM.d.ts → steps-CiNVJry_.d.ts} +2 -17
- package/dist/steps-CiNVJry_.d.ts.map +1 -0
- package/dist/{store-CT9YIIve.d.ts → store-B06JdC56.d.ts} +2 -2
- package/dist/{store-CT9YIIve.d.ts.map → store-B06JdC56.d.ts.map} +1 -1
- package/dist/{store-otlp-CDYWW_8N.js → store-otlp-C_Rq5I4D.js} +2 -2
- package/dist/{store-otlp-CDYWW_8N.js.map → store-otlp-C_Rq5I4D.js.map} +1 -1
- package/dist/{store-tool-spans-Br2_IUhm.d.ts → store-tool-spans-DPUG7UUY.d.ts} +6 -6
- package/dist/{store-tool-spans-Br2_IUhm.d.ts.map → store-tool-spans-DPUG7UUY.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CykkbOlv.js → store-tool-spans-Dlh9vkFK.js} +3 -3
- package/dist/{store-tool-spans-CykkbOlv.js.map → store-tool-spans-Dlh9vkFK.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-DW2bEpdB.js → summary-report-BI5hUtvK.js} +6 -16
- package/dist/summary-report-BI5hUtvK.js.map +1 -0
- package/dist/{summary-report-B__Y5ub3.d.ts → summary-report-CC07PhEL.d.ts} +6 -5
- package/dist/summary-report-CC07PhEL.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +3 -16
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +3 -3
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{task-failure-attributes--ZTP3tYO.js → task-failure-attributes-DTl-7-Kw.js} +3 -3
- package/dist/{task-failure-attributes--ZTP3tYO.js.map → task-failure-attributes-DTl-7-Kw.js.map} +1 -1
- package/dist/{tool-groups-B4tqh8jB.d.ts → tool-groups-Ci8i9ErB.d.ts} +3 -3
- package/dist/tool-groups-Ci8i9ErB.d.ts.map +1 -0
- package/dist/{tool-waste-C-VHSRwF.js → tool-waste-BqzmVdJk.js} +2 -2
- package/dist/{tool-waste-C-VHSRwF.js.map → tool-waste-BqzmVdJk.js.map} +1 -1
- package/dist/{tool-waste-DjRDEsuI.d.ts → tool-waste-Dro0gJi3.d.ts} +4 -4
- package/dist/{tool-waste-DjRDEsuI.d.ts.map → tool-waste-Dro0gJi3.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +4 -77
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +5 -15
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +13 -23
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -20
- package/dist/traces.js.map +1 -1
- package/dist/{trajectory-YC15QDYQ.d.ts → trajectory-Bi157Gun.d.ts} +3 -3
- package/dist/{trajectory-YC15QDYQ.d.ts.map → trajectory-Bi157Gun.d.ts.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +5 -5
- package/dist/trajectory-replay/index.js +5 -5
- package/dist/{provenance-oA4-zUqm.d.ts → transient-failure-DKF5Mofa.d.ts} +468 -13
- package/dist/transient-failure-DKF5Mofa.d.ts.map +1 -0
- package/dist/types-B3jzCp0p.js.map +1 -1
- package/dist/{types-CLAwnY-L.d.ts → types-BPb2Kf_C2.d.ts} +3 -3
- package/dist/types-BPb2Kf_C2.d.ts.map +1 -0
- package/dist/types-Bfk0uxRj.d.ts +443 -0
- package/dist/types-Bfk0uxRj.d.ts.map +1 -0
- package/dist/{types-DdFNuyxQ.d.ts → types-D4s7Z6nq.d.ts} +30 -6
- package/dist/types-D4s7Z6nq.d.ts.map +1 -0
- package/dist/{types-B2NsbrNy.d.ts → types-D9ssmxKL.d.ts} +3 -3
- package/dist/{types-B2NsbrNy.d.ts.map → types-D9ssmxKL.d.ts.map} +1 -1
- package/dist/{types-I5WwVzQ7.d.ts → types-DeIUdzNd.d.ts} +2 -2
- package/dist/{types-I5WwVzQ7.d.ts.map → types-DeIUdzNd.d.ts.map} +1 -1
- package/dist/{verdict-BndeTAh_.js → verdict-B0xltqu6.js} +2 -2
- package/dist/{verdict-BndeTAh_.js.map → verdict-B0xltqu6.js.map} +1 -1
- package/dist/verdict-cache-CdVVTVmn.js +88 -0
- package/dist/verdict-cache-CdVVTVmn.js.map +1 -0
- package/dist/wire/index.d.ts +21 -111
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/building-doctrine.md +3 -3
- package/docs/campaign-proposers.md +41 -10
- package/docs/design/statistics-decisions.md +89 -1
- package/docs/eval-surface-map.md +14 -0
- package/docs/experiment.md +19 -2
- package/docs/feedback-trajectories.md +1 -1
- package/docs/multishot-golden-records.md +4 -4
- package/docs/public-api.md +1616 -0
- package/docs/research-report-methodology.md +1 -1
- package/docs/search-history-receipts.md +39 -1
- package/docs/trace-analysis.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/trace-repair-continuation.md +1 -1
- package/docs/verdicts.md +24 -0
- package/package.json +6 -2
- package/dist/active-curriculum-C4mk67HP.js.map +0 -1
- package/dist/agent-profile-cell-BkcRDikH.d.ts.map +0 -1
- package/dist/backend-integrity-DOCa_QrR.d.ts.map +0 -1
- package/dist/benchmark-command-BU1Las59.js.map +0 -1
- package/dist/campaign-la-gEYNz.js.map +0 -1
- package/dist/canonical-D-XsTQ6_.js.map +0 -1
- package/dist/client-LIuo-KPv.js.map +0 -1
- package/dist/define-agent-eval-C8V8sMqP.js.map +0 -1
- package/dist/define-agent-eval-CEQWL9Hy.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-CBYlPvNy.js.map +0 -1
- package/dist/emitter-CPBAhxum.js.map +0 -1
- package/dist/eval-campaign-CQuZrLR_.js.map +0 -1
- package/dist/exec-y-DCLqK7.js.map +0 -1
- package/dist/exporters-q9iL-2Jf.js.map +0 -1
- package/dist/index-Aj3WO3_a.d.ts.map +0 -1
- package/dist/index-B8Ui1mr1.d.ts.map +0 -1
- package/dist/index-BNPtkBPf.d.ts.map +0 -1
- package/dist/index-C1ravkGA.d.ts +0 -1
- package/dist/index-IQccV3Ou.d.ts.map +0 -1
- package/dist/kind-factory-DmAa0h3K.js.map +0 -1
- package/dist/llm-client-Bg32RW0j.js.map +0 -1
- package/dist/llm-judge-BqqMS8t7.js.map +0 -1
- package/dist/matrix-DrVnRp4G.d.ts.map +0 -1
- package/dist/pre-registration-DakwTRXk.js +0 -96
- package/dist/pre-registration-DakwTRXk.js.map +0 -1
- package/dist/pre-registration-zFSLEiFU.d.ts.map +0 -1
- package/dist/produced-state-Bnq4FaDO.js.map +0 -1
- package/dist/provenance-oA4-zUqm.d.ts.map +0 -1
- package/dist/query-CJ_DX8vl.d.ts.map +0 -1
- package/dist/query-Di7eEQ79.js.map +0 -1
- package/dist/release-confidence-4XrqlpFD.d.ts.map +0 -1
- package/dist/release-confidence-BknrpBnO.js.map +0 -1
- package/dist/researcher-DJnoUE8c.d.ts.map +0 -1
- package/dist/rubric-predictive-validity-C7LnNvF2.d.ts.map +0 -1
- package/dist/rubric-predictive-validity-CzxLoZge.js.map +0 -1
- package/dist/schema-C6DW4ZHR.js.map +0 -1
- package/dist/schema-Cef2cFmb.d.ts.map +0 -1
- package/dist/semantic-concept-judge-laMCnTLn.js.map +0 -1
- package/dist/sequential-C458DXNf.js.map +0 -1
- package/dist/sequential-eprocess-CbUt2htw.js.map +0 -1
- package/dist/server-dIWwF3j_.js.map +0 -1
- package/dist/skillopt-optimization-method-CPBlTcj5.js.map +0 -1
- package/dist/statistical-heldout-_woZ9q9j.d.ts.map +0 -1
- package/dist/steps-AmkT-GIM.d.ts.map +0 -1
- package/dist/summary-report-B__Y5ub3.d.ts.map +0 -1
- package/dist/summary-report-DW2bEpdB.js.map +0 -1
- package/dist/tool-groups-B4tqh8jB.d.ts.map +0 -1
- package/dist/types-CLAwnY-L.d.ts.map +0 -1
- package/dist/types-DdFNuyxQ.d.ts.map +0 -1
- package/dist/types-jUBXJ7Iz.d.ts +0 -884
- package/dist/types-jUBXJ7Iz.d.ts.map +0 -1
- package/dist/verdict-cache-mZf5FEiY.js +0 -107
- package/dist/verdict-cache-mZf5FEiY.js.map +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-
|
|
3
|
-
import { a as JudgeInput, p as ChatClient } from "./types-
|
|
2
|
+
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-VVy4T9OW.js";
|
|
3
|
+
import { a as JudgeInput, p as ChatClient } from "./types-Bfk0uxRj.js";
|
|
4
4
|
import { RE2JS } from "re2js";
|
|
5
5
|
//#region src/trace-analyst/types.d.ts
|
|
6
6
|
/**
|
|
@@ -594,4 +594,4 @@ type AnalystRunEvent = {
|
|
|
594
594
|
};
|
|
595
595
|
//#endregion
|
|
596
596
|
export { SearchSpanResult as A, ViewSpansResult as B, TRACE_ANALYSIS_LIMITS as C, DatasetOverview as D, DEFAULT_TRACE_ANALYST_BUDGETS as E, TraceAnalystFilters as F, ViewTraceResult as H, TraceAnalystSpan as I, TraceAnalystSpanKind as L, SpanMatchRecord as M, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as N, ErrorCluster as O, TraceAnalystByteBudgets as P, TraceAnalystSpanStatus as R, BoundedTraceAnalysisStoreOptions as S, TraceAnalysisStoreContext as T, ViewTraceOversized as V, ProposalFinding as _, AnalystInputKind as a, makeFinding as b, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, ExecutionProbeRequest as g, ExecutionProbeOutcome as h, AnalystFinding as i, SearchTraceResult as j, QueryTracesPage as k, AnalystRunResult as l, ExecutionProbe as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, ProposalFindingOrigin as v, TraceAnalysisStore as w, makeProposalFinding as x, computeFindingId as y, TraceAnalystTraceSummary as z };
|
|
597
|
-
//# sourceMappingURL=types-
|
|
597
|
+
//# sourceMappingURL=types-D9ssmxKL.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-D9ssmxKL.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;KAgBY;KAUA;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cAS/B;;;cC7MA;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;;;;;;;UAUM;EACf,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;;;;UC/DH;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;KAGD;;KAGA;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { m as RolloutLine } from "./schema-BzWDXhOR.js";
|
|
2
2
|
//#region src/statistics/descriptive.d.ts
|
|
3
3
|
/**
|
|
4
4
|
* Descriptive statistics: means, bootstrap spread, correlation, and the
|
|
@@ -519,4 +519,4 @@ interface SupervisorRunTreeGap {
|
|
|
519
519
|
}
|
|
520
520
|
//#endregion
|
|
521
521
|
export { unavailable as A, weightedComposite as B, SupervisorRunTreeGap as C, WorkerLogSource as D, WallDistribution as E, partialCredit as F, pearsonR as I, ranks as L, WeightedCompositeInput as M, WeightedCompositeResult as N, isUnavailable as O, confidenceInterval as P, spearmanR as R, SupervisorRunTree as S, Unavailable as T, weightedMean as V, SupervisorRunNodeRole as _, OrchestrationMetrics as a, SupervisorRunRollup as b, PerWorkerRow as c, SUPERVISOR_RUN_ROLLUP_SCHEMA as d, SUPERVISOR_RUN_SCHEMA as f, SteerBreakdown as g, SpendMeasurements as h, NO_SOURCE_LIMITS as i, SeriesDistribution as j, showMeasured as k, RoleSpend as l, SpendMeasurement as m, EconomicsMetrics as n, OutcomeMetrics as o, SourceLimits as p, Measured as r, PatchStats as s, DecisionMetrics as t, RollupCellRow as u, SupervisorRunReader as v, SupervisorRunTreeGapCode as w, SupervisorRunSources as x, SupervisorRunReport as y, summarizeNumberSeries as z };
|
|
522
|
-
//# sourceMappingURL=types-
|
|
522
|
+
//# sourceMappingURL=types-DeIUdzNd.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-DeIUdzNd.d.ts","names":[],"sources":["../src/statistics/descriptive.ts","../src/supervisor-run/types.ts"],"mappings":";;;;;;;iBAQgB,aAAa;EAAU;EAAe;;;;;;;;;;iBAoBtC,mBACd,kBACA,qBACA;EAAQ;EAAe;;EACpB;EAAc;EAAe;;;iBAiClB,cAAc,iBAAiB;;UAM9B;WACN;WACA;WACA;WACA;WACA;WACA;;;;;;;;;iBAUK,sBAAsB,4BAA4B;;;;;iBA8BlD,MAAM;;;;;;iBAmBN,SAAS,aAAa;;;;;iBAuBtB,UAAU,aAAa;UAKtB;;EAEf,MAAM;;;;;EAKN,SAAS;;EAET;;UAGe;EACf;EACA;;;;;;;;;;;iBAYc,kBAAkB,OAAO,yBAAyB;;;;UC5JjD;WACN;;;KAIC,SAAS,KAAK,IAAI;iBAEd,YAAY,iBAAiB;iBAI7B,cAAc,aAAa,KAAK;;iBAKhC,aAAa,GAAG;;KAWpB;;;;;UAMK;;;;;WAKN;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA;WACA;WACA;;;;;;;;;;;;;UAcM;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;;cAIE,kBAAkB;;;;;;;;;;UAiBd;;WAEN;WACA;;WAEA;;WAEA;;;;;;WAMA;;;;;;WAMA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA,kBAAkB;;WAElB;;WAEA;;;;;;;WAOA;;WAEA;;WAEA;;WAEA;;;;;WAKA;IACP;IACA;IACA;IACA;;IAEA;IACA;;WAEO;;WAEA,QAAQ;;;;;;WAMR;;;;;WAKA;;;;;;UAOM;;WAEN;EACT,QAAQ,QAAQ;;cAOL;cACA;UAEI;;WAEN;WACA;;WAEA;;WAEA;;UAGM;WACN,gBAAgB;WAChB,gBAAgB;WAChB,kBAAkB;;WAElB,QAAQ;WACR,iBAAiB;WACjB,gBAAgB,kBAAkB;;WAElC,kBAAkB;;;;;;WAMlB,OAAO;WACP,WAAW;WACX,gBAAgB;;WAEhB,UAAU;;WAEV,gBAAgB;;WAEhB,iBAAiB;WACjB,oBAAoB;WACpB,kBAAkB;;;;;;;;;WASlB,sBAAsB;;WAEtB,QAAQ;WACR,SAAS;;WAET,mBAAmB;;UAGb;WACN,iBAAiB,SAAS;WAC1B,iBAAiB,SAAS;;WAE1B,UAAU;;WAEV,UAAU;;WAEV,WAAW;;WAEX,oBAAoB;;WAEpB,wBAAwB;;WAExB,eAAe;WACf,qBAAqB;;UAGf;WACN,UAAU;WACV,WAAW;;;;;;WAMX,WAAW;WACX,YAAY;WACZ,KAAK;WACL;;UAGM;;WAEN;WACA;;WAEA,MAAM;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;WACA;;WAEA;WACA;WACA;WACA;WACA;;WAEA;;;;;;;KAQC,mBAAmB;;UAGd;WACN,KAAK;;WAEL;;;;;;;;;;;;;;UAeM;WACN,gBAAgB;WAChB,aAAa;;UAGP;;WAEN,OAAO;;;;;;;;WAQP,kBAAkB;;WAElB,SAAS;;WAET,OAAO;;;;;;;WAOP,UAAU;;;;;;WAMV;WACA,yBAAyB;WACzB,0BAA0B,SAAS;WACnC,WAAW,kBAAkB;;UAGvB;WACN;WACA;WACA;WACA;;UAGM;WACN,WAAW;WACX,YAAY;WACZ,WAAW;WACX,eAAe;WACf,YAAY;WACZ,aAAa;WACb,YAAY;WACZ,YAAY;WACZ,UAAU;WACV,OAAO,SAAS;;WAEhB;;UAGM;WACN,eAAe;;WAEf;WACA;WACA;WACA,cAAc;WACd,yBAAyB;WACzB;WACA,eAAe;WACf,UAAU;WACV,WAAW;WACX,SAAS;;WAET;;WAEA;;UAGM;WACN;WACA;WACA,QAAQ;WACR,OAAO;WACP,aAAa;WACb,SAAS;WACT,UAAU;WACV,KAAK;;UAGC;WACN,eAAe;WACf;WACA,aAAa;WACb,iBAAiB;WACjB;WACA,WAAW;WACX,mBAAmB;WACnB,iBAAiB;WACjB,aAAa;WACb,qBAAqB;WACrB,eAAe;;;;;WAKf,UAAU;;;;;;;WAOV;aACE;eAA2B,OAAO;eAA2B;;aAC7D;eAAwB,OAAO;eAA2B;;;WAE5D,eAAe;WACf,kBAAkB;;;;;;;;UASZ;WACN;WACA,gBAAgB;;WAEhB,eAAe;;;KAId;UASK;WACN,MAAM;WACN;WACA;WACA"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as contentHash } from "./verdict-cache-
|
|
1
|
+
import { n as contentHash } from "./verdict-cache-CdVVTVmn.js";
|
|
2
2
|
//#region src/verdict.ts
|
|
3
3
|
/**
|
|
4
4
|
* Validator-output verdict — substrate primitive for "did this output pass,
|
|
@@ -58,4 +58,4 @@ function equivalenceVerdict(record) {
|
|
|
58
58
|
//#endregion
|
|
59
59
|
export { equivalenceVerdict as n, certificationEvidenceDigest as t };
|
|
60
60
|
|
|
61
|
-
//# sourceMappingURL=verdict-
|
|
61
|
+
//# sourceMappingURL=verdict-B0xltqu6.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict-
|
|
1
|
+
{"version":3,"file":"verdict-B0xltqu6.js","names":[],"sources":["../src/verdict.ts"],"sourcesContent":["/**\n * Validator-output verdict — substrate primitive for \"did this output pass,\n * and how well?\"\n *\n * Used by:\n * - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.\n * - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.\n * Runtime keeps `Validator` because it's coupled to runtime-shaped\n * `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE\n * itself is a substrate concept and lives here.\n *\n * Repo layering: agent-eval is the substrate (no upward deps). Both\n * agent-runtime and agent-knowledge consume this type FROM agent-eval —\n * never the other way around. See CLAUDE.md \"Repo layering\" for the rule.\n */\n\nimport { contentHash } from './verdict-cache'\nimport type {\n CheckerIdentity,\n EquivalenceRecord,\n VerificationStrategySource,\n} from './verification-strategy'\n\n/**\n * What certified a verdict — the epistemics a bare `valid` + `score` pair\n * cannot carry. A kernel-checked proof and an LLM judge can produce the\n * same `{ valid: true, score: 1 }`; this record is what tells them apart.\n *\n * Every field is required on purpose. A certification without a checker\n * identity or an evidence digest is exactly the unverifiable claim this\n * record exists to kill.\n */\nexport interface VerdictCertification {\n /**\n * Which verification-strategy member produced the certificate. Each\n * member's failure mode is documented on the family type and in\n * `VERIFICATION_STRATEGIES` (src/verification-strategy.ts).\n */\n strategy: VerificationStrategySource\n /**\n * Exact checker identity, e.g. `{ name: 'lean4', version: '4.33.0',\n * pins: { mathlib: 'db584cd6d46c' } }` for a kernel, or a judge model\n * id + snapshot for a judge.\n */\n checker: CheckerIdentity\n /**\n * Every step the certificate rests on that the checker did NOT verify,\n * e.g. 'diagonal embedding argued in prose' or 'agreement arms share\n * training-corpus blind spots'. An empty array is the producer's\n * explicit, auditable claim that there are none — it is not a default.\n */\n assumptions: string[]\n /** Digest (e.g. sha256) of the evidence artifact the checker consumed or produced. */\n evidenceDigest: string\n}\n\n/**\n * Minimal verdict shape — `valid` + `score` are required; `scores` +\n * `notes` are optional surface. Validators that need richer shapes\n * parameterise `Validator<Output, MyVerdict>` with their own type.\n *\n * Need structured extras? Extend DefaultVerdict with typed fields — never\n * serialize extras into `notes`.\n */\nexport interface DefaultVerdict {\n /** Whether the output meets the validator's pass criteria. */\n valid: boolean\n /** Aggregate score in [0, 1]. Drivers use this for winner selection. */\n score: number\n /** Per-dimension scores. Free-form; weighted into `score` by the validator. */\n scores?: Record<string, number>\n /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */\n notes?: string\n /**\n * What certified this verdict, when anything did. Absent = uncertified:\n * scored, but no strategy vouches for it. A consumer that cares reads\n * `certification.strategy` to tell a kernel-checked verdict from a\n * judge-scored one; a consumer that does not read it sees the exact\n * verdict it always saw.\n */\n certification?: VerdictCertification\n}\n\n/**\n * Digest a certification's evidence artifact: sha-256 over the canonical\n * JSON of the evidence AS SERIALIZED. The JSON round-trip drops\n * `undefined`-valued fields on purpose — certified verdicts persist as\n * JSON, so the digest must cover exactly what a reader of the persisted\n * artifact can re-digest, nothing the wire cannot carry. Throws when the\n * evidence has no JSON form at all.\n */\nexport function certificationEvidenceDigest(evidence: unknown): string {\n const serialized = JSON.stringify(evidence)\n if (serialized === undefined) {\n throw new Error(\n 'certificationEvidenceDigest: evidence has no JSON form — a certificate over nothing is the unverifiable claim this record exists to kill',\n )\n }\n return contentHash(JSON.parse(serialized))\n}\n\n/**\n * Land a finished statement-equivalence record in the shared verdict\n * spine. `valid` answers the protocol's question — are the two blind\n * statements equivalent — so a refutation is `valid: false` even though a\n * separating witness in hand is a successful protocol outcome.\n *\n * Certification is present exactly when the obligation was discharged\n * (proved or refuted): the checker vouches for the discharge either way.\n * An unresolved obligation certifies nothing and keeps its reason in\n * `notes`.\n */\nexport function equivalenceVerdict(record: EquivalenceRecord): DefaultVerdict {\n const { obligation, spec, arms } = record\n const discharged = obligation.status !== 'unresolved'\n const notes =\n obligation.status === 'proved'\n ? `statements equivalent: ${spec.artifact}`\n : obligation.status === 'refuted-with-separating-witness'\n ? `formalization gap witnessed: ${obligation.separatingWitness}`\n : `obligation undischarged: ${obligation.unresolvedReason}`\n return {\n valid: obligation.status === 'proved',\n score: obligation.status === 'proved' ? 1 : 0,\n notes,\n ...(discharged && obligation.evidenceDigest !== undefined\n ? {\n certification: {\n strategy: spec.source,\n checker: obligation.checker,\n // The arms' blindness declarations are attested by the arms,\n // never verified by the checker — the one step every\n // equivalence certificate rests on.\n assumptions: [\n `arm '${arms[0].armId}' and arm '${arms[1].armId}' self-declare blind derivation`,\n ],\n evidenceDigest: obligation.evidenceDigest,\n } satisfies VerdictCertification,\n }\n : {}),\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA2FA,SAAgB,4BAA4B,UAA2B;CACrE,MAAM,aAAa,KAAK,UAAU,QAAQ;CAC1C,IAAI,eAAe,KAAA,GACjB,MAAM,IAAI,MACR,0IACF;CAEF,OAAO,YAAY,KAAK,MAAM,UAAU,CAAC;AAC3C;;;;;;;;;;;;AAaA,SAAgB,mBAAmB,QAA2C;CAC5E,MAAM,EAAE,YAAY,MAAM,SAAS;CACnC,MAAM,aAAa,WAAW,WAAW;CACzC,MAAM,QACJ,WAAW,WAAW,WAClB,0BAA0B,KAAK,aAC/B,WAAW,WAAW,oCACpB,gCAAgC,WAAW,sBAC3C,4BAA4B,WAAW;CAC/C,OAAO;EACL,OAAO,WAAW,WAAW;EAC7B,OAAO,WAAW,WAAW,WAAW,IAAI;EAC5C;EACA,GAAI,cAAc,WAAW,mBAAmB,KAAA,IAC5C,EACE,eAAe;GACb,UAAU,KAAK;GACf,SAAS,WAAW;GAIpB,aAAa,CACX,QAAQ,KAAK,EAAE,CAAC,MAAM,aAAa,KAAK,EAAE,CAAC,MAAM,gCACnD;GACA,gBAAgB,WAAW;EAC7B,EACF,IACA,CAAC;CACP;AACF"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { i as hashCanonical, r as canonicalString } from "./canonical-IL-Bu-14.js";
|
|
2
|
+
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
|
3
|
+
//#region src/verdict-cache.ts
|
|
4
|
+
/**
|
|
5
|
+
* Content-addressed judge-verdict caching.
|
|
6
|
+
*
|
|
7
|
+
* LAW: cache JUDGE VERDICTS only — judging the same artifact with the same
|
|
8
|
+
* judge+rubric is pure. NEVER cache agent rollouts. (A router that cached
|
|
9
|
+
* identical fanout prompts silently destroyed best-of-N diversity; rollout
|
|
10
|
+
* caching reintroduces that failure class. Judging has no diversity to
|
|
11
|
+
* destroy — same artifact + same rubric ⇒ same verdict is the desired
|
|
12
|
+
* property, not a bug.)
|
|
13
|
+
*
|
|
14
|
+
* The cache key is `v2:` + a sha-256 over the canonical JSON of everything
|
|
15
|
+
* that can change a verdict: the artifact content, the scenario id, the judge
|
|
16
|
+
* name, the full dimension list (key + description — the description IS the
|
|
17
|
+
* rubric text shown to the judge), and a caller-supplied `judgeVersion`.
|
|
18
|
+
* `judgeVersion` is REQUIRED: a judge whose prompt/model/ensemble changes
|
|
19
|
+
* without a version bump would otherwise silently serve stale verdicts.
|
|
20
|
+
* The `v2:` prefix names the key scheme; a store written under another scheme
|
|
21
|
+
* misses once and repopulates.
|
|
22
|
+
*
|
|
23
|
+
* Canonical JSON is RFC 8785 from `ledger-core/canonical` — the package's one
|
|
24
|
+
* identity scheme. It refuses `undefined`, functions, symbols, non-finite
|
|
25
|
+
* numbers, class instances (including `Date`), `Map`, `Set`, cycles, and
|
|
26
|
+
* unpaired surrogates: an artifact that cannot be unambiguously serialized
|
|
27
|
+
* cannot be content-addressed, and coercing it would let two different
|
|
28
|
+
* artifacts collide on one key. Pass timestamps as ISO strings.
|
|
29
|
+
*/
|
|
30
|
+
/**
|
|
31
|
+
* Canonical JSON (RFC 8785): object keys sorted recursively, so two
|
|
32
|
+
* semantically equal values produce byte-identical output regardless of key
|
|
33
|
+
* insertion order. Throws `LedgerCanonicalizationError` on any value with no
|
|
34
|
+
* faithful JSON form — see `ledger-core/canonical`.
|
|
35
|
+
*/
|
|
36
|
+
function canonicalJson(value) {
|
|
37
|
+
return canonicalString(value);
|
|
38
|
+
}
|
|
39
|
+
/** Hex sha-256 over `canonicalJson(value)`. The content address used by the
|
|
40
|
+
* verdict cache and report attestation. Equals `hashCanonical(value)` with
|
|
41
|
+
* the `sha256:` prefix removed. */
|
|
42
|
+
function contentHash(value) {
|
|
43
|
+
return hashCanonical(value).slice(7);
|
|
44
|
+
}
|
|
45
|
+
function parseCacheLine(line, path, lineNo) {
|
|
46
|
+
let parsed;
|
|
47
|
+
try {
|
|
48
|
+
parsed = JSON.parse(line);
|
|
49
|
+
} catch (err) {
|
|
50
|
+
throw new Error(`fileVerdictCache: corrupt JSONL at ${path}:${lineNo} — ${err instanceof Error ? err.message : String(err)}`);
|
|
51
|
+
}
|
|
52
|
+
const rec = parsed;
|
|
53
|
+
if (typeof rec !== "object" || rec === null || typeof rec.key !== "string" || typeof rec.score !== "object" || rec.score === null || typeof rec.score.composite !== "number" || typeof rec.score.dimensions !== "object") throw new Error(`fileVerdictCache: invalid record shape at ${path}:${lineNo} — expected {key, score:{dimensions, composite, notes}}`);
|
|
54
|
+
return rec;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* JSONL-file-backed store: the full file is loaded into an in-memory index at
|
|
58
|
+
* construction; every `set` appends one line synchronously (durable before
|
|
59
|
+
* the verdict is returned). A corrupt or malformed line throws at load with
|
|
60
|
+
* file:line — a skipped line would silently re-judge (cost) or, worse, mask
|
|
61
|
+
* a half-written file that needs operator attention.
|
|
62
|
+
*/
|
|
63
|
+
function fileVerdictCache(path) {
|
|
64
|
+
const entries = /* @__PURE__ */ new Map();
|
|
65
|
+
if (existsSync(path)) {
|
|
66
|
+
const lines = readFileSync(path, "utf8").split("\n");
|
|
67
|
+
for (let i = 0; i < lines.length; i++) {
|
|
68
|
+
const line = lines[i];
|
|
69
|
+
if (line === void 0 || line.trim() === "") continue;
|
|
70
|
+
const rec = parseCacheLine(line, path, i + 1);
|
|
71
|
+
entries.set(rec.key, rec.score);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
get: (key) => entries.get(key),
|
|
76
|
+
set: (key, score) => {
|
|
77
|
+
appendFileSync(path, `${JSON.stringify({
|
|
78
|
+
key,
|
|
79
|
+
score
|
|
80
|
+
})}\n`, "utf8");
|
|
81
|
+
entries.set(key, score);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//#endregion
|
|
86
|
+
export { contentHash as n, fileVerdictCache as r, canonicalJson as t };
|
|
87
|
+
|
|
88
|
+
//# sourceMappingURL=verdict-cache-CdVVTVmn.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verdict-cache-CdVVTVmn.js","names":[],"sources":["../src/verdict-cache.ts"],"sourcesContent":["/**\n * Content-addressed judge-verdict caching.\n *\n * LAW: cache JUDGE VERDICTS only — judging the same artifact with the same\n * judge+rubric is pure. NEVER cache agent rollouts. (A router that cached\n * identical fanout prompts silently destroyed best-of-N diversity; rollout\n * caching reintroduces that failure class. Judging has no diversity to\n * destroy — same artifact + same rubric ⇒ same verdict is the desired\n * property, not a bug.)\n *\n * The cache key is `v2:` + a sha-256 over the canonical JSON of everything\n * that can change a verdict: the artifact content, the scenario id, the judge\n * name, the full dimension list (key + description — the description IS the\n * rubric text shown to the judge), and a caller-supplied `judgeVersion`.\n * `judgeVersion` is REQUIRED: a judge whose prompt/model/ensemble changes\n * without a version bump would otherwise silently serve stale verdicts.\n * The `v2:` prefix names the key scheme; a store written under another scheme\n * misses once and repopulates.\n *\n * Canonical JSON is RFC 8785 from `ledger-core/canonical` — the package's one\n * identity scheme. It refuses `undefined`, functions, symbols, non-finite\n * numbers, class instances (including `Date`), `Map`, `Set`, cycles, and\n * unpaired surrogates: an artifact that cannot be unambiguously serialized\n * cannot be content-addressed, and coercing it would let two different\n * artifacts collide on one key. Pass timestamps as ISO strings.\n */\n\nimport { appendFileSync, existsSync, readFileSync } from 'node:fs'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\nimport { canonicalString, hashCanonical } from './ledger-core/canonical'\n\n// ── canonical JSON + content hash ─────────────────────────────────────────\n\n/** Key scheme of every entry `cachedJudge` writes. */\nexport const VERDICT_CACHE_KEY_SCHEME = 'v2'\n\n/**\n * Canonical JSON (RFC 8785): object keys sorted recursively, so two\n * semantically equal values produce byte-identical output regardless of key\n * insertion order. Throws `LedgerCanonicalizationError` on any value with no\n * faithful JSON form — see `ledger-core/canonical`.\n */\nexport function canonicalJson(value: unknown): string {\n return canonicalString(value)\n}\n\n/** Hex sha-256 over `canonicalJson(value)`. The content address used by the\n * verdict cache and report attestation. Equals `hashCanonical(value)` with\n * the `sha256:` prefix removed. */\nexport function contentHash(value: unknown): string {\n return hashCanonical(value).slice('sha256:'.length)\n}\n\n// ── store contract ─────────────────────────────────────────────────────────\n\n/** Pluggable verdict store. Sync or async on both legs — `cachedJudge`\n * awaits the results either way. */\nexport interface VerdictCacheStore {\n get(key: string): Promise<JudgeScore | undefined> | JudgeScore | undefined\n set(key: string, score: JudgeScore): Promise<void> | void\n}\n\n/** Process-local Map-backed store. */\nexport function inMemoryVerdictCache(): VerdictCacheStore {\n const entries = new Map<string, JudgeScore>()\n return {\n get: (key) => entries.get(key),\n set: (key, score) => {\n entries.set(key, score)\n },\n }\n}\n\ninterface VerdictCacheLine {\n key: string\n score: JudgeScore\n}\n\nfunction parseCacheLine(line: string, path: string, lineNo: number): VerdictCacheLine {\n let parsed: unknown\n try {\n parsed = JSON.parse(line)\n } catch (err) {\n throw new Error(\n `fileVerdictCache: corrupt JSONL at ${path}:${lineNo} — ${err instanceof Error ? err.message : String(err)}`,\n )\n }\n const rec = parsed as Partial<VerdictCacheLine>\n if (\n typeof rec !== 'object' ||\n rec === null ||\n typeof rec.key !== 'string' ||\n typeof rec.score !== 'object' ||\n rec.score === null ||\n typeof rec.score.composite !== 'number' ||\n typeof rec.score.dimensions !== 'object'\n ) {\n throw new Error(\n `fileVerdictCache: invalid record shape at ${path}:${lineNo} — expected {key, score:{dimensions, composite, notes}}`,\n )\n }\n return rec as VerdictCacheLine\n}\n\n/**\n * JSONL-file-backed store: the full file is loaded into an in-memory index at\n * construction; every `set` appends one line synchronously (durable before\n * the verdict is returned). A corrupt or malformed line throws at load with\n * file:line — a skipped line would silently re-judge (cost) or, worse, mask\n * a half-written file that needs operator attention.\n */\nexport function fileVerdictCache(path: string): VerdictCacheStore {\n const entries = new Map<string, JudgeScore>()\n if (existsSync(path)) {\n const lines = readFileSync(path, 'utf8').split('\\n')\n for (let i = 0; i < lines.length; i++) {\n const line = lines[i]\n if (line === undefined || line.trim() === '') continue\n const rec = parseCacheLine(line, path, i + 1)\n entries.set(rec.key, rec.score)\n }\n }\n return {\n get: (key) => entries.get(key),\n set: (key, score) => {\n appendFileSync(path, `${JSON.stringify({ key, score })}\\n`, 'utf8')\n entries.set(key, score)\n },\n }\n}\n\n// ── cached judge wrapper ───────────────────────────────────────────────────\n\nexport interface VerdictCacheStats {\n hits: number\n misses: number\n}\n\nexport interface CachedJudgeOptions {\n /** REQUIRED — part of the cache key. Bump on any change to the judge's\n * prompt, model, ensemble, or scoring logic; silent judge upgrades must\n * never serve stale verdicts. */\n judgeVersion: string\n}\n\n/** The wrapped judge: same `JudgeConfig` seam, plus hit/miss observability. */\nexport type CachedJudge<TArtifact, TScenario extends Scenario = Scenario> = JudgeConfig<\n TArtifact,\n TScenario\n> & {\n stats(): VerdictCacheStats\n}\n\n/**\n * Wrap a `JudgeConfig` so repeat judgments of the same artifact are served\n * from the store instead of re-invoking `score()`. The wrapper is generic\n * over the judge's own type parameters and preserves `appliesTo` — it is a\n * drop-in replacement anywhere a `JudgeConfig` is accepted.\n *\n * A judge that throws is NOT cached: the error propagates and the next\n * attempt re-judges (caching a failure would pin a transient outage forever).\n */\nexport function cachedJudge<TArtifact, TScenario extends Scenario = Scenario>(\n judge: JudgeConfig<TArtifact, TScenario>,\n store: VerdictCacheStore,\n options: CachedJudgeOptions,\n): CachedJudge<TArtifact, TScenario> {\n if (typeof options.judgeVersion !== 'string' || options.judgeVersion.trim() === '') {\n throw new Error('cachedJudge: judgeVersion is required and must be a non-empty string')\n }\n const stats: VerdictCacheStats = { hits: 0, misses: 0 }\n const wrapped: CachedJudge<TArtifact, TScenario> = {\n name: judge.name,\n dimensions: judge.dimensions,\n judgeVersion: options.judgeVersion,\n async score(input) {\n const key = `${VERDICT_CACHE_KEY_SCHEME}:${contentHash({\n artifact: canonicalJson(input.artifact),\n scenarioId: input.scenario.id,\n judgeName: judge.name,\n dimensions: judge.dimensions,\n judgeVersion: options.judgeVersion,\n })}`\n const cached = await store.get(key)\n if (cached !== undefined) {\n stats.hits += 1\n return cached\n }\n const score = await judge.score(input)\n await store.set(key, score)\n stats.misses += 1\n return score\n },\n stats: () => ({ ...stats }),\n }\n if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo\n return wrapped\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,SAAgB,cAAc,OAAwB;CACpD,OAAO,gBAAgB,KAAK;AAC9B;;;;AAKA,SAAgB,YAAY,OAAwB;CAClD,OAAO,cAAc,KAAK,CAAC,CAAC,MAAM,CAAgB;AACpD;AA2BA,SAAS,eAAe,MAAc,MAAc,QAAkC;CACpF,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,IAAI;CAC1B,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,sCAAsC,KAAK,GAAG,OAAO,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GAC3G;CACF;CACA,MAAM,MAAM;CACZ,IACE,OAAO,QAAQ,YACf,QAAQ,QACR,OAAO,IAAI,QAAQ,YACnB,OAAO,IAAI,UAAU,YACrB,IAAI,UAAU,QACd,OAAO,IAAI,MAAM,cAAc,YAC/B,OAAO,IAAI,MAAM,eAAe,UAEhC,MAAM,IAAI,MACR,6CAA6C,KAAK,GAAG,OAAO,wDAC9D;CAEF,OAAO;AACT;;;;;;;;AASA,SAAgB,iBAAiB,MAAiC;CAChE,MAAM,0BAAU,IAAI,IAAwB;CAC5C,IAAI,WAAW,IAAI,GAAG;EACpB,MAAM,QAAQ,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,IAAI;EACnD,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;GACrC,MAAM,OAAO,MAAM;GACnB,IAAI,SAAS,KAAA,KAAa,KAAK,KAAK,MAAM,IAAI;GAC9C,MAAM,MAAM,eAAe,MAAM,MAAM,IAAI,CAAC;GAC5C,QAAQ,IAAI,IAAI,KAAK,IAAI,KAAK;EAChC;CACF;CACA,OAAO;EACL,MAAM,QAAQ,QAAQ,IAAI,GAAG;EAC7B,MAAM,KAAK,UAAU;GACnB,eAAe,MAAM,GAAG,KAAK,UAAU;IAAE;IAAK;GAAM,CAAC,EAAE,KAAK,MAAM;GAClE,QAAQ,IAAI,KAAK,KAAK;EACxB;CACF;AACF"}
|
package/dist/wire/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { c as CostLedgerHandle } from "../cost-ledger-DbQdN3nO.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-
|
|
1
|
+
import { b as CustomTokenPricing, c as CostLedgerHandle } from "../cost-ledger-DbQdN3nO.js";
|
|
2
|
+
import { s as TraceStore } from "../store-B06JdC56.js";
|
|
3
|
+
import { p as ChatClient } from "../types-Bfk0uxRj.js";
|
|
4
|
+
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-B3ZHaHV_.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { ServerType } from "@hono/node-server";
|
|
7
7
|
import { Hono } from "hono";
|
|
@@ -157,103 +157,6 @@ declare const TracesIngestResponseSchema: z.ZodObject<{
|
|
|
157
157
|
message: z.ZodString;
|
|
158
158
|
}, z.core.$strip>>>;
|
|
159
159
|
}, z.core.$strip>;
|
|
160
|
-
declare const FeedbackLabelSchema: z.ZodObject<{
|
|
161
|
-
id: z.ZodOptional<z.ZodString>;
|
|
162
|
-
source: z.ZodEnum<{
|
|
163
|
-
environment: "environment";
|
|
164
|
-
judge: "judge";
|
|
165
|
-
metric: "metric";
|
|
166
|
-
policy: "policy";
|
|
167
|
-
system: "system";
|
|
168
|
-
user: "user";
|
|
169
|
-
}>;
|
|
170
|
-
kind: z.ZodEnum<{
|
|
171
|
-
approve: "approve";
|
|
172
|
-
comment: "comment";
|
|
173
|
-
edit: "edit";
|
|
174
|
-
metric_outcome: "metric_outcome";
|
|
175
|
-
policy_block: "policy_block";
|
|
176
|
-
rank: "rank";
|
|
177
|
-
rate: "rate";
|
|
178
|
-
reject: "reject";
|
|
179
|
-
revision_request: "revision_request";
|
|
180
|
-
select: "select";
|
|
181
|
-
}>;
|
|
182
|
-
value: z.ZodUnknown;
|
|
183
|
-
reason: z.ZodOptional<z.ZodString>;
|
|
184
|
-
severity: z.ZodOptional<z.ZodEnum<{
|
|
185
|
-
critical: "critical";
|
|
186
|
-
error: "error";
|
|
187
|
-
info: "info";
|
|
188
|
-
warning: "warning";
|
|
189
|
-
}>>;
|
|
190
|
-
createdAt: z.ZodString;
|
|
191
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
192
|
-
}, z.core.$strip>;
|
|
193
|
-
declare const FeedbackAttemptSchema: z.ZodObject<{
|
|
194
|
-
id: z.ZodString;
|
|
195
|
-
stepIndex: z.ZodNumber;
|
|
196
|
-
artifactType: z.ZodEnum<{
|
|
197
|
-
action: "action";
|
|
198
|
-
code: "code";
|
|
199
|
-
data: "data";
|
|
200
|
-
decision: "decision";
|
|
201
|
-
other: "other";
|
|
202
|
-
plan: "plan";
|
|
203
|
-
research: "research";
|
|
204
|
-
text: "text";
|
|
205
|
-
ui: "ui";
|
|
206
|
-
}>;
|
|
207
|
-
artifact: z.ZodUnknown;
|
|
208
|
-
options: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
|
|
209
|
-
proposedAction: z.ZodOptional<z.ZodObject<{
|
|
210
|
-
type: z.ZodString;
|
|
211
|
-
risk: z.ZodOptional<z.ZodEnum<{
|
|
212
|
-
high: "high";
|
|
213
|
-
low: "low";
|
|
214
|
-
medium: "medium";
|
|
215
|
-
}>>;
|
|
216
|
-
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
217
|
-
externalSideEffect: z.ZodOptional<z.ZodBoolean>;
|
|
218
|
-
requiresApproval: z.ZodOptional<z.ZodBoolean>;
|
|
219
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
220
|
-
}, z.core.$strip>>;
|
|
221
|
-
feedback: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
222
|
-
id: z.ZodOptional<z.ZodString>;
|
|
223
|
-
source: z.ZodEnum<{
|
|
224
|
-
environment: "environment";
|
|
225
|
-
judge: "judge";
|
|
226
|
-
metric: "metric";
|
|
227
|
-
policy: "policy";
|
|
228
|
-
system: "system";
|
|
229
|
-
user: "user";
|
|
230
|
-
}>;
|
|
231
|
-
kind: z.ZodEnum<{
|
|
232
|
-
approve: "approve";
|
|
233
|
-
comment: "comment";
|
|
234
|
-
edit: "edit";
|
|
235
|
-
metric_outcome: "metric_outcome";
|
|
236
|
-
policy_block: "policy_block";
|
|
237
|
-
rank: "rank";
|
|
238
|
-
rate: "rate";
|
|
239
|
-
reject: "reject";
|
|
240
|
-
revision_request: "revision_request";
|
|
241
|
-
select: "select";
|
|
242
|
-
}>;
|
|
243
|
-
value: z.ZodUnknown;
|
|
244
|
-
reason: z.ZodOptional<z.ZodString>;
|
|
245
|
-
severity: z.ZodOptional<z.ZodEnum<{
|
|
246
|
-
critical: "critical";
|
|
247
|
-
error: "error";
|
|
248
|
-
info: "info";
|
|
249
|
-
warning: "warning";
|
|
250
|
-
}>>;
|
|
251
|
-
createdAt: z.ZodString;
|
|
252
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
253
|
-
}, z.core.$strip>>>;
|
|
254
|
-
createdAt: z.ZodString;
|
|
255
|
-
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
256
|
-
}, z.core.$strip>;
|
|
257
160
|
declare const FeedbackTrajectorySchema: z.ZodObject<{
|
|
258
161
|
id: z.ZodString;
|
|
259
162
|
projectId: z.ZodOptional<z.ZodString>;
|
|
@@ -426,9 +329,15 @@ declare class WireError extends Error {
|
|
|
426
329
|
interface HandleJudgeOptions {
|
|
427
330
|
costLedger?: CostLedgerHandle;
|
|
428
331
|
costPhase?: string;
|
|
429
|
-
|
|
332
|
+
/**
|
|
333
|
+
* Caller-owned transport for the judge call. agent-eval holds no provider
|
|
334
|
+
* credential: the process that serves this endpoint binds its own client.
|
|
335
|
+
* Omitting it makes `/v1/judge` refuse with `llm_not_configured`.
|
|
336
|
+
*/
|
|
337
|
+
chat?: ChatClient;
|
|
430
338
|
defaultModel?: string;
|
|
431
|
-
|
|
339
|
+
/** Endpoint rates used when the transport reports no billed amount. */
|
|
340
|
+
pricing?: CustomTokenPricing;
|
|
432
341
|
signal?: AbortSignal;
|
|
433
342
|
}
|
|
434
343
|
declare function handleJudge(req: JudgeRequest, options?: HandleJudgeOptions): Promise<JudgeResult>;
|
|
@@ -797,9 +706,9 @@ interface RpcError {
|
|
|
797
706
|
};
|
|
798
707
|
}
|
|
799
708
|
interface RpcOptions {
|
|
800
|
-
|
|
709
|
+
/** Caller-owned transport for `/v1/judge`. Without it the method refuses. */
|
|
710
|
+
chat?: ChatClient;
|
|
801
711
|
judgeModel?: string;
|
|
802
|
-
llmRouteRequirements?: LlmRouteRequirements;
|
|
803
712
|
}
|
|
804
713
|
declare function dispatchRpc(req: RpcRequest, options?: RpcOptions): Promise<RpcSuccess | RpcError>;
|
|
805
714
|
/** Read one JSON request from stdin, write one JSON response to stdout. */
|
|
@@ -808,7 +717,6 @@ declare function runRpcOnce(method?: string, options?: RpcOptions): Promise<numb
|
|
|
808
717
|
declare function runRpcBatch(method?: string, options?: RpcOptions): Promise<number>;
|
|
809
718
|
//#endregion
|
|
810
719
|
//#region src/wire/rubrics.d.ts
|
|
811
|
-
declare const BUILTIN_RUBRICS: Record<string, Rubric>;
|
|
812
720
|
/** Get a built-in rubric by name, or undefined. */
|
|
813
721
|
declare function getBuiltinRubric(name: string): Rubric | undefined;
|
|
814
722
|
/** List built-in rubrics with their stable versions. */
|
|
@@ -828,12 +736,14 @@ declare function listBuiltinRubrics(): {
|
|
|
828
736
|
interface CreateAppOptions {
|
|
829
737
|
/** Stores wired to the ingestion endpoints. */
|
|
830
738
|
stores?: IngestionStores;
|
|
831
|
-
/**
|
|
832
|
-
|
|
739
|
+
/**
|
|
740
|
+
* Caller-owned transport used by `/v1/judge`. agent-eval holds no provider
|
|
741
|
+
* credential; without a client the endpoint refuses with
|
|
742
|
+
* `llm_not_configured`.
|
|
743
|
+
*/
|
|
744
|
+
chat?: ChatClient;
|
|
833
745
|
/** Default judge model when a request does not provide one. */
|
|
834
746
|
judgeModel?: string;
|
|
835
|
-
/** Model route checks applied before every judge call. */
|
|
836
|
-
llmRouteRequirements?: LlmRouteRequirements;
|
|
837
747
|
/**
|
|
838
748
|
* Bearer-token auth. When provided, every endpoint EXCEPT `/healthz`
|
|
839
749
|
* and `/v1/version` requires `Authorization: Bearer <token>`. The
|
|
@@ -873,5 +783,5 @@ interface StartedServer {
|
|
|
873
783
|
*/
|
|
874
784
|
declare function startServerAsync(opts?: ServeOptions): Promise<StartedServer>;
|
|
875
785
|
//#endregion
|
|
876
|
-
export {
|
|
786
|
+
export { ErrorResponse, ErrorResponseSchema, FailureMode, FeedbackIngestResponse, FeedbackIngestResponseSchema, FeedbackTrajectory, FeedbackTrajectorySchema, HandleJudgeOptions, HealthResponseSchema, IngestionStores, JudgeRequest, JudgeRequestSchema, JudgeResult, JudgeResultSchema, ListRubricsResponse, ListRubricsResponseSchema, type RpcOptions, Rubric, RubricDimension, RubricDimensionSchema, RubricInfo, RubricSchema, type ServeOptions, type StartedServer, TraceEvent, TraceEventSchema, TracesIngestRequest, TracesIngestRequestSchema, TracesIngestResponse, TracesIngestResponseSchema, VersionResponse, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
|
|
877
787
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/wire/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/wire/schemas.ts","../../src/wire/handlers.ts","../../node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/specification-extension.d.ts","../../node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/oas-common.d.ts","../../node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/openapi31.d.ts","../../src/wire/openapi.ts","../../src/wire/rpc.ts","../../src/wire/rubrics.ts","../../src/wire/server.ts"],"x_google_ignoreList":[2,3,4],"mappings":";;;;;;;;cAmBa,uBAAqB,EAAA;;;;;;GAkBL,EAAA,KAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/wire/schemas.ts","../../src/wire/handlers.ts","../../node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/specification-extension.d.ts","../../node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/oas-common.d.ts","../../node_modules/.pnpm/openapi3-ts@4.6.0/node_modules/openapi3-ts/dist/model/openapi31.d.ts","../../src/wire/openapi.ts","../../src/wire/rpc.ts","../../src/wire/rubrics.ts","../../src/wire/server.ts"],"x_google_ignoreList":[2,3,4],"mappings":";;;;;;;;cAmBa,uBAAqB,EAAA;;;;;;GAkBL,EAAA,KAAA;cAEvB,mBAAiB,EAAA;;;GAKE,EAAA,KAAA;cAIZ,cAAY,EAAA;;;;;;;;;;;;;;;;;;;GA6BL,EAAA,KAAA;cAIP,oBAAkB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAwBL,EAAA,KAAA;cAEb,mBAAiB,EAAA;;;;;;;;;GA6BL,EAAA,KAAA;cAInB,kBAAgB,EAAA;;;;;;;;;;GAUE,EAAA,KAAA;cAEX,2BAAyB,EAAA;;;;;;;;;;;;GAIL,EAAA,KAAA;cAIpB,uBAAqB,EAAA;;;;;GAWL,EAAA,KAAA;cAEhB,sBAAoB,EAAA;;;GAKL,EAAA,KAAA;;;;;;;cAUf,kBAAgB,EAAA;;;;;;;;;;;;;;;;GA0BL,EAAA,KAAA;cAEX,2BAAyB,EAAA;;;;;;;;;;;;;;;;;;GAQL,EAAA,KAAA;cAEpB,4BAA0B,EAAA;;;;;;;GAiBL,EAAA,KAAA;cA2DrB,0BAAwB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BL,EAAA,KAAA;cAEnB,8BAA4B,EAAA;;;GAKL,EAAA,KAAA;KAExB,aAAa,EAAE,aAAa;KAC5B,sBAAsB,EAAE,aAAa;KACrC,uBAAuB,EAAE,aAAa;KACtC,qBAAqB,EAAE,aAAa;KACpC,yBAAyB,EAAE,aAAa;cAIvC,qBAAmB,EAAA;;;;;;GAcL,EAAA,KAAA;KAIf,kBAAkB,EAAE,aAAa;KACjC,cAAc,EAAE,aAAa;KAC7B,SAAS,EAAE,aAAa;KACxB,eAAe,EAAE,aAAa;KAC9B,cAAc,EAAE,aAAa;KAC7B,aAAa,EAAE,aAAa;KAC5B,sBAAsB,EAAE,aAAa;KACrC,kBAAkB,EAAE,aAAa;KACjC,gBAAgB,EAAE,aAAa;;;;;cAQ9B;;;;;iBAMG,WAAW,QAAQ;;;;cC/VtB,kBAAkB;WAEX;WAEA;WACA;EAJlB,YACkB,cAChB,iBACgB,iBACA;;UAgJH;EACf,aAAa;EACb;;;;;;EAMA,OAAO;EACP;;EAEA,UAAU;EACV,SAAS;;iBAGW,YACpB,KAAK,cACL,UAAS,qBACR,QAAQ;iBAuEK,qBAAqB;iBAMrB,iBAAiB;;;;;;;;;UAmBhB;EACf,aAAa;EACb,gBAAgB;;;;;;;;;;;;iBAaI,mBACpB,KAAK,qBACL,QAAQ,kBACP,QAAQ;;;;;;iBA8BW,qBACpB,KAAK,oBACL,QAAQ,kBACP,QAAQ;;;KC9VC;KACA;KACA;GACP,eAAe,iBAAiB;;;;UCFpB,qBAAqB;EAClC;EACA;EACA;KACK,YAAY;;;UAGJ,6BAA6B;EAC1C;EACA;EACA;;;;UCPa,sBAAsB;EACnC;EACA,MAAM;EACN,UAAU;EACV,QAAQ;EACR,aAAa;EACb,WAAW;EACX,OAAO;EACP,eAAe;EACf,WAAW;;UAEE,mBAAmB;EAChC;EACA;EACA;EACA,UAAU;EACV,UAAU;EACV;;UAEa,sBAAsB;EACnC;EACA;EACA;;UAEa,sBAAsB;EACnC;EACA;EACA;;UAEa,yBAAyB;EACtC;KACK,iBAAiB,eAAe;;EAErC;KACK,mBAAmB,iBAAiB;;EAEzC;KACK,oBAAoB,kBAAkB;;EAE3C;KACK,kBAAkB,gBAAgB;;EAEvC;KACK,kBAAkB,oBAAoB;;EAE3C;KACK,iBAAiB,eAAe;;EAErC;KACK,yBAAyB,uBAAuB;;EAErD;KACK,eAAe,aAAa;;EAEjC;KACK,mBAAmB,iBAAiB;;EAEzC;KACK,mBAAmB,iBAAiB;;;UAG5B,oBAAoB;GAChC,eAAe;;UAIH,uBAAuB;EACpC;EACA;EACA;EACA,MAAM;EACN,MAAM;EACN,OAAO;EACP,SAAS;EACT,UAAU;EACV,OAAO;EACP,QAAQ;EACR,QAAQ;EACR,UAAU;EACV,cAAc,kBAAkB;;UAEnB,wBAAwB;EACrC;EACA;EACA;EACA,eAAe;EACf;EACA,cAAc,kBAAkB;EAChC,cAAc,oBAAoB;EAClC,YAAY;EACZ,YAAY;EACZ;EACA,WAAW;EACX,UAAU;;UAEG,oCAAoC;EACjD;EACA;;KAEQ;KACA;UACK,4BAA4B;EACzC;EACA;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA,SAAS,eAAe;EACxB;KACK,gBAAgB,gBAAgB;;EAErC;EACA,UAAU;;UAEG,wBAAwB;EACrC;EACA,IAAI;;UAES,0BAA0B;EACvC;EACA,SAAS;EACT;;UAEa;GACZ,oBAAoB;;UAER,wBAAwB;EACrC,SAAS,eAAe;EACxB,WAAW;EACX;EACA,WAAW;;UAEE,uBAAuB;GACnC,mBAAmB;;UAEP;EACb;EACA;KACK,cAAc,eAAe;;EAElC;EACA;EACA;GACC;;UAEY,wBAAwB;EACrC,UAAU,iBAAiB;GAC1B,qBAAqB,iBAAiB;;UAE1B,uBAAuB;EACpC;EACA,UAAU;EACV,UAAU;EACV,QAAQ;;UAEK,wBAAwB;GACpC,eAAe,iBAAiB;;UAEpB,uBAAuB;GACnC,eAAe;;UAEH;GACZ,eAAe,eAAe;;UAElB;EACb;EACA;EACA;EACA;GACC;;UAEY;GACZ,eAAe,aAAa;;UAEhB,mBAAmB;EAChC;EACA;EACA,aAAa;EACb;EACA;EACA,SAAS;GACR;;UAEY;GACZ;;UAEY,qBAAqB;EAClC;;UAEa,kBAAkB;EAC/B;EACA;EACA,eAAe;GACd;;UAEY;GACZ,eAAe,gBAAgB;;UAEnB;EACb;EACA;EACA;;KAGQ;UACK,qBAAqB;EAClC;EACA,gBAAgB;EAChB;EACA;EACA,MAAM;EACN,eAAe;EACf;EACA;EACA;EACA,OAAO,mBAAmB;EAC1B;EACA,SAAS,eAAe;EACxB,SAAS,eAAe;EACxB,SAAS,eAAe;EACxB,MAAM,eAAe;EACrB,QAAQ,eAAe;EACvB;KACK,uBAAuB,eAAe;;EAE3C,uBAAuB,eAAe;EACtC,gBAAgB,eAAe;EAC/B;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,eAAe,eAAe;EAC9B;EACA;;UAMa;EACb;EACA;KACK;;;UAGQ,kBAAkB;EAC/B;EACA;EACA;EACA;EACA;;KAEQ;UACK,6BAA6B;EAC1C,MAAM;EACN;EACA;EACA;EACA;EACA;EACA,QAAQ;EACR;;UAEa,yBAAyB;EACtC,WAAW;EACX,WAAW;EACX,oBAAoB;EACpB,oBAAoB;;UAEP,wBAAwB;EACrC;EACA;EACA;EACA,QAAQ;;UAEK,qBAAqB;GACjC;;UAEY;GACZ;;;;iBChRW,aAAa,yBAAyB;;;UCX5C;EACR;EACA;;UAGQ;EACR;;UAGQ;EACR;IAAS;IAAc;IAAiB;;;UAGzB;;EAEf,OAAO;EACP;;iBAGoB,YACpB,KAAK,YACL,UAAS,aACR,QAAQ,aAAa;;iBAqDF,WAAW,iBAAiB,UAAS,aAAkB;;iBAuBvD,YAAY,iBAAiB,UAAS,aAAkB;;;;iBCjB9D,iBAAiB,eAAe;;iBAKhC;;;;;;;;;;;;;UCtEC;;EAEf,SAAS;;;;;;EAMT,OAAO;;EAEP;;;;;;;;;;EAUA;IACE,mBAAmB,4BAA4B;;;iBAMnC,UAAU,OAAM,mBAAqB,0BAAA,+BAAA;UA0IpC,qBAAqB;;EAEpC;;EAEA;;iBAGc,YAAY,OAAM,eAAoB;UAUrC;EACf,QAAQ;;;;EAIR;;EAEA;;EAEA,SAAS;;;;;;;iBAQK,iBAAiB,OAAM,eAAoB,QAAQ"}
|
package/dist/wire/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { A as
|
|
2
|
-
export {
|
|
1
|
+
import { A as hashRubric, C as RubricDimensionSchema, D as TracesIngestResponseSchema, E as TracesIngestRequestSchema, O as VersionResponseSchema, S as ListRubricsResponseSchema, T as TraceEventSchema, _ as FeedbackIngestResponseSchema, a as runRpcBatch, b as JudgeRequestSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as ErrorResponseSchema, h as listBuiltinRubrics, i as dispatchRpc, k as WIRE_VERSION, l as handleFeedbackIngest, m as getBuiltinRubric, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FeedbackTrajectorySchema, w as RubricSchema, x as JudgeResultSchema, y as HealthResponseSchema } from "../server-BtFd4uzB.js";
|
|
2
|
+
export { ErrorResponseSchema, FeedbackIngestResponseSchema, FeedbackTrajectorySchema, HealthResponseSchema, JudgeRequestSchema, JudgeResultSchema, ListRubricsResponseSchema, RubricDimensionSchema, RubricSchema, TraceEventSchema, TracesIngestRequestSchema, TracesIngestResponseSchema, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };
|
|
@@ -6,7 +6,7 @@ How every fleet agent that consumes `agent-eval` is built. Each rule is mechanic
|
|
|
6
6
|
|
|
7
7
|
Every hard-coded model id or endpoint default is verifiable against the live router. Membership in `{baseUrl}/models` is the free check; an optional 1-token probe per model confirms the router will actually serve it. A default the router cannot serve is a config bug caught before the run, not a runtime surprise that silently degrades into a stub. Backend ids are namespaced by binding: cli-bridge ids (`claude-code/*`, `kimi-code/*`, `opencode/*`) never appear as defaults in code reachable from production: bridge use is an explicit env opt-in, never an implicit fallback.
|
|
8
8
|
|
|
9
|
-
Enforced by: `preflightModels` (membership + optional probe) and `assertModelsServed` (gate that names every unreachable id with status + detail).
|
|
9
|
+
Enforced by: `preflightModels` (membership + optional probe, over a caller-owned `request` function) and `assertModelsServed` (gate that names every unreachable id with status + detail).
|
|
10
10
|
|
|
11
11
|
## 1a. Reachable is not the same as identified
|
|
12
12
|
|
|
@@ -20,14 +20,14 @@ Snapshot resolution (`gpt-4o-mini` → `gpt-4o-mini-2024-07-18`) pins a floating
|
|
|
20
20
|
Substitution (`gpt-4.1-mini` → `gemini-2.5-flash-lite`) mislabels every number the call produces.
|
|
21
21
|
A response that echoes no model id at all is unproven, which fails closed rather than defaulting to agreement.
|
|
22
22
|
|
|
23
|
-
Enforced by: `assertServedModel` / `assertServedModels` per call, `assertCrossFamilyServed` for panel diversity computed over the ids that answered, `
|
|
23
|
+
Enforced by: `assertServedModel` / `assertServedModels` per call, `assertCrossFamilyServed` for panel diversity computed over the ids that answered, `ChatResponse.servedModel` carrying the identity the caller's transport observed, and `assertModelsServed` (probe mode), which now fails a substituted id exactly as it fails a dead one.
|
|
24
24
|
`assertCrossFamily` reads requested ids and therefore proves configuration only — reach for the served-side check wherever the diversity claim is load-bearing.
|
|
25
25
|
|
|
26
26
|
## 2. Probe the platform before peeling client layers
|
|
27
27
|
|
|
28
28
|
When a request fails, one direct call against the live endpoint bisects platform-versus-client before any code-level debugging begins. A 401 from the router on a `model_not_found` is the platform telling you the default is dead; a connection refused is the platform being unreachable. Establish which side is at fault with a probe first, then debug only the side that is actually broken.
|
|
29
29
|
|
|
30
|
-
Enforced by: `preflightModels({ probe: true })`: the probe is the platform-side bisection, carrying the
|
|
30
|
+
Enforced by: `preflightModels({ probe: true, request })`: the probe is the platform-side bisection, carrying the endpoint's own `error.message` back to the caller.
|
|
31
31
|
|
|
32
32
|
## 3. Agent-produced findings are hypotheses
|
|
33
33
|
|
|
@@ -313,21 +313,30 @@ With `optimizer`, every recipe stage must use the standard `gepa` engine or a me
|
|
|
313
313
|
Agent Eval receives no provider key, enforces the declared request and token budget, and records the execution owner's exact usage and opaque finite JSON evidence.
|
|
314
314
|
`maxProposerCostUsd` also limits each individual GEPA engine stage.
|
|
315
315
|
|
|
316
|
-
|
|
316
|
+
`optimizer.call` is always caller code.
|
|
317
|
+
Agent Eval owns no model transport and never receives a provider credential.
|
|
317
318
|
|
|
318
|
-
|
|
319
|
-
import { createOpenAiCompatibleExecutionOwner } from '@tangle-network/agent-eval/campaign'
|
|
319
|
+
On agent-runtime, use `profileOptimizerModelCall`, which executes one exact `AgentProfile` and reports profile-digest evidence:
|
|
320
320
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
321
|
+
```ts
|
|
322
|
+
import { profileOptimizerModelCall } from '@tangle-network/agent-runtime/kernel'
|
|
323
|
+
|
|
324
|
+
const call = profileOptimizerModelCall({
|
|
325
|
+
profile: optimizerProfile,
|
|
326
|
+
context: 'prompt optimizer',
|
|
327
|
+
executor: {
|
|
328
|
+
backend: 'router',
|
|
329
|
+
routerBaseUrl: process.env.LLM_BASE_URL!,
|
|
330
|
+
routerKey: process.env.LLM_API_KEY!,
|
|
331
|
+
},
|
|
325
332
|
pricing: { inputUsdPerMillion: 0.4, outputUsdPerMillion: 1.6 },
|
|
326
333
|
})
|
|
327
334
|
```
|
|
328
335
|
|
|
329
|
-
|
|
330
|
-
|
|
336
|
+
Without agent-runtime, implement `ExternalOptimizerModelCall` over the OpenAI-compatible client you already have.
|
|
337
|
+
`examples/_shared/openai-compatible-owner.ts` is a complete minimal implementation to copy.
|
|
338
|
+
The callback resolves with one success or failure result and never rejects, because a rejection loses the execution record and fails the optimizer attempt.
|
|
339
|
+
The credential stays in your process; the proxy still enforces every budget and identity check.
|
|
331
340
|
|
|
332
341
|
### Metered agent CLI engines
|
|
333
342
|
|
|
@@ -355,7 +364,7 @@ Set `maxEvaluations` to at least the train-set size for every registering evalua
|
|
|
355
364
|
|
|
356
365
|
Without `optimizer`, an engine runs unproxied and can receive its own settings:
|
|
357
366
|
|
|
358
|
-
```
|
|
367
|
+
```text
|
|
359
368
|
recipe: {
|
|
360
369
|
kind: 'engine',
|
|
361
370
|
run: {
|
|
@@ -527,6 +536,28 @@ This includes the baseline, an earlier generation, and another candidate in the
|
|
|
527
536
|
The complete proposal is checked before candidate dispatch, so duplicates cannot consume candidate cells.
|
|
528
537
|
Use `reps` when one surface needs repeated measurements.
|
|
529
538
|
|
|
539
|
+
### Choose The Parent
|
|
540
|
+
|
|
541
|
+
`runOptimization()` is incumbent-anchored by default.
|
|
542
|
+
Every generation mutates the single best complete surface seen so far, and only a candidate that beats the incumbent becomes the next incumbent.
|
|
543
|
+
The recorded `parentSurfaceHash` lineage is then a chain.
|
|
544
|
+
Pass `selectParent` to draw the parent from the Pareto frontier instead.
|
|
545
|
+
The selector receives the frontier so far, the measured incumbent, the generation history, and the generation index, and returns one frontier parent.
|
|
546
|
+
The loop hands that parent to the proposer as `ctx.currentSurface` and `ctx.parentOutcome`, and records it as every candidate's `parentSurfaceHash`.
|
|
547
|
+
`ctx.incumbentOutcome` stays the global promotion bar, and a candidate still has to beat the incumbent to promote.
|
|
548
|
+
`crowdedFrontierParent({ seed })` is the provided policy: a seeded NSGA-II crowded tournament that prefers isolated frontier parents.
|
|
549
|
+
The loop refuses a parent it has not measured to completion.
|
|
550
|
+
`selfImprove({ selectParent })` forwards the same policy in proposer mode.
|
|
551
|
+
|
|
552
|
+
```ts
|
|
553
|
+
import { crowdedFrontierParent, runOptimization } from '@tangle-network/agent-eval/campaign'
|
|
554
|
+
|
|
555
|
+
const result = await runOptimization({
|
|
556
|
+
// ...scenarios, dispatchWithSurface, judges, proposer, populationSize, maxGenerations, runDir
|
|
557
|
+
selectParent: crowdedFrontierParent({ seed: 42 }),
|
|
558
|
+
})
|
|
559
|
+
```
|
|
560
|
+
|
|
530
561
|
## Data And Cost Rules
|
|
531
562
|
|
|
532
563
|
- Train and selection cases are visible to complete optimization methods.
|