@tangle-network/agent-eval 0.150.1 → 0.161.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +178 -1
- package/README.md +7 -3
- package/dist/{active-curriculum-C4mk67HP.js → active-curriculum-CD5TU2yW.js} +3 -13
- package/dist/active-curriculum-CD5TU2yW.js.map +1 -0
- package/dist/{agent-profile-cell-BkcRDikH.d.ts → agent-profile-cell-CTOZJUuE.d.ts} +4 -2
- package/dist/agent-profile-cell-CTOZJUuE.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +19 -36
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +8 -8
- package/dist/analyst/index.js.map +1 -1
- package/dist/{backend-integrity-DOCa_QrR.d.ts → backend-integrity-DxuQCu_A.d.ts} +4 -3
- package/dist/backend-integrity-DxuQCu_A.d.ts.map +1 -0
- package/dist/{benchmark-BtAWA8nT.d.ts → benchmark-CGPp-kDC.d.ts} +3 -3
- package/dist/{benchmark-BtAWA8nT.d.ts.map → benchmark-CGPp-kDC.d.ts.map} +1 -1
- package/dist/{benchmark-command-CAFwbH0L.js → benchmark-command-BVtaq_ve.js} +26 -31
- package/dist/benchmark-command-BVtaq_ve.js.map +1 -0
- package/dist/benchmarks/index.d.ts +6 -19
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +3 -3
- package/dist/campaign/index.d.ts +9 -9
- package/dist/campaign/index.js +7 -7
- package/dist/{campaign-CN_7xJdV.js → campaign-BSmOwskD.js} +77 -795
- package/dist/campaign-BSmOwskD.js.map +1 -0
- package/dist/{canonical-D-XsTQ6_.js → canonical-IL-Bu-14.js} +26 -2
- package/dist/canonical-IL-Bu-14.js.map +1 -0
- package/dist/{capture-fetch-BBVFzhkk.d.ts → capture-fetch-CqwsJkkG.d.ts} +3 -3
- package/dist/{capture-fetch-BBVFzhkk.d.ts.map → capture-fetch-CqwsJkkG.d.ts.map} +1 -1
- package/dist/{chat-client-2bVfrzhN.js → chat-client-DlMlAeYI.js} +5 -55
- package/dist/{chat-client-2bVfrzhN.js.map → chat-client-DlMlAeYI.js.map} +1 -1
- package/dist/chat-json-call-6g5sJobJ.js +53 -0
- package/dist/chat-json-call-6g5sJobJ.js.map +1 -0
- package/dist/cli.js +54 -19
- package/dist/cli.js.map +1 -1
- package/dist/{client-LIuo-KPv.js → client-CX7KqIdB.js} +3 -3
- package/dist/client-CX7KqIdB.js.map +1 -0
- package/dist/{client-kPQYT_56.d.ts → client-L9VVPkim.d.ts} +4 -4
- package/dist/{client-kPQYT_56.d.ts.map → client-L9VVPkim.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +13 -27
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +14 -17
- package/dist/contract/index.js.map +1 -1
- package/dist/{counterfactual--bpysZF0.d.ts → counterfactual-BaFUWK3H.d.ts} +4 -4
- package/dist/{counterfactual--bpysZF0.d.ts.map → counterfactual-BaFUWK3H.d.ts.map} +1 -1
- package/dist/{counterfactual-lDfCx0Uz.js → counterfactual-D_VWavVm.js} +2 -2
- package/dist/{counterfactual-lDfCx0Uz.js.map → counterfactual-D_VWavVm.js.map} +1 -1
- package/dist/{dataset-CJjKqQfA.d.ts → dataset-DQqhOCPt.d.ts} +5 -4
- package/dist/{dataset-CJjKqQfA.d.ts.map → dataset-DQqhOCPt.d.ts.map} +1 -1
- package/dist/{default-registry-Cw0Ohdoj.d.ts → default-registry-G9CKMNkc.d.ts} +7 -8
- package/dist/{default-registry-Cw0Ohdoj.d.ts.map → default-registry-G9CKMNkc.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CEQWL9Hy.d.ts → define-agent-eval-Dx1JnPEa.d.ts} +26 -6
- package/dist/define-agent-eval-Dx1JnPEa.d.ts.map +1 -0
- package/dist/{define-agent-eval-rqNyVhVV.js → define-agent-eval-h-s-sI-v.js} +17 -11
- package/dist/define-agent-eval-h-s-sI-v.js.map +1 -0
- package/dist/{descriptive-B5MwKfbf.js → descriptive-jDOuI6mz.js} +22 -2
- package/dist/descriptive-jDOuI6mz.js.map +1 -0
- package/dist/{dspy-rlm-engine-DHI0WrUU.js → dspy-rlm-engine-DptEII26.js} +95 -15
- package/dist/dspy-rlm-engine-DptEII26.js.map +1 -0
- package/dist/{emitter-CPBAhxum.js → emitter-BpYFQPj4.js} +2 -18
- package/dist/emitter-BpYFQPj4.js.map +1 -0
- package/dist/{emitter-DGQGoLyj.d.ts → emitter-D_jYSGRd.d.ts} +4 -20
- package/dist/{emitter-DGQGoLyj.d.ts.map → emitter-D_jYSGRd.d.ts.map} +1 -1
- package/dist/{engine-BLzhNzoY.d.ts → engine-Cu5qD5Fc.d.ts} +9 -11
- package/dist/{engine-BLzhNzoY.d.ts.map → engine-Cu5qD5Fc.d.ts.map} +1 -1
- package/dist/{eval-campaign-C4jmuM-b.js → eval-campaign-BsXWL2-2.js} +17 -28
- package/dist/eval-campaign-BsXWL2-2.js.map +1 -0
- package/dist/{exact-types-ccQAyut1.d.ts → exact-types-qnexxJ1Z.d.ts} +2 -2
- package/dist/{exact-types-ccQAyut1.d.ts.map → exact-types-qnexxJ1Z.d.ts.map} +1 -1
- package/dist/{exec-y-DCLqK7.js → exec-D9WpA2p-.js} +2 -2
- package/dist/exec-D9WpA2p-.js.map +1 -0
- package/dist/experiment/index.d.ts +45 -11
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +30 -11
- package/dist/experiment/index.js.map +1 -1
- package/dist/{experiment-tracker-0MhuPArU.d.ts → experiment-tracker-DCO6Cz4s.d.ts} +2 -2
- package/dist/{experiment-tracker-0MhuPArU.d.ts.map → experiment-tracker-DCO6Cz4s.d.ts.map} +1 -1
- package/dist/{exporters-q9iL-2Jf.js → exporters-Df7TgHFv.js} +3 -3
- package/dist/exporters-Df7TgHFv.js.map +1 -0
- package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts → external-optimizer-contracts-szBJ_1vh.d.ts} +2 -2
- package/dist/{external-optimizer-contracts-DbLsm4Po.d.ts.map → external-optimizer-contracts-szBJ_1vh.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-Bhmzngf-.js → external-optimizer-process-WosTBChy.js} +4 -4
- package/dist/{external-optimizer-process-Bhmzngf-.js.map → external-optimizer-process-WosTBChy.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-Dn90UqN2.js → external-optimizer-subprocess-BIWbHpgD.js} +10 -6
- package/dist/external-optimizer-subprocess-BIWbHpgD.js.map +1 -0
- package/dist/{failure-cluster-BLURuWG4.d.ts → failure-cluster-CXL8NbEw.d.ts} +3 -3
- package/dist/{failure-cluster-BLURuWG4.d.ts.map → failure-cluster-CXL8NbEw.d.ts.map} +1 -1
- package/dist/{feedback-trajectory-DpTTjo0q.d.ts → feedback-trajectory-B3ZHaHV_.d.ts} +7 -7
- package/dist/{feedback-trajectory-DpTTjo0q.d.ts.map → feedback-trajectory-B3ZHaHV_.d.ts.map} +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-XggBupCr.js → hf-dataset-D8_RNIis.js} +4 -4
- package/dist/{hf-dataset-XggBupCr.js.map → hf-dataset-D8_RNIis.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -14
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +2 -2
- package/dist/{index-CTKpu9ry.d.ts → index-CGtH1piv.d.ts} +48 -26
- package/dist/index-CGtH1piv.d.ts.map +1 -0
- package/dist/{index-BNPtkBPf.d.ts → index-D-IiQIBB.d.ts} +5 -10
- package/dist/index-D-IiQIBB.d.ts.map +1 -0
- package/dist/{index-IQccV3Ou.d.ts → index-D-V8gCs_.d.ts} +13 -90
- package/dist/index-D-V8gCs_.d.ts.map +1 -0
- package/dist/{index-B8Ui1mr1.d.ts → index-lfaSeKSD.d.ts} +18 -2
- package/dist/index-lfaSeKSD.d.ts.map +1 -0
- package/dist/index-vrJugRal.d.ts +1 -0
- package/dist/index.d.ts +68 -125
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +61 -112
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-BeT8KCgI.d.ts → insight-report-DRe8LB6d.d.ts} +4 -4
- package/dist/{insight-report-BeT8KCgI.d.ts.map → insight-report-DRe8LB6d.d.ts.map} +1 -1
- package/dist/{integrity-DysDBWDu.js → integrity-CyWSSoQS.js} +17 -4
- package/dist/integrity-CyWSSoQS.js.map +1 -0
- package/dist/{integrity-B0dZ96EO.d.ts → integrity-DUNX9Fao.d.ts} +3 -3
- package/dist/{integrity-B0dZ96EO.d.ts.map → integrity-DUNX9Fao.d.ts.map} +1 -1
- package/dist/{judge-calibration-DZkWrm5H.js → judge-calibration-zZjLz8hr.js} +2 -2
- package/dist/{judge-calibration-DZkWrm5H.js.map → judge-calibration-zZjLz8hr.js.map} +1 -1
- package/dist/{kind-factory-DmAa0h3K.js → kind-factory-DY8FdoXf.js} +3 -71
- package/dist/kind-factory-DY8FdoXf.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-DTae9rv_.js → ledger-core-BOzlRygb.js} +2 -2
- package/dist/{ledger-core-DTae9rv_.js.map → ledger-core-BOzlRygb.js.map} +1 -1
- package/dist/{llm-client-Bg32RW0j.js → llm-client-hgDieDNN.js} +53 -98
- package/dist/llm-client-hgDieDNN.js.map +1 -0
- package/dist/{llm-judge-CVq33oz1.js → llm-judge-BhasIPFT.js} +1136 -78
- package/dist/llm-judge-BhasIPFT.js.map +1 -0
- package/dist/{matrix-DrVnRp4G.d.ts → matrix-eXKRMHnL.d.ts} +74 -72
- package/dist/matrix-eXKRMHnL.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +8 -6
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +9 -7
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{mint-BV6tLVWl.js → mint-DfODW1KW.js} +3 -3
- package/dist/{mint-BV6tLVWl.js.map → mint-DfODW1KW.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +2 -8
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +56 -86
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +11 -46
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +30 -83
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-DJWAXLms.js → opencode-sqlite-eK6HW6dr.js} +2 -6
- package/dist/{opencode-sqlite-DJWAXLms.js.map → opencode-sqlite-eK6HW6dr.js.map} +1 -1
- package/dist/pipelines/index.d.ts +5 -5
- package/dist/pipelines/index.js +3 -3
- package/dist/{pre-registration-zFSLEiFU.d.ts → pre-registration-CzFCcwYk.d.ts} +55 -40
- package/dist/pre-registration-CzFCcwYk.d.ts.map +1 -0
- package/dist/pre-registration-KN9jkh58.js +110 -0
- package/dist/pre-registration-KN9jkh58.js.map +1 -0
- package/dist/{produced-state-jfk8Du3b.js → produced-state-DZ89riy5.js} +8 -8
- package/dist/produced-state-DZ89riy5.js.map +1 -0
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +31 -5
- package/dist/profile-cell.js.map +1 -1
- package/dist/{promotion-policy-DLOUkYhI.d.ts → promotion-policy-DtnOIZvk.d.ts} +2 -2
- package/dist/{promotion-policy-DLOUkYhI.d.ts.map → promotion-policy-DtnOIZvk.d.ts.map} +1 -1
- package/dist/{query-Di7eEQ79.js → query-CHmMP42p.js} +20 -11
- package/dist/query-CHmMP42p.js.map +1 -0
- package/dist/{query-CJ_DX8vl.d.ts → query-DxPYqpmT.d.ts} +10 -4
- package/dist/query-DxPYqpmT.d.ts.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/{registry-BQwrSYpC.d.ts → registry-8You7OK1.d.ts} +5 -7
- package/dist/{registry-BQwrSYpC.d.ts.map → registry-8You7OK1.d.ts.map} +1 -1
- package/dist/{release-confidence-BknrpBnO.js → release-confidence-DKfD2RYU.js} +28 -14
- package/dist/release-confidence-DKfD2RYU.js.map +1 -0
- package/dist/{release-confidence-4XrqlpFD.d.ts → release-confidence-Dqt0NFep.d.ts} +7 -6
- package/dist/release-confidence-Dqt0NFep.d.ts.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +3 -3
- package/dist/{researcher-DJnoUE8c.d.ts → researcher-Cz565b7D.d.ts} +34 -21
- package/dist/researcher-Cz565b7D.d.ts.map +1 -0
- package/dist/{reward-hacking-62tojkQd.d.ts → reward-hacking-MBf7qpSB.d.ts} +2 -2
- package/dist/{reward-hacking-62tojkQd.d.ts.map → reward-hacking-MBf7qpSB.d.ts.map} +1 -1
- package/dist/{reward-hacking-DKI9T52l.js → reward-hacking-t4lB1yt8.js} +3 -3
- package/dist/{reward-hacking-DKI9T52l.js.map → reward-hacking-t4lB1yt8.js.map} +1 -1
- package/dist/rl.d.ts +11 -42
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +41 -24
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +3 -3
- package/dist/rollout/index.js +7 -7
- package/dist/{rollout-ytVQ7WT8.js → rollout-Dm2tSdiQ.js} +6 -6
- package/dist/{rollout-ytVQ7WT8.js.map → rollout-Dm2tSdiQ.js.map} +1 -1
- package/dist/{rubric-predictive-validity-Cwwyd7ah.js → rubric-predictive-validity-CK8SCOg-.js} +6 -16
- package/dist/rubric-predictive-validity-CK8SCOg-.js.map +1 -0
- package/dist/{rubric-predictive-validity-C7LnNvF2.d.ts → rubric-predictive-validity-CxycqzX5.d.ts} +4 -3
- package/dist/rubric-predictive-validity-CxycqzX5.d.ts.map +1 -0
- package/dist/{run-record-D2lDdSAz.js → run-record-BC0ebuRP.js} +2 -2
- package/dist/{run-record-D2lDdSAz.js.map → run-record-BC0ebuRP.js.map} +1 -1
- package/dist/{run-record-DVV82Gwh.d.ts → run-record-VVy4T9OW.d.ts} +3 -3
- package/dist/{run-record-DVV82Gwh.d.ts.map → run-record-VVy4T9OW.d.ts.map} +1 -1
- package/dist/{schema-BtVldJ3T.d.ts → schema-Bjgdsn73.d.ts} +2 -4
- package/dist/{schema-BtVldJ3T.d.ts.map → schema-Bjgdsn73.d.ts.map} +1 -1
- package/dist/{schema-Cef2cFmb.d.ts → schema-BzWDXhOR.d.ts} +2 -5
- package/dist/schema-BzWDXhOR.d.ts.map +1 -0
- package/dist/{schema-C6DW4ZHR.js → schema-C1aaAxTf.js} +2 -2
- package/dist/schema-C1aaAxTf.js.map +1 -0
- package/dist/{schema-CRhEY1SO.js → schema-k6ZBftVv.js} +2 -8
- package/dist/{schema-CRhEY1SO.js.map → schema-k6ZBftVv.js.map} +1 -1
- package/dist/{semantic-concept-judge-laMCnTLn.js → semantic-concept-judge-BSkKKHeq.js} +14 -38
- package/dist/semantic-concept-judge-BSkKKHeq.js.map +1 -0
- package/dist/{sequential-eprocess-CbUt2htw.js → sequential-eprocess-D1jKoihe.js} +49 -2
- package/dist/sequential-eprocess-D1jKoihe.js.map +1 -0
- package/dist/{sequential-C458DXNf.js → sequential-rYW-Ophm.js} +41 -16
- package/dist/sequential-rYW-Ophm.js.map +1 -0
- package/dist/{series-convergence-BxKEgBwA.d.ts → series-convergence-D9WgpXGi.d.ts} +2 -2
- package/dist/{series-convergence-BxKEgBwA.d.ts.map → series-convergence-D9WgpXGi.d.ts.map} +1 -1
- package/dist/{server-dIWwF3j_.js → server-BtFd4uzB.js} +19 -42
- package/dist/server-BtFd4uzB.js.map +1 -0
- package/dist/{skillopt-optimization-method-DLeUcK-K.js → skillopt-optimization-method-DbaekMcn.js} +794 -8
- package/dist/skillopt-optimization-method-DbaekMcn.js.map +1 -0
- package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts → skillopt-optimization-method-x7TTF23P.d.ts} +20 -7
- package/dist/{skillopt-optimization-method-BDD_o1xE.d.ts.map → skillopt-optimization-method-x7TTF23P.d.ts.map} +1 -1
- package/dist/{statistical-heldout-_woZ9q9j.d.ts → statistical-heldout-Cy3EhjlC.d.ts} +21 -8
- package/dist/statistical-heldout-Cy3EhjlC.d.ts.map +1 -0
- package/dist/{steps-AmkT-GIM.d.ts → steps-CiNVJry_.d.ts} +2 -17
- package/dist/steps-CiNVJry_.d.ts.map +1 -0
- package/dist/{store-CT9YIIve.d.ts → store-B06JdC56.d.ts} +2 -2
- package/dist/{store-CT9YIIve.d.ts.map → store-B06JdC56.d.ts.map} +1 -1
- package/dist/{store-otlp-CDYWW_8N.js → store-otlp-C_Rq5I4D.js} +2 -2
- package/dist/{store-otlp-CDYWW_8N.js.map → store-otlp-C_Rq5I4D.js.map} +1 -1
- package/dist/{store-tool-spans-Br2_IUhm.d.ts → store-tool-spans-DPUG7UUY.d.ts} +6 -6
- package/dist/{store-tool-spans-Br2_IUhm.d.ts.map → store-tool-spans-DPUG7UUY.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CykkbOlv.js → store-tool-spans-Dlh9vkFK.js} +3 -3
- package/dist/{store-tool-spans-CykkbOlv.js.map → store-tool-spans-Dlh9vkFK.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-Blysd6Z2.js → summary-report-BI5hUtvK.js} +7 -17
- package/dist/summary-report-BI5hUtvK.js.map +1 -0
- package/dist/{summary-report-B__Y5ub3.d.ts → summary-report-CC07PhEL.d.ts} +6 -5
- package/dist/summary-report-CC07PhEL.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +27 -18
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +104 -26
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{task-failure-attributes--ZTP3tYO.js → task-failure-attributes-DTl-7-Kw.js} +3 -3
- package/dist/{task-failure-attributes--ZTP3tYO.js.map → task-failure-attributes-DTl-7-Kw.js.map} +1 -1
- package/dist/{tool-groups-B4tqh8jB.d.ts → tool-groups-Ci8i9ErB.d.ts} +3 -3
- package/dist/tool-groups-Ci8i9ErB.d.ts.map +1 -0
- package/dist/{tool-waste-BDdBZG1F.js → tool-waste-BqzmVdJk.js} +4 -4
- package/dist/{tool-waste-BDdBZG1F.js.map → tool-waste-BqzmVdJk.js.map} +1 -1
- package/dist/{tool-waste-DjRDEsuI.d.ts → tool-waste-Dro0gJi3.d.ts} +4 -4
- package/dist/{tool-waste-DjRDEsuI.d.ts.map → tool-waste-Dro0gJi3.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +4 -77
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +5 -15
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +13 -23
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -20
- package/dist/traces.js.map +1 -1
- package/dist/{trajectory-YC15QDYQ.d.ts → trajectory-Bi157Gun.d.ts} +3 -3
- package/dist/{trajectory-YC15QDYQ.d.ts.map → trajectory-Bi157Gun.d.ts.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +5 -5
- package/dist/trajectory-replay/index.js +5 -5
- package/dist/{provenance-oA4-zUqm.d.ts → transient-failure-DKF5Mofa.d.ts} +468 -13
- package/dist/transient-failure-DKF5Mofa.d.ts.map +1 -0
- package/dist/types-B3jzCp0p.js.map +1 -1
- package/dist/{types-CLAwnY-L.d.ts → types-BPb2Kf_C2.d.ts} +3 -3
- package/dist/types-BPb2Kf_C2.d.ts.map +1 -0
- package/dist/types-Bfk0uxRj.d.ts +443 -0
- package/dist/types-Bfk0uxRj.d.ts.map +1 -0
- package/dist/{types-DdFNuyxQ.d.ts → types-D4s7Z6nq.d.ts} +30 -6
- package/dist/types-D4s7Z6nq.d.ts.map +1 -0
- package/dist/{types-B2NsbrNy.d.ts → types-D9ssmxKL.d.ts} +3 -3
- package/dist/{types-B2NsbrNy.d.ts.map → types-D9ssmxKL.d.ts.map} +1 -1
- package/dist/{types-yLK8gXE9.d.ts → types-DeIUdzNd.d.ts} +160 -10
- package/dist/types-DeIUdzNd.d.ts.map +1 -0
- package/dist/{verdict-BndeTAh_.js → verdict-B0xltqu6.js} +2 -2
- package/dist/{verdict-BndeTAh_.js.map → verdict-B0xltqu6.js.map} +1 -1
- package/dist/verdict-cache-CdVVTVmn.js +88 -0
- package/dist/verdict-cache-CdVVTVmn.js.map +1 -0
- package/dist/wire/index.d.ts +21 -111
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/adapters-observability.md +9 -23
- package/docs/building-doctrine.md +3 -3
- package/docs/campaign-proposers.md +54 -15
- package/docs/concepts.md +3 -4
- package/docs/design/statistics-decisions.md +89 -1
- package/docs/eval-surface-map.md +14 -0
- package/docs/experiment.md +19 -2
- package/docs/feedback-trajectories.md +1 -1
- package/docs/multishot-golden-records.md +4 -4
- package/docs/public-api.md +1616 -0
- package/docs/research-report-methodology.md +1 -1
- package/docs/search-history-receipts.md +39 -1
- package/docs/trace-analysis.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/trace-repair-continuation.md +1 -1
- package/docs/verdicts.md +24 -0
- package/docs/wire-protocol.md +1 -1
- package/package.json +6 -2
- package/dist/active-curriculum-C4mk67HP.js.map +0 -1
- package/dist/agent-profile-cell-BkcRDikH.d.ts.map +0 -1
- package/dist/backend-integrity-DOCa_QrR.d.ts.map +0 -1
- package/dist/benchmark-command-CAFwbH0L.js.map +0 -1
- package/dist/campaign-CN_7xJdV.js.map +0 -1
- package/dist/canonical-D-XsTQ6_.js.map +0 -1
- package/dist/client-LIuo-KPv.js.map +0 -1
- package/dist/define-agent-eval-CEQWL9Hy.d.ts.map +0 -1
- package/dist/define-agent-eval-rqNyVhVV.js.map +0 -1
- package/dist/descriptive-B5MwKfbf.js.map +0 -1
- package/dist/dspy-rlm-engine-DHI0WrUU.js.map +0 -1
- package/dist/emitter-CPBAhxum.js.map +0 -1
- package/dist/eval-campaign-C4jmuM-b.js.map +0 -1
- package/dist/exec-y-DCLqK7.js.map +0 -1
- package/dist/exporters-q9iL-2Jf.js.map +0 -1
- package/dist/external-optimizer-subprocess-Dn90UqN2.js.map +0 -1
- package/dist/index-B8Ui1mr1.d.ts.map +0 -1
- package/dist/index-BNPtkBPf.d.ts.map +0 -1
- package/dist/index-C1ravkGA.d.ts +0 -1
- package/dist/index-CTKpu9ry.d.ts.map +0 -1
- package/dist/index-IQccV3Ou.d.ts.map +0 -1
- package/dist/integrity-DysDBWDu.js.map +0 -1
- package/dist/kind-factory-DmAa0h3K.js.map +0 -1
- package/dist/llm-client-Bg32RW0j.js.map +0 -1
- package/dist/llm-judge-CVq33oz1.js.map +0 -1
- package/dist/matrix-DrVnRp4G.d.ts.map +0 -1
- package/dist/pre-registration-DakwTRXk.js +0 -96
- package/dist/pre-registration-DakwTRXk.js.map +0 -1
- package/dist/pre-registration-zFSLEiFU.d.ts.map +0 -1
- package/dist/produced-state-jfk8Du3b.js.map +0 -1
- package/dist/provenance-oA4-zUqm.d.ts.map +0 -1
- package/dist/query-CJ_DX8vl.d.ts.map +0 -1
- package/dist/query-Di7eEQ79.js.map +0 -1
- package/dist/release-confidence-4XrqlpFD.d.ts.map +0 -1
- package/dist/release-confidence-BknrpBnO.js.map +0 -1
- package/dist/researcher-DJnoUE8c.d.ts.map +0 -1
- package/dist/rubric-predictive-validity-C7LnNvF2.d.ts.map +0 -1
- package/dist/rubric-predictive-validity-Cwwyd7ah.js.map +0 -1
- package/dist/schema-C6DW4ZHR.js.map +0 -1
- package/dist/schema-Cef2cFmb.d.ts.map +0 -1
- package/dist/semantic-concept-judge-laMCnTLn.js.map +0 -1
- package/dist/sequential-C458DXNf.js.map +0 -1
- package/dist/sequential-eprocess-CbUt2htw.js.map +0 -1
- package/dist/server-dIWwF3j_.js.map +0 -1
- package/dist/skillopt-optimization-method-DLeUcK-K.js.map +0 -1
- package/dist/statistical-heldout-_woZ9q9j.d.ts.map +0 -1
- package/dist/steps-AmkT-GIM.d.ts.map +0 -1
- package/dist/summary-report-B__Y5ub3.d.ts.map +0 -1
- package/dist/summary-report-Blysd6Z2.js.map +0 -1
- package/dist/tool-groups-B4tqh8jB.d.ts.map +0 -1
- package/dist/types-CLAwnY-L.d.ts.map +0 -1
- package/dist/types-DdFNuyxQ.d.ts.map +0 -1
- package/dist/types-jUBXJ7Iz.d.ts +0 -884
- package/dist/types-jUBXJ7Iz.d.ts.map +0 -1
- package/dist/types-yLK8gXE9.d.ts.map +0 -1
- package/dist/verdict-cache-mZf5FEiY.js +0 -107
- package/dist/verdict-cache-mZf5FEiY.js.map +0 -1
package/dist/{skillopt-optimization-method-DLeUcK-K.js → skillopt-optimization-method-DbaekMcn.js}
RENAMED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { r as canonicalString } from "./canonical-IL-Bu-14.js";
|
|
2
|
+
import { i as modelHasSnapshot } from "./run-record-BC0ebuRP.js";
|
|
3
|
+
import { n as contentHash, t as canonicalJson } from "./verdict-cache-CdVVTVmn.js";
|
|
4
|
+
import { D as costFromLedgerSummary, J as readGepaCandidatePopulationArtifact, N as surfaceContentHash, O as optimizationTokenUsageFromSummary, T as combineComparisonCosts, V as campaignBreakdown, nt as runCampaign, q as assertGepaCandidatePopulationSummary, t as llmJudge, y as recordCandidatePopulationSearch } from "./llm-judge-BhasIPFT.js";
|
|
5
|
+
import { t as FileLedgerJournal } from "./ledger-core-BOzlRygb.js";
|
|
6
|
+
import { C as SEARCH_LEDGER_FILE_CONTEXT, E as SearchLedgerIntegrityError, T as SearchLedgerError, _ as resolveExternalOptimizerCallbackLimits, b as createRunCostLedger, d as assertJsonValue, f as assertNoCredentialValues, g as removeCredentialEnvironment, h as isRecord, i as startExternalOptimizerModelProxy, m as isExternalTextCandidate, n as closeExternalOptimizerResources, p as isCandidateText, r as runWithCleanup, t as runExternalOptimizerProcess, u as assertExternalOptimizerModelBudget, v as resolveExternalOptimizerProcessLimits, w as SearchLedgerConflictError, x as fsCampaignStorage, y as safePathComponent } from "./external-optimizer-subprocess-BIWbHpgD.js";
|
|
4
7
|
import { a as heldoutSignificance, o as pairHoldout } from "./power-preflight-DEw-uC7q.js";
|
|
5
8
|
import { c as deepFreezeCanonicalJson } from "./types-BI4fT3HN.js";
|
|
6
|
-
import {
|
|
7
|
-
import { n as acquireSingleRunLock, t as startExternalOptimizerCallback } from "./external-optimizer-process-
|
|
9
|
+
import { g as assertServedModelPolicy } from "./llm-client-hgDieDNN.js";
|
|
10
|
+
import { n as acquireSingleRunLock, t as startExternalOptimizerCallback } from "./external-optimizer-process-WosTBChy.js";
|
|
8
11
|
import { createHash, randomBytes } from "node:crypto";
|
|
9
12
|
import { z } from "zod";
|
|
10
13
|
import { mkdir } from "node:fs/promises";
|
|
@@ -1408,6 +1411,746 @@ function isGepaCandidate(value, maxChars) {
|
|
|
1408
1411
|
return (typeof value === "string" ? value.length : JSON.stringify(value).length) <= maxChars;
|
|
1409
1412
|
}
|
|
1410
1413
|
//#endregion
|
|
1414
|
+
//#region src/campaign/search-ledger.ts
|
|
1415
|
+
/**
|
|
1416
|
+
* Durable append-only audit log for improvement searches.
|
|
1417
|
+
*
|
|
1418
|
+
* Existing campaign artifacts keep their own rich records: `RunRecord` owns a
|
|
1419
|
+
* measured run and `CostLedger` owns per-call accounting. This ledger does not
|
|
1420
|
+
* copy those structures. It binds their immutable ids and receipts into one replayable event stream so a
|
|
1421
|
+
* search can answer, after a crash, exactly which candidates and task attempts
|
|
1422
|
+
* existed, which surfaces actually fired, what they cost, and why they were
|
|
1423
|
+
* selected or rejected.
|
|
1424
|
+
*
|
|
1425
|
+
* The file format is canonical JSONL with a SHA-256 hash chain. Every append is
|
|
1426
|
+
* serialized across processes, fsynced before acknowledgement, and idempotent
|
|
1427
|
+
* by `eventId`. A malformed, non-canonical, truncated, reordered, or conflicting
|
|
1428
|
+
* log fails loudly; the implementation never skips a bad row.
|
|
1429
|
+
*
|
|
1430
|
+
* The journal machinery itself (hash chain, locking, fsync, idempotent append)
|
|
1431
|
+
* is the generic `ledger-core` journal; this module supplies the campaign
|
|
1432
|
+
* codec: event schemas, canonical event ordering, and the search state machine.
|
|
1433
|
+
*/
|
|
1434
|
+
const SEARCH_LEDGER_SCHEMA = "tangle.search-ledger.v1";
|
|
1435
|
+
const NON_EMPTY = z.string().min(1).refine((value) => value.trim() === value, "must not contain surrounding whitespace");
|
|
1436
|
+
const HASH = z.string().regex(/^sha256:[a-f0-9]{64}$/);
|
|
1437
|
+
const LINEAGE_NODE_ID = z.string().regex(/^[a-f0-9]{16}$/);
|
|
1438
|
+
const IMMUTABLE_REVISION = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64}|sha256:[a-f0-9]{64}|sha512:[A-Za-z0-9+/=]+)$/);
|
|
1439
|
+
const ISO_TIMESTAMP = z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/).refine((value) => Number.isFinite(Date.parse(value)), "invalid timestamp");
|
|
1440
|
+
const NON_NEGATIVE_INT = z.number().int().nonnegative().safe();
|
|
1441
|
+
const FINITE_NUMBER = z.number().finite();
|
|
1442
|
+
const ArtifactRefSchema = z.object({
|
|
1443
|
+
role: NON_EMPTY,
|
|
1444
|
+
uri: NON_EMPTY,
|
|
1445
|
+
sha256: HASH,
|
|
1446
|
+
byteLength: NON_NEGATIVE_INT
|
|
1447
|
+
}).strict();
|
|
1448
|
+
const SourceRefSchema = z.object({
|
|
1449
|
+
uri: NON_EMPTY,
|
|
1450
|
+
revision: IMMUTABLE_REVISION
|
|
1451
|
+
}).strict();
|
|
1452
|
+
const FailureReasonSchema = z.object({
|
|
1453
|
+
code: NON_EMPTY,
|
|
1454
|
+
message: NON_EMPTY
|
|
1455
|
+
}).strict();
|
|
1456
|
+
const EventBaseShape = {
|
|
1457
|
+
eventId: NON_EMPTY,
|
|
1458
|
+
occurredAt: ISO_TIMESTAMP,
|
|
1459
|
+
artifacts: z.array(ArtifactRefSchema).min(1)
|
|
1460
|
+
};
|
|
1461
|
+
const OperationKindSchema = z.enum([
|
|
1462
|
+
"candidate-generation",
|
|
1463
|
+
"analysis",
|
|
1464
|
+
"selection",
|
|
1465
|
+
"judge",
|
|
1466
|
+
"other"
|
|
1467
|
+
]);
|
|
1468
|
+
const CandidateSlotSchema = z.object({
|
|
1469
|
+
slotId: NON_EMPTY,
|
|
1470
|
+
generationOperationId: NON_EMPTY
|
|
1471
|
+
}).strict();
|
|
1472
|
+
const PlannedOperationSchema = z.object({
|
|
1473
|
+
operationId: NON_EMPTY,
|
|
1474
|
+
kind: OperationKindSchema
|
|
1475
|
+
}).strict();
|
|
1476
|
+
const SearchPlanExtendedSchema = z.object({
|
|
1477
|
+
...EventBaseShape,
|
|
1478
|
+
kind: z.literal("search-plan-extended"),
|
|
1479
|
+
extension: z.object({
|
|
1480
|
+
candidateSlots: z.array(CandidateSlotSchema),
|
|
1481
|
+
operations: z.array(PlannedOperationSchema)
|
|
1482
|
+
}).strict().superRefine((extension, ctx) => {
|
|
1483
|
+
if (extension.candidateSlots.length === 0 && extension.operations.length === 0) ctx.addIssue({
|
|
1484
|
+
code: "custom",
|
|
1485
|
+
message: "a plan extension must add slots or operations"
|
|
1486
|
+
});
|
|
1487
|
+
})
|
|
1488
|
+
}).strict();
|
|
1489
|
+
const SearchPlannedSchema = z.object({
|
|
1490
|
+
...EventBaseShape,
|
|
1491
|
+
kind: z.literal("search-planned"),
|
|
1492
|
+
plan: z.object({
|
|
1493
|
+
candidateSlots: z.array(CandidateSlotSchema).min(1),
|
|
1494
|
+
tasks: z.array(z.object({
|
|
1495
|
+
taskId: NON_EMPTY,
|
|
1496
|
+
source: SourceRefSchema,
|
|
1497
|
+
benchmark: SourceRefSchema,
|
|
1498
|
+
maxAttempts: z.number().int().positive().safe()
|
|
1499
|
+
}).strict()).min(1),
|
|
1500
|
+
operations: z.array(PlannedOperationSchema).min(1)
|
|
1501
|
+
}).strict()
|
|
1502
|
+
}).strict();
|
|
1503
|
+
const CandidateRegisteredSchema = z.object({
|
|
1504
|
+
...EventBaseShape,
|
|
1505
|
+
kind: z.literal("candidate-registered"),
|
|
1506
|
+
slotId: NON_EMPTY,
|
|
1507
|
+
generationOperationId: NON_EMPTY,
|
|
1508
|
+
candidateId: NON_EMPTY,
|
|
1509
|
+
lineage: z.object({
|
|
1510
|
+
lineageNodeId: LINEAGE_NODE_ID,
|
|
1511
|
+
parentCandidateIds: z.array(NON_EMPTY),
|
|
1512
|
+
generation: NON_NEGATIVE_INT,
|
|
1513
|
+
proposer: NON_EMPTY,
|
|
1514
|
+
proposerSource: SourceRefSchema
|
|
1515
|
+
}).strict(),
|
|
1516
|
+
surfaces: z.array(z.object({
|
|
1517
|
+
surfaceId: NON_EMPTY,
|
|
1518
|
+
kind: z.enum([
|
|
1519
|
+
"prompt",
|
|
1520
|
+
"tool-contract",
|
|
1521
|
+
"runtime-config",
|
|
1522
|
+
"memory",
|
|
1523
|
+
"knowledge",
|
|
1524
|
+
"agent-profile",
|
|
1525
|
+
"code",
|
|
1526
|
+
"deployment"
|
|
1527
|
+
]),
|
|
1528
|
+
artifact: ArtifactRefSchema
|
|
1529
|
+
}).strict()).min(1)
|
|
1530
|
+
}).strict();
|
|
1531
|
+
const CandidateSlotClosedSchema = z.object({
|
|
1532
|
+
...EventBaseShape,
|
|
1533
|
+
kind: z.literal("candidate-slot-closed"),
|
|
1534
|
+
slotId: NON_EMPTY,
|
|
1535
|
+
generationOperationId: NON_EMPTY,
|
|
1536
|
+
reason: FailureReasonSchema
|
|
1537
|
+
}).strict();
|
|
1538
|
+
const KnownTokensSchema = z.object({
|
|
1539
|
+
status: z.literal("known"),
|
|
1540
|
+
inputTokens: NON_NEGATIVE_INT,
|
|
1541
|
+
outputTokens: NON_NEGATIVE_INT,
|
|
1542
|
+
cachedTokens: NON_NEGATIVE_INT
|
|
1543
|
+
}).strict();
|
|
1544
|
+
const UnknownSchema = z.object({
|
|
1545
|
+
status: z.literal("unknown"),
|
|
1546
|
+
reason: NON_EMPTY
|
|
1547
|
+
}).strict();
|
|
1548
|
+
const KnownCostSchema = z.object({
|
|
1549
|
+
status: z.literal("known"),
|
|
1550
|
+
usd: z.number().finite().nonnegative(),
|
|
1551
|
+
source: z.enum([
|
|
1552
|
+
"provider",
|
|
1553
|
+
"pricing-table",
|
|
1554
|
+
"free"
|
|
1555
|
+
])
|
|
1556
|
+
}).strict().superRefine((cost, ctx) => {
|
|
1557
|
+
if (cost.source === "free" && cost.usd !== 0) ctx.addIssue({
|
|
1558
|
+
code: "custom",
|
|
1559
|
+
message: "free cost source must have usd 0"
|
|
1560
|
+
});
|
|
1561
|
+
});
|
|
1562
|
+
const UnknownCostSchema = z.object({
|
|
1563
|
+
status: z.literal("unknown"),
|
|
1564
|
+
knownLowerBoundUsd: z.number().finite().nonnegative(),
|
|
1565
|
+
reason: NON_EMPTY
|
|
1566
|
+
}).strict();
|
|
1567
|
+
const AccountingSchema = z.object({
|
|
1568
|
+
tokens: z.discriminatedUnion("status", [KnownTokensSchema, UnknownSchema]),
|
|
1569
|
+
cost: z.discriminatedUnion("status", [KnownCostSchema, UnknownCostSchema])
|
|
1570
|
+
}).strict();
|
|
1571
|
+
const MetricsSchema = z.record(NON_EMPTY, FINITE_NUMBER).superRefine((metrics, ctx) => {
|
|
1572
|
+
for (const key of Object.keys(metrics)) if (key === "__proto__" || key === "constructor" || key === "prototype") ctx.addIssue({
|
|
1573
|
+
code: "custom",
|
|
1574
|
+
message: `unsafe metric key ${key}`
|
|
1575
|
+
});
|
|
1576
|
+
});
|
|
1577
|
+
const OutcomeSchema = z.discriminatedUnion("status", [
|
|
1578
|
+
z.object({
|
|
1579
|
+
status: z.literal("passed"),
|
|
1580
|
+
score: FINITE_NUMBER,
|
|
1581
|
+
metrics: MetricsSchema
|
|
1582
|
+
}).strict(),
|
|
1583
|
+
z.object({
|
|
1584
|
+
status: z.literal("failed"),
|
|
1585
|
+
score: FINITE_NUMBER,
|
|
1586
|
+
metrics: MetricsSchema,
|
|
1587
|
+
failure: FailureReasonSchema
|
|
1588
|
+
}).strict(),
|
|
1589
|
+
z.object({
|
|
1590
|
+
status: z.literal("errored"),
|
|
1591
|
+
metrics: MetricsSchema,
|
|
1592
|
+
error: FailureReasonSchema.extend({ retryable: z.boolean() }).strict()
|
|
1593
|
+
}).strict()
|
|
1594
|
+
]);
|
|
1595
|
+
const EffectSchema = z.discriminatedUnion("status", [z.object({
|
|
1596
|
+
status: z.literal("measured"),
|
|
1597
|
+
metric: NON_EMPTY,
|
|
1598
|
+
baselineValue: FINITE_NUMBER,
|
|
1599
|
+
candidateValue: FINITE_NUMBER,
|
|
1600
|
+
delta: FINITE_NUMBER
|
|
1601
|
+
}).strict().superRefine((effect, ctx) => {
|
|
1602
|
+
const expected = effect.candidateValue - effect.baselineValue;
|
|
1603
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(expected), Math.abs(effect.delta)) * 8;
|
|
1604
|
+
if (Math.abs(effect.delta - expected) > tolerance) ctx.addIssue({
|
|
1605
|
+
code: "custom",
|
|
1606
|
+
message: "delta must equal candidateValue - baselineValue"
|
|
1607
|
+
});
|
|
1608
|
+
}), z.object({
|
|
1609
|
+
status: z.literal("not-measured"),
|
|
1610
|
+
reason: NON_EMPTY
|
|
1611
|
+
}).strict()]);
|
|
1612
|
+
const SurfaceEvidenceSchema = z.object({
|
|
1613
|
+
surfaceId: NON_EMPTY,
|
|
1614
|
+
fired: z.boolean(),
|
|
1615
|
+
firingCount: NON_NEGATIVE_INT,
|
|
1616
|
+
effect: EffectSchema,
|
|
1617
|
+
evidence: z.array(ArtifactRefSchema).min(1)
|
|
1618
|
+
}).strict().superRefine((evidence, ctx) => {
|
|
1619
|
+
if (evidence.fired && evidence.firingCount === 0) ctx.addIssue({
|
|
1620
|
+
code: "custom",
|
|
1621
|
+
message: "a fired surface must have firingCount >= 1"
|
|
1622
|
+
});
|
|
1623
|
+
if (!evidence.fired && evidence.firingCount !== 0) ctx.addIssue({
|
|
1624
|
+
code: "custom",
|
|
1625
|
+
message: "a surface that did not fire must have firingCount 0"
|
|
1626
|
+
});
|
|
1627
|
+
if (!evidence.fired && evidence.effect.status === "measured" && evidence.effect.delta !== 0) ctx.addIssue({
|
|
1628
|
+
code: "custom",
|
|
1629
|
+
message: "a surface that did not fire cannot claim non-zero effect"
|
|
1630
|
+
});
|
|
1631
|
+
});
|
|
1632
|
+
const TaskAttemptedSchema = z.object({
|
|
1633
|
+
...EventBaseShape,
|
|
1634
|
+
kind: z.literal("task-attempted"),
|
|
1635
|
+
candidateId: NON_EMPTY,
|
|
1636
|
+
runId: NON_EMPTY,
|
|
1637
|
+
attemptIndex: NON_NEGATIVE_INT,
|
|
1638
|
+
task: z.object({
|
|
1639
|
+
taskId: NON_EMPTY,
|
|
1640
|
+
source: SourceRefSchema
|
|
1641
|
+
}).strict(),
|
|
1642
|
+
identity: z.object({
|
|
1643
|
+
model: z.object({
|
|
1644
|
+
provider: NON_EMPTY,
|
|
1645
|
+
snapshot: NON_EMPTY.refine(modelHasSnapshot, "model must include an immutable snapshot")
|
|
1646
|
+
}).strict(),
|
|
1647
|
+
agent: SourceRefSchema,
|
|
1648
|
+
benchmark: SourceRefSchema
|
|
1649
|
+
}).strict(),
|
|
1650
|
+
outcome: OutcomeSchema,
|
|
1651
|
+
accounting: AccountingSchema,
|
|
1652
|
+
surfaceEvidence: z.array(SurfaceEvidenceSchema).min(1)
|
|
1653
|
+
}).strict();
|
|
1654
|
+
const SearchOperationRecordedSchema = z.object({
|
|
1655
|
+
...EventBaseShape,
|
|
1656
|
+
kind: z.literal("search-operation-recorded"),
|
|
1657
|
+
operationId: NON_EMPTY,
|
|
1658
|
+
operationKind: OperationKindSchema,
|
|
1659
|
+
execution: z.discriminatedUnion("kind", [z.object({
|
|
1660
|
+
kind: z.literal("model"),
|
|
1661
|
+
model: z.object({
|
|
1662
|
+
provider: NON_EMPTY,
|
|
1663
|
+
snapshot: NON_EMPTY.refine(modelHasSnapshot, "model must include an immutable snapshot")
|
|
1664
|
+
}).strict(),
|
|
1665
|
+
source: SourceRefSchema
|
|
1666
|
+
}).strict(), z.object({
|
|
1667
|
+
kind: z.literal("deterministic"),
|
|
1668
|
+
source: SourceRefSchema
|
|
1669
|
+
}).strict()]),
|
|
1670
|
+
outcome: z.discriminatedUnion("status", [
|
|
1671
|
+
z.object({ status: z.literal("completed") }).strict(),
|
|
1672
|
+
z.object({
|
|
1673
|
+
status: z.literal("partial"),
|
|
1674
|
+
failure: FailureReasonSchema
|
|
1675
|
+
}).strict(),
|
|
1676
|
+
z.object({
|
|
1677
|
+
status: z.literal("failed"),
|
|
1678
|
+
failure: FailureReasonSchema
|
|
1679
|
+
}).strict()
|
|
1680
|
+
]),
|
|
1681
|
+
accounting: AccountingSchema
|
|
1682
|
+
}).strict();
|
|
1683
|
+
const CandidateDecidedSchema = z.object({
|
|
1684
|
+
...EventBaseShape,
|
|
1685
|
+
kind: z.literal("candidate-decided"),
|
|
1686
|
+
candidateId: NON_EMPTY,
|
|
1687
|
+
decision: z.discriminatedUnion("status", [z.object({ status: z.literal("selected") }).strict(), z.object({
|
|
1688
|
+
status: z.literal("rejected"),
|
|
1689
|
+
reason: FailureReasonSchema
|
|
1690
|
+
}).strict()])
|
|
1691
|
+
}).strict();
|
|
1692
|
+
const SearchCompletedSchema = z.object({
|
|
1693
|
+
...EventBaseShape,
|
|
1694
|
+
kind: z.literal("search-completed"),
|
|
1695
|
+
result: z.discriminatedUnion("status", [z.object({
|
|
1696
|
+
status: z.literal("selected"),
|
|
1697
|
+
candidateId: NON_EMPTY
|
|
1698
|
+
}).strict(), z.object({
|
|
1699
|
+
status: z.literal("all-rejected"),
|
|
1700
|
+
reason: FailureReasonSchema
|
|
1701
|
+
}).strict()])
|
|
1702
|
+
}).strict();
|
|
1703
|
+
const EventSchema = z.discriminatedUnion("kind", [
|
|
1704
|
+
SearchPlannedSchema,
|
|
1705
|
+
SearchPlanExtendedSchema,
|
|
1706
|
+
CandidateRegisteredSchema,
|
|
1707
|
+
CandidateSlotClosedSchema,
|
|
1708
|
+
TaskAttemptedSchema,
|
|
1709
|
+
SearchOperationRecordedSchema,
|
|
1710
|
+
CandidateDecidedSchema,
|
|
1711
|
+
SearchCompletedSchema
|
|
1712
|
+
]);
|
|
1713
|
+
const EntrySchema = z.object({
|
|
1714
|
+
schema: z.literal(SEARCH_LEDGER_SCHEMA),
|
|
1715
|
+
campaignId: NON_EMPTY,
|
|
1716
|
+
sequence: NON_NEGATIVE_INT,
|
|
1717
|
+
previousHash: z.union([HASH, z.null()]),
|
|
1718
|
+
event: EventSchema,
|
|
1719
|
+
entryHash: HASH
|
|
1720
|
+
}).strict();
|
|
1721
|
+
/** Validate and return a canonical copy. Arrays whose order is not semantic are
|
|
1722
|
+
* sorted so retries from different processes produce byte-identical events. */
|
|
1723
|
+
function validateSearchLedgerEvent(input) {
|
|
1724
|
+
const parsed = EventSchema.safeParse(input);
|
|
1725
|
+
if (!parsed.success) throw new SearchLedgerError(`invalid search ledger event: ${formatZodError(parsed.error)}`);
|
|
1726
|
+
return normalizeEvent(parsed.data);
|
|
1727
|
+
}
|
|
1728
|
+
/** Open a durable filesystem search ledger. Construction performs no I/O; the
|
|
1729
|
+
* first `append` or `replay` validates the complete existing file. */
|
|
1730
|
+
function openSearchLedger(options) {
|
|
1731
|
+
if (options.path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
|
|
1732
|
+
return new FileSearchLedger(options.path, options.campaignId, options.trustedHead);
|
|
1733
|
+
}
|
|
1734
|
+
function searchLedgerCodec(campaignId) {
|
|
1735
|
+
return {
|
|
1736
|
+
...SEARCH_LEDGER_FILE_CONTEXT,
|
|
1737
|
+
header: {
|
|
1738
|
+
schema: SEARCH_LEDGER_SCHEMA,
|
|
1739
|
+
campaignId
|
|
1740
|
+
},
|
|
1741
|
+
conflictError: (message) => new SearchLedgerConflictError(message),
|
|
1742
|
+
parseEntry: parseSearchLedgerEntry,
|
|
1743
|
+
checkEntryHeader: (entry, index) => {
|
|
1744
|
+
if (entry.campaignId !== campaignId) throw new SearchLedgerIntegrityError(`entry ${index} belongs to campaign ${entry.campaignId}, expected ${campaignId}`);
|
|
1745
|
+
},
|
|
1746
|
+
createProjector: () => createSearchLedgerProjector(campaignId)
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1749
|
+
/** Append-only file-backed search ledger with idempotent writes and replay. */
|
|
1750
|
+
var FileSearchLedger = class {
|
|
1751
|
+
path;
|
|
1752
|
+
campaignId;
|
|
1753
|
+
trustedHeadPath;
|
|
1754
|
+
trustedHeadMode;
|
|
1755
|
+
journal;
|
|
1756
|
+
constructor(path, campaignId, trustedHead = "pin") {
|
|
1757
|
+
if (path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
|
|
1758
|
+
if (campaignId.length === 0) throw new SearchLedgerError("campaignId is empty");
|
|
1759
|
+
if (campaignId.trim() !== campaignId) throw new SearchLedgerError("campaignId must not contain surrounding whitespace");
|
|
1760
|
+
this.campaignId = campaignId;
|
|
1761
|
+
this.trustedHeadMode = trustedHead;
|
|
1762
|
+
this.journal = new FileLedgerJournal(path, searchLedgerCodec(campaignId), { requireTrustedHead: trustedHead === "require" });
|
|
1763
|
+
this.path = this.journal.path;
|
|
1764
|
+
this.trustedHeadPath = this.journal.trustedHeadPath;
|
|
1765
|
+
}
|
|
1766
|
+
async replay() {
|
|
1767
|
+
return (await this.journal.replay()).projection;
|
|
1768
|
+
}
|
|
1769
|
+
async append(input) {
|
|
1770
|
+
const event = validateSearchLedgerEvent(input);
|
|
1771
|
+
const { entry, appended, projection } = await this.journal.append(event, { pinHead: this.trustedHeadMode !== "off" });
|
|
1772
|
+
return {
|
|
1773
|
+
entry,
|
|
1774
|
+
appended,
|
|
1775
|
+
replay: projection
|
|
1776
|
+
};
|
|
1777
|
+
}
|
|
1778
|
+
async trustedHead() {
|
|
1779
|
+
return this.journal.trustedHead();
|
|
1780
|
+
}
|
|
1781
|
+
async pinTrustedHead() {
|
|
1782
|
+
return this.journal.pinTrustedHead();
|
|
1783
|
+
}
|
|
1784
|
+
async clearTrustedHead() {
|
|
1785
|
+
return this.journal.clearTrustedHead();
|
|
1786
|
+
}
|
|
1787
|
+
};
|
|
1788
|
+
function parseSearchLedgerEntry(raw, context) {
|
|
1789
|
+
const parsed = EntrySchema.safeParse(raw);
|
|
1790
|
+
if (!parsed.success) throw new SearchLedgerIntegrityError(`search ledger ${context.path} has a malformed entry at line ${context.line}: ${formatZodError(parsed.error)}`);
|
|
1791
|
+
const entry = parsed.data;
|
|
1792
|
+
if (canonicalString(validateSearchLedgerEvent(entry.event)) !== canonicalString(entry.event)) throw new SearchLedgerIntegrityError(`search ledger ${context.path} has non-canonical event ordering at line ${context.line}`);
|
|
1793
|
+
return entry;
|
|
1794
|
+
}
|
|
1795
|
+
/** Replay the campaign search state machine over chain-verified entries. The
|
|
1796
|
+
* generic journal owns sequence, hash, and eventId-uniqueness checks; this
|
|
1797
|
+
* projector owns every campaign invariant and builds the replay projection. */
|
|
1798
|
+
function createSearchLedgerProjector(campaignId) {
|
|
1799
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
1800
|
+
const plannedSlots = /* @__PURE__ */ new Map();
|
|
1801
|
+
const plannedOperations = /* @__PURE__ */ new Map();
|
|
1802
|
+
const planExtensions = [];
|
|
1803
|
+
const candidateBySlot = /* @__PURE__ */ new Map();
|
|
1804
|
+
const closedSlots = /* @__PURE__ */ new Map();
|
|
1805
|
+
const lineageNodes = /* @__PURE__ */ new Map();
|
|
1806
|
+
const runIds = /* @__PURE__ */ new Set();
|
|
1807
|
+
const attemptKeys = /* @__PURE__ */ new Set();
|
|
1808
|
+
const candidateEvents = [];
|
|
1809
|
+
const closedSlotEvents = [];
|
|
1810
|
+
const attempts = [];
|
|
1811
|
+
const operationEvents = [];
|
|
1812
|
+
const operationsById = /* @__PURE__ */ new Map();
|
|
1813
|
+
const decisions = [];
|
|
1814
|
+
let planEvent = null;
|
|
1815
|
+
let completion = null;
|
|
1816
|
+
let previousOccurredAt = Number.NEGATIVE_INFINITY;
|
|
1817
|
+
const apply = (entry, index) => {
|
|
1818
|
+
const event = entry.event;
|
|
1819
|
+
if (completion) throw new SearchLedgerIntegrityError(`event ${event.eventId} appears after terminal event ${completion.eventId}`);
|
|
1820
|
+
const occurredAt = Date.parse(event.occurredAt);
|
|
1821
|
+
if (occurredAt < previousOccurredAt) throw new SearchLedgerIntegrityError(`event ${event.eventId} occurred before the preceding durable event`);
|
|
1822
|
+
previousOccurredAt = occurredAt;
|
|
1823
|
+
assertUnique(event.artifacts.map(artifactKey), "artifact receipt", event.eventId);
|
|
1824
|
+
if (event.kind === "search-planned") {
|
|
1825
|
+
if (index !== 0 || planEvent) throw new SearchLedgerIntegrityError("search plan must be the first and only plan event");
|
|
1826
|
+
assertUnique(event.plan.candidateSlots.map((slot) => slot.slotId), "candidate slot", event.eventId);
|
|
1827
|
+
assertUnique(event.plan.tasks.map((task) => task.taskId), "planned taskId", event.eventId);
|
|
1828
|
+
assertUnique(event.plan.operations.map((operation) => operation.operationId), "planned operationId", event.eventId);
|
|
1829
|
+
for (const operation of event.plan.operations) plannedOperations.set(operation.operationId, operation);
|
|
1830
|
+
for (const slot of event.plan.candidateSlots) {
|
|
1831
|
+
assertSlotGenerationOperation(slot, plannedOperations);
|
|
1832
|
+
plannedSlots.set(slot.slotId, slot);
|
|
1833
|
+
}
|
|
1834
|
+
planEvent = event;
|
|
1835
|
+
return;
|
|
1836
|
+
}
|
|
1837
|
+
if (!planEvent) throw new SearchLedgerIntegrityError(`event ${event.eventId} appears before the required search plan`);
|
|
1838
|
+
if (event.kind === "search-plan-extended") {
|
|
1839
|
+
assertUnique(event.extension.candidateSlots.map((slot) => slot.slotId), "candidate slot", event.eventId);
|
|
1840
|
+
assertUnique(event.extension.operations.map((operation) => operation.operationId), "planned operationId", event.eventId);
|
|
1841
|
+
for (const operation of event.extension.operations) {
|
|
1842
|
+
if (plannedOperations.has(operation.operationId)) throw new SearchLedgerIntegrityError(`plan extension ${event.eventId} re-plans operation ${operation.operationId}`);
|
|
1843
|
+
plannedOperations.set(operation.operationId, operation);
|
|
1844
|
+
}
|
|
1845
|
+
for (const slot of event.extension.candidateSlots) {
|
|
1846
|
+
if (plannedSlots.has(slot.slotId)) throw new SearchLedgerIntegrityError(`plan extension ${event.eventId} re-plans candidate slot ${slot.slotId}`);
|
|
1847
|
+
assertSlotGenerationOperation(slot, plannedOperations);
|
|
1848
|
+
plannedSlots.set(slot.slotId, slot);
|
|
1849
|
+
}
|
|
1850
|
+
planExtensions.push(event);
|
|
1851
|
+
return;
|
|
1852
|
+
}
|
|
1853
|
+
if (event.kind === "candidate-registered") {
|
|
1854
|
+
if (candidates.has(event.candidateId)) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was registered twice`);
|
|
1855
|
+
const plannedSlot = plannedSlots.get(event.slotId);
|
|
1856
|
+
if (!plannedSlot) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} binds unknown slot ${event.slotId}`);
|
|
1857
|
+
if (candidateBySlot.has(event.slotId)) throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was bound twice`);
|
|
1858
|
+
if (closedSlots.has(event.slotId)) throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was already closed`);
|
|
1859
|
+
if (event.generationOperationId !== plannedSlot.generationOperationId) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`);
|
|
1860
|
+
const generationOperation = operationsById.get(event.generationOperationId);
|
|
1861
|
+
if (!generationOperation) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} precedes generation operation ${event.generationOperationId}`);
|
|
1862
|
+
if (generationOperation.outcome.status === "failed") throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} cannot bind failed generation operation ${event.generationOperationId}`);
|
|
1863
|
+
const previousCandidate = lineageNodes.get(event.lineage.lineageNodeId);
|
|
1864
|
+
if (previousCandidate) throw new SearchLedgerIntegrityError(`lineage node ${event.lineage.lineageNodeId} is already bound to ${previousCandidate}`);
|
|
1865
|
+
assertUnique(event.lineage.parentCandidateIds, "parentCandidateId", event.eventId);
|
|
1866
|
+
const parents = event.lineage.parentCandidateIds.map((id) => {
|
|
1867
|
+
const parent = candidates.get(id);
|
|
1868
|
+
if (!parent) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} references unknown parent ${id}`);
|
|
1869
|
+
return parent;
|
|
1870
|
+
});
|
|
1871
|
+
const expectedGeneration = parents.length === 0 ? 0 : Math.max(...parents.map((parent) => parent.registered.lineage.generation)) + 1;
|
|
1872
|
+
if (event.lineage.generation !== expectedGeneration) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} generation ${event.lineage.generation} does not follow its parents (expected ${expectedGeneration})`);
|
|
1873
|
+
assertUnique(event.surfaces.map((surface) => surface.surfaceId), "surfaceId", event.eventId);
|
|
1874
|
+
candidates.set(event.candidateId, {
|
|
1875
|
+
registered: event,
|
|
1876
|
+
attempts: [],
|
|
1877
|
+
decision: null
|
|
1878
|
+
});
|
|
1879
|
+
candidateBySlot.set(event.slotId, event.candidateId);
|
|
1880
|
+
lineageNodes.set(event.lineage.lineageNodeId, event.candidateId);
|
|
1881
|
+
candidateEvents.push(event);
|
|
1882
|
+
return;
|
|
1883
|
+
}
|
|
1884
|
+
if (event.kind === "task-attempted") {
|
|
1885
|
+
const candidate = candidates.get(event.candidateId);
|
|
1886
|
+
if (!candidate) throw new SearchLedgerIntegrityError(`attempt ${event.eventId} references unknown candidate ${event.candidateId}`);
|
|
1887
|
+
if (candidate.decision) throw new SearchLedgerIntegrityError(`attempt ${event.eventId} appears after candidate ${event.candidateId} was decided`);
|
|
1888
|
+
const plannedTask = planEvent.plan.tasks.find((task) => task.taskId === event.task.taskId);
|
|
1889
|
+
if (!plannedTask) throw new SearchLedgerIntegrityError(`attempt ${event.eventId} references unplanned task ${event.task.taskId}`);
|
|
1890
|
+
if (canonicalString(plannedTask.source) !== canonicalString(event.task.source) || canonicalString(plannedTask.benchmark) !== canonicalString(event.identity.benchmark)) throw new SearchLedgerIntegrityError(`task ${event.task.taskId} does not match its planned source identity`);
|
|
1891
|
+
if (event.attemptIndex >= plannedTask.maxAttempts) throw new SearchLedgerIntegrityError(`task ${event.task.taskId} attempt ${event.attemptIndex} exceeds planned maxAttempts ${plannedTask.maxAttempts}`);
|
|
1892
|
+
if (runIds.has(event.runId)) throw new SearchLedgerIntegrityError(`runId ${event.runId} was recorded twice`);
|
|
1893
|
+
runIds.add(event.runId);
|
|
1894
|
+
const attemptKey = canonicalString([
|
|
1895
|
+
event.candidateId,
|
|
1896
|
+
event.task.taskId,
|
|
1897
|
+
event.attemptIndex
|
|
1898
|
+
]);
|
|
1899
|
+
if (attemptKeys.has(attemptKey)) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} task ${event.task.taskId} attempt ${event.attemptIndex} was recorded twice`);
|
|
1900
|
+
const expectedAttemptIndex = candidate.attempts.filter((attempt) => attempt.task.taskId === event.task.taskId).length;
|
|
1901
|
+
if (event.attemptIndex !== expectedAttemptIndex) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} task ${event.task.taskId} attempt index ${event.attemptIndex} is not contiguous (expected ${expectedAttemptIndex})`);
|
|
1902
|
+
const previousAttempt = candidate.attempts.find((attempt) => attempt.task.taskId === event.task.taskId);
|
|
1903
|
+
if (previousAttempt?.outcome.status !== void 0 && previousAttempt.outcome.status !== "errored") throw new SearchLedgerIntegrityError(`task ${event.task.taskId} was retried after a measured outcome`);
|
|
1904
|
+
if (previousAttempt && canonicalString({
|
|
1905
|
+
task: previousAttempt.task,
|
|
1906
|
+
identity: previousAttempt.identity
|
|
1907
|
+
}) !== canonicalString({
|
|
1908
|
+
task: event.task,
|
|
1909
|
+
identity: event.identity
|
|
1910
|
+
})) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} task ${event.task.taskId} changed immutable execution identity between attempts`);
|
|
1911
|
+
attemptKeys.add(attemptKey);
|
|
1912
|
+
const declared = candidate.registered.surfaces.map((surface) => surface.surfaceId).sort();
|
|
1913
|
+
const observed = event.surfaceEvidence.map((surface) => surface.surfaceId).sort();
|
|
1914
|
+
assertUnique(observed, "surface evidence", event.eventId);
|
|
1915
|
+
for (const evidence of event.surfaceEvidence) assertUnique(evidence.evidence.map(artifactKey), "surface evidence receipt", event.eventId);
|
|
1916
|
+
if (canonicalString(declared) !== canonicalString(observed)) throw new SearchLedgerIntegrityError(`attempt ${event.eventId} surface evidence does not exactly cover candidate ${event.candidateId}`);
|
|
1917
|
+
candidate.attempts.push(event);
|
|
1918
|
+
attempts.push(event);
|
|
1919
|
+
return;
|
|
1920
|
+
}
|
|
1921
|
+
if (event.kind === "search-operation-recorded") {
|
|
1922
|
+
const plannedOperation = plannedOperations.get(event.operationId);
|
|
1923
|
+
if (!plannedOperation) throw new SearchLedgerIntegrityError(`operation ${event.operationId} was not declared in the search plan`);
|
|
1924
|
+
if (plannedOperation.kind !== event.operationKind) throw new SearchLedgerIntegrityError(`operation ${event.operationId} kind ${event.operationKind} does not match planned ${plannedOperation.kind}`);
|
|
1925
|
+
if (operationsById.has(event.operationId)) throw new SearchLedgerIntegrityError(`operation ${event.operationId} was recorded twice`);
|
|
1926
|
+
operationsById.set(event.operationId, event);
|
|
1927
|
+
operationEvents.push(event);
|
|
1928
|
+
return;
|
|
1929
|
+
}
|
|
1930
|
+
if (event.kind === "candidate-slot-closed") {
|
|
1931
|
+
const plannedSlot = plannedSlots.get(event.slotId);
|
|
1932
|
+
if (!plannedSlot) throw new SearchLedgerIntegrityError(`candidate slot closure ${event.eventId} references unknown slot ${event.slotId}`);
|
|
1933
|
+
if (event.generationOperationId !== plannedSlot.generationOperationId) throw new SearchLedgerIntegrityError(`candidate slot closure ${event.eventId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`);
|
|
1934
|
+
if (candidateBySlot.has(event.slotId)) throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was already bound to a candidate`);
|
|
1935
|
+
if (closedSlots.has(event.slotId)) throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was closed twice`);
|
|
1936
|
+
const operation = operationsById.get(event.generationOperationId);
|
|
1937
|
+
if (!operation) throw new SearchLedgerIntegrityError(`candidate slot closure ${event.eventId} precedes operation ${event.generationOperationId}`);
|
|
1938
|
+
if (operation.outcome.status === "completed") throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} cannot close from completed operation ${event.generationOperationId}`);
|
|
1939
|
+
closedSlots.set(event.slotId, event);
|
|
1940
|
+
closedSlotEvents.push(event);
|
|
1941
|
+
return;
|
|
1942
|
+
}
|
|
1943
|
+
if (event.kind === "candidate-decided") {
|
|
1944
|
+
const candidate = candidates.get(event.candidateId);
|
|
1945
|
+
if (!candidate) throw new SearchLedgerIntegrityError(`decision ${event.eventId} references unknown candidate ${event.candidateId}`);
|
|
1946
|
+
if (candidate.decision) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was decided twice`);
|
|
1947
|
+
if (event.decision.status === "selected") {
|
|
1948
|
+
if (!candidate.attempts.some((attempt) => attempt.outcome.status !== "errored")) throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} cannot be selected without a measured task outcome`);
|
|
1949
|
+
if (decisions.some((decision) => decision.decision.status === "selected")) throw new SearchLedgerIntegrityError("more than one candidate was selected");
|
|
1950
|
+
}
|
|
1951
|
+
candidate.decision = event;
|
|
1952
|
+
decisions.push(event);
|
|
1953
|
+
return;
|
|
1954
|
+
}
|
|
1955
|
+
const missingCandidateSlots = [...plannedSlots.values()].filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId);
|
|
1956
|
+
if (missingCandidateSlots.length > 0) throw new SearchLedgerIntegrityError(`search completed with missing candidate slots: ${missingCandidateSlots.join(", ")}`);
|
|
1957
|
+
const missingTaskOutcomes = plannedTaskOutcomeKeys(planEvent, candidates);
|
|
1958
|
+
if (missingTaskOutcomes.length > 0) throw new SearchLedgerIntegrityError(`search completed with missing task outcomes: ${missingTaskOutcomes.join(", ")}`);
|
|
1959
|
+
const missingOperations = [...plannedOperations.values()].filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId);
|
|
1960
|
+
if (missingOperations.length > 0) throw new SearchLedgerIntegrityError(`search completed with missing search operations: ${missingOperations.join(", ")}`);
|
|
1961
|
+
for (const operation of plannedOperations.values()) {
|
|
1962
|
+
if (operation.kind !== "candidate-generation") continue;
|
|
1963
|
+
const generationOutcome = operationsById.get(operation.operationId).outcome.status;
|
|
1964
|
+
const slots = [...plannedSlots.values()].filter((slot) => slot.generationOperationId === operation.operationId);
|
|
1965
|
+
if (slots.length === 0) continue;
|
|
1966
|
+
const registeredCount = slots.filter((slot) => candidateBySlot.has(slot.slotId)).length;
|
|
1967
|
+
const closedCount = slots.filter((slot) => closedSlots.has(slot.slotId)).length;
|
|
1968
|
+
if (generationOutcome === "completed" && closedCount > 0) throw new SearchLedgerIntegrityError(`completed generation operation ${operation.operationId} contains ${closedCount} closed slot(s)`);
|
|
1969
|
+
if (generationOutcome === "failed" && registeredCount > 0) throw new SearchLedgerIntegrityError(`failed generation operation ${operation.operationId} contains ${registeredCount} registered candidate(s)`);
|
|
1970
|
+
if (generationOutcome === "partial" && (registeredCount === 0 || closedCount === 0)) throw new SearchLedgerIntegrityError(`partial generation operation ${operation.operationId} must contain both a registered candidate and a closed slot`);
|
|
1971
|
+
}
|
|
1972
|
+
const pending = [...candidates.values()].filter((candidate) => candidate.decision === null);
|
|
1973
|
+
if (pending.length > 0) throw new SearchLedgerIntegrityError(`search completed with ${pending.length} candidate decision(s) missing`);
|
|
1974
|
+
const selected = decisions.filter((decision) => decision.decision.status === "selected");
|
|
1975
|
+
if (event.result.status === "selected") {
|
|
1976
|
+
if (selected.length !== 1 || selected[0].candidateId !== event.result.candidateId) throw new SearchLedgerIntegrityError(`search completion winner ${event.result.candidateId} does not match candidate decisions`);
|
|
1977
|
+
} else if (selected.length !== 0) throw new SearchLedgerIntegrityError("all-rejected completion contains a selected candidate");
|
|
1978
|
+
completion = event;
|
|
1979
|
+
};
|
|
1980
|
+
const finish = (entries) => {
|
|
1981
|
+
const selectedDecisions = decisions.filter((decision) => decision.decision.status === "selected");
|
|
1982
|
+
const rejectedDecisions = decisions.filter((decision) => decision.decision.status === "rejected");
|
|
1983
|
+
const outcomeCounts = {
|
|
1984
|
+
passed: 0,
|
|
1985
|
+
failed: 0,
|
|
1986
|
+
errored: 0
|
|
1987
|
+
};
|
|
1988
|
+
const operationOutcomeCounts = {
|
|
1989
|
+
completed: 0,
|
|
1990
|
+
partial: 0,
|
|
1991
|
+
failed: 0
|
|
1992
|
+
};
|
|
1993
|
+
let inputTokens = 0;
|
|
1994
|
+
let outputTokens = 0;
|
|
1995
|
+
let cachedTokens = 0;
|
|
1996
|
+
let costUsd = 0;
|
|
1997
|
+
const unknownTokenEventIds = [];
|
|
1998
|
+
const unknownCostEventIds = [];
|
|
1999
|
+
for (const attempt of attempts) outcomeCounts[attempt.outcome.status] += 1;
|
|
2000
|
+
for (const operation of operationEvents) operationOutcomeCounts[operation.outcome.status] += 1;
|
|
2001
|
+
for (const costedEvent of [...attempts, ...operationEvents]) {
|
|
2002
|
+
if (costedEvent.accounting.tokens.status === "known") {
|
|
2003
|
+
inputTokens += costedEvent.accounting.tokens.inputTokens;
|
|
2004
|
+
outputTokens += costedEvent.accounting.tokens.outputTokens;
|
|
2005
|
+
cachedTokens += costedEvent.accounting.tokens.cachedTokens;
|
|
2006
|
+
} else unknownTokenEventIds.push(costedEvent.eventId);
|
|
2007
|
+
if (costedEvent.accounting.cost.status === "known") costUsd += costedEvent.accounting.cost.usd;
|
|
2008
|
+
else {
|
|
2009
|
+
costUsd += costedEvent.accounting.cost.knownLowerBoundUsd;
|
|
2010
|
+
unknownCostEventIds.push(costedEvent.eventId);
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
const accounting = unknownTokenEventIds.length === 0 && unknownCostEventIds.length === 0 ? {
|
|
2014
|
+
status: "known",
|
|
2015
|
+
inputTokens,
|
|
2016
|
+
outputTokens,
|
|
2017
|
+
cachedTokens,
|
|
2018
|
+
costUsd
|
|
2019
|
+
} : {
|
|
2020
|
+
status: "partial",
|
|
2021
|
+
knownInputTokens: inputTokens,
|
|
2022
|
+
knownOutputTokens: outputTokens,
|
|
2023
|
+
knownCachedTokens: cachedTokens,
|
|
2024
|
+
knownCostUsd: costUsd,
|
|
2025
|
+
unknownTokenEventIds,
|
|
2026
|
+
unknownCostEventIds
|
|
2027
|
+
};
|
|
2028
|
+
const selectedCandidateId = completion?.result.status === "selected" ? completion.result.candidateId : null;
|
|
2029
|
+
const status = completion?.result.status === "selected" ? "selected" : completion?.result.status === "all-rejected" ? "all-rejected" : "in-progress";
|
|
2030
|
+
const missingCandidateSlots = [...plannedSlots.values()].filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId);
|
|
2031
|
+
const missingTaskOutcomes = planEvent ? plannedTaskOutcomeKeys(planEvent, candidates) : [];
|
|
2032
|
+
const missingOperations = [...plannedOperations.values()].filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId);
|
|
2033
|
+
return {
|
|
2034
|
+
entries: [...entries],
|
|
2035
|
+
plan: planEvent,
|
|
2036
|
+
planExtensions,
|
|
2037
|
+
candidates: candidateEvents,
|
|
2038
|
+
closedCandidateSlots: closedSlotEvents,
|
|
2039
|
+
attempts,
|
|
2040
|
+
operations: operationEvents,
|
|
2041
|
+
decisions,
|
|
2042
|
+
completion,
|
|
2043
|
+
audit: {
|
|
2044
|
+
campaignId,
|
|
2045
|
+
eventCount: entries.length,
|
|
2046
|
+
candidateCount: candidates.size,
|
|
2047
|
+
closedCandidateSlotCount: closedSlots.size,
|
|
2048
|
+
attemptCount: attempts.length,
|
|
2049
|
+
operationCount: operationEvents.length,
|
|
2050
|
+
outcomes: outcomeCounts,
|
|
2051
|
+
operationOutcomes: operationOutcomeCounts,
|
|
2052
|
+
decisions: {
|
|
2053
|
+
selected: selectedDecisions.length,
|
|
2054
|
+
rejected: rejectedDecisions.length,
|
|
2055
|
+
pending: candidates.size - decisions.length
|
|
2056
|
+
},
|
|
2057
|
+
expected: {
|
|
2058
|
+
candidateSlots: plannedSlots.size,
|
|
2059
|
+
taskOutcomes: candidates.size * (planEvent?.plan.tasks.length ?? 0),
|
|
2060
|
+
operations: plannedOperations.size,
|
|
2061
|
+
missingCandidateSlots,
|
|
2062
|
+
missingTaskOutcomes,
|
|
2063
|
+
missingOperations
|
|
2064
|
+
},
|
|
2065
|
+
status,
|
|
2066
|
+
selectedCandidateId,
|
|
2067
|
+
accounting,
|
|
2068
|
+
headHash: entries.at(-1)?.entryHash ?? null
|
|
2069
|
+
}
|
|
2070
|
+
};
|
|
2071
|
+
};
|
|
2072
|
+
return {
|
|
2073
|
+
apply,
|
|
2074
|
+
finish
|
|
2075
|
+
};
|
|
2076
|
+
}
|
|
2077
|
+
/** Every candidate slot must name a planned candidate-generation operation,
|
|
2078
|
+
* whether it arrives with the plan or with a later extension. */
|
|
2079
|
+
function assertSlotGenerationOperation(slot, plannedOperations) {
|
|
2080
|
+
if (plannedOperations.get(slot.generationOperationId)?.kind !== "candidate-generation") throw new SearchLedgerIntegrityError(`candidate slot ${slot.slotId} references unplanned candidate-generation operation ${slot.generationOperationId}`);
|
|
2081
|
+
}
|
|
2082
|
+
function plannedTaskOutcomeKeys(planEvent, candidates) {
|
|
2083
|
+
const missing = [];
|
|
2084
|
+
const registeredCandidates = [...candidates.values()].sort((a, b) => compareStrings(a.registered.slotId, b.registered.slotId));
|
|
2085
|
+
for (const candidate of registeredCandidates) {
|
|
2086
|
+
const slotId = candidate.registered.slotId;
|
|
2087
|
+
for (const task of planEvent.plan.tasks) if (!candidate.attempts.some((attempt) => attempt.task.taskId === task.taskId && attempt.outcome.status !== "errored")) missing.push(`${slotId}/${task.taskId}`);
|
|
2088
|
+
}
|
|
2089
|
+
return missing;
|
|
2090
|
+
}
|
|
2091
|
+
function normalizeEvent(event) {
|
|
2092
|
+
const artifacts = sortArtifacts(event.artifacts);
|
|
2093
|
+
if (event.kind === "search-planned") return {
|
|
2094
|
+
...event,
|
|
2095
|
+
artifacts,
|
|
2096
|
+
plan: {
|
|
2097
|
+
candidateSlots: [...event.plan.candidateSlots].sort((a, b) => compareStrings(a.slotId, b.slotId)),
|
|
2098
|
+
tasks: [...event.plan.tasks].sort((a, b) => compareStrings(a.taskId, b.taskId)),
|
|
2099
|
+
operations: [...event.plan.operations].sort((a, b) => compareStrings(a.operationId, b.operationId))
|
|
2100
|
+
}
|
|
2101
|
+
};
|
|
2102
|
+
if (event.kind === "search-plan-extended") return {
|
|
2103
|
+
...event,
|
|
2104
|
+
artifacts,
|
|
2105
|
+
extension: {
|
|
2106
|
+
candidateSlots: [...event.extension.candidateSlots].sort((a, b) => compareStrings(a.slotId, b.slotId)),
|
|
2107
|
+
operations: [...event.extension.operations].sort((a, b) => compareStrings(a.operationId, b.operationId))
|
|
2108
|
+
}
|
|
2109
|
+
};
|
|
2110
|
+
if (event.kind === "candidate-registered") return {
|
|
2111
|
+
...event,
|
|
2112
|
+
artifacts,
|
|
2113
|
+
lineage: {
|
|
2114
|
+
...event.lineage,
|
|
2115
|
+
parentCandidateIds: sortedStrings(event.lineage.parentCandidateIds)
|
|
2116
|
+
},
|
|
2117
|
+
surfaces: [...event.surfaces].map((surface) => ({
|
|
2118
|
+
...surface,
|
|
2119
|
+
artifact: { ...surface.artifact }
|
|
2120
|
+
})).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
|
|
2121
|
+
};
|
|
2122
|
+
if (event.kind === "task-attempted") return {
|
|
2123
|
+
...event,
|
|
2124
|
+
artifacts,
|
|
2125
|
+
surfaceEvidence: [...event.surfaceEvidence].map((evidence) => ({
|
|
2126
|
+
...evidence,
|
|
2127
|
+
evidence: sortArtifacts(evidence.evidence)
|
|
2128
|
+
})).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
|
|
2129
|
+
};
|
|
2130
|
+
return {
|
|
2131
|
+
...event,
|
|
2132
|
+
artifacts
|
|
2133
|
+
};
|
|
2134
|
+
}
|
|
2135
|
+
function sortArtifacts(artifacts) {
|
|
2136
|
+
return [...artifacts].map((artifact) => ({ ...artifact })).sort((a, b) => compareStrings(artifactKey(a), artifactKey(b)));
|
|
2137
|
+
}
|
|
2138
|
+
function artifactKey(artifact) {
|
|
2139
|
+
return canonicalString(artifact);
|
|
2140
|
+
}
|
|
2141
|
+
function compareStrings(a, b) {
|
|
2142
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
2143
|
+
}
|
|
2144
|
+
function sortedStrings(values) {
|
|
2145
|
+
return [...values].sort();
|
|
2146
|
+
}
|
|
2147
|
+
function assertUnique(values, label, eventId) {
|
|
2148
|
+
if (new Set(values).size !== values.length) throw new SearchLedgerIntegrityError(`event ${eventId} contains duplicate ${label} values`);
|
|
2149
|
+
}
|
|
2150
|
+
function formatZodError(error) {
|
|
2151
|
+
return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.join(".") : "<root>"}: ${issue.message}`).join("; ");
|
|
2152
|
+
}
|
|
2153
|
+
//#endregion
|
|
1411
2154
|
//#region src/campaign/gepa-optimization-method.ts
|
|
1412
2155
|
/**
|
|
1413
2156
|
* Turn an optional GEPA installation into an `OptimizationMethod`.
|
|
@@ -1662,14 +2405,32 @@ function gepaOptimizationMethod(config) {
|
|
|
1662
2405
|
resumed: result.resumed
|
|
1663
2406
|
});
|
|
1664
2407
|
if (callback.evaluations() !== result.totalEvaluations) throw new Error(`${name}: GEPA reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`);
|
|
2408
|
+
let searchHistory;
|
|
1665
2409
|
if (result.candidatePopulation) {
|
|
1666
|
-
const population = readGepaCandidatePopulationArtifact({
|
|
2410
|
+
const population = readGepaCandidatePopulationArtifact({
|
|
2411
|
+
summary: result.candidatePopulation,
|
|
2412
|
+
storage
|
|
2413
|
+
});
|
|
1667
2414
|
const selected = population.candidates[population.bestIndex];
|
|
1668
2415
|
const selectedHash = contentHash({
|
|
1669
2416
|
kind: "external-text-candidate",
|
|
1670
2417
|
candidate: result.bestCandidate
|
|
1671
2418
|
});
|
|
1672
2419
|
if (selected?.candidateHash !== selectedHash) throw new Error(`${name}: GEPA candidate population identifies a different winner`);
|
|
2420
|
+
if (config.searchLedger) searchHistory = await recordCandidatePopulationSearch({
|
|
2421
|
+
ledger: openSearchLedger({
|
|
2422
|
+
path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`,
|
|
2423
|
+
campaignId: runId
|
|
2424
|
+
}),
|
|
2425
|
+
storage,
|
|
2426
|
+
runDir,
|
|
2427
|
+
identity: config.searchLedger.identity,
|
|
2428
|
+
population,
|
|
2429
|
+
scenarios: input.selectionScenarios,
|
|
2430
|
+
generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd),
|
|
2431
|
+
producerId: name,
|
|
2432
|
+
runId
|
|
2433
|
+
});
|
|
1673
2434
|
}
|
|
1674
2435
|
const evaluationCost = costFromLedgerSummary(costLedger.summary({
|
|
1675
2436
|
phase: "gepa.external-evaluation",
|
|
@@ -1712,6 +2473,7 @@ function gepaOptimizationMethod(config) {
|
|
|
1712
2473
|
const externalTotalCostUsd = evaluationCost.totalCostUsd + reportedProposerCost;
|
|
1713
2474
|
return {
|
|
1714
2475
|
winnerSurface: decodeExternalTextCandidate(result.bestCandidate),
|
|
2476
|
+
...searchHistory ? { searchHistory } : {},
|
|
1715
2477
|
cost: modelProxy ? meteredCost : {
|
|
1716
2478
|
totalCostUsd: externalTotalCostUsd,
|
|
1717
2479
|
costProvenance: result.proposerCostAccounting === "reported" && evaluationCost.costProvenance.kind !== "uncaptured" ? {
|
|
@@ -1752,6 +2514,30 @@ function gepaOptimizationMethod(config) {
|
|
|
1752
2514
|
}
|
|
1753
2515
|
};
|
|
1754
2516
|
}
|
|
2517
|
+
/** Spend the optimizer booked to its own candidate generation. Unknown stays
|
|
2518
|
+
* unknown: the bridge reports proposer cost only when the engine measured it. */
|
|
2519
|
+
function optimizerAccounting(tokenUsage, proposerCostUsd) {
|
|
2520
|
+
return {
|
|
2521
|
+
tokens: tokenUsage?.inputTokens === void 0 || tokenUsage.outputTokens === void 0 ? {
|
|
2522
|
+
status: "unknown",
|
|
2523
|
+
reason: "the optimizer bridge reported no token usage"
|
|
2524
|
+
} : {
|
|
2525
|
+
status: "known",
|
|
2526
|
+
inputTokens: tokenUsage.inputTokens,
|
|
2527
|
+
outputTokens: tokenUsage.outputTokens,
|
|
2528
|
+
cachedTokens: 0
|
|
2529
|
+
},
|
|
2530
|
+
cost: proposerCostUsd === void 0 ? {
|
|
2531
|
+
status: "unknown",
|
|
2532
|
+
knownLowerBoundUsd: 0,
|
|
2533
|
+
reason: "the optimizer bridge reported no proposer cost"
|
|
2534
|
+
} : {
|
|
2535
|
+
status: "known",
|
|
2536
|
+
usd: proposerCostUsd,
|
|
2537
|
+
source: "provider"
|
|
2538
|
+
}
|
|
2539
|
+
};
|
|
2540
|
+
}
|
|
1755
2541
|
const MAX_TIMER_DELAY_MS = 2147483647;
|
|
1756
2542
|
function snapshotSkillOptOptimizationConfig(config) {
|
|
1757
2543
|
const runner = snapshotExternalOptimizerRunner(config.runner);
|
|
@@ -2095,6 +2881,6 @@ function skillOptOptimizationMethod(config) {
|
|
|
2095
2881
|
};
|
|
2096
2882
|
}
|
|
2097
2883
|
//#endregion
|
|
2098
|
-
export {
|
|
2884
|
+
export { validateSearchLedgerEvent as a, externalTextOptimizationMethod as c, REFERENCE_EQUIVALENCE_INPUT_LIMITS as d, REFERENCE_EQUIVALENCE_JUDGE_VERSION as f, openSearchLedger as i, decodeExternalTextCandidate as l, runReferenceEquivalenceJudge as m, gepaOptimizationMethod as n, heldOutGate as o, createReferenceEquivalenceJudge as p, FileSearchLedger as r, composeGate as s, skillOptOptimizationMethod as t, readExternalOptimizerObservationArtifact as u };
|
|
2099
2885
|
|
|
2100
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
2886
|
+
//# sourceMappingURL=skillopt-optimization-method-DbaekMcn.js.map
|