@tangle-network/agent-eval 0.216.0 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +141 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-DTt8uGP9.d.ts} +5 -18
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +37 -170
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-DqPJaJ4X.js} +763 -555
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-DyeNcenW.d.ts} +18 -16
- package/dist/analyze-runs-DyeNcenW.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-xQK8nOI6.js} +591 -109
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +8 -8
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BQ-Jujuj.js → campaign-CJGkxAQ6.js} +256 -765
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-OcUciD-N.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-BsZVgBhW.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-DthPOYGd.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-BXcZw-jo.js → claim-BPVuyHuy.js} +4 -4
- package/dist/claim-BPVuyHuy.js.map +1 -0
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +13 -13
- package/dist/contract/index.js +1041 -1473
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-D0352Z4d.js} +832 -817
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/define-agent-eval-BDxNlP3D.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +9 -9
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-CnU8Yq29.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-BvZ0R-36.d.ts} +18 -22
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-Dngq5h35.js → errors-BcOs5MKW.js} +6 -8
- package/dist/{errors-Dngq5h35.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-jTjC_QQD.d.ts} +8 -8
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +331 -33
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +303 -101
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-XIHBOSKt.js} +551 -553
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +8 -13
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-zQBG9hHU.js → findings-store-BMVQtEsm.js} +13 -255
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +20 -63
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-BLGqZ81N.d.ts} +11 -16
- package/dist/{index-oQ4aHziK.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DlH0Y9gI.d.ts → index-BiT6RtNH.d.ts} +31 -43
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-CX-GAUpw.d.ts} +4 -8
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-CPJJr98o.d.ts → index-CiRwLv2R.d.ts} +5 -68
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-NvGBiO8H.d.ts → index-ZuWx52WG.d.ts} +266 -495
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +2256 -2165
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +514 -1192
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-DOd9Q86O.js} +7 -12
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BvnVARVl.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BvnVARVl.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-CRHwS9iz.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/interval-Dq9gt7jx.js +219 -0
- package/dist/interval-Dq9gt7jx.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/judge-gate-Co5tuh57.js.map +1 -0
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-DsL0yFcb.d.ts} +9 -10
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-DQveoxSM.js → llm-client-BshCREn6.js} +90 -219
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-C8jMTNNG.js} +501 -931
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +4 -4
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-_BV9-Lk5.js → matrix-B8l-Nsxd.js} +7 -96
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DHe-dwVJ.d.ts} +10 -19
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-BbA_JCzW.js} +45 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -6
- package/dist/meta-eval/index.js +10 -10
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/mint-BRZXuwRi.d.ts +81 -0
- package/dist/mint-BRZXuwRi.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-Csor1BLJ.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +5 -8
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-BzsxidFD.js} +883 -861
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-BWG2DQIk.d.ts} +4 -22
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-DaY95yU6.d.ts} +105 -106
- package/dist/promotion-gate-DaY95yU6.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-B1D1qp2Z.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-Cveoq6fJ.d.ts → redact-DXdlFb_U.d.ts} +4 -21
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-Dk1ZdJEm.js → redact-mtkTiJVk.js} +92 -98
- package/dist/redact-mtkTiJVk.js.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-ssNoHzwD.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-y8H6RSX8.js → render-C-w2EZ6p.js} +331 -554
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-f6mZhh58.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/reward-DrZ71vnf.d.ts +55 -0
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -4
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-D9bKKBnZ.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-CaxbLq6Y.d.ts} +5 -11
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-ES07wfz9.js → run-record-CysKLB93.js} +3 -5
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-DlTZ_lqe.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +39 -53
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DxqjBxA-.d.ts} +18 -31
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-TUvB5gWH.js} +189 -199
- package/dist/search-ledger-TUvB5gWH.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-CBvz3u_S.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-D-1PPCmM.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-kyL94-ri.js} +428 -91
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-BteDAbHK.js} +135 -132
- package/dist/store-otlp-BteDAbHK.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-D5l_OfTJ.js} +9 -9
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DZS0HWVf.d.ts} +157 -165
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +21 -19
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-BsXJoYX5.js} +588 -598
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-DM7tlml1.d.ts} +16 -20
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +189 -153
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +58 -450
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-Bp_Ac0p1.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-78g6E4vh.js} +17 -13
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-CkjzWmeY.d.ts} +53 -56
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-xxwgsuMi.d.ts} +5 -10
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +15 -212
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -987
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -86
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-rXLW3M_4.js → tree-BGRglkKd.js} +2355 -2414
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-oKRhQAde.d.ts → types-CW8ewATZ.d.ts} +5 -114
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-D-mgRzlh.d.ts} +9 -72
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-B4C5nFw6.d.ts → types-D8GhJoHk.d.ts} +79 -75
- package/dist/types-D8GhJoHk.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-CfZMIzB_.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-DhPoXApk.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +32 -411
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/experiment.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-CIA2Atnt.d.ts.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agent-profile-cell-BYc0bxsL.js.map +0 -1
- package/dist/agreement-irr-DsTURxdr.js.map +0 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/baseline-Dm8966du.js +0 -259
- package/dist/baseline-Dm8966du.js.map +0 -1
- package/dist/benchmark-BywPeAUo.js +0 -562
- package/dist/benchmark-BywPeAUo.js.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/chat-json-call-BrKsyspg.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-DtMWvn7D.js +0 -98
- package/dist/counterfactual-DtMWvn7D.js.map +0 -1
- package/dist/default-registry-BrWh62zf.d.ts.map +0 -1
- package/dist/default-registry-Cn1LQ2Ve.js.map +0 -1
- package/dist/define-agent-eval-CEWkX5wZ.d.ts.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/dspy-rlm-engine-D-KX_e64.js.map +0 -1
- package/dist/effect-sizes-DiH8MGOH.js +0 -82
- package/dist/effect-sizes-DiH8MGOH.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-C_zNWfM8.d.ts.map +0 -1
- package/dist/equal-terms-Blx9Zk5I.js.map +0 -1
- package/dist/evaluation-Cn2dRZIz.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-nnsY5D_4.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-Ch9VxCpK.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/journal-DvKCvjty.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/judge-gate-DNcUrRFA.d.ts.map +0 -1
- package/dist/kind-factory-CbZ-6tvu.js.map +0 -1
- package/dist/ledger-core-B2hahzrt.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-CHMiebFx.d.ts.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-DzKn0TfY.js.map +0 -1
- package/dist/paired-binary-Q36bhn2R.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/prime-protocol-CEE4BR9N.d.ts.map +0 -1
- package/dist/prime-protocol-DcAK9pC8.js.map +0 -1
- package/dist/produced-state-ZCtDBxAE.js.map +0 -1
- package/dist/proposal-findings-BZiMTzAF.js.map +0 -1
- package/dist/query-B1D1qp2Z.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/release-confidence-Dbe9AS_X.js.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/reward-hacking-DHA7c1l8.d.ts.map +0 -1
- package/dist/rollout-D9bKKBnZ.js.map +0 -1
- package/dist/run-record-D1Hw-Jml.d.ts.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/search-ledger-file-BqOo-bVo.js.map +0 -1
- package/dist/search-shipper-er4pK5_Y.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/sequential-D-1PPCmM.js.map +0 -1
- package/dist/server-DbcC7xMl.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/statistical-heldout-1Gbmex7v.js.map +0 -1
- package/dist/statistical-heldout-BeR4bOdg.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/store-tool-spans-fZIigrpP.js.map +0 -1
- package/dist/student-t-BQeKJx2N.js +0 -39
- package/dist/student-t-BQeKJx2N.js.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/task-failure-attributes-DngaLdE4.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/types-oKRhQAde.d.ts.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-DhPoXApk.js.map +0 -1
- package/dist/verdict-cache-BQEsVy9n.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
import { i as JudgeError, s as ValidationError } from "./errors-
|
|
2
|
-
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { i as
|
|
8
|
-
import { n as contentHash } from "./verdict-cache-
|
|
9
|
-
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { m as mapConcurrent } from "./journal-DvKCvjty.js";
|
|
1
|
+
import { i as JudgeError, s as ValidationError } from "./errors-BcOs5MKW.js";
|
|
2
|
+
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-Dq2hOQ8L.js";
|
|
3
|
+
import { i as modelHasSnapshot, s as searchCellRunId } from "./run-record-DhEJgoPz.js";
|
|
4
|
+
import { t as mulberry32 } from "./random-DDDcC7YU.js";
|
|
5
|
+
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-CVNy48rR.js";
|
|
6
|
+
import { r as minimumPairsForPairedDeltaTest } from "./paired-promotion-decision-D0SHb-ZL.js";
|
|
7
|
+
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
|
|
8
|
+
import { n as contentHash } from "./verdict-cache-CL_7tePD.js";
|
|
9
|
+
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-CysKLB93.js";
|
|
10
|
+
import { n as CostAccountingIncompleteError, t as CostLedger } from "./cost-ledger-D0352Z4d.js";
|
|
11
|
+
import { A as surfaceHashMatches, D as surfaceContentHash, E as renderSurfaceDiff, F as assertCampaignSplitIdentity, I as assertCompleteCampaign, L as campaignCoverage, O as surfaceDispatchRef, P as assertCampaignDesign, R as campaignScenarioIdentity, V as formatCoverageFailures, b as FinalEvidenceConflictError, f as campaignBreakdown, j as BackendIntegrityError, k as surfaceHash, m as campaignMeanCompositeOrNull, p as campaignMeanComposite, x as FinalEvidenceError, z as campaignSplitDigest } from "./campaign-evidence-CQYw8_0H.js";
|
|
13
12
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
14
|
-
import {
|
|
15
|
-
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-
|
|
16
|
-
import {
|
|
17
|
-
import { t as detectRewardHacking } from "./reward-hacking-
|
|
18
|
-
import { t as assertProposalFindings } from "./proposal-findings-
|
|
19
|
-
import {
|
|
20
|
-
import { a as validateSearchLedgerEvent, d as
|
|
21
|
-
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-
|
|
22
|
-
import {
|
|
13
|
+
import { M as fsCampaignStorage, h as isExternalTextCandidate, j as createRunCostLedger } from "./external-optimizer-subprocess-XIHBOSKt.js";
|
|
14
|
+
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
|
|
15
|
+
import { d as detectPersonalData, s as redactText, t as REDACTION_VERSION, u as detectCredential } from "./redact-mtkTiJVk.js";
|
|
16
|
+
import { t as detectRewardHacking } from "./reward-hacking-ByuETaIb.js";
|
|
17
|
+
import { t as assertProposalFindings } from "./proposal-findings-BbWidvUk.js";
|
|
18
|
+
import { i as runSearch, m as searchExpansionIndex, n as paretoFrontier, o as searchPolicyView, p as SEARCH_KERNEL_SOURCE, r as paretoFrontierWithCrowding, s as EXPANSION_OPERATORS, w as estimateNode } from "./pareto-BzsxidFD.js";
|
|
19
|
+
import { a as validateSearchLedgerEvent, d as searchNodeId, f as searchTaskSetDigest, i as replaySearchLedgerText, n as openSearchLedger, u as searchCellId } from "./search-ledger-TUvB5gWH.js";
|
|
20
|
+
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-DQIvJsSB.js";
|
|
21
|
+
import { i as stripFencedJson, n as isRecord } from "./json-BhdsJaOQ.js";
|
|
22
|
+
import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm } from "./llm-client-BshCREn6.js";
|
|
23
23
|
import { createHash } from "node:crypto";
|
|
24
24
|
import { writeFileSync } from "node:fs";
|
|
25
25
|
import { z } from "zod";
|
|
@@ -1200,8 +1200,8 @@ function detectCalibrationDrift(runs, opts) {
|
|
|
1200
1200
|
alpha,
|
|
1201
1201
|
recentN: recent.length,
|
|
1202
1202
|
historyN: historical.length,
|
|
1203
|
-
recentMean: mean$
|
|
1204
|
-
historyMean: mean$
|
|
1203
|
+
recentMean: mean$1(recent),
|
|
1204
|
+
historyMean: mean$1(historical)
|
|
1205
1205
|
}
|
|
1206
1206
|
}],
|
|
1207
1207
|
evaluation: {
|
|
@@ -1438,7 +1438,7 @@ function chiSquareCritical(df, alpha) {
|
|
|
1438
1438
|
}
|
|
1439
1439
|
return TABLE[10][idx];
|
|
1440
1440
|
}
|
|
1441
|
-
function mean$
|
|
1441
|
+
function mean$1(xs) {
|
|
1442
1442
|
if (xs.length === 0) return 0;
|
|
1443
1443
|
return xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
1444
1444
|
}
|
|
@@ -3190,7 +3190,6 @@ function nonEmpty(value, label) {
|
|
|
3190
3190
|
* and a fact already in the ledger is not appended again, so a resumed search
|
|
3191
3191
|
* continues its ledger instead of conflicting with its own history.
|
|
3192
3192
|
*/
|
|
3193
|
-
const MAX_LABEL_CHARS = 200;
|
|
3194
3193
|
/** The default claim: development numbers over exactly the supplied tasks. */
|
|
3195
3194
|
function developmentClaim(populationId) {
|
|
3196
3195
|
return {
|
|
@@ -3554,7 +3553,7 @@ function declaredSurfaces(surface, artifact) {
|
|
|
3554
3553
|
}
|
|
3555
3554
|
function boundedLabel(label) {
|
|
3556
3555
|
const redacted = redactText(label.trim(), { profile: "share" });
|
|
3557
|
-
return redacted.length <=
|
|
3556
|
+
return redacted.length <= 200 ? redacted : `${redacted.slice(0, 199)}…`;
|
|
3558
3557
|
}
|
|
3559
3558
|
/** The header without its stamp, for comparing a resumed opening to the stored one. */
|
|
3560
3559
|
function headerMaterial(header) {
|
|
@@ -3709,7 +3708,7 @@ function crowdedFrontierParent(options) {
|
|
|
3709
3708
|
});
|
|
3710
3709
|
}
|
|
3711
3710
|
/**
|
|
3712
|
-
* AIDE's
|
|
3711
|
+
* AIDE's search_policy with three changes
|
|
3713
3712
|
* for noisy scores.
|
|
3714
3713
|
*
|
|
3715
3714
|
* 1. **Draft** while fewer than `drafts` nodes were drafted: the root is the
|
|
@@ -3968,7 +3967,7 @@ function hillClimb(spec) {
|
|
|
3968
3967
|
evidence: {
|
|
3969
3968
|
...chosen.evidence,
|
|
3970
3969
|
units: units.length,
|
|
3971
|
-
...units.length > 0 ? { mean: mean
|
|
3970
|
+
...units.length > 0 ? { mean: mean(units.map((unit) => unit.mean)) } : {},
|
|
3972
3971
|
isLeader: chosen.nodeId === kept ? 1 : 0
|
|
3973
3972
|
}
|
|
3974
3973
|
}
|
|
@@ -4024,7 +4023,7 @@ function paretoMembers(view) {
|
|
|
4024
4023
|
members.push({
|
|
4025
4024
|
nodeId,
|
|
4026
4025
|
ordinal,
|
|
4027
|
-
mean: sign * mean
|
|
4026
|
+
mean: sign * mean([...objectives.values()]),
|
|
4028
4027
|
objectives,
|
|
4029
4028
|
distance: 0
|
|
4030
4029
|
});
|
|
@@ -4046,7 +4045,7 @@ function compareCrowded(a, b) {
|
|
|
4046
4045
|
if (a.mean !== b.mean) return a.mean > b.mean ? -1 : 1;
|
|
4047
4046
|
return a.ordinal - b.ordinal;
|
|
4048
4047
|
}
|
|
4049
|
-
function mean
|
|
4048
|
+
function mean(values) {
|
|
4050
4049
|
let total = 0;
|
|
4051
4050
|
for (const value of values) total += value;
|
|
4052
4051
|
return total / values.length;
|
|
@@ -4058,285 +4057,9 @@ function mean$1(values) {
|
|
|
4058
4057
|
function isProposedCandidate(value) {
|
|
4059
4058
|
return typeof value === "object" && value !== null && "surface" in value && "label" in value && "rationale" in value;
|
|
4060
4059
|
}
|
|
4061
|
-
const LABEL_TRUST_RANK = {
|
|
4062
|
-
unverified: 0,
|
|
4063
|
-
"verified-signal": 1,
|
|
4064
|
-
"human-rated": 2
|
|
4065
|
-
};
|
|
4066
|
-
/** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
|
|
4067
|
-
function labelTrustRank(trust) {
|
|
4068
|
-
return LABEL_TRUST_RANK[trust ?? "unverified"];
|
|
4069
|
-
}
|
|
4070
4060
|
//#endregion
|
|
4071
|
-
//#region src/campaign/presets/
|
|
4072
|
-
/**
|
|
4073
|
-
* `runOptimization` improves a mutable surface with a caller-owned proposer,
|
|
4074
|
-
* as a search on the one kernel (`runSearch`).
|
|
4075
|
-
*
|
|
4076
|
-
* The baseline is the seeded root. The scenarios are the train split: the
|
|
4077
|
-
* proposer reads their scores, and the policy ranks on them because the loop
|
|
4078
|
-
* has no private selection split. A "generation" is one proposal of
|
|
4079
|
-
* `populationSize` candidates from the node the policy chose (by default the
|
|
4080
|
-
* incumbent, once every earlier candidate is measured). Every candidate is a
|
|
4081
|
-
* node measured on every scenario at every repeat, one cell at a time on the
|
|
4082
|
-
* run's lanes. The kept surface is the policy's leader: a budget decision on
|
|
4083
|
-
* the train split that claims nothing; `runImprovementLoop` compares it with
|
|
4084
|
-
* the baseline on separate cases.
|
|
4085
|
-
*
|
|
4086
|
-
* The search ledger is always written, beside the run (`<runDir>/search/`) or
|
|
4087
|
-
* where `searchLedger` puts it, and it is the run's only checkpoint: running
|
|
4088
|
-
* again on the same run directory and options continues an interrupted run
|
|
4089
|
-
* instead of starting over, and reruns nothing that settled.
|
|
4090
|
-
*/
|
|
4091
|
-
/** Improve a surface as a search on the kernel; see the module comment. */
|
|
4092
|
-
async function runOptimization(opts) {
|
|
4093
|
-
const candidateConcurrency = opts.candidateConcurrency ?? 1;
|
|
4094
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4095
|
-
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4096
|
-
for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
|
|
4097
|
-
const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
|
|
4098
|
-
const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
|
|
4099
|
-
const runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
4100
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
4101
|
-
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
4102
|
-
storage,
|
|
4103
|
-
runDir,
|
|
4104
|
-
costCeilingUsd: opts.costCeiling
|
|
4105
|
-
});
|
|
4106
|
-
const reps = opts.reps ?? 1;
|
|
4107
|
-
const seed = opts.seed ?? 42;
|
|
4108
|
-
const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
|
|
4109
|
-
const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
|
|
4110
|
-
input: opts.premeasuredBaseline,
|
|
4111
|
-
baselineSurface,
|
|
4112
|
-
scenarios: opts.scenarios,
|
|
4113
|
-
reps,
|
|
4114
|
-
seed,
|
|
4115
|
-
judges: opts.judges ?? [],
|
|
4116
|
-
dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
|
|
4117
|
-
requireJudgeScore
|
|
4118
|
-
}) : void 0;
|
|
4119
|
-
const splitDigest = campaignSplitDigest(opts.scenarios, reps);
|
|
4120
|
-
const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
|
|
4121
|
-
const { identity } = binding;
|
|
4122
|
-
const execution = {
|
|
4123
|
-
model: identity.model,
|
|
4124
|
-
agent: identity.agent,
|
|
4125
|
-
benchmark: {
|
|
4126
|
-
uri: "campaign://scenarios",
|
|
4127
|
-
revision: splitDigest
|
|
4128
|
-
}
|
|
4129
|
-
};
|
|
4130
|
-
const policy = opts.policy ?? incumbent();
|
|
4131
|
-
const allocation = uniform({ reps });
|
|
4132
|
-
const recorder = await SearchRecorder.open({
|
|
4133
|
-
ledger: binding.ledger,
|
|
4134
|
-
storage
|
|
4135
|
-
}, {
|
|
4136
|
-
subject: identity.subject ?? opts.proposer.kind,
|
|
4137
|
-
process: {
|
|
4138
|
-
name: "runOptimization",
|
|
4139
|
-
executionRef: identity.search
|
|
4140
|
-
},
|
|
4141
|
-
artifactKind: artifactKindOf(baselineSurface),
|
|
4142
|
-
objective: {
|
|
4143
|
-
metric: "composite",
|
|
4144
|
-
direction: "maximize",
|
|
4145
|
-
judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
|
|
4146
|
-
claim: identity.claim ?? developmentClaim(splitDigest)
|
|
4147
|
-
},
|
|
4148
|
-
splits: {
|
|
4149
|
-
train: opts.scenarios.map((scenario) => ({
|
|
4150
|
-
taskId: scenario.id,
|
|
4151
|
-
unitId: scenario.id,
|
|
4152
|
-
source: {
|
|
4153
|
-
uri: `scenario://${scenario.id}`,
|
|
4154
|
-
revision: hashCanonical(scenario)
|
|
4155
|
-
}
|
|
4156
|
-
})),
|
|
4157
|
-
selection: [],
|
|
4158
|
-
test: [],
|
|
4159
|
-
heldOutUnits: true
|
|
4160
|
-
},
|
|
4161
|
-
policy: {
|
|
4162
|
-
expansion: policy.name,
|
|
4163
|
-
allocation: allocation.name,
|
|
4164
|
-
seed
|
|
4165
|
-
},
|
|
4166
|
-
budget: {
|
|
4167
|
-
maxUsd: opts.costCeiling ?? null,
|
|
4168
|
-
maxCells: null,
|
|
4169
|
-
maxNodes: 1 + opts.populationSize * opts.maxGenerations,
|
|
4170
|
-
deadline: null,
|
|
4171
|
-
maxConcurrency: null,
|
|
4172
|
-
reservedClaimUsd: 0
|
|
4173
|
-
},
|
|
4174
|
-
containment: null,
|
|
4175
|
-
derivedFrom: null,
|
|
4176
|
-
identity: execution
|
|
4177
|
-
});
|
|
4178
|
-
const nodes = new SurfaceNodes({
|
|
4179
|
-
opts,
|
|
4180
|
-
recorder,
|
|
4181
|
-
runDir,
|
|
4182
|
-
costLedger,
|
|
4183
|
-
execution,
|
|
4184
|
-
premeasured
|
|
4185
|
-
});
|
|
4186
|
-
const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
|
|
4187
|
-
let findings = initialFindings;
|
|
4188
|
-
const executor = {
|
|
4189
|
-
lanes: () => [{
|
|
4190
|
-
name: "campaign",
|
|
4191
|
-
capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
|
|
4192
|
-
costCap: "estimate",
|
|
4193
|
-
cellUsd: 0
|
|
4194
|
-
}],
|
|
4195
|
-
place: () => "campaign",
|
|
4196
|
-
run: (work) => nodes.runCell(work, scenarioById),
|
|
4197
|
-
adopt: (work) => nodes.adoptCell(work)
|
|
4198
|
-
};
|
|
4199
|
-
const proposer = opts.proposer;
|
|
4200
|
-
const proposerSource = identity.proposer.source;
|
|
4201
|
-
const port = {
|
|
4202
|
-
name: proposer.kind,
|
|
4203
|
-
kind: "optimizer",
|
|
4204
|
-
source: proposerSource,
|
|
4205
|
-
execution: identity.proposer,
|
|
4206
|
-
childrenPerProposal: opts.populationSize,
|
|
4207
|
-
async propose(request) {
|
|
4208
|
-
const phases = [
|
|
4209
|
-
"search.proposal",
|
|
4210
|
-
"analysis.baseline",
|
|
4211
|
-
"analysis.generation"
|
|
4212
|
-
];
|
|
4213
|
-
const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map(receiptKey));
|
|
4214
|
-
const accounting = () => {
|
|
4215
|
-
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receiptKey(receipt)));
|
|
4216
|
-
return {
|
|
4217
|
-
accounting: searchReceiptAccounting(fresh),
|
|
4218
|
-
execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
|
|
4219
|
-
};
|
|
4220
|
-
};
|
|
4221
|
-
const earlier = await recorder.state();
|
|
4222
|
-
const root = earlier.rootNodeId;
|
|
4223
|
-
const rootCampaign = nodes.campaign(earlier, root);
|
|
4224
|
-
const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
|
|
4225
|
-
if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4226
|
-
const generation = nodes.generations(earlier).length;
|
|
4227
|
-
if (opts.analyzeGeneration) {
|
|
4228
|
-
const previous = generation - 1;
|
|
4229
|
-
const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
|
|
4230
|
-
const campaign = nodes.campaign(earlier, nodeId);
|
|
4231
|
-
return {
|
|
4232
|
-
surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
|
|
4233
|
-
campaign,
|
|
4234
|
-
composite: campaignMeanCompositeOrNull(campaign)
|
|
4235
|
-
};
|
|
4236
|
-
});
|
|
4237
|
-
const history = nodes.history(earlier, policy);
|
|
4238
|
-
if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
|
|
4239
|
-
const fresh = await opts.analyzeGeneration({
|
|
4240
|
-
generation: previous,
|
|
4241
|
-
runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
|
|
4242
|
-
candidates,
|
|
4243
|
-
history,
|
|
4244
|
-
costLedger,
|
|
4245
|
-
costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
|
|
4246
|
-
});
|
|
4247
|
-
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4248
|
-
findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
|
|
4249
|
-
}
|
|
4250
|
-
}
|
|
4251
|
-
const state = await recorder.state();
|
|
4252
|
-
const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
|
|
4253
|
-
const parent = request.parents[0];
|
|
4254
|
-
const context = Object.freeze({
|
|
4255
|
-
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4256
|
-
operator: request.operator,
|
|
4257
|
-
parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
|
|
4258
|
-
nodeId,
|
|
4259
|
-
artifact: immutableProposalSnapshot(artifact, "parent surface")
|
|
4260
|
-
})), "parents"),
|
|
4261
|
-
train: searchProposerView(state),
|
|
4262
|
-
summary: renderSearchSummary(state, { split: "train" }),
|
|
4263
|
-
history: proposalHistory,
|
|
4264
|
-
findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
|
|
4265
|
-
populationSize: opts.populationSize,
|
|
4266
|
-
generation,
|
|
4267
|
-
signal: request.signal,
|
|
4268
|
-
baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
|
|
4269
|
-
incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
|
|
4270
|
-
parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
|
|
4271
|
-
maxImprovementShots: opts.maxImprovementShots,
|
|
4272
|
-
paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
|
|
4273
|
-
costLedger,
|
|
4274
|
-
costPhase: "search.proposal"
|
|
4275
|
-
});
|
|
4276
|
-
if (proposer.decide?.({ history: proposalHistory }).stop) return {
|
|
4277
|
-
children: [],
|
|
4278
|
-
stop: "the proposer decided to stop",
|
|
4279
|
-
...accounting()
|
|
4280
|
-
};
|
|
4281
|
-
const proposed = await proposer.propose(context);
|
|
4282
|
-
if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
|
|
4283
|
-
return {
|
|
4284
|
-
children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
|
|
4285
|
-
artifact: candidate.surface,
|
|
4286
|
-
label: candidate.label,
|
|
4287
|
-
rationale: candidate.rationale,
|
|
4288
|
-
...candidate.attribution ? { attribution: candidate.attribution } : {}
|
|
4289
|
-
} : {
|
|
4290
|
-
artifact: candidate,
|
|
4291
|
-
label: "",
|
|
4292
|
-
rationale: ""
|
|
4293
|
-
}),
|
|
4294
|
-
...accounting()
|
|
4295
|
-
};
|
|
4296
|
-
}
|
|
4297
|
-
};
|
|
4298
|
-
const result = await runSearch({
|
|
4299
|
-
recorder,
|
|
4300
|
-
root: baselineSurface,
|
|
4301
|
-
codec: surfaceCodec,
|
|
4302
|
-
policy,
|
|
4303
|
-
allocation,
|
|
4304
|
-
proposer: port,
|
|
4305
|
-
executor,
|
|
4306
|
-
maxExpansions: opts.maxGenerations,
|
|
4307
|
-
signal: opts.signal
|
|
4308
|
-
});
|
|
4309
|
-
const state = result.state;
|
|
4310
|
-
const winnerSurface = nodes.surface(state, result.leader);
|
|
4311
|
-
const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
|
|
4312
|
-
const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
|
|
4313
|
-
const expansions = nodes.generations(state);
|
|
4314
|
-
return {
|
|
4315
|
-
generations: nodes.history(state, policy).map((record) => ({
|
|
4316
|
-
record,
|
|
4317
|
-
surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
|
|
4318
|
-
const surface = nodes.surface(state, nodeId);
|
|
4319
|
-
return {
|
|
4320
|
-
surfaceHash: surfaceHash(surface),
|
|
4321
|
-
surface,
|
|
4322
|
-
campaign: nodes.campaign(state, nodeId)
|
|
4323
|
-
};
|
|
4324
|
-
})
|
|
4325
|
-
})),
|
|
4326
|
-
baselineSurface,
|
|
4327
|
-
winnerSurface,
|
|
4328
|
-
winnerSurfaceHash: surfaceHash(winnerSurface),
|
|
4329
|
-
...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
|
|
4330
|
-
...winnerRationale ? { winnerRationale } : {},
|
|
4331
|
-
baselineCampaign: nodes.campaign(state, state.rootNodeId),
|
|
4332
|
-
cost: costLedger.summary(),
|
|
4333
|
-
searchHistory: recorder.receipt({
|
|
4334
|
-
producerId: proposer.kind,
|
|
4335
|
-
runId: runDir
|
|
4336
|
-
}),
|
|
4337
|
-
paretoFrontier: nodes.frontier(state)
|
|
4338
|
-
};
|
|
4339
|
-
}
|
|
4061
|
+
//#region src/campaign/presets/optimization-surface-nodes.ts
|
|
4062
|
+
/** Surface-node campaigns, cached cell adoption, and proposer history from the search ledger. */
|
|
4340
4063
|
/** A mutable surface as a search artifact. */
|
|
4341
4064
|
const surfaceCodec = {
|
|
4342
4065
|
node: surfaceNode,
|
|
@@ -4364,14 +4087,14 @@ var SurfaceNodes = class {
|
|
|
4364
4087
|
proposals = /* @__PURE__ */ new Map();
|
|
4365
4088
|
/** Cells this process ran or adopted, by node, keyed `scenarioId:rep`. */
|
|
4366
4089
|
cells = /* @__PURE__ */ new Map();
|
|
4367
|
-
constructor(
|
|
4368
|
-
this.opts =
|
|
4369
|
-
this.recorder =
|
|
4370
|
-
this.runDir =
|
|
4371
|
-
this.costLedger =
|
|
4372
|
-
this.execution =
|
|
4373
|
-
this.premeasured =
|
|
4374
|
-
this.storage =
|
|
4090
|
+
constructor(opts, recorder, runDir, costLedger, execution, premeasured) {
|
|
4091
|
+
this.opts = opts;
|
|
4092
|
+
this.recorder = recorder;
|
|
4093
|
+
this.runDir = runDir;
|
|
4094
|
+
this.costLedger = costLedger;
|
|
4095
|
+
this.execution = execution;
|
|
4096
|
+
this.premeasured = premeasured;
|
|
4097
|
+
this.storage = opts.storage ?? fsCampaignStorage();
|
|
4375
4098
|
}
|
|
4376
4099
|
/** `<runDir>/baseline` for the root, `<runDir>/candidates/<surfaceHash>`
|
|
4377
4100
|
* for every other node: content-addressed, so a cell's directory needs no
|
|
@@ -4406,7 +4129,10 @@ var SurfaceNodes = class {
|
|
|
4406
4129
|
const cell = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4407
4130
|
if (cell?.manifestHash !== this.manifestFor(work.artifact)) return null;
|
|
4408
4131
|
this.remember(work.nodeId, cell);
|
|
4409
|
-
return
|
|
4132
|
+
return campaignCellSearchResult(cell, {
|
|
4133
|
+
execution: this.execution,
|
|
4134
|
+
lane: work.lane
|
|
4135
|
+
});
|
|
4410
4136
|
}
|
|
4411
4137
|
/** Run one cell as a one-cell campaign in its node's directory; a cached
|
|
4412
4138
|
* cell is read back, not dispatched again. */
|
|
@@ -4421,7 +4147,10 @@ var SurfaceNodes = class {
|
|
|
4421
4147
|
})).cells[0];
|
|
4422
4148
|
if (!cell) throw new Error(`runOptimization: cell ${work.cellId} produced no campaign cell`);
|
|
4423
4149
|
this.remember(work.nodeId, cell);
|
|
4424
|
-
return
|
|
4150
|
+
return campaignCellSearchResult(cell, {
|
|
4151
|
+
execution: this.execution,
|
|
4152
|
+
lane: work.lane
|
|
4153
|
+
});
|
|
4425
4154
|
}
|
|
4426
4155
|
/** The node's campaign over the cells the ledger settled: the cells this
|
|
4427
4156
|
* process ran, else their cached results or failure receipts. */
|
|
@@ -4589,7 +4318,10 @@ var SurfaceNodes = class {
|
|
|
4589
4318
|
premeasuredCell(work) {
|
|
4590
4319
|
const cell = this.premeasured.cells.find((candidate) => candidate.scenarioId === work.taskId && candidate.rep === work.rep);
|
|
4591
4320
|
if (!cell) throw new Error(`runOptimization: premeasured baseline lacks ${work.taskId}:${work.rep}`);
|
|
4592
|
-
return
|
|
4321
|
+
return campaignCellSearchResult(cell, {
|
|
4322
|
+
execution: this.execution,
|
|
4323
|
+
lane: work.lane
|
|
4324
|
+
});
|
|
4593
4325
|
}
|
|
4594
4326
|
/** A cell's result as its campaign stored it for this surface: the cache of
|
|
4595
4327
|
* a scored cell, the final failure receipt of a failed one. A result another
|
|
@@ -4606,34 +4338,334 @@ var SurfaceNodes = class {
|
|
|
4606
4338
|
const cell = result !== void 0 ? JSON.parse(result) : receipt !== void 0 ? JSON.parse(receipt).cell : void 0;
|
|
4607
4339
|
return cell?.manifestHash === manifestHash ? cell : void 0;
|
|
4608
4340
|
}
|
|
4609
|
-
const cached = this.storage.read(cellCachePath(dir, cellId));
|
|
4610
|
-
const scored = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4611
|
-
if (scored?.manifestHash === manifestHash) return scored;
|
|
4612
|
-
const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
|
|
4613
|
-
const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
|
|
4614
|
-
return failed?.manifestHash === manifestHash ? failed : void 0;
|
|
4615
|
-
}
|
|
4616
|
-
manifestFor(surface) {
|
|
4617
|
-
return computeManifestHash({
|
|
4618
|
-
scenarios: this.opts.scenarios,
|
|
4619
|
-
judges: this.opts.judges ?? [],
|
|
4620
|
-
dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
|
|
4621
|
-
seed: this.opts.seed ?? 42,
|
|
4622
|
-
reps: this.opts.reps ?? 1
|
|
4623
|
-
});
|
|
4624
|
-
}
|
|
4625
|
-
remember(nodeId, cell) {
|
|
4626
|
-
const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
|
|
4627
|
-
cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
|
|
4628
|
-
this.cells.set(nodeId, cells);
|
|
4629
|
-
}
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4341
|
+
const cached = this.storage.read(cellCachePath(dir, cellId));
|
|
4342
|
+
const scored = cached === void 0 ? void 0 : JSON.parse(cached);
|
|
4343
|
+
if (scored?.manifestHash === manifestHash) return scored;
|
|
4344
|
+
const receipt = this.storage.read(join(cellDirectory(dir, cellId), "failure-receipt.json"));
|
|
4345
|
+
const failed = receipt === void 0 ? void 0 : JSON.parse(receipt).cell;
|
|
4346
|
+
return failed?.manifestHash === manifestHash ? failed : void 0;
|
|
4347
|
+
}
|
|
4348
|
+
manifestFor(surface) {
|
|
4349
|
+
return computeManifestHash({
|
|
4350
|
+
scenarios: this.opts.scenarios,
|
|
4351
|
+
judges: this.opts.judges ?? [],
|
|
4352
|
+
dispatchRef: surfaceDispatchRef(surface, this.opts.dispatchRef),
|
|
4353
|
+
seed: this.opts.seed ?? 42,
|
|
4354
|
+
reps: this.opts.reps ?? 1
|
|
4355
|
+
});
|
|
4356
|
+
}
|
|
4357
|
+
remember(nodeId, cell) {
|
|
4358
|
+
const cells = this.cells.get(nodeId) ?? /* @__PURE__ */ new Map();
|
|
4359
|
+
cells.set(`${cell.scenarioId}:${cell.rep}`, cell);
|
|
4360
|
+
this.cells.set(nodeId, cells);
|
|
4361
|
+
}
|
|
4362
|
+
};
|
|
4363
|
+
/** A scored campaign as a frontier member: per-scenario composite objectives. */
|
|
4364
|
+
function toParetoParent(surface, campaign, generation, label) {
|
|
4365
|
+
const objectives = {};
|
|
4366
|
+
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
|
|
4367
|
+
const parent = {
|
|
4368
|
+
surface,
|
|
4369
|
+
surfaceHash: surfaceHash(surface),
|
|
4370
|
+
objectives,
|
|
4371
|
+
composite: campaignMeanComposite(campaign),
|
|
4372
|
+
generation
|
|
4373
|
+
};
|
|
4374
|
+
if (label) parent.label = label;
|
|
4375
|
+
return parent;
|
|
4376
|
+
}
|
|
4377
|
+
/** The non-dominated set over the per-scenario objective vectors. */
|
|
4378
|
+
function computeParetoFrontier(scored) {
|
|
4379
|
+
if (scored.length <= 1) return [...scored];
|
|
4380
|
+
const ids = /* @__PURE__ */ new Set();
|
|
4381
|
+
for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
|
|
4382
|
+
if (ids.size === 0) return [...scored];
|
|
4383
|
+
const objectives = [...ids].map((id) => ({
|
|
4384
|
+
name: id,
|
|
4385
|
+
direction: "maximize",
|
|
4386
|
+
value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
|
|
4387
|
+
}));
|
|
4388
|
+
return paretoFrontier(scored, objectives).frontier;
|
|
4389
|
+
}
|
|
4390
|
+
function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
|
|
4391
|
+
const breakdown = campaignBreakdown(campaign);
|
|
4392
|
+
return {
|
|
4393
|
+
split: "search",
|
|
4394
|
+
generation,
|
|
4395
|
+
surfaceHash: hash,
|
|
4396
|
+
composite: campaignMeanComposite(campaign),
|
|
4397
|
+
dimensions: breakdown.dimensions,
|
|
4398
|
+
scenarios: breakdown.scenarios,
|
|
4399
|
+
coverage: {
|
|
4400
|
+
expectedCells: coverage.expectedCellIds.length,
|
|
4401
|
+
scorableCells: coverage.scorableCellIds.length
|
|
4402
|
+
}
|
|
4403
|
+
};
|
|
4404
|
+
}
|
|
4405
|
+
//#endregion
|
|
4406
|
+
//#region src/campaign/presets/run-optimization.ts
|
|
4407
|
+
/**
|
|
4408
|
+
* `runOptimization` improves a mutable surface with a caller-owned proposer,
|
|
4409
|
+
* as a search on the one kernel (`runSearch`).
|
|
4410
|
+
*
|
|
4411
|
+
* The baseline is the seeded root. The scenarios are the train split: the
|
|
4412
|
+
* proposer reads their scores, and the policy ranks on them because the loop
|
|
4413
|
+
* has no private selection split. A "generation" is one proposal of
|
|
4414
|
+
* `populationSize` candidates from the node the policy chose (by default the
|
|
4415
|
+
* incumbent, once every earlier candidate is measured). Every candidate is a
|
|
4416
|
+
* node measured on every scenario at every repeat, one cell at a time on the
|
|
4417
|
+
* run's lanes. The kept surface is the policy's leader: a budget decision on
|
|
4418
|
+
* the train split that claims nothing; `runImprovementLoop` compares it with
|
|
4419
|
+
* the baseline on separate cases.
|
|
4420
|
+
*
|
|
4421
|
+
* The search ledger is always written, beside the run (`<runDir>/search/`) or
|
|
4422
|
+
* where `searchLedger` puts it, and it is the run's only checkpoint: running
|
|
4423
|
+
* again on the same run directory and options continues an interrupted run
|
|
4424
|
+
* instead of starting over, and reruns nothing that settled.
|
|
4425
|
+
*/
|
|
4426
|
+
/** Compose the optimization proposer and cell executor on the shared search kernel. */
|
|
4427
|
+
/** Improve a surface as a search on the kernel; see the module comment. */
|
|
4428
|
+
async function runOptimization(opts) {
|
|
4429
|
+
const candidateConcurrency = opts.candidateConcurrency ?? 1;
|
|
4430
|
+
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("runOptimization: runDir is required and must be a non-empty string");
|
|
4431
|
+
if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) throw new Error("runOptimization: candidateConcurrency must be a positive integer");
|
|
4432
|
+
for (const [name, value] of [["populationSize", opts.populationSize], ["maxGenerations", opts.maxGenerations]]) if (!Number.isSafeInteger(value) || value < 0) throw new Error(`runOptimization: ${name} must be a non-negative integer`);
|
|
4433
|
+
const initialFindings = immutableProposalSnapshot(assertProposalFindings(opts.findings ?? [], "runOptimization initial proposal findings"), "initial findings");
|
|
4434
|
+
const baselineSurface = immutableProposalSnapshot(opts.baselineSurface, "baseline surface");
|
|
4435
|
+
const runDir = resolveRunDir(opts.runDir, opts.repo);
|
|
4436
|
+
const storage = opts.storage ?? fsCampaignStorage();
|
|
4437
|
+
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
4438
|
+
storage,
|
|
4439
|
+
runDir,
|
|
4440
|
+
costCeilingUsd: opts.costCeiling
|
|
4441
|
+
});
|
|
4442
|
+
const reps = opts.reps ?? 1;
|
|
4443
|
+
const seed = opts.seed ?? 42;
|
|
4444
|
+
const requireJudgeScore = (opts.judges?.length ?? 0) > 0;
|
|
4445
|
+
const premeasured = opts.premeasuredBaseline ? validatedPremeasuredBaseline({
|
|
4446
|
+
input: opts.premeasuredBaseline,
|
|
4447
|
+
baselineSurface,
|
|
4448
|
+
scenarios: opts.scenarios,
|
|
4449
|
+
reps,
|
|
4450
|
+
seed,
|
|
4451
|
+
judges: opts.judges ?? [],
|
|
4452
|
+
dispatchRef: surfaceDispatchRef(baselineSurface, opts.dispatchRef),
|
|
4453
|
+
requireJudgeScore
|
|
4454
|
+
}) : void 0;
|
|
4455
|
+
const splitDigest = campaignSplitDigest(opts.scenarios, reps);
|
|
4456
|
+
const binding = opts.searchLedger ?? await defaultBinding(opts, runDir, storage);
|
|
4457
|
+
const { identity } = binding;
|
|
4458
|
+
const execution = {
|
|
4459
|
+
model: identity.model,
|
|
4460
|
+
agent: identity.agent,
|
|
4461
|
+
benchmark: {
|
|
4462
|
+
uri: "campaign://scenarios",
|
|
4463
|
+
revision: splitDigest
|
|
4464
|
+
}
|
|
4465
|
+
};
|
|
4466
|
+
const policy = opts.policy ?? incumbent();
|
|
4467
|
+
const allocation = uniform({ reps });
|
|
4468
|
+
const recorder = await SearchRecorder.open({
|
|
4469
|
+
ledger: binding.ledger,
|
|
4470
|
+
storage
|
|
4471
|
+
}, {
|
|
4472
|
+
subject: identity.subject ?? opts.proposer.kind,
|
|
4473
|
+
process: {
|
|
4474
|
+
name: "runOptimization",
|
|
4475
|
+
executionRef: identity.search
|
|
4476
|
+
},
|
|
4477
|
+
artifactKind: typeof baselineSurface === "object" && baselineSurface.kind === "code" ? "code" : "prompt",
|
|
4478
|
+
objective: {
|
|
4479
|
+
metric: "composite",
|
|
4480
|
+
direction: "maximize",
|
|
4481
|
+
judge: identity.judge ?? { unknown: "runOptimization receives its judges as functions without a pinned source" },
|
|
4482
|
+
claim: identity.claim ?? developmentClaim(splitDigest)
|
|
4483
|
+
},
|
|
4484
|
+
splits: {
|
|
4485
|
+
train: opts.scenarios.map((scenario) => ({
|
|
4486
|
+
taskId: scenario.id,
|
|
4487
|
+
unitId: scenario.id,
|
|
4488
|
+
source: {
|
|
4489
|
+
uri: `scenario://${scenario.id}`,
|
|
4490
|
+
revision: hashCanonical(scenario)
|
|
4491
|
+
}
|
|
4492
|
+
})),
|
|
4493
|
+
selection: [],
|
|
4494
|
+
test: [],
|
|
4495
|
+
heldOutUnits: true
|
|
4496
|
+
},
|
|
4497
|
+
policy: {
|
|
4498
|
+
expansion: policy.name,
|
|
4499
|
+
allocation: allocation.name,
|
|
4500
|
+
seed
|
|
4501
|
+
},
|
|
4502
|
+
budget: {
|
|
4503
|
+
maxUsd: opts.costCeiling ?? null,
|
|
4504
|
+
maxCells: null,
|
|
4505
|
+
maxNodes: 1 + opts.populationSize * opts.maxGenerations,
|
|
4506
|
+
deadline: null,
|
|
4507
|
+
maxConcurrency: null,
|
|
4508
|
+
reservedClaimUsd: 0
|
|
4509
|
+
},
|
|
4510
|
+
containment: null,
|
|
4511
|
+
derivedFrom: null,
|
|
4512
|
+
identity: execution
|
|
4513
|
+
});
|
|
4514
|
+
const nodes = new SurfaceNodes(opts, recorder, runDir, costLedger, execution, premeasured);
|
|
4515
|
+
const scenarioById = new Map(opts.scenarios.map((scenario) => [scenario.id, scenario]));
|
|
4516
|
+
let findings = initialFindings;
|
|
4517
|
+
const executor = {
|
|
4518
|
+
lanes: () => [{
|
|
4519
|
+
name: "campaign",
|
|
4520
|
+
capacity: candidateConcurrency * (opts.maxConcurrency ?? 2),
|
|
4521
|
+
costCap: "estimate",
|
|
4522
|
+
cellUsd: 0
|
|
4523
|
+
}],
|
|
4524
|
+
place: () => "campaign",
|
|
4525
|
+
run: (work) => nodes.runCell(work, scenarioById),
|
|
4526
|
+
adopt: (work) => nodes.adoptCell(work)
|
|
4527
|
+
};
|
|
4528
|
+
const proposer = opts.proposer;
|
|
4529
|
+
const proposerSource = identity.proposer.source;
|
|
4530
|
+
const port = {
|
|
4531
|
+
name: proposer.kind,
|
|
4532
|
+
kind: "optimizer",
|
|
4533
|
+
source: proposerSource,
|
|
4534
|
+
execution: identity.proposer,
|
|
4535
|
+
childrenPerProposal: opts.populationSize,
|
|
4536
|
+
async propose(request) {
|
|
4537
|
+
const phases = [
|
|
4538
|
+
"search.proposal",
|
|
4539
|
+
"analysis.baseline",
|
|
4540
|
+
"analysis.generation"
|
|
4541
|
+
];
|
|
4542
|
+
const before = new Set(phases.flatMap((phase) => costLedger.list({ phase })).map((receipt) => receipt.callId));
|
|
4543
|
+
const accounting = () => {
|
|
4544
|
+
const fresh = phases.flatMap((phase) => costLedger.list({ phase })).filter((receipt) => !before.has(receipt.callId));
|
|
4545
|
+
return {
|
|
4546
|
+
accounting: searchReceiptAccounting(fresh),
|
|
4547
|
+
execution: opts.searchLedger ? identity.proposer : searchProposalExecution(fresh, identity.model.provider, proposerSource)
|
|
4548
|
+
};
|
|
4549
|
+
};
|
|
4550
|
+
const earlier = await recorder.state();
|
|
4551
|
+
const root = earlier.rootNodeId;
|
|
4552
|
+
const rootCampaign = nodes.campaign(earlier, root);
|
|
4553
|
+
const rootCoverage = campaignCoverage(rootCampaign.cells, opts.scenarios, reps, requireJudgeScore);
|
|
4554
|
+
if (!rootCoverage.complete) throw new Error(`runOptimization: ${premeasured ? "premeasured baseline" : "baseline"} is incomplete (${rootCoverage.scorableCellIds.length}/${rootCoverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(rootCoverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4555
|
+
const generation = nodes.generations(earlier).length;
|
|
4556
|
+
if (opts.analyzeGeneration) {
|
|
4557
|
+
const previous = generation - 1;
|
|
4558
|
+
const candidates = (previous < 0 ? [root] : nodes.expansionNodes(earlier, nodes.generations(earlier)[previous])).map((nodeId) => {
|
|
4559
|
+
const campaign = nodes.campaign(earlier, nodeId);
|
|
4560
|
+
return {
|
|
4561
|
+
surfaceHash: surfaceHash(nodes.surface(earlier, nodeId)),
|
|
4562
|
+
campaign,
|
|
4563
|
+
composite: campaignMeanCompositeOrNull(campaign)
|
|
4564
|
+
};
|
|
4565
|
+
});
|
|
4566
|
+
const history = nodes.history(earlier, policy);
|
|
4567
|
+
if (candidates.some(({ campaign }) => campaign.cells.length > 0)) {
|
|
4568
|
+
const fresh = await opts.analyzeGeneration({
|
|
4569
|
+
generation: previous,
|
|
4570
|
+
runDir: previous < 0 ? rootCampaign.runDir : `${runDir}/candidates`,
|
|
4571
|
+
candidates,
|
|
4572
|
+
history,
|
|
4573
|
+
costLedger,
|
|
4574
|
+
costPhase: previous < 0 ? "analysis.baseline" : "analysis.generation"
|
|
4575
|
+
});
|
|
4576
|
+
if (!Array.isArray(fresh)) throw new TypeError("runOptimization: analyzeGeneration must return an array");
|
|
4577
|
+
findings = immutableProposalSnapshot(assertProposalFindings(fresh, "runOptimization analysis findings"), "analysis findings");
|
|
4578
|
+
}
|
|
4579
|
+
}
|
|
4580
|
+
const state = await recorder.state();
|
|
4581
|
+
const proposalHistory = immutableProposalSnapshot(nodes.history(state, policy), "history");
|
|
4582
|
+
const parent = request.parents[0];
|
|
4583
|
+
const context = Object.freeze({
|
|
4584
|
+
currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
|
|
4585
|
+
operator: request.operator,
|
|
4586
|
+
parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
|
|
4587
|
+
nodeId,
|
|
4588
|
+
artifact: immutableProposalSnapshot(artifact, "parent surface")
|
|
4589
|
+
})), "parents"),
|
|
4590
|
+
train: searchProposerView(state),
|
|
4591
|
+
summary: renderSearchSummary(state, { split: "train" }),
|
|
4592
|
+
history: proposalHistory,
|
|
4593
|
+
findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
|
|
4594
|
+
populationSize: opts.populationSize,
|
|
4595
|
+
generation,
|
|
4596
|
+
signal: request.signal,
|
|
4597
|
+
baselineOutcome: immutableProposalSnapshot(nodes.outcome(state, root), "baseline"),
|
|
4598
|
+
incumbentOutcome: immutableProposalSnapshot(nodes.outcome(state, request.leader), "incumbent outcome"),
|
|
4599
|
+
parentOutcome: immutableProposalSnapshot(nodes.outcome(state, parent.nodeId), "parent outcome"),
|
|
4600
|
+
maxImprovementShots: opts.maxImprovementShots,
|
|
4601
|
+
paretoParents: immutableProposalSnapshot(nodes.frontier(state), "Pareto parents"),
|
|
4602
|
+
costLedger,
|
|
4603
|
+
costPhase: "search.proposal"
|
|
4604
|
+
});
|
|
4605
|
+
if (proposer.decide?.({ history: proposalHistory }).stop) return {
|
|
4606
|
+
children: [],
|
|
4607
|
+
stop: "the proposer decided to stop",
|
|
4608
|
+
...accounting()
|
|
4609
|
+
};
|
|
4610
|
+
const proposed = await proposer.propose(context);
|
|
4611
|
+
if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
|
|
4612
|
+
return {
|
|
4613
|
+
children: immutableProposalSnapshot(proposed, "candidate outputs").slice(0, opts.populationSize).map((candidate) => isProposedCandidate(candidate) ? {
|
|
4614
|
+
artifact: candidate.surface,
|
|
4615
|
+
label: candidate.label,
|
|
4616
|
+
rationale: candidate.rationale,
|
|
4617
|
+
...candidate.attribution ? { attribution: candidate.attribution } : {}
|
|
4618
|
+
} : {
|
|
4619
|
+
artifact: candidate,
|
|
4620
|
+
label: "",
|
|
4621
|
+
rationale: ""
|
|
4622
|
+
}),
|
|
4623
|
+
...accounting()
|
|
4624
|
+
};
|
|
4625
|
+
}
|
|
4626
|
+
};
|
|
4627
|
+
const result = await runSearch({
|
|
4628
|
+
recorder,
|
|
4629
|
+
root: baselineSurface,
|
|
4630
|
+
codec: surfaceCodec,
|
|
4631
|
+
policy,
|
|
4632
|
+
allocation,
|
|
4633
|
+
proposer: port,
|
|
4634
|
+
executor,
|
|
4635
|
+
maxExpansions: opts.maxGenerations,
|
|
4636
|
+
signal: opts.signal
|
|
4637
|
+
});
|
|
4638
|
+
const state = result.state;
|
|
4639
|
+
const winnerSurface = nodes.surface(state, result.leader);
|
|
4640
|
+
const firstEdge = state.edge(state.node(result.leader).edgeIds[0]);
|
|
4641
|
+
const winnerRationale = firstEdge.operator === "seed" || !("sha256" in firstEdge.rationale) ? void 0 : recorder.readBlob(firstEdge.rationale).text;
|
|
4642
|
+
const expansions = nodes.generations(state);
|
|
4643
|
+
return {
|
|
4644
|
+
generations: nodes.history(state, policy).map((record) => ({
|
|
4645
|
+
record,
|
|
4646
|
+
surfaces: nodes.expansionNodes(state, expansions[record.generationIndex]).map((nodeId) => {
|
|
4647
|
+
const surface = nodes.surface(state, nodeId);
|
|
4648
|
+
return {
|
|
4649
|
+
surfaceHash: surfaceHash(surface),
|
|
4650
|
+
surface,
|
|
4651
|
+
campaign: nodes.campaign(state, nodeId)
|
|
4652
|
+
};
|
|
4653
|
+
})
|
|
4654
|
+
})),
|
|
4655
|
+
baselineSurface,
|
|
4656
|
+
winnerSurface,
|
|
4657
|
+
winnerSurfaceHash: surfaceHash(winnerSurface),
|
|
4658
|
+
...firstEdge.operator !== "seed" && firstEdge.label ? { winnerLabel: firstEdge.label } : {},
|
|
4659
|
+
...winnerRationale ? { winnerRationale } : {},
|
|
4660
|
+
baselineCampaign: nodes.campaign(state, state.rootNodeId),
|
|
4661
|
+
cost: costLedger.summary(),
|
|
4662
|
+
searchHistory: recorder.receipt({
|
|
4663
|
+
producerId: proposer.kind,
|
|
4664
|
+
runId: runDir
|
|
4665
|
+
}),
|
|
4666
|
+
paretoFrontier: nodes.frontier(state)
|
|
4667
|
+
};
|
|
4668
|
+
}
|
|
4637
4669
|
/** The default ledger and identities when the caller binds none. Revisions are
|
|
4638
4670
|
* content digests of what the caller declared, and each uri names what its
|
|
4639
4671
|
* digest covers: the dispatch ref, the proposer kind, the kernel. */
|
|
@@ -4676,12 +4708,6 @@ async function defaultBinding(opts, runDir, storage) {
|
|
|
4676
4708
|
};
|
|
4677
4709
|
}
|
|
4678
4710
|
}
|
|
4679
|
-
function artifactKindOf(surface) {
|
|
4680
|
-
return typeof surface === "object" && surface.kind === "code" ? "code" : "prompt";
|
|
4681
|
-
}
|
|
4682
|
-
function receiptKey(receipt) {
|
|
4683
|
-
return receipt.callId;
|
|
4684
|
-
}
|
|
4685
4711
|
function immutableProposalSnapshot(value, label) {
|
|
4686
4712
|
try {
|
|
4687
4713
|
return deepFreeze(structuredClone(value));
|
|
@@ -4719,48 +4745,6 @@ function validatedPremeasuredBaseline(args) {
|
|
|
4719
4745
|
if (!coverage.complete) throw new Error(`runOptimization: premeasured baseline is incomplete (${coverage.scorableCellIds.length}/${coverage.expectedCellIds.length} designed cells scorable) — ${formatCoverageFailures(coverage)}. Refusing to optimize against an incomplete incumbent.`);
|
|
4720
4746
|
return campaign;
|
|
4721
4747
|
}
|
|
4722
|
-
/** A scored campaign as a frontier member: per-scenario composite objectives. */
|
|
4723
|
-
function toParetoParent(surface, campaign, generation, label) {
|
|
4724
|
-
const objectives = {};
|
|
4725
|
-
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) objectives[scenarioId] = composite;
|
|
4726
|
-
const parent = {
|
|
4727
|
-
surface,
|
|
4728
|
-
surfaceHash: surfaceHash(surface),
|
|
4729
|
-
objectives,
|
|
4730
|
-
composite: campaignMeanComposite(campaign),
|
|
4731
|
-
generation
|
|
4732
|
-
};
|
|
4733
|
-
if (label) parent.label = label;
|
|
4734
|
-
return parent;
|
|
4735
|
-
}
|
|
4736
|
-
/** The non-dominated set over the per-scenario objective vectors. */
|
|
4737
|
-
function computeParetoFrontier(scored) {
|
|
4738
|
-
if (scored.length <= 1) return [...scored];
|
|
4739
|
-
const ids = /* @__PURE__ */ new Set();
|
|
4740
|
-
for (const p of scored) for (const id of Object.keys(p.objectives)) ids.add(id);
|
|
4741
|
-
if (ids.size === 0) return [...scored];
|
|
4742
|
-
const objectives = [...ids].map((id) => ({
|
|
4743
|
-
name: id,
|
|
4744
|
-
direction: "maximize",
|
|
4745
|
-
value: (p) => p.objectives[id] ?? Number.NEGATIVE_INFINITY
|
|
4746
|
-
}));
|
|
4747
|
-
return paretoFrontier(scored, objectives).frontier;
|
|
4748
|
-
}
|
|
4749
|
-
function toScoredSurfaceOutcome(hash, campaign, coverage, generation) {
|
|
4750
|
-
const breakdown = campaignBreakdown(campaign);
|
|
4751
|
-
return {
|
|
4752
|
-
split: "search",
|
|
4753
|
-
generation,
|
|
4754
|
-
surfaceHash: hash,
|
|
4755
|
-
composite: campaignMeanComposite(campaign),
|
|
4756
|
-
dimensions: breakdown.dimensions,
|
|
4757
|
-
scenarios: breakdown.scenarios,
|
|
4758
|
-
coverage: {
|
|
4759
|
-
expectedCells: coverage.expectedCellIds.length,
|
|
4760
|
-
scorableCells: coverage.scorableCellIds.length
|
|
4761
|
-
}
|
|
4762
|
-
};
|
|
4763
|
-
}
|
|
4764
4748
|
//#endregion
|
|
4765
4749
|
//#region src/campaign/presets/run-improvement-loop.ts
|
|
4766
4750
|
/**
|
|
@@ -4830,10 +4814,126 @@ async function runImprovementLoop(opts) {
|
|
|
4830
4814
|
ghRepo: opts.ghRepo
|
|
4831
4815
|
});
|
|
4832
4816
|
return {
|
|
4833
|
-
...optimization,
|
|
4834
|
-
...comparison,
|
|
4835
|
-
prResult,
|
|
4836
|
-
cost: costLedger.summary()
|
|
4817
|
+
...optimization,
|
|
4818
|
+
...comparison,
|
|
4819
|
+
prResult,
|
|
4820
|
+
cost: costLedger.summary()
|
|
4821
|
+
};
|
|
4822
|
+
}
|
|
4823
|
+
//#endregion
|
|
4824
|
+
//#region src/campaign/optimization-cost.ts
|
|
4825
|
+
/** Attribute method calls while retaining the shared account's admission and read behavior. */
|
|
4826
|
+
function createMethodCostScope(account, methodName) {
|
|
4827
|
+
const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
|
|
4828
|
+
return {
|
|
4829
|
+
ledger: Object.freeze({
|
|
4830
|
+
costCeilingUsd: account.costCeilingUsd,
|
|
4831
|
+
runPaidCall: (input) => account.runPaidCall({
|
|
4832
|
+
...input,
|
|
4833
|
+
tags: {
|
|
4834
|
+
...input.tags,
|
|
4835
|
+
...tags
|
|
4836
|
+
}
|
|
4837
|
+
}),
|
|
4838
|
+
summary: account.summary.bind(account),
|
|
4839
|
+
list: account.list.bind(account),
|
|
4840
|
+
reconcile: account.reconcile.bind(account),
|
|
4841
|
+
markCompleted: account.markCompleted.bind(account),
|
|
4842
|
+
costPerCompletedTask: account.costPerCompletedTask.bind(account),
|
|
4843
|
+
...account.listPending ? { listPending: account.listPending.bind(account) } : {},
|
|
4844
|
+
...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
|
|
4845
|
+
}),
|
|
4846
|
+
reconcile(reported) {
|
|
4847
|
+
const summary = account.summary({ tags });
|
|
4848
|
+
if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
|
|
4849
|
+
const recorded = costFromLedgerSummary(summary);
|
|
4850
|
+
const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
|
|
4851
|
+
const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
|
|
4852
|
+
const combined = combineComparisonCosts([{
|
|
4853
|
+
label: "reported",
|
|
4854
|
+
cost: reported
|
|
4855
|
+
}, {
|
|
4856
|
+
label: "recorded",
|
|
4857
|
+
cost: recorded
|
|
4858
|
+
}]);
|
|
4859
|
+
const incompleteReasons = [
|
|
4860
|
+
...reported.incompleteReasons,
|
|
4861
|
+
...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
|
|
4862
|
+
...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
|
|
4863
|
+
];
|
|
4864
|
+
return {
|
|
4865
|
+
totalCostUsd,
|
|
4866
|
+
costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
|
|
4867
|
+
kind: combined.costProvenance.kind,
|
|
4868
|
+
usd: totalCostUsd
|
|
4869
|
+
},
|
|
4870
|
+
accountingComplete: incompleteReasons.length === 0,
|
|
4871
|
+
incompleteReasons
|
|
4872
|
+
};
|
|
4873
|
+
}
|
|
4874
|
+
};
|
|
4875
|
+
}
|
|
4876
|
+
/** Keep the cost fields a custom optimization method must report. */
|
|
4877
|
+
function costFromLedgerSummary(summary) {
|
|
4878
|
+
const cost = {
|
|
4879
|
+
totalCostUsd: summary.totalCostUsd,
|
|
4880
|
+
costProvenance: structuredClone(summary.costProvenance),
|
|
4881
|
+
accountingComplete: summary.accountingComplete,
|
|
4882
|
+
incompleteReasons: [...summary.incompleteReasons]
|
|
4883
|
+
};
|
|
4884
|
+
assertComparisonCost(cost, "cost ledger");
|
|
4885
|
+
return cost;
|
|
4886
|
+
}
|
|
4887
|
+
/** Combine method costs without turning one unknown bill into a known total. */
|
|
4888
|
+
function combineComparisonCosts(entries) {
|
|
4889
|
+
const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
|
|
4890
|
+
const cost = {
|
|
4891
|
+
totalCostUsd,
|
|
4892
|
+
costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
|
|
4893
|
+
kind: "uncaptured",
|
|
4894
|
+
usd: null
|
|
4895
|
+
} : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
|
|
4896
|
+
kind: "observed",
|
|
4897
|
+
usd: totalCostUsd
|
|
4898
|
+
} : {
|
|
4899
|
+
kind: "estimated",
|
|
4900
|
+
usd: totalCostUsd
|
|
4901
|
+
},
|
|
4902
|
+
accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
|
|
4903
|
+
incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
|
|
4904
|
+
};
|
|
4905
|
+
assertComparisonCost(cost, "combined cost");
|
|
4906
|
+
return cost;
|
|
4907
|
+
}
|
|
4908
|
+
function assertComparisonCost(cost, label) {
|
|
4909
|
+
if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
|
|
4910
|
+
if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
|
|
4911
|
+
const provenance = cost.costProvenance;
|
|
4912
|
+
if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
|
|
4913
|
+
if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
|
|
4914
|
+
if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
|
|
4915
|
+
if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
|
|
4916
|
+
if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
|
|
4917
|
+
if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
|
|
4918
|
+
}
|
|
4919
|
+
/** Preserve every optimizer token class while keeping total input and output explicit. */
|
|
4920
|
+
function optimizationTokenUsageFromSummary(summary, receipts) {
|
|
4921
|
+
if (!summary.usageComplete) return void 0;
|
|
4922
|
+
if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
|
|
4923
|
+
const cachedInputTokens = summary.cachedTokens;
|
|
4924
|
+
const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
|
|
4925
|
+
const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
|
|
4926
|
+
const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
|
|
4927
|
+
const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
|
|
4928
|
+
const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
|
|
4929
|
+
return {
|
|
4930
|
+
inputTokens,
|
|
4931
|
+
...cacheReadComplete ? { cachedInputTokens } : {},
|
|
4932
|
+
...cacheWriteComplete ? { cacheWriteInputTokens } : {},
|
|
4933
|
+
outputTokens: summary.outputTokens,
|
|
4934
|
+
...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
|
|
4935
|
+
totalTokens: inputTokens + summary.outputTokens,
|
|
4936
|
+
calls: summary.totalCalls
|
|
4837
4937
|
};
|
|
4838
4938
|
}
|
|
4839
4939
|
//#endregion
|
|
@@ -5022,102 +5122,6 @@ function assertPositiveSafeInteger(value, label) {
|
|
|
5022
5122
|
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`GEPA candidate population ${label} must be a positive safe integer`);
|
|
5023
5123
|
}
|
|
5024
5124
|
//#endregion
|
|
5025
|
-
//#region src/campaign/optimization-cost.ts
|
|
5026
|
-
/** Attribute method calls while retaining the shared account's admission and read behavior. */
|
|
5027
|
-
function createMethodCostScope(account, methodName) {
|
|
5028
|
-
const tags = { [`optimizationAttempt:${crypto.randomUUID()}`]: methodName };
|
|
5029
|
-
return {
|
|
5030
|
-
ledger: Object.freeze({
|
|
5031
|
-
costCeilingUsd: account.costCeilingUsd,
|
|
5032
|
-
runPaidCall: (input) => account.runPaidCall({
|
|
5033
|
-
...input,
|
|
5034
|
-
tags: {
|
|
5035
|
-
...input.tags,
|
|
5036
|
-
...tags
|
|
5037
|
-
}
|
|
5038
|
-
}),
|
|
5039
|
-
summary: account.summary.bind(account),
|
|
5040
|
-
list: account.list.bind(account),
|
|
5041
|
-
reconcile: account.reconcile.bind(account),
|
|
5042
|
-
markCompleted: account.markCompleted.bind(account),
|
|
5043
|
-
costPerCompletedTask: account.costPerCompletedTask.bind(account),
|
|
5044
|
-
...account.listPending ? { listPending: account.listPending.bind(account) } : {},
|
|
5045
|
-
...account.waitForIdle ? { waitForIdle: account.waitForIdle.bind(account) } : {}
|
|
5046
|
-
}),
|
|
5047
|
-
reconcile(reported) {
|
|
5048
|
-
const summary = account.summary({ tags });
|
|
5049
|
-
if (summary.pendingCalls > 0) throw new Error(`optimization method '${methodName}' returned with ${summary.pendingCalls} pending paid call(s)`);
|
|
5050
|
-
const recorded = costFromLedgerSummary(summary);
|
|
5051
|
-
const totalCostUsd = Math.max(reported.totalCostUsd, recorded.totalCostUsd);
|
|
5052
|
-
const roundingToleranceUsd = Number.EPSILON * Math.max(1, totalCostUsd) * (summary.totalCalls + 1);
|
|
5053
|
-
const combined = combineComparisonCosts([{
|
|
5054
|
-
label: "reported",
|
|
5055
|
-
cost: reported
|
|
5056
|
-
}, {
|
|
5057
|
-
label: "recorded",
|
|
5058
|
-
cost: recorded
|
|
5059
|
-
}]);
|
|
5060
|
-
const incompleteReasons = [
|
|
5061
|
-
...reported.incompleteReasons,
|
|
5062
|
-
...recorded.incompleteReasons.map((reason) => `recorded: ${reason}`),
|
|
5063
|
-
...recorded.totalCostUsd - reported.totalCostUsd > roundingToleranceUsd ? [`reported ${reported.totalCostUsd} USD below recorded ${recorded.totalCostUsd} USD`] : []
|
|
5064
|
-
];
|
|
5065
|
-
return {
|
|
5066
|
-
totalCostUsd,
|
|
5067
|
-
costProvenance: combined.costProvenance.kind === "uncaptured" ? combined.costProvenance : {
|
|
5068
|
-
kind: combined.costProvenance.kind,
|
|
5069
|
-
usd: totalCostUsd
|
|
5070
|
-
},
|
|
5071
|
-
accountingComplete: incompleteReasons.length === 0,
|
|
5072
|
-
incompleteReasons
|
|
5073
|
-
};
|
|
5074
|
-
}
|
|
5075
|
-
};
|
|
5076
|
-
}
|
|
5077
|
-
/** Keep the cost fields a custom optimization method must report. */
|
|
5078
|
-
function costFromLedgerSummary(summary) {
|
|
5079
|
-
const cost = {
|
|
5080
|
-
totalCostUsd: summary.totalCostUsd,
|
|
5081
|
-
costProvenance: structuredClone(summary.costProvenance),
|
|
5082
|
-
accountingComplete: summary.accountingComplete,
|
|
5083
|
-
incompleteReasons: [...summary.incompleteReasons]
|
|
5084
|
-
};
|
|
5085
|
-
assertComparisonCost(cost, "cost ledger");
|
|
5086
|
-
return cost;
|
|
5087
|
-
}
|
|
5088
|
-
/** Combine method costs without turning one unknown bill into a known total. */
|
|
5089
|
-
function combineComparisonCosts(entries) {
|
|
5090
|
-
const totalCostUsd = entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0);
|
|
5091
|
-
const cost = {
|
|
5092
|
-
totalCostUsd,
|
|
5093
|
-
costProvenance: entries.some((entry) => entry.cost.costProvenance.kind === "uncaptured") ? {
|
|
5094
|
-
kind: "uncaptured",
|
|
5095
|
-
usd: null
|
|
5096
|
-
} : entries.every((entry) => entry.cost.costProvenance.kind === "observed") ? {
|
|
5097
|
-
kind: "observed",
|
|
5098
|
-
usd: totalCostUsd
|
|
5099
|
-
} : {
|
|
5100
|
-
kind: "estimated",
|
|
5101
|
-
usd: totalCostUsd
|
|
5102
|
-
},
|
|
5103
|
-
accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
|
|
5104
|
-
incompleteReasons: entries.flatMap((entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`))
|
|
5105
|
-
};
|
|
5106
|
-
assertComparisonCost(cost, "combined cost");
|
|
5107
|
-
return cost;
|
|
5108
|
-
}
|
|
5109
|
-
function assertComparisonCost(cost, label) {
|
|
5110
|
-
if (!cost || typeof cost !== "object") throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
|
|
5111
|
-
if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
|
|
5112
|
-
const provenance = cost.costProvenance;
|
|
5113
|
-
if (!provenance || typeof provenance !== "object" || provenance.kind !== "observed" && provenance.kind !== "estimated" && provenance.kind !== "uncaptured" || (provenance.kind === "uncaptured" ? provenance.usd !== null : !Number.isFinite(provenance.usd) || provenance.usd < 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid costProvenance`);
|
|
5114
|
-
if (provenance.kind !== "uncaptured" && provenance.usd !== cost.totalCostUsd) throw new Error(`compareOptimizationMethods: ${label} returned costProvenance inconsistent with totalCostUsd`);
|
|
5115
|
-
if (typeof cost.accountingComplete !== "boolean") throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
|
|
5116
|
-
if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some((reason) => typeof reason !== "string" || reason.trim().length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
|
|
5117
|
-
if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) throw new Error(`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`);
|
|
5118
|
-
if (cost.accountingComplete && provenance.kind === "uncaptured") throw new Error(`compareOptimizationMethods: ${label} cannot mark uncaptured cost as complete`);
|
|
5119
|
-
}
|
|
5120
|
-
//#endregion
|
|
5121
5125
|
//#region src/campaign/optimization-method.ts
|
|
5122
5126
|
/** Both complete-method workflows use the same detached inputs, accounting, and evidence admission. */
|
|
5123
5127
|
async function executeOptimizationMethod(options) {
|
|
@@ -5266,440 +5270,6 @@ function assertOptimizationProvenance(methodName, value) {
|
|
|
5266
5270
|
if (value.optimizerModel === void 0 !== (value.optimizerCallRef === void 0) || value.optimizerModel === void 0 !== (value.modelExecutions === void 0)) fail("optimizerModel execution provenance");
|
|
5267
5271
|
}
|
|
5268
5272
|
//#endregion
|
|
5269
|
-
//#region src/campaign/presets/compare-optimization-methods.ts
|
|
5270
|
-
/**
|
|
5271
|
-
* Compare optimization methods on shared train, selection, and test data.
|
|
5272
|
-
* Optimizers receive only train and selection data. After every optimizer
|
|
5273
|
-
* finishes, their selected surfaces are measured on the same untouched test
|
|
5274
|
-
* data and compared with paired confidence intervals.
|
|
5275
|
-
*/
|
|
5276
|
-
/**
|
|
5277
|
-
* Compare complete optimization methods on disjoint train, selection, and final test data.
|
|
5278
|
-
*/
|
|
5279
|
-
async function compareOptimizationMethods(opts) {
|
|
5280
|
-
opts = {
|
|
5281
|
-
...opts,
|
|
5282
|
-
methods: opts.methods.map((method) => ({ ...method })),
|
|
5283
|
-
trainScenarios: structuredClone(opts.trainScenarios),
|
|
5284
|
-
selectionScenarios: structuredClone(opts.selectionScenarios),
|
|
5285
|
-
testScenarios: structuredClone(opts.testScenarios),
|
|
5286
|
-
judges: opts.judges.map(captureJudge),
|
|
5287
|
-
claim: opts.claim && defineEvaluationClaim(opts.claim),
|
|
5288
|
-
finalEvidence: opts.finalEvidence && captureFinalEvidencePolicy(opts.finalEvidence)
|
|
5289
|
-
};
|
|
5290
|
-
assertOptimizationMethods(opts.methods);
|
|
5291
|
-
assertComparisonPartitions(opts);
|
|
5292
|
-
const searchHistoryPolicy = opts.searchHistoryPolicy ?? "allow-missing";
|
|
5293
|
-
assertSearchHistoryAdmissionOptions(opts);
|
|
5294
|
-
const seed = opts.seed ?? 42;
|
|
5295
|
-
const confidence = opts.confidence ?? .95;
|
|
5296
|
-
assertConfidence(confidence);
|
|
5297
|
-
const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
|
|
5298
|
-
const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
|
|
5299
|
-
const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
|
|
5300
|
-
const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
|
|
5301
|
-
const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
|
|
5302
|
-
assertComparisonControls(opts, seed, resamples, confidence);
|
|
5303
|
-
const evidence = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
|
|
5304
|
-
const evidenceBySurface = /* @__PURE__ */ new Map();
|
|
5305
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
5306
|
-
const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
|
|
5307
|
-
const baselineSurface = structuredClone(opts.baselineSurface);
|
|
5308
|
-
const costLedger = opts.costLedger ?? createRunCostLedger({
|
|
5309
|
-
storage,
|
|
5310
|
-
runDir: `${resolvedRunDir}/cost`,
|
|
5311
|
-
costCeilingUsd: opts.costCeiling
|
|
5312
|
-
});
|
|
5313
|
-
const scoreOnTest = async (surface, tag, costPhase) => {
|
|
5314
|
-
const measuredSurface = structuredClone(surface);
|
|
5315
|
-
const campaign = await runCampaign({
|
|
5316
|
-
...opts,
|
|
5317
|
-
storage,
|
|
5318
|
-
costLedger,
|
|
5319
|
-
costPhase,
|
|
5320
|
-
scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
|
|
5321
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(structuredClone(measuredSurface), scenario, ctx),
|
|
5322
|
-
dispatchRef: finalDispatchRef(opts, measuredSurface),
|
|
5323
|
-
runDir: `${resolvedRunDir}/${tag}`
|
|
5324
|
-
});
|
|
5325
|
-
assertCompleteCampaign(campaign, opts.testScenarios, opts.reps ?? 1, true, `compareOptimizationMethods: ${tag} final comparison`);
|
|
5326
|
-
if (evidence) evidenceBySurface.set(surfaceContentHash(measuredSurface), createCampaignEvidenceReceipt({
|
|
5327
|
-
campaign,
|
|
5328
|
-
surface: measuredSurface,
|
|
5329
|
-
context: evidence
|
|
5330
|
-
}));
|
|
5331
|
-
const byScenario = {};
|
|
5332
|
-
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) byScenario[scenarioId] = composite;
|
|
5333
|
-
return byScenario;
|
|
5334
|
-
};
|
|
5335
|
-
const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
|
|
5336
|
-
const unitByScenario = opts.claim ? evaluationUnitMap(opts.claim, opts.testScenarios) : new Map(scenarioIds.map((id) => [id, id]));
|
|
5337
|
-
const units = opts.claim ? summarizeEvaluationUnits(opts.claim, opts.testScenarios) : {
|
|
5338
|
-
observations: scenarioIds.length,
|
|
5339
|
-
independentUnits: scenarioIds.length,
|
|
5340
|
-
units: scenarioIds.map((id) => ({
|
|
5341
|
-
id,
|
|
5342
|
-
observations: 1
|
|
5343
|
-
}))
|
|
5344
|
-
};
|
|
5345
|
-
const aggregateUnits = (arr) => {
|
|
5346
|
-
const values = /* @__PURE__ */ new Map();
|
|
5347
|
-
scenarioIds.forEach((id, index) => {
|
|
5348
|
-
const unitId = unitByScenario.get(id);
|
|
5349
|
-
const bucket = values.get(unitId) ?? [];
|
|
5350
|
-
bucket.push(arr[index]);
|
|
5351
|
-
values.set(unitId, bucket);
|
|
5352
|
-
});
|
|
5353
|
-
return units.units.map((unit) => mean(values.get(unit.id)));
|
|
5354
|
-
};
|
|
5355
|
-
const decisionOptions = {
|
|
5356
|
-
seed,
|
|
5357
|
-
resamples,
|
|
5358
|
-
confidence: intervalConfidence,
|
|
5359
|
-
threshold: opts.claim?.minimumEffect ?? 0
|
|
5360
|
-
};
|
|
5361
|
-
const align = (byScenario, label) => {
|
|
5362
|
-
const missing = scenarioIds.filter((id) => !(id in byScenario));
|
|
5363
|
-
if (missing.length > 0) throw new Error(`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(", ")}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`);
|
|
5364
|
-
return scenarioIds.map((id) => byScenario[id]);
|
|
5365
|
-
};
|
|
5366
|
-
if (opts.claim?.generalization === "new-units") assertIndependentEvaluationSplit(opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
|
|
5367
|
-
if (opts.finalEvidence) await reserveFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [...opts.trainScenarios, ...opts.selectionScenarios]);
|
|
5368
|
-
const optimizationOwner = new AbortController();
|
|
5369
|
-
const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
|
|
5370
|
-
try {
|
|
5371
|
-
const { selected: out, cost, history: searchHistoryCoverage } = await executeOptimizationMethod({
|
|
5372
|
-
method,
|
|
5373
|
-
input: createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed, baselineSurface, costLedger, optimizationOwner.signal),
|
|
5374
|
-
storage,
|
|
5375
|
-
searchHistoryPolicy: opts.searchHistoryPolicy,
|
|
5376
|
-
searchHistoryVerification: opts.searchHistoryVerification
|
|
5377
|
-
});
|
|
5378
|
-
const winnerSurface = out.winnerSurface;
|
|
5379
|
-
return {
|
|
5380
|
-
name: method.name,
|
|
5381
|
-
winnerSurface,
|
|
5382
|
-
cost,
|
|
5383
|
-
...out.durationMs === void 0 ? {} : { durationMs: out.durationMs },
|
|
5384
|
-
...out.composition === void 0 ? {} : { composition: out.composition },
|
|
5385
|
-
...out.provenance === void 0 ? {} : { provenance: out.provenance },
|
|
5386
|
-
searchHistoryCoverage
|
|
5387
|
-
};
|
|
5388
|
-
} catch (error) {
|
|
5389
|
-
if (!optimizationOwner.signal.aborted) optimizationOwner.abort(error);
|
|
5390
|
-
throw error;
|
|
5391
|
-
}
|
|
5392
|
-
});
|
|
5393
|
-
assertReportedCostWithinCeiling(combineComparisonCosts(optimized.map((result) => ({
|
|
5394
|
-
label: `method '${result.name}'`,
|
|
5395
|
-
cost: result.cost
|
|
5396
|
-
}))).totalCostUsd, opts.costCeiling, "optimization");
|
|
5397
|
-
const testCostPhase = finalCostPhase(opts, baselineSurface, optimized, seed);
|
|
5398
|
-
const finalEvidence = opts.finalEvidence ? await exposeFinalEvidence(opts.finalEvidence, opts.claim, opts.testScenarios, [baselineSurface, ...optimized.map((method) => method.winnerSurface)]) : void 0;
|
|
5399
|
-
const baselineArr = align(await scoreOnTest(baselineSurface, "test/baseline", testCostPhase), "baseline");
|
|
5400
|
-
const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
|
|
5401
|
-
const winners = [];
|
|
5402
|
-
for (const winner of optimized) {
|
|
5403
|
-
const surfaceKey = surfaceContentHash(winner.winnerSurface);
|
|
5404
|
-
let arr = testScoresBySurface.get(surfaceKey);
|
|
5405
|
-
if (!arr) {
|
|
5406
|
-
arr = align(await scoreOnTest(winner.winnerSurface, `test/methods/${slug(winner.name)}`, testCostPhase), `method "${winner.name}"`);
|
|
5407
|
-
testScoresBySurface.set(surfaceKey, arr);
|
|
5408
|
-
}
|
|
5409
|
-
winners.push({
|
|
5410
|
-
...winner,
|
|
5411
|
-
arr
|
|
5412
|
-
});
|
|
5413
|
-
}
|
|
5414
|
-
const scores = winners.map((w) => {
|
|
5415
|
-
const baselineUnits = aggregateUnits(baselineArr);
|
|
5416
|
-
const winnerUnits = aggregateUnits(w.arr);
|
|
5417
|
-
const decision = decidePairedPromotion(baselineUnits, winnerUnits, decisionOptions);
|
|
5418
|
-
const score = {
|
|
5419
|
-
name: w.name,
|
|
5420
|
-
baselineComposite: mean(baselineUnits),
|
|
5421
|
-
winnerComposite: mean(winnerUnits),
|
|
5422
|
-
lift: decision.delta,
|
|
5423
|
-
liftCi: {
|
|
5424
|
-
low: decision.low,
|
|
5425
|
-
high: decision.high
|
|
5426
|
-
},
|
|
5427
|
-
decision,
|
|
5428
|
-
optimizationCost: w.cost,
|
|
5429
|
-
scenarioScores: scenarioIds.map((scenarioId, index) => ({
|
|
5430
|
-
scenarioId,
|
|
5431
|
-
baselineComposite: baselineArr[index],
|
|
5432
|
-
winnerComposite: w.arr[index],
|
|
5433
|
-
lift: w.arr[index] - baselineArr[index]
|
|
5434
|
-
})),
|
|
5435
|
-
unitScores: units.units.map((unit, index) => ({
|
|
5436
|
-
unitId: unit.id,
|
|
5437
|
-
scenarios: unit.observations,
|
|
5438
|
-
baselineComposite: baselineUnits[index],
|
|
5439
|
-
winnerComposite: winnerUnits[index],
|
|
5440
|
-
lift: winnerUnits[index] - baselineUnits[index]
|
|
5441
|
-
})),
|
|
5442
|
-
winnerSurface: structuredClone(w.winnerSurface),
|
|
5443
|
-
rank: 0
|
|
5444
|
-
};
|
|
5445
|
-
if (evidence) {
|
|
5446
|
-
const baseline = evidenceBySurface.get(surfaceContentHash(baselineSurface));
|
|
5447
|
-
const winner = evidenceBySurface.get(surfaceContentHash(w.winnerSurface));
|
|
5448
|
-
if (!baseline || !winner) throw new Error("final measurement evidence is missing");
|
|
5449
|
-
score.evidence = {
|
|
5450
|
-
baseline,
|
|
5451
|
-
winner
|
|
5452
|
-
};
|
|
5453
|
-
}
|
|
5454
|
-
if (w.composition !== void 0) score.composition = structuredClone(w.composition);
|
|
5455
|
-
if (w.durationMs !== void 0) score.durationMs = w.durationMs;
|
|
5456
|
-
if (w.provenance !== void 0) score.provenance = structuredClone(w.provenance);
|
|
5457
|
-
return score;
|
|
5458
|
-
});
|
|
5459
|
-
scores.sort((a, b) => b.lift - a.lift);
|
|
5460
|
-
for (let start = 0; start < scores.length;) {
|
|
5461
|
-
let end = start + 1;
|
|
5462
|
-
while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
|
|
5463
|
-
const tied = scores.slice(start, end);
|
|
5464
|
-
if (tied.every((score) => score.optimizationCost.accountingComplete)) {
|
|
5465
|
-
tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
|
|
5466
|
-
scores.splice(start, tied.length, ...tied);
|
|
5467
|
-
}
|
|
5468
|
-
start = end;
|
|
5469
|
-
}
|
|
5470
|
-
scores.forEach((s, i) => {
|
|
5471
|
-
s.rank = i + 1;
|
|
5472
|
-
});
|
|
5473
|
-
const best = scores[0];
|
|
5474
|
-
const byName = new Map(winners.map((w) => [w.name, w]));
|
|
5475
|
-
const bestArr = aggregateUnits(byName.get(best.name).arr);
|
|
5476
|
-
const pairwise = scores.slice(1).map((other) => {
|
|
5477
|
-
const otherArr = aggregateUnits(byName.get(other.name).arr);
|
|
5478
|
-
const decision = decidePairedPromotion(otherArr, bestArr, decisionOptions);
|
|
5479
|
-
const reverse = decidePairedPromotion(bestArr, otherArr, decisionOptions);
|
|
5480
|
-
const favored = decision.promote ? best.name : reverse.promote ? other.name : null;
|
|
5481
|
-
return {
|
|
5482
|
-
a: best.name,
|
|
5483
|
-
b: other.name,
|
|
5484
|
-
deltaMean: decision.delta,
|
|
5485
|
-
low: decision.low,
|
|
5486
|
-
high: decision.high,
|
|
5487
|
-
favored,
|
|
5488
|
-
decision
|
|
5489
|
-
};
|
|
5490
|
-
});
|
|
5491
|
-
const optimizationCost = combineComparisonCosts(scores.map((score) => ({
|
|
5492
|
-
label: `method '${score.name}'`,
|
|
5493
|
-
cost: score.optimizationCost
|
|
5494
|
-
})));
|
|
5495
|
-
const testCost = costFromLedgerSummary(costLedger.summary({ phase: testCostPhase }));
|
|
5496
|
-
const totalCost = combineComparisonCosts([{
|
|
5497
|
-
label: "optimization",
|
|
5498
|
-
cost: optimizationCost
|
|
5499
|
-
}, {
|
|
5500
|
-
label: "final test",
|
|
5501
|
-
cost: testCost
|
|
5502
|
-
}]);
|
|
5503
|
-
assertReportedCostWithinCeiling(totalCost.totalCostUsd, opts.costCeiling, "total");
|
|
5504
|
-
return {
|
|
5505
|
-
scores,
|
|
5506
|
-
best,
|
|
5507
|
-
pairwise,
|
|
5508
|
-
testScenarioIds: scenarioIds,
|
|
5509
|
-
units,
|
|
5510
|
-
observationUnit: opts.claim ? "registered" : "scenario",
|
|
5511
|
-
pairedCellN: scenarioIds.length * (opts.reps ?? 1),
|
|
5512
|
-
...opts.claim ? { claim: opts.claim } : {},
|
|
5513
|
-
...finalEvidence ? { finalEvidence } : {},
|
|
5514
|
-
optimizationCost,
|
|
5515
|
-
testCost,
|
|
5516
|
-
totalCost,
|
|
5517
|
-
confidence,
|
|
5518
|
-
intervalConfidence,
|
|
5519
|
-
comparisonCount,
|
|
5520
|
-
seed,
|
|
5521
|
-
resamples,
|
|
5522
|
-
reps: opts.reps ?? 1,
|
|
5523
|
-
searchHistory: Object.freeze({
|
|
5524
|
-
policy: searchHistoryPolicy,
|
|
5525
|
-
allComplete: optimized.every((result) => result.searchHistoryCoverage.status === "complete"),
|
|
5526
|
-
producers: Object.freeze(optimized.map((result) => result.searchHistoryCoverage))
|
|
5527
|
-
})
|
|
5528
|
-
};
|
|
5529
|
-
}
|
|
5530
|
-
function assertReportedCostWithinCeiling(totalCostUsd, costCeiling, phase) {
|
|
5531
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(totalCostUsd), Math.abs(costCeiling ?? 0)) * 8;
|
|
5532
|
-
if (costCeiling !== void 0 && totalCostUsd > costCeiling + tolerance) throw new Error(`compareOptimizationMethods: reported ${phase} cost ${totalCostUsd} exceeds costCeiling ${costCeiling}`);
|
|
5533
|
-
}
|
|
5534
|
-
function assertOptimizationMethods(methods) {
|
|
5535
|
-
if (!Array.isArray(methods) || methods.length === 0) throw new Error("compareOptimizationMethods: no methods to compare");
|
|
5536
|
-
const names = /* @__PURE__ */ new Set();
|
|
5537
|
-
const pathOwners = /* @__PURE__ */ new Map();
|
|
5538
|
-
for (const method of methods) {
|
|
5539
|
-
if (!method || typeof method !== "object" || typeof method.optimize !== "function") throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
|
|
5540
|
-
if (!method.name || method.name.trim() !== method.name) throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
|
|
5541
|
-
if (names.has(method.name)) throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
|
|
5542
|
-
names.add(method.name);
|
|
5543
|
-
const pathKey = slug(method.name);
|
|
5544
|
-
const prior = pathOwners.get(pathKey);
|
|
5545
|
-
if (prior) throw new Error(`compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`);
|
|
5546
|
-
pathOwners.set(pathKey, method.name);
|
|
5547
|
-
}
|
|
5548
|
-
}
|
|
5549
|
-
function assertComparisonControls(opts, seed, resamples, confidence) {
|
|
5550
|
-
if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) throw new Error("compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring");
|
|
5551
|
-
if (!opts.judges || opts.judges.length === 0) throw new Error("compareOptimizationMethods: at least one judge is required");
|
|
5552
|
-
if (typeof opts.dispatchWithSurface !== "function") throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
|
|
5553
|
-
if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) throw new Error("compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided");
|
|
5554
|
-
const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
|
|
5555
|
-
if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) throw new Error("compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided");
|
|
5556
|
-
if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) throw new Error("compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided");
|
|
5557
|
-
try {
|
|
5558
|
-
surfaceContentHash(opts.baselineSurface);
|
|
5559
|
-
} catch (cause) {
|
|
5560
|
-
throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
|
|
5561
|
-
}
|
|
5562
|
-
const judgeNames = /* @__PURE__ */ new Set();
|
|
5563
|
-
for (const judge of opts.judges) {
|
|
5564
|
-
if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) throw new Error("compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)");
|
|
5565
|
-
if (judgeNames.has(judge.name)) throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
|
|
5566
|
-
judgeNames.add(judge.name);
|
|
5567
|
-
const dimensionKeys = /* @__PURE__ */ new Set();
|
|
5568
|
-
for (const dimension of judge.dimensions) {
|
|
5569
|
-
if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`);
|
|
5570
|
-
if (dimensionKeys.has(dimension.key)) throw new Error(`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`);
|
|
5571
|
-
dimensionKeys.add(dimension.key);
|
|
5572
|
-
}
|
|
5573
|
-
}
|
|
5574
|
-
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
|
|
5575
|
-
if (!Number.isSafeInteger(seed)) throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
|
|
5576
|
-
if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) throw new Error(`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`);
|
|
5577
|
-
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
|
|
5578
|
-
const minimumResamples = minimumBootstrapResamples(confidence, opts.methods.length * (opts.methods.length + 1) / 2);
|
|
5579
|
-
if (resamples < minimumResamples) throw new Error(`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`);
|
|
5580
|
-
if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) throw new Error("compareOptimizationMethods: optimizationConcurrency must be a positive safe integer");
|
|
5581
|
-
if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
|
|
5582
|
-
if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) throw new Error("compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647");
|
|
5583
|
-
if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) throw new Error("compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0");
|
|
5584
|
-
if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) throw new Error("compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling");
|
|
5585
|
-
}
|
|
5586
|
-
function assertComparisonPartitions(opts) {
|
|
5587
|
-
const partitions = [
|
|
5588
|
-
{
|
|
5589
|
-
name: "trainScenarios",
|
|
5590
|
-
scenarios: opts.trainScenarios
|
|
5591
|
-
},
|
|
5592
|
-
{
|
|
5593
|
-
name: "selectionScenarios",
|
|
5594
|
-
scenarios: opts.selectionScenarios
|
|
5595
|
-
},
|
|
5596
|
-
{
|
|
5597
|
-
name: "testScenarios",
|
|
5598
|
-
scenarios: opts.testScenarios
|
|
5599
|
-
}
|
|
5600
|
-
];
|
|
5601
|
-
const owner = /* @__PURE__ */ new Map();
|
|
5602
|
-
for (const partition of partitions) {
|
|
5603
|
-
if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
|
|
5604
|
-
if (partition.name === "testScenarios" && partition.scenarios.length < 2) throw new Error("compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty");
|
|
5605
|
-
const local = /* @__PURE__ */ new Set();
|
|
5606
|
-
const duplicates = /* @__PURE__ */ new Set();
|
|
5607
|
-
const overlaps = /* @__PURE__ */ new Map();
|
|
5608
|
-
for (const scenario of partition.scenarios) {
|
|
5609
|
-
if (local.has(scenario.id)) duplicates.add(scenario.id);
|
|
5610
|
-
local.add(scenario.id);
|
|
5611
|
-
const prior = owner.get(scenario.id);
|
|
5612
|
-
if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
|
|
5613
|
-
}
|
|
5614
|
-
if (duplicates.size > 0) throw new Error(`compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[...duplicates].join(", ")}]`);
|
|
5615
|
-
if (overlaps.size > 0) {
|
|
5616
|
-
const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} ∩ ${partition.name})`).join(", ");
|
|
5617
|
-
throw new Error(`compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`);
|
|
5618
|
-
}
|
|
5619
|
-
assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
|
|
5620
|
-
for (const id of local) owner.set(id, partition.name);
|
|
5621
|
-
}
|
|
5622
|
-
}
|
|
5623
|
-
function mean(xs) {
|
|
5624
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
5625
|
-
}
|
|
5626
|
-
function slug(name) {
|
|
5627
|
-
return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
|
|
5628
|
-
}
|
|
5629
|
-
function assertConfidence(confidence) {
|
|
5630
|
-
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) throw new Error(`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`);
|
|
5631
|
-
}
|
|
5632
|
-
function minimumBootstrapResamples(confidence, comparisonCount) {
|
|
5633
|
-
const exact = 2 * comparisonCount / (1 - confidence);
|
|
5634
|
-
return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
|
|
5635
|
-
}
|
|
5636
|
-
function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
|
|
5637
|
-
const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
|
|
5638
|
-
const signal = combineAbortSignals(opts.signal, opts.optimizationRunOptions?.signal, optimizationSignal);
|
|
5639
|
-
return Object.freeze({
|
|
5640
|
-
baselineSurface: structuredClone(baselineSurface),
|
|
5641
|
-
trainScenarios: opts.trainScenarios,
|
|
5642
|
-
selectionScenarios: opts.selectionScenarios,
|
|
5643
|
-
dispatchWithSurface: opts.dispatchWithSurface,
|
|
5644
|
-
judges: opts.judges,
|
|
5645
|
-
runDir: methodRunDir,
|
|
5646
|
-
seed,
|
|
5647
|
-
runOptions: Object.freeze({
|
|
5648
|
-
...opts.optimizationRunOptions ?? {},
|
|
5649
|
-
...opts.optimizationRunOptions?.dispatchRef === void 0 && opts.dispatchRef !== void 0 ? { dispatchRef: opts.dispatchRef } : {},
|
|
5650
|
-
...signal ? { signal } : {}
|
|
5651
|
-
}),
|
|
5652
|
-
costLedger
|
|
5653
|
-
});
|
|
5654
|
-
}
|
|
5655
|
-
function finalCostPhase(opts, baselineSurface, winners, seed) {
|
|
5656
|
-
return `compareOptimizationMethods.test:${contentHash({
|
|
5657
|
-
baseline: surfaceContentHash(baselineSurface),
|
|
5658
|
-
winners: winners.map((winner) => ({
|
|
5659
|
-
name: winner.name,
|
|
5660
|
-
surface: surfaceContentHash(winner.winnerSurface)
|
|
5661
|
-
})),
|
|
5662
|
-
testScenarios: opts.testScenarios,
|
|
5663
|
-
judges: opts.judges.map((judge) => ({
|
|
5664
|
-
name: judge.name,
|
|
5665
|
-
dimensions: judge.dimensions,
|
|
5666
|
-
version: judge.judgeVersion ?? contentHash({
|
|
5667
|
-
score: judge.score.toString(),
|
|
5668
|
-
appliesTo: judge.appliesTo?.toString() ?? null
|
|
5669
|
-
})
|
|
5670
|
-
})),
|
|
5671
|
-
dispatch: callerDispatchRef(opts),
|
|
5672
|
-
seed,
|
|
5673
|
-
reps: opts.reps ?? 1
|
|
5674
|
-
})}`;
|
|
5675
|
-
}
|
|
5676
|
-
function finalDispatchRef(opts, surface) {
|
|
5677
|
-
return `compareOptimizationMethods:${callerDispatchRef(opts)}:${surfaceContentHash(surface)}`;
|
|
5678
|
-
}
|
|
5679
|
-
function callerDispatchRef(opts) {
|
|
5680
|
-
return (opts.dispatchRef ?? opts.dispatchWithSurface.name) || "anonymous";
|
|
5681
|
-
}
|
|
5682
|
-
/** Preserve every optimizer token class while keeping total input and output explicit. */
|
|
5683
|
-
function optimizationTokenUsageFromSummary(summary, receipts) {
|
|
5684
|
-
if (!summary.usageComplete) return void 0;
|
|
5685
|
-
if (receipts.length !== summary.totalCalls) throw new Error("optimization token usage receipt count does not match the cost summary");
|
|
5686
|
-
const cachedInputTokens = summary.cachedTokens;
|
|
5687
|
-
const cacheWriteInputTokens = summary.cacheWriteTokens ?? 0;
|
|
5688
|
-
const inputTokens = summary.inputTokens + cachedInputTokens + cacheWriteInputTokens;
|
|
5689
|
-
const cacheReadComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cachedTokens !== void 0);
|
|
5690
|
-
const cacheWriteComplete = receipts.length === 0 || receipts.every((receipt) => receipt.cacheWriteTokens !== void 0);
|
|
5691
|
-
const reasoningComplete = receipts.length === 0 || receipts.every((receipt) => receipt.reasoningTokens !== void 0);
|
|
5692
|
-
return {
|
|
5693
|
-
inputTokens,
|
|
5694
|
-
...cacheReadComplete ? { cachedInputTokens } : {},
|
|
5695
|
-
...cacheWriteComplete ? { cacheWriteInputTokens } : {},
|
|
5696
|
-
outputTokens: summary.outputTokens,
|
|
5697
|
-
...reasoningComplete ? { reasoningTokens: summary.reasoningTokens ?? 0 } : {},
|
|
5698
|
-
totalTokens: inputTokens + summary.outputTokens,
|
|
5699
|
-
calls: summary.totalCalls
|
|
5700
|
-
};
|
|
5701
|
-
}
|
|
5702
|
-
//#endregion
|
|
5703
5273
|
//#region src/run-score.ts
|
|
5704
5274
|
const DEFAULT_RUN_SCORE_WEIGHTS = {
|
|
5705
5275
|
success: 4,
|
|
@@ -6172,6 +5742,6 @@ function firstString(value) {
|
|
|
6172
5742
|
return typeof value === "string" && value.trim() ? value : void 0;
|
|
6173
5743
|
}
|
|
6174
5744
|
//#endregion
|
|
6175
|
-
export {
|
|
5745
|
+
export { planCampaignRun as $, verifySearchHistoryReceipt as A, defaultProductionGate as B, searchEdgeId as C, assertSearchHistoryMatchesState as D, assertSearchHistoryAdmissionOptions as E, asha as F, runCanaries as G, redTeamDataset as H, uniform as I, evaluationUnitMap as J, assertIndependentEvaluationSplit as K, runFinalComparison as L, searchModelIdentity as M, searchProposalExecution as N, createSearchHistoryReceipt as O, searchReceiptAccounting as P, runCampaign as Q, openAutoPr as R, developmentClaim as S, surfaceNode as T, redTeamReport as U, DEFAULT_RED_TEAM_CORPUS as V, scoreRedTeamOutput as W, reserveFinalEvidence as X, exposeFinalEvidence as Y, runEval as Z, beam as _, aggregateRunScore as a, incumbentWithOperatorBandit as b, assertGepaCandidatePopulationSummary as c, costFromLedgerSummary as d, resolveRunDir as et, optimizationTokenUsageFromSummary as f, aide as g, isProposedCandidate as h, transientDispatchFailure as i, campaignCellSearchResult as j, verifySearchHistoryArtifact as k, readGepaCandidatePopulationArtifact as l, runOptimization as m, JudgeParseError as n, cellCachePath as nt, clamp01 as o, runImprovementLoop as p, captureFinalEvidencePolicy as q, isTransientTransportFailure as r, computeManifestHash as rt, executeOptimizationMethod as s, llmJudge as t, buildCellSchedule as tt, combineComparisonCosts as u, crowdedFrontierParent as v, surfaceDiff as w, SearchRecorder as x, incumbent as y, captureJudge as z };
|
|
6176
5746
|
|
|
6177
|
-
//# sourceMappingURL=llm-judge-
|
|
5747
|
+
//# sourceMappingURL=llm-judge-C8jMTNNG.js.map
|