@tangle-network/agent-eval 0.144.11 → 0.144.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/agent-profile-B9_GGsG8.d.ts +84 -0
- package/dist/agent-profile-B9_GGsG8.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts → agent-profile-cell-BkcRDikH.d.ts} +2 -2
- package/dist/{agent-profile-cell-BOP-iA9Q.d.ts.map → agent-profile-cell-BkcRDikH.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +134 -16
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +364 -10
- package/dist/analyst/index.js.map +1 -1
- package/dist/backend-integrity-CsVin_Wb.d.ts +280 -0
- package/dist/backend-integrity-CsVin_Wb.d.ts.map +1 -0
- package/dist/{baseline-C-GocmIW.js → baseline-BhPRQBVn.js} +3 -2
- package/dist/{baseline-C-GocmIW.js.map → baseline-BhPRQBVn.js.map} +1 -1
- package/dist/{benchmark-CWeqGl7x.js → benchmark-BhT16ep9.js} +2 -2
- package/dist/{benchmark-CWeqGl7x.js.map → benchmark-BhT16ep9.js.map} +1 -1
- package/dist/{agent-profile-DPi7IZg7.d.ts → benchmark-Ceoan7vk.d.ts} +4 -91
- package/dist/benchmark-Ceoan7vk.d.ts.map +1 -0
- package/dist/{benchmark-command-BteMFN62.js → benchmark-command-CY6cbhOB.js} +36 -79
- package/dist/benchmark-command-CY6cbhOB.js.map +1 -0
- package/dist/benchmarks/index.d.ts +244 -2
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/index.js +733 -1
- package/dist/benchmarks/index.js.map +1 -0
- package/dist/builder-eval/index.d.ts +23 -2
- package/dist/builder-eval/index.d.ts.map +1 -1
- package/dist/builder-eval/index.js +227 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +10 -8
- package/dist/campaign/index.js +9 -6
- package/dist/{campaign-C2TTzQII.js → campaign-BYjBAypg.js} +21 -634
- package/dist/campaign-BYjBAypg.js.map +1 -0
- package/dist/{canonical-D011XM8r.js → canonical-D-XsTQ6_.js} +2 -2
- package/dist/{canonical-D011XM8r.js.map → canonical-D-XsTQ6_.js.map} +1 -1
- package/dist/{index-CQTZ-4XN.d.ts → capture-fetch-DDvpjVRU.d.ts} +3 -3
- package/dist/capture-fetch-DDvpjVRU.d.ts.map +1 -0
- package/dist/{default-registry-BmktKy8r.js → chat-client-Bp4Ebuuc.js} +1276 -1276
- package/dist/chat-client-Bp4Ebuuc.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-C9gzZE59.d.ts → client-KUbGulm_.d.ts} +4 -4
- package/dist/{client-C9gzZE59.d.ts.map → client-KUbGulm_.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +13 -390
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +18 -542
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-DMFxsLKr.js → cost-ledger-BSe92yAV.js} +3 -3
- package/dist/{cost-ledger-DMFxsLKr.js.map → cost-ledger-BSe92yAV.js.map} +1 -1
- package/dist/{cost-ledger-Bv_e8XHY.d.ts → cost-ledger-DbQdN3nO.d.ts} +2 -2
- package/dist/{cost-ledger-Bv_e8XHY.d.ts.map → cost-ledger-DbQdN3nO.d.ts.map} +1 -1
- package/dist/{counterfactual-CxmxAONP.d.ts → counterfactual--bpysZF0.d.ts} +2 -14
- package/dist/{counterfactual-CxmxAONP.d.ts.map → counterfactual--bpysZF0.d.ts.map} +1 -1
- package/dist/{counterfactual-CWPTrMH7.js → counterfactual-lDfCx0Uz.js} +3 -31
- package/dist/{counterfactual-CWPTrMH7.js.map → counterfactual-lDfCx0Uz.js.map} +1 -1
- package/dist/{dataset-C8xaLXdY.d.ts → dataset-CJjKqQfA.d.ts} +2 -8
- package/dist/dataset-CJjKqQfA.d.ts.map +1 -0
- package/dist/{default-registry-Bf8Woqmq.d.ts → default-registry-Di6HP6pG.d.ts} +7 -12
- package/dist/default-registry-Di6HP6pG.d.ts.map +1 -0
- package/dist/{analyze-runs-C30yljDJ.js → define-agent-eval-iqjT--ZZ.js} +542 -130
- package/dist/define-agent-eval-iqjT--ZZ.js.map +1 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts +388 -0
- package/dist/define-agent-eval-sH24zBfM.d.ts.map +1 -0
- package/dist/descriptive-B5MwKfbf.js +144 -0
- package/dist/descriptive-B5MwKfbf.js.map +1 -0
- package/dist/{dspy-rlm-engine-DbTk4JdR.js → dspy-rlm-engine-B_qhSc21.js} +4 -4
- package/dist/{dspy-rlm-engine-DbTk4JdR.js.map → dspy-rlm-engine-B_qhSc21.js.map} +1 -1
- package/dist/effect-sizes-DiH8MGOH.js +82 -0
- package/dist/effect-sizes-DiH8MGOH.js.map +1 -0
- package/dist/{engine-3hL-XqwJ.d.ts → engine-otFpE2gF.d.ts} +10 -38
- package/dist/engine-otFpE2gF.d.ts.map +1 -0
- package/dist/{errors-CKPfb2aH.d.ts → errors-DEE6u6ot.d.ts} +2 -14
- package/dist/{errors-CKPfb2aH.d.ts.map → errors-DEE6u6ot.d.ts.map} +1 -1
- package/dist/{errors-D-LKuDhb.js → errors-Dngq5h35.js} +2 -8
- package/dist/{errors-D-LKuDhb.js.map → errors-Dngq5h35.js.map} +1 -1
- package/dist/{eval-campaign-B_7wcnav.js → eval-campaign-UB-usSQ2.js} +6 -6
- package/dist/{eval-campaign-B_7wcnav.js.map → eval-campaign-UB-usSQ2.js.map} +1 -1
- package/dist/{exact-types-DSFFpLLI.d.ts → exact-types-BH1twmAJ.d.ts} +2 -2
- package/dist/{exact-types-DSFFpLLI.d.ts.map → exact-types-BH1twmAJ.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +9 -6
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +11 -7
- package/dist/experiment/index.js.map +1 -1
- package/dist/experiment-tracker-C29gXM4B.js +269 -0
- package/dist/experiment-tracker-C29gXM4B.js.map +1 -0
- package/dist/{experiment-tracker-IMntXr6J.d.ts → experiment-tracker-DWHZBAYL.d.ts} +77 -121
- package/dist/experiment-tracker-DWHZBAYL.d.ts.map +1 -0
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts → external-optimizer-contracts-lixrOZdX.d.ts} +3 -3
- package/dist/{external-optimizer-contracts-CZuJNcT5.d.ts.map → external-optimizer-contracts-lixrOZdX.d.ts.map} +1 -1
- package/dist/external-optimizer-process-BTiNB-RH.js +301 -0
- package/dist/external-optimizer-process-BTiNB-RH.js.map +1 -0
- package/dist/{single-run-lock-DFWHEB09.js → external-optimizer-subprocess-DrJ9hR8u.js} +144 -438
- package/dist/external-optimizer-subprocess-DrJ9hR8u.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/{failure-cluster-CqcvCcdR.d.ts → failure-cluster-BLURuWG4.d.ts} +2 -3
- package/dist/failure-cluster-BLURuWG4.d.ts.map +1 -0
- package/dist/{feedback-trajectory-CacHpxsp.d.ts → feedback-trajectory-juOozjAc.d.ts} +5 -36
- package/dist/feedback-trajectory-juOozjAc.d.ts.map +1 -0
- package/dist/fuzz.d.ts +2 -2
- package/dist/fuzz.js +2 -2
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-CvSN3IG1.d.ts → index-B8Ui1mr1.d.ts} +2 -2
- package/dist/{index-CvSN3IG1.d.ts.map → index-B8Ui1mr1.d.ts.map} +1 -1
- package/dist/{skill-usage-CO9OLRBx.d.ts → index-BWDrSVfw.d.ts} +11 -136
- package/dist/index-BWDrSVfw.d.ts.map +1 -0
- package/dist/index-Ba3YrbAL.d.ts +1 -0
- package/dist/{index-BnEuDAK2.d.ts → index-COQYtuRF.d.ts} +3 -3
- package/dist/{index-BnEuDAK2.d.ts.map → index-COQYtuRF.d.ts.map} +1 -1
- package/dist/{index-C3ssXVLv.d.ts → index-CvjYbU0D.d.ts} +2 -2
- package/dist/{index-C3ssXVLv.d.ts.map → index-CvjYbU0D.d.ts.map} +1 -1
- package/dist/{index-DPPGNJ_R.d.ts → index-DSmEylT9.d.ts} +17 -115
- package/dist/index-DSmEylT9.d.ts.map +1 -0
- package/dist/index.d.ts +2397 -5308
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5914 -10496
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-CgX_s0Ez.d.ts → insight-report-CRi-Ufrj.d.ts} +4 -4
- package/dist/{insight-report-CgX_s0Ez.d.ts.map → insight-report-CRi-Ufrj.d.ts.map} +1 -1
- package/dist/{integrity-MLzHOfV9.js → integrity-Cy9WHAtb.js} +2 -2
- package/dist/{integrity-MLzHOfV9.js.map → integrity-Cy9WHAtb.js.map} +1 -1
- package/dist/{integrity-DY6tIbl0.js → integrity-DysDBWDu.js} +2 -2
- package/dist/{integrity-DY6tIbl0.js.map → integrity-DysDBWDu.js.map} +1 -1
- package/dist/{integrity-BuqEKu-x.d.ts → integrity-OrcI9Nau.d.ts} +3 -3
- package/dist/{integrity-BuqEKu-x.d.ts.map → integrity-OrcI9Nau.d.ts.map} +1 -1
- package/dist/internal-BDHPCnjk.js +230 -0
- package/dist/internal-BDHPCnjk.js.map +1 -0
- package/dist/judge-calibration-C5CbMYce.d.ts +117 -0
- package/dist/judge-calibration-C5CbMYce.d.ts.map +1 -0
- package/dist/judge-calibration-DZkWrm5H.js +317 -0
- package/dist/judge-calibration-DZkWrm5H.js.map +1 -0
- package/dist/{kind-factory-B8-r8-y8.js → kind-factory-CPmSd58s.js} +208 -208
- package/dist/{kind-factory-B8-r8-y8.js.map → kind-factory-CPmSd58s.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +1 -1
- package/dist/ledger-core/index.js +2 -2
- package/dist/{ledger-core-DXZIqu17.js → ledger-core-BmZt19oQ.js} +110 -110
- package/dist/{ledger-core-DXZIqu17.js.map → ledger-core-BmZt19oQ.js.map} +1 -1
- package/dist/{llm-client-DzvMUsS_.js → llm-client-d0-2TT1g.js} +4 -64
- package/dist/{llm-client-DzvMUsS_.js.map → llm-client-d0-2TT1g.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CQdVeM8k.js → llm-judge-dZ8P6nGI.js} +3349 -5401
- package/dist/llm-judge-dZ8P6nGI.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +3 -3
- package/dist/{metrics-C9YY1OcL.js → metrics-Cl0L1KUy.js} +2 -108
- package/dist/{metrics-C9YY1OcL.js.map → metrics-Cl0L1KUy.js.map} +1 -1
- package/dist/{mint-B2O60ACG.js → mint-Dj9Ww_3I.js} +4 -4
- package/dist/{mint-B2O60ACG.js.map → mint-Dj9Ww_3I.js.map} +1 -1
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts → multi-layer-verifier-DIguZc8Z.d.ts} +3 -3
- package/dist/{multi-layer-verifier-DnAqwl0h.d.ts.map → multi-layer-verifier-DIguZc8Z.d.ts.map} +1 -1
- package/dist/multiplicity-DIWHvysC.d.ts +43 -0
- package/dist/multiplicity-DIWHvysC.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +3 -3
- package/dist/multishot/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{opencode-sqlite-8r6WUfHc.js → opencode-sqlite-DJWAXLms.js} +2 -2
- package/dist/{opencode-sqlite-8r6WUfHc.js.map → opencode-sqlite-DJWAXLms.js.map} +1 -1
- package/dist/package-version-D7lQHt_-.js +34 -0
- package/dist/package-version-D7lQHt_-.js.map +1 -0
- package/dist/paired-arms-D-XRF_fy.js +1045 -0
- package/dist/paired-arms-D-XRF_fy.js.map +1 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts +251 -0
- package/dist/paired-promotion-decision-CGzg0cI_.d.ts.map +1 -0
- package/dist/paired-tests-BHIhYVdu.js +213 -0
- package/dist/paired-tests-BHIhYVdu.js.map +1 -0
- package/dist/pareto-BqNW3LJR.d.ts +117 -0
- package/dist/pareto-BqNW3LJR.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +3 -64
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -284
- package/dist/pipelines/index.js.map +1 -1
- package/dist/power-and-mde-CHIrXJll.js +195 -0
- package/dist/power-and-mde-CHIrXJll.js.map +1 -0
- package/dist/{promotion-policy-CrLrmys8.js → power-preflight-DEw-uC7q.js} +4 -184
- package/dist/power-preflight-DEw-uC7q.js.map +1 -0
- package/dist/pre-registration-CZwSFQS4.d.ts +577 -0
- package/dist/pre-registration-CZwSFQS4.d.ts.map +1 -0
- package/dist/{prime-protocol-BfSalTfR.js → prime-protocol-6tZTVsWm.js} +72 -21
- package/dist/prime-protocol-6tZTVsWm.js.map +1 -0
- package/dist/produced-state-DU79a81m.js +586 -0
- package/dist/produced-state-DU79a81m.js.map +1 -0
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/promotion-policy-D0nPhkSy.d.ts +134 -0
- package/dist/promotion-policy-D0nPhkSy.d.ts.map +1 -0
- package/dist/promotion-policy-xzA40Evo.js +186 -0
- package/dist/promotion-policy-xzA40Evo.js.map +1 -0
- package/dist/{skillopt-optimization-method-USDKhxSA.d.ts → provenance-CCdxgLDT.d.ts} +57 -478
- package/dist/provenance-CCdxgLDT.d.ts.map +1 -0
- package/dist/registry-oJeeI4-a.d.ts +178 -0
- package/dist/registry-oJeeI4-a.d.ts.map +1 -0
- package/dist/{release-report-DA2BCu5a.d.ts → release-confidence-BFRE5WSp.d.ts} +180 -112
- package/dist/release-confidence-BFRE5WSp.d.ts.map +1 -0
- package/dist/{release-report-BUYmoKo2.js → release-confidence-CxDuiAev.js} +125 -217
- package/dist/release-confidence-CxDuiAev.js.map +1 -0
- package/dist/reporting.d.ts +6 -5
- package/dist/reporting.js +7 -5
- package/dist/{researcher-BchpD55R.d.ts → researcher-DaN4GST-.d.ts} +7 -40
- package/dist/{researcher-BchpD55R.d.ts.map → researcher-DaN4GST-.d.ts.map} +1 -1
- package/dist/{reward-hacking-BDToousL.js → reward-hacking-DNgjilrV.js} +3 -3
- package/dist/reward-hacking-DNgjilrV.js.map +1 -0
- package/dist/{reward-hacking-BI0OMAlo.d.ts → reward-hacking-DSSTuI9r.d.ts} +5 -5
- package/dist/{reward-hacking-BI0OMAlo.d.ts.map → reward-hacking-DSSTuI9r.d.ts.map} +1 -1
- package/dist/rl.d.ts +7 -7
- package/dist/rl.js +11 -10
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +4 -4
- package/dist/{rollout-DRrksrcV.js → rollout-BWtw0I_6.js} +3 -3
- package/dist/{rollout-DRrksrcV.js.map → rollout-BWtw0I_6.js.map} +1 -1
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts → rubric-predictive-validity-BgxtKe4G.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-DvkjPYCe.d.ts.map → rubric-predictive-validity-BgxtKe4G.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BRR632r1.js → rubric-predictive-validity-Cwwyd7ah.js} +2 -2
- package/dist/{rubric-predictive-validity-BRR632r1.js.map → rubric-predictive-validity-Cwwyd7ah.js.map} +1 -1
- package/dist/{run-record-BmSPWXJR.js → run-record-BvHPVS-i.js} +3 -3
- package/dist/{run-record-BmSPWXJR.js.map → run-record-BvHPVS-i.js.map} +1 -1
- package/dist/{run-record-CF4Dwpxr.d.ts → run-record-CKiihE6f.d.ts} +4 -4
- package/dist/{run-record-CF4Dwpxr.d.ts.map → run-record-CKiihE6f.d.ts.map} +1 -1
- package/dist/{proposal-findings-2GIUo1et.js → run-score-lDzV0X8j.js} +31 -31
- package/dist/run-score-lDzV0X8j.js.map +1 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts +70 -0
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -0
- package/dist/{schema-Cef2cFmb2.d.ts → schema-Cef2cFmb.d.ts} +1 -1
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-BiJxScqe.js +406 -0
- package/dist/semantic-concept-judge-BiJxScqe.js.map +1 -0
- package/dist/{sequential-D-BLJBKU.js → sequential-C458DXNf.js} +4 -3
- package/dist/{sequential-D-BLJBKU.js.map → sequential-C458DXNf.js.map} +1 -1
- package/dist/sequential-eprocess-CbUt2htw.js +83 -0
- package/dist/sequential-eprocess-CbUt2htw.js.map +1 -0
- package/dist/series-convergence-D1cL1f-4.d.ts +129 -0
- package/dist/series-convergence-D1cL1f-4.d.ts.map +1 -0
- package/dist/{server-iu0ede49.js → server-ulsOdrTI.js} +5 -21
- package/dist/server-ulsOdrTI.js.map +1 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts +427 -0
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js +1999 -0
- package/dist/skillopt-optimization-method-jjdnc3YK.js.map +1 -0
- package/dist/{statistical-heldout-TQ-4CYiN.d.ts → statistical-heldout-DoFF5KJX.d.ts} +5 -278
- package/dist/statistical-heldout-DoFF5KJX.d.ts.map +1 -0
- package/dist/{store-otlp-Dw8PPIlL.js → store-otlp-CsptLYpN.js} +3 -3
- package/dist/{store-otlp-Dw8PPIlL.js.map → store-otlp-CsptLYpN.js.map} +1 -1
- package/dist/store-tool-spans-Cq9mFd-q.js +667 -0
- package/dist/store-tool-spans-Cq9mFd-q.js.map +1 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts +342 -0
- package/dist/store-tool-spans-DbDLOBOb.d.ts.map +1 -0
- package/dist/student-t-CvBq2mve.js +38 -0
- package/dist/student-t-CvBq2mve.js.map +1 -0
- package/dist/{summary-report-Lf-5I7xh.js → summary-report-Blysd6Z2.js} +6 -3
- package/dist/{summary-report-Lf-5I7xh.js.map → summary-report-Blysd6Z2.js.map} +1 -1
- package/dist/{summary-report-BNR7DWTj.d.ts → summary-report-CaL-Hnxt.d.ts} +5 -5
- package/dist/{summary-report-BNR7DWTj.d.ts.map → summary-report-CaL-Hnxt.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +391 -3
- package/dist/supervisor-run/index.d.ts.map +1 -0
- package/dist/supervisor-run/index.js +1689 -2
- package/dist/{supervisor-run-D_sokXcO.js.map → supervisor-run/index.js.map} +1 -1
- package/dist/{extract-usage-CdZdoj1s.js → task-failure-attributes-CpQ4y5RD.js} +5 -157
- package/dist/task-failure-attributes-CpQ4y5RD.js.map +1 -0
- package/dist/{tool-groups-CZPGGlHf.d.ts → tool-groups-Cteb03Ps.d.ts} +3 -3
- package/dist/tool-groups-Cteb03Ps.d.ts.map +1 -0
- package/dist/tool-waste-BDdBZG1F.js +803 -0
- package/dist/tool-waste-BDdBZG1F.js.map +1 -0
- package/dist/tool-waste-DjRDEsuI.d.ts +128 -0
- package/dist/tool-waste-DjRDEsuI.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +14 -5
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +35 -7
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +406 -7
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +1011 -10
- package/dist/traces.js.map +1 -0
- package/dist/trajectory-replay/index.d.ts +16 -3
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +52 -5
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/types-BEPZc6eo.d.ts +93 -0
- package/dist/types-BEPZc6eo.d.ts.map +1 -0
- package/dist/{usage-receipt-t7vAzCRQ.js → types-BI4fT3HN.js} +67 -67
- package/dist/types-BI4fT3HN.js.map +1 -0
- package/dist/{types-BnjdJ70P.d.ts → types-BZ59Ahr8.d.ts} +5 -5
- package/dist/{types-BnjdJ70P.d.ts.map → types-BZ59Ahr8.d.ts.map} +1 -1
- package/dist/{types-D216SgwM.d.ts → types-Cx3YUh2r.d.ts} +4 -241
- package/dist/types-Cx3YUh2r.d.ts.map +1 -0
- package/dist/{types-BvDKaULh.d.ts → types-DLQx4mKU.d.ts} +4 -4
- package/dist/{types-BvDKaULh.d.ts.map → types-DLQx4mKU.d.ts.map} +1 -1
- package/dist/{types-D4mog56g.d.ts → types-yLK8gXE9.d.ts} +2 -2
- package/dist/{types-D4mog56g.d.ts.map → types-yLK8gXE9.d.ts.map} +1 -1
- package/dist/verdict-BndeTAh_.js +61 -0
- package/dist/verdict-BndeTAh_.js.map +1 -0
- package/dist/verdict-E4eRNf7-.d.ts +392 -0
- package/dist/verdict-E4eRNf7-.d.ts.map +1 -0
- package/dist/{verdict-cache-BCcOh0kF.js → verdict-cache-mZf5FEiY.js} +3 -55
- package/dist/{verdict-cache-BCcOh0kF.js.map → verdict-cache-mZf5FEiY.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/charter.md +3 -3
- package/docs/control-runtime.md +3 -42
- package/docs/experiment.md +0 -1
- package/docs/feature-guide.md +2 -2
- package/docs/trace-repair-grader.md +1 -0
- package/docs/trajectory-replay.md +1 -0
- package/docs/verdicts.md +43 -0
- package/docs/verification-strategies.md +3 -2
- package/package.json +6 -11
- package/dist/agent-profile-DPi7IZg7.d.ts.map +0 -1
- package/dist/analyze-runs-C30yljDJ.js.map +0 -1
- package/dist/baseline-CavEbRyH.d.ts +0 -136
- package/dist/baseline-CavEbRyH.d.ts.map +0 -1
- package/dist/benchmark-command-BteMFN62.js.map +0 -1
- package/dist/benchmarks-Dzs8CKb1.js +0 -755
- package/dist/benchmarks-Dzs8CKb1.js.map +0 -1
- package/dist/campaign-C2TTzQII.js.map +0 -1
- package/dist/completion-verifier-DJA5BhPb.d.ts +0 -414
- package/dist/completion-verifier-DJA5BhPb.d.ts.map +0 -1
- package/dist/control.d.ts +0 -3
- package/dist/control.js +0 -2
- package/dist/dataset-C8xaLXdY.d.ts.map +0 -1
- package/dist/default-registry-Bf8Woqmq.d.ts.map +0 -1
- package/dist/default-registry-BmktKy8r.js.map +0 -1
- package/dist/engine-3hL-XqwJ.d.ts.map +0 -1
- package/dist/experiment-tracker-CnRICnMl.js +0 -500
- package/dist/experiment-tracker-CnRICnMl.js.map +0 -1
- package/dist/experiment-tracker-IMntXr6J.d.ts.map +0 -1
- package/dist/extract-usage-CdZdoj1s.js.map +0 -1
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +0 -1
- package/dist/feedback-trajectory-CacHpxsp.d.ts.map +0 -1
- package/dist/index-BZ3-y4YL.d.ts +0 -391
- package/dist/index-BZ3-y4YL.d.ts.map +0 -1
- package/dist/index-CQTZ-4XN.d.ts.map +0 -1
- package/dist/index-DPPGNJ_R.d.ts.map +0 -1
- package/dist/index-YE4KdKbO2.d.ts +0 -335
- package/dist/index-YE4KdKbO2.d.ts.map +0 -1
- package/dist/paired-arms-iZ08VFMN.js +0 -260
- package/dist/paired-arms-iZ08VFMN.js.map +0 -1
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts +0 -114
- package/dist/paired-promotion-decision-B6zJ3gYM.d.ts.map +0 -1
- package/dist/prime-protocol-BfSalTfR.js.map +0 -1
- package/dist/promotion-policy-Ckjhzg_4.d.ts +0 -289
- package/dist/promotion-policy-Ckjhzg_4.d.ts.map +0 -1
- package/dist/promotion-policy-CrLrmys8.js.map +0 -1
- package/dist/proposal-findings-2GIUo1et.js.map +0 -1
- package/dist/propose-review-control-dSNPjFUH.js +0 -1458
- package/dist/propose-review-control-dSNPjFUH.js.map +0 -1
- package/dist/release-report-BUYmoKo2.js.map +0 -1
- package/dist/release-report-DA2BCu5a.d.ts.map +0 -1
- package/dist/replay-CohS93nE.js +0 -1859
- package/dist/replay-CohS93nE.js.map +0 -1
- package/dist/replay-DbhZ4Ked.d.ts +0 -834
- package/dist/replay-DbhZ4Ked.d.ts.map +0 -1
- package/dist/reward-hacking-BDToousL.js.map +0 -1
- package/dist/run-evidence-Dhi3C81V.d.ts +0 -225
- package/dist/run-evidence-Dhi3C81V.d.ts.map +0 -1
- package/dist/schema-Cef2cFmb2.d.ts.map +0 -1
- package/dist/semantic-concept-judge-D1z-KepS.js +0 -767
- package/dist/semantic-concept-judge-D1z-KepS.js.map +0 -1
- package/dist/series-convergence-ofsqPWhs.d.ts +0 -35
- package/dist/series-convergence-ofsqPWhs.d.ts.map +0 -1
- package/dist/server-iu0ede49.js.map +0 -1
- package/dist/single-run-lock-DFWHEB09.js.map +0 -1
- package/dist/skill-usage-CO9OLRBx.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CQdVeM8k.js.map +0 -1
- package/dist/skillopt-optimization-method-USDKhxSA.d.ts.map +0 -1
- package/dist/statistical-heldout-TQ-4CYiN.d.ts.map +0 -1
- package/dist/statistics-ByxzSiOM.js +0 -2212
- package/dist/statistics-ByxzSiOM.js.map +0 -1
- package/dist/statistics-D6Uebe_4.d.ts +0 -968
- package/dist/statistics-D6Uebe_4.d.ts.map +0 -1
- package/dist/supervisor-run-D_sokXcO.js +0 -1690
- package/dist/test-graded-scenario-D1TaI2va.d.ts +0 -141
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +0 -1
- package/dist/test-graded-scenario-JHcKQNpq.js +0 -318
- package/dist/test-graded-scenario-JHcKQNpq.js.map +0 -1
- package/dist/tool-groups-CZPGGlHf.d.ts.map +0 -1
- package/dist/tool-use-metrics-DEGMKycK.js +0 -370
- package/dist/tool-use-metrics-DEGMKycK.js.map +0 -1
- package/dist/types-D216SgwM.d.ts.map +0 -1
- package/dist/usage-receipt-t7vAzCRQ.js.map +0 -1
- package/dist/verdict-DExhxfgR.d.ts +0 -201
- package/dist/verdict-DExhxfgR.d.ts.map +0 -1
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt, o as CostLedger, p as CostProvenance } from "./cost-ledger-
|
|
2
|
-
import { a as RunRecord } from "./run-record-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { _ as
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import { t as DetectRewardHackingInput } from "./reward-hacking-BI0OMAlo.js";
|
|
10
|
-
import { g as TraceSpanEvent, t as HostedClient } from "./client-C9gzZE59.js";
|
|
1
|
+
import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt, o as CostLedger, p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
|
|
2
|
+
import { a as RunRecord } from "./run-record-CKiihE6f.js";
|
|
3
|
+
import { p as ChatClient } from "./types-Cx3YUh2r.js";
|
|
4
|
+
import { _ as ProposalFinding } from "./types-DLQx4mKU.js";
|
|
5
|
+
import { g as ExternalTextCandidate, o as ExternalOptimizerEvaluationObservation } from "./external-optimizer-contracts-lixrOZdX.js";
|
|
6
|
+
import { C as JudgeDimension, H as SurfaceProposer, N as ParetoParent, R as Scenario, S as JudgeConfig, _ as GateDecision, a as CampaignResult, b as GenerationRecord, c as CampaignTraceWriter, d as DispatchContext, f as DispatchFn, g as GateContribution, j as MutableSurface, k as LabeledScenarioStore, p as Gate, r as CampaignCellResult, v as GateResult, y as GenerationCandidate } from "./types-BZ59Ahr8.js";
|
|
7
|
+
import { r as DatasetScenario, t as Dataset } from "./dataset-CJjKqQfA.js";
|
|
8
|
+
import { g as TraceSpanEvent, t as HostedClient } from "./client-KUbGulm_.js";
|
|
11
9
|
import { z } from "zod";
|
|
12
10
|
//#region src/llm-judge.d.ts
|
|
13
11
|
/** A rubric dimension as a bare key or the full `{ key, description }` shape. A
|
|
@@ -62,43 +60,6 @@ interface LlmJudgeOptions<TArtifact, TScenario extends Scenario = Scenario> {
|
|
|
62
60
|
*/
|
|
63
61
|
declare function llmJudge<TArtifact = unknown, TScenario extends Scenario = Scenario>(name: string, prompt: string, opts: LlmJudgeOptions<TArtifact, TScenario>): JudgeConfig<TArtifact, TScenario>;
|
|
64
62
|
//#endregion
|
|
65
|
-
//#region src/reference-equivalence-judge.d.ts
|
|
66
|
-
declare const REFERENCE_EQUIVALENCE_JUDGE_VERSION = "reference-equivalence-judge-v1-2026-07-13";
|
|
67
|
-
declare const REFERENCE_EQUIVALENCE_INPUT_LIMITS: {
|
|
68
|
-
readonly userRequest: 8000;
|
|
69
|
-
readonly expectedAnswer: 32000;
|
|
70
|
-
readonly candidateOutput: 32000;
|
|
71
|
-
};
|
|
72
|
-
interface ReferenceEquivalenceScenario extends Scenario {
|
|
73
|
-
userRequest: string;
|
|
74
|
-
expectedAnswer: string;
|
|
75
|
-
}
|
|
76
|
-
interface ReferenceEquivalenceJudgeInput {
|
|
77
|
-
userRequest: string;
|
|
78
|
-
expectedAnswer: string;
|
|
79
|
-
candidateOutput: string;
|
|
80
|
-
}
|
|
81
|
-
interface ReferenceEquivalenceJudgeOptions {
|
|
82
|
-
/** Injected transport. No implicit provider or credentials are selected. */
|
|
83
|
-
chat: ChatClient;
|
|
84
|
-
/** Falls back to the ChatClient's default model. */
|
|
85
|
-
model?: string;
|
|
86
|
-
/** Used only by the direct-call adapter. */
|
|
87
|
-
signal?: AbortSignal;
|
|
88
|
-
/** Optional receipt destination for direct calls; campaigns supply their own. */
|
|
89
|
-
costLedger?: CostLedgerHandle;
|
|
90
|
-
}
|
|
91
|
-
interface ReferenceEquivalenceJudgeResult extends LlmCallMetadata {
|
|
92
|
-
kind: 'reference-equivalence';
|
|
93
|
-
version: string;
|
|
94
|
-
score: number;
|
|
95
|
-
rationale: string;
|
|
96
|
-
}
|
|
97
|
-
/** Build the campaign-native expected-answer judge. */
|
|
98
|
-
declare function createReferenceEquivalenceJudge(options: ReferenceEquivalenceJudgeOptions): JudgeConfig<string, ReferenceEquivalenceScenario>;
|
|
99
|
-
/** Direct-call adapter over the campaign judge for product callers. */
|
|
100
|
-
declare function runReferenceEquivalenceJudge(input: ReferenceEquivalenceJudgeInput, options: ReferenceEquivalenceJudgeOptions): Promise<ReferenceEquivalenceJudgeResult>;
|
|
101
|
-
//#endregion
|
|
102
63
|
//#region src/campaign/auto-pr.d.ts
|
|
103
64
|
interface OpenAutoPrOptions<TArtifact, TScenario extends Scenario> {
|
|
104
65
|
/** Campaign result to attach to the PR. */
|
|
@@ -137,18 +98,6 @@ interface OpenAutoPrResult {
|
|
|
137
98
|
*/
|
|
138
99
|
declare function openAutoPr<TArtifact, TScenario extends Scenario>(options: OpenAutoPrOptions<TArtifact, TScenario>): OpenAutoPrResult;
|
|
139
100
|
//#endregion
|
|
140
|
-
//#region src/campaign/coverage.d.ts
|
|
141
|
-
/** Reject campaign designs whose denominator cannot be identified exactly. */
|
|
142
|
-
declare function assertCampaignDesign<TScenario extends Scenario>(scenarios: readonly TScenario[], reps: number): void;
|
|
143
|
-
/** Redacted but independently verifiable identity of one complete scenario. */
|
|
144
|
-
declare function campaignScenarioIdentity<TScenario extends Scenario>(scenario: TScenario): CampaignScenarioIdentity & Pick<TScenario, 'id' | 'kind'>;
|
|
145
|
-
/** Canonical split identity reconstructed from redacted scenario identities. */
|
|
146
|
-
declare function campaignSplitDigestFromIdentities(scenarios: readonly CampaignScenarioIdentity[], reps: number): `sha256:${string}`;
|
|
147
|
-
/** Canonical identity of the exact scenario payloads and replicate count. */
|
|
148
|
-
declare function campaignSplitDigest<TScenario extends Scenario>(scenarios: readonly TScenario[], reps: number): `sha256:${string}`;
|
|
149
|
-
/** Refuse a campaign whose retained task identities contradict its split digest. */
|
|
150
|
-
declare function assertCampaignSplitIdentity(scenarios: readonly CampaignScenarioIdentity[], reps: number, splitDigest: string): void;
|
|
151
|
-
//#endregion
|
|
152
101
|
//#region src/campaign/storage.d.ts
|
|
153
102
|
/**
|
|
154
103
|
* `CampaignStorage` — the filesystem seam `runCampaign` writes through
|
|
@@ -298,6 +247,53 @@ declare function readGepaCandidatePopulationArtifact(input: {
|
|
|
298
247
|
storage?: CampaignStorage;
|
|
299
248
|
}): GepaCandidatePopulationArtifact;
|
|
300
249
|
//#endregion
|
|
250
|
+
//#region src/campaign/cell-cache.d.ts
|
|
251
|
+
type CacheIssueReason = 'missing' | 'manifest-mismatch' | 'cell-mismatch' | 'missing-cost-provenance' | 'invalid-cost-provenance' | 'invalid-cost-receipts' | 'corrupt';
|
|
252
|
+
//#endregion
|
|
253
|
+
//#region src/campaign/plan-campaign-run.d.ts
|
|
254
|
+
interface CampaignRunPlanCell {
|
|
255
|
+
cellId: string;
|
|
256
|
+
scenarioId: string;
|
|
257
|
+
rep: number;
|
|
258
|
+
seed: number;
|
|
259
|
+
cachePath: string;
|
|
260
|
+
status: 'cached' | 'run' | 'blocked';
|
|
261
|
+
reason?: CacheIssueReason | 'resumable-off';
|
|
262
|
+
}
|
|
263
|
+
interface CampaignRunPlan {
|
|
264
|
+
manifestHash: string;
|
|
265
|
+
splitDigest: `sha256:${string}`;
|
|
266
|
+
totalCells: number;
|
|
267
|
+
cellsCached: number;
|
|
268
|
+
cellsBlocked: number;
|
|
269
|
+
cellsToRun: number;
|
|
270
|
+
cells: CampaignRunPlanCell[];
|
|
271
|
+
}
|
|
272
|
+
interface PlanCampaignRunOptions<TScenario extends Scenario, TArtifact> {
|
|
273
|
+
scenarios: TScenario[];
|
|
274
|
+
dispatch?: DispatchFn<TScenario, TArtifact>;
|
|
275
|
+
dispatchRef?: string;
|
|
276
|
+
judges?: JudgeConfig<TArtifact, TScenario>[];
|
|
277
|
+
seed?: number;
|
|
278
|
+
reps?: number;
|
|
279
|
+
resumable?: boolean;
|
|
280
|
+
/** See RunCampaignOptions.rerunInvalidCachedCells. */
|
|
281
|
+
rerunInvalidCachedCells?: boolean;
|
|
282
|
+
runDir: string;
|
|
283
|
+
/** Subject repo for the shared run-dir root (see RunCampaignOptions.repo). */
|
|
284
|
+
repo?: string;
|
|
285
|
+
storage?: CampaignStorage;
|
|
286
|
+
/** Spend account used to validate cached receipt identities. */
|
|
287
|
+
costLedger?: CostLedgerHandle;
|
|
288
|
+
/** Receipt tags used by the campaign that produced the cached cells. */
|
|
289
|
+
costTags?: Readonly<Record<string, string>>;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Plan a campaign WITHOUT dispatching: computes the manifest hash and the per-cell
|
|
293
|
+
* run-vs-cached schedule so callers can preview cost and resumability before spending.
|
|
294
|
+
*/
|
|
295
|
+
declare function planCampaignRun<TScenario extends Scenario, TArtifact>(opts: PlanCampaignRunOptions<TScenario, TArtifact>): CampaignRunPlan;
|
|
296
|
+
//#endregion
|
|
301
297
|
//#region src/campaign/run-campaign.d.ts
|
|
302
298
|
interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {
|
|
303
299
|
scenarios: TScenario[];
|
|
@@ -438,49 +434,6 @@ interface CampaignCellFailureReceipt<TArtifact = unknown> {
|
|
|
438
434
|
* Core campaign orchestrator: fan scenarios through dispatch, score with judges, aggregate bootstrap CIs, and persist reproducible `CampaignResult` records.
|
|
439
435
|
*/
|
|
440
436
|
declare function runCampaign<TScenario extends Scenario, TArtifact>(opts: RunCampaignOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
|
|
441
|
-
interface CampaignRunPlanCell {
|
|
442
|
-
cellId: string;
|
|
443
|
-
scenarioId: string;
|
|
444
|
-
rep: number;
|
|
445
|
-
seed: number;
|
|
446
|
-
cachePath: string;
|
|
447
|
-
status: 'cached' | 'run' | 'blocked';
|
|
448
|
-
reason?: CacheIssueReason | 'resumable-off';
|
|
449
|
-
}
|
|
450
|
-
interface CampaignRunPlan {
|
|
451
|
-
manifestHash: string;
|
|
452
|
-
splitDigest: `sha256:${string}`;
|
|
453
|
-
totalCells: number;
|
|
454
|
-
cellsCached: number;
|
|
455
|
-
cellsBlocked: number;
|
|
456
|
-
cellsToRun: number;
|
|
457
|
-
cells: CampaignRunPlanCell[];
|
|
458
|
-
}
|
|
459
|
-
interface PlanCampaignRunOptions<TScenario extends Scenario, TArtifact> {
|
|
460
|
-
scenarios: TScenario[];
|
|
461
|
-
dispatch?: DispatchFn<TScenario, TArtifact>;
|
|
462
|
-
dispatchRef?: string;
|
|
463
|
-
judges?: JudgeConfig<TArtifact, TScenario>[];
|
|
464
|
-
seed?: number;
|
|
465
|
-
reps?: number;
|
|
466
|
-
resumable?: boolean;
|
|
467
|
-
/** See RunCampaignOptions.rerunInvalidCachedCells. */
|
|
468
|
-
rerunInvalidCachedCells?: boolean;
|
|
469
|
-
runDir: string;
|
|
470
|
-
/** Subject repo for the shared run-dir root (see RunCampaignOptions.repo). */
|
|
471
|
-
repo?: string;
|
|
472
|
-
storage?: CampaignStorage;
|
|
473
|
-
/** Spend account used to validate cached receipt identities. */
|
|
474
|
-
costLedger?: CostLedgerHandle;
|
|
475
|
-
/** Receipt tags used by the campaign that produced the cached cells. */
|
|
476
|
-
costTags?: Readonly<Record<string, string>>;
|
|
477
|
-
}
|
|
478
|
-
/**
|
|
479
|
-
* Plan a campaign WITHOUT dispatching: computes the manifest hash and the per-cell
|
|
480
|
-
* run-vs-cached schedule so callers can preview cost and resumability before spending.
|
|
481
|
-
*/
|
|
482
|
-
declare function planCampaignRun<TScenario extends Scenario, TArtifact>(opts: PlanCampaignRunOptions<TScenario, TArtifact>): CampaignRunPlan;
|
|
483
|
-
type CacheIssueReason = 'missing' | 'manifest-mismatch' | 'cell-mismatch' | 'missing-cost-provenance' | 'invalid-cost-provenance' | 'invalid-cost-receipts' | 'corrupt';
|
|
484
437
|
//#endregion
|
|
485
438
|
//#region src/campaign/presets/compare-optimization-methods.d.ts
|
|
486
439
|
/** Shared campaign settings applied to every optimization method. */
|
|
@@ -696,96 +649,6 @@ declare function combineComparisonCosts(entries: ReadonlyArray<{
|
|
|
696
649
|
cost: ComparisonCost;
|
|
697
650
|
}>): ComparisonCost;
|
|
698
651
|
//#endregion
|
|
699
|
-
//#region src/campaign/external-text-evaluation.d.ts
|
|
700
|
-
interface ExternalOptimizationExample {
|
|
701
|
-
id: string;
|
|
702
|
-
data: unknown;
|
|
703
|
-
}
|
|
704
|
-
interface ExternalTextEvaluationResponse {
|
|
705
|
-
score: number;
|
|
706
|
-
info: {
|
|
707
|
-
scenarioId: string;
|
|
708
|
-
dimensions: Record<string, number>;
|
|
709
|
-
notes?: string;
|
|
710
|
-
artifact?: unknown;
|
|
711
|
-
};
|
|
712
|
-
}
|
|
713
|
-
declare function decodeExternalTextCandidate(candidate: ExternalTextCandidate): MutableSurface;
|
|
714
|
-
//#endregion
|
|
715
|
-
//#region src/campaign/external-text-optimization-contract.d.ts
|
|
716
|
-
interface ExternalTextOptimizerContext {
|
|
717
|
-
readonly runId: string;
|
|
718
|
-
readonly name: string;
|
|
719
|
-
readonly objective: string;
|
|
720
|
-
readonly evaluationId: string;
|
|
721
|
-
readonly background?: string;
|
|
722
|
-
readonly seedCandidate: ExternalTextCandidate;
|
|
723
|
-
readonly trainSet: readonly ExternalOptimizationExample[];
|
|
724
|
-
readonly selectionSet: readonly ExternalOptimizationExample[];
|
|
725
|
-
readonly maxEvaluations: number;
|
|
726
|
-
readonly seed: number;
|
|
727
|
-
/** Stable directory for optimizer checkpoints from compatible attempts. */
|
|
728
|
-
readonly stateDir: string;
|
|
729
|
-
readonly restoreRequested: boolean;
|
|
730
|
-
readonly artifactDir: string;
|
|
731
|
-
readonly signal: AbortSignal;
|
|
732
|
-
/** Record every optimizer-owned paid call through this attributed account. */
|
|
733
|
-
readonly cost: CampaignCostMeter;
|
|
734
|
-
readonly evaluate: (request: ExternalTextEvaluationRequest) => Promise<ExternalTextEvaluationResponse>;
|
|
735
|
-
}
|
|
736
|
-
interface ExternalTextOptimizerResult {
|
|
737
|
-
bestCandidate: ExternalTextCandidate;
|
|
738
|
-
resumed: boolean;
|
|
739
|
-
costAccounting: {
|
|
740
|
-
kind: 'metered';
|
|
741
|
-
} | {
|
|
742
|
-
kind: 'no-paid-work';
|
|
743
|
-
} | {
|
|
744
|
-
kind: 'external';
|
|
745
|
-
reason: string;
|
|
746
|
-
};
|
|
747
|
-
}
|
|
748
|
-
/**
|
|
749
|
-
* Configuration for adapting another text optimizer.
|
|
750
|
-
*
|
|
751
|
-
* `run` owns search. Agent Eval owns split isolation, bounded candidate
|
|
752
|
-
* evaluation, exact cost collection, provenance, and final comparison.
|
|
753
|
-
*/
|
|
754
|
-
interface ExternalTextOptimizationMethodConfig<TScenario extends Scenario, TArtifact = unknown> {
|
|
755
|
-
name: string;
|
|
756
|
-
source: Omit<OptimizationPackageSource, 'evidence'>;
|
|
757
|
-
objective: string;
|
|
758
|
-
evaluationId: string;
|
|
759
|
-
background?: string;
|
|
760
|
-
maxEvaluations: number;
|
|
761
|
-
/** Hard limit for calls made through `context.cost`. Use 0 for no paid work. */
|
|
762
|
-
maxOptimizerCostUsd: number;
|
|
763
|
-
/** Abort `context.signal` after this duration. Default: 30 minutes. */
|
|
764
|
-
timeoutMs?: number;
|
|
765
|
-
/** Default: `never`. Compatible runs reuse one state directory. */
|
|
766
|
-
resume?: ExternalOptimizerResumeMode;
|
|
767
|
-
maxCandidateChars?: number;
|
|
768
|
-
maxEvidenceChars?: number;
|
|
769
|
-
describeScenario?: (scenario: TScenario) => unknown;
|
|
770
|
-
describeArtifact?: (artifact: TArtifact, scenario: TScenario) => unknown;
|
|
771
|
-
run: (context: ExternalTextOptimizerContext) => Promise<ExternalTextOptimizerResult>;
|
|
772
|
-
}
|
|
773
|
-
//#endregion
|
|
774
|
-
//#region src/campaign/external-text-optimization.d.ts
|
|
775
|
-
/**
|
|
776
|
-
* Adapt a third-party text optimizer without reimplementing its search.
|
|
777
|
-
*
|
|
778
|
-
* The callback never receives final test cases. Calls to `evaluate` are
|
|
779
|
-
* counted before execution and stop at `maxEvaluations`.
|
|
780
|
-
*/
|
|
781
|
-
declare function externalTextOptimizationMethod<TScenario extends Scenario, TArtifact>(config: ExternalTextOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
782
|
-
//#endregion
|
|
783
|
-
//#region src/campaign/gates/compose.d.ts
|
|
784
|
-
/** Compose gates — all must `ship` for the composite to `ship`. First
|
|
785
|
-
* non-ship verdict short-circuits the composite verdict, but ALL gates run
|
|
786
|
-
* (so the result records every gate's reason — useful for diagnostics). */
|
|
787
|
-
declare function composeGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(...gates: Array<Gate<TArtifact, TScenario>>): Gate<TArtifact, TScenario>;
|
|
788
|
-
//#endregion
|
|
789
652
|
//#region src/canary.d.ts
|
|
790
653
|
type CanaryKind = 'silent_judge_fallback' | 'judge_calibration_drift' | 'distribution_shift';
|
|
791
654
|
type CanarySeverity = 'info' | 'warn' | 'error';
|
|
@@ -907,234 +770,6 @@ declare function redTeamDataset(extraCases?: RedTeamCase[]): Dataset;
|
|
|
907
770
|
declare function scoreRedTeamOutput(output: string, toolCalls: string[], rtCase: RedTeamCase): RedTeamFinding;
|
|
908
771
|
/** Aggregate red-team findings into per-category pass rates. */
|
|
909
772
|
declare function redTeamReport(findings: RedTeamFinding[]): RedTeamReport;
|
|
910
|
-
/**
|
|
911
|
-
* Extract the tool-call names from a corpus run — convenience for the
|
|
912
|
-
* common pipeline (run the scenario → score the run).
|
|
913
|
-
*/
|
|
914
|
-
declare function toolNamesForRun(store: TraceStore, runId: string): Promise<string[]>;
|
|
915
|
-
//#endregion
|
|
916
|
-
//#region src/campaign/gates/default-production-gate.d.ts
|
|
917
|
-
type DefaultProductionGateCheck = 'dimension-regression' | 'budget' | 'red-team' | 'reward-hacking' | 'canary';
|
|
918
|
-
type DefaultProductionRewardHackingOptions = Omit<DetectRewardHackingInput, 'runs' | 'truthOf'> & {
|
|
919
|
-
truthOf: NonNullable<DetectRewardHackingInput['truthOf']>;
|
|
920
|
-
};
|
|
921
|
-
interface DefaultProductionGateOptions {
|
|
922
|
-
/** Required: scenarios held out from training; substrate compares
|
|
923
|
-
* candidate-on-holdout vs baseline-on-holdout. */
|
|
924
|
-
holdoutScenarios: Scenario[];
|
|
925
|
-
/** Minimum held-out lift the **paired-bootstrap CI lower bound** must clear
|
|
926
|
-
* to ship — NOT a point estimate. Default 0 ⇒ "confidently positive at the
|
|
927
|
-
* confidence level". Interpreted in the judge's native composite scale (set
|
|
928
|
-
* e.g. 2 for a 0-100 rubric to require a ≥2-point significant gain). */
|
|
929
|
-
deltaThreshold?: number;
|
|
930
|
-
/** Confidence level for the held-out + dimension bootstraps. Default 0.95. */
|
|
931
|
-
confidence?: number;
|
|
932
|
-
/** Bootstrap resamples. Default 2000. */
|
|
933
|
-
bootstrapResamples?: number;
|
|
934
|
-
/** Fixed bootstrap seed for a deterministic verdict. Default 1337. */
|
|
935
|
-
bootstrapSeed?: number;
|
|
936
|
-
/** Minimum paired holdout observations (scenarios × reps) before a
|
|
937
|
-
* significance claim is allowed. The exact small-sample test may require
|
|
938
|
-
* more observations at the selected confidence. Default 3. */
|
|
939
|
-
minProductiveRuns?: number;
|
|
940
|
-
/** Ship statistic for the held-out significance test. Default `'mean'`
|
|
941
|
-
* (tie-robust — see `heldoutSignificance`). Pass `'median'` for
|
|
942
|
-
* outlier-robustness at the cost of tie-blindness. */
|
|
943
|
-
heldoutStatistic?: 'mean' | 'median';
|
|
944
|
-
/** Critical judge dimensions that must NOT significantly regress even when
|
|
945
|
-
* the net composite rises (anti-Goodhart). The gate HOLDS if any listed
|
|
946
|
-
* dimension's paired-delta CI lower bound < −`regressionTolerance`. E.g.
|
|
947
|
-
* `['hallucination_free']` for a legal agent. */
|
|
948
|
-
criticalDimensions?: string[];
|
|
949
|
-
/** Tolerance for the per-dimension regression guard, in the dimension's
|
|
950
|
-
* native scale. When omitted it auto-scales off observed magnitudes:
|
|
951
|
-
* 0.05 on [0,1], 5 on 0-100. */
|
|
952
|
-
regressionTolerance?: number;
|
|
953
|
-
/** Total $ budget for the complete improvement run. Requires
|
|
954
|
-
* `GateContext.costLedger`; missing or incomplete accounting holds. */
|
|
955
|
-
budgetUsd?: number;
|
|
956
|
-
/** Static artifact-screening cases. Only `expected: 'ignore'` cases without
|
|
957
|
-
* tool assertions are valid because this check does not dispatch case inputs
|
|
958
|
-
* or observe tool calls. */
|
|
959
|
-
redTeamBattery?: RedTeamCase[];
|
|
960
|
-
/** Shared run history, oldest first. Supplying history does not enable either
|
|
961
|
-
* monitoring check; configure `rewardHacking` and/or `canary` explicitly. */
|
|
962
|
-
recentRuns?: RunRecord[];
|
|
963
|
-
/** Enable reward-hacking monitoring with a caller-owned independent truth channel. */
|
|
964
|
-
rewardHacking?: DefaultProductionRewardHackingOptions;
|
|
965
|
-
/** Enable canary monitoring. Pass `{}` to use the canary defaults. */
|
|
966
|
-
canary?: CanaryOptions;
|
|
967
|
-
/** Optional checks that must be evaluated even when their normal input is
|
|
968
|
-
* absent. Configuring a check's input also makes that check required.
|
|
969
|
-
* Missing evidence always records `not_evaluated`; required unevaluated
|
|
970
|
-
* checks hold the release decision. Held-out significance is always required. */
|
|
971
|
-
requiredChecks?: DefaultProductionGateCheck[];
|
|
972
|
-
}
|
|
973
|
-
/**
|
|
974
|
-
* Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision.
|
|
975
|
-
*/
|
|
976
|
-
declare function defaultProductionGate<TArtifact, TScenario extends Scenario>(options: DefaultProductionGateOptions): Gate<TArtifact, TScenario>;
|
|
977
|
-
//#endregion
|
|
978
|
-
//#region src/campaign/gates/heldout-gate.d.ts
|
|
979
|
-
interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
|
|
980
|
-
scenarios: TScenario[];
|
|
981
|
-
/** Effect-size threshold the CI lower bound must clear, in the judge's native
|
|
982
|
-
* scale. Default 0.5. Equality holds; CI.low must be greater than this value. */
|
|
983
|
-
deltaThreshold?: number;
|
|
984
|
-
/** Bootstrap CI confidence. Default 0.95. */
|
|
985
|
-
confidence?: number;
|
|
986
|
-
/** Minimum paired holdout observations to claim significance. The exact
|
|
987
|
-
* small-sample test may require more observations at the selected
|
|
988
|
-
* confidence. Default 3. */
|
|
989
|
-
minProductiveRuns?: number;
|
|
990
|
-
/** Bootstrap resamples. Default 2000. */
|
|
991
|
-
resamples?: number;
|
|
992
|
-
/** Fixed bootstrap seed for deterministic verdicts. Default 1337. */
|
|
993
|
-
bootstrapSeed?: number;
|
|
994
|
-
}
|
|
995
|
-
/**
|
|
996
|
-
* Composable held-out gate: ships only when the lower bound of the DECIDING
|
|
997
|
-
* paired interval on the candidate-minus-baseline composite delta clears
|
|
998
|
-
* `deltaThreshold` — Tango's score interval on a pass/fail holdout, the mean
|
|
999
|
-
* bootstrap otherwise. See {@link decidePairedPromotion}.
|
|
1000
|
-
*/
|
|
1001
|
-
declare function heldOutGate<TArtifact, TScenario extends Scenario>(options: HeldOutGateOptions<TScenario>): Gate<TArtifact, TScenario>;
|
|
1002
|
-
//#endregion
|
|
1003
|
-
//#region src/campaign/optimizer-model.d.ts
|
|
1004
|
-
type OptimizerModelBudget = ExternalOptimizerModelBudget;
|
|
1005
|
-
/** One metered model path supplied by the package that owns execution. */
|
|
1006
|
-
interface OpenAICompatibleOptimizerModel {
|
|
1007
|
-
model: string;
|
|
1008
|
-
budget: OptimizerModelBudget;
|
|
1009
|
-
/** Caller-owned execution path, such as Runtime's exact AgentProfile adapter. */
|
|
1010
|
-
call: ExternalOptimizerModelCall;
|
|
1011
|
-
/** Stable public identity included in resumable-run compatibility. */
|
|
1012
|
-
callRef: string;
|
|
1013
|
-
}
|
|
1014
|
-
//#endregion
|
|
1015
|
-
//#region src/campaign/gepa-optimization-method.d.ts
|
|
1016
|
-
/** Shared settings for one bounded GEPA engine invocation. */
|
|
1017
|
-
interface GepaEngineOptions {
|
|
1018
|
-
/** GEPA engine name. GEPA validates names available in its Python runtime. */
|
|
1019
|
-
engine: string;
|
|
1020
|
-
/** Optional billed-USD stop. Omit when the execution owner reports USD as unknown. */
|
|
1021
|
-
maxProposerCostUsd?: number;
|
|
1022
|
-
/** Maximum concurrent evaluations inside this engine. Default: 1. */
|
|
1023
|
-
maxConcurrency?: number;
|
|
1024
|
-
/** Stop the engine after it reaches this score. */
|
|
1025
|
-
stopAtScore?: number;
|
|
1026
|
-
/** Isolate agent-based engines. Default: true. */
|
|
1027
|
-
sandbox?: boolean;
|
|
1028
|
-
/**
|
|
1029
|
-
* JSON-safe configuration for the registered GEPA engine.
|
|
1030
|
-
* Python callables and class instances cannot cross the process boundary.
|
|
1031
|
-
*/
|
|
1032
|
-
engineConfig?: Record<string, unknown>;
|
|
1033
|
-
}
|
|
1034
|
-
/** One independently budgeted GEPA engine invocation. */
|
|
1035
|
-
interface GepaEngineRun extends GepaEngineOptions {
|
|
1036
|
-
/** Maximum callback evaluations this engine may consume. */
|
|
1037
|
-
maxEvaluations: number;
|
|
1038
|
-
}
|
|
1039
|
-
/** An engine in an adaptive run. All engines share the recipe evaluation limit. */
|
|
1040
|
-
type GepaAdaptiveEngineRun = GepaEngineOptions;
|
|
1041
|
-
/**
|
|
1042
|
-
* A direct mapping to a GEPA optimization recipe.
|
|
1043
|
-
*
|
|
1044
|
-
* GEPA owns every search and composition operation represented here. Tangle
|
|
1045
|
-
* supplies the candidate, data, execution callback, judges, and budgets.
|
|
1046
|
-
*/
|
|
1047
|
-
type GepaOptimizationRecipe = {
|
|
1048
|
-
kind: 'engine';
|
|
1049
|
-
run: GepaEngineRun;
|
|
1050
|
-
} | {
|
|
1051
|
-
kind: 'sequential';
|
|
1052
|
-
runs: readonly GepaEngineRun[];
|
|
1053
|
-
} | {
|
|
1054
|
-
kind: 'adaptive-sequential';
|
|
1055
|
-
runs: readonly GepaAdaptiveEngineRun[];
|
|
1056
|
-
/** One evaluation budget shared by every adaptive stage. */
|
|
1057
|
-
maxEvaluations: number;
|
|
1058
|
-
/** Switch engines after this many evaluations without improvement. */
|
|
1059
|
-
plateauEvaluations: number;
|
|
1060
|
-
patience?: number;
|
|
1061
|
-
minEvaluationsPerStage?: number;
|
|
1062
|
-
improvementEpsilon?: number;
|
|
1063
|
-
cycle?: boolean;
|
|
1064
|
-
maxSwitches?: number;
|
|
1065
|
-
maxConcurrency?: number;
|
|
1066
|
-
} | {
|
|
1067
|
-
kind: 'best-of';
|
|
1068
|
-
runs: readonly GepaEngineRun[];
|
|
1069
|
-
maxWorkers?: number;
|
|
1070
|
-
} | {
|
|
1071
|
-
kind: 'vote';
|
|
1072
|
-
runs: readonly GepaEngineRun[];
|
|
1073
|
-
maxWorkers?: number;
|
|
1074
|
-
} | {
|
|
1075
|
-
kind: 'omni';
|
|
1076
|
-
explore: readonly GepaEngineRun[];
|
|
1077
|
-
continueWith: GepaEngineRun;
|
|
1078
|
-
maxWorkers?: number;
|
|
1079
|
-
};
|
|
1080
|
-
/** The command that runs the Python GEPA bridge. */
|
|
1081
|
-
type GepaRunnerCommand = ExternalOptimizerRunnerCommand;
|
|
1082
|
-
interface GepaOptimizationMethodConfig<TScenario extends Scenario, TArtifact = unknown> {
|
|
1083
|
-
/** Unique comparison-method name. Default identifies the GEPA recipe. */
|
|
1084
|
-
name?: string;
|
|
1085
|
-
/** A direct GEPA recipe. */
|
|
1086
|
-
recipe: GepaOptimizationRecipe;
|
|
1087
|
-
/** Plain-language goal shown to the external optimizer. */
|
|
1088
|
-
objective: string;
|
|
1089
|
-
/** Stable identity for the dispatch, judges, model settings, and scoring logic. */
|
|
1090
|
-
evaluationId: string;
|
|
1091
|
-
/** Optional bounded context about the surface and task. */
|
|
1092
|
-
background?: string;
|
|
1093
|
-
/**
|
|
1094
|
-
* Public dotted Python modules imported before GEPA resolves engine names.
|
|
1095
|
-
* Each module should call GEPA's official `register_engine()` API at import.
|
|
1096
|
-
*/
|
|
1097
|
-
engineModules?: readonly string[];
|
|
1098
|
-
/** Reject external candidates longer than this. Default: 200,000 characters. */
|
|
1099
|
-
maxCandidateChars?: number;
|
|
1100
|
-
/** Reject serialized score evidence longer than this. Default: 100,000 characters. */
|
|
1101
|
-
maxEvidenceChars?: number;
|
|
1102
|
-
/** Candidate-evaluation callback byte limits. Omitted fields use finite defaults. */
|
|
1103
|
-
evaluationCallbackLimits?: Partial<ExternalOptimizerCallbackLimits>;
|
|
1104
|
-
/** End the bridge process after this many milliseconds. Default: 30 minutes. */
|
|
1105
|
-
timeoutMs?: number;
|
|
1106
|
-
/**
|
|
1107
|
-
* OpenAI-compatible model used by standard GEPA reflection.
|
|
1108
|
-
* Calls pass through Agent Eval's local model proxy. Every recipe engine must
|
|
1109
|
-
* be `gepa` when this is set.
|
|
1110
|
-
*/
|
|
1111
|
-
optimizer?: OpenAICompatibleOptimizerModel;
|
|
1112
|
-
/**
|
|
1113
|
-
* Decide what the external optimizer may read for a train or selection case.
|
|
1114
|
-
* The returned value must be JSON-serializable. The final comparison cases
|
|
1115
|
-
* are not accepted by this API and cannot be serialized here.
|
|
1116
|
-
*/
|
|
1117
|
-
describeScenario?: (scenario: TScenario) => unknown;
|
|
1118
|
-
/** Optional bounded artifact evidence returned to GEPA after each evaluation. */
|
|
1119
|
-
describeArtifact?: (artifact: TArtifact, scenario: TScenario) => unknown;
|
|
1120
|
-
/** Default: `never`. Compatible runs resume only when explicitly enabled. */
|
|
1121
|
-
resume?: ExternalOptimizerResumeMode;
|
|
1122
|
-
/**
|
|
1123
|
-
* Required for resumable direct GEPA runs because upstream state uses Python
|
|
1124
|
-
* pickle. Enable only for state created locally in a directory you control.
|
|
1125
|
-
*/
|
|
1126
|
-
trustResumeState?: boolean;
|
|
1127
|
-
runner?: GepaRunnerCommand;
|
|
1128
|
-
}
|
|
1129
|
-
/**
|
|
1130
|
-
* Turn an optional GEPA installation into an `OptimizationMethod`.
|
|
1131
|
-
*
|
|
1132
|
-
* GEPA receives only serialized train and selection cases. The caller's final
|
|
1133
|
-
* test partition stays inside `compareOptimizationMethods`, which invokes this
|
|
1134
|
-
* method without a test-set field. The local callback routes every candidate
|
|
1135
|
-
* evaluation through the same dispatch and judges used by other methods.
|
|
1136
|
-
*/
|
|
1137
|
-
declare function gepaOptimizationMethod<TScenario extends Scenario, TArtifact>(config: GepaOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
1138
773
|
//#endregion
|
|
1139
774
|
//#region src/campaign/presets/run-eval.d.ts
|
|
1140
775
|
interface RunEvalOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'runDir'> {
|
|
@@ -1535,61 +1170,5 @@ interface EmitLoopProvenanceArgs<TArtifact, TScenario extends Scenario> extends
|
|
|
1535
1170
|
*/
|
|
1536
1171
|
declare function emitLoopProvenance<TArtifact, TScenario extends Scenario>(args: EmitLoopProvenanceArgs<TArtifact, TScenario>): Promise<EmitLoopProvenanceResult>;
|
|
1537
1172
|
//#endregion
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
epochs: number;
|
|
1541
|
-
batchSize: number;
|
|
1542
|
-
accumulation?: number;
|
|
1543
|
-
editBudget?: number;
|
|
1544
|
-
minEditBudget?: number;
|
|
1545
|
-
analystWorkers?: number;
|
|
1546
|
-
minibatchSize?: number;
|
|
1547
|
-
mergeBatchSize?: number;
|
|
1548
|
-
maxAnalystRounds?: number;
|
|
1549
|
-
evaluationWorkers?: number;
|
|
1550
|
-
learningRateSchedule?: 'constant' | 'linear' | 'cosine' | 'autonomous';
|
|
1551
|
-
learningRateControl?: 'fixed' | 'autonomous' | 'none';
|
|
1552
|
-
updateMode?: 'patch' | 'rewrite_from_suggestions' | 'full_rewrite_minibatch';
|
|
1553
|
-
failureOnly?: boolean;
|
|
1554
|
-
useSlowUpdate?: boolean;
|
|
1555
|
-
useMetaSkill?: boolean;
|
|
1556
|
-
/**
|
|
1557
|
-
* Additional flat SkillOpt trainer settings. Tangle overwrites data,
|
|
1558
|
-
* output, split, seed, validation, and activation settings.
|
|
1559
|
-
*/
|
|
1560
|
-
overrides?: Record<string, unknown>;
|
|
1561
|
-
}
|
|
1562
|
-
type SkillOptRunnerCommand = ExternalOptimizerRunnerCommand;
|
|
1563
|
-
interface SkillOptOptimizationMethodConfig<TScenario extends Scenario, TArtifact = unknown> {
|
|
1564
|
-
name?: string;
|
|
1565
|
-
/** Goal included with every described train and selection case. */
|
|
1566
|
-
objective: string;
|
|
1567
|
-
background?: string;
|
|
1568
|
-
/** Stable identity for the dispatch, judges, model settings, and scoring logic. */
|
|
1569
|
-
evaluationId: string;
|
|
1570
|
-
trainer: SkillOptTrainerConfig;
|
|
1571
|
-
/**
|
|
1572
|
-
* OpenAI-compatible model connection and hard limits for SkillOpt's own
|
|
1573
|
-
* optimizer calls.
|
|
1574
|
-
*/
|
|
1575
|
-
optimizer: OpenAICompatibleOptimizerModel;
|
|
1576
|
-
/** Hard cap on candidate-case callback requests. */
|
|
1577
|
-
maxEvaluations: number;
|
|
1578
|
-
/** Scores at or above this value count as hard successes. Default: 1. */
|
|
1579
|
-
hardScoreThreshold?: number;
|
|
1580
|
-
maxCandidateChars?: number;
|
|
1581
|
-
/** Maximum serialized scenario plus evaluation evidence. Default: 100,000. */
|
|
1582
|
-
maxEvidenceChars?: number;
|
|
1583
|
-
/** Candidate-evaluation callback byte limits. Omitted fields use finite defaults. */
|
|
1584
|
-
evaluationCallbackLimits?: Partial<ExternalOptimizerCallbackLimits>;
|
|
1585
|
-
timeoutMs?: number;
|
|
1586
|
-
describeScenario?: (scenario: TScenario) => unknown;
|
|
1587
|
-
describeArtifact?: (artifact: TArtifact, scenario: TScenario) => unknown;
|
|
1588
|
-
resume?: ExternalOptimizerResumeMode;
|
|
1589
|
-
runner?: SkillOptRunnerCommand;
|
|
1590
|
-
}
|
|
1591
|
-
/** Run Microsoft's SkillOpt trainer as a complete optimization method. */
|
|
1592
|
-
declare function skillOptOptimizationMethod<TScenario extends Scenario, TArtifact>(config: SkillOptOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
1593
|
-
//#endregion
|
|
1594
|
-
export { redTeamDataset as $, fsCampaignStorage as $t, RunEvalOptions as A, combineComparisonCosts as At, OptimizerModelBudget as B, runCampaign as Bt, RunImprovementLoopOptions as C, OptimizationMethodPairwise as Ct, RunOptimizationOptions as D, OptimizationMethodScore as Dt, PremeasuredOptimizationBaseline as E, OptimizationMethodRunOptions as Et, GepaOptimizationMethodConfig as F, CampaignRunPlan as Ft, DefaultProductionRewardHackingOptions as G, readGepaCandidatePopulationArtifact as Gt, heldOutGate as H, GepaCandidatePopulationCandidate as Ht, GepaOptimizationRecipe as I, CampaignRunPlanCell as It, RedTeamCase as J, ExternalOptimizerObservationSummary as Jt, defaultProductionGate as K, ExternalOptimizerExecutionSummary as Kt, GepaRunnerCommand as L, PlanCampaignRunOptions as Lt, GepaAdaptiveEngineRun as M, costFromLedgerSummary as Mt, GepaEngineOptions as N, optimizationTokenUsageFromSummary as Nt, RunOptimizationResult as O, OptimizationPackageSource as Ot, GepaEngineRun as P, CampaignCellFailureReceipt as Pt, RedTeamReport as Q, createRunCostLedger as Qt, gepaOptimizationMethod as R, RunCampaignOptions as Rt, verifyLoopProvenanceRecord as S, OptimizationMethodInput as St, runImprovementLoop as T, OptimizationMethodResult as Tt, DefaultProductionGateCheck as U, GepaCandidatePopulationSummary as Ut, HeldOutGateOptions as V, GepaCandidatePopulationArtifact as Vt, DefaultProductionGateOptions as W, GepaCandidateSelectionScore as Wt, RedTeamFinding as X, readExternalOptimizerObservationArtifact as Xt, RedTeamCategory as Y, ExternalOptimizerSubmittedCandidate as Yt, RedTeamPayload as Z, CampaignStorage as Zt, emitLoopProvenance as _, LlmJudgeDimension as _n, decodeExternalTextCandidate as _t, BuildLoopProvenanceArgs as a, campaignSplitDigestFromIdentities as an, CanaryKind as at, provenanceRecordPath as b, OptimizationMethod as bt, LoopProvenanceArgsFromResult as c, openAutoPr as cn, CanarySeverity as ct, LoopProvenanceEvidence as d, ReferenceEquivalenceJudgeInput as dn, externalTextOptimizationMethod as dt, inMemoryCampaignStorage as en, redTeamReport as et, LoopProvenanceOptimizationMethod as f, ReferenceEquivalenceJudgeOptions as fn, ExternalTextOptimizationMethodConfig as ft, canonicalDigest as g, runReferenceEquivalenceJudge as gn, ExternalTextEvaluationResponse as gt, campaignMeasurementDigest as h, createReferenceEquivalenceJudge as hn, ExternalOptimizationExample as ht, skillOptOptimizationMethod as i, campaignSplitDigest as in, CanaryEvaluation as it, runEval as j, compareOptimizationMethods as jt, runOptimization as k, OptimizationTokenUsage as kt, LoopProvenanceBackend as l, REFERENCE_EQUIVALENCE_INPUT_LIMITS as ln, runCanaries as lt, buildLoopProvenanceRecord as m, ReferenceEquivalenceScenario as mn, ExternalTextOptimizerResult as mt, SkillOptRunnerCommand as n, assertCampaignSplitIdentity as nn, toolNamesForRun as nt, EmitLoopProvenanceArgs as o, OpenAutoPrOptions as on, CanaryOptions as ot, LoopProvenanceRecord as p, ReferenceEquivalenceJudgeResult as pn, ExternalTextOptimizerContext as pt, DEFAULT_RED_TEAM_CORPUS as q, ExternalOptimizerObservationArtifact as qt, SkillOptTrainerConfig as r, campaignScenarioIdentity as rn, CanaryAlert as rt, EmitLoopProvenanceResult as s, OpenAutoPrResult as sn, CanaryReport as st, SkillOptOptimizationMethodConfig as t, assertCampaignDesign as tn, scoreRedTeamOutput as tt, LoopProvenanceCandidate as u, REFERENCE_EQUIVALENCE_JUDGE_VERSION as un, composeGate as ut, loopProvenanceArgsFromResult as v, LlmJudgeOptions as vn, CompareOptimizationMethodsOptions as vt, RunImprovementLoopResult as w, OptimizationMethodProvenance as wt, provenanceSpansPath as x, OptimizationMethodComparison as xt, loopProvenanceSpans as y, llmJudge as yn, ComparisonCost as yt, OpenAICompatibleOptimizerModel as z, planCampaignRun as zt };
|
|
1595
|
-
//# sourceMappingURL=skillopt-optimization-method-USDKhxSA.d.ts.map
|
|
1173
|
+
export { OptimizationTokenUsage as $, RedTeamCategory as A, llmJudge as At, CanaryReport as B, RunOptimizationOptions as C, fsCampaignStorage as Ct, runEval as D, openAutoPr as Dt, RunEvalOptions as E, OpenAutoPrResult as Et, scoreRedTeamOutput as F, OptimizationMethodComparison as G, CompareOptimizationMethodsOptions as H, CanaryAlert as I, OptimizationMethodProvenance as J, OptimizationMethodInput as K, CanaryEvaluation as L, RedTeamReport as M, redTeamDataset as N, DEFAULT_RED_TEAM_CORPUS as O, LlmJudgeDimension as Ot, redTeamReport as P, OptimizationPackageSource as Q, CanaryKind as R, PremeasuredOptimizationBaseline as S, createRunCostLedger as St, runOptimization as T, OpenAutoPrOptions as Tt, ComparisonCost as U, runCanaries as V, OptimizationMethod as W, OptimizationMethodRunOptions as X, OptimizationMethodResult as Y, OptimizationMethodScore as Z, provenanceSpansPath as _, ExternalOptimizerObservationArtifact as _t, LoopProvenanceBackend as a, RunCampaignOptions as at, RunImprovementLoopResult as b, readExternalOptimizerObservationArtifact as bt, LoopProvenanceOptimizationMethod as c, CampaignRunPlanCell as ct, campaignMeasurementDigest as d, GepaCandidatePopulationArtifact as dt, combineComparisonCosts as et, canonicalDigest as f, GepaCandidatePopulationCandidate as ft, provenanceRecordPath as g, ExternalOptimizerExecutionSummary as gt, loopProvenanceSpans as h, readGepaCandidatePopulationArtifact as ht, LoopProvenanceArgsFromResult as i, CampaignCellFailureReceipt as it, RedTeamFinding as j, RedTeamCase as k, LlmJudgeOptions as kt, LoopProvenanceRecord as l, PlanCampaignRunOptions as lt, loopProvenanceArgsFromResult as m, GepaCandidateSelectionScore as mt, EmitLoopProvenanceArgs as n, costFromLedgerSummary as nt, LoopProvenanceCandidate as o, runCampaign as ot, emitLoopProvenance as p, GepaCandidatePopulationSummary as pt, OptimizationMethodPairwise as q, EmitLoopProvenanceResult as r, optimizationTokenUsageFromSummary as rt, LoopProvenanceEvidence as s, CampaignRunPlan as st, BuildLoopProvenanceArgs as t, compareOptimizationMethods as tt, buildLoopProvenanceRecord as u, planCampaignRun as ut, verifyLoopProvenanceRecord as v, ExternalOptimizerObservationSummary as vt, RunOptimizationResult as w, inMemoryCampaignStorage as wt, runImprovementLoop as x, CampaignStorage as xt, RunImprovementLoopOptions as y, ExternalOptimizerSubmittedCandidate as yt, CanaryOptions as z };
|
|
1174
|
+
//# sourceMappingURL=provenance-CCdxgLDT.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provenance-CCdxgLDT.d.ts","names":[],"sources":["../src/llm-judge.ts","../src/campaign/auto-pr.ts","../src/campaign/storage.ts","../src/campaign/external-optimizer-observations.ts","../src/campaign/gepa-candidate-population.ts","../src/campaign/cell-cache.ts","../src/campaign/plan-campaign-run.ts","../src/campaign/run-campaign.ts","../src/campaign/presets/compare-optimization-methods.ts","../src/canary.ts","../src/red-team.ts","../src/campaign/presets/run-eval.ts","../src/campaign/presets/run-optimization.ts","../src/campaign/presets/run-improvement-loop.ts","../src/campaign/provenance.ts"],"mappings":";;;;;;;;;;;;KA6CY,6BAA6B;UAExB,gBAAgB,WAAW,kBAAkB,WAAW;;;;EAIvE,MAAM;;;EAGN,aAAa;;EAEb;;EAEA;EACA;EACA;;;EAGA,UAAU;;;;;EAKV;;EAEA,aAAa,UAAU;;;EAGvB,cAAc;IAAS,UAAU;IAAW,UAAU;;;EAEtD,aAAa;EACb;IAAmB;IAAc,QAAQ,EAAE;;;;;;;;;;;;;iBAoB7B,SAAS,qBAAqB,kBAAkB,WAAW,UACzE,cACA,gBACA,MAAM,gBAAgB,WAAW,aAChC,YAAY,WAAW;;;UClFT,kBAAkB,WAAW,kBAAkB;;EAE9D,QAAQ,eAAe,WAAW;;;EAGlC,MAAM;;;EAGN;;EAEA;EACA;;EAEA;;EAEA;;;EAGA;;EAEA,UAAU;IAAqB;IAAgB;IAAgB;;;UAGhD;EACf;EACA;EACA;EACA;;;;;iBAMc,WAAW,WAAW,kBAAkB,UACtD,SAAS,kBAAkB,WAAW,aACrC;;;;;;;;;;;;;;;;;;UCjCc;;EAEf,UAAU;;EAEV,OAAO;;EAEP,KAAK;;EAEL,MAAM,cAAc,kBAAkB;;;EAGtC,OAAO,cAAc,iBAAiB;;;;;;;;;iBAUxB,qBAAqB;;;;iBA0CrB,2BAA2B;;iBAmC3B,oBAAoB;EAClC,SAAS;EACT;EACA;;EAEA;IACE;;;UC/Ga;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;;WAEN,WAAW;;WAEX;WACA;WACA;;WAEA;aACE;aACA;;;UAII;WACN,SAAS;WACT,uBAAuB;;WAEvB,qBAAqB;;;;;;;;;;iBAWhB,yCAAyC;EACvD,SAAS;EACT,UAAU;IACR;;;UCrDa;WACN;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGM;WACN;WACA;;UAGM;;WAEN;WACA,WAAW;WACX;WACA;;WAEA;;WAEA;WACA,0BAA0B;WAC1B;;UAGM;WACN,SAAS;WACT;WACA;WACA,qBAAqB;;;;;;;;;;iBAWhB,oCAAoC;EAClD,SAAS;EACT,UAAU;IACR;;;KCuDQ;;;UCtGK;EACf;EACA;EACA;EACA;EACA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;UAGQ,uBAAuB,kBAAkB,UAAU;EAClE,WAAW;EACX,WAAW,WAAW,WAAW;EACjC;EACA,SAAS,YAAY,WAAW;EAChC;EACA;EACA;;EAEA;EACA;;EAEA;EACA,UAAU;;EAEV,aAAa;;EAEb,WAAW,SAAS;;;;;;iBAON,gBAAgB,kBAAkB,UAAU,WAC1D,MAAM,uBAAuB,WAAW,aACvC;;;UCvBc,mBAAmB,kBAAkB,UAAU;EAC9D,WAAW;EACX,UAAU,WAAW,WAAW;;EAEhC,SAAS;;;;;;EAMT;EACA,SAAS,YAAY,WAAW;;EAEhC;;;EAGA;;;EAGA;;;;;;EAMA;;;;EAIA,eAAe;EACf;EACA;;EAEA;;;EAGA,aAAa;;EAEb;;EAEA,WAAW,SAAS;;EAEpB;;;;;;;;EAQA;;;;;;;;;;EAUA;;;;;EAKA;;;;EAIA;;;EAGA;;;;EAIA;;;;;;;;;;;EAWA;;EAEA,YAAY;;EAEZ,oBAAoB,gBAAgB,gBAAgB;;;;;;EAMpD,UAAU;;;;;;;;;;;;EAYV,iBAAiB;IACf,UAAU;IACV;IACA;;;;;;;UAQa,2BAA2B;EAC1C;EACA;EACA;EACA;IACE;IACA;IACA;MACE;MACA;MACA;;;EAGJ,MAAM,mBAAmB;EACzB,MAAM;;;;;iBAMc,YAAY,kBAAkB,UAAU,WAC5D,MAAM,mBAAmB,WAAW,aACnC,QAAQ,eAAe,WAAW;;;;KC3IzB,6BAA6B,kBAAkB,UAAU,aAAa,KAChF,mBAAmB,WAAW;;UAKf;;EAEf;;EAEA,gBAAgB;EAChB;EACA;;UAGe;EACf;;EAEA;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;;EAEA;EACA;EACA;;UAGe;;EAEf,QAAQ;;EAER,SAAS;;EAET,UAAU;;EAEV,SAAS;;EAET;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,aAAa;;EAEb,eAAe;;EAEf,0BAA0B;;EAE1B,kBAAkB;;;UAIH,wBAAwB,kBAAkB,UAAU;;WAE1D,iBAAiB;;WAEjB,yBAAyB;;WAEzB,6BAA6B;;WAE7B,sBACP,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;;WAEJ,iBAAiB,YAAY,WAAW;;WAExC;WACA;;WAEA,YAAY,SAAS,6BAA6B,WAAW;;WAE7D,YAAY;;UAGN;;EAEf,eAAe;;EAEf,MAAM;;EAEN;;EAEA,aAAa;;;UAIE,mBAAmB,kBAAkB,WAAW,UAAU;;EAEzE;EACA,WACE,OAAO,wBAAwB,WAAW,eACvC,QAAQ;;UAGE;EACf;;EAEA;;EAEA;;EAEA;;;EAGA;IAAU;IAAa;;;EAEvB,kBAAkB;;EAElB;;EAEA,aAAa;;EAEb,gBAAgB;IACd;IACA;IACA;IACA;;EAEF,eAAe;;EAEf;;UAGe;;EAEf;EACA;;EAEA;EACA;EACA;;EAEA;;UAGe;;EAEf,QAAQ;EACR,MAAM;;EAEN,UAAU;EACV;;EAEA,kBAAkB;;EAElB,UAAU;;EAEV,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,kCAAkC,kBAAkB,UAAU,mBACrE,KAAK,mBAAmB,WAAW;EAC3C,SAAS,mBAAmB,WAAW;EACvC,iBAAiB;;EAEjB,gBAAgB;;EAEhB,oBAAoB;;EAEpB,eAAe;;EAEf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;EACb,QAAQ,YAAY,WAAW;;;EAG/B;;EAEA,yBAAyB,6BAA6B,WAAW;;EAEjE;;;EAGA;;EAEA;;;;;iBAMoB,2BAA2B,kBAAkB,UAAU,WAC3E,MAAM,kCAAkC,WAAW,aAClD,QAAQ;;iBAkwBK,sBAAsB,SAAS,oBAAoB;;iBAYnD,kCACd,SAAS,mBACT,mBAAmB,gBAClB;;iBA0Ba,uBACd,SAAS;EAAgB;EAAe,MAAM;KAC7C;;;KCzhCS;KAEA;UAEK;EACf,MAAM;EACN,UAAU;EACV;;;EAGA,UAAU;;UAGK;EACf,QAAQ;;EAER,QAAQ,OAAO;;EAEf,aAAa;;UAGE;EACf,MAAM;EACN;EACA;EACA;;UAGe;;;;;;;;;;EAUf;IACE;IACA;;IAEA;;;;;;;;;;;;;EAcF;IACE;IACA;IACA;IACA;;;;;;;;;;EAWF;IACE,WAAW,KAAK;IAChB;IACA;IACA;IACA;;;;;;;;iBASY,YAAY,MAAM,aAAa,OAAM,gBAAqB;;;KClG9D;UAUK;EACf,UAAU;;EAEV;;;;;EAKA;;EAEA;;EAEA;;UAGe,oBAAoB;EACnC,SAAS;;UAGM;EACf;EACA,UAAU;EACV;EACA;EACA;;UAGe;EACf,UAAU;EACV,oBAAoB,OAAO;EAC3B;;;cA0DW,yBAAyB;iBA6FtB,eAAe,aAAY,gBAAqB;;;;;iBAkBhD,mBACd,gBACA,qBACA,QAAQ,cACP;;iBA4Fa,cAAc,UAAU,mBAAmB;;;UCzT1C,eAAe,kBAAkB,UAAU,mBAClD,KAAK,mBAAmB,WAAW;EAC3C;;;;;iBAMoB,QAAQ,kBAAkB,UAAU,WACxD,MAAM,eAAe,WAAW,aAC/B,QAAQ,eAAe,WAAW;;;UC0BpB,gCAAgC,WAAW,kBAAkB;;EAE5E;;EAEA,UAAU,eAAe,WAAW;;UAGrB,2BAA2B,kBAAkB,UAAU,mBAC9D,KAAK,mBAAmB,WAAW;;EAE3C,iBAAiB;;;;;;;;;EASjB,sBAAsB,gCAAgC,WAAW;;EAEjE,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;;EAEb,UAAU,gBAAgB;EAC1B;EACA;;;EAGA;;;EAGA;;EAEA,WAAW,cAAc;;;;;;;;;;;EAWzB,qBAAqB;IACnB;IACA;IACA,YAAY;MACV;MACA,UAAU,eAAe,WAAW;MACpC;;IAEF,SAAS;;IAET,aAAa;IACb;QACI,QAAQ,cAAc;;;;;;;;;;;;;;;;;;EAkB5B,oBAAoB,UAAU,eAAe,WAAW;;KAG9C,uBACV,kBAAkB,UAClB,aACE,2BAA2B,WAAW;UAEzB,sBAAsB,WAAW,kBAAkB;EAClE,aAAa;IACX,QAAQ;IACR,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;;EAIxC,iBAAiB;EACjB,eAAe;EACf;;;;EAIA;;;;EAIA;EACA,kBAAkB,eAAe,WAAW;;EAE5C,MAAM;;;;;;EAMN,gBAAgB;;;;;iBAMI,gBAAgB,kBAAkB,UAAU,WAChE,MAAM,uBAAuB,WAAW,aACvC,QAAQ,sBAAsB,WAAW;;;KCpJhC,0BACV,kBAAkB,UAClB,aACE,uBAAuB,WAAW;;;EAGpC,kBAAkB;;;;;;;;;EASlB;;;;EAIA,MAAM,KAAK,WAAW;;;;;EAKtB;;EAEA;EACA;;;;;;;;EAQA,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;UAGlE,yBAAyB,WAAW,kBAAkB,kBAC7D,sBAAsB,WAAW;EACzC,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,uBAAuB,eAAe,WAAW;EACjD,qBAAqB;EACrB,YAAY,QAAQ,WAAW,KAAK,WAAW;;;;EAI/C;;;;;EAKA;EACA,WAAW,kBAAkB;;;;;iBAMT,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,0BAA0B,WAAW,aAC1C,QAAQ,yBAAyB,WAAW;;;UC1B9B;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,UAAU,YAAY;;EAEtB;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;EACA;EACA;;UAGe;EACf;IACE;IACA;;EAEF;IACE;IACA;IACA;IACA;MACE;MACA;MACA;MACA;;;EAGJ;;UAGe;EACf;EACA,MAAM;EACN;EACA,aAAa;;;;;;;UAQE;EACf;;EAEA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;EACA;;EAEA;;EAEA,YAAY;;EAEZ,qBAAqB;;EAErB,UAAU;;EAEV;;EAEA;IACE,UAAU;IACV;IACA;IACA,mBAAmB;;;;;;EAMrB;;EAEA;;EAEA;;;EAGA;;EAEA,SAAS;EACT;EACA;;UAGe,wBAAwB,WAAW,kBAAkB;EACpE;EACA;EACA;EACA,iBAAiB;EACjB,eAAe;EACf;EACA;;EAEA,wBAAwB,eAAe,WAAW;;EAElD,aAAa;IACX;IACA,YAAY;IACZ;;;IAGA,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;EAGxC,MAAM;;;;EAIN;EACA,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,qBAAqB;EACrB,uBAAuB,eAAe,WAAW;;EAEjD,cAAc,cAAc;EAC5B;EACA;EACA,qBAAqB;;UAGN,6BAA6B,WAAW,kBAAkB;EACzE;EACA;EACA;EACA,iBAAiB;EACjB,QAAQ,yBAAyB,WAAW;EAC5C,cAAc,cAAc;EAC5B;EACA;;;iBAIc,6BAA6B,WAAW,kBAAkB,UACxE,OAAO,6BAA6B,WAAW,aAC9C,wBAAwB,WAAW;;iBA4CtB,0BAA0B,WAAW,kBAAkB,UACrE,MAAM,wBAAwB,WAAW,aACxC;;iBAoRa,0BAA0B,WAAW,kBAAkB,UACrE,UAAU,eAAe,WAAW;;iBAqCtB,2BAA2B,QAAQ,uBAAuB;;iBAa1D,gBAAgB;;;;;;;;;;;;iBA2KhB,oBACd,QAAQ,sBACR;EAAQ;IACP;;iBAiJa,qBAAqB;;;;iBAMrB,oBAAoB;UAInB;EACf,QAAQ;EACR,OAAO;;EAEP;EACA;;UAGe,uBAAuB,WAAW,kBAAkB,kBAC3D,wBAAwB,WAAW;;EAE3C,SAAS;;;EAGT,eAAe;;;;;;;;;;;;iBA+EK,mBAAmB,WAAW,kBAAkB,UACpE,MAAM,uBAAuB,WAAW,aACvC,QAAQ"}
|