@tangle-network/agent-eval 0.216.0 → 0.216.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/adapters/http.d.ts +3 -3
- package/dist/{agent-profile-CIA2Atnt.d.ts → agent-profile-VFcQncbW.d.ts} +5 -5
- package/dist/{agent-profile-CIA2Atnt.d.ts.map → agent-profile-VFcQncbW.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-BYc0bxsL.js → agent-profile-cell-BK8mlqlq.js} +3 -3
- package/dist/{agent-profile-cell-BYc0bxsL.js.map → agent-profile-cell-BK8mlqlq.js.map} +1 -1
- package/dist/{agreement-irr-DsTURxdr.js → agreement-irr-BEWLIF3_.js} +3 -3
- package/dist/{agreement-irr-DsTURxdr.js.map → agreement-irr-BEWLIF3_.js.map} +1 -1
- package/dist/analyst/index.d.ts +14 -14
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +15 -15
- package/dist/{analyze-runs-3xpby3p2.js → analyze-runs-C84irm4i.js} +608 -604
- package/dist/analyze-runs-C84irm4i.js.map +1 -0
- package/dist/{analyze-runs-CSDAh8A7.d.ts → analyze-runs-D68fd-su.d.ts} +17 -15
- package/dist/analyze-runs-D68fd-su.d.ts.map +1 -0
- package/dist/{baseline-Dm8966du.js → baseline-Udi_PLuT.js} +5 -5
- package/dist/{baseline-Dm8966du.js.map → baseline-Udi_PLuT.js.map} +1 -1
- package/dist/{benchmark-BywPeAUo.js → benchmark-CXmVVHH2.js} +2 -2
- package/dist/{benchmark-BywPeAUo.js.map → benchmark-CXmVVHH2.js.map} +1 -1
- package/dist/{benchmark-command-CByl9dxG.js → benchmark-command-By0vh8q7.js} +62 -45
- package/dist/benchmark-command-By0vh8q7.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -5
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +13 -13
- package/dist/{campaign-BQ-Jujuj.js → campaign-BudZ70zo.js} +245 -247
- package/dist/campaign-BudZ70zo.js.map +1 -0
- package/dist/{campaign-evidence-OcUciD-N.js → campaign-evidence-Clrn4hBV.js} +8 -8
- package/dist/campaign-evidence-Clrn4hBV.js.map +1 -0
- package/dist/{canonical-BsZVgBhW.js → canonical-B2RqLofv.js} +2 -2
- package/dist/{canonical-BsZVgBhW.js.map → canonical-B2RqLofv.js.map} +1 -1
- package/dist/{chat-client-DthPOYGd.js → chat-client-l-MVDlDk.js} +2 -2
- package/dist/{chat-client-DthPOYGd.js.map → chat-client-l-MVDlDk.js.map} +1 -1
- package/dist/{chat-json-call-BrKsyspg.js → chat-json-call-puMArmtW.js} +2 -2
- package/dist/{chat-json-call-BrKsyspg.js.map → chat-json-call-puMArmtW.js.map} +1 -1
- package/dist/{claim-BXcZw-jo.js → claim-q4HQhK1G.js} +4 -4
- package/dist/claim-q4HQhK1G.js.map +1 -0
- package/dist/cli.js +9 -9
- package/dist/cli.js.map +1 -1
- package/dist/{cluster-bootstrap-B3JywQQn.js → cluster-bootstrap-qcUWYFl-.js} +3 -3
- package/dist/{cluster-bootstrap-B3JywQQn.js.map → cluster-bootstrap-qcUWYFl-.js.map} +1 -1
- package/dist/contract/index.d.ts +11 -11
- package/dist/contract/index.js +1022 -1078
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-D2FnJPBF.js → cost-ledger-BP8V4kUQ.js} +835 -816
- package/dist/cost-ledger-BP8V4kUQ.js.map +1 -0
- package/dist/{cost-ledger-CO7UyEcQ.d.ts → cost-ledger-Jk62OBfF.d.ts} +12 -47
- package/dist/cost-ledger-Jk62OBfF.d.ts.map +1 -0
- package/dist/{counterfactual-DtMWvn7D.js → counterfactual-Dn3veueY.js} +2 -2
- package/dist/{counterfactual-DtMWvn7D.js.map → counterfactual-Dn3veueY.js.map} +1 -1
- package/dist/{default-registry-Cn1LQ2Ve.js → default-registry-COhk9g36.js} +5 -5
- package/dist/{default-registry-Cn1LQ2Ve.js.map → default-registry-COhk9g36.js.map} +1 -1
- package/dist/{default-registry-BrWh62zf.d.ts → default-registry-DzHiGruW.d.ts} +6 -6
- package/dist/{default-registry-BrWh62zf.d.ts.map → default-registry-DzHiGruW.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Ch3MWX6k.js → define-agent-eval-CNTPzIkp.js} +10 -10
- package/dist/define-agent-eval-CNTPzIkp.js.map +1 -0
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts → define-agent-eval-WSck0CZJ.d.ts} +8 -8
- package/dist/{define-agent-eval-CEWkX5wZ.d.ts.map → define-agent-eval-WSck0CZJ.d.ts.map} +1 -1
- package/dist/{descriptive-CnU8Yq29.js → descriptive-DboxPZrG.js} +2 -2
- package/dist/{descriptive-CnU8Yq29.js.map → descriptive-DboxPZrG.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +3 -3
- package/dist/diagnosis/index.js +4 -4
- package/dist/{dspy-rlm-engine-D-KX_e64.js → dspy-rlm-engine-LRCu61m5.js} +3 -3
- package/dist/{dspy-rlm-engine-D-KX_e64.js.map → dspy-rlm-engine-LRCu61m5.js.map} +1 -1
- package/dist/{effect-sizes-DiH8MGOH.js → effect-sizes-BhzEpXMc.js} +2 -2
- package/dist/{effect-sizes-DiH8MGOH.js.map → effect-sizes-BhzEpXMc.js.map} +1 -1
- package/dist/{engine-C_zNWfM8.d.ts → engine-3A_4jOlI.d.ts} +5 -5
- package/dist/{engine-C_zNWfM8.d.ts.map → engine-3A_4jOlI.d.ts.map} +1 -1
- package/dist/{equal-terms-Blx9Zk5I.js → equal-terms-BtdXayCJ.js} +2 -2
- package/dist/{equal-terms-Blx9Zk5I.js.map → equal-terms-BtdXayCJ.js.map} +1 -1
- package/dist/{errors-Dngq5h35.js → errors-DVZiqS7K.js} +6 -2
- package/dist/{errors-Dngq5h35.js.map → errors-DVZiqS7K.js.map} +1 -1
- package/dist/{eval-campaign-4byTtqrf.js → eval-campaign-90jeaCAI.js} +6 -6
- package/dist/{eval-campaign-4byTtqrf.js.map → eval-campaign-90jeaCAI.js.map} +1 -1
- package/dist/{evaluation-Cn2dRZIz.d.ts → evaluation-6Yoi74gM.d.ts} +4 -4
- package/dist/{evaluation-Cn2dRZIz.d.ts.map → evaluation-6Yoi74gM.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -1
- package/dist/{exact-types-CD0lf5Qd.d.ts → exact-types-ClugMJfI.d.ts} +2 -2
- package/dist/{exact-types-CD0lf5Qd.d.ts.map → exact-types-ClugMJfI.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +404 -3
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +313 -19
- package/dist/experiment/index.js.map +1 -1
- package/dist/{external-optimizer-process-nnsY5D_4.js → external-optimizer-process-BjMGUuds.js} +3 -3
- package/dist/{external-optimizer-process-nnsY5D_4.js.map → external-optimizer-process-BjMGUuds.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-BqW0hHcI.js} +548 -548
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +1 -0
- package/dist/{feedback-trajectory-DkaDj0dj.d.ts → feedback-trajectory-C4EG0nNH.d.ts} +5 -8
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.js +1 -1
- package/dist/{index-CPJJr98o.d.ts → index-BFRO6chs.d.ts} +3 -2
- package/dist/index-BFRO6chs.d.ts.map +1 -0
- package/dist/{index-Ch9VxCpK.d.ts → index-BO41eGj5.d.ts} +2 -2
- package/dist/{index-Ch9VxCpK.d.ts.map → index-BO41eGj5.d.ts.map} +1 -1
- package/dist/{index-NvGBiO8H.d.ts → index-CyszSg4O.d.ts} +165 -162
- package/dist/index-CyszSg4O.d.ts.map +1 -0
- package/dist/{index-DlH0Y9gI.d.ts → index-DPcngtwt.d.ts} +30 -17
- package/dist/index-DPcngtwt.d.ts.map +1 -0
- package/dist/{index-oQ4aHziK.d.ts → index-DV-jSzxF.d.ts} +8 -8
- package/dist/{index-oQ4aHziK.d.ts.map → index-DV-jSzxF.d.ts.map} +1 -1
- package/dist/index.d.ts +1969 -1871
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +161 -201
- package/dist/index.js.map +1 -1
- package/dist/{integrity-BvnVARVl.js → integrity-BXOsAb3i.js} +2 -2
- package/dist/{integrity-BvnVARVl.js.map → integrity-BXOsAb3i.js.map} +1 -1
- package/dist/{integrity-CCAxGVK6.d.ts → integrity-DAMEDFaC.d.ts} +2 -2
- package/dist/{integrity-CCAxGVK6.d.ts.map → integrity-DAMEDFaC.d.ts.map} +1 -1
- package/dist/{integrity-BV1ZDf42.js → integrity-dLiDe2kD.js} +2 -5
- package/dist/integrity-dLiDe2kD.js.map +1 -0
- package/dist/{internal-CRHwS9iz.js → internal-Epl-YBZW.js} +3 -3
- package/dist/{internal-CRHwS9iz.js.map → internal-Epl-YBZW.js.map} +1 -1
- package/dist/interval-CELpye1n.js +219 -0
- package/dist/interval-CELpye1n.js.map +1 -0
- package/dist/interval-ohuOJurX.d.ts +47 -0
- package/dist/interval-ohuOJurX.d.ts.map +1 -0
- package/dist/{journal-DvKCvjty.js → journal-DfRz9kps.js} +2 -2
- package/dist/{journal-DvKCvjty.js.map → journal-DfRz9kps.js.map} +1 -1
- package/dist/{judge-calibration-N44V04Se.js → judge-calibration-BpCy5XQl.js} +3 -3
- package/dist/{judge-calibration-N44V04Se.js.map → judge-calibration-BpCy5XQl.js.map} +1 -1
- package/dist/{judge-gate-DNcUrRFA.d.ts → judge-gate-CE7CQAgt.d.ts} +3 -3
- package/dist/{judge-gate-DNcUrRFA.d.ts.map → judge-gate-CE7CQAgt.d.ts.map} +1 -1
- package/dist/{judge-gate-B1qYnkpi.js → judge-gate-iY9s4L44.js} +3 -3
- package/dist/judge-gate-iY9s4L44.js.map +1 -0
- package/dist/{kind-factory-CbZ-6tvu.js → kind-factory-BobI5Jy3.js} +4 -4
- package/dist/{kind-factory-CbZ-6tvu.js.map → kind-factory-BobI5Jy3.js.map} +1 -1
- package/dist/ledger-core/index.js +3 -3
- package/dist/{ledger-core-B2hahzrt.js → ledger-core-Gq7yp8lP.js} +4 -4
- package/dist/{ledger-core-B2hahzrt.js.map → ledger-core-Gq7yp8lP.js.map} +1 -1
- package/dist/{llm-client-DQveoxSM.js → llm-client-CyBAoYiX.js} +192 -219
- package/dist/llm-client-CyBAoYiX.js.map +1 -0
- package/dist/{llm-judge-PXV7Xljl.js → llm-judge-CG12SQJq.js} +499 -921
- package/dist/llm-judge-CG12SQJq.js.map +1 -0
- package/dist/matrix/index.d.ts +2 -2
- package/dist/matrix/index.js +1 -1
- package/dist/{matrix-CHMiebFx.d.ts → matrix-DSrSxjjy.d.ts} +4 -4
- package/dist/{matrix-CHMiebFx.d.ts.map → matrix-DSrSxjjy.d.ts.map} +1 -1
- package/dist/{matrix-_BV9-Lk5.js → matrix-Dm0BQW7U.js} +7 -11
- package/dist/matrix-Dm0BQW7U.js.map +1 -0
- package/dist/{paired-tests-jbwPdSz5.js → mean-DHoMiudD.js} +11 -5
- package/dist/mean-DHoMiudD.js.map +1 -0
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +9 -9
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/mint-DNU8QPG4.d.ts +81 -0
- package/dist/mint-DNU8QPG4.d.ts.map +1 -0
- package/dist/{mint-Csor1BLJ.js → mint-QaD5cA0u.js} +3 -3
- package/dist/{mint-Csor1BLJ.js.map → mint-QaD5cA0u.js.map} +1 -1
- package/dist/{multiplicity-D9cjmStq.js → multiplicity-DQUFcbW4.js} +2 -2
- package/dist/{multiplicity-D9cjmStq.js.map → multiplicity-DQUFcbW4.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -6
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/multishot/index.js +1 -1
- package/dist/nonnegative-number-DvdMxg_f.js +9 -0
- package/dist/nonnegative-number-DvdMxg_f.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-C8YIJWy1.js → otel-spans-B9tUXRHF.js} +3 -3
- package/dist/{otel-spans-C8YIJWy1.js.map → otel-spans-B9tUXRHF.js.map} +1 -1
- package/dist/{paired-arms-DzKn0TfY.js → paired-arms-CwMGJsrS.js} +5 -5
- package/dist/{paired-arms-DzKn0TfY.js.map → paired-arms-CwMGJsrS.js.map} +1 -1
- package/dist/{paired-binary-Q36bhn2R.js → paired-binary-CvpFHfKZ.js} +2 -2
- package/dist/{paired-binary-Q36bhn2R.js.map → paired-binary-CvpFHfKZ.js.map} +1 -1
- package/dist/{paired-promotion-decision-BN6IQTiu.js → paired-promotion-decision-DsAPYGCm.js} +3 -3
- package/dist/{paired-promotion-decision-BN6IQTiu.js.map → paired-promotion-decision-DsAPYGCm.js.map} +1 -1
- package/dist/{search-kernel-BztlERLv.js → pareto-Cm6ndrWN.js} +883 -861
- package/dist/pareto-Cm6ndrWN.js.map +1 -0
- package/dist/pipelines/index.d.ts +2 -2
- package/dist/pipelines/index.js +3 -3
- package/dist/{power-and-mde-fK5Qocwo.js → power-and-mde-Hj2ubb4B.js} +2 -2
- package/dist/{power-and-mde-fK5Qocwo.js.map → power-and-mde-Hj2ubb4B.js.map} +1 -1
- package/dist/predicate-FdKmElA_.js +38 -0
- package/dist/predicate-FdKmElA_.js.map +1 -0
- package/dist/{prime-protocol-DcAK9pC8.js → prime-protocol-CHKgA3a6.js} +2 -2
- package/dist/{prime-protocol-DcAK9pC8.js.map → prime-protocol-CHKgA3a6.js.map} +1 -1
- package/dist/{prime-protocol-CEE4BR9N.d.ts → prime-protocol-HiB9HX7E.d.ts} +3 -3
- package/dist/{prime-protocol-CEE4BR9N.d.ts.map → prime-protocol-HiB9HX7E.d.ts.map} +1 -1
- package/dist/{produced-state-ZCtDBxAE.js → produced-state-BsDmnwxc.js} +8 -8
- package/dist/{produced-state-ZCtDBxAE.js.map → produced-state-BsDmnwxc.js.map} +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{release-confidence-BK2QCB5M.d.ts → promotion-gate-BawrdPYr.d.ts} +102 -102
- package/dist/promotion-gate-BawrdPYr.d.ts.map +1 -0
- package/dist/{promotion-policy-CWW35UWq.js → promotion-policy-KO57gbyV.js} +4 -4
- package/dist/{promotion-policy-CWW35UWq.js.map → promotion-policy-KO57gbyV.js.map} +1 -1
- package/dist/{proposal-findings-BZiMTzAF.js → proposal-findings-CY1xKn_a.js} +2 -2
- package/dist/{proposal-findings-BZiMTzAF.js.map → proposal-findings-CY1xKn_a.js.map} +1 -1
- package/dist/{query-B1D1qp2Z.js → query-BUCZCK7F.js} +3 -3
- package/dist/{query-B1D1qp2Z.js.map → query-BUCZCK7F.js.map} +1 -1
- package/dist/{random-Dn5fPWkt.js → random-ByUSnV9w.js} +2 -2
- package/dist/{random-Dn5fPWkt.js.map → random-ByUSnV9w.js.map} +1 -1
- package/dist/{raw-provider-sink-rB2DYxr6.js → raw-provider-sink-DoHz4JNG.js} +2 -2
- package/dist/{raw-provider-sink-rB2DYxr6.js.map → raw-provider-sink-DoHz4JNG.js.map} +1 -1
- package/dist/{redact-Dk1ZdJEm.js → redact-B5xXKqQ3.js} +97 -98
- package/dist/redact-B5xXKqQ3.js.map +1 -0
- package/dist/{redact-Cveoq6fJ.d.ts → redact-mtCCcT8z.d.ts} +4 -20
- package/dist/redact-mtCCcT8z.d.ts.map +1 -0
- package/dist/{registry-JYfPxcVz.d.ts → registry-D7_hm2fK.d.ts} +5 -5
- package/dist/{registry-JYfPxcVz.d.ts.map → registry-D7_hm2fK.d.ts.map} +1 -1
- package/dist/{release-confidence-Dbe9AS_X.js → release-confidence-CpuP65S5.js} +5 -5
- package/dist/{release-confidence-Dbe9AS_X.js.map → release-confidence-CpuP65S5.js.map} +1 -1
- package/dist/{report-command-y8H6RSX8.js → report-command-SC9nqb4M.js} +106 -171
- package/dist/report-command-SC9nqb4M.js.map +1 -0
- package/dist/reporting-html-Kvlc8oMt.js +100 -0
- package/dist/reporting-html-Kvlc8oMt.js.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +6 -6
- package/dist/{researcher-f6mZhh58.d.ts → researcher-CEcfku-T.d.ts} +5 -5
- package/dist/{researcher-f6mZhh58.d.ts.map → researcher-CEcfku-T.d.ts.map} +1 -1
- package/dist/reward-CpCmAM8L.d.ts +78 -0
- package/dist/reward-CpCmAM8L.d.ts.map +1 -0
- package/dist/{reward-hacking-BFcQgkux.js → reward-hacking-E0d-WVcP.js} +2 -2
- package/dist/{reward-hacking-BFcQgkux.js.map → reward-hacking-E0d-WVcP.js.map} +1 -1
- package/dist/{reward-hacking-DHA7c1l8.d.ts → reward-hacking-RVdZ6Kkt.d.ts} +2 -2
- package/dist/{reward-hacking-DHA7c1l8.d.ts.map → reward-hacking-RVdZ6Kkt.d.ts.map} +1 -1
- package/dist/rl.d.ts +8 -8
- package/dist/rl.js +14 -14
- package/dist/rollout/index.d.ts +3 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-D9bKKBnZ.js → rollout-DZV1t2Wg.js} +2 -2
- package/dist/{rollout-D9bKKBnZ.js.map → rollout-DZV1t2Wg.js.map} +1 -1
- package/dist/round-decimal-COrKglMh.js +10 -0
- package/dist/round-decimal-COrKglMh.js.map +1 -0
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts → rubric-predictive-validity-C03EC7ZE.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-2SgHxoa5.d.ts.map → rubric-predictive-validity-C03EC7ZE.d.ts.map} +1 -1
- package/dist/{rubric-predictive-validity-BsgkBfT0.js → rubric-predictive-validity-DaFW70y9.js} +3 -3
- package/dist/{rubric-predictive-validity-BsgkBfT0.js.map → rubric-predictive-validity-DaFW70y9.js.map} +1 -1
- package/dist/{run-record-DlTZ_lqe.js → run-record-Boc5IUTp.js} +4 -4
- package/dist/{run-record-DlTZ_lqe.js.map → run-record-Boc5IUTp.js.map} +1 -1
- package/dist/{run-record-D1Hw-Jml.d.ts → run-record-C1E24G2A.d.ts} +2 -2
- package/dist/{run-record-D1Hw-Jml.d.ts.map → run-record-C1E24G2A.d.ts.map} +1 -1
- package/dist/{run-record-ES07wfz9.js → run-record-C5e7FjhJ.js} +3 -5
- package/dist/run-record-C5e7FjhJ.js.map +1 -0
- package/dist/search/index.d.ts +46 -32
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +3 -3
- package/dist/{search-kernel-CpcKH5Ih.d.ts → search-kernel-DyNEa6IU.d.ts} +14 -10
- package/dist/search-kernel-DyNEa6IU.d.ts.map +1 -0
- package/dist/{search-ledger-d7V8o9Lk.js → search-ledger-CVUGLDZp.js} +189 -199
- package/dist/search-ledger-CVUGLDZp.js.map +1 -0
- package/dist/{search-ledger-file-BqOo-bVo.js → search-ledger-file-DDQUgvCg.js} +4 -4
- package/dist/{search-ledger-file-BqOo-bVo.js.map → search-ledger-file-DDQUgvCg.js.map} +1 -1
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +1 -1
- package/dist/{search-shipper-er4pK5_Y.js → search-shipper-BOSD46qv.js} +3 -3
- package/dist/{search-shipper-er4pK5_Y.js.map → search-shipper-BOSD46qv.js.map} +1 -1
- package/dist/{search-summary-CBvz3u_S.js → search-summary-BQCSWf1g.js} +5 -5
- package/dist/{search-summary-CBvz3u_S.js.map → search-summary-BQCSWf1g.js.map} +1 -1
- package/dist/{semantic-concept-judge-zQBG9hHU.js → semantic-concept-judge-BQF8te0T.js} +12 -14
- package/dist/semantic-concept-judge-BQF8te0T.js.map +1 -0
- package/dist/{sequential-D-1PPCmM.js → sequential-Bf_pbE50.js} +5 -5
- package/dist/{sequential-D-1PPCmM.js.map → sequential-Bf_pbE50.js.map} +1 -1
- package/dist/{sequential-eprocess-D1jKoihe.js → sequential-eprocess-DM_qENCl.js} +2 -2
- package/dist/{sequential-eprocess-D1jKoihe.js.map → sequential-eprocess-DM_qENCl.js.map} +1 -1
- package/dist/{server-DbcC7xMl.js → server-e9oWO_bN.js} +4 -4
- package/dist/{server-DbcC7xMl.js.map → server-e9oWO_bN.js.map} +1 -1
- package/dist/{single-run-lock-fVu3MZWH.js → single-run-lock-CV9Eyd_q.js} +2 -2
- package/dist/{single-run-lock-fVu3MZWH.js.map → single-run-lock-CV9Eyd_q.js.map} +1 -1
- package/dist/{skillopt-optimization-method-CCtzralq.js → skillopt-optimization-method-BiwA8nfX.js} +427 -12
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +1 -0
- package/dist/{spans-D9_ehFuk.d.ts → spans-BHAUXMGN.d.ts} +2 -2
- package/dist/{spans-D9_ehFuk.d.ts.map → spans-BHAUXMGN.d.ts.map} +1 -1
- package/dist/{statistical-heldout-1Gbmex7v.js → statistical-heldout-42NlCNDe.js} +3 -3
- package/dist/{statistical-heldout-1Gbmex7v.js.map → statistical-heldout-42NlCNDe.js.map} +1 -1
- package/dist/{statistical-heldout-BeR4bOdg.d.ts → statistical-heldout-CYpGaoDH.d.ts} +3 -3
- package/dist/{statistical-heldout-BeR4bOdg.d.ts.map → statistical-heldout-CYpGaoDH.d.ts.map} +1 -1
- package/dist/{store-otlp-DtB4Slsr.js → store-otlp-DJ9dtA5x.js} +135 -132
- package/dist/store-otlp-DJ9dtA5x.js.map +1 -0
- package/dist/{store-tool-spans-fZIigrpP.js → store-tool-spans-CIX5arAG.js} +7 -7
- package/dist/{store-tool-spans-fZIigrpP.js.map → store-tool-spans-CIX5arAG.js.map} +1 -1
- package/dist/{store-tool-spans-DR1SrCLK.d.ts → store-tool-spans-DdCSPa7C.d.ts} +162 -161
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +20 -3
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{student-t-BQeKJx2N.js → student-t-DVPW7CdL.js} +2 -2
- package/dist/{student-t-BQeKJx2N.js.map → student-t-DVPW7CdL.js.map} +1 -1
- package/dist/{summary-report-DtdRucM0.js → summary-report-V_JACR4Y.js} +526 -614
- package/dist/summary-report-V_JACR4Y.js.map +1 -0
- package/dist/{summary-report-C-CtItCN.d.ts → summary-report-bdFDXq0b.d.ts} +10 -5
- package/dist/summary-report-bdFDXq0b.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +153 -149
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +65 -3
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-CpTLAlqb.d.ts → systemone-BomhHr2c.d.ts} +6 -6
- package/dist/{systemone-CpTLAlqb.d.ts.map → systemone-BomhHr2c.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-B_yo1xsd.d.ts → systemone-review-CUVcQJu6.d.ts} +2 -2
- package/dist/{systemone-review-B_yo1xsd.d.ts.map → systemone-review-CUVcQJu6.d.ts.map} +1 -1
- package/dist/{systemone-review-BZWfrtq1.js → systemone-review-D7sXwsqq.js} +2 -2
- package/dist/{systemone-review-BZWfrtq1.js.map → systemone-review-D7sXwsqq.js.map} +1 -1
- package/dist/systemone.d.ts +3 -3
- package/dist/systemone.js +4 -4
- package/dist/{task-failure-attributes-DngaLdE4.js → task-failure-attributes-hEmASjix.js} +2 -2
- package/dist/{task-failure-attributes-DngaLdE4.js.map → task-failure-attributes-hEmASjix.js.map} +1 -1
- package/dist/{terminal-record-BHocwFw1.js → terminal-record-DZyQZ3pz.js} +14 -10
- package/dist/terminal-record-DZyQZ3pz.js.map +1 -0
- package/dist/{tool-groups-mcxOXVlb.d.ts → tool-groups-C3NEcdCM.d.ts} +3 -3
- package/dist/tool-groups-C3NEcdCM.d.ts.map +1 -0
- package/dist/{tool-waste-Wi4g6Ekb.d.ts → tool-waste-BKgvd-TP.d.ts} +30 -30
- package/dist/tool-waste-BKgvd-TP.d.ts.map +1 -0
- package/dist/{tool-waste-D52PqU9Q.js → tool-waste-BlesqTqW.js} +4 -4
- package/dist/{tool-waste-D52PqU9Q.js.map → tool-waste-BlesqTqW.js.map} +1 -1
- package/dist/{trace-questions-DG7nqLQ7.d.ts → trace-questions-CM2CAocU.d.ts} +4 -4
- package/dist/{trace-questions-DG7nqLQ7.d.ts.map → trace-questions-CM2CAocU.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +3 -3
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +7 -13
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +76 -58
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +9 -9
- package/dist/trajectory-replay/index.js +3 -3
- package/dist/{tree-rXLW3M_4.js → tree-C1XYeXvy.js} +2277 -2339
- package/dist/tree-C1XYeXvy.js.map +1 -0
- package/dist/{types-DBZ7keha.d.ts → types-C02qtnG8.d.ts} +4 -46
- package/dist/types-C02qtnG8.d.ts.map +1 -0
- package/dist/{types-oKRhQAde.d.ts → types-C1tnkEgM.d.ts} +4 -4
- package/dist/{types-oKRhQAde.d.ts.map → types-C1tnkEgM.d.ts.map} +1 -1
- package/dist/{types-CfZMIzB_.d.ts → types-Cbe2fWBa.d.ts} +3 -3
- package/dist/{types-CfZMIzB_.d.ts.map → types-Cbe2fWBa.d.ts.map} +1 -1
- package/dist/{types-B4C5nFw6.d.ts → types-Cq8mSbq8.d.ts} +78 -72
- package/dist/types-Cq8mSbq8.d.ts.map +1 -0
- package/dist/{verdict-DhPoXApk.js → verdict-Dl0tiT1X.js} +2 -2
- package/dist/{verdict-DhPoXApk.js.map → verdict-Dl0tiT1X.js.map} +1 -1
- package/dist/{verdict-cache-BQEsVy9n.js → verdict-cache-Bm8EKL2s.js} +2 -2
- package/dist/{verdict-cache-BQEsVy9n.js.map → verdict-cache-Bm8EKL2s.js.map} +1 -1
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/design/mlbenchmarks-review/probes.mts +23 -31
- package/docs/experiment.md +1 -1
- package/package.json +1 -1
- package/dist/analyze-runs-3xpby3p2.js.map +0 -1
- package/dist/analyze-runs-CSDAh8A7.d.ts.map +0 -1
- package/dist/ast-DJ6Bc7ab.js +0 -558
- package/dist/ast-DJ6Bc7ab.js.map +0 -1
- package/dist/ast-hI-vjW6J.d.ts +0 -457
- package/dist/ast-hI-vjW6J.d.ts.map +0 -1
- package/dist/benchmark-command-CByl9dxG.js.map +0 -1
- package/dist/campaign-BQ-Jujuj.js.map +0 -1
- package/dist/campaign-evidence-OcUciD-N.js.map +0 -1
- package/dist/claim-BXcZw-jo.js.map +0 -1
- package/dist/cost-ledger-CO7UyEcQ.d.ts.map +0 -1
- package/dist/cost-ledger-D2FnJPBF.js.map +0 -1
- package/dist/define-agent-eval-Ch3MWX6k.js.map +0 -1
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +0 -1
- package/dist/feedback-trajectory-DkaDj0dj.d.ts.map +0 -1
- package/dist/index-CPJJr98o.d.ts.map +0 -1
- package/dist/index-DlH0Y9gI.d.ts.map +0 -1
- package/dist/index-NvGBiO8H.d.ts.map +0 -1
- package/dist/integrity-BV1ZDf42.js.map +0 -1
- package/dist/judge-gate-B1qYnkpi.js.map +0 -1
- package/dist/llm-client-DQveoxSM.js.map +0 -1
- package/dist/llm-judge-PXV7Xljl.js.map +0 -1
- package/dist/matrix-_BV9-Lk5.js.map +0 -1
- package/dist/mint-DCGoBYhB.d.ts +0 -155
- package/dist/mint-DCGoBYhB.d.ts.map +0 -1
- package/dist/paired-tests-jbwPdSz5.js.map +0 -1
- package/dist/redact-Cveoq6fJ.d.ts.map +0 -1
- package/dist/redact-Dk1ZdJEm.js.map +0 -1
- package/dist/release-confidence-BK2QCB5M.d.ts.map +0 -1
- package/dist/report-command-y8H6RSX8.js.map +0 -1
- package/dist/run-record-ES07wfz9.js.map +0 -1
- package/dist/search-kernel-BztlERLv.js.map +0 -1
- package/dist/search-kernel-CpcKH5Ih.d.ts.map +0 -1
- package/dist/search-ledger-d7V8o9Lk.js.map +0 -1
- package/dist/semantic-concept-judge-zQBG9hHU.js.map +0 -1
- package/dist/skillopt-optimization-method-CCtzralq.js.map +0 -1
- package/dist/store-otlp-DtB4Slsr.js.map +0 -1
- package/dist/store-tool-spans-DR1SrCLK.d.ts.map +0 -1
- package/dist/summary-report-C-CtItCN.d.ts.map +0 -1
- package/dist/summary-report-DtdRucM0.js.map +0 -1
- package/dist/terminal-record-BHocwFw1.js.map +0 -1
- package/dist/tool-groups-mcxOXVlb.d.ts.map +0 -1
- package/dist/tool-waste-Wi4g6Ekb.d.ts.map +0 -1
- package/dist/tree-rXLW3M_4.js.map +0 -1
- package/dist/types-B4C5nFw6.d.ts.map +0 -1
- package/dist/types-DBZ7keha.d.ts.map +0 -1
package/dist/{external-optimizer-subprocess-DcerlZSG.js → external-optimizer-subprocess-BqW0hHcI.js}
RENAMED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { l as errorMessage } from "./errors-DVZiqS7K.js";
|
|
2
|
+
import { t as canonicalJson } from "./verdict-cache-Bm8EKL2s.js";
|
|
3
|
+
import { c as costForTokenPricing, t as CostLedger } from "./cost-ledger-BP8V4kUQ.js";
|
|
4
|
+
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
5
|
+
import { n as appendSearchLedgerLine, r as tryWithSearchLedgerFileLock } from "./search-ledger-file-DDQUgvCg.js";
|
|
4
6
|
import { n as ANALYST_SEVERITIES, r as FINDING_SCOPE_FIELDS } from "./types-IXD2HJzu.js";
|
|
5
|
-
import {
|
|
7
|
+
import { c as ModelSubstitutionError, d as assertServedModel, f as assertServedModelPolicy } from "./llm-client-CyBAoYiX.js";
|
|
8
|
+
import { t as isNonnegativeFiniteNumber } from "./nonnegative-number-DvdMxg_f.js";
|
|
6
9
|
import { createRequire } from "node:module";
|
|
7
10
|
import { randomBytes } from "node:crypto";
|
|
8
11
|
import { z } from "zod";
|
|
@@ -485,6 +488,67 @@ function isCredentialKey(key) {
|
|
|
485
488
|
function isRecord(value) {
|
|
486
489
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
487
490
|
}
|
|
491
|
+
function freezeJsonSnapshot(value, label) {
|
|
492
|
+
assertJsonValue(value, label);
|
|
493
|
+
return deepFreezeCanonicalJson(JSON.parse(canonicalJson(value)));
|
|
494
|
+
}
|
|
495
|
+
//#endregion
|
|
496
|
+
//#region src/campaign/external-optimizer-http.ts
|
|
497
|
+
function listenLocal(server) {
|
|
498
|
+
return new Promise((resolvePromise, reject) => {
|
|
499
|
+
server.once("error", reject);
|
|
500
|
+
server.listen(0, "127.0.0.1", () => {
|
|
501
|
+
server.off("error", reject);
|
|
502
|
+
const address = server.address();
|
|
503
|
+
if (!address || typeof address === "string") {
|
|
504
|
+
reject(/* @__PURE__ */ new Error("external optimizer callback did not bind a TCP port"));
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
resolvePromise(address.port);
|
|
508
|
+
});
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
function closeServer(server) {
|
|
512
|
+
return new Promise((resolvePromise, reject) => {
|
|
513
|
+
server.close((error) => error ? reject(error) : resolvePromise());
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
function sendJson(response, status, body) {
|
|
517
|
+
response.writeHead(status, { "content-type": "application/json; charset=utf-8" });
|
|
518
|
+
response.end(JSON.stringify(body));
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Send a JSON body only while the response can still take one.
|
|
522
|
+
*
|
|
523
|
+
* A handler that lost its client — the request was aborted, or the response
|
|
524
|
+
* already ended — must not write again; Node throws `ERR_STREAM_WRITE_AFTER_END`
|
|
525
|
+
* and the throw escapes into the server's error path rather than the caller's.
|
|
526
|
+
*/
|
|
527
|
+
function sendJsonIfOpen(response, status, body) {
|
|
528
|
+
if (response.destroyed || response.writableEnded) return;
|
|
529
|
+
sendJson(response, status, body);
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* Wait until every in-flight handler has settled.
|
|
533
|
+
*
|
|
534
|
+
* Re-reads the set on each pass: a handler that is still running can register
|
|
535
|
+
* another, so awaiting one snapshot would return while work is outstanding and
|
|
536
|
+
* the caller would close the server under it.
|
|
537
|
+
*/
|
|
538
|
+
async function waitForActiveHandlers(activeHandlers) {
|
|
539
|
+
while (activeHandlers.size > 0) await Promise.allSettled([...activeHandlers]);
|
|
540
|
+
}
|
|
541
|
+
//#endregion
|
|
542
|
+
//#region src/campaign/external-optimizer-model-contract.ts
|
|
543
|
+
var RequestBodyTooLargeError = class extends Error {};
|
|
544
|
+
var ProviderResponseTooLargeError = class extends Error {
|
|
545
|
+
constructor() {
|
|
546
|
+
super("optimizer model response exceeds maxResponseBytes");
|
|
547
|
+
}
|
|
548
|
+
};
|
|
549
|
+
var MissingModelExecutionError = class extends Error {};
|
|
550
|
+
var ModelExecutionPersistenceError = class extends Error {};
|
|
551
|
+
var OwnerModelContractError = class extends Error {};
|
|
488
552
|
//#endregion
|
|
489
553
|
//#region src/campaign/external-optimizer-anthropic.ts
|
|
490
554
|
/**
|
|
@@ -953,349 +1017,126 @@ function refuse(message) {
|
|
|
953
1017
|
return new AnthropicRequestRefusal(400, "invalid_request_error", message);
|
|
954
1018
|
}
|
|
955
1019
|
//#endregion
|
|
956
|
-
//#region src/campaign/external-optimizer-
|
|
957
|
-
function
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
return;
|
|
966
|
-
}
|
|
967
|
-
resolvePromise(address.port);
|
|
968
|
-
});
|
|
969
|
-
});
|
|
970
|
-
}
|
|
971
|
-
function closeServer(server) {
|
|
972
|
-
return new Promise((resolvePromise, reject) => {
|
|
973
|
-
server.close((error) => error ? reject(error) : resolvePromise());
|
|
974
|
-
});
|
|
1020
|
+
//#region src/campaign/external-optimizer-model-response.ts
|
|
1021
|
+
function encodeAnthropicResponseBody(args) {
|
|
1022
|
+
const message = encodeAnthropicMessage(args.response, args.callId);
|
|
1023
|
+
const body = new TextEncoder().encode(args.stream ? renderAnthropicSseStream(message) : JSON.stringify(message));
|
|
1024
|
+
if (body.byteLength > args.maxBytes) throw new ProviderResponseTooLargeError();
|
|
1025
|
+
return {
|
|
1026
|
+
contentType: args.stream ? "text/event-stream; charset=utf-8" : "application/json",
|
|
1027
|
+
body
|
|
1028
|
+
};
|
|
975
1029
|
}
|
|
976
|
-
function
|
|
977
|
-
|
|
978
|
-
response.
|
|
1030
|
+
function encodeCanonicalModelResponse(args) {
|
|
1031
|
+
const usage = encodeProtocolUsage(args.path, args.response, args.receipt);
|
|
1032
|
+
const finishReason = args.response.finishReason === void 0 ? "stop" : args.response.finishReason;
|
|
1033
|
+
const body = args.path === "/v1/chat/completions" ? {
|
|
1034
|
+
id: `chatcmpl-${args.callId}`,
|
|
1035
|
+
object: "chat.completion",
|
|
1036
|
+
created: 0,
|
|
1037
|
+
model: args.response.model,
|
|
1038
|
+
choices: [{
|
|
1039
|
+
index: 0,
|
|
1040
|
+
message: {
|
|
1041
|
+
role: "assistant",
|
|
1042
|
+
content: args.response.content
|
|
1043
|
+
},
|
|
1044
|
+
finish_reason: finishReason
|
|
1045
|
+
}],
|
|
1046
|
+
...usage ? { usage } : {}
|
|
1047
|
+
} : {
|
|
1048
|
+
id: `resp-${args.callId}`,
|
|
1049
|
+
object: "response",
|
|
1050
|
+
created_at: 0,
|
|
1051
|
+
model: args.response.model,
|
|
1052
|
+
status: finishReason === "length" ? "incomplete" : "completed",
|
|
1053
|
+
error: null,
|
|
1054
|
+
incomplete_details: finishReason === "length" ? { reason: "max_output_tokens" } : null,
|
|
1055
|
+
output: [{
|
|
1056
|
+
id: `msg-${args.callId}`,
|
|
1057
|
+
type: "message",
|
|
1058
|
+
status: finishReason === "length" ? "incomplete" : "completed",
|
|
1059
|
+
role: "assistant",
|
|
1060
|
+
content: [{
|
|
1061
|
+
type: "output_text",
|
|
1062
|
+
text: args.response.content,
|
|
1063
|
+
annotations: []
|
|
1064
|
+
}]
|
|
1065
|
+
}],
|
|
1066
|
+
output_text: args.response.content,
|
|
1067
|
+
...usage ? { usage } : {}
|
|
1068
|
+
};
|
|
1069
|
+
const encoded = new TextEncoder().encode(JSON.stringify(body));
|
|
1070
|
+
if (encoded.byteLength > args.maxBytes) throw new ProviderResponseTooLargeError();
|
|
1071
|
+
return encoded;
|
|
979
1072
|
}
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
* A handler that lost its client — the request was aborted, or the response
|
|
984
|
-
* already ended — must not write again; Node throws `ERR_STREAM_WRITE_AFTER_END`
|
|
985
|
-
* and the throw escapes into the server's error path rather than the caller's.
|
|
986
|
-
*/
|
|
987
|
-
function sendJsonIfOpen(response, status, body) {
|
|
988
|
-
if (response.destroyed || response.writableEnded) return;
|
|
989
|
-
sendJson(response, status, body);
|
|
1073
|
+
function endpointFormat(path) {
|
|
1074
|
+
if (path === "/v1/messages") return "anthropic-messages";
|
|
1075
|
+
return path === "/v1/responses" ? "responses" : "chat-completions";
|
|
990
1076
|
}
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1077
|
+
function encodeProtocolUsage(path, response, receipt) {
|
|
1078
|
+
if (response.usage.captured === false) return void 0;
|
|
1079
|
+
const inputDetails = {
|
|
1080
|
+
...receipt.cachedTokens === void 0 ? {} : { cached_tokens: receipt.cachedTokens },
|
|
1081
|
+
...receipt.cacheWriteTokens === void 0 ? {} : path === "/v1/chat/completions" ? { cache_creation_tokens: receipt.cacheWriteTokens } : { cache_write_tokens: receipt.cacheWriteTokens }
|
|
1082
|
+
};
|
|
1083
|
+
const outputDetails = { ...receipt.reasoningTokens === void 0 ? {} : { reasoning_tokens: receipt.reasoningTokens } };
|
|
1084
|
+
return path === "/v1/chat/completions" ? {
|
|
1085
|
+
prompt_tokens: response.usage.promptTokens,
|
|
1086
|
+
completion_tokens: response.usage.completionTokens,
|
|
1087
|
+
total_tokens: response.usage.totalTokens,
|
|
1088
|
+
...Object.keys(inputDetails).length > 0 ? { prompt_tokens_details: inputDetails } : {},
|
|
1089
|
+
...Object.keys(outputDetails).length > 0 ? { completion_tokens_details: outputDetails } : {},
|
|
1090
|
+
...receipt.actualCostUsd === void 0 ? {} : { cost: receipt.actualCostUsd }
|
|
1091
|
+
} : {
|
|
1092
|
+
input_tokens: response.usage.promptTokens,
|
|
1093
|
+
output_tokens: response.usage.completionTokens,
|
|
1094
|
+
total_tokens: response.usage.totalTokens,
|
|
1095
|
+
...Object.keys(inputDetails).length > 0 ? { input_tokens_details: inputDetails } : {},
|
|
1096
|
+
...Object.keys(outputDetails).length > 0 ? { output_tokens_details: outputDetails } : {},
|
|
1097
|
+
...receipt.actualCostUsd === void 0 ? {} : { cost: receipt.actualCostUsd }
|
|
1098
|
+
};
|
|
1000
1099
|
}
|
|
1001
1100
|
//#endregion
|
|
1002
|
-
//#region src/campaign/external-optimizer-model-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
let
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
if (!accepting) {
|
|
1042
|
-
sendJsonIfOpen(response, 503, { error: "external optimizer model proxy is closing" });
|
|
1043
|
-
return;
|
|
1044
|
-
}
|
|
1045
|
-
const controller = new AbortController();
|
|
1046
|
-
const abortRequest = () => {
|
|
1047
|
-
request.destroy();
|
|
1048
|
-
response.destroy();
|
|
1049
|
-
};
|
|
1050
|
-
activeControllers.add(controller);
|
|
1051
|
-
controller.signal.addEventListener("abort", abortRequest, { once: true });
|
|
1052
|
-
let handler;
|
|
1053
|
-
handler = handleModelProxyRequest({
|
|
1054
|
-
request,
|
|
1055
|
-
response,
|
|
1056
|
-
controller,
|
|
1057
|
-
token,
|
|
1058
|
-
args,
|
|
1059
|
-
nextReservation: (maximumCostUsd, wire) => {
|
|
1060
|
-
if (totalRequestCount >= args.budget.maxRequests) return {
|
|
1061
|
-
accepted: false,
|
|
1062
|
-
reason: "optimizer model request limit reached"
|
|
1063
|
-
};
|
|
1064
|
-
if (args.budget.maxCostUsd !== void 0 && maximumCostUsd !== void 0 && committedForBudget + reservedForBudget + maximumCostUsd > args.budget.maxCostUsd + Number.EPSILON) return {
|
|
1065
|
-
accepted: false,
|
|
1066
|
-
reason: "optimizer model cost limit reached"
|
|
1067
|
-
};
|
|
1068
|
-
requestCount += 1;
|
|
1069
|
-
wireCounts[wire].requestAttempts += 1;
|
|
1070
|
-
totalRequestCount += 1;
|
|
1071
|
-
reservedForBudget += maximumCostUsd ?? 0;
|
|
1072
|
-
return { accepted: true };
|
|
1073
|
-
},
|
|
1074
|
-
settleReservation: (maximumCostUsd, chargedCostUsd) => {
|
|
1075
|
-
reservedForBudget = Math.max(0, reservedForBudget - (maximumCostUsd ?? 0));
|
|
1076
|
-
committedForBudget += chargedCostUsd ?? 0;
|
|
1077
|
-
},
|
|
1078
|
-
recordSuccessfulCompletion: (wire) => {
|
|
1079
|
-
successfulCompletionCount += 1;
|
|
1080
|
-
wireCounts[wire].successfulCompletions += 1;
|
|
1081
|
-
},
|
|
1082
|
-
recordExecutionReceipt: (observation) => {
|
|
1083
|
-
try {
|
|
1084
|
-
args.recordExecution({
|
|
1085
|
-
...observation,
|
|
1086
|
-
sequence: executionSequence + 1
|
|
1087
|
-
});
|
|
1088
|
-
} catch (error) {
|
|
1089
|
-
throw new ModelExecutionPersistenceError(`optimizer model execution evidence was not persisted: ${toErrorMessage(error)}`);
|
|
1090
|
-
}
|
|
1091
|
-
executionSequence += 1;
|
|
1092
|
-
executionRecordCount += 1;
|
|
1093
|
-
},
|
|
1094
|
-
recordModelCallInvocation: () => {
|
|
1095
|
-
modelCallInvocations += 1;
|
|
1096
|
-
},
|
|
1097
|
-
recordFailure: (error) => {
|
|
1098
|
-
failures.push(error);
|
|
1099
|
-
}
|
|
1100
|
-
}).finally(() => {
|
|
1101
|
-
controller.signal.removeEventListener("abort", abortRequest);
|
|
1102
|
-
activeControllers.delete(controller);
|
|
1103
|
-
activeHandlers.delete(handler);
|
|
1104
|
-
});
|
|
1105
|
-
activeHandlers.add(handler);
|
|
1106
|
-
handler.catch(() => void 0);
|
|
1107
|
-
});
|
|
1108
|
-
const port = await listenLocal(server);
|
|
1109
|
-
const close = () => {
|
|
1110
|
-
closePromise ??= closeModelProxy();
|
|
1111
|
-
return closePromise;
|
|
1112
|
-
};
|
|
1113
|
-
const onAbort = () => {
|
|
1114
|
-
for (const controller of activeControllers) controller.abort(args.signal?.reason);
|
|
1115
|
-
close().catch(() => void 0);
|
|
1116
|
-
};
|
|
1117
|
-
args.signal?.addEventListener("abort", onAbort, { once: true });
|
|
1118
|
-
if (args.signal?.aborted) onAbort();
|
|
1119
|
-
const assertExecutionComplete = () => {
|
|
1120
|
-
if (executionRecordCount !== modelCallInvocations) throw new Error(`external optimizer model callback returned ${executionRecordCount} execution records for ${modelCallInvocations} invoked calls`);
|
|
1121
|
-
};
|
|
1122
|
-
return {
|
|
1123
|
-
baseUrl: `http://127.0.0.1:${port}/v1`,
|
|
1124
|
-
apiKey: token,
|
|
1125
|
-
requestAttempts: () => requestCount,
|
|
1126
|
-
successfulCompletions: () => successfulCompletionCount,
|
|
1127
|
-
wireUsage: () => ({
|
|
1128
|
-
openai: { ...wireCounts.openai },
|
|
1129
|
-
anthropic: { ...wireCounts.anthropic }
|
|
1130
|
-
}),
|
|
1131
|
-
assertExecutionComplete,
|
|
1132
|
-
failures: () => [...failures],
|
|
1133
|
-
close
|
|
1134
|
-
};
|
|
1135
|
-
async function closeModelProxy() {
|
|
1136
|
-
args.signal?.removeEventListener("abort", onAbort);
|
|
1137
|
-
accepting = false;
|
|
1138
|
-
const closingServer = closeServer(server);
|
|
1139
|
-
server.closeIdleConnections?.();
|
|
1140
|
-
const [serverResult] = await Promise.allSettled([closingServer, waitForActiveHandlers(activeHandlers)]);
|
|
1141
|
-
if (activeControllers.size !== 0 || activeHandlers.size !== 0) throw new Error("external optimizer model proxy closed with active request work");
|
|
1142
|
-
let executionError;
|
|
1143
|
-
try {
|
|
1144
|
-
assertExecutionComplete();
|
|
1145
|
-
} catch (error) {
|
|
1146
|
-
executionError = error;
|
|
1147
|
-
}
|
|
1148
|
-
if (serverResult?.status === "rejected" && executionError !== void 0) throw new AggregateError([serverResult.reason, executionError], "external optimizer model proxy close and execution evidence both failed");
|
|
1149
|
-
if (serverResult?.status === "rejected") throw serverResult.reason;
|
|
1150
|
-
if (executionError !== void 0) throw executionError;
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
async function handleModelProxyRequest(args) {
|
|
1154
|
-
const { controller, request, response } = args;
|
|
1155
|
-
let wire = "openai";
|
|
1156
|
-
const respondError = (status, message, errorType) => {
|
|
1157
|
-
if (wire === "anthropic") {
|
|
1158
|
-
const mapped = status === 429 ? 402 : status;
|
|
1159
|
-
sendJsonIfOpen(response, mapped, anthropicErrorEnvelope(errorType ?? anthropicErrorTypeForStatus(mapped), message));
|
|
1160
|
-
return;
|
|
1161
|
-
}
|
|
1162
|
-
sendJsonIfOpen(response, status, { error: message });
|
|
1163
|
-
};
|
|
1164
|
-
try {
|
|
1165
|
-
const path = request.url ? new URL(request.url, "http://127.0.0.1").pathname : "";
|
|
1166
|
-
const anthropicPath = path === "/v1/messages" && args.args.anthropicEndpoint === true;
|
|
1167
|
-
if (request.method !== "POST" || !(MODEL_PROXY_PATHS.has(path) || anthropicPath)) {
|
|
1168
|
-
sendJsonIfOpen(response, 404, { error: "not found" });
|
|
1169
|
-
return;
|
|
1170
|
-
}
|
|
1171
|
-
if (anthropicPath) wire = "anthropic";
|
|
1172
|
-
const modelPath = path;
|
|
1173
|
-
if (!(request.headers.authorization === `Bearer ${args.token}` || wire === "anthropic" && request.headers["x-api-key"] === args.token)) {
|
|
1174
|
-
respondError(401, "unauthorized");
|
|
1175
|
-
return;
|
|
1176
|
-
}
|
|
1177
|
-
const body = await readBody(request, args.args.budget.maxRequestBytes);
|
|
1178
|
-
const parsed = parseModelProxyRequest(body, modelPath, args.args.model, args.args.budget);
|
|
1179
|
-
const maximumUsage = conservativeMaximumUsage(body.byteLength, parsed.maxOutputTokens + (args.args.budget.maxReasoningTokensPerRequest ?? 0), args.args.budget.pricing);
|
|
1180
|
-
const maximumCostUsd = args.args.budget.pricing ? costForTokenPricing(args.args.budget.pricing, maximumUsage) : void 0;
|
|
1181
|
-
const reservation = args.nextReservation(maximumCostUsd, wire);
|
|
1182
|
-
if (!reservation.accepted) {
|
|
1183
|
-
respondError(429, reservation.reason);
|
|
1184
|
-
return;
|
|
1185
|
-
}
|
|
1186
|
-
const timeout = setTimeout(() => controller.abort(), args.args.budget.requestTimeoutMs ?? 3e5);
|
|
1187
|
-
let chargedForBudget = maximumCostUsd;
|
|
1188
|
-
try {
|
|
1189
|
-
const requestTags = wire === "anthropic" ? {
|
|
1190
|
-
...args.args.tags ?? {},
|
|
1191
|
-
wire: "anthropic",
|
|
1192
|
-
...parsed.strippedFields === void 0 ? {} : { strippedFields: parsed.strippedFields.join(",") }
|
|
1193
|
-
} : args.args.tags;
|
|
1194
|
-
const paid = await args.args.costLedger.runPaidCall({
|
|
1195
|
-
...args.args.callId ? { callId: args.args.callId } : {},
|
|
1196
|
-
channel: args.args.channel ?? "optimizer",
|
|
1197
|
-
phase: args.args.phase,
|
|
1198
|
-
actor: args.args.actor,
|
|
1199
|
-
...requestTags ? { tags: requestTags } : {},
|
|
1200
|
-
model: args.args.model,
|
|
1201
|
-
...args.args.budget.pricing ? { maximumCharge: {
|
|
1202
|
-
customTokenPricing: args.args.budget.pricing,
|
|
1203
|
-
...maximumUsage
|
|
1204
|
-
} } : {},
|
|
1205
|
-
signal: controller.signal,
|
|
1206
|
-
execute: async (paidCallSignal, paidCallId) => forwardModelProxyRequest({
|
|
1207
|
-
call: args.args.call,
|
|
1208
|
-
callId: paidCallId,
|
|
1209
|
-
callRef: args.args.callRef,
|
|
1210
|
-
recordExecutionReceipt: args.recordExecutionReceipt,
|
|
1211
|
-
recordModelCallInvocation: args.recordModelCallInvocation,
|
|
1212
|
-
path: modelPath,
|
|
1213
|
-
request: parsed.request,
|
|
1214
|
-
...parsed.stream === void 0 ? {} : { anthropicStream: parsed.stream },
|
|
1215
|
-
model: args.args.model,
|
|
1216
|
-
allowWithinFamily: args.args.servedModelPolicy === "allow-within-family",
|
|
1217
|
-
maxOutputTokens: parsed.maxOutputTokens,
|
|
1218
|
-
...args.args.budget.maxReasoningTokensPerRequest === void 0 ? {} : { maxReasoningTokens: args.args.budget.maxReasoningTokensPerRequest },
|
|
1219
|
-
maxResponseBytes: args.args.budget.maxResponseBytes,
|
|
1220
|
-
signal: paidCallSignal
|
|
1221
|
-
}),
|
|
1222
|
-
receipt: (result) => result.receipt,
|
|
1223
|
-
receiptFromError: () => ({
|
|
1224
|
-
model: args.args.model,
|
|
1225
|
-
inputTokens: 0,
|
|
1226
|
-
outputTokens: 0,
|
|
1227
|
-
costUnknown: true,
|
|
1228
|
-
usageUnknown: true
|
|
1229
|
-
})
|
|
1230
|
-
});
|
|
1231
|
-
if (!paid.succeeded) {
|
|
1232
|
-
args.recordFailure(paid.error);
|
|
1233
|
-
chargedForBudget = paid.receipt ? paid.receipt.usageUnknown || paid.receipt.costUnknown ? maximumCostUsd : paid.receipt.costUsd : void 0;
|
|
1234
|
-
respondError(isAbortError(paid.error) ? 504 : paid.error instanceof AnthropicRequestRefusal ? paid.error.status : paid.error instanceof ProviderResponseTooLargeError || paid.error instanceof MissingModelExecutionError || paid.error instanceof ModelExecutionPersistenceError || paid.error instanceof OwnerModelContractError || paid.error instanceof ModelSubstitutionError ? 502 : 429, paid.error.message, paid.error instanceof AnthropicRequestRefusal ? paid.error.errorType : void 0);
|
|
1235
|
-
return;
|
|
1236
|
-
}
|
|
1237
|
-
if (paid.value.modelCallFailed) {
|
|
1238
|
-
respondError(502, paid.value.modelCallFailed);
|
|
1239
|
-
return;
|
|
1240
|
-
}
|
|
1241
|
-
chargedForBudget = paid.value.usageComplete ? paid.receipt.costUsd : maximumCostUsd;
|
|
1242
|
-
if (paid.value.usageRejected) {
|
|
1243
|
-
respondError(502, paid.value.usageRejected);
|
|
1244
|
-
return;
|
|
1245
|
-
}
|
|
1246
|
-
if (paid.value.status >= 200 && paid.value.status < 300) args.recordSuccessfulCompletion(wire);
|
|
1247
|
-
if (response.destroyed || response.writableEnded) return;
|
|
1248
|
-
response.writeHead(paid.value.status, {
|
|
1249
|
-
"content-type": paid.value.contentType,
|
|
1250
|
-
"content-length": String(paid.value.body.byteLength)
|
|
1251
|
-
});
|
|
1252
|
-
response.end(paid.value.body);
|
|
1253
|
-
} finally {
|
|
1254
|
-
clearTimeout(timeout);
|
|
1255
|
-
args.settleReservation(maximumCostUsd, chargedForBudget);
|
|
1256
|
-
}
|
|
1257
|
-
} catch (error) {
|
|
1258
|
-
respondError(error instanceof AnthropicRequestRefusal ? error.status : error instanceof RequestBodyTooLargeError ? 413 : error instanceof Error && isAbortError(error) ? 503 : 400, toErrorMessage(error), error instanceof AnthropicRequestRefusal ? error.errorType : void 0);
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
|
-
async function forwardModelProxyRequest(args) {
|
|
1262
|
-
args.recordModelCallInvocation();
|
|
1263
|
-
let called;
|
|
1264
|
-
try {
|
|
1265
|
-
called = await args.call({
|
|
1266
|
-
callId: args.callId,
|
|
1267
|
-
request: freezeJsonSnapshot(args.request, "optimizer model canonical request"),
|
|
1268
|
-
endpointFormat: endpointFormat(args.path),
|
|
1269
|
-
signal: args.signal
|
|
1270
|
-
});
|
|
1271
|
-
} catch (error) {
|
|
1272
|
-
throw new MissingModelExecutionError(`optimizer model callback rejected without execution evidence: ${toErrorMessage(error)}`);
|
|
1273
|
-
}
|
|
1274
|
-
if (!called || typeof called !== "object" || typeof called.succeeded !== "boolean") throw new MissingModelExecutionError("optimizer model callback returned no typed success/failure outcome");
|
|
1275
|
-
let execution;
|
|
1276
|
-
try {
|
|
1277
|
-
execution = freezeJsonSnapshot(called.execution, "optimizer model callback execution evidence");
|
|
1278
|
-
} catch (error) {
|
|
1279
|
-
throw new MissingModelExecutionError(`optimizer model callback returned invalid execution evidence: ${toErrorMessage(error)}`);
|
|
1280
|
-
}
|
|
1281
|
-
if (called.succeeded) {
|
|
1282
|
-
let authoritativeReceipt;
|
|
1283
|
-
let canonicalResponse;
|
|
1284
|
-
try {
|
|
1285
|
-
authoritativeReceipt = snapshotModelReceipt(called.receipt, args.model, args.allowWithinFamily);
|
|
1286
|
-
canonicalResponse = snapshotChatResponse(called.response, args.model, args.allowWithinFamily);
|
|
1287
|
-
assertResponseUsageMatchesReceipt(canonicalResponse, authoritativeReceipt);
|
|
1288
|
-
} catch (error) {
|
|
1289
|
-
args.recordExecutionReceipt({
|
|
1290
|
-
callId: args.callId,
|
|
1291
|
-
callRef: args.callRef,
|
|
1292
|
-
path: args.path,
|
|
1293
|
-
model: args.model,
|
|
1294
|
-
succeeded: false,
|
|
1295
|
-
error: toErrorMessage(error),
|
|
1296
|
-
execution
|
|
1297
|
-
});
|
|
1298
|
-
throw error;
|
|
1101
|
+
//#region src/campaign/external-optimizer-model-execution.ts
|
|
1102
|
+
async function forwardModelProxyRequest(args) {
|
|
1103
|
+
args.recordModelCallInvocation();
|
|
1104
|
+
let called;
|
|
1105
|
+
try {
|
|
1106
|
+
called = await args.call({
|
|
1107
|
+
callId: args.callId,
|
|
1108
|
+
request: freezeJsonSnapshot(args.request, "optimizer model canonical request"),
|
|
1109
|
+
endpointFormat: endpointFormat(args.path),
|
|
1110
|
+
signal: args.signal
|
|
1111
|
+
});
|
|
1112
|
+
} catch (error) {
|
|
1113
|
+
throw new MissingModelExecutionError(`optimizer model callback rejected without execution evidence: ${errorMessage(error)}`);
|
|
1114
|
+
}
|
|
1115
|
+
if (!called || typeof called !== "object" || typeof called.succeeded !== "boolean") throw new MissingModelExecutionError("optimizer model callback returned no typed success/failure outcome");
|
|
1116
|
+
let execution;
|
|
1117
|
+
try {
|
|
1118
|
+
execution = freezeJsonSnapshot(called.execution, "optimizer model callback execution evidence");
|
|
1119
|
+
} catch (error) {
|
|
1120
|
+
throw new MissingModelExecutionError(`optimizer model callback returned invalid execution evidence: ${errorMessage(error)}`);
|
|
1121
|
+
}
|
|
1122
|
+
if (called.succeeded) {
|
|
1123
|
+
let authoritativeReceipt;
|
|
1124
|
+
let canonicalResponse;
|
|
1125
|
+
try {
|
|
1126
|
+
authoritativeReceipt = snapshotModelReceipt(called.receipt, args.model, args.allowWithinFamily);
|
|
1127
|
+
canonicalResponse = snapshotChatResponse(called.response, args.model, args.allowWithinFamily);
|
|
1128
|
+
assertResponseUsageMatchesReceipt(canonicalResponse, authoritativeReceipt);
|
|
1129
|
+
} catch (error) {
|
|
1130
|
+
args.recordExecutionReceipt({
|
|
1131
|
+
callId: args.callId,
|
|
1132
|
+
callRef: args.callRef,
|
|
1133
|
+
path: args.path,
|
|
1134
|
+
model: args.model,
|
|
1135
|
+
succeeded: false,
|
|
1136
|
+
error: errorMessage(error),
|
|
1137
|
+
execution
|
|
1138
|
+
});
|
|
1139
|
+
throw error;
|
|
1299
1140
|
}
|
|
1300
1141
|
args.recordExecutionReceipt({
|
|
1301
1142
|
callId: args.callId,
|
|
@@ -1361,7 +1202,7 @@ function snapshotModelReceipt(value, expectedModel, allowWithinFamily) {
|
|
|
1361
1202
|
assertJsonValue(value, "optimizer model callback receipt");
|
|
1362
1203
|
snapshot = JSON.parse(canonicalJson(value));
|
|
1363
1204
|
} catch (error) {
|
|
1364
|
-
throw new OwnerModelContractError(`optimizer model callback returned an invalid receipt: ${
|
|
1205
|
+
throw new OwnerModelContractError(`optimizer model callback returned an invalid receipt: ${errorMessage(error)}`);
|
|
1365
1206
|
}
|
|
1366
1207
|
assertServedModel(expectedModel, snapshot.model, {
|
|
1367
1208
|
context: "optimizer model callback receipt",
|
|
@@ -1369,17 +1210,6 @@ function snapshotModelReceipt(value, expectedModel, allowWithinFamily) {
|
|
|
1369
1210
|
});
|
|
1370
1211
|
return snapshot;
|
|
1371
1212
|
}
|
|
1372
|
-
function freezeJsonSnapshot(value, label) {
|
|
1373
|
-
assertJsonValue(value, label);
|
|
1374
|
-
return deepFreezeJson(JSON.parse(canonicalJson(value)));
|
|
1375
|
-
}
|
|
1376
|
-
function deepFreezeJson(value) {
|
|
1377
|
-
if (value !== null && typeof value === "object") {
|
|
1378
|
-
for (const child of Object.values(value)) deepFreezeJson(child);
|
|
1379
|
-
Object.freeze(value);
|
|
1380
|
-
}
|
|
1381
|
-
return value;
|
|
1382
|
-
}
|
|
1383
1213
|
function assertResponseUsageMatchesReceipt(response, receipt) {
|
|
1384
1214
|
if (response.model !== receipt.model) throw new OwnerModelContractError("optimizer model response and execution receipt disagree about the served model");
|
|
1385
1215
|
if (response.usage.captured === false || receipt.usageUnknown === true) {
|
|
@@ -1394,10 +1224,80 @@ function assertResponseUsageMatchesReceipt(response, receipt) {
|
|
|
1394
1224
|
const receiptCostUsd = receipt.actualCostUsd ?? receipt.estimatedCostUsd ?? (receipt.customTokenPricing && receipt.usageUnknown !== true ? costForTokenPricing(receipt.customTokenPricing, receipt) : null);
|
|
1395
1225
|
if (response.costUsd !== receiptCostUsd) throw new OwnerModelContractError("optimizer model response billed cost disagrees with the execution receipt");
|
|
1396
1226
|
}
|
|
1397
|
-
function
|
|
1398
|
-
if (
|
|
1399
|
-
|
|
1400
|
-
|
|
1227
|
+
function snapshotChatResponse(value, expectedModel, allowWithinFamily) {
|
|
1228
|
+
if (!isRecord(value)) throw new OwnerModelContractError("optimizer model callback response must be an object");
|
|
1229
|
+
if (typeof value.content !== "string") throw new OwnerModelContractError("optimizer model callback response content must be a string");
|
|
1230
|
+
assertServedModel(expectedModel, typeof value.model === "string" ? value.model : null, {
|
|
1231
|
+
context: "optimizer model callback response",
|
|
1232
|
+
allowWithinFamily
|
|
1233
|
+
});
|
|
1234
|
+
if (!isNonnegativeFiniteNumber(value.durationMs)) throw new OwnerModelContractError("optimizer model callback response durationMs must be finite and non-negative");
|
|
1235
|
+
if (value.costUsd !== null && !isNonnegativeFiniteNumber(value.costUsd)) throw new OwnerModelContractError("optimizer model callback response costUsd must be null or finite and non-negative");
|
|
1236
|
+
if (value.finishReason !== void 0 && value.finishReason !== null && typeof value.finishReason !== "string") throw new OwnerModelContractError("optimizer model callback response finishReason must be a string or null");
|
|
1237
|
+
const toolCalls = snapshotResponseToolCalls(value.toolCalls);
|
|
1238
|
+
if (value.contentEmpty !== void 0 && (typeof value.contentEmpty !== "boolean" || value.contentEmpty !== (value.content.trim().length === 0))) throw new OwnerModelContractError("optimizer model callback response contentEmpty must match content");
|
|
1239
|
+
if (!isRecord(value.raw)) throw new OwnerModelContractError("optimizer model callback response raw must be an object");
|
|
1240
|
+
if (!isRecord(value.usage)) throw new OwnerModelContractError("optimizer model callback response usage must be an object");
|
|
1241
|
+
const usage = value.usage;
|
|
1242
|
+
for (const field of [
|
|
1243
|
+
"promptTokens",
|
|
1244
|
+
"completionTokens",
|
|
1245
|
+
"totalTokens"
|
|
1246
|
+
]) if (!isNonnegativeSafeInteger(usage[field])) throw new OwnerModelContractError(`optimizer model callback response usage.${field} must be a non-negative safe integer`);
|
|
1247
|
+
const promptTokens = usage.promptTokens;
|
|
1248
|
+
const completionTokens = usage.completionTokens;
|
|
1249
|
+
const totalTokens = usage.totalTokens;
|
|
1250
|
+
if (totalTokens !== promptTokens + completionTokens) throw new OwnerModelContractError("optimizer model callback response totalTokens must equal promptTokens plus completionTokens");
|
|
1251
|
+
if (usage.captured !== void 0 && typeof usage.captured !== "boolean") throw new OwnerModelContractError("optimizer model callback response usage.captured must be a boolean");
|
|
1252
|
+
if (usage.cachedPromptTokens !== void 0 && (!isNonnegativeSafeInteger(usage.cachedPromptTokens) || usage.cachedPromptTokens > promptTokens)) throw new OwnerModelContractError("optimizer model callback response cachedPromptTokens must be a subset of promptTokens");
|
|
1253
|
+
if (usage.reasoningTokens !== void 0 && (!isNonnegativeSafeInteger(usage.reasoningTokens) || usage.reasoningTokens > completionTokens)) throw new OwnerModelContractError("optimizer model callback response reasoningTokens must be a subset of completionTokens");
|
|
1254
|
+
let raw;
|
|
1255
|
+
try {
|
|
1256
|
+
raw = freezeJsonSnapshot(value.raw, "optimizer model callback canonical response.raw");
|
|
1257
|
+
} catch (error) {
|
|
1258
|
+
throw new OwnerModelContractError(`optimizer model callback returned invalid raw response evidence: ${errorMessage(error)}`);
|
|
1259
|
+
}
|
|
1260
|
+
return deepFreezeCanonicalJson({
|
|
1261
|
+
content: value.content,
|
|
1262
|
+
...toolCalls === void 0 ? {} : { toolCalls },
|
|
1263
|
+
usage: {
|
|
1264
|
+
promptTokens,
|
|
1265
|
+
completionTokens,
|
|
1266
|
+
totalTokens,
|
|
1267
|
+
...usage.captured === void 0 ? {} : { captured: usage.captured },
|
|
1268
|
+
...usage.reasoningTokens === void 0 ? {} : { reasoningTokens: usage.reasoningTokens },
|
|
1269
|
+
...usage.cachedPromptTokens === void 0 ? {} : { cachedPromptTokens: usage.cachedPromptTokens }
|
|
1270
|
+
},
|
|
1271
|
+
costUsd: value.costUsd,
|
|
1272
|
+
model: value.model,
|
|
1273
|
+
durationMs: value.durationMs,
|
|
1274
|
+
...value.finishReason === void 0 ? {} : { finishReason: value.finishReason },
|
|
1275
|
+
...value.contentEmpty === void 0 ? {} : { contentEmpty: value.contentEmpty },
|
|
1276
|
+
raw
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
/** Snapshot the owner's canonical tool calls onto the allowlisted response. */
|
|
1280
|
+
function snapshotResponseToolCalls(value) {
|
|
1281
|
+
if (value === void 0) return void 0;
|
|
1282
|
+
if (!Array.isArray(value) || value.length === 0) throw new OwnerModelContractError("optimizer model callback response toolCalls must be a non-empty array when present");
|
|
1283
|
+
return value.map((entry) => {
|
|
1284
|
+
if (!isRecord(entry) || typeof entry.id !== "string" || !entry.id || typeof entry.name !== "string" || !entry.name || typeof entry.argumentsJson !== "string") throw new OwnerModelContractError("optimizer model callback response toolCalls entries require string id, name, and argumentsJson");
|
|
1285
|
+
return {
|
|
1286
|
+
id: entry.id,
|
|
1287
|
+
name: entry.name,
|
|
1288
|
+
argumentsJson: entry.argumentsJson
|
|
1289
|
+
};
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
function isNonnegativeSafeInteger(value) {
|
|
1293
|
+
return Number.isSafeInteger(value) && value >= 0;
|
|
1294
|
+
}
|
|
1295
|
+
//#endregion
|
|
1296
|
+
//#region src/campaign/external-optimizer-model-request.ts
|
|
1297
|
+
function parseModelProxyRequest(body, path, expectedModel, budget) {
|
|
1298
|
+
if (path === "/v1/messages") {
|
|
1299
|
+
const translated = translateAnthropicMessagesRequest(body, expectedModel, budget.maxOutputTokensPerRequest);
|
|
1300
|
+
return {
|
|
1401
1301
|
maxOutputTokens: translated.maxOutputTokens,
|
|
1402
1302
|
stream: translated.stream,
|
|
1403
1303
|
...translated.strippedFields.length > 0 ? { strippedFields: translated.strippedFields } : {},
|
|
@@ -1653,169 +1553,309 @@ function assertAllowedKeys(value, allowed, label = "optimizer model request") {
|
|
|
1653
1553
|
const unknown = Object.keys(value).filter((key) => !allowedSet.has(key));
|
|
1654
1554
|
if (unknown.length > 0) throw new Error(`${label} contains unsupported fields: ${unknown.join(", ")}`);
|
|
1655
1555
|
}
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
const usage = value.usage;
|
|
1671
|
-
for (const field of [
|
|
1672
|
-
"promptTokens",
|
|
1673
|
-
"completionTokens",
|
|
1674
|
-
"totalTokens"
|
|
1675
|
-
]) if (!isNonnegativeSafeInteger(usage[field])) throw new OwnerModelContractError(`optimizer model callback response usage.${field} must be a non-negative safe integer`);
|
|
1676
|
-
const promptTokens = usage.promptTokens;
|
|
1677
|
-
const completionTokens = usage.completionTokens;
|
|
1678
|
-
const totalTokens = usage.totalTokens;
|
|
1679
|
-
if (totalTokens !== promptTokens + completionTokens) throw new OwnerModelContractError("optimizer model callback response totalTokens must equal promptTokens plus completionTokens");
|
|
1680
|
-
if (usage.captured !== void 0 && typeof usage.captured !== "boolean") throw new OwnerModelContractError("optimizer model callback response usage.captured must be a boolean");
|
|
1681
|
-
if (usage.cachedPromptTokens !== void 0 && (!isNonnegativeSafeInteger(usage.cachedPromptTokens) || usage.cachedPromptTokens > promptTokens)) throw new OwnerModelContractError("optimizer model callback response cachedPromptTokens must be a subset of promptTokens");
|
|
1682
|
-
if (usage.reasoningTokens !== void 0 && (!isNonnegativeSafeInteger(usage.reasoningTokens) || usage.reasoningTokens > completionTokens)) throw new OwnerModelContractError("optimizer model callback response reasoningTokens must be a subset of completionTokens");
|
|
1683
|
-
let raw;
|
|
1556
|
+
//#endregion
|
|
1557
|
+
//#region src/campaign/external-optimizer-model-handler.ts
|
|
1558
|
+
const MODEL_PROXY_PATHS = /* @__PURE__ */ new Set(["/v1/chat/completions", "/v1/responses"]);
|
|
1559
|
+
async function handleModelProxyRequest(args) {
|
|
1560
|
+
const { controller, request, response } = args;
|
|
1561
|
+
let wire = "openai";
|
|
1562
|
+
const respondError = (status, message, errorType) => {
|
|
1563
|
+
if (wire === "anthropic") {
|
|
1564
|
+
const mapped = status === 429 ? 402 : status;
|
|
1565
|
+
sendJsonIfOpen(response, mapped, anthropicErrorEnvelope(errorType ?? anthropicErrorTypeForStatus(mapped), message));
|
|
1566
|
+
return;
|
|
1567
|
+
}
|
|
1568
|
+
sendJsonIfOpen(response, status, { error: message });
|
|
1569
|
+
};
|
|
1684
1570
|
try {
|
|
1685
|
-
|
|
1571
|
+
const path = request.url ? new URL(request.url, "http://127.0.0.1").pathname : "";
|
|
1572
|
+
const anthropicPath = path === "/v1/messages" && args.args.anthropicEndpoint === true;
|
|
1573
|
+
if (request.method !== "POST" || !(MODEL_PROXY_PATHS.has(path) || anthropicPath)) {
|
|
1574
|
+
sendJsonIfOpen(response, 404, { error: "not found" });
|
|
1575
|
+
return;
|
|
1576
|
+
}
|
|
1577
|
+
if (anthropicPath) wire = "anthropic";
|
|
1578
|
+
const modelPath = path;
|
|
1579
|
+
if (!(request.headers.authorization === `Bearer ${args.token}` || wire === "anthropic" && request.headers["x-api-key"] === args.token)) {
|
|
1580
|
+
respondError(401, "unauthorized");
|
|
1581
|
+
return;
|
|
1582
|
+
}
|
|
1583
|
+
const body = await readBody(request, args.args.budget.maxRequestBytes);
|
|
1584
|
+
const parsed = parseModelProxyRequest(body, modelPath, args.args.model, args.args.budget);
|
|
1585
|
+
const maximumUsage = conservativeMaximumUsage(body.byteLength, parsed.maxOutputTokens + (args.args.budget.maxReasoningTokensPerRequest ?? 0), args.args.budget.pricing);
|
|
1586
|
+
const maximumCostUsd = args.args.budget.pricing ? costForTokenPricing(args.args.budget.pricing, maximumUsage) : void 0;
|
|
1587
|
+
const reservation = args.nextReservation(maximumCostUsd, wire);
|
|
1588
|
+
if (!reservation.accepted) {
|
|
1589
|
+
respondError(429, reservation.reason);
|
|
1590
|
+
return;
|
|
1591
|
+
}
|
|
1592
|
+
const timeout = setTimeout(() => controller.abort(), args.args.budget.requestTimeoutMs ?? 3e5);
|
|
1593
|
+
let chargedForBudget = maximumCostUsd;
|
|
1594
|
+
try {
|
|
1595
|
+
const requestTags = wire === "anthropic" ? {
|
|
1596
|
+
...args.args.tags ?? {},
|
|
1597
|
+
wire: "anthropic",
|
|
1598
|
+
...parsed.strippedFields === void 0 ? {} : { strippedFields: parsed.strippedFields.join(",") }
|
|
1599
|
+
} : args.args.tags;
|
|
1600
|
+
const paid = await args.args.costLedger.runPaidCall({
|
|
1601
|
+
...args.args.callId ? { callId: args.args.callId } : {},
|
|
1602
|
+
channel: args.args.channel ?? "optimizer",
|
|
1603
|
+
phase: args.args.phase,
|
|
1604
|
+
actor: args.args.actor,
|
|
1605
|
+
...requestTags ? { tags: requestTags } : {},
|
|
1606
|
+
model: args.args.model,
|
|
1607
|
+
...args.args.budget.pricing ? { maximumCharge: {
|
|
1608
|
+
customTokenPricing: args.args.budget.pricing,
|
|
1609
|
+
...maximumUsage
|
|
1610
|
+
} } : {},
|
|
1611
|
+
signal: controller.signal,
|
|
1612
|
+
execute: async (paidCallSignal, paidCallId) => forwardModelProxyRequest({
|
|
1613
|
+
call: args.args.call,
|
|
1614
|
+
callId: paidCallId,
|
|
1615
|
+
callRef: args.args.callRef,
|
|
1616
|
+
recordExecutionReceipt: args.recordExecutionReceipt,
|
|
1617
|
+
recordModelCallInvocation: args.recordModelCallInvocation,
|
|
1618
|
+
path: modelPath,
|
|
1619
|
+
request: parsed.request,
|
|
1620
|
+
...parsed.stream === void 0 ? {} : { anthropicStream: parsed.stream },
|
|
1621
|
+
model: args.args.model,
|
|
1622
|
+
allowWithinFamily: args.args.servedModelPolicy === "allow-within-family",
|
|
1623
|
+
maxOutputTokens: parsed.maxOutputTokens,
|
|
1624
|
+
...args.args.budget.maxReasoningTokensPerRequest === void 0 ? {} : { maxReasoningTokens: args.args.budget.maxReasoningTokensPerRequest },
|
|
1625
|
+
maxResponseBytes: args.args.budget.maxResponseBytes,
|
|
1626
|
+
signal: paidCallSignal
|
|
1627
|
+
}),
|
|
1628
|
+
receipt: (result) => result.receipt,
|
|
1629
|
+
receiptFromError: () => ({
|
|
1630
|
+
model: args.args.model,
|
|
1631
|
+
inputTokens: 0,
|
|
1632
|
+
outputTokens: 0,
|
|
1633
|
+
costUnknown: true,
|
|
1634
|
+
usageUnknown: true
|
|
1635
|
+
})
|
|
1636
|
+
});
|
|
1637
|
+
if (!paid.succeeded) {
|
|
1638
|
+
args.recordFailure(paid.error);
|
|
1639
|
+
chargedForBudget = paid.receipt ? paid.receipt.usageUnknown || paid.receipt.costUnknown ? maximumCostUsd : paid.receipt.costUsd : void 0;
|
|
1640
|
+
respondError(isAbortError(paid.error) ? 504 : paid.error instanceof AnthropicRequestRefusal ? paid.error.status : paid.error instanceof ProviderResponseTooLargeError || paid.error instanceof MissingModelExecutionError || paid.error instanceof ModelExecutionPersistenceError || paid.error instanceof OwnerModelContractError || paid.error instanceof ModelSubstitutionError ? 502 : 429, paid.error.message, paid.error instanceof AnthropicRequestRefusal ? paid.error.errorType : void 0);
|
|
1641
|
+
return;
|
|
1642
|
+
}
|
|
1643
|
+
if (paid.value.modelCallFailed) {
|
|
1644
|
+
respondError(502, paid.value.modelCallFailed);
|
|
1645
|
+
return;
|
|
1646
|
+
}
|
|
1647
|
+
chargedForBudget = paid.value.usageComplete ? paid.receipt.costUsd : maximumCostUsd;
|
|
1648
|
+
if (paid.value.usageRejected) {
|
|
1649
|
+
respondError(502, paid.value.usageRejected);
|
|
1650
|
+
return;
|
|
1651
|
+
}
|
|
1652
|
+
if (paid.value.status >= 200 && paid.value.status < 300) args.recordSuccessfulCompletion(wire);
|
|
1653
|
+
if (response.destroyed || response.writableEnded) return;
|
|
1654
|
+
response.writeHead(paid.value.status, {
|
|
1655
|
+
"content-type": paid.value.contentType,
|
|
1656
|
+
"content-length": String(paid.value.body.byteLength)
|
|
1657
|
+
});
|
|
1658
|
+
response.end(paid.value.body);
|
|
1659
|
+
} finally {
|
|
1660
|
+
clearTimeout(timeout);
|
|
1661
|
+
args.settleReservation(maximumCostUsd, chargedForBudget);
|
|
1662
|
+
}
|
|
1686
1663
|
} catch (error) {
|
|
1687
|
-
|
|
1664
|
+
respondError(error instanceof AnthropicRequestRefusal ? error.status : error instanceof RequestBodyTooLargeError ? 413 : error instanceof Error && isAbortError(error) ? 503 : 400, errorMessage(error), error instanceof AnthropicRequestRefusal ? error.errorType : void 0);
|
|
1688
1665
|
}
|
|
1689
|
-
return deepFreezeJson({
|
|
1690
|
-
content: value.content,
|
|
1691
|
-
...toolCalls === void 0 ? {} : { toolCalls },
|
|
1692
|
-
usage: {
|
|
1693
|
-
promptTokens,
|
|
1694
|
-
completionTokens,
|
|
1695
|
-
totalTokens,
|
|
1696
|
-
...usage.captured === void 0 ? {} : { captured: usage.captured },
|
|
1697
|
-
...usage.reasoningTokens === void 0 ? {} : { reasoningTokens: usage.reasoningTokens },
|
|
1698
|
-
...usage.cachedPromptTokens === void 0 ? {} : { cachedPromptTokens: usage.cachedPromptTokens }
|
|
1699
|
-
},
|
|
1700
|
-
costUsd: value.costUsd,
|
|
1701
|
-
model: value.model,
|
|
1702
|
-
durationMs: value.durationMs,
|
|
1703
|
-
...value.finishReason === void 0 ? {} : { finishReason: value.finishReason },
|
|
1704
|
-
...value.contentEmpty === void 0 ? {} : { contentEmpty: value.contentEmpty },
|
|
1705
|
-
raw
|
|
1706
|
-
});
|
|
1707
|
-
}
|
|
1708
|
-
/** Snapshot the owner's canonical tool calls onto the allowlisted response. */
|
|
1709
|
-
function snapshotResponseToolCalls(value) {
|
|
1710
|
-
if (value === void 0) return void 0;
|
|
1711
|
-
if (!Array.isArray(value) || value.length === 0) throw new OwnerModelContractError("optimizer model callback response toolCalls must be a non-empty array when present");
|
|
1712
|
-
return value.map((entry) => {
|
|
1713
|
-
if (!isRecord(entry) || typeof entry.id !== "string" || !entry.id || typeof entry.name !== "string" || !entry.name || typeof entry.argumentsJson !== "string") throw new OwnerModelContractError("optimizer model callback response toolCalls entries require string id, name, and argumentsJson");
|
|
1714
|
-
return {
|
|
1715
|
-
id: entry.id,
|
|
1716
|
-
name: entry.name,
|
|
1717
|
-
argumentsJson: entry.argumentsJson
|
|
1718
|
-
};
|
|
1719
|
-
});
|
|
1720
1666
|
}
|
|
1721
|
-
function
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1667
|
+
function conservativeMaximumUsage(inputTokenUpperBound, outputTokenUpperBound, pricing) {
|
|
1668
|
+
if (pricing === void 0) return {
|
|
1669
|
+
inputTokens: inputTokenUpperBound,
|
|
1670
|
+
outputTokens: outputTokenUpperBound
|
|
1671
|
+
};
|
|
1672
|
+
const inputRates = [pricing.inputUsdPerMillion, pricing.cachedInputUsdPerMillion ?? pricing.inputUsdPerMillion];
|
|
1673
|
+
const mostExpensiveInputClass = inputRates.indexOf(Math.max(...inputRates));
|
|
1725
1674
|
return {
|
|
1726
|
-
|
|
1727
|
-
|
|
1675
|
+
inputTokens: mostExpensiveInputClass === 0 ? inputTokenUpperBound : 0,
|
|
1676
|
+
...mostExpensiveInputClass === 1 ? { cachedTokens: inputTokenUpperBound } : {},
|
|
1677
|
+
cacheWriteTokens: inputTokenUpperBound,
|
|
1678
|
+
outputTokens: outputTokenUpperBound
|
|
1728
1679
|
};
|
|
1729
1680
|
}
|
|
1730
|
-
function
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
incomplete_details: finishReason === "length" ? { reason: "max_output_tokens" } : null,
|
|
1755
|
-
output: [{
|
|
1756
|
-
id: `msg-${args.callId}`,
|
|
1757
|
-
type: "message",
|
|
1758
|
-
status: finishReason === "length" ? "incomplete" : "completed",
|
|
1759
|
-
role: "assistant",
|
|
1760
|
-
content: [{
|
|
1761
|
-
type: "output_text",
|
|
1762
|
-
text: args.response.content,
|
|
1763
|
-
annotations: []
|
|
1764
|
-
}]
|
|
1765
|
-
}],
|
|
1766
|
-
output_text: args.response.content,
|
|
1767
|
-
...usage ? { usage } : {}
|
|
1768
|
-
};
|
|
1769
|
-
const encoded = new TextEncoder().encode(JSON.stringify(body));
|
|
1770
|
-
if (encoded.byteLength > args.maxBytes) throw new ProviderResponseTooLargeError();
|
|
1771
|
-
return encoded;
|
|
1681
|
+
function readBody(request, maximumBytes) {
|
|
1682
|
+
return new Promise((resolvePromise, reject) => {
|
|
1683
|
+
let size = 0;
|
|
1684
|
+
let tooLarge = false;
|
|
1685
|
+
const chunks = [];
|
|
1686
|
+
request.on("data", (chunk) => {
|
|
1687
|
+
if (tooLarge) return;
|
|
1688
|
+
size += chunk.byteLength;
|
|
1689
|
+
if (size > maximumBytes) {
|
|
1690
|
+
tooLarge = true;
|
|
1691
|
+
chunks.length = 0;
|
|
1692
|
+
return;
|
|
1693
|
+
}
|
|
1694
|
+
chunks.push(chunk);
|
|
1695
|
+
});
|
|
1696
|
+
request.on("error", reject);
|
|
1697
|
+
request.on("end", () => {
|
|
1698
|
+
if (tooLarge) {
|
|
1699
|
+
reject(new RequestBodyTooLargeError("optimizer model request body too large"));
|
|
1700
|
+
return;
|
|
1701
|
+
}
|
|
1702
|
+
resolvePromise(new Uint8Array(Buffer.concat(chunks)));
|
|
1703
|
+
});
|
|
1704
|
+
});
|
|
1772
1705
|
}
|
|
1773
|
-
function
|
|
1774
|
-
|
|
1775
|
-
return path === "/v1/responses" ? "responses" : "chat-completions";
|
|
1706
|
+
function isAbortError(error) {
|
|
1707
|
+
return error.name === "AbortError" || error.message.toLowerCase().includes("abort");
|
|
1776
1708
|
}
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1709
|
+
//#endregion
|
|
1710
|
+
//#region src/campaign/external-optimizer-model-proxy.ts
|
|
1711
|
+
/**
|
|
1712
|
+
* Put an OpenAI-compatible optimizer behind the shared cost ledger.
|
|
1713
|
+
*
|
|
1714
|
+
* The child process receives only a loopback URL and an ephemeral token. The
|
|
1715
|
+
* package that owns execution receives a validated immutable request through
|
|
1716
|
+
* `call`; Eval receives no provider credential. Every request reserves its
|
|
1717
|
+
* conservative byte-count input bound plus the declared output cap before the
|
|
1718
|
+
* owner is invoked exactly once.
|
|
1719
|
+
*/
|
|
1720
|
+
async function startExternalOptimizerModelProxy(args) {
|
|
1721
|
+
assertModelProxyConfig(args);
|
|
1722
|
+
args.signal?.throwIfAborted();
|
|
1723
|
+
const token = randomLocalToken();
|
|
1724
|
+
let requestCount = 0;
|
|
1725
|
+
let successfulCompletionCount = 0;
|
|
1726
|
+
const wireCounts = {
|
|
1727
|
+
openai: {
|
|
1728
|
+
requestAttempts: 0,
|
|
1729
|
+
successfulCompletions: 0
|
|
1730
|
+
},
|
|
1731
|
+
anthropic: {
|
|
1732
|
+
requestAttempts: 0,
|
|
1733
|
+
successfulCompletions: 0
|
|
1734
|
+
}
|
|
1782
1735
|
};
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1736
|
+
let modelCallInvocations = 0;
|
|
1737
|
+
let executionRecordCount = 0;
|
|
1738
|
+
let executionSequence = 0;
|
|
1739
|
+
const failures = [];
|
|
1740
|
+
let totalRequestCount = args.initialUsage?.requests ?? 0;
|
|
1741
|
+
let committedForBudget = args.initialUsage?.costUsd ?? 0;
|
|
1742
|
+
let reservedForBudget = 0;
|
|
1743
|
+
let accepting = true;
|
|
1744
|
+
let closePromise;
|
|
1745
|
+
const activeControllers = /* @__PURE__ */ new Set();
|
|
1746
|
+
const activeHandlers = /* @__PURE__ */ new Set();
|
|
1747
|
+
const server = createServer((request, response) => {
|
|
1748
|
+
if (!accepting) {
|
|
1749
|
+
sendJsonIfOpen(response, 503, { error: "external optimizer model proxy is closing" });
|
|
1750
|
+
return;
|
|
1751
|
+
}
|
|
1752
|
+
const controller = new AbortController();
|
|
1753
|
+
const abortRequest = () => {
|
|
1754
|
+
request.destroy();
|
|
1755
|
+
response.destroy();
|
|
1756
|
+
};
|
|
1757
|
+
activeControllers.add(controller);
|
|
1758
|
+
controller.signal.addEventListener("abort", abortRequest, { once: true });
|
|
1759
|
+
let handler;
|
|
1760
|
+
handler = handleModelProxyRequest({
|
|
1761
|
+
request,
|
|
1762
|
+
response,
|
|
1763
|
+
controller,
|
|
1764
|
+
token,
|
|
1765
|
+
args,
|
|
1766
|
+
nextReservation: (maximumCostUsd, wire) => {
|
|
1767
|
+
if (totalRequestCount >= args.budget.maxRequests) return {
|
|
1768
|
+
accepted: false,
|
|
1769
|
+
reason: "optimizer model request limit reached"
|
|
1770
|
+
};
|
|
1771
|
+
if (args.budget.maxCostUsd !== void 0 && maximumCostUsd !== void 0 && committedForBudget + reservedForBudget + maximumCostUsd > args.budget.maxCostUsd + Number.EPSILON) return {
|
|
1772
|
+
accepted: false,
|
|
1773
|
+
reason: "optimizer model cost limit reached"
|
|
1774
|
+
};
|
|
1775
|
+
requestCount += 1;
|
|
1776
|
+
wireCounts[wire].requestAttempts += 1;
|
|
1777
|
+
totalRequestCount += 1;
|
|
1778
|
+
reservedForBudget += maximumCostUsd ?? 0;
|
|
1779
|
+
return { accepted: true };
|
|
1780
|
+
},
|
|
1781
|
+
settleReservation: (maximumCostUsd, chargedCostUsd) => {
|
|
1782
|
+
reservedForBudget = Math.max(0, reservedForBudget - (maximumCostUsd ?? 0));
|
|
1783
|
+
committedForBudget += chargedCostUsd ?? 0;
|
|
1784
|
+
},
|
|
1785
|
+
recordSuccessfulCompletion: (wire) => {
|
|
1786
|
+
successfulCompletionCount += 1;
|
|
1787
|
+
wireCounts[wire].successfulCompletions += 1;
|
|
1788
|
+
},
|
|
1789
|
+
recordExecutionReceipt: (observation) => {
|
|
1790
|
+
try {
|
|
1791
|
+
args.recordExecution({
|
|
1792
|
+
...observation,
|
|
1793
|
+
sequence: executionSequence + 1
|
|
1794
|
+
});
|
|
1795
|
+
} catch (error) {
|
|
1796
|
+
throw new ModelExecutionPersistenceError(`optimizer model execution evidence was not persisted: ${errorMessage(error)}`);
|
|
1797
|
+
}
|
|
1798
|
+
executionSequence += 1;
|
|
1799
|
+
executionRecordCount += 1;
|
|
1800
|
+
},
|
|
1801
|
+
recordModelCallInvocation: () => {
|
|
1802
|
+
modelCallInvocations += 1;
|
|
1803
|
+
},
|
|
1804
|
+
recordFailure: (error) => {
|
|
1805
|
+
failures.push(error);
|
|
1806
|
+
}
|
|
1807
|
+
}).finally(() => {
|
|
1808
|
+
controller.signal.removeEventListener("abort", abortRequest);
|
|
1809
|
+
activeControllers.delete(controller);
|
|
1810
|
+
activeHandlers.delete(handler);
|
|
1811
|
+
});
|
|
1812
|
+
activeHandlers.add(handler);
|
|
1813
|
+
handler.catch(() => void 0);
|
|
1814
|
+
});
|
|
1815
|
+
const port = await listenLocal(server);
|
|
1816
|
+
const close = () => {
|
|
1817
|
+
closePromise ??= closeModelProxy();
|
|
1818
|
+
return closePromise;
|
|
1798
1819
|
};
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
}
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
if (pricing === void 0) return {
|
|
1808
|
-
inputTokens: inputTokenUpperBound,
|
|
1809
|
-
outputTokens: outputTokenUpperBound
|
|
1820
|
+
const onAbort = () => {
|
|
1821
|
+
for (const controller of activeControllers) controller.abort(args.signal?.reason);
|
|
1822
|
+
close().catch(() => void 0);
|
|
1823
|
+
};
|
|
1824
|
+
args.signal?.addEventListener("abort", onAbort, { once: true });
|
|
1825
|
+
if (args.signal?.aborted) onAbort();
|
|
1826
|
+
const assertExecutionComplete = () => {
|
|
1827
|
+
if (executionRecordCount !== modelCallInvocations) throw new Error(`external optimizer model callback returned ${executionRecordCount} execution records for ${modelCallInvocations} invoked calls`);
|
|
1810
1828
|
};
|
|
1811
|
-
const inputRates = [pricing.inputUsdPerMillion, pricing.cachedInputUsdPerMillion ?? pricing.inputUsdPerMillion];
|
|
1812
|
-
const mostExpensiveInputClass = inputRates.indexOf(Math.max(...inputRates));
|
|
1813
1829
|
return {
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1830
|
+
baseUrl: `http://127.0.0.1:${port}/v1`,
|
|
1831
|
+
apiKey: token,
|
|
1832
|
+
requestAttempts: () => requestCount,
|
|
1833
|
+
successfulCompletions: () => successfulCompletionCount,
|
|
1834
|
+
wireUsage: () => ({
|
|
1835
|
+
openai: { ...wireCounts.openai },
|
|
1836
|
+
anthropic: { ...wireCounts.anthropic }
|
|
1837
|
+
}),
|
|
1838
|
+
assertExecutionComplete,
|
|
1839
|
+
failures: () => [...failures],
|
|
1840
|
+
close
|
|
1818
1841
|
};
|
|
1842
|
+
async function closeModelProxy() {
|
|
1843
|
+
args.signal?.removeEventListener("abort", onAbort);
|
|
1844
|
+
accepting = false;
|
|
1845
|
+
const closingServer = closeServer(server);
|
|
1846
|
+
server.closeIdleConnections?.();
|
|
1847
|
+
const [serverResult] = await Promise.allSettled([closingServer, waitForActiveHandlers(activeHandlers)]);
|
|
1848
|
+
if (activeControllers.size !== 0 || activeHandlers.size !== 0) throw new Error("external optimizer model proxy closed with active request work");
|
|
1849
|
+
let executionError;
|
|
1850
|
+
try {
|
|
1851
|
+
assertExecutionComplete();
|
|
1852
|
+
} catch (error) {
|
|
1853
|
+
executionError = error;
|
|
1854
|
+
}
|
|
1855
|
+
if (serverResult?.status === "rejected" && executionError !== void 0) throw new AggregateError([serverResult.reason, executionError], "external optimizer model proxy close and execution evidence both failed");
|
|
1856
|
+
if (serverResult?.status === "rejected") throw serverResult.reason;
|
|
1857
|
+
if (executionError !== void 0) throw executionError;
|
|
1858
|
+
}
|
|
1819
1859
|
}
|
|
1820
1860
|
function assertModelProxyConfig(args) {
|
|
1821
1861
|
assertServedModelPolicy(args.servedModelPolicy, "external optimizer model proxy: servedModelPolicy");
|
|
@@ -1840,49 +1880,9 @@ function assertModelProxyConfig(args) {
|
|
|
1840
1880
|
if (args.initialUsage.requests > args.budget.maxRequests || args.budget.maxCostUsd !== void 0 && args.initialUsage.costUsd !== void 0 && args.initialUsage.costUsd > args.budget.maxCostUsd + Number.EPSILON) throw new Error("external optimizer model proxy: initialUsage exceeds the configured budget");
|
|
1841
1881
|
}
|
|
1842
1882
|
}
|
|
1843
|
-
var RequestBodyTooLargeError = class extends Error {};
|
|
1844
|
-
var ProviderResponseTooLargeError = class extends Error {
|
|
1845
|
-
constructor() {
|
|
1846
|
-
super("optimizer model response exceeds maxResponseBytes");
|
|
1847
|
-
}
|
|
1848
|
-
};
|
|
1849
|
-
var MissingModelExecutionError = class extends Error {};
|
|
1850
|
-
var ModelExecutionPersistenceError = class extends Error {};
|
|
1851
|
-
var OwnerModelContractError = class extends Error {};
|
|
1852
|
-
function readBody(request, maximumBytes) {
|
|
1853
|
-
return new Promise((resolvePromise, reject) => {
|
|
1854
|
-
let size = 0;
|
|
1855
|
-
let tooLarge = false;
|
|
1856
|
-
const chunks = [];
|
|
1857
|
-
request.on("data", (chunk) => {
|
|
1858
|
-
if (tooLarge) return;
|
|
1859
|
-
size += chunk.byteLength;
|
|
1860
|
-
if (size > maximumBytes) {
|
|
1861
|
-
tooLarge = true;
|
|
1862
|
-
chunks.length = 0;
|
|
1863
|
-
return;
|
|
1864
|
-
}
|
|
1865
|
-
chunks.push(chunk);
|
|
1866
|
-
});
|
|
1867
|
-
request.on("error", reject);
|
|
1868
|
-
request.on("end", () => {
|
|
1869
|
-
if (tooLarge) {
|
|
1870
|
-
reject(new RequestBodyTooLargeError("optimizer model request body too large"));
|
|
1871
|
-
return;
|
|
1872
|
-
}
|
|
1873
|
-
resolvePromise(new Uint8Array(Buffer.concat(chunks)));
|
|
1874
|
-
});
|
|
1875
|
-
});
|
|
1876
|
-
}
|
|
1877
1883
|
function randomLocalToken() {
|
|
1878
1884
|
return randomBytes(32).toString("hex");
|
|
1879
1885
|
}
|
|
1880
|
-
function isAbortError(error) {
|
|
1881
|
-
return error.name === "AbortError" || error.message.toLowerCase().includes("abort");
|
|
1882
|
-
}
|
|
1883
|
-
function toErrorMessage(error) {
|
|
1884
|
-
return error instanceof Error ? error.message : String(error);
|
|
1885
|
-
}
|
|
1886
1886
|
//#endregion
|
|
1887
1887
|
//#region src/campaign/external-optimizer-resources.ts
|
|
1888
1888
|
async function runWithCleanup(args) {
|
|
@@ -2162,4 +2162,4 @@ function summarizeProcessOutput(capture, max = 4e3) {
|
|
|
2162
2162
|
//#endregion
|
|
2163
2163
|
export { parseAnalystFinding as A, ProposalFindingSchema as C, lineageRoots as D, independentUnitCount as E, inMemoryCampaignStorage as F, spanningWindow as M, createRunCostLedger as N, mergeEvidenceUnits as O, fsCampaignStorage as P, FINDING_LAYERS as S, findingLayerRank as T, removeCredentialEnvironment as _, closeServer as a, safePathComponent as b, waitForActiveHandlers as c, assertExternalOptimizerModelBudget as d, assertJsonValue as f, isRecord as g, isExternalTextCandidate as h, startExternalOptimizerModelProxy as i, readFindingScope as j, mostSevere as k, DEFAULT_EXTERNAL_OPTIMIZER_CALLBACK_LIMITS as l, isCandidateText as m, closeExternalOptimizerResources as n, listenLocal as o, assertNoCredentialValues as p, runWithCleanup as r, sendJsonIfOpen as s, runExternalOptimizerProcess as t, DEFAULT_EXTERNAL_OPTIMIZER_PROCESS_LIMITS as u, resolveExternalOptimizerCallbackLimits as v, UNKNOWN_LINEAGE as w, AnalystFindingSchema as x, resolveExternalOptimizerProcessLimits as y };
|
|
2164
2164
|
|
|
2165
|
-
//# sourceMappingURL=external-optimizer-subprocess-
|
|
2165
|
+
//# sourceMappingURL=external-optimizer-subprocess-BqW0hHcI.js.map
|