@tangle-network/agent-eval 0.216.1 → 0.217.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DyeNcenW.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DyeNcenW.d.ts.map} +1 -1
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-xQK8nOI6.js} +554 -89
- package/dist/benchmark-command-xQK8nOI6.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CJGkxAQ6.js} +23 -530
- package/dist/campaign-CJGkxAQ6.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-CQYw8_0H.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-CQYw8_0H.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-ldqxYYTN.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-ldqxYYTN.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-DavzexSp.js} +3 -2
- package/dist/chat-json-call-DavzexSp.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-B_jw6C9p.js} +9 -64
- package/dist/completion-verifier-B_jw6C9p.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +25 -401
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-ClYjq3QE.js} +8 -11
- package/dist/default-registry-ClYjq3QE.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BDxNlP3D.js} +10 -10
- package/dist/{define-agent-eval-CNTPzIkp.js.map → define-agent-eval-BDxNlP3D.js.map} +1 -1
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BIkH_gAR.d.ts} +8 -8
- package/dist/define-agent-eval-BIkH_gAR.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-DhdCr1Dc.js} +5 -10
- package/dist/dspy-rlm-engine-DhdCr1Dc.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/{eval-campaign-90jeaCAI.js → eval-campaign-BLEfBGbA.js} +6 -6
- package/dist/{eval-campaign-90jeaCAI.js.map → eval-campaign-BLEfBGbA.js.map} +1 -1
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-jTjC_QQD.d.ts} +7 -7
- package/dist/evaluation-jTjC_QQD.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +149 -241
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-Cl6IeNAu.js} +3 -3
- package/dist/exporters-Cl6IeNAu.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-KVFGvYX1.js} +4 -3
- package/dist/external-optimizer-process-KVFGvYX1.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-XIHBOSKt.js} +9 -11
- package/dist/external-optimizer-subprocess-XIHBOSKt.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BMVQtEsm.js} +9 -249
- package/dist/findings-store-BMVQtEsm.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-D9f7-oX7.js} +4 -24
- package/dist/hf-dataset-D9f7-oX7.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-DHKEAv3s.d.ts → index-9jJ2fKSX.d.ts} +2 -64
- package/dist/index-9jJ2fKSX.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-DPcngtwt.d.ts → index-BiT6RtNH.d.ts} +11 -36
- package/dist/index-BiT6RtNH.d.ts.map +1 -0
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-CyszSg4O.d.ts → index-ZuWx52WG.d.ts} +115 -347
- package/dist/index-ZuWx52WG.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +402 -1040
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-DOd9Q86O.js} +7 -9
- package/dist/integrity-DOd9Q86O.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-CR4QikYp.js} +110 -110
- package/dist/journal-CR4QikYp.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-DrJhbms9.js} +5 -84
- package/dist/ledger-core-DrJhbms9.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-BshCREn6.js} +6 -108
- package/dist/llm-client-BshCREn6.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-C8jMTNNG.js} +24 -32
- package/dist/llm-judge-C8jMTNNG.js.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-DHe-dwVJ.d.ts} +9 -18
- package/dist/matrix-DHe-dwVJ.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-zszyYlpA.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-zszyYlpA.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-BzsxidFD.js} +11 -11
- package/dist/pareto-BzsxidFD.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-DaY95yU6.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-DaY95yU6.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-BbWidvUk.js} +3 -3
- package/dist/proposal-findings-BbWidvUk.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-C-w2EZ6p.js} +205 -363
- package/dist/render-C-w2EZ6p.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-0ABVIdGa.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-0ABVIdGa.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-D6C50mfA.js} +88 -13
- package/dist/rollout-D6C50mfA.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/{schema-BuiujZ-g.js → schema-DxcKTzbi.js} +2 -2
- package/dist/schema-DxcKTzbi.js.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-DxqjBxA-.d.ts} +10 -27
- package/dist/search-kernel-DxqjBxA-.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-TUvB5gWH.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-TUvB5gWH.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-UKCO2r8a.js} +4 -4
- package/dist/search-ledger-file-UKCO2r8a.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Cw2z3-td.js} +4 -4
- package/dist/search-shipper-Cw2z3-td.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-DQIvJsSB.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-DQIvJsSB.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-DoVbqqtJ.js} +101 -101
- package/dist/server-DoVbqqtJ.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-U58yf8N6.js} +2 -2
- package/dist/{single-run-lock-CV9Eyd_q.js.map → single-run-lock-U58yf8N6.js.map} +1 -1
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-kyL94-ri.js} +17 -95
- package/dist/skillopt-optimization-method-kyL94-ri.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-C9AbgREN.d.ts} +8 -67
- package/dist/statistical-heldout-C9AbgREN.d.ts.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-DM7tlml1.d.ts} +10 -19
- package/dist/summary-report-DM7tlml1.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-Bp_Ac0p1.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-Bp_Ac0p1.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-78g6E4vh.js} +5 -5
- package/dist/terminal-record-78g6E4vh.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +3 -117
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +98 -56
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-BGRglkKd.js} +13 -10
- package/dist/tree-BGRglkKd.js.map +1 -0
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-D8GhJoHk.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-D8GhJoHk.d.ts.map} +1 -1
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-DtMbFw6U.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-DtMbFw6U.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +17 -17
- package/docs/design.md +1 -1
- package/docs/evaluation-integrity.md +1 -1
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +138 -529
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +1 -1
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { s as ValidationError } from "./errors-
|
|
1
|
+
import { s as ValidationError } from "./errors-BcOs5MKW.js";
|
|
2
2
|
import { canonicalCandidateBytes, canonicalCandidateJson, sha256Bytes } from "@tangle-network/agent-interface";
|
|
3
3
|
//#region src/ledger-core/canonical.ts
|
|
4
4
|
/**
|
|
@@ -125,4 +125,4 @@ function compareCodeUnits(left, right) {
|
|
|
125
125
|
//#endregion
|
|
126
126
|
export { hashCanonical as a, compareCodeUnits as i, LedgerCanonicalizationError as n, jsonDocument as o, canonicalString as r, LEDGER_HASH_PATTERN as t };
|
|
127
127
|
|
|
128
|
-
//# sourceMappingURL=canonical-
|
|
128
|
+
//# sourceMappingURL=canonical-Dq2hOQ8L.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"canonical-
|
|
1
|
+
{"version":3,"file":"canonical-Dq2hOQ8L.js","names":[],"sources":["../src/ledger-core/canonical.ts"],"sourcesContent":["/**\n * Canonical bytes and digests for durable journal rows.\n *\n * Serialization is RFC 8785 through `@tangle-network/agent-interface` — the\n * stack's single identity scheme — so a value carries the same digest in every\n * package that produces or verifies one.\n *\n * Values canonical JSON cannot represent faithfully are refused, never coerced.\n * `NaN`, `Infinity`, `undefined`, class instances, cycles, and unpaired\n * surrogates each have a lossy encoding that maps distinct records onto one\n * digest, and in a hash chain the digest IS the record's identity: a coercion\n * there is a collision, not a convenience.\n */\n\nimport {\n canonicalCandidateBytes,\n canonicalCandidateJson,\n sha256Bytes,\n} from '@tangle-network/agent-interface'\nimport { ValidationError } from '../errors'\n\nexport type LedgerHash = `sha256:${string}`\n\n/** Shape of every ledger digest: `sha256:` followed by 64 lowercase hex chars. */\nexport const LEDGER_HASH_PATTERN = /^sha256:[a-f0-9]{64}$/\n\n/** A value has no faithful canonical-JSON form, so it cannot be stored or hashed. */\nexport class LedgerCanonicalizationError extends ValidationError {}\n\n/** Canonical JSON encoding (RFC 8785): object keys sorted by UTF-16 code unit,\n * recursively. This is the byte form that is hashed and stored, so equal values\n * always encode identically. */\nexport function canonicalString(value: unknown): string {\n try {\n return canonicalCandidateJson(value)\n } catch (error) {\n throw canonicalizationError(value, error)\n }\n}\n\nexport function hashCanonical(value: unknown): LedgerHash {\n try {\n return sha256Bytes(canonicalCandidateBytes(value))\n } catch (error) {\n throw canonicalizationError(value, error)\n }\n}\n\n/**\n * The JSON document form of `value` — the value as a JSON file carries it.\n *\n * A JSON document has no `undefined`, so an undefined-valued key is absent\n * from the document and absent from its digest. Use this, and say so, when a\n * record is digested BEFORE being written and re-digested AFTER being read\n * back: without it the two digests disagree for a record carrying an\n * undefined-valued key, and the reader cannot verify the writer.\n *\n * Only that one rule is applied. Every other non-canonical value — `NaN`, a\n * class instance, an `undefined` array item, a cycle — reaches\n * {@link canonicalString} unchanged and is refused there, so this form never\n * turns an ambiguous value into a valid-looking digest the way\n * `JSON.stringify` turns `NaN` into `null`.\n */\nexport function jsonDocument(value: unknown): unknown {\n if (value === null || typeof value !== 'object') return value\n if (Array.isArray(value)) return value.map(jsonDocument)\n const prototype = Object.getPrototypeOf(value)\n if (prototype !== Object.prototype && prototype !== null) return value\n // fromEntries creates own properties, including a caller's __proto__ key.\n return Object.fromEntries(\n Object.entries(value as Record<string, unknown>)\n .filter(([, entry]) => entry !== undefined)\n .map(([key, entry]) => [key, jsonDocument(entry)]),\n )\n}\n\n/** Name the offending value and path so a refusal is actionable. The RFC 8785\n * encoder above is the sole authority on acceptance; this walk only explains a\n * rejection it already made. */\nfunction canonicalizationError(value: unknown, cause: unknown): LedgerCanonicalizationError {\n const defect = firstNonCanonicalDefect(value, '$', new Set())\n const reason = defect !== '' ? defect : cause instanceof Error ? cause.message : String(cause)\n return new LedgerCanonicalizationError(`value has no canonical JSON form: ${reason}`, {\n cause,\n })\n}\n\nfunction firstNonCanonicalDefect(value: unknown, path: string, ancestors: Set<object>): string {\n if (value === null) return ''\n if (value === undefined) return `${path} is undefined`\n const type = typeof value\n if (type === 'number') {\n return Number.isFinite(value) ? '' : `${path} is ${String(value)}`\n }\n if (type === 'string') {\n return isWellFormedUtf16(value as string) ? '' : `${path} contains an unpaired surrogate`\n }\n if (type === 'boolean') return ''\n if (type !== 'object') return `${path} is a ${type}`\n const object = value as object\n if (ancestors.has(object)) return `${path} closes a reference cycle`\n const prototype = Object.getPrototypeOf(object)\n if (!Array.isArray(object) && prototype !== Object.prototype && prototype !== null) {\n return `${path} is a ${object.constructor?.name ?? 'non-plain'} instance, not plain JSON data`\n }\n const nested = new Set(ancestors).add(object)\n if (Array.isArray(object)) {\n for (let index = 0; index < object.length; index += 1) {\n const defect = firstNonCanonicalDefect(object[index], `${path}[${index}]`, nested)\n if (defect !== '') return defect\n }\n return ''\n }\n for (const [key, entry] of Object.entries(object)) {\n if (!isWellFormedUtf16(key)) return `${path}.${key} has a key with an unpaired surrogate`\n const defect = firstNonCanonicalDefect(entry, `${path}.${key}`, nested)\n if (defect !== '') return defect\n }\n return ''\n}\n\nfunction isWellFormedUtf16(value: string): boolean {\n for (let index = 0; index < value.length; index += 1) {\n const code = value.charCodeAt(index)\n if (code >= 0xd800 && code <= 0xdbff) {\n const next = value.charCodeAt(index + 1)\n if (!(next >= 0xdc00 && next <= 0xdfff)) return false\n index += 1\n } else if (code >= 0xdc00 && code <= 0xdfff) {\n return false\n }\n }\n return true\n}\n\n/**\n * Compare two strings by UTF-16 code unit — the order RFC 8785 uses for object\n * keys, and the order any array must be sorted into before it is canonicalized.\n *\n * `String.prototype.localeCompare` reads the host's collation, so it is a\n * property of the machine rather than of the value. RFC 8785 canonicalizes an\n * array BY POSITION, so a `localeCompare` sort in front of `canonicalString` or\n * `hashCanonical` lets the collation decide the digest bytes: the ids\n * `Accuracy, brevity, Clarity` order as `Accuracy,brevity,Clarity` under an\n * en-US collation and as `Accuracy,Clarity,brevity` by code unit, and the two\n * produce different digests for the same data.\n *\n * Use this for any ordering whose result reaches a digest, a stable\n * serialization, or a stored identity. A comparator for a report a human reads\n * is free to keep `localeCompare`.\n */\nexport function compareCodeUnits(left: string, right: string): number {\n if (left < right) return -1\n return left > right ? 1 : 0\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAa,sBAAsB;;AAGnC,IAAa,8BAAb,cAAiD,gBAAgB,CAAC;;;;AAKlE,SAAgB,gBAAgB,OAAwB;CACtD,IAAI;EACF,OAAO,uBAAuB,KAAK;CACrC,SAAS,OAAO;EACd,MAAM,sBAAsB,OAAO,KAAK;CAC1C;AACF;AAEA,SAAgB,cAAc,OAA4B;CACxD,IAAI;EACF,OAAO,YAAY,wBAAwB,KAAK,CAAC;CACnD,SAAS,OAAO;EACd,MAAM,sBAAsB,OAAO,KAAK;CAC1C;AACF;;;;;;;;;;;;;;;;AAiBA,SAAgB,aAAa,OAAyB;CACpD,IAAI,UAAU,QAAQ,OAAO,UAAU,UAAU,OAAO;CACxD,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO,MAAM,IAAI,YAAY;CACvD,MAAM,YAAY,OAAO,eAAe,KAAK;CAC7C,IAAI,cAAc,OAAO,aAAa,cAAc,MAAM,OAAO;CAEjE,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAgC,CAAC,CAC7C,QAAQ,GAAG,WAAW,UAAU,KAAA,CAAS,CAAC,CAC1C,KAAK,CAAC,KAAK,WAAW,CAAC,KAAK,aAAa,KAAK,CAAC,CAAC,CACrD;AACF;;;;AAKA,SAAS,sBAAsB,OAAgB,OAA6C;CAC1F,MAAM,SAAS,wBAAwB,OAAO,qBAAK,IAAI,IAAI,CAAC;CAE5D,OAAO,IAAI,4BAA4B,qCADxB,WAAW,KAAK,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KACP,EACpF,MACF,CAAC;AACH;AAEA,SAAS,wBAAwB,OAAgB,MAAc,WAAgC;CAC7F,IAAI,UAAU,MAAM,OAAO;CAC3B,IAAI,UAAU,KAAA,GAAW,OAAO,GAAG,KAAK;CACxC,MAAM,OAAO,OAAO;CACpB,IAAI,SAAS,UACX,OAAO,OAAO,SAAS,KAAK,IAAI,KAAK,GAAG,KAAK,MAAM,OAAO,KAAK;CAEjE,IAAI,SAAS,UACX,OAAO,kBAAkB,KAAe,IAAI,KAAK,GAAG,KAAK;CAE3D,IAAI,SAAS,WAAW,OAAO;CAC/B,IAAI,SAAS,UAAU,OAAO,GAAG,KAAK,QAAQ;CAC9C,MAAM,SAAS;CACf,IAAI,UAAU,IAAI,MAAM,GAAG,OAAO,GAAG,KAAK;CAC1C,MAAM,YAAY,OAAO,eAAe,MAAM;CAC9C,IAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,cAAc,OAAO,aAAa,cAAc,MAC5E,OAAO,GAAG,KAAK,QAAQ,OAAO,aAAa,QAAQ,YAAY;CAEjE,MAAM,SAAS,IAAI,IAAI,SAAS,CAAC,CAAC,IAAI,MAAM;CAC5C,IAAI,MAAM,QAAQ,MAAM,GAAG;EACzB,KAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;GACrD,MAAM,SAAS,wBAAwB,OAAO,QAAQ,GAAG,KAAK,GAAG,MAAM,IAAI,MAAM;GACjF,IAAI,WAAW,IAAI,OAAO;EAC5B;EACA,OAAO;CACT;CACA,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,MAAM,GAAG;EACjD,IAAI,CAAC,kBAAkB,GAAG,GAAG,OAAO,GAAG,KAAK,GAAG,IAAI;EACnD,MAAM,SAAS,wBAAwB,OAAO,GAAG,KAAK,GAAG,OAAO,MAAM;EACtE,IAAI,WAAW,IAAI,OAAO;CAC5B;CACA,OAAO;AACT;AAEA,SAAS,kBAAkB,OAAwB;CACjD,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS,GAAG;EACpD,MAAM,OAAO,MAAM,WAAW,KAAK;EACnC,IAAI,QAAQ,SAAU,QAAQ,OAAQ;GACpC,MAAM,OAAO,MAAM,WAAW,QAAQ,CAAC;GACvC,IAAI,EAAE,QAAQ,SAAU,QAAQ,QAAS,OAAO;GAChD,SAAS;EACX,OAAO,IAAI,QAAQ,SAAU,QAAQ,OACnC,OAAO;CAEX;CACA,OAAO;AACT;;;;;;;;;;;;;;;;;AAkBA,SAAgB,iBAAiB,MAAc,OAAuB;CACpE,IAAI,OAAO,OAAO,OAAO;CACzB,OAAO,OAAO,QAAQ,IAAI;AAC5B"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { t as isNonnegativeFiniteNumber } from "./nonnegative-number-DvdMxg_f.js";
|
|
2
|
+
//#region src/matrix/cell-spend.ts
|
|
3
|
+
/**
|
|
4
|
+
* Spend carrier for a cell that fails after it has already spent money.
|
|
5
|
+
*
|
|
6
|
+
* `runCell` reports what a cell cost by returning a `CellResult`. A throw
|
|
7
|
+
* carries no result, so the runner cannot know what the cell spent before it
|
|
8
|
+
* failed: the cumulative sum the cost ceiling reads, and `totalCostUsd`, both
|
|
9
|
+
* miss that money. A throw that carries a `CellSpend` closes the gap — the
|
|
10
|
+
* runner bills the failed cell for the spend the carrier declares, and records
|
|
11
|
+
* a cell that declared nothing as `uncaptured` instead of a zero that reads
|
|
12
|
+
* like "this cell spent nothing".
|
|
13
|
+
*
|
|
14
|
+
* The carrier is read structurally, never by `instanceof`, so a throw that
|
|
15
|
+
* crosses a package or realm boundary still bills.
|
|
16
|
+
*/
|
|
17
|
+
/** Property key the carrier occupies on a thrown object. Stable across
|
|
18
|
+
* versions — two copies of this package must agree on it. */
|
|
19
|
+
const CELL_SPEND_KEY = "__agentEvalCellSpend";
|
|
20
|
+
/**
|
|
21
|
+
* Return `error` carrying `spend`, for a throw site inside `runCell`:
|
|
22
|
+
* `throw withCellSpend(err, { costUsd, durationMs, kind })`.
|
|
23
|
+
*
|
|
24
|
+
* A value that cannot hold the carrier — a thrown string or number, or a
|
|
25
|
+
* frozen, sealed, or otherwise non-extensible object — is wrapped in an
|
|
26
|
+
* `Error` that keeps the original as `cause`, so the spend is never dropped.
|
|
27
|
+
* A later call overwrites an earlier carrier — the outer frame knows more
|
|
28
|
+
* than the inner one.
|
|
29
|
+
*
|
|
30
|
+
* Throws `TypeError` on a non-finite or negative amount. A poisoned amount
|
|
31
|
+
* would reach `cumulativeCost` and disable the cost ceiling for the whole run,
|
|
32
|
+
* because `NaN >= ceiling` is false.
|
|
33
|
+
*/
|
|
34
|
+
function withCellSpend(error, spend) {
|
|
35
|
+
assertAmount(spend.costUsd, "costUsd");
|
|
36
|
+
assertAmount(spend.durationMs, "durationMs");
|
|
37
|
+
if (spend.kind !== "observed" && spend.kind !== "estimated" && spend.kind !== "uncaptured") throw new TypeError(`withCellSpend: kind must be observed, estimated or uncaptured, received ${String(spend.kind)}`);
|
|
38
|
+
const carrier = {
|
|
39
|
+
costUsd: spend.costUsd,
|
|
40
|
+
durationMs: spend.durationMs,
|
|
41
|
+
kind: spend.kind
|
|
42
|
+
};
|
|
43
|
+
if (typeof error === "object" && error !== null && attach(error, carrier)) return error;
|
|
44
|
+
const wrapper = new Error(`cell failed: ${messageOf(error)}`, { cause: error });
|
|
45
|
+
attach(wrapper, carrier);
|
|
46
|
+
return wrapper;
|
|
47
|
+
}
|
|
48
|
+
/** True when the carrier landed. `defineProperty` throws on a non-extensible
|
|
49
|
+
* target, so the caller must be able to fall back. */
|
|
50
|
+
function attach(target, carrier) {
|
|
51
|
+
try {
|
|
52
|
+
Object.defineProperty(target, CELL_SPEND_KEY, {
|
|
53
|
+
value: carrier,
|
|
54
|
+
enumerable: false,
|
|
55
|
+
configurable: true,
|
|
56
|
+
writable: true
|
|
57
|
+
});
|
|
58
|
+
return true;
|
|
59
|
+
} catch {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function messageOf(error) {
|
|
64
|
+
if (error instanceof Error) return error.message;
|
|
65
|
+
return String(error);
|
|
66
|
+
}
|
|
67
|
+
/** Read the spend a thrown value carries. `undefined` when the value carries
|
|
68
|
+
* none, or carries one that does not satisfy `CellSpend` — the runner records
|
|
69
|
+
* both as `uncaptured`. */
|
|
70
|
+
function readCellSpend(error) {
|
|
71
|
+
if (typeof error !== "object" || error === null) return void 0;
|
|
72
|
+
const raw = error[CELL_SPEND_KEY];
|
|
73
|
+
if (typeof raw !== "object" || raw === null) return void 0;
|
|
74
|
+
const candidate = raw;
|
|
75
|
+
if (!isNonnegativeFiniteNumber(candidate.costUsd) || !isNonnegativeFiniteNumber(candidate.durationMs)) return void 0;
|
|
76
|
+
const kind = candidate.kind;
|
|
77
|
+
if (kind !== "observed" && kind !== "estimated" && kind !== "uncaptured") return void 0;
|
|
78
|
+
return {
|
|
79
|
+
costUsd: candidate.costUsd,
|
|
80
|
+
durationMs: candidate.durationMs,
|
|
81
|
+
kind
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
function assertAmount(value, field) {
|
|
85
|
+
if (!isNonnegativeFiniteNumber(value)) throw new TypeError(`withCellSpend: ${field} must be a finite number >= 0, received ${String(value)}`);
|
|
86
|
+
}
|
|
87
|
+
//#endregion
|
|
88
|
+
export { withCellSpend as n, readCellSpend as t };
|
|
89
|
+
|
|
90
|
+
//# sourceMappingURL=cell-spend-DEWR3nj-.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cell-spend-DEWR3nj-.js","names":["isAmount"],"sources":["../src/matrix/cell-spend.ts"],"sourcesContent":["/**\n * Spend carrier for a cell that fails after it has already spent money.\n *\n * `runCell` reports what a cell cost by returning a `CellResult`. A throw\n * carries no result, so the runner cannot know what the cell spent before it\n * failed: the cumulative sum the cost ceiling reads, and `totalCostUsd`, both\n * miss that money. A throw that carries a `CellSpend` closes the gap — the\n * runner bills the failed cell for the spend the carrier declares, and records\n * a cell that declared nothing as `uncaptured` instead of a zero that reads\n * like \"this cell spent nothing\".\n *\n * The carrier is read structurally, never by `instanceof`, so a throw that\n * crosses a package or realm boundary still bills.\n */\n\nimport { isNonnegativeFiniteNumber as isAmount } from '../math/nonnegative-number'\n\n/** Property key the carrier occupies on a thrown object. Stable across\n * versions — two copies of this package must agree on it. */\nconst CELL_SPEND_KEY = '__agentEvalCellSpend'\n\n/** Money and wall time a cell consumed before it threw. */\nexport interface CellSpend {\n /** KNOWN subtotal in USD. When `kind` is `uncaptured` the cell spent this\n * much AND an unknown amount more, so the value must not be read as the\n * cell's total. */\n costUsd: number\n /** Wall time the cell consumed before the throw, in milliseconds. */\n durationMs: number\n /** `observed` = provider-reported amounts, `estimated` = computed from token\n * prices, `uncaptured` = `costUsd` is a subtotal and part of the spend\n * could not be measured. */\n kind: 'observed' | 'estimated' | 'uncaptured'\n}\n\n/**\n * Return `error` carrying `spend`, for a throw site inside `runCell`:\n * `throw withCellSpend(err, { costUsd, durationMs, kind })`.\n *\n * A value that cannot hold the carrier — a thrown string or number, or a\n * frozen, sealed, or otherwise non-extensible object — is wrapped in an\n * `Error` that keeps the original as `cause`, so the spend is never dropped.\n * A later call overwrites an earlier carrier — the outer frame knows more\n * than the inner one.\n *\n * Throws `TypeError` on a non-finite or negative amount. A poisoned amount\n * would reach `cumulativeCost` and disable the cost ceiling for the whole run,\n * because `NaN >= ceiling` is false.\n */\nexport function withCellSpend(error: unknown, spend: CellSpend): unknown {\n assertAmount(spend.costUsd, 'costUsd')\n assertAmount(spend.durationMs, 'durationMs')\n if (spend.kind !== 'observed' && spend.kind !== 'estimated' && spend.kind !== 'uncaptured') {\n throw new TypeError(\n `withCellSpend: kind must be observed, estimated or uncaptured, received ${String(spend.kind)}`,\n )\n }\n const carrier: CellSpend = {\n costUsd: spend.costUsd,\n durationMs: spend.durationMs,\n kind: spend.kind,\n }\n if (typeof error === 'object' && error !== null && attach(error, carrier)) return error\n // A primitive cannot hold a property, and a frozen or sealed object refuses\n // one. Carry the spend on a fresh Error that keeps the original as `cause`\n // rather than lose it — a dropped carrier bills the cell as uncaptured,\n // which is the under-billing this module exists to prevent.\n const wrapper = new Error(`cell failed: ${messageOf(error)}`, { cause: error })\n attach(wrapper, carrier)\n return wrapper\n}\n\n/** True when the carrier landed. `defineProperty` throws on a non-extensible\n * target, so the caller must be able to fall back. */\nfunction attach(target: object, carrier: CellSpend): boolean {\n try {\n Object.defineProperty(target, CELL_SPEND_KEY, {\n value: carrier,\n enumerable: false,\n configurable: true,\n writable: true,\n })\n return true\n } catch {\n return false\n }\n}\n\nfunction messageOf(error: unknown): string {\n if (error instanceof Error) return error.message\n return String(error)\n}\n\n/** Read the spend a thrown value carries. `undefined` when the value carries\n * none, or carries one that does not satisfy `CellSpend` — the runner records\n * both as `uncaptured`. */\nexport function readCellSpend(error: unknown): CellSpend | undefined {\n if (typeof error !== 'object' || error === null) return undefined\n const raw = (error as Record<string, unknown>)[CELL_SPEND_KEY]\n if (typeof raw !== 'object' || raw === null) return undefined\n const candidate = raw as Record<string, unknown>\n if (!isAmount(candidate.costUsd) || !isAmount(candidate.durationMs)) return undefined\n const kind = candidate.kind\n if (kind !== 'observed' && kind !== 'estimated' && kind !== 'uncaptured') return undefined\n return { costUsd: candidate.costUsd, durationMs: candidate.durationMs, kind }\n}\n\n/** A number the matrix can add to a running cost or duration: finite and not\n * negative. Anything else would poison a sum the cost ceiling reads. */\nexport { isNonnegativeFiniteNumber as isAmount } from '../math/nonnegative-number'\n\nfunction assertAmount(value: number, field: string): void {\n if (!isAmount(value)) {\n throw new TypeError(\n `withCellSpend: ${field} must be a finite number >= 0, received ${String(value)}`,\n )\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAmBA,MAAM,iBAAiB;;;;;;;;;;;;;;;AA8BvB,SAAgB,cAAc,OAAgB,OAA2B;CACvE,aAAa,MAAM,SAAS,SAAS;CACrC,aAAa,MAAM,YAAY,YAAY;CAC3C,IAAI,MAAM,SAAS,cAAc,MAAM,SAAS,eAAe,MAAM,SAAS,cAC5E,MAAM,IAAI,UACR,2EAA2E,OAAO,MAAM,IAAI,GAC9F;CAEF,MAAM,UAAqB;EACzB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,MAAM,MAAM;CACd;CACA,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAO,OAAO,OAAO,GAAG,OAAO;CAKlF,MAAM,UAAU,IAAI,MAAM,gBAAgB,UAAU,KAAK,KAAK,EAAE,OAAO,MAAM,CAAC;CAC9E,OAAO,SAAS,OAAO;CACvB,OAAO;AACT;;;AAIA,SAAS,OAAO,QAAgB,SAA6B;CAC3D,IAAI;EACF,OAAO,eAAe,QAAQ,gBAAgB;GAC5C,OAAO;GACP,YAAY;GACZ,cAAc;GACd,UAAU;EACZ,CAAC;EACD,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,UAAU,OAAwB;CACzC,IAAI,iBAAiB,OAAO,OAAO,MAAM;CACzC,OAAO,OAAO,KAAK;AACrB;;;;AAKA,SAAgB,cAAc,OAAuC;CACnE,IAAI,OAAO,UAAU,YAAY,UAAU,MAAM,OAAO,KAAA;CACxD,MAAM,MAAO,MAAkC;CAC/C,IAAI,OAAO,QAAQ,YAAY,QAAQ,MAAM,OAAO,KAAA;CACpD,MAAM,YAAY;CAClB,IAAI,CAACA,0BAAS,UAAU,OAAO,KAAK,CAACA,0BAAS,UAAU,UAAU,GAAG,OAAO,KAAA;CAC5E,MAAM,OAAO,UAAU;CACvB,IAAI,SAAS,cAAc,SAAS,eAAe,SAAS,cAAc,OAAO,KAAA;CACjF,OAAO;EAAE,SAAS,UAAU;EAAS,YAAY,UAAU;EAAY;CAAK;AAC9E;AAMA,SAAS,aAAa,OAAe,OAAqB;CACxD,IAAI,CAACA,0BAAS,KAAK,GACjB,MAAM,IAAI,UACR,kBAAkB,MAAM,0CAA0C,OAAO,KAAK,GAChF;AAEJ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as LlmClient } from "./llm-client-
|
|
1
|
+
import { t as LlmClient } from "./llm-client-BshCREn6.js";
|
|
2
2
|
//#region src/analyst/chat-client.ts
|
|
3
3
|
/**
|
|
4
4
|
* Provider-neutral chat contract for every model call made by agent-eval.
|
|
@@ -114,4 +114,4 @@ function openAiCompatibleClient(opts) {
|
|
|
114
114
|
//#endregion
|
|
115
115
|
export { createChatClient as t };
|
|
116
116
|
|
|
117
|
-
//# sourceMappingURL=chat-client-
|
|
117
|
+
//# sourceMappingURL=chat-client-ldqxYYTN.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat-client-l-MVDlDk.js","names":[],"sources":["../src/analyst/chat-client.ts"],"sourcesContent":["/**\n * Provider-neutral chat contract for every model call made by agent-eval.\n *\n * The caller owns model execution. Evaluation code receives canonical requests\n * and results without importing a provider SDK, and agent-eval reads no\n * environment variable to find a provider credential: every transport is bound\n * at the call site.\n *\n * Two ways to bind one. `custom` and `sandbox-sdk` take a `chat` function the\n * caller wrote. `openai-compatible` takes an endpoint and a credential the\n * caller passes as values, and agent-eval issues the HTTP request itself.\n *\n * WHY THE THIRD ONE EXISTS. Every consumer that wanted a plain OpenAI-style\n * endpoint had to hand-write the same fetch loop — retry on 429 and 5xx,\n * `json_schema` degrade, fenced-JSON stripping, and, load-bearing here, the\n * `servedModel` echo that `assertServedModel` and `assertCrossFamilyServed`\n * read. A hand-written transport that omits `servedModel` makes both identity\n * checks read `unreported`, so the two-vendor judge rule they enforce silently\n * measures nothing. Measured motive: agent-eval 0.172.1 exported no such\n * transport, so a two-model judge panel in discovery-lab reached\n * `createChatClient({ transport: 'custom', baseUrl, apiKey })` — fields that\n * exist on no transport option — and every call threw\n * `TypeError: opts.chat is not a function`, which surfaced as\n * `aggregateJudgeVerdicts: all 2 judges failed`.\n *\n * The credential still never comes from agent-eval's environment. `baseUrl`\n * and the bearer are required arguments with no default and no fallback, so a\n * misconfigured caller gets a refusal at construction, never a call to an\n * endpoint it did not name.\n */\n\nimport {\n type LlmCallRequest,\n type LlmCallResult,\n LlmClient,\n type LlmClientOptions,\n} from '../llm-client'\n\n/**\n * Unified chat interface using the package's canonical LLM request and result.\n */\nexport interface ChatClient {\n /** Display name of the bound transport, included in telemetry. */\n readonly transport: ChatTransport\n /** Default model when the caller omits one. */\n readonly defaultModel?: string\n /** Total provider attempts this transport can make for one chat call. */\n readonly maximumAttempts?: number\n\n /** Implementations must enforce `req.maxTokens` when it is present. */\n chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>\n}\n\nexport type ChatTransport =\n | 'sandbox-sdk' // box.streamPrompt() — chat completion via sandbox SDK\n | 'custom' // caller-adapted SDK or transport\n | 'openai-compatible' // caller-named /v1 endpoint, driven by this package\n | 'mock' // test-time injection\n\nexport interface ChatRequest extends Omit<LlmCallRequest, 'model'> {\n /** Optional — falls back to ChatClient.defaultModel. */\n model?: string\n}\n\nexport type ChatResponse = LlmCallResult\n\nexport interface ChatCallOpts {\n /** Cancel the in-flight request. */\n signal?: AbortSignal\n /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */\n maxCostUsd?: number\n /** Correlation tag carried into request headers when the transport allows. */\n correlationId?: string\n /** Stable provider idempotency key for retries/redrives of one paid call. */\n idempotencyKey?: string\n}\n\n// ── Factory ─────────────────────────────────────────────────────────\n\nexport type CreateChatClientOpts =\n | SandboxSdkTransportOpts\n | CustomTransportOpts\n | OpenAiCompatibleTransportOpts\n | MockTransportOpts\n\ninterface BaseTransportOpts {\n defaultModel?: string\n /** Total provider attempts. Required for opaque transports used in capped runs. */\n maximumAttempts?: number\n}\n\n/**\n * Sandbox-SDK transport. The caller supplies a canonical chat function for an\n * already-configured Sandbox handle, so agent-eval does not import the SDK.\n */\nexport interface SandboxSdkTransportOpts extends BaseTransportOpts {\n transport: 'sandbox-sdk'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */\nexport interface CustomTransportOpts extends BaseTransportOpts {\n transport: 'custom'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * OpenAI-compatible HTTP transport: the caller names a `/v1` endpoint and\n * hands over a bearer, and this package drives `POST {baseUrl}/chat/completions`.\n *\n * Use this instead of hand-rolling a `custom` transport around `fetch`. The\n * result carries `servedModel` — the model id the provider echoed, verbatim —\n * which is the only field `assertServedModel` and `assertCrossFamilyServed`\n * can read to witness a gateway answering from a different model than the one\n * requested.\n *\n * `baseUrl` ends at the `/v1` prefix; the `/chat/completions` path is this\n * package's to append. A `baseUrl` that already carries the path is a\n * construction error, not a request to a doubled URL.\n *\n * Exactly one credential form is required — `apiKey`, `bearer`, or\n * `authHeader`. There is no environment fallback: agent-eval never goes\n * looking for a key.\n */\nexport interface OpenAiCompatibleTransportOpts\n extends BaseTransportOpts,\n Pick<\n LlmClientOptions,\n | 'assertServedModel'\n | 'customTokenPricing'\n | 'deadlineMs'\n | 'defaultTimeoutMs'\n | 'fetch'\n | 'jsonPayloadMode'\n | 'jsonSchemaTransport'\n | 'minIntervalMs'\n | 'provider'\n | 'rawSink'\n | 'signal'\n | 'thinking'\n > {\n transport: 'openai-compatible'\n /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */\n baseUrl: string\n /** Bearer credential. One of `apiKey`, `bearer`, or `authHeader` is required. */\n apiKey?: string\n /** Bearer credential, alternate spelling. */\n bearer?: string\n /** Non-bearer authorization header, for endpoints that want their own scheme. */\n authHeader?: { name: string; value: string }\n}\n\n/**\n * Mock transport for tests. The handler receives the request and returns\n * whatever the test wants. No retries, no JSON-schema degrade.\n */\nexport interface MockTransportOpts extends BaseTransportOpts {\n transport: 'mock'\n handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * Build a ChatClient bound to a specific transport. The returned client\n * is safe to share across analysts in a single registry run.\n */\nexport function createChatClient(opts: CreateChatClientOpts): ChatClient {\n switch (opts.transport) {\n case 'sandbox-sdk':\n return {\n transport: 'sandbox-sdk',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'custom':\n return {\n transport: 'custom',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'openai-compatible':\n return openAiCompatibleClient(opts)\n case 'mock':\n return {\n transport: 'mock',\n defaultModel: opts.defaultModel,\n maximumAttempts: 1,\n chat: async (req, callOpts) => opts.handler(resolveModel(req, opts.defaultModel), callOpts),\n }\n }\n}\n\nfunction resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest {\n if (req.model) return req\n if (!defaultModel) {\n throw new Error(\n 'ChatClient.chat: no model on request and no defaultModel on the client. ' +\n 'Either pass req.model or bind defaultModel at createChatClient().',\n )\n }\n return { ...req, model: defaultModel }\n}\n\n/**\n * Build the OpenAI-compatible client, refusing a half-configured one first.\n *\n * Both refusals are construction-time on purpose. A missing credential or a\n * doubled path is a caller mistake that a request would turn into a 401 or a\n * 404 from an endpoint the caller did not intend to reach — the failure then\n * reads as a provider problem instead of a configuration one.\n */\nfunction openAiCompatibleClient(opts: OpenAiCompatibleTransportOpts): ChatClient {\n const baseUrl = opts.baseUrl?.trim().replace(/\\/+$/, '') ?? ''\n if (!baseUrl) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl is required and has no default.\",\n )\n }\n if (/\\/chat\\/completions$/.test(baseUrl)) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl ends at the /v1 prefix; \" +\n `this package appends /chat/completions. Got ${baseUrl}`,\n )\n }\n if (!opts.apiKey?.trim() && !opts.bearer?.trim() && !opts.authHeader) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): one of apiKey, bearer, or authHeader \" +\n 'is required. agent-eval reads no environment variable for a credential.',\n )\n }\n\n const clientOptions: LlmClientOptions = {\n baseUrl,\n ...(opts.apiKey ? { apiKey: opts.apiKey } : {}),\n ...(opts.bearer ? { bearer: opts.bearer } : {}),\n ...(opts.authHeader ? { authHeader: opts.authHeader } : {}),\n ...(opts.maximumAttempts === undefined ? {} : { maximumAttempts: opts.maximumAttempts }),\n ...(opts.assertServedModel === undefined ? {} : { assertServedModel: opts.assertServedModel }),\n ...(opts.customTokenPricing === undefined\n ? {}\n : { customTokenPricing: opts.customTokenPricing }),\n ...(opts.deadlineMs === undefined ? {} : { deadlineMs: opts.deadlineMs }),\n ...(opts.defaultTimeoutMs === undefined ? {} : { defaultTimeoutMs: opts.defaultTimeoutMs }),\n ...(opts.fetch === undefined ? {} : { fetch: opts.fetch }),\n ...(opts.jsonPayloadMode === undefined ? {} : { jsonPayloadMode: opts.jsonPayloadMode }),\n ...(opts.jsonSchemaTransport === undefined\n ? {}\n : { jsonSchemaTransport: opts.jsonSchemaTransport }),\n ...(opts.minIntervalMs === undefined ? {} : { minIntervalMs: opts.minIntervalMs }),\n ...(opts.provider === undefined ? {} : { provider: opts.provider }),\n ...(opts.rawSink === undefined ? {} : { rawSink: opts.rawSink }),\n ...(opts.signal === undefined ? {} : { signal: opts.signal }),\n ...(opts.thinking === undefined ? {} : { thinking: opts.thinking }),\n }\n const client = new LlmClient(clientOptions)\n\n return {\n transport: 'openai-compatible',\n ...(opts.defaultModel ? { defaultModel: opts.defaultModel } : {}),\n maximumAttempts: client.maximumAttempts,\n chat: async (req, callOpts) => {\n const resolved = resolveModel(req, opts.defaultModel)\n return client.call(resolved as LlmCallRequest, {\n ...(callOpts?.signal ? { signal: callOpts.signal } : {}),\n ...(callOpts?.idempotencyKey ? { idempotencyKey: callOpts.idempotencyKey } : {}),\n })\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqKA,SAAgB,iBAAiB,MAAwC;CACvE,QAAQ,KAAK,WAAb;EACE,KAAK,eACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,UACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,qBACH,OAAO,uBAAuB,IAAI;EACpC,KAAK,QACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB;GACjB,MAAM,OAAO,KAAK,aAAa,KAAK,QAAQ,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EAC5F;CACJ;AACF;AAEA,SAAS,aAAa,KAAkB,cAA+C;CACrF,IAAI,IAAI,OAAO,OAAO;CACtB,IAAI,CAAC,cACH,MAAM,IAAI,MACR,2IAEF;CAEF,OAAO;EAAE,GAAG;EAAK,OAAO;CAAa;AACvC;;;;;;;;;AAUA,SAAS,uBAAuB,MAAiD;CAC/E,MAAM,UAAU,KAAK,SAAS,KAAK,CAAC,CAAC,QAAQ,QAAQ,EAAE,KAAK;CAC5D,IAAI,CAAC,SACH,MAAM,IAAI,MACR,+FACF;CAEF,IAAI,uBAAuB,KAAK,OAAO,GACrC,MAAM,IAAI,MACR,qIACiD,SACnD;CAEF,IAAI,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,YACxD,MAAM,IAAI,MACR,oKAEF;CAGF,MAAM,gBAAkC;EACtC;EACA,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,aAAa,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;EACzD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,sBAAsB,KAAA,IAAY,CAAC,IAAI,EAAE,mBAAmB,KAAK,kBAAkB;EAC5F,GAAI,KAAK,uBAAuB,KAAA,IAC5B,CAAC,IACD,EAAE,oBAAoB,KAAK,mBAAmB;EAClD,GAAI,KAAK,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,YAAY,KAAK,WAAW;EACvE,GAAI,KAAK,qBAAqB,KAAA,IAAY,CAAC,IAAI,EAAE,kBAAkB,KAAK,iBAAiB;EACzF,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EACxD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,wBAAwB,KAAA,IAC7B,CAAC,IACD,EAAE,qBAAqB,KAAK,oBAAoB;EACpD,GAAI,KAAK,kBAAkB,KAAA,IAAY,CAAC,IAAI,EAAE,eAAe,KAAK,cAAc;EAChF,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ;EAC9D,GAAI,KAAK,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO;EAC3D,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;CACnE;CACA,MAAM,SAAS,IAAI,UAAU,aAAa;CAE1C,OAAO;EACL,WAAW;EACX,GAAI,KAAK,eAAe,EAAE,cAAc,KAAK,aAAa,IAAI,CAAC;EAC/D,iBAAiB,OAAO;EACxB,MAAM,OAAO,KAAK,aAAa;GAC7B,MAAM,WAAW,aAAa,KAAK,KAAK,YAAY;GACpD,OAAO,OAAO,KAAK,UAA4B;IAC7C,GAAI,UAAU,SAAS,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;IACtD,GAAI,UAAU,iBAAiB,EAAE,gBAAgB,SAAS,eAAe,IAAI,CAAC;GAChF,CAAC;EACH;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"chat-client-ldqxYYTN.js","names":[],"sources":["../src/analyst/chat-client.ts"],"sourcesContent":["/**\n * Provider-neutral chat contract for every model call made by agent-eval.\n *\n * The caller owns model execution. Evaluation code receives canonical requests\n * and results without importing a provider SDK, and agent-eval reads no\n * environment variable to find a provider credential: every transport is bound\n * at the call site.\n *\n * Two ways to bind one. `custom` and `sandbox-sdk` take a `chat` function the\n * caller wrote. `openai-compatible` takes an endpoint and a credential the\n * caller passes as values, and agent-eval issues the HTTP request itself.\n *\n * WHY THE THIRD ONE EXISTS. Every consumer that wanted a plain OpenAI-style\n * endpoint had to hand-write the same fetch loop — retry on 429 and 5xx,\n * `json_schema` degrade, fenced-JSON stripping, and, load-bearing here, the\n * `servedModel` echo that `assertServedModel` and `assertCrossFamilyServed`\n * read. A hand-written transport that omits `servedModel` makes both identity\n * checks read `unreported`, so the two-vendor judge rule they enforce silently\n * measures nothing. Measured motive: agent-eval 0.172.1 exported no such\n * transport, so a two-model judge panel in discovery-lab reached\n * `createChatClient({ transport: 'custom', baseUrl, apiKey })` — fields that\n * exist on no transport option — and every call threw\n * `TypeError: opts.chat is not a function`, which surfaced as\n * `aggregateJudgeVerdicts: all 2 judges failed`.\n *\n * The credential still never comes from agent-eval's environment. `baseUrl`\n * and the bearer are required arguments with no default and no fallback, so a\n * misconfigured caller gets a refusal at construction, never a call to an\n * endpoint it did not name.\n */\n\nimport {\n type LlmCallRequest,\n type LlmCallResult,\n LlmClient,\n type LlmClientOptions,\n} from '../llm-client'\n\n/**\n * Unified chat interface using the package's canonical LLM request and result.\n */\nexport interface ChatClient {\n /** Display name of the bound transport, included in telemetry. */\n readonly transport: ChatTransport\n /** Default model when the caller omits one. */\n readonly defaultModel?: string\n /** Total provider attempts this transport can make for one chat call. */\n readonly maximumAttempts?: number\n\n /** Implementations must enforce `req.maxTokens` when it is present. */\n chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>\n}\n\nexport type ChatTransport =\n | 'sandbox-sdk' // box.streamPrompt() — chat completion via sandbox SDK\n | 'custom' // caller-adapted SDK or transport\n | 'openai-compatible' // caller-named /v1 endpoint, driven by this package\n | 'mock' // test-time injection\n\nexport interface ChatRequest extends Omit<LlmCallRequest, 'model'> {\n /** Optional — falls back to ChatClient.defaultModel. */\n model?: string\n}\n\nexport type ChatResponse = LlmCallResult\n\nexport interface ChatCallOpts {\n /** Cancel the in-flight request. */\n signal?: AbortSignal\n /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */\n maxCostUsd?: number\n /** Correlation tag carried into request headers when the transport allows. */\n correlationId?: string\n /** Stable provider idempotency key for retries/redrives of one paid call. */\n idempotencyKey?: string\n}\n\n// ── Factory ─────────────────────────────────────────────────────────\n\nexport type CreateChatClientOpts =\n | SandboxSdkTransportOpts\n | CustomTransportOpts\n | OpenAiCompatibleTransportOpts\n | MockTransportOpts\n\ninterface BaseTransportOpts {\n defaultModel?: string\n /** Total provider attempts. Required for opaque transports used in capped runs. */\n maximumAttempts?: number\n}\n\n/**\n * Sandbox-SDK transport. The caller supplies a canonical chat function for an\n * already-configured Sandbox handle, so agent-eval does not import the SDK.\n */\nexport interface SandboxSdkTransportOpts extends BaseTransportOpts {\n transport: 'sandbox-sdk'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */\nexport interface CustomTransportOpts extends BaseTransportOpts {\n transport: 'custom'\n chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * OpenAI-compatible HTTP transport: the caller names a `/v1` endpoint and\n * hands over a bearer, and this package drives `POST {baseUrl}/chat/completions`.\n *\n * Use this instead of hand-rolling a `custom` transport around `fetch`. The\n * result carries `servedModel` — the model id the provider echoed, verbatim —\n * which is the only field `assertServedModel` and `assertCrossFamilyServed`\n * can read to witness a gateway answering from a different model than the one\n * requested.\n *\n * `baseUrl` ends at the `/v1` prefix; the `/chat/completions` path is this\n * package's to append. A `baseUrl` that already carries the path is a\n * construction error, not a request to a doubled URL.\n *\n * Exactly one credential form is required — `apiKey`, `bearer`, or\n * `authHeader`. There is no environment fallback: agent-eval never goes\n * looking for a key.\n */\nexport interface OpenAiCompatibleTransportOpts\n extends BaseTransportOpts,\n Pick<\n LlmClientOptions,\n | 'assertServedModel'\n | 'customTokenPricing'\n | 'deadlineMs'\n | 'defaultTimeoutMs'\n | 'fetch'\n | 'jsonPayloadMode'\n | 'jsonSchemaTransport'\n | 'minIntervalMs'\n | 'provider'\n | 'rawSink'\n | 'signal'\n | 'thinking'\n > {\n transport: 'openai-compatible'\n /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */\n baseUrl: string\n /** Bearer credential. One of `apiKey`, `bearer`, or `authHeader` is required. */\n apiKey?: string\n /** Bearer credential, alternate spelling. */\n bearer?: string\n /** Non-bearer authorization header, for endpoints that want their own scheme. */\n authHeader?: { name: string; value: string }\n}\n\n/**\n * Mock transport for tests. The handler receives the request and returns\n * whatever the test wants. No retries, no JSON-schema degrade.\n */\nexport interface MockTransportOpts extends BaseTransportOpts {\n transport: 'mock'\n handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>\n}\n\n/**\n * Build a ChatClient bound to a specific transport. The returned client\n * is safe to share across analysts in a single registry run.\n */\nexport function createChatClient(opts: CreateChatClientOpts): ChatClient {\n switch (opts.transport) {\n case 'sandbox-sdk':\n return {\n transport: 'sandbox-sdk',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'custom':\n return {\n transport: 'custom',\n defaultModel: opts.defaultModel,\n maximumAttempts: opts.maximumAttempts,\n chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts),\n }\n case 'openai-compatible':\n return openAiCompatibleClient(opts)\n case 'mock':\n return {\n transport: 'mock',\n defaultModel: opts.defaultModel,\n maximumAttempts: 1,\n chat: async (req, callOpts) => opts.handler(resolveModel(req, opts.defaultModel), callOpts),\n }\n }\n}\n\nfunction resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest {\n if (req.model) return req\n if (!defaultModel) {\n throw new Error(\n 'ChatClient.chat: no model on request and no defaultModel on the client. ' +\n 'Either pass req.model or bind defaultModel at createChatClient().',\n )\n }\n return { ...req, model: defaultModel }\n}\n\n/**\n * Build the OpenAI-compatible client, refusing a half-configured one first.\n *\n * Both refusals are construction-time on purpose. A missing credential or a\n * doubled path is a caller mistake that a request would turn into a 401 or a\n * 404 from an endpoint the caller did not intend to reach — the failure then\n * reads as a provider problem instead of a configuration one.\n */\nfunction openAiCompatibleClient(opts: OpenAiCompatibleTransportOpts): ChatClient {\n const baseUrl = opts.baseUrl?.trim().replace(/\\/+$/, '') ?? ''\n if (!baseUrl) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl is required and has no default.\",\n )\n }\n if (/\\/chat\\/completions$/.test(baseUrl)) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): baseUrl ends at the /v1 prefix; \" +\n `this package appends /chat/completions. Got ${baseUrl}`,\n )\n }\n if (!opts.apiKey?.trim() && !opts.bearer?.trim() && !opts.authHeader) {\n throw new Error(\n \"createChatClient({ transport: 'openai-compatible' }): one of apiKey, bearer, or authHeader \" +\n 'is required. agent-eval reads no environment variable for a credential.',\n )\n }\n\n const clientOptions: LlmClientOptions = {\n baseUrl,\n ...(opts.apiKey ? { apiKey: opts.apiKey } : {}),\n ...(opts.bearer ? { bearer: opts.bearer } : {}),\n ...(opts.authHeader ? { authHeader: opts.authHeader } : {}),\n ...(opts.maximumAttempts === undefined ? {} : { maximumAttempts: opts.maximumAttempts }),\n ...(opts.assertServedModel === undefined ? {} : { assertServedModel: opts.assertServedModel }),\n ...(opts.customTokenPricing === undefined\n ? {}\n : { customTokenPricing: opts.customTokenPricing }),\n ...(opts.deadlineMs === undefined ? {} : { deadlineMs: opts.deadlineMs }),\n ...(opts.defaultTimeoutMs === undefined ? {} : { defaultTimeoutMs: opts.defaultTimeoutMs }),\n ...(opts.fetch === undefined ? {} : { fetch: opts.fetch }),\n ...(opts.jsonPayloadMode === undefined ? {} : { jsonPayloadMode: opts.jsonPayloadMode }),\n ...(opts.jsonSchemaTransport === undefined\n ? {}\n : { jsonSchemaTransport: opts.jsonSchemaTransport }),\n ...(opts.minIntervalMs === undefined ? {} : { minIntervalMs: opts.minIntervalMs }),\n ...(opts.provider === undefined ? {} : { provider: opts.provider }),\n ...(opts.rawSink === undefined ? {} : { rawSink: opts.rawSink }),\n ...(opts.signal === undefined ? {} : { signal: opts.signal }),\n ...(opts.thinking === undefined ? {} : { thinking: opts.thinking }),\n }\n const client = new LlmClient(clientOptions)\n\n return {\n transport: 'openai-compatible',\n ...(opts.defaultModel ? { defaultModel: opts.defaultModel } : {}),\n maximumAttempts: client.maximumAttempts,\n chat: async (req, callOpts) => {\n const resolved = resolveModel(req, opts.defaultModel)\n return client.call(resolved as LlmCallRequest, {\n ...(callOpts?.signal ? { signal: callOpts.signal } : {}),\n ...(callOpts?.idempotencyKey ? { idempotencyKey: callOpts.idempotencyKey } : {}),\n })\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqKA,SAAgB,iBAAiB,MAAwC;CACvE,QAAQ,KAAK,WAAb;EACE,KAAK,eACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,UACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB,KAAK;GACtB,MAAM,OAAO,KAAK,aAAa,KAAK,KAAK,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EACzF;EACF,KAAK,qBACH,OAAO,uBAAuB,IAAI;EACpC,KAAK,QACH,OAAO;GACL,WAAW;GACX,cAAc,KAAK;GACnB,iBAAiB;GACjB,MAAM,OAAO,KAAK,aAAa,KAAK,QAAQ,aAAa,KAAK,KAAK,YAAY,GAAG,QAAQ;EAC5F;CACJ;AACF;AAEA,SAAS,aAAa,KAAkB,cAA+C;CACrF,IAAI,IAAI,OAAO,OAAO;CACtB,IAAI,CAAC,cACH,MAAM,IAAI,MACR,2IAEF;CAEF,OAAO;EAAE,GAAG;EAAK,OAAO;CAAa;AACvC;;;;;;;;;AAUA,SAAS,uBAAuB,MAAiD;CAC/E,MAAM,UAAU,KAAK,SAAS,KAAK,CAAC,CAAC,QAAQ,QAAQ,EAAE,KAAK;CAC5D,IAAI,CAAC,SACH,MAAM,IAAI,MACR,+FACF;CAEF,IAAI,uBAAuB,KAAK,OAAO,GACrC,MAAM,IAAI,MACR,qIACiD,SACnD;CAEF,IAAI,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,CAAC,KAAK,YACxD,MAAM,IAAI,MACR,oKAEF;CAGF,MAAM,gBAAkC;EACtC;EACA,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;EAC7C,GAAI,KAAK,aAAa,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;EACzD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,sBAAsB,KAAA,IAAY,CAAC,IAAI,EAAE,mBAAmB,KAAK,kBAAkB;EAC5F,GAAI,KAAK,uBAAuB,KAAA,IAC5B,CAAC,IACD,EAAE,oBAAoB,KAAK,mBAAmB;EAClD,GAAI,KAAK,eAAe,KAAA,IAAY,CAAC,IAAI,EAAE,YAAY,KAAK,WAAW;EACvE,GAAI,KAAK,qBAAqB,KAAA,IAAY,CAAC,IAAI,EAAE,kBAAkB,KAAK,iBAAiB;EACzF,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EACxD,GAAI,KAAK,oBAAoB,KAAA,IAAY,CAAC,IAAI,EAAE,iBAAiB,KAAK,gBAAgB;EACtF,GAAI,KAAK,wBAAwB,KAAA,IAC7B,CAAC,IACD,EAAE,qBAAqB,KAAK,oBAAoB;EACpD,GAAI,KAAK,kBAAkB,KAAA,IAAY,CAAC,IAAI,EAAE,eAAe,KAAK,cAAc;EAChF,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;EACjE,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ;EAC9D,GAAI,KAAK,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO;EAC3D,GAAI,KAAK,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS;CACnE;CACA,MAAM,SAAS,IAAI,UAAU,aAAa;CAE1C,OAAO;EACL,WAAW;EACX,GAAI,KAAK,eAAe,EAAE,cAAc,KAAK,aAAa,IAAI,CAAC;EAC/D,iBAAiB,OAAO;EACxB,MAAM,OAAO,KAAK,aAAa;GAC7B,MAAM,WAAW,aAAa,KAAK,KAAK,YAAY;GACpD,OAAO,OAAO,KAAK,UAA4B;IAC7C,GAAI,UAAU,SAAS,EAAE,QAAQ,SAAS,OAAO,IAAI,CAAC;IACtD,GAAI,UAAU,iBAAiB,EAAE,gBAAgB,SAAS,eAAe,IAAI,CAAC;GAChF,CAAC;EACH;CACF;AACF"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as extractJsonPayload } from "./json-BhdsJaOQ.js";
|
|
2
|
+
import { a as maximumChargeForLlmRequest, i as costReceiptFromLlmError, r as costReceiptFromLlm } from "./llm-client-BshCREn6.js";
|
|
2
3
|
//#region src/chat-json-call.ts
|
|
3
4
|
/** Parse a JSON answer out of a model response. The transport may fence it. */
|
|
4
5
|
function parseJsonAnswer(response, actor) {
|
|
@@ -60,4 +61,4 @@ async function paidJsonChat(input) {
|
|
|
60
61
|
//#endregion
|
|
61
62
|
export { paidJsonChat as n, paidChat as t };
|
|
62
63
|
|
|
63
|
-
//# sourceMappingURL=chat-json-call-
|
|
64
|
+
//# sourceMappingURL=chat-json-call-DavzexSp.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chat-json-call-DavzexSp.js","names":[],"sources":["../src/chat-json-call.ts"],"sourcesContent":["/**\n * One paid model call through a caller-owned `ChatClient`, metered by the\n * cost ledger.\n *\n * agent-eval executes no paid model: the transport is supplied by the caller\n * and the credential never enters this package. What stays here is the\n * accounting around the call — the priced maximum reserved before it runs, the\n * stable call id forwarded as the provider idempotency key, the receipt\n * settled from the response, and an honest unknown-usage receipt when the\n * transport failed.\n *\n * The judges, the wire judge endpoint, and the in-process trace-analysis\n * engine all make the same call in the same order; this is the one copy of\n * that sequence. `paidChat` is the sequence; `paidJsonChat` adds the JSON\n * answer parse that a structured caller needs.\n */\n\nimport type { ChatClient, ChatResponse } from './analyst/chat-client'\nimport type { CostChannel, CostLedgerHandle, CostReceipt, CustomTokenPricing } from './cost-ledger'\nimport {\n costReceiptFromLlm,\n costReceiptFromLlmError,\n extractJsonPayload,\n type LlmCallRequest,\n maximumChargeForLlmRequest,\n} from './llm-client'\n\ninterface PaidChatInput {\n /** Caller-owned transport. One `chat()` call. */\n chat: ChatClient\n /** The exact canonical request, including its JSON-mode or schema fields. */\n request: LlmCallRequest\n ledger: CostLedgerHandle\n channel: CostChannel\n phase: string\n actor: string\n tags?: Record<string, string>\n signal?: AbortSignal\n /** Endpoint rates used when the transport reports no billed amount. */\n pricing?: CustomTokenPricing\n}\n\ntype PaidJsonChatInput = PaidChatInput\n\ntype PaidChatResult =\n | { succeeded: true; response: ChatResponse; receipt: CostReceipt }\n | { succeeded: false; error: Error; receipt?: CostReceipt }\n\ntype PaidJsonChatResult<T> =\n | { succeeded: true; value: T; response: ChatResponse; receipt: CostReceipt }\n | { succeeded: false; error: Error; receipt?: CostReceipt }\n\n/** Parse a JSON answer out of a model response. The transport may fence it. */\nfunction parseJsonAnswer<T>(response: ChatResponse, actor: string): T {\n try {\n return JSON.parse(extractJsonPayload(response.content)) as T\n } catch (error) {\n throw new Error(\n `${actor}: model answer was not JSON — ${error instanceof Error ? error.message : String(error)}`,\n )\n }\n}\n\n/** Reserve, execute, and settle one caller-owned chat call. */\nexport async function paidChat(input: PaidChatInput): Promise<PaidChatResult> {\n const paid = await input.ledger.runPaidCall({\n channel: input.channel,\n phase: input.phase,\n actor: input.actor,\n model: input.request.model,\n ...(input.tags && Object.keys(input.tags).length > 0 ? { tags: input.tags } : {}),\n maximumCharge: maximumChargeForLlmRequest(input.request, {\n ...(input.chat.maximumAttempts === undefined\n ? {}\n : { maximumAttempts: input.chat.maximumAttempts }),\n ...(input.pricing ? { customTokenPricing: input.pricing } : {}),\n }),\n ...(input.signal ? { signal: input.signal } : {}),\n execute: (signal, callId) => input.chat.chat(input.request, { signal, idempotencyKey: callId }),\n receipt: (response) => costReceiptFromLlm(response, input.pricing),\n receiptFromError: (error) => costReceiptFromLlmError(error, input.pricing),\n })\n if (!paid.succeeded) {\n return {\n succeeded: false,\n error: paid.error,\n ...(paid.receipt ? { receipt: paid.receipt } : {}),\n }\n }\n return { succeeded: true, response: paid.value, receipt: paid.receipt }\n}\n\nexport async function paidJsonChat<T>(input: PaidJsonChatInput): Promise<PaidJsonChatResult<T>> {\n const paid = await paidChat(input)\n if (!paid.succeeded) return paid\n // The call completed and was billed. A malformed answer is a contract\n // failure AFTER the money was spent, so it keeps the settled receipt instead\n // of reporting the spend as unknown.\n try {\n return {\n succeeded: true,\n value: parseJsonAnswer<T>(paid.response, input.actor),\n response: paid.response,\n receipt: paid.receipt,\n }\n } catch (error) {\n return {\n succeeded: false,\n error: error instanceof Error ? error : new Error(String(error)),\n receipt: paid.receipt,\n }\n }\n}\n"],"mappings":";;;;AAqDA,SAAS,gBAAmB,UAAwB,OAAkB;CACpE,IAAI;EACF,OAAO,KAAK,MAAM,mBAAmB,SAAS,OAAO,CAAC;CACxD,SAAS,OAAO;EACd,MAAM,IAAI,MACR,GAAG,MAAM,gCAAgC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAChG;CACF;AACF;;AAGA,eAAsB,SAAS,OAA+C;CAC5E,MAAM,OAAO,MAAM,MAAM,OAAO,YAAY;EAC1C,SAAS,MAAM;EACf,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM,QAAQ;EACrB,GAAI,MAAM,QAAQ,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;EAC/E,eAAe,2BAA2B,MAAM,SAAS;GACvD,GAAI,MAAM,KAAK,oBAAoB,KAAA,IAC/B,CAAC,IACD,EAAE,iBAAiB,MAAM,KAAK,gBAAgB;GAClD,GAAI,MAAM,UAAU,EAAE,oBAAoB,MAAM,QAAQ,IAAI,CAAC;EAC/D,CAAC;EACD,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;EAC/C,UAAU,QAAQ,WAAW,MAAM,KAAK,KAAK,MAAM,SAAS;GAAE;GAAQ,gBAAgB;EAAO,CAAC;EAC9F,UAAU,aAAa,mBAAmB,UAAU,MAAM,OAAO;EACjE,mBAAmB,UAAU,wBAAwB,OAAO,MAAM,OAAO;CAC3E,CAAC;CACD,IAAI,CAAC,KAAK,WACR,OAAO;EACL,WAAW;EACX,OAAO,KAAK;EACZ,GAAI,KAAK,UAAU,EAAE,SAAS,KAAK,QAAQ,IAAI,CAAC;CAClD;CAEF,OAAO;EAAE,WAAW;EAAM,UAAU,KAAK;EAAO,SAAS,KAAK;CAAQ;AACxE;AAEA,eAAsB,aAAgB,OAA0D;CAC9F,MAAM,OAAO,MAAM,SAAS,KAAK;CACjC,IAAI,CAAC,KAAK,WAAW,OAAO;CAI5B,IAAI;EACF,OAAO;GACL,WAAW;GACX,OAAO,gBAAmB,KAAK,UAAU,MAAM,KAAK;GACpD,UAAU,KAAK;GACf,SAAS,KAAK;EAChB;CACF,SAAS,OAAO;EACd,OAAO;GACL,WAAW;GACX,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GAC/D,SAAS,KAAK;EAChB;CACF;AACF"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { s as ValidationError } from "./errors-
|
|
2
|
-
import { i as compareCodeUnits } from "./canonical-
|
|
1
|
+
import { s as ValidationError } from "./errors-BcOs5MKW.js";
|
|
2
|
+
import { i as compareCodeUnits } from "./canonical-Dq2hOQ8L.js";
|
|
3
3
|
import { r as readField } from "./predicate-FdKmElA_.js";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
//#region src/experiment/claim.ts
|
|
@@ -51,4 +51,4 @@ function summarizeEvaluationUnits(claim, rows) {
|
|
|
51
51
|
//#endregion
|
|
52
52
|
export { evaluationClaimSchema as n, summarizeEvaluationUnits as r, defineEvaluationClaim as t };
|
|
53
53
|
|
|
54
|
-
//# sourceMappingURL=claim-
|
|
54
|
+
//# sourceMappingURL=claim-BPVuyHuy.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claim-
|
|
1
|
+
{"version":3,"file":"claim-BPVuyHuy.js","names":[],"sources":["../src/experiment/claim.ts"],"sourcesContent":["import { z } from 'zod'\nimport { ValidationError } from '../errors'\nimport { compareCodeUnits } from '../ledger-core/canonical'\nimport type { EvidenceRecord } from './ast/evidence'\nimport { readField } from './ast/predicate'\n\nconst nonEmpty = z\n .string()\n .min(1)\n .refine((value) => value.trim() === value)\n\n/** The claim's schema, shared by every record that binds a claim. */\nexport const evaluationClaimSchema = z\n .object({\n use: z.enum(['development', 'comparison', 'certification']),\n population: z.object({ id: nonEmpty, description: nonEmpty }).strict(),\n samplingFrame: nonEmpty,\n independentUnit: nonEmpty,\n generalization: z.enum(['fixed-roster', 'new-units']),\n minimumEffect: z.number().finite().positive().optional(),\n })\n .strict()\n\n/** The population and independent observations a measured result can describe. */\nexport type EvaluationClaim = z.infer<typeof evaluationClaimSchema>\n\n/** Validate a claim before binding it to a sealed experiment or final evidence. */\nexport function defineEvaluationClaim(input: EvaluationClaim): EvaluationClaim {\n const parsed = evaluationClaimSchema.safeParse(input)\n if (!parsed.success)\n throw new ValidationError(`invalid evaluation claim: ${parsed.error.message}`)\n return Object.freeze({ ...parsed.data, population: Object.freeze(parsed.data.population) })\n}\n\nexport interface EvaluationUnitSummary {\n observations: number\n independentUnits: number\n units: Array<{ id: string; observations: number }>\n}\n\n/** Repetitions retain their denominator without becoming additional independent units. */\nexport function summarizeEvaluationUnits(\n claim: EvaluationClaim,\n rows: readonly object[],\n): EvaluationUnitSummary {\n const validated = defineEvaluationClaim(claim)\n const counts = new Map<string, number>()\n for (const [index, row] of rows.entries()) {\n if (row === null || typeof row !== 'object' || Array.isArray(row)) {\n throw new ValidationError(`evaluation claim: row ${index} must be an object`)\n }\n const value = readField(row as EvidenceRecord, validated.independentUnit)\n if (typeof value !== 'string' || !value.trim() || value.trim() !== value) {\n throw new ValidationError(\n `evaluation claim: row ${index} needs a nonempty string at '${validated.independentUnit}'`,\n )\n }\n counts.set(value, (counts.get(value) ?? 0) + 1)\n }\n return {\n observations: rows.length,\n independentUnits: counts.size,\n units: [...counts]\n .sort(([left], [right]) => compareCodeUnits(left, right))\n .map(([id, observations]) => ({ id, observations })),\n }\n}\n"],"mappings":";;;;;AAMA,MAAM,WAAW,EACd,OAAO,CAAC,CACR,IAAI,CAAC,CAAC,CACN,QAAQ,UAAU,MAAM,KAAK,MAAM,KAAK;;AAG3C,MAAa,wBAAwB,EAClC,OAAO;CACN,KAAK,EAAE,KAAK;EAAC;EAAe;EAAc;CAAe,CAAC;CAC1D,YAAY,EAAE,OAAO;EAAE,IAAI;EAAU,aAAa;CAAS,CAAC,CAAC,CAAC,OAAO;CACrE,eAAe;CACf,iBAAiB;CACjB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,WAAW,CAAC;CACpD,eAAe,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;AACzD,CAAC,CAAC,CACD,OAAO;;AAMV,SAAgB,sBAAsB,OAAyC;CAC7E,MAAM,SAAS,sBAAsB,UAAU,KAAK;CACpD,IAAI,CAAC,OAAO,SACV,MAAM,IAAI,gBAAgB,6BAA6B,OAAO,MAAM,SAAS;CAC/E,OAAO,OAAO,OAAO;EAAE,GAAG,OAAO;EAAM,YAAY,OAAO,OAAO,OAAO,KAAK,UAAU;CAAE,CAAC;AAC5F;;AASA,SAAgB,yBACd,OACA,MACuB;CACvB,MAAM,YAAY,sBAAsB,KAAK;CAC7C,MAAM,yBAAS,IAAI,IAAoB;CACvC,KAAK,MAAM,CAAC,OAAO,QAAQ,KAAK,QAAQ,GAAG;EACzC,IAAI,QAAQ,QAAQ,OAAO,QAAQ,YAAY,MAAM,QAAQ,GAAG,GAC9D,MAAM,IAAI,gBAAgB,yBAAyB,MAAM,mBAAmB;EAE9E,MAAM,QAAQ,UAAU,KAAuB,UAAU,eAAe;EACxE,IAAI,OAAO,UAAU,YAAY,CAAC,MAAM,KAAK,KAAK,MAAM,KAAK,MAAM,OACjE,MAAM,IAAI,gBACR,yBAAyB,MAAM,+BAA+B,UAAU,gBAAgB,EAC1F;EAEF,OAAO,IAAI,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CAChD;CACA,OAAO;EACL,cAAc,KAAK;EACnB,kBAAkB,OAAO;EACzB,OAAO,CAAC,GAAG,MAAM,CAAC,CACf,MAAM,CAAC,OAAO,CAAC,WAAW,iBAAiB,MAAM,KAAK,CAAC,CAAC,CACxD,KAAK,CAAC,IAAI,mBAAmB;GAAE;GAAI;EAAa,EAAE;CACvD;AACF"}
|
package/dist/cli.js
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { t as
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { t as
|
|
2
|
+
import { c as errorMessage } from "./errors-BcOs5MKW.js";
|
|
3
|
+
import { l as operatorYield, u as INSUFFICIENT_FROM } from "./pareto-BzsxidFD.js";
|
|
4
|
+
import { i as replaySearchLedgerText } from "./search-ledger-TUvB5gWH.js";
|
|
5
|
+
import { t as renderSearchSummary } from "./search-summary-DQIvJsSB.js";
|
|
6
|
+
import { l as SEARCH_LEDGER_BATCH_MAX_BYTES, m as SearchRunKindSchema, r as shipSearchLedger, s as hostedTenantFromEnv, t as SearchShipConflictError } from "./search-shipper-Cw2z3-td.js";
|
|
7
|
+
import { n as runRolloutReleaseCli } from "./hf-dataset-D9f7-oX7.js";
|
|
8
|
+
import { t as createChatClient } from "./chat-client-ldqxYYTN.js";
|
|
9
|
+
import { t as runAnalystBenchmarkCommand } from "./benchmark-command-xQK8nOI6.js";
|
|
10
|
+
import { i as buildOpenApi, l as handleVersion, o as runRpcBatch, r as startServerAsync, s as runRpcOnce } from "./server-DoVbqqtJ.js";
|
|
11
|
+
import { b as editCreditText, f as landscape, g as surfaceTextEdits, l as metaSearch, m as formatLandscape, n as taskMatrix, o as skillManifold, r as formatSkillManifold, s as renderMetaSearchText, t as tree, v as front, y as editCredit } from "./tree-BGRglkKd.js";
|
|
12
|
+
import { i as analyzeSupervisorRun, n as renderSupervisorRunHeadline, r as renderSupervisorRunMarkdown } from "./render-C-w2EZ6p.js";
|
|
12
13
|
import { createHash } from "node:crypto";
|
|
13
14
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
14
|
-
import { dirname, join } from "node:path";
|
|
15
|
+
import { dirname, join, resolve } from "node:path";
|
|
15
16
|
import { fileURLToPath } from "node:url";
|
|
16
|
-
import { open, readFile } from "node:fs/promises";
|
|
17
|
+
import { open, readFile, stat } from "node:fs/promises";
|
|
17
18
|
//#region src/cli-config.ts
|
|
18
19
|
/**
|
|
19
20
|
* Provider configuration for the `agent-eval` binary.
|
|
@@ -435,6 +436,106 @@ async function firstLineSearchId(path) {
|
|
|
435
436
|
}
|
|
436
437
|
}
|
|
437
438
|
//#endregion
|
|
439
|
+
//#region src/supervisor-run/report-command.ts
|
|
440
|
+
/**
|
|
441
|
+
* `agent-eval supervisor-run report <runDir>` takes one run directory and
|
|
442
|
+
* prints the report the module already renders. The command reads through
|
|
443
|
+
* `analyzeSupervisorRun`, so a Runtime run directory takes the
|
|
444
|
+
* same path and the status comes from the record `terminal-record.ts` names.
|
|
445
|
+
*
|
|
446
|
+
* Exit codes follow the other self-parsing subcommands: 2 for a usage error,
|
|
447
|
+
* 1 when the directory cannot be read (missing, not a directory, or a corrupt
|
|
448
|
+
* journal or terminal record), 0 after the report was written to stdout.
|
|
449
|
+
*/
|
|
450
|
+
const FORMATS = [
|
|
451
|
+
"headline",
|
|
452
|
+
"markdown",
|
|
453
|
+
"json"
|
|
454
|
+
];
|
|
455
|
+
const SUPERVISOR_RUN_USAGE = "usage: agent-eval supervisor-run report <runDir> [--format headline|markdown|json]";
|
|
456
|
+
const PROCESS_IO = {
|
|
457
|
+
stdout: (text) => {
|
|
458
|
+
process.stdout.write(text);
|
|
459
|
+
},
|
|
460
|
+
stderr: (text) => {
|
|
461
|
+
process.stderr.write(text);
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
function isFormat(value) {
|
|
465
|
+
return FORMATS.includes(value);
|
|
466
|
+
}
|
|
467
|
+
function parseReportArgs(argv) {
|
|
468
|
+
let runDir = null;
|
|
469
|
+
let format = "headline";
|
|
470
|
+
for (let i = 0; i < argv.length; i++) {
|
|
471
|
+
const arg = argv[i];
|
|
472
|
+
if (arg === "--format") {
|
|
473
|
+
const raw = argv[++i];
|
|
474
|
+
if (raw === void 0 || !isFormat(raw)) throw new Error(`--format expects one of ${FORMATS.join("|")}, got ${JSON.stringify(raw ?? "")}`);
|
|
475
|
+
format = raw;
|
|
476
|
+
} else if (arg.startsWith("--format=")) {
|
|
477
|
+
const raw = arg.slice(9);
|
|
478
|
+
if (!isFormat(raw)) throw new Error(`--format expects one of ${FORMATS.join("|")}, got ${JSON.stringify(raw)}`);
|
|
479
|
+
format = raw;
|
|
480
|
+
} else if (arg.startsWith("--")) throw new Error(`unknown flag "${arg}"`);
|
|
481
|
+
else if (runDir === null) runDir = arg;
|
|
482
|
+
else throw new Error(`unexpected argument "${arg}"`);
|
|
483
|
+
}
|
|
484
|
+
if (runDir === null) throw new Error(SUPERVISOR_RUN_USAGE);
|
|
485
|
+
return {
|
|
486
|
+
runDir,
|
|
487
|
+
format
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* CLI driver for `agent-eval supervisor-run`. `argv` is everything after the
|
|
492
|
+
* command name, so `['report', runDir, ...flags]`. Returns the process exit code.
|
|
493
|
+
*/
|
|
494
|
+
async function runSupervisorRunCommand(argv, io = PROCESS_IO) {
|
|
495
|
+
const [verb, ...rest] = argv;
|
|
496
|
+
if (verb === void 0 || verb === "--help" || verb === "-h" || verb === "help") {
|
|
497
|
+
io.stdout(`${SUPERVISOR_RUN_USAGE}\n`);
|
|
498
|
+
return verb === void 0 ? 2 : 0;
|
|
499
|
+
}
|
|
500
|
+
if (verb !== "report") {
|
|
501
|
+
io.stderr(`unknown supervisor-run subcommand "${verb}"\n${SUPERVISOR_RUN_USAGE}\n`);
|
|
502
|
+
return 2;
|
|
503
|
+
}
|
|
504
|
+
if (rest.includes("--help") || rest.includes("-h")) {
|
|
505
|
+
io.stdout(`${SUPERVISOR_RUN_USAGE}\n`);
|
|
506
|
+
return 0;
|
|
507
|
+
}
|
|
508
|
+
let args;
|
|
509
|
+
try {
|
|
510
|
+
args = parseReportArgs(rest);
|
|
511
|
+
} catch (error) {
|
|
512
|
+
io.stderr(`${errorMessage(error)}\n`);
|
|
513
|
+
return 2;
|
|
514
|
+
}
|
|
515
|
+
const runDir = resolve(args.runDir);
|
|
516
|
+
const entry = await stat(runDir).catch(() => null);
|
|
517
|
+
if (entry === null || !entry.isDirectory()) {
|
|
518
|
+
io.stderr(`[agent-eval] supervisor-run report: ${runDir} is not a directory\n`);
|
|
519
|
+
return 1;
|
|
520
|
+
}
|
|
521
|
+
try {
|
|
522
|
+
const report = await analyzeSupervisorRun(runDir);
|
|
523
|
+
switch (args.format) {
|
|
524
|
+
case "headline":
|
|
525
|
+
io.stdout(`${renderSupervisorRunHeadline(report)}\n`);
|
|
526
|
+
break;
|
|
527
|
+
case "markdown":
|
|
528
|
+
io.stdout(renderSupervisorRunMarkdown(report));
|
|
529
|
+
break;
|
|
530
|
+
case "json": io.stdout(`${JSON.stringify(report, null, 2)}\n`);
|
|
531
|
+
}
|
|
532
|
+
return 0;
|
|
533
|
+
} catch (error) {
|
|
534
|
+
io.stderr(`[agent-eval] supervisor-run report failed for ${runDir}: ${errorMessage(error)}\n`);
|
|
535
|
+
return 1;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
//#endregion
|
|
438
539
|
//#region src/cli.ts
|
|
439
540
|
/**
|
|
440
541
|
* agent-eval CLI.
|