@tangle-network/agent-eval 0.216.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DjlZOVXx.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-BupdHRyf.js} +1225 -661
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CGvUTnmc.js} +324 -800
- package/dist/campaign-CGvUTnmc.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-DunzcJGv.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-Dm9c-b41.js} +3 -2
- package/dist/chat-json-call-Dm9c-b41.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-DbLVf3i8.js} +9 -64
- package/dist/completion-verifier-DbLVf3i8.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +107 -403
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-BJ-Pq6yL.js} +314 -285
- package/dist/default-registry-BJ-Pq6yL.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BGdFKlrc.js} +170 -110
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BlH10zEw.d.ts} +8 -8
- package/dist/define-agent-eval-BlH10zEw.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-CSMwudLO.js} +5 -10
- package/dist/dspy-rlm-engine-CSMwudLO.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-BOZ38vOH.d.ts} +7 -7
- package/dist/evaluation-BOZ38vOH.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +258 -311
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-DWvM16LE.js} +3 -3
- package/dist/exporters-DWvM16LE.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-C3Wm7Vhn.js} +4 -3
- package/dist/external-optimizer-process-C3Wm7Vhn.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-B5gdLRZv.js} +9 -11
- package/dist/external-optimizer-subprocess-B5gdLRZv.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BtTzPL0d.js} +9 -249
- package/dist/findings-store-BtTzPL0d.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-CKxIJQba.js} +4 -24
- package/dist/hf-dataset-CKxIJQba.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-CyszSg4O.d.ts → index-B16FNdW5.d.ts} +145 -360
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-DHKEAv3s.d.ts → index-CalAsXMR.d.ts} +8 -68
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-DPcngtwt.d.ts → index-DbMOXYnP.d.ts} +11 -36
- package/dist/index-DbMOXYnP.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +999 -1714
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-C8oUeB0g.js} +326 -203
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-C7om8Gum.js} +157 -147
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-CAxUdDk0.js} +5 -84
- package/dist/ledger-core-CAxUdDk0.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-_Y6ic-tf.js} +243 -355
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-Djxh-Uhs.js} +662 -482
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-CCSBna4q.d.ts} +9 -18
- package/dist/matrix-CCSBna4q.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-CbdwjTPW.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-CTv8vozZ.js} +11 -11
- package/dist/pareto-CTv8vozZ.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-CxEHRsZW.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-n-5WBbEx.js} +3 -3
- package/dist/proposal-findings-n-5WBbEx.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-Cs34y0iX.js} +625 -587
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-GU1tV4dn.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-Dd1ucT_M.js} +88 -13
- package/dist/rollout-Dd1ucT_M.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-BuiujZ-g.js → schema-CvhfLjtZ.js} +78 -48
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-p9JB3SmL.d.ts} +10 -27
- package/dist/search-kernel-p9JB3SmL.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-ClpN4I3e.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-Bd5Tioq6.js} +4 -4
- package/dist/search-ledger-file-Bd5Tioq6.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Bg2HW3Uv.js} +4 -4
- package/dist/search-shipper-Bg2HW3Uv.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-CrBuvVnf.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-C6M0qR8h.js} +101 -101
- package/dist/server-C6M0qR8h.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-5dlex7MJ.js} +1386 -1184
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-BHfLZHrm.d.ts} +8 -67
- package/dist/statistical-heldout-BHfLZHrm.d.ts.map +1 -0
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-CkbRcSuu.d.ts} +25 -19
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-BMzDJb1i.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-D4Rv-j1W.js} +244 -196
- package/dist/terminal-record-D4Rv-j1W.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +7 -118
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +921 -869
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-B5e1uAeL.js} +132 -21
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-BKCzzmit.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-DtkO1G9L.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +18 -18
- package/docs/design.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/evaluation-integrity.md +1 -1
- package/docs/profile-json-identity.md +1 -2
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +158 -538
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +7 -2
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-CNTPzIkp.js.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/eval-campaign-90jeaCAI.js +0 -343
- package/dist/eval-campaign-90jeaCAI.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/single-run-lock-CV9Eyd_q.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,42 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { l as ExactCapableAnalyst } from "./exact-types-
|
|
3
|
-
import { i as
|
|
4
|
-
import {
|
|
5
|
-
import { t as
|
|
6
|
-
//#region src/trace-analyst/behavioral-metrics.d.ts
|
|
7
|
-
type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
|
|
8
|
-
interface SuboptimalSignal {
|
|
9
|
-
code: SuboptimalCode;
|
|
10
|
-
severity: 'high' | 'medium' | 'low';
|
|
11
|
-
/** Human-readable claim, with the backing numbers inlined. */
|
|
12
|
-
detail: string;
|
|
13
|
-
/** The exact figures the detector fired on — auditable, no model in the loop. */
|
|
14
|
-
evidence: Record<string, number | string | boolean>;
|
|
15
|
-
}
|
|
16
|
-
interface BehavioralMetrics {
|
|
17
|
-
/** The only trace represented by these metrics; null when spans are empty. */
|
|
18
|
-
traceId: string | null;
|
|
19
|
-
llmCallCount: number;
|
|
20
|
-
/** Causally serial LLM timelines. Parallel branches are never joined. */
|
|
21
|
-
tokenSequences: BehavioralTokenSequence[];
|
|
22
|
-
/** Token values from the longest serial timeline, retained for convenience. */
|
|
23
|
-
inputTokenTrajectory: number[];
|
|
24
|
-
outputTokenTrajectory: number[];
|
|
25
|
-
toolHistogram: Record<string, number>;
|
|
26
|
-
totalToolCalls: number;
|
|
27
|
-
distinctTools: number;
|
|
28
|
-
/** distinct/total tool calls; 1.0 when there are no tool calls. */
|
|
29
|
-
toolDiversityRatio: number;
|
|
30
|
-
hasSelfVerification: boolean;
|
|
31
|
-
signals: SuboptimalSignal[];
|
|
32
|
-
}
|
|
33
|
-
interface BehavioralTokenSequence {
|
|
34
|
-
scopeId: string;
|
|
35
|
-
spanIds: string[];
|
|
36
|
-
inputTokenTrajectory: Array<number | null>;
|
|
37
|
-
outputTokenTrajectory: Array<number | null>;
|
|
38
|
-
}
|
|
39
|
-
//#endregion
|
|
1
|
+
import { W as TraceAnalysisStore, n as AnalystContext } from "./types-CW8ewATZ.js";
|
|
2
|
+
import { l as ExactCapableAnalyst } from "./exact-types-CflbJ9f3.js";
|
|
3
|
+
import { i as TraceAnalystLimits, r as TraceAnalysisEngineResult, s as RawAnalystFinding, t as TraceAnalysisEngine } from "./engine-BvZ0R-36.js";
|
|
4
|
+
import { t as TraceToolGroupName } from "./tool-groups-DyNz2JZ_.js";
|
|
5
|
+
import { i as AnalystRegistryOptions, t as AnalystRegistry } from "./registry-ssNoHzwD.js";
|
|
40
6
|
//#region src/analyst/behavioral-analyst.d.ts
|
|
41
7
|
interface BehavioralAnalystOptions {
|
|
42
8
|
/** Refuse larger unfiltered datasets instead of sampling them silently. Default 1,000. */
|
|
@@ -44,14 +10,6 @@ interface BehavioralAnalystOptions {
|
|
|
44
10
|
/** Evidence locations retained per finding. Counts still cover every analyzed trace. Default 20. */
|
|
45
11
|
maxEvidenceRefsPerFinding?: number;
|
|
46
12
|
}
|
|
47
|
-
/**
|
|
48
|
-
* Map computed signals → structured AnalystFindings. Pure: no LLM, no clock
|
|
49
|
-
* dependence beyond `produced_at` (overridable for deterministic tests).
|
|
50
|
-
*/
|
|
51
|
-
declare function deriveEfficiencyFindings(metrics: BehavioralMetrics, opts?: {
|
|
52
|
-
analystId?: string;
|
|
53
|
-
producedAt?: string;
|
|
54
|
-
}): AnalystFinding[];
|
|
55
13
|
/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */
|
|
56
14
|
declare function behavioralAnalyst(options?: BehavioralAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
|
|
57
15
|
//#endregion
|
|
@@ -91,7 +49,6 @@ declare function runTraceAnalyst(args: {
|
|
|
91
49
|
}): Promise<TraceAnalysisEngineResult>;
|
|
92
50
|
/** Adapt a research definition to the common Analyst registry contract. */
|
|
93
51
|
declare function createTraceAnalyst(definition: TraceAnalystDefinition, options: CreateTraceAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
|
|
94
|
-
declare function renderPriorFindings(prior: AnalystContext['priorFindings']): string;
|
|
95
52
|
//#endregion
|
|
96
53
|
//#region src/analyst/default-registry.d.ts
|
|
97
54
|
interface DefaultAnalystRegistryOptions {
|
|
@@ -109,5 +66,5 @@ interface DefaultAnalystRegistryOptions {
|
|
|
109
66
|
}
|
|
110
67
|
declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
|
|
111
68
|
//#endregion
|
|
112
|
-
export { createTraceAnalyst as a,
|
|
113
|
-
//# sourceMappingURL=default-registry-
|
|
69
|
+
export { createTraceAnalyst as a, behavioralAnalyst as c, TraceAnalystDefinition as i, buildDefaultAnalystRegistry as n, runTraceAnalyst as o, CreateTraceAnalystOptions as r, BehavioralAnalystOptions as s, DefaultAnalystRegistryOptions as t };
|
|
70
|
+
//# sourceMappingURL=default-registry-DXgLmoDI.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"default-registry-DXgLmoDI.d.ts","names":[],"sources":["../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;UA+BiB;;EAEf;;EAEA;;;iBAoGc,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgHI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;;;UCrKN;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { s as ValidationError } from "./errors-
|
|
2
|
-
import { i as modelHasSnapshot } from "./run-record-
|
|
3
|
-
import { t as analyzeRuns } from "./analyze-runs-
|
|
4
|
-
import {
|
|
5
|
-
import { n as campaignCellToRunRecord } from "./run-record-
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { t as defineEvaluationClaim } from "./claim-
|
|
9
|
-
import {
|
|
1
|
+
import { s as ValidationError } from "./errors-BcOs5MKW.js";
|
|
2
|
+
import { i as modelHasSnapshot } from "./run-record-DhEJgoPz.js";
|
|
3
|
+
import { t as analyzeRuns } from "./analyze-runs-DqPJaJ4X.js";
|
|
4
|
+
import { B as defaultProductionGate, E as assertSearchHistoryAdmissionOptions, J as evaluationUnitMap, K as assertIndependentEvaluationSplit, L as runFinalComparison, R as openAutoPr, X as reserveFinalEvidence, Z as runEval, d as costFromLedgerSummary, et as resolveRunDir, p as runImprovementLoop, q as captureFinalEvidencePolicy, s as executeOptimizationMethod, u as combineComparisonCosts, z as captureJudge } from "./llm-judge-Djxh-Uhs.js";
|
|
5
|
+
import { n as campaignCellToRunRecord } from "./run-record-CysKLB93.js";
|
|
6
|
+
import { D as surfaceContentHash, O as surfaceDispatchRef, c as campaignMeasurementDigest, d as loopProvenanceArgsFromResult, h as pairedCampaignComposites, k as surfaceHash, l as canonicalDigest, t as createCampaignEvidenceReceipt, u as emitLoopProvenance, y as powerPreflight, z as campaignSplitDigest } from "./campaign-evidence-DunzcJGv.js";
|
|
7
|
+
import { M as fsCampaignStorage, N as inMemoryCampaignStorage, j as createRunCostLedger } from "./external-optimizer-subprocess-B5gdLRZv.js";
|
|
8
|
+
import { t as defineEvaluationClaim } from "./claim-BPVuyHuy.js";
|
|
9
|
+
import { a as createHostedClient, i as startSearchShipper, n as shipBestEffort, r as shipSearchLedger } from "./search-shipper-Bg2HW3Uv.js";
|
|
10
10
|
import { fileURLToPath } from "node:url";
|
|
11
11
|
//#region src/contract/self-improve-reporting.ts
|
|
12
12
|
function pairedCompositeSummary(baseline, winner, independentUnitByScenarioId) {
|
|
@@ -68,16 +68,112 @@ function cellsToRunRecords(cells, candidateId, runId, surface, splitTag, fallbac
|
|
|
68
68
|
}
|
|
69
69
|
//#endregion
|
|
70
70
|
//#region src/contract/self-improve-method.ts
|
|
71
|
+
const METHOD_FINAL_PHASE = "selfImprove.method-final";
|
|
71
72
|
/** Complete methods own search and selection; this path only measures their final choice. */
|
|
72
73
|
async function runSelfImproveMethod(args) {
|
|
73
|
-
const { opts,
|
|
74
|
+
const { opts, holdout, costLedger, startedAt } = args;
|
|
74
75
|
const budget = opts.budget ?? {};
|
|
76
|
+
assertAutoPrTarget(opts);
|
|
77
|
+
const evidenceContext = captureEvidenceContext(opts, budget);
|
|
78
|
+
const baselineSurface = structuredClone(opts.baselineSurface);
|
|
79
|
+
const { selected, cost: methodCost, history } = await searchWithMethod(args, baselineSurface);
|
|
80
|
+
const totalCost = () => methodTotalCost(opts.method.name, methodCost, costLedger);
|
|
81
|
+
const comparison = await measureMethodWinner(args, baselineSurface, selected, totalCost);
|
|
82
|
+
const evidence = evidenceContext ? methodEvidenceReceipts(comparison, baselineSurface, selected, evidenceContext) : void 0;
|
|
83
|
+
const deferred = comparison.holdout === "deferred";
|
|
84
|
+
const report = deferred ? null : pairedCompositeSummary(comparison.baselineOnHoldout, comparison.winnerOnHoldout, opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0);
|
|
85
|
+
const baseline = report?.baseline ?? null;
|
|
86
|
+
const winner = report?.winner ?? null;
|
|
87
|
+
const lift = baseline && winner ? winner.compositeMean - baseline.compositeMean : void 0;
|
|
88
|
+
const insight = deferred ? void 0 : await analyzeMethodHoldout(args, comparison, baselineSurface, selected);
|
|
89
|
+
const cost = totalCost();
|
|
90
|
+
const optimization = {
|
|
91
|
+
name: opts.method.name,
|
|
92
|
+
cost: methodCost,
|
|
93
|
+
...selected.durationMs === void 0 ? {} : { durationMs: selected.durationMs },
|
|
94
|
+
...selected.provenance === void 0 ? {} : { provenance: selected.provenance }
|
|
95
|
+
};
|
|
96
|
+
const durationMs = Date.now() - startedAt;
|
|
97
|
+
const receipts = costLedger.list();
|
|
98
|
+
const provenance = recordMethodProvenance({
|
|
99
|
+
args,
|
|
100
|
+
comparison,
|
|
101
|
+
baselineSurface,
|
|
102
|
+
selected,
|
|
103
|
+
report,
|
|
104
|
+
lift,
|
|
105
|
+
optimization,
|
|
106
|
+
cost,
|
|
107
|
+
durationMs,
|
|
108
|
+
receipts
|
|
109
|
+
});
|
|
110
|
+
opts.onProvenance?.(provenance);
|
|
111
|
+
opts.onProgress?.({
|
|
112
|
+
kind: "gate.decided",
|
|
113
|
+
decision: comparison.gateResult.decision,
|
|
114
|
+
...lift === void 0 ? {} : { lift }
|
|
115
|
+
});
|
|
116
|
+
const prResult = opts.autoOnPromote === "pr" && comparison.gateResult.decision === "ship" ? openAutoPr({
|
|
117
|
+
result: comparison.winnerOnHoldout,
|
|
118
|
+
gate: comparison.gateResult,
|
|
119
|
+
promotedDiff: comparison.promotedDiff,
|
|
120
|
+
ghOwner: opts.ghOwner,
|
|
121
|
+
ghRepo: opts.ghRepo
|
|
122
|
+
}) : void 0;
|
|
123
|
+
const result = {
|
|
124
|
+
...evidence ? { evidence } : {},
|
|
125
|
+
mode: "method",
|
|
126
|
+
...opts.claim ? { claim: opts.claim } : {},
|
|
127
|
+
...comparison.finalEvidence ? { finalEvidence: comparison.finalEvidence } : {},
|
|
128
|
+
searchHistoryCoverage: history,
|
|
129
|
+
baseline,
|
|
130
|
+
winner: {
|
|
131
|
+
surface: selected.winnerSurface,
|
|
132
|
+
compositeMean: winner?.compositeMean ?? null,
|
|
133
|
+
perScenario: winner?.perScenario ?? {},
|
|
134
|
+
label: opts.method.name
|
|
135
|
+
},
|
|
136
|
+
...lift === void 0 ? {} : { lift },
|
|
137
|
+
diff: comparison.promotedDiff,
|
|
138
|
+
gateDecision: comparison.gateResult.decision,
|
|
139
|
+
provenance,
|
|
140
|
+
optimization,
|
|
141
|
+
cost,
|
|
142
|
+
ledgerCost: costLedger.summary(),
|
|
143
|
+
totalCostUsd: cost.totalCostUsd,
|
|
144
|
+
durationMs,
|
|
145
|
+
receipts,
|
|
146
|
+
...insight ? { insight } : {},
|
|
147
|
+
...selected.searchHistory ? { searchHistory: selected.searchHistory } : {},
|
|
148
|
+
raw: {
|
|
149
|
+
...comparison,
|
|
150
|
+
kind: "method",
|
|
151
|
+
baselineSurface,
|
|
152
|
+
winnerSurface: selected.winnerSurface,
|
|
153
|
+
winnerSurfaceHash: surfaceHash(selected.winnerSurface),
|
|
154
|
+
method: selected,
|
|
155
|
+
cost,
|
|
156
|
+
...prResult ? { prResult } : {}
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
await shipMethodSearchLedger(opts, selected);
|
|
160
|
+
return result;
|
|
161
|
+
}
|
|
162
|
+
/** An auto-PR needs a target repository before any spend. */
|
|
163
|
+
function assertAutoPrTarget(opts) {
|
|
75
164
|
if (opts.autoOnPromote === "pr" && (!opts.ghOwner || !opts.ghRepo)) throw new Error("selfImprove: autoOnPromote='pr' requires ghOwner + ghRepo");
|
|
165
|
+
}
|
|
166
|
+
/** Detach the caller's evidence context; receipts need a measured holdout. */
|
|
167
|
+
function captureEvidenceContext(opts, budget) {
|
|
76
168
|
const evidenceContext = opts.evidence === void 0 ? void 0 : structuredClone(opts.evidence);
|
|
77
169
|
if (evidenceContext && budget.holdout === "deferred") throw new Error("selfImprove: evidence receipts require measured holdout");
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
170
|
+
return evidenceContext;
|
|
171
|
+
}
|
|
172
|
+
/** Let the method search and select a winner on the train and selection splits. */
|
|
173
|
+
function searchWithMethod(args, baselineSurface) {
|
|
174
|
+
const { opts, train, selection, costLedger, storage, runDir } = args;
|
|
175
|
+
const budget = opts.budget ?? {};
|
|
176
|
+
return executeOptimizationMethod({
|
|
81
177
|
method: opts.method,
|
|
82
178
|
storage,
|
|
83
179
|
searchHistoryPolicy: opts.searchHistoryPolicy,
|
|
@@ -87,7 +183,7 @@ async function runSelfImproveMethod(args) {
|
|
|
87
183
|
trainScenarios: train,
|
|
88
184
|
selectionScenarios: selection,
|
|
89
185
|
dispatchWithSurface: opts.agent,
|
|
90
|
-
judges: [judge],
|
|
186
|
+
judges: [opts.judge],
|
|
91
187
|
runDir: `${runDir}/optimization/${opts.method.name.replace(/[^a-zA-Z0-9._-]/g, "_")}`,
|
|
92
188
|
seed: 42,
|
|
93
189
|
runOptions: {
|
|
@@ -105,29 +201,39 @@ async function runSelfImproveMethod(args) {
|
|
|
105
201
|
costLedger
|
|
106
202
|
}
|
|
107
203
|
});
|
|
108
|
-
|
|
109
|
-
|
|
204
|
+
}
|
|
205
|
+
/** Spend on the final holdout comparison, read back from the shared ledger. */
|
|
206
|
+
function methodFinalCost(costLedger) {
|
|
207
|
+
return combineComparisonCosts([
|
|
110
208
|
"holdout.baseline",
|
|
111
209
|
"holdout.winner",
|
|
112
210
|
"holdout.neutralized",
|
|
113
211
|
"promotion.gate"
|
|
114
212
|
].map((phase) => ({
|
|
115
213
|
label: phase,
|
|
116
|
-
cost: costFromLedgerSummary(costLedger.summary({ phase: `${
|
|
214
|
+
cost: costFromLedgerSummary(costLedger.summary({ phase: `${METHOD_FINAL_PHASE}.${phase}` }))
|
|
117
215
|
})));
|
|
118
|
-
|
|
119
|
-
|
|
216
|
+
}
|
|
217
|
+
/** The method's own spend plus the final comparison's. */
|
|
218
|
+
function methodTotalCost(methodName, methodCost, costLedger) {
|
|
219
|
+
return combineComparisonCosts([{
|
|
220
|
+
label: methodName,
|
|
120
221
|
cost: methodCost
|
|
121
222
|
}, {
|
|
122
223
|
label: "final comparison",
|
|
123
|
-
cost:
|
|
224
|
+
cost: methodFinalCost(costLedger)
|
|
124
225
|
}]);
|
|
226
|
+
}
|
|
227
|
+
/** Measure baseline and winner on the holdout; the gate holds when spend breaks the budget. */
|
|
228
|
+
function measureMethodWinner(args, baselineSurface, selected, totalCost) {
|
|
229
|
+
const { opts, holdout, costLedger, storage, runDir } = args;
|
|
230
|
+
const budget = opts.budget ?? {};
|
|
125
231
|
const gate = opts.gate ?? defaultProductionGate({
|
|
126
232
|
holdoutScenarios: holdout,
|
|
127
233
|
deltaThreshold: opts.claim?.minimumEffect ?? .05,
|
|
128
234
|
independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0
|
|
129
235
|
});
|
|
130
|
-
|
|
236
|
+
return runFinalComparison({
|
|
131
237
|
claim: opts.claim,
|
|
132
238
|
finalEvidence: opts.finalEvidence,
|
|
133
239
|
baselineSurface,
|
|
@@ -135,7 +241,7 @@ async function runSelfImproveMethod(args) {
|
|
|
135
241
|
scenarios: holdout,
|
|
136
242
|
dispatchWithSurface: opts.agent,
|
|
137
243
|
dispatchRef: opts.dispatchRef,
|
|
138
|
-
judges: [judge],
|
|
244
|
+
judges: [opts.judge],
|
|
139
245
|
holdout: budget.holdout,
|
|
140
246
|
neutralize: opts.neutralize,
|
|
141
247
|
gate: {
|
|
@@ -160,7 +266,7 @@ async function runSelfImproveMethod(args) {
|
|
|
160
266
|
runDir,
|
|
161
267
|
storage,
|
|
162
268
|
costLedger,
|
|
163
|
-
costPhase:
|
|
269
|
+
costPhase: METHOD_FINAL_PHASE,
|
|
164
270
|
reps: budget.reps,
|
|
165
271
|
maxConcurrency: budget.maxConcurrency ?? 2,
|
|
166
272
|
dispatchTimeoutMs: opts.dispatchTimeoutMs,
|
|
@@ -169,7 +275,10 @@ async function runSelfImproveMethod(args) {
|
|
|
169
275
|
expectUsage: opts.expectUsage ?? "assert",
|
|
170
276
|
label: "selfImprove"
|
|
171
277
|
});
|
|
172
|
-
|
|
278
|
+
}
|
|
279
|
+
/** Evidence receipts for the baseline and winner holdout campaigns. */
|
|
280
|
+
function methodEvidenceReceipts(comparison, baselineSurface, selected, evidenceContext) {
|
|
281
|
+
return {
|
|
173
282
|
baseline: createCampaignEvidenceReceipt({
|
|
174
283
|
campaign: comparison.baselineOnHoldout,
|
|
175
284
|
surface: baselineSurface,
|
|
@@ -180,28 +289,26 @@ async function runSelfImproveMethod(args) {
|
|
|
180
289
|
surface: selected.winnerSurface,
|
|
181
290
|
context: evidenceContext
|
|
182
291
|
})
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
const
|
|
188
|
-
|
|
189
|
-
const insight = deferred ? void 0 : await analyzeRuns({
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
/** Analyze the measured holdout runs; a winner that is the baseline adds no second candidate. */
|
|
295
|
+
function analyzeMethodHoldout(args, comparison, baselineSurface, selected) {
|
|
296
|
+
const { opts, holdout, runDir } = args;
|
|
297
|
+
return analyzeRuns({
|
|
190
298
|
runs: [...cellsToRunRecords(comparison.baselineOnHoldout.cells, "baseline", runDir, baselineSurface, "holdout", opts.model), ...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? [] : cellsToRunRecords(comparison.winnerOnHoldout.cells, "winner", runDir, selected.winnerSurface, "holdout", opts.model)],
|
|
191
299
|
baselineCandidateId: "baseline",
|
|
192
300
|
independentUnitByScenarioId: opts.claim ? evaluationUnitMap(opts.claim, holdout) : void 0,
|
|
193
301
|
decisionThreshold: opts.claim?.minimumEffect ?? .05,
|
|
194
302
|
...comparison.winnerOnHoldout === comparison.baselineOnHoldout ? {} : { candidateCandidateId: "winner" }
|
|
195
303
|
});
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
const
|
|
204
|
-
const receipts = costLedger.list();
|
|
304
|
+
}
|
|
305
|
+
/** Build, persist and return the digest-sealed provenance record. */
|
|
306
|
+
function recordMethodProvenance(input) {
|
|
307
|
+
const { args, comparison, baselineSurface, selected, report, lift, optimization } = input;
|
|
308
|
+
const { opts, train, selection, holdout, storage, runDir, startedAt } = args;
|
|
309
|
+
const budget = opts.budget ?? {};
|
|
310
|
+
const baseline = report?.baseline ?? null;
|
|
311
|
+
const winner = report?.winner ?? null;
|
|
205
312
|
const record = {
|
|
206
313
|
schema: "tangle.method-improvement",
|
|
207
314
|
runId: `${runDir}#${startedAt}`,
|
|
@@ -218,20 +325,20 @@ async function runSelfImproveMethod(args) {
|
|
|
218
325
|
holdoutSplitDigest: campaignSplitDigest(holdout, budget.reps ?? 1),
|
|
219
326
|
baselineCampaignDigest: campaignMeasurementDigest(comparison.baselineOnHoldout),
|
|
220
327
|
winnerCampaignDigest: campaignMeasurementDigest(comparison.winnerOnHoldout),
|
|
221
|
-
costReceiptsDigest: canonicalDigest(receipts),
|
|
328
|
+
costReceiptsDigest: canonicalDigest(input.receipts),
|
|
222
329
|
...report ? { holdoutObservations: report.observations } : {},
|
|
223
330
|
...comparison.neutralizedOnHoldout ? { neutralizedCampaignDigest: campaignMeasurementDigest(comparison.neutralizedOnHoldout) } : {}
|
|
224
331
|
},
|
|
225
332
|
gate: comparison.gateResult,
|
|
226
|
-
...deferred ? { holdout: "deferred" } : {},
|
|
333
|
+
...comparison.holdout === "deferred" ? { holdout: "deferred" } : {},
|
|
227
334
|
...baseline && winner ? {
|
|
228
335
|
baselineHoldoutComposite: baseline.compositeMean,
|
|
229
336
|
winnerHoldoutComposite: winner.compositeMean,
|
|
230
337
|
heldOutLift: lift
|
|
231
338
|
} : {},
|
|
232
|
-
cost,
|
|
233
|
-
totalCostUsd: cost.totalCostUsd,
|
|
234
|
-
totalDurationMs: durationMs
|
|
339
|
+
cost: input.cost,
|
|
340
|
+
totalCostUsd: input.cost.totalCostUsd,
|
|
341
|
+
totalDurationMs: input.durationMs
|
|
235
342
|
};
|
|
236
343
|
const serialized = JSON.parse(JSON.stringify(record));
|
|
237
344
|
const provenance = {
|
|
@@ -240,70 +347,23 @@ async function runSelfImproveMethod(args) {
|
|
|
240
347
|
};
|
|
241
348
|
storage.ensureDir(runDir);
|
|
242
349
|
storage.write(`${runDir}/method-provenance.json`, JSON.stringify(provenance, null, 2));
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
...comparison.finalEvidence ? { finalEvidence: comparison.finalEvidence } : {},
|
|
261
|
-
searchHistoryCoverage: history,
|
|
262
|
-
baseline,
|
|
263
|
-
winner: {
|
|
264
|
-
surface: selected.winnerSurface,
|
|
265
|
-
compositeMean: winner?.compositeMean ?? null,
|
|
266
|
-
perScenario: winner?.perScenario ?? {},
|
|
267
|
-
label: opts.method.name
|
|
268
|
-
},
|
|
269
|
-
...lift === void 0 ? {} : { lift },
|
|
270
|
-
diff: comparison.promotedDiff,
|
|
271
|
-
gateDecision: comparison.gateResult.decision,
|
|
272
|
-
provenance,
|
|
273
|
-
optimization,
|
|
274
|
-
cost,
|
|
275
|
-
ledgerCost: costLedger.summary(),
|
|
276
|
-
totalCostUsd: cost.totalCostUsd,
|
|
277
|
-
durationMs,
|
|
278
|
-
receipts,
|
|
279
|
-
...insight ? { insight } : {},
|
|
280
|
-
...selected.searchHistory ? { searchHistory: selected.searchHistory } : {},
|
|
281
|
-
raw: {
|
|
282
|
-
...comparison,
|
|
283
|
-
kind: "method",
|
|
284
|
-
baselineSurface,
|
|
285
|
-
winnerSurface: selected.winnerSurface,
|
|
286
|
-
winnerSurfaceHash: surfaceHash(selected.winnerSurface),
|
|
287
|
-
method: selected,
|
|
288
|
-
cost,
|
|
289
|
-
...prResult ? { prResult } : {}
|
|
290
|
-
}
|
|
291
|
-
};
|
|
292
|
-
if (opts.hostedTenant) {
|
|
293
|
-
const receipt = selected.searchHistory;
|
|
294
|
-
if (receipt) {
|
|
295
|
-
const path = fileURLToPath(receipt.ledger.uri);
|
|
296
|
-
await shipBestEffort(() => shipSearchLedger({
|
|
297
|
-
tenant: opts.hostedTenant,
|
|
298
|
-
ledger: {
|
|
299
|
-
path,
|
|
300
|
-
searchId: receipt.summary.searchId
|
|
301
|
-
},
|
|
302
|
-
runKind: "optimization"
|
|
303
|
-
}), path);
|
|
304
|
-
} else console.warn(`[agent-eval] hostedTenant: method ${opts.method.name} recorded no search ledger, so nothing was shipped`);
|
|
305
|
-
}
|
|
306
|
-
return result;
|
|
350
|
+
return provenance;
|
|
351
|
+
}
|
|
352
|
+
/** Ship the method's search ledger to the hosted tenant, when one is configured. */
|
|
353
|
+
async function shipMethodSearchLedger(opts, selected) {
|
|
354
|
+
if (!opts.hostedTenant) return;
|
|
355
|
+
const receipt = selected.searchHistory;
|
|
356
|
+
if (receipt) {
|
|
357
|
+
const path = fileURLToPath(receipt.ledger.uri);
|
|
358
|
+
await shipBestEffort(() => shipSearchLedger({
|
|
359
|
+
tenant: opts.hostedTenant,
|
|
360
|
+
ledger: {
|
|
361
|
+
path,
|
|
362
|
+
searchId: receipt.summary.searchId
|
|
363
|
+
},
|
|
364
|
+
runKind: "optimization"
|
|
365
|
+
}), path);
|
|
366
|
+
} else console.warn(`[agent-eval] hostedTenant: method ${opts.method.name} recorded no search ledger, so nothing was shipped`);
|
|
307
367
|
}
|
|
308
368
|
//#endregion
|
|
309
369
|
//#region src/contract/self-improve.ts
|
|
@@ -728,4 +788,4 @@ function requirePositiveInteger(value, field) {
|
|
|
728
788
|
//#endregion
|
|
729
789
|
export { SelfImproveRunError as n, selfImprove as r, defineAgentEval as t };
|
|
730
790
|
|
|
731
|
-
//# sourceMappingURL=define-agent-eval-
|
|
791
|
+
//# sourceMappingURL=define-agent-eval-BGdFKlrc.js.map
|