@tangle-network/agent-eval 0.216.1 → 0.218.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +137 -57
- package/README.md +2 -2
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-VFcQncbW.d.ts → agent-profile-DTt8uGP9.d.ts} +4 -17
- package/dist/agent-profile-DTt8uGP9.d.ts.map +1 -0
- package/dist/{agent-profile-cell-BK8mlqlq.js → agent-profile-cell-B1V1gUZs.js} +4 -53
- package/dist/agent-profile-cell-B1V1gUZs.js.map +1 -0
- package/dist/{agent-profile-cell-BNp-Js9F.d.ts → agent-profile-cell-BcpFbNd5.d.ts} +2 -22
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +1 -0
- package/dist/{agreement-irr-BEWLIF3_.js → agreement-irr-BhAqjluA.js} +5 -5
- package/dist/agreement-irr-BhAqjluA.js.map +1 -0
- package/dist/analyst/index.d.ts +36 -169
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +17 -192
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-D68fd-su.d.ts → analyze-runs-DjlZOVXx.d.ts} +5 -5
- package/dist/{analyze-runs-D68fd-su.d.ts.map → analyze-runs-DjlZOVXx.d.ts.map} +1 -1
- package/dist/{analyze-runs-C84irm4i.js → analyze-runs-DqPJaJ4X.js} +225 -21
- package/dist/analyze-runs-DqPJaJ4X.js.map +1 -0
- package/dist/authenticity/index.d.ts +0 -23
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -34
- package/dist/authenticity/index.js.map +1 -1
- package/dist/{benchmark-command-By0vh8q7.js → benchmark-command-BupdHRyf.js} +1225 -661
- package/dist/benchmark-command-BupdHRyf.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -6
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +15 -15
- package/dist/{campaign-BudZ70zo.js → campaign-CGvUTnmc.js} +324 -800
- package/dist/campaign-CGvUTnmc.js.map +1 -0
- package/dist/{campaign-evidence-Clrn4hBV.js → campaign-evidence-DunzcJGv.js} +10 -18
- package/dist/{campaign-evidence-Clrn4hBV.js.map → campaign-evidence-DunzcJGv.js.map} +1 -1
- package/dist/{canonical-B2RqLofv.js → canonical-Dq2hOQ8L.js} +2 -2
- package/dist/{canonical-B2RqLofv.js.map → canonical-Dq2hOQ8L.js.map} +1 -1
- package/dist/cell-spend-DEWR3nj-.js +90 -0
- package/dist/cell-spend-DEWR3nj-.js.map +1 -0
- package/dist/{chat-client-l-MVDlDk.js → chat-client-B2pXlPdx.js} +2 -2
- package/dist/{chat-client-l-MVDlDk.js.map → chat-client-B2pXlPdx.js.map} +1 -1
- package/dist/{chat-json-call-puMArmtW.js → chat-json-call-Dm9c-b41.js} +3 -2
- package/dist/chat-json-call-Dm9c-b41.js.map +1 -0
- package/dist/{claim-q4HQhK1G.js → claim-BPVuyHuy.js} +3 -3
- package/dist/{claim-q4HQhK1G.js.map → claim-BPVuyHuy.js.map} +1 -1
- package/dist/cli.js +113 -12
- package/dist/cli.js.map +1 -1
- package/dist/{client-BzL9niqi.d.ts → client-hwo3cJjR.d.ts} +3 -16
- package/dist/client-hwo3cJjR.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-qcUWYFl-.js → cluster-bootstrap-DEwwhl0C.js} +3 -3
- package/dist/{cluster-bootstrap-qcUWYFl-.js.map → cluster-bootstrap-DEwwhl0C.js.map} +1 -1
- package/dist/{produced-state-BsDmnwxc.js → completion-verifier-DbLVf3i8.js} +9 -64
- package/dist/completion-verifier-DbLVf3i8.js.map +1 -0
- package/dist/contract/index.d.ts +12 -12
- package/dist/contract/index.js +107 -403
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BP8V4kUQ.js → cost-ledger-D0352Z4d.js} +4 -8
- package/dist/cost-ledger-D0352Z4d.js.map +1 -0
- package/dist/dataset-DQqhOCPt.d.ts.map +1 -1
- package/dist/{default-registry-COhk9g36.js → default-registry-BJ-Pq6yL.js} +314 -285
- package/dist/default-registry-BJ-Pq6yL.js.map +1 -0
- package/dist/{default-registry-DzHiGruW.d.ts → default-registry-DXgLmoDI.d.ts} +7 -50
- package/dist/default-registry-DXgLmoDI.d.ts.map +1 -0
- package/dist/{define-agent-eval-CNTPzIkp.js → define-agent-eval-BGdFKlrc.js} +170 -110
- package/dist/define-agent-eval-BGdFKlrc.js.map +1 -0
- package/dist/{define-agent-eval-WSck0CZJ.d.ts → define-agent-eval-BlH10zEw.d.ts} +8 -8
- package/dist/define-agent-eval-BlH10zEw.d.ts.map +1 -0
- package/dist/{descriptive-DboxPZrG.js → descriptive-CVNy48rR.js} +2 -2
- package/dist/{descriptive-DboxPZrG.js.map → descriptive-CVNy48rR.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +4 -14
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +67 -28
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-LRCu61m5.js → dspy-rlm-engine-CSMwudLO.js} +5 -10
- package/dist/dspy-rlm-engine-CSMwudLO.js.map +1 -0
- package/dist/{emitter-HcCmFQl_.d.ts → emitter-DABYYTcH.d.ts} +5 -5
- package/dist/emitter-DABYYTcH.d.ts.map +1 -0
- package/dist/emitter-DbMXUIVP.js.map +1 -1
- package/dist/{engine-3A_4jOlI.d.ts → engine-BvZ0R-36.d.ts} +17 -21
- package/dist/engine-BvZ0R-36.d.ts.map +1 -0
- package/dist/{equal-terms-BtdXayCJ.js → equal-terms-DbPGcVtI.js} +2 -2
- package/dist/equal-terms-DbPGcVtI.js.map +1 -0
- package/dist/{errors-DVZiqS7K.js → errors-BcOs5MKW.js} +2 -8
- package/dist/{errors-DVZiqS7K.js.map → errors-BcOs5MKW.js.map} +1 -1
- package/dist/eval-campaign-BUMIxKTE.js +393 -0
- package/dist/eval-campaign-BUMIxKTE.js.map +1 -0
- package/dist/{evaluation-6Yoi74gM.d.ts → evaluation-BOZ38vOH.d.ts} +7 -7
- package/dist/evaluation-BOZ38vOH.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -2
- package/dist/evaluation.js +1 -1
- package/dist/evaluation.js.map +1 -1
- package/dist/{exact-types-ClugMJfI.d.ts → exact-types-CflbJ9f3.d.ts} +2 -2
- package/dist/{exact-types-ClugMJfI.d.ts.map → exact-types-CflbJ9f3.d.ts.map} +1 -1
- package/dist/{exec-ChJ6d5if.js → exec-CpbtOXVt.js} +2 -24
- package/dist/{exec-ChJ6d5if.js.map → exec-CpbtOXVt.js.map} +1 -1
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -1
- package/dist/experiment/index.d.ts +9 -112
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +258 -311
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-X4xwla8A.js → exporters-DWvM16LE.js} +3 -3
- package/dist/exporters-DWvM16LE.js.map +1 -0
- package/dist/{exporters-ChTnR90l.d.ts → exporters-Dk3oEbK_.d.ts} +2 -4
- package/dist/{exporters-ChTnR90l.d.ts.map → exporters-Dk3oEbK_.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-BjMGUuds.js → external-optimizer-process-C3Wm7Vhn.js} +4 -3
- package/dist/external-optimizer-process-C3Wm7Vhn.js.map +1 -0
- package/dist/{external-optimizer-subprocess-BqW0hHcI.js → external-optimizer-subprocess-B5gdLRZv.js} +9 -11
- package/dist/external-optimizer-subprocess-B5gdLRZv.js.map +1 -0
- package/dist/{extract-usage-BrnzRJ6H.js → extract-usage-BZLU01kR.js} +2 -23
- package/dist/{extract-usage-BrnzRJ6H.js.map → extract-usage-BZLU01kR.js.map} +1 -1
- package/dist/{failure-cluster-CbgEDcFJ.d.ts → failure-cluster-DN0t63Z0.d.ts} +4 -43
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +1 -0
- package/dist/{failure-taxonomy-CjabTATj.js → failure-taxonomy-Bx5oWqaW.js} +2 -19
- package/dist/{failure-taxonomy-CjabTATj.js.map → failure-taxonomy-Bx5oWqaW.js.map} +1 -1
- package/dist/{feedback-trajectory-C4EG0nNH.d.ts → feedback-trajectory-BWmhRBGu.d.ts} +7 -9
- package/dist/feedback-trajectory-BWmhRBGu.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-BQF8te0T.js → findings-store-BtTzPL0d.js} +9 -249
- package/dist/findings-store-BtTzPL0d.js.map +1 -0
- package/dist/fuzz.d.ts +19 -62
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +62 -108
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-BR2yhuNn.js → hf-dataset-CKxIJQba.js} +4 -24
- package/dist/hf-dataset-CKxIJQba.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +2 -2
- package/dist/{index-CyszSg4O.d.ts → index-B16FNdW5.d.ts} +145 -360
- package/dist/index-B16FNdW5.d.ts.map +1 -0
- package/dist/{index-DV-jSzxF.d.ts → index-BLGqZ81N.d.ts} +10 -15
- package/dist/{index-DV-jSzxF.d.ts.map → index-BLGqZ81N.d.ts.map} +1 -1
- package/dist/{index-BO41eGj5.d.ts → index-CX-GAUpw.d.ts} +3 -7
- package/dist/index-CX-GAUpw.d.ts.map +1 -0
- package/dist/{index-DHKEAv3s.d.ts → index-CalAsXMR.d.ts} +8 -68
- package/dist/index-CalAsXMR.d.ts.map +1 -0
- package/dist/{index-BFRO6chs.d.ts → index-CiRwLv2R.d.ts} +5 -69
- package/dist/index-CiRwLv2R.d.ts.map +1 -0
- package/dist/{index-DPcngtwt.d.ts → index-DbMOXYnP.d.ts} +11 -36
- package/dist/index-DbMOXYnP.d.ts.map +1 -0
- package/dist/index.d.ts +211 -218
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +999 -1714
- package/dist/index.js.map +1 -1
- package/dist/{integrity-dLiDe2kD.js → integrity-C8oUeB0g.js} +326 -203
- package/dist/integrity-C8oUeB0g.js.map +1 -0
- package/dist/{integrity-BXOsAb3i.js → integrity-W4CADcFc.js} +3 -7
- package/dist/{integrity-BXOsAb3i.js.map → integrity-W4CADcFc.js.map} +1 -1
- package/dist/{integrity-DAMEDFaC.d.ts → integrity-_qgZLA5p.d.ts} +4 -6
- package/dist/{integrity-DAMEDFaC.d.ts.map → integrity-_qgZLA5p.d.ts.map} +1 -1
- package/dist/{internal-Epl-YBZW.js → internal-BTK91Bqt.js} +3 -3
- package/dist/{internal-Epl-YBZW.js.map → internal-BTK91Bqt.js.map} +1 -1
- package/dist/{interval-CELpye1n.js → interval-Dq9gt7jx.js} +3 -3
- package/dist/{interval-CELpye1n.js.map → interval-Dq9gt7jx.js.map} +1 -1
- package/dist/{journal-DfRz9kps.js → journal-C7om8Gum.js} +157 -147
- package/dist/journal-C7om8Gum.js.map +1 -0
- package/dist/json-BhdsJaOQ.js +111 -0
- package/dist/json-BhdsJaOQ.js.map +1 -0
- package/dist/{judge-calibration-BpCy5XQl.js → judge-calibration-BkXMIWA4.js} +3 -3
- package/dist/{judge-calibration-BpCy5XQl.js.map → judge-calibration-BkXMIWA4.js.map} +1 -1
- package/dist/{judge-gate-iY9s4L44.js → judge-gate-Co5tuh57.js} +4 -27
- package/dist/{judge-gate-iY9s4L44.js.map → judge-gate-Co5tuh57.js.map} +1 -1
- package/dist/{judge-gate-CE7CQAgt.d.ts → judge-gate-DsL0yFcb.d.ts} +8 -9
- package/dist/judge-gate-DsL0yFcb.d.ts.map +1 -0
- package/dist/{kind-factory-BobI5Jy3.js → kind-factory-Gy4fXixd.js} +88 -17
- package/dist/kind-factory-Gy4fXixd.js.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +4 -4
- package/dist/ledger-core/lock-holder.d.ts +2 -0
- package/dist/ledger-core/lock-holder.js +181 -0
- package/dist/ledger-core/lock-holder.js.map +1 -0
- package/dist/{ledger-core-Gq7yp8lP.js → ledger-core-CAxUdDk0.js} +5 -84
- package/dist/ledger-core-CAxUdDk0.js.map +1 -0
- package/dist/{llm-client-CyBAoYiX.js → llm-client-_Y6ic-tf.js} +243 -355
- package/dist/llm-client-_Y6ic-tf.js.map +1 -0
- package/dist/{llm-judge-CG12SQJq.js → llm-judge-Djxh-Uhs.js} +662 -482
- package/dist/llm-judge-Djxh-Uhs.js.map +1 -0
- package/dist/lock-holder-DaJbfpci.d.ts +86 -0
- package/dist/lock-holder-DaJbfpci.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +3 -2
- package/dist/{matrix-Dm0BQW7U.js → matrix-B8l-Nsxd.js} +3 -88
- package/dist/matrix-B8l-Nsxd.js.map +1 -0
- package/dist/{matrix-DSrSxjjy.d.ts → matrix-CCSBna4q.d.ts} +9 -18
- package/dist/matrix-CCSBna4q.d.ts.map +1 -0
- package/dist/{mean-DHoMiudD.js → mean-BbA_JCzW.js} +39 -5
- package/dist/mean-BbA_JCzW.js.map +1 -0
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +10 -10
- package/dist/metrics-BKaPBtIG.js.map +1 -1
- package/dist/{mint-DNU8QPG4.d.ts → mint-BRZXuwRi.d.ts} +4 -4
- package/dist/{mint-DNU8QPG4.d.ts.map → mint-BRZXuwRi.d.ts.map} +1 -1
- package/dist/{mint-QaD5cA0u.js → mint-CbdwjTPW.js} +4 -4
- package/dist/{mint-QaD5cA0u.js.map → mint-CbdwjTPW.js.map} +1 -1
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts → multi-layer-verifier-DEgMUk6v.d.ts} +2 -2
- package/dist/{multi-layer-verifier-BUaQ4C17.d.ts.map → multi-layer-verifier-DEgMUk6v.d.ts.map} +1 -1
- package/dist/{multiplicity-DQUFcbW4.js → multiplicity-Sg0U7h8j.js} +2 -2
- package/dist/{multiplicity-DQUFcbW4.js.map → multiplicity-Sg0U7h8j.js.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -23
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +3 -3
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +3 -12
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +4 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +114 -10
- package/dist/{otel-spans-B9tUXRHF.js → otel-spans-CdN909k1.js} +3 -3
- package/dist/{otel-spans-B9tUXRHF.js.map → otel-spans-CdN909k1.js.map} +1 -1
- package/dist/{paired-arms-CwMGJsrS.js → paired-arms-Ct4LTKc1.js} +32 -9
- package/dist/paired-arms-Ct4LTKc1.js.map +1 -0
- package/dist/{paired-binary-CvpFHfKZ.js → paired-binary-B_nejA44.js} +3 -36
- package/dist/paired-binary-B_nejA44.js.map +1 -0
- package/dist/{paired-promotion-decision-DPKz8ZnF.d.ts → paired-promotion-decision-CQMJ5xGN.d.ts} +2 -34
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-DsAPYGCm.js → paired-promotion-decision-D0SHb-ZL.js} +3 -3
- package/dist/{paired-promotion-decision-DsAPYGCm.js.map → paired-promotion-decision-D0SHb-ZL.js.map} +1 -1
- package/dist/{pareto-Cm6ndrWN.js → pareto-CTv8vozZ.js} +11 -11
- package/dist/pareto-CTv8vozZ.js.map +1 -0
- package/dist/pipelines/index.d.ts +16 -19
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +4 -68
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-Hj2ubb4B.js → power-and-mde-N5nU4_Ly.js} +3 -3
- package/dist/{power-and-mde-Hj2ubb4B.js.map → power-and-mde-N5nU4_Ly.js.map} +1 -1
- package/dist/{prime-protocol-CHKgA3a6.js → prime-protocol-BFgGrEMP.js} +3 -3
- package/dist/prime-protocol-BFgGrEMP.js.map +1 -0
- package/dist/{prime-protocol-HiB9HX7E.d.ts → prime-protocol-BWG2DQIk.d.ts} +3 -21
- package/dist/prime-protocol-BWG2DQIk.d.ts.map +1 -0
- package/dist/profile-cell.d.ts +2 -2
- package/dist/profile-cell.js +2 -2
- package/dist/{promotion-gate-BawrdPYr.d.ts → promotion-gate-CxEHRsZW.d.ts} +7 -8
- package/dist/{promotion-gate-BawrdPYr.d.ts.map → promotion-gate-CxEHRsZW.d.ts.map} +1 -1
- package/dist/{promotion-policy-KO57gbyV.js → promotion-policy-6A8osGr_.js} +4 -4
- package/dist/{promotion-policy-KO57gbyV.js.map → promotion-policy-6A8osGr_.js.map} +1 -1
- package/dist/{proposal-findings-CY1xKn_a.js → proposal-findings-n-5WBbEx.js} +3 -3
- package/dist/proposal-findings-n-5WBbEx.js.map +1 -0
- package/dist/{query-BUCZCK7F.js → query-B_RZ36an.js} +5 -5
- package/dist/query-B_RZ36an.js.map +1 -0
- package/dist/{query-DK1nWcmp.d.ts → query-Cmv_U8vf.d.ts} +4 -24
- package/dist/{query-DK1nWcmp.d.ts.map → query-Cmv_U8vf.d.ts.map} +1 -1
- package/dist/{random-ByUSnV9w.js → random-DDDcC7YU.js} +2 -2
- package/dist/{random-ByUSnV9w.js.map → random-DDDcC7YU.js.map} +1 -1
- package/dist/{raw-provider-sink-DoHz4JNG.js → raw-provider-sink-75w4FWBj.js} +2 -2
- package/dist/{raw-provider-sink-DoHz4JNG.js.map → raw-provider-sink-75w4FWBj.js.map} +1 -1
- package/dist/{redact-mtCCcT8z.d.ts → redact-DXdlFb_U.d.ts} +2 -3
- package/dist/redact-DXdlFb_U.d.ts.map +1 -0
- package/dist/{redact-B5xXKqQ3.js → redact-mtkTiJVk.js} +2 -7
- package/dist/{redact-B5xXKqQ3.js.map → redact-mtkTiJVk.js.map} +1 -1
- package/dist/{registry-D7_hm2fK.d.ts → registry-ssNoHzwD.d.ts} +4 -4
- package/dist/{registry-D7_hm2fK.d.ts.map → registry-ssNoHzwD.d.ts.map} +1 -1
- package/dist/{release-confidence-CpuP65S5.js → release-confidence-CYIFH-fp.js} +6 -11
- package/dist/release-confidence-CYIFH-fp.js.map +1 -0
- package/dist/{report-command-SC9nqb4M.js → render-Cs34y0iX.js} +625 -587
- package/dist/render-Cs34y0iX.js.map +1 -0
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +8 -8
- package/dist/{researcher-CEcfku-T.d.ts → researcher-GU1tV4dn.d.ts} +9 -9
- package/dist/{researcher-CEcfku-T.d.ts.map → researcher-GU1tV4dn.d.ts.map} +1 -1
- package/dist/{reward-CpCmAM8L.d.ts → reward-DrZ71vnf.d.ts} +3 -26
- package/dist/reward-DrZ71vnf.d.ts.map +1 -0
- package/dist/{reward-hacking-E0d-WVcP.js → reward-hacking-ByuETaIb.js} +3 -3
- package/dist/{reward-hacking-E0d-WVcP.js.map → reward-hacking-ByuETaIb.js.map} +1 -1
- package/dist/{reward-hacking-RVdZ6Kkt.d.ts → reward-hacking-E52wq86n.d.ts} +5 -18
- package/dist/reward-hacking-E52wq86n.d.ts.map +1 -0
- package/dist/reward-nw2xZGZG.js.map +1 -1
- package/dist/rl.d.ts +117 -295
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +27 -747
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -5
- package/dist/rollout/index.js +7 -8
- package/dist/{rollout-DZV1t2Wg.js → rollout-Dd1ucT_M.js} +88 -13
- package/dist/rollout-Dd1ucT_M.js.map +1 -0
- package/dist/{rubric-predictive-validity-DaFW70y9.js → rubric-predictive-validity-D3_oI72N.js} +3 -3
- package/dist/{rubric-predictive-validity-DaFW70y9.js.map → rubric-predictive-validity-D3_oI72N.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts → rubric-predictive-validity-DiwT8PbL.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C03EC7ZE.d.ts.map → rubric-predictive-validity-DiwT8PbL.d.ts.map} +1 -1
- package/dist/{run-record-C1E24G2A.d.ts → run-record-CaxbLq6Y.d.ts} +4 -10
- package/dist/run-record-CaxbLq6Y.d.ts.map +1 -0
- package/dist/{run-record-C5e7FjhJ.js → run-record-CysKLB93.js} +3 -3
- package/dist/run-record-CysKLB93.js.map +1 -0
- package/dist/{run-record-Boc5IUTp.js → run-record-DhEJgoPz.js} +4 -4
- package/dist/{run-record-Boc5IUTp.js.map → run-record-DhEJgoPz.js.map} +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{schema-CSf6qWgZ.js → schema-Bt2vLFm3.js} +2 -14
- package/dist/schema-Bt2vLFm3.js.map +1 -0
- package/dist/schema-C-ziyJD7.d.ts.map +1 -1
- package/dist/{schema-BuiujZ-g.js → schema-CvhfLjtZ.js} +78 -48
- package/dist/schema-CvhfLjtZ.js.map +1 -0
- package/dist/{schema-2g7trUwH.d.ts → schema-D6EVKzTB.d.ts} +4 -5
- package/dist/schema-D6EVKzTB.d.ts.map +1 -0
- package/dist/search/index.d.ts +3 -31
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -4
- package/dist/{search-kernel-DyNEa6IU.d.ts → search-kernel-p9JB3SmL.d.ts} +10 -27
- package/dist/search-kernel-p9JB3SmL.d.ts.map +1 -0
- package/dist/{search-ledger-CVUGLDZp.js → search-ledger-ClpN4I3e.js} +7 -7
- package/dist/{search-ledger-CVUGLDZp.js.map → search-ledger-ClpN4I3e.js.map} +1 -1
- package/dist/{search-ledger-file-DDQUgvCg.js → search-ledger-file-Bd5Tioq6.js} +4 -4
- package/dist/search-ledger-file-Bd5Tioq6.js.map +1 -0
- package/dist/{search-shipper-BOSD46qv.js → search-shipper-Bg2HW3Uv.js} +4 -4
- package/dist/search-shipper-Bg2HW3Uv.js.map +1 -0
- package/dist/{search-summary-BQCSWf1g.js → search-summary-CrBuvVnf.js} +5 -5
- package/dist/{search-summary-BQCSWf1g.js.map → search-summary-CrBuvVnf.js.map} +1 -1
- package/dist/{sequential-CzK5DarL.js → sequential-BVv1sjsw.js} +2 -32
- package/dist/{sequential-CzK5DarL.js.map → sequential-BVv1sjsw.js.map} +1 -1
- package/dist/{sequential-Bf_pbE50.js → sequential-CimQrbjv.js} +5 -5
- package/dist/sequential-CimQrbjv.js.map +1 -0
- package/dist/{sequential-BhsrMupG.d.ts → sequential-KwGspfeR.d.ts} +2 -9
- package/dist/{sequential-BhsrMupG.d.ts.map → sequential-KwGspfeR.d.ts.map} +1 -1
- package/dist/{sequential-eprocess-DM_qENCl.js → sequential-eprocess-DodI6RoF.js} +2 -2
- package/dist/{sequential-eprocess-DM_qENCl.js.map → sequential-eprocess-DodI6RoF.js.map} +1 -1
- package/dist/{server-e9oWO_bN.js → server-C6M0qR8h.js} +101 -101
- package/dist/server-C6M0qR8h.js.map +1 -0
- package/dist/{single-run-lock-CV9Eyd_q.js → single-run-lock-Dekd8_Wv.js} +7 -7
- package/dist/single-run-lock-Dekd8_Wv.js.map +1 -0
- package/dist/{skillopt-optimization-method-BiwA8nfX.js → skillopt-optimization-method-5dlex7MJ.js} +1386 -1184
- package/dist/skillopt-optimization-method-5dlex7MJ.js.map +1 -0
- package/dist/{spans-BHAUXMGN.d.ts → spans-CVngGij3.d.ts} +2 -2
- package/dist/{spans-BHAUXMGN.d.ts.map → spans-CVngGij3.d.ts.map} +1 -1
- package/dist/{statistical-heldout-CYpGaoDH.d.ts → statistical-heldout-BHfLZHrm.d.ts} +8 -67
- package/dist/statistical-heldout-BHfLZHrm.d.ts.map +1 -0
- package/dist/{statistical-heldout-42NlCNDe.js → statistical-heldout-Bksv92Th.js} +3 -3
- package/dist/statistical-heldout-Bksv92Th.js.map +1 -0
- package/dist/{steps-Bg5-KVx1.d.ts → steps-DlepJPwD.d.ts} +2 -56
- package/dist/steps-DlepJPwD.d.ts.map +1 -0
- package/dist/{store-C4avFjIi.d.ts → store-B3nWRy3B.d.ts} +2 -2
- package/dist/{store-C4avFjIi.d.ts.map → store-B3nWRy3B.d.ts.map} +1 -1
- package/dist/{store-otlp-DJ9dtA5x.js → store-otlp-BteDAbHK.js} +3 -3
- package/dist/{store-otlp-DJ9dtA5x.js.map → store-otlp-BteDAbHK.js.map} +1 -1
- package/dist/{store-tool-spans-CIX5arAG.js → store-tool-spans-D5l_OfTJ.js} +8 -8
- package/dist/store-tool-spans-D5l_OfTJ.js.map +1 -0
- package/dist/{store-tool-spans-DdCSPa7C.d.ts → store-tool-spans-DZS0HWVf.d.ts} +9 -18
- package/dist/store-tool-spans-DZS0HWVf.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +4 -11
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +1 -16
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-V_JACR4Y.js → summary-report-BsXJoYX5.js} +88 -10
- package/dist/summary-report-BsXJoYX5.js.map +1 -0
- package/dist/{summary-report-bdFDXq0b.d.ts → summary-report-CkbRcSuu.d.ts} +25 -19
- package/dist/summary-report-CkbRcSuu.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +60 -28
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +4 -458
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{systemone-BomhHr2c.d.ts → systemone-BMzDJb1i.d.ts} +5 -5
- package/dist/{systemone-BomhHr2c.d.ts.map → systemone-BMzDJb1i.d.ts.map} +1 -1
- package/dist/systemone-protocol.js +1 -1
- package/dist/{systemone-review-CUVcQJu6.d.ts → systemone-review-CwE6JGIp.d.ts} +2 -2
- package/dist/{systemone-review-CUVcQJu6.d.ts.map → systemone-review-CwE6JGIp.d.ts.map} +1 -1
- package/dist/{systemone-review-D7sXwsqq.js → systemone-review-DzdOJ8I6.js} +3 -3
- package/dist/{systemone-review-D7sXwsqq.js.map → systemone-review-DzdOJ8I6.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -5
- package/dist/systemone.js.map +1 -1
- package/dist/{task-failure-attributes-hEmASjix.js → task-failure-attributes-BMsblaDh.js} +3 -3
- package/dist/task-failure-attributes-BMsblaDh.js.map +1 -0
- package/dist/{terminal-record-DZyQZ3pz.js → terminal-record-D4Rv-j1W.js} +244 -196
- package/dist/terminal-record-D4Rv-j1W.js.map +1 -0
- package/dist/{tool-groups-C3NEcdCM.d.ts → tool-groups-DyNz2JZ_.d.ts} +3 -3
- package/dist/tool-groups-DyNz2JZ_.d.ts.map +1 -0
- package/dist/{tool-waste-BKgvd-TP.d.ts → tool-waste-CkjzWmeY.d.ts} +25 -28
- package/dist/tool-waste-CkjzWmeY.d.ts.map +1 -0
- package/dist/{tool-waste-BlesqTqW.js → tool-waste-DLsjS_p-.js} +5 -5
- package/dist/{tool-waste-BlesqTqW.js.map → tool-waste-DLsjS_p-.js.map} +1 -1
- package/dist/{trace-questions-CM2CAocU.d.ts → trace-questions-xxwgsuMi.d.ts} +4 -9
- package/dist/{trace-questions-CM2CAocU.d.ts.map → trace-questions-xxwgsuMi.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +14 -211
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +865 -981
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +18 -46
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +13 -13
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.d.ts +7 -118
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +921 -869
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-C1XYeXvy.js → tree-B5e1uAeL.js} +132 -21
- package/dist/tree-B5e1uAeL.js.map +1 -0
- package/dist/{types-Cbe2fWBa.d.ts → types-BKCzzmit.d.ts} +3 -3
- package/dist/{types-Cbe2fWBa.d.ts.map → types-BKCzzmit.d.ts.map} +1 -1
- package/dist/{types-B3jzCp0p.js → types-BXgASBCL.js} +2 -2
- package/dist/{types-B3jzCp0p.js.map → types-BXgASBCL.js.map} +1 -1
- package/dist/{types-C1tnkEgM.d.ts → types-CW8ewATZ.d.ts} +4 -113
- package/dist/types-CW8ewATZ.d.ts.map +1 -0
- package/dist/{types-BqkVmHfv.d.ts → types-CmXY0PQf.d.ts} +2 -4
- package/dist/types-CmXY0PQf.d.ts.map +1 -0
- package/dist/{types-C02qtnG8.d.ts → types-D-mgRzlh.d.ts} +7 -28
- package/dist/types-D-mgRzlh.d.ts.map +1 -0
- package/dist/{types-IXD2HJzu.js → types-DqFMNTgK.js} +2 -4
- package/dist/types-DqFMNTgK.js.map +1 -0
- package/dist/{types-Cq8mSbq8.d.ts → types-DtkO1G9L.d.ts} +7 -9
- package/dist/{types-Cq8mSbq8.d.ts.map → types-DtkO1G9L.d.ts.map} +1 -1
- package/dist/{verdict-Dl0tiT1X.js → verdict-BGQmxQ8A.js} +3 -3
- package/dist/verdict-BGQmxQ8A.js.map +1 -0
- package/dist/{verdict-E4eRNf7-.d.ts → verdict-C24NELtN.d.ts} +2 -15
- package/dist/{verdict-E4eRNf7-.d.ts.map → verdict-C24NELtN.d.ts.map} +1 -1
- package/dist/{verdict-cache-Bm8EKL2s.js → verdict-cache-CL_7tePD.js} +2 -2
- package/dist/verdict-cache-CL_7tePD.js.map +1 -0
- package/dist/wire/index.d.ts +31 -410
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +2 -2
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/jev-product-adoption.md +1 -1
- package/docs/changes/jev-recorded-evidence.md +1 -1
- package/docs/design/statistics-decisions.md +18 -18
- package/docs/design.md +1 -1
- package/docs/eval-surface-map.md +1 -0
- package/docs/evaluation-integrity.md +1 -1
- package/docs/profile-json-identity.md +1 -2
- package/docs/proofs/campaign-attempt-retention-20261001/README.md +2 -2
- package/docs/public-api.md +158 -538
- package/docs/rollout.md +1 -1
- package/docs/trace-repair-admission.md +1 -1
- package/docs/wire-protocol.md +1 -1
- package/package.json +7 -2
- package/dist/active-curriculum-B_Lc9kfO.js +0 -61
- package/dist/active-curriculum-B_Lc9kfO.js.map +0 -1
- package/dist/agent-profile-VFcQncbW.d.ts.map +0 -1
- package/dist/agent-profile-cell-BK8mlqlq.js.map +0 -1
- package/dist/agent-profile-cell-BNp-Js9F.d.ts.map +0 -1
- package/dist/agreement-irr-BEWLIF3_.js.map +0 -1
- package/dist/analyze-runs-C84irm4i.js.map +0 -1
- package/dist/baseline-Udi_PLuT.js +0 -259
- package/dist/baseline-Udi_PLuT.js.map +0 -1
- package/dist/benchmark-CXmVVHH2.js +0 -562
- package/dist/benchmark-CXmVVHH2.js.map +0 -1
- package/dist/benchmark-command-By0vh8q7.js.map +0 -1
- package/dist/campaign-BudZ70zo.js.map +0 -1
- package/dist/chat-json-call-puMArmtW.js.map +0 -1
- package/dist/claude-jsonl-dZir12m6.js +0 -86
- package/dist/claude-jsonl-dZir12m6.js.map +0 -1
- package/dist/client-BzL9niqi.d.ts.map +0 -1
- package/dist/cost-ledger-BP8V4kUQ.js.map +0 -1
- package/dist/counterfactual-CPaWN99-.d.ts +0 -60
- package/dist/counterfactual-CPaWN99-.d.ts.map +0 -1
- package/dist/counterfactual-Dn3veueY.js +0 -98
- package/dist/counterfactual-Dn3veueY.js.map +0 -1
- package/dist/default-registry-COhk9g36.js.map +0 -1
- package/dist/default-registry-DzHiGruW.d.ts.map +0 -1
- package/dist/define-agent-eval-CNTPzIkp.js.map +0 -1
- package/dist/define-agent-eval-WSck0CZJ.d.ts.map +0 -1
- package/dist/dspy-rlm-engine-LRCu61m5.js.map +0 -1
- package/dist/effect-sizes-BhzEpXMc.js +0 -82
- package/dist/effect-sizes-BhzEpXMc.js.map +0 -1
- package/dist/emitter-HcCmFQl_.d.ts.map +0 -1
- package/dist/engine-3A_4jOlI.d.ts.map +0 -1
- package/dist/equal-terms-BtdXayCJ.js.map +0 -1
- package/dist/eval-campaign-90jeaCAI.js +0 -343
- package/dist/eval-campaign-90jeaCAI.js.map +0 -1
- package/dist/evaluation-6Yoi74gM.d.ts.map +0 -1
- package/dist/exporters-X4xwla8A.js.map +0 -1
- package/dist/external-optimizer-process-BjMGUuds.js.map +0 -1
- package/dist/external-optimizer-subprocess-BqW0hHcI.js.map +0 -1
- package/dist/failure-cluster-CbgEDcFJ.d.ts.map +0 -1
- package/dist/feedback-trajectory-C4EG0nNH.d.ts.map +0 -1
- package/dist/hf-dataset-BR2yhuNn.js.map +0 -1
- package/dist/index-BFRO6chs.d.ts.map +0 -1
- package/dist/index-BO41eGj5.d.ts.map +0 -1
- package/dist/index-CyszSg4O.d.ts.map +0 -1
- package/dist/index-DHKEAv3s.d.ts.map +0 -1
- package/dist/index-DPcngtwt.d.ts.map +0 -1
- package/dist/integrity-dLiDe2kD.js.map +0 -1
- package/dist/journal-DfRz9kps.js.map +0 -1
- package/dist/judge-gate-CE7CQAgt.d.ts.map +0 -1
- package/dist/kind-factory-BobI5Jy3.js.map +0 -1
- package/dist/ledger-core-Gq7yp8lP.js.map +0 -1
- package/dist/llm-client-CyBAoYiX.js.map +0 -1
- package/dist/llm-judge-CG12SQJq.js.map +0 -1
- package/dist/matrix-DSrSxjjy.d.ts.map +0 -1
- package/dist/matrix-Dm0BQW7U.js.map +0 -1
- package/dist/mean-DHoMiudD.js.map +0 -1
- package/dist/normal-DsJZtrJc.js +0 -25
- package/dist/normal-DsJZtrJc.js.map +0 -1
- package/dist/paired-arms-CwMGJsrS.js.map +0 -1
- package/dist/paired-binary-CvpFHfKZ.js.map +0 -1
- package/dist/paired-promotion-decision-DPKz8ZnF.d.ts.map +0 -1
- package/dist/pareto-Cm6ndrWN.js.map +0 -1
- package/dist/prime-protocol-CHKgA3a6.js.map +0 -1
- package/dist/prime-protocol-HiB9HX7E.d.ts.map +0 -1
- package/dist/produced-state-BsDmnwxc.js.map +0 -1
- package/dist/proposal-findings-CY1xKn_a.js.map +0 -1
- package/dist/query-BUCZCK7F.js.map +0 -1
- package/dist/redact-mtCCcT8z.d.ts.map +0 -1
- package/dist/release-confidence-CpuP65S5.js.map +0 -1
- package/dist/report-command-SC9nqb4M.js.map +0 -1
- package/dist/reward-CpCmAM8L.d.ts.map +0 -1
- package/dist/reward-hacking-RVdZ6Kkt.d.ts.map +0 -1
- package/dist/rollout-DZV1t2Wg.js.map +0 -1
- package/dist/run-record-C1E24G2A.d.ts.map +0 -1
- package/dist/run-record-C5e7FjhJ.js.map +0 -1
- package/dist/schema-2g7trUwH.d.ts.map +0 -1
- package/dist/schema-BuiujZ-g.js.map +0 -1
- package/dist/schema-CSf6qWgZ.js.map +0 -1
- package/dist/search-kernel-DyNEa6IU.d.ts.map +0 -1
- package/dist/search-ledger-file-DDQUgvCg.js.map +0 -1
- package/dist/search-shipper-BOSD46qv.js.map +0 -1
- package/dist/semantic-concept-judge-BQF8te0T.js.map +0 -1
- package/dist/sequential-Bf_pbE50.js.map +0 -1
- package/dist/server-e9oWO_bN.js.map +0 -1
- package/dist/single-run-lock-CV9Eyd_q.js.map +0 -1
- package/dist/skillopt-optimization-method-BiwA8nfX.js.map +0 -1
- package/dist/statistical-heldout-42NlCNDe.js.map +0 -1
- package/dist/statistical-heldout-CYpGaoDH.d.ts.map +0 -1
- package/dist/steps-Bg5-KVx1.d.ts.map +0 -1
- package/dist/store-tool-spans-CIX5arAG.js.map +0 -1
- package/dist/store-tool-spans-DdCSPa7C.d.ts.map +0 -1
- package/dist/student-t-DVPW7CdL.js +0 -39
- package/dist/student-t-DVPW7CdL.js.map +0 -1
- package/dist/summary-report-V_JACR4Y.js.map +0 -1
- package/dist/summary-report-bdFDXq0b.d.ts.map +0 -1
- package/dist/task-failure-attributes-hEmASjix.js.map +0 -1
- package/dist/terminal-record-DZyQZ3pz.js.map +0 -1
- package/dist/tool-groups-C3NEcdCM.d.ts.map +0 -1
- package/dist/tool-waste-BKgvd-TP.d.ts.map +0 -1
- package/dist/trajectory-Y1fmdlw7.d.ts +0 -24
- package/dist/trajectory-Y1fmdlw7.d.ts.map +0 -1
- package/dist/tree-C1XYeXvy.js.map +0 -1
- package/dist/types-BqkVmHfv.d.ts.map +0 -1
- package/dist/types-C02qtnG8.d.ts.map +0 -1
- package/dist/types-C1tnkEgM.d.ts.map +0 -1
- package/dist/types-IXD2HJzu.js.map +0 -1
- package/dist/usage-receipt-Cx3xNeyB.js +0 -76
- package/dist/usage-receipt-Cx3xNeyB.js.map +0 -1
- package/dist/verdict-Dl0tiT1X.js.map +0 -1
- package/dist/verdict-cache-Bm8EKL2s.js.map +0 -1
- package/docs/design/self-improvement-evidence-audit.md +0 -263
|
@@ -1,115 +1,29 @@
|
|
|
1
|
-
import { s as ValidationError, t as AgentEvalError } from "./errors-
|
|
2
|
-
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString } from "./canonical-
|
|
3
|
-
import { c as
|
|
4
|
-
import {
|
|
5
|
-
import { c as validateRunRecord, i as modelHasSnapshot, n as UNKNOWN_MODEL, o as runTaskScore } from "./run-record-
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { o as pairHoldout } from "./statistical-heldout-
|
|
9
|
-
import { n as contentHash, t as canonicalJson } from "./verdict-cache-
|
|
10
|
-
import {
|
|
11
|
-
import { n as campaignCellToRunRecord, r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import
|
|
16
|
-
import "./
|
|
17
|
-
import "./search-
|
|
18
|
-
import "./
|
|
19
|
-
import
|
|
20
|
-
import "./
|
|
21
|
-
import "./skillopt-optimization-method-BiwA8nfX.js";
|
|
22
|
-
import "./sequential-Bf_pbE50.js";
|
|
1
|
+
import { s as ValidationError, t as AgentEvalError } from "./errors-BcOs5MKW.js";
|
|
2
|
+
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString } from "./canonical-Dq2hOQ8L.js";
|
|
3
|
+
import { c as agentProfileModelId, o as agentProfileHash, r as verifyCompletion, s as agentProfileId, u as harnessAxisOf } from "./completion-verifier-DbLVf3i8.js";
|
|
4
|
+
import { i as buildAgentProfileCell } from "./agent-profile-cell-B1V1gUZs.js";
|
|
5
|
+
import { c as validateRunRecord, i as modelHasSnapshot, n as UNKNOWN_MODEL, o as runTaskScore } from "./run-record-DhEJgoPz.js";
|
|
6
|
+
import { t as comparePairedArms } from "./paired-arms-Ct4LTKc1.js";
|
|
7
|
+
import { i as pairedBootstrap } from "./mean-BbA_JCzW.js";
|
|
8
|
+
import { o as pairHoldout } from "./statistical-heldout-Bksv92Th.js";
|
|
9
|
+
import { n as contentHash, t as canonicalJson } from "./verdict-cache-CL_7tePD.js";
|
|
10
|
+
import { $ as planCampaignRun, Q as runCampaign, et as resolveRunDir, nt as cellCachePath, rt as computeManifestHash, s as executeOptimizationMethod, u as combineComparisonCosts } from "./llm-judge-Djxh-Uhs.js";
|
|
11
|
+
import { n as campaignCellToRunRecord, r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-CysKLB93.js";
|
|
12
|
+
import { C as assertCodeSurfaceIdentity, D as surfaceContentHash, M as assertRealBackend, N as summarizeBackendIntegrity, P as assertCampaignDesign, R as campaignScenarioIdentity } from "./campaign-evidence-DunzcJGv.js";
|
|
13
|
+
import { a as mapConcurrent } from "./journal-C7om8Gum.js";
|
|
14
|
+
import { M as fsCampaignStorage } from "./external-optimizer-subprocess-B5gdLRZv.js";
|
|
15
|
+
import "./pareto-CTv8vozZ.js";
|
|
16
|
+
import "./search-ledger-ClpN4I3e.js";
|
|
17
|
+
import "./search-summary-CrBuvVnf.js";
|
|
18
|
+
import "./external-optimizer-process-C3Wm7Vhn.js";
|
|
19
|
+
import "./skillopt-optimization-method-5dlex7MJ.js";
|
|
20
|
+
import "./sequential-CimQrbjv.js";
|
|
23
21
|
import { createHash } from "node:crypto";
|
|
24
|
-
import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync
|
|
22
|
+
import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
25
23
|
import { z } from "zod";
|
|
26
24
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
27
25
|
import { devNull, tmpdir } from "node:os";
|
|
28
26
|
import { execFileSync } from "node:child_process";
|
|
29
|
-
//#region src/campaign/analyst-surface.ts
|
|
30
|
-
function buildTraceAnalystSurfaceDispatch(options) {
|
|
31
|
-
return async (surface, scenario, context) => {
|
|
32
|
-
if (typeof surface !== "string") throw new TypeError(`buildTraceAnalystSurfaceDispatch requires a string prompt, received ${surface.kind}`);
|
|
33
|
-
return options.analyze({
|
|
34
|
-
instructions: surface,
|
|
35
|
-
traceStore: scenario.traceStore,
|
|
36
|
-
runId: `${context.cellId}:${scenario.id}`,
|
|
37
|
-
signal: context.signal
|
|
38
|
-
});
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
|
-
function traceAnalystQualityJudge() {
|
|
42
|
-
return {
|
|
43
|
-
name: "trace-analyst-quality",
|
|
44
|
-
judgeVersion: "agent-eval:trace-analyst-quality:v3",
|
|
45
|
-
dimensions: [
|
|
46
|
-
{
|
|
47
|
-
key: "issue_recall",
|
|
48
|
-
description: "share of labeled issues found"
|
|
49
|
-
},
|
|
50
|
-
{
|
|
51
|
-
key: "finding_precision",
|
|
52
|
-
description: "share of findings tied to a labeled issue"
|
|
53
|
-
},
|
|
54
|
-
{
|
|
55
|
-
key: "f1",
|
|
56
|
-
description: "harmonic mean of issue recall and finding precision"
|
|
57
|
-
},
|
|
58
|
-
{
|
|
59
|
-
key: "critical_step_accuracy",
|
|
60
|
-
description: "share of labeled causal steps cited"
|
|
61
|
-
},
|
|
62
|
-
{
|
|
63
|
-
key: "citation_coverage",
|
|
64
|
-
description: "share of findings with evidence citations"
|
|
65
|
-
},
|
|
66
|
-
{
|
|
67
|
-
key: "citation_label_agreement",
|
|
68
|
-
description: "share of citations matching a labeled case location"
|
|
69
|
-
},
|
|
70
|
-
{
|
|
71
|
-
key: "clean",
|
|
72
|
-
description: "1 when a clean case produces no findings"
|
|
73
|
-
}
|
|
74
|
-
],
|
|
75
|
-
appliesTo: (scenario) => scenario.kind === "trace-analyst" && scenario.labelState !== "unlabeled",
|
|
76
|
-
score({ artifact, scenario }) {
|
|
77
|
-
assertScorableScenario(scenario);
|
|
78
|
-
return traceAnalystJudgeScore(scoreAnalystFindings({
|
|
79
|
-
id: scenario.id,
|
|
80
|
-
expectedIssues: scenario.expectedIssues,
|
|
81
|
-
labeledEvidence: scenario.labeledEvidence
|
|
82
|
-
}, artifact.findings));
|
|
83
|
-
}
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
function assertScorableScenario(scenario) {
|
|
87
|
-
if (scenario.labelState === "unlabeled") throw new TypeError(`trace analyst scenario '${scenario.id}' has no quality labels`);
|
|
88
|
-
if (scenario.labelState === "positive" && scenario.expectedIssues.length === 0) throw new TypeError(`positive trace analyst scenario '${scenario.id}' requires expected issues`);
|
|
89
|
-
if (scenario.labelState === "trusted-negative" && scenario.expectedIssues.length > 0) throw new TypeError(`trusted-negative trace analyst scenario '${scenario.id}' cannot contain expected issues`);
|
|
90
|
-
}
|
|
91
|
-
function traceAnalystJudgeScore(score) {
|
|
92
|
-
const dimensions = {
|
|
93
|
-
issue_recall: score.issueRecall,
|
|
94
|
-
finding_precision: score.findingPrecision,
|
|
95
|
-
f1: score.f1,
|
|
96
|
-
clean: score.predictionOnLabelEmptyCase ? 0 : 1
|
|
97
|
-
};
|
|
98
|
-
if (score.criticalStepAccuracy !== null) dimensions.critical_step_accuracy = score.criticalStepAccuracy;
|
|
99
|
-
if (score.citationCoverage !== null) dimensions.citation_coverage = score.citationCoverage;
|
|
100
|
-
if (score.citationLabelAgreement !== null) dimensions.citation_label_agreement = score.citationLabelAgreement;
|
|
101
|
-
const notes = [
|
|
102
|
-
`${score.matchedIssueIds.length}/${score.expectedIssueCount} issues found`,
|
|
103
|
-
`${score.supportedFindingIndexes.length}/${score.supportedFindingIndexes.length + score.unsupportedFindingIndexes.length} findings supported`,
|
|
104
|
-
`${score.unlabeledEvidence.length} citations outside labeled locations`
|
|
105
|
-
].join("; ");
|
|
106
|
-
return {
|
|
107
|
-
dimensions,
|
|
108
|
-
composite: score.criticalStepAccuracy === null ? score.f1 : (score.f1 + score.criticalStepAccuracy) / 2,
|
|
109
|
-
notes
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
//#endregion
|
|
113
27
|
//#region src/campaign/cross-surface-context.ts
|
|
114
28
|
function validateCrossSurfaceInput(input) {
|
|
115
29
|
assertNonEmptyUnique(input.taskOrder, "taskOrder");
|
|
@@ -1045,264 +959,6 @@ function neutralizationGate(options) {
|
|
|
1045
959
|
};
|
|
1046
960
|
}
|
|
1047
961
|
//#endregion
|
|
1048
|
-
//#region src/campaign/grounded-reflection.ts
|
|
1049
|
-
/**
|
|
1050
|
-
* Deterministic per-field diff of call arguments between passing and failing
|
|
1051
|
-
* rollouts. A field set by failing rollouts but left unset by passing ones is
|
|
1052
|
-
* the classic poison-input signature; a field whose values differ across the
|
|
1053
|
-
* split points at the correct value. Feed `text` to the reviser verbatim.
|
|
1054
|
-
*/
|
|
1055
|
-
function rolloutArgumentDiff(rollouts, opts = {}) {
|
|
1056
|
-
const passThreshold = opts.passThreshold ?? 1;
|
|
1057
|
-
const maxValues = opts.maxValuesPerField ?? 4;
|
|
1058
|
-
const collect = (pass) => {
|
|
1059
|
-
const byField = /* @__PURE__ */ new Map();
|
|
1060
|
-
for (const r of rollouts) {
|
|
1061
|
-
if (pass !== r.score >= passThreshold) continue;
|
|
1062
|
-
for (const c of r.calls) for (const [k, v] of Object.entries(c.args)) {
|
|
1063
|
-
const set = byField.get(k) ?? /* @__PURE__ */ new Set();
|
|
1064
|
-
set.add(String(v));
|
|
1065
|
-
byField.set(k, set);
|
|
1066
|
-
}
|
|
1067
|
-
}
|
|
1068
|
-
return byField;
|
|
1069
|
-
};
|
|
1070
|
-
const passing = collect(true);
|
|
1071
|
-
const failing = collect(false);
|
|
1072
|
-
const lines = [];
|
|
1073
|
-
for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
|
|
1074
|
-
const pv = [...passing.get(field) ?? []].slice(0, maxValues);
|
|
1075
|
-
const fv = [...failing.get(field) ?? []].slice(0, maxValues);
|
|
1076
|
-
const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
|
|
1077
|
-
lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
|
|
1078
|
-
}
|
|
1079
|
-
const lower = (m) => {
|
|
1080
|
-
const out = /* @__PURE__ */ new Set();
|
|
1081
|
-
for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
|
|
1082
|
-
return out;
|
|
1083
|
-
};
|
|
1084
|
-
return {
|
|
1085
|
-
text: lines.join("\n") || " (no calls observed)",
|
|
1086
|
-
passingValues: lower(passing),
|
|
1087
|
-
failingValues: lower(failing)
|
|
1088
|
-
};
|
|
1089
|
-
}
|
|
1090
|
-
/**
|
|
1091
|
-
* Scan revised artifact text for single-quoted single-word literals (the
|
|
1092
|
-
* "use exactly 'new'" pattern) that appear in no passing rollout's argument
|
|
1093
|
-
* values. Multi-word quotes pass (they are prose, not prescriptions).
|
|
1094
|
-
* Callers should reject on `harmful` (with a bounded retry) and at most log
|
|
1095
|
-
* `ungrounded` - see the module header for why the severities differ.
|
|
1096
|
-
*/
|
|
1097
|
-
function classifyUngroundedLiterals(text, diff) {
|
|
1098
|
-
const ungrounded = /* @__PURE__ */ new Set();
|
|
1099
|
-
for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
|
|
1100
|
-
const w = m[1].toLowerCase();
|
|
1101
|
-
if (!diff.passingValues.has(w)) ungrounded.add(w);
|
|
1102
|
-
}
|
|
1103
|
-
const all = [...ungrounded];
|
|
1104
|
-
return {
|
|
1105
|
-
ungrounded: all,
|
|
1106
|
-
harmful: all.filter((w) => diff.failingValues.has(w))
|
|
1107
|
-
};
|
|
1108
|
-
}
|
|
1109
|
-
//#endregion
|
|
1110
|
-
//#region src/campaign/labeled-store/fs-adapter.ts
|
|
1111
|
-
/**
|
|
1112
|
-
* Filesystem `LabeledScenarioStore` adapter. The default capture sink for
|
|
1113
|
-
* traces + eval artifacts. Production deployments typically swap for a
|
|
1114
|
-
* Turso/SQLite adapter (same interface).
|
|
1115
|
-
*
|
|
1116
|
-
* Records land as one JSONL file per source under `<root>/<source>.jsonl`.
|
|
1117
|
-
* Each line is a `LabeledScenarioRecord`. Append-only — no in-place edits.
|
|
1118
|
-
*
|
|
1119
|
-
* Safety properties enforced at write-time:
|
|
1120
|
-
*
|
|
1121
|
-
* - **Provenance required**: writes without `source`, `sourceVersionHash`,
|
|
1122
|
-
* `capturedAt`, `redactionStatus` are rejected. Closes the alignment
|
|
1123
|
-
* reviewer's data-poisoning gap.
|
|
1124
|
-
* - **Per-source rate limits**: optional `rateLimitBucket` + `maxWritesPerMinute`
|
|
1125
|
-
* stops a single tenant/source from flooding the store.
|
|
1126
|
-
*
|
|
1127
|
-
* Safety properties enforced at sample-time:
|
|
1128
|
-
*
|
|
1129
|
-
* - **Required split + capturedBefore**: substrate refuses to sample without
|
|
1130
|
-
* an explicit `split` ('train' | 'test') AND a temporal cutoff. Eliminates
|
|
1131
|
-
* accidental train/test contamination.
|
|
1132
|
-
* - **Default training-source filter**: when the store is sampled with
|
|
1133
|
-
* `split: 'train'`, production-trace records are EXCLUDED unless the
|
|
1134
|
-
* caller passes `filter.source: 'production-trace'` explicitly. Closes
|
|
1135
|
-
* the contamination-by-default gap flagged by the senior eval engineer.
|
|
1136
|
-
*/
|
|
1137
|
-
/** Typed rejection from a labeled-scenario store (bad provenance, rate limit, invalid sample args) — carries a stable string `code`. */
|
|
1138
|
-
var LabeledScenarioStoreError = class extends Error {
|
|
1139
|
-
code;
|
|
1140
|
-
constructor(code, message) {
|
|
1141
|
-
super(message);
|
|
1142
|
-
this.code = code;
|
|
1143
|
-
this.name = "LabeledScenarioStoreError";
|
|
1144
|
-
}
|
|
1145
|
-
};
|
|
1146
|
-
/**
|
|
1147
|
-
* Filesystem `LabeledScenarioStore`: appends one JSONL file per source with provenance and
|
|
1148
|
-
* rate-limit guards. For tests, local dev, and small workloads — high-throughput lands in Turso.
|
|
1149
|
-
*/
|
|
1150
|
-
var FsLabeledScenarioStore = class {
|
|
1151
|
-
options;
|
|
1152
|
-
now;
|
|
1153
|
-
rateLimits = /* @__PURE__ */ new Map();
|
|
1154
|
-
constructor(options) {
|
|
1155
|
-
this.options = options;
|
|
1156
|
-
if (!existsSync(options.root)) mkdirSync(options.root, { recursive: true });
|
|
1157
|
-
this.now = options.now ?? Date.now;
|
|
1158
|
-
}
|
|
1159
|
-
async observe(write) {
|
|
1160
|
-
this.assertProvenance(write);
|
|
1161
|
-
this.assertRateLimit(write);
|
|
1162
|
-
const record = this.toRecord(write);
|
|
1163
|
-
appendLine(this.pathForSource(write.source), `${JSON.stringify(record)}\n`);
|
|
1164
|
-
}
|
|
1165
|
-
async sample(args) {
|
|
1166
|
-
if (!args.split) throw new LabeledScenarioStoreError("split_required", "sample() requires an explicit `split` (train | test) — substrate refuses ambiguous reads");
|
|
1167
|
-
if (!args.capturedBefore) throw new LabeledScenarioStoreError("capturedBefore_required", "sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline");
|
|
1168
|
-
const all = [];
|
|
1169
|
-
for (const source of ALL_SOURCES) {
|
|
1170
|
-
if (args.split === "train" && source === "production-trace") {
|
|
1171
|
-
if (!sourceFilterContains(args.filter?.source, "production-trace")) continue;
|
|
1172
|
-
}
|
|
1173
|
-
const path = this.pathForSource(source);
|
|
1174
|
-
if (!existsSync(path)) continue;
|
|
1175
|
-
const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
|
|
1176
|
-
for (const line of lines) {
|
|
1177
|
-
let record;
|
|
1178
|
-
try {
|
|
1179
|
-
record = JSON.parse(line);
|
|
1180
|
-
} catch {
|
|
1181
|
-
continue;
|
|
1182
|
-
}
|
|
1183
|
-
if (!matchesFilter(record, args, source)) continue;
|
|
1184
|
-
all.push(record);
|
|
1185
|
-
}
|
|
1186
|
-
}
|
|
1187
|
-
all.sort((a, b) => {
|
|
1188
|
-
if (a.capturedAt !== b.capturedAt) return compareCodeUnits(a.capturedAt, b.capturedAt);
|
|
1189
|
-
return compareCodeUnits(a.recordHash, b.recordHash);
|
|
1190
|
-
});
|
|
1191
|
-
return all.slice(0, args.count);
|
|
1192
|
-
}
|
|
1193
|
-
async size() {
|
|
1194
|
-
const bySource = {};
|
|
1195
|
-
const byTrust = {
|
|
1196
|
-
unverified: 0,
|
|
1197
|
-
"verified-signal": 0,
|
|
1198
|
-
"human-rated": 0
|
|
1199
|
-
};
|
|
1200
|
-
let total = 0;
|
|
1201
|
-
for (const source of ALL_SOURCES) {
|
|
1202
|
-
const path = this.pathForSource(source);
|
|
1203
|
-
if (!existsSync(path)) {
|
|
1204
|
-
bySource[source] = 0;
|
|
1205
|
-
continue;
|
|
1206
|
-
}
|
|
1207
|
-
const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
|
|
1208
|
-
bySource[source] = lines.length;
|
|
1209
|
-
total += lines.length;
|
|
1210
|
-
for (const line of lines) {
|
|
1211
|
-
let trust = "unverified";
|
|
1212
|
-
try {
|
|
1213
|
-
trust = JSON.parse(line).labelTrust ?? "unverified";
|
|
1214
|
-
} catch {}
|
|
1215
|
-
byTrust[trust] += 1;
|
|
1216
|
-
}
|
|
1217
|
-
}
|
|
1218
|
-
return {
|
|
1219
|
-
train: total,
|
|
1220
|
-
test: total,
|
|
1221
|
-
bySource,
|
|
1222
|
-
byTrust
|
|
1223
|
-
};
|
|
1224
|
-
}
|
|
1225
|
-
assertProvenance(write) {
|
|
1226
|
-
if (!write.source) throw new LabeledScenarioStoreError("missing_source", "LabeledScenarioWrite requires `source`");
|
|
1227
|
-
if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) throw new LabeledScenarioStoreError("missing_source_version", "LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)");
|
|
1228
|
-
if (!write.capturedAt) throw new LabeledScenarioStoreError("missing_captured_at", "LabeledScenarioWrite requires `capturedAt` ISO timestamp");
|
|
1229
|
-
if (!write.redactionStatus) throw new LabeledScenarioStoreError("missing_redaction_status", "LabeledScenarioWrite requires explicit `redactionStatus` — raw / redacted-pii / redacted-secrets / fully-redacted");
|
|
1230
|
-
if (!ALL_SOURCES.includes(write.source)) throw new LabeledScenarioStoreError("unknown_source", `LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`);
|
|
1231
|
-
}
|
|
1232
|
-
assertRateLimit(write) {
|
|
1233
|
-
const cap = this.options.maxWritesPerMinutePerBucket;
|
|
1234
|
-
if (!cap || !write.rateLimitBucket) return;
|
|
1235
|
-
const now = this.now();
|
|
1236
|
-
const windowMs = 6e4;
|
|
1237
|
-
let state = this.rateLimits.get(write.rateLimitBucket);
|
|
1238
|
-
if (!state || now - state.windowStartMs >= windowMs) {
|
|
1239
|
-
state = {
|
|
1240
|
-
bucket: write.rateLimitBucket,
|
|
1241
|
-
windowStartMs: now,
|
|
1242
|
-
count: 0
|
|
1243
|
-
};
|
|
1244
|
-
this.rateLimits.set(write.rateLimitBucket, state);
|
|
1245
|
-
}
|
|
1246
|
-
if (state.count >= cap) throw new LabeledScenarioStoreError("rate_limit_exceeded", `LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`);
|
|
1247
|
-
state.count += 1;
|
|
1248
|
-
}
|
|
1249
|
-
toRecord(write) {
|
|
1250
|
-
const recordHash = sha256$1(JSON.stringify({
|
|
1251
|
-
id: write.scenario.id,
|
|
1252
|
-
src: write.source,
|
|
1253
|
-
at: write.capturedAt,
|
|
1254
|
-
ver: write.sourceVersionHash
|
|
1255
|
-
}));
|
|
1256
|
-
return {
|
|
1257
|
-
...write,
|
|
1258
|
-
recordHash,
|
|
1259
|
-
split: "train"
|
|
1260
|
-
};
|
|
1261
|
-
}
|
|
1262
|
-
pathForSource(source) {
|
|
1263
|
-
return join(this.options.root, `${source}.jsonl`);
|
|
1264
|
-
}
|
|
1265
|
-
};
|
|
1266
|
-
const ALL_SOURCES = [
|
|
1267
|
-
"production-trace",
|
|
1268
|
-
"eval-run",
|
|
1269
|
-
"manual",
|
|
1270
|
-
"red-team",
|
|
1271
|
-
"synthetic"
|
|
1272
|
-
];
|
|
1273
|
-
function sourceFilterContains(filter, needle) {
|
|
1274
|
-
if (!filter) return false;
|
|
1275
|
-
if (Array.isArray(filter)) return filter.includes(needle);
|
|
1276
|
-
return filter === needle;
|
|
1277
|
-
}
|
|
1278
|
-
function matchesFilter(record, args, source) {
|
|
1279
|
-
if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
|
|
1280
|
-
if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
|
|
1281
|
-
const f = args.filter;
|
|
1282
|
-
if (!f) return true;
|
|
1283
|
-
if (f.kind && record.scenario.kind !== f.kind) return false;
|
|
1284
|
-
if (f.source) {
|
|
1285
|
-
if (!(Array.isArray(f.source) ? f.source : [f.source]).includes(source)) return false;
|
|
1286
|
-
}
|
|
1287
|
-
if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
|
|
1288
|
-
const composites = Object.values(record.judgeScores).map((s) => s.composite);
|
|
1289
|
-
const max = composites.length === 0 ? 0 : Math.max(...composites);
|
|
1290
|
-
if (f.minComposite !== void 0 && max < f.minComposite) return false;
|
|
1291
|
-
if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
|
|
1292
|
-
}
|
|
1293
|
-
if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) return false;
|
|
1294
|
-
return true;
|
|
1295
|
-
}
|
|
1296
|
-
function sha256$1(input) {
|
|
1297
|
-
return createHash("sha256").update(input).digest("hex").slice(0, 16);
|
|
1298
|
-
}
|
|
1299
|
-
function appendLine(path, line) {
|
|
1300
|
-
if (existsSync(path)) {
|
|
1301
|
-
const existing = readFileSync(path, "utf8");
|
|
1302
|
-
writeFileSync(path, existing + line);
|
|
1303
|
-
} else writeFileSync(path, line);
|
|
1304
|
-
}
|
|
1305
|
-
//#endregion
|
|
1306
962
|
//#region src/campaign/neutralize.ts
|
|
1307
963
|
/**
|
|
1308
964
|
* @module
|
|
@@ -1442,20 +1098,6 @@ function sequentialOptimizationMethod(options) {
|
|
|
1442
1098
|
//#endregion
|
|
1443
1099
|
//#region src/campaign/presets/playback.ts
|
|
1444
1100
|
/**
|
|
1445
|
-
* Adapt a `PlaybackDriver` into a `runProfileMatrix` dispatch. The artifact the
|
|
1446
|
-
* matrix scores is the `ProducedState` extracted from the driver's event
|
|
1447
|
-
* stream — grade it with `scoreUserStory` (or a judge wrapping it).
|
|
1448
|
-
*/
|
|
1449
|
-
function makePlaybackDispatch(driver) {
|
|
1450
|
-
return async (profile, scenario, ctx) => {
|
|
1451
|
-
const events = await driver.run(scenario, {
|
|
1452
|
-
...ctx,
|
|
1453
|
-
profile
|
|
1454
|
-
});
|
|
1455
|
-
return extractProducedState(events);
|
|
1456
|
-
};
|
|
1457
|
-
}
|
|
1458
|
-
/**
|
|
1459
1101
|
* Score one story's produced state against its requirements. Thin wrapper over
|
|
1460
1102
|
* `verifyCompletion` that builds the gold from the story and returns a
|
|
1461
1103
|
* per-requirement PASS/FAIL verdict. `checkCorrectness` is injected — a
|
|
@@ -2394,6 +2036,9 @@ const anchorEntrySchema = z.object({
|
|
|
2394
2036
|
bytes: z.number().int().positive(),
|
|
2395
2037
|
digest
|
|
2396
2038
|
}).strict();
|
|
2039
|
+
function fail(message) {
|
|
2040
|
+
throw new ValidationError(`retained dispatch: ${message}`);
|
|
2041
|
+
}
|
|
2397
2042
|
/**
|
|
2398
2043
|
* Persist dispatch intent before an external call and its validated result afterward.
|
|
2399
2044
|
* Replays settled calls; never repeats an uncertain call or refunds its allowance.
|
|
@@ -2405,157 +2050,192 @@ const anchorEntrySchema = z.object({
|
|
|
2405
2050
|
function createRetainedDispatch(options) {
|
|
2406
2051
|
if (!options.runDir.trim()) throw new ValidationError("retained dispatch requires a run directory");
|
|
2407
2052
|
const storage = options.storage ?? fsCampaignStorage();
|
|
2408
|
-
const parse = options.parse;
|
|
2409
2053
|
const header = headerSchema.parse({
|
|
2410
2054
|
kind: "retained-dispatch-v1",
|
|
2411
2055
|
scope: hashCanonical(options.scope),
|
|
2412
2056
|
limits: Object.entries(options.limits).sort(([a], [b]) => compareCodeUnits(a, b))
|
|
2413
2057
|
});
|
|
2414
2058
|
if (!header.limits.length) throw new ValidationError("retained dispatch requires named allowances");
|
|
2415
|
-
const
|
|
2416
|
-
|
|
2417
|
-
|
|
2059
|
+
const journal = {
|
|
2060
|
+
storage,
|
|
2061
|
+
parse: options.parse,
|
|
2062
|
+
header,
|
|
2063
|
+
limits: new Map(header.limits),
|
|
2064
|
+
counts: /* @__PURE__ */ new Map(),
|
|
2065
|
+
calls: /* @__PURE__ */ new Map(),
|
|
2066
|
+
indexPath: join(options.runDir, "dispatches.jsonl"),
|
|
2067
|
+
scopePath: join(options.runDir, "dispatch-scope.json"),
|
|
2068
|
+
anchorPath: join(options.runDir, "dispatch-anchor.jsonl"),
|
|
2069
|
+
receiptsDir: join(options.runDir, "dispatch-results"),
|
|
2070
|
+
scopeText: canonicalString(header),
|
|
2071
|
+
scopeDigest: hashCanonical(header),
|
|
2072
|
+
retainedText: "",
|
|
2073
|
+
revision: 0,
|
|
2074
|
+
chain: null,
|
|
2075
|
+
opened: false
|
|
2076
|
+
};
|
|
2418
2077
|
const active = /* @__PURE__ */ new Map();
|
|
2419
|
-
const indexPath = join(options.runDir, "dispatches.jsonl");
|
|
2420
|
-
const scopePath = join(options.runDir, "dispatch-scope.json");
|
|
2421
|
-
const anchorPath = join(options.runDir, "dispatch-anchor.jsonl");
|
|
2422
|
-
const scopeText = canonicalString(header);
|
|
2423
|
-
const scopeDigest = hashCanonical(header);
|
|
2424
|
-
const receiptsDir = join(options.runDir, "dispatch-results");
|
|
2425
|
-
const resultPath = (id) => join(receiptsDir, `${id.slice(7)}.json`);
|
|
2426
2078
|
storage.ensureDir(options.runDir);
|
|
2427
|
-
storage.ensureDir(receiptsDir);
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
const
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
if (content === void 0 && storage.exists(anchorPath)) fail("cannot read durable anchor");
|
|
2451
|
-
const text = content ?? "";
|
|
2452
|
-
if (text && !text.endsWith("\n")) fail("durable anchor is torn");
|
|
2453
|
-
const entries = [];
|
|
2454
|
-
let previous = 0;
|
|
2455
|
-
const lines = text.split("\n");
|
|
2456
|
-
if (lines.at(-1) === "") lines.pop();
|
|
2457
|
-
if (lines.some((line) => !line)) fail("durable anchor contains an empty record");
|
|
2458
|
-
for (const line of lines) {
|
|
2459
|
-
const entry = anchorEntrySchema.parse(JSON.parse(line));
|
|
2460
|
-
if (entry.scope !== scopeDigest) fail("durable anchor scope changed");
|
|
2461
|
-
if (entry.bytes <= previous) fail("durable anchor is not append-only");
|
|
2462
|
-
previous = entry.bytes;
|
|
2463
|
-
entries.push(entry);
|
|
2464
|
-
}
|
|
2465
|
-
return {
|
|
2466
|
-
text,
|
|
2467
|
-
entries
|
|
2468
|
-
};
|
|
2469
|
-
};
|
|
2470
|
-
const refresh = () => {
|
|
2471
|
-
if (storage.read(scopePath) !== scopeText) fail("scope anchor is missing or changed");
|
|
2472
|
-
const stored = storage.read(indexPath);
|
|
2473
|
-
if (stored === void 0 && storage.exists(indexPath)) fail("cannot read existing history");
|
|
2474
|
-
const text = stored ?? "";
|
|
2475
|
-
if (!text.startsWith(retainedText) || text && !text.endsWith("\n")) fail("history was truncated, replaced or torn");
|
|
2476
|
-
const { entries } = readAnchor();
|
|
2477
|
-
const bytes = Buffer.byteLength(text);
|
|
2478
|
-
const applied = Buffer.byteLength(retainedText);
|
|
2479
|
-
if (text && !entries.length) fail("durable anchor is missing");
|
|
2480
|
-
const highWater = entries[entries.length - 1];
|
|
2481
|
-
if (highWater && bytes !== highWater.bytes) {
|
|
2482
|
-
if (bytes < highWater.bytes) fail("history was truncated below its durable anchor");
|
|
2483
|
-
fail("history extends beyond its durable anchor");
|
|
2484
|
-
}
|
|
2485
|
-
let offset = 0;
|
|
2486
|
-
let running = null;
|
|
2487
|
-
let entryIndex = 0;
|
|
2488
|
-
let seenHeader = opened;
|
|
2489
|
-
for (const line of text.split("\n")) {
|
|
2490
|
-
if (!line) continue;
|
|
2491
|
-
running = hashCanonical([running, line]);
|
|
2492
|
-
offset += Buffer.byteLength(line) + 1;
|
|
2493
|
-
while (entryIndex < entries.length && entries[entryIndex].bytes === offset) {
|
|
2494
|
-
if (entries[entryIndex].digest !== running) fail("history diverges from its durable anchor");
|
|
2495
|
-
entryIndex += 1;
|
|
2496
|
-
}
|
|
2497
|
-
if (offset <= applied) continue;
|
|
2498
|
-
if (!seenHeader) {
|
|
2499
|
-
const value = JSON.parse(line);
|
|
2500
|
-
if (hashCanonical(headerSchema.parse(value)) !== hashCanonical(header)) fail("resume changed its scope or allowances");
|
|
2501
|
-
seenHeader = true;
|
|
2502
|
-
} else apply(eventSchema.parse(JSON.parse(line)));
|
|
2079
|
+
storage.ensureDir(journal.receiptsDir);
|
|
2080
|
+
openJournal(journal, options.requireExisting);
|
|
2081
|
+
return {
|
|
2082
|
+
/** The input binds all behavior-affecting request material, not just a caller-selected nonce. */
|
|
2083
|
+
run(request) {
|
|
2084
|
+
const id = hashCanonical([
|
|
2085
|
+
header.scope,
|
|
2086
|
+
request.lane,
|
|
2087
|
+
request.input
|
|
2088
|
+
]);
|
|
2089
|
+
const pending = active.get(id);
|
|
2090
|
+
if (pending) return pending;
|
|
2091
|
+
const promise = dispatchOnce(journal, id, request);
|
|
2092
|
+
active.set(id, promise);
|
|
2093
|
+
promise.finally(() => {
|
|
2094
|
+
active.delete(id);
|
|
2095
|
+
}).catch(() => {});
|
|
2096
|
+
return promise;
|
|
2097
|
+
},
|
|
2098
|
+
/** Counts include uncertain dispatches, which keep their reservation. No pricing is inferred. */
|
|
2099
|
+
committed() {
|
|
2100
|
+
refreshJournal(journal);
|
|
2101
|
+
return new Map(header.limits.map(([lane]) => [lane, journal.counts.get(lane) ?? 0]));
|
|
2503
2102
|
}
|
|
2504
|
-
if (entries.length && entryIndex !== entries.length) fail("history diverges from its durable anchor");
|
|
2505
|
-
chain = running;
|
|
2506
|
-
retainedText = text;
|
|
2507
|
-
revision = bytes;
|
|
2508
|
-
opened = seenHeader;
|
|
2509
2103
|
};
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2104
|
+
}
|
|
2105
|
+
/** Fold one journal event into the per-lane counts and pending-call map. */
|
|
2106
|
+
function applyEvent(journal, event) {
|
|
2107
|
+
const existing = journal.calls.get(event.id);
|
|
2108
|
+
if (event.kind === "started") {
|
|
2109
|
+
const limit = journal.limits.get(event.lane);
|
|
2110
|
+
const count = journal.counts.get(event.lane) ?? 0;
|
|
2111
|
+
if (existing || limit === void 0 || count >= limit) fail("invalid or over-budget dispatch history");
|
|
2112
|
+
journal.calls.set(event.id, { lane: event.lane });
|
|
2113
|
+
journal.counts.set(event.lane, count + 1);
|
|
2114
|
+
} else {
|
|
2115
|
+
if (!existing || existing.receipt) fail("settlement has no unique pending dispatch");
|
|
2116
|
+
existing.receipt = event.receipt;
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
/** Read the durable anchor file, refusing a torn, empty-record, foreign-scope or non-monotonic one. */
|
|
2120
|
+
function readAnchor(journal) {
|
|
2121
|
+
const { storage, anchorPath } = journal;
|
|
2122
|
+
const content = storage.read(anchorPath);
|
|
2123
|
+
if (content === void 0 && storage.exists(anchorPath)) fail("cannot read durable anchor");
|
|
2124
|
+
const text = content ?? "";
|
|
2125
|
+
if (text && !text.endsWith("\n")) fail("durable anchor is torn");
|
|
2126
|
+
const entries = [];
|
|
2127
|
+
let previous = 0;
|
|
2128
|
+
const lines = text.split("\n");
|
|
2129
|
+
if (lines.at(-1) === "") lines.pop();
|
|
2130
|
+
if (lines.some((line) => !line)) fail("durable anchor contains an empty record");
|
|
2131
|
+
for (const line of lines) {
|
|
2132
|
+
const entry = anchorEntrySchema.parse(JSON.parse(line));
|
|
2133
|
+
if (entry.scope !== journal.scopeDigest) fail("durable anchor scope changed");
|
|
2134
|
+
if (entry.bytes <= previous) fail("durable anchor is not append-only");
|
|
2135
|
+
previous = entry.bytes;
|
|
2136
|
+
entries.push(entry);
|
|
2137
|
+
}
|
|
2138
|
+
return {
|
|
2139
|
+
text,
|
|
2140
|
+
entries
|
|
2525
2141
|
};
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2142
|
+
}
|
|
2143
|
+
/** Re-read the journal, prove it extends what was applied and matches its anchor, and apply the new events. */
|
|
2144
|
+
function refreshJournal(journal) {
|
|
2145
|
+
const { storage } = journal;
|
|
2146
|
+
if (storage.read(journal.scopePath) !== journal.scopeText) fail("scope anchor is missing or changed");
|
|
2147
|
+
const stored = storage.read(journal.indexPath);
|
|
2148
|
+
if (stored === void 0 && storage.exists(journal.indexPath)) fail("cannot read existing history");
|
|
2149
|
+
const text = stored ?? "";
|
|
2150
|
+
if (!text.startsWith(journal.retainedText) || text && !text.endsWith("\n")) fail("history was truncated, replaced or torn");
|
|
2151
|
+
const { entries } = readAnchor(journal);
|
|
2152
|
+
const bytes = Buffer.byteLength(text);
|
|
2153
|
+
const applied = Buffer.byteLength(journal.retainedText);
|
|
2154
|
+
if (text && !entries.length) fail("durable anchor is missing");
|
|
2155
|
+
const highWater = entries[entries.length - 1];
|
|
2156
|
+
if (highWater && bytes !== highWater.bytes) {
|
|
2157
|
+
if (bytes < highWater.bytes) fail("history was truncated below its durable anchor");
|
|
2158
|
+
fail("history extends beyond its durable anchor");
|
|
2159
|
+
}
|
|
2160
|
+
let offset = 0;
|
|
2161
|
+
let running = null;
|
|
2162
|
+
let entryIndex = 0;
|
|
2163
|
+
let seenHeader = journal.opened;
|
|
2164
|
+
for (const line of text.split("\n")) {
|
|
2165
|
+
if (!line) continue;
|
|
2166
|
+
running = hashCanonical([running, line]);
|
|
2167
|
+
offset += Buffer.byteLength(line) + 1;
|
|
2168
|
+
while (entryIndex < entries.length && entries[entryIndex].bytes === offset) {
|
|
2169
|
+
if (entries[entryIndex].digest !== running) fail("history diverges from its durable anchor");
|
|
2170
|
+
entryIndex += 1;
|
|
2537
2171
|
}
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
})
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2172
|
+
if (offset <= applied) continue;
|
|
2173
|
+
if (!seenHeader) {
|
|
2174
|
+
const value = JSON.parse(line);
|
|
2175
|
+
if (hashCanonical(headerSchema.parse(value)) !== hashCanonical(journal.header)) fail("resume changed its scope or allowances");
|
|
2176
|
+
seenHeader = true;
|
|
2177
|
+
} else applyEvent(journal, eventSchema.parse(JSON.parse(line)));
|
|
2178
|
+
}
|
|
2179
|
+
if (entries.length && entryIndex !== entries.length) fail("history diverges from its durable anchor");
|
|
2180
|
+
journal.chain = running;
|
|
2181
|
+
journal.retainedText = text;
|
|
2182
|
+
journal.revision = bytes;
|
|
2183
|
+
journal.opened = seenHeader;
|
|
2184
|
+
}
|
|
2185
|
+
/** Compare-and-append one event; a lost race refreshes the journal and returns false. */
|
|
2186
|
+
function appendEvent(journal, event) {
|
|
2187
|
+
const encoded = canonicalString(event);
|
|
2188
|
+
const line = `${encoded}\n`;
|
|
2189
|
+
const next = journal.storage.append(journal.indexPath, line, journal.revision);
|
|
2190
|
+
if (next === void 0) {
|
|
2191
|
+
refreshJournal(journal);
|
|
2192
|
+
return false;
|
|
2193
|
+
}
|
|
2194
|
+
if (next !== journal.revision + Buffer.byteLength(line)) fail("storage returned an invalid append revision");
|
|
2195
|
+
journal.chain = hashCanonical([journal.chain, encoded]);
|
|
2196
|
+
journal.retainedText += line;
|
|
2197
|
+
journal.revision = next;
|
|
2198
|
+
if (event.kind === "retained-dispatch-v1") journal.opened = true;
|
|
2199
|
+
else applyEvent(journal, event);
|
|
2200
|
+
return true;
|
|
2201
|
+
}
|
|
2202
|
+
/** Record the journal's current length and digest in the durable anchor. */
|
|
2203
|
+
function advanceAnchor(journal) {
|
|
2204
|
+
const currentChain = journal.chain ?? fail("cannot anchor an empty history");
|
|
2205
|
+
if (journal.revision <= 0) fail("cannot anchor an empty history");
|
|
2206
|
+
const { text, entries } = readAnchor(journal);
|
|
2207
|
+
const last = entries[entries.length - 1];
|
|
2208
|
+
if (last) {
|
|
2209
|
+
if (last.bytes > journal.revision) fail("history is behind its durable anchor");
|
|
2210
|
+
if (last.bytes === journal.revision) {
|
|
2211
|
+
if (last.digest !== currentChain) fail("history diverges from its durable anchor");
|
|
2212
|
+
return;
|
|
2549
2213
|
}
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
if (
|
|
2214
|
+
}
|
|
2215
|
+
const line = `${canonicalString({
|
|
2216
|
+
kind: "retained-dispatch-anchor-v1",
|
|
2217
|
+
scope: journal.scopeDigest,
|
|
2218
|
+
bytes: journal.revision,
|
|
2219
|
+
digest: currentChain
|
|
2220
|
+
})}\n`;
|
|
2221
|
+
const next = journal.storage.append(journal.anchorPath, line, Buffer.byteLength(text));
|
|
2222
|
+
if (next === void 0) {
|
|
2223
|
+
const concurrent = readAnchor(journal).entries.at(-1);
|
|
2224
|
+
if (concurrent?.bytes === journal.revision && concurrent.digest === journal.chain) return;
|
|
2225
|
+
fail("durable anchor append is busy; retry with the same run directory");
|
|
2226
|
+
}
|
|
2227
|
+
if (next !== Buffer.byteLength(text) + Buffer.byteLength(line)) fail("storage returned an invalid durable anchor revision");
|
|
2228
|
+
}
|
|
2229
|
+
/** An event takes effect only after both its journal append and its anchor entry are durable. */
|
|
2230
|
+
function commitEvent(journal, event) {
|
|
2231
|
+
if (!appendEvent(journal, event)) return false;
|
|
2232
|
+
advanceAnchor(journal);
|
|
2233
|
+
return true;
|
|
2234
|
+
}
|
|
2235
|
+
/** Create or verify the scope anchor, load existing history, and commit the header on a fresh journal. */
|
|
2236
|
+
function openJournal(journal, requireExisting) {
|
|
2237
|
+
const { storage, scopePath, indexPath, anchorPath, scopeText } = journal;
|
|
2238
|
+
if (requireExisting && (!storage.exists(scopePath) || !storage.exists(indexPath) || !storage.exists(anchorPath))) fail("required retained history is missing");
|
|
2559
2239
|
if (storage.exists(scopePath)) {
|
|
2560
2240
|
if (storage.read(scopePath) !== scopeText) fail("resume changed its scope or allowances");
|
|
2561
2241
|
if (!storage.read(indexPath)?.trim()) fail("dispatch history is missing or empty");
|
|
@@ -2567,145 +2247,138 @@ function createRetainedDispatch(options) {
|
|
|
2567
2247
|
if (storage.read(scopePath) !== scopeText || !storage.read(indexPath)?.trim()) fail("scope initialization is busy; retry the same scope");
|
|
2568
2248
|
} else if (created !== Buffer.byteLength(scopeText)) fail("storage returned an invalid scope revision");
|
|
2569
2249
|
}
|
|
2570
|
-
|
|
2571
|
-
if (!opened && !
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2250
|
+
refreshJournal(journal);
|
|
2251
|
+
if (!journal.opened && !commitEvent(journal, journal.header) && !journal.opened) fail("history is busy; retry with the same run directory");
|
|
2252
|
+
}
|
|
2253
|
+
/** Read and verify a retained result file, or undefined when none was written. */
|
|
2254
|
+
function readResult(journal, id) {
|
|
2255
|
+
const content = journal.storage.read(resultPath(journal, id));
|
|
2256
|
+
if (content === void 0) return void 0;
|
|
2257
|
+
const result = resultSchema.parse(JSON.parse(content));
|
|
2258
|
+
if (result.id !== id) return fail("retained result dispatch id mismatch");
|
|
2259
|
+
if (hashCanonical(result.value) !== result.receipt) return fail("retained result digest mismatch");
|
|
2260
|
+
const value = journal.parse(result.value);
|
|
2261
|
+
if (hashCanonical(value) !== result.receipt) return fail("result decoder changed retained output");
|
|
2262
|
+
return {
|
|
2263
|
+
value,
|
|
2264
|
+
receipt: result.receipt
|
|
2584
2265
|
};
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
if (
|
|
2595
|
-
|
|
2596
|
-
id,
|
|
2597
|
-
receipt
|
|
2598
|
-
})) return;
|
|
2599
|
-
if (revision === previousRevision) fail("settlement is busy; retained result awaits reconciliation");
|
|
2266
|
+
}
|
|
2267
|
+
function resultPath(journal, id) {
|
|
2268
|
+
return join(journal.receiptsDir, `${id.slice(7)}.json`);
|
|
2269
|
+
}
|
|
2270
|
+
/** Commit the settlement of a dispatch whose result is already durable. */
|
|
2271
|
+
function settle(journal, id, receipt) {
|
|
2272
|
+
while (true) {
|
|
2273
|
+
const existing = journal.calls.get(id);
|
|
2274
|
+
if (existing?.receipt) {
|
|
2275
|
+
if (existing.receipt !== receipt) fail("settled result digest mismatch");
|
|
2276
|
+
return;
|
|
2600
2277
|
}
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2278
|
+
if (!existing) fail("settlement has no pending dispatch");
|
|
2279
|
+
const previousRevision = journal.revision;
|
|
2280
|
+
if (commitEvent(journal, {
|
|
2281
|
+
kind: "settled",
|
|
2282
|
+
id,
|
|
2283
|
+
receipt
|
|
2284
|
+
})) return;
|
|
2285
|
+
if (journal.revision === previousRevision) fail("settlement is busy; retained result awaits reconciliation");
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
/** Replay a settled dispatch from its retained result. */
|
|
2289
|
+
function loadSettled(journal, id, receipt) {
|
|
2290
|
+
const result = readResult(journal, id);
|
|
2291
|
+
if (!result) return fail("settled result is missing");
|
|
2292
|
+
if (result.receipt !== receipt) return fail("settled result digest mismatch");
|
|
2614
2293
|
return {
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
value: result.value,
|
|
2642
|
-
dispatchId: id,
|
|
2643
|
-
receiptDigest: result.receipt,
|
|
2644
|
-
replayed: true
|
|
2645
|
-
};
|
|
2646
|
-
}
|
|
2647
|
-
signal?.throwIfAborted();
|
|
2648
|
-
if ((counts.get(lane) ?? 0) >= limits.get(lane)) return {
|
|
2649
|
-
succeeded: false,
|
|
2650
|
-
reason: "allowance_exhausted",
|
|
2651
|
-
dispatchId: id
|
|
2652
|
-
};
|
|
2653
|
-
const previousRevision = revision;
|
|
2654
|
-
if (commit({
|
|
2655
|
-
kind: "started",
|
|
2656
|
-
id,
|
|
2657
|
-
lane
|
|
2658
|
-
})) break;
|
|
2659
|
-
if (revision === previousRevision) return fail("history is busy; retry with the same input");
|
|
2660
|
-
}
|
|
2661
|
-
let value;
|
|
2662
|
-
try {
|
|
2663
|
-
signal?.throwIfAborted();
|
|
2664
|
-
value = parse(await dispatch());
|
|
2665
|
-
} catch (error) {
|
|
2666
|
-
return {
|
|
2667
|
-
succeeded: false,
|
|
2668
|
-
reason: "outcome_unknown",
|
|
2669
|
-
dispatchId: id,
|
|
2670
|
-
diagnostic: error instanceof Error ? error.message : String(error)
|
|
2671
|
-
};
|
|
2672
|
-
}
|
|
2673
|
-
const encoded = canonicalString(value);
|
|
2674
|
-
const decoded = JSON.parse(encoded);
|
|
2675
|
-
const retainedValue = parse(decoded);
|
|
2676
|
-
if (canonicalString(retainedValue) !== encoded) return fail("result decoder is not stable across serialization");
|
|
2677
|
-
const receipt = hashCanonical(decoded);
|
|
2678
|
-
const retainedResult = canonicalString({
|
|
2679
|
-
kind: "retained-dispatch-result-v1",
|
|
2680
|
-
id,
|
|
2681
|
-
receipt,
|
|
2682
|
-
value: decoded
|
|
2683
|
-
});
|
|
2684
|
-
const file = resultPath(id);
|
|
2685
|
-
if (storage.append(file, retainedResult, 0) === void 0 && storage.read(file) !== retainedResult) fail("cannot retain dispatch result");
|
|
2686
|
-
settle(id, receipt);
|
|
2687
|
-
return {
|
|
2688
|
-
succeeded: true,
|
|
2689
|
-
value: retainedValue,
|
|
2690
|
-
dispatchId: id,
|
|
2691
|
-
receiptDigest: receipt,
|
|
2692
|
-
replayed: false
|
|
2693
|
-
};
|
|
2294
|
+
succeeded: true,
|
|
2295
|
+
value: result.value,
|
|
2296
|
+
dispatchId: id,
|
|
2297
|
+
receiptDigest: receipt,
|
|
2298
|
+
replayed: true
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2301
|
+
/** Replay a known dispatch, or commit its started intent; returns the outcome when no call should be made. */
|
|
2302
|
+
function claimDispatch(journal, id, lane, signal) {
|
|
2303
|
+
while (true) {
|
|
2304
|
+
const existing = journal.calls.get(id);
|
|
2305
|
+
if (existing) {
|
|
2306
|
+
if (existing.receipt) return loadSettled(journal, id, existing.receipt);
|
|
2307
|
+
const result = readResult(journal, id);
|
|
2308
|
+
if (!result) return {
|
|
2309
|
+
succeeded: false,
|
|
2310
|
+
reason: "outcome_unknown",
|
|
2311
|
+
dispatchId: id
|
|
2312
|
+
};
|
|
2313
|
+
settle(journal, id, result.receipt);
|
|
2314
|
+
return {
|
|
2315
|
+
succeeded: true,
|
|
2316
|
+
value: result.value,
|
|
2317
|
+
dispatchId: id,
|
|
2318
|
+
receiptDigest: result.receipt,
|
|
2319
|
+
replayed: true
|
|
2694
2320
|
};
|
|
2695
|
-
const promise = work();
|
|
2696
|
-
active.set(id, promise);
|
|
2697
|
-
promise.finally(() => {
|
|
2698
|
-
active.delete(id);
|
|
2699
|
-
}).catch(() => {});
|
|
2700
|
-
return promise;
|
|
2701
|
-
},
|
|
2702
|
-
/** Counts include uncertain dispatches, which keep their reservation. No pricing is inferred. */
|
|
2703
|
-
committed() {
|
|
2704
|
-
refresh();
|
|
2705
|
-
return new Map(header.limits.map(([lane]) => [lane, counts.get(lane) ?? 0]));
|
|
2706
2321
|
}
|
|
2322
|
+
signal?.throwIfAborted();
|
|
2323
|
+
if ((journal.counts.get(lane) ?? 0) >= journal.limits.get(lane)) return {
|
|
2324
|
+
succeeded: false,
|
|
2325
|
+
reason: "allowance_exhausted",
|
|
2326
|
+
dispatchId: id
|
|
2327
|
+
};
|
|
2328
|
+
const previousRevision = journal.revision;
|
|
2329
|
+
if (commitEvent(journal, {
|
|
2330
|
+
kind: "started",
|
|
2331
|
+
id,
|
|
2332
|
+
lane
|
|
2333
|
+
})) return void 0;
|
|
2334
|
+
if (journal.revision === previousRevision) return fail("history is busy; retry with the same input");
|
|
2335
|
+
}
|
|
2336
|
+
}
|
|
2337
|
+
/** Retain a dispatched value's result file, settle it, and return the decoded outcome. */
|
|
2338
|
+
function retainResult(journal, id, value) {
|
|
2339
|
+
const encoded = canonicalString(value);
|
|
2340
|
+
const decoded = JSON.parse(encoded);
|
|
2341
|
+
const retainedValue = journal.parse(decoded);
|
|
2342
|
+
if (canonicalString(retainedValue) !== encoded) return fail("result decoder is not stable across serialization");
|
|
2343
|
+
const receipt = hashCanonical(decoded);
|
|
2344
|
+
const retainedResult = canonicalString({
|
|
2345
|
+
kind: "retained-dispatch-result-v1",
|
|
2346
|
+
id,
|
|
2347
|
+
receipt,
|
|
2348
|
+
value: decoded
|
|
2349
|
+
});
|
|
2350
|
+
const file = resultPath(journal, id);
|
|
2351
|
+
if (journal.storage.append(file, retainedResult, 0) === void 0 && journal.storage.read(file) !== retainedResult) fail("cannot retain dispatch result");
|
|
2352
|
+
settle(journal, id, receipt);
|
|
2353
|
+
return {
|
|
2354
|
+
succeeded: true,
|
|
2355
|
+
value: retainedValue,
|
|
2356
|
+
dispatchId: id,
|
|
2357
|
+
receiptDigest: receipt,
|
|
2358
|
+
replayed: false
|
|
2707
2359
|
};
|
|
2708
2360
|
}
|
|
2361
|
+
/** Replay, claim an allowance, call the dispatch once, and retain its result. */
|
|
2362
|
+
async function dispatchOnce(journal, id, request) {
|
|
2363
|
+
const { lane, dispatch, signal } = request;
|
|
2364
|
+
if (!journal.limits.has(lane)) return fail("unknown allowance lane");
|
|
2365
|
+
refreshJournal(journal);
|
|
2366
|
+
const claimed = claimDispatch(journal, id, lane, signal);
|
|
2367
|
+
if (claimed) return claimed;
|
|
2368
|
+
let value;
|
|
2369
|
+
try {
|
|
2370
|
+
signal?.throwIfAborted();
|
|
2371
|
+
value = journal.parse(await dispatch());
|
|
2372
|
+
} catch (error) {
|
|
2373
|
+
return {
|
|
2374
|
+
succeeded: false,
|
|
2375
|
+
reason: "outcome_unknown",
|
|
2376
|
+
dispatchId: id,
|
|
2377
|
+
diagnostic: error instanceof Error ? error.message : String(error)
|
|
2378
|
+
};
|
|
2379
|
+
}
|
|
2380
|
+
return retainResult(journal, id, value);
|
|
2381
|
+
}
|
|
2709
2382
|
//#endregion
|
|
2710
2383
|
//#region src/campaign/scenario-selection.ts
|
|
2711
2384
|
const DEFAULT_SATURATION_CEILING = .999;
|
|
@@ -2761,155 +2434,6 @@ function scoreDiscrimination(signals, opts) {
|
|
|
2761
2434
|
};
|
|
2762
2435
|
}).sort(compareDiscrimination);
|
|
2763
2436
|
}
|
|
2764
|
-
/**
|
|
2765
|
-
* Select the top-`k` most discriminative scenario ids for a holdout, EXCLUDING
|
|
2766
|
-
* fully saturated ties when enough non-tied scenarios exist (a tie in the
|
|
2767
|
-
* holdout wastes a paired cell).
|
|
2768
|
-
*
|
|
2769
|
-
* Prefers non-tied scenarios; if fewer than `k` non-tied exist, fills with the
|
|
2770
|
-
* least-saturated tied ones (tied scenarios are already ordered least-saturated
|
|
2771
|
-
* first by `meanScore` asc). Deterministic. Throws if `k < 1`. If
|
|
2772
|
-
* `signals.length <= k`, returns all ids in discrimination order.
|
|
2773
|
-
*/
|
|
2774
|
-
function selectDiscriminative(signals, k, opts) {
|
|
2775
|
-
if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
|
|
2776
|
-
const ranked = scoreDiscrimination(signals, opts);
|
|
2777
|
-
if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
|
|
2778
|
-
const nonTied = ranked.filter((s) => !s.tied);
|
|
2779
|
-
if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
|
|
2780
|
-
const fill = ranked.filter((s) => s.tied).slice(0, k - nonTied.length);
|
|
2781
|
-
return [...nonTied, ...fill].map((s) => s.scenarioId);
|
|
2782
|
-
}
|
|
2783
|
-
//#endregion
|
|
2784
|
-
//#region src/campaign/upstream-evaluators.ts
|
|
2785
|
-
function phoenixEvaluatorJudge(evaluator, options) {
|
|
2786
|
-
if ((evaluator.optimizationDirection === "MINIMIZE" || evaluator.optimizationDirection === "NEUTRAL") && !options.toComposite) throw new TypeError(`phoenixEvaluatorJudge requires toComposite for a ${evaluator.optimizationDirection} evaluator`);
|
|
2787
|
-
const paidCall = validatePaidEvaluation(evaluator.name, evaluator.kind, options.paidCall);
|
|
2788
|
-
return upstreamJudge({
|
|
2789
|
-
name: options.name ?? evaluator.name,
|
|
2790
|
-
dimension: options.dimension,
|
|
2791
|
-
judgeVersion: options.judgeVersion,
|
|
2792
|
-
appliesTo: options.appliesTo,
|
|
2793
|
-
toComposite: options.toComposite
|
|
2794
|
-
}, async (input) => {
|
|
2795
|
-
const result = await runUpstreamEvaluation({
|
|
2796
|
-
name: evaluator.name,
|
|
2797
|
-
kind: evaluator.kind,
|
|
2798
|
-
signal: input.signal,
|
|
2799
|
-
costLedger: input.costLedger,
|
|
2800
|
-
costPhase: input.costPhase,
|
|
2801
|
-
costTags: input.costTags,
|
|
2802
|
-
scenarioId: input.scenario.id,
|
|
2803
|
-
paidCall,
|
|
2804
|
-
execute: (context) => evaluator.evaluate(options.mapInput({
|
|
2805
|
-
artifact: input.artifact,
|
|
2806
|
-
scenario: input.scenario
|
|
2807
|
-
}), context)
|
|
2808
|
-
});
|
|
2809
|
-
if (result.score === void 0 || !Number.isFinite(result.score)) throw new Error(`${evaluator.name}: Phoenix evaluator returned no finite score`);
|
|
2810
|
-
return {
|
|
2811
|
-
score: result.score,
|
|
2812
|
-
notes: [result.label, result.explanation].filter(Boolean).join(": ") || evaluator.name
|
|
2813
|
-
};
|
|
2814
|
-
});
|
|
2815
|
-
}
|
|
2816
|
-
function autoevalsScorerJudge(scorer, options) {
|
|
2817
|
-
const paidCall = validatePaidEvaluation(options.name, options.kind, options.paidCall);
|
|
2818
|
-
return upstreamJudge(options, async (input) => {
|
|
2819
|
-
const result = await runUpstreamEvaluation({
|
|
2820
|
-
name: options.name,
|
|
2821
|
-
kind: options.kind,
|
|
2822
|
-
signal: input.signal,
|
|
2823
|
-
costLedger: input.costLedger,
|
|
2824
|
-
costPhase: input.costPhase,
|
|
2825
|
-
costTags: input.costTags,
|
|
2826
|
-
scenarioId: input.scenario.id,
|
|
2827
|
-
paidCall,
|
|
2828
|
-
execute: (context) => scorer(options.mapInput({
|
|
2829
|
-
artifact: input.artifact,
|
|
2830
|
-
scenario: input.scenario
|
|
2831
|
-
}), context)
|
|
2832
|
-
});
|
|
2833
|
-
if (result.score === null || !Number.isFinite(result.score)) throw new Error(`${options.name}: Autoevals scorer returned no finite score`);
|
|
2834
|
-
return {
|
|
2835
|
-
score: result.score,
|
|
2836
|
-
notes: result.metadata ? JSON.stringify(result.metadata) : result.name
|
|
2837
|
-
};
|
|
2838
|
-
});
|
|
2839
|
-
}
|
|
2840
|
-
function upstreamJudge(options, evaluate) {
|
|
2841
|
-
const dimension = options.dimension ?? options.name;
|
|
2842
|
-
return {
|
|
2843
|
-
name: options.name,
|
|
2844
|
-
judgeVersion: options.judgeVersion,
|
|
2845
|
-
dimensions: [{
|
|
2846
|
-
key: dimension,
|
|
2847
|
-
description: `Score from ${options.name}`
|
|
2848
|
-
}],
|
|
2849
|
-
appliesTo: options.appliesTo,
|
|
2850
|
-
async score(input) {
|
|
2851
|
-
const result = await evaluate(input);
|
|
2852
|
-
const composite = options.toComposite ? options.toComposite(result.score) : result.score;
|
|
2853
|
-
if (!Number.isFinite(composite)) throw new Error(`${options.name}: toComposite returned a non-finite score`);
|
|
2854
|
-
return {
|
|
2855
|
-
dimensions: { [dimension]: result.score },
|
|
2856
|
-
composite,
|
|
2857
|
-
notes: result.notes
|
|
2858
|
-
};
|
|
2859
|
-
}
|
|
2860
|
-
};
|
|
2861
|
-
}
|
|
2862
|
-
function validatePaidEvaluation(name, kind, paidCall) {
|
|
2863
|
-
if (kind === "CODE") {
|
|
2864
|
-
if (paidCall) throw new TypeError(`${name}: CODE evaluator cannot declare paidCall`);
|
|
2865
|
-
return;
|
|
2866
|
-
}
|
|
2867
|
-
if (!paidCall) throw new TypeError(`${name}: LLM evaluator requires paidCall cost and token capture`);
|
|
2868
|
-
if (!paidCall.model.trim()) throw new TypeError(`${name}: paidCall.model must be non-empty`);
|
|
2869
|
-
return paidCall;
|
|
2870
|
-
}
|
|
2871
|
-
async function runUpstreamEvaluation(input) {
|
|
2872
|
-
if (input.kind === "CODE") return abortableEvaluation(input.signal, () => input.execute({ signal: input.signal }));
|
|
2873
|
-
if (!input.paidCall) throw new CostAccountingIncompleteError(`${input.name}: missing paid-call configuration`);
|
|
2874
|
-
if (!input.costLedger) throw new CostAccountingIncompleteError(`${input.name}: LLM evaluator requires the campaign cost ledger`);
|
|
2875
|
-
const paid = await input.costLedger.runPaidCall({
|
|
2876
|
-
channel: "judge",
|
|
2877
|
-
phase: input.costPhase ?? "judge",
|
|
2878
|
-
actor: input.name,
|
|
2879
|
-
model: input.paidCall.model,
|
|
2880
|
-
...input.paidCall.maximumCharge ? { maximumCharge: input.paidCall.maximumCharge } : {},
|
|
2881
|
-
tags: {
|
|
2882
|
-
...input.costTags,
|
|
2883
|
-
scenarioId: input.scenarioId
|
|
2884
|
-
},
|
|
2885
|
-
signal: input.signal,
|
|
2886
|
-
execute: (signal, callId) => Promise.resolve(input.execute({
|
|
2887
|
-
signal,
|
|
2888
|
-
callId
|
|
2889
|
-
})),
|
|
2890
|
-
receipt: input.paidCall.receipt,
|
|
2891
|
-
...input.paidCall.receiptFromError ? { receiptFromError: input.paidCall.receiptFromError } : {}
|
|
2892
|
-
});
|
|
2893
|
-
if (!paid.succeeded) throw paid.error;
|
|
2894
|
-
const recorded = input.costLedger.list().find((receipt) => receipt.callId === paid.callId);
|
|
2895
|
-
if (!recorded) throw new CostAccountingIncompleteError(`${input.name}: paid evaluator returned without a recorded cost receipt`);
|
|
2896
|
-
if (recorded.costUnknown || recorded.usageUnknown) throw new CostAccountingIncompleteError(`${input.name}: paid evaluator returned without complete cost and token usage`);
|
|
2897
|
-
return paid.value;
|
|
2898
|
-
}
|
|
2899
|
-
async function abortableEvaluation(signal, execute) {
|
|
2900
|
-
if (signal.aborted) throw signal.reason;
|
|
2901
|
-
let rejectAborted;
|
|
2902
|
-
const aborted = new Promise((_resolve, reject) => {
|
|
2903
|
-
rejectAborted = reject;
|
|
2904
|
-
});
|
|
2905
|
-
const onAbort = () => rejectAborted(signal.reason);
|
|
2906
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
2907
|
-
try {
|
|
2908
|
-
return await Promise.race([Promise.resolve().then(execute), aborted]);
|
|
2909
|
-
} finally {
|
|
2910
|
-
signal.removeEventListener("abort", onAbort);
|
|
2911
|
-
}
|
|
2912
|
-
}
|
|
2913
2437
|
//#endregion
|
|
2914
2438
|
//#region src/campaign/worktree/index.ts
|
|
2915
2439
|
/**
|
|
@@ -3432,6 +2956,6 @@ function resolveWorktreePath(surface, worktreeDir) {
|
|
|
3432
2956
|
return verifyCodeSurface(surface, worktreeDir).path;
|
|
3433
2957
|
}
|
|
3434
2958
|
//#endregion
|
|
3435
|
-
export {
|
|
2959
|
+
export { analyzeCrossSurfaceInteractions as S, neutralizationGate as _, createRetainedDispatch as a, loadEvalFixtureScenarios as b, runProfileMatrixSegment as c, scoreUserStory as d, scoreboardSummary as f, neutralizeText as g, sequentialOptimizationMethod as h, scoreDiscrimination as i, runProfileMatrix as l, scopedOptimizationMethod as m, resolveWorktreePath as n, createProfileMatrixPlan as o, userStoryScoreboard as p, verifyCodeSurface as r, finalizeProfileMatrix as s, gitWorktreeAdapter as t, renderScoreboardMarkdown as u, discoverEvalFixtures as v, planEvalFixtureRun as x, loadEvalFixture as y };
|
|
3436
2960
|
|
|
3437
|
-
//# sourceMappingURL=campaign-
|
|
2961
|
+
//# sourceMappingURL=campaign-CGvUTnmc.js.map
|