@tangle-network/agent-eval 0.221.8 → 0.222.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +168 -0
- package/dist/adapters/http.d.ts +3 -5
- package/dist/adapters/http.d.ts.map +1 -1
- package/dist/adapters/http.js +1 -2
- package/dist/{adversarial-smnADNFS.d.ts → adversarial-DnQQFqTR.d.ts} +1 -3
- package/dist/adversarial-DnQQFqTR.d.ts.map +1 -0
- package/dist/{agent-profile-CNmwfaqq.d.ts → agent-profile-CSEQRC9l.d.ts} +4 -10
- package/dist/agent-profile-CSEQRC9l.d.ts.map +1 -0
- package/dist/agent-profile-Dngk3LZ0.js +153 -0
- package/dist/agent-profile-Dngk3LZ0.js.map +1 -0
- package/dist/{agent-profile-cell-BcpFbNd5.d.ts → agent-profile-cell-3kYxUsK2.d.ts} +3 -5
- package/dist/agent-profile-cell-3kYxUsK2.d.ts.map +1 -0
- package/dist/{agent-profile-cell-B1V1gUZs.js → agent-profile-cell-CT5JYueM.js} +5 -7
- package/dist/{agent-profile-cell-B1V1gUZs.js.map → agent-profile-cell-CT5JYueM.js.map} +1 -1
- package/dist/{agreement-irr-BhAqjluA.js → agreement-irr-BTjDN0i6.js} +4 -5
- package/dist/{agreement-irr-BhAqjluA.js.map → agreement-irr-BTjDN0i6.js.map} +1 -1
- package/dist/analyst/index.d.ts +95 -159
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +28 -39
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DfX6dHP0.d.ts → analyze-runs-CGLRgZGd.d.ts} +7 -13
- package/dist/analyze-runs-CGLRgZGd.d.ts.map +1 -0
- package/dist/{analyze-runs-CHmsykNJ.js → analyze-runs-CTheASYV.js} +23 -32
- package/dist/{analyze-runs-CHmsykNJ.js.map → analyze-runs-CTheASYV.js.map} +1 -1
- package/dist/{attribute-vocabulary-CcXqHODT.d.ts → attribute-vocabulary-5oo8ykIx.d.ts} +2 -4
- package/dist/attribute-vocabulary-5oo8ykIx.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +0 -2
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -2
- package/dist/{benchmark-command-Cnjfo0Zh.js → benchmark-command-Cr7i7YSX.js} +61 -94
- package/dist/benchmark-command-Cr7i7YSX.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -15
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +27 -38
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/{bounded-process-Cj9YjXbE.js → bounded-process-BZrWXIKW.js} +2 -3
- package/dist/{bounded-process-Cj9YjXbE.js.map → bounded-process-BZrWXIKW.js.map} +1 -1
- package/dist/builder-eval/index.d.ts +5 -14
- package/dist/builder-eval/index.d.ts.map +1 -1
- package/dist/builder-eval/index.js +11 -17
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/{calibration-CTwxpJ-t.js → calibration-6usfVYYo.js} +4 -5
- package/dist/{calibration-CTwxpJ-t.js.map → calibration-6usfVYYo.js.map} +1 -1
- package/dist/campaign/index.d.ts +9 -9
- package/dist/campaign/index.js +15 -14
- package/dist/{campaign-uddTlFg5.js → campaign-ddr1N3yD.js} +61 -58
- package/dist/campaign-ddr1N3yD.js.map +1 -0
- package/dist/{campaign-evidence-DunzcJGv.js → campaign-evidence-DN8rWuBj.js} +20 -322
- package/dist/campaign-evidence-DN8rWuBj.js.map +1 -0
- package/dist/{canonical-Dq2hOQ8L.js → canonical-CicJlINA.js} +3 -4
- package/dist/{canonical-Dq2hOQ8L.js.map → canonical-CicJlINA.js.map} +1 -1
- package/dist/{canonical-CFpojCN5.d.ts → canonical-gLeHiqzd.d.ts} +2 -4
- package/dist/canonical-gLeHiqzd.d.ts.map +1 -0
- package/dist/{cell-spend-DEWR3nj-.js → cell-spend-zxmzg4GH.js} +3 -4
- package/dist/{cell-spend-DEWR3nj-.js.map → cell-spend-zxmzg4GH.js.map} +1 -1
- package/dist/{chat-client-BNATB3rJ.js → chat-client-CO3uB33i.js} +3 -4
- package/dist/{chat-client-BNATB3rJ.js.map → chat-client-CO3uB33i.js.map} +1 -1
- package/dist/{chat-json-call-BzWYp18J.js → chat-json-call-8yUv4LFO.js} +4 -5
- package/dist/{chat-json-call-BzWYp18J.js.map → chat-json-call-8yUv4LFO.js.map} +1 -1
- package/dist/{claim-BPVuyHuy.js → claim-Dod_mjk3.js} +5 -6
- package/dist/{claim-BPVuyHuy.js.map → claim-Dod_mjk3.js.map} +1 -1
- package/dist/cli.js +179 -23
- package/dist/cli.js.map +1 -1
- package/dist/{client-hwo3cJjR.d.ts → client-BIFwUg7I.d.ts} +6 -12
- package/dist/client-BIFwUg7I.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-DEwwhl0C.js → cluster-bootstrap-SbaeM44s.js} +4 -5
- package/dist/{cluster-bootstrap-DEwwhl0C.js.map → cluster-bootstrap-SbaeM44s.js.map} +1 -1
- package/dist/completion-verifier-7KEfroqx.d.ts +168 -0
- package/dist/completion-verifier-7KEfroqx.d.ts.map +1 -0
- package/dist/{completion-verifier-asaV1nSj.js → completion-verifier-rqbMh5-4.js} +9 -160
- package/dist/completion-verifier-rqbMh5-4.js.map +1 -0
- package/dist/contract/index.d.ts +13 -13
- package/dist/contract/index.js +41 -60
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BGZB6yY2.js → cost-ledger-CS8R3IZj.js} +10 -17
- package/dist/{cost-ledger-BGZB6yY2.js.map → cost-ledger-CS8R3IZj.js.map} +1 -1
- package/dist/{cost-ledger-C1W3ICmP.d.ts → cost-ledger-Czro9wmo.d.ts} +1 -5
- package/dist/cost-ledger-Czro9wmo.d.ts.map +1 -0
- package/dist/{dataset-DQqhOCPt.d.ts → dataset-DThC1S6G.d.ts} +1 -3
- package/dist/dataset-DThC1S6G.d.ts.map +1 -0
- package/dist/{decision-point-0nJubuoM.js → decision-point-lhMgODOR.js} +14 -18
- package/dist/{decision-point-0nJubuoM.js.map → decision-point-lhMgODOR.js.map} +1 -1
- package/dist/{deep-freeze-CurcOQg3.js → deep-freeze-CK8-2W7p.js} +2 -3
- package/dist/{deep-freeze-CurcOQg3.js.map → deep-freeze-CK8-2W7p.js.map} +1 -1
- package/dist/{default-registry-CLG5YqGS.d.ts → default-registry-1f4ukAIX.d.ts} +53 -61
- package/dist/default-registry-1f4ukAIX.d.ts.map +1 -0
- package/dist/{default-registry-BBTDgkDM.js → default-registry-C1tPEjZ4.js} +29 -45
- package/dist/{default-registry-BBTDgkDM.js.map → default-registry-C1tPEjZ4.js.map} +1 -1
- package/dist/{define-agent-eval-CCbT7Kfr.js → define-agent-eval-BLoP4Lf9.js} +15 -18
- package/dist/{define-agent-eval-CCbT7Kfr.js.map → define-agent-eval-BLoP4Lf9.js.map} +1 -1
- package/dist/{define-agent-eval-JKt3gMQr.d.ts → define-agent-eval-sUYh9BFA.d.ts} +19 -27
- package/dist/define-agent-eval-sUYh9BFA.d.ts.map +1 -0
- package/dist/{descriptive-CVNy48rR.js → descriptive-cPwITVfy.js} +30 -8
- package/dist/descriptive-cPwITVfy.js.map +1 -0
- package/dist/diagnosis/index.d.ts +5 -15
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +13 -19
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-DN1p_5jx.js → dspy-rlm-engine-jDiWkul6.js} +7 -10
- package/dist/{dspy-rlm-engine-DN1p_5jx.js.map → dspy-rlm-engine-jDiWkul6.js.map} +1 -1
- package/dist/{emitter-DbMXUIVP.js → emitter-DhQ65mza.js} +3 -4
- package/dist/{emitter-DbMXUIVP.js.map → emitter-DhQ65mza.js.map} +1 -1
- package/dist/{emitter-DABYYTcH.d.ts → emitter-mAXC5PSN.d.ts} +3 -5
- package/dist/emitter-mAXC5PSN.d.ts.map +1 -0
- package/dist/{engine-dtRVeuQs.d.ts → engine-BITiErg-.d.ts} +10 -18
- package/dist/engine-BITiErg-.d.ts.map +1 -0
- package/dist/{equal-terms-DbPGcVtI.js → equal-terms-K11VHmR3.js} +3 -4
- package/dist/{equal-terms-DbPGcVtI.js.map → equal-terms-K11VHmR3.js.map} +1 -1
- package/dist/{errors-BcOs5MKW.js → errors-DeEEV9Vx.js} +2 -3
- package/dist/{errors-BcOs5MKW.js.map → errors-DeEEV9Vx.js.map} +1 -1
- package/dist/{errors-DEE6u6ot.d.ts → errors-E9I2hldP.d.ts} +1 -3
- package/dist/errors-E9I2hldP.d.ts.map +1 -0
- package/dist/{eval-campaign-DoX5vz3E.js → eval-campaign-Cgixrvf7.js} +8 -9
- package/dist/{eval-campaign-DoX5vz3E.js.map → eval-campaign-Cgixrvf7.js.map} +1 -1
- package/dist/{evaluation-wdGyZ7jV.js → evaluation-B8H3GYej.js} +4 -6
- package/dist/{evaluation-wdGyZ7jV.js.map → evaluation-B8H3GYej.js.map} +1 -1
- package/dist/{evaluation-DblGquCB.d.ts → evaluation-Djq7-Ybj.d.ts} +4 -6
- package/dist/evaluation-Djq7-Ybj.d.ts.map +1 -0
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -2
- package/dist/{exact-types-Do63KoKn.d.ts → exact-types-CndDT_h7.d.ts} +8 -10
- package/dist/exact-types-CndDT_h7.d.ts.map +1 -0
- package/dist/{exec-CpbtOXVt.js → exec-C0_AkLUv.js} +3 -5
- package/dist/{exec-CpbtOXVt.js.map → exec-C0_AkLUv.js.map} +1 -1
- package/dist/{execution-tracks-CpgFPpS5.js → execution-tracks-DaDXYwAq.js} +2 -3
- package/dist/{execution-tracks-CpgFPpS5.js.map → execution-tracks-DaDXYwAq.js.map} +1 -1
- package/dist/experiment/index.d.ts +9 -37
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +29 -40
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-Dk3oEbK_.d.ts → exporters-B5Ze_RKh.d.ts} +4 -4
- package/dist/exporters-B5Ze_RKh.d.ts.map +1 -0
- package/dist/{exporters-DWvM16LE.js → exporters-ByLcILYU.js} +23 -5
- package/dist/exporters-ByLcILYU.js.map +1 -0
- package/dist/{external-optimizer-process-DEzJ2CGX.js → external-optimizer-process-BRVmaDFe.js} +5 -6
- package/dist/{external-optimizer-process-DEzJ2CGX.js.map → external-optimizer-process-BRVmaDFe.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-B_ukPtRD.js → external-optimizer-subprocess-U616hV_l.js} +24 -37
- package/dist/{external-optimizer-subprocess-B_ukPtRD.js.map → external-optimizer-subprocess-U616hV_l.js.map} +1 -1
- package/dist/{extract-usage-BZLU01kR.js → extract-usage-ISFqhfDz.js} +2 -3
- package/dist/{extract-usage-BZLU01kR.js.map → extract-usage-ISFqhfDz.js.map} +1 -1
- package/dist/{failure-cluster-DN0t63Z0.d.ts → failure-cluster-DOoa7CBg.d.ts} +3 -7
- package/dist/failure-cluster-DOoa7CBg.d.ts.map +1 -0
- package/dist/{failure-taxonomy-Bx5oWqaW.js → failure-taxonomy-9eQfXKoW.js} +2 -3
- package/dist/{failure-taxonomy-Bx5oWqaW.js.map → failure-taxonomy-9eQfXKoW.js.map} +1 -1
- package/dist/{feedback-trajectory-CYVJXolG.d.ts → feedback-trajectory-DsHR9krG.d.ts} +7 -13
- package/dist/feedback-trajectory-DsHR9krG.d.ts.map +1 -0
- package/dist/{finding-composition-8n0fFgVI.js → finding-composition-DlvE1Qul.js} +4 -6
- package/dist/{finding-composition-8n0fFgVI.js.map → finding-composition-DlvE1Qul.js.map} +1 -1
- package/dist/{findings-store-Bg4sdS-e.js → findings-store-B75Dspaq.js} +11 -14
- package/dist/{findings-store-Bg4sdS-e.js.map → findings-store-B75Dspaq.js.map} +1 -1
- package/dist/fuzz.d.ts +3 -17
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +30 -29
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-CKxIJQba.js → hf-dataset-DPVQXW5n.js} +362 -21
- package/dist/hf-dataset-DPVQXW5n.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -6
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +1 -2
- package/dist/{index-CalAsXMR.d.ts → index-B1Bm9oJA.d.ts} +3 -15
- package/dist/index-B1Bm9oJA.d.ts.map +1 -0
- package/dist/{index-D7fmC-1K.d.ts → index-BdKzYYkC.d.ts} +183 -36
- package/dist/index-BdKzYYkC.d.ts.map +1 -0
- package/dist/{index-CdEVr8-O.d.ts → index-Bdcpx2s8.d.ts} +24 -299
- package/dist/index-Bdcpx2s8.d.ts.map +1 -0
- package/dist/{index-jnuFLxOs.d.ts → index-Cu12qxJb.d.ts} +11 -39
- package/dist/index-Cu12qxJb.d.ts.map +1 -0
- package/dist/{index-CfJULcW5.d.ts → index-DAk2quYu.d.ts} +18 -38
- package/dist/index-DAk2quYu.d.ts.map +1 -0
- package/dist/{index-CHxwfW5g.d.ts → index-DwUGehqX.d.ts} +3 -9
- package/dist/index-DwUGehqX.d.ts.map +1 -0
- package/dist/{index-CTIRz-AB.d.ts → index-IzWTz5EP.d.ts} +14 -36
- package/dist/index-IzWTz5EP.d.ts.map +1 -0
- package/dist/index.d.ts +55 -188
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +122 -195
- package/dist/index.js.map +1 -1
- package/dist/{integrity-D-s2iHiT.js → integrity-B1V6klyO.js} +12 -363
- package/dist/integrity-B1V6klyO.js.map +1 -0
- package/dist/{integrity-D9lBnv_K.d.ts → integrity-BAdVzT8u.d.ts} +4 -6
- package/dist/integrity-BAdVzT8u.d.ts.map +1 -0
- package/dist/{integrity-W4CADcFc.js → integrity-DOHt1mwU.js} +3 -4
- package/dist/{integrity-W4CADcFc.js.map → integrity-DOHt1mwU.js.map} +1 -1
- package/dist/{internal-BTK91Bqt.js → internal-v6XfOeJW.js} +6 -8
- package/dist/{internal-BTK91Bqt.js.map → internal-v6XfOeJW.js.map} +1 -1
- package/dist/{interval-Dq9gt7jx.js → interval-Bd2f4CRT.js} +6 -8
- package/dist/{interval-Dq9gt7jx.js.map → interval-Bd2f4CRT.js.map} +1 -1
- package/dist/{interval-ohuOJurX.d.ts → interval-CWyxqDpE.d.ts} +1 -5
- package/dist/interval-CWyxqDpE.d.ts.map +1 -0
- package/dist/{journal-C7om8Gum.js → journal-C1uLJi0w.js} +115 -121
- package/dist/{journal-C7om8Gum.js.map → journal-C1uLJi0w.js.map} +1 -1
- package/dist/{json-BhdsJaOQ.js → json-DtvZNAU9.js} +4 -6
- package/dist/{json-BhdsJaOQ.js.map → json-DtvZNAU9.js.map} +1 -1
- package/dist/json-recovery-DK4MDm5r.js +296 -0
- package/dist/json-recovery-DK4MDm5r.js.map +1 -0
- package/dist/{judge-calibration-BkXMIWA4.js → judge-calibration-DGqlfWW8.js} +5 -6
- package/dist/{judge-calibration-BkXMIWA4.js.map → judge-calibration-DGqlfWW8.js.map} +1 -1
- package/dist/{judge-calibration-DFtEMlde.d.ts → judge-calibration-DnrbQaXI.d.ts} +1 -3
- package/dist/judge-calibration-DnrbQaXI.d.ts.map +1 -0
- package/dist/{judge-gate-BVvmP00P.js → judge-gate-C2BCOt-M.js} +8 -12
- package/dist/{judge-gate-BVvmP00P.js.map → judge-gate-C2BCOt-M.js.map} +1 -1
- package/dist/judges-BLxg5p8E.js +20 -0
- package/dist/judges-BLxg5p8E.js.map +1 -0
- package/dist/{kind-factory-Dl0PmNQL.js → kind-factory-VKysp1ds.js} +23 -39
- package/dist/{kind-factory-Dl0PmNQL.js.map → kind-factory-VKysp1ds.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +3 -3
- package/dist/ledger-core/lock-holder.d.ts +1 -1
- package/dist/ledger-core/lock-holder.js +1 -2
- package/dist/{ledger-core-CAxUdDk0.js → ledger-core-CfT-FzPf.js} +5 -6
- package/dist/{ledger-core-CAxUdDk0.js.map → ledger-core-CfT-FzPf.js.map} +1 -1
- package/dist/{llm-client-b1ysZVIU.js → llm-client-kJ9EOqjt.js} +22 -30
- package/dist/{llm-client-b1ysZVIU.js.map → llm-client-kJ9EOqjt.js.map} +1 -1
- package/dist/{llm-judge-Cybq29bL.js → llm-judge-cgbhTKQp.js} +108 -164
- package/dist/llm-judge-cgbhTKQp.js.map +1 -0
- package/dist/{lock-holder-DaJbfpci.d.ts → lock-holder-DoxlZFkQ.d.ts} +1 -3
- package/dist/lock-holder-DoxlZFkQ.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +2 -2
- package/dist/{matrix-DtkZLD_3.d.ts → matrix-8X6i5kJ6.d.ts} +4 -12
- package/dist/matrix-8X6i5kJ6.d.ts.map +1 -0
- package/dist/{matrix-B8l-Nsxd.js → matrix-CgpX8wdm.js} +8 -13
- package/dist/matrix-CgpX8wdm.js.map +1 -0
- package/dist/{mean-BbA_JCzW.js → mean-CdDYG-us.js} +6 -9
- package/dist/{mean-BbA_JCzW.js.map → mean-CdDYG-us.js.map} +1 -1
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +17 -21
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{metrics-BKaPBtIG.js → metrics-DNCAUtuB.js} +2 -3
- package/dist/{metrics-BKaPBtIG.js.map → metrics-DNCAUtuB.js.map} +1 -1
- package/dist/{mint-CbdwjTPW.js → mint-D9ZysYtY.js} +7 -8
- package/dist/{mint-CbdwjTPW.js.map → mint-D9ZysYtY.js.map} +1 -1
- package/dist/{multi-layer-verifier-DEgMUk6v.d.ts → multi-layer-verifier-lM4aVN78.d.ts} +2 -4
- package/dist/multi-layer-verifier-lM4aVN78.d.ts.map +1 -0
- package/dist/{multiplicity-Sg0U7h8j.js → multiplicity-DAd4xaDC.js} +3 -4
- package/dist/{multiplicity-Sg0U7h8j.js.map → multiplicity-DAd4xaDC.js.map} +1 -1
- package/dist/{multiplicity-DIWHvysC.d.ts → multiplicity-D_SEsc5j.d.ts} +1 -3
- package/dist/multiplicity-D_SEsc5j.d.ts.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -17
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +10 -17
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +2 -10
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +11 -18
- package/dist/multishot/index.js.map +1 -1
- package/dist/{nonnegative-number-DvdMxg_f.js → nonnegative-number-lEYbOsuh.js} +2 -3
- package/dist/{nonnegative-number-DvdMxg_f.js.map → nonnegative-number-lEYbOsuh.js.map} +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-64hmQqON.js → otel-spans-BsGzTWIK.js} +5 -6
- package/dist/{otel-spans-64hmQqON.js.map → otel-spans-BsGzTWIK.js.map} +1 -1
- package/dist/{otlp-attributes-D9ptWaIJ.js → otlp-attributes-Boeb7gtH.js} +2 -3
- package/dist/{otlp-attributes-D9ptWaIJ.js.map → otlp-attributes-Boeb7gtH.js.map} +1 -1
- package/dist/{outcome-observations-FLmRpgYa.js → outcome-observations-D4snOtUb.js} +4 -5
- package/dist/{outcome-observations-FLmRpgYa.js.map → outcome-observations-D4snOtUb.js.map} +1 -1
- package/dist/{outcome-store-DUcMj-3D.js → outcome-store-TarrcfHG.js} +2 -3
- package/dist/{outcome-store-DUcMj-3D.js.map → outcome-store-TarrcfHG.js.map} +1 -1
- package/dist/{package-version-D7lQHt_-.js → package-version-CJl_pY9W.js} +2 -3
- package/dist/{package-version-D7lQHt_-.js.map → package-version-CJl_pY9W.js.map} +1 -1
- package/dist/{paired-arms-B2ttLc0_.js → paired-arms-D-54hO-k.js} +10 -14
- package/dist/{paired-arms-B2ttLc0_.js.map → paired-arms-D-54hO-k.js.map} +1 -1
- package/dist/{paired-binary-B_nejA44.js → paired-binary-BR0nwHOA.js} +3 -4
- package/dist/{paired-binary-B_nejA44.js.map → paired-binary-BR0nwHOA.js.map} +1 -1
- package/dist/{paired-promotion-decision-CQMJ5xGN.d.ts → paired-promotion-decision-BDnqepZ1.d.ts} +1 -5
- package/dist/paired-promotion-decision-BDnqepZ1.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-D0SHb-ZL.js → paired-promotion-decision-FGvHA4p_.js} +5 -7
- package/dist/{paired-promotion-decision-D0SHb-ZL.js.map → paired-promotion-decision-FGvHA4p_.js.map} +1 -1
- package/dist/{pareto-BiBnQXAE.js → pareto-B9udAt2i.js} +28 -50
- package/dist/pareto-B9udAt2i.js.map +1 -0
- package/dist/pipelines/index.d.ts +5 -11
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +6 -8
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-CQX-0weG.js → power-and-mde-ClI6RWw4.js} +4 -5
- package/dist/{power-and-mde-CQX-0weG.js.map → power-and-mde-ClI6RWw4.js.map} +1 -1
- package/dist/{predicate-FdKmElA_.js → predicate-BYHA4MBn.js} +2 -3
- package/dist/{predicate-FdKmElA_.js.map → predicate-BYHA4MBn.js.map} +1 -1
- package/dist/{prime-bridge-transport-6feEglLf.d.ts → prime-bridge-transport-Cfqir_Im.d.ts} +1 -3
- package/dist/prime-bridge-transport-Cfqir_Im.d.ts.map +1 -0
- package/dist/{prime-bridge-transport-Z3cpKGoV.js → prime-bridge-transport-Csd5SQg7.js} +2 -3
- package/dist/{prime-bridge-transport-Z3cpKGoV.js.map → prime-bridge-transport-Csd5SQg7.js.map} +1 -1
- package/dist/{prime-protocol-BGxjfhM4.d.ts → prime-protocol-BpQlTowq.d.ts} +4 -8
- package/dist/prime-protocol-BpQlTowq.d.ts.map +1 -0
- package/dist/{prime-protocol-BFgGrEMP.js → prime-protocol-CrhpXRz9.js} +4 -6
- package/dist/{prime-protocol-BFgGrEMP.js.map → prime-protocol-CrhpXRz9.js.map} +1 -1
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -2
- package/dist/{promotion-gate-CkVZHagC.d.ts → promotion-gate-CixshGI8.d.ts} +5 -11
- package/dist/promotion-gate-CixshGI8.d.ts.map +1 -0
- package/dist/{promotion-policy-6A8osGr_.js → promotion-policy-CBYeD4zi.js} +5 -6
- package/dist/{promotion-policy-6A8osGr_.js.map → promotion-policy-CBYeD4zi.js.map} +1 -1
- package/dist/{proposal-findings-DqLUWsAG.js → proposal-findings-WFQrxnfe.js} +3 -4
- package/dist/{proposal-findings-DqLUWsAG.js.map → proposal-findings-WFQrxnfe.js.map} +1 -1
- package/dist/{quantile--0yb07pD.js → quantile-CncXP-Ss.js} +2 -3
- package/dist/{quantile--0yb07pD.js.map → quantile-CncXP-Ss.js.map} +1 -1
- package/dist/{query-B_RZ36an.js → query-CGXzO9d4.js} +5 -6
- package/dist/{query-B_RZ36an.js.map → query-CGXzO9d4.js.map} +1 -1
- package/dist/{query-Cmv_U8vf.d.ts → query-UD0B08Sb.d.ts} +4 -6
- package/dist/query-UD0B08Sb.d.ts.map +1 -0
- package/dist/{random-DGEyr3zG.d.ts → random-BN150Q_X.d.ts} +1 -5
- package/dist/random-BN150Q_X.d.ts.map +1 -0
- package/dist/{random-DDDcC7YU.js → random-DrzcgC7F.js} +3 -4
- package/dist/{random-DDDcC7YU.js.map → random-DrzcgC7F.js.map} +1 -1
- package/dist/{raw-provider-sink-75w4FWBj.js → raw-provider-sink-DDLkP_eF.js} +3 -4
- package/dist/{raw-provider-sink-75w4FWBj.js.map → raw-provider-sink-DDLkP_eF.js.map} +1 -1
- package/dist/{record-id-DUgsK5qp.js → record-id-CWZXFh58.js} +2 -3
- package/dist/{record-id-DUgsK5qp.js.map → record-id-CWZXFh58.js.map} +1 -1
- package/dist/{redact-DXdlFb_U.d.ts → redact-C_wzC4c_.d.ts} +1 -3
- package/dist/redact-C_wzC4c_.d.ts.map +1 -0
- package/dist/{redact-mtkTiJVk.js → redact-D6APJS1d.js} +3 -5
- package/dist/{redact-mtkTiJVk.js.map → redact-D6APJS1d.js.map} +1 -1
- package/dist/{registry-BcdJmyoU.d.ts → registry-D9A7Tybn.d.ts} +5 -11
- package/dist/registry-D9A7Tybn.d.ts.map +1 -0
- package/dist/{release-confidence-CYIFH-fp.js → release-confidence-CTUJz5mM.js} +10 -16
- package/dist/release-confidence-CTUJz5mM.js.map +1 -0
- package/dist/{render-BhJqXoBH.js → render-pfAZ7IR4.js} +266 -33
- package/dist/render-pfAZ7IR4.js.map +1 -0
- package/dist/{reporting-html-Kvlc8oMt.js → reporting-html-DMXIjATy.js} +2 -3
- package/dist/{reporting-html-Kvlc8oMt.js.map → reporting-html-DMXIjATy.js.map} +1 -1
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +7 -7
- package/dist/{researcher-DKWtoMWk.d.ts → researcher-CLnNJ7Or.d.ts} +8 -12
- package/dist/researcher-CLnNJ7Or.d.ts.map +1 -0
- package/dist/{reward-nw2xZGZG.js → reward-P0VD_Aft.js} +2 -3
- package/dist/{reward-nw2xZGZG.js.map → reward-P0VD_Aft.js.map} +1 -1
- package/dist/{reward-BeUgVVC_.d.ts → reward-ST4PXNXF.d.ts} +2 -4
- package/dist/reward-ST4PXNXF.d.ts.map +1 -0
- package/dist/{reward-hacking-WTxUbtbb.d.ts → reward-hacking-C9hPVT0Q.d.ts} +4 -8
- package/dist/reward-hacking-C9hPVT0Q.d.ts.map +1 -0
- package/dist/{reward-hacking-ByuETaIb.js → reward-hacking-DZzHZlS1.js} +5 -7
- package/dist/{reward-hacking-ByuETaIb.js.map → reward-hacking-DZzHZlS1.js.map} +1 -1
- package/dist/rl.d.ts +26 -63
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +35 -52
- package/dist/rl.js.map +1 -1
- package/dist/{rolldown-runtime-8H4AJuhK.js → rolldown-runtime-BJohB4LM.js} +1 -2
- package/dist/rollout/index.d.ts +6 -6
- package/dist/rollout/index.js +7 -7
- package/dist/{rollout-Dd1ucT_M.js → rollout-BhkMpAJT.js} +279 -16
- package/dist/rollout-BhkMpAJT.js.map +1 -0
- package/dist/{terminal-record-CB9BFtjS.js → rollout-nodes-BHR9Q5jP.js} +351 -11
- package/dist/rollout-nodes-BHR9Q5jP.js.map +1 -0
- package/dist/{round-decimal-COrKglMh.js → round-decimal-ZmjAQPo1.js} +2 -3
- package/dist/{round-decimal-COrKglMh.js.map → round-decimal-ZmjAQPo1.js.map} +1 -1
- package/dist/{rubric-predictive-validity-Cd5SCtAf.d.ts → rubric-predictive-validity-B3RzbwOx.d.ts} +2 -8
- package/dist/rubric-predictive-validity-B3RzbwOx.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-CeKW2GQN.js → rubric-predictive-validity-CQp_M0vy.js} +3 -4
- package/dist/{rubric-predictive-validity-CeKW2GQN.js.map → rubric-predictive-validity-CQp_M0vy.js.map} +1 -1
- package/dist/{run-record-CysKLB93.js → run-record-CZy4-TnZ.js} +4 -5
- package/dist/{run-record-CysKLB93.js.map → run-record-CZy4-TnZ.js.map} +1 -1
- package/dist/{run-record-C3awiXPf.d.ts → run-record-D2BFMPGl.d.ts} +5 -7
- package/dist/run-record-D2BFMPGl.d.ts.map +1 -0
- package/dist/{run-record-DhEJgoPz.js → run-record-DEjx43cI.js} +6 -7
- package/dist/{run-record-DhEJgoPz.js.map → run-record-DEjx43cI.js.map} +1 -1
- package/dist/{sandbox-harness-BlSOu4LX.d.ts → sandbox-harness-B3Kx1fOA.d.ts} +1 -3
- package/dist/sandbox-harness-B3Kx1fOA.d.ts.map +1 -0
- package/dist/{schema-D6EVKzTB.d.ts → schema-CkIvnmLU.d.ts} +2 -43
- package/dist/schema-CkIvnmLU.d.ts.map +1 -0
- package/dist/{schema-Bt2vLFm3.js → schema-D-xu3MU_.js} +2 -3
- package/dist/{schema-Bt2vLFm3.js.map → schema-D-xu3MU_.js.map} +1 -1
- package/dist/{schema-C-ziyJD7.d.ts → schema-DkFJr3Y8.d.ts} +111 -48
- package/dist/schema-DkFJr3Y8.d.ts.map +1 -0
- package/dist/{schema-CvhfLjtZ.js → schema-YUIXT8iW.js} +37 -7
- package/dist/schema-YUIXT8iW.js.map +1 -0
- package/dist/search/index.d.ts +3 -35
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -5
- package/dist/{search-kernel-CctpiDd2.d.ts → search-kernel-B77eE7Ac.d.ts} +10 -36
- package/dist/search-kernel-B77eE7Ac.d.ts.map +1 -0
- package/dist/{search-ledger-CZRqtFbW.js → search-ledger-BsVi-Ngv.js} +17 -26
- package/dist/{search-ledger-CZRqtFbW.js.map → search-ledger-BsVi-Ngv.js.map} +1 -1
- package/dist/{search-ledger-file-Bd5Tioq6.js → search-ledger-file-DTBD6X-y.js} +6 -8
- package/dist/{search-ledger-file-Bd5Tioq6.js.map → search-ledger-file-DTBD6X-y.js.map} +1 -1
- package/dist/{search-ledger-types-p2xKfiDJ.d.ts → search-ledger-types-qiYRUo_N.d.ts} +5 -9
- package/dist/search-ledger-types-qiYRUo_N.d.ts.map +1 -0
- package/dist/{search-shipper-Dn3NLsuH.js → search-shipper-pdg4viK7.js} +8 -13
- package/dist/{search-shipper-Dn3NLsuH.js.map → search-shipper-pdg4viK7.js.map} +1 -1
- package/dist/{search-summary-D4bwYnAE.js → search-summary-Ct7SrmVc.js} +6 -7
- package/dist/{search-summary-D4bwYnAE.js.map → search-summary-Ct7SrmVc.js.map} +1 -1
- package/dist/{sequential-CimQrbjv.js → sequential-B1I0tRfL.js} +6 -7
- package/dist/{sequential-CimQrbjv.js.map → sequential-B1I0tRfL.js.map} +1 -1
- package/dist/{sequential-KwGspfeR.d.ts → sequential-Bb_pdTqG.d.ts} +1 -3
- package/dist/sequential-Bb_pdTqG.d.ts.map +1 -0
- package/dist/{sequential-BVv1sjsw.js → sequential-Bjs5roiK.js} +2 -3
- package/dist/{sequential-BVv1sjsw.js.map → sequential-Bjs5roiK.js.map} +1 -1
- package/dist/{sequential-eprocess-DodI6RoF.js → sequential-eprocess-C1YLwFaz.js} +3 -4
- package/dist/{sequential-eprocess-DodI6RoF.js.map → sequential-eprocess-C1YLwFaz.js.map} +1 -1
- package/dist/{server-ChhXPCgX.js → server-DA6h3co5.js} +11 -17
- package/dist/{server-ChhXPCgX.js.map → server-DA6h3co5.js.map} +1 -1
- package/dist/{single-run-lock-Dekd8_Wv.js → single-run-lock-BGlW3OSk.js} +3 -4
- package/dist/{single-run-lock-Dekd8_Wv.js.map → single-run-lock-BGlW3OSk.js.map} +1 -1
- package/dist/{skillopt-optimization-method-sB_-yi1G.js → skillopt-optimization-method-qB1M554K.js} +38 -57
- package/dist/{skillopt-optimization-method-sB_-yi1G.js.map → skillopt-optimization-method-qB1M554K.js.map} +1 -1
- package/dist/{spans-CVngGij3.d.ts → spans-vZoW__3q.d.ts} +2 -4
- package/dist/spans-vZoW__3q.d.ts.map +1 -0
- package/dist/{statistical-heldout-BjzSOmLH.d.ts → statistical-heldout-BYxHe3KO.d.ts} +9 -37
- package/dist/statistical-heldout-BYxHe3KO.d.ts.map +1 -0
- package/dist/{statistical-heldout-Bksv92Th.js → statistical-heldout-TP6ndL0B.js} +4 -5
- package/dist/{statistical-heldout-Bksv92Th.js.map → statistical-heldout-TP6ndL0B.js.map} +1 -1
- package/dist/{steps-DlepJPwD.d.ts → steps-B8fYY_Vo.d.ts} +1 -3
- package/dist/steps-B8fYY_Vo.d.ts.map +1 -0
- package/dist/{store-B3nWRy3B.d.ts → store-CMs5wGXP.d.ts} +2 -4
- package/dist/store-CMs5wGXP.d.ts.map +1 -0
- package/dist/{store-BhNuLdJw.js → store-ZykIyQmy.js} +2 -3
- package/dist/{store-BhNuLdJw.js.map → store-ZykIyQmy.js.map} +1 -1
- package/dist/{store-otlp-CFf27E8_.js → store-otlp-DmLm65XW.js} +8 -13
- package/dist/{store-otlp-CFf27E8_.js.map → store-otlp-DmLm65XW.js.map} +1 -1
- package/dist/{store-tool-spans-BK_8FjGG.js → store-tool-spans-C2g8UYm-.js} +17 -41
- package/dist/{store-tool-spans-BK_8FjGG.js.map → store-tool-spans-C2g8UYm-.js.map} +1 -1
- package/dist/{store-tool-spans-DxsFwtdr.d.ts → store-tool-spans-DMM6wMwB.d.ts} +11 -29
- package/dist/store-tool-spans-DMM6wMwB.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +1 -5
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +3 -5
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-C8V49Jt3.d.ts → summary-report-0La5vejY.d.ts} +5 -13
- package/dist/summary-report-0La5vejY.d.ts.map +1 -0
- package/dist/{summary-report-DHDWxe6p.js → summary-report-CPfsrLac.js} +21 -30
- package/dist/summary-report-CPfsrLac.js.map +1 -0
- package/dist/supervisor-run/index.d.ts +30 -29
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +6 -7
- package/dist/{systemone-CHnqwmYg.d.ts → systemone-UHbKD6LH.d.ts} +12 -24
- package/dist/systemone-UHbKD6LH.d.ts.map +1 -0
- package/dist/{systemone-protocol-Bl2EI2KQ.d.ts → systemone-protocol-C4t7JH4Y.d.ts} +1 -3
- package/dist/systemone-protocol-C4t7JH4Y.d.ts.map +1 -0
- package/dist/systemone-protocol.d.ts +1 -1
- package/dist/systemone-protocol.js +2 -3
- package/dist/{systemone-review-cQ9acAsb.d.ts → systemone-review-Cn9Uj0xG.d.ts} +8 -10
- package/dist/systemone-review-Cn9Uj0xG.d.ts.map +1 -0
- package/dist/{systemone-review-DFGtIUcf.js → systemone-review-S49gsy_y.js} +5 -6
- package/dist/{systemone-review-DFGtIUcf.js.map → systemone-review-S49gsy_y.js.map} +1 -1
- package/dist/systemone.d.ts +4 -4
- package/dist/systemone.js +3 -3
- package/dist/{task-failure-attributes-oIK3BhCR.js → task-failure-attributes-CC6VuIpk.js} +7 -10
- package/dist/{task-failure-attributes-oIK3BhCR.js.map → task-failure-attributes-CC6VuIpk.js.map} +1 -1
- package/dist/{tool-groups-b3oU_D88.d.ts → tool-groups-CFWmAGCV.d.ts} +3 -5
- package/dist/tool-groups-CFWmAGCV.d.ts.map +1 -0
- package/dist/{tool-waste-CkjzWmeY.d.ts → tool-waste-BFG1vwTV.d.ts} +4 -16
- package/dist/tool-waste-BFG1vwTV.d.ts.map +1 -0
- package/dist/{tool-waste-DLsjS_p-.js → tool-waste-CLJ46AbT.js} +10 -26
- package/dist/{tool-waste-DLsjS_p-.js.map → tool-waste-CLJ46AbT.js.map} +1 -1
- package/dist/trace-attributes.d.ts +1 -1
- package/dist/trace-attributes.js +1 -2
- package/dist/{trace-questions-BmT8M6UQ.d.ts → trace-questions-BFCQAzoT.d.ts} +4 -6
- package/dist/trace-questions-BFCQAzoT.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +46 -92
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +32 -54
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +13 -29
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +18 -22
- package/dist/traces.js.map +1 -1
- package/dist/{trajectory-D_7rLrvE.js → trajectory-O3GDHmev.js} +2 -3
- package/dist/{trajectory-D_7rLrvE.js.map → trajectory-O3GDHmev.js.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +2 -22
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +19 -30
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-DSqwJpL9.js → tree-DrIChF4a.js} +39 -68
- package/dist/{tree-DSqwJpL9.js.map → tree-DrIChF4a.js.map} +1 -1
- package/dist/{types-B_IFcKp5.d.ts → types-0KglnQ9o.d.ts} +3 -15
- package/dist/types-0KglnQ9o.d.ts.map +1 -0
- package/dist/{types-6Lfdlshp.d.ts → types-BEvjDziM.d.ts} +50 -62
- package/dist/types-BEvjDziM.d.ts.map +1 -0
- package/dist/{types-CnWmj8NW.js → types-BK0blWhe.js} +3 -4
- package/dist/{types-CnWmj8NW.js.map → types-BK0blWhe.js.map} +1 -1
- package/dist/{types-CmXY0PQf.d.ts → types-C-aOqgt2.d.ts} +2 -5
- package/dist/types-C-aOqgt2.d.ts.map +1 -0
- package/dist/{types-BXgASBCL.js → types-DK4vNv5q.js} +2 -3
- package/dist/{types-BXgASBCL.js.map → types-DK4vNv5q.js.map} +1 -1
- package/dist/{types-DZ0mYVw9.d.ts → types-DUR0dqET.d.ts} +3 -5
- package/dist/types-DUR0dqET.d.ts.map +1 -0
- package/dist/{types-M6eF9m-r.d.ts → types-e3HGaI7P.d.ts} +84 -24
- package/dist/types-e3HGaI7P.d.ts.map +1 -0
- package/dist/{verdict-BGQmxQ8A.js → verdict-C8TcdwfD.js} +3 -4
- package/dist/{verdict-BGQmxQ8A.js.map → verdict-C8TcdwfD.js.map} +1 -1
- package/dist/{verdict-C24NELtN.d.ts → verdict-D_qiBQ6_.d.ts} +1 -5
- package/dist/verdict-D_qiBQ6_.d.ts.map +1 -0
- package/dist/{verdict-cache-CL_7tePD.js → verdict-cache-BVH-fl18.js} +3 -4
- package/dist/{verdict-cache-CL_7tePD.js.map → verdict-cache-BVH-fl18.js.map} +1 -1
- package/dist/wire/index.d.ts +43 -51
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +1 -2
- package/docs/eval-surface-map.md +15 -0
- package/docs/rollout.md +132 -6
- package/package.json +18 -18
- package/dist/adversarial-smnADNFS.d.ts.map +0 -1
- package/dist/agent-profile-CNmwfaqq.d.ts.map +0 -1
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +0 -1
- package/dist/analyze-runs-DfX6dHP0.d.ts.map +0 -1
- package/dist/attribute-vocabulary-CcXqHODT.d.ts.map +0 -1
- package/dist/benchmark-command-Cnjfo0Zh.js.map +0 -1
- package/dist/campaign-evidence-DunzcJGv.js.map +0 -1
- package/dist/campaign-uddTlFg5.js.map +0 -1
- package/dist/canonical-CFpojCN5.d.ts.map +0 -1
- package/dist/client-hwo3cJjR.d.ts.map +0 -1
- package/dist/completion-verifier-asaV1nSj.js.map +0 -1
- package/dist/cost-ledger-C1W3ICmP.d.ts.map +0 -1
- package/dist/dataset-DQqhOCPt.d.ts.map +0 -1
- package/dist/default-registry-CLG5YqGS.d.ts.map +0 -1
- package/dist/define-agent-eval-JKt3gMQr.d.ts.map +0 -1
- package/dist/descriptive-B2iPaT9J.d.ts +0 -89
- package/dist/descriptive-B2iPaT9J.d.ts.map +0 -1
- package/dist/descriptive-CVNy48rR.js.map +0 -1
- package/dist/emitter-DABYYTcH.d.ts.map +0 -1
- package/dist/engine-dtRVeuQs.d.ts.map +0 -1
- package/dist/errors-DEE6u6ot.d.ts.map +0 -1
- package/dist/evaluation-DblGquCB.d.ts.map +0 -1
- package/dist/exact-types-Do63KoKn.d.ts.map +0 -1
- package/dist/exporters-DWvM16LE.js.map +0 -1
- package/dist/exporters-Dk3oEbK_.d.ts.map +0 -1
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +0 -1
- package/dist/feedback-trajectory-CYVJXolG.d.ts.map +0 -1
- package/dist/hf-dataset-CKxIJQba.js.map +0 -1
- package/dist/index-CHxwfW5g.d.ts.map +0 -1
- package/dist/index-CTIRz-AB.d.ts.map +0 -1
- package/dist/index-CalAsXMR.d.ts.map +0 -1
- package/dist/index-CdEVr8-O.d.ts.map +0 -1
- package/dist/index-CfJULcW5.d.ts.map +0 -1
- package/dist/index-D7fmC-1K.d.ts.map +0 -1
- package/dist/index-jnuFLxOs.d.ts.map +0 -1
- package/dist/integrity-D-s2iHiT.js.map +0 -1
- package/dist/integrity-D9lBnv_K.d.ts.map +0 -1
- package/dist/interval-ohuOJurX.d.ts.map +0 -1
- package/dist/judge-calibration-DFtEMlde.d.ts.map +0 -1
- package/dist/llm-judge-Cybq29bL.js.map +0 -1
- package/dist/lock-holder-DaJbfpci.d.ts.map +0 -1
- package/dist/matrix-B8l-Nsxd.js.map +0 -1
- package/dist/matrix-DtkZLD_3.d.ts.map +0 -1
- package/dist/mint-CkzpVPdz.d.ts +0 -81
- package/dist/mint-CkzpVPdz.d.ts.map +0 -1
- package/dist/multi-layer-verifier-DEgMUk6v.d.ts.map +0 -1
- package/dist/multiplicity-DIWHvysC.d.ts.map +0 -1
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +0 -1
- package/dist/pareto-BiBnQXAE.js.map +0 -1
- package/dist/prime-bridge-transport-6feEglLf.d.ts.map +0 -1
- package/dist/prime-protocol-BGxjfhM4.d.ts.map +0 -1
- package/dist/promotion-gate-CkVZHagC.d.ts.map +0 -1
- package/dist/query-Cmv_U8vf.d.ts.map +0 -1
- package/dist/random-DGEyr3zG.d.ts.map +0 -1
- package/dist/redact-DXdlFb_U.d.ts.map +0 -1
- package/dist/registry-BcdJmyoU.d.ts.map +0 -1
- package/dist/release-confidence-CYIFH-fp.js.map +0 -1
- package/dist/render-BhJqXoBH.js.map +0 -1
- package/dist/researcher-DKWtoMWk.d.ts.map +0 -1
- package/dist/reward-BeUgVVC_.d.ts.map +0 -1
- package/dist/reward-hacking-WTxUbtbb.d.ts.map +0 -1
- package/dist/rollout-Dd1ucT_M.js.map +0 -1
- package/dist/rubric-predictive-validity-Cd5SCtAf.d.ts.map +0 -1
- package/dist/run-record-C3awiXPf.d.ts.map +0 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +0 -1
- package/dist/schema-C-ziyJD7.d.ts.map +0 -1
- package/dist/schema-CvhfLjtZ.js.map +0 -1
- package/dist/schema-D6EVKzTB.d.ts.map +0 -1
- package/dist/search-kernel-CctpiDd2.d.ts.map +0 -1
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +0 -1
- package/dist/sequential-KwGspfeR.d.ts.map +0 -1
- package/dist/spans-CVngGij3.d.ts.map +0 -1
- package/dist/statistical-heldout-BjzSOmLH.d.ts.map +0 -1
- package/dist/steps-DlepJPwD.d.ts.map +0 -1
- package/dist/store-B3nWRy3B.d.ts.map +0 -1
- package/dist/store-tool-spans-DxsFwtdr.d.ts.map +0 -1
- package/dist/summary-report-C8V49Jt3.d.ts.map +0 -1
- package/dist/summary-report-DHDWxe6p.js.map +0 -1
- package/dist/systemone-CHnqwmYg.d.ts.map +0 -1
- package/dist/systemone-protocol-Bl2EI2KQ.d.ts.map +0 -1
- package/dist/systemone-review-cQ9acAsb.d.ts.map +0 -1
- package/dist/terminal-record-CB9BFtjS.js.map +0 -1
- package/dist/tool-groups-b3oU_D88.d.ts.map +0 -1
- package/dist/tool-waste-CkjzWmeY.d.ts.map +0 -1
- package/dist/trace-questions-BmT8M6UQ.d.ts.map +0 -1
- package/dist/types-6Lfdlshp.d.ts.map +0 -1
- package/dist/types-B_IFcKp5.d.ts.map +0 -1
- package/dist/types-CmXY0PQf.d.ts.map +0 -1
- package/dist/types-DZ0mYVw9.d.ts.map +0 -1
- package/dist/types-M6eF9m-r.d.ts.map +0 -1
- package/dist/verdict-C24NELtN.d.ts.map +0 -1
|
@@ -1,28 +1,25 @@
|
|
|
1
|
-
import { d as CostReceipt, l as CostLedgerSummary, o as CostLedgerHandle, p as CostUsage, r as CostBreakdown, u as CostProvenance } from "./cost-ledger-
|
|
2
|
-
import { c as ValidationError, t as AgentEvalError } from "./errors-
|
|
3
|
-
import { a as RunRecord, c as RunSplitTag } from "./run-record-
|
|
4
|
-
import { J as TraceAnalysisStore, i as AnalystFinding, p as AnalystUsageReceipt, u as AnalystRunResult, v as ProposalFinding } from "./types-
|
|
5
|
-
import {
|
|
6
|
-
import { Dt as FinalEvidenceRecord, M as PairedArmsComparison, at as CampaignEvidenceContext, dt as EvidenceReceipt, wt as FinalEvidenceLedger } from "./statistical-heldout-
|
|
7
|
-
import { a as PairedPromotionDecision, u as PairedBootstrapResult } from "./paired-promotion-decision-
|
|
8
|
-
import { E as SearchExecutionIdentity, G as SearchSplit, V as SearchProposer, a as SearchAttemptAccounting, i as SearchArtifactRef, j as SearchModelIdentity, nt as EvaluationClaim, rt as EvaluationUnitSummary, tt as SearchUnknown, z as SearchOperationRecordedEvent } from "./search-ledger-types-
|
|
9
|
-
import { r as LedgerHash } from "./canonical-
|
|
10
|
-
import { C as JudgeDimension, H as SurfaceProposer, N as ParetoParent, O as LabeledScenarioSource, R as Scenario, S as JudgeConfig, _ as GateDecision, a as CampaignResult, b as GenerationRecord, c as CampaignTraceWriter, d as DispatchContext, f as DispatchFn, g as GateContribution, i as CampaignCostMeter, j as MutableSurface, k as LabeledScenarioStore, l as CodeSurface, o as CampaignScenarioIdentity, p as Gate, r as CampaignCellResult, u as ComponentSurface, v as GateResult, y as GenerationCandidate } from "./types-
|
|
11
|
-
import { At as SearchHistoryReceipt, Dt as SearchHistoryCoverage, I as SearchPolicy, Tt as SearchHistoryAdmissionOptions, Wt as CampaignStorage, gt as SearchRecorder, mt as SearchLedgerBinding, o as SearchCellResult, pt as RegisterSearchNodeInput, vt as SearchRunIdentity } from "./search-kernel-
|
|
12
|
-
import { E as AnalystIssueExpectation, G as ExternalOptimizerRunnerCommand, I as ExternalOptimizerEvaluationObservation, J as ExternalTextEvaluationRequest, K as ExternalOptimizerWireCounts, N as ExternalOptimizerCallbackLimits, R as ExternalOptimizerModelBudget, W as ExternalOptimizerResumeMode, m as AnalystBenchmarkLabelState, q as ExternalTextCandidate, t as AgentProfile$1, u as AnalystBenchmarkCase, z as ExternalOptimizerModelCall } from "./agent-profile-
|
|
13
|
-
import { r as DatasetScenario, t as Dataset } from "./dataset-
|
|
14
|
-
import {
|
|
15
|
-
import { t as
|
|
16
|
-
import {
|
|
1
|
+
import { d as CostReceipt, l as CostLedgerSummary, o as CostLedgerHandle, p as CostUsage, r as CostBreakdown, u as CostProvenance } from "./cost-ledger-Czro9wmo.js";
|
|
2
|
+
import { c as ValidationError, t as AgentEvalError } from "./errors-E9I2hldP.js";
|
|
3
|
+
import { a as RunRecord, c as RunSplitTag } from "./run-record-D2BFMPGl.js";
|
|
4
|
+
import { J as TraceAnalysisStore, i as AnalystFinding, p as AnalystUsageReceipt, u as AnalystRunResult, v as ProposalFinding } from "./types-BEvjDziM.js";
|
|
5
|
+
import { j as LlmCallMetadata, rt as ServedModelPolicy, x as ChatClient } from "./types-0KglnQ9o.js";
|
|
6
|
+
import { Dt as FinalEvidenceRecord, M as PairedArmsComparison, at as CampaignEvidenceContext, dt as EvidenceReceipt, wt as FinalEvidenceLedger } from "./statistical-heldout-BYxHe3KO.js";
|
|
7
|
+
import { a as PairedPromotionDecision, u as PairedBootstrapResult } from "./paired-promotion-decision-BDnqepZ1.js";
|
|
8
|
+
import { E as SearchExecutionIdentity, G as SearchSplit, V as SearchProposer, a as SearchAttemptAccounting, i as SearchArtifactRef, j as SearchModelIdentity, nt as EvaluationClaim, rt as EvaluationUnitSummary, tt as SearchUnknown, z as SearchOperationRecordedEvent } from "./search-ledger-types-qiYRUo_N.js";
|
|
9
|
+
import { r as LedgerHash } from "./canonical-gLeHiqzd.js";
|
|
10
|
+
import { C as JudgeDimension, H as SurfaceProposer, N as ParetoParent, O as LabeledScenarioSource, R as Scenario, S as JudgeConfig, _ as GateDecision, a as CampaignResult, b as GenerationRecord, c as CampaignTraceWriter, d as DispatchContext, f as DispatchFn, g as GateContribution, i as CampaignCostMeter, j as MutableSurface, k as LabeledScenarioStore, l as CodeSurface, o as CampaignScenarioIdentity, p as Gate, r as CampaignCellResult, u as ComponentSurface, v as GateResult, y as GenerationCandidate } from "./types-e3HGaI7P.js";
|
|
11
|
+
import { At as SearchHistoryReceipt, Dt as SearchHistoryCoverage, I as SearchPolicy, Tt as SearchHistoryAdmissionOptions, Wt as CampaignStorage, gt as SearchRecorder, mt as SearchLedgerBinding, o as SearchCellResult, pt as RegisterSearchNodeInput, vt as SearchRunIdentity } from "./search-kernel-B77eE7Ac.js";
|
|
12
|
+
import { E as AnalystIssueExpectation, G as ExternalOptimizerRunnerCommand, I as ExternalOptimizerEvaluationObservation, J as ExternalTextEvaluationRequest, K as ExternalOptimizerWireCounts, N as ExternalOptimizerCallbackLimits, R as ExternalOptimizerModelBudget, W as ExternalOptimizerResumeMode, m as AnalystBenchmarkLabelState, q as ExternalTextCandidate, t as AgentProfile$1, u as AnalystBenchmarkCase, z as ExternalOptimizerModelCall } from "./agent-profile-CSEQRC9l.js";
|
|
13
|
+
import { r as DatasetScenario, t as Dataset } from "./dataset-DThC1S6G.js";
|
|
14
|
+
import { t as DetectRewardHackingInput } from "./reward-hacking-C9hPVT0Q.js";
|
|
15
|
+
import { f as TraceSpanEvent, t as HostedClient } from "./client-BIFwUg7I.js";
|
|
16
|
+
import { n as CompletionVerdict, o as ProducedState, r as CorrectnessChecker, t as CompletionRequirement } from "./completion-verifier-7KEfroqx.js";
|
|
17
17
|
import { AgentProfile } from "@tangle-network/agent-interface";
|
|
18
18
|
import { z } from "zod";
|
|
19
|
-
//#region src/campaign/search-ledger-errors.d.ts
|
|
20
19
|
/** Base error for invalid search-ledger input or operations. */
|
|
21
20
|
declare class SearchLedgerError extends ValidationError {}
|
|
22
21
|
/** Error raised when durable search-ledger data fails an integrity check. */
|
|
23
22
|
declare class SearchLedgerIntegrityError extends SearchLedgerError {}
|
|
24
|
-
//#endregion
|
|
25
|
-
//#region src/campaign/coverage.d.ts
|
|
26
23
|
interface CampaignCoverage {
|
|
27
24
|
complete: boolean;
|
|
28
25
|
expectedCellIds: string[];
|
|
@@ -42,8 +39,6 @@ declare function campaignSplitDigestFromIdentities(scenarios: readonly CampaignS
|
|
|
42
39
|
declare function campaignSplitDigest<TScenario extends Scenario>(scenarios: readonly TScenario[], reps: number): `sha256:${string}`;
|
|
43
40
|
/** Exact designed-denominator receipt for one campaign. */
|
|
44
41
|
declare function campaignCoverage<TArtifact, TScenario extends Scenario>(cells: readonly CampaignCellResult<TArtifact>[], scenarios: readonly TScenario[], reps: number, requireJudgeScore: boolean): CampaignCoverage;
|
|
45
|
-
//#endregion
|
|
46
|
-
//#region src/campaign/final-evidence.d.ts
|
|
47
42
|
/** Durable final-data policy. Keep its ledger shared across related campaigns. */
|
|
48
43
|
interface FinalEvidencePolicy {
|
|
49
44
|
ledger: FinalEvidenceLedger;
|
|
@@ -54,8 +49,6 @@ interface FinalEvidenceUse {
|
|
|
54
49
|
claim: EvaluationClaim;
|
|
55
50
|
record: FinalEvidenceRecord;
|
|
56
51
|
}
|
|
57
|
-
//#endregion
|
|
58
|
-
//#region src/campaign/cell-schedule.d.ts
|
|
59
52
|
declare function buildCellSchedule<TScenario extends Scenario>(scenarios: TScenario[], seed: number, reps: number): Array<{
|
|
60
53
|
scenario: TScenario;
|
|
61
54
|
rep: number;
|
|
@@ -64,8 +57,6 @@ declare function buildCellSchedule<TScenario extends Scenario>(scenarios: TScena
|
|
|
64
57
|
}>;
|
|
65
58
|
type CellScheduleSlot<TScenario extends Scenario> = ReturnType<typeof buildCellSchedule<TScenario>>[number];
|
|
66
59
|
declare function cellCachePath(runDir: string, cellId: string): string;
|
|
67
|
-
//#endregion
|
|
68
|
-
//#region src/campaign/cell-cache.d.ts
|
|
69
60
|
type CacheIssueReason = 'missing' | 'manifest-mismatch' | 'cell-mismatch' | 'missing-cost-provenance' | 'invalid-cost-provenance' | 'invalid-cost-receipts' | 'corrupt';
|
|
70
61
|
type CacheRead<TArtifact> = {
|
|
71
62
|
status: 'hit';
|
|
@@ -74,8 +65,6 @@ type CacheRead<TArtifact> = {
|
|
|
74
65
|
status: 'miss';
|
|
75
66
|
reason: CacheIssueReason;
|
|
76
67
|
};
|
|
77
|
-
//#endregion
|
|
78
|
-
//#region src/campaign/plan-campaign-run.d.ts
|
|
79
68
|
interface CampaignRunPlanCell {
|
|
80
69
|
cellId: string;
|
|
81
70
|
scenarioId: string;
|
|
@@ -118,8 +107,6 @@ interface PlanCampaignRunOptions<TScenario extends Scenario, TArtifact> {
|
|
|
118
107
|
* run-vs-cached schedule so callers can preview cost and resumability before spending.
|
|
119
108
|
*/
|
|
120
109
|
declare function planCampaignRun<TScenario extends Scenario, TArtifact>(opts: PlanCampaignRunOptions<TScenario, TArtifact>): CampaignRunPlan;
|
|
121
|
-
//#endregion
|
|
122
|
-
//#region src/campaign/run-campaign.d.ts
|
|
123
110
|
interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {
|
|
124
111
|
scenarios: TScenario[];
|
|
125
112
|
dispatch: DispatchFn<TScenario, TArtifact>;
|
|
@@ -194,7 +181,9 @@ interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {
|
|
|
194
181
|
* manifest, and pairing are unchanged. An attempt that failed because the
|
|
195
182
|
* campaign was cancelled is never retried.
|
|
196
183
|
*/
|
|
197
|
-
cellRetry?: CampaignCellRetryPolicy
|
|
184
|
+
cellRetry?: CampaignCellRetryPolicy<TArtifact>;
|
|
185
|
+
/** Once per final persisted cell, including cache hits, in completion order. */
|
|
186
|
+
onCellSettled?: (cell: CampaignCellResult<TArtifact>, scenario: TScenario) => void | Promise<void>;
|
|
198
187
|
/**
|
|
199
188
|
* Per-cell dispatch deadline in ms. A `dispatch` that neither resolves nor
|
|
200
189
|
* rejects within this window is a hang (a stalled model request, an
|
|
@@ -287,7 +276,7 @@ interface CampaignCellFailureReceipt<TArtifact = unknown> {
|
|
|
287
276
|
* execution directory, including attempts resumed in a later invocation.
|
|
288
277
|
* The final cell records the retry count as `retryAttempts`.
|
|
289
278
|
*/
|
|
290
|
-
interface CampaignCellRetryPolicy {
|
|
279
|
+
interface CampaignCellRetryPolicy<TArtifact = unknown> {
|
|
291
280
|
/** Total attempts per cell, including the first. A positive safe integer. */
|
|
292
281
|
attempts: number;
|
|
293
282
|
/** Decides whether a failed attempt is dispatched again. Receives the
|
|
@@ -295,13 +284,13 @@ interface CampaignCellRetryPolicy {
|
|
|
295
284
|
* ready-made predicate for infrastructure hiccups (502/503/504, dropped
|
|
296
285
|
* streams, admission rejections). */
|
|
297
286
|
retryable: (failure: CampaignCellFailureReceipt['failure']) => boolean;
|
|
287
|
+
/** Retry a measured artifact without classifying it as a dispatch failure. */
|
|
288
|
+
retryResult?: (artifact: TArtifact) => boolean;
|
|
298
289
|
}
|
|
299
290
|
/**
|
|
300
291
|
* Core campaign orchestrator: fan scenarios through dispatch, score with judges, aggregate bootstrap CIs, and persist reproducible `CampaignResult` records.
|
|
301
292
|
*/
|
|
302
293
|
declare function runCampaign<TScenario extends Scenario, TArtifact>(opts: RunCampaignOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
|
|
303
|
-
//#endregion
|
|
304
|
-
//#region src/campaign/presets/run-eval.d.ts
|
|
305
294
|
interface RunEvalOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'runDir'> {
|
|
306
295
|
runDir: string;
|
|
307
296
|
}
|
|
@@ -309,8 +298,6 @@ interface RunEvalOptions<TScenario extends Scenario, TArtifact> extends Omit<Run
|
|
|
309
298
|
* Simplest evaluation preset: run scenarios through dispatch, score with judges, and return a `CampaignResult` — no optimizer, no gate, no PR.
|
|
310
299
|
*/
|
|
311
300
|
declare function runEval<TScenario extends Scenario, TArtifact>(opts: RunEvalOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
|
|
312
|
-
//#endregion
|
|
313
|
-
//#region src/campaign/auto-pr.d.ts
|
|
314
301
|
interface OpenAutoPrOptions<TArtifact, TScenario extends Scenario> {
|
|
315
302
|
/** Campaign result to attach to the PR. */
|
|
316
303
|
result: CampaignResult<TArtifact, TScenario>;
|
|
@@ -347,8 +334,6 @@ interface OpenAutoPrResult {
|
|
|
347
334
|
* Open a GitHub PR for a gate-approved surface promotion, attaching the manifest hash, gate verdict, and diff as the PR body.
|
|
348
335
|
*/
|
|
349
336
|
declare function openAutoPr<TArtifact, TScenario extends Scenario>(options: OpenAutoPrOptions<TArtifact, TScenario>): OpenAutoPrResult;
|
|
350
|
-
//#endregion
|
|
351
|
-
//#region src/campaign/presets/run-optimization.d.ts
|
|
352
337
|
/** Improve a surface as a search on the kernel; see the module comment. */
|
|
353
338
|
declare function runOptimization<TScenario extends Scenario, TArtifact>(opts: RunOptimizationOptions<TScenario, TArtifact>): Promise<RunOptimizationResult<TArtifact, TScenario>>;
|
|
354
339
|
/** Optimization inputs, immutable proposal snapshots, and imported baseline validation. */
|
|
@@ -446,8 +431,6 @@ interface RunOptimizationResult<TArtifact, TScenario extends Scenario> {
|
|
|
446
431
|
/** The non-dominated set of measured surfaces by per-scenario composite. */
|
|
447
432
|
paretoFrontier: ParetoParent[];
|
|
448
433
|
}
|
|
449
|
-
//#endregion
|
|
450
|
-
//#region src/campaign/presets/run-improvement-loop.d.ts
|
|
451
434
|
type RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> = RunOptimizationOptions<TScenario, TArtifact> & {
|
|
452
435
|
/** Holdout scenarios kept OUT of the training optimization pool — used
|
|
453
436
|
* ONLY to score baseline vs winner for the gate. */
|
|
@@ -507,8 +490,6 @@ interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extend
|
|
|
507
490
|
* Gated-promotion shell over `runOptimization`: scores the winner against the baseline on a holdout set, runs the release gate, and optionally opens a PR.
|
|
508
491
|
*/
|
|
509
492
|
declare function runImprovementLoop<TScenario extends Scenario, TArtifact>(opts: RunImprovementLoopOptions<TScenario, TArtifact>): Promise<RunImprovementLoopResult<TArtifact, TScenario>>;
|
|
510
|
-
//#endregion
|
|
511
|
-
//#region src/campaign/transient-failure.d.ts
|
|
512
493
|
interface TransientFailureOptions {
|
|
513
494
|
/**
|
|
514
495
|
* Treat full-duration timeouts ("timeout after 180000ms") as transient.
|
|
@@ -542,8 +523,6 @@ declare function isTransientTransportFailure(message: string | null | undefined,
|
|
|
542
523
|
* (`quotaExhaustedUntil`).
|
|
543
524
|
*/
|
|
544
525
|
declare function transientDispatchFailure(opts?: TransientFailureOptions): (failure: CampaignCellFailureReceipt['failure']) => boolean;
|
|
545
|
-
//#endregion
|
|
546
|
-
//#region src/llm-judge.d.ts
|
|
547
526
|
/** A rubric dimension as a bare key or the full `{ key, description }` shape. A
|
|
548
527
|
* bare string uses the key as its own description. */
|
|
549
528
|
type LlmJudgeDimension = string | JudgeDimension;
|
|
@@ -616,8 +595,6 @@ interface LlmJudgeOptions<TArtifact, TScenario extends Scenario = Scenario> {
|
|
|
616
595
|
* `weightedComposite`.
|
|
617
596
|
*/
|
|
618
597
|
declare function llmJudge<TArtifact = unknown, TScenario extends Scenario = Scenario>(name: string, prompt: string, opts: LlmJudgeOptions<TArtifact, TScenario>): JudgeConfig<TArtifact, TScenario>;
|
|
619
|
-
//#endregion
|
|
620
|
-
//#region src/reference-equivalence-judge.d.ts
|
|
621
598
|
interface ReferenceEquivalenceScenario extends Scenario {
|
|
622
599
|
userRequest: string;
|
|
623
600
|
expectedAnswer: string;
|
|
@@ -643,8 +620,6 @@ interface ReferenceEquivalenceJudgeResult extends LlmCallMetadata {
|
|
|
643
620
|
score: number;
|
|
644
621
|
rationale: string;
|
|
645
622
|
}
|
|
646
|
-
//#endregion
|
|
647
|
-
//#region src/campaign/external-optimizer-observations.d.ts
|
|
648
623
|
interface ExternalOptimizerObservationSummary {
|
|
649
624
|
scope: 'callback-submitted-candidates';
|
|
650
625
|
path: string;
|
|
@@ -692,8 +667,6 @@ declare function readExternalOptimizerObservationArtifact(input: {
|
|
|
692
667
|
summary: ExternalOptimizerObservationSummary;
|
|
693
668
|
storage?: CampaignStorage;
|
|
694
669
|
}): ExternalOptimizerObservationArtifact;
|
|
695
|
-
//#endregion
|
|
696
|
-
//#region src/campaign/gepa-candidate-population.d.ts
|
|
697
670
|
interface GepaCandidatePopulationSummary {
|
|
698
671
|
readonly scope: 'gepa-candidate-population';
|
|
699
672
|
readonly path: string;
|
|
@@ -742,8 +715,6 @@ declare function readGepaCandidatePopulationArtifact(input: {
|
|
|
742
715
|
summary: GepaCandidatePopulationSummary;
|
|
743
716
|
storage?: CampaignStorage;
|
|
744
717
|
}): GepaCandidatePopulationArtifact;
|
|
745
|
-
//#endregion
|
|
746
|
-
//#region src/campaign/optimization-cost.d.ts
|
|
747
718
|
/** Cost reported by a method or by final test scoring. */
|
|
748
719
|
interface ComparisonCost {
|
|
749
720
|
/** Known subtotal. Consult `costProvenance` before treating this as total spend. */
|
|
@@ -768,8 +739,6 @@ interface OptimizationTokenUsage {
|
|
|
768
739
|
totalTokens: number;
|
|
769
740
|
calls: number;
|
|
770
741
|
}
|
|
771
|
-
//#endregion
|
|
772
|
-
//#region src/campaign/optimization-method.d.ts
|
|
773
742
|
/** Shared campaign settings applied to every optimization method. */
|
|
774
743
|
type OptimizationMethodRunOptions<TScenario extends Scenario, TArtifact> = Omit<RunCampaignOptions<TScenario, TArtifact>, 'costCeiling' | 'costLedger' | 'dispatch' | 'judges' | 'runDir' | 'scenarios' | 'seed'>;
|
|
775
744
|
interface OptimizationPackageSource {
|
|
@@ -882,8 +851,6 @@ interface OptimizationMethod<TScenario extends Scenario = Scenario, TArtifact =
|
|
|
882
851
|
name: string;
|
|
883
852
|
optimize: (input: OptimizationMethodInput<TScenario, TArtifact>) => Promise<OptimizationMethodResult>;
|
|
884
853
|
}
|
|
885
|
-
//#endregion
|
|
886
|
-
//#region src/campaign/external-text-evaluation.d.ts
|
|
887
854
|
interface ExternalOptimizationExample {
|
|
888
855
|
id: string;
|
|
889
856
|
data: unknown;
|
|
@@ -907,8 +874,6 @@ interface ExternalTextEvaluationResponse {
|
|
|
907
874
|
};
|
|
908
875
|
}
|
|
909
876
|
declare function decodeExternalTextCandidate(candidate: ExternalTextCandidate): MutableSurface;
|
|
910
|
-
//#endregion
|
|
911
|
-
//#region src/campaign/external-text-optimization-contract.d.ts
|
|
912
877
|
interface ExternalTextOptimizerContext {
|
|
913
878
|
readonly runId: string;
|
|
914
879
|
readonly name: string;
|
|
@@ -966,8 +931,6 @@ interface ExternalTextOptimizationMethodConfig<TScenario extends Scenario, TArti
|
|
|
966
931
|
describeArtifact?: (artifact: TArtifact, scenario: TScenario) => unknown;
|
|
967
932
|
run: (context: ExternalTextOptimizerContext) => Promise<ExternalTextOptimizerResult>;
|
|
968
933
|
}
|
|
969
|
-
//#endregion
|
|
970
|
-
//#region src/campaign/external-text-optimization.d.ts
|
|
971
934
|
/**
|
|
972
935
|
* Adapt a third-party text optimizer without reimplementing its search.
|
|
973
936
|
*
|
|
@@ -975,8 +938,6 @@ interface ExternalTextOptimizationMethodConfig<TScenario extends Scenario, TArti
|
|
|
975
938
|
* counted before execution and stop at `maxEvaluations`.
|
|
976
939
|
*/
|
|
977
940
|
declare function externalTextOptimizationMethod<TScenario extends Scenario, TArtifact>(config: ExternalTextOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
978
|
-
//#endregion
|
|
979
|
-
//#region src/campaign/optimizer-model.d.ts
|
|
980
941
|
type OptimizerModelBudget = ExternalOptimizerModelBudget;
|
|
981
942
|
/** One metered model path supplied by the package that owns execution. */
|
|
982
943
|
interface OpenAICompatibleOptimizerModel {
|
|
@@ -1001,8 +962,6 @@ interface OpenAICompatibleOptimizerModel {
|
|
|
1001
962
|
*/
|
|
1002
963
|
anthropicEndpoint?: boolean;
|
|
1003
964
|
}
|
|
1004
|
-
//#endregion
|
|
1005
|
-
//#region src/campaign/gepa-optimization-method.d.ts
|
|
1006
965
|
/** Shared settings for one bounded GEPA engine invocation. */
|
|
1007
966
|
interface GepaEngineOptions {
|
|
1008
967
|
/** GEPA engine name. GEPA validates names available in its Python runtime. */
|
|
@@ -1148,8 +1107,6 @@ interface GepaOptimizationMethodConfig<TScenario extends Scenario, TArtifact = u
|
|
|
1148
1107
|
* stripped like any other credential.
|
|
1149
1108
|
*/
|
|
1150
1109
|
declare function gepaOptimizationMethod<TScenario extends Scenario, TArtifact>(config: GepaOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
1151
|
-
//#endregion
|
|
1152
|
-
//#region src/campaign/presets/compare-optimization-methods.d.ts
|
|
1153
1110
|
interface OptimizationMethodScore {
|
|
1154
1111
|
evidence?: {
|
|
1155
1112
|
baseline: EvidenceReceipt;
|
|
@@ -1273,8 +1230,6 @@ interface CompareOptimizationMethodsOptions<TScenario extends Scenario, TArtifac
|
|
|
1273
1230
|
* Compare complete optimization methods on disjoint train, selection, and final test data.
|
|
1274
1231
|
*/
|
|
1275
1232
|
declare function compareOptimizationMethods<TScenario extends Scenario, TArtifact>(rawOpts: CompareOptimizationMethodsOptions<TScenario, TArtifact>): Promise<OptimizationMethodComparison>;
|
|
1276
|
-
//#endregion
|
|
1277
|
-
//#region src/campaign/skillopt-optimization-method.d.ts
|
|
1278
1233
|
interface SkillOptTrainerConfig {
|
|
1279
1234
|
epochs: number;
|
|
1280
1235
|
batchSize: number;
|
|
@@ -1344,14 +1299,10 @@ interface SkillOptOptimizationMethodConfig<TScenario extends Scenario, TArtifact
|
|
|
1344
1299
|
}
|
|
1345
1300
|
/** Run Microsoft's SkillOpt trainer as a complete optimization method. */
|
|
1346
1301
|
declare function skillOptOptimizationMethod<TScenario extends Scenario, TArtifact>(config: SkillOptOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
1347
|
-
//#endregion
|
|
1348
|
-
//#region src/campaign/gates/compose.d.ts
|
|
1349
1302
|
/** Compose gates — all must `ship` for the composite to `ship`. First
|
|
1350
1303
|
* non-ship verdict short-circuits the composite verdict, but ALL gates run
|
|
1351
1304
|
* (so the result records every gate's reason — useful for diagnostics). */
|
|
1352
1305
|
declare function composeGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(...gates: Array<Gate<TArtifact, TScenario>>): Gate<TArtifact, TScenario>;
|
|
1353
|
-
//#endregion
|
|
1354
|
-
//#region src/canary.d.ts
|
|
1355
1306
|
type CanaryKind = 'silent_judge_fallback' | 'judge_calibration_drift' | 'distribution_shift';
|
|
1356
1307
|
type CanarySeverity = 'info' | 'warn' | 'error';
|
|
1357
1308
|
interface CanaryAlert {
|
|
@@ -1430,8 +1381,6 @@ interface CanaryOptions {
|
|
|
1430
1381
|
* the input is used to define "recent" vs "historical" windows.
|
|
1431
1382
|
*/
|
|
1432
1383
|
declare function runCanaries(runs: RunRecord[], opts?: CanaryOptions): CanaryReport;
|
|
1433
|
-
//#endregion
|
|
1434
|
-
//#region src/red-team.d.ts
|
|
1435
1384
|
type RedTeamCategory = 'prompt_injection_direct' | 'prompt_injection_indirect' | 'jailbreak_persona' | 'jailbreak_dan' | 'pii_leak' | 'permission_escalation' | 'data_exfil' | 'policy_override';
|
|
1436
1385
|
interface RedTeamPayload {
|
|
1437
1386
|
category: RedTeamCategory;
|
|
@@ -1472,8 +1421,6 @@ declare function redTeamDataset(extraCases?: RedTeamCase[]): Dataset;
|
|
|
1472
1421
|
declare function scoreRedTeamOutput(output: string, toolCalls: string[], rtCase: RedTeamCase): RedTeamFinding;
|
|
1473
1422
|
/** Aggregate red-team findings into per-category pass rates. */
|
|
1474
1423
|
declare function redTeamReport(findings: RedTeamFinding[]): RedTeamReport;
|
|
1475
|
-
//#endregion
|
|
1476
|
-
//#region src/campaign/gates/default-production-gate.d.ts
|
|
1477
1424
|
type DefaultProductionGateCheck = 'dimension-regression' | 'budget' | 'red-team' | 'reward-hacking' | 'canary';
|
|
1478
1425
|
type DefaultProductionRewardHackingOptions = Omit<DetectRewardHackingInput, 'runs' | 'truthOf'> & {
|
|
1479
1426
|
truthOf: NonNullable<DetectRewardHackingInput['truthOf']>;
|
|
@@ -1538,8 +1485,6 @@ interface DefaultProductionGateOptions {
|
|
|
1538
1485
|
* Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision.
|
|
1539
1486
|
*/
|
|
1540
1487
|
declare function defaultProductionGate<TArtifact, TScenario extends Scenario>(options: DefaultProductionGateOptions): Gate<TArtifact, TScenario>;
|
|
1541
|
-
//#endregion
|
|
1542
|
-
//#region src/campaign/gates/heldout-gate.d.ts
|
|
1543
1488
|
interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
|
|
1544
1489
|
scenarios: TScenario[];
|
|
1545
1490
|
/** Effect-size threshold the CI lower bound must clear, in the judge's native
|
|
@@ -1563,8 +1508,6 @@ interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
|
|
|
1563
1508
|
* bootstrap otherwise. See {@link decidePairedPromotion}.
|
|
1564
1509
|
*/
|
|
1565
1510
|
declare function heldOutGate<TArtifact, TScenario extends Scenario>(options: HeldOutGateOptions<TScenario>): Gate<TArtifact, TScenario>;
|
|
1566
|
-
//#endregion
|
|
1567
|
-
//#region src/campaign/score-utils.d.ts
|
|
1568
1511
|
interface CampaignComparisonUnits {
|
|
1569
1512
|
pairedCellN: number;
|
|
1570
1513
|
unitIds: string[];
|
|
@@ -1594,8 +1537,6 @@ interface CampaignBreakdown {
|
|
|
1594
1537
|
/** Per-candidate evidence a reflective/patch proposer grounds its next proposal
|
|
1595
1538
|
* on: mean score per judge dimension + per-scenario composite. */
|
|
1596
1539
|
declare function campaignBreakdown<TArtifact, TScenario extends Scenario>(campaign: CampaignResult<TArtifact, TScenario>): CampaignBreakdown;
|
|
1597
|
-
//#endregion
|
|
1598
|
-
//#region src/campaign/provenance.d.ts
|
|
1599
1540
|
interface LoopProvenanceCandidate {
|
|
1600
1541
|
/** Generation index this candidate was proposed in. */
|
|
1601
1542
|
generation: number;
|
|
@@ -1791,8 +1732,6 @@ interface EmitLoopProvenanceArgs<TArtifact, TScenario extends Scenario> extends
|
|
|
1791
1732
|
* collector never fails the loop (the durable artifact is the source of truth).
|
|
1792
1733
|
*/
|
|
1793
1734
|
declare function emitLoopProvenance<TArtifact, TScenario extends Scenario>(args: EmitLoopProvenanceArgs<TArtifact, TScenario>): Promise<EmitLoopProvenanceResult>;
|
|
1794
|
-
//#endregion
|
|
1795
|
-
//#region src/campaign/analyst-surface.d.ts
|
|
1796
1735
|
interface TraceAnalystScenario extends Scenario {
|
|
1797
1736
|
kind: 'trace-analyst';
|
|
1798
1737
|
traceStore: TraceAnalysisStore;
|
|
@@ -1813,8 +1752,6 @@ interface BuildTraceAnalystSurfaceDispatchOptions {
|
|
|
1813
1752
|
signal: AbortSignal;
|
|
1814
1753
|
}): Promise<TraceAnalystArtifact>;
|
|
1815
1754
|
}
|
|
1816
|
-
//#endregion
|
|
1817
|
-
//#region src/campaign/cost-breakdown.d.ts
|
|
1818
1755
|
/** Read the source rates for this execution, never a copied model price table.
|
|
1819
1756
|
* inputTokens includes cachedTokens. Seat credits are standard model credits;
|
|
1820
1757
|
* discounts and MCP charges require provider observations, not token guesses.
|
|
@@ -1837,8 +1774,6 @@ declare function measureCampaignCost(input: {
|
|
|
1837
1774
|
};
|
|
1838
1775
|
signal?: AbortSignal;
|
|
1839
1776
|
}): Promise<CostBreakdown>;
|
|
1840
|
-
//#endregion
|
|
1841
|
-
//#region src/campaign/cross-surface-types.d.ts
|
|
1842
1777
|
/** Whether one candidate attempt produced a usable executable outcome. */
|
|
1843
1778
|
type CrossSurfaceAttemptCompleteness = 'complete' | 'missing' | 'invalid';
|
|
1844
1779
|
/** One independently proposed change on one caller-defined surface. */
|
|
@@ -2098,16 +2033,12 @@ interface CrossSurfaceInteractionReport<TRow extends CrossSurfaceTaskRow = Cross
|
|
|
2098
2033
|
pairwise: CrossSurfacePairwiseEntry[];
|
|
2099
2034
|
selections: CrossSurfaceSelections;
|
|
2100
2035
|
}
|
|
2101
|
-
//#endregion
|
|
2102
|
-
//#region src/campaign/cross-surface-interaction.d.ts
|
|
2103
2036
|
/**
|
|
2104
2037
|
* Build the complete cross-surface evidence matrix and derive all three frozen
|
|
2105
2038
|
* candidates. The task/candidate/component orders are part of the input so
|
|
2106
2039
|
* neither insertion order nor an after-the-fact tie-break can change a result.
|
|
2107
2040
|
*/
|
|
2108
2041
|
declare function analyzeCrossSurfaceInteractions<TRow extends CrossSurfaceTaskRow>(input: AnalyzeCrossSurfaceInteractionsInput<TRow>): CrossSurfaceInteractionReport<TRow>;
|
|
2109
|
-
//#endregion
|
|
2110
|
-
//#region src/campaign/fixtures.d.ts
|
|
2111
2042
|
type EvalFixtureValidationMode = 'vitest' | 'none';
|
|
2112
2043
|
interface EvalFixtureFile {
|
|
2113
2044
|
path: string;
|
|
@@ -2167,8 +2098,6 @@ declare function loadEvalFixtureScenarios(evalsDir: string, options?: LoadEvalFi
|
|
|
2167
2098
|
* and returns the plan plus each fixture's name/path/fingerprint.
|
|
2168
2099
|
*/
|
|
2169
2100
|
declare function planEvalFixtureRun<TArtifact = unknown>(options: PlanEvalFixtureRunOptions<TArtifact>): EvalFixtureRunPlan;
|
|
2170
|
-
//#endregion
|
|
2171
|
-
//#region src/campaign/gates/neutralization-gate.d.ts
|
|
2172
2101
|
interface NeutralizationGateOptions<TScenario extends Scenario = Scenario> {
|
|
2173
2102
|
scenarios: TScenario[];
|
|
2174
2103
|
/** Reject when the neutralized (content-blanked, footprint-matched) variant
|
|
@@ -2182,8 +2111,6 @@ interface NeutralizationGateOptions<TScenario extends Scenario = Scenario> {
|
|
|
2182
2111
|
* mostly reproduced by a footprint-matched neutralized variant.
|
|
2183
2112
|
*/
|
|
2184
2113
|
declare function neutralizationGate<TArtifact, TScenario extends Scenario>(options: NeutralizationGateOptions<TScenario>): Gate<TArtifact, TScenario>;
|
|
2185
|
-
//#endregion
|
|
2186
|
-
//#region src/campaign/gepa-search-import.d.ts
|
|
2187
2114
|
/** The external optimizers whose searches this module records. */
|
|
2188
2115
|
type ExternalSearchOptimizer = 'GEPA' | 'SkillOpt';
|
|
2189
2116
|
interface GepaPopulationImport {
|
|
@@ -2234,8 +2161,6 @@ declare function importExternalEvaluations(input: {
|
|
|
2234
2161
|
}): Promise<GepaEvaluationImport>;
|
|
2235
2162
|
/** An optimizer candidate as the mutable surface it represents. */
|
|
2236
2163
|
declare function externalSurface(candidate: ExternalTextCandidate): MutableSurface;
|
|
2237
|
-
//#endregion
|
|
2238
|
-
//#region src/campaign/grounded-reflection.d.ts
|
|
2239
2164
|
/**
|
|
2240
2165
|
* Evidence grounding for reflective optimizers (GEPA-style revise loops).
|
|
2241
2166
|
*
|
|
@@ -2294,8 +2219,6 @@ interface UngroundedLiteralReport {
|
|
|
2294
2219
|
/** The subset failing rollouts actually used - prescribing these is proven harmful. */
|
|
2295
2220
|
readonly harmful: readonly string[];
|
|
2296
2221
|
}
|
|
2297
|
-
//#endregion
|
|
2298
|
-
//#region src/campaign/labeled-store/fs-adapter.d.ts
|
|
2299
2222
|
interface FsLabeledScenarioStoreOptions {
|
|
2300
2223
|
/** Root directory for JSONL files. Created if missing. */
|
|
2301
2224
|
root: string;
|
|
@@ -2305,8 +2228,6 @@ interface FsLabeledScenarioStoreOptions {
|
|
|
2305
2228
|
/** Test seam — override `Date.now()` for deterministic tests. */
|
|
2306
2229
|
now?: () => number;
|
|
2307
2230
|
}
|
|
2308
|
-
//#endregion
|
|
2309
|
-
//#region src/campaign/neutralize.d.ts
|
|
2310
2231
|
/**
|
|
2311
2232
|
* @module
|
|
2312
2233
|
* Footprint-matched neutralization — the placebo control for content-vs-footprint
|
|
@@ -2334,8 +2255,6 @@ interface FsLabeledScenarioStoreOptions {
|
|
|
2334
2255
|
* variant from being a true footprint match.
|
|
2335
2256
|
*/
|
|
2336
2257
|
declare function neutralizeText(content: string): string;
|
|
2337
|
-
//#endregion
|
|
2338
|
-
//#region src/campaign/optimization-method-composition.d.ts
|
|
2339
2258
|
/** Optimize a projection while evaluating every proposal in the complete candidate. */
|
|
2340
2259
|
declare function scopedOptimizationMethod<S extends Scenario, A>(options: {
|
|
2341
2260
|
name: string;
|
|
@@ -2348,175 +2267,6 @@ declare function sequentialOptimizationMethod<S extends Scenario, A>(options: {
|
|
|
2348
2267
|
name: string;
|
|
2349
2268
|
methods: readonly OptimizationMethod<S, A>[];
|
|
2350
2269
|
}): OptimizationMethod<S, A>;
|
|
2351
|
-
//#endregion
|
|
2352
|
-
//#region src/artifact-validator.d.ts
|
|
2353
|
-
/** An artifact an agent produced, as completion and produced-state checks read it. */
|
|
2354
|
-
interface ValidatedArtifact {
|
|
2355
|
-
/** Logical kind — validators type-guard on this */
|
|
2356
|
-
kind: 'file' | 'json' | 'text' | 'binary' | string;
|
|
2357
|
-
/** Filesystem-style path, optional */
|
|
2358
|
-
path?: string;
|
|
2359
|
-
/** String content for text/json/file kinds */
|
|
2360
|
-
content?: string;
|
|
2361
|
-
/** Binary content (if kind === 'binary') */
|
|
2362
|
-
bytes?: Uint8Array;
|
|
2363
|
-
/** Caller-supplied metadata (mimeType, sha256, size, etc.) */
|
|
2364
|
-
metadata?: Record<string, unknown>;
|
|
2365
|
-
}
|
|
2366
|
-
//#endregion
|
|
2367
|
-
//#region src/completion-verifier.d.ts
|
|
2368
|
-
/** What kind of produced state can satisfy a requirement structurally. */
|
|
2369
|
-
type SatisfiedBy = 'artifact' | 'proposal' | 'tool-call' | 'any';
|
|
2370
|
-
interface CompletionRequirement {
|
|
2371
|
-
/** Stable id from the task gold (e.g. a persona's `expected_requirements[].req_id`). */
|
|
2372
|
-
reqId: string;
|
|
2373
|
-
/** Human-readable description of the required deliverable. */
|
|
2374
|
-
title: string;
|
|
2375
|
-
/** Optional kind/category hint, matched against a produced item's kind. */
|
|
2376
|
-
category?: string;
|
|
2377
|
-
/** What produced state satisfies this requirement. Defaults to 'any'. */
|
|
2378
|
-
satisfiedBy?: SatisfiedBy;
|
|
2379
|
-
}
|
|
2380
|
-
interface TaskGold {
|
|
2381
|
-
taskId: string;
|
|
2382
|
-
requirements: CompletionRequirement[];
|
|
2383
|
-
}
|
|
2384
|
-
interface ProducedProposal {
|
|
2385
|
-
id: string;
|
|
2386
|
-
title: string;
|
|
2387
|
-
status: 'pending' | 'approved' | 'rejected';
|
|
2388
|
-
/** Optional persisted body — when present, enables a correctness check. */
|
|
2389
|
-
content?: string;
|
|
2390
|
-
}
|
|
2391
|
-
/** Everything observable about what a run actually produced. */
|
|
2392
|
-
interface ProducedState {
|
|
2393
|
-
/** Persisted vault artifacts. Reuses the shared `ValidatedArtifact` shape. */
|
|
2394
|
-
artifacts: ValidatedArtifact[];
|
|
2395
|
-
/** Proposals / filings the agent created. */
|
|
2396
|
-
proposals: ProducedProposal[];
|
|
2397
|
-
/** Names of tools the agent invoked. */
|
|
2398
|
-
toolCalls: string[];
|
|
2399
|
-
}
|
|
2400
|
-
interface RequirementCheck {
|
|
2401
|
-
reqId: string;
|
|
2402
|
-
title: string;
|
|
2403
|
-
/** A produced item of the right kind matched the requirement, non-empty. */
|
|
2404
|
-
structurallyPresent: boolean;
|
|
2405
|
-
/**
|
|
2406
|
-
* Whether the matched item actually fulfils the requirement. `null` when
|
|
2407
|
-
* not structurally present, when the matched item carries no content
|
|
2408
|
-
* to assess, or when the correctness check itself failed (`unmeasured`).
|
|
2409
|
-
*/
|
|
2410
|
-
correct: boolean | null;
|
|
2411
|
-
/** structurallyPresent && !unmeasured && correct !== false. */
|
|
2412
|
-
satisfied: boolean;
|
|
2413
|
-
/**
|
|
2414
|
-
* Set when the correctness check itself errored (LLM call failure or an
|
|
2415
|
-
* unparseable response after retry). The requirement's fulfilment is
|
|
2416
|
-
* UNKNOWN — `correct` stays null, `satisfied` is false, and
|
|
2417
|
-
* `completionVerdict` excludes the row from `completionRate`'s
|
|
2418
|
-
* denominator. Never folded into a zero: a synthetic zero is
|
|
2419
|
-
* indistinguishable from a real failure (see `JudgeParseError`).
|
|
2420
|
-
*/
|
|
2421
|
-
unmeasured?: true;
|
|
2422
|
-
/** Why the correctness check could not be measured (present iff `unmeasured`). */
|
|
2423
|
-
unmeasuredReason?: string;
|
|
2424
|
-
/** Human-readable evidence for the verdict. */
|
|
2425
|
-
evidence: string[];
|
|
2426
|
-
}
|
|
2427
|
-
/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
|
|
2428
|
-
* `score` = `completionRate` — derived in `completionVerdict()`, the one
|
|
2429
|
-
* place those equalities hold by construction. */
|
|
2430
|
-
interface CompletionVerdict extends DefaultVerdict {
|
|
2431
|
-
taskId: string;
|
|
2432
|
-
requirements: RequirementCheck[];
|
|
2433
|
-
/** satisfied / MEASURABLE requirements (unmeasured rows leave the denominator). */
|
|
2434
|
-
completionRate: number;
|
|
2435
|
-
/** Every measurable requirement satisfied (false when anything is unmeasured). */
|
|
2436
|
-
fullyComplete: boolean;
|
|
2437
|
-
/** Requirements whose correctness check errored — reported, never scored as zero. */
|
|
2438
|
-
unmeasuredCount: number;
|
|
2439
|
-
}
|
|
2440
|
-
/**
|
|
2441
|
-
* Construct a `CompletionVerdict` from the per-requirement checks, deriving
|
|
2442
|
-
* `completionRate` / `fullyComplete` and the spine fields (`valid` =
|
|
2443
|
-
* `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
|
|
2444
|
-
* requirements — a verdict over nothing is a misconfiguration, mirroring
|
|
2445
|
-
* `verifyCompletion`'s gold-spec guard.
|
|
2446
|
-
*/
|
|
2447
|
-
declare function completionVerdict(input: {
|
|
2448
|
-
taskId: string;
|
|
2449
|
-
requirements: RequirementCheck[];
|
|
2450
|
-
/** What certified the correctness stage, when anything did. Omitted =
|
|
2451
|
-
* an uncertified verdict — the honest default for a bare checker. */
|
|
2452
|
-
certification?: VerdictCertification;
|
|
2453
|
-
}): CompletionVerdict;
|
|
2454
|
-
/**
|
|
2455
|
-
* What a correctness checker declares about itself so the completion
|
|
2456
|
-
* verdict can carry a certification: which strategy member it discharges,
|
|
2457
|
-
* its exact identity, and the steps its answers rest on unverified.
|
|
2458
|
-
*/
|
|
2459
|
-
interface CorrectnessCheckerAttestation {
|
|
2460
|
-
strategy: VerificationStrategySource;
|
|
2461
|
-
checker: CheckerIdentity;
|
|
2462
|
-
assumptions: string[];
|
|
2463
|
-
}
|
|
2464
|
-
/**
|
|
2465
|
-
* Decides whether a produced item's content actually fulfils a requirement.
|
|
2466
|
-
* Injected so the structural verifier stays pure and unit-testable; the
|
|
2467
|
-
* production implementation is `createLlmCorrectnessChecker`.
|
|
2468
|
-
*
|
|
2469
|
-
* `attestation` is optional metadata on the function value: a checker that
|
|
2470
|
-
* carries one yields certified completion verdicts; a bare function yields
|
|
2471
|
-
* the same verdict uncertified. A plain arrow function remains a valid
|
|
2472
|
-
* checker.
|
|
2473
|
-
*/
|
|
2474
|
-
interface CorrectnessChecker {
|
|
2475
|
-
(requirement: CompletionRequirement, content: string): Promise<{
|
|
2476
|
-
correct: boolean;
|
|
2477
|
-
reason: string;
|
|
2478
|
-
}>;
|
|
2479
|
-
attestation?: CorrectnessCheckerAttestation;
|
|
2480
|
-
}
|
|
2481
|
-
/**
|
|
2482
|
-
* Verify whether a run completed the task. `checkCorrectness` is injected —
|
|
2483
|
-
* `createLlmCorrectnessChecker` for production, a deterministic stub in tests.
|
|
2484
|
-
*
|
|
2485
|
-
* Throws on a gold spec with no requirements: an eval task that requires
|
|
2486
|
-
* nothing is a misconfiguration, not a vacuously-complete task.
|
|
2487
|
-
*/
|
|
2488
|
-
declare function verifyCompletion(gold: TaskGold, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<CompletionVerdict>;
|
|
2489
|
-
interface LlmCorrectnessCheckerOpts {
|
|
2490
|
-
model?: string;
|
|
2491
|
-
/** Optional ledger for direct use. */
|
|
2492
|
-
costLedger?: CostLedgerHandle;
|
|
2493
|
-
costPhase?: string;
|
|
2494
|
-
costTags?: Record<string, string>;
|
|
2495
|
-
signal?: AbortSignal;
|
|
2496
|
-
/** Max chars of artifact content sent to the checker. */
|
|
2497
|
-
maxContentChars?: number;
|
|
2498
|
-
/**
|
|
2499
|
-
* Checker LLM calls per requirement before giving up (parse failures and
|
|
2500
|
-
* call errors both consume attempts). The failure then surfaces as an
|
|
2501
|
-
* `unmeasured` requirement, never a zero.
|
|
2502
|
-
*/
|
|
2503
|
-
maxAttempts?: number;
|
|
2504
|
-
/**
|
|
2505
|
-
* Forensic capture of every checker request/response/error — without it a
|
|
2506
|
-
* checker failure is unauditable (the agent-turn raws never contain the
|
|
2507
|
-
* checker's own calls). Same sink contract as `LlmClient`.
|
|
2508
|
-
*/
|
|
2509
|
-
rawSink?: RawProviderSink;
|
|
2510
|
-
}
|
|
2511
|
-
/**
|
|
2512
|
-
* Production `CorrectnessChecker` — one LLM call per matched artifact,
|
|
2513
|
-
* deterministic (temperature 0), structured JSON out. Judges fulfilment
|
|
2514
|
-
* only: a plan, a gesture, or a description of what should be done does not
|
|
2515
|
-
* fulfil a requirement — the artifact must BE the deliverable.
|
|
2516
|
-
*/
|
|
2517
|
-
declare function createLlmCorrectnessChecker(chat: ChatClient, opts?: LlmCorrectnessCheckerOpts): CorrectnessChecker;
|
|
2518
|
-
//#endregion
|
|
2519
|
-
//#region src/produced-state.d.ts
|
|
2520
2270
|
/** A tool the agent invoked. */
|
|
2521
2271
|
interface ToolCallEventLike {
|
|
2522
2272
|
type: 'tool_call';
|
|
@@ -2566,8 +2316,6 @@ type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLik
|
|
|
2566
2316
|
* recover dropped events, or establish freshness beyond the supplied stream.
|
|
2567
2317
|
*/
|
|
2568
2318
|
declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
|
|
2569
|
-
//#endregion
|
|
2570
|
-
//#region src/campaign/presets/playback.d.ts
|
|
2571
2319
|
/** One step of a user story — what the user does. The driver interprets
|
|
2572
2320
|
* `payload` (a Playwright selector + action, or a sandbox chat turn). */
|
|
2573
2321
|
interface PlaybackStep {
|
|
@@ -2664,8 +2412,6 @@ interface ScoreboardRenderOptions {
|
|
|
2664
2412
|
* rows in, same bytes out (no clock/random), so it is safe to snapshot.
|
|
2665
2413
|
*/
|
|
2666
2414
|
declare function renderScoreboardMarkdown(rows: readonly ScoreboardRow[], opts?: ScoreboardRenderOptions): string;
|
|
2667
|
-
//#endregion
|
|
2668
|
-
//#region src/integrity/backend-integrity.d.ts
|
|
2669
2415
|
interface BackendIntegrityReport {
|
|
2670
2416
|
/** Total records inspected. */
|
|
2671
2417
|
totalRecords: number;
|
|
@@ -2711,8 +2457,6 @@ declare function summarizeBackendIntegrity(records: ReadonlyArray<RunRecord>): B
|
|
|
2711
2457
|
declare function assertRealBackend(records: ReadonlyArray<RunRecord>, opts?: {
|
|
2712
2458
|
allowMixed?: boolean;
|
|
2713
2459
|
}): BackendIntegrityReport;
|
|
2714
|
-
//#endregion
|
|
2715
|
-
//#region src/campaign/presets/run-profile-matrix.d.ts
|
|
2716
2460
|
/** Dispatch for one cell: render `profile` against `scenario`, returning the
|
|
2717
2461
|
* artifact the judges score. Run LLM work through `ctx.cost.runPaidCall` —
|
|
2718
2462
|
* the integrity check depends on its receipt. */
|
|
@@ -2842,8 +2586,6 @@ interface RunProfileMatrixResult<TArtifact, TScenario extends Scenario> {
|
|
|
2842
2586
|
* Profile × scenario matrix runner: fan N agent profiles across M scenarios, project each cell to a validated `RunRecord` with real token usage, and enforce the backend-integrity guard before returning.
|
|
2843
2587
|
*/
|
|
2844
2588
|
declare function runProfileMatrix<TScenario extends Scenario, TArtifact>(opts: RunProfileMatrixOptions<TScenario, TArtifact>): Promise<RunProfileMatrixResult<TArtifact, TScenario>>;
|
|
2845
|
-
//#endregion
|
|
2846
|
-
//#region src/campaign/presets/segmented-profile-matrix.d.ts
|
|
2847
2589
|
interface ProfileMatrixRow {
|
|
2848
2590
|
rowId: string;
|
|
2849
2591
|
ordinal: number;
|
|
@@ -2922,8 +2664,6 @@ interface FinalizedProfileMatrixResult<TArtifact, TScenario extends Scenario> ex
|
|
|
2922
2664
|
declare function createProfileMatrixPlan<TScenario extends Scenario, TArtifact>(opts: CreateProfileMatrixPlanOptions<TScenario, TArtifact>): ProfileMatrixPlan<TScenario, TArtifact>;
|
|
2923
2665
|
declare function runProfileMatrixSegment<TScenario extends Scenario, TArtifact>(opts: RunProfileMatrixSegmentOptions<TScenario, TArtifact>): Promise<ProfileMatrixSegmentResult<TArtifact, TScenario>>;
|
|
2924
2666
|
declare function finalizeProfileMatrix<TScenario extends Scenario, TArtifact>(opts: FinalizeProfileMatrixOptions<TScenario, TArtifact>): Promise<FinalizedProfileMatrixResult<TArtifact, TScenario>>;
|
|
2925
|
-
//#endregion
|
|
2926
|
-
//#region src/campaign/retained-dispatch.d.ts
|
|
2927
2667
|
type RetainedDispatchOutcome<T> = {
|
|
2928
2668
|
succeeded: true;
|
|
2929
2669
|
value: T;
|
|
@@ -2972,15 +2712,11 @@ declare function createRetainedDispatch<Lane extends string, T>(options: Retaine
|
|
|
2972
2712
|
/** Counts include uncertain dispatches, which keep their reservation. No pricing is inferred. */
|
|
2973
2713
|
committed(): ReadonlyMap<Lane, number>;
|
|
2974
2714
|
};
|
|
2975
|
-
//#endregion
|
|
2976
|
-
//#region src/campaign/run-dir.d.ts
|
|
2977
2715
|
/** Resolve a campaign `runDir`. An absolute path is honored as-is (the caller
|
|
2978
2716
|
* chose an explicit location). A bare name is placed under the shared home root
|
|
2979
2717
|
* so bundles never pollute a repo working tree — the default the harness should
|
|
2980
2718
|
* compute so callers pass a *name*, not a path. */
|
|
2981
2719
|
declare function resolveRunDir(runDir: string, repo?: string): string;
|
|
2982
|
-
//#endregion
|
|
2983
|
-
//#region src/campaign/scenario-selection.d.ts
|
|
2984
2720
|
/**
|
|
2985
2721
|
* Discriminative scenario selection (research claim E2).
|
|
2986
2722
|
*
|
|
@@ -3021,8 +2757,6 @@ interface DiscriminationScore {
|
|
|
3021
2757
|
declare function scoreDiscrimination(signals: ScenarioSignal[], opts?: {
|
|
3022
2758
|
saturationCeiling?: number;
|
|
3023
2759
|
}): DiscriminationScore[];
|
|
3024
|
-
//#endregion
|
|
3025
|
-
//#region src/campaign/search-campaign-cell.d.ts
|
|
3026
2760
|
/**
|
|
3027
2761
|
* What `cell-settled` records for one campaign cell: `passed` with the
|
|
3028
2762
|
* composite (and each judge's) when every judge scored it, else `errored`,
|
|
@@ -3042,8 +2776,6 @@ declare function searchReceiptAccounting(receipts: ReadonlyArray<CostReceipt>):
|
|
|
3042
2776
|
/** What ran a proposal: a model when it made a paid call other than a judge's,
|
|
3043
2777
|
* else deterministic code. */
|
|
3044
2778
|
declare function searchProposalExecution(receipts: ReadonlyArray<CostReceipt>, provider: string, source: SearchOperationRecordedEvent['execution']['source']): SearchOperationRecordedEvent['execution'];
|
|
3045
|
-
//#endregion
|
|
3046
|
-
//#region src/campaign/single-run-lock.d.ts
|
|
3047
2779
|
/**
|
|
3048
2780
|
* Single-run lock for evaluations that share one mutable environment.
|
|
3049
2781
|
*
|
|
@@ -3079,8 +2811,6 @@ interface SingleRunLock {
|
|
|
3079
2811
|
* leaves a marker that fails closed instead of admitting overlapping runs.
|
|
3080
2812
|
*/
|
|
3081
2813
|
declare function acquireSingleRunLock(opts: SingleRunLockOptions): SingleRunLock;
|
|
3082
|
-
//#endregion
|
|
3083
|
-
//#region src/campaign/surface-identity.d.ts
|
|
3084
2814
|
/** Validate the immutable identity shape; the owning executor verifies the Git objects and patch. */
|
|
3085
2815
|
declare function assertCodeSurfaceIdentity(surface: unknown): asserts surface is CodeSurface;
|
|
3086
2816
|
/**
|
|
@@ -3105,8 +2835,6 @@ declare function surfaceHash(surface: MutableSurface): string;
|
|
|
3105
2835
|
declare function renderSurfaceDiff(winnerSurface: MutableSurface, baselineSurface: MutableSurface): string;
|
|
3106
2836
|
/** Bind a campaign cache entry to the exact surface and caller-owned execution revision. */
|
|
3107
2837
|
declare function surfaceDispatchRef(surface: MutableSurface, executionRef?: string): string;
|
|
3108
|
-
//#endregion
|
|
3109
|
-
//#region src/campaign/upstream-evaluators.d.ts
|
|
3110
2838
|
interface PhoenixEvaluationResultLike {
|
|
3111
2839
|
score?: number;
|
|
3112
2840
|
label?: string;
|
|
@@ -3128,8 +2856,6 @@ interface UpstreamEvaluationContext {
|
|
|
3128
2856
|
readonly signal: AbortSignal;
|
|
3129
2857
|
readonly callId?: string;
|
|
3130
2858
|
}
|
|
3131
|
-
//#endregion
|
|
3132
|
-
//#region src/campaign/worktree/index.d.ts
|
|
3133
2859
|
type GitOutput = string | Uint8Array;
|
|
3134
2860
|
type GitEnvironment = Readonly<Record<string, string>>;
|
|
3135
2861
|
type GitRunner = (args: string[], cwd: string, env?: GitEnvironment) => GitOutput;
|
|
@@ -3191,6 +2917,5 @@ declare function verifyCodeSurface(surface: CodeSurface, worktreeDir?: string):
|
|
|
3191
2917
|
/** Resolve a code candidate for evaluation only after verifying its immutable
|
|
3192
2918
|
* identity against the checkout at `worktreeRef`. */
|
|
3193
2919
|
declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
//# sourceMappingURL=index-CdEVr8-O.d.ts.map
|
|
2920
|
+
export { summarizeBackendIntegrity as $, RedTeamCategory as $n, ExternalOptimizerObservationArtifact as $r, CrossSurfaceCandidate as $t, RetainedDispatchOptions as A, CampaignRunPlanCell as Ai, BuildLoopProvenanceArgs as An, externalTextOptimizationMethod as Ar, importExternalEvaluations as At, RunProfileMatrixSegmentOptions as B, CampaignCoverage as Bi, emitLoopProvenance as Bn, OptimizationMethodProvenance as Br, LoadEvalFixtureScenariosOptions as Bt, searchModelIdentity as C, RunEvalOptions as Ci, CrossSurfaceSelectionPolicy as Cn, GepaEngineRun as Cr, RolloutCall as Ct, ScenarioSignal as D, RunCampaignOptions as Di, BuildTraceAnalystSurfaceDispatchOptions as Dn, gepaOptimizationMethod as Dr, GepaEvaluationImport as Dt, DiscriminationScore as E, CampaignCellRetryPolicy as Ei, measureCampaignCost as En, GepaRunnerCommand as Er, ExternalSearchOptimizer as Et, FinalizedProfileMatrixResult as F, CellScheduleSlot as Fi, LoopProvenanceCandidate as Fn, ExternalTextEvaluationResponse as Fr, EvalFixtureFile as Ft, ProfileSummary as G, campaignSplitDigestFromIdentities as Gi, HeldOutGateOptions as Gn, OptimizationTokenUsage as Gr, planEvalFixtureRun as Gt, finalizeProfileMatrix as H, campaignCoverage as Hi, CampaignComparisonUnits as Hn, OptimizationMethodRunOptions as Hr, discoverEvalFixtures as Ht, ProfileMatrixCoverage as I, buildCellSchedule as Ii, LoopProvenanceEvidence as In, decodeExternalTextCandidate as Ir, EvalFixtureLoadOptions as It, ScenarioRollup as J, DefaultProductionGateOptions as Jn, GepaCandidatePopulationCandidate as Jr, CrossSurfaceAdditionDecision as Jt, RunProfileMatrixOptions as K, SearchLedgerError as Ki, heldOutGate as Kn, costFromLedgerSummary as Kr, analyzeCrossSurfaceInteractions as Kt, ProfileMatrixPlan as L, cellCachePath as Li, LoopProvenanceOptimizationMethod as Ln, OptimizationMethod as Lr, EvalFixtureRunPlan as Lt, createRetainedDispatch as M, planCampaignRun as Mi, EmitLoopProvenanceResult as Mn, ExternalTextOptimizerContext as Mr, NeutralizationGateOptions as Mt, CreateProfileMatrixPlanOptions as N, CacheIssueReason as Ni, LoopProvenanceArgsFromResult as Nn, ExternalTextOptimizerResult as Nr, neutralizationGate as Nt, scoreDiscrimination as O, runCampaign as Oi, TraceAnalystArtifact as On, OpenAICompatibleOptimizerModel as Or, GepaPopulationImport as Ot, FinalizeProfileMatrixOptions as P, CacheRead as Pi, LoopProvenanceBackend as Pn, ExternalOptimizationExample as Pr, EvalFixture as Pt, assertRealBackend as Q, RedTeamCase as Qn, ExternalOptimizerExecutionSummary as Qr, CrossSurfaceBootstrapPolicy as Qt, ProfileMatrixRow as R, FinalEvidencePolicy as Ri, LoopProvenanceRecord as Rn, OptimizationMethodComposition as Rr, EvalFixtureScenario as Rt, campaignCellSearchResult as S, openAutoPr as Si, CrossSurfaceRelativeCost as Sn, GepaEngineOptions as Sr, RolloutArgumentDiffOptions as St, searchReceiptAccounting as T, CampaignCellFailureReceipt as Ti, CrossSurfaceTaskRow as Tn, GepaOptimizationRecipe as Tr, UngroundedLiteralReport as Tt, runProfileMatrixSegment as U, campaignScenarioIdentity as Ui, campaignBreakdown as Un, OptimizationPackageSource as Ur, loadEvalFixture as Ut, createProfileMatrixPlan as V, assertCampaignDesign as Vi, CampaignBreakdown as Vn, OptimizationMethodResult as Vr, PlanEvalFixtureRunOptions as Vt, ProfileDispatchFn as W, campaignSplitDigest as Wi, campaignMeanComposite as Wn, ComparisonCost as Wr, loadEvalFixtureScenarios as Wt, BackendIntegrityError as X, defaultProductionGate as Xn, GepaCandidateSelectionScore as Xr, CrossSurfaceAttemptCompleteness as Xt, runProfileMatrix as Y, DefaultProductionRewardHackingOptions as Yn, GepaCandidatePopulationSummary as Yr, CrossSurfaceAdditionRejectionReason as Yt, BackendIntegrityReport as Z, DEFAULT_RED_TEAM_CORPUS as Zn, readGepaCandidatePopulationArtifact as Zr, CrossSurfaceBestSingleSelection as Zt, surfaceDispatchRef as _, RunOptimizationOptions as _i, CrossSurfacePairCompatibility as _n, OptimizationMethodComparison as _r, scopedOptimizationMethod as _t, gitWorktreeAdapter as a, ReferenceEquivalenceJudgeResult as ai, CrossSurfaceComponentEvidence as an, CanaryAlert as ar, ScoreboardSummary as at, SingleRunLockOptions as b, OpenAutoPrOptions as bi, CrossSurfacePairwiseEntry as bn, compareOptimizationMethods as br, FsLabeledScenarioStoreOptions as bt, AutoevalsScoreLike as c, LlmJudgeOptions as ci, CrossSurfaceEligibility as cn, CanaryOptions as cr, renderScoreboardMarkdown as ct, PhoenixEvaluatorLike as d, isTransientTransportFailure as di, CrossSurfaceInteractionAwareSelection as dn, composeGate as dr, userStoryScoreboard as dt, ExternalOptimizerObservationSummary as ei, CrossSurfaceCandidateComparison as en, RedTeamFinding as er, PlaybackContext as et, assertCodeSurfaceIdentity as f, transientDispatchFailure as fi, CrossSurfaceInteractionEffect as fn, SkillOptOptimizationMethodConfig as fr, ArtifactEventLike as ft, surfaceContentHash as g, PremeasuredOptimizationBaseline as gi, CrossSurfaceNaiveStackSelection as gn, CompareOptimizationMethodsOptions as gr, extractProducedState as gt, renderSurfaceDiff as h, runImprovementLoop as hi, CrossSurfaceInteractionTask as hn, skillOptOptimizationMethod as hr, ToolCallEventLike as ht, WorktreeAdapter as i, ReferenceEquivalenceJudgeOptions as ii, CrossSurfaceComponent as in, scoreRedTeamOutput as ir, ScoreboardRow as it, RetainedDispatchOutcome as j, PlanCampaignRunOptions as ji, EmitLoopProvenanceArgs as jn, ExternalTextOptimizationMethodConfig as jr, importGepaPopulation as jt, resolveRunDir as k, CampaignRunPlan as ki, TraceAnalystScenario as kn, OptimizerModelBudget as kr, externalSurface as kt, AutoevalsScorerLike as l, llmJudge as li, CrossSurfaceEvidenceBreakdown as ln, CanaryReport as lr, scoreUserStory as lt, componentSurfaceIdentityMaterial as m, RunImprovementLoopResult as mi, CrossSurfaceInteractionReport as mn, SkillOptTrainerConfig as mr, RuntimeEventLike as mt, GitWorktreeAdapterOptions as n, readExternalOptimizerObservationArtifact as ni, CrossSurfaceCandidateOutcome as nn, redTeamDataset as nr, PlaybackStep as nt, resolveWorktreePath as o, ReferenceEquivalenceScenario as oi, CrossSurfaceCompositionStep as on, CanaryEvaluation as or, UserStory as ot, codeSurfaceIdentityMaterial as p, RunImprovementLoopOptions as pi, CrossSurfaceInteractionPath as pn, SkillOptRunnerCommand as pr, ProposalEventLike as pt, RunProfileMatrixResult as q, SearchLedgerIntegrityError as qi, DefaultProductionGateCheck as qn, GepaCandidatePopulationArtifact as qr, AnalyzeCrossSurfaceInteractionsInput as qt, Worktree as r, ReferenceEquivalenceJudgeInput as ri, CrossSurfaceCandidateSummary as rn, redTeamReport as rr, ScoreboardRenderOptions as rt, verifyCodeSurface as s, LlmJudgeDimension as si, CrossSurfaceDistribution as sn, CanaryKind as sr, UserStoryVerdict as st, CodeSurfaceVerification as t, ExternalOptimizerSubmittedCandidate as ti, CrossSurfaceCandidateEvidence as tn, RedTeamReport as tr, PlaybackDriver as tt, PhoenixEvaluationResultLike as u, TransientFailureOptions as ui, CrossSurfaceIneligibilityReason as un, runCanaries as ur, scoreboardSummary as ut, surfaceHash as v, RunOptimizationResult as vi, CrossSurfacePairEvidence as vn, OptimizationMethodPairwise as vr, sequentialOptimizationMethod as vt, searchProposalExecution as w, runEval as wi, CrossSurfaceSelections as wn, GepaOptimizationMethodConfig as wr, ScoredRollout as wt, acquireSingleRunLock as x, OpenAutoPrResult as xi, CrossSurfaceRankedSingle as xn, GepaAdaptiveEngineRun as xr, RolloutArgumentDiff as xt, SingleRunLock as y, runOptimization as yi, CrossSurfacePairIncompatibilityReason as yn, OptimizationMethodScore as yr, neutralizeText as yt, ProfileMatrixSegmentResult as z, FinalEvidenceUse as zi, canonicalDigest as zn, OptimizationMethodInput as zr, EvalFixtureValidationMode as zt };
|
|
2921
|
+
//# sourceMappingURL=index-Bdcpx2s8.d.ts.map
|