@tangle-network/agent-eval 0.221.7 → 0.222.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -0
- package/dist/adapters/http.d.ts +3 -5
- package/dist/adapters/http.d.ts.map +1 -1
- package/dist/adapters/http.js +1 -2
- package/dist/{adversarial-smnADNFS.d.ts → adversarial-DnQQFqTR.d.ts} +1 -3
- package/dist/adversarial-DnQQFqTR.d.ts.map +1 -0
- package/dist/{agent-profile-CNmwfaqq.d.ts → agent-profile-CoD1Dotf.d.ts} +4 -10
- package/dist/agent-profile-CoD1Dotf.d.ts.map +1 -0
- package/dist/agent-profile-Dngk3LZ0.js +153 -0
- package/dist/agent-profile-Dngk3LZ0.js.map +1 -0
- package/dist/{agent-profile-cell-BcpFbNd5.d.ts → agent-profile-cell-3kYxUsK2.d.ts} +3 -5
- package/dist/agent-profile-cell-3kYxUsK2.d.ts.map +1 -0
- package/dist/{agent-profile-cell-B1V1gUZs.js → agent-profile-cell-CT5JYueM.js} +5 -7
- package/dist/{agent-profile-cell-B1V1gUZs.js.map → agent-profile-cell-CT5JYueM.js.map} +1 -1
- package/dist/{agreement-irr-BhAqjluA.js → agreement-irr-BTjDN0i6.js} +4 -5
- package/dist/{agreement-irr-BhAqjluA.js.map → agreement-irr-BTjDN0i6.js.map} +1 -1
- package/dist/analyst/index.d.ts +95 -159
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +28 -39
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DfX6dHP0.d.ts → analyze-runs-C-pP1sK9.d.ts} +7 -13
- package/dist/analyze-runs-C-pP1sK9.d.ts.map +1 -0
- package/dist/{analyze-runs-CHmsykNJ.js → analyze-runs-CTheASYV.js} +23 -32
- package/dist/{analyze-runs-CHmsykNJ.js.map → analyze-runs-CTheASYV.js.map} +1 -1
- package/dist/{attribute-vocabulary-CcXqHODT.d.ts → attribute-vocabulary-5oo8ykIx.d.ts} +2 -4
- package/dist/attribute-vocabulary-5oo8ykIx.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +0 -2
- package/dist/authenticity/index.d.ts.map +1 -1
- package/dist/authenticity/index.js +1 -2
- package/dist/{benchmark-command-Cnjfo0Zh.js → benchmark-command-jYWR2gya.js} +61 -94
- package/dist/benchmark-command-jYWR2gya.js.map +1 -0
- package/dist/benchmarks/index.d.ts +5 -15
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +27 -38
- package/dist/benchmarks/index.js.map +1 -1
- package/dist/{bounded-process-Cj9YjXbE.js → bounded-process-BZrWXIKW.js} +2 -3
- package/dist/{bounded-process-Cj9YjXbE.js.map → bounded-process-BZrWXIKW.js.map} +1 -1
- package/dist/builder-eval/index.d.ts +5 -14
- package/dist/builder-eval/index.d.ts.map +1 -1
- package/dist/builder-eval/index.js +11 -17
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/{calibration-CTwxpJ-t.js → calibration-6usfVYYo.js} +4 -5
- package/dist/{calibration-CTwxpJ-t.js.map → calibration-6usfVYYo.js.map} +1 -1
- package/dist/campaign/index.d.ts +9 -9
- package/dist/campaign/index.js +15 -14
- package/dist/{campaign-uddTlFg5.js → campaign-ddr1N3yD.js} +61 -58
- package/dist/campaign-ddr1N3yD.js.map +1 -0
- package/dist/{campaign-evidence-DunzcJGv.js → campaign-evidence-DN8rWuBj.js} +20 -322
- package/dist/campaign-evidence-DN8rWuBj.js.map +1 -0
- package/dist/{canonical-Dq2hOQ8L.js → canonical-CicJlINA.js} +3 -4
- package/dist/{canonical-Dq2hOQ8L.js.map → canonical-CicJlINA.js.map} +1 -1
- package/dist/{canonical-CFpojCN5.d.ts → canonical-gLeHiqzd.d.ts} +2 -4
- package/dist/canonical-gLeHiqzd.d.ts.map +1 -0
- package/dist/{cell-spend-DEWR3nj-.js → cell-spend-zxmzg4GH.js} +3 -4
- package/dist/{cell-spend-DEWR3nj-.js.map → cell-spend-zxmzg4GH.js.map} +1 -1
- package/dist/{chat-client-BNATB3rJ.js → chat-client-CO3uB33i.js} +3 -4
- package/dist/{chat-client-BNATB3rJ.js.map → chat-client-CO3uB33i.js.map} +1 -1
- package/dist/{chat-json-call-BzWYp18J.js → chat-json-call-8yUv4LFO.js} +4 -5
- package/dist/{chat-json-call-BzWYp18J.js.map → chat-json-call-8yUv4LFO.js.map} +1 -1
- package/dist/{claim-BPVuyHuy.js → claim-Dod_mjk3.js} +5 -6
- package/dist/{claim-BPVuyHuy.js.map → claim-Dod_mjk3.js.map} +1 -1
- package/dist/cli.js +179 -23
- package/dist/cli.js.map +1 -1
- package/dist/{client-hwo3cJjR.d.ts → client-BIFwUg7I.d.ts} +6 -12
- package/dist/client-BIFwUg7I.d.ts.map +1 -0
- package/dist/{cluster-bootstrap-DEwwhl0C.js → cluster-bootstrap-SbaeM44s.js} +4 -5
- package/dist/{cluster-bootstrap-DEwwhl0C.js.map → cluster-bootstrap-SbaeM44s.js.map} +1 -1
- package/dist/completion-verifier-7KEfroqx.d.ts +168 -0
- package/dist/completion-verifier-7KEfroqx.d.ts.map +1 -0
- package/dist/{completion-verifier-asaV1nSj.js → completion-verifier-rqbMh5-4.js} +9 -160
- package/dist/completion-verifier-rqbMh5-4.js.map +1 -0
- package/dist/contract/index.d.ts +13 -13
- package/dist/contract/index.js +41 -60
- package/dist/contract/index.js.map +1 -1
- package/dist/{cost-ledger-BGZB6yY2.js → cost-ledger-CS8R3IZj.js} +10 -17
- package/dist/{cost-ledger-BGZB6yY2.js.map → cost-ledger-CS8R3IZj.js.map} +1 -1
- package/dist/{cost-ledger-C1W3ICmP.d.ts → cost-ledger-Czro9wmo.d.ts} +1 -5
- package/dist/cost-ledger-Czro9wmo.d.ts.map +1 -0
- package/dist/{dataset-DQqhOCPt.d.ts → dataset-DThC1S6G.d.ts} +1 -3
- package/dist/dataset-DThC1S6G.d.ts.map +1 -0
- package/dist/{decision-point-0nJubuoM.js → decision-point-lhMgODOR.js} +14 -18
- package/dist/{decision-point-0nJubuoM.js.map → decision-point-lhMgODOR.js.map} +1 -1
- package/dist/{deep-freeze-CurcOQg3.js → deep-freeze-CK8-2W7p.js} +2 -3
- package/dist/{deep-freeze-CurcOQg3.js.map → deep-freeze-CK8-2W7p.js.map} +1 -1
- package/dist/{default-registry-CLG5YqGS.d.ts → default-registry-B1DstPYg.d.ts} +53 -61
- package/dist/default-registry-B1DstPYg.d.ts.map +1 -0
- package/dist/{default-registry-BBTDgkDM.js → default-registry-C1tPEjZ4.js} +29 -45
- package/dist/{default-registry-BBTDgkDM.js.map → default-registry-C1tPEjZ4.js.map} +1 -1
- package/dist/{define-agent-eval-CCbT7Kfr.js → define-agent-eval-BLoP4Lf9.js} +15 -18
- package/dist/{define-agent-eval-CCbT7Kfr.js.map → define-agent-eval-BLoP4Lf9.js.map} +1 -1
- package/dist/{define-agent-eval-JKt3gMQr.d.ts → define-agent-eval-CAffiFbN.d.ts} +19 -27
- package/dist/define-agent-eval-CAffiFbN.d.ts.map +1 -0
- package/dist/{descriptive-CVNy48rR.js → descriptive-cPwITVfy.js} +30 -8
- package/dist/descriptive-cPwITVfy.js.map +1 -0
- package/dist/diagnosis/index.d.ts +5 -15
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +13 -19
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-DN1p_5jx.js → dspy-rlm-engine-jDiWkul6.js} +7 -10
- package/dist/{dspy-rlm-engine-DN1p_5jx.js.map → dspy-rlm-engine-jDiWkul6.js.map} +1 -1
- package/dist/{emitter-DbMXUIVP.js → emitter-DhQ65mza.js} +3 -4
- package/dist/{emitter-DbMXUIVP.js.map → emitter-DhQ65mza.js.map} +1 -1
- package/dist/{emitter-DABYYTcH.d.ts → emitter-mAXC5PSN.d.ts} +3 -5
- package/dist/emitter-mAXC5PSN.d.ts.map +1 -0
- package/dist/{engine-dtRVeuQs.d.ts → engine-CaHaCc3W.d.ts} +10 -18
- package/dist/engine-CaHaCc3W.d.ts.map +1 -0
- package/dist/{equal-terms-DbPGcVtI.js → equal-terms-K11VHmR3.js} +3 -4
- package/dist/{equal-terms-DbPGcVtI.js.map → equal-terms-K11VHmR3.js.map} +1 -1
- package/dist/{errors-BcOs5MKW.js → errors-DeEEV9Vx.js} +2 -3
- package/dist/{errors-BcOs5MKW.js.map → errors-DeEEV9Vx.js.map} +1 -1
- package/dist/{errors-DEE6u6ot.d.ts → errors-E9I2hldP.d.ts} +1 -3
- package/dist/errors-E9I2hldP.d.ts.map +1 -0
- package/dist/{eval-campaign-DoX5vz3E.js → eval-campaign-Cgixrvf7.js} +8 -9
- package/dist/{eval-campaign-DoX5vz3E.js.map → eval-campaign-Cgixrvf7.js.map} +1 -1
- package/dist/{evaluation-wdGyZ7jV.js → evaluation-B8H3GYej.js} +4 -6
- package/dist/{evaluation-wdGyZ7jV.js.map → evaluation-B8H3GYej.js.map} +1 -1
- package/dist/{evaluation-DblGquCB.d.ts → evaluation-BPqUvZ66.d.ts} +4 -6
- package/dist/evaluation-BPqUvZ66.d.ts.map +1 -0
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -2
- package/dist/{exact-types-Do63KoKn.d.ts → exact-types-D98BKpxo.d.ts} +8 -10
- package/dist/exact-types-D98BKpxo.d.ts.map +1 -0
- package/dist/{exec-CpbtOXVt.js → exec-C0_AkLUv.js} +3 -5
- package/dist/{exec-CpbtOXVt.js.map → exec-C0_AkLUv.js.map} +1 -1
- package/dist/{execution-tracks-CpgFPpS5.js → execution-tracks-DaDXYwAq.js} +2 -3
- package/dist/{execution-tracks-CpgFPpS5.js.map → execution-tracks-DaDXYwAq.js.map} +1 -1
- package/dist/experiment/index.d.ts +9 -37
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +29 -40
- package/dist/experiment/index.js.map +1 -1
- package/dist/{exporters-Dk3oEbK_.d.ts → exporters-B5Ze_RKh.d.ts} +4 -4
- package/dist/exporters-B5Ze_RKh.d.ts.map +1 -0
- package/dist/{exporters-DWvM16LE.js → exporters-ByLcILYU.js} +23 -5
- package/dist/exporters-ByLcILYU.js.map +1 -0
- package/dist/{external-optimizer-process-DEzJ2CGX.js → external-optimizer-process-BRVmaDFe.js} +5 -6
- package/dist/{external-optimizer-process-DEzJ2CGX.js.map → external-optimizer-process-BRVmaDFe.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-B_ukPtRD.js → external-optimizer-subprocess-U616hV_l.js} +24 -37
- package/dist/{external-optimizer-subprocess-B_ukPtRD.js.map → external-optimizer-subprocess-U616hV_l.js.map} +1 -1
- package/dist/{extract-usage-BZLU01kR.js → extract-usage-ISFqhfDz.js} +2 -3
- package/dist/{extract-usage-BZLU01kR.js.map → extract-usage-ISFqhfDz.js.map} +1 -1
- package/dist/{failure-cluster-DN0t63Z0.d.ts → failure-cluster-DOoa7CBg.d.ts} +3 -7
- package/dist/failure-cluster-DOoa7CBg.d.ts.map +1 -0
- package/dist/{failure-taxonomy-Bx5oWqaW.js → failure-taxonomy-9eQfXKoW.js} +2 -3
- package/dist/{failure-taxonomy-Bx5oWqaW.js.map → failure-taxonomy-9eQfXKoW.js.map} +1 -1
- package/dist/{feedback-trajectory-CYVJXolG.d.ts → feedback-trajectory-SBGncrCP.d.ts} +7 -13
- package/dist/feedback-trajectory-SBGncrCP.d.ts.map +1 -0
- package/dist/{finding-composition-8n0fFgVI.js → finding-composition-DlvE1Qul.js} +4 -6
- package/dist/{finding-composition-8n0fFgVI.js.map → finding-composition-DlvE1Qul.js.map} +1 -1
- package/dist/{findings-store-Bg4sdS-e.js → findings-store-B75Dspaq.js} +11 -14
- package/dist/{findings-store-Bg4sdS-e.js.map → findings-store-B75Dspaq.js.map} +1 -1
- package/dist/fuzz.d.ts +3 -17
- package/dist/fuzz.d.ts.map +1 -1
- package/dist/fuzz.js +30 -29
- package/dist/fuzz.js.map +1 -1
- package/dist/{hf-dataset-CKxIJQba.js → hf-dataset-DPVQXW5n.js} +362 -21
- package/dist/hf-dataset-DPVQXW5n.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -6
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +1 -2
- package/dist/{index-CalAsXMR.d.ts → index-B1Bm9oJA.d.ts} +3 -15
- package/dist/index-B1Bm9oJA.d.ts.map +1 -0
- package/dist/{index-CdEVr8-O.d.ts → index-B7v2QHlD.d.ts} +24 -299
- package/dist/index-B7v2QHlD.d.ts.map +1 -0
- package/dist/{index-CTIRz-AB.d.ts → index-CUjeTDPv.d.ts} +14 -36
- package/dist/index-CUjeTDPv.d.ts.map +1 -0
- package/dist/{index-CfJULcW5.d.ts → index-DAk2quYu.d.ts} +18 -38
- package/dist/index-DAk2quYu.d.ts.map +1 -0
- package/dist/{index-D7fmC-1K.d.ts → index-DUn0GNiZ.d.ts} +183 -36
- package/dist/index-DUn0GNiZ.d.ts.map +1 -0
- package/dist/{index-CHxwfW5g.d.ts → index-DwUGehqX.d.ts} +3 -9
- package/dist/index-DwUGehqX.d.ts.map +1 -0
- package/dist/{index-jnuFLxOs.d.ts → index-aDPVYBIM.d.ts} +11 -39
- package/dist/index-aDPVYBIM.d.ts.map +1 -0
- package/dist/index.d.ts +55 -188
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +122 -195
- package/dist/index.js.map +1 -1
- package/dist/{integrity-D-s2iHiT.js → integrity-B1V6klyO.js} +12 -363
- package/dist/integrity-B1V6klyO.js.map +1 -0
- package/dist/{integrity-D9lBnv_K.d.ts → integrity-BAdVzT8u.d.ts} +4 -6
- package/dist/integrity-BAdVzT8u.d.ts.map +1 -0
- package/dist/{integrity-W4CADcFc.js → integrity-DOHt1mwU.js} +3 -4
- package/dist/{integrity-W4CADcFc.js.map → integrity-DOHt1mwU.js.map} +1 -1
- package/dist/{internal-BTK91Bqt.js → internal-v6XfOeJW.js} +6 -8
- package/dist/{internal-BTK91Bqt.js.map → internal-v6XfOeJW.js.map} +1 -1
- package/dist/{interval-Dq9gt7jx.js → interval-Bd2f4CRT.js} +6 -8
- package/dist/{interval-Dq9gt7jx.js.map → interval-Bd2f4CRT.js.map} +1 -1
- package/dist/{interval-ohuOJurX.d.ts → interval-CWyxqDpE.d.ts} +1 -5
- package/dist/interval-CWyxqDpE.d.ts.map +1 -0
- package/dist/{journal-C7om8Gum.js → journal-C1uLJi0w.js} +115 -121
- package/dist/{journal-C7om8Gum.js.map → journal-C1uLJi0w.js.map} +1 -1
- package/dist/{json-BhdsJaOQ.js → json-DtvZNAU9.js} +4 -6
- package/dist/{json-BhdsJaOQ.js.map → json-DtvZNAU9.js.map} +1 -1
- package/dist/json-recovery-DK4MDm5r.js +296 -0
- package/dist/json-recovery-DK4MDm5r.js.map +1 -0
- package/dist/{judge-calibration-BkXMIWA4.js → judge-calibration-DGqlfWW8.js} +5 -6
- package/dist/{judge-calibration-BkXMIWA4.js.map → judge-calibration-DGqlfWW8.js.map} +1 -1
- package/dist/{judge-calibration-DFtEMlde.d.ts → judge-calibration-DnrbQaXI.d.ts} +1 -3
- package/dist/judge-calibration-DnrbQaXI.d.ts.map +1 -0
- package/dist/{judge-gate-BVvmP00P.js → judge-gate-C2BCOt-M.js} +8 -12
- package/dist/{judge-gate-BVvmP00P.js.map → judge-gate-C2BCOt-M.js.map} +1 -1
- package/dist/judges-BLxg5p8E.js +20 -0
- package/dist/judges-BLxg5p8E.js.map +1 -0
- package/dist/{kind-factory-Dl0PmNQL.js → kind-factory-VKysp1ds.js} +23 -39
- package/dist/{kind-factory-Dl0PmNQL.js.map → kind-factory-VKysp1ds.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +3 -3
- package/dist/ledger-core/lock-holder.d.ts +1 -1
- package/dist/ledger-core/lock-holder.js +1 -2
- package/dist/{ledger-core-CAxUdDk0.js → ledger-core-CfT-FzPf.js} +5 -6
- package/dist/{ledger-core-CAxUdDk0.js.map → ledger-core-CfT-FzPf.js.map} +1 -1
- package/dist/{llm-client-b1ysZVIU.js → llm-client-kJ9EOqjt.js} +22 -30
- package/dist/{llm-client-b1ysZVIU.js.map → llm-client-kJ9EOqjt.js.map} +1 -1
- package/dist/{llm-judge-Cybq29bL.js → llm-judge-cgbhTKQp.js} +108 -164
- package/dist/llm-judge-cgbhTKQp.js.map +1 -0
- package/dist/{lock-holder-DaJbfpci.d.ts → lock-holder-DoxlZFkQ.d.ts} +1 -3
- package/dist/lock-holder-DoxlZFkQ.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -3
- package/dist/matrix/index.js +2 -2
- package/dist/{matrix-DtkZLD_3.d.ts → matrix-CBUzIZs4.d.ts} +4 -12
- package/dist/matrix-CBUzIZs4.d.ts.map +1 -0
- package/dist/{matrix-B8l-Nsxd.js → matrix-CgpX8wdm.js} +8 -13
- package/dist/matrix-CgpX8wdm.js.map +1 -0
- package/dist/{mean-BbA_JCzW.js → mean-CdDYG-us.js} +6 -9
- package/dist/{mean-BbA_JCzW.js.map → mean-CdDYG-us.js.map} +1 -1
- package/dist/meta-eval/index.d.ts +3 -3
- package/dist/meta-eval/index.js +17 -21
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{metrics-BKaPBtIG.js → metrics-DNCAUtuB.js} +2 -3
- package/dist/{metrics-BKaPBtIG.js.map → metrics-DNCAUtuB.js.map} +1 -1
- package/dist/{mint-CbdwjTPW.js → mint-D9ZysYtY.js} +7 -8
- package/dist/{mint-CbdwjTPW.js.map → mint-D9ZysYtY.js.map} +1 -1
- package/dist/{multi-layer-verifier-DEgMUk6v.d.ts → multi-layer-verifier-lM4aVN78.d.ts} +2 -4
- package/dist/multi-layer-verifier-lM4aVN78.d.ts.map +1 -0
- package/dist/{multiplicity-Sg0U7h8j.js → multiplicity-DAd4xaDC.js} +3 -4
- package/dist/{multiplicity-Sg0U7h8j.js.map → multiplicity-DAd4xaDC.js.map} +1 -1
- package/dist/{multiplicity-DIWHvysC.d.ts → multiplicity-D_SEsc5j.d.ts} +1 -3
- package/dist/multiplicity-D_SEsc5j.d.ts.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -17
- package/dist/multishot/golden/index.d.ts.map +1 -1
- package/dist/multishot/golden/index.js +10 -17
- package/dist/multishot/golden/index.js.map +1 -1
- package/dist/multishot/index.d.ts +2 -10
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +11 -18
- package/dist/multishot/index.js.map +1 -1
- package/dist/{nonnegative-number-DvdMxg_f.js → nonnegative-number-lEYbOsuh.js} +2 -3
- package/dist/{nonnegative-number-DvdMxg_f.js.map → nonnegative-number-lEYbOsuh.js.map} +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{otel-spans-CdN909k1.js → otel-spans-BsGzTWIK.js} +5 -6
- package/dist/{otel-spans-CdN909k1.js.map → otel-spans-BsGzTWIK.js.map} +1 -1
- package/dist/{otlp-attributes-D9ptWaIJ.js → otlp-attributes-Boeb7gtH.js} +2 -3
- package/dist/{otlp-attributes-D9ptWaIJ.js.map → otlp-attributes-Boeb7gtH.js.map} +1 -1
- package/dist/{outcome-observations-FLmRpgYa.js → outcome-observations-D4snOtUb.js} +4 -5
- package/dist/{outcome-observations-FLmRpgYa.js.map → outcome-observations-D4snOtUb.js.map} +1 -1
- package/dist/{outcome-store-DUcMj-3D.js → outcome-store-TarrcfHG.js} +2 -3
- package/dist/{outcome-store-DUcMj-3D.js.map → outcome-store-TarrcfHG.js.map} +1 -1
- package/dist/{package-version-D7lQHt_-.js → package-version-CJl_pY9W.js} +2 -3
- package/dist/{package-version-D7lQHt_-.js.map → package-version-CJl_pY9W.js.map} +1 -1
- package/dist/{paired-arms-B2ttLc0_.js → paired-arms-D-54hO-k.js} +10 -14
- package/dist/{paired-arms-B2ttLc0_.js.map → paired-arms-D-54hO-k.js.map} +1 -1
- package/dist/{paired-binary-B_nejA44.js → paired-binary-BR0nwHOA.js} +3 -4
- package/dist/{paired-binary-B_nejA44.js.map → paired-binary-BR0nwHOA.js.map} +1 -1
- package/dist/{paired-promotion-decision-CQMJ5xGN.d.ts → paired-promotion-decision-BDnqepZ1.d.ts} +1 -5
- package/dist/paired-promotion-decision-BDnqepZ1.d.ts.map +1 -0
- package/dist/{paired-promotion-decision-D0SHb-ZL.js → paired-promotion-decision-FGvHA4p_.js} +5 -7
- package/dist/{paired-promotion-decision-D0SHb-ZL.js.map → paired-promotion-decision-FGvHA4p_.js.map} +1 -1
- package/dist/{pareto-BiBnQXAE.js → pareto-B9udAt2i.js} +28 -50
- package/dist/pareto-B9udAt2i.js.map +1 -0
- package/dist/pipelines/index.d.ts +5 -11
- package/dist/pipelines/index.d.ts.map +1 -1
- package/dist/pipelines/index.js +6 -8
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{power-and-mde-CQX-0weG.js → power-and-mde-ClI6RWw4.js} +4 -5
- package/dist/{power-and-mde-CQX-0weG.js.map → power-and-mde-ClI6RWw4.js.map} +1 -1
- package/dist/{predicate-FdKmElA_.js → predicate-BYHA4MBn.js} +2 -3
- package/dist/{predicate-FdKmElA_.js.map → predicate-BYHA4MBn.js.map} +1 -1
- package/dist/{prime-bridge-transport-6feEglLf.d.ts → prime-bridge-transport-Cfqir_Im.d.ts} +1 -3
- package/dist/prime-bridge-transport-Cfqir_Im.d.ts.map +1 -0
- package/dist/{prime-bridge-transport-Z3cpKGoV.js → prime-bridge-transport-Csd5SQg7.js} +2 -3
- package/dist/{prime-bridge-transport-Z3cpKGoV.js.map → prime-bridge-transport-Csd5SQg7.js.map} +1 -1
- package/dist/{prime-protocol-BGxjfhM4.d.ts → prime-protocol-COFkxas6.d.ts} +4 -8
- package/dist/prime-protocol-COFkxas6.d.ts.map +1 -0
- package/dist/{prime-protocol-BFgGrEMP.js → prime-protocol-CrhpXRz9.js} +4 -6
- package/dist/{prime-protocol-BFgGrEMP.js.map → prime-protocol-CrhpXRz9.js.map} +1 -1
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -2
- package/dist/{promotion-gate-CkVZHagC.d.ts → promotion-gate-CixshGI8.d.ts} +5 -11
- package/dist/promotion-gate-CixshGI8.d.ts.map +1 -0
- package/dist/{promotion-policy-6A8osGr_.js → promotion-policy-CBYeD4zi.js} +5 -6
- package/dist/{promotion-policy-6A8osGr_.js.map → promotion-policy-CBYeD4zi.js.map} +1 -1
- package/dist/{proposal-findings-DqLUWsAG.js → proposal-findings-WFQrxnfe.js} +3 -4
- package/dist/{proposal-findings-DqLUWsAG.js.map → proposal-findings-WFQrxnfe.js.map} +1 -1
- package/dist/{quantile--0yb07pD.js → quantile-CncXP-Ss.js} +2 -3
- package/dist/{quantile--0yb07pD.js.map → quantile-CncXP-Ss.js.map} +1 -1
- package/dist/{query-B_RZ36an.js → query-CGXzO9d4.js} +5 -6
- package/dist/{query-B_RZ36an.js.map → query-CGXzO9d4.js.map} +1 -1
- package/dist/{query-Cmv_U8vf.d.ts → query-UD0B08Sb.d.ts} +4 -6
- package/dist/query-UD0B08Sb.d.ts.map +1 -0
- package/dist/{random-DGEyr3zG.d.ts → random-BN150Q_X.d.ts} +1 -5
- package/dist/random-BN150Q_X.d.ts.map +1 -0
- package/dist/{random-DDDcC7YU.js → random-DrzcgC7F.js} +3 -4
- package/dist/{random-DDDcC7YU.js.map → random-DrzcgC7F.js.map} +1 -1
- package/dist/{raw-provider-sink-75w4FWBj.js → raw-provider-sink-DDLkP_eF.js} +3 -4
- package/dist/{raw-provider-sink-75w4FWBj.js.map → raw-provider-sink-DDLkP_eF.js.map} +1 -1
- package/dist/{record-id-DUgsK5qp.js → record-id-CWZXFh58.js} +2 -3
- package/dist/{record-id-DUgsK5qp.js.map → record-id-CWZXFh58.js.map} +1 -1
- package/dist/{redact-DXdlFb_U.d.ts → redact-C_wzC4c_.d.ts} +1 -3
- package/dist/redact-C_wzC4c_.d.ts.map +1 -0
- package/dist/{redact-mtkTiJVk.js → redact-D6APJS1d.js} +3 -5
- package/dist/{redact-mtkTiJVk.js.map → redact-D6APJS1d.js.map} +1 -1
- package/dist/{registry-BcdJmyoU.d.ts → registry-dhdDoAV8.d.ts} +5 -11
- package/dist/registry-dhdDoAV8.d.ts.map +1 -0
- package/dist/{release-confidence-CYIFH-fp.js → release-confidence-CTUJz5mM.js} +10 -16
- package/dist/release-confidence-CTUJz5mM.js.map +1 -0
- package/dist/{render-BhJqXoBH.js → render-pfAZ7IR4.js} +266 -33
- package/dist/render-pfAZ7IR4.js.map +1 -0
- package/dist/{reporting-html-Kvlc8oMt.js → reporting-html-DMXIjATy.js} +2 -3
- package/dist/{reporting-html-Kvlc8oMt.js.map → reporting-html-DMXIjATy.js.map} +1 -1
- package/dist/reporting.d.ts +6 -6
- package/dist/reporting.js +7 -7
- package/dist/{researcher-DKWtoMWk.d.ts → researcher-CLnNJ7Or.d.ts} +8 -12
- package/dist/researcher-CLnNJ7Or.d.ts.map +1 -0
- package/dist/{reward-nw2xZGZG.js → reward-P0VD_Aft.js} +2 -3
- package/dist/{reward-nw2xZGZG.js.map → reward-P0VD_Aft.js.map} +1 -1
- package/dist/{reward-BeUgVVC_.d.ts → reward-ST4PXNXF.d.ts} +2 -4
- package/dist/reward-ST4PXNXF.d.ts.map +1 -0
- package/dist/{reward-hacking-WTxUbtbb.d.ts → reward-hacking-C9hPVT0Q.d.ts} +4 -8
- package/dist/reward-hacking-C9hPVT0Q.d.ts.map +1 -0
- package/dist/{reward-hacking-ByuETaIb.js → reward-hacking-DZzHZlS1.js} +5 -7
- package/dist/{reward-hacking-ByuETaIb.js.map → reward-hacking-DZzHZlS1.js.map} +1 -1
- package/dist/rl.d.ts +26 -63
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +35 -52
- package/dist/rl.js.map +1 -1
- package/dist/{rolldown-runtime-8H4AJuhK.js → rolldown-runtime-BJohB4LM.js} +1 -2
- package/dist/rollout/index.d.ts +6 -6
- package/dist/rollout/index.js +7 -7
- package/dist/{rollout-Dd1ucT_M.js → rollout-BhkMpAJT.js} +279 -16
- package/dist/rollout-BhkMpAJT.js.map +1 -0
- package/dist/{terminal-record-CB9BFtjS.js → rollout-nodes-BHR9Q5jP.js} +351 -11
- package/dist/rollout-nodes-BHR9Q5jP.js.map +1 -0
- package/dist/{round-decimal-COrKglMh.js → round-decimal-ZmjAQPo1.js} +2 -3
- package/dist/{round-decimal-COrKglMh.js.map → round-decimal-ZmjAQPo1.js.map} +1 -1
- package/dist/{rubric-predictive-validity-Cd5SCtAf.d.ts → rubric-predictive-validity-B3RzbwOx.d.ts} +2 -8
- package/dist/rubric-predictive-validity-B3RzbwOx.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-CeKW2GQN.js → rubric-predictive-validity-CQp_M0vy.js} +3 -4
- package/dist/{rubric-predictive-validity-CeKW2GQN.js.map → rubric-predictive-validity-CQp_M0vy.js.map} +1 -1
- package/dist/{run-record-CysKLB93.js → run-record-CZy4-TnZ.js} +4 -5
- package/dist/{run-record-CysKLB93.js.map → run-record-CZy4-TnZ.js.map} +1 -1
- package/dist/{run-record-C3awiXPf.d.ts → run-record-D2BFMPGl.d.ts} +5 -7
- package/dist/run-record-D2BFMPGl.d.ts.map +1 -0
- package/dist/{run-record-DhEJgoPz.js → run-record-DEjx43cI.js} +6 -7
- package/dist/{run-record-DhEJgoPz.js.map → run-record-DEjx43cI.js.map} +1 -1
- package/dist/{sandbox-harness-BlSOu4LX.d.ts → sandbox-harness-B3Kx1fOA.d.ts} +1 -3
- package/dist/sandbox-harness-B3Kx1fOA.d.ts.map +1 -0
- package/dist/{schema-D6EVKzTB.d.ts → schema-CkIvnmLU.d.ts} +2 -43
- package/dist/schema-CkIvnmLU.d.ts.map +1 -0
- package/dist/{schema-Bt2vLFm3.js → schema-D-xu3MU_.js} +2 -3
- package/dist/{schema-Bt2vLFm3.js.map → schema-D-xu3MU_.js.map} +1 -1
- package/dist/{schema-C-ziyJD7.d.ts → schema-DkFJr3Y8.d.ts} +111 -48
- package/dist/schema-DkFJr3Y8.d.ts.map +1 -0
- package/dist/{schema-CvhfLjtZ.js → schema-YUIXT8iW.js} +37 -7
- package/dist/schema-YUIXT8iW.js.map +1 -0
- package/dist/search/index.d.ts +3 -35
- package/dist/search/index.d.ts.map +1 -1
- package/dist/search/index.js +4 -5
- package/dist/{search-kernel-CctpiDd2.d.ts → search-kernel-CBdwFZ05.d.ts} +10 -36
- package/dist/search-kernel-CBdwFZ05.d.ts.map +1 -0
- package/dist/{search-ledger-CZRqtFbW.js → search-ledger-BsVi-Ngv.js} +17 -26
- package/dist/{search-ledger-CZRqtFbW.js.map → search-ledger-BsVi-Ngv.js.map} +1 -1
- package/dist/{search-ledger-file-Bd5Tioq6.js → search-ledger-file-DTBD6X-y.js} +6 -8
- package/dist/{search-ledger-file-Bd5Tioq6.js.map → search-ledger-file-DTBD6X-y.js.map} +1 -1
- package/dist/{search-ledger-types-p2xKfiDJ.d.ts → search-ledger-types-qiYRUo_N.d.ts} +5 -9
- package/dist/search-ledger-types-qiYRUo_N.d.ts.map +1 -0
- package/dist/{search-shipper-Dn3NLsuH.js → search-shipper-pdg4viK7.js} +8 -13
- package/dist/{search-shipper-Dn3NLsuH.js.map → search-shipper-pdg4viK7.js.map} +1 -1
- package/dist/{search-summary-D4bwYnAE.js → search-summary-Ct7SrmVc.js} +6 -7
- package/dist/{search-summary-D4bwYnAE.js.map → search-summary-Ct7SrmVc.js.map} +1 -1
- package/dist/{sequential-CimQrbjv.js → sequential-B1I0tRfL.js} +6 -7
- package/dist/{sequential-CimQrbjv.js.map → sequential-B1I0tRfL.js.map} +1 -1
- package/dist/{sequential-KwGspfeR.d.ts → sequential-Bb_pdTqG.d.ts} +1 -3
- package/dist/sequential-Bb_pdTqG.d.ts.map +1 -0
- package/dist/{sequential-BVv1sjsw.js → sequential-Bjs5roiK.js} +2 -3
- package/dist/{sequential-BVv1sjsw.js.map → sequential-Bjs5roiK.js.map} +1 -1
- package/dist/{sequential-eprocess-DodI6RoF.js → sequential-eprocess-C1YLwFaz.js} +3 -4
- package/dist/{sequential-eprocess-DodI6RoF.js.map → sequential-eprocess-C1YLwFaz.js.map} +1 -1
- package/dist/{server-ChhXPCgX.js → server-DA6h3co5.js} +11 -17
- package/dist/{server-ChhXPCgX.js.map → server-DA6h3co5.js.map} +1 -1
- package/dist/{single-run-lock-Dekd8_Wv.js → single-run-lock-BGlW3OSk.js} +3 -4
- package/dist/{single-run-lock-Dekd8_Wv.js.map → single-run-lock-BGlW3OSk.js.map} +1 -1
- package/dist/{skillopt-optimization-method-sB_-yi1G.js → skillopt-optimization-method-qB1M554K.js} +38 -57
- package/dist/{skillopt-optimization-method-sB_-yi1G.js.map → skillopt-optimization-method-qB1M554K.js.map} +1 -1
- package/dist/{spans-CVngGij3.d.ts → spans-vZoW__3q.d.ts} +2 -4
- package/dist/spans-vZoW__3q.d.ts.map +1 -0
- package/dist/{statistical-heldout-BjzSOmLH.d.ts → statistical-heldout-Cei036nE.d.ts} +9 -37
- package/dist/statistical-heldout-Cei036nE.d.ts.map +1 -0
- package/dist/{statistical-heldout-Bksv92Th.js → statistical-heldout-TP6ndL0B.js} +4 -5
- package/dist/{statistical-heldout-Bksv92Th.js.map → statistical-heldout-TP6ndL0B.js.map} +1 -1
- package/dist/{steps-DlepJPwD.d.ts → steps-B8fYY_Vo.d.ts} +1 -3
- package/dist/steps-B8fYY_Vo.d.ts.map +1 -0
- package/dist/{store-B3nWRy3B.d.ts → store-CMs5wGXP.d.ts} +2 -4
- package/dist/store-CMs5wGXP.d.ts.map +1 -0
- package/dist/{store-BhNuLdJw.js → store-ZykIyQmy.js} +2 -3
- package/dist/{store-BhNuLdJw.js.map → store-ZykIyQmy.js.map} +1 -1
- package/dist/{store-otlp-CFf27E8_.js → store-otlp-DmLm65XW.js} +8 -13
- package/dist/{store-otlp-CFf27E8_.js.map → store-otlp-DmLm65XW.js.map} +1 -1
- package/dist/{store-tool-spans-DxsFwtdr.d.ts → store-tool-spans-B3i3fl0A.d.ts} +11 -29
- package/dist/store-tool-spans-B3i3fl0A.d.ts.map +1 -0
- package/dist/{store-tool-spans-BK_8FjGG.js → store-tool-spans-C2g8UYm-.js} +17 -41
- package/dist/{store-tool-spans-BK_8FjGG.js.map → store-tool-spans-C2g8UYm-.js.map} +1 -1
- package/dist/storyboard/index.d.ts +1 -5
- package/dist/storyboard/index.d.ts.map +1 -1
- package/dist/storyboard/index.js +3 -5
- package/dist/storyboard/index.js.map +1 -1
- package/dist/{summary-report-C8V49Jt3.d.ts → summary-report-0La5vejY.d.ts} +5 -13
- package/dist/summary-report-0La5vejY.d.ts.map +1 -0
- package/dist/{summary-report-DHDWxe6p.js → summary-report-CPfsrLac.js} +21 -30
- package/dist/summary-report-CPfsrLac.js.map +1 -0
- package/dist/supervisor-run/index.d.ts +30 -29
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +6 -7
- package/dist/{systemone-CHnqwmYg.d.ts → systemone-Cw2txK5m.d.ts} +12 -24
- package/dist/systemone-Cw2txK5m.d.ts.map +1 -0
- package/dist/{systemone-protocol-Bl2EI2KQ.d.ts → systemone-protocol-C4t7JH4Y.d.ts} +1 -3
- package/dist/systemone-protocol-C4t7JH4Y.d.ts.map +1 -0
- package/dist/systemone-protocol.d.ts +1 -1
- package/dist/systemone-protocol.js +2 -3
- package/dist/{systemone-review-cQ9acAsb.d.ts → systemone-review-BLgxt-4Q.d.ts} +8 -10
- package/dist/systemone-review-BLgxt-4Q.d.ts.map +1 -0
- package/dist/{systemone-review-DFGtIUcf.js → systemone-review-S49gsy_y.js} +5 -6
- package/dist/{systemone-review-DFGtIUcf.js.map → systemone-review-S49gsy_y.js.map} +1 -1
- package/dist/systemone.d.ts +4 -4
- package/dist/systemone.js +3 -3
- package/dist/{task-failure-attributes-BMsblaDh.js → task-failure-attributes-CC6VuIpk.js} +12 -11
- package/dist/task-failure-attributes-CC6VuIpk.js.map +1 -0
- package/dist/{tool-groups-b3oU_D88.d.ts → tool-groups-C3K-d3gd.d.ts} +3 -5
- package/dist/tool-groups-C3K-d3gd.d.ts.map +1 -0
- package/dist/{tool-waste-CkjzWmeY.d.ts → tool-waste-BFG1vwTV.d.ts} +4 -16
- package/dist/tool-waste-BFG1vwTV.d.ts.map +1 -0
- package/dist/{tool-waste-DLsjS_p-.js → tool-waste-CLJ46AbT.js} +10 -26
- package/dist/{tool-waste-DLsjS_p-.js.map → tool-waste-CLJ46AbT.js.map} +1 -1
- package/dist/trace-attributes.d.ts +1 -1
- package/dist/trace-attributes.js +1 -2
- package/dist/{trace-questions-BmT8M6UQ.d.ts → trace-questions-ej5AEJ5X.d.ts} +4 -6
- package/dist/trace-questions-ej5AEJ5X.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +46 -92
- package/dist/trace-repair/index.d.ts.map +1 -1
- package/dist/trace-repair/index.js +32 -54
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +13 -29
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +18 -22
- package/dist/traces.js.map +1 -1
- package/dist/{trajectory-D_7rLrvE.js → trajectory-O3GDHmev.js} +2 -3
- package/dist/{trajectory-D_7rLrvE.js.map → trajectory-O3GDHmev.js.map} +1 -1
- package/dist/trajectory-replay/index.d.ts +2 -22
- package/dist/trajectory-replay/index.d.ts.map +1 -1
- package/dist/trajectory-replay/index.js +19 -30
- package/dist/trajectory-replay/index.js.map +1 -1
- package/dist/{tree-DSqwJpL9.js → tree-DrIChF4a.js} +39 -68
- package/dist/{tree-DSqwJpL9.js.map → tree-DrIChF4a.js.map} +1 -1
- package/dist/{types-B_IFcKp5.d.ts → types-0KglnQ9o.d.ts} +3 -15
- package/dist/types-0KglnQ9o.d.ts.map +1 -0
- package/dist/{types-DZ0mYVw9.d.ts → types-0oMg7zee.d.ts} +3 -5
- package/dist/types-0oMg7zee.d.ts.map +1 -0
- package/dist/{types-CnWmj8NW.js → types-BK0blWhe.js} +3 -4
- package/dist/{types-CnWmj8NW.js.map → types-BK0blWhe.js.map} +1 -1
- package/dist/{types-CmXY0PQf.d.ts → types-C-aOqgt2.d.ts} +2 -5
- package/dist/types-C-aOqgt2.d.ts.map +1 -0
- package/dist/{types-M6eF9m-r.d.ts → types-C54Uk_a1.d.ts} +84 -24
- package/dist/types-C54Uk_a1.d.ts.map +1 -0
- package/dist/{types-BXgASBCL.js → types-DK4vNv5q.js} +2 -3
- package/dist/{types-BXgASBCL.js.map → types-DK4vNv5q.js.map} +1 -1
- package/dist/{types-6Lfdlshp.d.ts → types-DzCO-S1m.d.ts} +50 -62
- package/dist/types-DzCO-S1m.d.ts.map +1 -0
- package/dist/{verdict-BGQmxQ8A.js → verdict-C8TcdwfD.js} +3 -4
- package/dist/{verdict-BGQmxQ8A.js.map → verdict-C8TcdwfD.js.map} +1 -1
- package/dist/{verdict-C24NELtN.d.ts → verdict-D_qiBQ6_.d.ts} +1 -5
- package/dist/verdict-D_qiBQ6_.d.ts.map +1 -0
- package/dist/{verdict-cache-CL_7tePD.js → verdict-cache-BVH-fl18.js} +3 -4
- package/dist/{verdict-cache-CL_7tePD.js.map → verdict-cache-BVH-fl18.js.map} +1 -1
- package/dist/wire/index.d.ts +43 -51
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +1 -2
- package/docs/concepts.md +1 -0
- package/docs/eval-surface-map.md +15 -0
- package/docs/rollout.md +132 -6
- package/package.json +18 -18
- package/dist/adversarial-smnADNFS.d.ts.map +0 -1
- package/dist/agent-profile-CNmwfaqq.d.ts.map +0 -1
- package/dist/agent-profile-cell-BcpFbNd5.d.ts.map +0 -1
- package/dist/analyze-runs-DfX6dHP0.d.ts.map +0 -1
- package/dist/attribute-vocabulary-CcXqHODT.d.ts.map +0 -1
- package/dist/benchmark-command-Cnjfo0Zh.js.map +0 -1
- package/dist/campaign-evidence-DunzcJGv.js.map +0 -1
- package/dist/campaign-uddTlFg5.js.map +0 -1
- package/dist/canonical-CFpojCN5.d.ts.map +0 -1
- package/dist/client-hwo3cJjR.d.ts.map +0 -1
- package/dist/completion-verifier-asaV1nSj.js.map +0 -1
- package/dist/cost-ledger-C1W3ICmP.d.ts.map +0 -1
- package/dist/dataset-DQqhOCPt.d.ts.map +0 -1
- package/dist/default-registry-CLG5YqGS.d.ts.map +0 -1
- package/dist/define-agent-eval-JKt3gMQr.d.ts.map +0 -1
- package/dist/descriptive-B2iPaT9J.d.ts +0 -89
- package/dist/descriptive-B2iPaT9J.d.ts.map +0 -1
- package/dist/descriptive-CVNy48rR.js.map +0 -1
- package/dist/emitter-DABYYTcH.d.ts.map +0 -1
- package/dist/engine-dtRVeuQs.d.ts.map +0 -1
- package/dist/errors-DEE6u6ot.d.ts.map +0 -1
- package/dist/evaluation-DblGquCB.d.ts.map +0 -1
- package/dist/exact-types-Do63KoKn.d.ts.map +0 -1
- package/dist/exporters-DWvM16LE.js.map +0 -1
- package/dist/exporters-Dk3oEbK_.d.ts.map +0 -1
- package/dist/failure-cluster-DN0t63Z0.d.ts.map +0 -1
- package/dist/feedback-trajectory-CYVJXolG.d.ts.map +0 -1
- package/dist/hf-dataset-CKxIJQba.js.map +0 -1
- package/dist/index-CHxwfW5g.d.ts.map +0 -1
- package/dist/index-CTIRz-AB.d.ts.map +0 -1
- package/dist/index-CalAsXMR.d.ts.map +0 -1
- package/dist/index-CdEVr8-O.d.ts.map +0 -1
- package/dist/index-CfJULcW5.d.ts.map +0 -1
- package/dist/index-D7fmC-1K.d.ts.map +0 -1
- package/dist/index-jnuFLxOs.d.ts.map +0 -1
- package/dist/integrity-D-s2iHiT.js.map +0 -1
- package/dist/integrity-D9lBnv_K.d.ts.map +0 -1
- package/dist/interval-ohuOJurX.d.ts.map +0 -1
- package/dist/judge-calibration-DFtEMlde.d.ts.map +0 -1
- package/dist/llm-judge-Cybq29bL.js.map +0 -1
- package/dist/lock-holder-DaJbfpci.d.ts.map +0 -1
- package/dist/matrix-B8l-Nsxd.js.map +0 -1
- package/dist/matrix-DtkZLD_3.d.ts.map +0 -1
- package/dist/mint-CkzpVPdz.d.ts +0 -81
- package/dist/mint-CkzpVPdz.d.ts.map +0 -1
- package/dist/multi-layer-verifier-DEgMUk6v.d.ts.map +0 -1
- package/dist/multiplicity-DIWHvysC.d.ts.map +0 -1
- package/dist/paired-promotion-decision-CQMJ5xGN.d.ts.map +0 -1
- package/dist/pareto-BiBnQXAE.js.map +0 -1
- package/dist/prime-bridge-transport-6feEglLf.d.ts.map +0 -1
- package/dist/prime-protocol-BGxjfhM4.d.ts.map +0 -1
- package/dist/promotion-gate-CkVZHagC.d.ts.map +0 -1
- package/dist/query-Cmv_U8vf.d.ts.map +0 -1
- package/dist/random-DGEyr3zG.d.ts.map +0 -1
- package/dist/redact-DXdlFb_U.d.ts.map +0 -1
- package/dist/registry-BcdJmyoU.d.ts.map +0 -1
- package/dist/release-confidence-CYIFH-fp.js.map +0 -1
- package/dist/render-BhJqXoBH.js.map +0 -1
- package/dist/researcher-DKWtoMWk.d.ts.map +0 -1
- package/dist/reward-BeUgVVC_.d.ts.map +0 -1
- package/dist/reward-hacking-WTxUbtbb.d.ts.map +0 -1
- package/dist/rollout-Dd1ucT_M.js.map +0 -1
- package/dist/rubric-predictive-validity-Cd5SCtAf.d.ts.map +0 -1
- package/dist/run-record-C3awiXPf.d.ts.map +0 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +0 -1
- package/dist/schema-C-ziyJD7.d.ts.map +0 -1
- package/dist/schema-CvhfLjtZ.js.map +0 -1
- package/dist/schema-D6EVKzTB.d.ts.map +0 -1
- package/dist/search-kernel-CctpiDd2.d.ts.map +0 -1
- package/dist/search-ledger-types-p2xKfiDJ.d.ts.map +0 -1
- package/dist/sequential-KwGspfeR.d.ts.map +0 -1
- package/dist/spans-CVngGij3.d.ts.map +0 -1
- package/dist/statistical-heldout-BjzSOmLH.d.ts.map +0 -1
- package/dist/steps-DlepJPwD.d.ts.map +0 -1
- package/dist/store-B3nWRy3B.d.ts.map +0 -1
- package/dist/store-tool-spans-DxsFwtdr.d.ts.map +0 -1
- package/dist/summary-report-C8V49Jt3.d.ts.map +0 -1
- package/dist/summary-report-DHDWxe6p.js.map +0 -1
- package/dist/systemone-CHnqwmYg.d.ts.map +0 -1
- package/dist/systemone-protocol-Bl2EI2KQ.d.ts.map +0 -1
- package/dist/systemone-review-cQ9acAsb.d.ts.map +0 -1
- package/dist/task-failure-attributes-BMsblaDh.js.map +0 -1
- package/dist/terminal-record-CB9BFtjS.js.map +0 -1
- package/dist/tool-groups-b3oU_D88.d.ts.map +0 -1
- package/dist/tool-waste-CkjzWmeY.d.ts.map +0 -1
- package/dist/trace-questions-BmT8M6UQ.d.ts.map +0 -1
- package/dist/types-6Lfdlshp.d.ts.map +0 -1
- package/dist/types-B_IFcKp5.d.ts.map +0 -1
- package/dist/types-CmXY0PQf.d.ts.map +0 -1
- package/dist/types-DZ0mYVw9.d.ts.map +0 -1
- package/dist/types-M6eF9m-r.d.ts.map +0 -1
- package/dist/verdict-C24NELtN.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-record-DhEJgoPz.js","names":[],"sources":["../src/run-record.ts"],"sourcesContent":["/**\n * Paper-grade RunRecord schema + runtime validator.\n *\n * Every run that participates in a promotion gate, paper table, or\n * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory\n * fields are exactly those the paper \"Two Loops, Three Roles\" requires\n * for reproducibility: who/what/when/cost/seed/hash, plus the search vs\n * holdout split tag. A task score is optional because execution-only records\n * must preserve missing labels instead of converting errors into zero quality.\n *\n * This is intentionally NOT a replacement for the rich `Run` /\n * `ProposeReviewReport` / `ScenarioResult` types already in the\n * package. Those are runtime structures with full provenance. A\n * `RunRecord` is the analysis-time projection — the JSON-friendly\n * row you'd put in a parquet file or paste into a notebook.\n *\n * Validate at the boundary:\n *\n * const rec = validateRunRecord(rawJson) // throws on missing\n * const ok = isRunRecord(rawJson) // boolean check\n * const rec = parseRunRecordSafe(rawJson) // { ok, value | error }\n *\n * The validator runs in pure TS — zod is intentionally NOT a\n * dependency.\n */\n\nimport type { AgentProfileCell } from './agent-profile-cell'\nimport { validateAgentProfileCell } from './agent-profile-cell'\nimport type { CostProvenance } from './cost-ledger'\nimport { ValidationError } from './errors'\n// Value import of a leaf module that itself imports only this file's TYPES —\n// no runtime cycle. It keeps the raw split-score derivation spelled in exactly\n// one place (see `rollout/score-derivation-guard`).\nimport { observedScore } from './rollout/reward'\nimport { FAILURE_CLASSES, type FailureClass } from './trace/schema'\n\n/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the\n * combined train+test pool that the optimizer is allowed to read. */\nexport type RunSplitTag = 'search' | 'dev' | 'holdout'\n\n/**\n * Explicit execution-lifecycle result for a run.\n *\n * This is separate from task quality (`outcome`) and failure classification.\n * Producers set it only from root-run or process evidence.\n */\nexport type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown'\n\n/** Explicit model value for a row that never produced a served model snapshot. */\nexport const UNKNOWN_MODEL = 'unknown'\n\nexport interface RunTokenUsage {\n input: number\n /** All generated tokens charged as output, including reasoning tokens. */\n output: number\n /** Present only when one or more paid calls did not report token usage.\n * In that case, every numeric field is a known subtotal, not a measured total. */\n tokensKnown?: false\n /** Reasoning-token subset of `output`, when the provider reports it. */\n reasoning?: number\n /** Prompt tokens served from a provider cache. */\n cached?: number\n /** Prompt tokens written into a provider cache. */\n cacheWrite?: number\n}\n\n/**\n * How a run's USD amount was obtained.\n *\n * `costUsd` is the run's total, or null when the total is unknown. A run whose\n * receipts prove only part of its spend is a `lower-bound`: its total is\n * unknown, so `costUsd` and `usd` stay null, and the proven part is\n * `knownLowerBoundUsd`. A reader of the total therefore never takes a floor\n * for a total. A floor of $0 proves nothing, so it is written `uncaptured`.\n *\n * Only a RunRecord needs this kind. Every other cost shape keeps a known\n * subtotal beside `CostProvenance`, where `uncaptured` already means \"at\n * least the subtotal\"; a RunRecord's nullable `costUsd` cannot hold one.\n */\nexport type RunCostProvenance =\n | CostProvenance\n | { kind: 'lower-bound'; usd: null; knownLowerBoundUsd: number }\n\n/**\n * The run's place in a search: one attempt at one cell. The search ledger\n * holds the lineage (the node's parents, depth and order); a RunRecord only\n * points at it.\n */\nexport interface RunSearchCoordinates {\n searchId: string\n nodeId: string\n /** The cell id hashes the search, node, task, split and repeat, so it is\n * unique across candidates. */\n cellId: string\n /** Attempt number within the cell, counted from 1. */\n attempt: number\n}\n\n/** Where the run's execution record lives. */\nexport interface RunTraceRef {\n /** Trace id shared by every span of the run. */\n traceId: string\n /** Root run of the execution tree the run spawned, when one exists. */\n execRunId?: string\n}\n\n/**\n * The run id of one attempt at a search cell. It is unique across candidates\n * and attempts, unlike a `${scenario}:${rep}` id, which every candidate that\n * ran the same scenario shares.\n */\nexport function searchCellRunId(search: Pick<RunSearchCoordinates, 'cellId' | 'attempt'>): string {\n return `${search.cellId}:${search.attempt}`\n}\n\nexport interface RunJudgeMetadata {\n model: string\n promptVersion: string\n /** [0,1] confidence the judge declared. Constant judge confidence\n * across many runs is a fallback signal (see `canary.ts`). */\n confidence: number\n /** True if the judge degraded to a fallback path (rules-only,\n * prior-call cache, etc.). The canary uses this to alert. */\n fallback: boolean\n}\n\n/**\n * Per-judge / per-dimension breakdown for runs scored by an ensemble of\n * judges over a multi-dimensional rubric.\n *\n * The collapsed `outcome.searchScore` / `holdoutScore` carries the\n * composite the gate uses. The full breakdown belongs here so consumers\n * can answer \"which judge disagreed?\", \"which dimension dragged the\n * composite down?\", and \"did half the panel fail?\" without re-running.\n *\n * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and\n * `composite` are convenience projections — derivable but precomputed so\n * downstream IRR primitives (`interRaterReliability`,\n * `corpusInterRaterAgreement`) and reporters don't pay the same\n * aggregation twice.\n *\n * Fail-loud discipline: judges that errored out land in `failedJudges`\n * by id. A missing key in `perJudge` is ambiguous (silent zero vs not\n * run); the explicit list makes a partial-failure recorded as such.\n */\nexport interface JudgeScoresRecord {\n /** Per-judge per-dimension scores. `{ \"kimi-k2.6\": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */\n perJudge: Record<string, Record<string, number>>\n /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */\n perDimMean: Record<string, number>\n /** Composite mean across successful judges. Mirrors the task score only\n * when `failedJudges` is empty. */\n composite: number\n /** Judges that errored or returned an unparseable verdict. Recorded\n * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,\n * not inferred from missing keys in `perJudge`. */\n failedJudges?: string[]\n /** Free-form notes the judges emitted (joined across judges or\n * first-judge only — consumer's choice). */\n notes?: string\n}\n\nexport interface RunOutcome {\n /** Score on the search/optimization split. Optional for holdout-only and\n * execution-only records. */\n searchScore?: number\n /** Score on the held-out split. Optional for search-only and execution-only\n * records. When both scores are absent, the run is explicitly unlabeled. */\n holdoutScore?: number\n /** Bag of any other metric the run produced — judge dimensions,\n * pass/fail counters, latency stats, etc. Numeric only — keeps\n * reporters honest. */\n raw: Record<string, number>\n /** Per-judge / per-dim breakdown. Consumers writing ensemble\n * judgements populate this; substrate primitives like\n * `interRaterReliability` and `corpusInterRaterAgreement` accept\n * these records as input. Optional — single-judge or scalar-only\n * runs leave it unset. */\n judgeScores?: JudgeScoresRecord\n /** Authenticity / realness verdict — did the run build the REAL thing on the\n * intended infra, or fake it (see `./authenticity`)? Optional: only domains\n * with an authenticity config populate it. Carried in the corpus so the\n * flywheel / off-policy learning can optimize for real completion, not gamed\n * pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run\n * must not count as a real success regardless of `score`. */\n realness?: { score: number; gated: boolean; reason?: string }\n}\n\n/**\n * Mandatory paper-grade fields for a single evaluation run. Optional\n * fields are extension points; mandatory fields throw if missing.\n *\n * Hash discipline:\n * - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the\n * model (after any steering bundle merge).\n * - `configHash` is the sha256 of the effective run config (model,\n * temperature, tools, judges, splits). The pair (promptHash,\n * configHash) uniquely identifies an experiment cell.\n *\n * Model snapshot discipline:\n * - successful rows MUST encode a snapshot version. Bare aliases like\n * `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.\n * Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.\n * - a failed, cancelled, incomplete, or otherwise unknown row may use\n * `UNKNOWN_MODEL` when no served model was observed. This is an explicit\n * absence marker, not a fabricated snapshot.\n */\nexport interface RunRecord {\n /** UUID for the run. */\n runId: string\n /** Logical experiment grouping (a treatment vs a baseline within\n * the same sweep should share `experimentId`). */\n experimentId: string\n /** Stable identifier for the candidate (variant) being run. The\n * promotion gate compares two `candidateId`s on matched items. */\n candidateId: string\n /** RNG seed for the run. Always recorded — silent re-seeding is\n * the most common cause of non-reproducible numbers. */\n seed: number\n /** Model identifier WITH snapshot version. */\n model: string\n /** sha256 of the effective prompt (post-steering). */\n promptHash: string\n /** sha256 of the effective config. */\n configHash: string\n /** Git SHA the harness was run from. */\n commitSha: string\n /** End-to-end wall-clock duration in milliseconds. */\n wallMs: number\n /** Time spent queued before execution started, if known. */\n queueMs?: number\n /** Total USD cost, or null when the total is unknown. */\n costUsd: number | null\n /** Whether `costUsd` came from billing data or a price calculation, is only\n * proven down to a floor, or is unavailable. */\n costProvenance: RunCostProvenance\n /** Token usage breakdown. */\n tokenUsage: RunTokenUsage\n /** Root-run or process terminal result. Never inferred from a child span. */\n terminalOutcome: RunTerminalOutcome\n /** Root-run or process failure reason. Valid only for a failed, cancelled,\n * or incomplete terminal result; never populated from a child span. */\n terminalFailureReason?: string\n /** Judge-side metadata, if a judge was used. */\n judgeMetadata?: RunJudgeMetadata\n /** Per-split scores + raw bag. */\n outcome: RunOutcome\n /** Canonical task-failure class drawn from the shared\n * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result\n * evidence. Execution errors belong in\n * `outcome.raw.execution_error_count`. */\n failureClass?: FailureClass\n /** Free-form task-failure detail scoped under a non-success\n * `failureClass`. It is invalid without that class. */\n failureMode?: string\n /** Which split this run was drawn from. */\n splitTag: RunSplitTag\n /**\n * Stable scenario identifier the run observed or was scored against.\n * Comparison primitives match this identity rather than input order.\n */\n scenarioId: string\n /**\n * Canonical identity for the agent profile cell that produced this row:\n * profile artifact hash plus optional harness/model/prompt/reporting\n * dimensions. Use `agentProfile.cellId` to group persona sweeps and\n * longitudinal reports by the complete source profile, not by a loose\n * candidate label or opaque config hash.\n */\n agentProfile?: AgentProfileCell\n /** The search cell this run settled. When present, `runId` is\n * `searchCellRunId(search)`. */\n search?: RunSearchCoordinates\n /** The run's trace and execution tree. Absent means the producer did not\n * record where they are. */\n traceRef?: RunTraceRef\n}\n\n/**\n * Canonical task-result classification.\n *\n * A producer may omit classification, record explicit success, or attach\n * domain-specific detail to a non-success class. Detail can never stand alone.\n * Execution errors belong in `outcome.raw.execution_error_count`.\n */\nexport type RunTaskFailure =\n | { failureClass?: undefined; failureMode?: undefined }\n | { failureClass: 'success'; failureMode?: undefined }\n | {\n failureClass: Exclude<FailureClass, 'success'>\n failureMode?: string\n }\n\n/**\n * Return task quality, preferring held-out evidence when both scores exist.\n *\n * RAW: no realness protection is applied. Built on `observedScore` rather\n * than repeating the split derivation, so only `rollout/reward.ts` reads the\n * raw fields. Anything that becomes training data must use `trainingScore` or\n * `trainingReward` instead.\n */\nexport function runTaskScore(record: RunRecord): number | undefined {\n const score = observedScore(record)\n return typeof score === 'number' && Number.isFinite(score) ? score : undefined\n}\n\n// ── Validation ───────────────────────────────────────────────────────\n\nconst MANDATORY_TOP_LEVEL = [\n 'runId',\n 'experimentId',\n 'candidateId',\n 'seed',\n 'model',\n 'promptHash',\n 'configHash',\n 'commitSha',\n 'wallMs',\n 'costUsd',\n 'costProvenance',\n 'tokenUsage',\n 'terminalOutcome',\n 'outcome',\n 'splitTag',\n 'scenarioId',\n] as const\n\nconst SPLIT_TAGS: ReadonlyArray<RunSplitTag> = ['search', 'dev', 'holdout']\nconst TERMINAL_OUTCOMES: ReadonlyArray<RunTerminalOutcome> = [\n 'succeeded',\n 'failed',\n 'cancelled',\n 'incomplete',\n 'unknown',\n]\n\nexport class RunRecordValidationError extends ValidationError {\n readonly path: string\n constructor(message: string, path = '') {\n super(path ? `${message} (at ${path})` : message)\n this.path = path\n }\n}\n\n/**\n * Strict validator. Throws `RunRecordValidationError` on the first\n * missing or wrongly-typed field. Returns the input cast to\n * `RunRecord` on success — the validator does not coerce.\n */\nexport function validateRunRecord(input: unknown): RunRecord {\n if (input === null || typeof input !== 'object') {\n throw new RunRecordValidationError('expected object')\n }\n const obj = input as Record<string, unknown>\n\n for (const key of MANDATORY_TOP_LEVEL) {\n if (!(key in obj)) {\n throw new RunRecordValidationError(`missing mandatory field \"${key}\"`)\n }\n }\n\n expectString(obj.runId, 'runId')\n expectString(obj.experimentId, 'experimentId')\n expectString(obj.candidateId, 'candidateId')\n expectFiniteNumber(obj.seed, 'seed')\n expectString(obj.model, 'model')\n expectString(obj.promptHash, 'promptHash')\n expectString(obj.configHash, 'configHash')\n expectString(obj.commitSha, 'commitSha')\n expectNonNegativeNumber(obj.wallMs, 'wallMs')\n if (obj.queueMs !== undefined) expectNonNegativeNumber(obj.queueMs, 'queueMs')\n validateCost(obj.costUsd, obj.costProvenance)\n\n // Snapshot discipline: successful rows require a served model snapshot.\n // Non-success rows may carry the explicit absence marker when execution\n // stopped before a model identity was observed.\n if (\n !modelHasSnapshot(obj.model as string) &&\n !(obj.model === UNKNOWN_MODEL && obj.terminalOutcome !== 'succeeded')\n ) {\n throw new RunRecordValidationError(\n `model \"${obj.model}\" lacks a snapshot version (use 'name@YYYY-MM-DD' or 'name-YYYYMMDD', or '${UNKNOWN_MODEL}' for a non-success row without a served model)`,\n 'model',\n )\n }\n\n // Token usage.\n const tu = obj.tokenUsage\n if (tu === null || typeof tu !== 'object') {\n throw new RunRecordValidationError('tokenUsage must be an object', 'tokenUsage')\n }\n const tuRec = tu as Record<string, unknown>\n expectNonNegativeNumber(tuRec.input, 'tokenUsage.input')\n expectNonNegativeNumber(tuRec.output, 'tokenUsage.output')\n if (tuRec.tokensKnown !== undefined && tuRec.tokensKnown !== false) {\n throw new RunRecordValidationError(\n 'tokensKnown must be false when present; omit it when token usage is complete',\n 'tokenUsage.tokensKnown',\n )\n }\n if (tuRec.reasoning !== undefined) {\n expectNonNegativeNumber(tuRec.reasoning, 'tokenUsage.reasoning')\n if ((tuRec.reasoning as number) > (tuRec.output as number)) {\n throw new RunRecordValidationError(\n 'reasoning tokens must be a subset of output tokens',\n 'tokenUsage.reasoning',\n )\n }\n }\n if (tuRec.cached !== undefined) expectNonNegativeNumber(tuRec.cached, 'tokenUsage.cached')\n if (tuRec.cacheWrite !== undefined) {\n expectNonNegativeNumber(tuRec.cacheWrite, 'tokenUsage.cacheWrite')\n }\n\n // Judge metadata, optional.\n if (obj.judgeMetadata !== undefined) {\n const jm = obj.judgeMetadata\n if (jm === null || typeof jm !== 'object') {\n throw new RunRecordValidationError('judgeMetadata must be an object', 'judgeMetadata')\n }\n const jmRec = jm as Record<string, unknown>\n expectString(jmRec.model, 'judgeMetadata.model')\n expectString(jmRec.promptVersion, 'judgeMetadata.promptVersion')\n expectFiniteNumber(jmRec.confidence, 'judgeMetadata.confidence')\n if (typeof jmRec.fallback !== 'boolean') {\n throw new RunRecordValidationError(\n 'judgeMetadata.fallback must be boolean',\n 'judgeMetadata.fallback',\n )\n }\n }\n\n // Outcome.\n const out = obj.outcome\n if (out === null || typeof out !== 'object') {\n throw new RunRecordValidationError('outcome must be an object', 'outcome')\n }\n const outRec = out as Record<string, unknown>\n if (outRec.searchScore !== undefined)\n expectFiniteNumber(outRec.searchScore, 'outcome.searchScore')\n if (outRec.holdoutScore !== undefined)\n expectFiniteNumber(outRec.holdoutScore, 'outcome.holdoutScore')\n const raw = outRec.raw\n if (raw === null || typeof raw !== 'object') {\n throw new RunRecordValidationError('outcome.raw must be an object', 'outcome.raw')\n }\n for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {\n expectFiniteNumber(v, `outcome.raw.${k}`)\n }\n // Realness verdict, optional.\n if (outRec.realness !== undefined) {\n const r = outRec.realness\n if (r === null || typeof r !== 'object') {\n throw new RunRecordValidationError('outcome.realness must be an object', 'outcome.realness')\n }\n const rr = r as Record<string, unknown>\n expectFiniteNumber(rr.score, 'outcome.realness.score')\n if (typeof rr.gated !== 'boolean') {\n throw new RunRecordValidationError(\n 'outcome.realness.gated must be a boolean',\n 'outcome.realness.gated',\n )\n }\n }\n\n // Per-judge / per-dim breakdown, optional.\n if (outRec.judgeScores !== undefined) {\n validateJudgeScores(outRec.judgeScores, 'outcome.judgeScores')\n }\n\n // Failure mode optional.\n if (\n obj.failureClass !== undefined &&\n (typeof obj.failureClass !== 'string' ||\n !FAILURE_CLASSES.includes(obj.failureClass as FailureClass))\n ) {\n throw new RunRecordValidationError(\n `failureClass must be one of ${FAILURE_CLASSES.join(', ')}`,\n 'failureClass',\n )\n }\n if (obj.failureMode !== undefined) {\n expectString(obj.failureMode, 'failureMode')\n if (obj.failureClass === undefined || obj.failureClass === 'success') {\n throw new RunRecordValidationError(\n 'failureMode requires a non-success failureClass',\n 'failureMode',\n )\n }\n }\n\n if (\n typeof obj.terminalOutcome !== 'string' ||\n !TERMINAL_OUTCOMES.includes(obj.terminalOutcome as RunTerminalOutcome)\n ) {\n throw new RunRecordValidationError(\n `terminalOutcome must be one of ${TERMINAL_OUTCOMES.join(', ')}`,\n 'terminalOutcome',\n )\n }\n if (obj.terminalFailureReason !== undefined) {\n expectString(obj.terminalFailureReason, 'terminalFailureReason')\n if (\n obj.terminalOutcome !== 'failed' &&\n obj.terminalOutcome !== 'cancelled' &&\n obj.terminalOutcome !== 'incomplete'\n ) {\n throw new RunRecordValidationError(\n 'terminalFailureReason requires terminalOutcome failed, cancelled, or incomplete',\n 'terminalFailureReason',\n )\n }\n }\n\n if (obj.agentProfile !== undefined) {\n try {\n const profile = validateAgentProfileCell(obj.agentProfile)\n if (profile.model !== undefined && profile.model !== obj.model) {\n throw new RunRecordValidationError(\n `agentProfile.model \"${profile.model}\" does not match model \"${obj.model}\"`,\n 'agentProfile.model',\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== obj.promptHash) {\n throw new RunRecordValidationError(\n `agentProfile.promptHash \"${profile.promptHash}\" does not match promptHash \"${obj.promptHash}\"`,\n 'agentProfile.promptHash',\n )\n }\n } catch (error) {\n if (error instanceof RunRecordValidationError) throw error\n if (error instanceof Error) {\n throw new RunRecordValidationError(error.message, 'agentProfile')\n }\n throw error\n }\n }\n\n expectString(obj.scenarioId, 'scenarioId')\n\n if (obj.search !== undefined) validateSearchCoordinates(obj.search, obj.runId as string)\n if (obj.traceRef !== undefined) validateTraceRef(obj.traceRef)\n\n // Split tag.\n if (typeof obj.splitTag !== 'string' || !SPLIT_TAGS.includes(obj.splitTag as RunSplitTag)) {\n throw new RunRecordValidationError(\n `splitTag must be one of ${SPLIT_TAGS.join(', ')}, got ${String(obj.splitTag)}`,\n 'splitTag',\n )\n }\n\n return input as RunRecord\n}\n\nfunction validateCost(costUsd: unknown, provenance: unknown): void {\n if (provenance === null || typeof provenance !== 'object') {\n throw new RunRecordValidationError('costProvenance must be an object', 'costProvenance')\n }\n const value = provenance as Record<string, unknown>\n if (\n value.kind !== 'observed' &&\n value.kind !== 'estimated' &&\n value.kind !== 'lower-bound' &&\n value.kind !== 'uncaptured'\n ) {\n throw new RunRecordValidationError(\n 'costProvenance.kind must be observed, estimated, lower-bound, or uncaptured',\n 'costProvenance.kind',\n )\n }\n if (value.kind === 'lower-bound') {\n if (value.usd !== null) {\n throw new RunRecordValidationError(\n 'lower-bound costProvenance.usd must be null; the floor is knownLowerBoundUsd',\n 'costProvenance.usd',\n )\n }\n if (costUsd !== null) {\n throw new RunRecordValidationError(\n 'lower-bound cost requires costUsd to be null; a floor is not a total',\n 'costUsd',\n )\n }\n expectFiniteNumber(value.knownLowerBoundUsd, 'costProvenance.knownLowerBoundUsd')\n if ((value.knownLowerBoundUsd as number) <= 0) {\n throw new RunRecordValidationError(\n \"a lower bound must be positive; write a $0 floor as { kind: 'uncaptured', usd: null }\",\n 'costProvenance.knownLowerBoundUsd',\n )\n }\n return\n }\n // A record must never read as a total it cannot support, so an uncaptured\n // cost carries no number at all. A matrix `CellResult` keeps its known\n // subtotal instead, because a cost ceiling must charge the part it can see;\n // converting one to the other drops that subtotal.\n if (value.kind === 'uncaptured') {\n if (value.usd !== null) {\n throw new RunRecordValidationError(\n 'uncaptured costProvenance.usd must be null',\n 'costProvenance.usd',\n )\n }\n if (costUsd !== null) {\n throw new RunRecordValidationError('uncaptured cost requires costUsd to be null', 'costUsd')\n }\n return\n }\n expectNonNegativeNumber(costUsd, 'costUsd')\n expectNonNegativeNumber(value.usd, 'costProvenance.usd')\n if (value.usd !== costUsd) {\n throw new RunRecordValidationError(\n 'costProvenance.usd must equal costUsd',\n 'costProvenance.usd',\n )\n }\n}\n\nfunction validateSearchCoordinates(value: unknown, runId: string): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('search must be an object', 'search')\n }\n const search = value as Record<string, unknown>\n expectString(search.searchId, 'search.searchId')\n expectString(search.nodeId, 'search.nodeId')\n expectString(search.cellId, 'search.cellId')\n if (!Number.isSafeInteger(search.attempt) || (search.attempt as number) < 1) {\n throw new RunRecordValidationError('expected an integer from 1', 'search.attempt')\n }\n const expected = searchCellRunId(search as unknown as RunSearchCoordinates)\n if (runId !== expected) {\n throw new RunRecordValidationError(\n `runId \"${runId}\" must be \"${expected}\", the search cell and attempt it settled`,\n 'runId',\n )\n }\n}\n\nfunction validateTraceRef(value: unknown): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('traceRef must be an object', 'traceRef')\n }\n const ref = value as Record<string, unknown>\n expectString(ref.traceId, 'traceRef.traceId')\n if (ref.execRunId !== undefined) expectString(ref.execRunId, 'traceRef.execRunId')\n}\n\n/** Boolean validator — convenience for filtering arrays. */\nexport function isRunRecord(input: unknown): input is RunRecord {\n try {\n validateRunRecord(input)\n return true\n } catch {\n return false\n }\n}\n\n/** Non-throwing validator — returns a discriminated union. */\nexport function parseRunRecordSafe(\n input: unknown,\n): { ok: true; value: RunRecord } | { ok: false; error: RunRecordValidationError } {\n try {\n return { ok: true, value: validateRunRecord(input) }\n } catch (e) {\n if (e instanceof RunRecordValidationError) return { ok: false, error: e }\n throw e\n }\n}\n\n// ── Internals ────────────────────────────────────────────────────────\n\nfunction expectString(value: unknown, path: string): void {\n if (typeof value !== 'string' || value.length === 0) {\n throw new RunRecordValidationError(`expected non-empty string`, path)\n }\n}\n\nfunction expectFiniteNumber(value: unknown, path: string): void {\n if (typeof value !== 'number' || !Number.isFinite(value)) {\n throw new RunRecordValidationError(`expected finite number`, path)\n }\n}\n\nfunction expectNonNegativeNumber(value: unknown, path: string): void {\n expectFiniteNumber(value, path)\n if ((value as number) < 0) {\n throw new RunRecordValidationError('expected non-negative number', path)\n }\n}\n\nfunction validateJudgeScores(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('judgeScores must be an object', path)\n }\n const rec = value as Record<string, unknown>\n\n const perJudge = rec.perJudge\n if (perJudge === null || typeof perJudge !== 'object') {\n throw new RunRecordValidationError('perJudge must be an object', `${path}.perJudge`)\n }\n for (const [judgeId, dims] of Object.entries(perJudge as Record<string, unknown>)) {\n if (dims === null || typeof dims !== 'object') {\n throw new RunRecordValidationError(\n 'per-judge entry must be an object of dimension scores',\n `${path}.perJudge.${judgeId}`,\n )\n }\n for (const [dim, score] of Object.entries(dims as Record<string, unknown>)) {\n expectFiniteNumber(score, `${path}.perJudge.${judgeId}.${dim}`)\n }\n }\n\n const perDimMean = rec.perDimMean\n if (perDimMean === null || typeof perDimMean !== 'object') {\n throw new RunRecordValidationError('perDimMean must be an object', `${path}.perDimMean`)\n }\n for (const [dim, mean] of Object.entries(perDimMean as Record<string, unknown>)) {\n expectFiniteNumber(mean, `${path}.perDimMean.${dim}`)\n }\n\n expectFiniteNumber(rec.composite, `${path}.composite`)\n\n if (rec.failedJudges !== undefined) {\n if (!Array.isArray(rec.failedJudges)) {\n throw new RunRecordValidationError(\n 'failedJudges must be an array of strings',\n `${path}.failedJudges`,\n )\n }\n for (let i = 0; i < rec.failedJudges.length; i++) {\n const id = rec.failedJudges[i]\n if (typeof id !== 'string' || id.length === 0) {\n throw new RunRecordValidationError(\n 'failedJudges entry must be a non-empty string',\n `${path}.failedJudges[${i}]`,\n )\n }\n }\n }\n\n if (rec.notes !== undefined && typeof rec.notes !== 'string') {\n throw new RunRecordValidationError('notes must be a string', `${path}.notes`)\n }\n}\n\n/**\n * Snapshot check for provider model identifiers. Accepts ISO and compact\n * dates, Router's `-MMDD` snapshots, one opaque `@token`, and Vertex-style\n * `:date-token` suffixes. Routing selectors such as `@preset/name` are not\n * immutable model identities.\n */\nexport function modelHasSnapshot(model: string): boolean {\n if (model.length === 0 || model.trim() !== model) return false\n\n const opaqueAt = model.lastIndexOf('@')\n if (opaqueAt > 0) {\n const base = model.slice(0, opaqueAt)\n const token = model.slice(opaqueAt + 1)\n if (!base.includes('@') && /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(token)) {\n return true\n }\n }\n\n const isoDate = model.match(/-(\\d{4})-(\\d{2})-(\\d{2})$/u)\n if (isoDate && validSnapshotDate(isoDate[1]!, isoDate[2]!, isoDate[3]!)) return true\n\n const compactDate = model.match(/-(\\d{4})(\\d{2})(\\d{2})$/u)\n if (compactDate && validSnapshotDate(compactDate[1]!, compactDate[2]!, compactDate[3]!)) {\n return true\n }\n\n const routerDate = model.match(/-(\\d{2})(\\d{2})$/u)\n if (routerDate && validSnapshotDate(undefined, routerDate[1]!, routerDate[2]!)) return true\n\n return /:date-[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(model)\n}\n\nfunction validSnapshotDate(year: string | undefined, month: string, day: string): boolean {\n const monthNumber = Number(month)\n const dayNumber = Number(day)\n if (!Number.isInteger(monthNumber) || monthNumber < 1 || monthNumber > 12) return false\n\n const yearNumber = year === undefined ? undefined : Number(year)\n const leapYear =\n yearNumber === undefined ||\n (yearNumber % 4 === 0 && (yearNumber % 100 !== 0 || yearNumber % 400 === 0))\n const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]\n return Number.isInteger(dayNumber) && dayNumber >= 1 && dayNumber <= daysInMonth[monthNumber - 1]!\n}\n"],"mappings":";;;;;;AAiDA,MAAa,gBAAgB;;;;;;AA8D7B,SAAgB,gBAAgB,QAAkE;CAChG,OAAO,GAAG,OAAO,OAAO,GAAG,OAAO;AACpC;;;;;;;;;AA4LA,SAAgB,aAAa,QAAuC;CAClE,MAAM,QAAQ,cAAc,MAAM;CAClC,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;AACvE;AAIA,MAAM,sBAAsB;CAC1B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,aAAyC;CAAC;CAAU;CAAO;AAAS;AAC1E,MAAM,oBAAuD;CAC3D;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,2BAAb,cAA8C,gBAAgB;CAC5D;CACA,YAAY,SAAiB,OAAO,IAAI;EACtC,MAAM,OAAO,GAAG,QAAQ,OAAO,KAAK,KAAK,OAAO;EAChD,KAAK,OAAO;CACd;AACF;;;;;;AAOA,SAAgB,kBAAkB,OAA2B;CAC3D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iBAAiB;CAEtD,MAAM,MAAM;CAEZ,KAAK,MAAM,OAAO,qBAChB,IAAI,EAAE,OAAO,MACX,MAAM,IAAI,yBAAyB,4BAA4B,IAAI,EAAE;CAIzE,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,cAAc,cAAc;CAC7C,aAAa,IAAI,aAAa,aAAa;CAC3C,mBAAmB,IAAI,MAAM,MAAM;CACnC,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,WAAW,WAAW;CACvC,wBAAwB,IAAI,QAAQ,QAAQ;CAC5C,IAAI,IAAI,YAAY,KAAA,GAAW,wBAAwB,IAAI,SAAS,SAAS;CAC7E,aAAa,IAAI,SAAS,IAAI,cAAc;CAK5C,IACE,CAAC,iBAAiB,IAAI,KAAe,KACrC,EAAE,IAAI,UAAA,aAA2B,IAAI,oBAAoB,cAEzD,MAAM,IAAI,yBACR,UAAU,IAAI,MAAM,4EAA4E,cAAc,kDAC9G,OACF;CAIF,MAAM,KAAK,IAAI;CACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,gCAAgC,YAAY;CAEjF,MAAM,QAAQ;CACd,wBAAwB,MAAM,OAAO,kBAAkB;CACvD,wBAAwB,MAAM,QAAQ,mBAAmB;CACzD,IAAI,MAAM,gBAAgB,KAAA,KAAa,MAAM,gBAAgB,OAC3D,MAAM,IAAI,yBACR,gFACA,wBACF;CAEF,IAAI,MAAM,cAAc,KAAA,GAAW;EACjC,wBAAwB,MAAM,WAAW,sBAAsB;EAC/D,IAAK,MAAM,YAAwB,MAAM,QACvC,MAAM,IAAI,yBACR,sDACA,sBACF;CAEJ;CACA,IAAI,MAAM,WAAW,KAAA,GAAW,wBAAwB,MAAM,QAAQ,mBAAmB;CACzF,IAAI,MAAM,eAAe,KAAA,GACvB,wBAAwB,MAAM,YAAY,uBAAuB;CAInE,IAAI,IAAI,kBAAkB,KAAA,GAAW;EACnC,MAAM,KAAK,IAAI;EACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,mCAAmC,eAAe;EAEvF,MAAM,QAAQ;EACd,aAAa,MAAM,OAAO,qBAAqB;EAC/C,aAAa,MAAM,eAAe,6BAA6B;EAC/D,mBAAmB,MAAM,YAAY,0BAA0B;EAC/D,IAAI,OAAO,MAAM,aAAa,WAC5B,MAAM,IAAI,yBACR,0CACA,wBACF;CAEJ;CAGA,MAAM,MAAM,IAAI;CAChB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,6BAA6B,SAAS;CAE3E,MAAM,SAAS;CACf,IAAI,OAAO,gBAAgB,KAAA,GACzB,mBAAmB,OAAO,aAAa,qBAAqB;CAC9D,IAAI,OAAO,iBAAiB,KAAA,GAC1B,mBAAmB,OAAO,cAAc,sBAAsB;CAChE,MAAM,MAAM,OAAO;CACnB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,iCAAiC,aAAa;CAEnF,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,GAA8B,GAChE,mBAAmB,GAAG,eAAe,GAAG;CAG1C,IAAI,OAAO,aAAa,KAAA,GAAW;EACjC,MAAM,IAAI,OAAO;EACjB,IAAI,MAAM,QAAQ,OAAO,MAAM,UAC7B,MAAM,IAAI,yBAAyB,sCAAsC,kBAAkB;EAE7F,MAAM,KAAK;EACX,mBAAmB,GAAG,OAAO,wBAAwB;EACrD,IAAI,OAAO,GAAG,UAAU,WACtB,MAAM,IAAI,yBACR,4CACA,wBACF;CAEJ;CAGA,IAAI,OAAO,gBAAgB,KAAA,GACzB,oBAAoB,OAAO,aAAa,qBAAqB;CAI/D,IACE,IAAI,iBAAiB,KAAA,MACpB,OAAO,IAAI,iBAAiB,YAC3B,CAAC,gBAAgB,SAAS,IAAI,YAA4B,IAE5D,MAAM,IAAI,yBACR,+BAA+B,gBAAgB,KAAK,IAAI,KACxD,cACF;CAEF,IAAI,IAAI,gBAAgB,KAAA,GAAW;EACjC,aAAa,IAAI,aAAa,aAAa;EAC3C,IAAI,IAAI,iBAAiB,KAAA,KAAa,IAAI,iBAAiB,WACzD,MAAM,IAAI,yBACR,mDACA,aACF;CAEJ;CAEA,IACE,OAAO,IAAI,oBAAoB,YAC/B,CAAC,kBAAkB,SAAS,IAAI,eAAqC,GAErE,MAAM,IAAI,yBACR,kCAAkC,kBAAkB,KAAK,IAAI,KAC7D,iBACF;CAEF,IAAI,IAAI,0BAA0B,KAAA,GAAW;EAC3C,aAAa,IAAI,uBAAuB,uBAAuB;EAC/D,IACE,IAAI,oBAAoB,YACxB,IAAI,oBAAoB,eACxB,IAAI,oBAAoB,cAExB,MAAM,IAAI,yBACR,mFACA,uBACF;CAEJ;CAEA,IAAI,IAAI,iBAAiB,KAAA,GACvB,IAAI;EACF,MAAM,UAAU,yBAAyB,IAAI,YAAY;EACzD,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,IAAI,OACvD,MAAM,IAAI,yBACR,uBAAuB,QAAQ,MAAM,0BAA0B,IAAI,MAAM,IACzE,oBACF;EAEF,IAAI,QAAQ,eAAe,KAAA,KAAa,QAAQ,eAAe,IAAI,YACjE,MAAM,IAAI,yBACR,4BAA4B,QAAQ,WAAW,+BAA+B,IAAI,WAAW,IAC7F,yBACF;CAEJ,SAAS,OAAO;EACd,IAAI,iBAAiB,0BAA0B,MAAM;EACrD,IAAI,iBAAiB,OACnB,MAAM,IAAI,yBAAyB,MAAM,SAAS,cAAc;EAElE,MAAM;CACR;CAGF,aAAa,IAAI,YAAY,YAAY;CAEzC,IAAI,IAAI,WAAW,KAAA,GAAW,0BAA0B,IAAI,QAAQ,IAAI,KAAe;CACvF,IAAI,IAAI,aAAa,KAAA,GAAW,iBAAiB,IAAI,QAAQ;CAG7D,IAAI,OAAO,IAAI,aAAa,YAAY,CAAC,WAAW,SAAS,IAAI,QAAuB,GACtF,MAAM,IAAI,yBACR,2BAA2B,WAAW,KAAK,IAAI,EAAE,QAAQ,OAAO,IAAI,QAAQ,KAC5E,UACF;CAGF,OAAO;AACT;AAEA,SAAS,aAAa,SAAkB,YAA2B;CACjE,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,oCAAoC,gBAAgB;CAEzF,MAAM,QAAQ;CACd,IACE,MAAM,SAAS,cACf,MAAM,SAAS,eACf,MAAM,SAAS,iBACf,MAAM,SAAS,cAEf,MAAM,IAAI,yBACR,+EACA,qBACF;CAEF,IAAI,MAAM,SAAS,eAAe;EAChC,IAAI,MAAM,QAAQ,MAChB,MAAM,IAAI,yBACR,gFACA,oBACF;EAEF,IAAI,YAAY,MACd,MAAM,IAAI,yBACR,wEACA,SACF;EAEF,mBAAmB,MAAM,oBAAoB,mCAAmC;EAChF,IAAK,MAAM,sBAAiC,GAC1C,MAAM,IAAI,yBACR,yFACA,mCACF;EAEF;CACF;CAKA,IAAI,MAAM,SAAS,cAAc;EAC/B,IAAI,MAAM,QAAQ,MAChB,MAAM,IAAI,yBACR,8CACA,oBACF;EAEF,IAAI,YAAY,MACd,MAAM,IAAI,yBAAyB,+CAA+C,SAAS;EAE7F;CACF;CACA,wBAAwB,SAAS,SAAS;CAC1C,wBAAwB,MAAM,KAAK,oBAAoB;CACvD,IAAI,MAAM,QAAQ,SAChB,MAAM,IAAI,yBACR,yCACA,oBACF;AAEJ;AAEA,SAAS,0BAA0B,OAAgB,OAAqB;CACtE,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,4BAA4B,QAAQ;CAEzE,MAAM,SAAS;CACf,aAAa,OAAO,UAAU,iBAAiB;CAC/C,aAAa,OAAO,QAAQ,eAAe;CAC3C,aAAa,OAAO,QAAQ,eAAe;CAC3C,IAAI,CAAC,OAAO,cAAc,OAAO,OAAO,KAAM,OAAO,UAAqB,GACxE,MAAM,IAAI,yBAAyB,8BAA8B,gBAAgB;CAEnF,MAAM,WAAW,gBAAgB,MAAyC;CAC1E,IAAI,UAAU,UACZ,MAAM,IAAI,yBACR,UAAU,MAAM,aAAa,SAAS,4CACtC,OACF;AAEJ;AAEA,SAAS,iBAAiB,OAAsB;CAC9C,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,8BAA8B,UAAU;CAE7E,MAAM,MAAM;CACZ,aAAa,IAAI,SAAS,kBAAkB;CAC5C,IAAI,IAAI,cAAc,KAAA,GAAW,aAAa,IAAI,WAAW,oBAAoB;AACnF;;AAGA,SAAgB,YAAY,OAAoC;CAC9D,IAAI;EACF,kBAAkB,KAAK;EACvB,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;AAGA,SAAgB,mBACd,OACiF;CACjF,IAAI;EACF,OAAO;GAAE,IAAI;GAAM,OAAO,kBAAkB,KAAK;EAAE;CACrD,SAAS,GAAG;EACV,IAAI,aAAa,0BAA0B,OAAO;GAAE,IAAI;GAAO,OAAO;EAAE;EACxE,MAAM;CACR;AACF;AAIA,SAAS,aAAa,OAAgB,MAAoB;CACxD,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,MAAM,IAAI,yBAAyB,6BAA6B,IAAI;AAExE;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;CAC9D,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GACrD,MAAM,IAAI,yBAAyB,0BAA0B,IAAI;AAErE;AAEA,SAAS,wBAAwB,OAAgB,MAAoB;CACnE,mBAAmB,OAAO,IAAI;CAC9B,IAAK,QAAmB,GACtB,MAAM,IAAI,yBAAyB,gCAAgC,IAAI;AAE3E;AAEA,SAAS,oBAAoB,OAAgB,MAAoB;CAC/D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iCAAiC,IAAI;CAE1E,MAAM,MAAM;CAEZ,MAAM,WAAW,IAAI;CACrB,IAAI,aAAa,QAAQ,OAAO,aAAa,UAC3C,MAAM,IAAI,yBAAyB,8BAA8B,GAAG,KAAK,UAAU;CAErF,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,QAAmC,GAAG;EACjF,IAAI,SAAS,QAAQ,OAAO,SAAS,UACnC,MAAM,IAAI,yBACR,yDACA,GAAG,KAAK,YAAY,SACtB;EAEF,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,IAA+B,GACvE,mBAAmB,OAAO,GAAG,KAAK,YAAY,QAAQ,GAAG,KAAK;CAElE;CAEA,MAAM,aAAa,IAAI;CACvB,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,gCAAgC,GAAG,KAAK,YAAY;CAEzF,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAqC,GAC5E,mBAAmB,MAAM,GAAG,KAAK,cAAc,KAAK;CAGtD,mBAAmB,IAAI,WAAW,GAAG,KAAK,WAAW;CAErD,IAAI,IAAI,iBAAiB,KAAA,GAAW;EAClC,IAAI,CAAC,MAAM,QAAQ,IAAI,YAAY,GACjC,MAAM,IAAI,yBACR,4CACA,GAAG,KAAK,cACV;EAEF,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,aAAa,QAAQ,KAAK;GAChD,MAAM,KAAK,IAAI,aAAa;GAC5B,IAAI,OAAO,OAAO,YAAY,GAAG,WAAW,GAC1C,MAAM,IAAI,yBACR,iDACA,GAAG,KAAK,gBAAgB,EAAE,EAC5B;EAEJ;CACF;CAEA,IAAI,IAAI,UAAU,KAAA,KAAa,OAAO,IAAI,UAAU,UAClD,MAAM,IAAI,yBAAyB,0BAA0B,GAAG,KAAK,OAAO;AAEhF;;;;;;;AAQA,SAAgB,iBAAiB,OAAwB;CACvD,IAAI,MAAM,WAAW,KAAK,MAAM,KAAK,MAAM,OAAO,OAAO;CAEzD,MAAM,WAAW,MAAM,YAAY,GAAG;CACtC,IAAI,WAAW,GAAG;EAChB,MAAM,OAAO,MAAM,MAAM,GAAG,QAAQ;EACpC,MAAM,QAAQ,MAAM,MAAM,WAAW,CAAC;EACtC,IAAI,CAAC,KAAK,SAAS,GAAG,KAAK,gDAAgD,KAAK,KAAK,GACnF,OAAO;CAEX;CAEA,MAAM,UAAU,MAAM,MAAM,4BAA4B;CACxD,IAAI,WAAW,kBAAkB,QAAQ,IAAK,QAAQ,IAAK,QAAQ,EAAG,GAAG,OAAO;CAEhF,MAAM,cAAc,MAAM,MAAM,0BAA0B;CAC1D,IAAI,eAAe,kBAAkB,YAAY,IAAK,YAAY,IAAK,YAAY,EAAG,GACpF,OAAO;CAGT,MAAM,aAAa,MAAM,MAAM,mBAAmB;CAClD,IAAI,cAAc,kBAAkB,KAAA,GAAW,WAAW,IAAK,WAAW,EAAG,GAAG,OAAO;CAEvF,OAAO,qDAAqD,KAAK,KAAK;AACxE;AAEA,SAAS,kBAAkB,MAA0B,OAAe,KAAsB;CACxF,MAAM,cAAc,OAAO,KAAK;CAChC,MAAM,YAAY,OAAO,GAAG;CAC5B,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,KAAK,cAAc,IAAI,OAAO;CAElF,MAAM,aAAa,SAAS,KAAA,IAAY,KAAA,IAAY,OAAO,IAAI;CAI/D,MAAM,cAAc;EAAC;EAFnB,eAAe,KAAA,KACd,aAAa,MAAM,MAAM,aAAa,QAAQ,KAAK,aAAa,QAAQ,KACvC,KAAK;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;CAAE;CACnF,OAAO,OAAO,UAAU,SAAS,KAAK,aAAa,KAAK,aAAa,YAAY,cAAc;AACjG"}
|
|
1
|
+
{"version":3,"file":"run-record-DEjx43cI.js","names":[],"sources":["../src/run-record.ts"],"sourcesContent":["/**\n * Paper-grade RunRecord schema + runtime validator.\n *\n * Every run that participates in a promotion gate, paper table, or\n * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory\n * fields are exactly those the paper \"Two Loops, Three Roles\" requires\n * for reproducibility: who/what/when/cost/seed/hash, plus the search vs\n * holdout split tag. A task score is optional because execution-only records\n * must preserve missing labels instead of converting errors into zero quality.\n *\n * This is intentionally NOT a replacement for the rich `Run` /\n * `ProposeReviewReport` / `ScenarioResult` types already in the\n * package. Those are runtime structures with full provenance. A\n * `RunRecord` is the analysis-time projection — the JSON-friendly\n * row you'd put in a parquet file or paste into a notebook.\n *\n * Validate at the boundary:\n *\n * const rec = validateRunRecord(rawJson) // throws on missing\n * const ok = isRunRecord(rawJson) // boolean check\n * const rec = parseRunRecordSafe(rawJson) // { ok, value | error }\n *\n * The validator runs in pure TS — zod is intentionally NOT a\n * dependency.\n */\n\nimport type { AgentProfileCell } from './agent-profile-cell'\nimport { validateAgentProfileCell } from './agent-profile-cell'\nimport type { CostProvenance } from './cost-ledger'\nimport { ValidationError } from './errors'\n// Value import of a leaf module that itself imports only this file's TYPES —\n// no runtime cycle. It keeps the raw split-score derivation spelled in exactly\n// one place (see `rollout/score-derivation-guard`).\nimport { observedScore } from './rollout/reward'\nimport { FAILURE_CLASSES, type FailureClass } from './trace/schema'\n\n/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the\n * combined train+test pool that the optimizer is allowed to read. */\nexport type RunSplitTag = 'search' | 'dev' | 'holdout'\n\n/**\n * Explicit execution-lifecycle result for a run.\n *\n * This is separate from task quality (`outcome`) and failure classification.\n * Producers set it only from root-run or process evidence.\n */\nexport type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown'\n\n/** Explicit model value for a row that never produced a served model snapshot. */\nexport const UNKNOWN_MODEL = 'unknown'\n\nexport interface RunTokenUsage {\n input: number\n /** All generated tokens charged as output, including reasoning tokens. */\n output: number\n /** Present only when one or more paid calls did not report token usage.\n * In that case, every numeric field is a known subtotal, not a measured total. */\n tokensKnown?: false\n /** Reasoning-token subset of `output`, when the provider reports it. */\n reasoning?: number\n /** Prompt tokens served from a provider cache. */\n cached?: number\n /** Prompt tokens written into a provider cache. */\n cacheWrite?: number\n}\n\n/**\n * How a run's USD amount was obtained.\n *\n * `costUsd` is the run's total, or null when the total is unknown. A run whose\n * receipts prove only part of its spend is a `lower-bound`: its total is\n * unknown, so `costUsd` and `usd` stay null, and the proven part is\n * `knownLowerBoundUsd`. A reader of the total therefore never takes a floor\n * for a total. A floor of $0 proves nothing, so it is written `uncaptured`.\n *\n * Only a RunRecord needs this kind. Every other cost shape keeps a known\n * subtotal beside `CostProvenance`, where `uncaptured` already means \"at\n * least the subtotal\"; a RunRecord's nullable `costUsd` cannot hold one.\n */\nexport type RunCostProvenance =\n | CostProvenance\n | { kind: 'lower-bound'; usd: null; knownLowerBoundUsd: number }\n\n/**\n * The run's place in a search: one attempt at one cell. The search ledger\n * holds the lineage (the node's parents, depth and order); a RunRecord only\n * points at it.\n */\nexport interface RunSearchCoordinates {\n searchId: string\n nodeId: string\n /** The cell id hashes the search, node, task, split and repeat, so it is\n * unique across candidates. */\n cellId: string\n /** Attempt number within the cell, counted from 1. */\n attempt: number\n}\n\n/** Where the run's execution record lives. */\nexport interface RunTraceRef {\n /** Trace id shared by every span of the run. */\n traceId: string\n /** Root run of the execution tree the run spawned, when one exists. */\n execRunId?: string\n}\n\n/**\n * The run id of one attempt at a search cell. It is unique across candidates\n * and attempts, unlike a `${scenario}:${rep}` id, which every candidate that\n * ran the same scenario shares.\n */\nexport function searchCellRunId(search: Pick<RunSearchCoordinates, 'cellId' | 'attempt'>): string {\n return `${search.cellId}:${search.attempt}`\n}\n\nexport interface RunJudgeMetadata {\n model: string\n promptVersion: string\n /** [0,1] confidence the judge declared. Constant judge confidence\n * across many runs is a fallback signal (see `canary.ts`). */\n confidence: number\n /** True if the judge degraded to a fallback path (rules-only,\n * prior-call cache, etc.). The canary uses this to alert. */\n fallback: boolean\n}\n\n/**\n * Per-judge / per-dimension breakdown for runs scored by an ensemble of\n * judges over a multi-dimensional rubric.\n *\n * The collapsed `outcome.searchScore` / `holdoutScore` carries the\n * composite the gate uses. The full breakdown belongs here so consumers\n * can answer \"which judge disagreed?\", \"which dimension dragged the\n * composite down?\", and \"did half the panel fail?\" without re-running.\n *\n * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and\n * `composite` are convenience projections — derivable but precomputed so\n * downstream IRR primitives (`interRaterReliability`,\n * `corpusInterRaterAgreement`) and reporters don't pay the same\n * aggregation twice.\n *\n * Fail-loud discipline: judges that errored out land in `failedJudges`\n * by id. A missing key in `perJudge` is ambiguous (silent zero vs not\n * run); the explicit list makes a partial-failure recorded as such.\n */\nexport interface JudgeScoresRecord {\n /** Per-judge per-dimension scores. `{ \"kimi-k2.6\": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */\n perJudge: Record<string, Record<string, number>>\n /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */\n perDimMean: Record<string, number>\n /** Composite mean across successful judges. Mirrors the task score only\n * when `failedJudges` is empty. */\n composite: number\n /** Judges that errored or returned an unparseable verdict. Recorded\n * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,\n * not inferred from missing keys in `perJudge`. */\n failedJudges?: string[]\n /** Free-form notes the judges emitted (joined across judges or\n * first-judge only — consumer's choice). */\n notes?: string\n}\n\nexport interface RunOutcome {\n /** Score on the search/optimization split. Optional for holdout-only and\n * execution-only records. */\n searchScore?: number\n /** Score on the held-out split. Optional for search-only and execution-only\n * records. When both scores are absent, the run is explicitly unlabeled. */\n holdoutScore?: number\n /** Bag of any other metric the run produced — judge dimensions,\n * pass/fail counters, latency stats, etc. Numeric only — keeps\n * reporters honest. */\n raw: Record<string, number>\n /** Per-judge / per-dim breakdown. Consumers writing ensemble\n * judgements populate this; substrate primitives like\n * `interRaterReliability` and `corpusInterRaterAgreement` accept\n * these records as input. Optional — single-judge or scalar-only\n * runs leave it unset. */\n judgeScores?: JudgeScoresRecord\n /** Authenticity / realness verdict — did the run build the REAL thing on the\n * intended infra, or fake it (see `./authenticity`)? Optional: only domains\n * with an authenticity config populate it. Carried in the corpus so the\n * flywheel / off-policy learning can optimize for real completion, not gamed\n * pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run\n * must not count as a real success regardless of `score`. */\n realness?: { score: number; gated: boolean; reason?: string }\n}\n\n/**\n * Mandatory paper-grade fields for a single evaluation run. Optional\n * fields are extension points; mandatory fields throw if missing.\n *\n * Hash discipline:\n * - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the\n * model (after any steering bundle merge).\n * - `configHash` is the sha256 of the effective run config (model,\n * temperature, tools, judges, splits). The pair (promptHash,\n * configHash) uniquely identifies an experiment cell.\n *\n * Model snapshot discipline:\n * - successful rows MUST encode a snapshot version. Bare aliases like\n * `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.\n * Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.\n * - a failed, cancelled, incomplete, or otherwise unknown row may use\n * `UNKNOWN_MODEL` when no served model was observed. This is an explicit\n * absence marker, not a fabricated snapshot.\n */\nexport interface RunRecord {\n /** UUID for the run. */\n runId: string\n /** Logical experiment grouping (a treatment vs a baseline within\n * the same sweep should share `experimentId`). */\n experimentId: string\n /** Stable identifier for the candidate (variant) being run. The\n * promotion gate compares two `candidateId`s on matched items. */\n candidateId: string\n /** RNG seed for the run. Always recorded — silent re-seeding is\n * the most common cause of non-reproducible numbers. */\n seed: number\n /** Model identifier WITH snapshot version. */\n model: string\n /** sha256 of the effective prompt (post-steering). */\n promptHash: string\n /** sha256 of the effective config. */\n configHash: string\n /** Git SHA the harness was run from. */\n commitSha: string\n /** End-to-end wall-clock duration in milliseconds. */\n wallMs: number\n /** Time spent queued before execution started, if known. */\n queueMs?: number\n /** Total USD cost, or null when the total is unknown. */\n costUsd: number | null\n /** Whether `costUsd` came from billing data or a price calculation, is only\n * proven down to a floor, or is unavailable. */\n costProvenance: RunCostProvenance\n /** Token usage breakdown. */\n tokenUsage: RunTokenUsage\n /** Root-run or process terminal result. Never inferred from a child span. */\n terminalOutcome: RunTerminalOutcome\n /** Root-run or process failure reason. Valid only for a failed, cancelled,\n * or incomplete terminal result; never populated from a child span. */\n terminalFailureReason?: string\n /** Judge-side metadata, if a judge was used. */\n judgeMetadata?: RunJudgeMetadata\n /** Per-split scores + raw bag. */\n outcome: RunOutcome\n /** Canonical task-failure class drawn from the shared\n * `FAILURE_CLASSES` taxonomy. Producers set it only from task-result\n * evidence. Execution errors belong in\n * `outcome.raw.execution_error_count`. */\n failureClass?: FailureClass\n /** Free-form task-failure detail scoped under a non-success\n * `failureClass`. It is invalid without that class. */\n failureMode?: string\n /** Which split this run was drawn from. */\n splitTag: RunSplitTag\n /**\n * Stable scenario identifier the run observed or was scored against.\n * Comparison primitives match this identity rather than input order.\n */\n scenarioId: string\n /**\n * Canonical identity for the agent profile cell that produced this row:\n * profile artifact hash plus optional harness/model/prompt/reporting\n * dimensions. Use `agentProfile.cellId` to group persona sweeps and\n * longitudinal reports by the complete source profile, not by a loose\n * candidate label or opaque config hash.\n */\n agentProfile?: AgentProfileCell\n /** The search cell this run settled. When present, `runId` is\n * `searchCellRunId(search)`. */\n search?: RunSearchCoordinates\n /** The run's trace and execution tree. Absent means the producer did not\n * record where they are. */\n traceRef?: RunTraceRef\n}\n\n/**\n * Canonical task-result classification.\n *\n * A producer may omit classification, record explicit success, or attach\n * domain-specific detail to a non-success class. Detail can never stand alone.\n * Execution errors belong in `outcome.raw.execution_error_count`.\n */\nexport type RunTaskFailure =\n | { failureClass?: undefined; failureMode?: undefined }\n | { failureClass: 'success'; failureMode?: undefined }\n | {\n failureClass: Exclude<FailureClass, 'success'>\n failureMode?: string\n }\n\n/**\n * Return task quality, preferring held-out evidence when both scores exist.\n *\n * RAW: no realness protection is applied. Built on `observedScore` rather\n * than repeating the split derivation, so only `rollout/reward.ts` reads the\n * raw fields. Anything that becomes training data must use `trainingScore` or\n * `trainingReward` instead.\n */\nexport function runTaskScore(record: RunRecord): number | undefined {\n const score = observedScore(record)\n return typeof score === 'number' && Number.isFinite(score) ? score : undefined\n}\n\n// ── Validation ───────────────────────────────────────────────────────\n\nconst MANDATORY_TOP_LEVEL = [\n 'runId',\n 'experimentId',\n 'candidateId',\n 'seed',\n 'model',\n 'promptHash',\n 'configHash',\n 'commitSha',\n 'wallMs',\n 'costUsd',\n 'costProvenance',\n 'tokenUsage',\n 'terminalOutcome',\n 'outcome',\n 'splitTag',\n 'scenarioId',\n] as const\n\nconst SPLIT_TAGS: ReadonlyArray<RunSplitTag> = ['search', 'dev', 'holdout']\nconst TERMINAL_OUTCOMES: ReadonlyArray<RunTerminalOutcome> = [\n 'succeeded',\n 'failed',\n 'cancelled',\n 'incomplete',\n 'unknown',\n]\n\nexport class RunRecordValidationError extends ValidationError {\n readonly path: string\n constructor(message: string, path = '') {\n super(path ? `${message} (at ${path})` : message)\n this.path = path\n }\n}\n\n/**\n * Strict validator. Throws `RunRecordValidationError` on the first\n * missing or wrongly-typed field. Returns the input cast to\n * `RunRecord` on success — the validator does not coerce.\n */\nexport function validateRunRecord(input: unknown): RunRecord {\n if (input === null || typeof input !== 'object') {\n throw new RunRecordValidationError('expected object')\n }\n const obj = input as Record<string, unknown>\n\n for (const key of MANDATORY_TOP_LEVEL) {\n if (!(key in obj)) {\n throw new RunRecordValidationError(`missing mandatory field \"${key}\"`)\n }\n }\n\n expectString(obj.runId, 'runId')\n expectString(obj.experimentId, 'experimentId')\n expectString(obj.candidateId, 'candidateId')\n expectFiniteNumber(obj.seed, 'seed')\n expectString(obj.model, 'model')\n expectString(obj.promptHash, 'promptHash')\n expectString(obj.configHash, 'configHash')\n expectString(obj.commitSha, 'commitSha')\n expectNonNegativeNumber(obj.wallMs, 'wallMs')\n if (obj.queueMs !== undefined) expectNonNegativeNumber(obj.queueMs, 'queueMs')\n validateCost(obj.costUsd, obj.costProvenance)\n\n // Snapshot discipline: successful rows require a served model snapshot.\n // Non-success rows may carry the explicit absence marker when execution\n // stopped before a model identity was observed.\n if (\n !modelHasSnapshot(obj.model as string) &&\n !(obj.model === UNKNOWN_MODEL && obj.terminalOutcome !== 'succeeded')\n ) {\n throw new RunRecordValidationError(\n `model \"${obj.model}\" lacks a snapshot version (use 'name@YYYY-MM-DD' or 'name-YYYYMMDD', or '${UNKNOWN_MODEL}' for a non-success row without a served model)`,\n 'model',\n )\n }\n\n // Token usage.\n const tu = obj.tokenUsage\n if (tu === null || typeof tu !== 'object') {\n throw new RunRecordValidationError('tokenUsage must be an object', 'tokenUsage')\n }\n const tuRec = tu as Record<string, unknown>\n expectNonNegativeNumber(tuRec.input, 'tokenUsage.input')\n expectNonNegativeNumber(tuRec.output, 'tokenUsage.output')\n if (tuRec.tokensKnown !== undefined && tuRec.tokensKnown !== false) {\n throw new RunRecordValidationError(\n 'tokensKnown must be false when present; omit it when token usage is complete',\n 'tokenUsage.tokensKnown',\n )\n }\n if (tuRec.reasoning !== undefined) {\n expectNonNegativeNumber(tuRec.reasoning, 'tokenUsage.reasoning')\n if ((tuRec.reasoning as number) > (tuRec.output as number)) {\n throw new RunRecordValidationError(\n 'reasoning tokens must be a subset of output tokens',\n 'tokenUsage.reasoning',\n )\n }\n }\n if (tuRec.cached !== undefined) expectNonNegativeNumber(tuRec.cached, 'tokenUsage.cached')\n if (tuRec.cacheWrite !== undefined) {\n expectNonNegativeNumber(tuRec.cacheWrite, 'tokenUsage.cacheWrite')\n }\n\n // Judge metadata, optional.\n if (obj.judgeMetadata !== undefined) {\n const jm = obj.judgeMetadata\n if (jm === null || typeof jm !== 'object') {\n throw new RunRecordValidationError('judgeMetadata must be an object', 'judgeMetadata')\n }\n const jmRec = jm as Record<string, unknown>\n expectString(jmRec.model, 'judgeMetadata.model')\n expectString(jmRec.promptVersion, 'judgeMetadata.promptVersion')\n expectFiniteNumber(jmRec.confidence, 'judgeMetadata.confidence')\n if (typeof jmRec.fallback !== 'boolean') {\n throw new RunRecordValidationError(\n 'judgeMetadata.fallback must be boolean',\n 'judgeMetadata.fallback',\n )\n }\n }\n\n // Outcome.\n const out = obj.outcome\n if (out === null || typeof out !== 'object') {\n throw new RunRecordValidationError('outcome must be an object', 'outcome')\n }\n const outRec = out as Record<string, unknown>\n if (outRec.searchScore !== undefined)\n expectFiniteNumber(outRec.searchScore, 'outcome.searchScore')\n if (outRec.holdoutScore !== undefined)\n expectFiniteNumber(outRec.holdoutScore, 'outcome.holdoutScore')\n const raw = outRec.raw\n if (raw === null || typeof raw !== 'object') {\n throw new RunRecordValidationError('outcome.raw must be an object', 'outcome.raw')\n }\n for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {\n expectFiniteNumber(v, `outcome.raw.${k}`)\n }\n // Realness verdict, optional.\n if (outRec.realness !== undefined) {\n const r = outRec.realness\n if (r === null || typeof r !== 'object') {\n throw new RunRecordValidationError('outcome.realness must be an object', 'outcome.realness')\n }\n const rr = r as Record<string, unknown>\n expectFiniteNumber(rr.score, 'outcome.realness.score')\n if (typeof rr.gated !== 'boolean') {\n throw new RunRecordValidationError(\n 'outcome.realness.gated must be a boolean',\n 'outcome.realness.gated',\n )\n }\n }\n\n // Per-judge / per-dim breakdown, optional.\n if (outRec.judgeScores !== undefined) {\n validateJudgeScores(outRec.judgeScores, 'outcome.judgeScores')\n }\n\n // Failure mode optional.\n if (\n obj.failureClass !== undefined &&\n (typeof obj.failureClass !== 'string' ||\n !FAILURE_CLASSES.includes(obj.failureClass as FailureClass))\n ) {\n throw new RunRecordValidationError(\n `failureClass must be one of ${FAILURE_CLASSES.join(', ')}`,\n 'failureClass',\n )\n }\n if (obj.failureMode !== undefined) {\n expectString(obj.failureMode, 'failureMode')\n if (obj.failureClass === undefined || obj.failureClass === 'success') {\n throw new RunRecordValidationError(\n 'failureMode requires a non-success failureClass',\n 'failureMode',\n )\n }\n }\n\n if (\n typeof obj.terminalOutcome !== 'string' ||\n !TERMINAL_OUTCOMES.includes(obj.terminalOutcome as RunTerminalOutcome)\n ) {\n throw new RunRecordValidationError(\n `terminalOutcome must be one of ${TERMINAL_OUTCOMES.join(', ')}`,\n 'terminalOutcome',\n )\n }\n if (obj.terminalFailureReason !== undefined) {\n expectString(obj.terminalFailureReason, 'terminalFailureReason')\n if (\n obj.terminalOutcome !== 'failed' &&\n obj.terminalOutcome !== 'cancelled' &&\n obj.terminalOutcome !== 'incomplete'\n ) {\n throw new RunRecordValidationError(\n 'terminalFailureReason requires terminalOutcome failed, cancelled, or incomplete',\n 'terminalFailureReason',\n )\n }\n }\n\n if (obj.agentProfile !== undefined) {\n try {\n const profile = validateAgentProfileCell(obj.agentProfile)\n if (profile.model !== undefined && profile.model !== obj.model) {\n throw new RunRecordValidationError(\n `agentProfile.model \"${profile.model}\" does not match model \"${obj.model}\"`,\n 'agentProfile.model',\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== obj.promptHash) {\n throw new RunRecordValidationError(\n `agentProfile.promptHash \"${profile.promptHash}\" does not match promptHash \"${obj.promptHash}\"`,\n 'agentProfile.promptHash',\n )\n }\n } catch (error) {\n if (error instanceof RunRecordValidationError) throw error\n if (error instanceof Error) {\n throw new RunRecordValidationError(error.message, 'agentProfile')\n }\n throw error\n }\n }\n\n expectString(obj.scenarioId, 'scenarioId')\n\n if (obj.search !== undefined) validateSearchCoordinates(obj.search, obj.runId as string)\n if (obj.traceRef !== undefined) validateTraceRef(obj.traceRef)\n\n // Split tag.\n if (typeof obj.splitTag !== 'string' || !SPLIT_TAGS.includes(obj.splitTag as RunSplitTag)) {\n throw new RunRecordValidationError(\n `splitTag must be one of ${SPLIT_TAGS.join(', ')}, got ${String(obj.splitTag)}`,\n 'splitTag',\n )\n }\n\n return input as RunRecord\n}\n\nfunction validateCost(costUsd: unknown, provenance: unknown): void {\n if (provenance === null || typeof provenance !== 'object') {\n throw new RunRecordValidationError('costProvenance must be an object', 'costProvenance')\n }\n const value = provenance as Record<string, unknown>\n if (\n value.kind !== 'observed' &&\n value.kind !== 'estimated' &&\n value.kind !== 'lower-bound' &&\n value.kind !== 'uncaptured'\n ) {\n throw new RunRecordValidationError(\n 'costProvenance.kind must be observed, estimated, lower-bound, or uncaptured',\n 'costProvenance.kind',\n )\n }\n if (value.kind === 'lower-bound') {\n if (value.usd !== null) {\n throw new RunRecordValidationError(\n 'lower-bound costProvenance.usd must be null; the floor is knownLowerBoundUsd',\n 'costProvenance.usd',\n )\n }\n if (costUsd !== null) {\n throw new RunRecordValidationError(\n 'lower-bound cost requires costUsd to be null; a floor is not a total',\n 'costUsd',\n )\n }\n expectFiniteNumber(value.knownLowerBoundUsd, 'costProvenance.knownLowerBoundUsd')\n if ((value.knownLowerBoundUsd as number) <= 0) {\n throw new RunRecordValidationError(\n \"a lower bound must be positive; write a $0 floor as { kind: 'uncaptured', usd: null }\",\n 'costProvenance.knownLowerBoundUsd',\n )\n }\n return\n }\n // A record must never read as a total it cannot support, so an uncaptured\n // cost carries no number at all. A matrix `CellResult` keeps its known\n // subtotal instead, because a cost ceiling must charge the part it can see;\n // converting one to the other drops that subtotal.\n if (value.kind === 'uncaptured') {\n if (value.usd !== null) {\n throw new RunRecordValidationError(\n 'uncaptured costProvenance.usd must be null',\n 'costProvenance.usd',\n )\n }\n if (costUsd !== null) {\n throw new RunRecordValidationError('uncaptured cost requires costUsd to be null', 'costUsd')\n }\n return\n }\n expectNonNegativeNumber(costUsd, 'costUsd')\n expectNonNegativeNumber(value.usd, 'costProvenance.usd')\n if (value.usd !== costUsd) {\n throw new RunRecordValidationError(\n 'costProvenance.usd must equal costUsd',\n 'costProvenance.usd',\n )\n }\n}\n\nfunction validateSearchCoordinates(value: unknown, runId: string): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('search must be an object', 'search')\n }\n const search = value as Record<string, unknown>\n expectString(search.searchId, 'search.searchId')\n expectString(search.nodeId, 'search.nodeId')\n expectString(search.cellId, 'search.cellId')\n if (!Number.isSafeInteger(search.attempt) || (search.attempt as number) < 1) {\n throw new RunRecordValidationError('expected an integer from 1', 'search.attempt')\n }\n const expected = searchCellRunId(search as unknown as RunSearchCoordinates)\n if (runId !== expected) {\n throw new RunRecordValidationError(\n `runId \"${runId}\" must be \"${expected}\", the search cell and attempt it settled`,\n 'runId',\n )\n }\n}\n\nfunction validateTraceRef(value: unknown): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('traceRef must be an object', 'traceRef')\n }\n const ref = value as Record<string, unknown>\n expectString(ref.traceId, 'traceRef.traceId')\n if (ref.execRunId !== undefined) expectString(ref.execRunId, 'traceRef.execRunId')\n}\n\n/** Boolean validator — convenience for filtering arrays. */\nexport function isRunRecord(input: unknown): input is RunRecord {\n try {\n validateRunRecord(input)\n return true\n } catch {\n return false\n }\n}\n\n/** Non-throwing validator — returns a discriminated union. */\nexport function parseRunRecordSafe(\n input: unknown,\n): { ok: true; value: RunRecord } | { ok: false; error: RunRecordValidationError } {\n try {\n return { ok: true, value: validateRunRecord(input) }\n } catch (e) {\n if (e instanceof RunRecordValidationError) return { ok: false, error: e }\n throw e\n }\n}\n\n// ── Internals ────────────────────────────────────────────────────────\n\nfunction expectString(value: unknown, path: string): void {\n if (typeof value !== 'string' || value.length === 0) {\n throw new RunRecordValidationError(`expected non-empty string`, path)\n }\n}\n\nfunction expectFiniteNumber(value: unknown, path: string): void {\n if (typeof value !== 'number' || !Number.isFinite(value)) {\n throw new RunRecordValidationError(`expected finite number`, path)\n }\n}\n\nfunction expectNonNegativeNumber(value: unknown, path: string): void {\n expectFiniteNumber(value, path)\n if ((value as number) < 0) {\n throw new RunRecordValidationError('expected non-negative number', path)\n }\n}\n\nfunction validateJudgeScores(value: unknown, path: string): void {\n if (value === null || typeof value !== 'object') {\n throw new RunRecordValidationError('judgeScores must be an object', path)\n }\n const rec = value as Record<string, unknown>\n\n const perJudge = rec.perJudge\n if (perJudge === null || typeof perJudge !== 'object') {\n throw new RunRecordValidationError('perJudge must be an object', `${path}.perJudge`)\n }\n for (const [judgeId, dims] of Object.entries(perJudge as Record<string, unknown>)) {\n if (dims === null || typeof dims !== 'object') {\n throw new RunRecordValidationError(\n 'per-judge entry must be an object of dimension scores',\n `${path}.perJudge.${judgeId}`,\n )\n }\n for (const [dim, score] of Object.entries(dims as Record<string, unknown>)) {\n expectFiniteNumber(score, `${path}.perJudge.${judgeId}.${dim}`)\n }\n }\n\n const perDimMean = rec.perDimMean\n if (perDimMean === null || typeof perDimMean !== 'object') {\n throw new RunRecordValidationError('perDimMean must be an object', `${path}.perDimMean`)\n }\n for (const [dim, mean] of Object.entries(perDimMean as Record<string, unknown>)) {\n expectFiniteNumber(mean, `${path}.perDimMean.${dim}`)\n }\n\n expectFiniteNumber(rec.composite, `${path}.composite`)\n\n if (rec.failedJudges !== undefined) {\n if (!Array.isArray(rec.failedJudges)) {\n throw new RunRecordValidationError(\n 'failedJudges must be an array of strings',\n `${path}.failedJudges`,\n )\n }\n for (let i = 0; i < rec.failedJudges.length; i++) {\n const id = rec.failedJudges[i]\n if (typeof id !== 'string' || id.length === 0) {\n throw new RunRecordValidationError(\n 'failedJudges entry must be a non-empty string',\n `${path}.failedJudges[${i}]`,\n )\n }\n }\n }\n\n if (rec.notes !== undefined && typeof rec.notes !== 'string') {\n throw new RunRecordValidationError('notes must be a string', `${path}.notes`)\n }\n}\n\n/**\n * Snapshot check for provider model identifiers. Accepts ISO and compact\n * dates, Router's `-MMDD` snapshots, one opaque `@token`, and Vertex-style\n * `:date-token` suffixes. Routing selectors such as `@preset/name` are not\n * immutable model identities.\n */\nexport function modelHasSnapshot(model: string): boolean {\n if (model.length === 0 || model.trim() !== model) return false\n\n const opaqueAt = model.lastIndexOf('@')\n if (opaqueAt > 0) {\n const base = model.slice(0, opaqueAt)\n const token = model.slice(opaqueAt + 1)\n if (!base.includes('@') && /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(token)) {\n return true\n }\n }\n\n const isoDate = model.match(/-(\\d{4})-(\\d{2})-(\\d{2})$/u)\n if (isoDate && validSnapshotDate(isoDate[1]!, isoDate[2]!, isoDate[3]!)) return true\n\n const compactDate = model.match(/-(\\d{4})(\\d{2})(\\d{2})$/u)\n if (compactDate && validSnapshotDate(compactDate[1]!, compactDate[2]!, compactDate[3]!)) {\n return true\n }\n\n const routerDate = model.match(/-(\\d{2})(\\d{2})$/u)\n if (routerDate && validSnapshotDate(undefined, routerDate[1]!, routerDate[2]!)) return true\n\n return /:date-[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(model)\n}\n\nfunction validSnapshotDate(year: string | undefined, month: string, day: string): boolean {\n const monthNumber = Number(month)\n const dayNumber = Number(day)\n if (!Number.isInteger(monthNumber) || monthNumber < 1 || monthNumber > 12) return false\n\n const yearNumber = year === undefined ? undefined : Number(year)\n const leapYear =\n yearNumber === undefined ||\n (yearNumber % 4 === 0 && (yearNumber % 100 !== 0 || yearNumber % 400 === 0))\n const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]\n return Number.isInteger(dayNumber) && dayNumber >= 1 && dayNumber <= daysInMonth[monthNumber - 1]!\n}\n"],"mappings":";;;;;;AAiDA,MAAa,gBAAgB;;;;;;AA8D7B,SAAgB,gBAAgB,QAAkE;CAChG,OAAO,GAAG,OAAO,OAAO,GAAG,OAAO;AACpC;;;;;;;;;AA4LA,SAAgB,aAAa,QAAuC;CAClE,MAAM,QAAQ,cAAc,MAAM;CAClC,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ,KAAA;AACvE;AAIA,MAAM,sBAAsB;CAC1B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,MAAM,aAAyC;CAAC;CAAU;CAAO;AAAS;AAC1E,MAAM,oBAAuD;CAC3D;CACA;CACA;CACA;CACA;AACF;AAEA,IAAa,2BAAb,cAA8C,gBAAgB;CAC5D;CACA,YAAY,SAAiB,OAAO,IAAI;EACtC,MAAM,OAAO,GAAG,QAAQ,OAAO,KAAK,KAAK,OAAO;EAChD,KAAK,OAAO;CACd;AACF;;;;;;AAOA,SAAgB,kBAAkB,OAA2B;CAC3D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iBAAiB;CAEtD,MAAM,MAAM;CAEZ,KAAK,MAAM,OAAO,qBAChB,IAAI,EAAE,OAAO,MACX,MAAM,IAAI,yBAAyB,4BAA4B,IAAI,EAAE;CAIzE,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,cAAc,cAAc;CAC7C,aAAa,IAAI,aAAa,aAAa;CAC3C,mBAAmB,IAAI,MAAM,MAAM;CACnC,aAAa,IAAI,OAAO,OAAO;CAC/B,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,YAAY,YAAY;CACzC,aAAa,IAAI,WAAW,WAAW;CACvC,wBAAwB,IAAI,QAAQ,QAAQ;CAC5C,IAAI,IAAI,YAAY,KAAA,GAAW,wBAAwB,IAAI,SAAS,SAAS;CAC7E,aAAa,IAAI,SAAS,IAAI,cAAc;CAK5C,IACE,CAAC,iBAAiB,IAAI,KAAe,KACrC,EAAE,IAAI,UAAA,aAA2B,IAAI,oBAAoB,cAEzD,MAAM,IAAI,yBACR,UAAU,IAAI,MAAM,4EAA4E,cAAc,kDAC9G,OACF;CAIF,MAAM,KAAK,IAAI;CACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,gCAAgC,YAAY;CAEjF,MAAM,QAAQ;CACd,wBAAwB,MAAM,OAAO,kBAAkB;CACvD,wBAAwB,MAAM,QAAQ,mBAAmB;CACzD,IAAI,MAAM,gBAAgB,KAAA,KAAa,MAAM,gBAAgB,OAC3D,MAAM,IAAI,yBACR,gFACA,wBACF;CAEF,IAAI,MAAM,cAAc,KAAA,GAAW;EACjC,wBAAwB,MAAM,WAAW,sBAAsB;EAC/D,IAAK,MAAM,YAAwB,MAAM,QACvC,MAAM,IAAI,yBACR,sDACA,sBACF;CAEJ;CACA,IAAI,MAAM,WAAW,KAAA,GAAW,wBAAwB,MAAM,QAAQ,mBAAmB;CACzF,IAAI,MAAM,eAAe,KAAA,GACvB,wBAAwB,MAAM,YAAY,uBAAuB;CAInE,IAAI,IAAI,kBAAkB,KAAA,GAAW;EACnC,MAAM,KAAK,IAAI;EACf,IAAI,OAAO,QAAQ,OAAO,OAAO,UAC/B,MAAM,IAAI,yBAAyB,mCAAmC,eAAe;EAEvF,MAAM,QAAQ;EACd,aAAa,MAAM,OAAO,qBAAqB;EAC/C,aAAa,MAAM,eAAe,6BAA6B;EAC/D,mBAAmB,MAAM,YAAY,0BAA0B;EAC/D,IAAI,OAAO,MAAM,aAAa,WAC5B,MAAM,IAAI,yBACR,0CACA,wBACF;CAEJ;CAGA,MAAM,MAAM,IAAI;CAChB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,6BAA6B,SAAS;CAE3E,MAAM,SAAS;CACf,IAAI,OAAO,gBAAgB,KAAA,GACzB,mBAAmB,OAAO,aAAa,qBAAqB;CAC9D,IAAI,OAAO,iBAAiB,KAAA,GAC1B,mBAAmB,OAAO,cAAc,sBAAsB;CAChE,MAAM,MAAM,OAAO;CACnB,IAAI,QAAQ,QAAQ,OAAO,QAAQ,UACjC,MAAM,IAAI,yBAAyB,iCAAiC,aAAa;CAEnF,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,GAA8B,GAChE,mBAAmB,GAAG,eAAe,GAAG;CAG1C,IAAI,OAAO,aAAa,KAAA,GAAW;EACjC,MAAM,IAAI,OAAO;EACjB,IAAI,MAAM,QAAQ,OAAO,MAAM,UAC7B,MAAM,IAAI,yBAAyB,sCAAsC,kBAAkB;EAE7F,MAAM,KAAK;EACX,mBAAmB,GAAG,OAAO,wBAAwB;EACrD,IAAI,OAAO,GAAG,UAAU,WACtB,MAAM,IAAI,yBACR,4CACA,wBACF;CAEJ;CAGA,IAAI,OAAO,gBAAgB,KAAA,GACzB,oBAAoB,OAAO,aAAa,qBAAqB;CAI/D,IACE,IAAI,iBAAiB,KAAA,MACpB,OAAO,IAAI,iBAAiB,YAC3B,CAAC,gBAAgB,SAAS,IAAI,YAA4B,IAE5D,MAAM,IAAI,yBACR,+BAA+B,gBAAgB,KAAK,IAAI,KACxD,cACF;CAEF,IAAI,IAAI,gBAAgB,KAAA,GAAW;EACjC,aAAa,IAAI,aAAa,aAAa;EAC3C,IAAI,IAAI,iBAAiB,KAAA,KAAa,IAAI,iBAAiB,WACzD,MAAM,IAAI,yBACR,mDACA,aACF;CAEJ;CAEA,IACE,OAAO,IAAI,oBAAoB,YAC/B,CAAC,kBAAkB,SAAS,IAAI,eAAqC,GAErE,MAAM,IAAI,yBACR,kCAAkC,kBAAkB,KAAK,IAAI,KAC7D,iBACF;CAEF,IAAI,IAAI,0BAA0B,KAAA,GAAW;EAC3C,aAAa,IAAI,uBAAuB,uBAAuB;EAC/D,IACE,IAAI,oBAAoB,YACxB,IAAI,oBAAoB,eACxB,IAAI,oBAAoB,cAExB,MAAM,IAAI,yBACR,mFACA,uBACF;CAEJ;CAEA,IAAI,IAAI,iBAAiB,KAAA,GACvB,IAAI;EACF,MAAM,UAAU,yBAAyB,IAAI,YAAY;EACzD,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,IAAI,OACvD,MAAM,IAAI,yBACR,uBAAuB,QAAQ,MAAM,0BAA0B,IAAI,MAAM,IACzE,oBACF;EAEF,IAAI,QAAQ,eAAe,KAAA,KAAa,QAAQ,eAAe,IAAI,YACjE,MAAM,IAAI,yBACR,4BAA4B,QAAQ,WAAW,+BAA+B,IAAI,WAAW,IAC7F,yBACF;CAEJ,SAAS,OAAO;EACd,IAAI,iBAAiB,0BAA0B,MAAM;EACrD,IAAI,iBAAiB,OACnB,MAAM,IAAI,yBAAyB,MAAM,SAAS,cAAc;EAElE,MAAM;CACR;CAGF,aAAa,IAAI,YAAY,YAAY;CAEzC,IAAI,IAAI,WAAW,KAAA,GAAW,0BAA0B,IAAI,QAAQ,IAAI,KAAe;CACvF,IAAI,IAAI,aAAa,KAAA,GAAW,iBAAiB,IAAI,QAAQ;CAG7D,IAAI,OAAO,IAAI,aAAa,YAAY,CAAC,WAAW,SAAS,IAAI,QAAuB,GACtF,MAAM,IAAI,yBACR,2BAA2B,WAAW,KAAK,IAAI,EAAE,QAAQ,OAAO,IAAI,QAAQ,KAC5E,UACF;CAGF,OAAO;AACT;AAEA,SAAS,aAAa,SAAkB,YAA2B;CACjE,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,oCAAoC,gBAAgB;CAEzF,MAAM,QAAQ;CACd,IACE,MAAM,SAAS,cACf,MAAM,SAAS,eACf,MAAM,SAAS,iBACf,MAAM,SAAS,cAEf,MAAM,IAAI,yBACR,+EACA,qBACF;CAEF,IAAI,MAAM,SAAS,eAAe;EAChC,IAAI,MAAM,QAAQ,MAChB,MAAM,IAAI,yBACR,gFACA,oBACF;EAEF,IAAI,YAAY,MACd,MAAM,IAAI,yBACR,wEACA,SACF;EAEF,mBAAmB,MAAM,oBAAoB,mCAAmC;EAChF,IAAK,MAAM,sBAAiC,GAC1C,MAAM,IAAI,yBACR,yFACA,mCACF;EAEF;CACF;CAKA,IAAI,MAAM,SAAS,cAAc;EAC/B,IAAI,MAAM,QAAQ,MAChB,MAAM,IAAI,yBACR,8CACA,oBACF;EAEF,IAAI,YAAY,MACd,MAAM,IAAI,yBAAyB,+CAA+C,SAAS;EAE7F;CACF;CACA,wBAAwB,SAAS,SAAS;CAC1C,wBAAwB,MAAM,KAAK,oBAAoB;CACvD,IAAI,MAAM,QAAQ,SAChB,MAAM,IAAI,yBACR,yCACA,oBACF;AAEJ;AAEA,SAAS,0BAA0B,OAAgB,OAAqB;CACtE,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,4BAA4B,QAAQ;CAEzE,MAAM,SAAS;CACf,aAAa,OAAO,UAAU,iBAAiB;CAC/C,aAAa,OAAO,QAAQ,eAAe;CAC3C,aAAa,OAAO,QAAQ,eAAe;CAC3C,IAAI,CAAC,OAAO,cAAc,OAAO,OAAO,KAAM,OAAO,UAAqB,GACxE,MAAM,IAAI,yBAAyB,8BAA8B,gBAAgB;CAEnF,MAAM,WAAW,gBAAgB,MAAyC;CAC1E,IAAI,UAAU,UACZ,MAAM,IAAI,yBACR,UAAU,MAAM,aAAa,SAAS,4CACtC,OACF;AAEJ;AAEA,SAAS,iBAAiB,OAAsB;CAC9C,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,8BAA8B,UAAU;CAE7E,MAAM,MAAM;CACZ,aAAa,IAAI,SAAS,kBAAkB;CAC5C,IAAI,IAAI,cAAc,KAAA,GAAW,aAAa,IAAI,WAAW,oBAAoB;AACnF;;AAGA,SAAgB,YAAY,OAAoC;CAC9D,IAAI;EACF,kBAAkB,KAAK;EACvB,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;AAGA,SAAgB,mBACd,OACiF;CACjF,IAAI;EACF,OAAO;GAAE,IAAI;GAAM,OAAO,kBAAkB,KAAK;EAAE;CACrD,SAAS,GAAG;EACV,IAAI,aAAa,0BAA0B,OAAO;GAAE,IAAI;GAAO,OAAO;EAAE;EACxE,MAAM;CACR;AACF;AAIA,SAAS,aAAa,OAAgB,MAAoB;CACxD,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,MAAM,IAAI,yBAAyB,6BAA6B,IAAI;AAExE;AAEA,SAAS,mBAAmB,OAAgB,MAAoB;CAC9D,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GACrD,MAAM,IAAI,yBAAyB,0BAA0B,IAAI;AAErE;AAEA,SAAS,wBAAwB,OAAgB,MAAoB;CACnE,mBAAmB,OAAO,IAAI;CAC9B,IAAK,QAAmB,GACtB,MAAM,IAAI,yBAAyB,gCAAgC,IAAI;AAE3E;AAEA,SAAS,oBAAoB,OAAgB,MAAoB;CAC/D,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,MAAM,IAAI,yBAAyB,iCAAiC,IAAI;CAE1E,MAAM,MAAM;CAEZ,MAAM,WAAW,IAAI;CACrB,IAAI,aAAa,QAAQ,OAAO,aAAa,UAC3C,MAAM,IAAI,yBAAyB,8BAA8B,GAAG,KAAK,UAAU;CAErF,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,QAAmC,GAAG;EACjF,IAAI,SAAS,QAAQ,OAAO,SAAS,UACnC,MAAM,IAAI,yBACR,yDACA,GAAG,KAAK,YAAY,SACtB;EAEF,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,IAA+B,GACvE,mBAAmB,OAAO,GAAG,KAAK,YAAY,QAAQ,GAAG,KAAK;CAElE;CAEA,MAAM,aAAa,IAAI;CACvB,IAAI,eAAe,QAAQ,OAAO,eAAe,UAC/C,MAAM,IAAI,yBAAyB,gCAAgC,GAAG,KAAK,YAAY;CAEzF,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAqC,GAC5E,mBAAmB,MAAM,GAAG,KAAK,cAAc,KAAK;CAGtD,mBAAmB,IAAI,WAAW,GAAG,KAAK,WAAW;CAErD,IAAI,IAAI,iBAAiB,KAAA,GAAW;EAClC,IAAI,CAAC,MAAM,QAAQ,IAAI,YAAY,GACjC,MAAM,IAAI,yBACR,4CACA,GAAG,KAAK,cACV;EAEF,KAAK,IAAI,IAAI,GAAG,IAAI,IAAI,aAAa,QAAQ,KAAK;GAChD,MAAM,KAAK,IAAI,aAAa;GAC5B,IAAI,OAAO,OAAO,YAAY,GAAG,WAAW,GAC1C,MAAM,IAAI,yBACR,iDACA,GAAG,KAAK,gBAAgB,EAAE,EAC5B;EAEJ;CACF;CAEA,IAAI,IAAI,UAAU,KAAA,KAAa,OAAO,IAAI,UAAU,UAClD,MAAM,IAAI,yBAAyB,0BAA0B,GAAG,KAAK,OAAO;AAEhF;;;;;;;AAQA,SAAgB,iBAAiB,OAAwB;CACvD,IAAI,MAAM,WAAW,KAAK,MAAM,KAAK,MAAM,OAAO,OAAO;CAEzD,MAAM,WAAW,MAAM,YAAY,GAAG;CACtC,IAAI,WAAW,GAAG;EAChB,MAAM,OAAO,MAAM,MAAM,GAAG,QAAQ;EACpC,MAAM,QAAQ,MAAM,MAAM,WAAW,CAAC;EACtC,IAAI,CAAC,KAAK,SAAS,GAAG,KAAK,gDAAgD,KAAK,KAAK,GACnF,OAAO;CAEX;CAEA,MAAM,UAAU,MAAM,MAAM,4BAA4B;CACxD,IAAI,WAAW,kBAAkB,QAAQ,IAAK,QAAQ,IAAK,QAAQ,EAAG,GAAG,OAAO;CAEhF,MAAM,cAAc,MAAM,MAAM,0BAA0B;CAC1D,IAAI,eAAe,kBAAkB,YAAY,IAAK,YAAY,IAAK,YAAY,EAAG,GACpF,OAAO;CAGT,MAAM,aAAa,MAAM,MAAM,mBAAmB;CAClD,IAAI,cAAc,kBAAkB,KAAA,GAAW,WAAW,IAAK,WAAW,EAAG,GAAG,OAAO;CAEvF,OAAO,qDAAqD,KAAK,KAAK;AACxE;AAEA,SAAS,kBAAkB,MAA0B,OAAe,KAAsB;CACxF,MAAM,cAAc,OAAO,KAAK;CAChC,MAAM,YAAY,OAAO,GAAG;CAC5B,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,KAAK,cAAc,IAAI,OAAO;CAElF,MAAM,aAAa,SAAS,KAAA,IAAY,KAAA,IAAY,OAAO,IAAI;CAI/D,MAAM,cAAc;EAAC;EAFnB,eAAe,KAAA,KACd,aAAa,MAAM,MAAM,aAAa,QAAQ,KAAK,aAAa,QAAQ,KACvC,KAAK;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;EAAI;CAAE;CACnF,OAAO,OAAO,UAAU,SAAS,KAAK,aAAa,KAAK,aAAa,YAAY,cAAc;AACjG"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
//#region src/sandbox-harness.d.ts
|
|
2
1
|
interface HarnessConfig {
|
|
3
2
|
/** Setup command (e.g. "pnpm install"). Non-zero exit fails the run. */
|
|
4
3
|
setupCommand?: string;
|
|
@@ -65,6 +64,5 @@ interface SandboxHarnessResult {
|
|
|
65
64
|
/** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */
|
|
66
65
|
score: number;
|
|
67
66
|
}
|
|
68
|
-
//#endregion
|
|
69
67
|
export { SandboxDriver as n, SandboxHarnessResult as r, HarnessConfig as t };
|
|
70
|
-
//# sourceMappingURL=sandbox-harness-
|
|
68
|
+
//# sourceMappingURL=sandbox-harness-B3Kx1fOA.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sandbox-harness-B3Kx1fOA.d.ts","names":[],"sources":["../src/sandbox-harness.ts"],"mappings":"A,UAoBiB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;;EAEA;;EAEA,MAAM;;EAEN,aAAa;;UAGE;EACf;EACA,MACE,gBACA,gBACA;IACG;IAAoB;;;UAGV;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;;;;;EAOA;;;;;;EAMA;;UAGe;EACf;EACA,KACE,OAAO,wBACP,iBACA,QAAQ,gBACP,QAAQ;;UA0II;EACf;EACA,QAAQ;EACR,MAAM;EACN,OAAO;EACP;;EAEA"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
//#region src/trace/schema.d.ts
|
|
2
1
|
/**
|
|
3
2
|
* TraceSchema v1 — the canonical data model for agent-eval.
|
|
4
3
|
*
|
|
@@ -228,50 +227,10 @@ interface Artifact {
|
|
|
228
227
|
* `host-executor: acquire timeout after 900000ms (in_flight=10/10)` — the
|
|
229
228
|
* bridge's host lane had saturated.
|
|
230
229
|
*/
|
|
231
|
-
declare const FAILURE_CLASSES: readonly [
|
|
232
|
-
/** The bridge refused the connection or dropped it before a terminal event. */
|
|
233
|
-
'bridge_unreachable',
|
|
234
|
-
/** The bridge accepted the request and its execution lane was already full. */
|
|
235
|
-
'bridge_lane_saturated',
|
|
236
|
-
/** The router could not reconcile a completion with its own recorded usage. */
|
|
237
|
-
'router_usage_mismatch',
|
|
238
|
-
/** A session was bound to one AgentProfile and asked to serve another. */
|
|
239
|
-
'session_profile_conflict',
|
|
240
|
-
/** The agent runtime restarted underneath a live run. */
|
|
241
|
-
'runtime_restart',
|
|
242
|
-
/** The root driver of a recursive run failed before it could dispatch work. */
|
|
243
|
-
'root_driver_failed',
|
|
244
|
-
/** A sandbox box never finished provisioning. Distinct from `sandbox_failure`,
|
|
245
|
-
* which is a command the agent ran exiting non-zero inside a live box. */
|
|
246
|
-
'box_provision_failure',
|
|
247
|
-
/** A sandbox box did not confirm destruction after the run. */
|
|
248
|
-
'box_teardown_failure',
|
|
249
|
-
/** The harness refused an agent-authored profile: an unknown model, a
|
|
250
|
-
* workspace file it may not overwrite, a system prompt it may not replace. */
|
|
251
|
-
'profile_refused',
|
|
252
|
-
/** The model was reached and returned no visible output. */
|
|
253
|
-
'empty_completion',
|
|
254
|
-
/** The response stream carried malformed protocol content. */
|
|
255
|
-
'stream_protocol_error',
|
|
256
|
-
/** The response stream ended before a terminal event. */
|
|
257
|
-
'stream_incomplete',
|
|
258
|
-
/** A message was appended to a conversation the provider had already sealed. */
|
|
259
|
-
'message_sealed',
|
|
260
|
-
/** The model process was killed by a signal or exited with no code. */
|
|
261
|
-
'process_killed',
|
|
262
|
-
/** The run was cancelled by an operator. */
|
|
263
|
-
'cancelled',
|
|
264
|
-
/** The run was terminated without a more specific reason. */
|
|
265
|
-
'terminated',
|
|
266
|
-
/** The agent exhausted its traversal or iteration cap. */
|
|
267
|
-
'traversal_cap_exhausted',
|
|
268
|
-
/** No reason was recorded at all. Distinct from `unknown`, which means a
|
|
269
|
-
* reason exists and no rule matched it. */
|
|
270
|
-
'unreported', 'unknown'];
|
|
230
|
+
declare const FAILURE_CLASSES: readonly ["success", "reasoning_error", "tool_selection_error", "tool_argument_error", "tool_recovery_failure", "hallucination", "instruction_following", "safety_refusal_miss", "policy_violation", "budget_exceeded", "format_drift", "permission_escalation", "pii_leak", "cost_overrun", "timeout", "sandbox_failure", "missing_user_data", "missing_domain_data", "missing_codebase_context", "missing_runtime_context", "missing_credentials", "missing_integration_connection", "missing_integration_scope", "integration_approval_required", "integration_auth_expired", "integration_provider_failure", "bad_integration_manifest", "unsafe_integration_write_denied", "stale_external_data", "bad_retrieval", "insufficient_evidence", "contradictory_evidence", "ambiguous_user_intent", "knowledge_readiness_blocked", "bridge_unreachable", "bridge_lane_saturated", "router_usage_mismatch", "session_profile_conflict", "runtime_restart", "root_driver_failed", "box_provision_failure", "box_teardown_failure", "profile_refused", "empty_completion", "stream_protocol_error", "stream_incomplete", "message_sealed", "process_killed", "cancelled", "terminated", "traversal_cap_exhausted", "unreported", "unknown"];
|
|
271
231
|
type FailureClass = (typeof FAILURE_CLASSES)[number];
|
|
272
232
|
declare function isLlmSpan(s: Span): s is LlmSpan;
|
|
273
233
|
declare function isToolSpan(s: Span): s is ToolSpan;
|
|
274
234
|
declare function isJudgeSpan(s: Span): s is JudgeSpan;
|
|
275
|
-
//#endregion
|
|
276
235
|
export { TraceRunOutcome as C, isToolSpan as E, TraceEvent as S, isLlmSpan as T, Span as _, FAILURE_CLASSES as a, SpanStatus as b, JudgeSpan as c, RetrievalSpan as d, Run as f, SandboxSpan as g, RunStatus as h, EventKind as i, LlmSpan as l, RunLayer as m, BudgetLedgerEntry as n, FailureClass as o, RunCaptureReport as p, BudgetSpec as r, GenericSpan as s, Artifact as t, Message as u, SpanBase as v, isJudgeSpan as w, ToolSpan as x, SpanKind as y };
|
|
277
|
-
//# sourceMappingURL=schema-
|
|
236
|
+
//# sourceMappingURL=schema-CkIvnmLU.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-CkIvnmLU.d.ts","names":[],"sources":["../src/trace/schema.ts"],"mappings":";;;;;;;;;;;KAgBY;UAEK;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,eAAe;EACf;;;;;;;;;KAUU;UAEK;EACf;;;;;;;;;;;EAWA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;EAEA,iBAAiB;;EAEjB;;;EAGA;;EAEA;;EAEA;;EAEA,QAAQ;EACR;EACA;EACA,QAAQ;EACR,UAAU;EACV,SAAS;;EAET,OAAO;;;;;;EAMP,UAAU;;;UAIK;;EAEf;;EAEA;;EAEA;;KAKU;KAEA;UAEK;EACf;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,SAAS;EACT;;EAEA,aAAa;;UAGE;EACf;EACA;EACA;;EAEA,SAAS;IAAQ;IAAqB;IAAc;;;UAGrC,gBAAgB;EAC/B;EACA;EACA,UAAU;EACV;EACA;;EAEA;EACA;EACA;;EAEA;EACA;EACA;;UAGe,iBAAiB;EAChC;EACA;EACA;;EAEA;EACA;EACA;;UAGe,sBAAsB;EACrC;EACA;EACA,MAAM;IAAQ;IAAe;IAAe;;;UAG7B,kBAAkB;EACjC;EACA;;EAEA;EACA;;EAEA;EACA;EACA;;UAGe,oBAAoB;EACnC;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;UAGe,oBAAoB;EACnC;;KAGU,OAAO,UAAU,WAAW,gBAAgB,YAAY,cAAc;KAItE;UAUK;EACf;EACA;EACA;EACA,MAAM;EACN;EACA,SAAS;;UAKM;EACf;EACA,iBAAiB;EACjB;EACA;EACA;EACA;EACA;;EAEA;;UAKe;EACf;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;;;;;;;cAyBW;KAgFD,uBAAuB;iBAInB,UAAU,GAAG,OAAO,KAAK;iBAGzB,WAAW,GAAG,OAAO,KAAK;iBAG1B,YAAY,GAAG,OAAO,KAAK"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
// src/trace/schema.ts
|
|
2
2
|
/**
|
|
3
3
|
* The failure taxonomy. `FailureClass` derives from this array, so the type
|
|
4
4
|
* and the runtime list cannot name different sets.
|
|
@@ -83,7 +83,6 @@ function isToolSpan(s) {
|
|
|
83
83
|
function isJudgeSpan(s) {
|
|
84
84
|
return s.kind === "judge";
|
|
85
85
|
}
|
|
86
|
-
//#endregion
|
|
87
86
|
export { isToolSpan as i, isJudgeSpan as n, isLlmSpan as r, FAILURE_CLASSES as t };
|
|
88
87
|
|
|
89
|
-
//# sourceMappingURL=schema-
|
|
88
|
+
//# sourceMappingURL=schema-D-xu3MU_.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema-Bt2vLFm3.js","names":[],"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\n\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n tokens?: number\n wallMs?: number\n calls?: number\n usd?: number\n}\n\nexport interface TraceRunOutcome {\n score?: number\n pass?: boolean\n failureClass?: FailureClass\n notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n runId: string\n /**\n * Stable identifier of the scenario being executed.\n *\n * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n * input WITHOUT this field, substituting a sensible default\n * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n * keeps the persisted shape unambiguous for downstream filters + aggregations\n * while removing the boilerplate of inventing placeholder ids at the call site.\n */\n scenarioId: string\n variantId?: string\n datasetVersion?: string\n /** Git SHA of agent code at run time. */\n codeSha?: string\n /** Hash of the prompt template + any system prompt. */\n promptSha?: string\n /** Model id + date + system-prompt hash, concatenated. */\n modelFingerprint?: string\n seed?: number\n /** Arbitrary environment markers (shell, docker version, tz). */\n envFingerprint?: Record<string, string>\n /** Version of the redaction rules applied to this run. */\n redactionVersion?: string\n /** Parent run in a nested build workflow. A builder run's children are\n * app-build runs; those children are app-runtime runs. */\n parentRunId?: string\n /** Stable project identifier — groups runs across chats + sessions. */\n projectId?: string\n /** Chat/conversation identifier within a project. */\n chatId?: string\n /** Layer classification — hint for aggregation; not enforced. */\n layer?: RunLayer\n startedAt: number\n endedAt?: number\n status: RunStatus\n outcome?: TraceRunOutcome\n budget?: BudgetSpec\n /** Free-form labels for downstream grouping. */\n tags?: Record<string, string>\n /**\n * The producer's own account of its store writes for this run, written when\n * the run ends. Absent when the producer kept no account, which is unknown,\n * not complete.\n */\n capture?: RunCaptureReport\n}\n\n/** Store writes a producer made for one run. */\nexport interface RunCaptureReport {\n /** Writes that landed. */\n written: number\n /** Writes that failed; those records are missing from the store. */\n dropped: number\n /** The most recent failure, prefixed with the write that failed. */\n lastError?: string\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n spanId: string\n parentSpanId?: string\n runId: string\n kind: SpanKind\n name: string\n startedAt: number\n endedAt?: number\n status?: SpanStatus\n error?: string\n /** Anything not covered by typed fields. Kept deliberately free-form. */\n attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n role: 'system' | 'user' | 'assistant' | 'tool'\n content: string\n tokens?: number\n /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n kind: 'llm'\n model: string\n messages: Message[]\n output?: string\n inputTokens?: number\n /** All generated tokens, including the reasoning subset when present. */\n outputTokens?: number\n cachedTokens?: number\n cacheWriteTokens?: number\n /** Reasoning-token subset of `outputTokens`. */\n reasoningTokens?: number\n costUsd?: number\n finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n kind: 'tool'\n toolName: string\n args: unknown\n /** False when the source observed the call but did not capture its arguments. */\n argsCaptured?: boolean\n result?: unknown\n latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n kind: 'retrieval'\n query: string\n hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n kind: 'judge'\n judgeId: string\n /** Span this judgment applies to. */\n targetSpanId: string\n dimension: string\n /** Numeric score (free-range; interpretation up to the judge). */\n score: number\n rationale?: string\n evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n kind: 'sandbox'\n image?: string\n command?: string\n exitCode?: number\n testsTotal?: number\n testsPassed?: number\n stdoutHash?: string\n stderrHash?: string\n /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n | 'log'\n | 'error'\n | 'budget_decrement'\n | 'budget_breach'\n | 'state_mutation'\n | 'policy_violation'\n | 'redaction_applied'\n | 'custom'\n\nexport interface TraceEvent {\n eventId: string\n runId: string\n spanId?: string\n kind: EventKind\n timestamp: number\n payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n runId: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n remaining: number\n timestamp: number\n breached: boolean\n /** Span that triggered this entry, if any. */\n spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n artifactId: string\n runId: string\n spanId?: string\n contentType: string\n sizeBytes: number\n /** sha256 in hex. */\n hash: string\n /** External storage URL (R2, S3, filesystem path). */\n storageUrl?: string\n /** Inline content for small blobs — keep under ~64KB. */\n inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\n/**\n * The failure taxonomy. `FailureClass` derives from this array, so the type\n * and the runtime list cannot name different sets.\n *\n * Two layers, and the split is what `FAILURE_BLAME` reads\n * (`src/failure-taxonomy.ts`):\n * - behaviour classes name what the agent did wrong, and a run that ends in\n * one is evidence about the agent;\n * - transport classes name a death before or around the model turn — the\n * bridge refused, the box never provisioned, the stream tore — and a run\n * that ends in one of the machine-blamed members is not evidence about the\n * agent at all.\n *\n * The transport layer exists because a taxonomy with no name for \"the run\n * never reached a model\" charges an infrastructure outage to the agent under\n * test. Measured: six director cells settled with the runtime journal's own\n * `infra` flag reading FALSE while the real reason was\n * `host-executor: acquire timeout after 900000ms (in_flight=10/10)` — the\n * bridge's host lane had saturated.\n */\nexport const FAILURE_CLASSES = [\n 'success',\n 'reasoning_error',\n 'tool_selection_error',\n 'tool_argument_error',\n 'tool_recovery_failure',\n 'hallucination',\n 'instruction_following',\n 'safety_refusal_miss',\n 'policy_violation',\n 'budget_exceeded',\n 'format_drift',\n 'permission_escalation',\n 'pii_leak',\n 'cost_overrun',\n 'timeout',\n 'sandbox_failure',\n 'missing_user_data',\n 'missing_domain_data',\n 'missing_codebase_context',\n 'missing_runtime_context',\n 'missing_credentials',\n 'missing_integration_connection',\n 'missing_integration_scope',\n 'integration_approval_required',\n 'integration_auth_expired',\n 'integration_provider_failure',\n 'bad_integration_manifest',\n 'unsafe_integration_write_denied',\n 'stale_external_data',\n 'bad_retrieval',\n 'insufficient_evidence',\n 'contradictory_evidence',\n 'ambiguous_user_intent',\n 'knowledge_readiness_blocked',\n\n // ── Transport: the run never reached a model, or the turn never completed ──\n /** The bridge refused the connection or dropped it before a terminal event. */\n 'bridge_unreachable',\n /** The bridge accepted the request and its execution lane was already full. */\n 'bridge_lane_saturated',\n /** The router could not reconcile a completion with its own recorded usage. */\n 'router_usage_mismatch',\n /** A session was bound to one AgentProfile and asked to serve another. */\n 'session_profile_conflict',\n /** The agent runtime restarted underneath a live run. */\n 'runtime_restart',\n /** The root driver of a recursive run failed before it could dispatch work. */\n 'root_driver_failed',\n /** A sandbox box never finished provisioning. Distinct from `sandbox_failure`,\n * which is a command the agent ran exiting non-zero inside a live box. */\n 'box_provision_failure',\n /** A sandbox box did not confirm destruction after the run. */\n 'box_teardown_failure',\n /** The harness refused an agent-authored profile: an unknown model, a\n * workspace file it may not overwrite, a system prompt it may not replace. */\n 'profile_refused',\n /** The model was reached and returned no visible output. */\n 'empty_completion',\n /** The response stream carried malformed protocol content. */\n 'stream_protocol_error',\n /** The response stream ended before a terminal event. */\n 'stream_incomplete',\n /** A message was appended to a conversation the provider had already sealed. */\n 'message_sealed',\n /** The model process was killed by a signal or exited with no code. */\n 'process_killed',\n /** The run was cancelled by an operator. */\n 'cancelled',\n /** The run was terminated without a more specific reason. */\n 'terminated',\n /** The agent exhausted its traversal or iteration cap. */\n 'traversal_cap_exhausted',\n\n /** No reason was recorded at all. Distinct from `unknown`, which means a\n * reason exists and no rule matched it. */\n 'unreported',\n 'unknown',\n] as const\n\nexport type FailureClass = (typeof FAILURE_CLASSES)[number]\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n return s.kind === 'tool'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n return s.kind === 'judge'\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAyQA,MAAa,kBAAkB;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA;CAGA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAIA;CACA;AACF;AAMA,SAAgB,UAAU,GAAuB;CAC/C,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,WAAW,GAAwB;CACjD,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,YAAY,GAAyB;CACnD,OAAO,EAAE,SAAS;AACpB"}
|
|
1
|
+
{"version":3,"file":"schema-D-xu3MU_.js","names":[],"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\n\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n tokens?: number\n wallMs?: number\n calls?: number\n usd?: number\n}\n\nexport interface TraceRunOutcome {\n score?: number\n pass?: boolean\n failureClass?: FailureClass\n notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n runId: string\n /**\n * Stable identifier of the scenario being executed.\n *\n * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n * input WITHOUT this field, substituting a sensible default\n * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n * keeps the persisted shape unambiguous for downstream filters + aggregations\n * while removing the boilerplate of inventing placeholder ids at the call site.\n */\n scenarioId: string\n variantId?: string\n datasetVersion?: string\n /** Git SHA of agent code at run time. */\n codeSha?: string\n /** Hash of the prompt template + any system prompt. */\n promptSha?: string\n /** Model id + date + system-prompt hash, concatenated. */\n modelFingerprint?: string\n seed?: number\n /** Arbitrary environment markers (shell, docker version, tz). */\n envFingerprint?: Record<string, string>\n /** Version of the redaction rules applied to this run. */\n redactionVersion?: string\n /** Parent run in a nested build workflow. A builder run's children are\n * app-build runs; those children are app-runtime runs. */\n parentRunId?: string\n /** Stable project identifier — groups runs across chats + sessions. */\n projectId?: string\n /** Chat/conversation identifier within a project. */\n chatId?: string\n /** Layer classification — hint for aggregation; not enforced. */\n layer?: RunLayer\n startedAt: number\n endedAt?: number\n status: RunStatus\n outcome?: TraceRunOutcome\n budget?: BudgetSpec\n /** Free-form labels for downstream grouping. */\n tags?: Record<string, string>\n /**\n * The producer's own account of its store writes for this run, written when\n * the run ends. Absent when the producer kept no account, which is unknown,\n * not complete.\n */\n capture?: RunCaptureReport\n}\n\n/** Store writes a producer made for one run. */\nexport interface RunCaptureReport {\n /** Writes that landed. */\n written: number\n /** Writes that failed; those records are missing from the store. */\n dropped: number\n /** The most recent failure, prefixed with the write that failed. */\n lastError?: string\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n spanId: string\n parentSpanId?: string\n runId: string\n kind: SpanKind\n name: string\n startedAt: number\n endedAt?: number\n status?: SpanStatus\n error?: string\n /** Anything not covered by typed fields. Kept deliberately free-form. */\n attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n role: 'system' | 'user' | 'assistant' | 'tool'\n content: string\n tokens?: number\n /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n kind: 'llm'\n model: string\n messages: Message[]\n output?: string\n inputTokens?: number\n /** All generated tokens, including the reasoning subset when present. */\n outputTokens?: number\n cachedTokens?: number\n cacheWriteTokens?: number\n /** Reasoning-token subset of `outputTokens`. */\n reasoningTokens?: number\n costUsd?: number\n finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n kind: 'tool'\n toolName: string\n args: unknown\n /** False when the source observed the call but did not capture its arguments. */\n argsCaptured?: boolean\n result?: unknown\n latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n kind: 'retrieval'\n query: string\n hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n kind: 'judge'\n judgeId: string\n /** Span this judgment applies to. */\n targetSpanId: string\n dimension: string\n /** Numeric score (free-range; interpretation up to the judge). */\n score: number\n rationale?: string\n evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n kind: 'sandbox'\n image?: string\n command?: string\n exitCode?: number\n testsTotal?: number\n testsPassed?: number\n stdoutHash?: string\n stderrHash?: string\n /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n | 'log'\n | 'error'\n | 'budget_decrement'\n | 'budget_breach'\n | 'state_mutation'\n | 'policy_violation'\n | 'redaction_applied'\n | 'custom'\n\nexport interface TraceEvent {\n eventId: string\n runId: string\n spanId?: string\n kind: EventKind\n timestamp: number\n payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n runId: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n remaining: number\n timestamp: number\n breached: boolean\n /** Span that triggered this entry, if any. */\n spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n artifactId: string\n runId: string\n spanId?: string\n contentType: string\n sizeBytes: number\n /** sha256 in hex. */\n hash: string\n /** External storage URL (R2, S3, filesystem path). */\n storageUrl?: string\n /** Inline content for small blobs — keep under ~64KB. */\n inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\n/**\n * The failure taxonomy. `FailureClass` derives from this array, so the type\n * and the runtime list cannot name different sets.\n *\n * Two layers, and the split is what `FAILURE_BLAME` reads\n * (`src/failure-taxonomy.ts`):\n * - behaviour classes name what the agent did wrong, and a run that ends in\n * one is evidence about the agent;\n * - transport classes name a death before or around the model turn — the\n * bridge refused, the box never provisioned, the stream tore — and a run\n * that ends in one of the machine-blamed members is not evidence about the\n * agent at all.\n *\n * The transport layer exists because a taxonomy with no name for \"the run\n * never reached a model\" charges an infrastructure outage to the agent under\n * test. Measured: six director cells settled with the runtime journal's own\n * `infra` flag reading FALSE while the real reason was\n * `host-executor: acquire timeout after 900000ms (in_flight=10/10)` — the\n * bridge's host lane had saturated.\n */\nexport const FAILURE_CLASSES = [\n 'success',\n 'reasoning_error',\n 'tool_selection_error',\n 'tool_argument_error',\n 'tool_recovery_failure',\n 'hallucination',\n 'instruction_following',\n 'safety_refusal_miss',\n 'policy_violation',\n 'budget_exceeded',\n 'format_drift',\n 'permission_escalation',\n 'pii_leak',\n 'cost_overrun',\n 'timeout',\n 'sandbox_failure',\n 'missing_user_data',\n 'missing_domain_data',\n 'missing_codebase_context',\n 'missing_runtime_context',\n 'missing_credentials',\n 'missing_integration_connection',\n 'missing_integration_scope',\n 'integration_approval_required',\n 'integration_auth_expired',\n 'integration_provider_failure',\n 'bad_integration_manifest',\n 'unsafe_integration_write_denied',\n 'stale_external_data',\n 'bad_retrieval',\n 'insufficient_evidence',\n 'contradictory_evidence',\n 'ambiguous_user_intent',\n 'knowledge_readiness_blocked',\n\n // ── Transport: the run never reached a model, or the turn never completed ──\n /** The bridge refused the connection or dropped it before a terminal event. */\n 'bridge_unreachable',\n /** The bridge accepted the request and its execution lane was already full. */\n 'bridge_lane_saturated',\n /** The router could not reconcile a completion with its own recorded usage. */\n 'router_usage_mismatch',\n /** A session was bound to one AgentProfile and asked to serve another. */\n 'session_profile_conflict',\n /** The agent runtime restarted underneath a live run. */\n 'runtime_restart',\n /** The root driver of a recursive run failed before it could dispatch work. */\n 'root_driver_failed',\n /** A sandbox box never finished provisioning. Distinct from `sandbox_failure`,\n * which is a command the agent ran exiting non-zero inside a live box. */\n 'box_provision_failure',\n /** A sandbox box did not confirm destruction after the run. */\n 'box_teardown_failure',\n /** The harness refused an agent-authored profile: an unknown model, a\n * workspace file it may not overwrite, a system prompt it may not replace. */\n 'profile_refused',\n /** The model was reached and returned no visible output. */\n 'empty_completion',\n /** The response stream carried malformed protocol content. */\n 'stream_protocol_error',\n /** The response stream ended before a terminal event. */\n 'stream_incomplete',\n /** A message was appended to a conversation the provider had already sealed. */\n 'message_sealed',\n /** The model process was killed by a signal or exited with no code. */\n 'process_killed',\n /** The run was cancelled by an operator. */\n 'cancelled',\n /** The run was terminated without a more specific reason. */\n 'terminated',\n /** The agent exhausted its traversal or iteration cap. */\n 'traversal_cap_exhausted',\n\n /** No reason was recorded at all. Distinct from `unknown`, which means a\n * reason exists and no rule matched it. */\n 'unreported',\n 'unknown',\n] as const\n\nexport type FailureClass = (typeof FAILURE_CLASSES)[number]\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n return s.kind === 'tool'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n return s.kind === 'judge'\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAyQA,MAAa,kBAAkB;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAIA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA;CAGA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAIA;CACA;AACF;AAMA,SAAgB,UAAU,GAAuB;CAC/C,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,WAAW,GAAwB;CACjD,OAAO,EAAE,SAAS;AACpB;AACA,SAAgB,YAAY,GAAyB;CACnD,OAAO,EAAE,SAAS;AACpB"}
|
|
@@ -1,52 +1,114 @@
|
|
|
1
|
-
//#region src/rollout/schema.d.ts
|
|
2
1
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
*
|
|
13
|
-
* - the bench rollout-ledger (agent-runtime PR #591): the wire shape —
|
|
14
|
-
* role, task.split/rep, parent_rollout_id, policy provenance, capture
|
|
15
|
-
* provenance, inline canonical chat-with-tools messages.
|
|
16
|
-
* Where the two conflicted, RunRecord-derived semantics won; the wire
|
|
17
|
-
* field names follow the ledger (snake_case). See `docs/rollout.md` for
|
|
18
|
-
* the field-by-field decision table.
|
|
19
|
-
*
|
|
20
|
-
* Messages are inlined — never referenced — because every harness store a
|
|
21
|
-
* rollout can be recovered from is mutable or garbage-collected. A line
|
|
22
|
-
* must stay a complete training/eval example on its own.
|
|
23
|
-
*
|
|
24
|
-
* `outcome.reward` is THE single scalar (null = no verdict exists — a
|
|
25
|
-
* labeled gap, never 0). `outcome.realness_gated` is the anti-Goodhart
|
|
26
|
-
* flag: a gated line must never export as a positive training example.
|
|
27
|
-
*
|
|
28
|
-
* That last sentence is enforced here, by `validateRolloutLine`, not merely
|
|
29
|
-
* documented. Validating `reward` and `realness_gated` independently — each a
|
|
30
|
-
* well-typed field, their COMBINATION unchecked — is what let a line claiming
|
|
31
|
-
* `{reward: 0.95, realness_gated: true}` validate clean and walk into every
|
|
32
|
-
* training export. The relationship between the two IS the invariant, so it is
|
|
33
|
-
* checked where every other structural claim about a line is checked.
|
|
2
|
+
* Descriptive statistics: means, bootstrap spread, correlation, and the
|
|
3
|
+
* weighted judge-dimension composite. Nothing here is a significance test.
|
|
4
|
+
*/
|
|
5
|
+
/** Weighted mean — falls back to uniform weights when omitted */
|
|
6
|
+
declare function weightedMean(scores: {
|
|
7
|
+
score: number;
|
|
8
|
+
weight?: number;
|
|
9
|
+
}[]): number;
|
|
10
|
+
/**
|
|
11
|
+
* Percentile bootstrap confidence interval on the mean of `scores`.
|
|
34
12
|
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
13
|
+
* Descriptive spread. It is not a significance test, and at small n its bounds
|
|
14
|
+
* are anti-conservative in the same way {@link pairedBootstrap}'s are — see
|
|
15
|
+
* {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from
|
|
16
|
+
* the scores themselves, so the interval is reproducible either way.
|
|
17
|
+
*/
|
|
18
|
+
declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
|
|
19
|
+
seed?: number;
|
|
20
|
+
resamples?: number;
|
|
21
|
+
}): {
|
|
22
|
+
mean: number;
|
|
23
|
+
lower: number;
|
|
24
|
+
upper: number;
|
|
25
|
+
};
|
|
26
|
+
/** Partial credit: returns 0-1 ratio of current toward target */
|
|
27
|
+
declare function partialCredit(current: number, target: number): number;
|
|
28
|
+
/** Distribution summary of a number series: count, extremes, quantiles, sum. */
|
|
29
|
+
interface SeriesDistribution {
|
|
30
|
+
readonly n: number;
|
|
31
|
+
readonly min: number;
|
|
32
|
+
readonly p50: number;
|
|
33
|
+
readonly p90: number;
|
|
34
|
+
readonly max: number;
|
|
35
|
+
readonly sum: number;
|
|
36
|
+
readonly percentiles?: Readonly<Record<number, number>>;
|
|
37
|
+
readonly medianInterval?: {
|
|
38
|
+
readonly method: 'normal-order-statistics';
|
|
39
|
+
readonly criticalValue: number;
|
|
40
|
+
readonly sampleSize: number;
|
|
41
|
+
readonly lowRank: number;
|
|
42
|
+
readonly highRank: number;
|
|
43
|
+
readonly low: number;
|
|
44
|
+
readonly high: number;
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
interface SeriesDistributionOptions {
|
|
48
|
+
/** Nearest-rank percentiles on a 0–100 scale, including fractional tails. */
|
|
49
|
+
percentiles?: readonly number[];
|
|
50
|
+
/** Descriptive median rank band, not a bootstrap interval or promotion test. */
|
|
51
|
+
medianInterval?: {
|
|
52
|
+
criticalValue: number;
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Fold a number series into its distribution summary. Quantiles use the
|
|
57
|
+
* nearest-rank definition — the `ceil(q·n)`-th order statistic — so every
|
|
58
|
+
* reported quantile is a value from the series. Returns `null` for an empty
|
|
59
|
+
* series: an empty series has no distribution, and a zero-filled summary
|
|
60
|
+
* would read as a measured all-zero series.
|
|
61
|
+
*/
|
|
62
|
+
declare function summarizeNumberSeries(values: readonly number[], options?: SeriesDistributionOptions): SeriesDistribution | null;
|
|
63
|
+
/**
|
|
64
|
+
* Average-rank-with-ties transform (1-indexed). Tied values receive the mean
|
|
65
|
+
* of the ranks they span, the standard correction for Spearman's ρ.
|
|
66
|
+
*/
|
|
67
|
+
declare function ranks(xs: number[]): number[];
|
|
68
|
+
/**
|
|
69
|
+
* Pearson product-moment correlation coefficient r ∈ [-1, 1] between two
|
|
70
|
+
* equal-length series. See the edge-case contract above: NaN for n < 2 or
|
|
71
|
+
* unequal lengths, 1 when both series are constant, 0 when exactly one is.
|
|
72
|
+
*/
|
|
73
|
+
declare function pearsonR(a: number[], b: number[]): number;
|
|
74
|
+
/**
|
|
75
|
+
* Spearman's rank correlation ρ — Pearson over the average-rank-with-ties
|
|
76
|
+
* transform of each series. Same edge-case contract as {@link pearsonR}.
|
|
77
|
+
*/
|
|
78
|
+
declare function spearmanR(a: number[], b: number[]): number;
|
|
79
|
+
interface WeightedCompositeInput {
|
|
80
|
+
/** Per-dimension scores (typically 0..1). */
|
|
81
|
+
dims: Record<string, number>;
|
|
82
|
+
/** Weight per dimension. Every weighted dimension MUST be present in
|
|
83
|
+
* `dims` — a weight for an absent dimension is a config error and throws,
|
|
84
|
+
* because silently dropping it would renormalise the composite onto a
|
|
85
|
+
* different denominator than intended. */
|
|
86
|
+
weights: Record<string, number>;
|
|
87
|
+
/** Optional pass threshold; when set, the result reports `pass`. */
|
|
88
|
+
threshold?: number;
|
|
89
|
+
}
|
|
90
|
+
interface WeightedCompositeResult {
|
|
91
|
+
composite: number;
|
|
92
|
+
pass?: boolean;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Weighted composite over judge dimensions: `Σ(score_d · w_d) / Σ(w_d)` across
|
|
96
|
+
* the weighted dimensions. The canonical replacement for the per-consumer
|
|
97
|
+
* hand-rolled composite math (tax/legal/creative/gtm each ship a copy).
|
|
42
98
|
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* `gateErrors` with their declared policy, so a check added to that list applies
|
|
47
|
-
* here without anyone editing this file, and a check deliberately skipped has to
|
|
48
|
-
* name itself there.
|
|
99
|
+
* Fail-loud: throws if a weighted dimension is missing from `dims`, if any
|
|
100
|
+
* weight is negative, or if the weights sum to 0 — none of which can produce
|
|
101
|
+
* a meaningful composite.
|
|
49
102
|
*/
|
|
103
|
+
declare function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult;
|
|
104
|
+
type LicenseClass = 'open-weight-api' | 'restricted' | 'evaluation-only' | 'human';
|
|
105
|
+
declare const LICENSE_CLASSES: readonly LicenseClass[];
|
|
106
|
+
/** Classify a model or harness name; `null` when it names no known family. */
|
|
107
|
+
declare function licenseClassOfName(name: string | null | undefined): LicenseClass | null;
|
|
108
|
+
/** The line's stated class, else the class its model or harness implies. */
|
|
109
|
+
declare function licenseClassOf(line: Pick<RolloutLine, 'policy'>): LicenseClass | null;
|
|
110
|
+
/** Every line passes without `allowed`; with it, the line's class must be listed. */
|
|
111
|
+
declare function isLicenseEligible(line: Pick<RolloutLine, 'policy'>, allowed?: readonly LicenseClass[]): boolean;
|
|
50
112
|
declare const ROLLOUT_SCHEMA = "tangle.rollout.v1";
|
|
51
113
|
/** `agent` = a solo evaluation run (no multi-agent topology). */
|
|
52
114
|
type RolloutRole = 'agent' | 'supervisor' | 'worker' | 'proposer' | 'judge' | 'analyst';
|
|
@@ -150,6 +212,8 @@ interface RolloutPolicy {
|
|
|
150
212
|
agent_profile_cell_id?: string | null;
|
|
151
213
|
/** Sampling params (temperature, top_p, max_tokens…); null = not recorded. */
|
|
152
214
|
sampling: Record<string, unknown> | null;
|
|
215
|
+
/** Terms the producer's outputs carry; absent = inferred (see `./license`). */
|
|
216
|
+
license_class?: LicenseClass;
|
|
153
217
|
}
|
|
154
218
|
interface RolloutOutcome {
|
|
155
219
|
/**
|
|
@@ -437,6 +501,5 @@ type MintedRolloutLine = Omit<RolloutLine, 'outcome'> & {
|
|
|
437
501
|
declare function assertMinted(value: unknown, context?: string): MintedRolloutLine;
|
|
438
502
|
/** `assertMinted` over a batch, naming the offending index in the error. */
|
|
439
503
|
declare function assertMintedLines(values: readonly unknown[], context?: string): MintedRolloutLine[];
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
//# sourceMappingURL=schema-C-ziyJD7.d.ts.map
|
|
504
|
+
export { isTrainableSplit as A, WeightedCompositeResult as B, RolloutWorkspaceCapture as C, assertRolloutLine as D, assertMintedLines as E, licenseClassOf as F, spearmanR as G, partialCredit as H, licenseClassOfName as I, weightedMean as J, summarizeNumberSeries as K, SeriesDistribution as L, LICENSE_CLASSES as M, LicenseClass as N, gateGamedOutcome as O, isLicenseEligible as P, SeriesDistributionOptions as R, RolloutTask as S, assertMinted as T, pearsonR as U, confidenceInterval as V, ranks as W, RolloutPolicy as _, MintedRolloutLine as a, RolloutSplit as b, ROLLOUT_ROLES as c, RolloutArtifacts as d, RolloutCapture as f, RolloutOutcome as g, RolloutLine as h, GatedEvidence as i, validateRolloutLine as j, isRolloutLine as k, ROLLOUT_SCHEMA as l, RolloutEvidenceAttempt as m, ChatRole as n, MintedRolloutOutcome as o, RolloutCostBlock as p, weightedComposite as q, ChatToolCall as r, ROLLOUT_CAPTURES as s, ChatMessage as t, ROLLOUT_SPLITS as u, RolloutProvenance as v, ToolDef as w, RolloutStep as x, RolloutRole as y, WeightedCompositeInput as z };
|
|
505
|
+
//# sourceMappingURL=schema-DkFJr3Y8.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-DkFJr3Y8.d.ts","names":[],"sources":["../src/statistics/descriptive.ts","../src/rollout/license.ts","../src/rollout/schema.ts"],"mappings":";;;;;iBASgB,aAAa;EAAU;EAAe;;;;;;;;;;iBAoBtC,mBACd,kBACA,qBACA;EAAQ;EAAe;;EACpB;EAAc;EAAe;;;iBAiClB,cAAc,iBAAiB;;UAM9B;WACN;WACA;WACA;WACA;WACA;WACA;WACA,cAAc,SAAS;WACvB;aACE;aACA;aACA;aACA;aACA;aACA;aACA;;;UAII;;EAEf;;EAEA;IAAmB;;;;;;;;;;iBAUL,sBACd,2BACA,UAAU,4BACT;;;;;iBA6Ea,MAAM;;;;;;iBAmBN,SAAS,aAAa;;;;;iBAuBtB,UAAU,aAAa;UAKtB;;EAEf,MAAM;;;;;EAKN,SAAS;;EAET;;UAGe;EACf;EACA;;;;;;;;;;;iBAYc,kBAAkB,OAAO,yBAAyB;KCzOtD;cACC,0BAA0B;;iBAYvB,mBAAmB,kCAAkC;;iBAQrD,eAAe,MAAM,KAAK,yBAAyB;;iBASnD,kBACd,MAAM,KAAK,wBACX,mBAAmB;cCLR;;KAGD;cACC,wBAAwB;;KAUzB;cACC,yBAAyB;iBAItB,iBAAiB,OAAO;;KAK5B;cACC,2BAA2B;KAM5B;UAGK;EACf;EACA;EACA;IACE;;IAEA;;;UAIa;EACf,MAAM;EACN;;EAEA;EACA,aAAa;;EAEb;EACA;;;;;;;;EAQA;;UAGe;EACf;EACA;IACE;IACA;IACA,aAAa;;;;;;;;UASA;EACf;EACA;;EAEA;;EAEA;EACA;EACA;;;;;EAUA;;EAEA;;EAEA;;;;;;EAMA;;UAOe;;EAEf;EACA;EACA,OAAO;;EAEP;;EAEA;;UAGe;;EAEf;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,UAAU;;EAEV,gBAAgB;;UAGD;;;;;EAKf;;EAEA;;EAEA;;EAEA,SAAS;EACT;EACA;EACA;;;;;;;;;;;;;EAaA;;;;;;;;;;;;;;;;;;;;;EAqBA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;;;EAKA;;UAGe;EACf;EACA;;;EAGA;;;;;;;UAQe;;EAEf,UAAU;;EAEV;;;;;;;;;EASA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;WACN;WACA;;WAEA;WACA;WACA;;WAEA;WACA;WACA;;UAGM;EACf;EACA,SAAS;;EAET;IAAW;IAAkB;IAAgB;IAAgB;;;EAE7D,WAAW;;EAEX,8BAA8B;;EAE9B;;;;;;;EAOA;;;;;;;EAOA,iBAAiB;;UAGF;EACf,eAAe;EACf;;;EAGA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;EACA,MAAM;EACN,MAAM;EACN,QAAQ;;EAER,UAAU;EACV,WAAW;;EAEX,QAAQ;EACR,SAAS;EACT,MAAM;EACN,WAAW;EACX,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA0GE,iBAAiB,MAAM,cAAc;iBAgXrC,oBAAoB;iBAmBpB,kBACd,gBACA,2BACS,SAAS;iBAOJ,cAAc,iBAAiB,SAAS;;;;;;cAa1C;;;;;;UAOG,6BAA6B;EAC5C;;;;;;;;;;;;;;;;;;;;;;KAuBU,oBAAoB,KAAK;YACzB;EACV,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAsCK,aAAa,gBAAgB,mBAA2B;;iBAiBxD,kBACd,4BACA,mBACC"}
|