@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/chunk-UB2LOJ6Q.js
DELETED
|
@@ -1,4461 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
JudgeParseError,
|
|
3
|
-
assertCodeSurfaceIdentity,
|
|
4
|
-
costReceiptFromTCloud,
|
|
5
|
-
labelTrustRank,
|
|
6
|
-
maximumChargeForTCloudRequest,
|
|
7
|
-
pairHoldout,
|
|
8
|
-
recoverTruncatedJson,
|
|
9
|
-
surfaceContentHash
|
|
10
|
-
} from "./chunk-NKAGIDE2.js";
|
|
11
|
-
import {
|
|
12
|
-
SearchLedgerConflictError,
|
|
13
|
-
SearchLedgerError,
|
|
14
|
-
SearchLedgerIntegrityError,
|
|
15
|
-
appendSearchLedgerLine,
|
|
16
|
-
assertRealBackend,
|
|
17
|
-
contentHash,
|
|
18
|
-
planCampaignRun,
|
|
19
|
-
runCampaign,
|
|
20
|
-
summarizeBackendIntegrity,
|
|
21
|
-
withSearchLedgerFileLock
|
|
22
|
-
} from "./chunk-EZJEIH2R.js";
|
|
23
|
-
import {
|
|
24
|
-
Mutex
|
|
25
|
-
} from "./chunk-WGXIEX7P.js";
|
|
26
|
-
import {
|
|
27
|
-
campaignCellToRunRecord
|
|
28
|
-
} from "./chunk-2MKQIFS4.js";
|
|
29
|
-
import {
|
|
30
|
-
comparePairedArms
|
|
31
|
-
} from "./chunk-DPUHNQLN.js";
|
|
32
|
-
import {
|
|
33
|
-
eProcess,
|
|
34
|
-
mulberry32,
|
|
35
|
-
pairedBootstrap
|
|
36
|
-
} from "./chunk-MHELPNRP.js";
|
|
37
|
-
import {
|
|
38
|
-
CostLedger
|
|
39
|
-
} from "./chunk-WS3NZZQQ.js";
|
|
40
|
-
import {
|
|
41
|
-
analyzeTraces
|
|
42
|
-
} from "./chunk-5DTSBUL2.js";
|
|
43
|
-
import {
|
|
44
|
-
buildAgentProfileCell,
|
|
45
|
-
canonicalize,
|
|
46
|
-
modelHasSnapshot,
|
|
47
|
-
runTaskScore,
|
|
48
|
-
validateRunRecord
|
|
49
|
-
} from "./chunk-2JX3CFMB.js";
|
|
50
|
-
import {
|
|
51
|
-
AgentEvalError,
|
|
52
|
-
ValidationError
|
|
53
|
-
} from "./chunk-ONWEPEDO.js";
|
|
54
|
-
|
|
55
|
-
// src/campaign/analyst-surface.ts
|
|
56
|
-
function surfaceToText(surface) {
|
|
57
|
-
if (typeof surface === "string") return surface;
|
|
58
|
-
throw new Error(
|
|
59
|
-
`buildAnalystSurfaceDispatch: the analyst surface must be a string actorDescription, got a ${surface.kind}-tier surface. The analyst prompt is prompt-tier.`
|
|
60
|
-
);
|
|
61
|
-
}
|
|
62
|
-
function buildAnalystSurfaceDispatch(opts) {
|
|
63
|
-
const analyze = opts.analyze ?? analyzeTraces;
|
|
64
|
-
return async (surface, scenario, _ctx) => {
|
|
65
|
-
const actorDescription = surfaceToText(surface);
|
|
66
|
-
const res = await analyze(
|
|
67
|
-
{ question: scenario.question },
|
|
68
|
-
{ ...opts.analystOptions, actorDescription, source: scenario.source }
|
|
69
|
-
);
|
|
70
|
-
return {
|
|
71
|
-
answer: res.answer,
|
|
72
|
-
findings: res.findings,
|
|
73
|
-
actorPromptVersion: res.actorPromptVersion
|
|
74
|
-
};
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
function failureModeRecallJudge(opts = {}) {
|
|
78
|
-
const recallWeight = opts.recallWeight ?? 0.5;
|
|
79
|
-
return {
|
|
80
|
-
name: "failure-mode-recall",
|
|
81
|
-
dimensions: [
|
|
82
|
-
{ key: "recall", description: "fraction of ground-truth failure modes the analyst surfaced" },
|
|
83
|
-
{
|
|
84
|
-
key: "precision",
|
|
85
|
-
description: "1 \u2212 share of findings that named a failure/tool/error absent from this corpus"
|
|
86
|
-
}
|
|
87
|
-
],
|
|
88
|
-
appliesTo: (s) => s.kind === "analyst-surface",
|
|
89
|
-
score({ artifact, scenario }) {
|
|
90
|
-
const modes = scenario.expectedFailureModes;
|
|
91
|
-
if (modes.length === 0) {
|
|
92
|
-
throw new Error(
|
|
93
|
-
`failureModeRecallJudge: scenario '${scenario.id}' has no expectedFailureModes \u2014 refusing to score (a vacuous 1.0 would corrupt the comparison)`
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
const hay = artifact.findings.join("\n").toLowerCase();
|
|
97
|
-
const matched = modes.filter((m) => m.cues.some((c) => hay.includes(c.toLowerCase())));
|
|
98
|
-
const recall = matched.length / modes.length;
|
|
99
|
-
const forbidden = (scenario.forbiddenCues ?? []).map((c) => c.toLowerCase());
|
|
100
|
-
let precision = 1;
|
|
101
|
-
let hallucinated = 0;
|
|
102
|
-
if (forbidden.length > 0) {
|
|
103
|
-
const denom = Math.max(1, artifact.findings.length);
|
|
104
|
-
hallucinated = artifact.findings.filter(
|
|
105
|
-
(f) => forbidden.some((c) => f.toLowerCase().includes(c))
|
|
106
|
-
).length;
|
|
107
|
-
precision = 1 - hallucinated / denom;
|
|
108
|
-
}
|
|
109
|
-
const composite = forbidden.length > 0 ? recallWeight * recall + (1 - recallWeight) * precision : recall;
|
|
110
|
-
const missed = modes.filter((m) => !matched.includes(m)).map((m) => m.id);
|
|
111
|
-
const notes = `matched ${matched.length}/${modes.length} failure modes` + (missed.length ? `; missed [${missed.join(", ")}]` : "") + (hallucinated ? `; ${hallucinated} out-of-corpus finding(s)` : "");
|
|
112
|
-
return { dimensions: { recall, precision }, composite, notes };
|
|
113
|
-
}
|
|
114
|
-
};
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
// src/campaign/cross-surface-context.ts
|
|
118
|
-
function validateCrossSurfaceInput(input) {
|
|
119
|
-
assertNonEmptyUnique(input.taskOrder, "taskOrder");
|
|
120
|
-
assertNonEmptyUnique(input.componentOrder, "componentOrder");
|
|
121
|
-
if (input.componentOrder.length < 2) {
|
|
122
|
-
throw new ValidationError(
|
|
123
|
-
"analyzeCrossSurfaceInteractions: componentOrder must contain at least two surfaces"
|
|
124
|
-
);
|
|
125
|
-
}
|
|
126
|
-
assertNonEmptyUnique(input.candidateOrder, "candidateOrder");
|
|
127
|
-
assertNonEmptyUnique(input.costMetricOrder, "costMetricOrder");
|
|
128
|
-
if (input.costMetricOrder.includes("score")) {
|
|
129
|
-
throw new ValidationError(
|
|
130
|
-
`analyzeCrossSurfaceInteractions: costMetricOrder cannot contain reserved metric 'score'`
|
|
131
|
-
);
|
|
132
|
-
}
|
|
133
|
-
validateBootstrap(input);
|
|
134
|
-
validateSelection(input);
|
|
135
|
-
const componentById = indexUnique(
|
|
136
|
-
input.components,
|
|
137
|
-
(component) => component.componentId,
|
|
138
|
-
"component"
|
|
139
|
-
);
|
|
140
|
-
assertExactSet(input.componentOrder, componentById.keys(), "componentOrder", "components");
|
|
141
|
-
const surfaces = /* @__PURE__ */ new Set();
|
|
142
|
-
for (const component of componentById.values()) {
|
|
143
|
-
assertNonEmpty(component.componentId, "componentId");
|
|
144
|
-
assertNonEmpty(component.surfaceId, `surfaceId for component '${component.componentId}'`);
|
|
145
|
-
if (surfaces.has(component.surfaceId)) {
|
|
146
|
-
throw new ValidationError(
|
|
147
|
-
`analyzeCrossSurfaceInteractions: surfaceId '${component.surfaceId}' has more than one component; the interaction stage requires one independently selected finalist per surface`
|
|
148
|
-
);
|
|
149
|
-
}
|
|
150
|
-
surfaces.add(component.surfaceId);
|
|
151
|
-
if (typeof component.bestSingleEligible !== "boolean") {
|
|
152
|
-
throw new ValidationError(
|
|
153
|
-
`analyzeCrossSurfaceInteractions: component '${component.componentId}' must declare bestSingleEligible`
|
|
154
|
-
);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
const componentIndex = new Map(input.componentOrder.map((id, index) => [id, index]));
|
|
158
|
-
const candidateById = indexUnique(
|
|
159
|
-
input.candidates,
|
|
160
|
-
(candidate) => candidate.candidateId,
|
|
161
|
-
"candidate"
|
|
162
|
-
);
|
|
163
|
-
assertExactSet(input.candidateOrder, candidateById.keys(), "candidateOrder", "candidates");
|
|
164
|
-
if (!candidateById.has(input.baselineCandidateId)) {
|
|
165
|
-
throw new ValidationError(
|
|
166
|
-
`analyzeCrossSurfaceInteractions: unknown baselineCandidateId '${input.baselineCandidateId}'`
|
|
167
|
-
);
|
|
168
|
-
}
|
|
169
|
-
const candidateByComponents = /* @__PURE__ */ new Map();
|
|
170
|
-
const singleByComponent = /* @__PURE__ */ new Map();
|
|
171
|
-
for (const candidate of candidateById.values()) {
|
|
172
|
-
validateCandidate(candidate, componentById, componentIndex, input.baselineCandidateId);
|
|
173
|
-
const key = crossSurfaceComponentSetKey(candidate.componentIds);
|
|
174
|
-
const duplicate = candidateByComponents.get(key);
|
|
175
|
-
if (duplicate) {
|
|
176
|
-
throw new ValidationError(
|
|
177
|
-
`analyzeCrossSurfaceInteractions: candidates '${duplicate.candidateId}' and '${candidate.candidateId}' materialize the same component set`
|
|
178
|
-
);
|
|
179
|
-
}
|
|
180
|
-
candidateByComponents.set(key, candidate);
|
|
181
|
-
if (candidate.componentIds.length === 1) {
|
|
182
|
-
singleByComponent.set(candidate.componentIds[0], candidate);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
const baseline = candidateById.get(input.baselineCandidateId);
|
|
186
|
-
if (baseline.componentIds.length !== 0) {
|
|
187
|
-
throw new ValidationError(
|
|
188
|
-
`analyzeCrossSurfaceInteractions: baseline '${baseline.candidateId}' must have zero components`
|
|
189
|
-
);
|
|
190
|
-
}
|
|
191
|
-
for (const componentId of input.componentOrder) {
|
|
192
|
-
if (!singleByComponent.has(componentId)) {
|
|
193
|
-
throw new ValidationError(
|
|
194
|
-
`analyzeCrossSurfaceInteractions: no single-surface candidate for component '${componentId}'`
|
|
195
|
-
);
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
for (let left = 0; left < input.componentOrder.length; left++) {
|
|
199
|
-
for (let right = left + 1; right < input.componentOrder.length; right++) {
|
|
200
|
-
const ids = [input.componentOrder[left], input.componentOrder[right]];
|
|
201
|
-
if (!candidateByComponents.has(crossSurfaceComponentSetKey(ids))) {
|
|
202
|
-
throw new ValidationError(
|
|
203
|
-
`analyzeCrossSurfaceInteractions: missing pair candidate for components [${ids.join(", ")}]`
|
|
204
|
-
);
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
const rowsByCandidate = /* @__PURE__ */ new Map();
|
|
209
|
-
const taskIds = new Set(input.taskOrder);
|
|
210
|
-
for (const row of input.rows) {
|
|
211
|
-
const candidate = candidateById.get(row.candidateId);
|
|
212
|
-
if (!candidate) {
|
|
213
|
-
throw new ValidationError(
|
|
214
|
-
`analyzeCrossSurfaceInteractions: row for task '${row.taskId}' names unknown candidate '${row.candidateId}'`
|
|
215
|
-
);
|
|
216
|
-
}
|
|
217
|
-
if (!taskIds.has(row.taskId)) {
|
|
218
|
-
throw new ValidationError(
|
|
219
|
-
`analyzeCrossSurfaceInteractions: row for candidate '${row.candidateId}' names task '${row.taskId}' outside the declared taskOrder`
|
|
220
|
-
);
|
|
221
|
-
}
|
|
222
|
-
validateRow(input, row, candidate);
|
|
223
|
-
const byTask = rowsByCandidate.get(row.candidateId) ?? /* @__PURE__ */ new Map();
|
|
224
|
-
if (byTask.has(row.taskId)) {
|
|
225
|
-
throw new ValidationError(
|
|
226
|
-
`analyzeCrossSurfaceInteractions: duplicate row for candidate '${row.candidateId}' and task '${row.taskId}'`
|
|
227
|
-
);
|
|
228
|
-
}
|
|
229
|
-
byTask.set(row.taskId, row);
|
|
230
|
-
rowsByCandidate.set(row.candidateId, byTask);
|
|
231
|
-
}
|
|
232
|
-
for (const candidateId of input.candidateOrder) {
|
|
233
|
-
const byTask = rowsByCandidate.get(candidateId);
|
|
234
|
-
const missing = input.taskOrder.filter((taskId) => !byTask?.has(taskId));
|
|
235
|
-
if (missing.length > 0) {
|
|
236
|
-
throw new ValidationError(
|
|
237
|
-
`analyzeCrossSurfaceInteractions: candidate '${candidateId}' is missing declared task row(s) [${missing.join(", ")}]; encode failed attempts explicitly instead of changing the task axis`
|
|
238
|
-
);
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
const expectedRows = input.candidateOrder.length * input.taskOrder.length;
|
|
242
|
-
if (input.rows.length !== expectedRows) {
|
|
243
|
-
throw new ValidationError(
|
|
244
|
-
`analyzeCrossSurfaceInteractions: expected exactly ${expectedRows} candidate \xD7 task rows, got ${input.rows.length}`
|
|
245
|
-
);
|
|
246
|
-
}
|
|
247
|
-
return {
|
|
248
|
-
input,
|
|
249
|
-
components: input.componentOrder.map((id) => componentById.get(id)),
|
|
250
|
-
candidates: input.candidateOrder.map((id) => candidateById.get(id)),
|
|
251
|
-
componentById,
|
|
252
|
-
candidateById,
|
|
253
|
-
candidateByComponents,
|
|
254
|
-
singleByComponent,
|
|
255
|
-
rowsByCandidate,
|
|
256
|
-
componentIndex,
|
|
257
|
-
candidateIndex: new Map(input.candidateOrder.map((id, index) => [id, index]))
|
|
258
|
-
};
|
|
259
|
-
}
|
|
260
|
-
function crossSurfaceRowsFor(context, candidateId) {
|
|
261
|
-
return context.input.taskOrder.map((taskId) => crossSurfaceRowFor(context, candidateId, taskId));
|
|
262
|
-
}
|
|
263
|
-
function crossSurfaceRowFor(context, candidateId, taskId) {
|
|
264
|
-
return context.rowsByCandidate.get(candidateId).get(taskId);
|
|
265
|
-
}
|
|
266
|
-
function canonicalCrossSurfaceComponents(context, componentIds) {
|
|
267
|
-
return [...componentIds].sort(
|
|
268
|
-
(left, right) => context.componentIndex.get(left) - context.componentIndex.get(right)
|
|
269
|
-
);
|
|
270
|
-
}
|
|
271
|
-
function crossSurfaceComponentSetKey(componentIds) {
|
|
272
|
-
return JSON.stringify(componentIds);
|
|
273
|
-
}
|
|
274
|
-
function validateBootstrap(input) {
|
|
275
|
-
const { seed, resamples, confidence } = input.bootstrap;
|
|
276
|
-
if (!Number.isInteger(seed)) {
|
|
277
|
-
throw new ValidationError(`analyzeCrossSurfaceInteractions: bootstrap.seed must be an integer`);
|
|
278
|
-
}
|
|
279
|
-
if (!Number.isInteger(resamples) || resamples <= 0) {
|
|
280
|
-
throw new ValidationError(
|
|
281
|
-
`analyzeCrossSurfaceInteractions: bootstrap.resamples must be a positive integer`
|
|
282
|
-
);
|
|
283
|
-
}
|
|
284
|
-
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
|
|
285
|
-
throw new ValidationError(
|
|
286
|
-
`analyzeCrossSurfaceInteractions: bootstrap.confidence must be in (0,1)`
|
|
287
|
-
);
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
function validateSelection(input) {
|
|
291
|
-
const policy = input.selection;
|
|
292
|
-
for (const [name, value] of [
|
|
293
|
-
["minimumFiringTasks", policy.minimumFiringTasks],
|
|
294
|
-
["minimumEffectTasks", policy.minimumEffectTasks]
|
|
295
|
-
]) {
|
|
296
|
-
if (!Number.isInteger(value) || value < 0 || value > input.taskOrder.length) {
|
|
297
|
-
throw new ValidationError(
|
|
298
|
-
`analyzeCrossSurfaceInteractions: selection.${name} must be an integer in [0,${input.taskOrder.length}]`
|
|
299
|
-
);
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
if (!Number.isInteger(policy.minimumBundleComponents) || policy.minimumBundleComponents < 2 || policy.minimumBundleComponents > input.componentOrder.length) {
|
|
303
|
-
throw new ValidationError(
|
|
304
|
-
`analyzeCrossSurfaceInteractions: selection.minimumBundleComponents must be an integer in [2,${input.componentOrder.length}]`
|
|
305
|
-
);
|
|
306
|
-
}
|
|
307
|
-
for (const [metric, limit] of Object.entries(policy.maximumMedianCostRatioToBaseline)) {
|
|
308
|
-
if (!input.costMetricOrder.includes(metric)) {
|
|
309
|
-
throw new ValidationError(
|
|
310
|
-
`analyzeCrossSurfaceInteractions: cost limit names undeclared metric '${metric}'`
|
|
311
|
-
);
|
|
312
|
-
}
|
|
313
|
-
if (!Number.isFinite(limit) || limit <= 0) {
|
|
314
|
-
throw new ValidationError(
|
|
315
|
-
`analyzeCrossSurfaceInteractions: cost ratio limit for '${metric}' must be positive and finite`
|
|
316
|
-
);
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
function validateCandidate(candidate, componentById, componentIndex, baselineCandidateId) {
|
|
321
|
-
assertNonEmpty(candidate.candidateId, "candidateId");
|
|
322
|
-
assertNonEmpty(candidate.contentHash, `contentHash for candidate '${candidate.candidateId}'`);
|
|
323
|
-
if (!Number.isInteger(candidate.artifactBytes) || candidate.artifactBytes < 0) {
|
|
324
|
-
throw new ValidationError(
|
|
325
|
-
`analyzeCrossSurfaceInteractions: artifactBytes for candidate '${candidate.candidateId}' must be a non-negative integer`
|
|
326
|
-
);
|
|
327
|
-
}
|
|
328
|
-
assertUnique(candidate.componentIds, `componentIds for candidate '${candidate.candidateId}'`);
|
|
329
|
-
for (const componentId of candidate.componentIds) {
|
|
330
|
-
if (!componentById.has(componentId)) {
|
|
331
|
-
throw new ValidationError(
|
|
332
|
-
`analyzeCrossSurfaceInteractions: candidate '${candidate.candidateId}' names unknown component '${componentId}'`
|
|
333
|
-
);
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
const canonical = [...candidate.componentIds].sort(
|
|
337
|
-
(left, right) => componentIndex.get(left) - componentIndex.get(right)
|
|
338
|
-
);
|
|
339
|
-
if (!arraysEqual(candidate.componentIds, canonical)) {
|
|
340
|
-
throw new ValidationError(
|
|
341
|
-
`analyzeCrossSurfaceInteractions: candidate '${candidate.candidateId}' components must follow componentOrder`
|
|
342
|
-
);
|
|
343
|
-
}
|
|
344
|
-
if (candidate.candidateId !== baselineCandidateId && candidate.componentIds.length === 0) {
|
|
345
|
-
throw new ValidationError(
|
|
346
|
-
`analyzeCrossSurfaceInteractions: only baseline '${baselineCandidateId}' may have zero components`
|
|
347
|
-
);
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
function validateRow(input, row, candidate) {
|
|
351
|
-
if (!arraysEqual(row.componentIds, candidate.componentIds)) {
|
|
352
|
-
throw new ValidationError(
|
|
353
|
-
`analyzeCrossSurfaceInteractions: row '${row.candidateId}/${row.taskId}' componentIds do not match its candidate`
|
|
354
|
-
);
|
|
355
|
-
}
|
|
356
|
-
if (!["complete", "missing", "invalid"].includes(row.completeness)) {
|
|
357
|
-
throw new ValidationError(
|
|
358
|
-
`analyzeCrossSurfaceInteractions: row '${row.candidateId}/${row.taskId}' has unknown completeness '${String(row.completeness)}'`
|
|
359
|
-
);
|
|
360
|
-
}
|
|
361
|
-
if (row.completeness === "complete") {
|
|
362
|
-
if (typeof row.pass !== "boolean" || !Number.isFinite(row.score)) {
|
|
363
|
-
throw new ValidationError(
|
|
364
|
-
`analyzeCrossSurfaceInteractions: complete row '${row.candidateId}/${row.taskId}' requires boolean pass and finite score`
|
|
365
|
-
);
|
|
366
|
-
}
|
|
367
|
-
if (row.rejectReason !== null) {
|
|
368
|
-
throw new ValidationError(
|
|
369
|
-
`analyzeCrossSurfaceInteractions: complete row '${row.candidateId}/${row.taskId}' cannot carry rejectReason`
|
|
370
|
-
);
|
|
371
|
-
}
|
|
372
|
-
} else {
|
|
373
|
-
if (row.pass !== null || row.score !== null) {
|
|
374
|
-
throw new ValidationError(
|
|
375
|
-
`analyzeCrossSurfaceInteractions: ${row.completeness} row '${row.candidateId}/${row.taskId}' must use null pass and score`
|
|
376
|
-
);
|
|
377
|
-
}
|
|
378
|
-
if (typeof row.rejectReason !== "string" || row.rejectReason.trim() === "") {
|
|
379
|
-
throw new ValidationError(
|
|
380
|
-
`analyzeCrossSurfaceInteractions: ${row.completeness} row '${row.candidateId}/${row.taskId}' requires rejectReason`
|
|
381
|
-
);
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
assertExactSet(
|
|
385
|
-
input.costMetricOrder,
|
|
386
|
-
Object.keys(row.cost),
|
|
387
|
-
`cost keys for row '${row.candidateId}/${row.taskId}'`,
|
|
388
|
-
"costMetricOrder"
|
|
389
|
-
);
|
|
390
|
-
for (const metric of input.costMetricOrder) {
|
|
391
|
-
const value = row.cost[metric];
|
|
392
|
-
if (value === null || value === void 0 || !Number.isFinite(value) || value < 0) {
|
|
393
|
-
throw new ValidationError(
|
|
394
|
-
`analyzeCrossSurfaceInteractions: unknown or invalid cost '${metric}' on row '${row.candidateId}/${row.taskId}'; every attempt must report a non-negative finite value`
|
|
395
|
-
);
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
const evidenceByComponent = indexUnique(
|
|
399
|
-
row.componentEvidence,
|
|
400
|
-
(evidence) => evidence.componentId,
|
|
401
|
-
`componentEvidence on row '${row.candidateId}/${row.taskId}'`
|
|
402
|
-
);
|
|
403
|
-
assertExactSet(
|
|
404
|
-
candidate.componentIds,
|
|
405
|
-
evidenceByComponent.keys(),
|
|
406
|
-
`componentEvidence on row '${row.candidateId}/${row.taskId}'`,
|
|
407
|
-
"candidate components"
|
|
408
|
-
);
|
|
409
|
-
for (const evidence of evidenceByComponent.values()) {
|
|
410
|
-
assertTriState(evidence.fired, "fired", row);
|
|
411
|
-
assertTriState(evidence.effectObserved, "effectObserved", row);
|
|
412
|
-
if (evidence.effectObserved === true && evidence.fired !== true) {
|
|
413
|
-
throw new ValidationError(
|
|
414
|
-
`analyzeCrossSurfaceInteractions: effectObserved=true requires fired=true for component '${evidence.componentId}' on row '${row.candidateId}/${row.taskId}'`
|
|
415
|
-
);
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
function assertTriState(value, field, row) {
|
|
420
|
-
if (value !== true && value !== false && value !== null) {
|
|
421
|
-
throw new ValidationError(
|
|
422
|
-
`analyzeCrossSurfaceInteractions: ${field} on row '${row.candidateId}/${row.taskId}' must be boolean or null`
|
|
423
|
-
);
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
function indexUnique(items, id, label) {
|
|
427
|
-
const result = /* @__PURE__ */ new Map();
|
|
428
|
-
for (const item of items) {
|
|
429
|
-
const key = id(item);
|
|
430
|
-
assertNonEmpty(key, `${label} id`);
|
|
431
|
-
if (result.has(key)) {
|
|
432
|
-
throw new ValidationError(`analyzeCrossSurfaceInteractions: duplicate ${label} id '${key}'`);
|
|
433
|
-
}
|
|
434
|
-
result.set(key, item);
|
|
435
|
-
}
|
|
436
|
-
return result;
|
|
437
|
-
}
|
|
438
|
-
function assertNonEmptyUnique(values, label) {
|
|
439
|
-
if (values.length === 0) {
|
|
440
|
-
throw new ValidationError(`analyzeCrossSurfaceInteractions: ${label} is empty`);
|
|
441
|
-
}
|
|
442
|
-
assertUnique(values, label);
|
|
443
|
-
for (const value of values) assertNonEmpty(value, label);
|
|
444
|
-
}
|
|
445
|
-
function assertUnique(values, label) {
|
|
446
|
-
if (new Set(values).size !== values.length) {
|
|
447
|
-
throw new ValidationError(`analyzeCrossSurfaceInteractions: ${label} contains duplicates`);
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
function assertNonEmpty(value, label) {
|
|
451
|
-
if (typeof value !== "string" || value.trim() === "") {
|
|
452
|
-
throw new ValidationError(
|
|
453
|
-
`analyzeCrossSurfaceInteractions: ${label} must be a non-empty string`
|
|
454
|
-
);
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
function assertExactSet(expected, actualIterable, actualLabel, expectedLabel) {
|
|
458
|
-
const actual = [...actualIterable];
|
|
459
|
-
const expectedSet = new Set(expected);
|
|
460
|
-
const actualSet = new Set(actual);
|
|
461
|
-
const missing = expected.filter((value) => !actualSet.has(value));
|
|
462
|
-
const extra = actual.filter((value) => !expectedSet.has(value));
|
|
463
|
-
if (missing.length > 0 || extra.length > 0 || actual.length !== expected.length) {
|
|
464
|
-
throw new ValidationError(
|
|
465
|
-
`analyzeCrossSurfaceInteractions: ${actualLabel} does not match ${expectedLabel}; missing=[${missing.join(", ")}], extra=[${extra.join(", ")}]`
|
|
466
|
-
);
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
function arraysEqual(left, right) {
|
|
470
|
-
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
// src/campaign/cross-surface-interaction.ts
|
|
474
|
-
function analyzeCrossSurfaceInteractions(input) {
|
|
475
|
-
const context = validateCrossSurfaceInput(input);
|
|
476
|
-
const baseline = context.candidateById.get(input.baselineCandidateId);
|
|
477
|
-
const preliminary = context.candidates.map(
|
|
478
|
-
(candidate) => summarizeCandidate(context, candidate, baseline)
|
|
479
|
-
);
|
|
480
|
-
const baselineSummary = preliminary.find(
|
|
481
|
-
(summary) => summary.candidate.candidateId === input.baselineCandidateId
|
|
482
|
-
);
|
|
483
|
-
const summaries = preliminary.map(
|
|
484
|
-
(summary) => summary.candidate.candidateId === input.baselineCandidateId ? summary : {
|
|
485
|
-
...summary,
|
|
486
|
-
eligibility: candidateEligibility(context, summary, baselineSummary)
|
|
487
|
-
}
|
|
488
|
-
);
|
|
489
|
-
const summaryById = new Map(summaries.map((summary) => [summary.candidate.candidateId, summary]));
|
|
490
|
-
const eligibleSingles = rankedEligibleSingles(context, summaryById);
|
|
491
|
-
const interactionReadySingles = rankedInteractionReadySingles(context, summaryById);
|
|
492
|
-
const pairwise = buildPairwise(context, summaryById, interactionReadySingles, baselineSummary);
|
|
493
|
-
const selections = selectCandidates(
|
|
494
|
-
context,
|
|
495
|
-
summaryById,
|
|
496
|
-
eligibleSingles,
|
|
497
|
-
interactionReadySingles,
|
|
498
|
-
pairwise
|
|
499
|
-
);
|
|
500
|
-
const rows = context.candidates.flatMap(
|
|
501
|
-
(candidate) => input.taskOrder.map(
|
|
502
|
-
(taskId) => context.rowsByCandidate.get(candidate.candidateId).get(taskId)
|
|
503
|
-
)
|
|
504
|
-
);
|
|
505
|
-
return {
|
|
506
|
-
taskIds: [...input.taskOrder],
|
|
507
|
-
componentIds: [...input.componentOrder],
|
|
508
|
-
candidateIds: [...input.candidateOrder],
|
|
509
|
-
costMetrics: [...input.costMetricOrder],
|
|
510
|
-
rows,
|
|
511
|
-
missingAttempts: rows.filter((row) => row.completeness === "missing"),
|
|
512
|
-
invalidAttempts: rows.filter((row) => row.completeness === "invalid"),
|
|
513
|
-
candidates: summaries,
|
|
514
|
-
pairwise,
|
|
515
|
-
selections
|
|
516
|
-
};
|
|
517
|
-
}
|
|
518
|
-
function summarizeCandidate(context, candidate, baseline) {
|
|
519
|
-
const rows = crossSurfaceRowsFor(context, candidate.candidateId);
|
|
520
|
-
const baselineRows = crossSurfaceRowsFor(context, baseline.candidateId);
|
|
521
|
-
const outcome = summarizeOutcome(
|
|
522
|
-
context,
|
|
523
|
-
rows,
|
|
524
|
-
baselineRows,
|
|
525
|
-
candidate.candidateId === baseline.candidateId
|
|
526
|
-
);
|
|
527
|
-
const completeScores = rows.filter((row) => row.completeness === "complete").map((row) => row.score);
|
|
528
|
-
const costs = Object.fromEntries(
|
|
529
|
-
context.input.costMetricOrder.map((metric) => [
|
|
530
|
-
metric,
|
|
531
|
-
distribution(rows.map((row) => row.cost[metric]))
|
|
532
|
-
])
|
|
533
|
-
);
|
|
534
|
-
return {
|
|
535
|
-
candidate,
|
|
536
|
-
outcome,
|
|
537
|
-
score: completeScores.length === 0 ? null : distribution(completeScores),
|
|
538
|
-
costs,
|
|
539
|
-
firing: summarizeCandidateEvidence(rows, candidate.componentIds, "fired"),
|
|
540
|
-
effect: summarizeCandidateEvidence(rows, candidate.componentIds, "effectObserved"),
|
|
541
|
-
comparisonToBaseline: candidate.candidateId === baseline.candidateId ? null : pairedComparison(context, baseline.candidateId, candidate.candidateId),
|
|
542
|
-
eligibility: null
|
|
543
|
-
};
|
|
544
|
-
}
|
|
545
|
-
function summarizeOutcome(context, rows, baselineRows, isBaseline) {
|
|
546
|
-
const resolvedTaskIds = [];
|
|
547
|
-
const failedTaskIds = [];
|
|
548
|
-
const missingTaskIds = [];
|
|
549
|
-
const invalidTaskIds = [];
|
|
550
|
-
const benefitTaskIds = [];
|
|
551
|
-
const regressionTaskIds = [];
|
|
552
|
-
const comparisonMissingTaskIds = [];
|
|
553
|
-
for (let index = 0; index < context.input.taskOrder.length; index++) {
|
|
554
|
-
const row = rows[index];
|
|
555
|
-
const baseline = baselineRows[index];
|
|
556
|
-
if (row.completeness === "missing") missingTaskIds.push(row.taskId);
|
|
557
|
-
else if (row.completeness === "invalid") invalidTaskIds.push(row.taskId);
|
|
558
|
-
else if (row.pass) resolvedTaskIds.push(row.taskId);
|
|
559
|
-
else failedTaskIds.push(row.taskId);
|
|
560
|
-
if (isBaseline) continue;
|
|
561
|
-
if (row.completeness !== "complete" || baseline.completeness !== "complete") {
|
|
562
|
-
comparisonMissingTaskIds.push(row.taskId);
|
|
563
|
-
} else if (row.pass && !baseline.pass) benefitTaskIds.push(row.taskId);
|
|
564
|
-
else if (!row.pass && baseline.pass) regressionTaskIds.push(row.taskId);
|
|
565
|
-
}
|
|
566
|
-
return {
|
|
567
|
-
resolvedTaskIds,
|
|
568
|
-
failedTaskIds,
|
|
569
|
-
missingTaskIds,
|
|
570
|
-
invalidTaskIds,
|
|
571
|
-
benefitTaskIds,
|
|
572
|
-
regressionTaskIds,
|
|
573
|
-
comparisonMissingTaskIds,
|
|
574
|
-
netBenefit: benefitTaskIds.length - regressionTaskIds.length
|
|
575
|
-
};
|
|
576
|
-
}
|
|
577
|
-
function summarizeCandidateEvidence(rows, componentIds, field) {
|
|
578
|
-
if (componentIds.length === 0) {
|
|
579
|
-
return {
|
|
580
|
-
byComponent: [],
|
|
581
|
-
allObservedTaskIds: [],
|
|
582
|
-
someObservedTaskIds: [],
|
|
583
|
-
noneObservedTaskIds: [],
|
|
584
|
-
unobservedTaskIds: []
|
|
585
|
-
};
|
|
586
|
-
}
|
|
587
|
-
const byComponent = componentIds.map((componentId) => {
|
|
588
|
-
const observedTaskIds = [];
|
|
589
|
-
const notObservedTaskIds = [];
|
|
590
|
-
const unobservedTaskIds2 = [];
|
|
591
|
-
for (const row of rows) {
|
|
592
|
-
const value = evidenceValue(row, componentId, field);
|
|
593
|
-
if (value === true) observedTaskIds.push(row.taskId);
|
|
594
|
-
else if (value === false) notObservedTaskIds.push(row.taskId);
|
|
595
|
-
else unobservedTaskIds2.push(row.taskId);
|
|
596
|
-
}
|
|
597
|
-
return { componentId, observedTaskIds, notObservedTaskIds, unobservedTaskIds: unobservedTaskIds2 };
|
|
598
|
-
});
|
|
599
|
-
const allObservedTaskIds = [];
|
|
600
|
-
const someObservedTaskIds = [];
|
|
601
|
-
const noneObservedTaskIds = [];
|
|
602
|
-
const unobservedTaskIds = [];
|
|
603
|
-
for (const row of rows) {
|
|
604
|
-
const values = componentIds.map((componentId) => evidenceValue(row, componentId, field));
|
|
605
|
-
if (values.some((value) => value === null)) unobservedTaskIds.push(row.taskId);
|
|
606
|
-
else if (values.every(Boolean)) allObservedTaskIds.push(row.taskId);
|
|
607
|
-
else if (values.some(Boolean)) someObservedTaskIds.push(row.taskId);
|
|
608
|
-
else noneObservedTaskIds.push(row.taskId);
|
|
609
|
-
}
|
|
610
|
-
return {
|
|
611
|
-
byComponent,
|
|
612
|
-
allObservedTaskIds,
|
|
613
|
-
someObservedTaskIds,
|
|
614
|
-
noneObservedTaskIds,
|
|
615
|
-
unobservedTaskIds
|
|
616
|
-
};
|
|
617
|
-
}
|
|
618
|
-
function candidateEligibility(context, summary, baseline) {
|
|
619
|
-
const reasons = [];
|
|
620
|
-
if (summary.outcome.missingTaskIds.length > 0) reasons.push("missing_attempt");
|
|
621
|
-
if (summary.outcome.invalidTaskIds.length > 0) reasons.push("invalid_attempt");
|
|
622
|
-
if (summary.outcome.comparisonMissingTaskIds.length > 0) reasons.push("baseline_outcome_missing");
|
|
623
|
-
if (summary.outcome.benefitTaskIds.length <= summary.outcome.regressionTaskIds.length) {
|
|
624
|
-
reasons.push("benefit_not_greater_than_regression");
|
|
625
|
-
}
|
|
626
|
-
appendEvidenceEligibilityReasons(
|
|
627
|
-
reasons,
|
|
628
|
-
summary.firing,
|
|
629
|
-
context.input.selection.minimumFiringTasks,
|
|
630
|
-
context.input.selection.requireObservedFiring,
|
|
631
|
-
"firing_below_minimum",
|
|
632
|
-
"firing_unobserved"
|
|
633
|
-
);
|
|
634
|
-
appendEvidenceEligibilityReasons(
|
|
635
|
-
reasons,
|
|
636
|
-
summary.effect,
|
|
637
|
-
context.input.selection.minimumEffectTasks,
|
|
638
|
-
context.input.selection.requireObservedEffect,
|
|
639
|
-
"effect_below_minimum",
|
|
640
|
-
"effect_unobserved"
|
|
641
|
-
);
|
|
642
|
-
if (!withinCostLimits(context, summary, baseline)) reasons.push("cost_limit_exceeded");
|
|
643
|
-
return { eligible: reasons.length === 0, reasons: unique(reasons) };
|
|
644
|
-
}
|
|
645
|
-
function appendEvidenceEligibilityReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
|
|
646
|
-
if (evidence.byComponent.some((component) => component.observedTaskIds.length < minimum)) {
|
|
647
|
-
reasons.push(belowReason);
|
|
648
|
-
}
|
|
649
|
-
if (requireObserved && evidence.byComponent.some((component) => component.unobservedTaskIds.length > 0)) {
|
|
650
|
-
reasons.push(unobservedReason);
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
function rankedEligibleSingles(context, summaryById) {
|
|
654
|
-
return [...context.singleByComponent.values()].map((candidate) => summaryById.get(candidate.candidateId)).filter((summary) => summary.eligibility?.eligible).sort((left, right) => compareSingleSummaries(context, left, right));
|
|
655
|
-
}
|
|
656
|
-
function rankedInteractionReadySingles(context, summaryById) {
|
|
657
|
-
return [...context.singleByComponent.values()].map((candidate) => summaryById.get(candidate.candidateId)).filter(
|
|
658
|
-
(summary) => summary.outcome.regressionTaskIds.length === 0 && summary.eligibility?.reasons.every(
|
|
659
|
-
(reason) => reason === "benefit_not_greater_than_regression"
|
|
660
|
-
)
|
|
661
|
-
).sort((left, right) => compareSingleSummaries(context, left, right));
|
|
662
|
-
}
|
|
663
|
-
function compareSingleSummaries(context, left, right) {
|
|
664
|
-
const byNetBenefit = right.outcome.netBenefit - left.outcome.netBenefit;
|
|
665
|
-
if (byNetBenefit !== 0) return byNetBenefit;
|
|
666
|
-
const byBenefit = right.outcome.benefitTaskIds.length - left.outcome.benefitTaskIds.length;
|
|
667
|
-
if (byBenefit !== 0) return byBenefit;
|
|
668
|
-
const byRegression = left.outcome.regressionTaskIds.length - right.outcome.regressionTaskIds.length;
|
|
669
|
-
if (byRegression !== 0) return byRegression;
|
|
670
|
-
for (const metric of context.input.costMetricOrder) {
|
|
671
|
-
const byCost = left.costs[metric].median - right.costs[metric].median;
|
|
672
|
-
if (byCost !== 0) return byCost;
|
|
673
|
-
}
|
|
674
|
-
const byBytes = left.candidate.artifactBytes - right.candidate.artifactBytes;
|
|
675
|
-
if (byBytes !== 0) return byBytes;
|
|
676
|
-
return context.candidateIndex.get(left.candidate.candidateId) - context.candidateIndex.get(right.candidate.candidateId);
|
|
677
|
-
}
|
|
678
|
-
function buildPairwise(context, summaryById, interactionReadySingles, baseline) {
|
|
679
|
-
const interactionReadyIds = new Set(
|
|
680
|
-
interactionReadySingles.map((summary) => summary.candidate.candidateId)
|
|
681
|
-
);
|
|
682
|
-
const entries = [];
|
|
683
|
-
for (let leftIndex = 0; leftIndex < context.components.length; leftIndex++) {
|
|
684
|
-
for (let rightIndex = leftIndex + 1; rightIndex < context.components.length; rightIndex++) {
|
|
685
|
-
const leftComponent = context.components[leftIndex];
|
|
686
|
-
const rightComponent = context.components[rightIndex];
|
|
687
|
-
const leftSingle = context.singleByComponent.get(leftComponent.componentId);
|
|
688
|
-
const rightSingle = context.singleByComponent.get(rightComponent.componentId);
|
|
689
|
-
const pair = context.candidateByComponents.get(
|
|
690
|
-
crossSurfaceComponentSetKey([leftComponent.componentId, rightComponent.componentId])
|
|
691
|
-
);
|
|
692
|
-
const pairSummary = summaryById.get(pair.candidateId);
|
|
693
|
-
const leftSummary = summaryById.get(leftSingle.candidateId);
|
|
694
|
-
const rightSummary = summaryById.get(rightSingle.candidateId);
|
|
695
|
-
const synergyTaskIds = [];
|
|
696
|
-
const interferenceTaskIds = [];
|
|
697
|
-
for (const taskId of context.input.taskOrder) {
|
|
698
|
-
const pairRow = crossSurfaceRowFor(context, pair.candidateId, taskId);
|
|
699
|
-
const leftRow = crossSurfaceRowFor(context, leftSingle.candidateId, taskId);
|
|
700
|
-
const rightRow = crossSurfaceRowFor(context, rightSingle.candidateId, taskId);
|
|
701
|
-
if (![pairRow, leftRow, rightRow].every((row) => row.completeness === "complete")) continue;
|
|
702
|
-
if (pairRow.pass && !leftRow.pass && !rightRow.pass) synergyTaskIds.push(taskId);
|
|
703
|
-
if (!pairRow.pass && (leftRow.pass || rightRow.pass)) interferenceTaskIds.push(taskId);
|
|
704
|
-
}
|
|
705
|
-
const incrementalVsConstituents = [
|
|
706
|
-
compareCandidates(context, summaryById, leftSingle.candidateId, pair.candidateId),
|
|
707
|
-
compareCandidates(context, summaryById, rightSingle.candidateId, pair.candidateId)
|
|
708
|
-
];
|
|
709
|
-
const betterSingle = compareSingleSummaries(context, leftSummary, rightSummary) <= 0 ? leftSummary : rightSummary;
|
|
710
|
-
const firing = summarizePairEvidence(
|
|
711
|
-
crossSurfaceRowsFor(context, pair.candidateId),
|
|
712
|
-
leftComponent.componentId,
|
|
713
|
-
rightComponent.componentId,
|
|
714
|
-
"fired"
|
|
715
|
-
);
|
|
716
|
-
const effect = summarizePairEvidence(
|
|
717
|
-
crossSurfaceRowsFor(context, pair.candidateId),
|
|
718
|
-
leftComponent.componentId,
|
|
719
|
-
rightComponent.componentId,
|
|
720
|
-
"effectObserved"
|
|
721
|
-
);
|
|
722
|
-
const compatibility = pairCompatibility(
|
|
723
|
-
context,
|
|
724
|
-
pairSummary,
|
|
725
|
-
baseline,
|
|
726
|
-
betterSingle,
|
|
727
|
-
incrementalVsConstituents,
|
|
728
|
-
interactionReadyIds.has(leftSingle.candidateId) && interactionReadyIds.has(rightSingle.candidateId),
|
|
729
|
-
interferenceTaskIds,
|
|
730
|
-
firing,
|
|
731
|
-
effect
|
|
732
|
-
);
|
|
733
|
-
entries.push({
|
|
734
|
-
componentIds: [leftComponent.componentId, rightComponent.componentId],
|
|
735
|
-
singleCandidateIds: [leftSingle.candidateId, rightSingle.candidateId],
|
|
736
|
-
compositionCandidateId: pair.candidateId,
|
|
737
|
-
benefitTaskIds: [...pairSummary.outcome.benefitTaskIds],
|
|
738
|
-
regressionTaskIds: [...pairSummary.outcome.regressionTaskIds],
|
|
739
|
-
synergyTaskIds,
|
|
740
|
-
interferenceTaskIds,
|
|
741
|
-
incrementalVsConstituents,
|
|
742
|
-
relativeCostToBaseline: relativeCosts(context, pairSummary, baseline),
|
|
743
|
-
firing,
|
|
744
|
-
effect,
|
|
745
|
-
interaction: interactionEffect(
|
|
746
|
-
context,
|
|
747
|
-
baseline.candidate,
|
|
748
|
-
[leftSingle, rightSingle],
|
|
749
|
-
pair
|
|
750
|
-
),
|
|
751
|
-
compatibility
|
|
752
|
-
});
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
|
-
return entries;
|
|
756
|
-
}
|
|
757
|
-
function pairCompatibility(context, pair, baseline, betterSingle, comparisons, constituentsReady, interferenceTaskIds, firing, effect) {
|
|
758
|
-
const reasons = [];
|
|
759
|
-
if (!constituentsReady) reasons.push("constituent_not_ready");
|
|
760
|
-
if (pair.outcome.missingTaskIds.length > 0 || pair.outcome.invalidTaskIds.length > 0 || pair.outcome.comparisonMissingTaskIds.length > 0) {
|
|
761
|
-
reasons.push("pair_incomplete");
|
|
762
|
-
}
|
|
763
|
-
if (pair.outcome.regressionTaskIds.length > 0) reasons.push("baseline_regression");
|
|
764
|
-
if (interferenceTaskIds.length > 0) reasons.push("interference");
|
|
765
|
-
const betterComparison = comparisons.find(
|
|
766
|
-
(comparison) => comparison.comparatorCandidateId === betterSingle.candidate.candidateId
|
|
767
|
-
);
|
|
768
|
-
if (betterComparison.winsTaskIds.length === 0) reasons.push("no_incremental_resolution");
|
|
769
|
-
appendPairEvidenceReasons(
|
|
770
|
-
reasons,
|
|
771
|
-
firing,
|
|
772
|
-
context.input.selection.minimumFiringTasks,
|
|
773
|
-
context.input.selection.requireObservedFiring,
|
|
774
|
-
"firing_below_minimum",
|
|
775
|
-
"firing_unobserved"
|
|
776
|
-
);
|
|
777
|
-
appendPairEvidenceReasons(
|
|
778
|
-
reasons,
|
|
779
|
-
effect,
|
|
780
|
-
context.input.selection.minimumEffectTasks,
|
|
781
|
-
context.input.selection.requireObservedEffect,
|
|
782
|
-
"effect_below_minimum",
|
|
783
|
-
"effect_unobserved"
|
|
784
|
-
);
|
|
785
|
-
if (!withinCostLimits(context, pair, baseline)) reasons.push("cost_limit_exceeded");
|
|
786
|
-
return {
|
|
787
|
-
compatible: reasons.length === 0,
|
|
788
|
-
reasons: unique(reasons),
|
|
789
|
-
betterSingleCandidateId: betterSingle.candidate.candidateId
|
|
790
|
-
};
|
|
791
|
-
}
|
|
792
|
-
function appendPairEvidenceReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
|
|
793
|
-
const leftObserved = evidence.bothTaskIds.length + evidence.leftOnlyTaskIds.length;
|
|
794
|
-
const rightObserved = evidence.bothTaskIds.length + evidence.rightOnlyTaskIds.length;
|
|
795
|
-
if (leftObserved < minimum || rightObserved < minimum) reasons.push(belowReason);
|
|
796
|
-
if (requireObserved && evidence.unobservedTaskIds.length > 0) reasons.push(unobservedReason);
|
|
797
|
-
}
|
|
798
|
-
function interactionEffect(context, baseline, singles, composition) {
|
|
799
|
-
const perTask = context.input.taskOrder.map((taskId) => {
|
|
800
|
-
const baselineRow = crossSurfaceRowFor(context, baseline.candidateId, taskId);
|
|
801
|
-
const singleRows = singles.map((single) => crossSurfaceRowFor(context, single.candidateId, taskId));
|
|
802
|
-
const compositionRow = crossSurfaceRowFor(context, composition.candidateId, taskId);
|
|
803
|
-
const rows = [baselineRow, ...singleRows, compositionRow];
|
|
804
|
-
if (!rows.every((row) => row.completeness === "complete")) {
|
|
805
|
-
return { taskId, passInteraction: null, scoreInteraction: null };
|
|
806
|
-
}
|
|
807
|
-
const additivePass = singleRows.reduce((sum, row) => sum + Number(row.pass), 0) - (singles.length - 1) * Number(baselineRow.pass);
|
|
808
|
-
const additiveScore = singleRows.reduce((sum, row) => sum + row.score, 0) - (singles.length - 1) * baselineRow.score;
|
|
809
|
-
return {
|
|
810
|
-
taskId,
|
|
811
|
-
passInteraction: Number(compositionRow.pass) - additivePass,
|
|
812
|
-
scoreInteraction: compositionRow.score - additiveScore
|
|
813
|
-
};
|
|
814
|
-
});
|
|
815
|
-
const complete = perTask.filter(
|
|
816
|
-
(row) => row.passInteraction !== null && row.scoreInteraction !== null
|
|
817
|
-
);
|
|
818
|
-
const passInteractions = complete.map((row) => row.passInteraction);
|
|
819
|
-
const scoreInteractions = complete.map((row) => row.scoreInteraction);
|
|
820
|
-
return {
|
|
821
|
-
perTask,
|
|
822
|
-
n: complete.length,
|
|
823
|
-
nMissing: perTask.length - complete.length,
|
|
824
|
-
meanPassInteraction: meanOrNull(passInteractions),
|
|
825
|
-
meanScoreInteraction: meanOrNull(scoreInteractions),
|
|
826
|
-
passBootstrap: bootstrapInteraction(passInteractions, context),
|
|
827
|
-
scoreBootstrap: bootstrapInteraction(scoreInteractions, context)
|
|
828
|
-
};
|
|
829
|
-
}
|
|
830
|
-
function bootstrapInteraction(interactions, context) {
|
|
831
|
-
if (interactions.length === 0) return null;
|
|
832
|
-
return pairedBootstrap(new Array(interactions.length).fill(0), interactions, {
|
|
833
|
-
...context.input.bootstrap,
|
|
834
|
-
statistic: "mean"
|
|
835
|
-
});
|
|
836
|
-
}
|
|
837
|
-
function selectCandidates(context, summaryById, eligibleSingles, interactionReadySingles, pairwise) {
|
|
838
|
-
const bestSingleRanking = eligibleSingles.filter((summary) => {
|
|
839
|
-
const componentId = summary.candidate.componentIds[0];
|
|
840
|
-
return context.componentById.get(componentId).bestSingleEligible;
|
|
841
|
-
});
|
|
842
|
-
const ranking = bestSingleRanking.map((summary, index) => ({
|
|
843
|
-
rank: index + 1,
|
|
844
|
-
candidateId: summary.candidate.candidateId,
|
|
845
|
-
componentId: summary.candidate.componentIds[0]
|
|
846
|
-
}));
|
|
847
|
-
const best = bestSingleRanking[0];
|
|
848
|
-
const bestSingle = best ? {
|
|
849
|
-
candidateId: best.candidate.candidateId,
|
|
850
|
-
componentId: best.candidate.componentIds[0],
|
|
851
|
-
ranking
|
|
852
|
-
} : null;
|
|
853
|
-
const naiveStack = selectNaiveStack(context, eligibleSingles);
|
|
854
|
-
const interactionAware = selectInteractionAware(
|
|
855
|
-
context,
|
|
856
|
-
summaryById,
|
|
857
|
-
interactionReadySingles,
|
|
858
|
-
pairwise
|
|
859
|
-
);
|
|
860
|
-
return { bestSingle, naiveStack, interactionAware };
|
|
861
|
-
}
|
|
862
|
-
function selectNaiveStack(context, eligibleSingles) {
|
|
863
|
-
if (eligibleSingles.length === 0) return null;
|
|
864
|
-
const componentIds = eligibleSingles.map((summary) => summary.candidate.componentIds[0]).sort((left, right) => context.componentIndex.get(left) - context.componentIndex.get(right));
|
|
865
|
-
const candidate = context.candidateByComponents.get(crossSurfaceComponentSetKey(componentIds));
|
|
866
|
-
if (!candidate) {
|
|
867
|
-
throw new ValidationError(
|
|
868
|
-
`analyzeCrossSurfaceInteractions: naive stack [${componentIds.join(", ")}] was not evaluated`
|
|
869
|
-
);
|
|
870
|
-
}
|
|
871
|
-
return { candidateId: candidate.candidateId, componentIds };
|
|
872
|
-
}
|
|
873
|
-
function selectInteractionAware(context, summaryById, interactionReadySingles, pairwise) {
|
|
874
|
-
const baseline = summaryById.get(context.input.baselineCandidateId);
|
|
875
|
-
const pairByComponents = new Map(
|
|
876
|
-
pairwise.map((entry) => [crossSurfaceComponentSetKey(entry.componentIds), entry])
|
|
877
|
-
);
|
|
878
|
-
const paths = pairwise.filter((entry) => entry.compatibility.compatible).map(
|
|
879
|
-
(entry) => growInteractionPath(
|
|
880
|
-
context,
|
|
881
|
-
summaryById,
|
|
882
|
-
interactionReadySingles,
|
|
883
|
-
pairByComponents,
|
|
884
|
-
baseline,
|
|
885
|
-
summaryById.get(entry.compositionCandidateId)
|
|
886
|
-
)
|
|
887
|
-
);
|
|
888
|
-
if (paths.length === 0) return null;
|
|
889
|
-
const qualified = paths.filter((path) => path.qualified).sort((left, right) => compareInteractionPaths(context, summaryById, left, right));
|
|
890
|
-
const winning = qualified[0] ?? [...paths].sort((left, right) => compareInteractionPaths(context, summaryById, left, right))[0];
|
|
891
|
-
return {
|
|
892
|
-
seedCandidateId: winning.seedCandidateId,
|
|
893
|
-
terminalCandidateId: winning.terminalCandidateId,
|
|
894
|
-
terminalComponentIds: [...winning.terminalComponentIds],
|
|
895
|
-
selectedCandidateId: qualified.length > 0 ? winning.terminalCandidateId : null,
|
|
896
|
-
qualified: qualified.length > 0,
|
|
897
|
-
evaluatedPaths: paths,
|
|
898
|
-
steps: winning.steps
|
|
899
|
-
};
|
|
900
|
-
}
|
|
901
|
-
function growInteractionPath(context, summaryById, interactionReadySingles, pairByComponents, baseline, seed) {
|
|
902
|
-
let current = seed;
|
|
903
|
-
let retained = [...seed.candidate.componentIds];
|
|
904
|
-
let remaining = interactionReadySingles.filter(
|
|
905
|
-
(summary) => !retained.includes(summary.candidate.componentIds[0])
|
|
906
|
-
);
|
|
907
|
-
const steps = [];
|
|
908
|
-
while (remaining.length > 0) {
|
|
909
|
-
const decisions = remaining.map(
|
|
910
|
-
(addition) => evaluateAddition(
|
|
911
|
-
context,
|
|
912
|
-
summaryById,
|
|
913
|
-
pairByComponents,
|
|
914
|
-
baseline,
|
|
915
|
-
current,
|
|
916
|
-
retained,
|
|
917
|
-
addition
|
|
918
|
-
)
|
|
919
|
-
);
|
|
920
|
-
const eligible = decisions.filter((decision) => decision.eligible).sort((left, right) => compareAdditionDecisions(context, summaryById, left, right));
|
|
921
|
-
const selected = eligible[0];
|
|
922
|
-
if (selected) selected.selected = true;
|
|
923
|
-
steps.push({
|
|
924
|
-
fromCandidateId: current.candidate.candidateId,
|
|
925
|
-
retainedComponentIds: [...retained],
|
|
926
|
-
considered: decisions.sort(
|
|
927
|
-
(left, right) => context.candidateIndex.get(left.additionCandidateId) - context.candidateIndex.get(right.additionCandidateId)
|
|
928
|
-
),
|
|
929
|
-
selectedCandidateId: selected?.bundleCandidateId ?? null
|
|
930
|
-
});
|
|
931
|
-
if (!selected?.bundleCandidateId) break;
|
|
932
|
-
current = summaryById.get(selected.bundleCandidateId);
|
|
933
|
-
retained = [...current.candidate.componentIds];
|
|
934
|
-
remaining = remaining.filter(
|
|
935
|
-
(summary) => summary.candidate.candidateId !== selected.additionCandidateId
|
|
936
|
-
);
|
|
937
|
-
}
|
|
938
|
-
const qualified = retained.length >= context.input.selection.minimumBundleComponents;
|
|
939
|
-
return {
|
|
940
|
-
seedCandidateId: seed.candidate.candidateId,
|
|
941
|
-
terminalCandidateId: current.candidate.candidateId,
|
|
942
|
-
terminalComponentIds: retained,
|
|
943
|
-
qualified,
|
|
944
|
-
steps
|
|
945
|
-
};
|
|
946
|
-
}
|
|
947
|
-
function compareInteractionPaths(context, summaryById, left, right) {
|
|
948
|
-
const leftSummary = summaryById.get(left.terminalCandidateId);
|
|
949
|
-
const rightSummary = summaryById.get(right.terminalCandidateId);
|
|
950
|
-
const byNetBenefit = rightSummary.outcome.netBenefit - leftSummary.outcome.netBenefit;
|
|
951
|
-
if (byNetBenefit !== 0) return byNetBenefit;
|
|
952
|
-
const byBenefit = rightSummary.outcome.benefitTaskIds.length - leftSummary.outcome.benefitTaskIds.length;
|
|
953
|
-
if (byBenefit !== 0) return byBenefit;
|
|
954
|
-
const byRegression = leftSummary.outcome.regressionTaskIds.length - rightSummary.outcome.regressionTaskIds.length;
|
|
955
|
-
if (byRegression !== 0) return byRegression;
|
|
956
|
-
for (const metric of context.input.costMetricOrder) {
|
|
957
|
-
const byCost = leftSummary.costs[metric].median - rightSummary.costs[metric].median;
|
|
958
|
-
if (byCost !== 0) return byCost;
|
|
959
|
-
}
|
|
960
|
-
const byBytes = leftSummary.candidate.artifactBytes - rightSummary.candidate.artifactBytes;
|
|
961
|
-
if (byBytes !== 0) return byBytes;
|
|
962
|
-
const byCandidateOrder = context.candidateIndex.get(left.terminalCandidateId) - context.candidateIndex.get(right.terminalCandidateId);
|
|
963
|
-
if (byCandidateOrder !== 0) return byCandidateOrder;
|
|
964
|
-
return context.candidateIndex.get(left.seedCandidateId) - context.candidateIndex.get(right.seedCandidateId);
|
|
965
|
-
}
|
|
966
|
-
function evaluateAddition(context, summaryById, pairByComponents, baseline, current, retained, addition) {
|
|
967
|
-
const additionComponentId = addition.candidate.componentIds[0];
|
|
968
|
-
const reasons = [];
|
|
969
|
-
for (const retainedComponentId of retained) {
|
|
970
|
-
const pair = pairByComponents.get(
|
|
971
|
-
crossSurfaceComponentSetKey(canonicalCrossSurfaceComponents(context, [retainedComponentId, additionComponentId]))
|
|
972
|
-
);
|
|
973
|
-
if (!pair?.compatibility.compatible) reasons.push("pair_incompatible");
|
|
974
|
-
}
|
|
975
|
-
const bundleComponents = canonicalCrossSurfaceComponents(context, [...retained, additionComponentId]);
|
|
976
|
-
const bundle = context.candidateByComponents.get(crossSurfaceComponentSetKey(bundleComponents));
|
|
977
|
-
if (!bundle) {
|
|
978
|
-
reasons.push("full_bundle_not_evaluated");
|
|
979
|
-
return emptyAdditionDecision(addition, additionComponentId, reasons);
|
|
980
|
-
}
|
|
981
|
-
const bundleSummary = summaryById.get(bundle.candidateId);
|
|
982
|
-
if (bundleSummary.outcome.missingTaskIds.length > 0 || bundleSummary.outcome.invalidTaskIds.length > 0 || bundleSummary.outcome.comparisonMissingTaskIds.length > 0) {
|
|
983
|
-
reasons.push("bundle_incomplete");
|
|
984
|
-
}
|
|
985
|
-
if (bundleSummary.outcome.regressionTaskIds.length > 0) reasons.push("baseline_regression");
|
|
986
|
-
const comparison = compareCandidates(
|
|
987
|
-
context,
|
|
988
|
-
summaryById,
|
|
989
|
-
current.candidate.candidateId,
|
|
990
|
-
bundle.candidateId
|
|
991
|
-
);
|
|
992
|
-
if (comparison.winsTaskIds.length === 0) reasons.push("no_incremental_resolution");
|
|
993
|
-
if (comparison.regressionTaskIds.length > 0) reasons.push("incremental_regression");
|
|
994
|
-
appendBundleEvidenceReasons(
|
|
995
|
-
reasons,
|
|
996
|
-
bundleSummary.firing,
|
|
997
|
-
context.input.selection.minimumFiringTasks,
|
|
998
|
-
context.input.selection.requireObservedFiring,
|
|
999
|
-
"firing_below_minimum",
|
|
1000
|
-
"firing_unobserved"
|
|
1001
|
-
);
|
|
1002
|
-
appendBundleEvidenceReasons(
|
|
1003
|
-
reasons,
|
|
1004
|
-
bundleSummary.effect,
|
|
1005
|
-
context.input.selection.minimumEffectTasks,
|
|
1006
|
-
context.input.selection.requireObservedEffect,
|
|
1007
|
-
"effect_below_minimum",
|
|
1008
|
-
"effect_unobserved"
|
|
1009
|
-
);
|
|
1010
|
-
if (!withinCostLimits(context, bundleSummary, baseline)) reasons.push("cost_limit_exceeded");
|
|
1011
|
-
return {
|
|
1012
|
-
additionCandidateId: addition.candidate.candidateId,
|
|
1013
|
-
additionComponentId,
|
|
1014
|
-
bundleCandidateId: bundle.candidateId,
|
|
1015
|
-
incrementalResolutionTaskIds: comparison.winsTaskIds,
|
|
1016
|
-
incrementalRegressionTaskIds: comparison.regressionTaskIds,
|
|
1017
|
-
incrementalMedianCost: Object.fromEntries(
|
|
1018
|
-
context.input.costMetricOrder.map((metric) => [
|
|
1019
|
-
metric,
|
|
1020
|
-
bundleSummary.costs[metric].median - current.costs[metric].median
|
|
1021
|
-
])
|
|
1022
|
-
),
|
|
1023
|
-
eligible: reasons.length === 0,
|
|
1024
|
-
selected: false,
|
|
1025
|
-
reasons: unique(reasons)
|
|
1026
|
-
};
|
|
1027
|
-
}
|
|
1028
|
-
function emptyAdditionDecision(addition, additionComponentId, reasons) {
|
|
1029
|
-
return {
|
|
1030
|
-
additionCandidateId: addition.candidate.candidateId,
|
|
1031
|
-
additionComponentId,
|
|
1032
|
-
bundleCandidateId: null,
|
|
1033
|
-
incrementalResolutionTaskIds: [],
|
|
1034
|
-
incrementalRegressionTaskIds: [],
|
|
1035
|
-
incrementalMedianCost: null,
|
|
1036
|
-
eligible: false,
|
|
1037
|
-
selected: false,
|
|
1038
|
-
reasons: unique(reasons)
|
|
1039
|
-
};
|
|
1040
|
-
}
|
|
1041
|
-
function appendBundleEvidenceReasons(reasons, evidence, minimum, requireObserved, belowReason, unobservedReason) {
|
|
1042
|
-
if (evidence.byComponent.some((component) => component.observedTaskIds.length < minimum)) {
|
|
1043
|
-
reasons.push(belowReason);
|
|
1044
|
-
}
|
|
1045
|
-
if (requireObserved && evidence.byComponent.some((component) => component.unobservedTaskIds.length > 0)) {
|
|
1046
|
-
reasons.push(unobservedReason);
|
|
1047
|
-
}
|
|
1048
|
-
}
|
|
1049
|
-
function compareAdditionDecisions(context, summaryById, left, right) {
|
|
1050
|
-
const byWins = right.incrementalResolutionTaskIds.length - left.incrementalResolutionTaskIds.length;
|
|
1051
|
-
if (byWins !== 0) return byWins;
|
|
1052
|
-
for (const metric of context.input.costMetricOrder) {
|
|
1053
|
-
const byCost = left.incrementalMedianCost[metric] - right.incrementalMedianCost[metric];
|
|
1054
|
-
if (byCost !== 0) return byCost;
|
|
1055
|
-
}
|
|
1056
|
-
const leftBytes = summaryById.get(left.additionCandidateId).candidate.artifactBytes;
|
|
1057
|
-
const rightBytes = summaryById.get(right.additionCandidateId).candidate.artifactBytes;
|
|
1058
|
-
const byBytes = leftBytes - rightBytes;
|
|
1059
|
-
if (byBytes !== 0) return byBytes;
|
|
1060
|
-
return context.candidateIndex.get(left.additionCandidateId) - context.candidateIndex.get(right.additionCandidateId);
|
|
1061
|
-
}
|
|
1062
|
-
function compareCandidates(context, summaryById, comparatorCandidateId, treatmentCandidateId) {
|
|
1063
|
-
const winsTaskIds = [];
|
|
1064
|
-
const regressionTaskIds = [];
|
|
1065
|
-
const missingTaskIds = [];
|
|
1066
|
-
for (const taskId of context.input.taskOrder) {
|
|
1067
|
-
const comparator = crossSurfaceRowFor(context, comparatorCandidateId, taskId);
|
|
1068
|
-
const treatment = crossSurfaceRowFor(context, treatmentCandidateId, taskId);
|
|
1069
|
-
if (comparator.completeness !== "complete" || treatment.completeness !== "complete") {
|
|
1070
|
-
missingTaskIds.push(taskId);
|
|
1071
|
-
} else if (treatment.pass && !comparator.pass) winsTaskIds.push(taskId);
|
|
1072
|
-
else if (!treatment.pass && comparator.pass) regressionTaskIds.push(taskId);
|
|
1073
|
-
}
|
|
1074
|
-
return {
|
|
1075
|
-
comparatorCandidateId,
|
|
1076
|
-
treatmentCandidateId,
|
|
1077
|
-
winsTaskIds,
|
|
1078
|
-
regressionTaskIds,
|
|
1079
|
-
missingTaskIds,
|
|
1080
|
-
paired: pairedComparison(context, comparatorCandidateId, treatmentCandidateId),
|
|
1081
|
-
relativeCost: relativeCosts(
|
|
1082
|
-
context,
|
|
1083
|
-
summaryById.get(treatmentCandidateId),
|
|
1084
|
-
summaryById.get(comparatorCandidateId)
|
|
1085
|
-
)
|
|
1086
|
-
};
|
|
1087
|
-
}
|
|
1088
|
-
function pairedComparison(context, comparatorCandidateId, treatmentCandidateId) {
|
|
1089
|
-
const rows = [];
|
|
1090
|
-
for (const taskId of context.input.taskOrder) {
|
|
1091
|
-
rows.push(toPairedArmRow(context, crossSurfaceRowFor(context, comparatorCandidateId, taskId)));
|
|
1092
|
-
rows.push(toPairedArmRow(context, crossSurfaceRowFor(context, treatmentCandidateId, taskId)));
|
|
1093
|
-
}
|
|
1094
|
-
return comparePairedArms(rows, {
|
|
1095
|
-
baselineArm: comparatorCandidateId,
|
|
1096
|
-
treatmentArm: treatmentCandidateId,
|
|
1097
|
-
metricNames: ["score", ...context.input.costMetricOrder],
|
|
1098
|
-
bootstrap: { ...context.input.bootstrap, statistic: "mean" }
|
|
1099
|
-
});
|
|
1100
|
-
}
|
|
1101
|
-
function toPairedArmRow(context, row) {
|
|
1102
|
-
const metrics = Object.fromEntries(
|
|
1103
|
-
context.input.costMetricOrder.map((metric) => [metric, row.cost[metric]])
|
|
1104
|
-
);
|
|
1105
|
-
if (row.completeness === "complete") metrics.score = row.score;
|
|
1106
|
-
return {
|
|
1107
|
-
pairKey: row.taskId,
|
|
1108
|
-
arm: row.candidateId,
|
|
1109
|
-
...row.completeness === "complete" ? { pass: row.pass } : {},
|
|
1110
|
-
metrics
|
|
1111
|
-
};
|
|
1112
|
-
}
|
|
1113
|
-
function summarizePairEvidence(rows, leftComponentId, rightComponentId, field) {
|
|
1114
|
-
const bothTaskIds = [];
|
|
1115
|
-
const leftOnlyTaskIds = [];
|
|
1116
|
-
const rightOnlyTaskIds = [];
|
|
1117
|
-
const neitherTaskIds = [];
|
|
1118
|
-
const unobservedTaskIds = [];
|
|
1119
|
-
for (const row of rows) {
|
|
1120
|
-
const left = evidenceValue(row, leftComponentId, field);
|
|
1121
|
-
const right = evidenceValue(row, rightComponentId, field);
|
|
1122
|
-
if (left === null || right === null) unobservedTaskIds.push(row.taskId);
|
|
1123
|
-
else if (left && right) bothTaskIds.push(row.taskId);
|
|
1124
|
-
else if (left) leftOnlyTaskIds.push(row.taskId);
|
|
1125
|
-
else if (right) rightOnlyTaskIds.push(row.taskId);
|
|
1126
|
-
else neitherTaskIds.push(row.taskId);
|
|
1127
|
-
}
|
|
1128
|
-
return {
|
|
1129
|
-
bothTaskIds,
|
|
1130
|
-
leftOnlyTaskIds,
|
|
1131
|
-
rightOnlyTaskIds,
|
|
1132
|
-
neitherTaskIds,
|
|
1133
|
-
unobservedTaskIds
|
|
1134
|
-
};
|
|
1135
|
-
}
|
|
1136
|
-
function relativeCosts(context, treatment, comparator) {
|
|
1137
|
-
return Object.fromEntries(
|
|
1138
|
-
context.input.costMetricOrder.map((metric) => {
|
|
1139
|
-
const treatmentMedian = treatment.costs[metric].median;
|
|
1140
|
-
const comparatorMedian = comparator.costs[metric].median;
|
|
1141
|
-
return [
|
|
1142
|
-
metric,
|
|
1143
|
-
{
|
|
1144
|
-
treatmentMedian,
|
|
1145
|
-
comparatorMedian,
|
|
1146
|
-
medianDelta: treatmentMedian - comparatorMedian,
|
|
1147
|
-
medianRatio: comparatorMedian === 0 ? treatmentMedian === 0 ? 1 : null : treatmentMedian / comparatorMedian
|
|
1148
|
-
}
|
|
1149
|
-
];
|
|
1150
|
-
})
|
|
1151
|
-
);
|
|
1152
|
-
}
|
|
1153
|
-
function withinCostLimits(context, treatment, baseline) {
|
|
1154
|
-
const relative3 = relativeCosts(context, treatment, baseline);
|
|
1155
|
-
return Object.entries(context.input.selection.maximumMedianCostRatioToBaseline).every(
|
|
1156
|
-
([metric, limit]) => {
|
|
1157
|
-
const ratio = relative3[metric].medianRatio;
|
|
1158
|
-
return ratio !== null && ratio <= limit;
|
|
1159
|
-
}
|
|
1160
|
-
);
|
|
1161
|
-
}
|
|
1162
|
-
function distribution(values) {
|
|
1163
|
-
const sorted = [...values].sort((left, right) => left - right);
|
|
1164
|
-
const total = sorted.reduce((sum, value) => sum + value, 0);
|
|
1165
|
-
const middle = Math.floor(sorted.length / 2);
|
|
1166
|
-
const median = sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
1167
|
-
return {
|
|
1168
|
-
n: sorted.length,
|
|
1169
|
-
min: sorted[0],
|
|
1170
|
-
median,
|
|
1171
|
-
mean: total / sorted.length,
|
|
1172
|
-
max: sorted[sorted.length - 1],
|
|
1173
|
-
total
|
|
1174
|
-
};
|
|
1175
|
-
}
|
|
1176
|
-
function evidenceValue(row, componentId, field) {
|
|
1177
|
-
return row.componentEvidence.find((evidence) => evidence.componentId === componentId)[field];
|
|
1178
|
-
}
|
|
1179
|
-
function meanOrNull(values) {
|
|
1180
|
-
return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
1181
|
-
}
|
|
1182
|
-
function unique(values) {
|
|
1183
|
-
return [...new Set(values)];
|
|
1184
|
-
}
|
|
1185
|
-
|
|
1186
|
-
// src/campaign/fixtures.ts
|
|
1187
|
-
import { createHash } from "crypto";
|
|
1188
|
-
import { existsSync, readdirSync, readFileSync, statSync } from "fs";
|
|
1189
|
-
import { isAbsolute, join, relative, resolve } from "path";
|
|
1190
|
-
function discoverEvalFixtures(evalsDir) {
|
|
1191
|
-
const root = resolve(evalsDir);
|
|
1192
|
-
if (!existsSync(root)) throw new Error(`discoverEvalFixtures: evalsDir not found: ${root}`);
|
|
1193
|
-
const fixtures = [];
|
|
1194
|
-
const walk = (dir, base = "") => {
|
|
1195
|
-
for (const entry of readdirSync(dir).sort()) {
|
|
1196
|
-
if (entry.startsWith(".") || entry === "node_modules") continue;
|
|
1197
|
-
const fullPath = join(dir, entry);
|
|
1198
|
-
if (!statSync(fullPath).isDirectory()) continue;
|
|
1199
|
-
const name = base ? `${base}/${entry}` : entry;
|
|
1200
|
-
if (existsWithExactCase(fullPath, "PROMPT.md")) {
|
|
1201
|
-
fixtures.push(name);
|
|
1202
|
-
} else {
|
|
1203
|
-
walk(fullPath, name);
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
};
|
|
1207
|
-
walk(root);
|
|
1208
|
-
return fixtures;
|
|
1209
|
-
}
|
|
1210
|
-
function loadEvalFixture(evalsDir, name, options = {}) {
|
|
1211
|
-
const validation = options.validation ?? "vitest";
|
|
1212
|
-
const root = resolve(evalsDir);
|
|
1213
|
-
const fixturePath = resolve(root, name);
|
|
1214
|
-
assertInside(root, fixturePath, name);
|
|
1215
|
-
if (!existsSync(fixturePath) || !statSync(fixturePath).isDirectory()) {
|
|
1216
|
-
throw new Error(`loadEvalFixture: fixture not found: ${name}`);
|
|
1217
|
-
}
|
|
1218
|
-
if (!existsWithExactCase(fixturePath, "PROMPT.md")) {
|
|
1219
|
-
throw new Error(`loadEvalFixture: ${name} is missing exact-case PROMPT.md`);
|
|
1220
|
-
}
|
|
1221
|
-
const promptPath = join(fixturePath, "PROMPT.md");
|
|
1222
|
-
const evalPath = resolveEvalPath(fixturePath);
|
|
1223
|
-
const packageJsonPath = existsWithExactCase(fixturePath, "package.json") ? join(fixturePath, "package.json") : void 0;
|
|
1224
|
-
if (validation !== "none") {
|
|
1225
|
-
if (!evalPath)
|
|
1226
|
-
throw new Error(`loadEvalFixture: ${name} is missing exact-case EVAL.ts or EVAL.tsx`);
|
|
1227
|
-
if (!packageJsonPath)
|
|
1228
|
-
throw new Error(`loadEvalFixture: ${name} is missing exact-case package.json`);
|
|
1229
|
-
assertModulePackage(packageJsonPath, name);
|
|
1230
|
-
}
|
|
1231
|
-
const files = collectFixtureFiles(fixturePath);
|
|
1232
|
-
return {
|
|
1233
|
-
name,
|
|
1234
|
-
path: fixturePath,
|
|
1235
|
-
promptPath,
|
|
1236
|
-
evalPath,
|
|
1237
|
-
packageJsonPath,
|
|
1238
|
-
prompt: readFileSync(promptPath, "utf8"),
|
|
1239
|
-
files,
|
|
1240
|
-
fingerprint: contentHash({
|
|
1241
|
-
files,
|
|
1242
|
-
config: options.fingerprintConfig ?? null
|
|
1243
|
-
})
|
|
1244
|
-
};
|
|
1245
|
-
}
|
|
1246
|
-
function loadEvalFixtureScenarios(evalsDir, options = {}) {
|
|
1247
|
-
const names = options.names ?? discoverEvalFixtures(evalsDir);
|
|
1248
|
-
return names.map((name) => {
|
|
1249
|
-
const fixture = loadEvalFixture(evalsDir, name, options);
|
|
1250
|
-
return {
|
|
1251
|
-
id: fixture.name,
|
|
1252
|
-
kind: "eval-fixture",
|
|
1253
|
-
tags: ["eval-fixture"],
|
|
1254
|
-
fixtureName: fixture.name,
|
|
1255
|
-
fixturePath: fixture.path,
|
|
1256
|
-
promptPath: fixture.promptPath,
|
|
1257
|
-
evalPath: fixture.evalPath,
|
|
1258
|
-
packageJsonPath: fixture.packageJsonPath,
|
|
1259
|
-
prompt: fixture.prompt,
|
|
1260
|
-
fingerprint: fixture.fingerprint
|
|
1261
|
-
};
|
|
1262
|
-
});
|
|
1263
|
-
}
|
|
1264
|
-
function planEvalFixtureRun(options) {
|
|
1265
|
-
const scenarios = loadEvalFixtureScenarios(options.evalsDir, {
|
|
1266
|
-
names: options.names,
|
|
1267
|
-
validation: options.validation,
|
|
1268
|
-
fingerprintConfig: options.fingerprintConfig
|
|
1269
|
-
});
|
|
1270
|
-
const plan = planCampaignRun({
|
|
1271
|
-
scenarios,
|
|
1272
|
-
dispatchRef: options.dispatchRef ?? "eval-fixture-dispatch",
|
|
1273
|
-
judges: options.judges,
|
|
1274
|
-
seed: options.seed,
|
|
1275
|
-
reps: options.reps,
|
|
1276
|
-
resumable: options.resumable,
|
|
1277
|
-
runDir: options.runDir,
|
|
1278
|
-
storage: options.storage
|
|
1279
|
-
});
|
|
1280
|
-
return {
|
|
1281
|
-
...plan,
|
|
1282
|
-
fixtures: scenarios.map((scenario) => ({
|
|
1283
|
-
fixtureName: scenario.fixtureName,
|
|
1284
|
-
fixturePath: scenario.fixturePath,
|
|
1285
|
-
fingerprint: scenario.fingerprint
|
|
1286
|
-
}))
|
|
1287
|
-
};
|
|
1288
|
-
}
|
|
1289
|
-
function existsWithExactCase(dirPath, fileName) {
|
|
1290
|
-
try {
|
|
1291
|
-
return readdirSync(dirPath).includes(fileName);
|
|
1292
|
-
} catch {
|
|
1293
|
-
return false;
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
function resolveEvalPath(fixturePath) {
|
|
1297
|
-
if (existsWithExactCase(fixturePath, "EVAL.ts")) return join(fixturePath, "EVAL.ts");
|
|
1298
|
-
if (existsWithExactCase(fixturePath, "EVAL.tsx")) return join(fixturePath, "EVAL.tsx");
|
|
1299
|
-
return void 0;
|
|
1300
|
-
}
|
|
1301
|
-
function assertModulePackage(packageJsonPath, name) {
|
|
1302
|
-
let parsed;
|
|
1303
|
-
try {
|
|
1304
|
-
parsed = JSON.parse(readFileSync(packageJsonPath, "utf8"));
|
|
1305
|
-
} catch (err) {
|
|
1306
|
-
throw new Error(
|
|
1307
|
-
`loadEvalFixture: ${name} package.json is invalid JSON: ${err instanceof Error ? err.message : String(err)}`
|
|
1308
|
-
);
|
|
1309
|
-
}
|
|
1310
|
-
if (typeof parsed !== "object" || parsed === null || parsed.type !== "module") {
|
|
1311
|
-
throw new Error(`loadEvalFixture: ${name} package.json must set "type": "module"`);
|
|
1312
|
-
}
|
|
1313
|
-
}
|
|
1314
|
-
function collectFixtureFiles(fixturePath, base = "") {
|
|
1315
|
-
const files = [];
|
|
1316
|
-
for (const entry of readdirSync(join(fixturePath, base)).sort()) {
|
|
1317
|
-
if (entry === "node_modules" || entry === ".git") continue;
|
|
1318
|
-
const relativePath = base ? `${base}/${entry}` : entry;
|
|
1319
|
-
const fullPath = join(fixturePath, relativePath);
|
|
1320
|
-
const stat = statSync(fullPath);
|
|
1321
|
-
if (stat.isDirectory()) {
|
|
1322
|
-
files.push(...collectFixtureFiles(fixturePath, relativePath));
|
|
1323
|
-
continue;
|
|
1324
|
-
}
|
|
1325
|
-
const bytes = readFileSync(fullPath);
|
|
1326
|
-
files.push({
|
|
1327
|
-
path: relativePath,
|
|
1328
|
-
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
1329
|
-
bytes: bytes.byteLength
|
|
1330
|
-
});
|
|
1331
|
-
}
|
|
1332
|
-
return files;
|
|
1333
|
-
}
|
|
1334
|
-
function assertInside(root, target, label) {
|
|
1335
|
-
const rel = relative(root, target);
|
|
1336
|
-
if (rel === "" || !rel.startsWith("..") && !isAbsolute(rel)) return;
|
|
1337
|
-
throw new Error(`loadEvalFixture: fixture path escapes evalsDir: ${label}`);
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
// src/campaign/gates/neutralization-gate.ts
|
|
1341
|
-
function mean(xs) {
|
|
1342
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
1343
|
-
}
|
|
1344
|
-
function pairedLift(arm, baseline, scenarioIds) {
|
|
1345
|
-
const paired = pairHoldout(arm, baseline, scenarioIds, (s) => s.composite);
|
|
1346
|
-
const deltas = paired.after.map((a, i) => a - (paired.before[i] ?? 0));
|
|
1347
|
-
return { lift: mean(deltas), n: deltas.length };
|
|
1348
|
-
}
|
|
1349
|
-
function neutralizationGate(options) {
|
|
1350
|
-
const maxDecorativeFraction = options.maxDecorativeFraction ?? 0.5;
|
|
1351
|
-
return {
|
|
1352
|
-
name: "neutralizationGate",
|
|
1353
|
-
async decide(ctx) {
|
|
1354
|
-
if (!ctx.baselineJudgeScores) {
|
|
1355
|
-
throw new Error(
|
|
1356
|
-
"neutralizationGate: ctx.baselineJudgeScores is required \u2014 the placebo control measures lift OVER baseline."
|
|
1357
|
-
);
|
|
1358
|
-
}
|
|
1359
|
-
if (!ctx.neutralizedJudgeScores) {
|
|
1360
|
-
throw new Error(
|
|
1361
|
-
"neutralizationGate: ctx.neutralizedJudgeScores is required. It is populated by runImprovementLoop only when a `neutralize` function is supplied \u2014 composing this gate without that wiring would pass an unproven candidate."
|
|
1362
|
-
);
|
|
1363
|
-
}
|
|
1364
|
-
const scenarioIds = new Set(options.scenarios.map((s) => s.id));
|
|
1365
|
-
const cand = pairedLift(
|
|
1366
|
-
ctx.judgeScores,
|
|
1367
|
-
ctx.baselineJudgeScores,
|
|
1368
|
-
scenarioIds
|
|
1369
|
-
);
|
|
1370
|
-
const neut = pairedLift(
|
|
1371
|
-
ctx.neutralizedJudgeScores,
|
|
1372
|
-
ctx.baselineJudgeScores,
|
|
1373
|
-
scenarioIds
|
|
1374
|
-
);
|
|
1375
|
-
if (cand.lift <= 0) {
|
|
1376
|
-
return {
|
|
1377
|
-
decision: "hold",
|
|
1378
|
-
reasons: [
|
|
1379
|
-
`neutralization: candidate held-out lift ${cand.lift.toFixed(3)} \u2264 0 \u2014 no positive lift to attribute to content`
|
|
1380
|
-
],
|
|
1381
|
-
contributingGates: [
|
|
1382
|
-
{
|
|
1383
|
-
name: "neutralizationGate",
|
|
1384
|
-
status: "fail",
|
|
1385
|
-
detail: { candidateLift: cand.lift, neutralizedLift: neut.lift, n: cand.n }
|
|
1386
|
-
}
|
|
1387
|
-
],
|
|
1388
|
-
delta: cand.lift
|
|
1389
|
-
};
|
|
1390
|
-
}
|
|
1391
|
-
const decorativeFraction = neut.lift / cand.lift;
|
|
1392
|
-
const passed = decorativeFraction < maxDecorativeFraction;
|
|
1393
|
-
const pct = (decorativeFraction * 100).toFixed(0);
|
|
1394
|
-
return {
|
|
1395
|
-
decision: passed ? "ship" : "hold",
|
|
1396
|
-
reasons: passed ? [
|
|
1397
|
-
`neutralization: content is causal \u2014 blanked variant reproduces ${pct}% of the lift (< ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
|
|
1398
|
-
] : [
|
|
1399
|
-
`neutralization: lift is DECORATIVE \u2014 blanking the content (footprint-matched) reproduces ${pct}% of the lift (\u2265 ${(maxDecorativeFraction * 100).toFixed(0)}%); candidate \u0394 ${cand.lift.toFixed(3)}, neutralized \u0394 ${neut.lift.toFixed(3)}`
|
|
1400
|
-
],
|
|
1401
|
-
contributingGates: [
|
|
1402
|
-
{
|
|
1403
|
-
name: "neutralizationGate",
|
|
1404
|
-
status: passed ? "pass" : "fail",
|
|
1405
|
-
detail: {
|
|
1406
|
-
candidateLift: cand.lift,
|
|
1407
|
-
neutralizedLift: neut.lift,
|
|
1408
|
-
decorativeFraction,
|
|
1409
|
-
maxDecorativeFraction,
|
|
1410
|
-
n: cand.n
|
|
1411
|
-
}
|
|
1412
|
-
}
|
|
1413
|
-
],
|
|
1414
|
-
delta: cand.lift
|
|
1415
|
-
};
|
|
1416
|
-
}
|
|
1417
|
-
};
|
|
1418
|
-
}
|
|
1419
|
-
|
|
1420
|
-
// src/campaign/gates/sequential.ts
|
|
1421
|
-
import { createHash as createHash2 } from "crypto";
|
|
1422
|
-
function verifyManifestSync(m) {
|
|
1423
|
-
if (m.algo !== void 0 && m.algo !== "sha256-content") {
|
|
1424
|
-
throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
|
|
1425
|
-
}
|
|
1426
|
-
const { contentHash: contentHash2, algo: _algo, ...rest } = m;
|
|
1427
|
-
void _algo;
|
|
1428
|
-
const bytes = JSON.stringify(canonicalize(rest));
|
|
1429
|
-
const hash = createHash2("sha256").update(bytes, "utf8").digest("hex");
|
|
1430
|
-
return hash === contentHash2;
|
|
1431
|
-
}
|
|
1432
|
-
function resolveConfig(opts) {
|
|
1433
|
-
const m = opts.preRegistration;
|
|
1434
|
-
let alpha;
|
|
1435
|
-
let maxN;
|
|
1436
|
-
let direction;
|
|
1437
|
-
let minEffect;
|
|
1438
|
-
if (m) {
|
|
1439
|
-
if (!verifyManifestSync(m)) {
|
|
1440
|
-
throw new Error(
|
|
1441
|
-
`sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
|
|
1442
|
-
);
|
|
1443
|
-
}
|
|
1444
|
-
if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
|
|
1445
|
-
throw new Error(
|
|
1446
|
-
`sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
|
|
1447
|
-
);
|
|
1448
|
-
}
|
|
1449
|
-
if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
|
|
1450
|
-
throw new Error(
|
|
1451
|
-
`sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
|
|
1452
|
-
);
|
|
1453
|
-
}
|
|
1454
|
-
alpha = m.alpha;
|
|
1455
|
-
maxN = m.preRegisteredN;
|
|
1456
|
-
direction = m.direction;
|
|
1457
|
-
minEffect = m.minEffect;
|
|
1458
|
-
} else {
|
|
1459
|
-
alpha = opts.alpha ?? 0.05;
|
|
1460
|
-
if (opts.maxN === void 0) {
|
|
1461
|
-
throw new Error(
|
|
1462
|
-
"sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
|
|
1463
|
-
);
|
|
1464
|
-
}
|
|
1465
|
-
maxN = opts.maxN;
|
|
1466
|
-
direction = "increase";
|
|
1467
|
-
minEffect = 0;
|
|
1468
|
-
}
|
|
1469
|
-
const scale = opts.scale ?? 1;
|
|
1470
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
1471
|
-
throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
|
|
1472
|
-
}
|
|
1473
|
-
if (!Number.isInteger(maxN) || maxN < 1) {
|
|
1474
|
-
throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
|
|
1475
|
-
}
|
|
1476
|
-
const minN = opts.minN ?? 5;
|
|
1477
|
-
if (!Number.isInteger(minN) || minN < 1 || minN > maxN) {
|
|
1478
|
-
throw new Error(
|
|
1479
|
-
`sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`
|
|
1480
|
-
);
|
|
1481
|
-
}
|
|
1482
|
-
if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) {
|
|
1483
|
-
throw new Error(
|
|
1484
|
-
`sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`
|
|
1485
|
-
);
|
|
1486
|
-
}
|
|
1487
|
-
const nullMean = 0.5 + minEffect / (2 * scale);
|
|
1488
|
-
return {
|
|
1489
|
-
alpha,
|
|
1490
|
-
minN,
|
|
1491
|
-
maxN,
|
|
1492
|
-
maxBet: opts.maxBet ?? 0.5,
|
|
1493
|
-
scale,
|
|
1494
|
-
shuffleSeed: opts.shuffleSeed ?? 1337,
|
|
1495
|
-
direction,
|
|
1496
|
-
nullMean,
|
|
1497
|
-
minEffect
|
|
1498
|
-
};
|
|
1499
|
-
}
|
|
1500
|
-
function makeStream(cfg) {
|
|
1501
|
-
const proc = eProcess({ alpha: cfg.alpha, maxBet: cfg.maxBet, nullMean: cfg.nullMean });
|
|
1502
|
-
const threshold = 1 / cfg.alpha;
|
|
1503
|
-
let terminal;
|
|
1504
|
-
return {
|
|
1505
|
-
observe(delta) {
|
|
1506
|
-
if (terminal === "undecided-at-maxN") {
|
|
1507
|
-
throw new Error(
|
|
1508
|
-
`sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted \u2014 extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`
|
|
1509
|
-
);
|
|
1510
|
-
}
|
|
1511
|
-
if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) {
|
|
1512
|
-
throw new Error(
|
|
1513
|
-
`sequentialPairedGate: delta ${delta} outside \xB1scale=${cfg.scale} \u2014 pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`
|
|
1514
|
-
);
|
|
1515
|
-
}
|
|
1516
|
-
const d = cfg.direction === "decrease" ? -delta : delta;
|
|
1517
|
-
const step = proc.update((d / cfg.scale + 1) / 2);
|
|
1518
|
-
if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) {
|
|
1519
|
-
terminal = "promote";
|
|
1520
|
-
}
|
|
1521
|
-
if (terminal === "promote") {
|
|
1522
|
-
return {
|
|
1523
|
-
decision: "promote",
|
|
1524
|
-
eValue: step.wealth,
|
|
1525
|
-
n: step.n,
|
|
1526
|
-
reason: `e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level \u03B1=${cfg.alpha}`
|
|
1527
|
-
};
|
|
1528
|
-
}
|
|
1529
|
-
if (step.n >= cfg.maxN) {
|
|
1530
|
-
terminal = "undecided-at-maxN";
|
|
1531
|
-
return {
|
|
1532
|
-
decision: "undecided-at-maxN",
|
|
1533
|
-
eValue: step.wealth,
|
|
1534
|
-
n: step.n,
|
|
1535
|
-
reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)}). This is NOT evidence of no effect \u2014 the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
|
|
1536
|
-
};
|
|
1537
|
-
}
|
|
1538
|
-
return {
|
|
1539
|
-
decision: "continue",
|
|
1540
|
-
eValue: step.wealth,
|
|
1541
|
-
n: step.n,
|
|
1542
|
-
reason: `e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} \u2014 keep observing`
|
|
1543
|
-
};
|
|
1544
|
-
},
|
|
1545
|
-
state() {
|
|
1546
|
-
return { ...proc.state(), decision: terminal ?? "continue" };
|
|
1547
|
-
}
|
|
1548
|
-
};
|
|
1549
|
-
}
|
|
1550
|
-
function seededShuffle(items, seed) {
|
|
1551
|
-
const rng = mulberry32(seed);
|
|
1552
|
-
for (let i = items.length - 1; i > 0; i--) {
|
|
1553
|
-
const j = Math.floor(rng() * (i + 1));
|
|
1554
|
-
const tmp = items[i];
|
|
1555
|
-
items[i] = items[j];
|
|
1556
|
-
items[j] = tmp;
|
|
1557
|
-
}
|
|
1558
|
-
return items;
|
|
1559
|
-
}
|
|
1560
|
-
function sequentialPairedGate(options) {
|
|
1561
|
-
const cfg = resolveConfig(options);
|
|
1562
|
-
const name = options.name ?? "sequentialPairedGate";
|
|
1563
|
-
const manifest = options.preRegistration;
|
|
1564
|
-
const observeStream = makeStream(cfg);
|
|
1565
|
-
return {
|
|
1566
|
-
name,
|
|
1567
|
-
observe(delta) {
|
|
1568
|
-
return observeStream.observe(delta);
|
|
1569
|
-
},
|
|
1570
|
-
state() {
|
|
1571
|
-
return observeStream.state();
|
|
1572
|
-
},
|
|
1573
|
-
async decide(ctx) {
|
|
1574
|
-
if (!ctx.baselineJudgeScores) {
|
|
1575
|
-
throw new Error(
|
|
1576
|
-
`${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
|
|
1577
|
-
);
|
|
1578
|
-
}
|
|
1579
|
-
const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
|
|
1580
|
-
const paired = pairHoldout(
|
|
1581
|
-
ctx.judgeScores,
|
|
1582
|
-
ctx.baselineJudgeScores,
|
|
1583
|
-
scenarioIds,
|
|
1584
|
-
(s) => s.composite
|
|
1585
|
-
);
|
|
1586
|
-
const deltas = paired.after.map((a, i) => a - paired.before[i]);
|
|
1587
|
-
seededShuffle(deltas, cfg.shuffleSeed);
|
|
1588
|
-
const stream = makeStream(cfg);
|
|
1589
|
-
let last;
|
|
1590
|
-
for (const d of deltas) {
|
|
1591
|
-
last = stream.observe(d);
|
|
1592
|
-
if (last.decision !== "continue") break;
|
|
1593
|
-
}
|
|
1594
|
-
const detail = {
|
|
1595
|
-
...stream.state(),
|
|
1596
|
-
minN: cfg.minN,
|
|
1597
|
-
maxN: cfg.maxN,
|
|
1598
|
-
scale: cfg.scale,
|
|
1599
|
-
shuffleSeed: cfg.shuffleSeed,
|
|
1600
|
-
direction: cfg.direction,
|
|
1601
|
-
minEffect: cfg.minEffect,
|
|
1602
|
-
pairedN: deltas.length,
|
|
1603
|
-
...manifest ? { preRegisteredId: manifest.id, metric: manifest.metric } : {}
|
|
1604
|
-
};
|
|
1605
|
-
const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
|
|
1606
|
-
if (last === void 0) {
|
|
1607
|
-
return {
|
|
1608
|
-
decision: "need_more_work",
|
|
1609
|
-
reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
|
|
1610
|
-
contributingGates: [{ name, status: "not_evaluated", detail }]
|
|
1611
|
-
};
|
|
1612
|
-
}
|
|
1613
|
-
const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
|
|
1614
|
-
return {
|
|
1615
|
-
decision,
|
|
1616
|
-
reasons: [`${name}: ${last.reason}`],
|
|
1617
|
-
contributingGates: [
|
|
1618
|
-
{
|
|
1619
|
-
name,
|
|
1620
|
-
status: decision === "ship" ? "pass" : decision === "need_more_work" ? "not_evaluated" : "fail",
|
|
1621
|
-
detail
|
|
1622
|
-
}
|
|
1623
|
-
],
|
|
1624
|
-
delta: meanDelta
|
|
1625
|
-
};
|
|
1626
|
-
}
|
|
1627
|
-
};
|
|
1628
|
-
}
|
|
1629
|
-
function sequentialDecide(options = {}) {
|
|
1630
|
-
const alpha = options.alpha ?? 0.05;
|
|
1631
|
-
const minN = options.minN ?? 5;
|
|
1632
|
-
const scale = options.scale ?? 1;
|
|
1633
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
1634
|
-
throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
|
|
1635
|
-
}
|
|
1636
|
-
const proc = eProcess({ alpha, maxBet: options.maxBet ?? 0.5, nullMean: 0.5 });
|
|
1637
|
-
const threshold = 1 / alpha;
|
|
1638
|
-
let processedGenerations = 0;
|
|
1639
|
-
let reference;
|
|
1640
|
-
let stopped;
|
|
1641
|
-
const topCandidate = (record) => {
|
|
1642
|
-
if (record.candidates.length === 0) {
|
|
1643
|
-
throw new Error(
|
|
1644
|
-
`sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
|
|
1645
|
-
);
|
|
1646
|
-
}
|
|
1647
|
-
return record.candidates.reduce((best, candidate) => {
|
|
1648
|
-
if (best.composite === null) return candidate;
|
|
1649
|
-
if (candidate.composite === null) return best;
|
|
1650
|
-
return candidate.composite > best.composite ? candidate : best;
|
|
1651
|
-
});
|
|
1652
|
-
};
|
|
1653
|
-
const decide = ({ history }) => {
|
|
1654
|
-
if (stopped) return stopped;
|
|
1655
|
-
if (history.length === 0) return { stop: false };
|
|
1656
|
-
if (reference === void 0) {
|
|
1657
|
-
reference = new Map(
|
|
1658
|
-
topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
|
|
1659
|
-
);
|
|
1660
|
-
}
|
|
1661
|
-
for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
|
|
1662
|
-
const top = topCandidate(history[g]);
|
|
1663
|
-
const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
|
|
1664
|
-
for (const [scenarioId, refComposite] of reference) {
|
|
1665
|
-
const candComposite = byScenario.get(scenarioId);
|
|
1666
|
-
if (candComposite === void 0) {
|
|
1667
|
-
throw new Error(
|
|
1668
|
-
`sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
|
|
1669
|
-
);
|
|
1670
|
-
}
|
|
1671
|
-
const delta = candComposite - refComposite;
|
|
1672
|
-
if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
|
|
1673
|
-
throw new Error(
|
|
1674
|
-
`sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
|
|
1675
|
-
);
|
|
1676
|
-
}
|
|
1677
|
-
const step = proc.update((delta / scale + 1) / 2);
|
|
1678
|
-
if (step.n >= minN && step.wealth >= threshold) {
|
|
1679
|
-
stopped = {
|
|
1680
|
-
stop: true,
|
|
1681
|
-
reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
|
|
1682
|
-
};
|
|
1683
|
-
processedGenerations = history.length;
|
|
1684
|
-
return stopped;
|
|
1685
|
-
}
|
|
1686
|
-
}
|
|
1687
|
-
}
|
|
1688
|
-
processedGenerations = history.length;
|
|
1689
|
-
return { stop: false };
|
|
1690
|
-
};
|
|
1691
|
-
decide.state = () => proc.state();
|
|
1692
|
-
return decide;
|
|
1693
|
-
}
|
|
1694
|
-
|
|
1695
|
-
// src/campaign/grounded-reflection.ts
|
|
1696
|
-
function rolloutArgumentDiff(rollouts, opts = {}) {
|
|
1697
|
-
const passThreshold = opts.passThreshold ?? 1;
|
|
1698
|
-
const maxValues = opts.maxValuesPerField ?? 4;
|
|
1699
|
-
const collect = (pass) => {
|
|
1700
|
-
const byField = /* @__PURE__ */ new Map();
|
|
1701
|
-
for (const r of rollouts) {
|
|
1702
|
-
if (pass !== r.score >= passThreshold) continue;
|
|
1703
|
-
for (const c of r.calls) {
|
|
1704
|
-
for (const [k, v] of Object.entries(c.args)) {
|
|
1705
|
-
const set = byField.get(k) ?? /* @__PURE__ */ new Set();
|
|
1706
|
-
set.add(String(v));
|
|
1707
|
-
byField.set(k, set);
|
|
1708
|
-
}
|
|
1709
|
-
}
|
|
1710
|
-
}
|
|
1711
|
-
return byField;
|
|
1712
|
-
};
|
|
1713
|
-
const passing = collect(true);
|
|
1714
|
-
const failing = collect(false);
|
|
1715
|
-
const lines = [];
|
|
1716
|
-
for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
|
|
1717
|
-
const pv = [...passing.get(field) ?? []].slice(0, maxValues);
|
|
1718
|
-
const fv = [...failing.get(field) ?? []].slice(0, maxValues);
|
|
1719
|
-
const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
|
|
1720
|
-
lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
|
|
1721
|
-
}
|
|
1722
|
-
const lower = (m) => {
|
|
1723
|
-
const out = /* @__PURE__ */ new Set();
|
|
1724
|
-
for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
|
|
1725
|
-
return out;
|
|
1726
|
-
};
|
|
1727
|
-
return {
|
|
1728
|
-
text: lines.join("\n") || " (no calls observed)",
|
|
1729
|
-
passingValues: lower(passing),
|
|
1730
|
-
failingValues: lower(failing)
|
|
1731
|
-
};
|
|
1732
|
-
}
|
|
1733
|
-
function classifyUngroundedLiterals(text, diff) {
|
|
1734
|
-
const ungrounded = /* @__PURE__ */ new Set();
|
|
1735
|
-
for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
|
|
1736
|
-
const w = m[1].toLowerCase();
|
|
1737
|
-
if (!diff.passingValues.has(w)) ungrounded.add(w);
|
|
1738
|
-
}
|
|
1739
|
-
const all = [...ungrounded];
|
|
1740
|
-
return {
|
|
1741
|
-
ungrounded: all,
|
|
1742
|
-
harmful: all.filter((w) => diff.failingValues.has(w))
|
|
1743
|
-
};
|
|
1744
|
-
}
|
|
1745
|
-
|
|
1746
|
-
// src/campaign/labeled-store/fs-adapter.ts
|
|
1747
|
-
import { createHash as createHash3 } from "crypto";
|
|
1748
|
-
import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
1749
|
-
import { join as join2 } from "path";
|
|
1750
|
-
var LabeledScenarioStoreError = class extends Error {
|
|
1751
|
-
constructor(code, message) {
|
|
1752
|
-
super(message);
|
|
1753
|
-
this.code = code;
|
|
1754
|
-
this.name = "LabeledScenarioStoreError";
|
|
1755
|
-
}
|
|
1756
|
-
code;
|
|
1757
|
-
};
|
|
1758
|
-
var FsLabeledScenarioStore = class {
|
|
1759
|
-
constructor(options) {
|
|
1760
|
-
this.options = options;
|
|
1761
|
-
if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
|
|
1762
|
-
this.now = options.now ?? Date.now;
|
|
1763
|
-
}
|
|
1764
|
-
options;
|
|
1765
|
-
now;
|
|
1766
|
-
rateLimits = /* @__PURE__ */ new Map();
|
|
1767
|
-
async observe(write) {
|
|
1768
|
-
this.assertProvenance(write);
|
|
1769
|
-
this.assertRateLimit(write);
|
|
1770
|
-
const record = this.toRecord(write);
|
|
1771
|
-
const path = this.pathForSource(write.source);
|
|
1772
|
-
const line = `${JSON.stringify(record)}
|
|
1773
|
-
`;
|
|
1774
|
-
appendLine(path, line);
|
|
1775
|
-
}
|
|
1776
|
-
async sample(args) {
|
|
1777
|
-
if (!args.split) {
|
|
1778
|
-
throw new LabeledScenarioStoreError(
|
|
1779
|
-
"split_required",
|
|
1780
|
-
"sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
|
|
1781
|
-
);
|
|
1782
|
-
}
|
|
1783
|
-
if (!args.capturedBefore) {
|
|
1784
|
-
throw new LabeledScenarioStoreError(
|
|
1785
|
-
"capturedBefore_required",
|
|
1786
|
-
"sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
|
|
1787
|
-
);
|
|
1788
|
-
}
|
|
1789
|
-
const all = [];
|
|
1790
|
-
for (const source of ALL_SOURCES) {
|
|
1791
|
-
if (args.split === "train" && source === "production-trace") {
|
|
1792
|
-
const explicit = sourceFilterContains(args.filter?.source, "production-trace");
|
|
1793
|
-
if (!explicit) continue;
|
|
1794
|
-
}
|
|
1795
|
-
const path = this.pathForSource(source);
|
|
1796
|
-
if (!existsSync2(path)) continue;
|
|
1797
|
-
const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
|
|
1798
|
-
for (const line of lines) {
|
|
1799
|
-
let record;
|
|
1800
|
-
try {
|
|
1801
|
-
record = JSON.parse(line);
|
|
1802
|
-
} catch {
|
|
1803
|
-
continue;
|
|
1804
|
-
}
|
|
1805
|
-
if (!matchesFilter(record, args, source)) continue;
|
|
1806
|
-
all.push(record);
|
|
1807
|
-
}
|
|
1808
|
-
}
|
|
1809
|
-
all.sort((a, b) => {
|
|
1810
|
-
if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
|
|
1811
|
-
return a.recordHash.localeCompare(b.recordHash);
|
|
1812
|
-
});
|
|
1813
|
-
return all.slice(0, args.count);
|
|
1814
|
-
}
|
|
1815
|
-
async size() {
|
|
1816
|
-
const bySource = {};
|
|
1817
|
-
const byTrust = {
|
|
1818
|
-
unverified: 0,
|
|
1819
|
-
"verified-signal": 0,
|
|
1820
|
-
"human-rated": 0
|
|
1821
|
-
};
|
|
1822
|
-
let total = 0;
|
|
1823
|
-
for (const source of ALL_SOURCES) {
|
|
1824
|
-
const path = this.pathForSource(source);
|
|
1825
|
-
if (!existsSync2(path)) {
|
|
1826
|
-
bySource[source] = 0;
|
|
1827
|
-
continue;
|
|
1828
|
-
}
|
|
1829
|
-
const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
|
|
1830
|
-
bySource[source] = lines.length;
|
|
1831
|
-
total += lines.length;
|
|
1832
|
-
for (const line of lines) {
|
|
1833
|
-
let trust = "unverified";
|
|
1834
|
-
try {
|
|
1835
|
-
trust = JSON.parse(line).labelTrust ?? "unverified";
|
|
1836
|
-
} catch {
|
|
1837
|
-
}
|
|
1838
|
-
byTrust[trust] += 1;
|
|
1839
|
-
}
|
|
1840
|
-
}
|
|
1841
|
-
return { train: total, test: total, bySource, byTrust };
|
|
1842
|
-
}
|
|
1843
|
-
assertProvenance(write) {
|
|
1844
|
-
if (!write.source) {
|
|
1845
|
-
throw new LabeledScenarioStoreError(
|
|
1846
|
-
"missing_source",
|
|
1847
|
-
"LabeledScenarioWrite requires `source`"
|
|
1848
|
-
);
|
|
1849
|
-
}
|
|
1850
|
-
if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
|
|
1851
|
-
throw new LabeledScenarioStoreError(
|
|
1852
|
-
"missing_source_version",
|
|
1853
|
-
"LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
|
|
1854
|
-
);
|
|
1855
|
-
}
|
|
1856
|
-
if (!write.capturedAt) {
|
|
1857
|
-
throw new LabeledScenarioStoreError(
|
|
1858
|
-
"missing_captured_at",
|
|
1859
|
-
"LabeledScenarioWrite requires `capturedAt` ISO timestamp"
|
|
1860
|
-
);
|
|
1861
|
-
}
|
|
1862
|
-
if (!write.redactionStatus) {
|
|
1863
|
-
throw new LabeledScenarioStoreError(
|
|
1864
|
-
"missing_redaction_status",
|
|
1865
|
-
"LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
|
|
1866
|
-
);
|
|
1867
|
-
}
|
|
1868
|
-
if (!ALL_SOURCES.includes(write.source)) {
|
|
1869
|
-
throw new LabeledScenarioStoreError(
|
|
1870
|
-
"unknown_source",
|
|
1871
|
-
`LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
|
|
1872
|
-
);
|
|
1873
|
-
}
|
|
1874
|
-
}
|
|
1875
|
-
assertRateLimit(write) {
|
|
1876
|
-
const cap = this.options.maxWritesPerMinutePerBucket;
|
|
1877
|
-
if (!cap || !write.rateLimitBucket) return;
|
|
1878
|
-
const now = this.now();
|
|
1879
|
-
const windowMs = 6e4;
|
|
1880
|
-
let state = this.rateLimits.get(write.rateLimitBucket);
|
|
1881
|
-
if (!state || now - state.windowStartMs >= windowMs) {
|
|
1882
|
-
state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
|
|
1883
|
-
this.rateLimits.set(write.rateLimitBucket, state);
|
|
1884
|
-
}
|
|
1885
|
-
if (state.count >= cap) {
|
|
1886
|
-
throw new LabeledScenarioStoreError(
|
|
1887
|
-
"rate_limit_exceeded",
|
|
1888
|
-
`LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
|
|
1889
|
-
);
|
|
1890
|
-
}
|
|
1891
|
-
state.count += 1;
|
|
1892
|
-
}
|
|
1893
|
-
toRecord(write) {
|
|
1894
|
-
const recordHash = sha256(
|
|
1895
|
-
JSON.stringify({
|
|
1896
|
-
id: write.scenario.id,
|
|
1897
|
-
src: write.source,
|
|
1898
|
-
at: write.capturedAt,
|
|
1899
|
-
ver: write.sourceVersionHash
|
|
1900
|
-
})
|
|
1901
|
-
);
|
|
1902
|
-
return {
|
|
1903
|
-
...write,
|
|
1904
|
-
recordHash,
|
|
1905
|
-
split: "train"
|
|
1906
|
-
};
|
|
1907
|
-
}
|
|
1908
|
-
pathForSource(source) {
|
|
1909
|
-
return join2(this.options.root, `${source}.jsonl`);
|
|
1910
|
-
}
|
|
1911
|
-
};
|
|
1912
|
-
var ALL_SOURCES = [
|
|
1913
|
-
"production-trace",
|
|
1914
|
-
"eval-run",
|
|
1915
|
-
"manual",
|
|
1916
|
-
"red-team",
|
|
1917
|
-
"synthetic"
|
|
1918
|
-
];
|
|
1919
|
-
function sourceFilterContains(filter, needle) {
|
|
1920
|
-
if (!filter) return false;
|
|
1921
|
-
if (Array.isArray(filter)) return filter.includes(needle);
|
|
1922
|
-
return filter === needle;
|
|
1923
|
-
}
|
|
1924
|
-
function matchesFilter(record, args, source) {
|
|
1925
|
-
if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
|
|
1926
|
-
if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
|
|
1927
|
-
const f = args.filter;
|
|
1928
|
-
if (!f) return true;
|
|
1929
|
-
if (f.kind && record.scenario.kind !== f.kind) return false;
|
|
1930
|
-
if (f.source) {
|
|
1931
|
-
const sources = Array.isArray(f.source) ? f.source : [f.source];
|
|
1932
|
-
if (!sources.includes(source)) return false;
|
|
1933
|
-
}
|
|
1934
|
-
if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
|
|
1935
|
-
const composites = Object.values(record.judgeScores).map((s) => s.composite);
|
|
1936
|
-
const max = composites.length === 0 ? 0 : Math.max(...composites);
|
|
1937
|
-
if (f.minComposite !== void 0 && max < f.minComposite) return false;
|
|
1938
|
-
if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
|
|
1939
|
-
}
|
|
1940
|
-
if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
|
|
1941
|
-
return false;
|
|
1942
|
-
}
|
|
1943
|
-
return true;
|
|
1944
|
-
}
|
|
1945
|
-
function sha256(input) {
|
|
1946
|
-
return createHash3("sha256").update(input).digest("hex").slice(0, 16);
|
|
1947
|
-
}
|
|
1948
|
-
function appendLine(path, line) {
|
|
1949
|
-
if (existsSync2(path)) {
|
|
1950
|
-
const existing = readFileSync2(path, "utf8");
|
|
1951
|
-
writeFileSync(path, existing + line);
|
|
1952
|
-
} else {
|
|
1953
|
-
writeFileSync(path, line);
|
|
1954
|
-
}
|
|
1955
|
-
}
|
|
1956
|
-
|
|
1957
|
-
// src/campaign/neutralize.ts
|
|
1958
|
-
var FILLER = "#";
|
|
1959
|
-
function neutralizeText(content) {
|
|
1960
|
-
return content.replace(/\S/g, FILLER);
|
|
1961
|
-
}
|
|
1962
|
-
|
|
1963
|
-
// src/completion-verifier.ts
|
|
1964
|
-
import { randomUUID } from "crypto";
|
|
1965
|
-
function completionVerdict(input) {
|
|
1966
|
-
if (input.requirements.length === 0) {
|
|
1967
|
-
throw new Error(
|
|
1968
|
-
`completionVerdict: task '${input.taskId}' has no requirement checks \u2014 nothing to derive a verdict from`
|
|
1969
|
-
);
|
|
1970
|
-
}
|
|
1971
|
-
const measurable = input.requirements.filter((r) => !r.unmeasured);
|
|
1972
|
-
const unmeasuredCount = input.requirements.length - measurable.length;
|
|
1973
|
-
if (measurable.length === 0) {
|
|
1974
|
-
throw new Error(
|
|
1975
|
-
`completionVerdict: task '${input.taskId}' has no measurable requirements \u2014 all ${input.requirements.length} correctness checks failed (${input.requirements[0]?.unmeasuredReason ?? "unknown reason"})`
|
|
1976
|
-
);
|
|
1977
|
-
}
|
|
1978
|
-
const satisfiedCount = measurable.filter((r) => r.satisfied).length;
|
|
1979
|
-
const completionRate = satisfiedCount / measurable.length;
|
|
1980
|
-
const fullyComplete = unmeasuredCount === 0 && satisfiedCount === measurable.length;
|
|
1981
|
-
return {
|
|
1982
|
-
taskId: input.taskId,
|
|
1983
|
-
requirements: input.requirements,
|
|
1984
|
-
completionRate,
|
|
1985
|
-
fullyComplete,
|
|
1986
|
-
unmeasuredCount,
|
|
1987
|
-
valid: fullyComplete,
|
|
1988
|
-
score: completionRate
|
|
1989
|
-
};
|
|
1990
|
-
}
|
|
1991
|
-
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
1992
|
-
"the",
|
|
1993
|
-
"a",
|
|
1994
|
-
"an",
|
|
1995
|
-
"of",
|
|
1996
|
-
"for",
|
|
1997
|
-
"and",
|
|
1998
|
-
"or",
|
|
1999
|
-
"to",
|
|
2000
|
-
"in",
|
|
2001
|
-
"on",
|
|
2002
|
-
"with",
|
|
2003
|
-
"by"
|
|
2004
|
-
]);
|
|
2005
|
-
var REQUIREMENT_FORM_STOPWORDS = /* @__PURE__ */ new Set([
|
|
2006
|
-
"generated",
|
|
2007
|
-
"generate",
|
|
2008
|
-
"view",
|
|
2009
|
-
"render",
|
|
2010
|
-
"rendered",
|
|
2011
|
-
"persisted",
|
|
2012
|
-
"persist",
|
|
2013
|
-
"artifact",
|
|
2014
|
-
"file",
|
|
2015
|
-
"document",
|
|
2016
|
-
"note",
|
|
2017
|
-
"proposal",
|
|
2018
|
-
"deliverable",
|
|
2019
|
-
"output",
|
|
2020
|
-
"created",
|
|
2021
|
-
"create",
|
|
2022
|
-
"produce",
|
|
2023
|
-
"produced",
|
|
2024
|
-
"flag"
|
|
2025
|
-
]);
|
|
2026
|
-
var MATCH_THRESHOLD = 0.5;
|
|
2027
|
-
var MIN_CONTENT_CHARS = 50;
|
|
2028
|
-
function tokens(s, extraStop) {
|
|
2029
|
-
return new Set(
|
|
2030
|
-
s.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !STOPWORDS.has(t) && !extraStop?.has(t))
|
|
2031
|
-
);
|
|
2032
|
-
}
|
|
2033
|
-
function tokenRecall(requirementText, candidateText) {
|
|
2034
|
-
const req = tokens(requirementText, REQUIREMENT_FORM_STOPWORDS);
|
|
2035
|
-
if (req.size === 0) return 0;
|
|
2036
|
-
const cand = tokens(candidateText);
|
|
2037
|
-
let hit = 0;
|
|
2038
|
-
for (const t of req) if (cand.has(t)) hit++;
|
|
2039
|
-
return hit / req.size;
|
|
2040
|
-
}
|
|
2041
|
-
function artifactCandidates(req, reqIndex, artifacts) {
|
|
2042
|
-
const reqText = `${req.title} ${req.category ?? ""}`;
|
|
2043
|
-
const out = [];
|
|
2044
|
-
artifacts.forEach((a, i) => {
|
|
2045
|
-
if ((a.content ?? "").trim().length < MIN_CONTENT_CHARS) return;
|
|
2046
|
-
let score = tokenRecall(
|
|
2047
|
-
reqText,
|
|
2048
|
-
`${a.path ?? ""} ${a.kind} ${(a.content ?? "").slice(0, 4e3)}`
|
|
2049
|
-
);
|
|
2050
|
-
if (req.category && a.kind && req.category.toLowerCase() === a.kind.toLowerCase()) {
|
|
2051
|
-
score = Math.max(score, 1);
|
|
2052
|
-
}
|
|
2053
|
-
if (score < MATCH_THRESHOLD) return;
|
|
2054
|
-
out.push({
|
|
2055
|
-
reqIndex,
|
|
2056
|
-
itemKey: `artifact:${i}`,
|
|
2057
|
-
score,
|
|
2058
|
-
evidence: `artifact '${a.path ?? a.kind}' matched (token recall ${score.toFixed(2)})`,
|
|
2059
|
-
content: a.content ?? null
|
|
2060
|
-
});
|
|
2061
|
-
});
|
|
2062
|
-
return out;
|
|
2063
|
-
}
|
|
2064
|
-
function proposalCandidates(req, reqIndex, proposals) {
|
|
2065
|
-
const reqText = `${req.title} ${req.category ?? ""}`;
|
|
2066
|
-
const out = [];
|
|
2067
|
-
for (const p of proposals) {
|
|
2068
|
-
if (p.status !== "approved") continue;
|
|
2069
|
-
const body = (p.content ?? "").trim();
|
|
2070
|
-
if (body.length < MIN_CONTENT_CHARS) continue;
|
|
2071
|
-
const score = tokenRecall(reqText, `${p.title} ${body}`);
|
|
2072
|
-
if (score < MATCH_THRESHOLD) continue;
|
|
2073
|
-
out.push({
|
|
2074
|
-
reqIndex,
|
|
2075
|
-
itemKey: `proposal:${p.id}`,
|
|
2076
|
-
score,
|
|
2077
|
-
evidence: `approved proposal '${p.title}' matched (token recall ${score.toFixed(2)})`,
|
|
2078
|
-
content: body
|
|
2079
|
-
});
|
|
2080
|
-
}
|
|
2081
|
-
return out;
|
|
2082
|
-
}
|
|
2083
|
-
function toolCallCandidates(req, reqIndex, toolCalls) {
|
|
2084
|
-
const out = [];
|
|
2085
|
-
toolCalls.forEach((name, i) => {
|
|
2086
|
-
const score = tokenRecall(req.title, name);
|
|
2087
|
-
if (score < MATCH_THRESHOLD) return;
|
|
2088
|
-
out.push({
|
|
2089
|
-
reqIndex,
|
|
2090
|
-
itemKey: `tool:${i}`,
|
|
2091
|
-
score,
|
|
2092
|
-
evidence: `tool call '${name}' matched (token recall ${score.toFixed(2)})`,
|
|
2093
|
-
content: null
|
|
2094
|
-
});
|
|
2095
|
-
});
|
|
2096
|
-
return out;
|
|
2097
|
-
}
|
|
2098
|
-
async function verifyCompletion(gold, state, checkCorrectness) {
|
|
2099
|
-
if (gold.requirements.length === 0) {
|
|
2100
|
-
throw new Error(
|
|
2101
|
-
`verifyCompletion: task '${gold.taskId}' has no requirements \u2014 malformed gold spec`
|
|
2102
|
-
);
|
|
2103
|
-
}
|
|
2104
|
-
const candidates = [];
|
|
2105
|
-
gold.requirements.forEach((req, i) => {
|
|
2106
|
-
const by = req.satisfiedBy ?? "any";
|
|
2107
|
-
if (by === "artifact" || by === "any") {
|
|
2108
|
-
candidates.push(...artifactCandidates(req, i, state.artifacts));
|
|
2109
|
-
}
|
|
2110
|
-
if (by === "proposal" || by === "any") {
|
|
2111
|
-
candidates.push(...proposalCandidates(req, i, state.proposals));
|
|
2112
|
-
}
|
|
2113
|
-
if (by === "tool-call" || by === "any") {
|
|
2114
|
-
candidates.push(...toolCallCandidates(req, i, state.toolCalls));
|
|
2115
|
-
}
|
|
2116
|
-
});
|
|
2117
|
-
candidates.sort((a, b) => b.score - a.score);
|
|
2118
|
-
const assigned = /* @__PURE__ */ new Map();
|
|
2119
|
-
const itemTaken = /* @__PURE__ */ new Set();
|
|
2120
|
-
for (const c of candidates) {
|
|
2121
|
-
if (assigned.has(c.reqIndex) || itemTaken.has(c.itemKey)) continue;
|
|
2122
|
-
assigned.set(c.reqIndex, c);
|
|
2123
|
-
itemTaken.add(c.itemKey);
|
|
2124
|
-
}
|
|
2125
|
-
const requirements = [];
|
|
2126
|
-
for (let i = 0; i < gold.requirements.length; i++) {
|
|
2127
|
-
const req = gold.requirements[i];
|
|
2128
|
-
const match = assigned.get(i);
|
|
2129
|
-
const evidence = [];
|
|
2130
|
-
let correct = null;
|
|
2131
|
-
let unmeasuredReason;
|
|
2132
|
-
if (match) {
|
|
2133
|
-
evidence.push(match.evidence);
|
|
2134
|
-
if (match.content !== null) {
|
|
2135
|
-
try {
|
|
2136
|
-
const r = await checkCorrectness(req, match.content);
|
|
2137
|
-
correct = r.correct;
|
|
2138
|
-
evidence.push(`correctness: ${r.correct ? "pass" : "fail"} \u2014 ${r.reason}`);
|
|
2139
|
-
} catch (err) {
|
|
2140
|
-
unmeasuredReason = err instanceof JudgeParseError ? `checker response unparseable after retry: ${err.raw.slice(0, 200)}` : `checker call failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
2141
|
-
evidence.push(`correctness: UNMEASURED \u2014 ${unmeasuredReason}`);
|
|
2142
|
-
}
|
|
2143
|
-
} else {
|
|
2144
|
-
evidence.push("correctness: not assessed \u2014 matched item carries no content");
|
|
2145
|
-
}
|
|
2146
|
-
} else {
|
|
2147
|
-
const by = req.satisfiedBy ?? "any";
|
|
2148
|
-
const kind = by === "any" ? "artifact/proposal/tool-call" : by;
|
|
2149
|
-
evidence.push(`no produced ${kind} matched this requirement`);
|
|
2150
|
-
}
|
|
2151
|
-
const structurallyPresent = match !== void 0;
|
|
2152
|
-
const unmeasured = unmeasuredReason !== void 0;
|
|
2153
|
-
const satisfied = structurallyPresent && !unmeasured && correct !== false;
|
|
2154
|
-
requirements.push({
|
|
2155
|
-
reqId: req.reqId,
|
|
2156
|
-
title: req.title,
|
|
2157
|
-
structurallyPresent,
|
|
2158
|
-
correct,
|
|
2159
|
-
satisfied,
|
|
2160
|
-
...unmeasured ? { unmeasured: true, unmeasuredReason } : {},
|
|
2161
|
-
evidence
|
|
2162
|
-
});
|
|
2163
|
-
}
|
|
2164
|
-
return completionVerdict({ taskId: gold.taskId, requirements });
|
|
2165
|
-
}
|
|
2166
|
-
function parseCorrectnessResponse(raw) {
|
|
2167
|
-
const readVerdict = (candidate) => {
|
|
2168
|
-
if (candidate === null || typeof candidate !== "object") return null;
|
|
2169
|
-
const { correct, reason } = candidate;
|
|
2170
|
-
if (typeof correct !== "boolean") return null;
|
|
2171
|
-
return { correct, reason: typeof reason === "string" ? reason : "" };
|
|
2172
|
-
};
|
|
2173
|
-
const match = raw.match(/\{[\s\S]*\}/);
|
|
2174
|
-
if (match) {
|
|
2175
|
-
try {
|
|
2176
|
-
const strict = readVerdict(JSON.parse(match[0]));
|
|
2177
|
-
if (strict) return strict;
|
|
2178
|
-
} catch {
|
|
2179
|
-
}
|
|
2180
|
-
}
|
|
2181
|
-
const start = raw.indexOf("{");
|
|
2182
|
-
if (start !== -1) {
|
|
2183
|
-
const recovered = readVerdict(recoverTruncatedJson(raw.slice(start)));
|
|
2184
|
-
if (recovered) return recovered;
|
|
2185
|
-
}
|
|
2186
|
-
throw new JudgeParseError("correctness-checker", raw);
|
|
2187
|
-
}
|
|
2188
|
-
function createLlmCorrectnessChecker(tc, opts = {}) {
|
|
2189
|
-
const model = opts.model ?? "claude-sonnet-4-6";
|
|
2190
|
-
const maxContentChars = opts.maxContentChars ?? 8e3;
|
|
2191
|
-
const maxAttempts = opts.maxAttempts ?? 2;
|
|
2192
|
-
const costLedger = opts.costLedger ?? new CostLedger();
|
|
2193
|
-
const sink = opts.rawSink;
|
|
2194
|
-
const record = async (event) => {
|
|
2195
|
-
try {
|
|
2196
|
-
await sink?.record(event);
|
|
2197
|
-
} catch {
|
|
2198
|
-
}
|
|
2199
|
-
};
|
|
2200
|
-
return async (requirement, content) => {
|
|
2201
|
-
const request = {
|
|
2202
|
-
model,
|
|
2203
|
-
messages: [
|
|
2204
|
-
{
|
|
2205
|
-
role: "system",
|
|
2206
|
-
content: 'You verify whether a produced work artifact actually fulfils a stated requirement. Judge fulfilment only \u2014 is the deliverable substantively present and on-point \u2014 not polish. A plan to do it later, a vague gesture, or a description of what should be done does NOT fulfil a requirement; the artifact must BE the deliverable. Respond with a single JSON object: {"correct": boolean, "reason": string (<= 30 words)}.'
|
|
2207
|
-
},
|
|
2208
|
-
{
|
|
2209
|
-
role: "user",
|
|
2210
|
-
content: `Requirement: ${requirement.title}
|
|
2211
|
-
${requirement.category ? `Category: ${requirement.category}
|
|
2212
|
-
` : ""}
|
|
2213
|
-
Produced artifact:
|
|
2214
|
-
${content.slice(0, maxContentChars)}`
|
|
2215
|
-
}
|
|
2216
|
-
],
|
|
2217
|
-
temperature: 0,
|
|
2218
|
-
maxTokens: 200
|
|
2219
|
-
};
|
|
2220
|
-
let lastErr;
|
|
2221
|
-
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
2222
|
-
const started = Date.now();
|
|
2223
|
-
await record({
|
|
2224
|
-
eventId: randomUUID(),
|
|
2225
|
-
provider: "correctness-checker",
|
|
2226
|
-
model,
|
|
2227
|
-
endpoint: "/chat",
|
|
2228
|
-
baseUrl: "",
|
|
2229
|
-
attemptIndex: attempt,
|
|
2230
|
-
direction: "request",
|
|
2231
|
-
timestamp: started,
|
|
2232
|
-
requestBody: request,
|
|
2233
|
-
redactedFields: []
|
|
2234
|
-
});
|
|
2235
|
-
try {
|
|
2236
|
-
const paid = await costLedger.runPaidCall({
|
|
2237
|
-
channel: "verifier",
|
|
2238
|
-
phase: opts.costPhase ?? "completion.correctness",
|
|
2239
|
-
actor: "correctness-checker",
|
|
2240
|
-
model,
|
|
2241
|
-
maximumCharge: maximumChargeForTCloudRequest(request, opts.tcloudMaximumAttempts),
|
|
2242
|
-
tags: {
|
|
2243
|
-
...opts.costTags,
|
|
2244
|
-
requirementId: requirement.reqId,
|
|
2245
|
-
attempt: String(attempt)
|
|
2246
|
-
},
|
|
2247
|
-
signal: opts.signal,
|
|
2248
|
-
execute: () => tc.chat(request),
|
|
2249
|
-
receipt: (response) => costReceiptFromTCloud(response, model),
|
|
2250
|
-
receiptFromError: (error) => opts.receiptFromError?.(error, attempt)
|
|
2251
|
-
});
|
|
2252
|
-
if (!paid.succeeded) throw paid.error;
|
|
2253
|
-
const resp = paid.value;
|
|
2254
|
-
const raw = resp.choices?.[0]?.message?.content ?? "";
|
|
2255
|
-
await record({
|
|
2256
|
-
eventId: randomUUID(),
|
|
2257
|
-
provider: "correctness-checker",
|
|
2258
|
-
model,
|
|
2259
|
-
endpoint: "/chat",
|
|
2260
|
-
baseUrl: "",
|
|
2261
|
-
attemptIndex: attempt,
|
|
2262
|
-
direction: "response",
|
|
2263
|
-
timestamp: Date.now(),
|
|
2264
|
-
durationMs: Date.now() - started,
|
|
2265
|
-
responseBody: resp,
|
|
2266
|
-
redactedFields: []
|
|
2267
|
-
});
|
|
2268
|
-
return parseCorrectnessResponse(raw);
|
|
2269
|
-
} catch (err) {
|
|
2270
|
-
lastErr = err;
|
|
2271
|
-
await record({
|
|
2272
|
-
eventId: randomUUID(),
|
|
2273
|
-
provider: "correctness-checker",
|
|
2274
|
-
model,
|
|
2275
|
-
endpoint: "/chat",
|
|
2276
|
-
baseUrl: "",
|
|
2277
|
-
attemptIndex: attempt,
|
|
2278
|
-
direction: "error",
|
|
2279
|
-
timestamp: Date.now(),
|
|
2280
|
-
durationMs: Date.now() - started,
|
|
2281
|
-
errorMessage: err instanceof Error ? err.message : String(err),
|
|
2282
|
-
redactedFields: []
|
|
2283
|
-
});
|
|
2284
|
-
}
|
|
2285
|
-
}
|
|
2286
|
-
throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
|
|
2287
|
-
};
|
|
2288
|
-
}
|
|
2289
|
-
var TITLE_STOPWORDS = /* @__PURE__ */ new Set([
|
|
2290
|
-
"the",
|
|
2291
|
-
"a",
|
|
2292
|
-
"an",
|
|
2293
|
-
"and",
|
|
2294
|
-
"or",
|
|
2295
|
-
"for",
|
|
2296
|
-
"to",
|
|
2297
|
-
"of",
|
|
2298
|
-
"in",
|
|
2299
|
-
"on",
|
|
2300
|
-
"with",
|
|
2301
|
-
"review",
|
|
2302
|
-
"update",
|
|
2303
|
-
"new",
|
|
2304
|
-
"proposed"
|
|
2305
|
-
]);
|
|
2306
|
-
function createTokenRecallChecker(opts = {}) {
|
|
2307
|
-
const minRecall = opts.minRecall ?? 0.5;
|
|
2308
|
-
const minLen = opts.minContentLength ?? 120;
|
|
2309
|
-
return async (requirement, content) => {
|
|
2310
|
-
const body = content.trim();
|
|
2311
|
-
if (body.length < minLen)
|
|
2312
|
-
return {
|
|
2313
|
-
correct: false,
|
|
2314
|
-
reason: `content too thin (${body.length} chars) to be the deliverable`
|
|
2315
|
-
};
|
|
2316
|
-
const titleTokens = requirement.title.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 2 && !TITLE_STOPWORDS.has(t));
|
|
2317
|
-
if (titleTokens.length === 0)
|
|
2318
|
-
return {
|
|
2319
|
-
correct: true,
|
|
2320
|
-
reason: "requirement title has no significant tokens \u2014 structural match accepted"
|
|
2321
|
-
};
|
|
2322
|
-
const lower = body.toLowerCase();
|
|
2323
|
-
const hits = titleTokens.filter((t) => lower.includes(t)).length;
|
|
2324
|
-
const recall = hits / titleTokens.length;
|
|
2325
|
-
return recall >= minRecall ? {
|
|
2326
|
-
correct: true,
|
|
2327
|
-
reason: `content recalls ${hits}/${titleTokens.length} requirement tokens`
|
|
2328
|
-
} : {
|
|
2329
|
-
correct: false,
|
|
2330
|
-
reason: `content recalls only ${hits}/${titleTokens.length} requirement tokens`
|
|
2331
|
-
};
|
|
2332
|
-
};
|
|
2333
|
-
}
|
|
2334
|
-
|
|
2335
|
-
// src/produced-state.ts
|
|
2336
|
-
function artifactKind(mimeType) {
|
|
2337
|
-
if (!mimeType) return "file";
|
|
2338
|
-
if (mimeType.includes("json")) return "json";
|
|
2339
|
-
if (mimeType.startsWith("text/")) return "text";
|
|
2340
|
-
return "file";
|
|
2341
|
-
}
|
|
2342
|
-
function extractProducedState(events) {
|
|
2343
|
-
const artifacts = [];
|
|
2344
|
-
const proposals = [];
|
|
2345
|
-
const toolCalls = [];
|
|
2346
|
-
const seenTools = /* @__PURE__ */ new Set();
|
|
2347
|
-
for (const ev of events) {
|
|
2348
|
-
if (ev.type === "tool_call") {
|
|
2349
|
-
const name = ev.toolName;
|
|
2350
|
-
if (name && !seenTools.has(name)) {
|
|
2351
|
-
seenTools.add(name);
|
|
2352
|
-
toolCalls.push(name);
|
|
2353
|
-
}
|
|
2354
|
-
} else if (ev.type === "artifact") {
|
|
2355
|
-
const a = ev;
|
|
2356
|
-
artifacts.push({
|
|
2357
|
-
kind: artifactKind(a.mimeType),
|
|
2358
|
-
path: a.name ?? a.uri ?? a.artifactId,
|
|
2359
|
-
content: a.content ?? ""
|
|
2360
|
-
});
|
|
2361
|
-
} else if (ev.type === "proposal_created") {
|
|
2362
|
-
const p = ev;
|
|
2363
|
-
proposals.push({
|
|
2364
|
-
id: p.proposalId,
|
|
2365
|
-
title: p.title,
|
|
2366
|
-
status: p.status ?? "pending",
|
|
2367
|
-
...p.content !== void 0 ? { content: p.content } : {}
|
|
2368
|
-
});
|
|
2369
|
-
}
|
|
2370
|
-
}
|
|
2371
|
-
return { artifacts, proposals, toolCalls };
|
|
2372
|
-
}
|
|
2373
|
-
|
|
2374
|
-
// src/campaign/presets/playback.ts
|
|
2375
|
-
function makePlaybackDispatch(driver) {
|
|
2376
|
-
return async (profile, scenario, ctx) => {
|
|
2377
|
-
const events = await driver.run(scenario, { ...ctx, profile });
|
|
2378
|
-
return extractProducedState(events);
|
|
2379
|
-
};
|
|
2380
|
-
}
|
|
2381
|
-
async function scoreUserStory(story, state, checkCorrectness) {
|
|
2382
|
-
const verdict = await verifyCompletion(
|
|
2383
|
-
{ taskId: story.id, requirements: story.requirements },
|
|
2384
|
-
state,
|
|
2385
|
-
checkCorrectness
|
|
2386
|
-
);
|
|
2387
|
-
return { ...verdict, title: story.title };
|
|
2388
|
-
}
|
|
2389
|
-
function userStoryScoreboard(verdicts) {
|
|
2390
|
-
const rows = [];
|
|
2391
|
-
for (const v of verdicts) {
|
|
2392
|
-
for (const r of v.requirements) {
|
|
2393
|
-
rows.push({
|
|
2394
|
-
storyId: v.taskId,
|
|
2395
|
-
storyTitle: v.title,
|
|
2396
|
-
reqId: r.reqId,
|
|
2397
|
-
reqTitle: r.title,
|
|
2398
|
-
status: r.satisfied ? "PASS" : "FAIL",
|
|
2399
|
-
evidence: r.evidence
|
|
2400
|
-
});
|
|
2401
|
-
}
|
|
2402
|
-
}
|
|
2403
|
-
return rows;
|
|
2404
|
-
}
|
|
2405
|
-
function scoreboardSummary(rows) {
|
|
2406
|
-
const byStory = /* @__PURE__ */ new Map();
|
|
2407
|
-
let passed = 0;
|
|
2408
|
-
for (const r of rows) {
|
|
2409
|
-
const s = byStory.get(r.storyId) ?? { total: 0, passed: 0 };
|
|
2410
|
-
s.total++;
|
|
2411
|
-
if (r.status === "PASS") {
|
|
2412
|
-
s.passed++;
|
|
2413
|
-
passed++;
|
|
2414
|
-
}
|
|
2415
|
-
byStory.set(r.storyId, s);
|
|
2416
|
-
}
|
|
2417
|
-
let storiesFullyComplete = 0;
|
|
2418
|
-
for (const s of byStory.values()) if (s.total > 0 && s.passed === s.total) storiesFullyComplete++;
|
|
2419
|
-
return {
|
|
2420
|
-
stories: byStory.size,
|
|
2421
|
-
storiesFullyComplete,
|
|
2422
|
-
requirements: rows.length,
|
|
2423
|
-
passed,
|
|
2424
|
-
failed: rows.length - passed,
|
|
2425
|
-
passRate: rows.length === 0 ? 0 : passed / rows.length
|
|
2426
|
-
};
|
|
2427
|
-
}
|
|
2428
|
-
function escapeCell(s) {
|
|
2429
|
-
return s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
|
|
2430
|
-
}
|
|
2431
|
-
function truncate(s, max) {
|
|
2432
|
-
return s.length <= max ? s : `${s.slice(0, Math.max(0, max - 1))}\u2026`;
|
|
2433
|
-
}
|
|
2434
|
-
function renderScoreboardMarkdown(rows, opts = {}) {
|
|
2435
|
-
const maxEv = opts.maxEvidenceChars ?? 160;
|
|
2436
|
-
const sum = scoreboardSummary(rows);
|
|
2437
|
-
const pct = (n) => `${Math.round(n * 100)}%`;
|
|
2438
|
-
const ev = (e) => escapeCell(truncate(e.join("; "), maxEv)) || "\u2014";
|
|
2439
|
-
const out = [`# ${opts.title ?? "Product-flow playback scoreboard"}`, ""];
|
|
2440
|
-
if (opts.meta) {
|
|
2441
|
-
for (const [k, v] of Object.entries(opts.meta)) out.push(`- **${k}:** ${v}`);
|
|
2442
|
-
out.push("");
|
|
2443
|
-
}
|
|
2444
|
-
out.push(
|
|
2445
|
-
`**${sum.storiesFullyComplete}/${sum.stories}** user stories fully shipped \xB7 **${sum.passed}/${sum.requirements}** requirements passing (${pct(sum.passRate)}) \xB7 **${sum.failed}** open`,
|
|
2446
|
-
""
|
|
2447
|
-
);
|
|
2448
|
-
const fails = rows.filter((r) => r.status === "FAIL");
|
|
2449
|
-
if (fails.length > 0) {
|
|
2450
|
-
out.push("## Open tickets", "", "| Story | Requirement | Evidence |", "| --- | --- | --- |");
|
|
2451
|
-
for (const r of fails) {
|
|
2452
|
-
out.push(`| ${escapeCell(r.storyTitle)} | ${escapeCell(r.reqTitle)} | ${ev(r.evidence)} |`);
|
|
2453
|
-
}
|
|
2454
|
-
out.push("");
|
|
2455
|
-
} else {
|
|
2456
|
-
out.push("_All requirements passing \u2014 no open tickets._", "");
|
|
2457
|
-
}
|
|
2458
|
-
out.push("## Per-story tick-off", "");
|
|
2459
|
-
for (const storyId of [...new Set(rows.map((r) => r.storyId))]) {
|
|
2460
|
-
const storyRows = rows.filter((r) => r.storyId === storyId);
|
|
2461
|
-
const passed = storyRows.filter((r) => r.status === "PASS").length;
|
|
2462
|
-
const mark = passed === storyRows.length ? "\u2705" : "\u26A0\uFE0F";
|
|
2463
|
-
out.push(
|
|
2464
|
-
`### ${escapeCell(storyRows[0].storyTitle)} \u2014 ${passed}/${storyRows.length} ${mark}`,
|
|
2465
|
-
"",
|
|
2466
|
-
"| Requirement | Status | Evidence |",
|
|
2467
|
-
"| --- | --- | --- |"
|
|
2468
|
-
);
|
|
2469
|
-
for (const r of storyRows) {
|
|
2470
|
-
out.push(
|
|
2471
|
-
`| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
|
|
2472
|
-
);
|
|
2473
|
-
}
|
|
2474
|
-
out.push("");
|
|
2475
|
-
}
|
|
2476
|
-
return out.join("\n");
|
|
2477
|
-
}
|
|
2478
|
-
|
|
2479
|
-
// src/campaign/presets/run-profile-matrix.ts
|
|
2480
|
-
import { createHash as createHash5 } from "crypto";
|
|
2481
|
-
import { join as join3 } from "path";
|
|
2482
|
-
|
|
2483
|
-
// src/agent-profile.ts
|
|
2484
|
-
import { createHash as createHash4 } from "crypto";
|
|
2485
|
-
import { harnessSupportsModel } from "@tangle-network/agent-interface";
|
|
2486
|
-
var CODING_HARNESSES = [
|
|
2487
|
-
"opencode",
|
|
2488
|
-
"claude-code",
|
|
2489
|
-
"codex",
|
|
2490
|
-
"kimi-code"
|
|
2491
|
-
];
|
|
2492
|
-
var HARNESS_NATIVE_MODEL = "default";
|
|
2493
|
-
function expandProfileAxes(spec) {
|
|
2494
|
-
const harnesses = spec.harnesses ?? CODING_HARNESSES;
|
|
2495
|
-
if (harnesses.length === 0) throw new ValidationError("expandProfileAxes: no harnesses to sweep");
|
|
2496
|
-
const baseModel = spec.base.model?.default;
|
|
2497
|
-
const models = spec.models ?? (baseModel ? [baseModel] : []);
|
|
2498
|
-
if (models.length === 0) {
|
|
2499
|
-
throw new ValidationError(
|
|
2500
|
-
"expandProfileAxes: no models to sweep \u2014 base profile has no model.default and none were supplied"
|
|
2501
|
-
);
|
|
2502
|
-
}
|
|
2503
|
-
const out = [];
|
|
2504
|
-
const seen = /* @__PURE__ */ new Set();
|
|
2505
|
-
for (const harness of harnesses) {
|
|
2506
|
-
const supported = spec.keepIncompatible ? models : models.filter((model) => harnessSupportsModel(harness, model));
|
|
2507
|
-
const effective = supported.length > 0 ? supported : [HARNESS_NATIVE_MODEL];
|
|
2508
|
-
for (const model of effective) {
|
|
2509
|
-
const profile = {
|
|
2510
|
-
...spec.base,
|
|
2511
|
-
name: `${spec.base.name ?? "agent"}/${harness}/${model}`,
|
|
2512
|
-
model: { ...spec.base.model, default: model },
|
|
2513
|
-
metadata: { ...spec.base.metadata ?? {}, harness, harnessModel: model }
|
|
2514
|
-
};
|
|
2515
|
-
const id = agentProfileId(profile);
|
|
2516
|
-
if (seen.has(id)) continue;
|
|
2517
|
-
seen.add(id);
|
|
2518
|
-
out.push(profile);
|
|
2519
|
-
}
|
|
2520
|
-
}
|
|
2521
|
-
if (out.length === 0) {
|
|
2522
|
-
throw new ValidationError(
|
|
2523
|
-
`expandProfileAxes: produced no profiles (harnesses=[${harnesses.join(", ")}], models=[${models.join(", ")}]).`
|
|
2524
|
-
);
|
|
2525
|
-
}
|
|
2526
|
-
return out;
|
|
2527
|
-
}
|
|
2528
|
-
function harnessAxisOf(profile) {
|
|
2529
|
-
const m = profile.metadata;
|
|
2530
|
-
const harness = m?.harness;
|
|
2531
|
-
const model = m?.harnessModel;
|
|
2532
|
-
if (typeof harness === "string" && typeof model === "string") {
|
|
2533
|
-
return { harness, model };
|
|
2534
|
-
}
|
|
2535
|
-
return void 0;
|
|
2536
|
-
}
|
|
2537
|
-
function agentProfileId(profile) {
|
|
2538
|
-
const label = pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? "profile";
|
|
2539
|
-
return `${label}-${agentProfileHash(profile).slice(0, 16)}`;
|
|
2540
|
-
}
|
|
2541
|
-
function agentProfileModelId(profile) {
|
|
2542
|
-
const model = profile.model?.default?.trim();
|
|
2543
|
-
if (!model) {
|
|
2544
|
-
const label = agentProfileDisplayLabel(profile) ?? "unnamed profile";
|
|
2545
|
-
throw new ValidationError(
|
|
2546
|
-
`AgentProfile "${label}" has no model.default \u2014 cannot record eval run`
|
|
2547
|
-
);
|
|
2548
|
-
}
|
|
2549
|
-
return model;
|
|
2550
|
-
}
|
|
2551
|
-
function agentProfileDisplayLabel(profile) {
|
|
2552
|
-
return profile.name?.trim() || profile.version?.trim() || void 0;
|
|
2553
|
-
}
|
|
2554
|
-
function pathSafeProfileLabel(label) {
|
|
2555
|
-
const safe = label?.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
2556
|
-
return safe || void 0;
|
|
2557
|
-
}
|
|
2558
|
-
function compact(input) {
|
|
2559
|
-
const out = {};
|
|
2560
|
-
for (const [key, value] of Object.entries(input)) {
|
|
2561
|
-
if (value !== void 0) out[key] = value;
|
|
2562
|
-
}
|
|
2563
|
-
return out;
|
|
2564
|
-
}
|
|
2565
|
-
function agentProfileHash(profile) {
|
|
2566
|
-
const model = agentProfileModelId(profile);
|
|
2567
|
-
const behaviour = {
|
|
2568
|
-
...profile,
|
|
2569
|
-
name: void 0,
|
|
2570
|
-
description: void 0,
|
|
2571
|
-
tags: profile.tags ? [...profile.tags].sort() : void 0,
|
|
2572
|
-
model: compact({ ...profile.model, default: model })
|
|
2573
|
-
};
|
|
2574
|
-
return createHash4("sha256").update(JSON.stringify(canonicalize(behaviour))).digest("hex");
|
|
2575
|
-
}
|
|
2576
|
-
|
|
2577
|
-
// src/campaign/presets/run-profile-matrix.ts
|
|
2578
|
-
var ProfileMatrixError = class extends AgentEvalError {
|
|
2579
|
-
constructor(message) {
|
|
2580
|
-
super("profile_matrix", message);
|
|
2581
|
-
}
|
|
2582
|
-
};
|
|
2583
|
-
function sanitize(id) {
|
|
2584
|
-
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
2585
|
-
}
|
|
2586
|
-
function sha(input) {
|
|
2587
|
-
return createHash5("sha256").update(JSON.stringify(input)).digest("hex");
|
|
2588
|
-
}
|
|
2589
|
-
function mean2(xs) {
|
|
2590
|
-
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
2591
|
-
}
|
|
2592
|
-
function requireResolvedModel(cell, profileId) {
|
|
2593
|
-
const resolved = cell.resolvedModel?.trim();
|
|
2594
|
-
if (!resolved) {
|
|
2595
|
-
throw new ProfileMatrixError(
|
|
2596
|
-
`profile '${profileId}' declared the '${HARNESS_NATIVE_MODEL}' runtime-resolved model but its dispatch reported no resolved model for cell '${cell.cellId}' \u2014 return it in the ctx.cost.runPaidCall receipt so the RunRecord pins the real model (never records '${HARNESS_NATIVE_MODEL}')`
|
|
2597
|
-
);
|
|
2598
|
-
}
|
|
2599
|
-
if (!modelHasSnapshot(resolved)) {
|
|
2600
|
-
throw new ProfileMatrixError(
|
|
2601
|
-
`profile '${profileId}' resolved to model '${resolved}' for cell '${cell.cellId}', which lacks a snapshot version \u2014 pin it (name@YYYY-MM-DD or name-YYYYMMDD) in the paid-call receipt`
|
|
2602
|
-
);
|
|
2603
|
-
}
|
|
2604
|
-
return resolved;
|
|
2605
|
-
}
|
|
2606
|
-
function buildRunRecord(args) {
|
|
2607
|
-
const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
|
|
2608
|
-
const profileId = agentProfileId(profile);
|
|
2609
|
-
const declaredModel = agentProfileModelId(profile);
|
|
2610
|
-
const model = declaredModel === HARNESS_NATIVE_MODEL ? requireResolvedModel(cell, profileId) : declaredModel;
|
|
2611
|
-
const record = campaignCellToRunRecord(cell, {
|
|
2612
|
-
runId: `${matrixId}:${profileId}:${cell.cellId}`,
|
|
2613
|
-
experimentId,
|
|
2614
|
-
candidateId: profileId,
|
|
2615
|
-
model,
|
|
2616
|
-
promptHash: profileHash,
|
|
2617
|
-
configHash,
|
|
2618
|
-
commitSha,
|
|
2619
|
-
splitTag,
|
|
2620
|
-
agentProfile: args.agentProfileCell
|
|
2621
|
-
});
|
|
2622
|
-
if (args.corpusText && args.scenario) {
|
|
2623
|
-
try {
|
|
2624
|
-
const text = args.corpusText(cell.artifact, args.scenario);
|
|
2625
|
-
if (text && typeof text.prompt === "string" && typeof text.completion === "string") {
|
|
2626
|
-
record.prompt = text.prompt;
|
|
2627
|
-
record.completion = text.completion;
|
|
2628
|
-
}
|
|
2629
|
-
} catch {
|
|
2630
|
-
}
|
|
2631
|
-
}
|
|
2632
|
-
return record;
|
|
2633
|
-
}
|
|
2634
|
-
async function runProfileMatrix(opts) {
|
|
2635
|
-
if (opts.profiles.length === 0) throw new ProfileMatrixError("profiles must not be empty");
|
|
2636
|
-
if (opts.scenarios.length === 0) throw new ProfileMatrixError("scenarios must not be empty");
|
|
2637
|
-
const splitTag = opts.splitTag ?? "search";
|
|
2638
|
-
const seed = opts.seed ?? 42;
|
|
2639
|
-
const validate = opts.validate ?? true;
|
|
2640
|
-
const integrityMode = opts.integrity ?? "assert";
|
|
2641
|
-
const profileIds = opts.profiles.map(agentProfileId);
|
|
2642
|
-
const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}`;
|
|
2643
|
-
const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}`;
|
|
2644
|
-
const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
|
|
2645
|
-
for (const profile of opts.profiles) {
|
|
2646
|
-
const profileHash = agentProfileHash(profile);
|
|
2647
|
-
const profileId = agentProfileId(profile);
|
|
2648
|
-
const declaredModel = agentProfileModelId(profile);
|
|
2649
|
-
const model = declaredModel === HARNESS_NATIVE_MODEL ? `${HARNESS_NATIVE_MODEL}@runtime-resolved` : declaredModel;
|
|
2650
|
-
try {
|
|
2651
|
-
validateRunRecord({
|
|
2652
|
-
runId: `${matrixId}:${profileId}:probe`,
|
|
2653
|
-
experimentId,
|
|
2654
|
-
candidateId: profileId,
|
|
2655
|
-
seed,
|
|
2656
|
-
model,
|
|
2657
|
-
promptHash: profileHash,
|
|
2658
|
-
configHash: profileHash,
|
|
2659
|
-
commitSha: opts.commitSha,
|
|
2660
|
-
wallMs: 0,
|
|
2661
|
-
costUsd: null,
|
|
2662
|
-
costProvenance: { kind: "uncaptured", usd: null },
|
|
2663
|
-
tokenUsage: { input: 0, output: 0 },
|
|
2664
|
-
terminalOutcome: "succeeded",
|
|
2665
|
-
outcome: { raw: { execution_error_count: 0 } },
|
|
2666
|
-
splitTag,
|
|
2667
|
-
scenarioId: "recordability-probe"
|
|
2668
|
-
});
|
|
2669
|
-
} catch (err) {
|
|
2670
|
-
throw new ProfileMatrixError(
|
|
2671
|
-
`profile '${profileId}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
|
|
2672
|
-
);
|
|
2673
|
-
}
|
|
2674
|
-
}
|
|
2675
|
-
const records = [];
|
|
2676
|
-
const campaigns = {};
|
|
2677
|
-
const byProfile = {};
|
|
2678
|
-
for (const profile of opts.profiles) {
|
|
2679
|
-
const profileHash = agentProfileHash(profile);
|
|
2680
|
-
const profileId = agentProfileId(profile);
|
|
2681
|
-
const declaredModel = agentProfileModelId(profile);
|
|
2682
|
-
const configHash = sha({
|
|
2683
|
-
profile: profileHash,
|
|
2684
|
-
judges: (opts.judges ?? []).map((j) => j.name),
|
|
2685
|
-
seed,
|
|
2686
|
-
splitTag
|
|
2687
|
-
});
|
|
2688
|
-
const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
|
|
2689
|
-
Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
|
|
2690
|
-
const campaign = await runCampaign({
|
|
2691
|
-
scenarios: opts.scenarios,
|
|
2692
|
-
dispatch,
|
|
2693
|
-
judges: opts.judges,
|
|
2694
|
-
seed,
|
|
2695
|
-
reps: opts.reps,
|
|
2696
|
-
maxConcurrency: opts.maxConcurrency,
|
|
2697
|
-
costCeiling: opts.costCeiling,
|
|
2698
|
-
labeledStore: opts.labeledStore,
|
|
2699
|
-
captureSource: opts.captureSource,
|
|
2700
|
-
storage: opts.storage,
|
|
2701
|
-
now: opts.now,
|
|
2702
|
-
runDir: join3(opts.runDir, sanitize(profileId))
|
|
2703
|
-
});
|
|
2704
|
-
const axis = harnessAxisOf(profile);
|
|
2705
|
-
const buildCellIdentity = (cellModel) => buildAgentProfileCell({
|
|
2706
|
-
profileId,
|
|
2707
|
-
sourceProfile: { kind: "agent-interface-profile", hash: profileHash },
|
|
2708
|
-
model: cellModel,
|
|
2709
|
-
...axis ? { harness: { id: axis.harness } } : {}
|
|
2710
|
-
});
|
|
2711
|
-
const sharedCellIdentity = declaredModel === HARNESS_NATIVE_MODEL ? void 0 : await buildCellIdentity(declaredModel);
|
|
2712
|
-
const profileRecords = [];
|
|
2713
|
-
for (const cell of campaign.cells) {
|
|
2714
|
-
const agentProfileCell = sharedCellIdentity ?? await buildCellIdentity(requireResolvedModel(cell, profileId));
|
|
2715
|
-
const record = buildRunRecord({
|
|
2716
|
-
cell,
|
|
2717
|
-
profile,
|
|
2718
|
-
profileHash,
|
|
2719
|
-
configHash,
|
|
2720
|
-
experimentId,
|
|
2721
|
-
splitTag,
|
|
2722
|
-
commitSha: opts.commitSha,
|
|
2723
|
-
matrixId,
|
|
2724
|
-
agentProfileCell,
|
|
2725
|
-
scenario: scenarioById.get(cell.scenarioId),
|
|
2726
|
-
corpusText: opts.corpusText
|
|
2727
|
-
});
|
|
2728
|
-
if (validate) validateRunRecord(record);
|
|
2729
|
-
profileRecords.push(record);
|
|
2730
|
-
records.push(record);
|
|
2731
|
-
}
|
|
2732
|
-
const totalCostUsd = campaign.aggregates.totalCostUsd;
|
|
2733
|
-
campaigns[profileId] = campaign;
|
|
2734
|
-
byProfile[profileId] = {
|
|
2735
|
-
profileId,
|
|
2736
|
-
profileHash,
|
|
2737
|
-
// The declared model, unless it snapped to the sentinel — then the
|
|
2738
|
-
// resolved model the cells actually ran on (all cells of a profile share
|
|
2739
|
-
// one harness, so the first record's model is representative).
|
|
2740
|
-
model: declaredModel === HARNESS_NATIVE_MODEL ? profileRecords[0]?.model ?? declaredModel : declaredModel,
|
|
2741
|
-
records: profileRecords.length,
|
|
2742
|
-
meanComposite: meanOrNull2(
|
|
2743
|
-
profileRecords.map(scoreOf).filter((score) => score !== void 0)
|
|
2744
|
-
),
|
|
2745
|
-
totalCostUsd,
|
|
2746
|
-
integrity: summarizeBackendIntegrity(profileRecords)
|
|
2747
|
-
};
|
|
2748
|
-
}
|
|
2749
|
-
const integrity = summarizeBackendIntegrity(records);
|
|
2750
|
-
if (integrityMode === "assert") {
|
|
2751
|
-
assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
|
|
2752
|
-
} else if (integrityMode === "warn" && integrity.verdict !== "real") {
|
|
2753
|
-
console.warn(
|
|
2754
|
-
`[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
|
|
2755
|
-
);
|
|
2756
|
-
}
|
|
2757
|
-
const byScenario = rollup(records, (r) => r.scenarioId);
|
|
2758
|
-
const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
|
|
2759
|
-
return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
|
|
2760
|
-
}
|
|
2761
|
-
function scoreOf(r) {
|
|
2762
|
-
return runTaskScore(r);
|
|
2763
|
-
}
|
|
2764
|
-
function meanOrNull2(values) {
|
|
2765
|
-
return values.length === 0 ? null : mean2(values);
|
|
2766
|
-
}
|
|
2767
|
-
function rollup(records, keyOf) {
|
|
2768
|
-
const groups = /* @__PURE__ */ new Map();
|
|
2769
|
-
for (const r of records) {
|
|
2770
|
-
const key = keyOf(r);
|
|
2771
|
-
if (key === void 0) continue;
|
|
2772
|
-
const score = scoreOf(r);
|
|
2773
|
-
if (score === void 0) continue;
|
|
2774
|
-
const arr = groups.get(key) ?? [];
|
|
2775
|
-
arr.push(score);
|
|
2776
|
-
groups.set(key, arr);
|
|
2777
|
-
}
|
|
2778
|
-
const out = {};
|
|
2779
|
-
for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
|
|
2780
|
-
return out;
|
|
2781
|
-
}
|
|
2782
|
-
function rollupByPersona(records, scenarios, personaOf) {
|
|
2783
|
-
const personaByScenarioId = /* @__PURE__ */ new Map();
|
|
2784
|
-
for (const s of scenarios) personaByScenarioId.set(s.id, personaOf(s));
|
|
2785
|
-
return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
|
|
2786
|
-
}
|
|
2787
|
-
|
|
2788
|
-
// src/campaign/scenario-selection.ts
|
|
2789
|
-
var DEFAULT_SATURATION_CEILING = 0.999;
|
|
2790
|
-
var VARIANCE_EPSILON = 1e-9;
|
|
2791
|
-
function moments(scores) {
|
|
2792
|
-
const n = scores.length;
|
|
2793
|
-
if (n === 0) return { meanScore: 0, variance: 0 };
|
|
2794
|
-
let sum = 0;
|
|
2795
|
-
for (const s of scores) sum += s;
|
|
2796
|
-
const meanScore = sum / n;
|
|
2797
|
-
let sqDev = 0;
|
|
2798
|
-
for (const s of scores) {
|
|
2799
|
-
const d = s - meanScore;
|
|
2800
|
-
sqDev += d * d;
|
|
2801
|
-
}
|
|
2802
|
-
return { meanScore, variance: sqDev / n };
|
|
2803
|
-
}
|
|
2804
|
-
function compareDiscrimination(a, b) {
|
|
2805
|
-
if (b.discrimination !== a.discrimination) return b.discrimination - a.discrimination;
|
|
2806
|
-
if (a.meanScore !== b.meanScore) return a.meanScore - b.meanScore;
|
|
2807
|
-
return a.scenarioId < b.scenarioId ? -1 : a.scenarioId > b.scenarioId ? 1 : 0;
|
|
2808
|
-
}
|
|
2809
|
-
function scoreDiscrimination(signals, opts) {
|
|
2810
|
-
const saturationCeiling = opts?.saturationCeiling ?? DEFAULT_SATURATION_CEILING;
|
|
2811
|
-
const scored = signals.map((signal) => {
|
|
2812
|
-
const { meanScore, variance } = moments(signal.scores);
|
|
2813
|
-
const tied = variance < VARIANCE_EPSILON && meanScore >= saturationCeiling;
|
|
2814
|
-
return {
|
|
2815
|
-
scenarioId: signal.scenarioId,
|
|
2816
|
-
discrimination: variance,
|
|
2817
|
-
meanScore,
|
|
2818
|
-
variance,
|
|
2819
|
-
tied
|
|
2820
|
-
};
|
|
2821
|
-
});
|
|
2822
|
-
return scored.sort(compareDiscrimination);
|
|
2823
|
-
}
|
|
2824
|
-
function selectDiscriminative(signals, k, opts) {
|
|
2825
|
-
if (k < 1) throw new Error(`selectDiscriminative: k must be >= 1 (got ${k})`);
|
|
2826
|
-
const ranked = scoreDiscrimination(signals, opts);
|
|
2827
|
-
if (ranked.length <= k) return ranked.map((s) => s.scenarioId);
|
|
2828
|
-
const nonTied = ranked.filter((s) => !s.tied);
|
|
2829
|
-
if (nonTied.length >= k) return nonTied.slice(0, k).map((s) => s.scenarioId);
|
|
2830
|
-
const tied = ranked.filter((s) => s.tied);
|
|
2831
|
-
const fill = tied.slice(0, k - nonTied.length);
|
|
2832
|
-
return [...nonTied, ...fill].map((s) => s.scenarioId);
|
|
2833
|
-
}
|
|
2834
|
-
|
|
2835
|
-
// src/campaign/search-ledger.ts
|
|
2836
|
-
import { createHash as createHash6 } from "crypto";
|
|
2837
|
-
import { existsSync as existsSync3, readFileSync as readFileSync3 } from "fs";
|
|
2838
|
-
import { resolve as resolve2 } from "path";
|
|
2839
|
-
import { z } from "zod";
|
|
2840
|
-
var SEARCH_LEDGER_SCHEMA = "tangle.search-ledger.v1";
|
|
2841
|
-
var NON_EMPTY = z.string().min(1).refine((value) => value.trim() === value, "must not contain surrounding whitespace");
|
|
2842
|
-
var HASH = z.string().regex(/^sha256:[a-f0-9]{64}$/);
|
|
2843
|
-
var LINEAGE_NODE_ID = z.string().regex(/^[a-f0-9]{16}$/);
|
|
2844
|
-
var IMMUTABLE_REVISION = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64}|sha256:[a-f0-9]{64}|sha512:[A-Za-z0-9+/=]+)$/);
|
|
2845
|
-
var ISO_TIMESTAMP = z.string().regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/).refine((value) => Number.isFinite(Date.parse(value)), "invalid timestamp");
|
|
2846
|
-
var NON_NEGATIVE_INT = z.number().int().nonnegative().safe();
|
|
2847
|
-
var FINITE_NUMBER = z.number().finite();
|
|
2848
|
-
var ArtifactRefSchema = z.object({
|
|
2849
|
-
role: NON_EMPTY,
|
|
2850
|
-
uri: NON_EMPTY,
|
|
2851
|
-
sha256: HASH,
|
|
2852
|
-
byteLength: NON_NEGATIVE_INT
|
|
2853
|
-
}).strict();
|
|
2854
|
-
var SourceRefSchema = z.object({
|
|
2855
|
-
uri: NON_EMPTY,
|
|
2856
|
-
revision: IMMUTABLE_REVISION
|
|
2857
|
-
}).strict();
|
|
2858
|
-
var FailureReasonSchema = z.object({
|
|
2859
|
-
code: NON_EMPTY,
|
|
2860
|
-
message: NON_EMPTY
|
|
2861
|
-
}).strict();
|
|
2862
|
-
var EventBaseShape = {
|
|
2863
|
-
eventId: NON_EMPTY,
|
|
2864
|
-
occurredAt: ISO_TIMESTAMP,
|
|
2865
|
-
artifacts: z.array(ArtifactRefSchema).min(1)
|
|
2866
|
-
};
|
|
2867
|
-
var OperationKindSchema = z.enum([
|
|
2868
|
-
"candidate-generation",
|
|
2869
|
-
"analysis",
|
|
2870
|
-
"selection",
|
|
2871
|
-
"judge",
|
|
2872
|
-
"other"
|
|
2873
|
-
]);
|
|
2874
|
-
var SearchPlannedSchema = z.object({
|
|
2875
|
-
...EventBaseShape,
|
|
2876
|
-
kind: z.literal("search-planned"),
|
|
2877
|
-
plan: z.object({
|
|
2878
|
-
candidateSlots: z.array(
|
|
2879
|
-
z.object({
|
|
2880
|
-
slotId: NON_EMPTY,
|
|
2881
|
-
generationOperationId: NON_EMPTY
|
|
2882
|
-
}).strict()
|
|
2883
|
-
).min(1),
|
|
2884
|
-
tasks: z.array(
|
|
2885
|
-
z.object({
|
|
2886
|
-
taskId: NON_EMPTY,
|
|
2887
|
-
source: SourceRefSchema,
|
|
2888
|
-
benchmark: SourceRefSchema,
|
|
2889
|
-
maxAttempts: z.number().int().positive().safe()
|
|
2890
|
-
}).strict()
|
|
2891
|
-
).min(1),
|
|
2892
|
-
operations: z.array(
|
|
2893
|
-
z.object({
|
|
2894
|
-
operationId: NON_EMPTY,
|
|
2895
|
-
kind: OperationKindSchema
|
|
2896
|
-
}).strict()
|
|
2897
|
-
).min(1)
|
|
2898
|
-
}).strict()
|
|
2899
|
-
}).strict();
|
|
2900
|
-
var CandidateRegisteredSchema = z.object({
|
|
2901
|
-
...EventBaseShape,
|
|
2902
|
-
kind: z.literal("candidate-registered"),
|
|
2903
|
-
slotId: NON_EMPTY,
|
|
2904
|
-
generationOperationId: NON_EMPTY,
|
|
2905
|
-
candidateId: NON_EMPTY,
|
|
2906
|
-
lineage: z.object({
|
|
2907
|
-
lineageNodeId: LINEAGE_NODE_ID,
|
|
2908
|
-
parentCandidateIds: z.array(NON_EMPTY),
|
|
2909
|
-
generation: NON_NEGATIVE_INT,
|
|
2910
|
-
proposer: NON_EMPTY,
|
|
2911
|
-
proposerSource: SourceRefSchema
|
|
2912
|
-
}).strict(),
|
|
2913
|
-
surfaces: z.array(
|
|
2914
|
-
z.object({
|
|
2915
|
-
surfaceId: NON_EMPTY,
|
|
2916
|
-
kind: z.enum([
|
|
2917
|
-
"prompt",
|
|
2918
|
-
"tool-contract",
|
|
2919
|
-
"runtime-config",
|
|
2920
|
-
"memory",
|
|
2921
|
-
"knowledge",
|
|
2922
|
-
"agent-profile",
|
|
2923
|
-
"code",
|
|
2924
|
-
"deployment"
|
|
2925
|
-
]),
|
|
2926
|
-
artifact: ArtifactRefSchema
|
|
2927
|
-
}).strict()
|
|
2928
|
-
).min(1)
|
|
2929
|
-
}).strict();
|
|
2930
|
-
var CandidateSlotClosedSchema = z.object({
|
|
2931
|
-
...EventBaseShape,
|
|
2932
|
-
kind: z.literal("candidate-slot-closed"),
|
|
2933
|
-
slotId: NON_EMPTY,
|
|
2934
|
-
generationOperationId: NON_EMPTY,
|
|
2935
|
-
reason: FailureReasonSchema
|
|
2936
|
-
}).strict();
|
|
2937
|
-
var KnownTokensSchema = z.object({
|
|
2938
|
-
status: z.literal("known"),
|
|
2939
|
-
inputTokens: NON_NEGATIVE_INT,
|
|
2940
|
-
outputTokens: NON_NEGATIVE_INT,
|
|
2941
|
-
cachedTokens: NON_NEGATIVE_INT
|
|
2942
|
-
}).strict();
|
|
2943
|
-
var UnknownSchema = z.object({
|
|
2944
|
-
status: z.literal("unknown"),
|
|
2945
|
-
reason: NON_EMPTY
|
|
2946
|
-
}).strict();
|
|
2947
|
-
var KnownCostSchema = z.object({
|
|
2948
|
-
status: z.literal("known"),
|
|
2949
|
-
usd: z.number().finite().nonnegative(),
|
|
2950
|
-
source: z.enum(["provider", "pricing-table", "free"])
|
|
2951
|
-
}).strict().superRefine((cost, ctx) => {
|
|
2952
|
-
if (cost.source === "free" && cost.usd !== 0) {
|
|
2953
|
-
ctx.addIssue({ code: "custom", message: "free cost source must have usd 0" });
|
|
2954
|
-
}
|
|
2955
|
-
});
|
|
2956
|
-
var UnknownCostSchema = z.object({
|
|
2957
|
-
status: z.literal("unknown"),
|
|
2958
|
-
knownLowerBoundUsd: z.number().finite().nonnegative(),
|
|
2959
|
-
reason: NON_EMPTY
|
|
2960
|
-
}).strict();
|
|
2961
|
-
var AccountingSchema = z.object({
|
|
2962
|
-
tokens: z.discriminatedUnion("status", [KnownTokensSchema, UnknownSchema]),
|
|
2963
|
-
cost: z.discriminatedUnion("status", [KnownCostSchema, UnknownCostSchema])
|
|
2964
|
-
}).strict();
|
|
2965
|
-
var MetricsSchema = z.record(NON_EMPTY, FINITE_NUMBER).superRefine((metrics, ctx) => {
|
|
2966
|
-
for (const key of Object.keys(metrics)) {
|
|
2967
|
-
if (key === "__proto__" || key === "constructor" || key === "prototype") {
|
|
2968
|
-
ctx.addIssue({ code: "custom", message: `unsafe metric key ${key}` });
|
|
2969
|
-
}
|
|
2970
|
-
}
|
|
2971
|
-
});
|
|
2972
|
-
var OutcomeSchema = z.discriminatedUnion("status", [
|
|
2973
|
-
z.object({
|
|
2974
|
-
status: z.literal("passed"),
|
|
2975
|
-
score: FINITE_NUMBER,
|
|
2976
|
-
metrics: MetricsSchema
|
|
2977
|
-
}).strict(),
|
|
2978
|
-
z.object({
|
|
2979
|
-
status: z.literal("failed"),
|
|
2980
|
-
score: FINITE_NUMBER,
|
|
2981
|
-
metrics: MetricsSchema,
|
|
2982
|
-
failure: FailureReasonSchema
|
|
2983
|
-
}).strict(),
|
|
2984
|
-
z.object({
|
|
2985
|
-
status: z.literal("errored"),
|
|
2986
|
-
metrics: MetricsSchema,
|
|
2987
|
-
error: FailureReasonSchema.extend({ retryable: z.boolean() }).strict()
|
|
2988
|
-
}).strict()
|
|
2989
|
-
]);
|
|
2990
|
-
var EffectSchema = z.discriminatedUnion("status", [
|
|
2991
|
-
z.object({
|
|
2992
|
-
status: z.literal("measured"),
|
|
2993
|
-
metric: NON_EMPTY,
|
|
2994
|
-
baselineValue: FINITE_NUMBER,
|
|
2995
|
-
candidateValue: FINITE_NUMBER,
|
|
2996
|
-
delta: FINITE_NUMBER
|
|
2997
|
-
}).strict().superRefine((effect, ctx) => {
|
|
2998
|
-
const expected = effect.candidateValue - effect.baselineValue;
|
|
2999
|
-
const tolerance = Number.EPSILON * Math.max(1, Math.abs(expected), Math.abs(effect.delta)) * 8;
|
|
3000
|
-
if (Math.abs(effect.delta - expected) > tolerance) {
|
|
3001
|
-
ctx.addIssue({ code: "custom", message: "delta must equal candidateValue - baselineValue" });
|
|
3002
|
-
}
|
|
3003
|
-
}),
|
|
3004
|
-
z.object({
|
|
3005
|
-
status: z.literal("not-measured"),
|
|
3006
|
-
reason: NON_EMPTY
|
|
3007
|
-
}).strict()
|
|
3008
|
-
]);
|
|
3009
|
-
var SurfaceEvidenceSchema = z.object({
|
|
3010
|
-
surfaceId: NON_EMPTY,
|
|
3011
|
-
fired: z.boolean(),
|
|
3012
|
-
firingCount: NON_NEGATIVE_INT,
|
|
3013
|
-
effect: EffectSchema,
|
|
3014
|
-
evidence: z.array(ArtifactRefSchema).min(1)
|
|
3015
|
-
}).strict().superRefine((evidence, ctx) => {
|
|
3016
|
-
if (evidence.fired && evidence.firingCount === 0) {
|
|
3017
|
-
ctx.addIssue({ code: "custom", message: "a fired surface must have firingCount >= 1" });
|
|
3018
|
-
}
|
|
3019
|
-
if (!evidence.fired && evidence.firingCount !== 0) {
|
|
3020
|
-
ctx.addIssue({
|
|
3021
|
-
code: "custom",
|
|
3022
|
-
message: "a surface that did not fire must have firingCount 0"
|
|
3023
|
-
});
|
|
3024
|
-
}
|
|
3025
|
-
if (!evidence.fired && evidence.effect.status === "measured" && evidence.effect.delta !== 0) {
|
|
3026
|
-
ctx.addIssue({
|
|
3027
|
-
code: "custom",
|
|
3028
|
-
message: "a surface that did not fire cannot claim non-zero effect"
|
|
3029
|
-
});
|
|
3030
|
-
}
|
|
3031
|
-
});
|
|
3032
|
-
var TaskAttemptedSchema = z.object({
|
|
3033
|
-
...EventBaseShape,
|
|
3034
|
-
kind: z.literal("task-attempted"),
|
|
3035
|
-
candidateId: NON_EMPTY,
|
|
3036
|
-
runId: NON_EMPTY,
|
|
3037
|
-
attemptIndex: NON_NEGATIVE_INT,
|
|
3038
|
-
task: z.object({ taskId: NON_EMPTY, source: SourceRefSchema }).strict(),
|
|
3039
|
-
identity: z.object({
|
|
3040
|
-
model: z.object({
|
|
3041
|
-
provider: NON_EMPTY,
|
|
3042
|
-
snapshot: NON_EMPTY.refine(
|
|
3043
|
-
modelHasSnapshot,
|
|
3044
|
-
"model must include an immutable snapshot"
|
|
3045
|
-
)
|
|
3046
|
-
}).strict(),
|
|
3047
|
-
agent: SourceRefSchema,
|
|
3048
|
-
benchmark: SourceRefSchema
|
|
3049
|
-
}).strict(),
|
|
3050
|
-
outcome: OutcomeSchema,
|
|
3051
|
-
accounting: AccountingSchema,
|
|
3052
|
-
surfaceEvidence: z.array(SurfaceEvidenceSchema).min(1)
|
|
3053
|
-
}).strict();
|
|
3054
|
-
var SearchOperationRecordedSchema = z.object({
|
|
3055
|
-
...EventBaseShape,
|
|
3056
|
-
kind: z.literal("search-operation-recorded"),
|
|
3057
|
-
operationId: NON_EMPTY,
|
|
3058
|
-
operationKind: OperationKindSchema,
|
|
3059
|
-
execution: z.discriminatedUnion("kind", [
|
|
3060
|
-
z.object({
|
|
3061
|
-
kind: z.literal("model"),
|
|
3062
|
-
model: z.object({
|
|
3063
|
-
provider: NON_EMPTY,
|
|
3064
|
-
snapshot: NON_EMPTY.refine(
|
|
3065
|
-
modelHasSnapshot,
|
|
3066
|
-
"model must include an immutable snapshot"
|
|
3067
|
-
)
|
|
3068
|
-
}).strict(),
|
|
3069
|
-
source: SourceRefSchema
|
|
3070
|
-
}).strict(),
|
|
3071
|
-
z.object({
|
|
3072
|
-
kind: z.literal("deterministic"),
|
|
3073
|
-
source: SourceRefSchema
|
|
3074
|
-
}).strict()
|
|
3075
|
-
]),
|
|
3076
|
-
outcome: z.discriminatedUnion("status", [
|
|
3077
|
-
z.object({ status: z.literal("completed") }).strict(),
|
|
3078
|
-
z.object({
|
|
3079
|
-
status: z.literal("partial"),
|
|
3080
|
-
failure: FailureReasonSchema
|
|
3081
|
-
}).strict(),
|
|
3082
|
-
z.object({
|
|
3083
|
-
status: z.literal("failed"),
|
|
3084
|
-
failure: FailureReasonSchema
|
|
3085
|
-
}).strict()
|
|
3086
|
-
]),
|
|
3087
|
-
accounting: AccountingSchema
|
|
3088
|
-
}).strict();
|
|
3089
|
-
var CandidateDecidedSchema = z.object({
|
|
3090
|
-
...EventBaseShape,
|
|
3091
|
-
kind: z.literal("candidate-decided"),
|
|
3092
|
-
candidateId: NON_EMPTY,
|
|
3093
|
-
decision: z.discriminatedUnion("status", [
|
|
3094
|
-
z.object({ status: z.literal("selected") }).strict(),
|
|
3095
|
-
z.object({
|
|
3096
|
-
status: z.literal("rejected"),
|
|
3097
|
-
reason: FailureReasonSchema
|
|
3098
|
-
}).strict()
|
|
3099
|
-
])
|
|
3100
|
-
}).strict();
|
|
3101
|
-
var SearchCompletedSchema = z.object({
|
|
3102
|
-
...EventBaseShape,
|
|
3103
|
-
kind: z.literal("search-completed"),
|
|
3104
|
-
result: z.discriminatedUnion("status", [
|
|
3105
|
-
z.object({
|
|
3106
|
-
status: z.literal("selected"),
|
|
3107
|
-
candidateId: NON_EMPTY
|
|
3108
|
-
}).strict(),
|
|
3109
|
-
z.object({
|
|
3110
|
-
status: z.literal("all-rejected"),
|
|
3111
|
-
reason: FailureReasonSchema
|
|
3112
|
-
}).strict()
|
|
3113
|
-
])
|
|
3114
|
-
}).strict();
|
|
3115
|
-
var EventSchema = z.discriminatedUnion("kind", [
|
|
3116
|
-
SearchPlannedSchema,
|
|
3117
|
-
CandidateRegisteredSchema,
|
|
3118
|
-
CandidateSlotClosedSchema,
|
|
3119
|
-
TaskAttemptedSchema,
|
|
3120
|
-
SearchOperationRecordedSchema,
|
|
3121
|
-
CandidateDecidedSchema,
|
|
3122
|
-
SearchCompletedSchema
|
|
3123
|
-
]);
|
|
3124
|
-
var EntrySchema = z.object({
|
|
3125
|
-
schema: z.literal(SEARCH_LEDGER_SCHEMA),
|
|
3126
|
-
campaignId: NON_EMPTY,
|
|
3127
|
-
sequence: NON_NEGATIVE_INT,
|
|
3128
|
-
previousHash: z.union([HASH, z.null()]),
|
|
3129
|
-
event: EventSchema,
|
|
3130
|
-
entryHash: HASH
|
|
3131
|
-
}).strict();
|
|
3132
|
-
function validateSearchLedgerEvent(input) {
|
|
3133
|
-
const parsed = EventSchema.safeParse(input);
|
|
3134
|
-
if (!parsed.success) {
|
|
3135
|
-
throw new SearchLedgerError(`invalid search ledger event: ${formatZodError(parsed.error)}`);
|
|
3136
|
-
}
|
|
3137
|
-
return normalizeEvent(parsed.data);
|
|
3138
|
-
}
|
|
3139
|
-
function openSearchLedger(options) {
|
|
3140
|
-
if (options.path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
|
|
3141
|
-
return new FileSearchLedger(options.path, options.campaignId);
|
|
3142
|
-
}
|
|
3143
|
-
var ledgerMutexes = /* @__PURE__ */ new Map();
|
|
3144
|
-
function mutexFor(path) {
|
|
3145
|
-
const existing = ledgerMutexes.get(path);
|
|
3146
|
-
if (existing) return existing;
|
|
3147
|
-
const mutex = new Mutex();
|
|
3148
|
-
ledgerMutexes.set(path, mutex);
|
|
3149
|
-
return mutex;
|
|
3150
|
-
}
|
|
3151
|
-
var FileSearchLedger = class {
|
|
3152
|
-
path;
|
|
3153
|
-
campaignId;
|
|
3154
|
-
mutex;
|
|
3155
|
-
constructor(path, campaignId) {
|
|
3156
|
-
if (path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
|
|
3157
|
-
if (campaignId.length === 0) throw new SearchLedgerError("campaignId is empty");
|
|
3158
|
-
if (campaignId.trim() !== campaignId) {
|
|
3159
|
-
throw new SearchLedgerError("campaignId must not contain surrounding whitespace");
|
|
3160
|
-
}
|
|
3161
|
-
this.path = resolve2(path);
|
|
3162
|
-
this.campaignId = campaignId;
|
|
3163
|
-
this.mutex = mutexFor(this.path);
|
|
3164
|
-
}
|
|
3165
|
-
async replay() {
|
|
3166
|
-
return this.mutex.runExclusive(
|
|
3167
|
-
() => withSearchLedgerFileLock(this.path, () => replayFile(this.path, this.campaignId))
|
|
3168
|
-
);
|
|
3169
|
-
}
|
|
3170
|
-
async append(input) {
|
|
3171
|
-
const event = validateSearchLedgerEvent(input);
|
|
3172
|
-
return this.mutex.runExclusive(
|
|
3173
|
-
() => withSearchLedgerFileLock(this.path, () => {
|
|
3174
|
-
const before = replayFile(this.path, this.campaignId);
|
|
3175
|
-
const existing = before.entries.find((entry2) => entry2.event.eventId === event.eventId);
|
|
3176
|
-
if (existing) {
|
|
3177
|
-
if (canonicalString(existing.event) !== canonicalString(event)) {
|
|
3178
|
-
throw new SearchLedgerConflictError(
|
|
3179
|
-
`eventId ${event.eventId} already exists with different content`
|
|
3180
|
-
);
|
|
3181
|
-
}
|
|
3182
|
-
return { entry: existing, appended: false, replay: before };
|
|
3183
|
-
}
|
|
3184
|
-
const previousHash = before.audit.headHash;
|
|
3185
|
-
const material = {
|
|
3186
|
-
schema: SEARCH_LEDGER_SCHEMA,
|
|
3187
|
-
campaignId: this.campaignId,
|
|
3188
|
-
sequence: before.entries.length,
|
|
3189
|
-
previousHash,
|
|
3190
|
-
event
|
|
3191
|
-
};
|
|
3192
|
-
const entry = {
|
|
3193
|
-
...material,
|
|
3194
|
-
entryHash: hashCanonical(material)
|
|
3195
|
-
};
|
|
3196
|
-
const replay = replayEntries([...before.entries, entry], this.campaignId);
|
|
3197
|
-
appendSearchLedgerLine(this.path, `${canonicalString(entry)}
|
|
3198
|
-
`);
|
|
3199
|
-
return { entry, appended: true, replay };
|
|
3200
|
-
})
|
|
3201
|
-
);
|
|
3202
|
-
}
|
|
3203
|
-
};
|
|
3204
|
-
function replayFile(path, campaignId) {
|
|
3205
|
-
if (!existsSync3(path)) return replayEntries([], campaignId);
|
|
3206
|
-
const text = readFileSync3(path, "utf8");
|
|
3207
|
-
if (text.length === 0) return replayEntries([], campaignId);
|
|
3208
|
-
if (!text.endsWith("\n")) {
|
|
3209
|
-
throw new SearchLedgerIntegrityError(
|
|
3210
|
-
`search ledger ${path} has a truncated final record (missing newline)`
|
|
3211
|
-
);
|
|
3212
|
-
}
|
|
3213
|
-
const lines = text.slice(0, -1).split("\n");
|
|
3214
|
-
const entries = [];
|
|
3215
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
3216
|
-
const line = lines[index];
|
|
3217
|
-
if (line.length === 0) {
|
|
3218
|
-
throw new SearchLedgerIntegrityError(
|
|
3219
|
-
`search ledger ${path} has a blank row at line ${index + 1}`
|
|
3220
|
-
);
|
|
3221
|
-
}
|
|
3222
|
-
let raw;
|
|
3223
|
-
try {
|
|
3224
|
-
raw = JSON.parse(line);
|
|
3225
|
-
} catch (error) {
|
|
3226
|
-
throw new SearchLedgerIntegrityError(
|
|
3227
|
-
`search ledger ${path} has invalid JSON at line ${index + 1}`,
|
|
3228
|
-
{ cause: error }
|
|
3229
|
-
);
|
|
3230
|
-
}
|
|
3231
|
-
const parsed = EntrySchema.safeParse(raw);
|
|
3232
|
-
if (!parsed.success) {
|
|
3233
|
-
throw new SearchLedgerIntegrityError(
|
|
3234
|
-
`search ledger ${path} has a malformed entry at line ${index + 1}: ${formatZodError(parsed.error)}`
|
|
3235
|
-
);
|
|
3236
|
-
}
|
|
3237
|
-
const entry = parsed.data;
|
|
3238
|
-
const normalizedEvent = validateSearchLedgerEvent(entry.event);
|
|
3239
|
-
if (canonicalString(normalizedEvent) !== canonicalString(entry.event)) {
|
|
3240
|
-
throw new SearchLedgerIntegrityError(
|
|
3241
|
-
`search ledger ${path} has non-canonical event ordering at line ${index + 1}`
|
|
3242
|
-
);
|
|
3243
|
-
}
|
|
3244
|
-
if (line !== canonicalString(entry)) {
|
|
3245
|
-
throw new SearchLedgerIntegrityError(
|
|
3246
|
-
`search ledger ${path} has non-canonical bytes at line ${index + 1}`
|
|
3247
|
-
);
|
|
3248
|
-
}
|
|
3249
|
-
entries.push(entry);
|
|
3250
|
-
}
|
|
3251
|
-
return replayEntries(entries, campaignId);
|
|
3252
|
-
}
|
|
3253
|
-
function replayEntries(entries, campaignId) {
|
|
3254
|
-
const candidates = /* @__PURE__ */ new Map();
|
|
3255
|
-
const candidateBySlot = /* @__PURE__ */ new Map();
|
|
3256
|
-
const closedSlots = /* @__PURE__ */ new Map();
|
|
3257
|
-
const lineageNodes = /* @__PURE__ */ new Map();
|
|
3258
|
-
const eventIds = /* @__PURE__ */ new Set();
|
|
3259
|
-
const runIds = /* @__PURE__ */ new Set();
|
|
3260
|
-
const attemptKeys = /* @__PURE__ */ new Set();
|
|
3261
|
-
const candidateEvents = [];
|
|
3262
|
-
const closedSlotEvents = [];
|
|
3263
|
-
const attempts = [];
|
|
3264
|
-
const operationEvents = [];
|
|
3265
|
-
const operationsById = /* @__PURE__ */ new Map();
|
|
3266
|
-
const decisions = [];
|
|
3267
|
-
let planEvent = null;
|
|
3268
|
-
let completion = null;
|
|
3269
|
-
let expectedPrevious = null;
|
|
3270
|
-
let previousOccurredAt = Number.NEGATIVE_INFINITY;
|
|
3271
|
-
for (let index = 0; index < entries.length; index += 1) {
|
|
3272
|
-
const entry = entries[index];
|
|
3273
|
-
if (entry.campaignId !== campaignId) {
|
|
3274
|
-
throw new SearchLedgerIntegrityError(
|
|
3275
|
-
`entry ${index} belongs to campaign ${entry.campaignId}, expected ${campaignId}`
|
|
3276
|
-
);
|
|
3277
|
-
}
|
|
3278
|
-
if (entry.sequence !== index) {
|
|
3279
|
-
throw new SearchLedgerIntegrityError(
|
|
3280
|
-
`entry ${entry.event.eventId} has sequence ${entry.sequence}, expected ${index}`
|
|
3281
|
-
);
|
|
3282
|
-
}
|
|
3283
|
-
if (entry.previousHash !== expectedPrevious) {
|
|
3284
|
-
throw new SearchLedgerIntegrityError(
|
|
3285
|
-
`entry ${entry.event.eventId} does not extend the previous hash`
|
|
3286
|
-
);
|
|
3287
|
-
}
|
|
3288
|
-
const { entryHash: _entryHash, ...material } = entry;
|
|
3289
|
-
const expectedHash = hashCanonical(material);
|
|
3290
|
-
if (entry.entryHash !== expectedHash) {
|
|
3291
|
-
throw new SearchLedgerIntegrityError(
|
|
3292
|
-
`entry ${entry.event.eventId} hash mismatch: expected ${expectedHash}, got ${entry.entryHash}`
|
|
3293
|
-
);
|
|
3294
|
-
}
|
|
3295
|
-
expectedPrevious = entry.entryHash;
|
|
3296
|
-
const event = entry.event;
|
|
3297
|
-
if (eventIds.has(event.eventId)) {
|
|
3298
|
-
throw new SearchLedgerIntegrityError(`duplicate eventId ${event.eventId} in durable ledger`);
|
|
3299
|
-
}
|
|
3300
|
-
eventIds.add(event.eventId);
|
|
3301
|
-
if (completion) {
|
|
3302
|
-
throw new SearchLedgerIntegrityError(
|
|
3303
|
-
`event ${event.eventId} appears after terminal event ${completion.eventId}`
|
|
3304
|
-
);
|
|
3305
|
-
}
|
|
3306
|
-
const occurredAt = Date.parse(event.occurredAt);
|
|
3307
|
-
if (occurredAt < previousOccurredAt) {
|
|
3308
|
-
throw new SearchLedgerIntegrityError(
|
|
3309
|
-
`event ${event.eventId} occurred before the preceding durable event`
|
|
3310
|
-
);
|
|
3311
|
-
}
|
|
3312
|
-
previousOccurredAt = occurredAt;
|
|
3313
|
-
assertUnique2(event.artifacts.map(artifactKey), "artifact receipt", event.eventId);
|
|
3314
|
-
if (event.kind === "search-planned") {
|
|
3315
|
-
if (index !== 0 || planEvent) {
|
|
3316
|
-
throw new SearchLedgerIntegrityError("search plan must be the first and only plan event");
|
|
3317
|
-
}
|
|
3318
|
-
assertUnique2(
|
|
3319
|
-
event.plan.candidateSlots.map((slot) => slot.slotId),
|
|
3320
|
-
"candidate slot",
|
|
3321
|
-
event.eventId
|
|
3322
|
-
);
|
|
3323
|
-
assertUnique2(
|
|
3324
|
-
event.plan.tasks.map((task) => task.taskId),
|
|
3325
|
-
"planned taskId",
|
|
3326
|
-
event.eventId
|
|
3327
|
-
);
|
|
3328
|
-
assertUnique2(
|
|
3329
|
-
event.plan.operations.map((operation) => operation.operationId),
|
|
3330
|
-
"planned operationId",
|
|
3331
|
-
event.eventId
|
|
3332
|
-
);
|
|
3333
|
-
for (const slot of event.plan.candidateSlots) {
|
|
3334
|
-
const generationOperation = event.plan.operations.find(
|
|
3335
|
-
(operation) => operation.operationId === slot.generationOperationId
|
|
3336
|
-
);
|
|
3337
|
-
if (generationOperation?.kind !== "candidate-generation") {
|
|
3338
|
-
throw new SearchLedgerIntegrityError(
|
|
3339
|
-
`candidate slot ${slot.slotId} references unplanned candidate-generation operation ${slot.generationOperationId}`
|
|
3340
|
-
);
|
|
3341
|
-
}
|
|
3342
|
-
}
|
|
3343
|
-
planEvent = event;
|
|
3344
|
-
continue;
|
|
3345
|
-
}
|
|
3346
|
-
if (!planEvent) {
|
|
3347
|
-
throw new SearchLedgerIntegrityError(
|
|
3348
|
-
`event ${event.eventId} appears before the required search plan`
|
|
3349
|
-
);
|
|
3350
|
-
}
|
|
3351
|
-
if (event.kind === "candidate-registered") {
|
|
3352
|
-
if (candidates.has(event.candidateId)) {
|
|
3353
|
-
throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was registered twice`);
|
|
3354
|
-
}
|
|
3355
|
-
const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId);
|
|
3356
|
-
if (!plannedSlot) {
|
|
3357
|
-
throw new SearchLedgerIntegrityError(
|
|
3358
|
-
`candidate ${event.candidateId} binds unknown slot ${event.slotId}`
|
|
3359
|
-
);
|
|
3360
|
-
}
|
|
3361
|
-
if (candidateBySlot.has(event.slotId)) {
|
|
3362
|
-
throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was bound twice`);
|
|
3363
|
-
}
|
|
3364
|
-
if (closedSlots.has(event.slotId)) {
|
|
3365
|
-
throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was already closed`);
|
|
3366
|
-
}
|
|
3367
|
-
if (event.generationOperationId !== plannedSlot.generationOperationId) {
|
|
3368
|
-
throw new SearchLedgerIntegrityError(
|
|
3369
|
-
`candidate ${event.candidateId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`
|
|
3370
|
-
);
|
|
3371
|
-
}
|
|
3372
|
-
const generationOperation = operationsById.get(event.generationOperationId);
|
|
3373
|
-
if (!generationOperation) {
|
|
3374
|
-
throw new SearchLedgerIntegrityError(
|
|
3375
|
-
`candidate ${event.candidateId} precedes generation operation ${event.generationOperationId}`
|
|
3376
|
-
);
|
|
3377
|
-
}
|
|
3378
|
-
if (generationOperation.outcome.status === "failed") {
|
|
3379
|
-
throw new SearchLedgerIntegrityError(
|
|
3380
|
-
`candidate ${event.candidateId} cannot bind failed generation operation ${event.generationOperationId}`
|
|
3381
|
-
);
|
|
3382
|
-
}
|
|
3383
|
-
const previousCandidate = lineageNodes.get(event.lineage.lineageNodeId);
|
|
3384
|
-
if (previousCandidate) {
|
|
3385
|
-
throw new SearchLedgerIntegrityError(
|
|
3386
|
-
`lineage node ${event.lineage.lineageNodeId} is already bound to ${previousCandidate}`
|
|
3387
|
-
);
|
|
3388
|
-
}
|
|
3389
|
-
assertUnique2(event.lineage.parentCandidateIds, "parentCandidateId", event.eventId);
|
|
3390
|
-
const parents = event.lineage.parentCandidateIds.map((id) => {
|
|
3391
|
-
const parent = candidates.get(id);
|
|
3392
|
-
if (!parent) {
|
|
3393
|
-
throw new SearchLedgerIntegrityError(
|
|
3394
|
-
`candidate ${event.candidateId} references unknown parent ${id}`
|
|
3395
|
-
);
|
|
3396
|
-
}
|
|
3397
|
-
return parent;
|
|
3398
|
-
});
|
|
3399
|
-
const expectedGeneration = parents.length === 0 ? 0 : Math.max(...parents.map((parent) => parent.registered.lineage.generation)) + 1;
|
|
3400
|
-
if (event.lineage.generation !== expectedGeneration) {
|
|
3401
|
-
throw new SearchLedgerIntegrityError(
|
|
3402
|
-
`candidate ${event.candidateId} generation ${event.lineage.generation} does not follow its parents (expected ${expectedGeneration})`
|
|
3403
|
-
);
|
|
3404
|
-
}
|
|
3405
|
-
assertUnique2(
|
|
3406
|
-
event.surfaces.map((surface) => surface.surfaceId),
|
|
3407
|
-
"surfaceId",
|
|
3408
|
-
event.eventId
|
|
3409
|
-
);
|
|
3410
|
-
candidates.set(event.candidateId, { registered: event, attempts: [], decision: null });
|
|
3411
|
-
candidateBySlot.set(event.slotId, event.candidateId);
|
|
3412
|
-
lineageNodes.set(event.lineage.lineageNodeId, event.candidateId);
|
|
3413
|
-
candidateEvents.push(event);
|
|
3414
|
-
continue;
|
|
3415
|
-
}
|
|
3416
|
-
if (event.kind === "task-attempted") {
|
|
3417
|
-
const candidate = candidates.get(event.candidateId);
|
|
3418
|
-
if (!candidate) {
|
|
3419
|
-
throw new SearchLedgerIntegrityError(
|
|
3420
|
-
`attempt ${event.eventId} references unknown candidate ${event.candidateId}`
|
|
3421
|
-
);
|
|
3422
|
-
}
|
|
3423
|
-
if (candidate.decision) {
|
|
3424
|
-
throw new SearchLedgerIntegrityError(
|
|
3425
|
-
`attempt ${event.eventId} appears after candidate ${event.candidateId} was decided`
|
|
3426
|
-
);
|
|
3427
|
-
}
|
|
3428
|
-
const plannedTask = planEvent.plan.tasks.find((task) => task.taskId === event.task.taskId);
|
|
3429
|
-
if (!plannedTask) {
|
|
3430
|
-
throw new SearchLedgerIntegrityError(
|
|
3431
|
-
`attempt ${event.eventId} references unplanned task ${event.task.taskId}`
|
|
3432
|
-
);
|
|
3433
|
-
}
|
|
3434
|
-
if (canonicalString(plannedTask.source) !== canonicalString(event.task.source) || canonicalString(plannedTask.benchmark) !== canonicalString(event.identity.benchmark)) {
|
|
3435
|
-
throw new SearchLedgerIntegrityError(
|
|
3436
|
-
`task ${event.task.taskId} does not match its planned source identity`
|
|
3437
|
-
);
|
|
3438
|
-
}
|
|
3439
|
-
if (event.attemptIndex >= plannedTask.maxAttempts) {
|
|
3440
|
-
throw new SearchLedgerIntegrityError(
|
|
3441
|
-
`task ${event.task.taskId} attempt ${event.attemptIndex} exceeds planned maxAttempts ${plannedTask.maxAttempts}`
|
|
3442
|
-
);
|
|
3443
|
-
}
|
|
3444
|
-
if (runIds.has(event.runId)) {
|
|
3445
|
-
throw new SearchLedgerIntegrityError(`runId ${event.runId} was recorded twice`);
|
|
3446
|
-
}
|
|
3447
|
-
runIds.add(event.runId);
|
|
3448
|
-
const attemptKey = canonicalString([event.candidateId, event.task.taskId, event.attemptIndex]);
|
|
3449
|
-
if (attemptKeys.has(attemptKey)) {
|
|
3450
|
-
throw new SearchLedgerIntegrityError(
|
|
3451
|
-
`candidate ${event.candidateId} task ${event.task.taskId} attempt ${event.attemptIndex} was recorded twice`
|
|
3452
|
-
);
|
|
3453
|
-
}
|
|
3454
|
-
const expectedAttemptIndex = candidate.attempts.filter(
|
|
3455
|
-
(attempt) => attempt.task.taskId === event.task.taskId
|
|
3456
|
-
).length;
|
|
3457
|
-
if (event.attemptIndex !== expectedAttemptIndex) {
|
|
3458
|
-
throw new SearchLedgerIntegrityError(
|
|
3459
|
-
`candidate ${event.candidateId} task ${event.task.taskId} attempt index ${event.attemptIndex} is not contiguous (expected ${expectedAttemptIndex})`
|
|
3460
|
-
);
|
|
3461
|
-
}
|
|
3462
|
-
const previousAttempt = candidate.attempts.find(
|
|
3463
|
-
(attempt) => attempt.task.taskId === event.task.taskId
|
|
3464
|
-
);
|
|
3465
|
-
if (previousAttempt?.outcome.status !== void 0 && previousAttempt.outcome.status !== "errored") {
|
|
3466
|
-
throw new SearchLedgerIntegrityError(
|
|
3467
|
-
`task ${event.task.taskId} was retried after a measured outcome`
|
|
3468
|
-
);
|
|
3469
|
-
}
|
|
3470
|
-
if (previousAttempt && canonicalString({ task: previousAttempt.task, identity: previousAttempt.identity }) !== canonicalString({ task: event.task, identity: event.identity })) {
|
|
3471
|
-
throw new SearchLedgerIntegrityError(
|
|
3472
|
-
`candidate ${event.candidateId} task ${event.task.taskId} changed immutable execution identity between attempts`
|
|
3473
|
-
);
|
|
3474
|
-
}
|
|
3475
|
-
attemptKeys.add(attemptKey);
|
|
3476
|
-
const declared = candidate.registered.surfaces.map((surface) => surface.surfaceId).sort();
|
|
3477
|
-
const observed = event.surfaceEvidence.map((surface) => surface.surfaceId).sort();
|
|
3478
|
-
assertUnique2(observed, "surface evidence", event.eventId);
|
|
3479
|
-
for (const evidence of event.surfaceEvidence) {
|
|
3480
|
-
assertUnique2(evidence.evidence.map(artifactKey), "surface evidence receipt", event.eventId);
|
|
3481
|
-
}
|
|
3482
|
-
if (canonicalString(declared) !== canonicalString(observed)) {
|
|
3483
|
-
throw new SearchLedgerIntegrityError(
|
|
3484
|
-
`attempt ${event.eventId} surface evidence does not exactly cover candidate ${event.candidateId}`
|
|
3485
|
-
);
|
|
3486
|
-
}
|
|
3487
|
-
candidate.attempts.push(event);
|
|
3488
|
-
attempts.push(event);
|
|
3489
|
-
continue;
|
|
3490
|
-
}
|
|
3491
|
-
if (event.kind === "search-operation-recorded") {
|
|
3492
|
-
const plannedOperation = planEvent.plan.operations.find(
|
|
3493
|
-
(operation) => operation.operationId === event.operationId
|
|
3494
|
-
);
|
|
3495
|
-
if (!plannedOperation) {
|
|
3496
|
-
throw new SearchLedgerIntegrityError(
|
|
3497
|
-
`operation ${event.operationId} was not declared in the search plan`
|
|
3498
|
-
);
|
|
3499
|
-
}
|
|
3500
|
-
if (plannedOperation.kind !== event.operationKind) {
|
|
3501
|
-
throw new SearchLedgerIntegrityError(
|
|
3502
|
-
`operation ${event.operationId} kind ${event.operationKind} does not match planned ${plannedOperation.kind}`
|
|
3503
|
-
);
|
|
3504
|
-
}
|
|
3505
|
-
if (operationsById.has(event.operationId)) {
|
|
3506
|
-
throw new SearchLedgerIntegrityError(`operation ${event.operationId} was recorded twice`);
|
|
3507
|
-
}
|
|
3508
|
-
operationsById.set(event.operationId, event);
|
|
3509
|
-
operationEvents.push(event);
|
|
3510
|
-
continue;
|
|
3511
|
-
}
|
|
3512
|
-
if (event.kind === "candidate-slot-closed") {
|
|
3513
|
-
const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId);
|
|
3514
|
-
if (!plannedSlot) {
|
|
3515
|
-
throw new SearchLedgerIntegrityError(
|
|
3516
|
-
`candidate slot closure ${event.eventId} references unknown slot ${event.slotId}`
|
|
3517
|
-
);
|
|
3518
|
-
}
|
|
3519
|
-
if (event.generationOperationId !== plannedSlot.generationOperationId) {
|
|
3520
|
-
throw new SearchLedgerIntegrityError(
|
|
3521
|
-
`candidate slot closure ${event.eventId} generation operation ${event.generationOperationId} does not match slot ${event.slotId} plan ${plannedSlot.generationOperationId}`
|
|
3522
|
-
);
|
|
3523
|
-
}
|
|
3524
|
-
if (candidateBySlot.has(event.slotId)) {
|
|
3525
|
-
throw new SearchLedgerIntegrityError(
|
|
3526
|
-
`candidate slot ${event.slotId} was already bound to a candidate`
|
|
3527
|
-
);
|
|
3528
|
-
}
|
|
3529
|
-
if (closedSlots.has(event.slotId)) {
|
|
3530
|
-
throw new SearchLedgerIntegrityError(`candidate slot ${event.slotId} was closed twice`);
|
|
3531
|
-
}
|
|
3532
|
-
const operation = operationsById.get(event.generationOperationId);
|
|
3533
|
-
if (!operation) {
|
|
3534
|
-
throw new SearchLedgerIntegrityError(
|
|
3535
|
-
`candidate slot closure ${event.eventId} precedes operation ${event.generationOperationId}`
|
|
3536
|
-
);
|
|
3537
|
-
}
|
|
3538
|
-
if (operation.outcome.status === "completed") {
|
|
3539
|
-
throw new SearchLedgerIntegrityError(
|
|
3540
|
-
`candidate slot ${event.slotId} cannot close from completed operation ${event.generationOperationId}`
|
|
3541
|
-
);
|
|
3542
|
-
}
|
|
3543
|
-
closedSlots.set(event.slotId, event);
|
|
3544
|
-
closedSlotEvents.push(event);
|
|
3545
|
-
continue;
|
|
3546
|
-
}
|
|
3547
|
-
if (event.kind === "candidate-decided") {
|
|
3548
|
-
const candidate = candidates.get(event.candidateId);
|
|
3549
|
-
if (!candidate) {
|
|
3550
|
-
throw new SearchLedgerIntegrityError(
|
|
3551
|
-
`decision ${event.eventId} references unknown candidate ${event.candidateId}`
|
|
3552
|
-
);
|
|
3553
|
-
}
|
|
3554
|
-
if (candidate.decision) {
|
|
3555
|
-
throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was decided twice`);
|
|
3556
|
-
}
|
|
3557
|
-
if (event.decision.status === "selected") {
|
|
3558
|
-
if (!candidate.attempts.some((attempt) => attempt.outcome.status !== "errored")) {
|
|
3559
|
-
throw new SearchLedgerIntegrityError(
|
|
3560
|
-
`candidate ${event.candidateId} cannot be selected without a measured task outcome`
|
|
3561
|
-
);
|
|
3562
|
-
}
|
|
3563
|
-
if (decisions.some((decision) => decision.decision.status === "selected")) {
|
|
3564
|
-
throw new SearchLedgerIntegrityError("more than one candidate was selected");
|
|
3565
|
-
}
|
|
3566
|
-
}
|
|
3567
|
-
candidate.decision = event;
|
|
3568
|
-
decisions.push(event);
|
|
3569
|
-
continue;
|
|
3570
|
-
}
|
|
3571
|
-
const missingCandidateSlots2 = planEvent.plan.candidateSlots.filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId);
|
|
3572
|
-
if (missingCandidateSlots2.length > 0) {
|
|
3573
|
-
throw new SearchLedgerIntegrityError(
|
|
3574
|
-
`search completed with missing candidate slots: ${missingCandidateSlots2.join(", ")}`
|
|
3575
|
-
);
|
|
3576
|
-
}
|
|
3577
|
-
const missingTaskOutcomes2 = plannedTaskOutcomeKeys(planEvent, candidates);
|
|
3578
|
-
if (missingTaskOutcomes2.length > 0) {
|
|
3579
|
-
throw new SearchLedgerIntegrityError(
|
|
3580
|
-
`search completed with missing task outcomes: ${missingTaskOutcomes2.join(", ")}`
|
|
3581
|
-
);
|
|
3582
|
-
}
|
|
3583
|
-
const missingOperations2 = planEvent.plan.operations.filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId);
|
|
3584
|
-
if (missingOperations2.length > 0) {
|
|
3585
|
-
throw new SearchLedgerIntegrityError(
|
|
3586
|
-
`search completed with missing search operations: ${missingOperations2.join(", ")}`
|
|
3587
|
-
);
|
|
3588
|
-
}
|
|
3589
|
-
for (const operation of planEvent.plan.operations) {
|
|
3590
|
-
if (operation.kind !== "candidate-generation") continue;
|
|
3591
|
-
const generationOutcome = operationsById.get(operation.operationId).outcome.status;
|
|
3592
|
-
const slots = planEvent.plan.candidateSlots.filter(
|
|
3593
|
-
(slot) => slot.generationOperationId === operation.operationId
|
|
3594
|
-
);
|
|
3595
|
-
if (slots.length === 0) continue;
|
|
3596
|
-
const registeredCount = slots.filter((slot) => candidateBySlot.has(slot.slotId)).length;
|
|
3597
|
-
const closedCount = slots.filter((slot) => closedSlots.has(slot.slotId)).length;
|
|
3598
|
-
if (generationOutcome === "completed" && closedCount > 0) {
|
|
3599
|
-
throw new SearchLedgerIntegrityError(
|
|
3600
|
-
`completed generation operation ${operation.operationId} contains ${closedCount} closed slot(s)`
|
|
3601
|
-
);
|
|
3602
|
-
}
|
|
3603
|
-
if (generationOutcome === "failed" && registeredCount > 0) {
|
|
3604
|
-
throw new SearchLedgerIntegrityError(
|
|
3605
|
-
`failed generation operation ${operation.operationId} contains ${registeredCount} registered candidate(s)`
|
|
3606
|
-
);
|
|
3607
|
-
}
|
|
3608
|
-
if (generationOutcome === "partial" && (registeredCount === 0 || closedCount === 0)) {
|
|
3609
|
-
throw new SearchLedgerIntegrityError(
|
|
3610
|
-
`partial generation operation ${operation.operationId} must contain both a registered candidate and a closed slot`
|
|
3611
|
-
);
|
|
3612
|
-
}
|
|
3613
|
-
}
|
|
3614
|
-
const pending = [...candidates.values()].filter((candidate) => candidate.decision === null);
|
|
3615
|
-
if (pending.length > 0) {
|
|
3616
|
-
throw new SearchLedgerIntegrityError(
|
|
3617
|
-
`search completed with ${pending.length} candidate decision(s) missing`
|
|
3618
|
-
);
|
|
3619
|
-
}
|
|
3620
|
-
const selected = decisions.filter((decision) => decision.decision.status === "selected");
|
|
3621
|
-
if (event.result.status === "selected") {
|
|
3622
|
-
if (selected.length !== 1 || selected[0].candidateId !== event.result.candidateId) {
|
|
3623
|
-
throw new SearchLedgerIntegrityError(
|
|
3624
|
-
`search completion winner ${event.result.candidateId} does not match candidate decisions`
|
|
3625
|
-
);
|
|
3626
|
-
}
|
|
3627
|
-
} else if (selected.length !== 0) {
|
|
3628
|
-
throw new SearchLedgerIntegrityError("all-rejected completion contains a selected candidate");
|
|
3629
|
-
}
|
|
3630
|
-
completion = event;
|
|
3631
|
-
}
|
|
3632
|
-
const selectedDecisions = decisions.filter((decision) => decision.decision.status === "selected");
|
|
3633
|
-
const rejectedDecisions = decisions.filter((decision) => decision.decision.status === "rejected");
|
|
3634
|
-
const outcomeCounts = { passed: 0, failed: 0, errored: 0 };
|
|
3635
|
-
const operationOutcomeCounts = { completed: 0, partial: 0, failed: 0 };
|
|
3636
|
-
let inputTokens = 0;
|
|
3637
|
-
let outputTokens = 0;
|
|
3638
|
-
let cachedTokens = 0;
|
|
3639
|
-
let costUsd = 0;
|
|
3640
|
-
const unknownTokenEventIds = [];
|
|
3641
|
-
const unknownCostEventIds = [];
|
|
3642
|
-
for (const attempt of attempts) {
|
|
3643
|
-
outcomeCounts[attempt.outcome.status] += 1;
|
|
3644
|
-
}
|
|
3645
|
-
for (const operation of operationEvents) {
|
|
3646
|
-
operationOutcomeCounts[operation.outcome.status] += 1;
|
|
3647
|
-
}
|
|
3648
|
-
for (const costedEvent of [...attempts, ...operationEvents]) {
|
|
3649
|
-
if (costedEvent.accounting.tokens.status === "known") {
|
|
3650
|
-
inputTokens += costedEvent.accounting.tokens.inputTokens;
|
|
3651
|
-
outputTokens += costedEvent.accounting.tokens.outputTokens;
|
|
3652
|
-
cachedTokens += costedEvent.accounting.tokens.cachedTokens;
|
|
3653
|
-
} else {
|
|
3654
|
-
unknownTokenEventIds.push(costedEvent.eventId);
|
|
3655
|
-
}
|
|
3656
|
-
if (costedEvent.accounting.cost.status === "known") {
|
|
3657
|
-
costUsd += costedEvent.accounting.cost.usd;
|
|
3658
|
-
} else {
|
|
3659
|
-
costUsd += costedEvent.accounting.cost.knownLowerBoundUsd;
|
|
3660
|
-
unknownCostEventIds.push(costedEvent.eventId);
|
|
3661
|
-
}
|
|
3662
|
-
}
|
|
3663
|
-
const accounting = unknownTokenEventIds.length === 0 && unknownCostEventIds.length === 0 ? {
|
|
3664
|
-
status: "known",
|
|
3665
|
-
inputTokens,
|
|
3666
|
-
outputTokens,
|
|
3667
|
-
cachedTokens,
|
|
3668
|
-
costUsd
|
|
3669
|
-
} : {
|
|
3670
|
-
status: "partial",
|
|
3671
|
-
knownInputTokens: inputTokens,
|
|
3672
|
-
knownOutputTokens: outputTokens,
|
|
3673
|
-
knownCachedTokens: cachedTokens,
|
|
3674
|
-
knownCostUsd: costUsd,
|
|
3675
|
-
unknownTokenEventIds,
|
|
3676
|
-
unknownCostEventIds
|
|
3677
|
-
};
|
|
3678
|
-
const selectedCandidateId = completion?.result.status === "selected" ? completion.result.candidateId : null;
|
|
3679
|
-
const status = completion?.result.status === "selected" ? "selected" : completion?.result.status === "all-rejected" ? "all-rejected" : "in-progress";
|
|
3680
|
-
const missingCandidateSlots = planEvent?.plan.candidateSlots.filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)).map((slot) => slot.slotId) ?? [];
|
|
3681
|
-
const missingTaskOutcomes = planEvent ? plannedTaskOutcomeKeys(planEvent, candidates) : [];
|
|
3682
|
-
const missingOperations = planEvent?.plan.operations.filter((operation) => !operationsById.has(operation.operationId)).map((operation) => operation.operationId) ?? [];
|
|
3683
|
-
return {
|
|
3684
|
-
entries: [...entries],
|
|
3685
|
-
plan: planEvent,
|
|
3686
|
-
candidates: candidateEvents,
|
|
3687
|
-
closedCandidateSlots: closedSlotEvents,
|
|
3688
|
-
attempts,
|
|
3689
|
-
operations: operationEvents,
|
|
3690
|
-
decisions,
|
|
3691
|
-
completion,
|
|
3692
|
-
audit: {
|
|
3693
|
-
campaignId,
|
|
3694
|
-
eventCount: entries.length,
|
|
3695
|
-
candidateCount: candidates.size,
|
|
3696
|
-
closedCandidateSlotCount: closedSlots.size,
|
|
3697
|
-
attemptCount: attempts.length,
|
|
3698
|
-
operationCount: operationEvents.length,
|
|
3699
|
-
outcomes: outcomeCounts,
|
|
3700
|
-
operationOutcomes: operationOutcomeCounts,
|
|
3701
|
-
decisions: {
|
|
3702
|
-
selected: selectedDecisions.length,
|
|
3703
|
-
rejected: rejectedDecisions.length,
|
|
3704
|
-
pending: candidates.size - decisions.length
|
|
3705
|
-
},
|
|
3706
|
-
expected: {
|
|
3707
|
-
candidateSlots: planEvent?.plan.candidateSlots.length ?? 0,
|
|
3708
|
-
taskOutcomes: candidates.size * (planEvent?.plan.tasks.length ?? 0),
|
|
3709
|
-
operations: planEvent?.plan.operations.length ?? 0,
|
|
3710
|
-
missingCandidateSlots,
|
|
3711
|
-
missingTaskOutcomes,
|
|
3712
|
-
missingOperations
|
|
3713
|
-
},
|
|
3714
|
-
status,
|
|
3715
|
-
selectedCandidateId,
|
|
3716
|
-
accounting,
|
|
3717
|
-
headHash: entries.at(-1)?.entryHash ?? null
|
|
3718
|
-
}
|
|
3719
|
-
};
|
|
3720
|
-
}
|
|
3721
|
-
function plannedTaskOutcomeKeys(planEvent, candidates) {
|
|
3722
|
-
const missing = [];
|
|
3723
|
-
const registeredCandidates = [...candidates.values()].sort(
|
|
3724
|
-
(a, b) => compareStrings(a.registered.slotId, b.registered.slotId)
|
|
3725
|
-
);
|
|
3726
|
-
for (const candidate of registeredCandidates) {
|
|
3727
|
-
const slotId = candidate.registered.slotId;
|
|
3728
|
-
for (const task of planEvent.plan.tasks) {
|
|
3729
|
-
const measured = candidate.attempts.some(
|
|
3730
|
-
(attempt) => attempt.task.taskId === task.taskId && attempt.outcome.status !== "errored"
|
|
3731
|
-
);
|
|
3732
|
-
if (!measured) missing.push(`${slotId}/${task.taskId}`);
|
|
3733
|
-
}
|
|
3734
|
-
}
|
|
3735
|
-
return missing;
|
|
3736
|
-
}
|
|
3737
|
-
function normalizeEvent(event) {
|
|
3738
|
-
const artifacts = sortArtifacts(event.artifacts);
|
|
3739
|
-
if (event.kind === "search-planned") {
|
|
3740
|
-
return {
|
|
3741
|
-
...event,
|
|
3742
|
-
artifacts,
|
|
3743
|
-
plan: {
|
|
3744
|
-
candidateSlots: [...event.plan.candidateSlots].sort(
|
|
3745
|
-
(a, b) => compareStrings(a.slotId, b.slotId)
|
|
3746
|
-
),
|
|
3747
|
-
tasks: [...event.plan.tasks].sort((a, b) => compareStrings(a.taskId, b.taskId)),
|
|
3748
|
-
operations: [...event.plan.operations].sort(
|
|
3749
|
-
(a, b) => compareStrings(a.operationId, b.operationId)
|
|
3750
|
-
)
|
|
3751
|
-
}
|
|
3752
|
-
};
|
|
3753
|
-
}
|
|
3754
|
-
if (event.kind === "candidate-registered") {
|
|
3755
|
-
return {
|
|
3756
|
-
...event,
|
|
3757
|
-
artifacts,
|
|
3758
|
-
lineage: {
|
|
3759
|
-
...event.lineage,
|
|
3760
|
-
parentCandidateIds: sortedStrings(event.lineage.parentCandidateIds)
|
|
3761
|
-
},
|
|
3762
|
-
surfaces: [...event.surfaces].map((surface) => ({ ...surface, artifact: { ...surface.artifact } })).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
|
|
3763
|
-
};
|
|
3764
|
-
}
|
|
3765
|
-
if (event.kind === "task-attempted") {
|
|
3766
|
-
return {
|
|
3767
|
-
...event,
|
|
3768
|
-
artifacts,
|
|
3769
|
-
surfaceEvidence: [...event.surfaceEvidence].map((evidence) => ({ ...evidence, evidence: sortArtifacts(evidence.evidence) })).sort((a, b) => compareStrings(a.surfaceId, b.surfaceId))
|
|
3770
|
-
};
|
|
3771
|
-
}
|
|
3772
|
-
return { ...event, artifacts };
|
|
3773
|
-
}
|
|
3774
|
-
function sortArtifacts(artifacts) {
|
|
3775
|
-
return [...artifacts].map((artifact) => ({ ...artifact })).sort((a, b) => compareStrings(artifactKey(a), artifactKey(b)));
|
|
3776
|
-
}
|
|
3777
|
-
function artifactKey(artifact) {
|
|
3778
|
-
return canonicalString(artifact);
|
|
3779
|
-
}
|
|
3780
|
-
function compareStrings(a, b) {
|
|
3781
|
-
return a < b ? -1 : a > b ? 1 : 0;
|
|
3782
|
-
}
|
|
3783
|
-
function sortedStrings(values) {
|
|
3784
|
-
return [...values].sort();
|
|
3785
|
-
}
|
|
3786
|
-
function assertUnique2(values, label, eventId) {
|
|
3787
|
-
if (new Set(values).size !== values.length) {
|
|
3788
|
-
throw new SearchLedgerIntegrityError(`event ${eventId} contains duplicate ${label} values`);
|
|
3789
|
-
}
|
|
3790
|
-
}
|
|
3791
|
-
function canonicalString(value) {
|
|
3792
|
-
return JSON.stringify(canonicalize(value));
|
|
3793
|
-
}
|
|
3794
|
-
function hashCanonical(value) {
|
|
3795
|
-
return `sha256:${createHash6("sha256").update(canonicalString(value)).digest("hex")}`;
|
|
3796
|
-
}
|
|
3797
|
-
function formatZodError(error) {
|
|
3798
|
-
return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.join(".") : "<root>"}: ${issue.message}`).join("; ");
|
|
3799
|
-
}
|
|
3800
|
-
|
|
3801
|
-
// src/campaign/transient-failure.ts
|
|
3802
|
-
var BASE_TRANSIENT = /\b50[234]\b|no stream output|produced no stream|admission timed out|admission_rejected|queue_timeout|fetch failed|ECONNRESET|This operation was aborted/i;
|
|
3803
|
-
var TIMEOUT_PATTERN = /timeout after \d+ ?ms|cli-bridge timeout/i;
|
|
3804
|
-
function isTransientTransportFailure(message, opts = {}) {
|
|
3805
|
-
if (!message) return false;
|
|
3806
|
-
if (BASE_TRANSIENT.test(message)) return true;
|
|
3807
|
-
if ((opts.retryFullDurationTimeouts ?? false) && TIMEOUT_PATTERN.test(message)) return true;
|
|
3808
|
-
for (const p of opts.extraPatterns ?? []) if (p.test(message)) return true;
|
|
3809
|
-
return false;
|
|
3810
|
-
}
|
|
3811
|
-
|
|
3812
|
-
// src/campaign/worktree/index.ts
|
|
3813
|
-
import { execFileSync } from "child_process";
|
|
3814
|
-
import { createHash as createHash7 } from "crypto";
|
|
3815
|
-
import {
|
|
3816
|
-
closeSync,
|
|
3817
|
-
existsSync as existsSync4,
|
|
3818
|
-
constants as fsConstants,
|
|
3819
|
-
fstatSync,
|
|
3820
|
-
lstatSync,
|
|
3821
|
-
mkdirSync as mkdirSync2,
|
|
3822
|
-
mkdtempSync,
|
|
3823
|
-
openSync,
|
|
3824
|
-
readlinkSync,
|
|
3825
|
-
readSync,
|
|
3826
|
-
realpathSync,
|
|
3827
|
-
rmSync
|
|
3828
|
-
} from "fs";
|
|
3829
|
-
import { devNull, tmpdir } from "os";
|
|
3830
|
-
import { basename, dirname, isAbsolute as isAbsolute2, join as join4, relative as relative2, resolve as resolve3, sep } from "path";
|
|
3831
|
-
var MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024;
|
|
3832
|
-
var FILE_HASH_CHUNK_BYTES = 1024 * 1024;
|
|
3833
|
-
var GIT_REPOSITORY_ENV = /* @__PURE__ */ new Set([
|
|
3834
|
-
"GIT_ALTERNATE_OBJECT_DIRECTORIES",
|
|
3835
|
-
"GIT_COMMON_DIR",
|
|
3836
|
-
"GIT_DIR",
|
|
3837
|
-
"GIT_INDEX_FILE",
|
|
3838
|
-
"GIT_NAMESPACE",
|
|
3839
|
-
"GIT_OBJECT_DIRECTORY",
|
|
3840
|
-
"GIT_PREFIX",
|
|
3841
|
-
"GIT_QUARANTINE_PATH",
|
|
3842
|
-
"GIT_WORK_TREE"
|
|
3843
|
-
]);
|
|
3844
|
-
var WorktreeAdapterError = class extends Error {
|
|
3845
|
-
constructor(message, cause) {
|
|
3846
|
-
super(message);
|
|
3847
|
-
this.cause = cause;
|
|
3848
|
-
this.name = "WorktreeAdapterError";
|
|
3849
|
-
}
|
|
3850
|
-
cause;
|
|
3851
|
-
};
|
|
3852
|
-
function defaultGit(args, cwd, overrides) {
|
|
3853
|
-
try {
|
|
3854
|
-
const env = { ...process.env };
|
|
3855
|
-
for (const key of Object.keys(env)) {
|
|
3856
|
-
if (GIT_REPOSITORY_ENV.has(key) || key === "GIT_CONFIG" || key.startsWith("GIT_CONFIG_") || key.startsWith("GIT_ATTR_") || key === "GIT_DIFF_OPTS" || key === "GIT_EXTERNAL_DIFF") {
|
|
3857
|
-
delete env[key];
|
|
3858
|
-
}
|
|
3859
|
-
}
|
|
3860
|
-
Object.assign(env, overrides);
|
|
3861
|
-
env.GIT_NO_REPLACE_OBJECTS = "1";
|
|
3862
|
-
env.LC_ALL = "C";
|
|
3863
|
-
env.LANG = "C";
|
|
3864
|
-
return execFileSync("git", args, { cwd, env, maxBuffer: MAX_GIT_OUTPUT_BYTES });
|
|
3865
|
-
} catch (err) {
|
|
3866
|
-
const stderr = err && typeof err === "object" && "stderr" in err ? String(err.stderr) : "";
|
|
3867
|
-
throw new WorktreeAdapterError(`git ${args.join(" ")} failed: ${stderr || String(err)}`, err);
|
|
3868
|
-
}
|
|
3869
|
-
}
|
|
3870
|
-
function gitBytes(git, args, cwd, env) {
|
|
3871
|
-
try {
|
|
3872
|
-
const output = git(args, cwd, env);
|
|
3873
|
-
return typeof output === "string" ? Buffer.from(output, "utf8") : Buffer.from(output);
|
|
3874
|
-
} catch (err) {
|
|
3875
|
-
if (err instanceof WorktreeAdapterError) throw err;
|
|
3876
|
-
throw new WorktreeAdapterError(`git ${args.join(" ")} failed: ${String(err)}`, err);
|
|
3877
|
-
}
|
|
3878
|
-
}
|
|
3879
|
-
function gitText(git, args, cwd, env) {
|
|
3880
|
-
return gitBytes(git, args, cwd, env).toString("utf8").trim();
|
|
3881
|
-
}
|
|
3882
|
-
function hasRegisteredWorktree(git, repoRoot, path) {
|
|
3883
|
-
const expected = Buffer.from(`worktree ${resolve3(path)}`, "utf8");
|
|
3884
|
-
const records = gitBytes(git, ["worktree", "list", "--porcelain", "-z"], repoRoot);
|
|
3885
|
-
let start = 0;
|
|
3886
|
-
while (start < records.length) {
|
|
3887
|
-
const end = records.indexOf(0, start);
|
|
3888
|
-
if (end < 0) {
|
|
3889
|
-
throw new WorktreeAdapterError("Git worktree list output was not NUL-terminated");
|
|
3890
|
-
}
|
|
3891
|
-
if (records.subarray(start, end).equals(expected)) return true;
|
|
3892
|
-
start = end + 1;
|
|
3893
|
-
}
|
|
3894
|
-
return false;
|
|
3895
|
-
}
|
|
3896
|
-
function hasLocalBranch(git, repoRoot, branch) {
|
|
3897
|
-
const ref = `refs/heads/${branch}`;
|
|
3898
|
-
return gitText(git, ["for-each-ref", "--format=%(refname)", "--", ref], repoRoot).split("\n").some((candidate) => candidate === ref);
|
|
3899
|
-
}
|
|
3900
|
-
function reconcileAbsent(exists, remove) {
|
|
3901
|
-
try {
|
|
3902
|
-
if (!exists()) return void 0;
|
|
3903
|
-
} catch (err) {
|
|
3904
|
-
return err;
|
|
3905
|
-
}
|
|
3906
|
-
try {
|
|
3907
|
-
remove();
|
|
3908
|
-
return void 0;
|
|
3909
|
-
} catch (removeError) {
|
|
3910
|
-
try {
|
|
3911
|
-
if (!exists()) return void 0;
|
|
3912
|
-
} catch (recheckError) {
|
|
3913
|
-
return new AggregateError(
|
|
3914
|
-
[removeError, recheckError],
|
|
3915
|
-
"Removal failed and the resulting resource state could not be checked"
|
|
3916
|
-
);
|
|
3917
|
-
}
|
|
3918
|
-
return removeError;
|
|
3919
|
-
}
|
|
3920
|
-
}
|
|
3921
|
-
function sha2562(bytes) {
|
|
3922
|
-
return `sha256:${createHash7("sha256").update(bytes).digest("hex")}`;
|
|
3923
|
-
}
|
|
3924
|
-
var PATCH_ARGS = [
|
|
3925
|
-
// Git binary patches embed a zlib stream. Pin no-compression at command
|
|
3926
|
-
// scope so repository/global config and compressor heuristics cannot change
|
|
3927
|
-
// the candidate identity for the same base and final tree.
|
|
3928
|
-
"-c",
|
|
3929
|
-
"core.compression=0",
|
|
3930
|
-
"-c",
|
|
3931
|
-
`core.attributesFile=${devNull}`,
|
|
3932
|
-
"-c",
|
|
3933
|
-
"core.quotePath=true",
|
|
3934
|
-
"-c",
|
|
3935
|
-
"diff.suppressBlankEmpty=false",
|
|
3936
|
-
"diff",
|
|
3937
|
-
"--unified=3",
|
|
3938
|
-
"--inter-hunk-context=0",
|
|
3939
|
-
`-O${devNull}`,
|
|
3940
|
-
"--binary",
|
|
3941
|
-
"--full-index",
|
|
3942
|
-
"--no-color",
|
|
3943
|
-
"--no-ext-diff",
|
|
3944
|
-
"--no-textconv",
|
|
3945
|
-
"--no-renames",
|
|
3946
|
-
"--diff-algorithm=myers",
|
|
3947
|
-
"--no-indent-heuristic",
|
|
3948
|
-
"--src-prefix=a/",
|
|
3949
|
-
"--dst-prefix=b/"
|
|
3950
|
-
];
|
|
3951
|
-
var CANONICAL_GIT_ENV = {
|
|
3952
|
-
GIT_ATTR_GLOBAL: devNull,
|
|
3953
|
-
GIT_ATTR_NOSYSTEM: "1",
|
|
3954
|
-
GIT_CONFIG_GLOBAL: devNull,
|
|
3955
|
-
GIT_CONFIG_NOSYSTEM: "1",
|
|
3956
|
-
GIT_CONFIG_SYSTEM: devNull
|
|
3957
|
-
};
|
|
3958
|
-
function patchBytes(git, cwd, baseCommit, candidateCommit) {
|
|
3959
|
-
const scratch = mkdtempSync(join4(tmpdir(), "agent-eval-patch-"));
|
|
3960
|
-
const bareRepo = join4(scratch, "repo.git");
|
|
3961
|
-
const emptyTemplate = join4(scratch, "empty-template");
|
|
3962
|
-
mkdirSync2(emptyTemplate);
|
|
3963
|
-
try {
|
|
3964
|
-
const objectFormat = gitObjectHashAlgorithm(candidateCommit);
|
|
3965
|
-
const sourceObjects = realpathSync(gitText(git, ["rev-parse", "--git-path", "objects"], cwd));
|
|
3966
|
-
gitText(
|
|
3967
|
-
git,
|
|
3968
|
-
[
|
|
3969
|
-
"init",
|
|
3970
|
-
"--bare",
|
|
3971
|
-
"--quiet",
|
|
3972
|
-
...objectFormat === "sha256" ? ["--object-format=sha256"] : [],
|
|
3973
|
-
`--template=${emptyTemplate}`,
|
|
3974
|
-
bareRepo
|
|
3975
|
-
],
|
|
3976
|
-
scratch,
|
|
3977
|
-
CANONICAL_GIT_ENV
|
|
3978
|
-
);
|
|
3979
|
-
return gitBytes(
|
|
3980
|
-
git,
|
|
3981
|
-
[`--git-dir=${bareRepo}`, ...PATCH_ARGS, baseCommit, candidateCommit, "--"],
|
|
3982
|
-
scratch,
|
|
3983
|
-
{
|
|
3984
|
-
...CANONICAL_GIT_ENV,
|
|
3985
|
-
GIT_ALTERNATE_OBJECT_DIRECTORIES: sourceObjects
|
|
3986
|
-
}
|
|
3987
|
-
);
|
|
3988
|
-
} finally {
|
|
3989
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
3990
|
-
}
|
|
3991
|
-
}
|
|
3992
|
-
function resolveCommit(git, cwd, ref) {
|
|
3993
|
-
return gitText(git, ["rev-parse", "--verify", `${ref}^{commit}`], cwd);
|
|
3994
|
-
}
|
|
3995
|
-
function unresolvedWorktreePath(surface, worktreeDir) {
|
|
3996
|
-
if (isAbsolute2(surface.worktreeRef)) return surface.worktreeRef;
|
|
3997
|
-
if (worktreeDir) return join4(worktreeDir, basename(surface.worktreeRef));
|
|
3998
|
-
return surface.worktreeRef;
|
|
3999
|
-
}
|
|
4000
|
-
function displayGitPath(path) {
|
|
4001
|
-
return JSON.stringify(path);
|
|
4002
|
-
}
|
|
4003
|
-
function parseGitTreeEntries(bytes) {
|
|
4004
|
-
const input = Buffer.from(bytes);
|
|
4005
|
-
const entries = [];
|
|
4006
|
-
let start = 0;
|
|
4007
|
-
while (start < input.length) {
|
|
4008
|
-
const end = input.indexOf(0, start);
|
|
4009
|
-
if (end < 0) throw new WorktreeAdapterError("Git tree output was not NUL-terminated");
|
|
4010
|
-
const record = input.subarray(start, end);
|
|
4011
|
-
start = end + 1;
|
|
4012
|
-
if (record.length === 0) continue;
|
|
4013
|
-
const tab = record.indexOf(9);
|
|
4014
|
-
if (tab < 0) throw new WorktreeAdapterError("Git tree entry did not contain a path");
|
|
4015
|
-
const metadata = record.subarray(0, tab).toString("ascii").split(" ");
|
|
4016
|
-
if (metadata.length !== 3) throw new WorktreeAdapterError("Git tree entry was malformed");
|
|
4017
|
-
const mode = metadata[0];
|
|
4018
|
-
const type = metadata[1];
|
|
4019
|
-
const objectId = metadata[2];
|
|
4020
|
-
if (!mode || !type || !objectId) {
|
|
4021
|
-
throw new WorktreeAdapterError("Git tree entry was malformed");
|
|
4022
|
-
}
|
|
4023
|
-
const pathBytes = record.subarray(tab + 1);
|
|
4024
|
-
const path = pathBytes.toString("utf8");
|
|
4025
|
-
if (!Buffer.from(path, "utf8").equals(pathBytes)) {
|
|
4026
|
-
throw new WorktreeAdapterError("CodeSurface paths must be valid UTF-8");
|
|
4027
|
-
}
|
|
4028
|
-
if (type !== "blob" && type !== "commit") {
|
|
4029
|
-
throw new WorktreeAdapterError(
|
|
4030
|
-
`CodeSurface contains unsupported Git object type ${type} at ${displayGitPath(path)}`
|
|
4031
|
-
);
|
|
4032
|
-
}
|
|
4033
|
-
entries.push({ mode, objectId, path });
|
|
4034
|
-
}
|
|
4035
|
-
return entries;
|
|
4036
|
-
}
|
|
4037
|
-
function parseGitPathList(bytes) {
|
|
4038
|
-
const input = Buffer.from(bytes);
|
|
4039
|
-
const paths = [];
|
|
4040
|
-
let start = 0;
|
|
4041
|
-
while (start < input.length) {
|
|
4042
|
-
const end = input.indexOf(0, start);
|
|
4043
|
-
if (end < 0) throw new WorktreeAdapterError("Git path output was not NUL-terminated");
|
|
4044
|
-
const pathBytes = input.subarray(start, end);
|
|
4045
|
-
start = end + 1;
|
|
4046
|
-
if (pathBytes.length === 0) continue;
|
|
4047
|
-
const path = pathBytes.toString("utf8");
|
|
4048
|
-
if (!Buffer.from(path, "utf8").equals(pathBytes)) {
|
|
4049
|
-
throw new WorktreeAdapterError("CodeSurface paths must be valid UTF-8");
|
|
4050
|
-
}
|
|
4051
|
-
paths.push(path);
|
|
4052
|
-
}
|
|
4053
|
-
return paths;
|
|
4054
|
-
}
|
|
4055
|
-
function isWithinRoot(root, candidate) {
|
|
4056
|
-
const rel = relative2(root, candidate);
|
|
4057
|
-
return rel === "" || !isAbsolute2(rel) && rel !== ".." && !rel.startsWith(`..${sep}`);
|
|
4058
|
-
}
|
|
4059
|
-
function assertSafeRelativePath(root, path) {
|
|
4060
|
-
if (path.length === 0 || isAbsolute2(path)) {
|
|
4061
|
-
throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
|
|
4062
|
-
}
|
|
4063
|
-
const absolutePath = resolve3(root, path);
|
|
4064
|
-
if (!isWithinRoot(root, absolutePath) || absolutePath === root) {
|
|
4065
|
-
throw new WorktreeAdapterError(`CodeSurface path escapes its worktree: ${displayGitPath(path)}`);
|
|
4066
|
-
}
|
|
4067
|
-
const segments = path.split(/[\\/]/u);
|
|
4068
|
-
let parent = root;
|
|
4069
|
-
for (const segment of segments.slice(0, -1)) {
|
|
4070
|
-
if (segment.length === 0 || segment === "." || segment === "..") {
|
|
4071
|
-
throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
|
|
4072
|
-
}
|
|
4073
|
-
parent = join4(parent, segment);
|
|
4074
|
-
const stat = lstatSync(parent);
|
|
4075
|
-
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
4076
|
-
throw new WorktreeAdapterError(
|
|
4077
|
-
`CodeSurface path traverses a non-directory or symbolic link: ${displayGitPath(path)}`
|
|
4078
|
-
);
|
|
4079
|
-
}
|
|
4080
|
-
}
|
|
4081
|
-
return absolutePath;
|
|
4082
|
-
}
|
|
4083
|
-
function gitObjectHashAlgorithm(objectId) {
|
|
4084
|
-
if (/^[a-f0-9]{40}$/.test(objectId)) return "sha1";
|
|
4085
|
-
if (/^[a-f0-9]{64}$/.test(objectId)) return "sha256";
|
|
4086
|
-
throw new WorktreeAdapterError(`Unsupported Git object id: ${objectId}`);
|
|
4087
|
-
}
|
|
4088
|
-
function hashGitBlobBytes(bytes, objectId) {
|
|
4089
|
-
const hash = createHash7(gitObjectHashAlgorithm(objectId));
|
|
4090
|
-
hash.update(`blob ${bytes.byteLength}\0`);
|
|
4091
|
-
hash.update(bytes);
|
|
4092
|
-
return hash.digest("hex");
|
|
4093
|
-
}
|
|
4094
|
-
function hashGitBlobFile(path, objectId) {
|
|
4095
|
-
const before = lstatSync(path);
|
|
4096
|
-
if (!before.isFile() || before.isSymbolicLink()) {
|
|
4097
|
-
throw new WorktreeAdapterError(`CodeSurface expected a regular file at ${displayGitPath(path)}`);
|
|
4098
|
-
}
|
|
4099
|
-
const noFollow = process.platform === "win32" ? 0 : fsConstants.O_NOFOLLOW;
|
|
4100
|
-
const fd = openSync(path, fsConstants.O_RDONLY | noFollow);
|
|
4101
|
-
try {
|
|
4102
|
-
const opened = fstatSync(fd);
|
|
4103
|
-
if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) {
|
|
4104
|
-
throw new WorktreeAdapterError(
|
|
4105
|
-
`CodeSurface file changed while it was being verified: ${displayGitPath(path)}`
|
|
4106
|
-
);
|
|
4107
|
-
}
|
|
4108
|
-
const hash = createHash7(gitObjectHashAlgorithm(objectId));
|
|
4109
|
-
hash.update(`blob ${opened.size}\0`);
|
|
4110
|
-
const chunk = Buffer.allocUnsafe(FILE_HASH_CHUNK_BYTES);
|
|
4111
|
-
let total = 0;
|
|
4112
|
-
while (true) {
|
|
4113
|
-
const count = readSync(fd, chunk, 0, chunk.length, null);
|
|
4114
|
-
if (count === 0) break;
|
|
4115
|
-
total += count;
|
|
4116
|
-
hash.update(chunk.subarray(0, count));
|
|
4117
|
-
}
|
|
4118
|
-
if (total !== opened.size) {
|
|
4119
|
-
throw new WorktreeAdapterError(
|
|
4120
|
-
`CodeSurface file changed while it was being verified: ${displayGitPath(path)}`
|
|
4121
|
-
);
|
|
4122
|
-
}
|
|
4123
|
-
return { hash: hash.digest("hex"), executable: (opened.mode & 73) !== 0 };
|
|
4124
|
-
} finally {
|
|
4125
|
-
closeSync(fd);
|
|
4126
|
-
}
|
|
4127
|
-
}
|
|
4128
|
-
function assertSymlinkTargetIsBound(root, linkPath, trackedPaths) {
|
|
4129
|
-
const targetBytes = readlinkSync(linkPath, { encoding: "buffer" });
|
|
4130
|
-
const target = targetBytes.toString("utf8");
|
|
4131
|
-
if (!Buffer.from(target, "utf8").equals(targetBytes)) {
|
|
4132
|
-
throw new WorktreeAdapterError(
|
|
4133
|
-
`CodeSurface symbolic link has an unsafe target at ${displayGitPath(linkPath)}`
|
|
4134
|
-
);
|
|
4135
|
-
}
|
|
4136
|
-
if (isAbsolute2(target)) {
|
|
4137
|
-
throw new WorktreeAdapterError(
|
|
4138
|
-
`CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
|
|
4139
|
-
);
|
|
4140
|
-
}
|
|
4141
|
-
const lexicalTarget = resolve3(dirname(linkPath), target);
|
|
4142
|
-
if (!isWithinRoot(root, lexicalTarget)) {
|
|
4143
|
-
throw new WorktreeAdapterError(
|
|
4144
|
-
`CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
|
|
4145
|
-
);
|
|
4146
|
-
}
|
|
4147
|
-
let resolvedTarget;
|
|
4148
|
-
try {
|
|
4149
|
-
resolvedTarget = realpathSync(linkPath);
|
|
4150
|
-
} catch (err) {
|
|
4151
|
-
throw new WorktreeAdapterError(
|
|
4152
|
-
`CodeSurface symbolic link target is missing or cyclic at ${displayGitPath(linkPath)}`,
|
|
4153
|
-
err
|
|
4154
|
-
);
|
|
4155
|
-
}
|
|
4156
|
-
if (!isWithinRoot(root, resolvedTarget)) {
|
|
4157
|
-
throw new WorktreeAdapterError(
|
|
4158
|
-
`CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`
|
|
4159
|
-
);
|
|
4160
|
-
}
|
|
4161
|
-
const targetRelative = relative2(root, resolvedTarget).split(sep).join("/");
|
|
4162
|
-
const targetIsTracked = trackedPaths.has(targetRelative) || [...trackedPaths].some((trackedPath) => trackedPath.startsWith(`${targetRelative}/`));
|
|
4163
|
-
if (!targetIsTracked) {
|
|
4164
|
-
throw new WorktreeAdapterError(
|
|
4165
|
-
`CodeSurface symbolic link resolves to untracked content at ${displayGitPath(linkPath)}`
|
|
4166
|
-
);
|
|
4167
|
-
}
|
|
4168
|
-
}
|
|
4169
|
-
function assertRawTreeMatchesWorktree(git, root, candidateCommit) {
|
|
4170
|
-
const entries = parseGitTreeEntries(
|
|
4171
|
-
gitBytes(git, ["ls-tree", "-r", "-z", "--full-tree", candidateCommit], root)
|
|
4172
|
-
);
|
|
4173
|
-
const trackedPaths = new Set(entries.map((entry) => entry.path));
|
|
4174
|
-
for (const entry of entries) {
|
|
4175
|
-
const absolutePath = assertSafeRelativePath(root, entry.path);
|
|
4176
|
-
if (entry.mode === "160000" || entry.mode === "040000") {
|
|
4177
|
-
throw new WorktreeAdapterError(
|
|
4178
|
-
`CodeSurface contains a Git submodule whose executable bytes are not bound: ${displayGitPath(entry.path)}`
|
|
4179
|
-
);
|
|
4180
|
-
}
|
|
4181
|
-
if (entry.mode === "120000") {
|
|
4182
|
-
const stat = lstatSync(absolutePath);
|
|
4183
|
-
if (!stat.isSymbolicLink()) {
|
|
4184
|
-
throw new WorktreeAdapterError(
|
|
4185
|
-
`CodeSurface expected a symbolic link at ${displayGitPath(entry.path)}`
|
|
4186
|
-
);
|
|
4187
|
-
}
|
|
4188
|
-
const targetBytes = readlinkSync(absolutePath, { encoding: "buffer" });
|
|
4189
|
-
const actualObjectId = hashGitBlobBytes(targetBytes, entry.objectId);
|
|
4190
|
-
if (actualObjectId !== entry.objectId) {
|
|
4191
|
-
throw new WorktreeAdapterError(
|
|
4192
|
-
`CodeSurface raw symbolic-link bytes differ from the candidate tree at ${displayGitPath(entry.path)}`
|
|
4193
|
-
);
|
|
4194
|
-
}
|
|
4195
|
-
assertSymlinkTargetIsBound(root, absolutePath, trackedPaths);
|
|
4196
|
-
continue;
|
|
4197
|
-
}
|
|
4198
|
-
if (entry.mode !== "100644" && entry.mode !== "100755") {
|
|
4199
|
-
throw new WorktreeAdapterError(
|
|
4200
|
-
`CodeSurface contains unsupported Git mode ${entry.mode} at ${displayGitPath(entry.path)}`
|
|
4201
|
-
);
|
|
4202
|
-
}
|
|
4203
|
-
const actual = hashGitBlobFile(absolutePath, entry.objectId);
|
|
4204
|
-
if (actual.hash !== entry.objectId) {
|
|
4205
|
-
throw new WorktreeAdapterError(
|
|
4206
|
-
`CodeSurface raw tracked bytes differ from the candidate tree because the worktree changed after finalization at ${displayGitPath(entry.path)}`
|
|
4207
|
-
);
|
|
4208
|
-
}
|
|
4209
|
-
if (process.platform !== "win32" && actual.executable !== (entry.mode === "100755")) {
|
|
4210
|
-
throw new WorktreeAdapterError(
|
|
4211
|
-
`CodeSurface executable mode differs from the candidate tree at ${displayGitPath(entry.path)}`
|
|
4212
|
-
);
|
|
4213
|
-
}
|
|
4214
|
-
}
|
|
4215
|
-
}
|
|
4216
|
-
function verifyCodeSurfaceWithGit(surface, path, git) {
|
|
4217
|
-
assertCodeSurfaceIdentity(surface);
|
|
4218
|
-
if (!existsSync4(path)) {
|
|
4219
|
-
throw new WorktreeAdapterError(`CodeSurface worktree does not exist: ${path}`);
|
|
4220
|
-
}
|
|
4221
|
-
const lexicalRoot = resolve3(path);
|
|
4222
|
-
const canonicalRoot = realpathSync(path);
|
|
4223
|
-
if (lstatSync(lexicalRoot).isSymbolicLink() || process.platform !== "win32" && lexicalRoot !== canonicalRoot) {
|
|
4224
|
-
throw new WorktreeAdapterError(
|
|
4225
|
-
`CodeSurface worktree locator must not contain a symbolic link: ${path}`
|
|
4226
|
-
);
|
|
4227
|
-
}
|
|
4228
|
-
const repoRoot = gitText(git, ["rev-parse", "--show-toplevel"], path);
|
|
4229
|
-
const canonicalRepoRoot = realpathSync(repoRoot);
|
|
4230
|
-
if (canonicalRepoRoot !== canonicalRoot) {
|
|
4231
|
-
throw new WorktreeAdapterError(
|
|
4232
|
-
`CodeSurface worktree locator is not the repository root: expected ${repoRoot}, got ${path}`
|
|
4233
|
-
);
|
|
4234
|
-
}
|
|
4235
|
-
const indexFlags = gitText(git, ["ls-files", "-v"], path).split("\n").filter((line) => line.length > 0 && (line.startsWith("S ") || /^[a-z]/.test(line)));
|
|
4236
|
-
if (indexFlags.length > 0) {
|
|
4237
|
-
throw new WorktreeAdapterError(
|
|
4238
|
-
`CodeSurface worktree uses hidden index entries that cannot be verified: ${path}
|
|
4239
|
-
${indexFlags.join("\n")}`
|
|
4240
|
-
);
|
|
4241
|
-
}
|
|
4242
|
-
const extraPaths = [
|
|
4243
|
-
...parseGitPathList(gitBytes(git, ["ls-files", "--others", "--exclude-standard", "-z"], path)),
|
|
4244
|
-
...parseGitPathList(
|
|
4245
|
-
gitBytes(git, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], path)
|
|
4246
|
-
)
|
|
4247
|
-
];
|
|
4248
|
-
if (extraPaths.length > 0) {
|
|
4249
|
-
throw new WorktreeAdapterError(
|
|
4250
|
-
`CodeSurface worktree changed after finalization: ${path}
|
|
4251
|
-
${extraPaths.join("\n")}`
|
|
4252
|
-
);
|
|
4253
|
-
}
|
|
4254
|
-
const candidateCommit = resolveCommit(git, path, "HEAD");
|
|
4255
|
-
if (candidateCommit !== surface.candidateCommit) {
|
|
4256
|
-
throw new WorktreeAdapterError(
|
|
4257
|
-
`CodeSurface candidate commit mismatch: expected ${surface.candidateCommit}, got ${candidateCommit}`
|
|
4258
|
-
);
|
|
4259
|
-
}
|
|
4260
|
-
const baseCommit = resolveCommit(git, path, surface.baseCommit);
|
|
4261
|
-
if (baseCommit !== surface.baseCommit) {
|
|
4262
|
-
throw new WorktreeAdapterError(
|
|
4263
|
-
`CodeSurface base commit mismatch: expected ${surface.baseCommit}, got ${baseCommit}`
|
|
4264
|
-
);
|
|
4265
|
-
}
|
|
4266
|
-
const baseTree = gitText(git, ["rev-parse", "--verify", `${surface.baseCommit}^{tree}`], path);
|
|
4267
|
-
if (baseTree !== surface.baseTree) {
|
|
4268
|
-
throw new WorktreeAdapterError(
|
|
4269
|
-
`CodeSurface base tree mismatch: expected ${surface.baseTree}, got ${baseTree}`
|
|
4270
|
-
);
|
|
4271
|
-
}
|
|
4272
|
-
const candidateTree = gitText(
|
|
4273
|
-
git,
|
|
4274
|
-
["rev-parse", "--verify", `${surface.candidateCommit}^{tree}`],
|
|
4275
|
-
path
|
|
4276
|
-
);
|
|
4277
|
-
if (candidateTree !== surface.candidateTree) {
|
|
4278
|
-
throw new WorktreeAdapterError(
|
|
4279
|
-
`CodeSurface tree mismatch: expected ${surface.candidateTree}, got ${candidateTree}`
|
|
4280
|
-
);
|
|
4281
|
-
}
|
|
4282
|
-
try {
|
|
4283
|
-
gitText(
|
|
4284
|
-
git,
|
|
4285
|
-
[
|
|
4286
|
-
"diff-index",
|
|
4287
|
-
"--cached",
|
|
4288
|
-
"--quiet",
|
|
4289
|
-
"--no-ext-diff",
|
|
4290
|
-
"--no-textconv",
|
|
4291
|
-
surface.candidateCommit,
|
|
4292
|
-
"--"
|
|
4293
|
-
],
|
|
4294
|
-
path
|
|
4295
|
-
);
|
|
4296
|
-
} catch (err) {
|
|
4297
|
-
throw new WorktreeAdapterError(
|
|
4298
|
-
`CodeSurface index differs from finalized candidate: ${path}`,
|
|
4299
|
-
err
|
|
4300
|
-
);
|
|
4301
|
-
}
|
|
4302
|
-
try {
|
|
4303
|
-
assertRawTreeMatchesWorktree(git, canonicalRepoRoot, surface.candidateCommit);
|
|
4304
|
-
} catch (err) {
|
|
4305
|
-
if (err instanceof WorktreeAdapterError) throw err;
|
|
4306
|
-
throw new WorktreeAdapterError(`CodeSurface raw tree verification failed: ${path}`, err);
|
|
4307
|
-
}
|
|
4308
|
-
const mergeBase = gitText(git, ["merge-base", surface.baseCommit, surface.candidateCommit], path);
|
|
4309
|
-
if (mergeBase !== surface.baseCommit) {
|
|
4310
|
-
throw new WorktreeAdapterError(
|
|
4311
|
-
`CodeSurface candidate ${surface.candidateCommit} does not descend from base ${surface.baseCommit}`
|
|
4312
|
-
);
|
|
4313
|
-
}
|
|
4314
|
-
const patch = patchBytes(git, path, surface.baseCommit, surface.candidateCommit);
|
|
4315
|
-
const actualPatchHash = sha2562(patch);
|
|
4316
|
-
if (actualPatchHash !== surface.patch.sha256 || patch.byteLength !== surface.patch.byteLength) {
|
|
4317
|
-
throw new WorktreeAdapterError(
|
|
4318
|
-
`CodeSurface patch mismatch: expected ${surface.patch.sha256}/${surface.patch.byteLength} bytes, got ${actualPatchHash}/${patch.byteLength} bytes`
|
|
4319
|
-
);
|
|
4320
|
-
}
|
|
4321
|
-
return {
|
|
4322
|
-
path,
|
|
4323
|
-
repoRoot: canonicalRepoRoot,
|
|
4324
|
-
contentHash: surfaceContentHash(surface),
|
|
4325
|
-
patchBytes: new Uint8Array(patch)
|
|
4326
|
-
};
|
|
4327
|
-
}
|
|
4328
|
-
function slug(label) {
|
|
4329
|
-
return label.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "candidate";
|
|
4330
|
-
}
|
|
4331
|
-
function gitWorktreeAdapter(opts) {
|
|
4332
|
-
const git = opts.git ?? defaultGit;
|
|
4333
|
-
const worktreeDir = opts.worktreeDir ?? join4(opts.repoRoot, ".worktrees");
|
|
4334
|
-
const branchPrefix = opts.branchPrefix ?? "improve";
|
|
4335
|
-
return {
|
|
4336
|
-
async create({ baseRef, label }) {
|
|
4337
|
-
const id = `${slug(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
|
|
4338
|
-
const branch = `${branchPrefix}/${id}`;
|
|
4339
|
-
const path = join4(worktreeDir, id);
|
|
4340
|
-
const baseCommit = resolveCommit(git, opts.repoRoot, baseRef);
|
|
4341
|
-
const baseTree = gitText(
|
|
4342
|
-
git,
|
|
4343
|
-
["rev-parse", "--verify", `${baseCommit}^{tree}`],
|
|
4344
|
-
opts.repoRoot
|
|
4345
|
-
);
|
|
4346
|
-
gitText(git, ["worktree", "add", "-b", branch, path, baseCommit], opts.repoRoot);
|
|
4347
|
-
return { path, branch, baseRef, baseCommit, baseTree };
|
|
4348
|
-
},
|
|
4349
|
-
async finalize(worktree, summary) {
|
|
4350
|
-
const status = gitText(
|
|
4351
|
-
git,
|
|
4352
|
-
["status", "--porcelain=v1", "--untracked-files=all"],
|
|
4353
|
-
worktree.path
|
|
4354
|
-
);
|
|
4355
|
-
if (status.length > 0) {
|
|
4356
|
-
gitText(git, ["add", "-A"], worktree.path);
|
|
4357
|
-
gitText(git, ["commit", "-m", summary], worktree.path);
|
|
4358
|
-
}
|
|
4359
|
-
const candidateCommit = resolveCommit(git, worktree.path, "HEAD");
|
|
4360
|
-
const candidateTree = gitText(
|
|
4361
|
-
git,
|
|
4362
|
-
["rev-parse", "--verify", `${candidateCommit}^{tree}`],
|
|
4363
|
-
worktree.path
|
|
4364
|
-
);
|
|
4365
|
-
const patch = patchBytes(git, worktree.path, worktree.baseCommit, candidateCommit);
|
|
4366
|
-
const surface = {
|
|
4367
|
-
kind: "code",
|
|
4368
|
-
worktreeRef: worktree.path,
|
|
4369
|
-
baseRef: worktree.baseRef,
|
|
4370
|
-
baseCommit: worktree.baseCommit,
|
|
4371
|
-
baseTree: worktree.baseTree,
|
|
4372
|
-
candidateCommit,
|
|
4373
|
-
candidateTree,
|
|
4374
|
-
patch: {
|
|
4375
|
-
format: "git-diff-binary",
|
|
4376
|
-
sha256: sha2562(patch),
|
|
4377
|
-
byteLength: patch.byteLength
|
|
4378
|
-
},
|
|
4379
|
-
summary
|
|
4380
|
-
};
|
|
4381
|
-
verifyCodeSurfaceWithGit(surface, worktree.path, git);
|
|
4382
|
-
return surface;
|
|
4383
|
-
},
|
|
4384
|
-
async discard(worktree) {
|
|
4385
|
-
const failures = [
|
|
4386
|
-
reconcileAbsent(
|
|
4387
|
-
() => hasRegisteredWorktree(git, opts.repoRoot, worktree.path),
|
|
4388
|
-
() => gitText(git, ["worktree", "remove", "--force", "--", worktree.path], opts.repoRoot)
|
|
4389
|
-
),
|
|
4390
|
-
reconcileAbsent(
|
|
4391
|
-
() => hasLocalBranch(git, opts.repoRoot, worktree.branch),
|
|
4392
|
-
() => gitText(git, ["branch", "-D", "--", worktree.branch], opts.repoRoot)
|
|
4393
|
-
)
|
|
4394
|
-
].filter((failure) => failure !== void 0);
|
|
4395
|
-
if (failures.length > 0) {
|
|
4396
|
-
const cause = failures.length === 1 ? failures[0] : new AggregateError(failures, "Multiple Git resources could not be removed");
|
|
4397
|
-
throw new WorktreeAdapterError(
|
|
4398
|
-
`Failed to discard worktree ${worktree.path} and branch ${worktree.branch}`,
|
|
4399
|
-
cause
|
|
4400
|
-
);
|
|
4401
|
-
}
|
|
4402
|
-
}
|
|
4403
|
-
};
|
|
4404
|
-
}
|
|
4405
|
-
function verifyCodeSurface(surface, worktreeDir) {
|
|
4406
|
-
const path = unresolvedWorktreePath(surface, worktreeDir);
|
|
4407
|
-
return verifyCodeSurfaceWithGit(surface, path, defaultGit);
|
|
4408
|
-
}
|
|
4409
|
-
function resolveWorktreePath(surface, worktreeDir) {
|
|
4410
|
-
return verifyCodeSurface(surface, worktreeDir).path;
|
|
4411
|
-
}
|
|
4412
|
-
|
|
4413
|
-
export {
|
|
4414
|
-
completionVerdict,
|
|
4415
|
-
verifyCompletion,
|
|
4416
|
-
parseCorrectnessResponse,
|
|
4417
|
-
createLlmCorrectnessChecker,
|
|
4418
|
-
createTokenRecallChecker,
|
|
4419
|
-
extractProducedState,
|
|
4420
|
-
CODING_HARNESSES,
|
|
4421
|
-
HARNESS_NATIVE_MODEL,
|
|
4422
|
-
expandProfileAxes,
|
|
4423
|
-
harnessAxisOf,
|
|
4424
|
-
agentProfileId,
|
|
4425
|
-
agentProfileModelId,
|
|
4426
|
-
agentProfileHash,
|
|
4427
|
-
buildAnalystSurfaceDispatch,
|
|
4428
|
-
failureModeRecallJudge,
|
|
4429
|
-
analyzeCrossSurfaceInteractions,
|
|
4430
|
-
discoverEvalFixtures,
|
|
4431
|
-
loadEvalFixture,
|
|
4432
|
-
loadEvalFixtureScenarios,
|
|
4433
|
-
planEvalFixtureRun,
|
|
4434
|
-
neutralizationGate,
|
|
4435
|
-
sequentialPairedGate,
|
|
4436
|
-
sequentialDecide,
|
|
4437
|
-
rolloutArgumentDiff,
|
|
4438
|
-
classifyUngroundedLiterals,
|
|
4439
|
-
LabeledScenarioStoreError,
|
|
4440
|
-
FsLabeledScenarioStore,
|
|
4441
|
-
neutralizeText,
|
|
4442
|
-
makePlaybackDispatch,
|
|
4443
|
-
scoreUserStory,
|
|
4444
|
-
userStoryScoreboard,
|
|
4445
|
-
scoreboardSummary,
|
|
4446
|
-
renderScoreboardMarkdown,
|
|
4447
|
-
ProfileMatrixError,
|
|
4448
|
-
runProfileMatrix,
|
|
4449
|
-
scoreDiscrimination,
|
|
4450
|
-
selectDiscriminative,
|
|
4451
|
-
SEARCH_LEDGER_SCHEMA,
|
|
4452
|
-
validateSearchLedgerEvent,
|
|
4453
|
-
openSearchLedger,
|
|
4454
|
-
FileSearchLedger,
|
|
4455
|
-
isTransientTransportFailure,
|
|
4456
|
-
WorktreeAdapterError,
|
|
4457
|
-
gitWorktreeAdapter,
|
|
4458
|
-
verifyCodeSurface,
|
|
4459
|
-
resolveWorktreePath
|
|
4460
|
-
};
|
|
4461
|
-
//# sourceMappingURL=chunk-UB2LOJ6Q.js.map
|