@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
|
@@ -1,1912 +1,1819 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
import {
|
|
10
|
-
calibrationFromPairs
|
|
11
|
-
} from "../chunk-NPCTHQIO.js";
|
|
12
|
-
import {
|
|
13
|
-
projectRuntimeTrajectoryEvidence
|
|
14
|
-
} from "../chunk-T4SQEITX.js";
|
|
15
|
-
import {
|
|
16
|
-
offPolicyEstimateAll
|
|
17
|
-
} from "../chunk-VGRCHJON.js";
|
|
18
|
-
import {
|
|
19
|
-
confidenceInterval
|
|
20
|
-
} from "../chunk-MHELPNRP.js";
|
|
21
|
-
import "../chunk-VI2UW6B6.js";
|
|
22
|
-
import "../chunk-PXE2VKMX.js";
|
|
23
|
-
import {
|
|
24
|
-
ValidationError
|
|
25
|
-
} from "../chunk-ONWEPEDO.js";
|
|
26
|
-
import "../chunk-PZ5AY32C.js";
|
|
27
|
-
|
|
28
|
-
// src/belief-state/calibration.ts
|
|
1
|
+
import { s as ValidationError } from "../errors-8YnH8WlF.js";
|
|
2
|
+
import { a as confidenceInterval } from "../statistics-CnnxdpOg.js";
|
|
3
|
+
import { s as trainingScore } from "../reward-nw2xZGZG.js";
|
|
4
|
+
import { n as projectRuntimeTrajectoryEvidence } from "../runtime-trajectory-1gyaTOoC.js";
|
|
5
|
+
import { r as offPolicyEstimateAll } from "../off-policy-DvgzvtIx.js";
|
|
6
|
+
import { n as calibrationFromPairs } from "../calibration-CNWWA6K8.js";
|
|
7
|
+
import { a as fromPiSession, c as observeCodeAgentSession, i as fromOpenCodeSession, n as fromCodexSession, r as fromKimiCodeSession, t as fromClaudeCodeSession } from "../code-agent-session-BjkMTQ7H.js";
|
|
8
|
+
//#region src/belief-state/calibration.ts
|
|
29
9
|
function calibrateBeliefDecisions(points, options = {}) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
10
|
+
const pairs = filterCalibrationRegion(points, options).filter((point) => typeof point.confidence === "number" && point.outcome).map((point) => ({
|
|
11
|
+
evalScore: point.confidence,
|
|
12
|
+
outcome: outcomeScore$1(point)
|
|
13
|
+
})).filter((pair) => Number.isFinite(pair.outcome));
|
|
14
|
+
const minPairs = options.minPairs ?? 10;
|
|
15
|
+
if (pairs.length < minPairs) return null;
|
|
16
|
+
return calibrationFromPairs(pairs, "belief-confidence", "decision-outcome", {
|
|
17
|
+
bins: options.bins ?? 5,
|
|
18
|
+
range: {
|
|
19
|
+
lo: 0,
|
|
20
|
+
hi: 1
|
|
21
|
+
}
|
|
22
|
+
});
|
|
41
23
|
}
|
|
42
24
|
function filterCalibrationRegion(points, options) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
return Number.NaN;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// src/belief-state/ope.ts
|
|
25
|
+
const region = options.region ?? "all";
|
|
26
|
+
if (region === "all") return points;
|
|
27
|
+
const policy = options.policy;
|
|
28
|
+
if (!policy) throw new ValidationError(`calibrateBeliefDecisions: policy is required when region is "${region}"`);
|
|
29
|
+
return points.filter((point) => {
|
|
30
|
+
const accepted = policy.decide(point).action === "accept";
|
|
31
|
+
return region === "accepted" ? accepted : !accepted;
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
function outcomeScore$1(point) {
|
|
35
|
+
if (typeof point.outcome?.reward === "number") return point.outcome.reward;
|
|
36
|
+
if (typeof point.outcome?.score === "number") return point.outcome.score;
|
|
37
|
+
if (point.outcome?.success === true) return 1;
|
|
38
|
+
if (point.outcome?.success === false) return 0;
|
|
39
|
+
return NaN;
|
|
40
|
+
}
|
|
41
|
+
//#endregion
|
|
42
|
+
//#region src/belief-state/ope.ts
|
|
65
43
|
function embeddedBeliefOpeTargetPolicy(id = "embedded-target-prob") {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return point.qHat;
|
|
79
|
-
}
|
|
80
|
-
};
|
|
44
|
+
return {
|
|
45
|
+
id,
|
|
46
|
+
targetProbOf(point) {
|
|
47
|
+
return point.targetProb;
|
|
48
|
+
},
|
|
49
|
+
qHatChosenOf(point) {
|
|
50
|
+
return point.qHatChosen;
|
|
51
|
+
},
|
|
52
|
+
vHatTargetOf(point) {
|
|
53
|
+
return point.vHatTarget;
|
|
54
|
+
}
|
|
55
|
+
};
|
|
81
56
|
}
|
|
82
57
|
function beliefDecisionsToOffPolicyTrajectories(points, targetPolicy, options = {}) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
...qHatChosen !== void 0 ? { qHatChosen } : {},
|
|
135
|
-
...vHatTarget !== void 0 ? { vHatTarget } : {},
|
|
136
|
-
qHat
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
|
-
return {
|
|
140
|
-
targetPolicyId: targetPolicy.id,
|
|
141
|
-
trajectories,
|
|
142
|
-
dropped: points.length - trajectories.length,
|
|
143
|
-
diagnostics: compactDiagnostics(diagnostics, options.maxDiagnostics ?? 20)
|
|
144
|
-
};
|
|
58
|
+
const trajectories = [];
|
|
59
|
+
const diagnostics = [];
|
|
60
|
+
for (const point of points) {
|
|
61
|
+
if (!point.outcome) {
|
|
62
|
+
diagnostics.push(`${point.id}: missing outcome`);
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (!isBehaviorProbability(point.behaviorProb)) {
|
|
66
|
+
diagnostics.push(`${point.id}: invalid behaviorProb ${formatProbability(point.behaviorProb)}`);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
let targetProb;
|
|
70
|
+
let qHatChosen;
|
|
71
|
+
let vHatTarget;
|
|
72
|
+
try {
|
|
73
|
+
targetProb = targetPolicy.targetProbOf(point);
|
|
74
|
+
qHatChosen = targetPolicy.qHatChosenOf?.(point);
|
|
75
|
+
vHatTarget = targetPolicy.vHatTargetOf?.(point);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
diagnostics.push(`${point.id}: target policy ${targetPolicy.id} threw (${errorMessage$1(error)})`);
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
if (!isTargetProbability(targetProb)) {
|
|
81
|
+
diagnostics.push(`${point.id}: invalid targetProb ${formatProbability(targetProb)}`);
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
const hasQHatChosen = qHatChosen !== null && qHatChosen !== void 0;
|
|
85
|
+
const hasVHatTarget = vHatTarget !== null && vHatTarget !== void 0;
|
|
86
|
+
if (hasQHatChosen !== hasVHatTarget) {
|
|
87
|
+
diagnostics.push(`${point.id}: qHatChosen and vHatTarget must be supplied together`);
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (hasQHatChosen && hasVHatTarget && (!isTargetProbability(qHatChosen) || !isTargetProbability(vHatTarget))) {
|
|
91
|
+
diagnostics.push(`${point.id}: invalid contextual Q pair qHatChosen=${formatProbability(qHatChosen)} vHatTarget=${formatProbability(vHatTarget)}`);
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
trajectories.push({
|
|
95
|
+
runId: point.id,
|
|
96
|
+
reward: rewardOf$1(point),
|
|
97
|
+
behaviorProb: point.behaviorProb,
|
|
98
|
+
targetProb,
|
|
99
|
+
...qHatChosen !== void 0 ? { qHatChosen } : {},
|
|
100
|
+
...vHatTarget !== void 0 ? { vHatTarget } : {}
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
targetPolicyId: targetPolicy.id,
|
|
105
|
+
trajectories,
|
|
106
|
+
dropped: points.length - trajectories.length,
|
|
107
|
+
diagnostics: compactDiagnostics(diagnostics, options.maxDiagnostics ?? 20)
|
|
108
|
+
};
|
|
145
109
|
}
|
|
146
110
|
function evaluateBeliefOffPolicy(points, targetPolicy, options = {}) {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
111
|
+
const trajectoryReport = beliefDecisionsToOffPolicyTrajectories(points, targetPolicy, options);
|
|
112
|
+
const { trajectories } = trajectoryReport;
|
|
113
|
+
const estimates = offPolicyEstimateAll(trajectories, options);
|
|
114
|
+
const support = supportDiagnostics(estimates.dr, {
|
|
115
|
+
minEffectiveSampleSize: options.minEffectiveSampleSize ?? 30,
|
|
116
|
+
minEffectiveSampleRatio: options.minEffectiveSampleRatio ?? .25,
|
|
117
|
+
dropped: trajectoryReport.dropped,
|
|
118
|
+
diagnostics: trajectoryReport.diagnostics
|
|
119
|
+
});
|
|
120
|
+
return {
|
|
121
|
+
targetPolicyId: targetPolicy.id,
|
|
122
|
+
...estimates,
|
|
123
|
+
support
|
|
124
|
+
};
|
|
158
125
|
}
|
|
159
126
|
function supportDiagnostics(estimate, options) {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
}
|
|
183
|
-
if (estimate.maxImportanceWeight > 10) {
|
|
184
|
-
reasons.push(`max importance weight ${estimate.maxImportanceWeight.toFixed(2)} is high`);
|
|
185
|
-
}
|
|
186
|
-
return {
|
|
187
|
-
supported: reasons.length === 0,
|
|
188
|
-
n: estimate.n,
|
|
189
|
-
dropped: options.dropped,
|
|
190
|
-
effectiveSampleSize: estimate.effectiveSampleSize,
|
|
191
|
-
effectiveSampleRatio: ratio2,
|
|
192
|
-
maxImportanceWeight: estimate.maxImportanceWeight,
|
|
193
|
-
reasons
|
|
194
|
-
};
|
|
195
|
-
}
|
|
196
|
-
function rewardOf(point) {
|
|
197
|
-
if (typeof point.outcome?.reward === "number") return point.outcome.reward;
|
|
198
|
-
if (typeof point.outcome?.score === "number") return point.outcome.score;
|
|
199
|
-
if (point.outcome?.success === true) return 1;
|
|
200
|
-
return 0;
|
|
127
|
+
const ratio = estimate.n > 0 ? estimate.effectiveSampleSize / estimate.n : 0;
|
|
128
|
+
const reasons = [...options.diagnostics];
|
|
129
|
+
if (estimate.n === 0) reasons.push("no valid OPE trajectories");
|
|
130
|
+
if (options.dropped > 0) reasons.push(`dropped ${options.dropped} unsupported decision(s)`);
|
|
131
|
+
if (estimate.effectiveSampleSize < options.minEffectiveSampleSize) reasons.push(`effective sample size ${estimate.effectiveSampleSize.toFixed(2)} below ${options.minEffectiveSampleSize}`);
|
|
132
|
+
if (ratio < options.minEffectiveSampleRatio) reasons.push(`effective sample ratio ${ratio.toFixed(2)} below ${options.minEffectiveSampleRatio}`);
|
|
133
|
+
if (estimate.maxImportanceWeight > 10) reasons.push(`max importance weight ${estimate.maxImportanceWeight.toFixed(2)} is high`);
|
|
134
|
+
return {
|
|
135
|
+
supported: reasons.length === 0,
|
|
136
|
+
n: estimate.n,
|
|
137
|
+
dropped: options.dropped,
|
|
138
|
+
effectiveSampleSize: estimate.effectiveSampleSize,
|
|
139
|
+
effectiveSampleRatio: ratio,
|
|
140
|
+
maxImportanceWeight: estimate.maxImportanceWeight,
|
|
141
|
+
reasons
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
function rewardOf$1(point) {
|
|
145
|
+
if (typeof point.outcome?.reward === "number") return point.outcome.reward;
|
|
146
|
+
if (typeof point.outcome?.score === "number") return point.outcome.score;
|
|
147
|
+
if (point.outcome?.success === true) return 1;
|
|
148
|
+
return 0;
|
|
201
149
|
}
|
|
202
150
|
function isBehaviorProbability(value) {
|
|
203
|
-
|
|
151
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1;
|
|
204
152
|
}
|
|
205
153
|
function isTargetProbability(value) {
|
|
206
|
-
|
|
154
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
207
155
|
}
|
|
208
156
|
function formatProbability(value) {
|
|
209
|
-
|
|
157
|
+
return typeof value === "number" ? String(value) : String(value ?? "missing");
|
|
210
158
|
}
|
|
211
|
-
function errorMessage(error) {
|
|
212
|
-
|
|
159
|
+
function errorMessage$1(error) {
|
|
160
|
+
return error instanceof Error ? error.message : String(error);
|
|
213
161
|
}
|
|
214
162
|
function compactDiagnostics(diagnostics, maxDiagnostics) {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
retryCost: 0.1,
|
|
230
|
-
stopUtility: 0,
|
|
231
|
-
costWeight: 1
|
|
163
|
+
if (diagnostics.length <= maxDiagnostics) return diagnostics;
|
|
164
|
+
return [...diagnostics.slice(0, maxDiagnostics), `${diagnostics.length - maxDiagnostics} additional OPE diagnostic(s) omitted`];
|
|
165
|
+
}
|
|
166
|
+
//#endregion
|
|
167
|
+
//#region src/belief-state/selective.ts
|
|
168
|
+
const DEFAULT_UTILITY = {
|
|
169
|
+
successUtility: 1,
|
|
170
|
+
failureUtility: -1,
|
|
171
|
+
deferUtility: 0,
|
|
172
|
+
verifyCost: .05,
|
|
173
|
+
askCost: .05,
|
|
174
|
+
retryCost: .1,
|
|
175
|
+
stopUtility: 0,
|
|
176
|
+
costWeight: 1
|
|
232
177
|
};
|
|
233
178
|
function thresholdSelectivePolicy(options) {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
qHat: point.qHat,
|
|
252
|
-
reason: confidence >= threshold ? "confidence threshold passed" : "confidence threshold failed"
|
|
253
|
-
};
|
|
254
|
-
}
|
|
255
|
-
};
|
|
179
|
+
const threshold = options.confidenceThreshold;
|
|
180
|
+
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) throw new ValidationError(`thresholdSelectivePolicy: confidenceThreshold must be in [0, 1], got ${threshold}`);
|
|
181
|
+
const belowThresholdAction = options.belowThresholdAction ?? "verify";
|
|
182
|
+
return {
|
|
183
|
+
id: options.id ?? `confidence>=${threshold}`,
|
|
184
|
+
decide(point) {
|
|
185
|
+
const confidence = point.confidence ?? 0;
|
|
186
|
+
return {
|
|
187
|
+
action: confidence >= threshold ? "accept" : belowThresholdAction,
|
|
188
|
+
confidence,
|
|
189
|
+
targetProb: point.targetProb,
|
|
190
|
+
qHatChosen: point.qHatChosen,
|
|
191
|
+
vHatTarget: point.vHatTarget,
|
|
192
|
+
reason: confidence >= threshold ? "confidence threshold passed" : "confidence threshold failed"
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
};
|
|
256
196
|
}
|
|
257
197
|
function evaluateBeliefSelectivePolicy(points, policy, options = {}) {
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
reasons
|
|
311
|
-
};
|
|
198
|
+
const utility = {
|
|
199
|
+
...DEFAULT_UTILITY,
|
|
200
|
+
...options.utility ?? {}
|
|
201
|
+
};
|
|
202
|
+
const scored = points.filter((point) => point.outcome);
|
|
203
|
+
const minN = options.minN ?? 30;
|
|
204
|
+
const minAccepted = options.minAccepted ?? 5;
|
|
205
|
+
const minUtilityDelta = options.minUtilityDelta ?? 0;
|
|
206
|
+
const deltas = [];
|
|
207
|
+
const acceptedRewards = [];
|
|
208
|
+
const rejectedRewards = [];
|
|
209
|
+
let baselineUtility = 0;
|
|
210
|
+
let policyUtility = 0;
|
|
211
|
+
let accepted = 0;
|
|
212
|
+
let acceptedErrors = 0;
|
|
213
|
+
for (const point of scored) {
|
|
214
|
+
const baseline = acceptUtility(point, utility);
|
|
215
|
+
const decision = policy.decide(point);
|
|
216
|
+
const candidate = policyDecisionUtility(point, decision.action, utility);
|
|
217
|
+
const reward = rewardOf(point, utility);
|
|
218
|
+
baselineUtility += baseline;
|
|
219
|
+
policyUtility += candidate;
|
|
220
|
+
deltas.push(candidate - baseline);
|
|
221
|
+
if (decision.action === "accept") {
|
|
222
|
+
accepted++;
|
|
223
|
+
acceptedRewards.push(reward);
|
|
224
|
+
if (reward < 0) acceptedErrors++;
|
|
225
|
+
} else rejectedRewards.push(reward);
|
|
226
|
+
}
|
|
227
|
+
const n = scored.length;
|
|
228
|
+
const rejected = Math.max(0, n - accepted);
|
|
229
|
+
const ci = confidenceInterval(deltas, .95, { seed: options.seed ?? 17 });
|
|
230
|
+
const reasons = [];
|
|
231
|
+
if (n < minN) reasons.push(`need at least ${minN} scored decisions, got ${n}`);
|
|
232
|
+
if (accepted < minAccepted) reasons.push(`need at least ${minAccepted} accepted decisions, got ${accepted}`);
|
|
233
|
+
if (ci.lower <= minUtilityDelta) reasons.push(`utility CI lower bound ${ci.lower.toFixed(4)} does not clear ${minUtilityDelta}`);
|
|
234
|
+
const recommendation = n < minN || accepted < minAccepted ? "need_more_data" : ci.lower > minUtilityDelta ? "ship" : "hold";
|
|
235
|
+
return {
|
|
236
|
+
policyId: policy.id,
|
|
237
|
+
n,
|
|
238
|
+
accepted,
|
|
239
|
+
rejected,
|
|
240
|
+
coverage: n > 0 ? accepted / n : 0,
|
|
241
|
+
acceptedErrorRate: accepted > 0 ? acceptedErrors / accepted : 0,
|
|
242
|
+
baselineUtility,
|
|
243
|
+
policyUtility,
|
|
244
|
+
utilityDelta: policyUtility - baselineUtility,
|
|
245
|
+
utilityCi95: ci,
|
|
246
|
+
rejectedMeanReward: rejectedRewards.length > 0 ? mean$2(rejectedRewards) : null,
|
|
247
|
+
recommendation,
|
|
248
|
+
reasons
|
|
249
|
+
};
|
|
312
250
|
}
|
|
313
251
|
function acceptUtility(point, utility) {
|
|
314
|
-
|
|
252
|
+
return rewardOf(point, utility) - utility.costWeight * (point.costUsd ?? point.outcome?.costUsd ?? 0);
|
|
315
253
|
}
|
|
316
254
|
function policyDecisionUtility(point, action, utility) {
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
}
|
|
324
|
-
function
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
}
|
|
333
|
-
function mean(values) {
|
|
334
|
-
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
|
|
255
|
+
if (action === "accept") return acceptUtility(point, utility);
|
|
256
|
+
if (action === "verify") return utility.deferUtility - utility.verifyCost;
|
|
257
|
+
if (action === "ask") return utility.deferUtility - utility.askCost;
|
|
258
|
+
if (action === "retry") return utility.deferUtility - utility.retryCost;
|
|
259
|
+
if (action === "stop") return utility.stopUtility;
|
|
260
|
+
return utility.deferUtility;
|
|
261
|
+
}
|
|
262
|
+
function rewardOf(point, utility) {
|
|
263
|
+
const outcome = point.outcome;
|
|
264
|
+
if (!outcome) return utility.failureUtility;
|
|
265
|
+
if (typeof outcome.reward === "number") return 2 * outcome.reward - 1;
|
|
266
|
+
if (typeof outcome.score === "number") return 2 * outcome.score - 1;
|
|
267
|
+
if (outcome.success === true) return utility.successUtility;
|
|
268
|
+
if (outcome.success === false) return utility.failureUtility;
|
|
269
|
+
return utility.failureUtility;
|
|
270
|
+
}
|
|
271
|
+
function mean$2(values) {
|
|
272
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
273
|
+
}
|
|
274
|
+
//#endregion
|
|
275
|
+
//#region src/belief-state/report.ts
|
|
338
276
|
function analyzeBeliefPolicy(options) {
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
};
|
|
277
|
+
const selective = evaluateBeliefSelectivePolicy(options.points, options.policy, options.selective);
|
|
278
|
+
const calibration = calibrateBeliefDecisions(options.points, options.calibration);
|
|
279
|
+
const opeTargetPolicy = options.ope?.targetPolicy;
|
|
280
|
+
const ope = opeTargetPolicy ? evaluateBeliefOffPolicy(options.points, opeTargetPolicy, options.ope) : null;
|
|
281
|
+
const diagnostics = [];
|
|
282
|
+
const selectiveStatus = selective.recommendation;
|
|
283
|
+
const calibrationStatus = calibration ? "supported" : "unsupported";
|
|
284
|
+
const opeRequested = options.requireOpe === true || options.ope !== void 0;
|
|
285
|
+
const opeStatus = ope ? ope.support.supported ? "supported" : "unsupported" : opeRequested ? "unsupported" : "not_requested";
|
|
286
|
+
if (!calibration) diagnostics.push("calibration unsupported: not enough confidence/outcome pairs");
|
|
287
|
+
if (opeRequested && !opeTargetPolicy) diagnostics.push("OPE unsupported: missing target policy");
|
|
288
|
+
else if (ope && !ope.support.supported) diagnostics.push(...ope.support.reasons.map((reason) => `OPE unsupported: ${reason}`));
|
|
289
|
+
const status = overallStatus({
|
|
290
|
+
selectiveStatus,
|
|
291
|
+
hasCalibration: calibration !== null,
|
|
292
|
+
opeStatus,
|
|
293
|
+
opeRequested
|
|
294
|
+
});
|
|
295
|
+
return {
|
|
296
|
+
policyId: options.policy.id,
|
|
297
|
+
n: options.points.length,
|
|
298
|
+
status,
|
|
299
|
+
selectiveStatus,
|
|
300
|
+
calibrationStatus,
|
|
301
|
+
opeStatus,
|
|
302
|
+
...ope ? { opeTargetPolicyId: ope.targetPolicyId } : {},
|
|
303
|
+
selective,
|
|
304
|
+
...calibration ? { calibration } : {},
|
|
305
|
+
...ope ? { ope } : {},
|
|
306
|
+
diagnostics
|
|
307
|
+
};
|
|
371
308
|
}
|
|
372
309
|
function overallStatus(options) {
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
310
|
+
if (options.selectiveStatus === "need_more_data" || !options.hasCalibration) return "need_more_data";
|
|
311
|
+
if (options.selectiveStatus === "hold") return "hold";
|
|
312
|
+
if (options.opeRequested && options.opeStatus !== "supported") return "hold";
|
|
313
|
+
return "ship";
|
|
314
|
+
}
|
|
315
|
+
//#endregion
|
|
316
|
+
//#region src/belief-state/code-agent-corpus.ts
|
|
317
|
+
const FAILURE_RECOVERY_ACTIONS = [
|
|
318
|
+
"retry",
|
|
319
|
+
"verify",
|
|
320
|
+
"continue",
|
|
321
|
+
"stop"
|
|
322
|
+
];
|
|
323
|
+
const TARGET_LABELS = {
|
|
324
|
+
"failure-recovery": "Failure recovery after tool or patch failure",
|
|
325
|
+
"tool-selection": "Tool/action selection",
|
|
326
|
+
"graph-completion": "Graph completion decision"
|
|
387
327
|
};
|
|
388
328
|
function extractCodeAgentBeliefDecisionPoints(options) {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
severity: "info",
|
|
420
|
-
reason: `no belief decision points extracted from ${options.source} entries`
|
|
421
|
-
});
|
|
422
|
-
}
|
|
423
|
-
return { decisions, diagnostics };
|
|
329
|
+
const entries = options.entries.filter(isRecord$2);
|
|
330
|
+
const diagnostics = [];
|
|
331
|
+
const observed = observedActionsFor(options.source, entries, options);
|
|
332
|
+
const decisions = [];
|
|
333
|
+
for (const action of observed) {
|
|
334
|
+
if (action.kind === "tool" || action.kind === "patch") decisions.push(toolSelectionDecision(action, options));
|
|
335
|
+
if (action.kind === "graph-completion") decisions.push(graphCompletionDecision(action, options));
|
|
336
|
+
}
|
|
337
|
+
for (const failed of observed) {
|
|
338
|
+
if (failed.kind !== "tool" && failed.kind !== "patch" || failed.success !== false) continue;
|
|
339
|
+
const next = observed.find((candidate) => candidate.stepIndex > failed.stepIndex && (candidate.kind === "tool" || candidate.kind === "patch" || candidate.kind === "terminal"));
|
|
340
|
+
if (!next) {
|
|
341
|
+
diagnostics.push({
|
|
342
|
+
runId: options.run.runId,
|
|
343
|
+
severity: "warning",
|
|
344
|
+
reason: `${failed.id}: failed action has no observable follow-up decision`
|
|
345
|
+
});
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
decisions.push(failureRecoveryDecision(failed, next, options));
|
|
349
|
+
}
|
|
350
|
+
if (decisions.length === 0) diagnostics.push({
|
|
351
|
+
runId: options.run.runId,
|
|
352
|
+
severity: "info",
|
|
353
|
+
reason: `no belief decision points extracted from ${options.source} entries`
|
|
354
|
+
});
|
|
355
|
+
return {
|
|
356
|
+
decisions,
|
|
357
|
+
diagnostics
|
|
358
|
+
};
|
|
424
359
|
}
|
|
425
360
|
function inventoryBeliefDecisionPoints(points) {
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
return { n: points.length, byKind, byTarget, diagnostics };
|
|
361
|
+
const byKind = [...groupBy(points, (point) => point.kind).entries()].map(([kind, bucketPoints]) => bucketFor(kind, bucketPoints, { kind })).sort(sortBuckets);
|
|
362
|
+
const byTarget = [...groupBy(points, targetIdOf).entries()].filter((entry) => {
|
|
363
|
+
return entry[0] !== void 0;
|
|
364
|
+
}).map(([targetId, bucketPoints]) => bucketFor(targetId, bucketPoints, { targetId })).sort(sortBuckets);
|
|
365
|
+
const diagnostics = [];
|
|
366
|
+
if (points.length === 0) diagnostics.push("no decision points available");
|
|
367
|
+
for (const bucket of byTarget) {
|
|
368
|
+
if (bucket.withOutcome < bucket.n) diagnostics.push(`${bucket.id}: ${bucket.n - bucket.withOutcome} decision(s) missing outcome`);
|
|
369
|
+
if (bucket.withBehaviorProb < bucket.n || bucket.withTargetProb < bucket.n) diagnostics.push(`${bucket.id}: OPE support incomplete`);
|
|
370
|
+
}
|
|
371
|
+
return {
|
|
372
|
+
n: points.length,
|
|
373
|
+
byKind,
|
|
374
|
+
byTarget,
|
|
375
|
+
diagnostics
|
|
376
|
+
};
|
|
443
377
|
}
|
|
444
378
|
function selectBeliefDecisionTarget(points, options = {}) {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
reasons
|
|
472
|
-
};
|
|
473
|
-
}
|
|
474
|
-
return null;
|
|
379
|
+
const minN = options.minN ?? 10;
|
|
380
|
+
const minOutcomeCoverage = options.minOutcomeCoverage ?? .8;
|
|
381
|
+
const preferredTargets = options.preferredTargets ?? [
|
|
382
|
+
"failure-recovery",
|
|
383
|
+
"tool-selection",
|
|
384
|
+
"graph-completion"
|
|
385
|
+
];
|
|
386
|
+
const inventory = inventoryBeliefDecisionPoints(points);
|
|
387
|
+
for (const targetId of preferredTargets) {
|
|
388
|
+
const support = inventory.byTarget.find((bucket) => bucket.targetId === targetId);
|
|
389
|
+
if (!support) continue;
|
|
390
|
+
const reasons = [];
|
|
391
|
+
if (support.n < minN) reasons.push(`need at least ${minN} decisions, got ${support.n}`);
|
|
392
|
+
const outcomeCoverage = support.n > 0 ? support.withOutcome / support.n : 0;
|
|
393
|
+
if (outcomeCoverage < minOutcomeCoverage) reasons.push(`outcome coverage ${outcomeCoverage.toFixed(2)} below ${minOutcomeCoverage.toFixed(2)}`);
|
|
394
|
+
if (reasons.length > 0) continue;
|
|
395
|
+
const targetPoints = points.filter((point) => targetIdOf(point) === targetId);
|
|
396
|
+
return {
|
|
397
|
+
id: targetId,
|
|
398
|
+
label: TARGET_LABELS[targetId],
|
|
399
|
+
points: targetPoints,
|
|
400
|
+
support,
|
|
401
|
+
reasons
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
return null;
|
|
475
405
|
}
|
|
476
406
|
function analyzeBeliefDecisionCorpus(options) {
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
407
|
+
const inventory = inventoryBeliefDecisionPoints(options.points);
|
|
408
|
+
const diagnostics = [...inventory.diagnostics];
|
|
409
|
+
const target = options.targetId !== void 0 ? targetSelectionFor(options.points, options.targetId, options) : selectBeliefDecisionTarget(options.points, options);
|
|
410
|
+
if (!target) {
|
|
411
|
+
diagnostics.push("no decision target has enough support for policy evaluation");
|
|
412
|
+
return {
|
|
413
|
+
inventory,
|
|
414
|
+
diagnostics
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
const policy = options.policy ?? thresholdSelectivePolicy({
|
|
418
|
+
id: `${target.id}:confidence>=${options.confidenceThreshold ?? .5}`,
|
|
419
|
+
confidenceThreshold: options.confidenceThreshold ?? .5,
|
|
420
|
+
belowThresholdAction: "verify"
|
|
421
|
+
});
|
|
422
|
+
const minN = options.minN ?? 10;
|
|
423
|
+
return {
|
|
424
|
+
inventory,
|
|
425
|
+
target,
|
|
426
|
+
policy,
|
|
427
|
+
evaluation: analyzeBeliefPolicy({
|
|
428
|
+
points: target.points,
|
|
429
|
+
policy,
|
|
430
|
+
selective: {
|
|
431
|
+
minN,
|
|
432
|
+
minAccepted: options.minAccepted ?? Math.min(5, minN),
|
|
433
|
+
minUtilityDelta: 0,
|
|
434
|
+
...options.policyOptions?.selective ?? {}
|
|
435
|
+
},
|
|
436
|
+
calibration: {
|
|
437
|
+
minPairs: Math.min(10, minN),
|
|
438
|
+
policy,
|
|
439
|
+
region: "all",
|
|
440
|
+
...options.policyOptions?.calibration ?? {}
|
|
441
|
+
},
|
|
442
|
+
ope: {
|
|
443
|
+
targetPolicy: embeddedBeliefOpeTargetPolicy(`${target.id}:embedded-target-prob`),
|
|
444
|
+
minEffectiveSampleSize: minN,
|
|
445
|
+
...options.policyOptions?.ope ?? {}
|
|
446
|
+
},
|
|
447
|
+
requireOpe: options.requireOpe ?? true
|
|
448
|
+
}),
|
|
449
|
+
diagnostics
|
|
450
|
+
};
|
|
513
451
|
}
|
|
514
452
|
function observedActionsFor(source, entries, options) {
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
453
|
+
const observation = options.observation ?? observeCodeAgentSession({
|
|
454
|
+
source,
|
|
455
|
+
entries,
|
|
456
|
+
sourcePath: options.sourcePath
|
|
457
|
+
});
|
|
458
|
+
if (observation.source !== source) throw new Error("code-agent observation source does not match extraction source");
|
|
459
|
+
return observation.actions.map((action) => observedActionFromSession(action, options));
|
|
520
460
|
}
|
|
521
461
|
function observedActionFromSession(action, options) {
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
462
|
+
return observedAction({
|
|
463
|
+
options,
|
|
464
|
+
localId: action.id,
|
|
465
|
+
stepIndex: action.stepIndex,
|
|
466
|
+
kind: action.kind,
|
|
467
|
+
action: action.name,
|
|
468
|
+
timestamp: action.timestampMs,
|
|
469
|
+
success: action.status === "completed" ? true : action.status === "failed" ? false : void 0,
|
|
470
|
+
costUsd: action.costUsd,
|
|
471
|
+
metadata: {
|
|
472
|
+
surface: action.surface,
|
|
473
|
+
status: action.status,
|
|
474
|
+
...action.metadata
|
|
475
|
+
}
|
|
476
|
+
});
|
|
533
477
|
}
|
|
534
478
|
function toolSelectionDecision(action, options) {
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
479
|
+
return {
|
|
480
|
+
id: `${options.run.runId}:tool-selection:${action.localId}`,
|
|
481
|
+
runId: options.run.runId,
|
|
482
|
+
scenarioId: options.run.scenarioId,
|
|
483
|
+
stepIndex: action.stepIndex,
|
|
484
|
+
kind: "tool-select",
|
|
485
|
+
chosenAction: action.action,
|
|
486
|
+
candidateActions: [action.action],
|
|
487
|
+
confidence: .65,
|
|
488
|
+
costUsd: action.costUsd,
|
|
489
|
+
evidence: action.evidence,
|
|
490
|
+
outcome: outcomeFromAction(action, options.run),
|
|
491
|
+
metadata: {
|
|
492
|
+
target: "tool-selection",
|
|
493
|
+
source: options.source,
|
|
494
|
+
actionKind: action.kind,
|
|
495
|
+
confidenceSource: "fixed-observed-action-prior",
|
|
496
|
+
...action.metadata
|
|
497
|
+
}
|
|
498
|
+
};
|
|
555
499
|
}
|
|
556
500
|
function graphCompletionDecision(action, options) {
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
501
|
+
return {
|
|
502
|
+
id: `${options.run.runId}:graph-completion:${action.localId}`,
|
|
503
|
+
runId: options.run.runId,
|
|
504
|
+
scenarioId: options.run.scenarioId,
|
|
505
|
+
stepIndex: action.stepIndex,
|
|
506
|
+
kind: "stop",
|
|
507
|
+
chosenAction: "complete",
|
|
508
|
+
candidateActions: [
|
|
509
|
+
"complete",
|
|
510
|
+
"continue",
|
|
511
|
+
"verify"
|
|
512
|
+
],
|
|
513
|
+
confidence: .75,
|
|
514
|
+
evidence: action.evidence,
|
|
515
|
+
outcome: outcomeFromAction(action, options.run),
|
|
516
|
+
metadata: {
|
|
517
|
+
target: "graph-completion",
|
|
518
|
+
source: options.source,
|
|
519
|
+
confidenceSource: "fixed-graph-completion-prior",
|
|
520
|
+
...action.metadata
|
|
521
|
+
}
|
|
522
|
+
};
|
|
575
523
|
}
|
|
576
524
|
function failureRecoveryDecision(failed, next, options) {
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
525
|
+
const chosenAction = classifyFailureRecovery(failed, next);
|
|
526
|
+
return {
|
|
527
|
+
id: `${options.run.runId}:failure-recovery:${failed.localId}`,
|
|
528
|
+
runId: options.run.runId,
|
|
529
|
+
scenarioId: options.run.scenarioId,
|
|
530
|
+
stepIndex: failed.stepIndex,
|
|
531
|
+
kind: "retry",
|
|
532
|
+
chosenAction,
|
|
533
|
+
candidateActions: [...FAILURE_RECOVERY_ACTIONS],
|
|
534
|
+
confidence: recoveryConfidence(chosenAction),
|
|
535
|
+
evidence: [...failed.evidence, ...next.evidence],
|
|
536
|
+
outcome: outcomeFromAction(next, options.run),
|
|
537
|
+
metadata: {
|
|
538
|
+
target: "failure-recovery",
|
|
539
|
+
source: options.source,
|
|
540
|
+
failedActionKind: failed.kind,
|
|
541
|
+
failedAction: failed.action,
|
|
542
|
+
nextActionKind: next.kind,
|
|
543
|
+
nextAction: next.action,
|
|
544
|
+
confidenceSource: "heuristic-observed-follow-up"
|
|
545
|
+
}
|
|
546
|
+
};
|
|
599
547
|
}
|
|
600
548
|
function classifyFailureRecovery(failed, next) {
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
549
|
+
if (next.kind === "terminal") return "stop";
|
|
550
|
+
if (isVerificationAction(next.action)) return "verify";
|
|
551
|
+
if (next.kind === failed.kind && next.action === failed.action) return "retry";
|
|
552
|
+
return "continue";
|
|
605
553
|
}
|
|
606
554
|
function recoveryConfidence(action) {
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
555
|
+
if (action === "verify") return .8;
|
|
556
|
+
if (action === "retry") return .6;
|
|
557
|
+
if (action === "stop") return .55;
|
|
558
|
+
return .35;
|
|
611
559
|
}
|
|
612
560
|
function isVerificationAction(action) {
|
|
613
|
-
|
|
614
|
-
|
|
561
|
+
const normalized = action.toLowerCase();
|
|
562
|
+
return normalized.includes("verify") || normalized.includes("test") || normalized.includes("check") || normalized.includes("lint") || normalized.includes("build") || normalized.includes("typecheck") || normalized.includes("pytest") || normalized.includes("vitest") || normalized.includes("tsc");
|
|
615
563
|
}
|
|
616
564
|
function outcomeFromAction(action, run) {
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
565
|
+
const runScore = scoreFromRun(run);
|
|
566
|
+
const success = action.success ?? (runScore !== null ? runScore >= .5 : void 0);
|
|
567
|
+
const score = action.success === void 0 ? runScore ?? void 0 : action.success ? 1 : 0;
|
|
568
|
+
if (success === void 0 && score === void 0) return void 0;
|
|
569
|
+
return {
|
|
570
|
+
...success !== void 0 ? { success } : {},
|
|
571
|
+
...score !== void 0 ? {
|
|
572
|
+
score,
|
|
573
|
+
reward: score
|
|
574
|
+
} : {},
|
|
575
|
+
...action.costUsd !== void 0 ? { costUsd: action.costUsd } : {},
|
|
576
|
+
metadata: { outcomeSource: action.success === void 0 ? "run-score" : "observed-action-status" }
|
|
577
|
+
};
|
|
629
578
|
}
|
|
630
579
|
function observedAction(input) {
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
metadata: input.metadata ?? {}
|
|
655
|
-
};
|
|
580
|
+
const id = `${input.options.run.runId}:${input.options.source}:${input.localId}`;
|
|
581
|
+
return {
|
|
582
|
+
id,
|
|
583
|
+
localId: input.localId,
|
|
584
|
+
stepIndex: input.stepIndex,
|
|
585
|
+
kind: input.kind,
|
|
586
|
+
action: input.action,
|
|
587
|
+
timestamp: input.timestamp,
|
|
588
|
+
success: input.success,
|
|
589
|
+
costUsd: input.costUsd,
|
|
590
|
+
evidence: [{
|
|
591
|
+
source: "event",
|
|
592
|
+
id,
|
|
593
|
+
runId: input.options.run.runId,
|
|
594
|
+
detail: input.action,
|
|
595
|
+
metadata: {
|
|
596
|
+
source: input.options.source,
|
|
597
|
+
sourcePath: input.options.sourcePath,
|
|
598
|
+
...input.metadata
|
|
599
|
+
}
|
|
600
|
+
}],
|
|
601
|
+
metadata: input.metadata ?? {}
|
|
602
|
+
};
|
|
656
603
|
}
|
|
657
604
|
function targetSelectionFor(points, targetId, options) {
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
605
|
+
const targetPoints = points.filter((point) => targetIdOf(point) === targetId);
|
|
606
|
+
if (targetPoints.length === 0) return null;
|
|
607
|
+
const support = bucketFor(targetId, targetPoints, { targetId });
|
|
608
|
+
const minN = options.minN ?? 10;
|
|
609
|
+
const minOutcomeCoverage = options.minOutcomeCoverage ?? .8;
|
|
610
|
+
const reasons = [];
|
|
611
|
+
if (support.n < minN) reasons.push(`need at least ${minN} decisions, got ${support.n}`);
|
|
612
|
+
const outcomeCoverage = support.n > 0 ? support.withOutcome / support.n : 0;
|
|
613
|
+
if (outcomeCoverage < minOutcomeCoverage) reasons.push(`outcome coverage ${outcomeCoverage.toFixed(2)} below ${minOutcomeCoverage.toFixed(2)}`);
|
|
614
|
+
if (reasons.length > 0) return null;
|
|
615
|
+
return {
|
|
616
|
+
id: targetId,
|
|
617
|
+
label: TARGET_LABELS[targetId],
|
|
618
|
+
points: targetPoints,
|
|
619
|
+
support,
|
|
620
|
+
reasons
|
|
621
|
+
};
|
|
673
622
|
}
|
|
674
623
|
function bucketFor(id, points, identity) {
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
meanConfidence: confidences.length > 0 ? mean2(confidences) : null
|
|
694
|
-
};
|
|
624
|
+
const outcomes = points.filter((point) => point.outcome);
|
|
625
|
+
const scores = outcomes.map((point) => outcomeScore(point.outcome)).filter((score) => score !== null);
|
|
626
|
+
const confidences = points.map((point) => point.confidence).filter((confidence) => typeof confidence === "number");
|
|
627
|
+
const successes = outcomes.filter((point) => point.outcome?.success === true).length;
|
|
628
|
+
const successDenominator = outcomes.filter((point) => typeof point.outcome?.success === "boolean").length;
|
|
629
|
+
return {
|
|
630
|
+
id,
|
|
631
|
+
...identity,
|
|
632
|
+
n: points.length,
|
|
633
|
+
withOutcome: outcomes.length,
|
|
634
|
+
withConfidence: confidences.length,
|
|
635
|
+
withCandidateActions: points.filter((point) => (point.candidateActions?.length ?? 0) > 0).length,
|
|
636
|
+
withBehaviorProb: points.filter((point) => point.behaviorProb !== void 0).length,
|
|
637
|
+
withTargetProb: points.filter((point) => point.targetProb !== void 0).length,
|
|
638
|
+
successRate: successDenominator > 0 ? successes / successDenominator : null,
|
|
639
|
+
meanScore: scores.length > 0 ? mean$1(scores) : null,
|
|
640
|
+
meanConfidence: confidences.length > 0 ? mean$1(confidences) : null
|
|
641
|
+
};
|
|
695
642
|
}
|
|
696
643
|
function targetIdOf(point) {
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
644
|
+
const target = point.metadata?.target;
|
|
645
|
+
if (target === "failure-recovery" || target === "tool-selection" || target === "graph-completion") return target;
|
|
646
|
+
}
|
|
647
|
+
function outcomeScore(outcome) {
|
|
648
|
+
if (!outcome) return null;
|
|
649
|
+
if (typeof outcome.score === "number") return outcome.score;
|
|
650
|
+
if (typeof outcome.reward === "number") return outcome.reward;
|
|
651
|
+
if (outcome.success === true) return 1;
|
|
652
|
+
if (outcome.success === false) return 0;
|
|
653
|
+
return null;
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* GATED (`trainingScore`). The number this returns becomes a belief-decision
|
|
657
|
+
* point's `outcome.score` AND its `outcome.reward` — corpus labels, i.e.
|
|
658
|
+
* training data by another name. A run flagged as gamed would otherwise label
|
|
659
|
+
* every decision on its trajectory a success and teach a belief model to
|
|
660
|
+
* predict that the gaming path works.
|
|
661
|
+
*/
|
|
710
662
|
function scoreFromRun(run) {
|
|
711
|
-
|
|
712
|
-
if (typeof run.outcome.searchScore === "number") return run.outcome.searchScore;
|
|
713
|
-
return null;
|
|
663
|
+
return trainingScore(run) ?? null;
|
|
714
664
|
}
|
|
715
665
|
function sortBuckets(a, b) {
|
|
716
|
-
|
|
666
|
+
return b.n - a.n || a.id.localeCompare(b.id);
|
|
717
667
|
}
|
|
718
668
|
function groupBy(values, keyOf) {
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
}
|
|
728
|
-
function
|
|
729
|
-
|
|
730
|
-
}
|
|
731
|
-
function isRecord(value) {
|
|
732
|
-
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
|
|
669
|
+
const map = /* @__PURE__ */ new Map();
|
|
670
|
+
for (const value of values) {
|
|
671
|
+
const key = keyOf(value);
|
|
672
|
+
const bucket = map.get(key);
|
|
673
|
+
if (bucket) bucket.push(value);
|
|
674
|
+
else map.set(key, [value]);
|
|
675
|
+
}
|
|
676
|
+
return map;
|
|
677
|
+
}
|
|
678
|
+
function mean$1(values) {
|
|
679
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
680
|
+
}
|
|
681
|
+
function isRecord$2(value) {
|
|
682
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
683
|
+
}
|
|
684
|
+
//#endregion
|
|
685
|
+
//#region src/belief-state/research-evidence.ts
|
|
736
686
|
function buildBeliefDecisionResearchEvidencePacket(options) {
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
blockers: unique(gates.flatMap((gate) => gate.blockers)),
|
|
759
|
-
caveats
|
|
760
|
-
};
|
|
687
|
+
const claimScope = options.claimScope ?? "counterfactual";
|
|
688
|
+
const requireOpe = claimScope === "counterfactual";
|
|
689
|
+
const analysis = analyzeBeliefDecisionCorpus({
|
|
690
|
+
...options,
|
|
691
|
+
requireOpe: options.requireOpe ?? requireOpe
|
|
692
|
+
});
|
|
693
|
+
const gates = [
|
|
694
|
+
corpusGate(analysis),
|
|
695
|
+
selectiveGate(analysis),
|
|
696
|
+
calibrationGate(analysis),
|
|
697
|
+
...requireOpe ? [opeGate(analysis)] : []
|
|
698
|
+
];
|
|
699
|
+
const caveats = unique([...gates.flatMap((gate) => gate.caveats), ...claimScope === "selective" ? ["counterfactual claims excluded: OPE support was not required"] : []]);
|
|
700
|
+
return {
|
|
701
|
+
claimScope,
|
|
702
|
+
status: gates.every((gate) => gate.status === "supported") ? "supported" : "blocked",
|
|
703
|
+
analysis,
|
|
704
|
+
gates,
|
|
705
|
+
blockers: unique(gates.flatMap((gate) => gate.blockers)),
|
|
706
|
+
caveats
|
|
707
|
+
};
|
|
761
708
|
}
|
|
762
709
|
function corpusGate(analysis) {
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
710
|
+
const support = analysis.target?.support;
|
|
711
|
+
if (!support) return blocked("corpus", "no decision target has enough outcome support");
|
|
712
|
+
return {
|
|
713
|
+
id: "corpus",
|
|
714
|
+
status: "supported",
|
|
715
|
+
blockers: [],
|
|
716
|
+
caveats: support.withBehaviorProb < support.n || support.withTargetProb < support.n ? ["propensity support incomplete; counterfactual claims will require OPE support"] : []
|
|
717
|
+
};
|
|
769
718
|
}
|
|
770
719
|
function selectiveGate(analysis) {
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
);
|
|
781
|
-
}
|
|
782
|
-
return { id: "selective", status: "supported", blockers: [], caveats: [] };
|
|
720
|
+
const evaluation = analysis.evaluation;
|
|
721
|
+
if (!evaluation) return blocked("selective", "no policy evaluation was produced");
|
|
722
|
+
if (evaluation.selectiveStatus !== "ship") return blocked("selective", ...orDefault(evaluation.selective.reasons, `selective status is ${evaluation.selectiveStatus}`));
|
|
723
|
+
return {
|
|
724
|
+
id: "selective",
|
|
725
|
+
status: "supported",
|
|
726
|
+
blockers: [],
|
|
727
|
+
caveats: []
|
|
728
|
+
};
|
|
783
729
|
}
|
|
784
730
|
function calibrationGate(analysis) {
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
731
|
+
const evaluation = analysis.evaluation;
|
|
732
|
+
if (!evaluation) return blocked("calibration", "no policy evaluation was produced");
|
|
733
|
+
if (evaluation.calibrationStatus !== "supported") return blocked("calibration", "not enough confidence/outcome pairs for calibration");
|
|
734
|
+
return {
|
|
735
|
+
id: "calibration",
|
|
736
|
+
status: "supported",
|
|
737
|
+
blockers: [],
|
|
738
|
+
caveats: []
|
|
739
|
+
};
|
|
791
740
|
}
|
|
792
741
|
function opeGate(analysis) {
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
742
|
+
const evaluation = analysis.evaluation;
|
|
743
|
+
if (!evaluation) return blocked("ope", "no policy evaluation was produced");
|
|
744
|
+
if (evaluation.opeStatus !== "supported") return blocked("ope", ...orDefault(evaluation.ope?.support.reasons ?? evaluation.diagnostics.filter((diagnostic) => diagnostic.includes("OPE")), "missing OPE support"));
|
|
745
|
+
return {
|
|
746
|
+
id: "ope",
|
|
747
|
+
status: "supported",
|
|
748
|
+
blockers: [],
|
|
749
|
+
caveats: []
|
|
750
|
+
};
|
|
800
751
|
}
|
|
801
752
|
function blocked(id, ...blockers) {
|
|
802
|
-
|
|
753
|
+
return {
|
|
754
|
+
id,
|
|
755
|
+
status: "blocked",
|
|
756
|
+
blockers,
|
|
757
|
+
caveats: []
|
|
758
|
+
};
|
|
803
759
|
}
|
|
804
760
|
function orDefault(values, fallback) {
|
|
805
|
-
|
|
761
|
+
return values.length > 0 ? values : [fallback];
|
|
806
762
|
}
|
|
807
763
|
function unique(values) {
|
|
808
|
-
|
|
764
|
+
return [...new Set(values)];
|
|
809
765
|
}
|
|
810
|
-
|
|
811
|
-
|
|
766
|
+
//#endregion
|
|
767
|
+
//#region src/belief-state/code-agent-evidence.ts
|
|
812
768
|
function buildCodeAgentBeliefEvidenceCorpus(options) {
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
769
|
+
const { sessions, ...evidenceOptions } = options;
|
|
770
|
+
const runs = [];
|
|
771
|
+
const metrics = [];
|
|
772
|
+
const intakeDiagnostics = [];
|
|
773
|
+
const extractionDiagnostics = [];
|
|
774
|
+
const decisions = [];
|
|
775
|
+
for (const session of sessions) {
|
|
776
|
+
const intake = fromCodeAgentBeliefSession(session);
|
|
777
|
+
runs.push(...intake.runs);
|
|
778
|
+
metrics.push(...intake.metrics);
|
|
779
|
+
intakeDiagnostics.push(...intake.diagnostics);
|
|
780
|
+
for (const [index, run] of intake.runs.entries()) {
|
|
781
|
+
const extraction = extractCodeAgentBeliefDecisionPoints({
|
|
782
|
+
source: session.source,
|
|
783
|
+
entries: session.entries,
|
|
784
|
+
observation: intake.observations[index],
|
|
785
|
+
run,
|
|
786
|
+
sourcePath: session.sourcePath
|
|
787
|
+
});
|
|
788
|
+
decisions.push(...extraction.decisions);
|
|
789
|
+
extractionDiagnostics.push(...extraction.diagnostics);
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
const evidence = buildBeliefDecisionResearchEvidencePacket({
|
|
793
|
+
...evidenceOptions,
|
|
794
|
+
points: decisions
|
|
795
|
+
});
|
|
796
|
+
return {
|
|
797
|
+
runs,
|
|
798
|
+
metrics,
|
|
799
|
+
intakeDiagnostics,
|
|
800
|
+
extractionDiagnostics,
|
|
801
|
+
decisions,
|
|
802
|
+
inventory: inventoryBeliefDecisionPoints(decisions),
|
|
803
|
+
evidence
|
|
804
|
+
};
|
|
849
805
|
}
|
|
850
806
|
function fromCodeAgentBeliefSession(session) {
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
"memory-read",
|
|
874
|
-
"tool-select",
|
|
875
|
-
"skill-select",
|
|
876
|
-
"workflow-select",
|
|
877
|
-
"surface-promote"
|
|
807
|
+
switch (session.source) {
|
|
808
|
+
case "codex": return fromCodexSession(session);
|
|
809
|
+
case "claude-code": return fromClaudeCodeSession(session);
|
|
810
|
+
case "opencode": return fromOpenCodeSession(session);
|
|
811
|
+
case "kimi-code": return fromKimiCodeSession(session);
|
|
812
|
+
case "pi": return fromPiSession(session);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
//#endregion
|
|
816
|
+
//#region src/belief-state/types.ts
|
|
817
|
+
const BELIEF_DECISION_KINDS = [
|
|
818
|
+
"continue",
|
|
819
|
+
"verify",
|
|
820
|
+
"ask",
|
|
821
|
+
"retry",
|
|
822
|
+
"stop",
|
|
823
|
+
"memory-write",
|
|
824
|
+
"memory-read",
|
|
825
|
+
"tool-select",
|
|
826
|
+
"skill-select",
|
|
827
|
+
"workflow-select",
|
|
828
|
+
"surface-promote"
|
|
878
829
|
];
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
830
|
+
const BELIEF_EVIDENCE_SOURCES = [
|
|
831
|
+
"run",
|
|
832
|
+
"span",
|
|
833
|
+
"event",
|
|
834
|
+
"finding",
|
|
835
|
+
"memory",
|
|
836
|
+
"knowledge",
|
|
837
|
+
"policy"
|
|
887
838
|
];
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
839
|
+
const BELIEF_EVIDENCE_QUALITIES = [
|
|
840
|
+
"direct",
|
|
841
|
+
"derived",
|
|
842
|
+
"self-reported",
|
|
843
|
+
"unverified",
|
|
844
|
+
"stale",
|
|
845
|
+
"contradicted"
|
|
895
846
|
];
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
847
|
+
const BELIEF_EVALUATION_CRITERIA = [
|
|
848
|
+
{
|
|
849
|
+
id: "capture-integrity",
|
|
850
|
+
label: "Capture integrity",
|
|
851
|
+
reasonCodes: [
|
|
852
|
+
"trace-missing",
|
|
853
|
+
"run-record-missing",
|
|
854
|
+
"backend-integrity-missing"
|
|
855
|
+
]
|
|
856
|
+
},
|
|
857
|
+
{
|
|
858
|
+
id: "decision-completeness",
|
|
859
|
+
label: "Decision completeness",
|
|
860
|
+
reasonCodes: [
|
|
861
|
+
"candidate-actions-missing",
|
|
862
|
+
"chosen-action-missing",
|
|
863
|
+
"decision-evidence-missing"
|
|
864
|
+
]
|
|
865
|
+
},
|
|
866
|
+
{
|
|
867
|
+
id: "evidence-quality",
|
|
868
|
+
label: "Evidence quality",
|
|
869
|
+
reasonCodes: [
|
|
870
|
+
"evidence-stale",
|
|
871
|
+
"evidence-contradictory",
|
|
872
|
+
"evidence-unverified",
|
|
873
|
+
"evidence-self-reported"
|
|
874
|
+
]
|
|
875
|
+
},
|
|
876
|
+
{
|
|
877
|
+
id: "outcome-quality",
|
|
878
|
+
label: "Outcome quality",
|
|
879
|
+
reasonCodes: [
|
|
880
|
+
"outcome-missing",
|
|
881
|
+
"outcome-delayed",
|
|
882
|
+
"cost-missing"
|
|
883
|
+
]
|
|
884
|
+
},
|
|
885
|
+
{
|
|
886
|
+
id: "calibration",
|
|
887
|
+
label: "Calibration",
|
|
888
|
+
reasonCodes: [
|
|
889
|
+
"confidence-missing",
|
|
890
|
+
"calibration-unsupported",
|
|
891
|
+
"calibration-gap-high"
|
|
892
|
+
]
|
|
893
|
+
},
|
|
894
|
+
{
|
|
895
|
+
id: "accepted-region-risk",
|
|
896
|
+
label: "Accepted-region risk",
|
|
897
|
+
reasonCodes: ["accepted-error-high", "coverage-too-low"]
|
|
898
|
+
},
|
|
899
|
+
{
|
|
900
|
+
id: "policy-value",
|
|
901
|
+
label: "Policy value",
|
|
902
|
+
reasonCodes: [
|
|
903
|
+
"utility-lift-missing",
|
|
904
|
+
"baseline-dominates",
|
|
905
|
+
"cost-too-high"
|
|
906
|
+
]
|
|
907
|
+
},
|
|
908
|
+
{
|
|
909
|
+
id: "ope-support",
|
|
910
|
+
label: "OPE support",
|
|
911
|
+
reasonCodes: [
|
|
912
|
+
"behavior-propensity-missing",
|
|
913
|
+
"behavior-propensity-invalid",
|
|
914
|
+
"target-propensity-missing",
|
|
915
|
+
"target-propensity-invalid",
|
|
916
|
+
"effective-sample-size-low",
|
|
917
|
+
"importance-weight-high"
|
|
918
|
+
]
|
|
919
|
+
},
|
|
920
|
+
{
|
|
921
|
+
id: "memory-health",
|
|
922
|
+
label: "Memory health",
|
|
923
|
+
reasonCodes: [
|
|
924
|
+
"memory-stale",
|
|
925
|
+
"memory-poisoning-risk",
|
|
926
|
+
"context-bloat",
|
|
927
|
+
"memory-write-unverified"
|
|
928
|
+
]
|
|
929
|
+
},
|
|
930
|
+
{
|
|
931
|
+
id: "surface-attribution",
|
|
932
|
+
label: "Surface attribution",
|
|
933
|
+
reasonCodes: ["surface-claim-unsupported", "causal-attribution-missing"]
|
|
934
|
+
},
|
|
935
|
+
{
|
|
936
|
+
id: "generalization",
|
|
937
|
+
label: "Generalization",
|
|
938
|
+
reasonCodes: [
|
|
939
|
+
"split-missing",
|
|
940
|
+
"holdout-regression",
|
|
941
|
+
"task-family-coverage-low",
|
|
942
|
+
"leakage-risk"
|
|
943
|
+
]
|
|
944
|
+
},
|
|
945
|
+
{
|
|
946
|
+
id: "promotion",
|
|
947
|
+
label: "Promotion",
|
|
948
|
+
reasonCodes: [
|
|
949
|
+
"negative-control-failed",
|
|
950
|
+
"promotion-gate-failed",
|
|
951
|
+
"human-review-required"
|
|
952
|
+
]
|
|
953
|
+
}
|
|
983
954
|
];
|
|
984
955
|
function isBeliefDecisionKind(value) {
|
|
985
|
-
|
|
956
|
+
return typeof value === "string" && BELIEF_DECISION_KINDS.includes(value);
|
|
986
957
|
}
|
|
987
958
|
function isBeliefEvidenceSource(value) {
|
|
988
|
-
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
959
|
+
return typeof value === "string" && BELIEF_EVIDENCE_SOURCES.includes(value);
|
|
960
|
+
}
|
|
961
|
+
//#endregion
|
|
962
|
+
//#region src/belief-state/extract.ts
|
|
963
|
+
const DECISION_MARKERS = /* @__PURE__ */ new Set([
|
|
964
|
+
"belief_decision",
|
|
965
|
+
"belief.decision",
|
|
966
|
+
"decision_point"
|
|
967
|
+
]);
|
|
993
968
|
async function extractBeliefDecisionPoints(store, options = {}) {
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
969
|
+
const runs = options.runIds ? (await Promise.all(options.runIds.map((runId) => store.getRun(runId)))).filter(Boolean) : await store.listRuns();
|
|
970
|
+
const decisions = [];
|
|
971
|
+
const diagnostics = [];
|
|
972
|
+
for (const run of runs) {
|
|
973
|
+
if (!run) continue;
|
|
974
|
+
const events = await store.events({ runId: run.runId });
|
|
975
|
+
const spans = await store.spans({ runId: run.runId });
|
|
976
|
+
const spanIds = new Set(spans.map((span) => span.spanId));
|
|
977
|
+
let stepIndex = 0;
|
|
978
|
+
for (const event of [...events].sort((a, b) => a.timestamp - b.timestamp)) {
|
|
979
|
+
const parsed = parseDecisionEvent(event, {
|
|
980
|
+
scenarioId: run.scenarioId,
|
|
981
|
+
stepIndex,
|
|
982
|
+
spanExists: event.spanId ? spanIds.has(event.spanId) : false
|
|
983
|
+
});
|
|
984
|
+
if (!parsed) continue;
|
|
985
|
+
if ("diagnostic" in parsed) {
|
|
986
|
+
diagnostics.push(parsed.diagnostic);
|
|
987
|
+
continue;
|
|
988
|
+
}
|
|
989
|
+
decisions.push(parsed.decision);
|
|
990
|
+
stepIndex++;
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return {
|
|
994
|
+
decisions,
|
|
995
|
+
diagnostics
|
|
996
|
+
};
|
|
1019
997
|
}
|
|
1020
998
|
function parseDecisionEvent(event, context) {
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
kind: decisionKind,
|
|
1071
|
-
chosenAction,
|
|
1072
|
-
candidateActions: stringArrayField(payload, "candidateActions"),
|
|
1073
|
-
confidence: finiteUnitField(payload, "confidence"),
|
|
1074
|
-
behaviorProb: numberField(payload, "behaviorProb"),
|
|
1075
|
-
targetProb: numberField(payload, "targetProb"),
|
|
1076
|
-
qHatChosen: finiteUnitField(payload, "qHatChosen"),
|
|
1077
|
-
vHatTarget: finiteUnitField(payload, "vHatTarget"),
|
|
1078
|
-
qHat: finiteUnitField(payload, "qHat"),
|
|
1079
|
-
costUsd: nonNegativeNumberField(payload, "costUsd"),
|
|
1080
|
-
evidence,
|
|
1081
|
-
outcome: parseOutcome(payload),
|
|
1082
|
-
metadata: recordField(payload, "metadata")
|
|
1083
|
-
}
|
|
1084
|
-
};
|
|
999
|
+
const payload = event.payload;
|
|
1000
|
+
const marker = stringField(payload, "kind") ?? stringField(payload, "type");
|
|
1001
|
+
if (!marker || !DECISION_MARKERS.has(marker)) return null;
|
|
1002
|
+
const decisionKind = stringField(payload, "decisionKind");
|
|
1003
|
+
if (!isBeliefDecisionKind(decisionKind)) return { diagnostic: {
|
|
1004
|
+
runId: event.runId,
|
|
1005
|
+
eventId: event.eventId,
|
|
1006
|
+
severity: "warning",
|
|
1007
|
+
reason: `belief decision event has unsupported decisionKind "${decisionKind ?? ""}"`
|
|
1008
|
+
} };
|
|
1009
|
+
const chosenAction = stringField(payload, "chosenAction");
|
|
1010
|
+
if (!chosenAction) return { diagnostic: {
|
|
1011
|
+
runId: event.runId,
|
|
1012
|
+
eventId: event.eventId,
|
|
1013
|
+
severity: "warning",
|
|
1014
|
+
reason: "belief decision event is missing chosenAction"
|
|
1015
|
+
} };
|
|
1016
|
+
const evidence = [{
|
|
1017
|
+
source: "event",
|
|
1018
|
+
id: event.eventId,
|
|
1019
|
+
runId: event.runId,
|
|
1020
|
+
eventId: event.eventId,
|
|
1021
|
+
quality: "direct"
|
|
1022
|
+
}];
|
|
1023
|
+
if (event.spanId && context.spanExists) evidence.push({
|
|
1024
|
+
source: "span",
|
|
1025
|
+
id: event.spanId,
|
|
1026
|
+
runId: event.runId,
|
|
1027
|
+
spanId: event.spanId,
|
|
1028
|
+
quality: "direct"
|
|
1029
|
+
});
|
|
1030
|
+
return { decision: {
|
|
1031
|
+
id: stringField(payload, "id") ?? event.eventId,
|
|
1032
|
+
runId: event.runId,
|
|
1033
|
+
scenarioId: stringField(payload, "scenarioId") ?? context.scenarioId,
|
|
1034
|
+
stepIndex: numberField(payload, "stepIndex") ?? context.stepIndex,
|
|
1035
|
+
kind: decisionKind,
|
|
1036
|
+
chosenAction,
|
|
1037
|
+
candidateActions: stringArrayField(payload, "candidateActions"),
|
|
1038
|
+
confidence: finiteUnitField(payload, "confidence"),
|
|
1039
|
+
behaviorProb: numberField(payload, "behaviorProb"),
|
|
1040
|
+
targetProb: numberField(payload, "targetProb"),
|
|
1041
|
+
qHatChosen: finiteUnitField(payload, "qHatChosen"),
|
|
1042
|
+
vHatTarget: finiteUnitField(payload, "vHatTarget"),
|
|
1043
|
+
costUsd: nonNegativeNumberField(payload, "costUsd"),
|
|
1044
|
+
evidence,
|
|
1045
|
+
outcome: parseOutcome(payload),
|
|
1046
|
+
metadata: recordField(payload, "metadata")
|
|
1047
|
+
} };
|
|
1085
1048
|
}
|
|
1086
1049
|
function parseOutcome(payload) {
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1050
|
+
const value = recordField(payload, "outcome");
|
|
1051
|
+
if (!value) return void 0;
|
|
1052
|
+
return {
|
|
1053
|
+
success: typeof value.success === "boolean" ? value.success : void 0,
|
|
1054
|
+
score: finiteUnitField(value, "score"),
|
|
1055
|
+
reward: finiteUnitField(value, "reward"),
|
|
1056
|
+
costUsd: nonNegativeNumberField(value, "costUsd"),
|
|
1057
|
+
observedAt: stringField(value, "observedAt"),
|
|
1058
|
+
metadata: recordField(value, "metadata")
|
|
1059
|
+
};
|
|
1097
1060
|
}
|
|
1098
1061
|
function stringField(obj, key) {
|
|
1099
|
-
|
|
1100
|
-
|
|
1062
|
+
const value = obj[key];
|
|
1063
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
1101
1064
|
}
|
|
1102
1065
|
function numberField(obj, key) {
|
|
1103
|
-
|
|
1104
|
-
|
|
1066
|
+
const value = obj[key];
|
|
1067
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
1105
1068
|
}
|
|
1106
1069
|
function finiteUnitField(obj, key) {
|
|
1107
|
-
|
|
1108
|
-
|
|
1070
|
+
const value = numberField(obj, key);
|
|
1071
|
+
return value === void 0 ? void 0 : Math.max(0, Math.min(1, value));
|
|
1109
1072
|
}
|
|
1110
1073
|
function nonNegativeNumberField(obj, key) {
|
|
1111
|
-
|
|
1112
|
-
|
|
1074
|
+
const value = numberField(obj, key);
|
|
1075
|
+
return value === void 0 ? void 0 : Math.max(0, value);
|
|
1113
1076
|
}
|
|
1114
1077
|
function stringArrayField(obj, key) {
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
);
|
|
1120
|
-
return strings.length > 0 ? strings : void 0;
|
|
1078
|
+
const value = obj[key];
|
|
1079
|
+
if (!Array.isArray(value)) return void 0;
|
|
1080
|
+
const strings = value.filter((item) => typeof item === "string" && item.length > 0);
|
|
1081
|
+
return strings.length > 0 ? strings : void 0;
|
|
1121
1082
|
}
|
|
1122
1083
|
function recordField(obj, key) {
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1084
|
+
const value = obj[key];
|
|
1085
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
1086
|
+
return value;
|
|
1087
|
+
}
|
|
1088
|
+
//#endregion
|
|
1089
|
+
//#region src/belief-state/runtime-hooks.ts
|
|
1090
|
+
const DEFAULT_MAX_CONTEXT_CHARS$1 = 12e3;
|
|
1091
|
+
const DEFAULT_PAYLOAD_PREVIEW_CHARS = 2e3;
|
|
1131
1092
|
function runtimeDecisionPointToBeliefShadowProbeInput(point, options) {
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1093
|
+
const diagnostics = [];
|
|
1094
|
+
const decisionKind = resolveDecisionKind(point, options.decisionKind, diagnostics);
|
|
1095
|
+
if (!decisionKind) return { diagnostics };
|
|
1096
|
+
const lifecycleEvidence = runtimeHookEventsToEvidenceRefs(point, options);
|
|
1097
|
+
const evidence = [...point.evidence ?? [], ...lifecycleEvidence];
|
|
1098
|
+
return {
|
|
1099
|
+
input: {
|
|
1100
|
+
probeId: options.probeId,
|
|
1101
|
+
decisionId: point.id,
|
|
1102
|
+
runId: point.runId,
|
|
1103
|
+
scenarioId: point.scenarioId,
|
|
1104
|
+
stepIndex: point.stepIndex,
|
|
1105
|
+
decisionKind,
|
|
1106
|
+
candidateActions: uniqueStrings$1(point.candidateActions ?? []),
|
|
1107
|
+
evidence: evidence.map((ref) => ({
|
|
1108
|
+
id: ref.id,
|
|
1109
|
+
source: ref.source,
|
|
1110
|
+
...options.includeEvidenceDetail && ref.detail ? { detail: ref.detail } : {},
|
|
1111
|
+
...ref.quality ? { quality: ref.quality } : {}
|
|
1112
|
+
})),
|
|
1113
|
+
context: trimText$1(point.context, options.maxContextChars),
|
|
1114
|
+
metadata: mergeMetadata(point.metadata, lifecycleMetadata(lifecycleEvidence))
|
|
1115
|
+
},
|
|
1116
|
+
diagnostics
|
|
1117
|
+
};
|
|
1157
1118
|
}
|
|
1158
1119
|
function runtimeDecisionPointToBeliefDecisionPoint(point, options) {
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
outcome: options.outcome,
|
|
1198
|
-
metadata: mergeMetadata(
|
|
1199
|
-
mergeMetadata(point.metadata, lifecycleMetadata(lifecycleEvidence)),
|
|
1200
|
-
options.metadata
|
|
1201
|
-
)
|
|
1202
|
-
},
|
|
1203
|
-
diagnostics
|
|
1204
|
-
};
|
|
1120
|
+
const diagnostics = [];
|
|
1121
|
+
const decisionKind = resolveDecisionKind(point, options.decisionKind, diagnostics);
|
|
1122
|
+
const chosenAction = stringOrUndefined(options.chosenAction);
|
|
1123
|
+
if (!chosenAction) diagnostics.push({
|
|
1124
|
+
decisionId: point.id,
|
|
1125
|
+
severity: "error",
|
|
1126
|
+
reason: "missing chosenAction"
|
|
1127
|
+
});
|
|
1128
|
+
const candidateActions = uniqueStrings$1(point.candidateActions ?? []);
|
|
1129
|
+
if (chosenAction && candidateActions.length > 0 && !candidateActions.includes(chosenAction)) diagnostics.push({
|
|
1130
|
+
decisionId: point.id,
|
|
1131
|
+
severity: "warning",
|
|
1132
|
+
reason: `chosenAction ${chosenAction} is not in candidateActions`
|
|
1133
|
+
});
|
|
1134
|
+
if (!decisionKind || !chosenAction) return { diagnostics };
|
|
1135
|
+
const lifecycleEvidence = runtimeHookEventsToEvidenceRefs(point, options);
|
|
1136
|
+
const evidence = [...point.evidence ?? [], ...lifecycleEvidence];
|
|
1137
|
+
return {
|
|
1138
|
+
point: {
|
|
1139
|
+
id: point.id,
|
|
1140
|
+
runId: point.runId,
|
|
1141
|
+
scenarioId: point.scenarioId,
|
|
1142
|
+
stepIndex: point.stepIndex,
|
|
1143
|
+
kind: decisionKind,
|
|
1144
|
+
chosenAction,
|
|
1145
|
+
candidateActions,
|
|
1146
|
+
confidence: unitProbabilityOrUndefined(options.confidence),
|
|
1147
|
+
behaviorProb: finiteNumberOrUndefined(options.behaviorProb),
|
|
1148
|
+
targetProb: finiteNumberOrUndefined(options.targetProb),
|
|
1149
|
+
qHatChosen: options.qHatChosen === null ? null : unitProbabilityOrUndefined(options.qHatChosen),
|
|
1150
|
+
vHatTarget: options.vHatTarget === null ? null : unitProbabilityOrUndefined(options.vHatTarget),
|
|
1151
|
+
costUsd: nonNegativeNumberOrUndefined(options.costUsd),
|
|
1152
|
+
evidence: evidence.map((ref) => runtimeEvidenceToBeliefEvidence(ref, point)),
|
|
1153
|
+
outcome: options.outcome,
|
|
1154
|
+
metadata: mergeMetadata(mergeMetadata(point.metadata, lifecycleMetadata(lifecycleEvidence)), options.metadata)
|
|
1155
|
+
},
|
|
1156
|
+
diagnostics
|
|
1157
|
+
};
|
|
1205
1158
|
}
|
|
1206
1159
|
function createBeliefRuntimeHookCollector(defaults) {
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1160
|
+
const decisions = [];
|
|
1161
|
+
const events = [];
|
|
1162
|
+
return {
|
|
1163
|
+
hooks: {
|
|
1164
|
+
onEvent: (event) => {
|
|
1165
|
+
events.push(snapshotRuntimeHookEvent(event));
|
|
1166
|
+
},
|
|
1167
|
+
onDecisionPoint: (point) => {
|
|
1168
|
+
decisions.push(snapshotRuntimeDecisionPoint(point));
|
|
1169
|
+
}
|
|
1170
|
+
},
|
|
1171
|
+
decisions,
|
|
1172
|
+
events,
|
|
1173
|
+
toShadowProbeInputs: (options = {}) => {
|
|
1174
|
+
const inputs = [];
|
|
1175
|
+
const diagnostics = [];
|
|
1176
|
+
const includeLifecycleEvidence = options.includeLifecycleEvidence ?? defaults.includeLifecycleEvidence;
|
|
1177
|
+
for (const point of decisions) {
|
|
1178
|
+
const report = runtimeDecisionPointToBeliefShadowProbeInput(point, {
|
|
1179
|
+
...defaults,
|
|
1180
|
+
...options,
|
|
1181
|
+
includeLifecycleEvidence,
|
|
1182
|
+
lifecycleEvents: includeLifecycleEvidence === false ? void 0 : options.lifecycleEvents ?? defaults.lifecycleEvents ?? events
|
|
1183
|
+
});
|
|
1184
|
+
if (report.input) inputs.push(report.input);
|
|
1185
|
+
diagnostics.push(...report.diagnostics);
|
|
1186
|
+
}
|
|
1187
|
+
return {
|
|
1188
|
+
inputs,
|
|
1189
|
+
diagnostics
|
|
1190
|
+
};
|
|
1191
|
+
},
|
|
1192
|
+
clear: () => {
|
|
1193
|
+
decisions.length = 0;
|
|
1194
|
+
events.length = 0;
|
|
1195
|
+
}
|
|
1196
|
+
};
|
|
1241
1197
|
}
|
|
1242
1198
|
function resolveDecisionKind(point, override, diagnostics) {
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
return void 0;
|
|
1199
|
+
const kind = override ?? point.kind;
|
|
1200
|
+
if (isBeliefDecisionKind(kind)) return kind;
|
|
1201
|
+
diagnostics.push({
|
|
1202
|
+
decisionId: point.id,
|
|
1203
|
+
severity: "error",
|
|
1204
|
+
reason: `unsupported decisionKind "${kind}"`
|
|
1205
|
+
});
|
|
1251
1206
|
}
|
|
1252
1207
|
function runtimeEvidenceToBeliefEvidence(ref, point) {
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
metadata: mergeMetadata({ runtimeSource: ref.source }, ref.metadata)
|
|
1270
|
-
};
|
|
1208
|
+
if (isBeliefEvidenceSource(ref.source)) return {
|
|
1209
|
+
source: ref.source,
|
|
1210
|
+
id: ref.id,
|
|
1211
|
+
runId: point.runId,
|
|
1212
|
+
detail: ref.detail,
|
|
1213
|
+
quality: ref.quality,
|
|
1214
|
+
metadata: ref.metadata
|
|
1215
|
+
};
|
|
1216
|
+
return {
|
|
1217
|
+
source: "event",
|
|
1218
|
+
id: ref.id,
|
|
1219
|
+
runId: point.runId,
|
|
1220
|
+
detail: ref.detail,
|
|
1221
|
+
quality: ref.quality,
|
|
1222
|
+
metadata: mergeMetadata({ runtimeSource: ref.source }, ref.metadata)
|
|
1223
|
+
};
|
|
1271
1224
|
}
|
|
1272
1225
|
function runtimeHookEventsToEvidenceRefs(point, options) {
|
|
1273
|
-
|
|
1274
|
-
|
|
1226
|
+
if (options.includeLifecycleEvidence === false) return [];
|
|
1227
|
+
return (options.lifecycleEvents ?? []).filter((event) => runtimeHookEventMatchesDecision(point, event)).map(runtimeHookEventToEvidenceRef);
|
|
1275
1228
|
}
|
|
1276
1229
|
function runtimeHookEventMatchesDecision(point, event) {
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1230
|
+
if (event.runId !== point.runId) return false;
|
|
1231
|
+
if (event.scenarioId && point.scenarioId && event.scenarioId !== point.scenarioId) return false;
|
|
1232
|
+
return event.stepIndex === void 0 || event.stepIndex === point.stepIndex;
|
|
1280
1233
|
}
|
|
1281
1234
|
function runtimeHookEventToEvidenceRef(event) {
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
event.metadata
|
|
1297
|
-
)
|
|
1298
|
-
};
|
|
1235
|
+
return {
|
|
1236
|
+
source: "runtime_event",
|
|
1237
|
+
id: event.id,
|
|
1238
|
+
detail: `${event.target}:${event.phase}`,
|
|
1239
|
+
quality: "direct",
|
|
1240
|
+
metadata: mergeMetadata(compactMetadata$1({
|
|
1241
|
+
target: event.target,
|
|
1242
|
+
phase: event.phase,
|
|
1243
|
+
timestamp: event.timestamp,
|
|
1244
|
+
stepIndex: event.stepIndex,
|
|
1245
|
+
parentId: event.parentId,
|
|
1246
|
+
payloadPreview: previewUnknown(event.payload)
|
|
1247
|
+
}), event.metadata)
|
|
1248
|
+
};
|
|
1299
1249
|
}
|
|
1300
1250
|
function lifecycleMetadata(refs) {
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1251
|
+
if (refs.length === 0) return void 0;
|
|
1252
|
+
return {
|
|
1253
|
+
lifecycleEventCount: refs.length,
|
|
1254
|
+
lifecycleEventIds: refs.map((ref) => ref.id)
|
|
1255
|
+
};
|
|
1306
1256
|
}
|
|
1307
1257
|
function snapshotRuntimeHookEvent(event) {
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1258
|
+
return {
|
|
1259
|
+
id: event.id,
|
|
1260
|
+
runId: event.runId,
|
|
1261
|
+
scenarioId: event.scenarioId,
|
|
1262
|
+
target: event.target,
|
|
1263
|
+
phase: event.phase,
|
|
1264
|
+
timestamp: event.timestamp,
|
|
1265
|
+
stepIndex: event.stepIndex,
|
|
1266
|
+
parentId: event.parentId,
|
|
1267
|
+
payload: snapshotUnknown(event.payload),
|
|
1268
|
+
metadata: event.metadata ? { ...event.metadata } : void 0
|
|
1269
|
+
};
|
|
1320
1270
|
}
|
|
1321
1271
|
function snapshotRuntimeDecisionPoint(point) {
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1272
|
+
return {
|
|
1273
|
+
id: point.id,
|
|
1274
|
+
runId: point.runId,
|
|
1275
|
+
scenarioId: point.scenarioId,
|
|
1276
|
+
stepIndex: point.stepIndex,
|
|
1277
|
+
kind: point.kind,
|
|
1278
|
+
candidateActions: [...point.candidateActions ?? []],
|
|
1279
|
+
context: point.context,
|
|
1280
|
+
evidence: (point.evidence ?? []).map((ref) => ({
|
|
1281
|
+
source: ref.source,
|
|
1282
|
+
id: ref.id,
|
|
1283
|
+
detail: ref.detail,
|
|
1284
|
+
quality: ref.quality,
|
|
1285
|
+
metadata: ref.metadata ? { ...ref.metadata } : void 0
|
|
1286
|
+
})),
|
|
1287
|
+
metadata: point.metadata ? { ...point.metadata } : void 0
|
|
1288
|
+
};
|
|
1339
1289
|
}
|
|
1340
1290
|
function mergeMetadata(base, extra) {
|
|
1341
|
-
|
|
1342
|
-
|
|
1291
|
+
if (!base && !extra) return void 0;
|
|
1292
|
+
return {
|
|
1293
|
+
...base ?? {},
|
|
1294
|
+
...extra ?? {}
|
|
1295
|
+
};
|
|
1343
1296
|
}
|
|
1344
|
-
function compactMetadata(values) {
|
|
1345
|
-
|
|
1346
|
-
|
|
1297
|
+
function compactMetadata$1(values) {
|
|
1298
|
+
const entries = Object.entries(values).filter(([, value]) => value !== void 0);
|
|
1299
|
+
return entries.length > 0 ? Object.fromEntries(entries) : void 0;
|
|
1347
1300
|
}
|
|
1348
1301
|
function previewUnknown(value, maxChars = DEFAULT_PAYLOAD_PREVIEW_CHARS) {
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1302
|
+
if (value === void 0) return void 0;
|
|
1303
|
+
if (typeof value === "string") return trimText$1(value, maxChars);
|
|
1304
|
+
try {
|
|
1305
|
+
return trimText$1(JSON.stringify(value), maxChars);
|
|
1306
|
+
} catch {
|
|
1307
|
+
return trimText$1(String(value), maxChars);
|
|
1308
|
+
}
|
|
1356
1309
|
}
|
|
1357
1310
|
function snapshotUnknown(value) {
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1311
|
+
if (Array.isArray(value)) return [...value];
|
|
1312
|
+
if (isRecord$1(value)) return { ...value };
|
|
1313
|
+
return value;
|
|
1361
1314
|
}
|
|
1362
|
-
function
|
|
1363
|
-
|
|
1315
|
+
function isRecord$1(value) {
|
|
1316
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1364
1317
|
}
|
|
1365
|
-
function uniqueStrings(values) {
|
|
1366
|
-
|
|
1318
|
+
function uniqueStrings$1(values) {
|
|
1319
|
+
return [...new Set(values.filter((value) => value.length > 0))];
|
|
1367
1320
|
}
|
|
1368
|
-
function trimText(value, maxChars = DEFAULT_MAX_CONTEXT_CHARS) {
|
|
1369
|
-
|
|
1370
|
-
|
|
1321
|
+
function trimText$1(value, maxChars = DEFAULT_MAX_CONTEXT_CHARS$1) {
|
|
1322
|
+
if (!value) return void 0;
|
|
1323
|
+
return value.length > maxChars ? value.slice(value.length - maxChars) : value;
|
|
1371
1324
|
}
|
|
1372
1325
|
function stringOrUndefined(value) {
|
|
1373
|
-
|
|
1326
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
1374
1327
|
}
|
|
1375
1328
|
function finiteNumberOrUndefined(value) {
|
|
1376
|
-
|
|
1329
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
1377
1330
|
}
|
|
1378
1331
|
function unitProbabilityOrUndefined(value) {
|
|
1379
|
-
|
|
1380
|
-
|
|
1332
|
+
const number = finiteNumberOrUndefined(value);
|
|
1333
|
+
return number !== void 0 && number >= 0 && number <= 1 ? number : void 0;
|
|
1381
1334
|
}
|
|
1382
1335
|
function nonNegativeNumberOrUndefined(value) {
|
|
1383
|
-
|
|
1384
|
-
|
|
1336
|
+
const number = finiteNumberOrUndefined(value);
|
|
1337
|
+
return number !== void 0 && number >= 0 ? number : void 0;
|
|
1385
1338
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1339
|
+
//#endregion
|
|
1340
|
+
//#region src/belief-state/phase0-measurement.ts
|
|
1341
|
+
const DEFAULT_BASELINE_POLICY_ID = "always-accept-observed-action";
|
|
1389
1342
|
function buildRuntimeBeliefPhase0Measurement(options) {
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
}),
|
|
1450
|
-
diagnostics
|
|
1451
|
-
};
|
|
1343
|
+
const runsById = new Map(options.runs.map((run) => [run.runId, run]));
|
|
1344
|
+
const labelsByDecisionId = /* @__PURE__ */ new Map();
|
|
1345
|
+
const diagnostics = [];
|
|
1346
|
+
for (const label of options.labels) {
|
|
1347
|
+
if (labelsByDecisionId.has(label.decisionId)) diagnostics.push(`${label.decisionId}: duplicate label; using the last label`);
|
|
1348
|
+
labelsByDecisionId.set(label.decisionId, label);
|
|
1349
|
+
}
|
|
1350
|
+
const points = [];
|
|
1351
|
+
let missingRunRecordCount = 0;
|
|
1352
|
+
let missingLabelCount = 0;
|
|
1353
|
+
for (const decision of options.decisions) {
|
|
1354
|
+
const run = runsById.get(decision.runId);
|
|
1355
|
+
if (!run) {
|
|
1356
|
+
missingRunRecordCount += 1;
|
|
1357
|
+
diagnostics.push(`${decision.id}: missing RunRecord join for runId ${decision.runId}`);
|
|
1358
|
+
continue;
|
|
1359
|
+
}
|
|
1360
|
+
const label = labelsByDecisionId.get(decision.id);
|
|
1361
|
+
if (!label) {
|
|
1362
|
+
missingLabelCount += 1;
|
|
1363
|
+
diagnostics.push(`${decision.id}: missing observed action/outcome label`);
|
|
1364
|
+
continue;
|
|
1365
|
+
}
|
|
1366
|
+
const splitTag = label.splitTag ?? run.splitTag;
|
|
1367
|
+
const report = runtimeDecisionPointToBeliefDecisionPoint({
|
|
1368
|
+
...decision,
|
|
1369
|
+
scenarioId: decision.scenarioId ?? run.scenarioId
|
|
1370
|
+
}, {
|
|
1371
|
+
chosenAction: label.chosenAction,
|
|
1372
|
+
confidence: label.confidence,
|
|
1373
|
+
behaviorProb: label.behaviorProb,
|
|
1374
|
+
targetProb: label.targetProb,
|
|
1375
|
+
qHatChosen: label.qHatChosen,
|
|
1376
|
+
vHatTarget: label.vHatTarget,
|
|
1377
|
+
costUsd: label.costUsd,
|
|
1378
|
+
outcome: label.outcome,
|
|
1379
|
+
lifecycleEvents: options.events,
|
|
1380
|
+
metadata: compactMetadata({
|
|
1381
|
+
baselinePolicyId: options.baselinePolicyId ?? DEFAULT_BASELINE_POLICY_ID,
|
|
1382
|
+
splitTag,
|
|
1383
|
+
...label.metadata
|
|
1384
|
+
})
|
|
1385
|
+
});
|
|
1386
|
+
diagnostics.push(...report.diagnostics.map((item) => `${item.decisionId}: ${item.reason}`));
|
|
1387
|
+
if (report.point) points.push(report.point);
|
|
1388
|
+
}
|
|
1389
|
+
const packet = buildBeliefDecisionResearchEvidencePacket({
|
|
1390
|
+
...options,
|
|
1391
|
+
points
|
|
1392
|
+
});
|
|
1393
|
+
return {
|
|
1394
|
+
points,
|
|
1395
|
+
packet,
|
|
1396
|
+
summary: summarizePhase0Measurement(options, points, packet, {
|
|
1397
|
+
missingRunRecordCount,
|
|
1398
|
+
missingLabelCount
|
|
1399
|
+
}),
|
|
1400
|
+
diagnostics
|
|
1401
|
+
};
|
|
1452
1402
|
}
|
|
1453
1403
|
function summarizePhase0Measurement(options, points, packet, counts) {
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1404
|
+
const producerDecisionCount = options.decisions.length;
|
|
1405
|
+
return {
|
|
1406
|
+
runCount: options.runs.length,
|
|
1407
|
+
producerDecisionCount,
|
|
1408
|
+
lifecycleEventCount: options.events?.length ?? 0,
|
|
1409
|
+
labelCount: options.labels.length,
|
|
1410
|
+
completedPointCount: points.length,
|
|
1411
|
+
runJoinRate: ratio(producerDecisionCount - counts.missingRunRecordCount, producerDecisionCount),
|
|
1412
|
+
labelJoinRate: ratio(points.length, producerDecisionCount),
|
|
1413
|
+
missingRunRecordCount: counts.missingRunRecordCount,
|
|
1414
|
+
missingLabelCount: counts.missingLabelCount,
|
|
1415
|
+
withEvidence: points.filter((point) => point.evidence.length > 0).length,
|
|
1416
|
+
withOutcome: points.filter((point) => point.outcome).length,
|
|
1417
|
+
withSplit: points.filter((point) => typeof point.metadata?.splitTag === "string").length,
|
|
1418
|
+
withBehaviorProb: points.filter((point) => point.behaviorProb !== void 0).length,
|
|
1419
|
+
withTargetProb: points.filter((point) => point.targetProb !== void 0).length,
|
|
1420
|
+
baselinePolicyId: options.baselinePolicyId ?? DEFAULT_BASELINE_POLICY_ID,
|
|
1421
|
+
packetStatus: packet.status,
|
|
1422
|
+
claimScope: packet.claimScope
|
|
1423
|
+
};
|
|
1474
1424
|
}
|
|
1475
1425
|
function ratio(numerator, denominator) {
|
|
1476
|
-
|
|
1426
|
+
return denominator > 0 ? numerator / denominator : 0;
|
|
1477
1427
|
}
|
|
1478
|
-
function
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
}
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1428
|
+
function compactMetadata(values) {
|
|
1429
|
+
const entries = Object.entries(values).filter(([, value]) => value !== void 0);
|
|
1430
|
+
return entries.length > 0 ? Object.fromEntries(entries) : void 0;
|
|
1431
|
+
}
|
|
1432
|
+
//#endregion
|
|
1433
|
+
//#region src/belief-state/runtime-benchmark-corpus.ts
|
|
1434
|
+
const MAX_STRING_LENGTH = 12e3;
|
|
1435
|
+
const MAX_CONTEXT_LENGTH = 2e4;
|
|
1436
|
+
const MAX_EVIDENCE_DETAIL_LENGTH = 2e3;
|
|
1437
|
+
const MAX_CANDIDATE_ACTIONS = 50;
|
|
1438
|
+
const MAX_EVIDENCE_REFS = 50;
|
|
1439
|
+
const MAX_METADATA_DEPTH = 4;
|
|
1440
|
+
const MAX_METADATA_KEYS = 100;
|
|
1441
|
+
const SENSITIVE_KEY_RE = /(?:authorization|api[_-]?key|token|secret|password|cookie|credential|bearer)/i;
|
|
1442
|
+
const SENSITIVE_VALUE_RES = [
|
|
1443
|
+
/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi,
|
|
1444
|
+
/\b(?:sk|gh[pousr])_[A-Za-z0-9_]{20,}\b/g,
|
|
1445
|
+
/\b(?:sk|ghp|gho|ghu|ghs|ghr)-[A-Za-z0-9_-]{20,}\b/g
|
|
1496
1446
|
];
|
|
1497
|
-
|
|
1447
|
+
const SENSITIVE_ASSIGNMENT_RE = /\b(api[_-]?key|token|secret|password|cookie)\s*[:=]\s*["']?[^"'\s,;}]+/gi;
|
|
1498
1448
|
function buildRuntimeBenchmarkBeliefPhase0Measurement(options) {
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
labelCount: labels.length
|
|
1535
|
-
},
|
|
1536
|
-
diagnostics: [...trajectory.diagnostics, ...diagnostics, ...measurement.diagnostics]
|
|
1537
|
-
};
|
|
1449
|
+
const diagnostics = [];
|
|
1450
|
+
const trajectory = projectRuntimeTrajectoryEvidence({
|
|
1451
|
+
records: options.records,
|
|
1452
|
+
defaultSplitTag: options.defaultSplitTag,
|
|
1453
|
+
recordIdOf: runtimeBenchmarkRecordId,
|
|
1454
|
+
scenarioIdOf: runtimeBenchmarkScenarioId
|
|
1455
|
+
});
|
|
1456
|
+
const decisions = options.decisions ?? runtimeBenchmarkDecisionPoints(options.records, diagnostics);
|
|
1457
|
+
const labels = options.labels ?? [];
|
|
1458
|
+
if (decisions.length === 0) diagnostics.push("no runtime decision points supplied or found on records; benchmark lifecycle events alone cannot produce belief decision rows");
|
|
1459
|
+
if (labels.length === 0 && decisions.length > 0) diagnostics.push("no decision labels supplied; observed action/outcome joins will be incomplete");
|
|
1460
|
+
const measurement = buildRuntimeBeliefPhase0Measurement({
|
|
1461
|
+
...options,
|
|
1462
|
+
runs: trajectory.runs,
|
|
1463
|
+
events: trajectory.events,
|
|
1464
|
+
decisions,
|
|
1465
|
+
labels
|
|
1466
|
+
});
|
|
1467
|
+
return {
|
|
1468
|
+
runs: trajectory.runs,
|
|
1469
|
+
events: trajectory.events,
|
|
1470
|
+
decisions,
|
|
1471
|
+
labels,
|
|
1472
|
+
trajectory,
|
|
1473
|
+
measurement,
|
|
1474
|
+
summary: {
|
|
1475
|
+
decisionCount: decisions.length,
|
|
1476
|
+
labelCount: labels.length
|
|
1477
|
+
},
|
|
1478
|
+
diagnostics: [
|
|
1479
|
+
...trajectory.diagnostics,
|
|
1480
|
+
...diagnostics,
|
|
1481
|
+
...measurement.diagnostics
|
|
1482
|
+
]
|
|
1483
|
+
};
|
|
1538
1484
|
}
|
|
1539
1485
|
function runtimeBenchmarkRecordId(record) {
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1486
|
+
const parts = [
|
|
1487
|
+
nonEmptyString(record.benchmark),
|
|
1488
|
+
nonEmptyString(record.instanceId),
|
|
1489
|
+
nonEmptyString(record.condition)
|
|
1490
|
+
].filter((part) => part !== void 0);
|
|
1491
|
+
return parts.length > 0 ? parts.join(":") : void 0;
|
|
1546
1492
|
}
|
|
1547
1493
|
function runtimeBenchmarkScenarioId(record) {
|
|
1548
|
-
|
|
1494
|
+
return nonEmptyString(record.instanceId);
|
|
1549
1495
|
}
|
|
1550
1496
|
function runtimeBenchmarkDecisionPoints(records, diagnostics) {
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
}
|
|
1575
|
-
return decisions;
|
|
1497
|
+
const decisions = [];
|
|
1498
|
+
for (let recordIndex = 0; recordIndex < records.length; recordIndex += 1) {
|
|
1499
|
+
const record = records[recordIndex];
|
|
1500
|
+
const raw = record.runtimeDecisionPoints;
|
|
1501
|
+
if (raw === void 0) continue;
|
|
1502
|
+
const recordId = runtimeBenchmarkRecordId(record) ?? `record[${recordIndex}]`;
|
|
1503
|
+
if (!Array.isArray(raw)) {
|
|
1504
|
+
diagnostics.push(`${recordId}: runtimeDecisionPoints is not an array`);
|
|
1505
|
+
continue;
|
|
1506
|
+
}
|
|
1507
|
+
for (let pointIndex = 0; pointIndex < raw.length; pointIndex += 1) {
|
|
1508
|
+
const point = runtimeBenchmarkDecisionPoint(raw[pointIndex], {
|
|
1509
|
+
diagnostics,
|
|
1510
|
+
path: `${recordId}: runtimeDecisionPoints[${pointIndex}]`
|
|
1511
|
+
});
|
|
1512
|
+
if (!point) {
|
|
1513
|
+
diagnostics.push(`${recordId}: runtimeDecisionPoints[${pointIndex}] is not a RuntimeDecisionPoint`);
|
|
1514
|
+
continue;
|
|
1515
|
+
}
|
|
1516
|
+
decisions.push(point);
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
return decisions;
|
|
1576
1520
|
}
|
|
1577
1521
|
function runtimeBenchmarkDecisionPoint(input, context) {
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
metadata: sanitizeMetadataRecord(input.metadata)
|
|
1599
|
-
};
|
|
1522
|
+
if (!isRecord(input)) return null;
|
|
1523
|
+
if (typeof input.id !== "string" || input.id.length === 0) return null;
|
|
1524
|
+
if (typeof input.runId !== "string" || input.runId.length === 0) return null;
|
|
1525
|
+
if (typeof input.stepIndex !== "number" || !Number.isInteger(input.stepIndex) || input.stepIndex < 0) return null;
|
|
1526
|
+
if (typeof input.kind !== "string" || input.kind.length === 0) return null;
|
|
1527
|
+
return {
|
|
1528
|
+
id: sanitizeString(input.id, MAX_STRING_LENGTH),
|
|
1529
|
+
runId: sanitizeString(input.runId, MAX_STRING_LENGTH),
|
|
1530
|
+
scenarioId: sanitizeOptionalString(input.scenarioId, MAX_STRING_LENGTH),
|
|
1531
|
+
stepIndex: input.stepIndex,
|
|
1532
|
+
kind: sanitizeString(input.kind, MAX_STRING_LENGTH),
|
|
1533
|
+
candidateActions: stringArray(input.candidateActions, {
|
|
1534
|
+
...context,
|
|
1535
|
+
maxItems: MAX_CANDIDATE_ACTIONS,
|
|
1536
|
+
label: "candidateActions"
|
|
1537
|
+
}),
|
|
1538
|
+
context: sanitizeOptionalString(input.context, MAX_CONTEXT_LENGTH),
|
|
1539
|
+
evidence: runtimeBenchmarkEvidence(input.evidence, context),
|
|
1540
|
+
metadata: sanitizeMetadataRecord(input.metadata)
|
|
1541
|
+
};
|
|
1600
1542
|
}
|
|
1601
1543
|
function runtimeBenchmarkEvidence(input, context) {
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
metadata: sanitizeMetadataRecord(item.metadata)
|
|
1617
|
-
}
|
|
1618
|
-
];
|
|
1619
|
-
});
|
|
1544
|
+
if (!Array.isArray(input)) return [];
|
|
1545
|
+
if (input.length > MAX_EVIDENCE_REFS) context.diagnostics.push(`${context.path}: evidence truncated to ${MAX_EVIDENCE_REFS} refs`);
|
|
1546
|
+
return input.slice(0, MAX_EVIDENCE_REFS).flatMap((item) => {
|
|
1547
|
+
if (!isRecord(item)) return [];
|
|
1548
|
+
const source = sanitizeOptionalString(item.source, MAX_STRING_LENGTH);
|
|
1549
|
+
const id = sanitizeOptionalString(item.id, MAX_STRING_LENGTH);
|
|
1550
|
+
if (!source || !id) return [];
|
|
1551
|
+
return [{
|
|
1552
|
+
source,
|
|
1553
|
+
id,
|
|
1554
|
+
detail: sanitizeOptionalString(item.detail, MAX_EVIDENCE_DETAIL_LENGTH),
|
|
1555
|
+
metadata: sanitizeMetadataRecord(item.metadata)
|
|
1556
|
+
}];
|
|
1557
|
+
});
|
|
1620
1558
|
}
|
|
1621
1559
|
function stringArray(input, context) {
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
const values = input.slice(0, context.maxItems).filter((value) => typeof value === "string" && value.length > 0).map((value) => sanitizeString(value, MAX_STRING_LENGTH));
|
|
1627
|
-
return values.length > 0 ? values : void 0;
|
|
1560
|
+
if (!Array.isArray(input)) return void 0;
|
|
1561
|
+
if (input.length > context.maxItems) context.diagnostics.push(`${context.path}: ${context.label} truncated to ${context.maxItems}`);
|
|
1562
|
+
const values = input.slice(0, context.maxItems).filter((value) => typeof value === "string" && value.length > 0).map((value) => sanitizeString(value, MAX_STRING_LENGTH));
|
|
1563
|
+
return values.length > 0 ? values : void 0;
|
|
1628
1564
|
}
|
|
1629
1565
|
function sanitizeMetadataRecord(metadata) {
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1566
|
+
if (!isRecord(metadata)) return void 0;
|
|
1567
|
+
const sanitized = sanitizeMetadata(metadata);
|
|
1568
|
+
if (!sanitized || typeof sanitized !== "object" || Array.isArray(sanitized)) return void 0;
|
|
1569
|
+
return sanitized;
|
|
1634
1570
|
}
|
|
1635
1571
|
function sanitizeMetadata(value, depth = 0) {
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
}
|
|
1649
|
-
return sanitized;
|
|
1572
|
+
if (value == null) return value;
|
|
1573
|
+
if (typeof value === "string") return sanitizeString(value, MAX_STRING_LENGTH);
|
|
1574
|
+
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
1575
|
+
if (Array.isArray(value)) {
|
|
1576
|
+
if (depth >= MAX_METADATA_DEPTH) return "[MaxDepth]";
|
|
1577
|
+
return value.slice(0, MAX_METADATA_KEYS).map((item) => sanitizeMetadata(item, depth + 1));
|
|
1578
|
+
}
|
|
1579
|
+
if (!isRecord(value)) return void 0;
|
|
1580
|
+
if (depth >= MAX_METADATA_DEPTH) return "[MaxDepth]";
|
|
1581
|
+
const sanitized = {};
|
|
1582
|
+
for (const [key, nested] of Object.entries(value).slice(0, MAX_METADATA_KEYS)) sanitized[key] = SENSITIVE_KEY_RE.test(key) ? "[REDACTED]" : sanitizeMetadata(nested, depth + 1);
|
|
1583
|
+
return sanitized;
|
|
1650
1584
|
}
|
|
1651
1585
|
function sanitizeOptionalString(value, maxLength) {
|
|
1652
|
-
|
|
1586
|
+
return typeof value === "string" && value.length > 0 ? sanitizeString(value, maxLength) : void 0;
|
|
1653
1587
|
}
|
|
1654
1588
|
function sanitizeString(value, maxLength) {
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
if (sanitized.length <= maxLength) return sanitized;
|
|
1664
|
-
return sanitized.slice(0, maxLength);
|
|
1665
|
-
}
|
|
1666
|
-
function isRecord3(value) {
|
|
1667
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1589
|
+
let sanitized = value;
|
|
1590
|
+
for (const pattern of SENSITIVE_VALUE_RES) sanitized = sanitized.replace(pattern, "[REDACTED]");
|
|
1591
|
+
sanitized = sanitized.replace(SENSITIVE_ASSIGNMENT_RE, (_match, key) => `${key}=[REDACTED]`);
|
|
1592
|
+
if (sanitized.length <= maxLength) return sanitized;
|
|
1593
|
+
return sanitized.slice(0, maxLength);
|
|
1594
|
+
}
|
|
1595
|
+
function isRecord(value) {
|
|
1596
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1668
1597
|
}
|
|
1669
1598
|
function nonEmptyString(value) {
|
|
1670
|
-
|
|
1599
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
1671
1600
|
}
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1601
|
+
//#endregion
|
|
1602
|
+
//#region src/belief-state/shadow-probe.ts
|
|
1603
|
+
const DEFAULT_CONCURRENCY = 4;
|
|
1604
|
+
const DEFAULT_MAX_CONTEXT_CHARS = 12e3;
|
|
1676
1605
|
async function runBeliefShadowProbe(options) {
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1606
|
+
const concurrency = boundedInteger(options.concurrency ?? DEFAULT_CONCURRENCY, 1, 32);
|
|
1607
|
+
const records = [];
|
|
1608
|
+
const diagnostics = [];
|
|
1609
|
+
let next = 0;
|
|
1610
|
+
async function worker() {
|
|
1611
|
+
while (next < options.points.length) {
|
|
1612
|
+
const index = next;
|
|
1613
|
+
next += 1;
|
|
1614
|
+
const point = options.points[index];
|
|
1615
|
+
if (!point) continue;
|
|
1616
|
+
const result = await probePoint(point, options);
|
|
1617
|
+
records[index] = result.record;
|
|
1618
|
+
diagnostics.push(...result.diagnostics);
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, options.points.length) }, worker));
|
|
1622
|
+
const completed = records.filter((record) => !!record);
|
|
1623
|
+
return {
|
|
1624
|
+
probeId: options.probeId,
|
|
1625
|
+
records: completed,
|
|
1626
|
+
diagnostics,
|
|
1627
|
+
summary: summarizeShadowProbe(options.points.length, completed)
|
|
1628
|
+
};
|
|
1700
1629
|
}
|
|
1701
1630
|
function formatBeliefShadowProbePrompt(input) {
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1631
|
+
return [
|
|
1632
|
+
"Return only JSON. Do not include chain-of-thought.",
|
|
1633
|
+
"Infer the agent belief state at this decision boundary using only the context below.",
|
|
1634
|
+
"",
|
|
1635
|
+
`decisionKind: ${input.decisionKind}`,
|
|
1636
|
+
`candidateActions: ${JSON.stringify(input.candidateActions)}`,
|
|
1637
|
+
input.observedAction ? `observedAction: ${JSON.stringify(input.observedAction)}` : "",
|
|
1638
|
+
input.context ? `context:\n${input.context}` : "",
|
|
1639
|
+
"",
|
|
1640
|
+
"Schema:",
|
|
1641
|
+
JSON.stringify({
|
|
1642
|
+
predictedAction: "one candidate action",
|
|
1643
|
+
confidence: "number in [0,1]",
|
|
1644
|
+
beliefSummary: "short outcome-blind summary",
|
|
1645
|
+
uncertainty: ["short uncertainty"],
|
|
1646
|
+
evidenceRefs: ["evidence id"],
|
|
1647
|
+
wouldChangeMindIf: ["observable evidence"],
|
|
1648
|
+
targetProb: "optional number in [0,1]",
|
|
1649
|
+
qHatChosen: "optional number in [0,1], paired with vHatTarget",
|
|
1650
|
+
vHatTarget: "optional number in [0,1], paired with qHatChosen"
|
|
1651
|
+
})
|
|
1652
|
+
].filter(Boolean).join("\n");
|
|
1724
1653
|
}
|
|
1725
1654
|
async function probePoint(point, options) {
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1655
|
+
const diagnostics = [];
|
|
1656
|
+
const candidateActions = uniqueStrings(point.candidateActions ?? []);
|
|
1657
|
+
if ((options.requireCandidateActions ?? true) && candidateActions.length === 0) {
|
|
1658
|
+
diagnostics.push({
|
|
1659
|
+
decisionId: point.id,
|
|
1660
|
+
severity: "warning",
|
|
1661
|
+
reason: "missing candidateActions"
|
|
1662
|
+
});
|
|
1663
|
+
return { diagnostics };
|
|
1664
|
+
}
|
|
1665
|
+
let response;
|
|
1666
|
+
try {
|
|
1667
|
+
response = await options.probe({
|
|
1668
|
+
probeId: options.probeId,
|
|
1669
|
+
decisionId: point.id,
|
|
1670
|
+
runId: point.runId,
|
|
1671
|
+
scenarioId: point.scenarioId,
|
|
1672
|
+
stepIndex: point.stepIndex,
|
|
1673
|
+
decisionKind: point.kind,
|
|
1674
|
+
candidateActions,
|
|
1675
|
+
...options.includeObservedAction ? { observedAction: point.chosenAction } : {},
|
|
1676
|
+
evidence: point.evidence.map((ref) => ({
|
|
1677
|
+
id: ref.id,
|
|
1678
|
+
source: ref.source,
|
|
1679
|
+
...options.includeEvidenceDetail && ref.detail ? { detail: ref.detail } : {},
|
|
1680
|
+
...ref.quality ? { quality: ref.quality } : {}
|
|
1681
|
+
})),
|
|
1682
|
+
context: trimText(await options.contextOf?.(point), options.maxContextChars),
|
|
1683
|
+
metadata: await options.metadataOf?.(point)
|
|
1684
|
+
});
|
|
1685
|
+
} catch (error) {
|
|
1686
|
+
diagnostics.push({
|
|
1687
|
+
decisionId: point.id,
|
|
1688
|
+
severity: "error",
|
|
1689
|
+
reason: `probe threw: ${errorMessage(error)}`
|
|
1690
|
+
});
|
|
1691
|
+
return { diagnostics };
|
|
1692
|
+
}
|
|
1693
|
+
const normalized = normalizeProbeResponse(response, {
|
|
1694
|
+
point,
|
|
1695
|
+
candidateActions,
|
|
1696
|
+
allowOutOfSetActions: options.allowOutOfSetActions ?? false
|
|
1697
|
+
});
|
|
1698
|
+
if (!normalized.record) {
|
|
1699
|
+
diagnostics.push(...normalized.diagnostics);
|
|
1700
|
+
return { diagnostics };
|
|
1701
|
+
}
|
|
1702
|
+
return {
|
|
1703
|
+
record: {
|
|
1704
|
+
probeId: options.probeId,
|
|
1705
|
+
decisionId: point.id,
|
|
1706
|
+
runId: point.runId,
|
|
1707
|
+
scenarioId: point.scenarioId,
|
|
1708
|
+
stepIndex: point.stepIndex,
|
|
1709
|
+
decisionKind: point.kind,
|
|
1710
|
+
candidateActions,
|
|
1711
|
+
observedAction: point.chosenAction,
|
|
1712
|
+
agreesWithObservedAction: normalized.record.predictedAction === point.chosenAction,
|
|
1713
|
+
...options.includeOutcomeInRecord === false ? {} : { outcome: point.outcome },
|
|
1714
|
+
...normalized.record
|
|
1715
|
+
},
|
|
1716
|
+
diagnostics
|
|
1717
|
+
};
|
|
1789
1718
|
}
|
|
1790
1719
|
function normalizeProbeResponse(response, options) {
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1720
|
+
const diagnostics = [];
|
|
1721
|
+
const predictedAction = stringOrNull(response.predictedAction);
|
|
1722
|
+
if (!predictedAction) diagnostics.push({
|
|
1723
|
+
decisionId: options.point.id,
|
|
1724
|
+
severity: "error",
|
|
1725
|
+
reason: "missing predictedAction"
|
|
1726
|
+
});
|
|
1727
|
+
else if (!options.allowOutOfSetActions && options.candidateActions.length > 0 && !options.candidateActions.includes(predictedAction)) diagnostics.push({
|
|
1728
|
+
decisionId: options.point.id,
|
|
1729
|
+
severity: "error",
|
|
1730
|
+
reason: `predictedAction ${predictedAction} is not in candidateActions`
|
|
1731
|
+
});
|
|
1732
|
+
if (!isUnitProbability(response.confidence)) diagnostics.push({
|
|
1733
|
+
decisionId: options.point.id,
|
|
1734
|
+
severity: "error",
|
|
1735
|
+
reason: `invalid confidence ${String(response.confidence)}`
|
|
1736
|
+
});
|
|
1737
|
+
if (response.targetProb !== void 0 && !isUnitProbability(response.targetProb)) diagnostics.push({
|
|
1738
|
+
decisionId: options.point.id,
|
|
1739
|
+
severity: "error",
|
|
1740
|
+
reason: `invalid targetProb ${String(response.targetProb)}`
|
|
1741
|
+
});
|
|
1742
|
+
const hasQHatChosen = response.qHatChosen !== void 0 && response.qHatChosen !== null;
|
|
1743
|
+
const hasVHatTarget = response.vHatTarget !== void 0 && response.vHatTarget !== null;
|
|
1744
|
+
if (hasQHatChosen !== hasVHatTarget) diagnostics.push({
|
|
1745
|
+
decisionId: options.point.id,
|
|
1746
|
+
severity: "error",
|
|
1747
|
+
reason: "qHatChosen and vHatTarget must be supplied together"
|
|
1748
|
+
});
|
|
1749
|
+
if (hasQHatChosen && !isUnitProbability(response.qHatChosen)) diagnostics.push({
|
|
1750
|
+
decisionId: options.point.id,
|
|
1751
|
+
severity: "error",
|
|
1752
|
+
reason: `invalid qHatChosen ${String(response.qHatChosen)}`
|
|
1753
|
+
});
|
|
1754
|
+
if (hasVHatTarget && !isUnitProbability(response.vHatTarget)) diagnostics.push({
|
|
1755
|
+
decisionId: options.point.id,
|
|
1756
|
+
severity: "error",
|
|
1757
|
+
reason: `invalid vHatTarget ${String(response.vHatTarget)}`
|
|
1758
|
+
});
|
|
1759
|
+
if (diagnostics.length > 0 || !predictedAction) return { diagnostics };
|
|
1760
|
+
return {
|
|
1761
|
+
record: {
|
|
1762
|
+
predictedAction,
|
|
1763
|
+
confidence: response.confidence,
|
|
1764
|
+
...response.beliefSummary ? { beliefSummary: trimText(response.beliefSummary, 2e3) } : {},
|
|
1765
|
+
uncertainty: compactStrings(response.uncertainty),
|
|
1766
|
+
evidenceRefs: compactStrings(response.evidenceRefs),
|
|
1767
|
+
wouldChangeMindIf: compactStrings(response.wouldChangeMindIf),
|
|
1768
|
+
...response.targetProb !== void 0 ? { targetProb: response.targetProb } : {},
|
|
1769
|
+
...response.qHatChosen !== void 0 ? { qHatChosen: response.qHatChosen } : {},
|
|
1770
|
+
...response.vHatTarget !== void 0 ? { vHatTarget: response.vHatTarget } : {},
|
|
1771
|
+
...response.metadata ? { metadata: response.metadata } : {}
|
|
1772
|
+
},
|
|
1773
|
+
diagnostics
|
|
1774
|
+
};
|
|
1842
1775
|
}
|
|
1843
1776
|
function summarizeShadowProbe(attempted, records) {
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1777
|
+
const confidences = records.map((record) => record.confidence);
|
|
1778
|
+
const agreements = records.filter((record) => record.agreesWithObservedAction).length;
|
|
1779
|
+
return {
|
|
1780
|
+
attempted,
|
|
1781
|
+
completed: records.length,
|
|
1782
|
+
dropped: attempted - records.length,
|
|
1783
|
+
withOutcome: records.filter((record) => record.outcome !== void 0).length,
|
|
1784
|
+
withTargetProb: records.filter((record) => record.targetProb !== void 0).length,
|
|
1785
|
+
meanConfidence: confidences.length > 0 ? mean(confidences) : null,
|
|
1786
|
+
observedAgreementRate: records.length > 0 ? agreements / records.length : null
|
|
1787
|
+
};
|
|
1855
1788
|
}
|
|
1856
1789
|
function isUnitProbability(value) {
|
|
1857
|
-
|
|
1790
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
|
|
1858
1791
|
}
|
|
1859
1792
|
function boundedInteger(value, min, max) {
|
|
1860
|
-
|
|
1861
|
-
|
|
1793
|
+
if (!Number.isFinite(value)) return min;
|
|
1794
|
+
return Math.max(min, Math.min(max, Math.floor(value)));
|
|
1862
1795
|
}
|
|
1863
1796
|
function compactStrings(values, maxItems = 12) {
|
|
1864
|
-
|
|
1865
|
-
|
|
1797
|
+
if (!Array.isArray(values)) return [];
|
|
1798
|
+
return values.filter((value) => typeof value === "string" && value.length > 0).slice(0, maxItems).map((value) => trimText(value, 500) ?? "").filter(Boolean);
|
|
1866
1799
|
}
|
|
1867
|
-
function
|
|
1868
|
-
|
|
1800
|
+
function uniqueStrings(values) {
|
|
1801
|
+
return [...new Set(values.filter((value) => value.length > 0))];
|
|
1869
1802
|
}
|
|
1870
1803
|
function stringOrNull(value) {
|
|
1871
|
-
|
|
1872
|
-
}
|
|
1873
|
-
function
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
}
|
|
1877
|
-
function
|
|
1878
|
-
|
|
1879
|
-
}
|
|
1880
|
-
function
|
|
1881
|
-
|
|
1882
|
-
}
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
BELIEF_EVIDENCE_QUALITIES,
|
|
1887
|
-
BELIEF_EVIDENCE_SOURCES,
|
|
1888
|
-
analyzeBeliefDecisionCorpus,
|
|
1889
|
-
analyzeBeliefPolicy,
|
|
1890
|
-
beliefDecisionsToOffPolicyTrajectories,
|
|
1891
|
-
buildBeliefDecisionResearchEvidencePacket,
|
|
1892
|
-
buildCodeAgentBeliefEvidenceCorpus,
|
|
1893
|
-
buildRuntimeBeliefPhase0Measurement,
|
|
1894
|
-
buildRuntimeBenchmarkBeliefPhase0Measurement,
|
|
1895
|
-
calibrateBeliefDecisions,
|
|
1896
|
-
createBeliefRuntimeHookCollector,
|
|
1897
|
-
embeddedBeliefOpeTargetPolicy,
|
|
1898
|
-
evaluateBeliefOffPolicy,
|
|
1899
|
-
evaluateBeliefSelectivePolicy,
|
|
1900
|
-
extractBeliefDecisionPoints,
|
|
1901
|
-
extractCodeAgentBeliefDecisionPoints,
|
|
1902
|
-
formatBeliefShadowProbePrompt,
|
|
1903
|
-
inventoryBeliefDecisionPoints,
|
|
1904
|
-
isBeliefDecisionKind,
|
|
1905
|
-
isBeliefEvidenceSource,
|
|
1906
|
-
runBeliefShadowProbe,
|
|
1907
|
-
runtimeDecisionPointToBeliefDecisionPoint,
|
|
1908
|
-
runtimeDecisionPointToBeliefShadowProbeInput,
|
|
1909
|
-
selectBeliefDecisionTarget,
|
|
1910
|
-
thresholdSelectivePolicy
|
|
1911
|
-
};
|
|
1804
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
1805
|
+
}
|
|
1806
|
+
function trimText(value, maxChars = DEFAULT_MAX_CONTEXT_CHARS) {
|
|
1807
|
+
if (!value) return void 0;
|
|
1808
|
+
return value.length > maxChars ? value.slice(value.length - maxChars) : value;
|
|
1809
|
+
}
|
|
1810
|
+
function mean(values) {
|
|
1811
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
1812
|
+
}
|
|
1813
|
+
function errorMessage(error) {
|
|
1814
|
+
return error instanceof Error ? error.message : String(error);
|
|
1815
|
+
}
|
|
1816
|
+
//#endregion
|
|
1817
|
+
export { BELIEF_DECISION_KINDS, BELIEF_EVALUATION_CRITERIA, BELIEF_EVIDENCE_QUALITIES, BELIEF_EVIDENCE_SOURCES, analyzeBeliefDecisionCorpus, analyzeBeliefPolicy, beliefDecisionsToOffPolicyTrajectories, buildBeliefDecisionResearchEvidencePacket, buildCodeAgentBeliefEvidenceCorpus, buildRuntimeBeliefPhase0Measurement, buildRuntimeBenchmarkBeliefPhase0Measurement, calibrateBeliefDecisions, createBeliefRuntimeHookCollector, embeddedBeliefOpeTargetPolicy, evaluateBeliefOffPolicy, evaluateBeliefSelectivePolicy, extractBeliefDecisionPoints, extractCodeAgentBeliefDecisionPoints, formatBeliefShadowProbePrompt, inventoryBeliefDecisionPoints, isBeliefDecisionKind, isBeliefEvidenceSource, runBeliefShadowProbe, runtimeDecisionPointToBeliefDecisionPoint, runtimeDecisionPointToBeliefShadowProbeInput, selectBeliefDecisionTarget, thresholdSelectivePolicy };
|
|
1818
|
+
|
|
1912
1819
|
//# sourceMappingURL=index.js.map
|