@tangle-network/agent-eval 0.128.2 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +279 -0
- package/README.md +19 -0
- package/dist/active-curriculum-C4mk67HP.js +214 -0
- package/dist/active-curriculum-C4mk67HP.js.map +1 -0
- package/dist/adversarial-smnADNFS.d.ts +21 -0
- package/dist/adversarial-smnADNFS.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +83 -2932
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +319 -364
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyst-BkTS3C58.d.ts +89 -0
- package/dist/analyst-BkTS3C58.d.ts.map +1 -0
- package/dist/analyst-LsnNpSkm.js +152 -0
- package/dist/analyst-LsnNpSkm.js.map +1 -0
- package/dist/analyze-runs-C1CavBMk.js +1067 -0
- package/dist/analyze-runs-C1CavBMk.js.map +1 -0
- package/dist/analyze-runs-FsgYCinh.d.ts +72 -0
- package/dist/analyze-runs-FsgYCinh.d.ts.map +1 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts +54 -0
- package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +81 -79
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.js +209 -193
- package/dist/authenticity/index.js.map +1 -1
- package/dist/baseline-HsBvw_dk.js +550 -0
- package/dist/baseline-HsBvw_dk.js.map +1 -0
- package/dist/baseline-hG3K85h4.d.ts +125 -0
- package/dist/baseline-hG3K85h4.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +448 -1205
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1617 -1710
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +2 -894
- package/dist/benchmarks/index.js +2 -59
- package/dist/benchmarks-DviOvUNr.js +754 -0
- package/dist/benchmarks-DviOvUNr.js.map +1 -0
- package/dist/builder-eval/index.d.ts +150 -662
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +356 -345
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/calibration-CNWWA6K8.js +94 -0
- package/dist/calibration-CNWWA6K8.js.map +1 -0
- package/dist/campaign/index.d.ts +5 -6390
- package/dist/campaign/index.js +3 -212
- package/dist/campaign-CBKZvQ1H.js +3885 -0
- package/dist/campaign-CBKZvQ1H.js.map +1 -0
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +164 -174
- package/dist/cli.js.map +1 -1
- package/dist/client-C97NMzqi.d.ts +581 -0
- package/dist/client-C97NMzqi.d.ts.map +1 -0
- package/dist/client-CYzbdJOZ.js +637 -0
- package/dist/client-CYzbdJOZ.js.map +1 -0
- package/dist/code-agent-session-BjkMTQ7H.js +1390 -0
- package/dist/code-agent-session-BjkMTQ7H.js.map +1 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts +143 -0
- package/dist/code-agent-session-DqqgOJaz.d.ts.map +1 -0
- package/dist/concurrency-DIxRZF_J.js +85 -0
- package/dist/concurrency-DIxRZF_J.js.map +1 -0
- package/dist/contract/index.d.ts +645 -5605
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +1730 -1937
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +3 -1030
- package/dist/control.js +2 -32
- package/dist/cost-ledger-DIgQUFZZ.js +801 -0
- package/dist/cost-ledger-DIgQUFZZ.js.map +1 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts +236 -0
- package/dist/cost-ledger-Dye6jCgg.d.ts.map +1 -0
- package/dist/dataset-BvtnC8Dc.d.ts +115 -0
- package/dist/dataset-BvtnC8Dc.d.ts.map +1 -0
- package/dist/default-registry-C-vFCSEc.js +2579 -0
- package/dist/default-registry-C-vFCSEc.js.map +1 -0
- package/dist/default-registry-CNPo-Vsb.d.ts +540 -0
- package/dist/default-registry-CNPo-Vsb.d.ts.map +1 -0
- package/dist/emitter-CPBAhxum.js +266 -0
- package/dist/emitter-CPBAhxum.js.map +1 -0
- package/dist/emitter-DGQGoLyj.d.ts +113 -0
- package/dist/emitter-DGQGoLyj.d.ts.map +1 -0
- package/dist/errors-8YnH8WlF.js +64 -0
- package/dist/errors-8YnH8WlF.js.map +1 -0
- package/dist/errors-CEk209JS.d.ts +76 -0
- package/dist/errors-CEk209JS.d.ts.map +1 -0
- package/dist/eval-campaign-DEm6c8ru.js +349 -0
- package/dist/eval-campaign-DEm6c8ru.js.map +1 -0
- package/dist/execution-tracks-CpgFPpS5.js +93 -0
- package/dist/execution-tracks-CpgFPpS5.js.map +1 -0
- package/dist/exporters-q9iL-2Jf.js +148 -0
- package/dist/exporters-q9iL-2Jf.js.map +1 -0
- package/dist/extract-usage-BrQ8mCLX.js +155 -0
- package/dist/extract-usage-BrQ8mCLX.js.map +1 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts +59 -0
- package/dist/failure-cluster-CqcvCcdR.d.ts.map +1 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts +340 -0
- package/dist/feedback-trajectory-CVaeREXV.d.ts.map +1 -0
- package/dist/fuzz.d.ts +320 -646
- package/dist/fuzz.d.ts.map +1 -0
- package/dist/fuzz.js +670 -617
- package/dist/fuzz.js.map +1 -1
- package/dist/hf-dataset-DBJXXoY1.js +763 -0
- package/dist/hf-dataset-DBJXXoY1.js.map +1 -0
- package/dist/hosted/index.d.ts +11 -831
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/index.js +2 -37
- package/dist/index-2JJSA6-r2.d.ts +926 -0
- package/dist/index-2JJSA6-r2.d.ts.map +1 -0
- package/dist/index-6N0aYmpW.d.ts +217 -0
- package/dist/index-6N0aYmpW.d.ts.map +1 -0
- package/dist/index-BAvgST_9.d.ts +131 -0
- package/dist/index-BAvgST_9.d.ts.map +1 -0
- package/dist/index-BvnJuTGD.d.ts +68 -0
- package/dist/index-BvnJuTGD.d.ts.map +1 -0
- package/dist/index-C61Wi7yg.d.ts +547 -0
- package/dist/index-C61Wi7yg.d.ts.map +1 -0
- package/dist/index-CAPUUKaM.d.ts +335 -0
- package/dist/index-CAPUUKaM.d.ts.map +1 -0
- package/dist/index-DE5fb3EC.d.ts +2244 -0
- package/dist/index-DE5fb3EC.d.ts.map +1 -0
- package/dist/index-DSC51roc.d.ts +102 -0
- package/dist/index-DSC51roc.d.ts.map +1 -0
- package/dist/index.d.ts +3776 -15120
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11185 -11191
- package/dist/index.js.map +1 -1
- package/dist/integrity-BzRbCHzi.js +424 -0
- package/dist/integrity-BzRbCHzi.js.map +1 -0
- package/dist/integrity-rmVhXWA7.d.ts +61 -0
- package/dist/integrity-rmVhXWA7.d.ts.map +1 -0
- package/dist/judge-calibration-DFtEMlde.d.ts +146 -0
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -0
- package/dist/ledger-core/index.d.ts +2 -0
- package/dist/ledger-core/index.js +2 -0
- package/dist/ledger-core-DtZz1RG0.js +388 -0
- package/dist/ledger-core-DtZz1RG0.js.map +1 -0
- package/dist/llm-client--GR4JbZE.js +687 -0
- package/dist/llm-client--GR4JbZE.js.map +1 -0
- package/dist/llm-client-B_nIBlYo.d.ts +290 -0
- package/dist/llm-client-B_nIBlYo.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +3 -155
- package/dist/matrix/index.js +2 -8
- package/dist/matrix-BzQnu2S6.js +270 -0
- package/dist/matrix-BzQnu2S6.js.map +1 -0
- package/dist/meta-eval/index.d.ts +4 -1027
- package/dist/meta-eval/index.js +393 -390
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/metrics-C9YY1OcL.js +239 -0
- package/dist/metrics-C9YY1OcL.js.map +1 -0
- package/dist/mint-yN2M2eh0.js +201 -0
- package/dist/mint-yN2M2eh0.js.map +1 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts +138 -0
- package/dist/multi-layer-verifier-BHY1gWAc.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +273 -481
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js +595 -548
- package/dist/multishot/index.js.map +1 -1
- package/dist/off-policy-DvgzvtIx.js +220 -0
- package/dist/off-policy-DvgzvtIx.js.map +1 -0
- package/dist/off-policy-mskQw8Mb.d.ts +153 -0
- package/dist/off-policy-mskQw8Mb.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-BGrHeDu3.js +318 -0
- package/dist/opencode-sqlite-BGrHeDu3.js.map +1 -0
- package/dist/outcome-store-BYHIuO0e.d.ts +65 -0
- package/dist/outcome-store-BYHIuO0e.d.ts.map +1 -0
- package/dist/outcome-store-ChBKlTd_.js +75 -0
- package/dist/outcome-store-ChBKlTd_.js.map +1 -0
- package/dist/paired-arms-D9D0wXj2.js +260 -0
- package/dist/paired-arms-D9D0wXj2.js.map +1 -0
- package/dist/pipelines/index.d.ts +95 -532
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +497 -478
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pre-registration-DakwTRXk.js +96 -0
- package/dist/pre-registration-DakwTRXk.js.map +1 -0
- package/dist/propose-review-control-LhIWGzHl.js +1458 -0
- package/dist/propose-review-control-LhIWGzHl.js.map +1 -0
- package/dist/query-CJ_DX8vl.d.ts +28 -0
- package/dist/query-CJ_DX8vl.d.ts.map +1 -0
- package/dist/query-Di7eEQ79.js +83 -0
- package/dist/query-Di7eEQ79.js.map +1 -0
- package/dist/raw-provider-sink-BQd7mzyT.js +201 -0
- package/dist/raw-provider-sink-BQd7mzyT.js.map +1 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts +134 -0
- package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +1 -0
- package/dist/redact-7Aq1ukl-.js +107 -0
- package/dist/redact-7Aq1ukl-.js.map +1 -0
- package/dist/release-report-Cz9NKH39.js +603 -0
- package/dist/release-report-Cz9NKH39.js.map +1 -0
- package/dist/release-report-mvB2G4_J.d.ts +244 -0
- package/dist/release-report-mvB2G4_J.d.ts.map +1 -0
- package/dist/replay-GnyotH0J.js +1741 -0
- package/dist/replay-GnyotH0J.js.map +1 -0
- package/dist/replay-OoidtG1E.d.ts +749 -0
- package/dist/replay-OoidtG1E.d.ts.map +1 -0
- package/dist/reporting.d.ts +6 -1298
- package/dist/reporting.js +6 -50
- package/dist/researcher-CwTdwXG1.d.ts +314 -0
- package/dist/researcher-CwTdwXG1.d.ts.map +1 -0
- package/dist/reward-hacking-eAnOsynk.d.ts +224 -0
- package/dist/reward-hacking-eAnOsynk.d.ts.map +1 -0
- package/dist/reward-hacking-qipEpKvY.js +596 -0
- package/dist/reward-hacking-qipEpKvY.js.map +1 -0
- package/dist/reward-nw2xZGZG.js +137 -0
- package/dist/reward-nw2xZGZG.js.map +1 -0
- package/dist/rl.d.ts +916 -3596
- package/dist/rl.d.ts.map +1 -0
- package/dist/rl.js +2362 -1751
- package/dist/rl.js.map +1 -1
- package/dist/rolldown-runtime-8H4AJuhK.js +14 -0
- package/dist/rollout/index.d.ts +3 -1048
- package/dist/rollout/index.js +8 -110
- package/dist/rollout-BOYjemfR.js +624 -0
- package/dist/rollout-BOYjemfR.js.map +1 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js +141 -0
- package/dist/rubric-predictive-validity-B3xmbmS1.js.map +1 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts +74 -0
- package/dist/rubric-predictive-validity-Ku_clp_1.d.ts.map +1 -0
- package/dist/run-evidence-ClRX_8A9.d.ts +225 -0
- package/dist/run-evidence-ClRX_8A9.d.ts.map +1 -0
- package/dist/run-record-BuoE80Dq.js +467 -0
- package/dist/run-record-BuoE80Dq.js.map +1 -0
- package/dist/run-record-CnZu_gjl.d.ts +357 -0
- package/dist/run-record-CnZu_gjl.d.ts.map +1 -0
- package/dist/run-score-iEEAWiBY.js +41 -0
- package/dist/run-score-iEEAWiBY.js.map +1 -0
- package/dist/runtime-trajectory-1gyaTOoC.js +93 -0
- package/dist/runtime-trajectory-1gyaTOoC.js.map +1 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts +50 -0
- package/dist/runtime-trajectory-BvSZcCHD.d.ts.map +1 -0
- package/dist/schema-BtVldJ3T.d.ts +206 -0
- package/dist/schema-BtVldJ3T.d.ts.map +1 -0
- package/dist/schema-C6DW4ZHR.js +821 -0
- package/dist/schema-C6DW4ZHR.js.map +1 -0
- package/dist/schema-CRhEY1SO.js +69 -0
- package/dist/schema-CRhEY1SO.js.map +1 -0
- package/dist/schema-Cef2cFmb.d.ts +408 -0
- package/dist/schema-Cef2cFmb.d.ts.map +1 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js +725 -0
- package/dist/semantic-concept-judge-B6cWNJ2K.js.map +1 -0
- package/dist/sequential-Br0mAPHA.js +148 -0
- package/dist/sequential-Br0mAPHA.js.map +1 -0
- package/dist/sequential-CYwq6Ff_.d.ts +141 -0
- package/dist/sequential-CYwq6Ff_.d.ts.map +1 -0
- package/dist/series-convergence-CjO2QdRW.js +43 -0
- package/dist/series-convergence-CjO2QdRW.js.map +1 -0
- package/dist/series-convergence-ofsqPWhs.d.ts +35 -0
- package/dist/series-convergence-ofsqPWhs.d.ts.map +1 -0
- package/dist/server-m5D9cvnG.js +1040 -0
- package/dist/server-m5D9cvnG.js.map +1 -0
- package/dist/skill-usage-C_pXm7lP.d.ts +534 -0
- package/dist/skill-usage-C_pXm7lP.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts +1739 -0
- package/dist/skillopt-optimization-method-B7wX7XkF.d.ts.map +1 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js +7712 -0
- package/dist/skillopt-optimization-method-D4ODwFVV.js.map +1 -0
- package/dist/statistics-Cmj6nynr.d.ts +514 -0
- package/dist/statistics-Cmj6nynr.d.ts.map +1 -0
- package/dist/statistics-CnnxdpOg.js +1437 -0
- package/dist/statistics-CnnxdpOg.js.map +1 -0
- package/dist/store-CT9YIIve.d.ts +117 -0
- package/dist/store-CT9YIIve.d.ts.map +1 -0
- package/dist/store-CxJry_cs.d.ts +229 -0
- package/dist/store-CxJry_cs.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +84 -203
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.js +609 -542
- package/dist/storyboard/index.js.map +1 -1
- package/dist/summary-report-BNs5nmXI.js +862 -0
- package/dist/summary-report-BNs5nmXI.js.map +1 -0
- package/dist/summary-report-Cj9gdw4i.d.ts +407 -0
- package/dist/summary-report-Cj9gdw4i.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -849
- package/dist/supervisor-run/index.js +2 -64
- package/dist/supervisor-run-_lnTLM3z.js +1679 -0
- package/dist/supervisor-run-_lnTLM3z.js.map +1 -0
- package/dist/task-failure-attributes-CQZlB3et.js +311 -0
- package/dist/task-failure-attributes-CQZlB3et.js.map +1 -0
- package/dist/test-graded-scenario-BsqWLmPt.js +318 -0
- package/dist/test-graded-scenario-BsqWLmPt.js.map +1 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts +141 -0
- package/dist/test-graded-scenario-D1TaI2va.d.ts.map +1 -0
- package/dist/tools-BmuN627J.js +1085 -0
- package/dist/tools-BmuN627J.js.map +1 -0
- package/dist/trace-attributes.d.ts +2 -52
- package/dist/trace-attributes.js +131 -61
- package/dist/trace-attributes.js.map +1 -1
- package/dist/traces.d.ts +12 -2365
- package/dist/traces.js +12 -251
- package/dist/trajectory-D_7rLrvE.js +56 -0
- package/dist/trajectory-D_7rLrvE.js.map +1 -0
- package/dist/types-DGsxbAEd.d.ts +387 -0
- package/dist/types-DGsxbAEd.d.ts.map +1 -0
- package/dist/types-k9tZGKUg.d.ts +640 -0
- package/dist/types-k9tZGKUg.d.ts.map +1 -0
- package/dist/verdict-Dps8_okt.d.ts +37 -0
- package/dist/verdict-Dps8_okt.d.ts.map +1 -0
- package/dist/wire/index.d.ts +702 -1174
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -81
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +18 -10
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-2JX3CFMB.js +0 -695
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-2MKQIFS4.js +0 -183
- package/dist/chunk-2MKQIFS4.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-5DTSBUL2.js +0 -159
- package/dist/chunk-5DTSBUL2.js.map +0 -1
- package/dist/chunk-7ZZMD7UK.js +0 -386
- package/dist/chunk-7ZZMD7UK.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-BYT7ELPS.js +0 -1553
- package/dist/chunk-BYT7ELPS.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js +0 -2428
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-DPUHNQLN.js +0 -232
- package/dist/chunk-DPUHNQLN.js.map +0 -1
- package/dist/chunk-DRYIUNWY.js +0 -622
- package/dist/chunk-DRYIUNWY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js +0 -617
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js +0 -2001
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js +0 -1559
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js +0 -171
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MHELPNRP.js +0 -1212
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js +0 -1040
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js +0 -7633
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js +0 -332
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-P5W7RQKK.js +0 -576
- package/dist/chunk-P5W7RQKK.js.map +0 -1
- package/dist/chunk-P6FYH6K4.js +0 -1161
- package/dist/chunk-P6FYH6K4.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js +0 -669
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-S5YLIBFX.js +0 -136
- package/dist/chunk-S5YLIBFX.js.map +0 -1
- package/dist/chunk-SZLVEKMJ.js +0 -1446
- package/dist/chunk-SZLVEKMJ.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TBL77AUT.js +0 -355
- package/dist/chunk-TBL77AUT.js.map +0 -1
- package/dist/chunk-TSN7JT6D.js +0 -1646
- package/dist/chunk-TSN7JT6D.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js +0 -4461
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js +0 -291
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js +0 -163
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js +0 -908
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WGXIEX7P.js +0 -116
- package/dist/chunk-WGXIEX7P.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js +0 -929
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js +0 -695
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js +0 -766
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/chunk-YJBNWCAA.js +0 -1056
- package/dist/chunk-YJBNWCAA.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZUUWPZCV.js +0 -752
- package/dist/chunk-ZUUWPZCV.js.map +0 -1
- package/dist/control.js.map +0 -1
- package/dist/hosted/index.js.map +0 -1
- package/dist/matrix/index.js.map +0 -1
- package/dist/reporting.js.map +0 -1
- package/dist/rollout/index.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- package/dist/run-campaign-ISHFZ7FJ.js.map +0 -1
- package/dist/supervisor-run/index.js.map +0 -1
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.js.map +0 -1
package/dist/chunk-EJGRPCO3.js
DELETED
|
@@ -1,617 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
ROLLOUT_ROLES,
|
|
3
|
-
ROLLOUT_SCHEMA,
|
|
4
|
-
assertRolloutLine,
|
|
5
|
-
isTrainableSplit
|
|
6
|
-
} from "./chunk-UWZZKKU7.js";
|
|
7
|
-
|
|
8
|
-
// src/rollout/exporters.ts
|
|
9
|
-
function toSftRows(lines, options = {}) {
|
|
10
|
-
return lines.filter((line) => isTrainingLineEligible(line, options) && line.messages.length > 0).map((line) => ({
|
|
11
|
-
messages: line.messages,
|
|
12
|
-
metadata: {
|
|
13
|
-
rollout_id: line.rollout_id,
|
|
14
|
-
run_id: line.run_id,
|
|
15
|
-
candidate_id: line.candidate_id ?? null,
|
|
16
|
-
instance_id: line.task.instance_id,
|
|
17
|
-
reward: line.outcome.reward
|
|
18
|
-
}
|
|
19
|
-
}));
|
|
20
|
-
}
|
|
21
|
-
function toRewardRows(lines, options = {}) {
|
|
22
|
-
return lines.filter(
|
|
23
|
-
(line) => isTrainingLineEligible(line, options) && line.messages.some(
|
|
24
|
-
(message) => message.role === "user" && typeof message.content === "string" && message.content.length > 0
|
|
25
|
-
)
|
|
26
|
-
).map((line) => ({
|
|
27
|
-
prompt: line.messages.find((m) => m.role === "user")?.content ?? "",
|
|
28
|
-
steps: line.steps ?? [],
|
|
29
|
-
reward: line.outcome.reward,
|
|
30
|
-
metadata: {
|
|
31
|
-
rollout_id: line.rollout_id,
|
|
32
|
-
run_id: line.run_id,
|
|
33
|
-
candidate_id: line.candidate_id ?? null,
|
|
34
|
-
instance_id: line.task.instance_id,
|
|
35
|
-
split: line.task.split
|
|
36
|
-
}
|
|
37
|
-
}));
|
|
38
|
-
}
|
|
39
|
-
function firstAssistantIndex(messages) {
|
|
40
|
-
const index = messages.findIndex((m) => m.role === "assistant");
|
|
41
|
-
return index === -1 ? messages.length : index;
|
|
42
|
-
}
|
|
43
|
-
function toVerifiersRolloutOutput(line) {
|
|
44
|
-
const split = firstAssistantIndex(line.messages);
|
|
45
|
-
return {
|
|
46
|
-
prompt: line.messages.slice(0, split),
|
|
47
|
-
completion: line.messages.slice(split),
|
|
48
|
-
reward: line.outcome.reward,
|
|
49
|
-
metrics: line.outcome.metrics,
|
|
50
|
-
tool_defs: line.tool_defs,
|
|
51
|
-
token_usage: {
|
|
52
|
-
input_tokens: line.cost.tokens_in,
|
|
53
|
-
output_tokens: line.cost.tokens_out,
|
|
54
|
-
reasoning_tokens: line.cost.tokens_reasoning,
|
|
55
|
-
cache_read_tokens: line.cost.cache_read,
|
|
56
|
-
cache_write_tokens: line.cost.cache_write
|
|
57
|
-
},
|
|
58
|
-
info: {
|
|
59
|
-
task: line.task,
|
|
60
|
-
policy: line.policy,
|
|
61
|
-
rollout_id: line.rollout_id,
|
|
62
|
-
run_id: line.run_id,
|
|
63
|
-
experiment_id: line.experiment_id ?? null,
|
|
64
|
-
candidate_id: line.candidate_id ?? null,
|
|
65
|
-
generation: line.generation,
|
|
66
|
-
candidate_index: line.candidate_index,
|
|
67
|
-
role: line.role
|
|
68
|
-
}
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
|
-
function toVerifiersRolloutOutputs(lines, options = {}) {
|
|
72
|
-
return lines.filter(
|
|
73
|
-
(line) => isTrainingLineEligible(line, options) && firstAssistantIndex(line.messages) > 0 && firstAssistantIndex(line.messages) < line.messages.length
|
|
74
|
-
).map(toVerifiersRolloutOutput);
|
|
75
|
-
}
|
|
76
|
-
function toRftItem(line) {
|
|
77
|
-
const split = firstAssistantIndex(line.messages);
|
|
78
|
-
return {
|
|
79
|
-
messages: line.messages.slice(0, split),
|
|
80
|
-
reference: {
|
|
81
|
-
reward: line.outcome.reward,
|
|
82
|
-
reward_source: line.outcome.reward_source,
|
|
83
|
-
verdict: line.outcome.verdict,
|
|
84
|
-
instance_id: line.task.instance_id,
|
|
85
|
-
suite: line.task.suite,
|
|
86
|
-
split: line.task.split,
|
|
87
|
-
rollout_id: line.rollout_id
|
|
88
|
-
}
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
function toRftItems(lines, options = {}) {
|
|
92
|
-
return lines.filter(
|
|
93
|
-
(line) => isTrainingLineEligible(line, options) && line.messages.length > 0 && firstAssistantIndex(line.messages) > 0
|
|
94
|
-
).map(toRftItem);
|
|
95
|
-
}
|
|
96
|
-
function toJsonl(rows) {
|
|
97
|
-
return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length ? "\n" : "");
|
|
98
|
-
}
|
|
99
|
-
function isTrainingLineEligible(line, options) {
|
|
100
|
-
const minimumQualityExclusive = options.minimumQualityExclusive ?? 0;
|
|
101
|
-
if (!Number.isFinite(minimumQualityExclusive)) {
|
|
102
|
-
throw new Error("minimumQualityExclusive must be finite");
|
|
103
|
-
}
|
|
104
|
-
const reward = line.outcome.reward;
|
|
105
|
-
if (reward === null) return false;
|
|
106
|
-
if (!Number.isFinite(reward)) {
|
|
107
|
-
throw new Error(`training reward for rollout "${line.rollout_id}" must be finite`);
|
|
108
|
-
}
|
|
109
|
-
if (reward <= minimumQualityExclusive) return false;
|
|
110
|
-
if (!line.outcome.is_completed || line.outcome.is_truncated || line.outcome.error !== null) {
|
|
111
|
-
return false;
|
|
112
|
-
}
|
|
113
|
-
if (line.outcome.realness_gated === true) return false;
|
|
114
|
-
if (line.task.split === "search") return true;
|
|
115
|
-
return line.task.split === "holdout" && options.allowHeldOutTrainingData === true;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
// src/rollout/ledger.ts
|
|
119
|
-
import { appendFile, mkdir, readFile, writeFile } from "fs/promises";
|
|
120
|
-
import { dirname } from "path";
|
|
121
|
-
function serialize(lines) {
|
|
122
|
-
for (const [i, line] of lines.entries()) assertRolloutLine(line, `rollout line [${i}]`);
|
|
123
|
-
return lines.map((line) => JSON.stringify(line)).join("\n") + (lines.length > 0 ? "\n" : "");
|
|
124
|
-
}
|
|
125
|
-
async function writeRolloutLedger(path, lines) {
|
|
126
|
-
const payload = serialize(lines);
|
|
127
|
-
await mkdir(dirname(path), { recursive: true });
|
|
128
|
-
await writeFile(path, payload);
|
|
129
|
-
}
|
|
130
|
-
async function appendRolloutLines(path, lines) {
|
|
131
|
-
if (lines.length === 0) return;
|
|
132
|
-
const payload = serialize(lines);
|
|
133
|
-
await mkdir(dirname(path), { recursive: true });
|
|
134
|
-
await appendFile(path, payload);
|
|
135
|
-
}
|
|
136
|
-
async function readRolloutLedger(path) {
|
|
137
|
-
const raw = await readFile(path, "utf8");
|
|
138
|
-
const lines = [];
|
|
139
|
-
const rawLines = raw.split("\n");
|
|
140
|
-
for (let i = 0; i < rawLines.length; i++) {
|
|
141
|
-
const text = rawLines[i];
|
|
142
|
-
if (!text?.trim()) continue;
|
|
143
|
-
let parsed;
|
|
144
|
-
try {
|
|
145
|
-
parsed = JSON.parse(text);
|
|
146
|
-
} catch (error) {
|
|
147
|
-
throw new Error(
|
|
148
|
-
`${path}:${i + 1}: malformed JSON \u2014 ${error instanceof Error ? error.message : String(error)}`
|
|
149
|
-
);
|
|
150
|
-
}
|
|
151
|
-
assertRolloutLine(parsed, `${path}:${i + 1}`);
|
|
152
|
-
lines.push(parsed);
|
|
153
|
-
}
|
|
154
|
-
return lines;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
// src/rollout/release/card.ts
|
|
158
|
-
var RELEASE_FORMATS = ["sft", "verifiers", "rft", "raw"];
|
|
159
|
-
var FORMAT_FILES = {
|
|
160
|
-
sft: "sft/train.jsonl",
|
|
161
|
-
verifiers: "verifiers/train.jsonl",
|
|
162
|
-
rft: "rft/train.jsonl",
|
|
163
|
-
raw: "raw/train.jsonl"
|
|
164
|
-
};
|
|
165
|
-
var FORMAT_DESCRIPTIONS = {
|
|
166
|
-
sft: "Successful trainable-split transcripts (`reward >= 1`, never realness-gated) as `{messages, metadata}` chat JSONL.",
|
|
167
|
-
verifiers: "Prime Intellect verifiers `RolloutOutput`: prompt/completion split at the first assistant turn, plus reward, metrics, tool defs, and token usage.",
|
|
168
|
-
rft: "OpenAI RFT items: prompt turns plus `reference.*` verdict fields for a grader (completions are re-sampled during RFT).",
|
|
169
|
-
raw: `Full \`${ROLLOUT_SCHEMA}\` ledger lines (scrubbed), one per agent invocation.`
|
|
170
|
-
};
|
|
171
|
-
function unique(values) {
|
|
172
|
-
return [...new Set(values.filter((v) => v !== null))].sort();
|
|
173
|
-
}
|
|
174
|
-
function formatReward(reward) {
|
|
175
|
-
if (reward === null) return "null";
|
|
176
|
-
return Number.isInteger(reward) ? String(reward) : reward.toFixed(4);
|
|
177
|
-
}
|
|
178
|
-
function markdownTable(header, rows) {
|
|
179
|
-
return [
|
|
180
|
-
`| ${header.join(" | ")} |`,
|
|
181
|
-
`| ${header.map(() => "---").join(" | ")} |`,
|
|
182
|
-
...rows.map((row) => `| ${row.join(" | ")} |`)
|
|
183
|
-
].join("\n");
|
|
184
|
-
}
|
|
185
|
-
function roleRewardRows(lines) {
|
|
186
|
-
const counts = /* @__PURE__ */ new Map();
|
|
187
|
-
for (const line of lines) {
|
|
188
|
-
const byReward = counts.get(line.role) ?? /* @__PURE__ */ new Map();
|
|
189
|
-
const key = formatReward(line.outcome.reward);
|
|
190
|
-
byReward.set(key, (byReward.get(key) ?? 0) + 1);
|
|
191
|
-
counts.set(line.role, byReward);
|
|
192
|
-
}
|
|
193
|
-
const rows = [];
|
|
194
|
-
for (const role of ROLLOUT_ROLES) {
|
|
195
|
-
const byReward = counts.get(role);
|
|
196
|
-
if (!byReward) continue;
|
|
197
|
-
const keys = [...byReward.keys()].sort((a, b) => {
|
|
198
|
-
if (a === "null") return 1;
|
|
199
|
-
if (b === "null") return -1;
|
|
200
|
-
return Number(b) - Number(a);
|
|
201
|
-
});
|
|
202
|
-
for (const key of keys) rows.push([role, key, String(byReward.get(key))]);
|
|
203
|
-
}
|
|
204
|
-
return rows;
|
|
205
|
-
}
|
|
206
|
-
function buildDatasetCard(inputs) {
|
|
207
|
-
const { lines, formats, includeProposers, sourceFiles, scrubTotals, excluded, formatCounts } = inputs;
|
|
208
|
-
const runIds = unique(lines.map((line) => line.run_id));
|
|
209
|
-
const generations = [...new Set(lines.map((line) => line.generation))].filter((g) => g !== null).sort((a, b) => a - b);
|
|
210
|
-
const models = unique(lines.map((line) => line.policy.model));
|
|
211
|
-
const harnesses = unique(lines.map((line) => line.policy.harness));
|
|
212
|
-
const captures = unique(lines.map((line) => line.provenance.capture));
|
|
213
|
-
const rewardSources = unique(lines.map((line) => line.outcome.reward_source));
|
|
214
|
-
const gapLines = lines.filter((line) => line.messages.length === 0).length;
|
|
215
|
-
const configs = formats.map(
|
|
216
|
-
(format) => [
|
|
217
|
-
` - config_name: ${format}`,
|
|
218
|
-
" data_files:",
|
|
219
|
-
" - split: train",
|
|
220
|
-
` path: ${FORMAT_FILES[format]}`
|
|
221
|
-
].join("\n")
|
|
222
|
-
).join("\n");
|
|
223
|
-
const frontmatter = [
|
|
224
|
-
"---",
|
|
225
|
-
"license: unknown",
|
|
226
|
-
"pretty_name: Tangle rollout ledger \u2014 agent trajectories",
|
|
227
|
-
"configs:",
|
|
228
|
-
configs,
|
|
229
|
-
"---"
|
|
230
|
-
].join("\n");
|
|
231
|
-
const formatsTable = markdownTable(
|
|
232
|
-
["config", "path", "rows", "contents"],
|
|
233
|
-
formats.map((format) => [
|
|
234
|
-
format,
|
|
235
|
-
`\`${FORMAT_FILES[format]}\``,
|
|
236
|
-
String(formatCounts[format] ?? 0),
|
|
237
|
-
FORMAT_DESCRIPTIONS[format]
|
|
238
|
-
])
|
|
239
|
-
);
|
|
240
|
-
const countsTable = markdownTable(["role", "reward", "lines"], roleRewardRows(lines));
|
|
241
|
-
const scrubTable = markdownTable(
|
|
242
|
-
["rule", "rewrites"],
|
|
243
|
-
Object.entries(scrubTotals).map(([rule, count]) => [rule, String(count)])
|
|
244
|
-
);
|
|
245
|
-
const proposerNote = includeProposers ? "Proposer sessions are INCLUDED (`--include-proposers`); their transcripts contain improvement-loop harness source." : `Proposer sessions are excluded by default (${excluded.proposers} lines dropped); they contain improvement-loop harness source. Rebuild with \`--include-proposers\` to keep them.`;
|
|
246
|
-
return `${frontmatter}
|
|
247
|
-
|
|
248
|
-
# Tangle rollout ledger \u2014 agent trajectories
|
|
249
|
-
|
|
250
|
-
One line per agent invocation (supervisor episode, worker session, proposer shot, judge call, analyst pass) captured by the \`${ROLLOUT_SCHEMA}\` rollout ledger, labeled with improvement-loop coordinates and the official-judge reward, with the full message transcript inline.
|
|
251
|
-
|
|
252
|
-
This release contains the **trainable split only** (\`search\`). Holdout, dev, and canary splits are structurally excluded at build time, and the build additionally drops any non-trainable line as a fail-closed filter (${excluded.nonTrain} dropped here).
|
|
253
|
-
|
|
254
|
-
## Formats
|
|
255
|
-
|
|
256
|
-
${formatsTable}
|
|
257
|
-
|
|
258
|
-
## Schema (\`${ROLLOUT_SCHEMA}\`)
|
|
259
|
-
|
|
260
|
-
Each raw line carries:
|
|
261
|
-
|
|
262
|
-
- \`rollout_id\` / \`parent_rollout_id\` \u2014 invocation identity; workers point at their spawning supervisor episode.
|
|
263
|
-
- \`run_id\`, \`experiment_id\`, \`candidate_id\` \u2014 run/experiment/candidate identity from the producing RunRecord, when present.
|
|
264
|
-
- \`generation\`, \`candidate_index\` \u2014 improvement-loop coordinates (\`-1\` = baseline campaign; \`null\` = not an improvement loop).
|
|
265
|
-
- \`role\` \u2014 one of ${ROLLOUT_ROLES.map((role) => `\`${role}\``).join(", ")}.
|
|
266
|
-
- \`task\` \u2014 suite, instance id, split, seed, replicate index.
|
|
267
|
-
- \`policy\` \u2014 harness, model, provider, profile commit, prompt/config hashes, sampling params.
|
|
268
|
-
- \`messages\` / \`tool_defs\` \u2014 full transcript in canonical OpenAI chat-with-tools form (including \`reasoning_content\`). An empty \`messages\` array is a labeled gap line; \`provenance.gap\` says why the transcript could not be recovered (${gapLines} gap lines in this release).
|
|
269
|
-
- \`outcome\` \u2014 \`reward\` (the single scalar), \`reward_source\`, the verbatim judge \`verdict\`, non-scalar \`metrics\`, and \`realness_gated\` (anti-Goodhart flag: reward forced to 0, never SFT-eligible).
|
|
270
|
-
- \`cost\` \u2014 usd, token counts, wall time.
|
|
271
|
-
- \`artifacts\` / \`provenance\` \u2014 patch/run-dir/transcript pointers (scrubbed) and capture metadata.
|
|
272
|
-
|
|
273
|
-
## Provenance
|
|
274
|
-
|
|
275
|
-
- Source ledgers: ${sourceFiles.map((file) => `\`${file}\``).join(", ")}
|
|
276
|
-
- Run ids: ${runIds.map((id) => `\`${id}\``).join(", ")}
|
|
277
|
-
- Generations: ${generations.join(", ")} (\`-1\` = baseline campaign)
|
|
278
|
-
- Models: ${models.map((m) => `\`${m}\``).join(", ")}
|
|
279
|
-
- Harnesses: ${harnesses.map((h) => `\`${h}\``).join(", ")}
|
|
280
|
-
- Capture modes: ${captures.join(", ")}
|
|
281
|
-
- Every reward traces to a named source (reward sources in this release: ${rewardSources.map((s) => `\`${s}\``).join(", ")}).
|
|
282
|
-
|
|
283
|
-
${proposerNote}
|
|
284
|
-
|
|
285
|
-
## Reward semantics per role
|
|
286
|
-
|
|
287
|
-
- **agent** \u2014 the producing RunRecord's holdout/search score, with the realness gate forcing gamed successes to 0.
|
|
288
|
-
- **supervisor** \u2014 the official-judge verdict on the episode's delivered artifact (1 = resolved, 0 = not).
|
|
289
|
-
- **worker** \u2014 INHERITED from the parent supervisor episode (\`\u2026/inherited\`). Caveat: reward 1 does not establish this worker's individual contribution (sibling workers in the same episode share the episode outcome), and reward 0 does not prove this worker failed.
|
|
290
|
-
- **proposer** \u2014 the fraction of improvement-set instances the proposed candidate resolved (\`\u2026/candidate-resolved-fraction\`); a scalar in [0, 1], not a binary verdict.
|
|
291
|
-
- **judge / analyst** \u2014 carry the episode verdict where one applies; otherwise \`reward: null\` (a labeled gap, never 0).
|
|
292
|
-
|
|
293
|
-
## Counts
|
|
294
|
-
|
|
295
|
-
${countsTable}
|
|
296
|
-
|
|
297
|
-
Total lines: ${lines.length}
|
|
298
|
-
|
|
299
|
-
## Scrubbing
|
|
300
|
-
|
|
301
|
-
Absolute home paths were rewritten to \`$WORK\`, credential-shaped strings were replaced with \`[REDACTED:<kind>]\` markers, internal hostnames were normalized to \`*.internal.example\`, and username-bearing incidentals (\`ls -l\` owner columns, per-user pytest tmpdirs) were normalized to \`user\` / \`$USER\`. Rewrite counts for this release (full per-file breakdown in \`scrub-report.json\`):
|
|
302
|
-
|
|
303
|
-
${scrubTable}
|
|
304
|
-
|
|
305
|
-
## License
|
|
306
|
-
|
|
307
|
-
\`license: unknown\` is a placeholder \u2014 the releasing operator must set the real SPDX license id in the frontmatter above before publishing.
|
|
308
|
-
|
|
309
|
-
## Citation
|
|
310
|
-
|
|
311
|
-
\`\`\`bibtex
|
|
312
|
-
@misc{tangle_rollout_ledger,
|
|
313
|
-
title = {Tangle rollout ledger \u2014 agent trajectories},
|
|
314
|
-
author = {{Tangle Network}},
|
|
315
|
-
howpublished = {HuggingFace Datasets},
|
|
316
|
-
note = {Operator: fill in the repository URL, authors, and year before publishing}
|
|
317
|
-
}
|
|
318
|
-
\`\`\`
|
|
319
|
-
`;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
// src/rollout/release/scrub.ts
|
|
323
|
-
var SCRUB_RULES = [
|
|
324
|
-
{
|
|
325
|
-
// /home/<user> and /Users/<user> prefixes → $WORK ($HOME-derived paths).
|
|
326
|
-
name: "home-path",
|
|
327
|
-
pattern: /\/(?:home|Users)\/[A-Za-z0-9._-]+/g,
|
|
328
|
-
rewrite: () => "$WORK"
|
|
329
|
-
},
|
|
330
|
-
{
|
|
331
|
-
// Harness stores encode cwd as a dashed path segment (-home-drew-…);
|
|
332
|
-
// strip the leading -home-<user> so the username never ships.
|
|
333
|
-
name: "home-path-encoded",
|
|
334
|
-
pattern: /(?<=\/)-(?:home|Users)-[A-Za-z0-9_.]+/g,
|
|
335
|
-
rewrite: () => "$WORK"
|
|
336
|
-
},
|
|
337
|
-
{
|
|
338
|
-
// pytest names its tmpdir after the invoking user (/tmp/pytest-of-drew).
|
|
339
|
-
name: "tmp-user-dir",
|
|
340
|
-
pattern: /\/tmp\/pytest-of-[A-Za-z0-9._-]+/g,
|
|
341
|
-
rewrite: () => "/tmp/pytest-of-$USER"
|
|
342
|
-
},
|
|
343
|
-
{
|
|
344
|
-
// `ls -l` owner/group columns leak the username in captured tool output.
|
|
345
|
-
// The `(?!user user…)` guard keeps a second pass at zero rewrites.
|
|
346
|
-
name: "ls-owner",
|
|
347
|
-
pattern: /([-bcdlps][-rwxsStT]{9}[.+@]?\s+\d+\s+)(?!user user(?=\s))[A-Za-z0-9._-]+\s+[A-Za-z0-9._-]+(?=\s)/g,
|
|
348
|
-
rewrite: (_match, prefix) => `${prefix}user user`
|
|
349
|
-
},
|
|
350
|
-
{
|
|
351
|
-
// Env-var-shaped secrets: NAME=value where NAME looks credential-bearing.
|
|
352
|
-
// The (?!\[REDACTED:) guard keeps the rule idempotent on its own output.
|
|
353
|
-
name: "env-secret",
|
|
354
|
-
pattern: /\b([A-Z][A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?))=(?!\[REDACTED:)("[^"]*"|'[^']*'|[^\s"']+)/g,
|
|
355
|
-
rewrite: (_match, name) => `${name}=[REDACTED:env]`
|
|
356
|
-
},
|
|
357
|
-
{
|
|
358
|
-
name: "bearer-token",
|
|
359
|
-
pattern: /\b(Bearer|Basic)\s+(?!\[REDACTED:)[A-Za-z0-9\-._~+/=]{8,}/g,
|
|
360
|
-
rewrite: (_match, scheme) => `${scheme} [REDACTED:bearer]`
|
|
361
|
-
},
|
|
362
|
-
{
|
|
363
|
-
// Bare provider-prefixed keys (OpenAI/Anthropic sk-, GitHub ghp_/gho_/…,
|
|
364
|
-
// fine-grained PATs, HuggingFace hf_, Slack xox*, AWS AKIA).
|
|
365
|
-
name: "api-key",
|
|
366
|
-
pattern: /\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|hf_[A-Za-z0-9]{16,}|xox[baprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16})\b/g,
|
|
367
|
-
rewrite: () => "[REDACTED:api-key]"
|
|
368
|
-
},
|
|
369
|
-
{
|
|
370
|
-
// Our infra hostnames → placeholder domain, subdomain preserved
|
|
371
|
-
// (router.tangle.tools → router.internal.example).
|
|
372
|
-
name: "infra-host",
|
|
373
|
-
pattern: /(?<![A-Za-z0-9.-])((?:[A-Za-z0-9-]+\.)*)tangle\.(?:tools|network)(?![A-Za-z0-9-])/g,
|
|
374
|
-
rewrite: (_match, prefix) => `${prefix ?? ""}internal.example`
|
|
375
|
-
},
|
|
376
|
-
{
|
|
377
|
-
// Known workstation hostnames leak through email Message-IDs and fqdn
|
|
378
|
-
// lookups inside captured test output; extend the list as machines join.
|
|
379
|
-
name: "machine-host",
|
|
380
|
-
pattern: /\b[A-Za-z0-9]+-GTR-Pro\b/g,
|
|
381
|
-
rewrite: () => "workstation"
|
|
382
|
-
}
|
|
383
|
-
];
|
|
384
|
-
function emptyScrubCounts() {
|
|
385
|
-
return Object.fromEntries(SCRUB_RULES.map((rule) => [rule.name, 0]));
|
|
386
|
-
}
|
|
387
|
-
function addScrubCounts(into, from) {
|
|
388
|
-
for (const [name, count] of Object.entries(from)) into[name] = (into[name] ?? 0) + count;
|
|
389
|
-
return into;
|
|
390
|
-
}
|
|
391
|
-
function scrubText(text, counts) {
|
|
392
|
-
let out = text;
|
|
393
|
-
for (const rule of SCRUB_RULES) {
|
|
394
|
-
out = out.replace(rule.pattern, (match, ...rest) => {
|
|
395
|
-
counts[rule.name] = (counts[rule.name] ?? 0) + 1;
|
|
396
|
-
return rule.rewrite(match, typeof rest[0] === "string" ? rest[0] : void 0);
|
|
397
|
-
});
|
|
398
|
-
}
|
|
399
|
-
return out;
|
|
400
|
-
}
|
|
401
|
-
function scrubValue(value, counts) {
|
|
402
|
-
if (typeof value === "string") return scrubText(value, counts);
|
|
403
|
-
if (Array.isArray(value)) return value.map((item) => scrubValue(item, counts));
|
|
404
|
-
if (value !== null && typeof value === "object") {
|
|
405
|
-
const out = {};
|
|
406
|
-
for (const [key, item] of Object.entries(value)) out[key] = scrubValue(item, counts);
|
|
407
|
-
return out;
|
|
408
|
-
}
|
|
409
|
-
return value;
|
|
410
|
-
}
|
|
411
|
-
function scrubRolloutLine(line, counts) {
|
|
412
|
-
return scrubValue(line, counts);
|
|
413
|
-
}
|
|
414
|
-
function scrubLines(lines) {
|
|
415
|
-
const counts = emptyScrubCounts();
|
|
416
|
-
return { lines: lines.map((line) => scrubRolloutLine(line, counts)), counts };
|
|
417
|
-
}
|
|
418
|
-
function defaultRolloutScrubber(text) {
|
|
419
|
-
return scrubText(text, emptyScrubCounts());
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
// src/rollout/release/hf-dataset.ts
|
|
423
|
-
import { spawnSync } from "child_process";
|
|
424
|
-
import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
|
|
425
|
-
import { basename, dirname as dirname2, join } from "path";
|
|
426
|
-
async function buildHfDataset(inputs, options) {
|
|
427
|
-
if (inputs.length === 0) throw new Error("no input ledgers given");
|
|
428
|
-
if (options.formats.length === 0) throw new Error("no formats selected");
|
|
429
|
-
const report = {
|
|
430
|
-
files: {},
|
|
431
|
-
totals: emptyScrubCounts(),
|
|
432
|
-
excluded: { proposers: 0, nonTrain: 0 }
|
|
433
|
-
};
|
|
434
|
-
const kept = [];
|
|
435
|
-
let read = 0;
|
|
436
|
-
for (const input of inputs) {
|
|
437
|
-
const lines = await readRolloutLedger(input);
|
|
438
|
-
read += lines.length;
|
|
439
|
-
const shippable = lines.filter((line) => {
|
|
440
|
-
if (!isTrainableSplit(line.task.split)) {
|
|
441
|
-
report.excluded.nonTrain += 1;
|
|
442
|
-
return false;
|
|
443
|
-
}
|
|
444
|
-
if (!options.includeProposers && line.role === "proposer") {
|
|
445
|
-
report.excluded.proposers += 1;
|
|
446
|
-
return false;
|
|
447
|
-
}
|
|
448
|
-
return true;
|
|
449
|
-
});
|
|
450
|
-
const scrubbed = scrubLines(shippable);
|
|
451
|
-
report.files[input] = scrubbed.counts;
|
|
452
|
-
addScrubCounts(report.totals, scrubbed.counts);
|
|
453
|
-
kept.push(...scrubbed.lines);
|
|
454
|
-
}
|
|
455
|
-
const formatCounts = {};
|
|
456
|
-
const files = [];
|
|
457
|
-
for (const format of options.formats) {
|
|
458
|
-
const path = join(options.out, FORMAT_FILES[format]);
|
|
459
|
-
await mkdir2(dirname2(path), { recursive: true });
|
|
460
|
-
if (format === "raw") {
|
|
461
|
-
await writeRolloutLedger(path, kept);
|
|
462
|
-
formatCounts.raw = kept.length;
|
|
463
|
-
} else if (format === "sft") {
|
|
464
|
-
const rows = toSftRows(kept);
|
|
465
|
-
await writeFile2(path, toJsonl(rows));
|
|
466
|
-
formatCounts.sft = rows.length;
|
|
467
|
-
} else if (format === "verifiers") {
|
|
468
|
-
const outputs = toVerifiersRolloutOutputs(kept);
|
|
469
|
-
await writeFile2(path, toJsonl(outputs));
|
|
470
|
-
formatCounts.verifiers = outputs.length;
|
|
471
|
-
} else {
|
|
472
|
-
const items = toRftItems(kept);
|
|
473
|
-
await writeFile2(path, toJsonl(items));
|
|
474
|
-
formatCounts.rft = items.length;
|
|
475
|
-
}
|
|
476
|
-
files.push(path);
|
|
477
|
-
}
|
|
478
|
-
const reportPath = join(options.out, "scrub-report.json");
|
|
479
|
-
await writeFile2(reportPath, `${JSON.stringify(report, null, 2)}
|
|
480
|
-
`);
|
|
481
|
-
files.push(reportPath);
|
|
482
|
-
const cardPath = join(options.out, "README.md");
|
|
483
|
-
await writeFile2(
|
|
484
|
-
cardPath,
|
|
485
|
-
buildDatasetCard({
|
|
486
|
-
lines: kept,
|
|
487
|
-
formats: options.formats,
|
|
488
|
-
includeProposers: options.includeProposers,
|
|
489
|
-
sourceFiles: inputs.map((input) => basename(input)),
|
|
490
|
-
scrubTotals: report.totals,
|
|
491
|
-
excluded: report.excluded,
|
|
492
|
-
formatCounts
|
|
493
|
-
})
|
|
494
|
-
);
|
|
495
|
-
files.push(cardPath);
|
|
496
|
-
return { inputs, read, kept: kept.length, scrub: report, formatCounts, files };
|
|
497
|
-
}
|
|
498
|
-
function planPushCommand(repo, outDir) {
|
|
499
|
-
return ["huggingface-cli", "upload", repo, outDir, ".", "--repo-type", "dataset"];
|
|
500
|
-
}
|
|
501
|
-
function pushDataset(repo, outDir) {
|
|
502
|
-
const found = spawnSync("which", ["huggingface-cli"], { stdio: "ignore" });
|
|
503
|
-
if (found.status !== 0) {
|
|
504
|
-
throw new Error(
|
|
505
|
-
"huggingface-cli not found on PATH \u2014 install huggingface_hub[cli] before --push"
|
|
506
|
-
);
|
|
507
|
-
}
|
|
508
|
-
if (!process.env.HF_TOKEN) {
|
|
509
|
-
throw new Error("HF_TOKEN not present in env \u2014 refusing to push");
|
|
510
|
-
}
|
|
511
|
-
const [command, ...args] = planPushCommand(repo, outDir);
|
|
512
|
-
const run = spawnSync(command, args, { stdio: "inherit" });
|
|
513
|
-
if (run.status !== 0) throw new Error(`huggingface-cli upload exited ${String(run.status)}`);
|
|
514
|
-
}
|
|
515
|
-
var ROLLOUT_RELEASE_USAGE = "usage: agent-eval rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]";
|
|
516
|
-
function parseRolloutReleaseArgs(argv) {
|
|
517
|
-
const args = {
|
|
518
|
-
inputs: [],
|
|
519
|
-
out: "",
|
|
520
|
-
formats: [...RELEASE_FORMATS],
|
|
521
|
-
includeProposers: false,
|
|
522
|
-
push: null
|
|
523
|
-
};
|
|
524
|
-
for (let i = 0; i < argv.length; i++) {
|
|
525
|
-
const arg = argv[i];
|
|
526
|
-
if (arg === "--out") {
|
|
527
|
-
args.out = argv[++i] ?? "";
|
|
528
|
-
} else if (arg === "--formats") {
|
|
529
|
-
const raw = (argv[++i] ?? "").split(",").filter(Boolean);
|
|
530
|
-
for (const format of raw) {
|
|
531
|
-
if (!RELEASE_FORMATS.includes(format)) {
|
|
532
|
-
throw new Error(
|
|
533
|
-
`unknown format "${format}" \u2014 expected one of ${RELEASE_FORMATS.join(",")}`
|
|
534
|
-
);
|
|
535
|
-
}
|
|
536
|
-
}
|
|
537
|
-
args.formats = raw;
|
|
538
|
-
} else if (arg === "--include-proposers") {
|
|
539
|
-
args.includeProposers = true;
|
|
540
|
-
} else if (arg === "--push") {
|
|
541
|
-
args.push = argv[++i] ?? null;
|
|
542
|
-
} else if (arg.startsWith("--")) {
|
|
543
|
-
throw new Error(`unknown flag "${arg}"`);
|
|
544
|
-
} else {
|
|
545
|
-
args.inputs.push(arg);
|
|
546
|
-
}
|
|
547
|
-
}
|
|
548
|
-
if (args.inputs.length === 0 || !args.out) {
|
|
549
|
-
throw new Error(ROLLOUT_RELEASE_USAGE);
|
|
550
|
-
}
|
|
551
|
-
if (args.push !== null && !/^[\w.-]+\/[\w.-]+$/.test(args.push)) {
|
|
552
|
-
throw new Error(`--push expects <org/name>, got "${args.push}"`);
|
|
553
|
-
}
|
|
554
|
-
return args;
|
|
555
|
-
}
|
|
556
|
-
async function runRolloutReleaseCli(argv) {
|
|
557
|
-
let args;
|
|
558
|
-
try {
|
|
559
|
-
args = parseRolloutReleaseArgs(argv);
|
|
560
|
-
} catch (error) {
|
|
561
|
-
process.stderr.write(`${error instanceof Error ? error.message : String(error)}
|
|
562
|
-
`);
|
|
563
|
-
return 2;
|
|
564
|
-
}
|
|
565
|
-
const summary = await buildHfDataset(args.inputs, args);
|
|
566
|
-
process.stdout.write(
|
|
567
|
-
`${JSON.stringify(
|
|
568
|
-
{
|
|
569
|
-
read: summary.read,
|
|
570
|
-
kept: summary.kept,
|
|
571
|
-
formatCounts: summary.formatCounts,
|
|
572
|
-
scrub: summary.scrub
|
|
573
|
-
},
|
|
574
|
-
null,
|
|
575
|
-
2
|
|
576
|
-
)}
|
|
577
|
-
`
|
|
578
|
-
);
|
|
579
|
-
process.stdout.write(`dataset \u2192 ${args.out} (${summary.files.length} files)
|
|
580
|
-
`);
|
|
581
|
-
if (args.push !== null) {
|
|
582
|
-
pushDataset(args.push, args.out);
|
|
583
|
-
process.stdout.write(`pushed \u2192 ${args.push}
|
|
584
|
-
`);
|
|
585
|
-
}
|
|
586
|
-
return 0;
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
export {
|
|
590
|
-
toSftRows,
|
|
591
|
-
toRewardRows,
|
|
592
|
-
toVerifiersRolloutOutput,
|
|
593
|
-
toVerifiersRolloutOutputs,
|
|
594
|
-
toRftItem,
|
|
595
|
-
toRftItems,
|
|
596
|
-
toJsonl,
|
|
597
|
-
writeRolloutLedger,
|
|
598
|
-
appendRolloutLines,
|
|
599
|
-
readRolloutLedger,
|
|
600
|
-
RELEASE_FORMATS,
|
|
601
|
-
FORMAT_FILES,
|
|
602
|
-
buildDatasetCard,
|
|
603
|
-
SCRUB_RULES,
|
|
604
|
-
emptyScrubCounts,
|
|
605
|
-
addScrubCounts,
|
|
606
|
-
scrubText,
|
|
607
|
-
scrubRolloutLine,
|
|
608
|
-
scrubLines,
|
|
609
|
-
defaultRolloutScrubber,
|
|
610
|
-
buildHfDataset,
|
|
611
|
-
planPushCommand,
|
|
612
|
-
pushDataset,
|
|
613
|
-
ROLLOUT_RELEASE_USAGE,
|
|
614
|
-
parseRolloutReleaseArgs,
|
|
615
|
-
runRolloutReleaseCli
|
|
616
|
-
};
|
|
617
|
-
//# sourceMappingURL=chunk-EJGRPCO3.js.map
|