devlyn-cli 3.2.1 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CLAUDE.md +2 -2
- package/README.md +12 -102
- package/bin/devlyn.js +13 -225
- package/bin/instruction-templates.json +8 -0
- package/config/skills/_shared/adapters/grok.md +1 -2
- package/config/skills/_shared/archive_run.py +15 -3
- package/config/skills/_shared/judge-role-evidence.py +21 -1
- package/config/skills/_shared/runtime-principles.md +1 -1
- package/config/skills/_shared/spec-verify-check.py +0 -653
- package/config/skills/_shared/state-phase-write.py +351 -25
- package/config/skills/_shared/task-complete.py +660 -150
- package/config/skills/_shared/terminal-claim-check.py +229 -4
- package/config/skills/_shared/verify-merge-findings.py +93 -2
- package/config/skills/devlyn:ideate/SKILL.md +3 -4
- package/config/skills/devlyn:ideate/references/elicitation.md +2 -35
- package/config/skills/devlyn:ideate/references/from-spec-mode.md +3 -18
- package/config/skills/devlyn:ideate/references/project-mode.md +1 -16
- package/config/skills/devlyn:ideate/references/spec-template.md +0 -2
- package/config/skills/devlyn:queue/SKILL.md +6 -4
- package/config/skills/devlyn:resolve/SKILL.md +16 -25
- package/config/skills/devlyn:resolve/references/free-form-mode.md +0 -14
- package/config/skills/devlyn:resolve/references/outer-loop.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/cleanup.md +3 -3
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +1 -16
- package/config/skills/devlyn:resolve/references/phases/verify.md +4 -9
- package/config/skills/devlyn:resolve/references/state-schema.md +9 -7
- package/config/skills/devlyn:resolve/references/task-completion.md +47 -38
- package/{config/skills → optional-skills}/code-health-standards/SKILL.md +4 -13
- package/{config/skills → optional-skills}/code-review-standards/SKILL.md +4 -12
- package/{config/skills → optional-skills}/root-cause-analysis/SKILL.md +4 -12
- package/{config/skills → optional-skills}/ui-implementation-standards/SKILL.md +4 -14
- package/package.json +1 -57
- package/benchmark/auto-resolve/BENCHMARK-DESIGN.md +0 -289
- package/benchmark/auto-resolve/BENCHMARK-RESULTS.md +0 -341
- package/benchmark/auto-resolve/README.md +0 -696
- package/benchmark/auto-resolve/RUBRIC.md +0 -177
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/spec.md +0 -44
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.ko.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/NOTES.md +0 -68
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/expected.json +0 -60
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/invalid.js +0 -29
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/parallel.js +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/NOTES.md +0 -75
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/expected.json +0 -52
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/setup.sh +0 -171
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/spec.md +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/NOTES.md +0 -89
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/setup.sh +0 -251
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/spec.md +0 -57
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/task.txt +0 -13
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/verifiers/replay-malformed-body.js +0 -64
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md +0 -104
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/expected.json +0 -46
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/setup.sh +0 -336
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/setup.sh +0 -32
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/spec.md +0 -63
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/exact-success.js +0 -54
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/no-hardcoded-pricing.js +0 -47
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/stock-error.js +0 -45
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected-pair-plan-registry.json +0 -170
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected.json +0 -84
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/metadata.json +0 -21
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-fail.json +0 -214
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-pass.json +0 -223
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/setup.sh +0 -5
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/task.txt +0 -14
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/expected.json +0 -62
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/error-order.js +0 -55
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/priority-blocked.js +0 -48
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/spec.md +0 -64
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/conflicting-duplicate.js +0 -34
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/idempotent-close.js +0 -41
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/spec.md +0 -76
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/priority-rollback.js +0 -64
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/single-warehouse-fefo.js +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/NOTES.md +0 -40
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/expected.json +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/spec.md +0 -71
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/catalog-source.js +0 -57
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/exact-success.js +0 -63
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/stock-error.js +0 -34
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/NOTES.md +0 -31
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/spec.md +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/conflicting-duplicate.js +0 -29
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/exact-payout.js +0 -58
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/rules-source.js +0 -56
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected-pair-plan-registry.json +0 -162
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/metadata.json +0 -19
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/duplicate-event-error.js +0 -35
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js +0 -53
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/spec.md +0 -70
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/task.txt +0 -3
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/duplicate-renewal-error.js +0 -42
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js +0 -70
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/NOTES.md +0 -47
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/setup.sh +0 -6
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/NOTES.md +0 -45
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/setup.sh +0 -55
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md +0 -43
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/expected.json +0 -77
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/task.txt +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/expected.json +0 -76
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/spec.md +0 -45
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md +0 -53
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/expected.json +0 -63
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/task.txt +0 -1
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md +0 -105
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/SCHEMA.md +0 -188
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md +0 -37
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md +0 -13
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/spec.md +0 -69
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/exact-proration.js +0 -48
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/rules-source-and-conflict.js +0 -79
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/policy-precedence.js +0 -72
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-and-immutability.js +0 -43
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-boundary.js +0 -116
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/RETIRED.md +0 -12
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/expected.json +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/spec.md +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/task.txt +0 -17
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/mixed-idempotent-settlement.js +0 -53
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/rejection-boundaries.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/NOTES.md +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/RETIRED.md +0 -15
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/setup.sh +0 -50
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/ordered-settlement.js +0 -88
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/reject-invalid.js +0 -45
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/NOTES.md +0 -30
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/setup.sh +0 -21
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/dup-op.js +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/priority-apply.js +0 -82
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/rollback.js +0 -77
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/NOTES.md +0 -99
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/RETIRED.md +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/spec.md +0 -61
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/correctness-small.js +0 -135
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/scale.js +0 -100
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/validation.js +0 -75
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/NOTES.md +0 -93
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/spec.md +0 -66
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/correctness-small.js +0 -84
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/scale.js +0 -184
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/validation.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/NOTES.md +0 -60
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/RETIRED.md +0 -29
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/expected.json +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/test-repo/README.md +0 -12
- package/benchmark/auto-resolve/fixtures/test-repo/bin/cli.js +0 -63
- package/benchmark/auto-resolve/fixtures/test-repo/package-lock.json +0 -823
- package/benchmark/auto-resolve/fixtures/test-repo/package.json +0 -22
- package/benchmark/auto-resolve/fixtures/test-repo/playwright.config.js +0 -17
- package/benchmark/auto-resolve/fixtures/test-repo/server/index.js +0 -37
- package/benchmark/auto-resolve/fixtures/test-repo/tests/cli.test.js +0 -25
- package/benchmark/auto-resolve/fixtures/test-repo/tests/server.test.js +0 -58
- package/benchmark/auto-resolve/fixtures/test-repo/web/index.html +0 -37
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +0 -25
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +0 -10
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +0 -30
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json +0 -82
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md +0 -18
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json +0 -46
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md +0 -17
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +0 -32
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +0 -44
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +0 -16
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/run-real-benchmark.md +0 -303
- package/benchmark/auto-resolve/scripts/audit-headroom-rejections.py +0 -441
- package/benchmark/auto-resolve/scripts/audit-pair-evidence.py +0 -1256
- package/benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py +0 -306
- package/benchmark/auto-resolve/scripts/check-f9-artifacts.py +0 -268
- package/benchmark/auto-resolve/scripts/collect-swebench-predictions.py +0 -108
- package/benchmark/auto-resolve/scripts/compile-report.py +0 -493
- package/benchmark/auto-resolve/scripts/fetch-swebench-instances.py +0 -129
- package/benchmark/auto-resolve/scripts/frozen-verify-gate.py +0 -434
- package/benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py +0 -612
- package/benchmark/auto-resolve/scripts/headroom-gate.py +0 -378
- package/benchmark/auto-resolve/scripts/iter-0033c-compare.py +0 -683
- package/benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py +0 -97
- package/benchmark/auto-resolve/scripts/judge-opus-pass.sh +0 -542
- package/benchmark/auto-resolve/scripts/judge.sh +0 -565
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-a.py +0 -256
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py +0 -287
- package/benchmark/auto-resolve/scripts/oracle-test-fidelity.py +0 -328
- package/benchmark/auto-resolve/scripts/pair-candidate-frontier.py +0 -469
- package/benchmark/auto-resolve/scripts/pair-plan-idgen.py +0 -401
- package/benchmark/auto-resolve/scripts/pair-plan-lint.py +0 -475
- package/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh +0 -103
- package/benchmark/auto-resolve/scripts/pair_evidence_contract.py +0 -270
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py +0 -273
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py +0 -148
- package/benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py +0 -210
- package/benchmark/auto-resolve/scripts/recent-benchmark-summary.py +0 -232
- package/benchmark/auto-resolve/scripts/run-fixture.sh +0 -1024
- package/benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh +0 -683
- package/benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh +0 -458
- package/benchmark/auto-resolve/scripts/run-headroom-candidate.sh +0 -336
- package/benchmark/auto-resolve/scripts/run-iter-0033c.sh +0 -208
- package/benchmark/auto-resolve/scripts/run-suite.sh +0 -251
- package/benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh +0 -310
- package/benchmark/auto-resolve/scripts/run-swebench-solver-batch.sh +0 -257
- package/benchmark/auto-resolve/scripts/ship-gate.py +0 -391
- package/benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py +0 -53
- package/benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py +0 -77
- package/benchmark/auto-resolve/scripts/swebench-frozen-matrix.py +0 -478
- package/benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh +0 -288
- package/benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh +0 -1672
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +0 -1015
- package/benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh +0 -491
- package/benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh +0 -91
- package/benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh +0 -517
- package/benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh +0 -610
- package/benchmark/auto-resolve/scripts/test-headroom-gate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh +0 -525
- package/benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh +0 -254
- package/benchmark/auto-resolve/scripts/test-lint-fixtures.sh +0 -580
- package/benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh +0 -591
- package/benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh +0 -497
- package/benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh +0 -111
- package/benchmark/auto-resolve/scripts/test-ship-gate.sh +0 -1189
- package/benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh +0 -1221
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/NOTES.md +0 -28
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/expected.json +0 -63
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/spec.md +0 -47
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/expected.json +0 -53
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/spec.md +0 -50
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/duplicate-order-error.js +0 -27
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/priority-stock-reservation.js +0 -44
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/spec.md +0 -52
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/duplicate-ticket-error.js +0 -29
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/priority-agent-assignment.js +0 -48
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/spec.md +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/duplicate-return-error.js +0 -43
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/priority-return-routing.js +0 -70
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/NOTES.md +0 -37
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/expected.json +0 -54
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/credit-ledger-priority.js +0 -98
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/duplicate-charge-error.js +0 -38
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/NOTES.md +0 -36
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/expected.json +0 -56
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/duplicate-refund-error.js +0 -41
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/priority-refund-ledger.js +0 -65
- package/config/skills/_shared/pair-plan-schema.md +0 -298
- package/scripts/lint-fixtures.sh +0 -352
- package/scripts/lint-shadow-fixtures.sh +0 -58
- package/scripts/lint-skills.sh +0 -4265
- package/scripts/test-owner-phases.py +0 -238
|
@@ -1,270 +0,0 @@
|
|
|
1
|
-
"""Shared pair-evidence contract for benchmark audits."""
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import json
|
|
5
|
-
import math
|
|
6
|
-
import re
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
from typing import Any
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}
|
|
12
|
-
CANONICAL_PAIR_TRIGGER_REASONS = {
|
|
13
|
-
"pair.default",
|
|
14
|
-
"mode.verify-only",
|
|
15
|
-
"mode.pair-verify",
|
|
16
|
-
"complexity.high",
|
|
17
|
-
"complexity.large",
|
|
18
|
-
"spec.complexity.high",
|
|
19
|
-
"spec.complexity.large",
|
|
20
|
-
"spec.solo_headroom_hypothesis",
|
|
21
|
-
"risk.high",
|
|
22
|
-
"risk_probes.enabled",
|
|
23
|
-
"risk_probes.present",
|
|
24
|
-
"coverage.failed",
|
|
25
|
-
"mechanical.warning",
|
|
26
|
-
"judge.warning",
|
|
27
|
-
}
|
|
28
|
-
HISTORICAL_PAIR_TRIGGER_REASON_ALIASES = {
|
|
29
|
-
"risk_profile.high_risk",
|
|
30
|
-
"risk_probes_enabled",
|
|
31
|
-
}
|
|
32
|
-
HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES = {
|
|
33
|
-
"complexity.high.spec.frontmatter",
|
|
34
|
-
"frontmatter.complexity.high",
|
|
35
|
-
"high.complexity.spec",
|
|
36
|
-
"high.risk.profile",
|
|
37
|
-
"spec.frontmatter.complexity.high",
|
|
38
|
-
"state.complexity.high",
|
|
39
|
-
}
|
|
40
|
-
# Benchmark readers accept historical aliases only for archived artifacts.
|
|
41
|
-
# Runtime /devlyn:resolve state must continue to emit canonical reasons.
|
|
42
|
-
KNOWN_PAIR_TRIGGER_REASONS = (
|
|
43
|
-
CANONICAL_PAIR_TRIGGER_REASONS | HISTORICAL_PAIR_TRIGGER_REASON_ALIASES
|
|
44
|
-
)
|
|
45
|
-
OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")
|
|
46
|
-
BACKTICKED_TEXT_RE = re.compile(r"`[^`\n]+`")
|
|
47
|
-
RESERVED_BACKTICK_TERMS = {"solo-headroom hypothesis", "solo_claude", "miss"}
|
|
48
|
-
COMMAND_PREFIXES = {
|
|
49
|
-
"bash",
|
|
50
|
-
"bun",
|
|
51
|
-
"cargo",
|
|
52
|
-
"git",
|
|
53
|
-
"go",
|
|
54
|
-
"jest",
|
|
55
|
-
"make",
|
|
56
|
-
"node",
|
|
57
|
-
"npm",
|
|
58
|
-
"pnpm",
|
|
59
|
-
"printf",
|
|
60
|
-
"pytest",
|
|
61
|
-
"python",
|
|
62
|
-
"python3",
|
|
63
|
-
"ruff",
|
|
64
|
-
"sh",
|
|
65
|
-
"uv",
|
|
66
|
-
"vitest",
|
|
67
|
-
"yarn",
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def reject_json_constant(token: str) -> None:
|
|
72
|
-
raise ValueError(f"invalid JSON numeric constant: {token}")
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def loads_strict_json_object(text: str) -> dict[str, Any]:
|
|
76
|
-
data = json.loads(text, parse_constant=reject_json_constant)
|
|
77
|
-
if not isinstance(data, dict):
|
|
78
|
-
raise ValueError("top-level JSON value must be an object")
|
|
79
|
-
return data
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def normalized_pair_trigger_reason(reason: str) -> str:
|
|
83
|
-
return re.sub(r"[^a-z0-9]+", ".", reason.lower()).strip(".")
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def is_known_pair_trigger_reason(reason: str) -> bool:
|
|
87
|
-
normalized = normalized_pair_trigger_reason(reason)
|
|
88
|
-
return (
|
|
89
|
-
reason in CANONICAL_PAIR_TRIGGER_REASONS
|
|
90
|
-
or reason in HISTORICAL_PAIR_TRIGGER_REASON_ALIASES
|
|
91
|
-
or normalized in HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES
|
|
92
|
-
)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
def is_canonical_pair_trigger_reason(reason: str) -> bool:
|
|
96
|
-
return reason in CANONICAL_PAIR_TRIGGER_REASONS
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def is_historical_pair_trigger_reason(reason: str) -> bool:
|
|
100
|
-
normalized = normalized_pair_trigger_reason(reason)
|
|
101
|
-
return (
|
|
102
|
-
reason in HISTORICAL_PAIR_TRIGGER_REASON_ALIASES
|
|
103
|
-
or normalized in HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES
|
|
104
|
-
)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def has_known_pair_trigger_reason(reasons: list[str]) -> bool:
|
|
108
|
-
return any(is_known_pair_trigger_reason(reason) for reason in reasons)
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def all_known_pair_trigger_reasons(reasons: list[str]) -> bool:
|
|
112
|
-
return all(is_known_pair_trigger_reason(reason) for reason in reasons)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def has_canonical_pair_trigger_reason(reasons: list[str]) -> bool:
|
|
116
|
-
return any(is_canonical_pair_trigger_reason(reason) for reason in reasons)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def has_historical_pair_trigger_reason(reasons: list[str]) -> bool:
|
|
120
|
-
return any(is_historical_pair_trigger_reason(reason) for reason in reasons)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def is_command_like_backtick(value: str) -> bool:
|
|
124
|
-
stripped = value.strip()
|
|
125
|
-
lower = stripped.lower()
|
|
126
|
-
if not stripped or lower in RESERVED_BACKTICK_TERMS:
|
|
127
|
-
return False
|
|
128
|
-
first = lower.split(maxsplit=1)[0]
|
|
129
|
-
return (
|
|
130
|
-
first in COMMAND_PREFIXES
|
|
131
|
-
or any(marker in stripped for marker in ("/", "$", "=", "|", "&&", ";"))
|
|
132
|
-
or stripped.endswith((".js", ".py", ".sh"))
|
|
133
|
-
)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
def actionable_observable_commands(text: str) -> list[str]:
|
|
137
|
-
commands: list[str] = []
|
|
138
|
-
for line in text.splitlines():
|
|
139
|
-
lower = line.lower()
|
|
140
|
-
if "miss" not in lower or not any(marker in lower for marker in OBSERVABLE_COMMAND_MARKERS):
|
|
141
|
-
continue
|
|
142
|
-
for match in BACKTICKED_TEXT_RE.finditer(line):
|
|
143
|
-
value = match.group(0).strip("`")
|
|
144
|
-
if is_command_like_backtick(value):
|
|
145
|
-
commands.append(value)
|
|
146
|
-
return commands
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
def has_actionable_solo_headroom_hypothesis_text(text: str) -> bool:
|
|
150
|
-
lower = text.lower()
|
|
151
|
-
return (
|
|
152
|
-
"solo-headroom hypothesis" in lower
|
|
153
|
-
and "solo_claude" in lower
|
|
154
|
-
and "miss" in lower
|
|
155
|
-
and bool(actionable_observable_commands(text))
|
|
156
|
-
)
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
def path_has_actionable_solo_headroom_hypothesis(path: Path) -> bool:
|
|
160
|
-
try:
|
|
161
|
-
text = path.read_text(encoding="utf-8")
|
|
162
|
-
except OSError:
|
|
163
|
-
return False
|
|
164
|
-
return has_actionable_solo_headroom_hypothesis_text(text)
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def normalize_pair_evidence_row(
|
|
168
|
-
*,
|
|
169
|
-
fixture: str,
|
|
170
|
-
run_id: str,
|
|
171
|
-
pair_arm: object,
|
|
172
|
-
row: dict[str, Any],
|
|
173
|
-
) -> dict[str, Any] | None:
|
|
174
|
-
bare_score = row.get("bare_score")
|
|
175
|
-
solo_score = row.get("solo_score")
|
|
176
|
-
pair_score = row.get("pair_score")
|
|
177
|
-
pair_margin = row.get("pair_margin")
|
|
178
|
-
pair_mode = row.get("pair_mode")
|
|
179
|
-
pair_trigger_eligible = row.get("pair_trigger_eligible")
|
|
180
|
-
pair_trigger_reasons = row.get("pair_trigger_reasons")
|
|
181
|
-
wall_ratio = row.get("pair_solo_wall_ratio")
|
|
182
|
-
if not fixture or not run_id:
|
|
183
|
-
return None
|
|
184
|
-
if not isinstance(pair_arm, str) or pair_arm not in ALLOWED_PAIR_ARMS:
|
|
185
|
-
return None
|
|
186
|
-
if not all(is_score(value) for value in [bare_score, solo_score, pair_score]):
|
|
187
|
-
return None
|
|
188
|
-
if not is_strict_int(pair_margin):
|
|
189
|
-
return None
|
|
190
|
-
if pair_margin != pair_score - solo_score:
|
|
191
|
-
return None
|
|
192
|
-
if pair_mode is not True:
|
|
193
|
-
return None
|
|
194
|
-
if pair_trigger_eligible is not True:
|
|
195
|
-
return None
|
|
196
|
-
if not (
|
|
197
|
-
isinstance(pair_trigger_reasons, list)
|
|
198
|
-
and pair_trigger_reasons
|
|
199
|
-
and all(isinstance(reason, str) for reason in pair_trigger_reasons)
|
|
200
|
-
and all_known_pair_trigger_reasons(pair_trigger_reasons)
|
|
201
|
-
and has_canonical_pair_trigger_reason(pair_trigger_reasons)
|
|
202
|
-
):
|
|
203
|
-
return None
|
|
204
|
-
if not is_strict_number(wall_ratio):
|
|
205
|
-
return None
|
|
206
|
-
normalized = {
|
|
207
|
-
"run_id": run_id,
|
|
208
|
-
"pair_arm": pair_arm,
|
|
209
|
-
"bare_score": bare_score,
|
|
210
|
-
"solo_score": solo_score,
|
|
211
|
-
"pair_score": pair_score,
|
|
212
|
-
"pair_margin": pair_margin,
|
|
213
|
-
"pair_mode": pair_mode,
|
|
214
|
-
"pair_trigger_eligible": pair_trigger_eligible,
|
|
215
|
-
"pair_trigger_reasons": pair_trigger_reasons,
|
|
216
|
-
"pair_trigger_has_canonical_reason": True,
|
|
217
|
-
"pair_trigger_has_hypothesis_reason": (
|
|
218
|
-
"spec.solo_headroom_hypothesis" in pair_trigger_reasons
|
|
219
|
-
),
|
|
220
|
-
"pair_solo_wall_ratio": wall_ratio,
|
|
221
|
-
}
|
|
222
|
-
return normalized
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
def best_pair_evidence(evidence: list[object]) -> dict[str, Any] | None:
|
|
226
|
-
candidates = [
|
|
227
|
-
normalized
|
|
228
|
-
for item in evidence
|
|
229
|
-
if isinstance(item, dict)
|
|
230
|
-
if isinstance(item.get("run_id"), str)
|
|
231
|
-
for normalized in [
|
|
232
|
-
normalize_pair_evidence_row(
|
|
233
|
-
fixture="_",
|
|
234
|
-
run_id=item["run_id"],
|
|
235
|
-
pair_arm=item.get("pair_arm"),
|
|
236
|
-
row=item,
|
|
237
|
-
)
|
|
238
|
-
]
|
|
239
|
-
if normalized is not None
|
|
240
|
-
]
|
|
241
|
-
if not candidates:
|
|
242
|
-
return None
|
|
243
|
-
|
|
244
|
-
def key(item: dict[str, Any]) -> tuple[int, int, str]:
|
|
245
|
-
margin = item.get("pair_margin")
|
|
246
|
-
pair_score = item.get("pair_score")
|
|
247
|
-
return (
|
|
248
|
-
margin if isinstance(margin, int) else -10_000,
|
|
249
|
-
pair_score if isinstance(pair_score, int) else -10_000,
|
|
250
|
-
str(item.get("run_id") or ""),
|
|
251
|
-
)
|
|
252
|
-
|
|
253
|
-
return max(candidates, key=key)
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
def is_strict_int(value: object) -> bool:
|
|
257
|
-
return isinstance(value, int) and not isinstance(value, bool)
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
def is_score(value: object) -> bool:
|
|
261
|
-
return is_strict_int(value) and 0 <= value <= 100
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
def is_strict_number(value: object) -> bool:
|
|
265
|
-
return (
|
|
266
|
-
isinstance(value, (int, float))
|
|
267
|
-
and not isinstance(value, bool)
|
|
268
|
-
and math.isfinite(value)
|
|
269
|
-
and value > 0
|
|
270
|
-
)
|
|
@@ -1,273 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Prepare a SWE-bench instance for frozen VERIFY solo-vs-pair review.
|
|
3
|
-
|
|
4
|
-
The script does not run models and does not evaluate SWE-bench correctness.
|
|
5
|
-
It converts one official SWE-bench-style instance plus one candidate patch into
|
|
6
|
-
the case layout consumed by run-frozen-verify-pair.sh.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
from __future__ import annotations
|
|
10
|
-
|
|
11
|
-
import argparse
|
|
12
|
-
import json
|
|
13
|
-
import re
|
|
14
|
-
import shlex
|
|
15
|
-
import shutil
|
|
16
|
-
import subprocess
|
|
17
|
-
from pathlib import Path
|
|
18
|
-
from typing import Any
|
|
19
|
-
|
|
20
|
-
from pair_evidence_contract import loads_strict_json_object
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
SAFE_ID = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
24
|
-
SAFE_REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
|
|
25
|
-
SAFE_COMMIT = re.compile(r"^[0-9a-fA-F]{7,40}$")
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def run(cmd: list[str], cwd: Path | None = None) -> None:
|
|
29
|
-
subprocess.run(cmd, cwd=cwd, check=True)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def read_json(path: Path) -> dict[str, Any]:
|
|
33
|
-
try:
|
|
34
|
-
return loads_strict_json_object(path.read_text(encoding="utf8"))
|
|
35
|
-
except ValueError as exc:
|
|
36
|
-
if str(exc) == "top-level JSON value must be an object":
|
|
37
|
-
raise ValueError(f"expected JSON object: {path}") from exc
|
|
38
|
-
raise
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
def require_text(instance: dict[str, Any], key: str) -> str:
|
|
42
|
-
value = instance.get(key)
|
|
43
|
-
if not isinstance(value, str) or not value.strip():
|
|
44
|
-
raise ValueError(f"SWE-bench instance missing non-empty {key!r}")
|
|
45
|
-
return value.strip()
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def positive_int(value: str) -> int:
|
|
49
|
-
try:
|
|
50
|
-
parsed = int(value)
|
|
51
|
-
except ValueError as exc:
|
|
52
|
-
raise argparse.ArgumentTypeError("must be an integer") from exc
|
|
53
|
-
if parsed <= 0:
|
|
54
|
-
raise argparse.ArgumentTypeError("must be > 0")
|
|
55
|
-
return parsed
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def require_safe_repo(instance: dict[str, Any]) -> str:
|
|
59
|
-
repo = require_text(instance, "repo")
|
|
60
|
-
if not SAFE_REPO.match(repo):
|
|
61
|
-
raise ValueError(f"unsafe SWE-bench repo: {repo!r}")
|
|
62
|
-
return repo
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def require_safe_base_commit(instance: dict[str, Any]) -> str:
|
|
66
|
-
base_commit = require_text(instance, "base_commit")
|
|
67
|
-
if not SAFE_COMMIT.match(base_commit):
|
|
68
|
-
raise ValueError(f"unsafe SWE-bench base_commit: {base_commit!r}")
|
|
69
|
-
return base_commit
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def repo_cache_name(repo: str, base_commit: str) -> str:
|
|
73
|
-
safe_repo = repo.replace("/", "__")
|
|
74
|
-
return f"{safe_repo}-{base_commit[:12]}"
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def prepare_repo(instance: dict[str, Any], repo_dir: Path | None, repos_root: Path) -> Path:
|
|
78
|
-
repo = require_safe_repo(instance)
|
|
79
|
-
base_commit = require_safe_base_commit(instance)
|
|
80
|
-
repos_root.mkdir(parents=True, exist_ok=True)
|
|
81
|
-
dest = repos_root / repo_cache_name(repo, base_commit)
|
|
82
|
-
|
|
83
|
-
if repo_dir is not None:
|
|
84
|
-
if dest.exists():
|
|
85
|
-
shutil.rmtree(dest)
|
|
86
|
-
run(["git", "clone", "--quiet", "--no-hardlinks", str(repo_dir), str(dest)])
|
|
87
|
-
elif not dest.exists():
|
|
88
|
-
run(["git", "clone", "--quiet", f"https://github.com/{repo}.git", str(dest)])
|
|
89
|
-
|
|
90
|
-
run(["git", "fetch", "--quiet", "--all", "--tags"], cwd=dest)
|
|
91
|
-
run(["git", "checkout", "--quiet", base_commit], cwd=dest)
|
|
92
|
-
run(["git", "reset", "--hard", "--quiet"], cwd=dest)
|
|
93
|
-
run(["git", "clean", "-ffdqx"], cwd=dest)
|
|
94
|
-
return dest
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
def write_case_files(
|
|
98
|
-
instance: dict[str, Any],
|
|
99
|
-
case_dir: Path,
|
|
100
|
-
patch_text: str,
|
|
101
|
-
timeout_seconds: int,
|
|
102
|
-
) -> None:
|
|
103
|
-
instance_id = require_text(instance, "instance_id")
|
|
104
|
-
repo = require_safe_repo(instance)
|
|
105
|
-
base_commit = require_safe_base_commit(instance)
|
|
106
|
-
problem = require_text(instance, "problem_statement")
|
|
107
|
-
case_dir.mkdir(parents=True, exist_ok=True)
|
|
108
|
-
|
|
109
|
-
metadata = {
|
|
110
|
-
"id": instance_id,
|
|
111
|
-
"category": "high-risk",
|
|
112
|
-
"difficulty": instance.get("difficulty") or "swebench",
|
|
113
|
-
"timeout_seconds": timeout_seconds,
|
|
114
|
-
"required_tools": ["git", "python3"],
|
|
115
|
-
"browser": False,
|
|
116
|
-
"deps_change_expected": True,
|
|
117
|
-
"intent": f"SWE-bench issue for {repo} at {base_commit}: resolve the supplied problem statement without using the gold patch.",
|
|
118
|
-
"source": {
|
|
119
|
-
"benchmark": "SWE-bench",
|
|
120
|
-
"repo": repo,
|
|
121
|
-
"base_commit": base_commit,
|
|
122
|
-
"issue_url": instance.get("issue_url"),
|
|
123
|
-
"pr_url": instance.get("pr_url"),
|
|
124
|
-
"version": instance.get("version"),
|
|
125
|
-
},
|
|
126
|
-
}
|
|
127
|
-
(case_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf8")
|
|
128
|
-
|
|
129
|
-
spec = f"""---
|
|
130
|
-
id: "{instance_id}"
|
|
131
|
-
title: "SWE-bench {instance_id}"
|
|
132
|
-
status: planned
|
|
133
|
-
complexity: high
|
|
134
|
-
depends-on: []
|
|
135
|
-
---
|
|
136
|
-
|
|
137
|
-
# SWE-bench {instance_id}
|
|
138
|
-
|
|
139
|
-
## Context
|
|
140
|
-
|
|
141
|
-
Repository: `{repo}`
|
|
142
|
-
Base commit: `{base_commit}`
|
|
143
|
-
|
|
144
|
-
This case is imported from a SWE-bench-style instance. Treat the problem
|
|
145
|
-
statement below as the visible user contract. Do not use the gold `patch` or
|
|
146
|
-
`test_patch` fields as implementation guidance during model generation or
|
|
147
|
-
review.
|
|
148
|
-
|
|
149
|
-
## Requirements
|
|
150
|
-
|
|
151
|
-
- [ ] Resolve the reported issue described in the problem statement.
|
|
152
|
-
- [ ] Preserve existing behavior outside the issue's scope.
|
|
153
|
-
- [ ] Keep the implementation consistent with the repository's local style and
|
|
154
|
-
dependency policy.
|
|
155
|
-
- [ ] Surface failures explicitly; do not hide errors behind silent fallbacks.
|
|
156
|
-
|
|
157
|
-
## Problem Statement
|
|
158
|
-
|
|
159
|
-
{problem}
|
|
160
|
-
|
|
161
|
-
## Constraints
|
|
162
|
-
|
|
163
|
-
- Do not inspect or rely on the SWE-bench gold solution patch while producing
|
|
164
|
-
or judging a candidate patch.
|
|
165
|
-
- Do not add broad rewrites, unrelated formatting churn, or new dependencies
|
|
166
|
-
unless the problem statement strictly requires them.
|
|
167
|
-
- Frozen VERIFY compares reviewers on the same already-applied candidate patch;
|
|
168
|
-
it is review evidence, not a full SWE-bench solve-rate measurement.
|
|
169
|
-
|
|
170
|
-
## Verification
|
|
171
|
-
|
|
172
|
-
- Run the official SWE-bench evaluator separately for solve-rate evidence.
|
|
173
|
-
- Use `/devlyn:resolve --verify-only` here only to compare solo vs gated pair
|
|
174
|
-
review of the frozen candidate patch against the visible problem statement.
|
|
175
|
-
"""
|
|
176
|
-
(case_dir / "spec.md").write_text(spec, encoding="utf8")
|
|
177
|
-
(case_dir / "task.txt").write_text(problem + "\n", encoding="utf8")
|
|
178
|
-
(case_dir / "expected.json").write_text(
|
|
179
|
-
json.dumps(
|
|
180
|
-
{
|
|
181
|
-
"verification_commands": [],
|
|
182
|
-
"forbidden_patterns": [],
|
|
183
|
-
"required_files": [],
|
|
184
|
-
"forbidden_files": [],
|
|
185
|
-
"tier_a_waivers": [],
|
|
186
|
-
"spec_output_files": [],
|
|
187
|
-
"max_deps_added": 999,
|
|
188
|
-
},
|
|
189
|
-
indent=2,
|
|
190
|
-
)
|
|
191
|
-
+ "\n",
|
|
192
|
-
encoding="utf8",
|
|
193
|
-
)
|
|
194
|
-
(case_dir / "setup.sh").write_text("#!/usr/bin/env bash\nset -euo pipefail\n", encoding="utf8")
|
|
195
|
-
(case_dir / "setup.sh").chmod(0o755)
|
|
196
|
-
notes = f"""# {instance_id} — SWE-bench Frozen VERIFY Case
|
|
197
|
-
|
|
198
|
-
Source repo: `{repo}`
|
|
199
|
-
Base commit: `{base_commit}`
|
|
200
|
-
|
|
201
|
-
This case exists to measure whether gated pair VERIFY catches verdict-binding
|
|
202
|
-
review issues that solo VERIFY misses on a fixed candidate patch. It does not
|
|
203
|
-
replace official SWE-bench pass/fail evaluation.
|
|
204
|
-
"""
|
|
205
|
-
(case_dir / "NOTES.md").write_text(notes, encoding="utf8")
|
|
206
|
-
(case_dir / "model.patch").write_text(patch_text, encoding="utf8")
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
def main() -> int:
|
|
210
|
-
parser = argparse.ArgumentParser()
|
|
211
|
-
parser.add_argument("--instance-json", required=True, type=Path)
|
|
212
|
-
parser.add_argument("--model-patch", required=True, type=Path)
|
|
213
|
-
parser.add_argument(
|
|
214
|
-
"--cases-root",
|
|
215
|
-
default=Path("benchmark/auto-resolve/external/swebench/cases"),
|
|
216
|
-
type=Path,
|
|
217
|
-
)
|
|
218
|
-
parser.add_argument(
|
|
219
|
-
"--repos-root",
|
|
220
|
-
default=Path("benchmark/auto-resolve/external/swebench/repos"),
|
|
221
|
-
type=Path,
|
|
222
|
-
)
|
|
223
|
-
parser.add_argument(
|
|
224
|
-
"--repo-dir",
|
|
225
|
-
type=Path,
|
|
226
|
-
help="Local clone/source repo to copy instead of cloning GitHub; useful for tests and cached runs.",
|
|
227
|
-
)
|
|
228
|
-
parser.add_argument("--timeout-seconds", type=positive_int, default=2400)
|
|
229
|
-
args = parser.parse_args()
|
|
230
|
-
|
|
231
|
-
instance = read_json(args.instance_json)
|
|
232
|
-
instance_id = require_text(instance, "instance_id")
|
|
233
|
-
if not SAFE_ID.match(instance_id):
|
|
234
|
-
raise ValueError(f"unsafe instance_id for path/spec use: {instance_id!r}")
|
|
235
|
-
patch_text = args.model_patch.read_text(encoding="utf8")
|
|
236
|
-
if not patch_text.strip():
|
|
237
|
-
raise ValueError(f"model patch is empty: {args.model_patch}")
|
|
238
|
-
|
|
239
|
-
repo_path = prepare_repo(instance, args.repo_dir, args.repos_root)
|
|
240
|
-
case_dir = args.cases_root / instance_id
|
|
241
|
-
write_case_files(instance, case_dir, patch_text, args.timeout_seconds)
|
|
242
|
-
|
|
243
|
-
command = [
|
|
244
|
-
"bash",
|
|
245
|
-
"benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh",
|
|
246
|
-
"--fixture",
|
|
247
|
-
instance_id,
|
|
248
|
-
"--fixtures-root",
|
|
249
|
-
str(args.cases_root),
|
|
250
|
-
"--base-repo",
|
|
251
|
-
str(repo_path),
|
|
252
|
-
"--diff",
|
|
253
|
-
str(case_dir / "model.patch"),
|
|
254
|
-
"--pair-mode",
|
|
255
|
-
"gated",
|
|
256
|
-
]
|
|
257
|
-
(case_dir / "run-command.txt").write_text(shlex.join(command) + "\n", encoding="utf8")
|
|
258
|
-
print(
|
|
259
|
-
json.dumps(
|
|
260
|
-
{
|
|
261
|
-
"instance_id": instance_id,
|
|
262
|
-
"case_dir": str(case_dir),
|
|
263
|
-
"repo_dir": str(repo_path),
|
|
264
|
-
"run_command": command,
|
|
265
|
-
},
|
|
266
|
-
indent=2,
|
|
267
|
-
)
|
|
268
|
-
)
|
|
269
|
-
return 0
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
if __name__ == "__main__":
|
|
273
|
-
raise SystemExit(main())
|
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Prepare a SWE-bench prediction JSONL as frozen VERIFY review cases."""
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
import argparse
|
|
7
|
-
import json
|
|
8
|
-
import subprocess
|
|
9
|
-
import tempfile
|
|
10
|
-
from pathlib import Path
|
|
11
|
-
from typing import Any
|
|
12
|
-
|
|
13
|
-
from pair_evidence_contract import loads_strict_json_object, reject_json_constant
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
17
|
-
rows: list[dict[str, Any]] = []
|
|
18
|
-
with path.open(encoding="utf8") as f:
|
|
19
|
-
for line_no, line in enumerate(f, start=1):
|
|
20
|
-
if not line.strip():
|
|
21
|
-
continue
|
|
22
|
-
value = json.loads(line, parse_constant=reject_json_constant)
|
|
23
|
-
if not isinstance(value, dict):
|
|
24
|
-
raise ValueError(f"{path}:{line_no}: expected JSON object")
|
|
25
|
-
rows.append(value)
|
|
26
|
-
return rows
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def require_text(row: dict[str, Any], key: str, source: str) -> str:
|
|
30
|
-
value = row.get(key)
|
|
31
|
-
if not isinstance(value, str) or not value.strip():
|
|
32
|
-
raise ValueError(f"{source} missing non-empty {key!r}")
|
|
33
|
-
return value.strip()
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def positive_int(value: str) -> int:
|
|
37
|
-
try:
|
|
38
|
-
parsed = int(value)
|
|
39
|
-
except ValueError as exc:
|
|
40
|
-
raise argparse.ArgumentTypeError("must be an integer") from exc
|
|
41
|
-
if parsed <= 0:
|
|
42
|
-
raise argparse.ArgumentTypeError("must be > 0")
|
|
43
|
-
return parsed
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def parse_prepared_case(stdout: str, source: str) -> dict[str, Any]:
|
|
47
|
-
try:
|
|
48
|
-
value = loads_strict_json_object(stdout)
|
|
49
|
-
except ValueError as exc:
|
|
50
|
-
if str(exc) == "top-level JSON value must be an object":
|
|
51
|
-
raise ValueError(f"{source}: expected JSON object") from exc
|
|
52
|
-
raise
|
|
53
|
-
for key in ("instance_id", "case_dir", "repo_dir", "run_command"):
|
|
54
|
-
if key == "run_command":
|
|
55
|
-
if not isinstance(value.get(key), list) or not value[key]:
|
|
56
|
-
raise ValueError(f"{source}: missing non-empty {key!r}")
|
|
57
|
-
elif not isinstance(value.get(key), str) or not value[key].strip():
|
|
58
|
-
raise ValueError(f"{source}: missing non-empty {key!r}")
|
|
59
|
-
return value
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def main() -> int:
|
|
63
|
-
parser = argparse.ArgumentParser()
|
|
64
|
-
parser.add_argument("--instances-jsonl", required=True, type=Path)
|
|
65
|
-
parser.add_argument("--predictions-jsonl", required=True, type=Path)
|
|
66
|
-
parser.add_argument(
|
|
67
|
-
"--cases-root",
|
|
68
|
-
default=Path("benchmark/auto-resolve/external/swebench/cases"),
|
|
69
|
-
type=Path,
|
|
70
|
-
)
|
|
71
|
-
parser.add_argument(
|
|
72
|
-
"--repos-root",
|
|
73
|
-
default=Path("benchmark/auto-resolve/external/swebench/repos"),
|
|
74
|
-
type=Path,
|
|
75
|
-
)
|
|
76
|
-
parser.add_argument("--repo-dir", type=Path, help="Use one local repo clone for every selected instance.")
|
|
77
|
-
parser.add_argument("--instance-id", action="append", help="Prepare only these instance ids.")
|
|
78
|
-
parser.add_argument("--limit", type=positive_int, help="Prepare at most N matched instances after filtering.")
|
|
79
|
-
parser.add_argument("--timeout-seconds", type=positive_int, default=2400)
|
|
80
|
-
parser.add_argument("--out-manifest", type=Path)
|
|
81
|
-
args = parser.parse_args()
|
|
82
|
-
|
|
83
|
-
instances = {require_text(row, "instance_id", f"{args.instances_jsonl}"): row for row in read_jsonl(args.instances_jsonl)}
|
|
84
|
-
predictions: dict[str, dict[str, Any]] = {}
|
|
85
|
-
for row in read_jsonl(args.predictions_jsonl):
|
|
86
|
-
instance_id = require_text(row, "instance_id", f"{args.predictions_jsonl}")
|
|
87
|
-
if instance_id in predictions:
|
|
88
|
-
raise ValueError(f"duplicate prediction for {instance_id}")
|
|
89
|
-
predictions[instance_id] = row
|
|
90
|
-
|
|
91
|
-
selected_ids = args.instance_id or list(predictions)
|
|
92
|
-
if not selected_ids:
|
|
93
|
-
raise ValueError("no prediction instances selected")
|
|
94
|
-
script = Path(__file__).with_name("prepare-swebench-frozen-case.py")
|
|
95
|
-
prepared: list[dict[str, Any]] = []
|
|
96
|
-
with tempfile.TemporaryDirectory() as tmp:
|
|
97
|
-
tmp_dir = Path(tmp)
|
|
98
|
-
for instance_id in selected_ids:
|
|
99
|
-
if args.limit is not None and len(prepared) >= args.limit:
|
|
100
|
-
break
|
|
101
|
-
if instance_id not in instances:
|
|
102
|
-
raise ValueError(f"prediction instance not found in instances JSONL: {instance_id}")
|
|
103
|
-
prediction = predictions.get(instance_id)
|
|
104
|
-
if prediction is None:
|
|
105
|
-
raise ValueError(f"selected instance missing prediction: {instance_id}")
|
|
106
|
-
patch_value = prediction.get("model_patch")
|
|
107
|
-
if not isinstance(patch_value, str) or not patch_value.strip():
|
|
108
|
-
raise ValueError(f"prediction {instance_id} missing non-empty 'model_patch'")
|
|
109
|
-
instance_path = tmp_dir / f"{instance_id}.instance.json"
|
|
110
|
-
patch_path = tmp_dir / f"{instance_id}.patch"
|
|
111
|
-
instance_path.write_text(json.dumps(instances[instance_id], indent=2) + "\n", encoding="utf8")
|
|
112
|
-
patch_path.write_text(patch_value, encoding="utf8")
|
|
113
|
-
|
|
114
|
-
cmd = [
|
|
115
|
-
"python3",
|
|
116
|
-
str(script),
|
|
117
|
-
"--instance-json",
|
|
118
|
-
str(instance_path),
|
|
119
|
-
"--model-patch",
|
|
120
|
-
str(patch_path),
|
|
121
|
-
"--cases-root",
|
|
122
|
-
str(args.cases_root),
|
|
123
|
-
"--repos-root",
|
|
124
|
-
str(args.repos_root),
|
|
125
|
-
"--timeout-seconds",
|
|
126
|
-
str(args.timeout_seconds),
|
|
127
|
-
]
|
|
128
|
-
if args.repo_dir is not None:
|
|
129
|
-
cmd.extend(["--repo-dir", str(args.repo_dir)])
|
|
130
|
-
completed = subprocess.run(cmd, check=True, text=True, capture_output=True)
|
|
131
|
-
prepared.append(parse_prepared_case(completed.stdout, f"prepared case {instance_id}"))
|
|
132
|
-
|
|
133
|
-
manifest = {
|
|
134
|
-
"instances_jsonl": str(args.instances_jsonl),
|
|
135
|
-
"predictions_jsonl": str(args.predictions_jsonl),
|
|
136
|
-
"cases_root": str(args.cases_root),
|
|
137
|
-
"repos_root": str(args.repos_root),
|
|
138
|
-
"prepared_count": len(prepared),
|
|
139
|
-
"prepared": prepared,
|
|
140
|
-
}
|
|
141
|
-
if args.out_manifest:
|
|
142
|
-
args.out_manifest.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf8")
|
|
143
|
-
print(json.dumps(manifest, indent=2))
|
|
144
|
-
return 0
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
if __name__ == "__main__":
|
|
148
|
-
raise SystemExit(main())
|