devlyn-cli 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CLAUDE.md +2 -2
- package/README.md +22 -113
- package/bin/devlyn.js +13 -225
- package/bin/instruction-templates.json +752 -1
- package/bin/instructions.js +101 -17
- package/config/skills/_shared/adapters/grok.md +1 -2
- package/config/skills/_shared/archive_run.py +15 -3
- package/config/skills/_shared/judge-role-evidence.py +21 -1
- package/config/skills/_shared/runtime-principles.md +1 -1
- package/config/skills/_shared/spec-verify-check.py +0 -653
- package/config/skills/_shared/state-phase-write.py +371 -30
- package/config/skills/_shared/task-complete.py +635 -143
- package/config/skills/_shared/terminal-claim-check.py +229 -4
- package/config/skills/_shared/verify-merge-findings.py +93 -2
- package/config/skills/devlyn:ideate/SKILL.md +3 -4
- package/config/skills/devlyn:ideate/references/elicitation.md +2 -35
- package/config/skills/devlyn:ideate/references/from-spec-mode.md +3 -18
- package/config/skills/devlyn:ideate/references/project-mode.md +1 -16
- package/config/skills/devlyn:ideate/references/spec-template.md +0 -2
- package/config/skills/devlyn:queue/SKILL.md +6 -4
- package/config/skills/devlyn:resolve/SKILL.md +16 -25
- package/config/skills/devlyn:resolve/references/free-form-mode.md +0 -14
- package/config/skills/devlyn:resolve/references/outer-loop.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/cleanup.md +3 -3
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +1 -16
- package/config/skills/devlyn:resolve/references/phases/verify.md +4 -9
- package/config/skills/devlyn:resolve/references/state-schema.md +9 -7
- package/config/skills/devlyn:resolve/references/task-completion.md +46 -37
- package/{config/skills → optional-skills}/code-health-standards/SKILL.md +4 -13
- package/{config/skills → optional-skills}/code-review-standards/SKILL.md +4 -12
- package/{config/skills → optional-skills}/root-cause-analysis/SKILL.md +4 -12
- package/{config/skills → optional-skills}/ui-implementation-standards/SKILL.md +4 -14
- package/package.json +1 -57
- package/benchmark/auto-resolve/BENCHMARK-DESIGN.md +0 -289
- package/benchmark/auto-resolve/BENCHMARK-RESULTS.md +0 -341
- package/benchmark/auto-resolve/README.md +0 -696
- package/benchmark/auto-resolve/RUBRIC.md +0 -177
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/spec.md +0 -44
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.ko.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/NOTES.md +0 -68
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/expected.json +0 -60
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/invalid.js +0 -29
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/parallel.js +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/NOTES.md +0 -75
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/expected.json +0 -52
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/setup.sh +0 -171
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/spec.md +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/NOTES.md +0 -89
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/setup.sh +0 -251
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/spec.md +0 -57
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/task.txt +0 -13
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/verifiers/replay-malformed-body.js +0 -64
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md +0 -104
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/expected.json +0 -46
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/setup.sh +0 -336
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/setup.sh +0 -32
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/spec.md +0 -63
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/exact-success.js +0 -54
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/no-hardcoded-pricing.js +0 -47
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/stock-error.js +0 -45
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected-pair-plan-registry.json +0 -170
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected.json +0 -84
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/metadata.json +0 -21
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-fail.json +0 -214
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-pass.json +0 -223
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/setup.sh +0 -5
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/task.txt +0 -14
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/expected.json +0 -62
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/error-order.js +0 -55
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/priority-blocked.js +0 -48
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/spec.md +0 -64
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/conflicting-duplicate.js +0 -34
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/idempotent-close.js +0 -41
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/spec.md +0 -76
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/priority-rollback.js +0 -64
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/single-warehouse-fefo.js +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/NOTES.md +0 -40
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/expected.json +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/spec.md +0 -71
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/catalog-source.js +0 -57
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/exact-success.js +0 -63
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/stock-error.js +0 -34
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/NOTES.md +0 -31
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/spec.md +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/conflicting-duplicate.js +0 -29
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/exact-payout.js +0 -58
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/rules-source.js +0 -56
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected-pair-plan-registry.json +0 -162
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/metadata.json +0 -19
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/duplicate-event-error.js +0 -35
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js +0 -53
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/spec.md +0 -70
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/task.txt +0 -3
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/duplicate-renewal-error.js +0 -42
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js +0 -70
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/NOTES.md +0 -47
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/setup.sh +0 -6
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/NOTES.md +0 -45
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/setup.sh +0 -55
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md +0 -43
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/expected.json +0 -77
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/task.txt +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/expected.json +0 -76
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/spec.md +0 -45
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md +0 -53
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/expected.json +0 -63
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/task.txt +0 -1
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md +0 -105
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/SCHEMA.md +0 -188
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md +0 -37
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md +0 -13
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/spec.md +0 -69
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/exact-proration.js +0 -48
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/rules-source-and-conflict.js +0 -79
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/policy-precedence.js +0 -72
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-and-immutability.js +0 -43
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-boundary.js +0 -116
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/RETIRED.md +0 -12
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/expected.json +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/spec.md +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/task.txt +0 -17
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/mixed-idempotent-settlement.js +0 -53
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/rejection-boundaries.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/NOTES.md +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/RETIRED.md +0 -15
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/setup.sh +0 -50
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/ordered-settlement.js +0 -88
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/reject-invalid.js +0 -45
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/NOTES.md +0 -30
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/setup.sh +0 -21
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/dup-op.js +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/priority-apply.js +0 -82
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/rollback.js +0 -77
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/NOTES.md +0 -99
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/RETIRED.md +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/spec.md +0 -61
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/correctness-small.js +0 -135
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/scale.js +0 -100
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/validation.js +0 -75
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/NOTES.md +0 -93
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/spec.md +0 -66
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/correctness-small.js +0 -84
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/scale.js +0 -184
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/validation.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/NOTES.md +0 -60
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/RETIRED.md +0 -29
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/expected.json +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/test-repo/README.md +0 -12
- package/benchmark/auto-resolve/fixtures/test-repo/bin/cli.js +0 -63
- package/benchmark/auto-resolve/fixtures/test-repo/package-lock.json +0 -823
- package/benchmark/auto-resolve/fixtures/test-repo/package.json +0 -22
- package/benchmark/auto-resolve/fixtures/test-repo/playwright.config.js +0 -17
- package/benchmark/auto-resolve/fixtures/test-repo/server/index.js +0 -37
- package/benchmark/auto-resolve/fixtures/test-repo/tests/cli.test.js +0 -25
- package/benchmark/auto-resolve/fixtures/test-repo/tests/server.test.js +0 -58
- package/benchmark/auto-resolve/fixtures/test-repo/web/index.html +0 -37
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +0 -25
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +0 -10
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +0 -30
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json +0 -82
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md +0 -18
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json +0 -46
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md +0 -17
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +0 -32
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +0 -44
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +0 -16
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/run-real-benchmark.md +0 -303
- package/benchmark/auto-resolve/scripts/audit-headroom-rejections.py +0 -441
- package/benchmark/auto-resolve/scripts/audit-pair-evidence.py +0 -1256
- package/benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py +0 -306
- package/benchmark/auto-resolve/scripts/check-f9-artifacts.py +0 -268
- package/benchmark/auto-resolve/scripts/collect-swebench-predictions.py +0 -108
- package/benchmark/auto-resolve/scripts/compile-report.py +0 -493
- package/benchmark/auto-resolve/scripts/fetch-swebench-instances.py +0 -129
- package/benchmark/auto-resolve/scripts/frozen-verify-gate.py +0 -434
- package/benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py +0 -612
- package/benchmark/auto-resolve/scripts/headroom-gate.py +0 -378
- package/benchmark/auto-resolve/scripts/iter-0033c-compare.py +0 -683
- package/benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py +0 -97
- package/benchmark/auto-resolve/scripts/judge-opus-pass.sh +0 -542
- package/benchmark/auto-resolve/scripts/judge.sh +0 -565
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-a.py +0 -256
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py +0 -287
- package/benchmark/auto-resolve/scripts/oracle-test-fidelity.py +0 -328
- package/benchmark/auto-resolve/scripts/pair-candidate-frontier.py +0 -469
- package/benchmark/auto-resolve/scripts/pair-plan-idgen.py +0 -401
- package/benchmark/auto-resolve/scripts/pair-plan-lint.py +0 -475
- package/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh +0 -103
- package/benchmark/auto-resolve/scripts/pair_evidence_contract.py +0 -270
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py +0 -273
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py +0 -148
- package/benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py +0 -210
- package/benchmark/auto-resolve/scripts/recent-benchmark-summary.py +0 -232
- package/benchmark/auto-resolve/scripts/run-fixture.sh +0 -1024
- package/benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh +0 -683
- package/benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh +0 -458
- package/benchmark/auto-resolve/scripts/run-headroom-candidate.sh +0 -336
- package/benchmark/auto-resolve/scripts/run-iter-0033c.sh +0 -208
- package/benchmark/auto-resolve/scripts/run-suite.sh +0 -251
- package/benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh +0 -310
- package/benchmark/auto-resolve/scripts/run-swebench-solver-batch.sh +0 -257
- package/benchmark/auto-resolve/scripts/ship-gate.py +0 -391
- package/benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py +0 -53
- package/benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py +0 -77
- package/benchmark/auto-resolve/scripts/swebench-frozen-matrix.py +0 -478
- package/benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh +0 -288
- package/benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh +0 -1672
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +0 -1015
- package/benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh +0 -491
- package/benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh +0 -91
- package/benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh +0 -517
- package/benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh +0 -610
- package/benchmark/auto-resolve/scripts/test-headroom-gate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh +0 -525
- package/benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh +0 -254
- package/benchmark/auto-resolve/scripts/test-lint-fixtures.sh +0 -580
- package/benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh +0 -591
- package/benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh +0 -497
- package/benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh +0 -111
- package/benchmark/auto-resolve/scripts/test-ship-gate.sh +0 -1189
- package/benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh +0 -1221
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/NOTES.md +0 -28
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/expected.json +0 -63
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/spec.md +0 -47
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/expected.json +0 -53
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/spec.md +0 -50
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/duplicate-order-error.js +0 -27
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/priority-stock-reservation.js +0 -44
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/spec.md +0 -52
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/duplicate-ticket-error.js +0 -29
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/priority-agent-assignment.js +0 -48
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/spec.md +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/duplicate-return-error.js +0 -43
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/priority-return-routing.js +0 -70
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/NOTES.md +0 -37
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/expected.json +0 -54
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/credit-ledger-priority.js +0 -98
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/duplicate-charge-error.js +0 -38
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/NOTES.md +0 -36
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/expected.json +0 -56
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/duplicate-refund-error.js +0 -41
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/priority-refund-ledger.js +0 -65
- package/config/skills/_shared/pair-plan-schema.md +0 -298
- package/scripts/lint-fixtures.sh +0 -352
- package/scripts/lint-shadow-fixtures.sh +0 -58
- package/scripts/lint-skills.sh +0 -4265
- package/scripts/test-owner-phases.py +0 -193
|
@@ -244,38 +244,6 @@ RISK_PROBE_INTEGRITY_FIX_HINT = (
|
|
|
244
244
|
".devlyn/risk-probes.jsonl or .devlyn/probes/."
|
|
245
245
|
)
|
|
246
246
|
INLINE_JSON_OBJECT_RE = re.compile(r'`?\{\s*"[^"\n]+"\s*:', re.IGNORECASE)
|
|
247
|
-
BACKTICKED_TEXT_RE = re.compile(r"`[^`\n]+`")
|
|
248
|
-
OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")
|
|
249
|
-
RESERVED_BACKTICK_TERMS = {"solo-headroom hypothesis", "solo_claude", "miss"}
|
|
250
|
-
SOLO_CEILING_CONTROL_RE = re.compile(
|
|
251
|
-
r'\bS[2-6]\b|S2-S6|solo-saturated|rejected controls?|solo ceiling',
|
|
252
|
-
re.IGNORECASE,
|
|
253
|
-
)
|
|
254
|
-
SOLO_CEILING_DIFFERENCE_RE = re.compile(
|
|
255
|
-
r'\bdiffer(?:s|ent|ence)?\b|\bunlike\b|\bbecause\b|\bpreserve\b|\bheadroom\b',
|
|
256
|
-
re.IGNORECASE,
|
|
257
|
-
)
|
|
258
|
-
COMMAND_PREFIXES = {
|
|
259
|
-
"bash",
|
|
260
|
-
"bun",
|
|
261
|
-
"cargo",
|
|
262
|
-
"git",
|
|
263
|
-
"go",
|
|
264
|
-
"jest",
|
|
265
|
-
"make",
|
|
266
|
-
"node",
|
|
267
|
-
"npm",
|
|
268
|
-
"pnpm",
|
|
269
|
-
"printf",
|
|
270
|
-
"pytest",
|
|
271
|
-
"python",
|
|
272
|
-
"python3",
|
|
273
|
-
"ruff",
|
|
274
|
-
"sh",
|
|
275
|
-
"uv",
|
|
276
|
-
"vitest",
|
|
277
|
-
"yarn",
|
|
278
|
-
}
|
|
279
247
|
LOCAL_URL_HOSTS = {
|
|
280
248
|
'localhost',
|
|
281
249
|
'127.0.0.1',
|
|
@@ -442,135 +410,6 @@ def validate_present_spec_complexity(text: str) -> str | None:
|
|
|
442
410
|
return f"frontmatter complexity must be one of: {values}"
|
|
443
411
|
|
|
444
412
|
|
|
445
|
-
def backticked_observable_miss_commands(text: str) -> list[str]:
|
|
446
|
-
commands: list[str] = []
|
|
447
|
-
for line in text.splitlines():
|
|
448
|
-
lower = line.lower()
|
|
449
|
-
if "miss" not in lower or not any(marker in lower for marker in OBSERVABLE_COMMAND_MARKERS):
|
|
450
|
-
continue
|
|
451
|
-
for match in BACKTICKED_TEXT_RE.finditer(line):
|
|
452
|
-
value = match.group(0).strip("`")
|
|
453
|
-
if is_command_like_backtick(value):
|
|
454
|
-
commands.append(value)
|
|
455
|
-
return commands
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
def is_command_like_backtick(value: str) -> bool:
|
|
459
|
-
stripped = value.strip()
|
|
460
|
-
lower = stripped.lower()
|
|
461
|
-
if not stripped or lower in RESERVED_BACKTICK_TERMS:
|
|
462
|
-
return False
|
|
463
|
-
first = lower.split(maxsplit=1)[0]
|
|
464
|
-
return (
|
|
465
|
-
first in COMMAND_PREFIXES
|
|
466
|
-
or any(marker in stripped for marker in ("/", "$", "=", "|", "&&", ";"))
|
|
467
|
-
or stripped.endswith((".js", ".py", ".sh"))
|
|
468
|
-
)
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
def has_backticked_observable_miss_command(text: str) -> bool:
|
|
472
|
-
return bool(backticked_observable_miss_commands(text))
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
def validate_present_solo_headroom_hypothesis(text: str) -> str | None:
|
|
476
|
-
lower = text.lower()
|
|
477
|
-
if "solo-headroom hypothesis" not in lower and not ("solo_claude" in lower and "miss" in lower):
|
|
478
|
-
return None
|
|
479
|
-
if (
|
|
480
|
-
"solo-headroom hypothesis" in lower
|
|
481
|
-
and "solo_claude" in lower
|
|
482
|
-
and "miss" in lower
|
|
483
|
-
and has_backticked_observable_miss_command(text)
|
|
484
|
-
):
|
|
485
|
-
return None
|
|
486
|
-
return (
|
|
487
|
-
"solo-headroom hypothesis must include `solo-headroom hypothesis`, "
|
|
488
|
-
"`solo_claude`, `miss`, and a backticked command/observable line "
|
|
489
|
-
"that exposes the miss"
|
|
490
|
-
)
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
def validate_present_solo_ceiling_avoidance(text: str) -> str | None:
|
|
494
|
-
lower = text.lower()
|
|
495
|
-
if "solo ceiling avoidance" not in lower:
|
|
496
|
-
return None
|
|
497
|
-
if (
|
|
498
|
-
"solo_claude" in lower
|
|
499
|
-
and SOLO_CEILING_CONTROL_RE.search(text)
|
|
500
|
-
and SOLO_CEILING_DIFFERENCE_RE.search(text)
|
|
501
|
-
):
|
|
502
|
-
return None
|
|
503
|
-
return (
|
|
504
|
-
"solo ceiling avoidance must include `solo ceiling avoidance`, "
|
|
505
|
-
"`solo_claude`, and a concrete difference from rejected or "
|
|
506
|
-
"solo-saturated controls such as `S2`-`S6`"
|
|
507
|
-
)
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
def validate_solo_headroom_commands_against_expected(
|
|
511
|
-
spec_text: str,
|
|
512
|
-
commands: object,
|
|
513
|
-
expected_label: str,
|
|
514
|
-
) -> str | None:
|
|
515
|
-
lower = spec_text.lower()
|
|
516
|
-
if "solo-headroom hypothesis" not in lower and not ("solo_claude" in lower and "miss" in lower):
|
|
517
|
-
return None
|
|
518
|
-
expected_cmds = {
|
|
519
|
-
command.get("cmd")
|
|
520
|
-
for command in commands
|
|
521
|
-
if isinstance(command, dict) and isinstance(command.get("cmd"), str)
|
|
522
|
-
} if isinstance(commands, list) else set()
|
|
523
|
-
hypothesis_cmds = backticked_observable_miss_commands(spec_text)
|
|
524
|
-
if any(command in expected_cmds for command in hypothesis_cmds):
|
|
525
|
-
return None
|
|
526
|
-
return (
|
|
527
|
-
"solo-headroom hypothesis observable command must match "
|
|
528
|
-
f"{expected_label} verification_commands[].cmd"
|
|
529
|
-
)
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
def command_contains_expected(actual: str, expected: str) -> bool:
|
|
533
|
-
normalized_actual = " ".join(actual.split())
|
|
534
|
-
normalized_expected = " ".join(expected.split())
|
|
535
|
-
if not normalized_expected:
|
|
536
|
-
return False
|
|
537
|
-
pattern = re.compile(
|
|
538
|
-
rf"(?<![A-Za-z0-9_.:/=-]){re.escape(normalized_expected)}(?![A-Za-z0-9_.:/=-])"
|
|
539
|
-
)
|
|
540
|
-
return bool(pattern.search(normalized_actual))
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
def validate_risk_probes_cover_solo_headroom_hypothesis(
|
|
544
|
-
probes: list[dict],
|
|
545
|
-
verification_text: str,
|
|
546
|
-
) -> str | None:
|
|
547
|
-
hypothesis_cmds = backticked_observable_miss_commands(verification_text)
|
|
548
|
-
if not hypothesis_cmds:
|
|
549
|
-
return None
|
|
550
|
-
if not probes:
|
|
551
|
-
return None
|
|
552
|
-
derived_from = probes[0].get("derived_from")
|
|
553
|
-
if not (
|
|
554
|
-
isinstance(derived_from, str)
|
|
555
|
-
and "solo-headroom hypothesis" in derived_from.lower()
|
|
556
|
-
and any(command_contains_expected(derived_from, hypothesis_cmd) for hypothesis_cmd in hypothesis_cmds)
|
|
557
|
-
):
|
|
558
|
-
return (
|
|
559
|
-
"risk-probes[0].derived_from must reference the solo-headroom "
|
|
560
|
-
"hypothesis bullet and observable command"
|
|
561
|
-
)
|
|
562
|
-
cmd = probes[0].get("cmd")
|
|
563
|
-
if isinstance(cmd, str) and any(
|
|
564
|
-
command_contains_expected(cmd, hypothesis_cmd)
|
|
565
|
-
for hypothesis_cmd in hypothesis_cmds
|
|
566
|
-
):
|
|
567
|
-
return None
|
|
568
|
-
return (
|
|
569
|
-
"risk-probes[0].cmd must contain a "
|
|
570
|
-
"solo-headroom hypothesis observable command"
|
|
571
|
-
)
|
|
572
|
-
|
|
573
|
-
|
|
574
413
|
def external_url_hosts(text: str) -> list[str]:
|
|
575
414
|
hosts: list[str] = []
|
|
576
415
|
for match in EXTERNAL_URL_RE.finditer(text or ''):
|
|
@@ -868,24 +707,7 @@ def validate_expected_against_sibling_spec(spec_path: Path, data: object) -> str
|
|
|
868
707
|
return None
|
|
869
708
|
if not spec_path.is_file():
|
|
870
709
|
return None
|
|
871
|
-
try:
|
|
872
|
-
spec_text = spec_path.read_text(encoding="utf-8")
|
|
873
|
-
except OSError:
|
|
874
|
-
return None
|
|
875
|
-
solo_headroom_err = validate_present_solo_headroom_hypothesis(spec_text)
|
|
876
|
-
if solo_headroom_err:
|
|
877
|
-
return solo_headroom_err
|
|
878
|
-
solo_ceiling_err = validate_present_solo_ceiling_avoidance(spec_text)
|
|
879
|
-
if solo_ceiling_err:
|
|
880
|
-
return solo_ceiling_err
|
|
881
710
|
commands = data.get("verification_commands", [])
|
|
882
|
-
solo_headroom_command_err = validate_solo_headroom_commands_against_expected(
|
|
883
|
-
spec_text,
|
|
884
|
-
commands,
|
|
885
|
-
"spec.expected.json",
|
|
886
|
-
)
|
|
887
|
-
if solo_headroom_command_err:
|
|
888
|
-
return solo_headroom_command_err
|
|
889
711
|
pure_design = data.get("pure_design") is True
|
|
890
712
|
if commands:
|
|
891
713
|
if pure_design:
|
|
@@ -1099,12 +921,6 @@ def load_risk_probes(
|
|
|
1099
921
|
if missing:
|
|
1100
922
|
formatted = "; ".join(f"{r['tag']} (derived_from={r['derived_from']!r})" for r in missing)
|
|
1101
923
|
return ([], f"risk-probes.jsonl missing required probe(s): {formatted}")
|
|
1102
|
-
solo_headroom_probe_err = validate_risk_probes_cover_solo_headroom_hypothesis(
|
|
1103
|
-
probes,
|
|
1104
|
-
verification_text,
|
|
1105
|
-
)
|
|
1106
|
-
if solo_headroom_probe_err:
|
|
1107
|
-
return ([], solo_headroom_probe_err)
|
|
1108
924
|
return (probes, None)
|
|
1109
925
|
|
|
1110
926
|
|
|
@@ -1907,14 +1723,6 @@ def run_check_mode(md_path: Path) -> int:
|
|
|
1907
1723
|
if frontmatter_err:
|
|
1908
1724
|
print(f"[spec-verify --check] {md_path}: {frontmatter_err}", file=sys.stderr)
|
|
1909
1725
|
return 2
|
|
1910
|
-
solo_headroom_err = validate_present_solo_headroom_hypothesis(text)
|
|
1911
|
-
if solo_headroom_err:
|
|
1912
|
-
print(f"[spec-verify --check] {md_path}: {solo_headroom_err}", file=sys.stderr)
|
|
1913
|
-
return 2
|
|
1914
|
-
solo_ceiling_err = validate_present_solo_ceiling_avoidance(text)
|
|
1915
|
-
if solo_ceiling_err:
|
|
1916
|
-
print(f"[spec-verify --check] {md_path}: {solo_ceiling_err}", file=sys.stderr)
|
|
1917
|
-
return 2
|
|
1918
1726
|
expected_path = md_path.with_name("spec.expected.json")
|
|
1919
1727
|
if expected_path.is_file():
|
|
1920
1728
|
data, err = load_expected_contract(expected_path)
|
|
@@ -1949,14 +1757,6 @@ def run_check_mode(md_path: Path) -> int:
|
|
|
1949
1757
|
if err:
|
|
1950
1758
|
print(f"[spec-verify --check] {md_path}: shape error: {err}", file=sys.stderr)
|
|
1951
1759
|
return 2
|
|
1952
|
-
solo_headroom_command_err = validate_solo_headroom_commands_against_expected(
|
|
1953
|
-
text,
|
|
1954
|
-
data.get("verification_commands", []),
|
|
1955
|
-
"`## Verification` JSON carrier",
|
|
1956
|
-
)
|
|
1957
|
-
if solo_headroom_command_err:
|
|
1958
|
-
print(f"[spec-verify --check] {md_path}: {solo_headroom_command_err}", file=sys.stderr)
|
|
1959
|
-
return 2
|
|
1960
1760
|
return 0
|
|
1961
1761
|
|
|
1962
1762
|
|
|
@@ -2615,169 +2415,6 @@ def run_self_test() -> int:
|
|
|
2615
2415
|
print(bad_complexity_check.stderr, file=sys.stderr)
|
|
2616
2416
|
return 1
|
|
2617
2417
|
|
|
2618
|
-
weak_solo_headroom = work / "weak-solo-headroom.md"
|
|
2619
|
-
weak_solo_headroom.write_text(
|
|
2620
|
-
"# Weak\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2621
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling.\n"
|
|
2622
|
-
"- Observable command: `node check.js` exposes behavior.\n",
|
|
2623
|
-
encoding="utf-8",
|
|
2624
|
-
)
|
|
2625
|
-
weak_solo_check = subprocess.run(
|
|
2626
|
-
[sys.executable, script_path, "--check", str(weak_solo_headroom)],
|
|
2627
|
-
cwd=work,
|
|
2628
|
-
capture_output=True,
|
|
2629
|
-
text=True,
|
|
2630
|
-
encoding="utf-8",
|
|
2631
|
-
)
|
|
2632
|
-
if weak_solo_check.returncode == 0:
|
|
2633
|
-
print("weak solo-headroom hypothesis was accepted by --check", file=sys.stderr)
|
|
2634
|
-
return 1
|
|
2635
|
-
if "backticked command/observable line that exposes the miss" not in weak_solo_check.stderr:
|
|
2636
|
-
print("--check did not report weak solo-headroom hypothesis", file=sys.stderr)
|
|
2637
|
-
print(weak_solo_check.stderr, file=sys.stderr)
|
|
2638
|
-
return 1
|
|
2639
|
-
|
|
2640
|
-
weak_descriptive_backtick = work / "weak-descriptive-backtick.md"
|
|
2641
|
-
weak_descriptive_backtick.write_text(
|
|
2642
|
-
"# Weak descriptive backtick\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2643
|
-
"- solo-headroom hypothesis: solo_claude should miss behavior where observable `priority rollback` exposes the miss.\n",
|
|
2644
|
-
encoding="utf-8",
|
|
2645
|
-
)
|
|
2646
|
-
weak_descriptive_check = subprocess.run(
|
|
2647
|
-
[sys.executable, script_path, "--check", str(weak_descriptive_backtick)],
|
|
2648
|
-
cwd=work,
|
|
2649
|
-
capture_output=True,
|
|
2650
|
-
text=True,
|
|
2651
|
-
encoding="utf-8",
|
|
2652
|
-
)
|
|
2653
|
-
if weak_descriptive_check.returncode == 0:
|
|
2654
|
-
print("descriptive backtick solo-headroom hypothesis was accepted by --check", file=sys.stderr)
|
|
2655
|
-
return 1
|
|
2656
|
-
|
|
2657
|
-
strong_solo_headroom = work / "strong-solo-headroom.md"
|
|
2658
|
-
strong_solo_headroom.write_text(
|
|
2659
|
-
"# Strong\n\n## Verification\n\n"
|
|
2660
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling exposed by `node check.js`.\n",
|
|
2661
|
-
encoding="utf-8",
|
|
2662
|
-
)
|
|
2663
|
-
strong_solo_check = subprocess.run(
|
|
2664
|
-
[sys.executable, script_path, "--check", str(strong_solo_headroom)],
|
|
2665
|
-
cwd=work,
|
|
2666
|
-
capture_output=True,
|
|
2667
|
-
text=True,
|
|
2668
|
-
encoding="utf-8",
|
|
2669
|
-
)
|
|
2670
|
-
if strong_solo_check.returncode != 0:
|
|
2671
|
-
print("actionable solo-headroom hypothesis was rejected by --check", file=sys.stderr)
|
|
2672
|
-
print(strong_solo_check.stderr, file=sys.stderr)
|
|
2673
|
-
return 1
|
|
2674
|
-
|
|
2675
|
-
docs_style_solo_headroom = work / "docs-style-solo-headroom.md"
|
|
2676
|
-
docs_style_solo_headroom.write_text(
|
|
2677
|
-
"# Docs style\n\n## Verification\n\n"
|
|
2678
|
-
"- Solo-headroom hypothesis: the spec must literally contain `solo_claude`, `miss`, and an observable command; "
|
|
2679
|
-
"`node check.js` exposes the miss.\n",
|
|
2680
|
-
encoding="utf-8",
|
|
2681
|
-
)
|
|
2682
|
-
docs_style_solo_check = subprocess.run(
|
|
2683
|
-
[sys.executable, script_path, "--check", str(docs_style_solo_headroom)],
|
|
2684
|
-
cwd=work,
|
|
2685
|
-
capture_output=True,
|
|
2686
|
-
text=True,
|
|
2687
|
-
encoding="utf-8",
|
|
2688
|
-
)
|
|
2689
|
-
if docs_style_solo_check.returncode != 0:
|
|
2690
|
-
print("docs-style solo-headroom hypothesis was rejected by --check", file=sys.stderr)
|
|
2691
|
-
print(docs_style_solo_check.stderr, file=sys.stderr)
|
|
2692
|
-
return 1
|
|
2693
|
-
|
|
2694
|
-
weak_solo_ceiling = work / "weak-solo-ceiling.md"
|
|
2695
|
-
weak_solo_ceiling.write_text(
|
|
2696
|
-
"# Weak ceiling\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2697
|
-
"- solo ceiling avoidance: this is not like the previous ones.\n",
|
|
2698
|
-
encoding="utf-8",
|
|
2699
|
-
)
|
|
2700
|
-
weak_solo_ceiling_check = subprocess.run(
|
|
2701
|
-
[sys.executable, script_path, "--check", str(weak_solo_ceiling)],
|
|
2702
|
-
cwd=work,
|
|
2703
|
-
capture_output=True,
|
|
2704
|
-
text=True,
|
|
2705
|
-
encoding="utf-8",
|
|
2706
|
-
)
|
|
2707
|
-
if weak_solo_ceiling_check.returncode == 0:
|
|
2708
|
-
print("weak solo ceiling avoidance was accepted by --check", file=sys.stderr)
|
|
2709
|
-
return 1
|
|
2710
|
-
if "concrete difference from rejected or solo-saturated controls" not in weak_solo_ceiling_check.stderr:
|
|
2711
|
-
print("--check did not report weak solo ceiling avoidance", file=sys.stderr)
|
|
2712
|
-
print(weak_solo_ceiling_check.stderr, file=sys.stderr)
|
|
2713
|
-
return 1
|
|
2714
|
-
|
|
2715
|
-
strong_solo_ceiling = work / "strong-solo-ceiling.md"
|
|
2716
|
-
strong_solo_ceiling.write_text(
|
|
2717
|
-
"# Strong ceiling\n\n## Verification\n\n"
|
|
2718
|
-
"- solo ceiling avoidance: unlike solo-saturated `S2`-`S6`, this uses a cross-run "
|
|
2719
|
-
"state leak because solo_claude headroom should be preserved.\n",
|
|
2720
|
-
encoding="utf-8",
|
|
2721
|
-
)
|
|
2722
|
-
strong_solo_ceiling_check = subprocess.run(
|
|
2723
|
-
[sys.executable, script_path, "--check", str(strong_solo_ceiling)],
|
|
2724
|
-
cwd=work,
|
|
2725
|
-
capture_output=True,
|
|
2726
|
-
text=True,
|
|
2727
|
-
encoding="utf-8",
|
|
2728
|
-
)
|
|
2729
|
-
if strong_solo_ceiling_check.returncode != 0:
|
|
2730
|
-
print("actionable solo ceiling avoidance was rejected by --check", file=sys.stderr)
|
|
2731
|
-
print(strong_solo_ceiling_check.stderr, file=sys.stderr)
|
|
2732
|
-
return 1
|
|
2733
|
-
|
|
2734
|
-
inline_mismatched_solo = work / "inline-mismatched-solo.md"
|
|
2735
|
-
inline_mismatched_solo.write_text(
|
|
2736
|
-
"# Inline mismatch\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2737
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
2738
|
-
"`node check.js` exposes the miss.\n\n"
|
|
2739
|
-
"```json\n"
|
|
2740
|
-
+ json.dumps({"verification_commands": [{"cmd": "printf ok"}]})
|
|
2741
|
-
+ "\n```\n",
|
|
2742
|
-
encoding="utf-8",
|
|
2743
|
-
)
|
|
2744
|
-
inline_mismatched_check = subprocess.run(
|
|
2745
|
-
[sys.executable, script_path, "--check", str(inline_mismatched_solo)],
|
|
2746
|
-
cwd=work,
|
|
2747
|
-
capture_output=True,
|
|
2748
|
-
text=True,
|
|
2749
|
-
encoding="utf-8",
|
|
2750
|
-
)
|
|
2751
|
-
if inline_mismatched_check.returncode == 0:
|
|
2752
|
-
print("mismatched inline solo-headroom command was accepted by --check", file=sys.stderr)
|
|
2753
|
-
return 1
|
|
2754
|
-
if "observable command must match `## Verification` JSON carrier" not in inline_mismatched_check.stderr:
|
|
2755
|
-
print("--check did not report mismatched inline solo-headroom command", file=sys.stderr)
|
|
2756
|
-
print(inline_mismatched_check.stderr, file=sys.stderr)
|
|
2757
|
-
return 1
|
|
2758
|
-
|
|
2759
|
-
inline_matched_solo = work / "inline-matched-solo.md"
|
|
2760
|
-
inline_matched_solo.write_text(
|
|
2761
|
-
"# Inline match\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
2762
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
2763
|
-
"`printf ok` exposes the miss.\n\n"
|
|
2764
|
-
"```json\n"
|
|
2765
|
-
+ json.dumps({"verification_commands": [{"cmd": "printf ok"}]})
|
|
2766
|
-
+ "\n```\n",
|
|
2767
|
-
encoding="utf-8",
|
|
2768
|
-
)
|
|
2769
|
-
inline_matched_check = subprocess.run(
|
|
2770
|
-
[sys.executable, script_path, "--check", str(inline_matched_solo)],
|
|
2771
|
-
cwd=work,
|
|
2772
|
-
capture_output=True,
|
|
2773
|
-
text=True,
|
|
2774
|
-
encoding="utf-8",
|
|
2775
|
-
)
|
|
2776
|
-
if inline_matched_check.returncode != 0:
|
|
2777
|
-
print("matched inline solo-headroom command was rejected by --check", file=sys.stderr)
|
|
2778
|
-
print(inline_matched_check.stderr, file=sys.stderr)
|
|
2779
|
-
return 1
|
|
2780
|
-
|
|
2781
2418
|
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
2782
2419
|
"id": "P2",
|
|
2783
2420
|
"derived_from": "probe must pass visible marker.",
|
|
@@ -3019,180 +2656,6 @@ def run_self_test() -> int:
|
|
|
3019
2656
|
print("error_contract with unrelated derived_from was accepted", file=sys.stderr)
|
|
3020
2657
|
return 1
|
|
3021
2658
|
|
|
3022
|
-
spec_md.write_text(
|
|
3023
|
-
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3024
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
3025
|
-
"`printf ok` exposes the miss.\n"
|
|
3026
|
-
"- probe must pass visible marker.\n",
|
|
3027
|
-
encoding="utf-8",
|
|
3028
|
-
)
|
|
3029
|
-
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3030
|
-
"id": "P4",
|
|
3031
|
-
"derived_from": "solo-headroom hypothesis: solo_claude should miss duplicate handling; `printf ok` exposes the miss.",
|
|
3032
|
-
"cmd": "printf unrelated",
|
|
3033
|
-
"exit_code": 0,
|
|
3034
|
-
"stdout_contains": ["unrelated"],
|
|
3035
|
-
"stdout_not_contains": [],
|
|
3036
|
-
"tags": ["shape_contract"],
|
|
3037
|
-
"tag_evidence": {
|
|
3038
|
-
"shape_contract": [
|
|
3039
|
-
"uses_visible_input_key_names",
|
|
3040
|
-
"asserts_visible_output_key_names",
|
|
3041
|
-
"asserts_no_unexpected_output_keys",
|
|
3042
|
-
],
|
|
3043
|
-
},
|
|
3044
|
-
}) + "\n", encoding="utf-8")
|
|
3045
|
-
bad_solo_headroom_probe = subprocess.run(
|
|
3046
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
3047
|
-
cwd=work,
|
|
3048
|
-
env=env,
|
|
3049
|
-
capture_output=True,
|
|
3050
|
-
text=True,
|
|
3051
|
-
encoding="utf-8",
|
|
3052
|
-
)
|
|
3053
|
-
if bad_solo_headroom_probe.returncode == 0:
|
|
3054
|
-
print("risk probe missing solo-headroom command coverage was accepted", file=sys.stderr)
|
|
3055
|
-
return 1
|
|
3056
|
-
if "risk-probes[0].cmd must contain a solo-headroom hypothesis observable command" not in bad_solo_headroom_probe.stderr:
|
|
3057
|
-
print("solo-headroom risk-probe coverage failure had the wrong error", file=sys.stderr)
|
|
3058
|
-
print(bad_solo_headroom_probe.stderr, file=sys.stderr)
|
|
3059
|
-
return 1
|
|
3060
|
-
|
|
3061
|
-
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3062
|
-
"id": "P4a",
|
|
3063
|
-
"derived_from": "probe must pass visible marker.",
|
|
3064
|
-
"cmd": "bash -lc 'printf ok'",
|
|
3065
|
-
"exit_code": 0,
|
|
3066
|
-
"stdout_contains": ["ok"],
|
|
3067
|
-
"stdout_not_contains": [],
|
|
3068
|
-
"tags": ["shape_contract"],
|
|
3069
|
-
"tag_evidence": {
|
|
3070
|
-
"shape_contract": [
|
|
3071
|
-
"uses_visible_input_key_names",
|
|
3072
|
-
"asserts_visible_output_key_names",
|
|
3073
|
-
"asserts_no_unexpected_output_keys",
|
|
3074
|
-
],
|
|
3075
|
-
},
|
|
3076
|
-
}) + "\n", encoding="utf-8")
|
|
3077
|
-
bad_solo_headroom_derived_from = subprocess.run(
|
|
3078
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
3079
|
-
cwd=work,
|
|
3080
|
-
env=env,
|
|
3081
|
-
capture_output=True,
|
|
3082
|
-
text=True,
|
|
3083
|
-
encoding="utf-8",
|
|
3084
|
-
)
|
|
3085
|
-
if bad_solo_headroom_derived_from.returncode == 0:
|
|
3086
|
-
print("risk probe with unrelated solo-headroom derived_from was accepted", file=sys.stderr)
|
|
3087
|
-
return 1
|
|
3088
|
-
if "risk-probes[0].derived_from must reference the solo-headroom hypothesis bullet" not in bad_solo_headroom_derived_from.stderr:
|
|
3089
|
-
print("solo-headroom risk-probe derived_from failure had the wrong error", file=sys.stderr)
|
|
3090
|
-
print(bad_solo_headroom_derived_from.stderr, file=sys.stderr)
|
|
3091
|
-
return 1
|
|
3092
|
-
|
|
3093
|
-
(devlyn / "risk-probes.jsonl").write_text(
|
|
3094
|
-
json.dumps({
|
|
3095
|
-
"id": "P5a",
|
|
3096
|
-
"derived_from": "solo-headroom hypothesis: solo_claude should miss duplicate handling; `printf ok` exposes the miss.",
|
|
3097
|
-
"cmd": "printf first-unrelated",
|
|
3098
|
-
"exit_code": 0,
|
|
3099
|
-
"stdout_contains": ["first-unrelated"],
|
|
3100
|
-
"stdout_not_contains": [],
|
|
3101
|
-
"tags": ["shape_contract"],
|
|
3102
|
-
"tag_evidence": {
|
|
3103
|
-
"shape_contract": [
|
|
3104
|
-
"uses_visible_input_key_names",
|
|
3105
|
-
"asserts_visible_output_key_names",
|
|
3106
|
-
"asserts_no_unexpected_output_keys",
|
|
3107
|
-
],
|
|
3108
|
-
},
|
|
3109
|
-
}) + "\n" + json.dumps({
|
|
3110
|
-
"id": "P5b",
|
|
3111
|
-
"derived_from": "probe must pass visible marker.",
|
|
3112
|
-
"cmd": "bash -lc 'printf ok'",
|
|
3113
|
-
"exit_code": 0,
|
|
3114
|
-
"stdout_contains": ["ok"],
|
|
3115
|
-
"stdout_not_contains": [],
|
|
3116
|
-
"tags": ["shape_contract"],
|
|
3117
|
-
"tag_evidence": {
|
|
3118
|
-
"shape_contract": [
|
|
3119
|
-
"uses_visible_input_key_names",
|
|
3120
|
-
"asserts_visible_output_key_names",
|
|
3121
|
-
"asserts_no_unexpected_output_keys",
|
|
3122
|
-
],
|
|
3123
|
-
},
|
|
3124
|
-
}) + "\n",
|
|
3125
|
-
encoding="utf-8",
|
|
3126
|
-
)
|
|
3127
|
-
late_solo_headroom_probe = subprocess.run(
|
|
3128
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
3129
|
-
cwd=work,
|
|
3130
|
-
env=env,
|
|
3131
|
-
capture_output=True,
|
|
3132
|
-
text=True,
|
|
3133
|
-
encoding="utf-8",
|
|
3134
|
-
)
|
|
3135
|
-
if late_solo_headroom_probe.returncode == 0:
|
|
3136
|
-
print("solo-headroom command in a later risk probe was accepted", file=sys.stderr)
|
|
3137
|
-
return 1
|
|
3138
|
-
|
|
3139
|
-
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3140
|
-
"id": "P5c",
|
|
3141
|
-
"derived_from": "solo-headroom hypothesis: solo_claude should miss duplicate handling; `printf ok` exposes the miss.",
|
|
3142
|
-
"cmd": "printf ok2",
|
|
3143
|
-
"exit_code": 0,
|
|
3144
|
-
"stdout_contains": ["ok2"],
|
|
3145
|
-
"stdout_not_contains": [],
|
|
3146
|
-
"tags": ["shape_contract"],
|
|
3147
|
-
"tag_evidence": {
|
|
3148
|
-
"shape_contract": [
|
|
3149
|
-
"uses_visible_input_key_names",
|
|
3150
|
-
"asserts_visible_output_key_names",
|
|
3151
|
-
"asserts_no_unexpected_output_keys",
|
|
3152
|
-
],
|
|
3153
|
-
},
|
|
3154
|
-
}) + "\n", encoding="utf-8")
|
|
3155
|
-
prefix_solo_headroom_probe = subprocess.run(
|
|
3156
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
3157
|
-
cwd=work,
|
|
3158
|
-
env=env,
|
|
3159
|
-
capture_output=True,
|
|
3160
|
-
text=True,
|
|
3161
|
-
encoding="utf-8",
|
|
3162
|
-
)
|
|
3163
|
-
if prefix_solo_headroom_probe.returncode == 0:
|
|
3164
|
-
print("solo-headroom command prefix match was accepted", file=sys.stderr)
|
|
3165
|
-
return 1
|
|
3166
|
-
|
|
3167
|
-
(devlyn / "risk-probes.jsonl").write_text(json.dumps({
|
|
3168
|
-
"id": "P5",
|
|
3169
|
-
"derived_from": "solo-headroom hypothesis: solo_claude should miss duplicate handling; `printf ok` exposes the miss.",
|
|
3170
|
-
"cmd": "bash -lc 'printf ok'",
|
|
3171
|
-
"exit_code": 0,
|
|
3172
|
-
"stdout_contains": ["ok"],
|
|
3173
|
-
"stdout_not_contains": [],
|
|
3174
|
-
"tags": ["shape_contract"],
|
|
3175
|
-
"tag_evidence": {
|
|
3176
|
-
"shape_contract": [
|
|
3177
|
-
"uses_visible_input_key_names",
|
|
3178
|
-
"asserts_visible_output_key_names",
|
|
3179
|
-
"asserts_no_unexpected_output_keys",
|
|
3180
|
-
],
|
|
3181
|
-
},
|
|
3182
|
-
}) + "\n", encoding="utf-8")
|
|
3183
|
-
good_solo_headroom_probe = subprocess.run(
|
|
3184
|
-
[sys.executable, script_path, "--validate-risk-probes"],
|
|
3185
|
-
cwd=work,
|
|
3186
|
-
env=env,
|
|
3187
|
-
capture_output=True,
|
|
3188
|
-
text=True,
|
|
3189
|
-
encoding="utf-8",
|
|
3190
|
-
)
|
|
3191
|
-
if good_solo_headroom_probe.returncode != 0:
|
|
3192
|
-
print("risk probe covering solo-headroom command was rejected", file=sys.stderr)
|
|
3193
|
-
print(good_solo_headroom_probe.stderr, file=sys.stderr)
|
|
3194
|
-
return 1
|
|
3195
|
-
|
|
3196
2659
|
expected_json = work / "spec.expected.json"
|
|
3197
2660
|
expected_json.write_text(json.dumps({
|
|
3198
2661
|
"verification_commands": [
|
|
@@ -3209,122 +2672,6 @@ def run_self_test() -> int:
|
|
|
3209
2672
|
"forbidden_files": [],
|
|
3210
2673
|
"max_deps_added": 0,
|
|
3211
2674
|
}) + "\n", encoding="utf-8")
|
|
3212
|
-
spec_md.write_text(
|
|
3213
|
-
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3214
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling.\n"
|
|
3215
|
-
"- Observable command: `node check.js` exposes behavior.\n",
|
|
3216
|
-
encoding="utf-8",
|
|
3217
|
-
)
|
|
3218
|
-
weak_sibling_solo = subprocess.run(
|
|
3219
|
-
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
3220
|
-
cwd=work,
|
|
3221
|
-
capture_output=True,
|
|
3222
|
-
text=True,
|
|
3223
|
-
encoding="utf-8",
|
|
3224
|
-
)
|
|
3225
|
-
if weak_sibling_solo.returncode == 0:
|
|
3226
|
-
print("weak sibling solo-headroom hypothesis was accepted by --check-expected", file=sys.stderr)
|
|
3227
|
-
return 1
|
|
3228
|
-
if "backticked command/observable line that exposes the miss" not in weak_sibling_solo.stderr:
|
|
3229
|
-
print("--check-expected did not report weak sibling solo-headroom hypothesis", file=sys.stderr)
|
|
3230
|
-
print(weak_sibling_solo.stderr, file=sys.stderr)
|
|
3231
|
-
return 1
|
|
3232
|
-
|
|
3233
|
-
spec_md.write_text(
|
|
3234
|
-
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3235
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
3236
|
-
"`node check.js` exposes the miss.\n",
|
|
3237
|
-
encoding="utf-8",
|
|
3238
|
-
)
|
|
3239
|
-
mismatched_sibling_solo = subprocess.run(
|
|
3240
|
-
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
3241
|
-
cwd=work,
|
|
3242
|
-
capture_output=True,
|
|
3243
|
-
text=True,
|
|
3244
|
-
encoding="utf-8",
|
|
3245
|
-
)
|
|
3246
|
-
if mismatched_sibling_solo.returncode == 0:
|
|
3247
|
-
print("mismatched sibling solo-headroom command was accepted by --check-expected", file=sys.stderr)
|
|
3248
|
-
return 1
|
|
3249
|
-
if "observable command must match spec.expected.json" not in mismatched_sibling_solo.stderr:
|
|
3250
|
-
print("--check-expected did not report mismatched sibling solo-headroom command", file=sys.stderr)
|
|
3251
|
-
print(mismatched_sibling_solo.stderr, file=sys.stderr)
|
|
3252
|
-
return 1
|
|
3253
|
-
|
|
3254
|
-
spec_md.write_text(
|
|
3255
|
-
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3256
|
-
"- solo-headroom hypothesis: solo_claude should miss duplicate handling; "
|
|
3257
|
-
"`printf ok` exposes the miss.\n",
|
|
3258
|
-
encoding="utf-8",
|
|
3259
|
-
)
|
|
3260
|
-
matched_sibling_solo = subprocess.run(
|
|
3261
|
-
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
3262
|
-
cwd=work,
|
|
3263
|
-
capture_output=True,
|
|
3264
|
-
text=True,
|
|
3265
|
-
encoding="utf-8",
|
|
3266
|
-
)
|
|
3267
|
-
if matched_sibling_solo.returncode != 0:
|
|
3268
|
-
print("matched sibling solo-headroom command was rejected by --check-expected", file=sys.stderr)
|
|
3269
|
-
print(matched_sibling_solo.stderr, file=sys.stderr)
|
|
3270
|
-
return 1
|
|
3271
|
-
|
|
3272
|
-
spec_md.write_text(
|
|
3273
|
-
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3274
|
-
"- Solo-headroom hypothesis: the spec must literally contain `solo_claude`, `miss`, and an observable command; "
|
|
3275
|
-
"`printf ok` exposes the miss.\n",
|
|
3276
|
-
encoding="utf-8",
|
|
3277
|
-
)
|
|
3278
|
-
docs_style_sibling_solo = subprocess.run(
|
|
3279
|
-
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
3280
|
-
cwd=work,
|
|
3281
|
-
capture_output=True,
|
|
3282
|
-
text=True,
|
|
3283
|
-
encoding="utf-8",
|
|
3284
|
-
)
|
|
3285
|
-
if docs_style_sibling_solo.returncode != 0:
|
|
3286
|
-
print("docs-style sibling solo-headroom command was rejected by --check-expected", file=sys.stderr)
|
|
3287
|
-
print(docs_style_sibling_solo.stderr, file=sys.stderr)
|
|
3288
|
-
return 1
|
|
3289
|
-
|
|
3290
|
-
spec_md.write_text(
|
|
3291
|
-
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3292
|
-
"- solo ceiling avoidance: this differs from controls but omits the required baseline.\n",
|
|
3293
|
-
encoding="utf-8",
|
|
3294
|
-
)
|
|
3295
|
-
weak_sibling_solo_ceiling = subprocess.run(
|
|
3296
|
-
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
3297
|
-
cwd=work,
|
|
3298
|
-
capture_output=True,
|
|
3299
|
-
text=True,
|
|
3300
|
-
encoding="utf-8",
|
|
3301
|
-
)
|
|
3302
|
-
if weak_sibling_solo_ceiling.returncode == 0:
|
|
3303
|
-
print("weak sibling solo ceiling avoidance was accepted by --check-expected", file=sys.stderr)
|
|
3304
|
-
return 1
|
|
3305
|
-
if "concrete difference from rejected or solo-saturated controls" not in weak_sibling_solo_ceiling.stderr:
|
|
3306
|
-
print("--check-expected did not report weak sibling solo ceiling avoidance", file=sys.stderr)
|
|
3307
|
-
print(weak_sibling_solo_ceiling.stderr, file=sys.stderr)
|
|
3308
|
-
return 1
|
|
3309
|
-
|
|
3310
|
-
spec_md.write_text(
|
|
3311
|
-
"# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n"
|
|
3312
|
-
"- solo ceiling avoidance: unlike solo-saturated `S2`-`S6`, this includes a "
|
|
3313
|
-
"multi-run temporal dependency because solo_claude headroom should remain.\n",
|
|
3314
|
-
encoding="utf-8",
|
|
3315
|
-
)
|
|
3316
|
-
strong_sibling_solo_ceiling = subprocess.run(
|
|
3317
|
-
[sys.executable, script_path, "--check-expected", str(expected_json)],
|
|
3318
|
-
cwd=work,
|
|
3319
|
-
capture_output=True,
|
|
3320
|
-
text=True,
|
|
3321
|
-
encoding="utf-8",
|
|
3322
|
-
)
|
|
3323
|
-
if strong_sibling_solo_ceiling.returncode != 0:
|
|
3324
|
-
print("actionable sibling solo ceiling avoidance was rejected by --check-expected", file=sys.stderr)
|
|
3325
|
-
print(strong_sibling_solo_ceiling.stderr, file=sys.stderr)
|
|
3326
|
-
return 1
|
|
3327
|
-
|
|
3328
2675
|
spec_md.write_text("# Spec\n\n<!-- devlyn:verification -->\n## Verification\n\n- probe must pass visible marker.\n", encoding="utf-8")
|
|
3329
2676
|
expected_good = subprocess.run(
|
|
3330
2677
|
[sys.executable, script_path, "--check-expected", str(expected_json)],
|