devlyn-cli 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CLAUDE.md +2 -2
- package/README.md +22 -113
- package/bin/devlyn.js +13 -225
- package/bin/instruction-templates.json +752 -1
- package/bin/instructions.js +101 -17
- package/config/skills/_shared/adapters/grok.md +1 -2
- package/config/skills/_shared/archive_run.py +15 -3
- package/config/skills/_shared/judge-role-evidence.py +21 -1
- package/config/skills/_shared/runtime-principles.md +1 -1
- package/config/skills/_shared/spec-verify-check.py +0 -653
- package/config/skills/_shared/state-phase-write.py +371 -30
- package/config/skills/_shared/task-complete.py +635 -143
- package/config/skills/_shared/terminal-claim-check.py +229 -4
- package/config/skills/_shared/verify-merge-findings.py +93 -2
- package/config/skills/devlyn:ideate/SKILL.md +3 -4
- package/config/skills/devlyn:ideate/references/elicitation.md +2 -35
- package/config/skills/devlyn:ideate/references/from-spec-mode.md +3 -18
- package/config/skills/devlyn:ideate/references/project-mode.md +1 -16
- package/config/skills/devlyn:ideate/references/spec-template.md +0 -2
- package/config/skills/devlyn:queue/SKILL.md +6 -4
- package/config/skills/devlyn:resolve/SKILL.md +16 -25
- package/config/skills/devlyn:resolve/references/free-form-mode.md +0 -14
- package/config/skills/devlyn:resolve/references/outer-loop.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/build-gate.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/cleanup.md +3 -3
- package/config/skills/devlyn:resolve/references/phases/implement.md +1 -1
- package/config/skills/devlyn:resolve/references/phases/probe-derive.md +1 -16
- package/config/skills/devlyn:resolve/references/phases/verify.md +4 -9
- package/config/skills/devlyn:resolve/references/state-schema.md +9 -7
- package/config/skills/devlyn:resolve/references/task-completion.md +46 -37
- package/{config/skills → optional-skills}/code-health-standards/SKILL.md +4 -13
- package/{config/skills → optional-skills}/code-review-standards/SKILL.md +4 -12
- package/{config/skills → optional-skills}/root-cause-analysis/SKILL.md +4 -12
- package/{config/skills → optional-skills}/ui-implementation-standards/SKILL.md +4 -14
- package/package.json +1 -57
- package/benchmark/auto-resolve/BENCHMARK-DESIGN.md +0 -289
- package/benchmark/auto-resolve/BENCHMARK-RESULTS.md +0 -341
- package/benchmark/auto-resolve/README.md +0 -696
- package/benchmark/auto-resolve/RUBRIC.md +0 -177
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/spec.md +0 -44
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.ko.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/task.txt +0 -8
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/NOTES.md +0 -68
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/expected.json +0 -60
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/invalid.js +0 -29
- package/benchmark/auto-resolve/fixtures/F10-persist-write-collision/verifiers/parallel.js +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/NOTES.md +0 -75
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/expected.json +0 -52
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/setup.sh +0 -171
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/spec.md +0 -50
- package/benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/NOTES.md +0 -89
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/setup.sh +0 -251
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/spec.md +0 -57
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/task.txt +0 -13
- package/benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/verifiers/replay-malformed-body.js +0 -64
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md +0 -104
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/expected.json +0 -46
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/setup.sh +0 -336
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/spec.md +0 -51
- package/benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/setup.sh +0 -32
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/spec.md +0 -63
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/exact-success.js +0 -54
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/no-hardcoded-pricing.js +0 -47
- package/benchmark/auto-resolve/fixtures/F16-cli-quote-tax-rules/verifiers/stock-error.js +0 -45
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected-pair-plan-registry.json +0 -170
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/expected.json +0 -84
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/metadata.json +0 -21
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-fail.json +0 -214
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/pair-plan.sample-pass.json +0 -223
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/setup.sh +0 -5
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/task.txt +0 -14
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/expected.json +0 -62
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/error-order.js +0 -55
- package/benchmark/auto-resolve/fixtures/F21-cli-scheduler-priority/verifiers/priority-blocked.js +0 -48
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/spec.md +0 -64
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/conflicting-duplicate.js +0 -34
- package/benchmark/auto-resolve/fixtures/F22-cli-ledger-close/verifiers/idempotent-close.js +0 -41
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/NOTES.md +0 -39
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/spec.md +0 -76
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/priority-rollback.js +0 -64
- package/benchmark/auto-resolve/fixtures/F23-cli-fulfillment-wave/verifiers/single-warehouse-fefo.js +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/NOTES.md +0 -40
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/expected.json +0 -66
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/spec.md +0 -71
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/catalog-source.js +0 -57
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/exact-success.js +0 -63
- package/benchmark/auto-resolve/fixtures/F25-cli-cart-promotion-rules/verifiers/stock-error.js +0 -34
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/NOTES.md +0 -31
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/expected.json +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/setup.sh +0 -17
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/spec.md +0 -68
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/conflicting-duplicate.js +0 -29
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/exact-payout.js +0 -58
- package/benchmark/auto-resolve/fixtures/F26-cli-payout-ledger-rules/verifiers/rules-source.js +0 -56
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected-pair-plan-registry.json +0 -162
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/metadata.json +0 -19
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/spec.md +0 -55
- package/benchmark/auto-resolve/fixtures/F3-backend-contract-risk/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md +0 -34
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/duplicate-event-error.js +0 -35
- package/benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js +0 -53
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md +0 -38
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/spec.md +0 -70
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/task.txt +0 -3
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/duplicate-renewal-error.js +0 -42
- package/benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js +0 -70
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/NOTES.md +0 -47
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/setup.sh +0 -6
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F4-web-browser-design/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/NOTES.md +0 -45
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/setup.sh +0 -55
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md +0 -43
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/expected.json +0 -77
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/task.txt +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md +0 -57
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/expected.json +0 -76
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/setup.sh +0 -36
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/spec.md +0 -45
- package/benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md +0 -53
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/expected.json +0 -63
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/spec.md +0 -48
- package/benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/task.txt +0 -1
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md +0 -105
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/expected.json +0 -74
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/SCHEMA.md +0 -188
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md +0 -37
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md +0 -13
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/expected.json +0 -56
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/spec.md +0 -69
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/task.txt +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/exact-proration.js +0 -48
- package/benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/verifiers/rules-source-and-conflict.js +0 -79
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md +0 -7
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/expected.json +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/spec.md +0 -67
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/policy-precedence.js +0 -72
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-and-immutability.js +0 -43
- package/benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/verifiers/validation-boundary.js +0 -116
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/NOTES.md +0 -35
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/RETIRED.md +0 -12
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/expected.json +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/setup.sh +0 -2
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/spec.md +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/task.txt +0 -17
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/mixed-idempotent-settlement.js +0 -53
- package/benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/verifiers/rejection-boundaries.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/NOTES.md +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/RETIRED.md +0 -15
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/expected.json +0 -57
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/setup.sh +0 -50
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/spec.md +0 -59
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/ordered-settlement.js +0 -88
- package/benchmark/auto-resolve/fixtures/retired/F34-cli-settle-inflight/verifiers/reject-invalid.js +0 -45
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/NOTES.md +0 -30
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/setup.sh +0 -21
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/dup-op.js +0 -54
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/priority-apply.js +0 -82
- package/benchmark/auto-resolve/fixtures/retired/F35-cli-apply-journal/verifiers/rollback.js +0 -77
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/NOTES.md +0 -99
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/RETIRED.md +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/expected.json +0 -64
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/setup.sh +0 -4
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/spec.md +0 -61
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/correctness-small.js +0 -135
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/scale.js +0 -100
- package/benchmark/auto-resolve/fixtures/retired/F36-cli-session-admission/verifiers/validation.js +0 -75
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/NOTES.md +0 -93
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/RETIRED.md +0 -16
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/expected.json +0 -65
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/setup.sh +0 -18
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/spec.md +0 -66
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/task.txt +0 -9
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/correctness-small.js +0 -84
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/scale.js +0 -184
- package/benchmark/auto-resolve/fixtures/retired/F37-cli-rule-lookup/verifiers/validation.js +0 -74
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/NOTES.md +0 -60
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/RETIRED.md +0 -29
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/expected.json +0 -73
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/metadata.json +0 -10
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/setup.sh +0 -28
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/spec.md +0 -58
- package/benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/task.txt +0 -5
- package/benchmark/auto-resolve/fixtures/test-repo/README.md +0 -12
- package/benchmark/auto-resolve/fixtures/test-repo/bin/cli.js +0 -63
- package/benchmark/auto-resolve/fixtures/test-repo/package-lock.json +0 -823
- package/benchmark/auto-resolve/fixtures/test-repo/package.json +0 -22
- package/benchmark/auto-resolve/fixtures/test-repo/playwright.config.js +0 -17
- package/benchmark/auto-resolve/fixtures/test-repo/server/index.js +0 -37
- package/benchmark/auto-resolve/fixtures/test-repo/tests/cli.test.js +0 -25
- package/benchmark/auto-resolve/fixtures/test-repo/tests/server.test.js +0 -58
- package/benchmark/auto-resolve/fixtures/test-repo/web/index.html +0 -37
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.json +0 -25
- package/benchmark/auto-resolve/results/20260507-f10-f11-tier1-full-pipeline/headroom-gate.md +0 -10
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f22-exact-error-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260508-f26-headroom/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json +0 -28
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json +0 -30
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json +0 -82
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md +0 -18
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json +0 -46
- package/benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md +0 -17
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f12-webhook-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f15-concurrency-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json +0 -32
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json +0 -44
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md +0 -16
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json +0 -18
- package/benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md +0 -9
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260511-f3-http-error-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f4-web-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f5-fixloop-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f6-checksum-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f7-scope-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.json +0 -28
- package/benchmark/auto-resolve/results/20260512-f9-e2e-headroom/headroom-gate.md +0 -15
- package/benchmark/auto-resolve/run-real-benchmark.md +0 -303
- package/benchmark/auto-resolve/scripts/audit-headroom-rejections.py +0 -441
- package/benchmark/auto-resolve/scripts/audit-pair-evidence.py +0 -1256
- package/benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py +0 -306
- package/benchmark/auto-resolve/scripts/check-f9-artifacts.py +0 -268
- package/benchmark/auto-resolve/scripts/collect-swebench-predictions.py +0 -108
- package/benchmark/auto-resolve/scripts/compile-report.py +0 -493
- package/benchmark/auto-resolve/scripts/fetch-swebench-instances.py +0 -129
- package/benchmark/auto-resolve/scripts/frozen-verify-gate.py +0 -434
- package/benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py +0 -612
- package/benchmark/auto-resolve/scripts/headroom-gate.py +0 -378
- package/benchmark/auto-resolve/scripts/iter-0033c-compare.py +0 -683
- package/benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py +0 -97
- package/benchmark/auto-resolve/scripts/judge-opus-pass.sh +0 -542
- package/benchmark/auto-resolve/scripts/judge.sh +0 -565
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-a.py +0 -256
- package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py +0 -287
- package/benchmark/auto-resolve/scripts/oracle-test-fidelity.py +0 -328
- package/benchmark/auto-resolve/scripts/pair-candidate-frontier.py +0 -469
- package/benchmark/auto-resolve/scripts/pair-plan-idgen.py +0 -401
- package/benchmark/auto-resolve/scripts/pair-plan-lint.py +0 -475
- package/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh +0 -103
- package/benchmark/auto-resolve/scripts/pair_evidence_contract.py +0 -270
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py +0 -273
- package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py +0 -148
- package/benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py +0 -210
- package/benchmark/auto-resolve/scripts/recent-benchmark-summary.py +0 -232
- package/benchmark/auto-resolve/scripts/run-fixture.sh +0 -1024
- package/benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh +0 -683
- package/benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh +0 -458
- package/benchmark/auto-resolve/scripts/run-headroom-candidate.sh +0 -336
- package/benchmark/auto-resolve/scripts/run-iter-0033c.sh +0 -208
- package/benchmark/auto-resolve/scripts/run-suite.sh +0 -251
- package/benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh +0 -310
- package/benchmark/auto-resolve/scripts/run-swebench-solver-batch.sh +0 -257
- package/benchmark/auto-resolve/scripts/ship-gate.py +0 -391
- package/benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py +0 -53
- package/benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py +0 -77
- package/benchmark/auto-resolve/scripts/swebench-frozen-matrix.py +0 -478
- package/benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh +0 -288
- package/benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh +0 -1672
- package/benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh +0 -1015
- package/benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh +0 -491
- package/benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh +0 -91
- package/benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh +0 -517
- package/benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh +0 -610
- package/benchmark/auto-resolve/scripts/test-headroom-gate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh +0 -525
- package/benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh +0 -254
- package/benchmark/auto-resolve/scripts/test-lint-fixtures.sh +0 -580
- package/benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh +0 -591
- package/benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh +0 -497
- package/benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh +0 -401
- package/benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh +0 -111
- package/benchmark/auto-resolve/scripts/test-ship-gate.sh +0 -1189
- package/benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh +0 -1221
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/NOTES.md +0 -28
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/expected.json +0 -63
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/spec.md +0 -47
- package/benchmark/auto-resolve/shadow-fixtures/S1-cli-lang-flag/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/expected.json +0 -53
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/spec.md +0 -50
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/duplicate-order-error.js +0 -27
- package/benchmark/auto-resolve/shadow-fixtures/S2-cli-inventory-reservation/verifiers/priority-stock-reservation.js +0 -44
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/spec.md +0 -52
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/duplicate-ticket-error.js +0 -29
- package/benchmark/auto-resolve/shadow-fixtures/S3-cli-ticket-assignment/verifiers/priority-agent-assignment.js +0 -48
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/NOTES.md +0 -34
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/expected.json +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/spec.md +0 -55
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/duplicate-return-error.js +0 -43
- package/benchmark/auto-resolve/shadow-fixtures/S4-cli-return-routing/verifiers/priority-return-routing.js +0 -70
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/NOTES.md +0 -37
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/expected.json +0 -54
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/credit-ledger-priority.js +0 -98
- package/benchmark/auto-resolve/shadow-fixtures/S5-cli-credit-grant-ledger/verifiers/duplicate-charge-error.js +0 -38
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/NOTES.md +0 -36
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/expected.json +0 -56
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/metadata.json +0 -10
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/setup.sh +0 -3
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/spec.md +0 -59
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/task.txt +0 -1
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/duplicate-refund-error.js +0 -41
- package/benchmark/auto-resolve/shadow-fixtures/S6-cli-refund-window-ledger/verifiers/priority-refund-ledger.js +0 -65
- package/config/skills/_shared/pair-plan-schema.md +0 -298
- package/scripts/lint-fixtures.sh +0 -352
- package/scripts/lint-shadow-fixtures.sh +0 -58
- package/scripts/lint-skills.sh +0 -4265
- package/scripts/test-owner-phases.py +0 -193
package/scripts/lint-skills.sh
DELETED
|
@@ -1,4265 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# lint-skills.sh — structural quality checks for the devlyn harness.
|
|
3
|
-
#
|
|
4
|
-
# Gates the three things that have drifted in the past:
|
|
5
|
-
# 1. Forbidden MCP / stale-model references in skills, README, installer.
|
|
6
|
-
# 2. Missing `name:` in skill frontmatter (Anthropic spec violation).
|
|
7
|
-
# 3. Source ↔ installed mirror drift on the harness critical path.
|
|
8
|
-
#
|
|
9
|
-
# Exit 0 = clean. Non-zero = fails; prints offending file:line per check.
|
|
10
|
-
|
|
11
|
-
set -u
|
|
12
|
-
cd "$(dirname "$0")/.."
|
|
13
|
-
|
|
14
|
-
red=$(printf '\033[31m'); green=$(printf '\033[32m'); dim=$(printf '\033[2m'); reset=$(printf '\033[0m')
|
|
15
|
-
fail=0
|
|
16
|
-
|
|
17
|
-
section() { printf '\n%s=== %s ===%s\n' "$dim" "$1" "$reset"; }
|
|
18
|
-
ok() { printf ' %s✓%s %s\n' "$green" "$reset" "$1"; }
|
|
19
|
-
bad() { printf ' %s✗%s %s\n' "$red" "$reset" "$1"; fail=1; }
|
|
20
|
-
|
|
21
|
-
make_temp_file() {
|
|
22
|
-
local __var="$1"
|
|
23
|
-
shift || true
|
|
24
|
-
local path
|
|
25
|
-
if ! path=$(command mktemp "$@"); then
|
|
26
|
-
bad "mktemp failed: ${*:-<default>}"
|
|
27
|
-
return 1
|
|
28
|
-
fi
|
|
29
|
-
printf -v "$__var" '%s' "$path"
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
make_temp_dir() {
|
|
33
|
-
local __var="$1"
|
|
34
|
-
shift || true
|
|
35
|
-
local path
|
|
36
|
-
if ! path=$(command mktemp -d "$@"); then
|
|
37
|
-
bad "mktemp -d failed: ${*:-<default>}"
|
|
38
|
-
return 1
|
|
39
|
-
fi
|
|
40
|
-
printf -v "$__var" '%s' "$path"
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
section "Check 0a: Temp allocation fails closed"
|
|
44
|
-
direct_mktemp=$(grep -nE '(^|[ =])mktemp( |$)|\$\([[:space:]]*mktemp' scripts/lint-skills.sh \
|
|
45
|
-
| grep -v 'command mktemp' \
|
|
46
|
-
| grep -v 'make_temp_' \
|
|
47
|
-
| grep -v 'direct_mktemp=' || true)
|
|
48
|
-
if [ -z "$direct_mktemp" ]; then
|
|
49
|
-
ok "lint-skills.sh uses temp allocation helpers instead of direct mktemp"
|
|
50
|
-
else
|
|
51
|
-
while IFS= read -r f; do bad "$f"; done <<< "$direct_mktemp"
|
|
52
|
-
fi
|
|
53
|
-
|
|
54
|
-
# iter-0034 Phase 4 cutover (2026-05-03): legacy skill paths dropped.
|
|
55
|
-
# Surface is the 2-skill product (`/devlyn:resolve` + `/devlyn:ideate`)
|
|
56
|
-
# plus the `_shared/` kernel. Keep this list single-source so all installed
|
|
57
|
-
# mirror parity checks cover the same files.
|
|
58
|
-
critical_path_files=$(cat <<'EOF'
|
|
59
|
-
_shared/.ruff.toml
|
|
60
|
-
_shared/process-evidence.py
|
|
61
|
-
_shared/platform-support.py
|
|
62
|
-
_shared/invocation-receipt.py
|
|
63
|
-
_shared/role-config.py
|
|
64
|
-
_shared/judge-role-evidence.py
|
|
65
|
-
_shared/spec-verify-check.py
|
|
66
|
-
_shared/judge-output-parser.py
|
|
67
|
-
_shared/collect-codex-findings.py
|
|
68
|
-
_shared/verify-merge-findings.py
|
|
69
|
-
_shared/archive_run.py
|
|
70
|
-
_shared/task-complete.py
|
|
71
|
-
_shared/state-phase-write.py
|
|
72
|
-
_shared/terminal-claim-check.py
|
|
73
|
-
_shared/resolve-stop-hook.py
|
|
74
|
-
_shared/resolve-bootstrap.py
|
|
75
|
-
devlyn:ideate/SKILL.md
|
|
76
|
-
devlyn:ideate/references/spec-template.md
|
|
77
|
-
devlyn:ideate/references/elicitation.md
|
|
78
|
-
devlyn:ideate/references/project-mode.md
|
|
79
|
-
devlyn:ideate/references/from-spec-mode.md
|
|
80
|
-
devlyn:resolve/SKILL.md
|
|
81
|
-
devlyn:queue/SKILL.md
|
|
82
|
-
devlyn:engines/SKILL.md
|
|
83
|
-
devlyn:resolve/references/state-schema.md
|
|
84
|
-
devlyn:resolve/references/task-completion.md
|
|
85
|
-
devlyn:resolve/references/outer-loop.md
|
|
86
|
-
devlyn:resolve/references/free-form-mode.md
|
|
87
|
-
devlyn:resolve/references/phases/plan.md
|
|
88
|
-
devlyn:resolve/references/phases/probe-derive.md
|
|
89
|
-
devlyn:resolve/references/phases/implement.md
|
|
90
|
-
devlyn:resolve/references/phases/surface-close.md
|
|
91
|
-
devlyn:resolve/references/phases/build-gate.md
|
|
92
|
-
devlyn:resolve/references/phases/cleanup.md
|
|
93
|
-
devlyn:resolve/references/phases/verify.md
|
|
94
|
-
_shared/expected.schema.json
|
|
95
|
-
_shared/adapters/README.md
|
|
96
|
-
_shared/adapters/claude.md
|
|
97
|
-
_shared/adapters/codex.md
|
|
98
|
-
_shared/adapters/grok.md
|
|
99
|
-
_shared/adapters/omp.md
|
|
100
|
-
_shared/codex-config.md
|
|
101
|
-
_shared/codex-monitored.sh
|
|
102
|
-
_shared/engine-preflight.md
|
|
103
|
-
_shared/pair-plan-schema.md
|
|
104
|
-
_shared/runtime-principles.md
|
|
105
|
-
EOF
|
|
106
|
-
)
|
|
107
|
-
|
|
108
|
-
check_skill_mirror_parity() {
|
|
109
|
-
local target_dir="$1"
|
|
110
|
-
local skip_msg="$2"
|
|
111
|
-
local missing_prefix="$3"
|
|
112
|
-
local differ_suffix="$4"
|
|
113
|
-
local ok_msg="$5"
|
|
114
|
-
local drift=0
|
|
115
|
-
local rel src dst
|
|
116
|
-
|
|
117
|
-
if [ ! -d "$target_dir" ]; then
|
|
118
|
-
ok "$skip_msg"
|
|
119
|
-
return
|
|
120
|
-
fi
|
|
121
|
-
|
|
122
|
-
while IFS= read -r rel; do
|
|
123
|
-
[ -n "$rel" ] || continue
|
|
124
|
-
src="config/skills/$rel"
|
|
125
|
-
dst="$target_dir/$rel"
|
|
126
|
-
if [ ! -f "$src" ] || [ ! -f "$dst" ]; then
|
|
127
|
-
bad "$missing_prefix: $rel"; drift=1; continue
|
|
128
|
-
fi
|
|
129
|
-
if ! diff -q "$src" "$dst" >/dev/null 2>&1; then
|
|
130
|
-
bad "$rel — $differ_suffix"
|
|
131
|
-
drift=1
|
|
132
|
-
fi
|
|
133
|
-
done <<< "$critical_path_files"
|
|
134
|
-
|
|
135
|
-
# iter-0009: codex-monitored.sh must be executable in installed mirrors
|
|
136
|
-
# (skills trees get copied into work dirs for variant arms; bash refuses to
|
|
137
|
-
# run a non-executable wrapper).
|
|
138
|
-
if [ -f "$target_dir/_shared/codex-monitored.sh" ] \
|
|
139
|
-
&& [ ! -x "$target_dir/_shared/codex-monitored.sh" ]; then
|
|
140
|
-
bad "_shared/codex-monitored.sh — not executable in ${target_dir} mirror"
|
|
141
|
-
drift=1
|
|
142
|
-
fi
|
|
143
|
-
if [ $drift -eq 0 ]; then
|
|
144
|
-
ok "$ok_msg"
|
|
145
|
-
fi
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
# ---------------------------------------------------------------------------
|
|
149
|
-
# 1. No MCP references in managed source or user-facing docs.
|
|
150
|
-
# ---------------------------------------------------------------------------
|
|
151
|
-
section "Check 1: No mcp__codex-cli__ outside _shared / archive"
|
|
152
|
-
# Legal places: config/skills/_shared/codex-config.md (explicitly says "MCP is not used"),
|
|
153
|
-
# archival snapshots, and tests.
|
|
154
|
-
offenders=$(git grep -Il -- 'mcp__codex-cli__' -- \
|
|
155
|
-
config/skills \
|
|
156
|
-
benchmark \
|
|
157
|
-
README.md \
|
|
158
|
-
CLAUDE.md \
|
|
159
|
-
bin/ \
|
|
160
|
-
':!config/skills/_shared/codex-config.md' \
|
|
161
|
-
':!config/skills/roadmap-archival-workspace/**' \
|
|
162
|
-
':!config/skills/devlyn:auto-resolve-workspace/**' \
|
|
163
|
-
':!config/skills/devlyn:ideate-workspace/**' \
|
|
164
|
-
':!config/skills/preflight-workspace/**' \
|
|
165
|
-
':!benchmark/auto-resolve/external/**' \
|
|
166
|
-
':!benchmark/auto-resolve/results/**' \
|
|
167
|
-
':!benchmark/ceiling/results/**' \
|
|
168
|
-
':!benchmark/auto-resolve/PILOT-RESULTS*' \
|
|
169
|
-
2>/dev/null || true)
|
|
170
|
-
if [ -z "$offenders" ]; then
|
|
171
|
-
ok "no MCP references in managed files"
|
|
172
|
-
else
|
|
173
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
174
|
-
fi
|
|
175
|
-
|
|
176
|
-
# ---------------------------------------------------------------------------
|
|
177
|
-
# 2. No "Requires Codex MCP" prose.
|
|
178
|
-
# ---------------------------------------------------------------------------
|
|
179
|
-
section "Check 2: No 'Requires Codex MCP' prose"
|
|
180
|
-
offenders=$(git grep -Il -- 'Requires Codex MCP\|Codex MCP server\|Codex MCP available\|Codex MCP disconnected' -- \
|
|
181
|
-
config/skills \
|
|
182
|
-
benchmark \
|
|
183
|
-
README.md \
|
|
184
|
-
CLAUDE.md \
|
|
185
|
-
bin/ \
|
|
186
|
-
':!config/skills/roadmap-archival-workspace/**' \
|
|
187
|
-
':!config/skills/devlyn:auto-resolve-workspace/**' \
|
|
188
|
-
':!config/skills/devlyn:ideate-workspace/**' \
|
|
189
|
-
':!config/skills/preflight-workspace/**' \
|
|
190
|
-
':!benchmark/auto-resolve/external/**' \
|
|
191
|
-
':!benchmark/auto-resolve/results/**' \
|
|
192
|
-
':!benchmark/ceiling/results/**' \
|
|
193
|
-
':!benchmark/auto-resolve/PILOT-RESULTS*' \
|
|
194
|
-
2>/dev/null || true)
|
|
195
|
-
if [ -z "$offenders" ]; then
|
|
196
|
-
ok "no Codex MCP prose"
|
|
197
|
-
else
|
|
198
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
199
|
-
fi
|
|
200
|
-
|
|
201
|
-
# ---------------------------------------------------------------------------
|
|
202
|
-
# 3. No stale model strings (gpt-5.0..5.4 hardcoded outside config).
|
|
203
|
-
# ---------------------------------------------------------------------------
|
|
204
|
-
section "Check 3: No hardcoded pre-5.5 model strings"
|
|
205
|
-
offenders=$(grep -RInE 'gpt-5\.[0-4][^.]' \
|
|
206
|
-
config/skills CLAUDE.md README.md 2>/dev/null \
|
|
207
|
-
| grep -v 'config/skills/_shared/codex-config.md' \
|
|
208
|
-
| grep -v 'config/skills/roadmap-archival-workspace/' \
|
|
209
|
-
| grep -v 'config/skills/devlyn:auto-resolve-workspace/' \
|
|
210
|
-
| grep -v 'config/skills/devlyn:ideate-workspace/' \
|
|
211
|
-
| grep -v 'config/skills/preflight-workspace/' \
|
|
212
|
-
| grep -v 'evals\.json' \
|
|
213
|
-
|| true)
|
|
214
|
-
if [ -z "$offenders" ]; then
|
|
215
|
-
ok "no hardcoded pre-5.5 strings"
|
|
216
|
-
else
|
|
217
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
218
|
-
fi
|
|
219
|
-
|
|
220
|
-
# ---------------------------------------------------------------------------
|
|
221
|
-
# 4. No model-pinned Claude references, any generation, except the
|
|
222
|
-
# validated explicit-effort capability declaration.
|
|
223
|
-
# ---------------------------------------------------------------------------
|
|
224
|
-
section "Check 4: No model-pinned Claude references"
|
|
225
|
-
offenders=$(grep -RInE 'Claude (Opus|Sonnet|Haiku|Fable)|claude-(opus|sonnet|haiku|fable)-[0-9]' \
|
|
226
|
-
config/skills 2>/dev/null \
|
|
227
|
-
| grep -v 'config/skills/roadmap-archival-workspace/' \
|
|
228
|
-
| grep -v 'config/skills/devlyn:auto-resolve-workspace/' \
|
|
229
|
-
| grep -v 'config/skills/devlyn:ideate-workspace/' \
|
|
230
|
-
| grep -v 'config/skills/preflight-workspace/' \
|
|
231
|
-
| grep -vE '^config/skills/_shared/adapters/claude\.md:[0-9]+:<!-- devlyn-effort 2\.1\.263 claude-fable-5-1 low,medium,high,xhigh,max -->$' \
|
|
232
|
-
| cut -d: -f1 \
|
|
233
|
-
| sort -u \
|
|
234
|
-
|| true)
|
|
235
|
-
if [ -z "$offenders" ]; then
|
|
236
|
-
ok "no model-pinned Claude references"
|
|
237
|
-
else
|
|
238
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
239
|
-
fi
|
|
240
|
-
|
|
241
|
-
# ---------------------------------------------------------------------------
|
|
242
|
-
# 5. Every devlyn:* skill has `name:` in frontmatter.
|
|
243
|
-
# ---------------------------------------------------------------------------
|
|
244
|
-
section "Check 5: devlyn:* SKILL.md has name: field"
|
|
245
|
-
missing=0
|
|
246
|
-
for skill in config/skills/devlyn:*/SKILL.md; do
|
|
247
|
-
[ -f "$skill" ] || continue
|
|
248
|
-
if ! head -20 "$skill" | grep -q '^name:'; then
|
|
249
|
-
bad "$skill — missing 'name:' in frontmatter"
|
|
250
|
-
missing=1
|
|
251
|
-
fi
|
|
252
|
-
done
|
|
253
|
-
if [ $missing -eq 0 ]; then
|
|
254
|
-
ok "all devlyn:* skills have name: field"
|
|
255
|
-
fi
|
|
256
|
-
|
|
257
|
-
# ---------------------------------------------------------------------------
|
|
258
|
-
# 5a. devlyn:design-ui is a required skill, not an optional addon.
|
|
259
|
-
# ---------------------------------------------------------------------------
|
|
260
|
-
section "Check 5a: devlyn:design-ui is required"
|
|
261
|
-
if [ -f "config/skills/devlyn:design-ui/SKILL.md" ]; then
|
|
262
|
-
ok "devlyn:design-ui source lives in config/skills"
|
|
263
|
-
else
|
|
264
|
-
bad "devlyn:design-ui must be a required skill under config/skills"
|
|
265
|
-
fi
|
|
266
|
-
if [ ! -e "optional-skills/devlyn:design-ui" ]; then
|
|
267
|
-
ok "devlyn:design-ui is not in optional-skills"
|
|
268
|
-
else
|
|
269
|
-
bad "devlyn:design-ui must not be installed as an optional addon"
|
|
270
|
-
fi
|
|
271
|
-
if grep -Fq "const DEVLYN_CORE_SKILLS = ['devlyn:resolve', 'devlyn:ideate', 'devlyn:design-ui', 'devlyn:engines', 'devlyn:queue', '_shared'];" bin/devlyn.js \
|
|
272
|
-
&& grep -Fq "skillsToInstall: DEVLYN_CORE_SKILLS" bin/devlyn.js; then
|
|
273
|
-
ok "Codex install includes devlyn:design-ui (via shared DEVLYN_CORE_SKILLS bundle)"
|
|
274
|
-
else
|
|
275
|
-
bad "Codex skillsToInstall must include devlyn:design-ui (shared DEVLYN_CORE_SKILLS bundle)"
|
|
276
|
-
fi
|
|
277
|
-
if ! grep -F "name: 'devlyn:design-ui'" bin/devlyn.js >/dev/null 2>&1; then
|
|
278
|
-
ok "devlyn:design-ui is absent from OPTIONAL_ADDONS"
|
|
279
|
-
else
|
|
280
|
-
bad "devlyn:design-ui must not be listed in OPTIONAL_ADDONS"
|
|
281
|
-
fi
|
|
282
|
-
|
|
283
|
-
# ---------------------------------------------------------------------------
|
|
284
|
-
# 5b. Every completed managed-skills install writes its own version marker.
|
|
285
|
-
# ---------------------------------------------------------------------------
|
|
286
|
-
section "Check 5b: Installed skill roots carry exact version markers"
|
|
287
|
-
if make_temp_dir tmp_install_marker /tmp/devlyn-install-marker.XXXXXX; then
|
|
288
|
-
marker_home="$tmp_install_marker/home"
|
|
289
|
-
marker_project="$tmp_install_marker/project"
|
|
290
|
-
timeout_low="$tmp_install_marker/timeout-low"
|
|
291
|
-
timeout_high="$tmp_install_marker/timeout-high"
|
|
292
|
-
installer="$PWD/bin/devlyn.js"
|
|
293
|
-
mkdir -p "$marker_home" "$marker_project" "$timeout_low/.claude" "$timeout_high/.claude"
|
|
294
|
-
printf '{"env":{"BASH_MAX_TIMEOUT_MS":"600000"}}\n' > "$timeout_low/.claude/settings.json"
|
|
295
|
-
printf '{"env":{"BASH_MAX_TIMEOUT_MS":"7200000"}}\n' > "$timeout_high/.claude/settings.json"
|
|
296
|
-
|
|
297
|
-
if (cd "$marker_project" \
|
|
298
|
-
&& HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/claude.log" 2>&1 \
|
|
299
|
-
&& HOME="$marker_home" node "$installer" agents all >"$tmp_install_marker/agents.log" 2>&1) \
|
|
300
|
-
&& (cd "$timeout_low" && HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/timeout-low.log" 2>&1) \
|
|
301
|
-
&& cp "$timeout_low/.claude/settings.json" "$tmp_install_marker/timeout-low-first.json" \
|
|
302
|
-
&& (cd "$timeout_low" && HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/timeout-low-second.log" 2>&1) \
|
|
303
|
-
&& cmp -s "$timeout_low/.claude/settings.json" "$tmp_install_marker/timeout-low-first.json" \
|
|
304
|
-
&& (cd "$timeout_high" && HOME="$marker_home" node "$installer" -y >"$tmp_install_marker/timeout-high.log" 2>&1); then
|
|
305
|
-
if python3 - "$installer" "$marker_home" "$marker_project" "$timeout_low" "$timeout_high" <<'PY'
|
|
306
|
-
import json
|
|
307
|
-
import pathlib
|
|
308
|
-
import stat
|
|
309
|
-
import sys
|
|
310
|
-
|
|
311
|
-
installer = pathlib.Path(sys.argv[1])
|
|
312
|
-
home = pathlib.Path(sys.argv[2])
|
|
313
|
-
project = pathlib.Path(sys.argv[3])
|
|
314
|
-
timeout_low = pathlib.Path(sys.argv[4])
|
|
315
|
-
timeout_high = pathlib.Path(sys.argv[5])
|
|
316
|
-
package = json.loads((installer.parent.parent / "package.json").read_text())
|
|
317
|
-
expected = {"schemaVersion": 1, "package": package["name"], "version": package["version"]}
|
|
318
|
-
roots = [
|
|
319
|
-
project / ".claude" / "skills",
|
|
320
|
-
home / ".codex" / "skills",
|
|
321
|
-
home / ".agents" / "skills",
|
|
322
|
-
home / ".grok" / "skills",
|
|
323
|
-
]
|
|
324
|
-
for root in roots:
|
|
325
|
-
marker = root / ".devlyn-install.json"
|
|
326
|
-
actual = json.loads(marker.read_text())
|
|
327
|
-
if actual != expected:
|
|
328
|
-
raise SystemExit(f"{marker}: expected {expected}, got {actual}")
|
|
329
|
-
if stat.S_IMODE(marker.stat().st_mode) != 0o600:
|
|
330
|
-
raise SystemExit(f"{marker}: expected mode 0600")
|
|
331
|
-
if list(root.glob(".devlyn-install.json.*.tmp")):
|
|
332
|
-
raise SystemExit(f"{root}: stale marker temp file")
|
|
333
|
-
for target, expected_timeout in (
|
|
334
|
-
(project, "3600000"),
|
|
335
|
-
(timeout_low, "3600000"),
|
|
336
|
-
(timeout_high, "7200000"),
|
|
337
|
-
):
|
|
338
|
-
env = json.loads((target / ".claude" / "settings.json").read_text())["env"]
|
|
339
|
-
if env.get("BASH_MAX_TIMEOUT_MS") != expected_timeout:
|
|
340
|
-
raise SystemExit(f"{target}: expected BASH_MAX_TIMEOUT_MS={expected_timeout}")
|
|
341
|
-
if "BASH_DEFAULT_TIMEOUT_MS" in env:
|
|
342
|
-
raise SystemExit(f"{target}: BASH_DEFAULT_TIMEOUT_MS must not be installed")
|
|
343
|
-
PY
|
|
344
|
-
then
|
|
345
|
-
if grep -Fxq '.claude/skills/.devlyn-install.json' "$marker_project/.gitignore"; then
|
|
346
|
-
ok "managed roots have exact 0600 markers; Bash max is installed, raised, preserved, and idempotent"
|
|
347
|
-
else
|
|
348
|
-
bad "Claude project install must ignore its local marker"
|
|
349
|
-
fi
|
|
350
|
-
else
|
|
351
|
-
bad "completed installs must write the exact per-root marker"
|
|
352
|
-
fi
|
|
353
|
-
else
|
|
354
|
-
bad "isolated managed-skills install failed"
|
|
355
|
-
fi
|
|
356
|
-
|
|
357
|
-
incomplete="$tmp_install_marker/incomplete"
|
|
358
|
-
mkdir -p "$incomplete/package" "$incomplete/home/.codex/skills" "$incomplete/claude/.claude/skills" "$incomplete/codex"
|
|
359
|
-
cp -R bin config package.json AGENTS.md CLAUDE.md "$incomplete/package/"
|
|
360
|
-
rm -rf "$incomplete/package/config/skills/devlyn:queue"
|
|
361
|
-
printf '{"version":"stale"}\n' > "$incomplete/home/.codex/skills/.devlyn-install.json"
|
|
362
|
-
printf '{"version":"stale"}\n' > "$incomplete/claude/.claude/skills/.devlyn-install.json"
|
|
363
|
-
if ! (cd "$incomplete/claude" \
|
|
364
|
-
&& HOME="$incomplete/home" node "$incomplete/package/bin/devlyn.js" -y >"$incomplete/claude.log" 2>&1) \
|
|
365
|
-
&& ! (cd "$incomplete/codex" \
|
|
366
|
-
&& HOME="$incomplete/home" node "$incomplete/package/bin/devlyn.js" agents codex >"$incomplete/codex.log" 2>&1) \
|
|
367
|
-
&& grep -Fq 'Incomplete devlyn skill install; missing: devlyn:queue' "$incomplete/claude.log" \
|
|
368
|
-
&& grep -Fq 'Incomplete devlyn skill install; missing: devlyn:queue' "$incomplete/codex.log" \
|
|
369
|
-
&& [ -d "$incomplete/claude/.claude/skills/devlyn:resolve" ] \
|
|
370
|
-
&& [ -d "$incomplete/home/.codex/skills/devlyn:resolve" ] \
|
|
371
|
-
&& [ ! -e "$incomplete/claude/.claude/skills/.devlyn-install.json" ] \
|
|
372
|
-
&& [ ! -e "$incomplete/home/.codex/skills/.devlyn-install.json" ]; then
|
|
373
|
-
ok "incomplete Claude and Codex copies fail visibly without stale or replacement markers"
|
|
374
|
-
else
|
|
375
|
-
bad "incomplete Claude and Codex copies must fail visibly without an install marker"
|
|
376
|
-
fi
|
|
377
|
-
rm -rf "$tmp_install_marker"
|
|
378
|
-
else
|
|
379
|
-
bad "could not allocate install-marker smoke directory"
|
|
380
|
-
fi
|
|
381
|
-
|
|
382
|
-
# ---------------------------------------------------------------------------
|
|
383
|
-
# 6. Source ↔ installed mirror parity on critical path.
|
|
384
|
-
# Only runs if .claude/skills exists (i.e. installer has been run).
|
|
385
|
-
# ---------------------------------------------------------------------------
|
|
386
|
-
section "Check 6: Source ↔ installed mirror parity (critical path)"
|
|
387
|
-
check_skill_mirror_parity \
|
|
388
|
-
".claude/skills" \
|
|
389
|
-
"no .claude/skills (fresh checkout) — skipping parity check" \
|
|
390
|
-
"missing file on critical path" \
|
|
391
|
-
"source and installed differ" \
|
|
392
|
-
"critical path parity clean"
|
|
393
|
-
|
|
394
|
-
# Codex / agent runtimes in this repo can also expose a project-local
|
|
395
|
-
# `.agents/skills` mirror. If it exists, keep the same critical path in parity;
|
|
396
|
-
# otherwise a session can read stale pair/risk-probe contracts even while the
|
|
397
|
-
# source and `.claude/skills` mirrors are clean.
|
|
398
|
-
section "Check 6a: Source ↔ .agents mirror parity (critical path)"
|
|
399
|
-
check_skill_mirror_parity \
|
|
400
|
-
".agents/skills" \
|
|
401
|
-
"no .agents/skills — skipping parity check" \
|
|
402
|
-
"missing .agents critical-path file" \
|
|
403
|
-
"source and .agents mirror differ" \
|
|
404
|
-
".agents critical path parity clean"
|
|
405
|
-
|
|
406
|
-
# ---------------------------------------------------------------------------
|
|
407
|
-
# 6b. VERIFY merge verdict binding self-test.
|
|
408
|
-
for helper in role-config judge-role-evidence task-complete; do
|
|
409
|
-
if python3 "config/skills/_shared/$helper.py" --self-test; then
|
|
410
|
-
ok "$helper.py self-test passed"
|
|
411
|
-
else
|
|
412
|
-
bad "$helper.py self-test failed"
|
|
413
|
-
fi
|
|
414
|
-
done
|
|
415
|
-
|
|
416
|
-
# F23 full-pipeline prompt-fix rerun exposed a real failure where Codex
|
|
417
|
-
# pair-JUDGE emitted HIGH findings but state kept pair_judge as
|
|
418
|
-
# PASS_WITH_ISSUES. Routing severity must be deterministic, not prose.
|
|
419
|
-
# ---------------------------------------------------------------------------
|
|
420
|
-
section "Check 6b: VERIFY merge makes pair HIGH verdict-binding"
|
|
421
|
-
if python3 config/skills/_shared/verify-merge-findings.py --self-test >/dev/null 2>&1; then
|
|
422
|
-
ok "verify-merge-findings.py self-test passed"
|
|
423
|
-
else
|
|
424
|
-
bad "verify-merge-findings.py self-test failed"
|
|
425
|
-
fi
|
|
426
|
-
|
|
427
|
-
if ! grep -Fq 'def pair_trigger_skip_contract_violation' config/skills/_shared/verify-merge-findings.py \
|
|
428
|
-
|| ! grep -Fq 'def pair_trigger_missing_contract_violation' config/skills/_shared/verify-merge-findings.py \
|
|
429
|
-
|| ! grep -Fq 'def pair_trigger_reason_completeness_violation' config/skills/_shared/verify-merge-findings.py \
|
|
430
|
-
|| ! grep -Fq 'def pair_trigger_present' config/skills/_shared/verify-merge-findings.py \
|
|
431
|
-
|| ! grep -Fq 'KNOWN_PAIR_TRIGGER_REASONS = {' config/skills/_shared/verify-merge-findings.py \
|
|
432
|
-
|| ! grep -Fq 'mode.pair-verify' config/skills/_shared/verify-merge-findings.py \
|
|
433
|
-
|| ! grep -Fq 'state.get("pair_verify") is True' config/skills/_shared/verify-merge-findings.py \
|
|
434
|
-
|| ! grep -Fq 'def pair_flag_contract_violation' config/skills/_shared/verify-merge-findings.py \
|
|
435
|
-
|| ! grep -Fq 'verify-pair-trigger-conflicting-pair-flags' config/skills/_shared/verify-merge-findings.py \
|
|
436
|
-
|| ! grep -Fq -- '--pair-verify and --no-pair are mutually exclusive' config/skills/_shared/verify-merge-findings.py \
|
|
437
|
-
|| ! grep -Fq 'def has_known_pair_trigger_reason' config/skills/_shared/verify-merge-findings.py \
|
|
438
|
-
|| ! grep -Fq 'def all_known_pair_trigger_reasons' config/skills/_shared/verify-merge-findings.py \
|
|
439
|
-
|| ! grep -Fq 'return reason in KNOWN_PAIR_TRIGGER_REASONS' config/skills/_shared/verify-merge-findings.py \
|
|
440
|
-
|| ! grep -Fq '"reasons": ["risk.high", 3]' config/skills/_shared/verify-merge-findings.py \
|
|
441
|
-
|| ! grep -Fq '"reasons": ["risk.high", "looks-hard"]' config/skills/_shared/verify-merge-findings.py \
|
|
442
|
-
|| ! grep -Fq '"reasons": ["risk high"]' config/skills/_shared/verify-merge-findings.py \
|
|
443
|
-
|| ! grep -Fq '"reasons": ["risk_profile.high_risk", "risk_probes_enabled"]' config/skills/_shared/verify-merge-findings.py \
|
|
444
|
-
|| ! grep -Fq 'verify-pair-trigger-reasons-unknown' config/skills/_shared/verify-merge-findings.py \
|
|
445
|
-
|| ! grep -Fq 'verify-pair-trigger-reasons-incomplete' config/skills/_shared/verify-merge-findings.py \
|
|
446
|
-
|| ! grep -Fq 'pair_trigger.reasons is missing applicable canonical reason(s)' config/skills/_shared/verify-merge-findings.py \
|
|
447
|
-
|| ! grep -Fq 'pair_trigger.reasons must include a known pair-trigger reason' config/skills/_shared/verify-merge-findings.py \
|
|
448
|
-
|| ! grep -Fq 'pair_trigger.reasons must only include known pair-trigger reasons' config/skills/_shared/verify-merge-findings.py \
|
|
449
|
-
|| ! grep -Fq 'verify-pair-trigger-ineligible-unjustified' config/skills/_shared/verify-merge-findings.py \
|
|
450
|
-
|| ! grep -Fq 'VERIFY state requires a pair decision' config/skills/_shared/verify-merge-findings.py \
|
|
451
|
-
|| ! grep -Fq 'verify-pair-trigger-user-no-pair-unsupported' config/skills/_shared/verify-merge-findings.py \
|
|
452
|
-
|| ! grep -Fq 'risk_profile.pair_default_enabled false from an explicit --no-pair opt-out' config/skills/_shared/verify-merge-findings.py \
|
|
453
|
-
|| ! grep -Fq 'def risk_profile_contract_violation' config/skills/_shared/verify-merge-findings.py \
|
|
454
|
-
|| ! grep -Fq 'verify-risk-profile-malformed' config/skills/_shared/verify-merge-findings.py \
|
|
455
|
-
|| ! grep -Fq 'risk_profile.risk_probes_enabled must be a boolean' config/skills/_shared/verify-merge-findings.py \
|
|
456
|
-
|| ! grep -Fq 'risk_profile.reasons must be a list of strings' config/skills/_shared/verify-merge-findings.py \
|
|
457
|
-
|| ! grep -Fq 'def spec_frontmatter_complexity' config/skills/_shared/verify-merge-findings.py \
|
|
458
|
-
|| ! grep -Fq '"complexity": "large"' config/skills/_shared/verify-merge-findings.py \
|
|
459
|
-
|| ! grep -Fq 'complexity.large' config/skills/_shared/verify-merge-findings.py \
|
|
460
|
-
|| ! grep -Fq 'spec.complexity.high' config/skills/_shared/verify-merge-findings.py \
|
|
461
|
-
|| ! grep -Fq 'spec.complexity.large' config/skills/_shared/verify-merge-findings.py \
|
|
462
|
-
|| ! grep -Fq 'verify-pair-trigger-required-missing' config/skills/_shared/verify-merge-findings.py \
|
|
463
|
-
|| ! grep -Fq 'verify-pair-trigger-skipped-reason-unsupported' config/skills/_shared/verify-merge-findings.py \
|
|
464
|
-
|| ! grep -Fq 'verify-pair-trigger-mechanical-blocker-unsupported' config/skills/_shared/verify-merge-findings.py \
|
|
465
|
-
|| ! grep -Fq 'verify-pair-trigger-primary-judge-blocker-unsupported' config/skills/_shared/verify-merge-findings.py \
|
|
466
|
-
|| ! grep -Fq 'Missing, contradictory, incomplete, or unknown trigger state BLOCKs VERIFY' config/skills/devlyn:resolve/references/state-schema.md \
|
|
467
|
-
|| ! grep -Fq 'Canonical reasons are `pair.default`, `mode.verify-only`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
468
|
-
|| ! grep -Fq '`mode.pair-verify`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
469
|
-
|| ! grep -Fq 'Eligible schema-v3 state must contain `pair.default` and every other applicable canonical' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
470
|
-
|| ! grep -Fq 'with `pair.default` plus every applicable outcome-independent canonical reason' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
471
|
-
|| ! grep -Fq '`mode.pair-verify`' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
472
|
-
|| ! grep -Fq '"pair_verify": false' config/skills/devlyn:resolve/references/state-schema.md \
|
|
473
|
-
|| ! grep -Fq 'Schema v3.0 eligible state requires `pair.default` plus every applicable telemetry reason' config/skills/devlyn:resolve/references/state-schema.md \
|
|
474
|
-
|| ! grep -Fq 'Ineligible new-run state has empty reasons and only' config/skills/devlyn:resolve/references/state-schema.md \
|
|
475
|
-
|| ! grep -Fq 'True only for `--pair-verify`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
476
|
-
|| ! grep -Fq 'pair_verify: true` only when `--pair-verify` was passed' config/skills/devlyn:resolve/SKILL.md \
|
|
477
|
-
|| ! grep -Fq 'persist `pair_trigger` before spawn with `pair.default` plus every applicable outcome-independent reason' config/skills/devlyn:resolve/SKILL.md \
|
|
478
|
-
|| ! grep -Fq '`--pair-verify` and `--no-pair` are mutually exclusive' config/skills/devlyn:resolve/SKILL.md \
|
|
479
|
-
|| ! grep -Fq 'mutually exclusive with `risk_profile.pair_default_enabled == false`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
480
|
-
|| ! grep -Fq 'if both are present, stop with `BLOCKED:invalid-flags`' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
481
|
-
|| ! grep -Fq 'Malformed risk or trigger state BLOCKs VERIFY' config/skills/devlyn:resolve/SKILL.md \
|
|
482
|
-
|| ! grep -Fq 'pair_default_enabled` is false only for explicit `--no-pair`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
483
|
-
|| ! grep -Fq 'def reject_json_constant' config/skills/_shared/verify-merge-findings.py \
|
|
484
|
-
|| ! grep -Fq 'loads_strict_json(raw)' config/skills/_shared/verify-merge-findings.py \
|
|
485
|
-
|| ! grep -Fq 'invalid JSON numeric constant: NaN' config/skills/_shared/verify-merge-findings.py; then
|
|
486
|
-
bad "verify-merge-findings.py must block missing pair_trigger and unsupported skip reasons"
|
|
487
|
-
fi
|
|
488
|
-
verify_merge_risk_profile_guard_missing=0
|
|
489
|
-
for file in \
|
|
490
|
-
config/skills/_shared/verify-merge-findings.py \
|
|
491
|
-
.agents/skills/_shared/verify-merge-findings.py; do
|
|
492
|
-
if ! grep -Fq 'def risk_profile_contract_violation' "$file" \
|
|
493
|
-
|| ! grep -Fq 'verify-risk-profile-malformed' "$file" \
|
|
494
|
-
|| ! grep -Fq 'risk_profile.risk_probes_enabled must be a boolean' "$file" \
|
|
495
|
-
|| ! grep -Fq 'risk_profile.reasons must be a list of strings' "$file"; then
|
|
496
|
-
bad "$file — verify-merge-findings.py must fail closed on malformed risk_profile"
|
|
497
|
-
verify_merge_risk_profile_guard_missing=1
|
|
498
|
-
fi
|
|
499
|
-
done
|
|
500
|
-
if [ $verify_merge_risk_profile_guard_missing -eq 0 ]; then
|
|
501
|
-
ok "verify-merge-findings.py risk_profile shape guard is mirrored"
|
|
502
|
-
fi
|
|
503
|
-
|
|
504
|
-
section "Check 6b1: Process evidence runner is sealed"
|
|
505
|
-
if python3 config/skills/_shared/process-evidence.py --self-test >/dev/null 2>&1; then
|
|
506
|
-
ok "process-evidence.py self-test passed"
|
|
507
|
-
else
|
|
508
|
-
bad "process-evidence.py self-test failed"
|
|
509
|
-
fi
|
|
510
|
-
if ! grep -Fq 'PHASES = {"implement", "build_gate", "verify"}' config/skills/_shared/process-evidence.py \
|
|
511
|
-
|| ! grep -Fq 'classification' config/skills/_shared/process-evidence.py \
|
|
512
|
-
|| ! grep -Fq 'validate_bound_carrier' config/skills/_shared/process-evidence.py \
|
|
513
|
-
|| ! grep -Fq 'validate_summary_commands' config/skills/_shared/process-evidence.py; then
|
|
514
|
-
bad "process-evidence.py must preserve phase identity, capability classification, and bound-byte validation"
|
|
515
|
-
fi
|
|
516
|
-
|
|
517
|
-
section "Check 6b2: Codex invocation receipts are phase-owned"
|
|
518
|
-
if python3 config/skills/_shared/invocation-receipt.py --self-test; then
|
|
519
|
-
ok "invocation-receipt.py self-test passed"
|
|
520
|
-
else
|
|
521
|
-
bad "invocation-receipt.py self-test failed"
|
|
522
|
-
fi
|
|
523
|
-
if ! grep -Fq 'sandbox must remain workspace-write' config/skills/_shared/invocation-receipt.py \
|
|
524
|
-
|| ! grep -Fq 'SCHEMA_VERSION = "2.0"' config/skills/_shared/invocation-receipt.py \
|
|
525
|
-
|| ! grep -Fq 'sandbox_workspace_write.network_access' config/skills/_shared/invocation-receipt.py \
|
|
526
|
-
|| ! grep -Fq 'forbidden Codex bypass flag' config/skills/_shared/invocation-receipt.py \
|
|
527
|
-
|| ! grep -Fq 'forbidden Codex sandbox: danger-full-access' config/skills/_shared/codex-monitored.sh \
|
|
528
|
-
|| ! grep -Fq 'DEVLYN_INVOCATION_RECEIPT' config/skills/_shared/codex-monitored.sh; then
|
|
529
|
-
bad "Codex invocation receipts must bind phase identity and reject authority widening"
|
|
530
|
-
fi
|
|
531
|
-
|
|
532
|
-
section "Check 6c: Codex stdout collection writes canonical pair findings"
|
|
533
|
-
if python3 config/skills/_shared/collect-codex-findings.py --self-test >/dev/null 2>&1; then
|
|
534
|
-
ok "collect-codex-findings.py self-test passed"
|
|
535
|
-
else
|
|
536
|
-
bad "collect-codex-findings.py self-test failed"
|
|
537
|
-
fi
|
|
538
|
-
# iter-0082: the frozen collector contract — every negative through every ingress
|
|
539
|
-
# path, plus real judge captures replayed against hand-verified expectations.
|
|
540
|
-
# The self-test above is unit-shaped; this is the integration bar against real bytes.
|
|
541
|
-
if python3 benchmark/ceiling/probes/r-weld-0082/test-collector-contract.py >/dev/null 2>&1; then
|
|
542
|
-
ok "collector contract (iter-0082 negatives + real-capture corpus) passed"
|
|
543
|
-
else
|
|
544
|
-
bad "collector contract failed — run benchmark/ceiling/probes/r-weld-0082/test-collector-contract.py"
|
|
545
|
-
fi
|
|
546
|
-
if ! grep -Fq 'def reject_json_constant' config/skills/_shared/collect-codex-findings.py \
|
|
547
|
-
|| ! grep -Fq 'PARSER["collect_stdout"]' config/skills/_shared/collect-codex-findings.py \
|
|
548
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' config/skills/_shared/judge-output-parser.py \
|
|
549
|
-
|| ! grep -Fq 'NaN pair-JUDGE stdout finding' config/skills/_shared/collect-codex-findings.py; then
|
|
550
|
-
bad "collect-codex-findings.py must reject non-standard JSON constants in pair-JUDGE stdout"
|
|
551
|
-
fi
|
|
552
|
-
|
|
553
|
-
section "Check 6c1: Archive preserves pair/risk-probe artifacts safely"
|
|
554
|
-
if python3 config/skills/_shared/archive_run.py --self-test >/dev/null 2>&1; then
|
|
555
|
-
ok "archive_run.py self-test passed"
|
|
556
|
-
else
|
|
557
|
-
bad "archive_run.py self-test failed"
|
|
558
|
-
fi
|
|
559
|
-
if ! grep -Fq 'SAFE_RUN_ID_RE' config/skills/_shared/archive_run.py \
|
|
560
|
-
|| ! grep -Fq 'run_id must match [A-Za-z0-9_.-]+' config/skills/_shared/archive_run.py \
|
|
561
|
-
|| ! grep -Fq 'Archive devlyn:resolve run artifacts' config/skills/_shared/archive_run.py \
|
|
562
|
-
|| grep -Fq 'Archive auto-resolve run artifacts' config/skills/_shared/archive_run.py \
|
|
563
|
-
|| ! grep -Fq 'invalid JSON numeric constant: NaN' config/skills/_shared/archive_run.py \
|
|
564
|
-
|| ! grep -Fq '"verify.pair.findings.jsonl"' config/skills/_shared/archive_run.py \
|
|
565
|
-
|| ! grep -Fq '"verify-merge.summary.json"' config/skills/_shared/archive_run.py \
|
|
566
|
-
|| ! grep -Fq '"verify.primary.timeout.json"' config/skills/_shared/archive_run.py \
|
|
567
|
-
|| ! grep -Fq '"surface-close.output.json"' config/skills/_shared/archive_run.py \
|
|
568
|
-
|| ! grep -Fq '"*-judge.*"' config/skills/_shared/archive_run.py; then
|
|
569
|
-
bad "archive_run.py must safely archive pair/risk-probe evidence and reject unsafe run ids"
|
|
570
|
-
fi
|
|
571
|
-
|
|
572
|
-
section "Check 6c1: Owner phase lifecycle and scope boundaries"
|
|
573
|
-
if python3 scripts/test-owner-phases.py; then
|
|
574
|
-
ok "owner phase CLI and Git regression tests passed"
|
|
575
|
-
else
|
|
576
|
-
bad "owner phase CLI and Git regression tests failed"
|
|
577
|
-
fi
|
|
578
|
-
|
|
579
|
-
section "Check 6c2: SURFACE_CLOSE v6 dispatch is complete and mirrored"
|
|
580
|
-
surface_close_missing=0
|
|
581
|
-
if python3 config/skills/_shared/state-phase-write.py --self-test >/dev/null 2>&1; then
|
|
582
|
-
ok "SURFACE_CLOSE state/envelope/adjudication/audit/rollback self-test passed"
|
|
583
|
-
else
|
|
584
|
-
bad "state-phase-write.py SURFACE_CLOSE self-test failed"
|
|
585
|
-
fi
|
|
586
|
-
if python3 - <<'PY'
|
|
587
|
-
import ast
|
|
588
|
-
import runpy
|
|
589
|
-
|
|
590
|
-
entries = ["src/{a,b}/**"]
|
|
591
|
-
paths = ("src/a/example.py", "src/b/example.py", "src/c/example.py")
|
|
592
|
-
expected = (True, True, False)
|
|
593
|
-
sources = {"path_matches_surface": [], "validate_surface_brace_glob": []}
|
|
594
|
-
for name in ("state-phase-write.py", "spec-verify-check.py"):
|
|
595
|
-
path = f"config/skills/_shared/{name}"
|
|
596
|
-
module = runpy.run_path(path)
|
|
597
|
-
actual = tuple(module["path_matches_surface"](path, entries) for path in paths)
|
|
598
|
-
assert actual == expected, (name, actual)
|
|
599
|
-
tree = ast.parse(open(path, encoding="utf-8").read())
|
|
600
|
-
text = open(path, encoding="utf-8").read()
|
|
601
|
-
for function_name in sources:
|
|
602
|
-
function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == function_name)
|
|
603
|
-
sources[function_name].append(ast.get_source_segment(text, function))
|
|
604
|
-
for invalid in ("src/{a,b", "src/{a,}/**", "src/{a,{b,c}}", "src/{a,**}"):
|
|
605
|
-
try:
|
|
606
|
-
module["path_matches_surface"]("src/a/example.py", [invalid])
|
|
607
|
-
except ValueError as exc:
|
|
608
|
-
assert invalid in str(exc) and "supported form" in str(exc)
|
|
609
|
-
else:
|
|
610
|
-
raise AssertionError((name, invalid))
|
|
611
|
-
for function_name, bodies in sources.items():
|
|
612
|
-
assert bodies[0] == bodies[1], f"{function_name} bodies differ"
|
|
613
|
-
PY
|
|
614
|
-
then
|
|
615
|
-
ok "brace-glob surface matching and errors are byte-parity checked in both consumers"
|
|
616
|
-
else
|
|
617
|
-
bad "brace-glob surface matching or errors drifted between state and spec checks"
|
|
618
|
-
fi
|
|
619
|
-
if ! grep -Fq 'rollback_surface_delta' config/skills/_shared/state-phase-write.py \
|
|
620
|
-
|| ! grep -Fq 'validate_surface_adjudication' config/skills/_shared/state-phase-write.py \
|
|
621
|
-
|| ! grep -Fq 'validate_surface_execution' config/skills/_shared/state-phase-write.py \
|
|
622
|
-
|| ! grep -Fq 'phases.surface_close spawn requires --engine claude' config/skills/_shared/state-phase-write.py \
|
|
623
|
-
|| ! grep -Fq 'SURFACE_CLOSE requires canonical native JSON' config/skills/_shared/state-phase-write.py \
|
|
624
|
-
|| ! grep -Fq 'BLOCKED:surface-close-input-mismatch' config/skills/_shared/state-phase-write.py \
|
|
625
|
-
|| ! grep -Fq 'surface-check' config/skills/_shared/state-phase-write.py \
|
|
626
|
-
|| ! grep -Fq 'surface-rollback' config/skills/_shared/state-phase-write.py; then
|
|
627
|
-
bad "state-phase-write.py must mechanically adjudicate, audit, guard, and roll back SURFACE_CLOSE"
|
|
628
|
-
fi
|
|
629
|
-
for tree in config/skills .agents/skills; do
|
|
630
|
-
skill="$tree/devlyn:resolve/SKILL.md"
|
|
631
|
-
phase="$tree/devlyn:resolve/references/phases/surface-close.md"
|
|
632
|
-
if ! grep -Fq '## PHASE 2.5: SURFACE_CLOSE' "$skill" \
|
|
633
|
-
|| ! grep -Fq '`state.source.type == "generated"` and complexity is trivial/medium' "$skill" \
|
|
634
|
-
|| ! grep -Fq 'Engine is Claude always' "$skill" \
|
|
635
|
-
|| ! grep -Fq 'executor flag/pin' "$skill" \
|
|
636
|
-
|| ! grep -Fq 'auto_surface_close_claude_unavailable' "$skill" \
|
|
637
|
-
|| ! grep -Fq 'canonical body VERBATIM' "$skill" \
|
|
638
|
-
|| ! grep -Fq 'run-bounded.py 600 --stdin-file .devlyn/surface-close.prompt.<round> --record-transport -- claude -p' "$skill" \
|
|
639
|
-
|| ! grep -Fq -- '--tools "Read,Grep,Glob,Edit,Write" --dangerously-skip-permissions --output-format json --strict-mcp-config --mcp-config '\''{"mcpServers":{}}'\''' "$skill" \
|
|
640
|
-
|| ! grep -Fq '.devlyn/surface-close.output.json' "$skill" \
|
|
641
|
-
|| ! grep -Fq '**Common post-fix checkpoint (BUILD_GATE and VERIFY):**' "$skill" \
|
|
642
|
-
|| ! grep -Fq 'durability-enforce --round <n> --origin-phase <build_gate|cleanup|verify>' "$skill" \
|
|
643
|
-
|| ! grep -Fq '`surface-rollback`' "$skill" \
|
|
644
|
-
|| grep -Fq 'Supplied digests' "$phase" \
|
|
645
|
-
|| grep -Fq 'Hash both artifacts first' "$phase" \
|
|
646
|
-
|| ! grep -Fq 'Never modify inputs or read state, PLAN, or IMPLEMENT transcript/reasoning.' "$phase" \
|
|
647
|
-
|| ! grep -Fq 'optionally followed by ` — <one-line evidence>`' "$phase" \
|
|
648
|
-
|| ! grep -Fq 'N/A <authorized-file>[:<line>] — <one-line evidence-based relationship judgment>' "$phase"; then
|
|
649
|
-
bad "$tree — SURFACE_CLOSE v6 dispatch/timeout/adjudication/rollback contract missing"
|
|
650
|
-
surface_close_missing=1
|
|
651
|
-
fi
|
|
652
|
-
done
|
|
653
|
-
if [ $surface_close_missing -eq 0 ]; then
|
|
654
|
-
ok "SURFACE_CLOSE v6 dispatch/timeout/adjudication/rollback contract is mirrored"
|
|
655
|
-
fi
|
|
656
|
-
|
|
657
|
-
section "Check 6d: Spec verification executes hidden-blind risk probes"
|
|
658
|
-
if python3 config/skills/_shared/spec-verify-check.py --self-test >/dev/null 2>&1; then
|
|
659
|
-
ok "spec-verify-check.py risk-probe and expected-contract self-test passed"
|
|
660
|
-
else
|
|
661
|
-
bad "spec-verify-check.py risk-probe / expected-contract self-test failed"
|
|
662
|
-
fi
|
|
663
|
-
if ! grep -Fq 'def validate_present_spec_complexity' config/skills/_shared/spec-verify-check.py \
|
|
664
|
-
|| ! grep -Fq 'SPEC_COMPLEXITY_VALUES = {"trivial", "medium", "high", "large"}' config/skills/_shared/spec-verify-check.py \
|
|
665
|
-
|| ! grep -Fq 'frontmatter complexity must be one of' config/skills/_shared/spec-verify-check.py \
|
|
666
|
-
|| ! grep -Fq 'def validate_sibling_spec_complexity' config/skills/_shared/spec-verify-check.py \
|
|
667
|
-
|| ! grep -Fq 'unsupported spec complexity was accepted' config/skills/_shared/spec-verify-check.py \
|
|
668
|
-
|| ! grep -Fq 'unsupported sibling spec complexity was accepted by --check-expected' config/skills/_shared/spec-verify-check.py; then
|
|
669
|
-
bad "spec-verify-check.py checks must reject unsupported spec complexity values"
|
|
670
|
-
else
|
|
671
|
-
ok "spec-verify-check.py checks reject unsupported spec complexity values"
|
|
672
|
-
fi
|
|
673
|
-
if ! grep -Fq 'generated criteria carrier was not staged into .devlyn/spec-verify.json' config/skills/_shared/spec-verify-check.py \
|
|
674
|
-
|| ! grep -Fq 'spec source with mismatched source.spec_sha256 was accepted' config/skills/_shared/spec-verify-check.py \
|
|
675
|
-
|| ! grep -Fq 'spec source with matching source.spec_sha256 was not staged' config/skills/_shared/spec-verify-check.py \
|
|
676
|
-
|| ! grep -Fq 'source.spec_sha256 mismatch' config/skills/_shared/spec-verify-check.py \
|
|
677
|
-
|| ! grep -Fq 'generated criteria without a JSON carrier was accepted' config/skills/_shared/spec-verify-check.py \
|
|
678
|
-
|| ! grep -Fq 'generated criteria without source.criteria_sha256 was accepted' config/skills/_shared/spec-verify-check.py \
|
|
679
|
-
|| ! grep -Fq 'generated criteria with mismatched source.criteria_sha256 was accepted' config/skills/_shared/spec-verify-check.py \
|
|
680
|
-
|| ! grep -Fq 'def source_integrity_error' config/skills/_shared/spec-verify-check.py \
|
|
681
|
-
|| ! grep -Fq 'source.criteria_sha256 mismatch' config/skills/_shared/spec-verify-check.py \
|
|
682
|
-
|| ! grep -Fq 'Generated criteria were written without one' config/skills/_shared/spec-verify-check.py \
|
|
683
|
-
|| ! grep -Fq 'generated criteria carrier was not staged into .devlyn/spec-verify.json' .agents/skills/_shared/spec-verify-check.py \
|
|
684
|
-
|| ! grep -Fq 'spec source with mismatched source.spec_sha256 was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
685
|
-
|| ! grep -Fq 'spec source with matching source.spec_sha256 was not staged' .agents/skills/_shared/spec-verify-check.py \
|
|
686
|
-
|| ! grep -Fq 'source.spec_sha256 mismatch' .agents/skills/_shared/spec-verify-check.py \
|
|
687
|
-
|| ! grep -Fq 'generated criteria without a JSON carrier was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
688
|
-
|| ! grep -Fq 'generated criteria without source.criteria_sha256 was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
689
|
-
|| ! grep -Fq 'generated criteria with mismatched source.criteria_sha256 was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
690
|
-
|| ! grep -Fq 'def source_integrity_error' .agents/skills/_shared/spec-verify-check.py \
|
|
691
|
-
|| ! grep -Fq 'source.criteria_sha256 mismatch' .agents/skills/_shared/spec-verify-check.py \
|
|
692
|
-
|| ! grep -Fq 'Generated criteria were written without one' .agents/skills/_shared/spec-verify-check.py \
|
|
693
|
-
|| ! grep -Fq '"criteria_sha256": generated_hash' .agents/skills/_shared/spec-verify-check.py; then
|
|
694
|
-
bad "spec-verify-check.py self-test must cover generated criteria source extraction"
|
|
695
|
-
else
|
|
696
|
-
ok "spec-verify-check.py covers generated criteria source extraction"
|
|
697
|
-
fi
|
|
698
|
-
if ! grep -Fq 'def validate_present_solo_headroom_hypothesis' config/skills/_shared/spec-verify-check.py \
|
|
699
|
-
|| ! grep -Fq 'def state_requires_risk_probes' config/skills/_shared/spec-verify-check.py \
|
|
700
|
-
|| ! grep -Fq 'def risk_probes_state_error' config/skills/_shared/spec-verify-check.py \
|
|
701
|
-
|| ! grep -Fq -- '--include-risk-probes accepted missing required risk-probes.jsonl' config/skills/_shared/spec-verify-check.py \
|
|
702
|
-
|| ! grep -Fq -- '--include-risk-probes accepted non-boolean risk_probes_enabled' config/skills/_shared/spec-verify-check.py \
|
|
703
|
-
|| ! grep -Fq -- '--include-risk-probes accepted non-object risk_profile' config/skills/_shared/spec-verify-check.py \
|
|
704
|
-
|| ! grep -Fq 'def validate_risk_probes_cover_solo_headroom_hypothesis' config/skills/_shared/spec-verify-check.py \
|
|
705
|
-
|| ! grep -Fq 'def has_backticked_observable_miss_command' config/skills/_shared/spec-verify-check.py \
|
|
706
|
-
|| ! grep -Fq 'backticked command/observable line that exposes the miss' config/skills/_shared/spec-verify-check.py \
|
|
707
|
-
|| ! grep -Fq 'weak solo-headroom hypothesis was accepted by --check' config/skills/_shared/spec-verify-check.py \
|
|
708
|
-
|| ! grep -Fq 'descriptive backtick solo-headroom hypothesis was accepted by --check' config/skills/_shared/spec-verify-check.py \
|
|
709
|
-
|| ! grep -Fq 'risk probe missing solo-headroom command coverage was accepted' config/skills/_shared/spec-verify-check.py \
|
|
710
|
-
|| ! grep -Fq 'risk probe with unrelated solo-headroom derived_from was accepted' config/skills/_shared/spec-verify-check.py \
|
|
711
|
-
|| ! grep -Fq 'risk-probes[0].derived_from must reference the solo-headroom hypothesis bullet' config/skills/_shared/spec-verify-check.py \
|
|
712
|
-
|| ! grep -Fq 'solo-headroom command in a later risk probe was accepted' config/skills/_shared/spec-verify-check.py \
|
|
713
|
-
|| ! grep -Fq 'solo-headroom command prefix match was accepted' config/skills/_shared/spec-verify-check.py \
|
|
714
|
-
|| ! grep -Fq '(?<![A-Za-z0-9_.:/=-])' config/skills/_shared/spec-verify-check.py \
|
|
715
|
-
|| ! grep -Fq 'risk-probes[0].cmd must contain a solo-headroom hypothesis observable command' config/skills/_shared/spec-verify-check.py \
|
|
716
|
-
|| ! grep -Fq 'weak sibling solo-headroom hypothesis was accepted by --check-expected' config/skills/_shared/spec-verify-check.py \
|
|
717
|
-
|| ! grep -Fq 'docs-style solo-headroom hypothesis was rejected by --check' config/skills/_shared/spec-verify-check.py \
|
|
718
|
-
|| ! grep -Fq 'docs-style sibling solo-headroom command was rejected by --check-expected' config/skills/_shared/spec-verify-check.py \
|
|
719
|
-
|| ! grep -Fq 'def validate_present_solo_headroom_hypothesis' .agents/skills/_shared/spec-verify-check.py \
|
|
720
|
-
|| ! grep -Fq 'def state_requires_risk_probes' .agents/skills/_shared/spec-verify-check.py \
|
|
721
|
-
|| ! grep -Fq 'def risk_probes_state_error' .agents/skills/_shared/spec-verify-check.py \
|
|
722
|
-
|| ! grep -Fq -- '--include-risk-probes accepted missing required risk-probes.jsonl' .agents/skills/_shared/spec-verify-check.py \
|
|
723
|
-
|| ! grep -Fq -- '--include-risk-probes accepted non-boolean risk_probes_enabled' .agents/skills/_shared/spec-verify-check.py \
|
|
724
|
-
|| ! grep -Fq -- '--include-risk-probes accepted non-object risk_profile' .agents/skills/_shared/spec-verify-check.py \
|
|
725
|
-
|| ! grep -Fq 'def validate_risk_probes_cover_solo_headroom_hypothesis' .agents/skills/_shared/spec-verify-check.py \
|
|
726
|
-
|| ! grep -Fq 'def has_backticked_observable_miss_command' .agents/skills/_shared/spec-verify-check.py \
|
|
727
|
-
|| ! grep -Fq 'backticked command/observable line that exposes the miss' .agents/skills/_shared/spec-verify-check.py \
|
|
728
|
-
|| ! grep -Fq 'weak solo-headroom hypothesis was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
|
|
729
|
-
|| ! grep -Fq 'descriptive backtick solo-headroom hypothesis was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
|
|
730
|
-
|| ! grep -Fq 'risk probe missing solo-headroom command coverage was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
731
|
-
|| ! grep -Fq 'risk probe with unrelated solo-headroom derived_from was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
732
|
-
|| ! grep -Fq 'risk-probes[0].derived_from must reference the solo-headroom hypothesis bullet' .agents/skills/_shared/spec-verify-check.py \
|
|
733
|
-
|| ! grep -Fq 'solo-headroom command in a later risk probe was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
734
|
-
|| ! grep -Fq 'solo-headroom command prefix match was accepted' .agents/skills/_shared/spec-verify-check.py \
|
|
735
|
-
|| ! grep -Fq '(?<![A-Za-z0-9_.:/=-])' .agents/skills/_shared/spec-verify-check.py \
|
|
736
|
-
|| ! grep -Fq 'risk-probes[0].cmd must contain a solo-headroom hypothesis observable command' .agents/skills/_shared/spec-verify-check.py \
|
|
737
|
-
|| ! grep -Fq 'weak sibling solo-headroom hypothesis was accepted by --check-expected' .agents/skills/_shared/spec-verify-check.py \
|
|
738
|
-
|| ! grep -Fq 'docs-style solo-headroom hypothesis was rejected by --check' .agents/skills/_shared/spec-verify-check.py \
|
|
739
|
-
|| ! grep -Fq 'docs-style sibling solo-headroom command was rejected by --check-expected' .agents/skills/_shared/spec-verify-check.py; then
|
|
740
|
-
bad "spec-verify-check.py --check and --check-expected must reject weak solo-headroom hypotheses"
|
|
741
|
-
else
|
|
742
|
-
ok "spec-verify-check.py rejects weak solo-headroom hypotheses"
|
|
743
|
-
fi
|
|
744
|
-
if ! grep -Fq 'requires `.devlyn/risk-probes.jsonl`' config/skills/devlyn:resolve/SKILL.md \
|
|
745
|
-
|| ! grep -Fq 'requires `.devlyn/risk-probes.jsonl`' .agents/skills/devlyn:resolve/SKILL.md \
|
|
746
|
-
|| ! grep -Fq 'missing `.devlyn/risk-probes.jsonl` is a CRITICAL mechanical blocker' config/skills/devlyn:resolve/SKILL.md \
|
|
747
|
-
|| ! grep -Fq 'missing `.devlyn/risk-probes.jsonl` is a CRITICAL mechanical blocker' .agents/skills/devlyn:resolve/SKILL.md \
|
|
748
|
-
|| ! grep -Fq 'requires that file when `state.risk_profile.risk_probes_enabled == true`' config/skills/devlyn:resolve/references/phases/build-gate.md \
|
|
749
|
-
|| ! grep -Fq 'requires that file when `state.risk_profile.risk_probes_enabled == true`' .agents/skills/devlyn:resolve/references/phases/build-gate.md \
|
|
750
|
-
|| ! grep -Fq 'Malformed `state.risk_profile` is also CRITICAL because it can hide enabled risk probes' config/skills/devlyn:resolve/references/phases/build-gate.md \
|
|
751
|
-
|| ! grep -Fq 'Malformed `state.risk_profile` is also CRITICAL because it can hide enabled risk probes' .agents/skills/devlyn:resolve/references/phases/build-gate.md \
|
|
752
|
-
|| ! grep -Fq 'When `state.risk_profile.risk_probes_enabled == true`, missing `.devlyn/risk-probes.jsonl` is also CRITICAL' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
753
|
-
|| ! grep -Fq 'When `state.risk_profile.risk_probes_enabled == true`, missing `.devlyn/risk-probes.jsonl` is also CRITICAL' .agents/skills/devlyn:resolve/references/phases/verify.md; then
|
|
754
|
-
bad "BUILD_GATE and VERIFY must fail closed when enabled risk probes are missing"
|
|
755
|
-
else
|
|
756
|
-
ok "BUILD_GATE and VERIFY require enabled risk probes"
|
|
757
|
-
fi
|
|
758
|
-
if grep -Fq 'or any(char.isspace() for char in stripped)' config/skills/_shared/spec-verify-check.py \
|
|
759
|
-
|| grep -Fq 'or any(char.isspace() for char in stripped)' .agents/skills/_shared/spec-verify-check.py \
|
|
760
|
-
|| grep -Fq 'or any(char.isspace() for char in stripped)' config/skills/_shared/verify-merge-findings.py \
|
|
761
|
-
|| grep -Fq 'or any(char.isspace() for char in stripped)' .agents/skills/_shared/verify-merge-findings.py \
|
|
762
|
-
|| grep -Fq 'or any(char.isspace() for char in stripped)' benchmark/auto-resolve/scripts/pair_evidence_contract.py; then
|
|
763
|
-
bad "solo-headroom command detection must not treat descriptive whitespace as a command"
|
|
764
|
-
else
|
|
765
|
-
ok "solo-headroom command detection rejects descriptive whitespace"
|
|
766
|
-
fi
|
|
767
|
-
if ! grep -Fq '"printf",' config/skills/_shared/spec-verify-check.py \
|
|
768
|
-
|| ! grep -Fq '"printf",' .agents/skills/_shared/spec-verify-check.py \
|
|
769
|
-
|| ! grep -Fq '"printf",' config/skills/_shared/verify-merge-findings.py \
|
|
770
|
-
|| ! grep -Fq '"printf",' .agents/skills/_shared/verify-merge-findings.py \
|
|
771
|
-
|| ! grep -Fq '"printf",' benchmark/auto-resolve/scripts/pair_evidence_contract.py; then
|
|
772
|
-
bad "solo-headroom command detection must keep explicit printf command support"
|
|
773
|
-
else
|
|
774
|
-
ok "solo-headroom command detection keeps explicit printf command support"
|
|
775
|
-
fi
|
|
776
|
-
if python3 - <<'PY'
|
|
777
|
-
import ast
|
|
778
|
-
import pathlib
|
|
779
|
-
import sys
|
|
780
|
-
|
|
781
|
-
files = [
|
|
782
|
-
pathlib.Path("benchmark/auto-resolve/scripts/pair_evidence_contract.py"),
|
|
783
|
-
pathlib.Path("config/skills/_shared/spec-verify-check.py"),
|
|
784
|
-
pathlib.Path(".agents/skills/_shared/spec-verify-check.py"),
|
|
785
|
-
pathlib.Path("config/skills/_shared/verify-merge-findings.py"),
|
|
786
|
-
pathlib.Path(".agents/skills/_shared/verify-merge-findings.py"),
|
|
787
|
-
]
|
|
788
|
-
names = [
|
|
789
|
-
"COMMAND_PREFIXES",
|
|
790
|
-
"RESERVED_BACKTICK_TERMS",
|
|
791
|
-
"OBSERVABLE_COMMAND_MARKERS",
|
|
792
|
-
]
|
|
793
|
-
|
|
794
|
-
def extract(path: pathlib.Path, name: str) -> tuple[str, ...]:
|
|
795
|
-
tree = ast.parse(path.read_text(encoding="utf-8"))
|
|
796
|
-
for node in tree.body:
|
|
797
|
-
if isinstance(node, ast.Assign):
|
|
798
|
-
if any(isinstance(target, ast.Name) and target.id == name for target in node.targets):
|
|
799
|
-
value = ast.literal_eval(node.value)
|
|
800
|
-
return tuple(sorted(value))
|
|
801
|
-
raise AssertionError(f"{path}: missing {name}")
|
|
802
|
-
|
|
803
|
-
baseline = {name: extract(files[0], name) for name in names}
|
|
804
|
-
for path in files[1:]:
|
|
805
|
-
for name in names:
|
|
806
|
-
value = extract(path, name)
|
|
807
|
-
if value != baseline[name]:
|
|
808
|
-
print(f"{path}: {name} drifted from {files[0]}", file=sys.stderr)
|
|
809
|
-
print(f"expected={baseline[name]!r}", file=sys.stderr)
|
|
810
|
-
print(f"actual={value!r}", file=sys.stderr)
|
|
811
|
-
sys.exit(1)
|
|
812
|
-
PY
|
|
813
|
-
then
|
|
814
|
-
ok "solo-headroom command detection constants stay in parity"
|
|
815
|
-
else
|
|
816
|
-
bad "solo-headroom command detection constants must stay in parity"
|
|
817
|
-
fi
|
|
818
|
-
if ! grep -Fq 'def resolve_required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
|
|
819
|
-
|| ! grep -Fq 'def validate_required_risk_probe_requirement' config/skills/_shared/spec-verify-check.py \
|
|
820
|
-
|| ! grep -Fq 'missing required probe(s)' config/skills/_shared/spec-verify-check.py \
|
|
821
|
-
|| ! grep -Fq 'risk-probes.jsonl missing a declared required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
|
|
822
|
-
|| ! grep -Fq 'risk-probes.jsonl covering a declared required_risk_probe_requirements' config/skills/_shared/spec-verify-check.py \
|
|
823
|
-
|| ! grep -Fq 'required_risk_probe_requirements with an unknown tag was accepted' config/skills/_shared/spec-verify-check.py \
|
|
824
|
-
|| ! grep -Fq 'required_risk_probe_requirements.derived_from not present in the' config/skills/_shared/spec-verify-check.py \
|
|
825
|
-
|| ! grep -Fq 'required_risk_probe_requirements' config/skills/_shared/expected.schema.json \
|
|
826
|
-
|| ! grep -Fq 'required_risk_probe_requirements' config/skills/devlyn:ideate/references/spec-template.md; then
|
|
827
|
-
bad "spec-verify-check.py must enforce declared required_risk_probe_requirements (iter-0049 language-neutral F3 replacement)"
|
|
828
|
-
fi
|
|
829
|
-
if ! grep -Fq 'spec.expected.json top-level array produced a traceback' config/skills/_shared/spec-verify-check.py \
|
|
830
|
-
|| ! grep -Fq 'invalid spec.expected.json produced a traceback' config/skills/_shared/spec-verify-check.py \
|
|
831
|
-
|| ! grep -Fq 'NaN spec.expected.json did not report invalid numeric constant' config/skills/_shared/spec-verify-check.py \
|
|
832
|
-
|| ! grep -Fq 'NaN risk-probes JSONL did not report invalid numeric constant' config/skills/_shared/spec-verify-check.py \
|
|
833
|
-
|| ! grep -Fq 'def reject_json_constant' config/skills/_shared/spec-verify-check.py \
|
|
834
|
-
|| ! grep -Fq 'loads_strict_json(line)' config/skills/_shared/spec-verify-check.py \
|
|
835
|
-
|| ! grep -Fq 'loads_strict_json(expected_path.read_text(encoding="utf-8"))' config/skills/_shared/spec-verify-check.py \
|
|
836
|
-
|| ! grep -Fq 'top-level must be a JSON object' config/skills/_shared/spec-verify-check.py \
|
|
837
|
-
|| ! grep -Fq 'has invalid JSON' config/skills/_shared/spec-verify-check.py; then
|
|
838
|
-
bad "spec-verify-check.py self-test must fail malformed spec.expected.json cleanly without traceback"
|
|
839
|
-
fi
|
|
840
|
-
if ! grep -Fq '"asserts_named_stream_output"' config/skills/_shared/spec-verify-check.py \
|
|
841
|
-
|| ! grep -Fq '"asserts_error_payload_or_stderr"' config/skills/_shared/spec-verify-check.py \
|
|
842
|
-
|| ! grep -Fq '"asserts_nonzero_or_exit_2"' config/skills/_shared/spec-verify-check.py \
|
|
843
|
-
|| ! grep -Fq 'error_contract without exit-code evidence was accepted' config/skills/_shared/spec-verify-check.py \
|
|
844
|
-
|| ! grep -Fq 'stdout_stderr_contract without stream evidence was accepted' config/skills/_shared/spec-verify-check.py \
|
|
845
|
-
|| ! grep -Fq '`stdout_stderr_contract`: `asserts_named_stream_output`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
846
|
-
|| ! grep -Fq '`error_contract`: `asserts_error_payload_or_stderr`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
847
|
-
|| ! grep -Fq '`asserts_nonzero_or_exit_2`' config/skills/devlyn:resolve/SKILL.md; then
|
|
848
|
-
bad "risk-probe error/stdout-stderr tags must require concrete tag_evidence markers in validator and prompt contract"
|
|
849
|
-
fi
|
|
850
|
-
if ! grep -Fq '"http_error_contract"' config/skills/_shared/spec-verify-check.py \
|
|
851
|
-
|| ! grep -Fq 'asserts_http_error_status' config/skills/_shared/spec-verify-check.py \
|
|
852
|
-
|| ! grep -Fq 'http_error_contract without payload evidence was accepted' config/skills/_shared/spec-verify-check.py \
|
|
853
|
-
|| ! grep -Fq 'exact error body shape_contract without exact object evidence was accepted' config/skills/_shared/spec-verify-check.py \
|
|
854
|
-
|| ! grep -Fq 'exact error body shape_contract with exact object evidence was rejected' config/skills/_shared/spec-verify-check.py \
|
|
855
|
-
|| ! grep -Fq '`http_error_contract`: `asserts_http_error_status`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
856
|
-
|| ! grep -Fq '`http_error_contract` must include `asserts_http_error_status`' config/skills/devlyn:resolve/SKILL.md; then
|
|
857
|
-
bad "risk-probe HTTP error contracts must require concrete status and payload markers"
|
|
858
|
-
fi
|
|
859
|
-
if ! grep -Fq '"uses_visible_input_key_names"' config/skills/_shared/spec-verify-check.py \
|
|
860
|
-
|| ! grep -Fq '"asserts_visible_output_key_names"' config/skills/_shared/spec-verify-check.py \
|
|
861
|
-
|| ! grep -Fq '"asserts_no_unexpected_output_keys"' config/skills/_shared/spec-verify-check.py \
|
|
862
|
-
|| ! grep -Fq 'shape_contract claiming visible exact error object without' config/skills/_shared/spec-verify-check.py \
|
|
863
|
-
|| ! grep -Fq 'visible_text_names_exact_json_error_object' config/skills/_shared/spec-verify-check.py \
|
|
864
|
-
|| ! grep -Fq 'JSON error object shape_contract with exact object evidence was rejected' config/skills/_shared/spec-verify-check.py \
|
|
865
|
-
|| ! grep -Fq 'shape_contract without any evidence was accepted' config/skills/_shared/spec-verify-check.py \
|
|
866
|
-
|| ! grep -Fq 'shape_contract with exact key evidence was rejected' config/skills/_shared/spec-verify-check.py \
|
|
867
|
-
|| ! grep -Fq '`shape_contract` when the visible text names exact keys' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
868
|
-
|| ! grep -Fq 'visible_text_names_exact_json_error_object' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
869
|
-
|| ! grep -Fq '`shape_contract` must' config/skills/devlyn:resolve/SKILL.md \
|
|
870
|
-
|| ! grep -Fq 'visible_text_names_exact_json_error_object' config/skills/devlyn:resolve/SKILL.md; then
|
|
871
|
-
bad "risk-probe shape contracts must require exact visible input/output key evidence unconditionally, and asserts_exact_error_object when the probe claims the visible text names an exact JSON error object"
|
|
872
|
-
fi
|
|
873
|
-
if ! grep -Fq '"auth_signature_contract"' config/skills/_shared/spec-verify-check.py \
|
|
874
|
-
|| ! grep -Fq '"idempotency_replay"' config/skills/_shared/spec-verify-check.py \
|
|
875
|
-
|| ! grep -Fq 'asserts_signature_over_exact_bytes' config/skills/_shared/spec-verify-check.py \
|
|
876
|
-
|| ! grep -Fq 'duplicate_id_rejected_regardless_of_body' config/skills/_shared/spec-verify-check.py \
|
|
877
|
-
|| ! grep -Fq '`auth_signature_contract`: `asserts_signature_over_exact_bytes`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
878
|
-
|| ! grep -Fq '`idempotency_replay`: `first_delivery_then_duplicate`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
879
|
-
|| ! grep -Fq '`auth_signature_contract` must include `asserts_signature_over_exact_bytes`' config/skills/devlyn:resolve/SKILL.md; then
|
|
880
|
-
bad "risk-probe webhook/signature/replay contracts must require concrete auth_signature_contract and idempotency_replay tags"
|
|
881
|
-
fi
|
|
882
|
-
if ! grep -Fq '"concurrent_state_consistency"' config/skills/_shared/spec-verify-check.py \
|
|
883
|
-
|| ! grep -Fq 'overlapping_mutations_exercised' config/skills/_shared/spec-verify-check.py \
|
|
884
|
-
|| ! grep -Fq '`concurrent_state_consistency`: `overlapping_mutations_exercised`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
885
|
-
|| ! grep -Fq '`concurrent_state_consistency` must' config/skills/devlyn:resolve/SKILL.md; then
|
|
886
|
-
bad "risk-probe concurrent state contracts must require concrete concurrent_state_consistency markers"
|
|
887
|
-
fi
|
|
888
|
-
if ! grep -Fq '"atomic_batch_state"' config/skills/_shared/spec-verify-check.py \
|
|
889
|
-
|| ! grep -Fq 'mixed_valid_invalid_batch' config/skills/_shared/spec-verify-check.py \
|
|
890
|
-
|| ! grep -Fq 'atomic_batch_state without success-order evidence was accepted' config/skills/_shared/spec-verify-check.py \
|
|
891
|
-
|| ! grep -Fq '`atomic_batch_state`: `mixed_valid_invalid_batch`' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
892
|
-
|| ! grep -Fq '`atomic_batch_state` must include `mixed_valid_invalid_batch`' config/skills/devlyn:resolve/SKILL.md; then
|
|
893
|
-
bad "risk-probe atomic batch contracts must require concrete mixed-failure and success-order markers"
|
|
894
|
-
fi
|
|
895
|
-
|
|
896
|
-
section "Check 6f: ideate validates sibling spec.expected.json"
|
|
897
|
-
expected_check_missing=0
|
|
898
|
-
for file in \
|
|
899
|
-
config/skills/devlyn:ideate/SKILL.md \
|
|
900
|
-
config/skills/devlyn:ideate/references/elicitation.md \
|
|
901
|
-
config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
902
|
-
config/skills/devlyn:ideate/references/project-mode.md \
|
|
903
|
-
config/skills/devlyn:ideate/references/spec-template.md
|
|
904
|
-
do
|
|
905
|
-
if ! grep -Fq -- '--check-expected <expected-path>' "$file"; then
|
|
906
|
-
bad "$file — missing spec.expected.json mechanical validation command"
|
|
907
|
-
expected_check_missing=1
|
|
908
|
-
fi
|
|
909
|
-
done
|
|
910
|
-
if [ $expected_check_missing -eq 0 ]; then
|
|
911
|
-
ok "ideate docs require --check-expected for sibling expected contracts"
|
|
912
|
-
fi
|
|
913
|
-
if ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:ideate/SKILL.md \
|
|
914
|
-
|| ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:ideate/SKILL.md \
|
|
915
|
-
|| ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:ideate/references/elicitation.md \
|
|
916
|
-
|| ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
917
|
-
|| ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:ideate/references/project-mode.md \
|
|
918
|
-
|| ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/project-mode.md \
|
|
919
|
-
|| ! grep -Fq 'any present actionable solo-headroom hypothesis' config/skills/devlyn:resolve/SKILL.md \
|
|
920
|
-
|| ! grep -Fq 'any present actionable solo-headroom hypothesis' .agents/skills/devlyn:resolve/SKILL.md \
|
|
921
|
-
|| ! grep -Fq "legacy inline \`## Verification\` JSON carrier" config/skills/devlyn:ideate/SKILL.md \
|
|
922
|
-
|| ! grep -Fq "legacy inline \`## Verification\` JSON carrier" .agents/skills/devlyn:ideate/SKILL.md \
|
|
923
|
-
|| ! grep -Fq "legacy inline \`## Verification\` JSON carrier" config/skills/devlyn:ideate/references/elicitation.md \
|
|
924
|
-
|| ! grep -Fq "legacy inline \`## Verification\` JSON carrier" .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
925
|
-
|| ! grep -Fq "legacy inline \`## Verification\` JSON carrier" config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
926
|
-
|| ! grep -Fq "legacy inline \`## Verification\` JSON carrier" .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
927
|
-
|| ! grep -Fq "inline \`## Verification\` JSON carrier" config/skills/devlyn:resolve/SKILL.md \
|
|
928
|
-
|| ! grep -Fq "inline \`## Verification\` JSON carrier" .agents/skills/devlyn:resolve/SKILL.md \
|
|
929
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/SKILL.md \
|
|
930
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/SKILL.md \
|
|
931
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/elicitation.md \
|
|
932
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
933
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/project-mode.md \
|
|
934
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/project-mode.md \
|
|
935
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:resolve/SKILL.md \
|
|
936
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:resolve/SKILL.md; then
|
|
937
|
-
bad "ideate/resolve docs must describe mechanical solo-headroom validation"
|
|
938
|
-
else
|
|
939
|
-
ok "ideate/resolve docs describe mechanical solo-headroom validation"
|
|
940
|
-
fi
|
|
941
|
-
if ! grep -Fq 'def validate_expected_against_sibling_spec' config/skills/_shared/spec-verify-check.py \
|
|
942
|
-
|| ! grep -Fq 'empty verification_commands should fail for runtime specs' config/skills/_shared/spec-verify-check.py \
|
|
943
|
-
|| ! grep -Fq 'empty verification_commands should be valid for pure-design specs' config/skills/_shared/spec-verify-check.py \
|
|
944
|
-
|| ! grep -Fq 'backticked_observable_miss_commands(spec_text)' config/skills/_shared/spec-verify-check.py \
|
|
945
|
-
|| ! grep -Fq 'observable command must match spec.expected.json' config/skills/_shared/spec-verify-check.py \
|
|
946
|
-
|| ! grep -Fq 'observable command must match `## Verification` JSON carrier' config/skills/_shared/spec-verify-check.py \
|
|
947
|
-
|| ! grep -Fq 'mismatched inline solo-headroom command was accepted by --check' config/skills/_shared/spec-verify-check.py \
|
|
948
|
-
|| ! grep -Fq 'matched inline solo-headroom command was rejected by --check' config/skills/_shared/spec-verify-check.py \
|
|
949
|
-
|| ! grep -Fq 'mismatched sibling solo-headroom command was accepted by --check-expected' config/skills/_shared/spec-verify-check.py \
|
|
950
|
-
|| ! grep -Fq 'matched sibling solo-headroom command was rejected by --check-expected' config/skills/_shared/spec-verify-check.py \
|
|
951
|
-
|| ! grep -Fq 'def validate_present_solo_ceiling_avoidance' config/skills/_shared/spec-verify-check.py \
|
|
952
|
-
|| ! grep -Fq 'weak solo ceiling avoidance was accepted by --check' config/skills/_shared/spec-verify-check.py \
|
|
953
|
-
|| ! grep -Fq 'actionable solo ceiling avoidance was rejected by --check' config/skills/_shared/spec-verify-check.py \
|
|
954
|
-
|| ! grep -Fq 'weak sibling solo ceiling avoidance was accepted by --check-expected' config/skills/_shared/spec-verify-check.py \
|
|
955
|
-
|| ! grep -Fq 'actionable sibling solo ceiling avoidance was rejected by --check-expected' config/skills/_shared/spec-verify-check.py \
|
|
956
|
-
|| ! grep -Fq 'backticked_observable_miss_commands(spec_text)' .agents/skills/_shared/spec-verify-check.py \
|
|
957
|
-
|| ! grep -Fq 'observable command must match spec.expected.json' .agents/skills/_shared/spec-verify-check.py \
|
|
958
|
-
|| ! grep -Fq 'observable command must match `## Verification` JSON carrier' .agents/skills/_shared/spec-verify-check.py \
|
|
959
|
-
|| ! grep -Fq 'mismatched inline solo-headroom command was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
|
|
960
|
-
|| ! grep -Fq 'matched inline solo-headroom command was rejected by --check' .agents/skills/_shared/spec-verify-check.py \
|
|
961
|
-
|| ! grep -Fq 'mismatched sibling solo-headroom command was accepted by --check-expected' .agents/skills/_shared/spec-verify-check.py \
|
|
962
|
-
|| ! grep -Fq 'matched sibling solo-headroom command was rejected by --check-expected' .agents/skills/_shared/spec-verify-check.py \
|
|
963
|
-
|| ! grep -Fq 'def validate_present_solo_ceiling_avoidance' .agents/skills/_shared/spec-verify-check.py \
|
|
964
|
-
|| ! grep -Fq 'weak solo ceiling avoidance was accepted by --check' .agents/skills/_shared/spec-verify-check.py \
|
|
965
|
-
|| ! grep -Fq 'actionable solo ceiling avoidance was rejected by --check' .agents/skills/_shared/spec-verify-check.py \
|
|
966
|
-
|| ! grep -Fq 'weak sibling solo ceiling avoidance was accepted by --check-expected' .agents/skills/_shared/spec-verify-check.py \
|
|
967
|
-
|| ! grep -Fq 'actionable sibling solo ceiling avoidance was rejected by --check-expected' .agents/skills/_shared/spec-verify-check.py; then
|
|
968
|
-
bad "spec-verify-check.py must reject empty expected runtime contracts, weak solo ceiling avoidance, and preserve pure-design escape"
|
|
969
|
-
fi
|
|
970
|
-
if ! grep -Fq 'Verification includes at least one compound scenario that exercises the interaction end-to-end' \
|
|
971
|
-
config/skills/devlyn:ideate/references/spec-template.md \
|
|
972
|
-
|| ! grep -Fq 'Verification includes at least one compound scenario that exercises the interaction end-to-end' .agents/skills/devlyn:ideate/references/spec-template.md; then
|
|
973
|
-
bad "ideate spec template must require compound interaction verification for pair-relevant high-risk specs"
|
|
974
|
-
else
|
|
975
|
-
ok "ideate spec template requires compound interaction verification for pair-relevant specs"
|
|
976
|
-
fi
|
|
977
|
-
if ! grep -Fq 'ask for one concrete compound' config/skills/devlyn:ideate/references/elicitation.md \
|
|
978
|
-
|| ! grep -Fq 'ask for one concrete compound' .agents/skills/devlyn:ideate/references/elicitation.md; then
|
|
979
|
-
bad "ideate elicitation must ask for compound interaction scenarios when pair-relevant risks appear"
|
|
980
|
-
else
|
|
981
|
-
ok "ideate elicitation asks for compound interaction scenarios when pair-relevant risks appear"
|
|
982
|
-
fi
|
|
983
|
-
if ! grep -Fq 'solo-headroom hypothesis inside `## Verification`' config/skills/devlyn:ideate/references/spec-template.md \
|
|
984
|
-
|| ! grep -Fq 'solo-headroom hypothesis inside `## Verification`' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
985
|
-
|| ! grep -Fq 'ask for one solo-headroom hypothesis' config/skills/devlyn:ideate/references/elicitation.md \
|
|
986
|
-
|| ! grep -Fq 'ask for one solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/elicitation.md; then
|
|
987
|
-
bad "ideate must require a visible solo-headroom hypothesis for benchmark and pair-evidence specs"
|
|
988
|
-
else
|
|
989
|
-
ok "ideate requires solo-headroom hypothesis for benchmark and pair-evidence specs"
|
|
990
|
-
fi
|
|
991
|
-
if ! grep -Fq 'must literally contain `solo-headroom hypothesis`' config/skills/devlyn:ideate/references/spec-template.md \
|
|
992
|
-
|| ! grep -Fq 'must literally contain `solo-headroom hypothesis`' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
993
|
-
|| ! grep -Fq 'backticked line itself must contain `miss`' config/skills/devlyn:ideate/references/spec-template.md \
|
|
994
|
-
|| ! grep -Fq 'backticked line itself must contain `miss`' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
995
|
-
|| ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' config/skills/devlyn:ideate/references/spec-template.md \
|
|
996
|
-
|| ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
997
|
-
|| ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/spec-template.md \
|
|
998
|
-
|| ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
999
|
-
|| ! grep -Fq 'must literally contain `solo-headroom hypothesis`' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1000
|
-
|| ! grep -Fq 'must literally contain `solo-headroom hypothesis`' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1001
|
-
|| ! grep -Fq 'line itself must contain `miss`' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1002
|
-
|| ! grep -Fq 'line itself must contain `miss`' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1003
|
-
|| ! grep -Fq 'Do not write a benchmark/risk-probe/pair-evidence spec until this' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1004
|
-
|| ! grep -Fq 'Do not write a benchmark/risk-probe/pair-evidence spec until this' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1005
|
-
|| ! grep -Fq 'spec not ready — solo-headroom hypothesis required' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1006
|
-
|| ! grep -Fq 'spec not ready — solo-headroom hypothesis required' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1007
|
-
|| ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1008
|
-
|| ! grep -Fq '`solo_claude`, `miss`, and a backticked observable command' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1009
|
-
|| ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1010
|
-
|| ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1011
|
-
|| ! grep -Fq 'Verification literally contains `solo-headroom hypothesis`, `solo_claude`' config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1012
|
-
|| ! grep -Fq 'Verification literally contains `solo-headroom hypothesis`, `solo_claude`' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1013
|
-
|| ! grep -Fq 'backticked line itself must contain `miss`' config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1014
|
-
|| ! grep -Fq 'backticked line itself must contain `miss`' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1015
|
-
|| ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1016
|
-
|| ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1017
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1018
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1019
|
-
|| ! grep -Fq "source for VERIFY's canonical \`spec.solo_headroom_hypothesis\` trigger reason" config/skills/devlyn:ideate/references/spec-template.md \
|
|
1020
|
-
|| ! grep -Fq "source for VERIFY's canonical \`spec.solo_headroom_hypothesis\` trigger reason" .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
1021
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' config/skills/devlyn:ideate/references/spec-template.md \
|
|
1022
|
-
|| ! grep -Fq 'spec.expected.json.verification_commands[].cmd' .agents/skills/devlyn:ideate/references/spec-template.md; then
|
|
1023
|
-
bad "ideate solo-headroom hypothesis prompt must match the actionable checker contract"
|
|
1024
|
-
else
|
|
1025
|
-
ok "ideate solo-headroom hypothesis prompt matches checker contract"
|
|
1026
|
-
fi
|
|
1027
|
-
if ! grep -Fq 'quick mode must not infer a solo-headroom hypothesis' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1028
|
-
|| ! grep -Fq 'quick mode must not infer a solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1029
|
-
|| ! grep -Fq 'do not infer a solo-headroom hypothesis' config/skills/devlyn:ideate/SKILL.md \
|
|
1030
|
-
|| ! grep -Fq 'do not infer a solo-headroom hypothesis' .agents/skills/devlyn:ideate/SKILL.md; then
|
|
1031
|
-
bad "ideate quick mode must not invent solo-headroom hypotheses"
|
|
1032
|
-
else
|
|
1033
|
-
ok "ideate quick mode does not invent solo-headroom hypotheses"
|
|
1034
|
-
fi
|
|
1035
|
-
if ! grep -Fq 'solo ceiling avoidance' config/skills/devlyn:ideate/references/spec-template.md \
|
|
1036
|
-
|| ! grep -Fq 'solo ceiling avoidance' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
1037
|
-
|| ! grep -Fq 'rejected or solo-saturated controls such as `S2`-`S6`' config/skills/devlyn:ideate/references/spec-template.md \
|
|
1038
|
-
|| ! grep -Fq 'rejected or solo-saturated controls such as `S2`-`S6`' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
1039
|
-
|| ! grep -Fq 'Solo ceiling avoidance' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1040
|
-
|| ! grep -Fq 'Solo ceiling avoidance' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1041
|
-
|| ! grep -Fq 'spec not ready — solo ceiling avoidance required' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1042
|
-
|| ! grep -Fq 'spec not ready — solo ceiling avoidance required' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1043
|
-
|| ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states solo ceiling avoidance' config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1044
|
-
|| ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states solo ceiling avoidance' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1045
|
-
|| ! grep -Fq 'also do not infer solo ceiling avoidance' config/skills/devlyn:ideate/SKILL.md \
|
|
1046
|
-
|| ! grep -Fq 'also do not infer solo ceiling avoidance' .agents/skills/devlyn:ideate/SKILL.md \
|
|
1047
|
-
|| ! grep -Fq 'per-feature Verification must also include a solo ceiling avoidance note' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1048
|
-
|| ! grep -Fq 'per-feature Verification must also include a solo ceiling avoidance note' .agents/skills/devlyn:ideate/references/project-mode.md; then
|
|
1049
|
-
bad "ideate must require solo ceiling avoidance for new unmeasured pair candidates"
|
|
1050
|
-
else
|
|
1051
|
-
ok "ideate requires solo ceiling avoidance for new unmeasured pair candidates"
|
|
1052
|
-
fi
|
|
1053
|
-
if ! grep -Fq 'complexity: medium' config/skills/devlyn:ideate/references/spec-template.md \
|
|
1054
|
-
|| ! grep -Fq 'complexity: medium' .agents/skills/devlyn:ideate/references/spec-template.md \
|
|
1055
|
-
|| ! grep -Fq 'Complexity signal' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1056
|
-
|| ! grep -Fq 'Complexity signal' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1057
|
-
|| ! grep -Fq 'downstream VERIFY pair-trigger signal' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1058
|
-
|| ! grep -Fq 'downstream VERIFY pair-trigger signal' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1059
|
-
|| ! grep -Fq 'complexity=medium default' config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1060
|
-
|| ! grep -Fq 'complexity=medium default' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1061
|
-
|| ! grep -Fq 'supported `complexity` frontmatter' config/skills/devlyn:ideate/SKILL.md \
|
|
1062
|
-
|| ! grep -Fq 'supported `complexity` frontmatter' .agents/skills/devlyn:ideate/SKILL.md \
|
|
1063
|
-
|| ! grep -Fq 'supported `complexity` frontmatter' config/skills/devlyn:ideate/references/elicitation.md \
|
|
1064
|
-
|| ! grep -Fq 'supported `complexity` frontmatter' .agents/skills/devlyn:ideate/references/elicitation.md \
|
|
1065
|
-
|| ! grep -Fq 'supported `complexity` frontmatter' config/skills/devlyn:resolve/SKILL.md \
|
|
1066
|
-
|| ! grep -Fq 'supported `complexity` frontmatter' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1067
|
-
|| ! grep -Fq 'sibling spec `complexity` frontmatter' config/skills/devlyn:ideate/SKILL.md \
|
|
1068
|
-
|| ! grep -Fq 'sibling spec `complexity` frontmatter' .agents/skills/devlyn:ideate/SKILL.md \
|
|
1069
|
-
|| ! grep -Fq 'sibling spec `complexity` frontmatter' config/skills/devlyn:resolve/SKILL.md \
|
|
1070
|
-
|| ! grep -Fq 'sibling spec `complexity` frontmatter' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1071
|
-
|| ! grep -Fq 'Frontmatter has `id`, `title`, `kind`, `status: planned`, `complexity`' config/skills/devlyn:ideate/SKILL.md \
|
|
1072
|
-
|| ! grep -Fq 'Frontmatter has `id`, `title`, `kind`, `status: planned`, `complexity`' .agents/skills/devlyn:ideate/SKILL.md; then
|
|
1073
|
-
bad "ideate specs must emit complexity frontmatter for resolve pair triggers"
|
|
1074
|
-
else
|
|
1075
|
-
ok "ideate specs emit complexity frontmatter for resolve pair triggers"
|
|
1076
|
-
fi
|
|
1077
|
-
if ! grep -Fq 'warning: Verification may need one compound end-to-end scenario before pair-relevant risks are measurable' \
|
|
1078
|
-
config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1079
|
-
|| ! grep -Fq 'warning: Verification may need one compound end-to-end scenario before pair-relevant risks are measurable' .agents/skills/devlyn:ideate/references/from-spec-mode.md; then
|
|
1080
|
-
bad "ideate from-spec mode must warn when preserved high-risk specs lack compound verification"
|
|
1081
|
-
else
|
|
1082
|
-
ok "ideate from-spec mode warns on pair-relevant specs with weak verification"
|
|
1083
|
-
fi
|
|
1084
|
-
if ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states a solo-headroom hypothesis' \
|
|
1085
|
-
config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1086
|
-
|| ! grep -Fq 'pair-evidence not ready — Pair-candidate headroom is unproven until the spec states a solo-headroom hypothesis' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1087
|
-
|| ! grep -Fq 'Do not call' config/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1088
|
-
|| ! grep -Fq 'Do not call' .agents/skills/devlyn:ideate/references/from-spec-mode.md \
|
|
1089
|
-
|| ! grep -Fq 'announcement must say `pair-evidence not ready`' config/skills/devlyn:ideate/SKILL.md \
|
|
1090
|
-
|| ! grep -Fq 'announcement must say `pair-evidence not ready`' .agents/skills/devlyn:ideate/SKILL.md; then
|
|
1091
|
-
bad "ideate from-spec mode must warn when pair-candidate specs lack solo-headroom hypothesis"
|
|
1092
|
-
else
|
|
1093
|
-
ok "ideate from-spec mode warns on missing solo-headroom hypothesis"
|
|
1094
|
-
fi
|
|
1095
|
-
if ! grep -Fq 'per-feature Verification must' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1096
|
-
|| ! grep -Fq 'per-feature Verification must' .agents/skills/devlyn:ideate/references/project-mode.md; then
|
|
1097
|
-
bad "ideate project mode must require compound verification inside each pair-relevant feature spec"
|
|
1098
|
-
else
|
|
1099
|
-
ok "ideate project mode keeps compound verification inside pair-relevant feature specs"
|
|
1100
|
-
fi
|
|
1101
|
-
if ! grep -Fq 'per-feature Verification must include a solo-headroom' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1102
|
-
|| ! grep -Fq 'per-feature Verification must include a solo-headroom' .agents/skills/devlyn:ideate/references/project-mode.md \
|
|
1103
|
-
|| grep -rq 'Context or Verification' config/skills/devlyn:ideate/references .agents/skills/devlyn:ideate/references; then
|
|
1104
|
-
bad "ideate project mode must keep solo-headroom hypothesis inside each pair-candidate feature spec"
|
|
1105
|
-
else
|
|
1106
|
-
ok "ideate project mode keeps solo-headroom hypothesis inside each pair-candidate feature spec"
|
|
1107
|
-
fi
|
|
1108
|
-
if ! grep -Fq 'feature spec must literally contain' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1109
|
-
|| ! grep -Fq 'feature spec must literally contain' .agents/skills/devlyn:ideate/references/project-mode.md \
|
|
1110
|
-
|| ! grep -Fq '`solo-headroom hypothesis`, `solo_claude`, `miss`, and a backticked' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1111
|
-
|| ! grep -Fq '`solo-headroom hypothesis`, `solo_claude`, `miss`, and a backticked' .agents/skills/devlyn:ideate/references/project-mode.md \
|
|
1112
|
-
|| ! grep -Fq 'behavior a capable' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1113
|
-
|| ! grep -Fq 'behavior a capable' .agents/skills/devlyn:ideate/references/project-mode.md \
|
|
1114
|
-
|| ! grep -Fq 'backticked line itself must' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1115
|
-
|| ! grep -Fq 'backticked line itself must' .agents/skills/devlyn:ideate/references/project-mode.md \
|
|
1116
|
-
|| ! grep -Fq 'command/observable' config/skills/devlyn:ideate/references/project-mode.md \
|
|
1117
|
-
|| ! grep -Fq 'command/observable' .agents/skills/devlyn:ideate/references/project-mode.md; then
|
|
1118
|
-
bad "ideate project mode solo-headroom prompt must keep the actionable checker contract"
|
|
1119
|
-
else
|
|
1120
|
-
ok "ideate project mode solo-headroom prompt keeps checker contract"
|
|
1121
|
-
fi
|
|
1122
|
-
|
|
1123
|
-
if ! grep -Fq 'If the visible spec includes a solo-headroom hypothesis, the first probe must' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1124
|
-
|| ! grep -Fq 'If the visible spec includes a solo-headroom hypothesis, the first probe must' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1125
|
-
|| ! grep -Fq 'When the visible spec includes a solo-headroom hypothesis, the first probe must' config/skills/devlyn:resolve/SKILL.md \
|
|
1126
|
-
|| ! grep -Fq 'When the visible spec includes a solo-headroom hypothesis, the first probe must' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1127
|
-
|| ! grep -Fq 'its `cmd` must contain the hypothesis'\''s backticked' config/skills/devlyn:resolve/SKILL.md \
|
|
1128
|
-
|| ! grep -Fq 'its `cmd` must contain the hypothesis'\''s backticked' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1129
|
-
|| ! grep -Fq 'its `derived_from` must reference the hypothesis bullet' config/skills/devlyn:resolve/SKILL.md \
|
|
1130
|
-
|| ! grep -Fq 'its `derived_from` must reference the hypothesis bullet' .agents/skills/devlyn:resolve/SKILL.md; then
|
|
1131
|
-
bad "resolve risk-probe prompts must consume solo-headroom hypothesis before pair-evidence work"
|
|
1132
|
-
else
|
|
1133
|
-
ok "resolve risk-probe prompts consume solo-headroom hypothesis"
|
|
1134
|
-
fi
|
|
1135
|
-
if ! grep -Fq 'the behavior the spec says `solo_claude` is expected to miss' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1136
|
-
|| ! grep -Fq 'the behavior the spec says `solo_claude` is expected to miss' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1137
|
-
|| ! grep -Fq 'exercise the stated `solo_claude` miss' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1138
|
-
|| ! grep -Fq 'exercise the stated `solo_claude` miss' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1139
|
-
|| ! grep -Fq 'with a `cmd` containing the hypothesis'\''s' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1140
|
-
|| ! grep -Fq 'with a `cmd` containing the hypothesis'\''s' .agents/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1141
|
-
|| ! grep -Fq '`derived_from` pointing at the hypothesis' config/skills/devlyn:resolve/references/phases/probe-derive.md \
|
|
1142
|
-
|| ! grep -Fq '`derived_from` pointing at the hypothesis' .agents/skills/devlyn:resolve/references/phases/probe-derive.md; then
|
|
1143
|
-
bad "resolve risk-probe solo-headroom prompt must target the stated solo_claude miss"
|
|
1144
|
-
else
|
|
1145
|
-
ok "resolve risk-probe solo-headroom prompt targets the stated solo_claude miss"
|
|
1146
|
-
fi
|
|
1147
|
-
if ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
|
|
1148
|
-
config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1149
|
-
|| ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
|
|
1150
|
-
.agents/skills/devlyn:resolve/references/phases/verify.md \
|
|
1151
|
-
|| ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
|
|
1152
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1153
|
-
|| ! grep -Fq 'If the spec includes a solo-headroom hypothesis, one targeted review must use' \
|
|
1154
|
-
.agents/skills/devlyn:resolve/SKILL.md \
|
|
1155
|
-
|| ! grep -Fq 'hypothesis'\''s backticked observable command as the exact anchor' \
|
|
1156
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1157
|
-
|| ! grep -Fq 'hypothesis'\''s backticked observable command as the exact anchor' \
|
|
1158
|
-
.agents/skills/devlyn:resolve/SKILL.md \
|
|
1159
|
-
|| ! grep -Fq 'hypothesis'\''s backticked observable command as its exact anchor' \
|
|
1160
|
-
config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1161
|
-
|| ! grep -Fq 'hypothesis'\''s backticked observable command as its exact anchor' \
|
|
1162
|
-
.agents/skills/devlyn:resolve/references/phases/verify.md; then
|
|
1163
|
-
bad "resolve pair-JUDGE prompts must prioritize solo-headroom hypothesis"
|
|
1164
|
-
else
|
|
1165
|
-
ok "resolve pair-JUDGE prompts prioritize solo-headroom hypothesis"
|
|
1166
|
-
fi
|
|
1167
|
-
if ! grep -Fq 'The `--engine` flag does not disable default pairing' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1168
|
-
|| ! grep -Fq 'The `--engine` flag does not disable default pairing' .agents/skills/devlyn:resolve/references/phases/verify.md \
|
|
1169
|
-
|| ! grep -Fq 'the second judge uses the OTHER engine by default when available' config/skills/devlyn:resolve/SKILL.md \
|
|
1170
|
-
|| ! grep -Fq 'the second judge uses the OTHER engine by default when available' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1171
|
-
|| ! grep -Fq '_shared/adapters/<name>.md' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1172
|
-
|| ! grep -Fq '`## Invocation`' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1173
|
-
|| ! grep -Fq '_shared/adapters/<name>.md` `## Invocation' config/skills/devlyn:resolve/SKILL.md \
|
|
1174
|
-
|| ! grep -Fq '## Invocation' config/skills/_shared/adapters/claude.md \
|
|
1175
|
-
|| ! grep -Fq '.devlyn/claude-judge.stdout' config/skills/_shared/adapters/claude.md; then
|
|
1176
|
-
bad "engine-neutral pair-judge dual declaration (iter-0060) out of sync"
|
|
1177
|
-
else
|
|
1178
|
-
ok "engine-neutral pair-judge dual declaration (iter-0060) in sync"
|
|
1179
|
-
fi
|
|
1180
|
-
if ! grep -Fq '`complexity.high`, `complexity.large`' config/skills/devlyn:resolve/SKILL.md \
|
|
1181
|
-
|| ! grep -Fq '`complexity.high`, `complexity.large`' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1182
|
-
|| ! grep -Fq '`spec.complexity.high`, `spec.complexity.large`' config/skills/devlyn:resolve/SKILL.md \
|
|
1183
|
-
|| ! grep -Fq '`spec.complexity.high`, `spec.complexity.large`' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1184
|
-
|| ! grep -Fq 'Legacy complexity values remain accepted only for archived compatibility' config/skills/devlyn:resolve/references/state-schema.md \
|
|
1185
|
-
|| ! grep -Fq 'Legacy complexity values remain accepted only for archived compatibility' .agents/skills/devlyn:resolve/references/state-schema.md \
|
|
1186
|
-
|| ! grep -Fq '`complexity.high`, `complexity.large`, `spec.complexity.high`, `spec.complexity.large`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
1187
|
-
|| ! grep -Fq '`complexity.high`, `complexity.large`, `spec.complexity.high`, `spec.complexity.large`' .agents/skills/devlyn:resolve/references/state-schema.md \
|
|
1188
|
-
|| ! grep -Fq '`complexity.large`, `spec.complexity.high`, `spec.complexity.large`,' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1189
|
-
|| ! grep -Fq '`complexity.large`, `spec.complexity.high`, `spec.complexity.large`,' .agents/skills/devlyn:resolve/references/phases/verify.md \
|
|
1190
|
-
|| ! grep -Fq '`spec.solo_headroom_hypothesis`, `risk.high`, `risk_probes.enabled`,' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1191
|
-
|| ! grep -Fq '`spec.solo_headroom_hypothesis`, `risk.high`, `risk_probes.enabled`,' .agents/skills/devlyn:resolve/references/phases/verify.md; then
|
|
1192
|
-
bad "resolve VERIFY docs must distinguish current large complexity, legacy high state, and legacy large spec compatibility"
|
|
1193
|
-
else
|
|
1194
|
-
ok "resolve VERIFY docs distinguish current large complexity, legacy high state, and legacy large spec compatibility"
|
|
1195
|
-
fi
|
|
1196
|
-
if ! grep -Fq 'def spec_has_solo_headroom_hypothesis' config/skills/_shared/verify-merge-findings.py \
|
|
1197
|
-
|| ! grep -Fq 'spec.solo_headroom_hypothesis' config/skills/_shared/verify-merge-findings.py \
|
|
1198
|
-
|| ! grep -Fq 'and "solo_claude" in lower' config/skills/_shared/verify-merge-findings.py \
|
|
1199
|
-
|| ! grep -Fq 'and "miss" in lower' config/skills/_shared/verify-merge-findings.py \
|
|
1200
|
-
|| ! grep -Fq 'OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")' config/skills/_shared/verify-merge-findings.py \
|
|
1201
|
-
|| ! grep -Fq 'def is_command_like_backtick' config/skills/_shared/verify-merge-findings.py \
|
|
1202
|
-
|| ! grep -Fq 'for key in ("spec_path", "criteria_path")' config/skills/_shared/verify-merge-findings.py \
|
|
1203
|
-
|| ! grep -Fq 'has_backticked_observable_command(text)' config/skills/_shared/verify-merge-findings.py \
|
|
1204
|
-
|| ! grep -Fq 'Observable command: `node check.js` exposes behavior' config/skills/_shared/verify-merge-findings.py \
|
|
1205
|
-
|| ! grep -Fq 'observable `SOLO_CLAUDE` exposes the miss' config/skills/_shared/verify-merge-findings.py \
|
|
1206
|
-
|| ! grep -Fq 'observable `priority rollback` exposes the miss' config/skills/_shared/verify-merge-findings.py \
|
|
1207
|
-
|| ! grep -Fq 'implementation token `rollback`' config/skills/_shared/verify-merge-findings.py \
|
|
1208
|
-
|| ! grep -Fq '`SOLO_CLAUDE` should miss' config/skills/_shared/verify-merge-findings.py \
|
|
1209
|
-
|| ! grep -Fq '{"source": {"criteria_path": str(criteria_path)}}' config/skills/_shared/verify-merge-findings.py \
|
|
1210
|
-
|| ! grep -Fq 'spec_has_solo_headroom_hypothesis(' config/skills/_shared/verify-merge-findings.py \
|
|
1211
|
-
|| ! grep -Fq ') is False' config/skills/_shared/verify-merge-findings.py \
|
|
1212
|
-
|| ! grep -Fq ') is True' config/skills/_shared/verify-merge-findings.py \
|
|
1213
|
-
|| ! grep -Fq 'def spec_has_solo_headroom_hypothesis' .agents/skills/_shared/verify-merge-findings.py \
|
|
1214
|
-
|| ! grep -Fq 'spec.solo_headroom_hypothesis' .agents/skills/_shared/verify-merge-findings.py \
|
|
1215
|
-
|| ! grep -Fq 'and "solo_claude" in lower' .agents/skills/_shared/verify-merge-findings.py \
|
|
1216
|
-
|| ! grep -Fq 'and "miss" in lower' .agents/skills/_shared/verify-merge-findings.py \
|
|
1217
|
-
|| ! grep -Fq 'OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")' .agents/skills/_shared/verify-merge-findings.py \
|
|
1218
|
-
|| ! grep -Fq 'def is_command_like_backtick' .agents/skills/_shared/verify-merge-findings.py \
|
|
1219
|
-
|| ! grep -Fq 'for key in ("spec_path", "criteria_path")' .agents/skills/_shared/verify-merge-findings.py \
|
|
1220
|
-
|| ! grep -Fq 'has_backticked_observable_command(text)' .agents/skills/_shared/verify-merge-findings.py \
|
|
1221
|
-
|| ! grep -Fq 'Observable command: `node check.js` exposes behavior' .agents/skills/_shared/verify-merge-findings.py \
|
|
1222
|
-
|| ! grep -Fq 'observable `SOLO_CLAUDE` exposes the miss' .agents/skills/_shared/verify-merge-findings.py \
|
|
1223
|
-
|| ! grep -Fq 'observable `priority rollback` exposes the miss' .agents/skills/_shared/verify-merge-findings.py \
|
|
1224
|
-
|| ! grep -Fq 'implementation token `rollback`' .agents/skills/_shared/verify-merge-findings.py \
|
|
1225
|
-
|| ! grep -Fq '`SOLO_CLAUDE` should miss' .agents/skills/_shared/verify-merge-findings.py \
|
|
1226
|
-
|| ! grep -Fq '{"source": {"criteria_path": str(criteria_path)}}' .agents/skills/_shared/verify-merge-findings.py \
|
|
1227
|
-
|| ! grep -Fq 'spec_has_solo_headroom_hypothesis(' .agents/skills/_shared/verify-merge-findings.py \
|
|
1228
|
-
|| ! grep -Fq ') is False' .agents/skills/_shared/verify-merge-findings.py \
|
|
1229
|
-
|| ! grep -Fq ') is True' .agents/skills/_shared/verify-merge-findings.py \
|
|
1230
|
-
|| ! grep -Fq '`spec.solo_headroom_hypothesis`' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1231
|
-
|| ! grep -Fq '`spec.solo_headroom_hypothesis`' .agents/skills/devlyn:resolve/references/phases/verify.md \
|
|
1232
|
-
|| ! grep -Fq '`spec.solo_headroom_hypothesis`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
1233
|
-
|| ! grep -Fq '`spec.solo_headroom_hypothesis`' .agents/skills/devlyn:resolve/references/state-schema.md \
|
|
1234
|
-
|| ! grep -Fq 'same actionable solo-headroom hypothesis is a VERIFY pair-trigger reason' config/skills/devlyn:resolve/SKILL.md \
|
|
1235
|
-
|| ! grep -Fq 'same actionable solo-headroom hypothesis is a VERIFY pair-trigger reason' .agents/skills/devlyn:resolve/SKILL.md; then
|
|
1236
|
-
bad "resolve VERIFY pair trigger must include actionable solo-headroom hypothesis specs"
|
|
1237
|
-
else
|
|
1238
|
-
ok "resolve VERIFY pair trigger includes actionable solo-headroom hypothesis specs"
|
|
1239
|
-
fi
|
|
1240
|
-
if ! grep -Fq 'pair_evidence_intent' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1241
|
-
|| ! grep -Fq 'pair_evidence_intent' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1242
|
-
|| ! grep -Fq 'has_actionable_solo_headroom' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1243
|
-
|| ! grep -Fq 'has_actionable_solo_headroom' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1244
|
-
|| ! grep -Fq 'state.source.type = "generated"' config/skills/devlyn:resolve/SKILL.md \
|
|
1245
|
-
|| ! grep -Fq 'state.source.type = "generated"' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1246
|
-
|| ! grep -Fq 'state.source.criteria_path = ".devlyn/criteria.generated.md"' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1247
|
-
|| ! grep -Fq 'state.source.criteria_path = ".devlyn/criteria.generated.md"' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1248
|
-
|| ! grep -Fq 'state.source.criteria_sha256' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1249
|
-
|| ! grep -Fq 'state.source.criteria_sha256' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1250
|
-
|| ! grep -Fq 'state.source.criteria_sha256` for generated free-form mode' config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1251
|
-
|| ! grep -Fq 'state.source.criteria_sha256` for generated free-form mode' .agents/skills/devlyn:resolve/references/phases/verify.md \
|
|
1252
|
-
|| ! grep -Fq 'Free-form sets `type: "generated"`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
1253
|
-
|| ! grep -Fq 'Free-form sets `type: "generated"`' .agents/skills/devlyn:resolve/references/state-schema.md \
|
|
1254
|
-
|| ! grep -Fq 'backticked observable command line that itself contains `miss`' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1255
|
-
|| ! grep -Fq 'backticked observable command line that itself contains `miss`' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1256
|
-
|| ! grep -Fq 'BLOCKED:solo-headroom-hypothesis-required' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1257
|
-
|| ! grep -Fq 'BLOCKED:solo-headroom-hypothesis-required' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1258
|
-
|| ! grep -Fq 'pair-evidence intent without an actionable solo-headroom hypothesis must halt' config/skills/devlyn:resolve/SKILL.md \
|
|
1259
|
-
|| ! grep -Fq 'pair-evidence intent without an actionable solo-headroom hypothesis must halt' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1260
|
-
|| ! grep -Fq 'preserve that literal hypothesis in `.devlyn/criteria.generated.md`' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1261
|
-
|| ! grep -Fq 'preserve that literal hypothesis in `.devlyn/criteria.generated.md`' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1262
|
-
|| ! grep -Fq 'emit the canonical `spec.solo_headroom_hypothesis` pair trigger reason' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1263
|
-
|| ! grep -Fq 'emit the canonical `spec.solo_headroom_hypothesis` pair trigger reason' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1264
|
-
|| ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-headroom-hypothesis-required`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
1265
|
-
|| ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-headroom-hypothesis-required`' .agents/skills/devlyn:resolve/references/state-schema.md \
|
|
1266
|
-
|| ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-headroom-hypothesis-required`' config/skills/devlyn:resolve/SKILL.md \
|
|
1267
|
-
|| ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-headroom-hypothesis-required`' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1268
|
-
|| ! grep -Fq 'Free-form goals that ask for benchmark evidence, pair-evidence, risk-probe' README.md \
|
|
1269
|
-
|| ! grep -Fq '`/devlyn:resolve` stops with `BLOCKED:solo-headroom-hypothesis-required`' README.md \
|
|
1270
|
-
|| ! grep -Fq 'points you to `/devlyn:ideate` instead of inventing a weak hypothesis' README.md; then
|
|
1271
|
-
bad "resolve free-form mode must block pair-evidence goals without actionable solo-headroom hypothesis"
|
|
1272
|
-
else
|
|
1273
|
-
ok "resolve free-form mode blocks pair-evidence goals without actionable solo-headroom hypothesis"
|
|
1274
|
-
fi
|
|
1275
|
-
if ! grep -Fq 'unmeasured_pair_candidate_intent' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1276
|
-
|| ! grep -Fq 'unmeasured_pair_candidate_intent' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1277
|
-
|| ! grep -Fq 'has_solo_ceiling_avoidance' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1278
|
-
|| ! grep -Fq 'has_solo_ceiling_avoidance' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1279
|
-
|| ! grep -Fq 'BLOCKED:solo-ceiling-avoidance-required' config/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1280
|
-
|| ! grep -Fq 'BLOCKED:solo-ceiling-avoidance-required' .agents/skills/devlyn:resolve/references/free-form-mode.md \
|
|
1281
|
-
|| ! grep -Fq 'unmeasured pair-candidate intent without solo ceiling avoidance must halt' config/skills/devlyn:resolve/SKILL.md \
|
|
1282
|
-
|| ! grep -Fq 'unmeasured pair-candidate intent without solo ceiling avoidance must halt' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1283
|
-
|| ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-ceiling-avoidance-required`' config/skills/devlyn:resolve/SKILL.md \
|
|
1284
|
-
|| ! grep -Fq '/devlyn:ideate` guidance after `BLOCKED:solo-ceiling-avoidance-required`' .agents/skills/devlyn:resolve/SKILL.md \
|
|
1285
|
-
|| ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-ceiling-avoidance-required`' config/skills/devlyn:resolve/references/state-schema.md \
|
|
1286
|
-
|| ! grep -Fq '/devlyn:ideate` guidance for `BLOCKED:solo-ceiling-avoidance-required`' .agents/skills/devlyn:resolve/references/state-schema.md \
|
|
1287
|
-
|| ! grep -Fq '`/devlyn:resolve` stops with `BLOCKED:solo-ceiling-avoidance-required`' README.md; then
|
|
1288
|
-
bad "resolve free-form mode must block new unmeasured pair candidates without solo ceiling avoidance"
|
|
1289
|
-
else
|
|
1290
|
-
ok "resolve free-form mode blocks new unmeasured pair candidates without solo ceiling avoidance"
|
|
1291
|
-
fi
|
|
1292
|
-
|
|
1293
|
-
section "Check 6g: resolve consumes sibling spec.expected.json"
|
|
1294
|
-
sibling_consume_missing=0
|
|
1295
|
-
for file in \
|
|
1296
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1297
|
-
config/skills/devlyn:resolve/references/phases/build-gate.md \
|
|
1298
|
-
config/skills/devlyn:resolve/references/phases/verify.md
|
|
1299
|
-
do
|
|
1300
|
-
if ! grep -Fq 'sibling `spec.expected.json`' "$file"; then
|
|
1301
|
-
bad "$file — missing sibling spec.expected.json consumption contract"
|
|
1302
|
-
sibling_consume_missing=1
|
|
1303
|
-
fi
|
|
1304
|
-
done
|
|
1305
|
-
for pattern in \
|
|
1306
|
-
'def stage_from_expected' \
|
|
1307
|
-
'stage_from_expected(' \
|
|
1308
|
-
'contract_found, _staged, expected_error, expected_path' \
|
|
1309
|
-
'def expected_contract_findings' \
|
|
1310
|
-
'correctness.forbidden-pattern' \
|
|
1311
|
-
'scope.max-deps-added-exceeded' \
|
|
1312
|
-
'SPEC_VERIFY_FINDINGS_FILE'
|
|
1313
|
-
do
|
|
1314
|
-
if ! grep -Fq "$pattern" config/skills/_shared/spec-verify-check.py; then
|
|
1315
|
-
bad "spec-verify-check.py missing sibling expected staging implementation: $pattern"
|
|
1316
|
-
sibling_consume_missing=1
|
|
1317
|
-
fi
|
|
1318
|
-
done
|
|
1319
|
-
if [ $sibling_consume_missing -eq 0 ]; then
|
|
1320
|
-
ok "resolve self-stages and mechanically checks sibling spec.expected.json"
|
|
1321
|
-
fi
|
|
1322
|
-
|
|
1323
|
-
section "Check 6i: VERIFY mechanical findings are merge-visible"
|
|
1324
|
-
verify_mech_missing=0
|
|
1325
|
-
for pattern in \
|
|
1326
|
-
'SPEC_VERIFY_PHASE=verify_mechanical' \
|
|
1327
|
-
'SPEC_VERIFY_FINDINGS_FILE=verify-mechanical.findings.jsonl' \
|
|
1328
|
-
'SPEC_VERIFY_FINDING_PREFIX=VERIFY-MECH'
|
|
1329
|
-
do
|
|
1330
|
-
if ! grep -Fq "$pattern" config/skills/devlyn:resolve/SKILL.md \
|
|
1331
|
-
|| ! grep -Fq "$pattern" config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1332
|
-
|| ! grep -Fq "$pattern" config/skills/_shared/spec-verify-check.py; then
|
|
1333
|
-
bad "VERIFY mechanical output contract missing: $pattern"
|
|
1334
|
-
verify_mech_missing=1
|
|
1335
|
-
fi
|
|
1336
|
-
done
|
|
1337
|
-
if ! grep -Fq '("mechanical", "verify-mechanical.findings.jsonl")' \
|
|
1338
|
-
config/skills/_shared/verify-merge-findings.py; then
|
|
1339
|
-
bad "verify-merge-findings.py does not consume verify-mechanical.findings.jsonl"
|
|
1340
|
-
verify_mech_missing=1
|
|
1341
|
-
fi
|
|
1342
|
-
if grep -Fq 'All paths emit a CRITICAL finding to' config/skills/_shared/spec-verify-check.py; then
|
|
1343
|
-
bad "spec-verify-check.py has stale single-output CRITICAL finding wording"
|
|
1344
|
-
verify_mech_missing=1
|
|
1345
|
-
fi
|
|
1346
|
-
if [ $verify_mech_missing -eq 0 ]; then
|
|
1347
|
-
ok "spec-verify VERIFY output routes into verify-merge-findings.py"
|
|
1348
|
-
fi
|
|
1349
|
-
|
|
1350
|
-
section "Check 6i1: Verification provenance contract is wired"
|
|
1351
|
-
verification_provenance_missing=0
|
|
1352
|
-
for file in \
|
|
1353
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1354
|
-
.agents/skills/devlyn:resolve/SKILL.md
|
|
1355
|
-
do
|
|
1356
|
-
if ! grep -Fq 'orchestrator commands with no separate model' "$file" \
|
|
1357
|
-
|| ! grep -Fq 'execution_kind: "orchestrator_commands"' "$file" \
|
|
1358
|
-
|| grep -Fq 'resolved BUILD_GATE engine through the fresh-worker route' "$file" \
|
|
1359
|
-
|| grep -Fq 'Codex-routed IMPLEMENT, BUILD_GATE, or CLEANUP spawn' "$file" \
|
|
1360
|
-
|| ! grep -Fq 'BLOCKED:build-env-underprovisioned' "$file" \
|
|
1361
|
-
|| ! grep -Fq '.devlyn/<primary-engine>-judge.stdout' "$file" \
|
|
1362
|
-
|| ! grep -Fq '.devlyn/<other-engine>-judge.stdout' "$file" \
|
|
1363
|
-
|| ! grep -Fq 'state-bound process-evidence manifest/raw stream' "$file"; then
|
|
1364
|
-
bad "$file — orchestrator provenance/capability/archive contract missing"
|
|
1365
|
-
verification_provenance_missing=1
|
|
1366
|
-
fi
|
|
1367
|
-
done
|
|
1368
|
-
for file in \
|
|
1369
|
-
config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1370
|
-
.agents/skills/devlyn:resolve/references/phases/verify.md
|
|
1371
|
-
do
|
|
1372
|
-
if ! grep -Fq 'executable verification belongs exclusively to MECHANICAL' "$file" \
|
|
1373
|
-
|| ! grep -Fq 'native read/search tools or non-mutating shell commands' "$file" \
|
|
1374
|
-
|| ! grep -Fq 'JUDGE does not execute literal verification, lint, test, build, risk-probe, or' "$file" \
|
|
1375
|
-
|| ! grep -Fq 'newly invented interaction commands.' "$file" \
|
|
1376
|
-
|| ! grep -Fq '.devlyn/<primary-engine>-judge.stdout' "$file" \
|
|
1377
|
-
|| ! grep -Fq '.devlyn/<other-engine>-judge.stdout' "$file" \
|
|
1378
|
-
|| ! grep -Fq 'process-evidence manifest and raw stdout/stderr streams' "$file"; then
|
|
1379
|
-
bad "$file — JUDGE sealed-evidence contract missing"
|
|
1380
|
-
verification_provenance_missing=1
|
|
1381
|
-
fi
|
|
1382
|
-
if grep -Fq 'execute at least one combined adversarial check' "$file" \
|
|
1383
|
-
|| grep -Fq 'Execute at most two targeted probes' "$file" \
|
|
1384
|
-
|| grep -Fq "repo's CLI/API/test runner" "$file"; then
|
|
1385
|
-
bad "$file — stale JUDGE command-execution instruction remains"
|
|
1386
|
-
verification_provenance_missing=1
|
|
1387
|
-
fi
|
|
1388
|
-
done
|
|
1389
|
-
for file in \
|
|
1390
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1391
|
-
.agents/skills/devlyn:resolve/SKILL.md \
|
|
1392
|
-
config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1393
|
-
.agents/skills/devlyn:resolve/references/phases/verify.md
|
|
1394
|
-
do
|
|
1395
|
-
if grep -Eiq 'executes? no commands|no code-mutation or command-execution tools|do not receive command-execution tools' "$file"; then
|
|
1396
|
-
bad "$file — blanket JUDGE inspection prohibition remains"
|
|
1397
|
-
verification_provenance_missing=1
|
|
1398
|
-
fi
|
|
1399
|
-
done
|
|
1400
|
-
if [ $verification_provenance_missing -eq 0 ]; then
|
|
1401
|
-
ok "BUILD_GATE, sealed JUDGE inputs, canonical captures, and archive flow are wired"
|
|
1402
|
-
fi
|
|
1403
|
-
|
|
1404
|
-
section "Check 6j: VERIFY default pair dispatches concurrently"
|
|
1405
|
-
pair_trigger_order_missing=0
|
|
1406
|
-
for file in \
|
|
1407
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1408
|
-
.agents/skills/devlyn:resolve/SKILL.md
|
|
1409
|
-
do
|
|
1410
|
-
if ! grep -Fq 'Verify dual-judge is default-when-available' "$file" \
|
|
1411
|
-
|| ! grep -Fq 'persist `pair_trigger` before spawn with `pair.default`' "$file" \
|
|
1412
|
-
|| ! grep -Fq 'dispatch both judges concurrently against the same frozen diff' "$file" \
|
|
1413
|
-
|| ! grep -Fq 'A primary blocker never cancels the pair judge' "$file" \
|
|
1414
|
-
|| ! grep -Fq 'run the same two required judges sequentially' "$file" \
|
|
1415
|
-
|| ! grep -Fq 'Eligible schema-v3 state must include `pair.default`' "$file" \
|
|
1416
|
-
|| ! grep -Fq '`primary_judge_blocker` is parser-recognized only for archived v2.0 replay; new runs never write it' "$file"; then
|
|
1417
|
-
bad "$file — VERIFY default pair rule missing."
|
|
1418
|
-
pair_trigger_order_missing=1
|
|
1419
|
-
fi
|
|
1420
|
-
done
|
|
1421
|
-
for file in \
|
|
1422
|
-
config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1423
|
-
.agents/skills/devlyn:resolve/references/phases/verify.md
|
|
1424
|
-
do
|
|
1425
|
-
if ! grep -Fq 'Pair-mode (default when OTHER engine is available)' "$file" \
|
|
1426
|
-
|| ! grep -Fq 'with `pair.default` plus every applicable' "$file" \
|
|
1427
|
-
|| ! grep -Fq 'dispatch the primary and pair JUDGEs concurrently against the same frozen' "$file" \
|
|
1428
|
-
|| ! grep -Fq 'A primary blocker does not cancel the pair-JUDGE' "$file" \
|
|
1429
|
-
|| ! grep -Fq 'runs the same two required judges' "$file" \
|
|
1430
|
-
|| ! grep -Fq 'Eligible schema-v3 state must contain `pair.default`' "$file" \
|
|
1431
|
-
|| ! grep -Fq 'New runs never' "$file"; then
|
|
1432
|
-
bad "$file — VERIFY default pair dispatch missing"
|
|
1433
|
-
pair_trigger_order_missing=1
|
|
1434
|
-
fi
|
|
1435
|
-
done
|
|
1436
|
-
for file in \
|
|
1437
|
-
config/skills/devlyn:resolve/references/state-schema.md \
|
|
1438
|
-
.agents/skills/devlyn:resolve/references/state-schema.md
|
|
1439
|
-
do
|
|
1440
|
-
if ! grep -Fq '"version": "3.0"' "$file" \
|
|
1441
|
-
|| ! grep -Fq 'pair.default` dispatches both judges concurrently' "$file" \
|
|
1442
|
-
|| ! grep -Fq 'runs the same two required judges sequentially' "$file" \
|
|
1443
|
-
|| ! grep -Fq 'a primary blocker never skips the pair' "$file" \
|
|
1444
|
-
|| ! grep -Fq 'Schema v3.0 eligible state requires `pair.default`' "$file" \
|
|
1445
|
-
|| ! grep -Fq 'archived v2.0 replay' "$file" \
|
|
1446
|
-
|| ! grep -Fq 'Missing, contradictory, incomplete, or unknown trigger state BLOCKs VERIFY' "$file"; then
|
|
1447
|
-
bad "$file — schema-v3 default pair contract missing"
|
|
1448
|
-
pair_trigger_order_missing=1
|
|
1449
|
-
fi
|
|
1450
|
-
done
|
|
1451
|
-
if [ $pair_trigger_order_missing -eq 0 ]; then
|
|
1452
|
-
ok "VERIFY default pair dispatches concurrently"
|
|
1453
|
-
fi
|
|
1454
|
-
|
|
1455
|
-
section "Check 6k: PLAN route is orchestrator-fixed"
|
|
1456
|
-
plan_route_ok=1
|
|
1457
|
-
for plan_route_file in "config/skills/devlyn:resolve/SKILL.md" CLAUDE.md AGENTS.md \
|
|
1458
|
-
"config/skills/devlyn:engines/SKILL.md" ".agents/skills/devlyn:resolve/SKILL.md" \
|
|
1459
|
-
".agents/skills/devlyn:engines/SKILL.md"; do
|
|
1460
|
-
[ -f "$plan_route_file" ] || plan_route_ok=0
|
|
1461
|
-
done
|
|
1462
|
-
sed -n '/^<engine_routing>$/,/^<\/engine_routing>$/p' "config/skills/devlyn:resolve/SKILL.md" \
|
|
1463
|
-
| grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine`, an executor pin, or `state.engine`' || plan_route_ok=0
|
|
1464
|
-
sed -n '/^## PHASE 1: PLAN$/,/^## PHASE 1.5/p' "config/skills/devlyn:resolve/SKILL.md" \
|
|
1465
|
-
| grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine`, an executor pin, or `state.engine`' || plan_route_ok=0
|
|
1466
|
-
grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine` or an executor pin' CLAUDE.md || plan_route_ok=0
|
|
1467
|
-
grep -Fq 'PLAN is orchestrator-fixed and never inherits `--engine` or the executor pin' AGENTS.md || plan_route_ok=0
|
|
1468
|
-
grep -Fq 'PLAN is orchestrator-fixed and never follows the pin' config/skills/devlyn:engines/SKILL.md || plan_route_ok=0
|
|
1469
|
-
if grep -En '([Ee]xecutor|Default engine: Claude for)[^[:cntrl:]]{0,80}PLAN[[:space:]]*/[[:space:]]*IMPLEMENT([[:space:]]*/[[:space:]]*BUILD_GATE)?[[:space:]]*/[[:space:]]*CLEANUP' \
|
|
1470
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1471
|
-
CLAUDE.md \
|
|
1472
|
-
AGENTS.md \
|
|
1473
|
-
README.md \
|
|
1474
|
-
config/skills/devlyn:engines/SKILL.md \
|
|
1475
|
-
.agents/skills/devlyn:resolve/SKILL.md \
|
|
1476
|
-
.agents/skills/devlyn:engines/SKILL.md \
|
|
1477
|
-
>/dev/null 2>&1; then
|
|
1478
|
-
plan_route_ok=0
|
|
1479
|
-
fi
|
|
1480
|
-
if [ $plan_route_ok -eq 1 ]; then
|
|
1481
|
-
ok "PLAN is orchestrator-fixed: exact declarations locked on all surfaces, no executor-role enumeration"
|
|
1482
|
-
else
|
|
1483
|
-
bad "PLAN route contract violated: a locked orchestrator-fixed declaration is missing or an executor-role enumeration re-teaches PLAN"
|
|
1484
|
-
fi
|
|
1485
|
-
|
|
1486
|
-
section "Check 6h: No undocumented spec.expected.json.browser_flows field"
|
|
1487
|
-
browser_flow_refs=$(grep -RInF 'spec.expected.json.browser_flows' \
|
|
1488
|
-
config/skills README.md bin/ package.json 2>/dev/null || true)
|
|
1489
|
-
if [ -z "$browser_flow_refs" ]; then
|
|
1490
|
-
ok "active docs do not advertise unsupported browser_flows schema field"
|
|
1491
|
-
else
|
|
1492
|
-
while IFS= read -r f; do bad "$f"; done <<< "$browser_flow_refs"
|
|
1493
|
-
fi
|
|
1494
|
-
|
|
1495
|
-
section "Check 6e: All-or-nothing probes prove mutable rollback"
|
|
1496
|
-
probe_doc="config/skills/devlyn:resolve/references/phases/probe-derive.md"
|
|
1497
|
-
if grep -Fq "pre-rejected by a whole-order availability shortcut" "$probe_doc" \
|
|
1498
|
-
&& grep -Fq "must allocate a scarce" "$probe_doc" \
|
|
1499
|
-
&& grep -Fq "must request the same scarce first-line SKU" "$probe_doc"; then
|
|
1500
|
-
ok "all-or-nothing probe contract preserves mutable rollback evidence"
|
|
1501
|
-
else
|
|
1502
|
-
bad "$probe_doc — missing mutable rollback probe contract"
|
|
1503
|
-
fi
|
|
1504
|
-
|
|
1505
|
-
# ---------------------------------------------------------------------------
|
|
1506
|
-
# 8. CRITIC security sub-pass must be native, not Dual.
|
|
1507
|
-
# Catches the specific drift where a section updates but a cross-reference doesn't.
|
|
1508
|
-
# ---------------------------------------------------------------------------
|
|
1509
|
-
section "Check 8: CRITIC security is native (no stale Dual references)"
|
|
1510
|
-
# Match only the concrete bad patterns used when CRITIC security was routed to
|
|
1511
|
-
# Dual (v3.4 and earlier):
|
|
1512
|
-
# 1. Markdown table cell `| **Dual** |`
|
|
1513
|
-
# 2. Prose `Dual (Claude + Codex parallel, merged)`
|
|
1514
|
-
# Retrospective mentions like "drops the Dual-model token cost" are fine.
|
|
1515
|
-
offenders=$(grep -RInE '\|\s*\*\*Dual\*\*\s*\||Dual\s*\(Claude\s*\+\s*Codex' \
|
|
1516
|
-
config/skills 2>/dev/null \
|
|
1517
|
-
| grep -v 'roadmap-archival-workspace/' \
|
|
1518
|
-
| grep -v 'devlyn:auto-resolve-workspace/' \
|
|
1519
|
-
| grep -v 'devlyn:ideate-workspace/' \
|
|
1520
|
-
| grep -v 'preflight-workspace/' \
|
|
1521
|
-
|| true)
|
|
1522
|
-
if [ -z "$offenders" ]; then
|
|
1523
|
-
ok "CRITIC security uses native (no Dual stragglers)"
|
|
1524
|
-
else
|
|
1525
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
1526
|
-
fi
|
|
1527
|
-
|
|
1528
|
-
# ---------------------------------------------------------------------------
|
|
1529
|
-
# 10. No raw `codex exec` invocation in skill prompts (iter-0010).
|
|
1530
|
-
# iter-0009 wrapper + iter-0010 production rollout require every Codex
|
|
1531
|
-
# invocation in skill SKILL.md / references to use codex-monitored.sh.
|
|
1532
|
-
# Raw `codex exec ...` in a prompt re-introduces the iter-0008 byte-watchdog
|
|
1533
|
-
# starvation: orchestrator pattern-primes from the doc and emits the raw
|
|
1534
|
-
# shape, which can collapse into `... | tail -200` and starve the outer API
|
|
1535
|
-
# stream. Descriptive phrases like "passes args through to `codex exec`
|
|
1536
|
-
# verbatim" are allowed — only invocation-shaped uses are forbidden.
|
|
1537
|
-
#
|
|
1538
|
-
# Pattern: `codex exec[[:space:]]+\S` — catches any invocation shape
|
|
1539
|
-
# (whitespace then a non-space character after `exec`). Passes backtick-
|
|
1540
|
-
# closed descriptive prose like `` `codex exec` `` because the closing
|
|
1541
|
-
# backtick is non-whitespace adjacent to `exec`, not whitespace.
|
|
1542
|
-
# Concrete shapes caught:
|
|
1543
|
-
# - single-line flag: `codex exec -C ...`
|
|
1544
|
-
# - resume form: `codex exec resume --last`
|
|
1545
|
-
# - multi-line cont.: `codex exec \` (space + `\` at EOL)
|
|
1546
|
-
# - quoted prompt: `codex exec "prompt"` ← iter-0011
|
|
1547
|
-
# - variable expansion: `codex exec $PROMPT` ← iter-0011
|
|
1548
|
-
# - literal token: `codex exec prompt` ← iter-0011
|
|
1549
|
-
# Excludes: _shared/codex-config.md (canonical doc may discuss the rule
|
|
1550
|
-
# itself), workspace/, archive snapshots.
|
|
1551
|
-
# ---------------------------------------------------------------------------
|
|
1552
|
-
section "Check 10: No raw codex exec invocation in skill prompts"
|
|
1553
|
-
offenders=$(grep -RInE 'codex exec[[:space:]]+[^[:space:]]' \
|
|
1554
|
-
config/skills 2>/dev/null \
|
|
1555
|
-
| grep -v 'config/skills/_shared/codex-config.md' \
|
|
1556
|
-
| grep -v 'config/skills/_shared/codex-monitored.sh' \
|
|
1557
|
-
| grep -v 'roadmap-archival-workspace/' \
|
|
1558
|
-
| grep -v 'devlyn:auto-resolve-workspace/' \
|
|
1559
|
-
| grep -v 'devlyn:ideate-workspace/' \
|
|
1560
|
-
| grep -v 'preflight-workspace/' \
|
|
1561
|
-
|| true)
|
|
1562
|
-
if [ -z "$offenders" ]; then
|
|
1563
|
-
ok "no raw codex exec invocations in skill prompts (wrapper-form everywhere)"
|
|
1564
|
-
else
|
|
1565
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
1566
|
-
fi
|
|
1567
|
-
|
|
1568
|
-
# ---------------------------------------------------------------------------
|
|
1569
|
-
# 10a. Bounded Codex probe/judge calls must run isolated.
|
|
1570
|
-
# Pair/risk-probe paths are measured read-only judges, not implementation
|
|
1571
|
-
# phases. They must not inherit user config, AGENTS.md, hooks, or other
|
|
1572
|
-
# local rules that can add hidden context or transcript side effects.
|
|
1573
|
-
# The wrapper owns the flag expansion; skill docs own requiring
|
|
1574
|
-
# CODEX_MONITORED_ISOLATED=1 for probe-derive and pair-JUDGE.
|
|
1575
|
-
# ---------------------------------------------------------------------------
|
|
1576
|
-
section "Check 10a0: Codex primary VERIFY budget is bounded and mirrored"
|
|
1577
|
-
primary_verify_missing=0
|
|
1578
|
-
primary_verify_route='DEVLYN_CODEX_PROMPT_FILE="<primary-prompt-file>" CODEX_MONITORED_ISOLATED=1 CODEX_MONITORED_TIMEOUT_SEC=600 bash "$CODEX_MONITORED_PATH" -C "$PWD" -s read-only -c model_reasoning_effort=high - >.devlyn/codex-judge.stdout 2>.devlyn/codex-judge.stderr'
|
|
1579
|
-
for file in \
|
|
1580
|
-
config/skills/devlyn:resolve/SKILL.md \
|
|
1581
|
-
.agents/skills/devlyn:resolve/SKILL.md \
|
|
1582
|
-
config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1583
|
-
.agents/skills/devlyn:resolve/references/phases/verify.md
|
|
1584
|
-
do
|
|
1585
|
-
if { ! grep -Fxq "$primary_verify_route" "$file" \
|
|
1586
|
-
&& ! grep -Fq "\`$primary_verify_route\`" "$file"; } \
|
|
1587
|
-
|| ! grep -Fq '.devlyn/verify.primary.timeout.json' "$file" \
|
|
1588
|
-
|| ! grep -Fq 'pair-style `TIMEOUT`' "$file"; then
|
|
1589
|
-
bad "$file — Codex primary-JUDGE route/timeout authority missing"
|
|
1590
|
-
primary_verify_missing=1
|
|
1591
|
-
fi
|
|
1592
|
-
done
|
|
1593
|
-
for file in \
|
|
1594
|
-
config/skills/devlyn:resolve/references/phases/verify.md \
|
|
1595
|
-
.agents/skills/devlyn:resolve/references/phases/verify.md
|
|
1596
|
-
do
|
|
1597
|
-
for needle in \
|
|
1598
|
-
'one broad pass over the' \
|
|
1599
|
-
'It then makes one' \
|
|
1600
|
-
'targeted interaction pass over the clauses the broad pass left unresolved.' \
|
|
1601
|
-
'Before any third pass' \
|
|
1602
|
-
'verdict-binding BLOCKED coverage finding'
|
|
1603
|
-
do
|
|
1604
|
-
if ! grep -Fq "$needle" "$file"; then
|
|
1605
|
-
bad "$file — bounded primary review instruction missing: $needle"
|
|
1606
|
-
primary_verify_missing=1
|
|
1607
|
-
fi
|
|
1608
|
-
done
|
|
1609
|
-
done
|
|
1610
|
-
for file in \
|
|
1611
|
-
config/skills/_shared/adapters/codex.md \
|
|
1612
|
-
.agents/skills/_shared/adapters/codex.md
|
|
1613
|
-
do
|
|
1614
|
-
for needle in \
|
|
1615
|
-
'batch related reads' \
|
|
1616
|
-
'sealed parity proves' \
|
|
1617
|
-
'Open self-test bodies or MECHANICAL raw streams only for a'
|
|
1618
|
-
do
|
|
1619
|
-
if ! grep -Fq "$needle" "$file"; then
|
|
1620
|
-
bad "$file — Codex bounded retrieval tactic missing: $needle"
|
|
1621
|
-
primary_verify_missing=1
|
|
1622
|
-
fi
|
|
1623
|
-
done
|
|
1624
|
-
done
|
|
1625
|
-
for file in \
|
|
1626
|
-
config/skills/_shared/verify-merge-findings.py \
|
|
1627
|
-
.agents/skills/_shared/verify-merge-findings.py \
|
|
1628
|
-
config/skills/_shared/archive_run.py \
|
|
1629
|
-
.agents/skills/_shared/archive_run.py \
|
|
1630
|
-
config/skills/_shared/resolve-bootstrap.py \
|
|
1631
|
-
.agents/skills/_shared/resolve-bootstrap.py
|
|
1632
|
-
do
|
|
1633
|
-
if ! grep -Fq 'verify.primary.timeout.json' "$file"; then
|
|
1634
|
-
bad "$file — primary timeout deterministic/archive contract missing"
|
|
1635
|
-
primary_verify_missing=1
|
|
1636
|
-
fi
|
|
1637
|
-
done
|
|
1638
|
-
if [ $primary_verify_missing -eq 0 ]; then
|
|
1639
|
-
ok "Codex primary VERIFY is bounded, fail-closed, archived, and mirrored"
|
|
1640
|
-
fi
|
|
1641
|
-
|
|
1642
|
-
section "Check 10a: Bounded Codex calls use isolated wrapper mode"
|
|
1643
|
-
isolation_missing=0
|
|
1644
|
-
for needle in \
|
|
1645
|
-
'CODEX_MONITORED_ISOLATED=1 bash "$CODEX_MONITORED_PATH"' \
|
|
1646
|
-
'CODEX_MONITORED_ISOLATED=1 CODEX_MONITORED_TIMEOUT_SEC=600` and `-c model_reasoning_effort=medium' \
|
|
1647
|
-
'CODEX_MONITORED_PATH="$DEVLYN_SHARED_DIR/codex-monitored.sh"'
|
|
1648
|
-
do
|
|
1649
|
-
if ! grep -RInF "$needle" config/skills >/dev/null 2>&1; then
|
|
1650
|
-
bad "missing isolated Codex invocation contract: $needle"
|
|
1651
|
-
isolation_missing=1
|
|
1652
|
-
fi
|
|
1653
|
-
done
|
|
1654
|
-
for flag in \
|
|
1655
|
-
'--ignore-user-config' \
|
|
1656
|
-
'--ignore-rules' \
|
|
1657
|
-
'--ephemeral' \
|
|
1658
|
-
'--disable codex_hooks' \
|
|
1659
|
-
'--disable hooks'
|
|
1660
|
-
do
|
|
1661
|
-
if ! grep -F -- "$flag" config/skills/_shared/codex-monitored.sh >/dev/null 2>&1; then
|
|
1662
|
-
bad "codex-monitored.sh missing isolation flag expansion: $flag"
|
|
1663
|
-
isolation_missing=1
|
|
1664
|
-
fi
|
|
1665
|
-
done
|
|
1666
|
-
for helper in require_positive_int require_nonnegative_int; do
|
|
1667
|
-
if ! grep -F "$helper" config/skills/_shared/codex-monitored.sh >/dev/null 2>&1; then
|
|
1668
|
-
bad "codex-monitored.sh missing numeric env validator: $helper"
|
|
1669
|
-
isolation_missing=1
|
|
1670
|
-
fi
|
|
1671
|
-
done
|
|
1672
|
-
|
|
1673
|
-
if make_temp_dir tmp_env /tmp/codex-monitored-env.XXXXXX; then
|
|
1674
|
-
if CODEX_MONITORED_HEARTBEAT=0 CODEX_BIN=/bin/true \
|
|
1675
|
-
bash config/skills/_shared/codex-monitored.sh prompt \
|
|
1676
|
-
>"$tmp_env/heartbeat.stdout" 2>"$tmp_env/heartbeat.stderr"; then
|
|
1677
|
-
bad "codex-monitored.sh accepted CODEX_MONITORED_HEARTBEAT=0"
|
|
1678
|
-
isolation_missing=1
|
|
1679
|
-
elif ! grep -F 'CODEX_MONITORED_HEARTBEAT must be > 0' "$tmp_env/heartbeat.stderr" >/dev/null 2>&1; then
|
|
1680
|
-
bad "codex-monitored.sh heartbeat validation emitted wrong error"
|
|
1681
|
-
isolation_missing=1
|
|
1682
|
-
fi
|
|
1683
|
-
if CODEX_MONITORED_TIMEOUT_SEC=abc CODEX_BIN=/bin/true \
|
|
1684
|
-
bash config/skills/_shared/codex-monitored.sh prompt \
|
|
1685
|
-
>"$tmp_env/timeout.stdout" 2>"$tmp_env/timeout.stderr"; then
|
|
1686
|
-
bad "codex-monitored.sh accepted non-numeric CODEX_MONITORED_TIMEOUT_SEC"
|
|
1687
|
-
isolation_missing=1
|
|
1688
|
-
elif ! grep -F 'CODEX_MONITORED_TIMEOUT_SEC must be a non-negative integer' "$tmp_env/timeout.stderr" >/dev/null 2>&1; then
|
|
1689
|
-
bad "codex-monitored.sh timeout validation emitted wrong error"
|
|
1690
|
-
isolation_missing=1
|
|
1691
|
-
fi
|
|
1692
|
-
rm -rf "$tmp_env"
|
|
1693
|
-
else
|
|
1694
|
-
isolation_missing=1
|
|
1695
|
-
fi
|
|
1696
|
-
|
|
1697
|
-
if make_temp_dir tmp_iso /tmp/codex-monitored-isolated.XXXXXX; then
|
|
1698
|
-
cat > "$tmp_iso/codex" <<'EOF'
|
|
1699
|
-
#!/usr/bin/env bash
|
|
1700
|
-
printf '%s\n' "$@" > "$CODEX_FAKE_ARGS_OUT"
|
|
1701
|
-
EOF
|
|
1702
|
-
chmod +x "$tmp_iso/codex"
|
|
1703
|
-
(
|
|
1704
|
-
unset DEVLYN_INVOCATION_RUN_ID DEVLYN_INVOCATION_PHASE \
|
|
1705
|
-
DEVLYN_INVOCATION_ROUND DEVLYN_INVOCATION_WORKDIR \
|
|
1706
|
-
DEVLYN_INVOCATION_PROMPT_FILE DEVLYN_INVOCATION_SESSION_FILE \
|
|
1707
|
-
DEVLYN_INVOCATION_RECEIPT
|
|
1708
|
-
CODEX_FAKE_ARGS_OUT="$tmp_iso/args.txt" \
|
|
1709
|
-
CODEX_MONITORED_ISOLATED=1 \
|
|
1710
|
-
CODEX_MONITORED_HEARTBEAT=999 \
|
|
1711
|
-
CODEX_BIN="$tmp_iso/codex" \
|
|
1712
|
-
bash config/skills/_shared/codex-monitored.sh -s read-only prompt \
|
|
1713
|
-
>"$tmp_iso/stdout.txt" 2>"$tmp_iso/stderr.txt"
|
|
1714
|
-
)
|
|
1715
|
-
iso_exit=$?
|
|
1716
|
-
if [ $iso_exit -ne 0 ]; then
|
|
1717
|
-
bad "codex-monitored.sh isolated fake invocation exited $iso_exit"
|
|
1718
|
-
isolation_missing=1
|
|
1719
|
-
else
|
|
1720
|
-
for expected in exec --ignore-user-config --ignore-rules --ephemeral --disable codex_hooks --disable hooks -s read-only prompt; do
|
|
1721
|
-
if ! grep -Fx -- "$expected" "$tmp_iso/args.txt" >/dev/null 2>&1; then
|
|
1722
|
-
bad "codex-monitored.sh isolated fake invocation missing arg: $expected"
|
|
1723
|
-
isolation_missing=1
|
|
1724
|
-
fi
|
|
1725
|
-
done
|
|
1726
|
-
if ! grep -F '[codex-monitored] isolated=1' "$tmp_iso/stderr.txt" >/dev/null 2>&1; then
|
|
1727
|
-
bad "codex-monitored.sh isolated fake invocation missing lifecycle marker"
|
|
1728
|
-
isolation_missing=1
|
|
1729
|
-
fi
|
|
1730
|
-
fi
|
|
1731
|
-
rm -rf "$tmp_iso"
|
|
1732
|
-
else
|
|
1733
|
-
isolation_missing=1
|
|
1734
|
-
fi
|
|
1735
|
-
if [ $isolation_missing -eq 0 ]; then
|
|
1736
|
-
ok "bounded Codex probe/judge calls require isolated wrapper mode"
|
|
1737
|
-
fi
|
|
1738
|
-
|
|
1739
|
-
section "Check 10a1: Shared helper paths resolve from the invoked skill"
|
|
1740
|
-
shared_path_offenders=$(grep -RInF '.claude/skills/_shared' config/skills 2>/dev/null \
|
|
1741
|
-
| grep -v 'roadmap-archival-workspace/' \
|
|
1742
|
-
| grep -v 'devlyn:auto-resolve-workspace/' \
|
|
1743
|
-
| grep -v 'devlyn:ideate-workspace/' \
|
|
1744
|
-
| grep -v 'preflight-workspace/' \
|
|
1745
|
-
|| true)
|
|
1746
|
-
if [ -z "$shared_path_offenders" ]; then
|
|
1747
|
-
ok "no project-relative .claude/skills/_shared paths in managed skills"
|
|
1748
|
-
else
|
|
1749
|
-
while IFS= read -r f; do bad "$f"; done <<< "$shared_path_offenders"
|
|
1750
|
-
fi
|
|
1751
|
-
|
|
1752
|
-
section "Check 10a2: Bounded non-interactive calls discard inherited input"
|
|
1753
|
-
if python3 - <<'PY'
|
|
1754
|
-
import json, subprocess, sys
|
|
1755
|
-
|
|
1756
|
-
prompt = "sealed argv prompt\n두 번째 줄"
|
|
1757
|
-
poison = b"OPERATOR_HEREDOC_SHOULD_NOT_BECOME_USER_INPUT\n"
|
|
1758
|
-
child = (
|
|
1759
|
-
"import json,sys,time; "
|
|
1760
|
-
"print(json.dumps({'argv':sys.argv[1:],'stdin_hex':sys.stdin.buffer.read().hex()},ensure_ascii=False),flush=True); "
|
|
1761
|
-
"print('stderr preserved',file=sys.stderr,flush=True); "
|
|
1762
|
-
"time.sleep(float(sys.argv[2])); raise SystemExit(int(sys.argv[3]))"
|
|
1763
|
-
)
|
|
1764
|
-
for label, delay, code, expected_exit in (("normal", 0, 0, 0), ("nonzero", 0, 17, 17), ("timeout", 30, 0, 124)):
|
|
1765
|
-
args = [prompt, str(delay), str(code)]
|
|
1766
|
-
result = subprocess.run(
|
|
1767
|
-
[sys.executable, "config/skills/_shared/run-bounded.py", "1", "--", sys.executable, "-c", child, *args],
|
|
1768
|
-
input=poison, capture_output=True, timeout=9,
|
|
1769
|
-
)
|
|
1770
|
-
assert result.returncode == expected_exit, (label, result.returncode)
|
|
1771
|
-
assert json.loads(result.stdout) == {"argv": args, "stdin_hex": ""}, (label, result.stdout)
|
|
1772
|
-
assert result.stderr == b"stderr preserved\n", (label, result.stderr)
|
|
1773
|
-
PY
|
|
1774
|
-
then
|
|
1775
|
-
ok "run-bounded.py closes child stdin and preserves argv, streams, exits and timeout"
|
|
1776
|
-
else
|
|
1777
|
-
bad "run-bounded.py inherited-input/exit/timeout regression failed"
|
|
1778
|
-
fi
|
|
1779
|
-
|
|
1780
|
-
# ---------------------------------------------------------------------------
|
|
1781
|
-
# 10b. Shared routing docs must describe the current 2-skill surface.
|
|
1782
|
-
# A stale auto-resolve/preflight/ideate-CHALLENGE reference can misroute
|
|
1783
|
-
# bounded pair work back into unisolated or retired Codex paths.
|
|
1784
|
-
# ---------------------------------------------------------------------------
|
|
1785
|
-
section "Check 10b: Shared routing docs avoid retired skill surfaces"
|
|
1786
|
-
offenders=$(grep -RInE 'auto-resolve/SKILL\.md|preflight/SKILL\.md|challenge-rubric\.md|ideate CHALLENGE phase|does NOT consume this file|cross-model challenge phases when configured|phase-1-build\.md|phase-2-evaluate\.md|phase-3-critic\.md' \
|
|
1787
|
-
config/skills/_shared 2>/dev/null || true)
|
|
1788
|
-
if [ -z "$offenders" ]; then
|
|
1789
|
-
ok "shared routing docs reference current devlyn:ideate/devlyn:resolve surface"
|
|
1790
|
-
else
|
|
1791
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
1792
|
-
fi
|
|
1793
|
-
|
|
1794
|
-
# ---------------------------------------------------------------------------
|
|
1795
|
-
# 10c. User-facing current docs must not advertise retired skills.
|
|
1796
|
-
# Historical archive mentions are allowed, but install/package copy and
|
|
1797
|
-
# active helper scripts should describe ideate -> resolve, not
|
|
1798
|
-
# ideate -> auto-resolve -> preflight or ideate CHALLENGE.
|
|
1799
|
-
# ---------------------------------------------------------------------------
|
|
1800
|
-
section "Check 10c: User-facing current docs avoid retired skill surfaces"
|
|
1801
|
-
offenders=$(
|
|
1802
|
-
{
|
|
1803
|
-
sed '/<!-- legacy-surface-map:begin/,/<!-- legacy-surface-map:end/s/.*//' README.md 2>/dev/null \
|
|
1804
|
-
| grep -nE '/devlyn:auto-resolve|ideate CHALLENGE|--with-codex|Quick Start pointing to ideate → auto-resolve → preflight|auto-resolve'\''s build agent|Core pipeline skills \(`ideate`, `auto-resolve`, `preflight`\)' || true
|
|
1805
|
-
grep -nE '"description": .*auto-resolve|"description": .*preflight' package.json 2>/dev/null || true
|
|
1806
|
-
grep -nE 'so auto-resolve doesn'\''t prompt' bin/devlyn.js 2>/dev/null || true
|
|
1807
|
-
grep -nE 'devlyn:auto-resolve|phase-1-build\.md|phase-2-evaluate\.md|phase-3-critic\.md' scripts/static-ab.sh 2>/dev/null || true
|
|
1808
|
-
} | sed -E 's#^#user-facing retired-surface reference: #'
|
|
1809
|
-
)
|
|
1810
|
-
if [ -z "$offenders" ]; then
|
|
1811
|
-
ok "README/package/helper copy describes current ideate -> resolve surface"
|
|
1812
|
-
else
|
|
1813
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
1814
|
-
fi
|
|
1815
|
-
|
|
1816
|
-
# ---------------------------------------------------------------------------
|
|
1817
|
-
# 10d. Prompt adapters cite the current official model guidance.
|
|
1818
|
-
# Prompt edits must be model-specific where tactics differ: OpenAI guidance
|
|
1819
|
-
# for GPT/Codex, Anthropic guidance for Claude. This check keeps that
|
|
1820
|
-
# contract from becoming oral tradition.
|
|
1821
|
-
# ---------------------------------------------------------------------------
|
|
1822
|
-
section "Check 10d: Prompt adapters cite official model guidance"
|
|
1823
|
-
adapter_missing=0
|
|
1824
|
-
if ! grep -Fq 'https://developers.openai.com/api/docs/guides/prompt-guidance?model=gpt-5.5' \
|
|
1825
|
-
config/skills/_shared/adapters/codex.md; then
|
|
1826
|
-
bad "codex adapter missing official OpenAI prompt guidance URL"
|
|
1827
|
-
adapter_missing=1
|
|
1828
|
-
fi
|
|
1829
|
-
if ! grep -Fq 'https://platform.claude.com/docs/en/build-with-claude/prompt-engineering/claude-prompting-best-practices' \
|
|
1830
|
-
config/skills/_shared/adapters/claude.md; then
|
|
1831
|
-
bad "claude adapter missing official Claude prompting best-practices URL"
|
|
1832
|
-
adapter_missing=1
|
|
1833
|
-
fi
|
|
1834
|
-
for pattern in 'Use Markdown only where it carries structure' 'metaprompter for itself'; do
|
|
1835
|
-
if ! grep -Fq "$pattern" config/skills/_shared/adapters/codex.md; then
|
|
1836
|
-
bad "codex adapter missing official-guidance tactic: $pattern"
|
|
1837
|
-
adapter_missing=1
|
|
1838
|
-
fi
|
|
1839
|
-
done
|
|
1840
|
-
for pattern in 'high` or `xhigh` effort' 'report every issue you find' 'do not filter for importance or confidence' 'prefer concise positive examples' '<example>'; do
|
|
1841
|
-
if ! grep -Fq "$pattern" config/skills/_shared/adapters/claude.md; then
|
|
1842
|
-
bad "claude adapter missing official-guidance tactic: $pattern"
|
|
1843
|
-
adapter_missing=1
|
|
1844
|
-
fi
|
|
1845
|
-
done
|
|
1846
|
-
for file in config/skills/devlyn:resolve/SKILL.md config/skills/devlyn:ideate/SKILL.md; do
|
|
1847
|
-
if ! grep -Fq '_shared/adapters/<engine>.md' "$file"; then
|
|
1848
|
-
bad "$file — missing per-engine adapter injection contract"
|
|
1849
|
-
adapter_missing=1
|
|
1850
|
-
fi
|
|
1851
|
-
done
|
|
1852
|
-
if [ $adapter_missing -eq 0 ]; then
|
|
1853
|
-
ok "adapters cite official GPT/Claude guidance, carry model-specific tactics, and both skills inject them"
|
|
1854
|
-
fi
|
|
1855
|
-
|
|
1856
|
-
# ---------------------------------------------------------------------------
|
|
1857
|
-
# 10d1. Opus sidecar must fail closed on score-source mapping before provider calls.
|
|
1858
|
-
# ---------------------------------------------------------------------------
|
|
1859
|
-
section "Check 10d1: Opus judge sidecar validates blind mapping before provider calls"
|
|
1860
|
-
if ! grep -Fq 'judge blind mapping missing arm(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1861
|
-
|| ! grep -Fq 'scores_by_arm without blind mapping' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1862
|
-
|| ! grep -Fq 'scores_by_arm malformed score(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1863
|
-
|| ! grep -Fq 'gpt judge.json _blind_mapping must be an object' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1864
|
-
|| ! grep -Fq 'def is_score(value):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1865
|
-
|| ! grep -Fq 'not isinstance(value, bool) and 0 <= value <= 100' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1866
|
-
|| ! grep -Fq 'invalid opus score value(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1867
|
-
|| ! grep -Fq 'invalid opus disqualifier value(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1868
|
-
|| ! grep -Fq 'opus-invalid-generated-dq' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1869
|
-
|| ! grep -Fq 'def blind_mapping(j):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1870
|
-
|| ! grep -Fq 'def mapped_arm_set(j):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1871
|
-
|| ! grep -Fq '{"solo_claude", "bare"}.issubset(mapped_arms)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1872
|
-
|| ! grep -Fq 'def mapped_scores(j):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1873
|
-
|| ! grep -Fq 'if arm in mapped_arms and is_score(score)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1874
|
-
|| ! grep -Fq 'def margin_from_scores(scores, left, right):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1875
|
-
|| ! grep -Fq 'def mapped_winner(j, scores):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1876
|
-
|| ! grep -Fq 'def fmt_metric(value):' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1877
|
-
|| ! grep -Fq 'g_l1_l0 = margin_from_scores(g_scores, "solo_claude", "bare")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1878
|
-
|| ! grep -Fq 'g_v_l0 = margin_from_scores(g_scores, "variant", "bare")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1879
|
-
|| ! grep -Fq 'g_winner = mapped_winner(g, g_scores)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1880
|
-
|| ! grep -Fq '"winner_agree": g_winner is not None and o_winner is not None and g_winner == o_winner' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1881
|
-
|| ! grep -Fq '"gpt_scores": g_scores' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1882
|
-
|| ! grep -Fq '"opus_scores": o_scores' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1883
|
-
|| ! grep -Fq 'opus-bad-mapping' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1884
|
-
|| ! grep -Fq 'opus-malformed-mapping' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1885
|
-
|| ! grep -Fq 'opus-malformed-score' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1886
|
-
|| ! grep -Fq 'opus-invalid-generated-score' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1887
|
-
|| ! grep -Fq 'arg-parse-opus-summary-mapping' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1888
|
-
|| ! grep -Fq 'arg-parse-opus-summary-null-margin' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1889
|
-
|| ! grep -Fq 'assert row["gpt_margin_l1_l0"] == 10' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1890
|
-
|| ! grep -Fq 'assert row["gpt_winner"] is None' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1891
|
-
|| ! grep -Fq "gpt_l1_l0_avg=na" benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
1892
|
-
|| ! grep -Fq 'solo_claude-bare={chosen' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1893
|
-
|| ! grep -Fq 'variant-solo_claude={chosen' benchmark/auto-resolve/scripts/judge-opus-pass.sh; then
|
|
1894
|
-
offenders="${offenders}"$'\n'"judge-opus-pass.sh must validate _blind_mapping locally before invoking Claude"
|
|
1895
|
-
fi
|
|
1896
|
-
|
|
1897
|
-
# ---------------------------------------------------------------------------
|
|
1898
|
-
# 10e. Benchmark docs must describe the current solo-vs-pair topology.
|
|
1899
|
-
# Pair evidence work depends on the bare/solo_claude/pair contract being current:
|
|
1900
|
-
# bare, solo_claude, and the selected pair arm are measured today.
|
|
1901
|
-
# ---------------------------------------------------------------------------
|
|
1902
|
-
section "Check 10e: Benchmark docs describe current 3-arm pair topology"
|
|
1903
|
-
offenders=$(grep -RInE 'L1 .*queued|cannot directly verify the L1|auto-resolve → preflight|ideate → auto-resolve|all fixtures × 2 arms|9 fixtures × 2 arms|9 fixtures × 3 arms|≥ 7 of 9|7/9 fixtures|variant` / `bare|variant/\{input|preflight; bare|Audited by preflight|Variant'\''s CRITIC|future enhancement|release-blocker|Today the suite runs `variant`|/devlyn:auto-resolve|auto-resolve-ready|REAL auto-resolve|real auto-resolve|auto-resolve run|auto-resolve runs|Claude/auto-resolve|variant − bare|Margin \(variant|Ship thresholds use margin \(variant|both arms improve together|both arms|benchmark --n 3|run-suite\.sh --n 3|higher confidence for ship decisions|3 runs per fixture for ship decisions|One-command A/B benchmark|A/B randomized|A/B benchmark suite vs bare|not vs bare — bare is the opponent' \
|
|
1904
|
-
benchmark/auto-resolve/BENCHMARK-DESIGN.md \
|
|
1905
|
-
benchmark/auto-resolve/README.md \
|
|
1906
|
-
benchmark/auto-resolve/RUBRIC.md \
|
|
1907
|
-
benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
1908
|
-
benchmark/auto-resolve/run-real-benchmark.md \
|
|
1909
|
-
benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
1910
|
-
benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
1911
|
-
2>/dev/null || true)
|
|
1912
|
-
fixture_note_stale=$(git grep -InE -- 'both arms|both solo and pair arms|bare or solo consistently reaches ceiling' -- \
|
|
1913
|
-
'benchmark/auto-resolve/fixtures/**/NOTES.md' \
|
|
1914
|
-
':!benchmark/auto-resolve/fixtures/retired/**' \
|
|
1915
|
-
2>/dev/null || true)
|
|
1916
|
-
if [ -n "$fixture_note_stale" ]; then
|
|
1917
|
-
offenders="${offenders}"$'\n'"fixture NOTES must name bare and solo_claude explicitly instead of ambiguous arm wording:"$'\n'"$fixture_note_stale"
|
|
1918
|
-
fi
|
|
1919
|
-
shadow_note_stale=$(git grep -In -- 'solo headroom' -- \
|
|
1920
|
-
'benchmark/auto-resolve/shadow-fixtures/**/NOTES.md' \
|
|
1921
|
-
2>/dev/null || true)
|
|
1922
|
-
if [ -n "$shadow_note_stale" ]; then
|
|
1923
|
-
offenders="${offenders}"$'\n'"shadow fixture NOTES must name solo_claude headroom explicitly:"$'\n'"$shadow_note_stale"
|
|
1924
|
-
fi
|
|
1925
|
-
active_doc_stale_solo_scores=$(git grep -InE -- '(^|[^[:alnum:]_])solo [0-9]+|/ solo$|, solo [0-9]+|vs solo [0-9]+' -- \
|
|
1926
|
-
benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
1927
|
-
benchmark/auto-resolve/README.md \
|
|
1928
|
-
benchmark/auto-resolve/run-real-benchmark.md \
|
|
1929
|
-
2>/dev/null || true)
|
|
1930
|
-
if [ -n "$active_doc_stale_solo_scores" ]; then
|
|
1931
|
-
offenders="${offenders}"$'\n'"active benchmark score evidence must name solo_claude instead of shorthand solo:"$'\n'"$active_doc_stale_solo_scores"
|
|
1932
|
-
fi
|
|
1933
|
-
active_stale_margin_labels=$(git grep -InE -- 'l1-l0=|v-l1=|L1-L0 disagreement|Per-axis L1-L0|Suite-level per-axis L1-L0|Suite avg L1-L0|L2-L1 margin' -- \
|
|
1934
|
-
benchmark/auto-resolve/BENCHMARK-DESIGN.md \
|
|
1935
|
-
benchmark/auto-resolve/README.md \
|
|
1936
|
-
benchmark/auto-resolve/RUBRIC.md \
|
|
1937
|
-
benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
1938
|
-
2>/dev/null || true)
|
|
1939
|
-
if [ -n "$active_stale_margin_labels" ]; then
|
|
1940
|
-
offenders="${offenders}"$'\n'"active benchmark docs/stdout must use key-aligned margin labels:"$'\n'"$active_stale_margin_labels"
|
|
1941
|
-
fi
|
|
1942
|
-
if grep -RInE 'gate: ≥ 7 of 9|Hard floor 3: ≥ 7 of 9|7-of-9 L1 floor|all 9 fixtures produced' \
|
|
1943
|
-
benchmark/auto-resolve/scripts/compile-report.py \
|
|
1944
|
-
benchmark/auto-resolve/scripts/ship-gate.py \
|
|
1945
|
-
benchmark/auto-resolve/scripts/judge-opus-pass.sh >/dev/null 2>&1; then
|
|
1946
|
-
offenders="${offenders}"$'\n'"benchmark scripts must describe the current extended fixture count as an explicit selected/gated set, not stale 9-fixture wording"
|
|
1947
|
-
fi
|
|
1948
|
-
if grep -Fq 'PAIR_ARM="l2_gated"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
|
|
1949
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: default pair arm must stay on current measured l2_risk_probes path"
|
|
1950
|
-
fi
|
|
1951
|
-
if ! grep -Fq 'for required in result.json verify.json diff.patch; do' \
|
|
1952
|
-
benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
1953
|
-
|| ! grep -Fq 'reuse source missing $required' \
|
|
1954
|
-
benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
|
|
1955
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: calibrated-arm reuse must fail closed on missing result.json, verify.json, and diff.patch"
|
|
1956
|
-
fi
|
|
1957
|
-
if ! grep -Fq 'reuse destination incomplete $required' \
|
|
1958
|
-
benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
1959
|
-
|| ! grep -Fq 'reuse destination is not a directory' \
|
|
1960
|
-
benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
|
|
1961
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: calibrated-arm reuse must fail closed on incomplete existing destination"
|
|
1962
|
-
fi
|
|
1963
|
-
if ! grep -Fq 'diff.patch missing' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
1964
|
-
|| ! grep -Fq 'diff.patch missing' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
|
|
1965
|
-
offenders="${offenders}"$'\n'"benchmark pair gates must require diff.patch artifacts for measured arms"
|
|
1966
|
-
fi
|
|
1967
|
-
if ! grep -Fq 'pair-arm must be l2_risk_probes or l2_gated' \
|
|
1968
|
-
benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
1969
|
-
|| ! grep -Fq 'pair-arm l2_forced is retired' \
|
|
1970
|
-
benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
|
|
1971
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: pair arm selection must fail closed before fixture execution"
|
|
1972
|
-
fi
|
|
1973
|
-
if ! grep -Fq 'l2_risk_probes|l2_gated) ;;' \
|
|
1974
|
-
benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
|
|
1975
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh: runner allowlist must remain l2_risk_probes|l2_gated only"
|
|
1976
|
-
fi
|
|
1977
|
-
if grep -Fq 'default="l2_gated"' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
|
|
1978
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py: --pair-arm default must stay on current measured l2_risk_probes path"
|
|
1979
|
-
fi
|
|
1980
|
-
if ! grep -Fq 'from pair_evidence_contract import (' \
|
|
1981
|
-
benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
1982
|
-
|| ! grep -Fq 'ALLOWED_PAIR_ARMS,' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
1983
|
-
|| ! grep -Fq 'ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}' \
|
|
1984
|
-
benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
1985
|
-
|| ! grep -Fq 'pair-arm l2_forced is retired' \
|
|
1986
|
-
benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
|
|
1987
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py: pair arm selection must fail closed inside the gate"
|
|
1988
|
-
fi
|
|
1989
|
-
if ! grep -Fq '"l2_risk_probes"' benchmark/auto-resolve/scripts/check-f9-artifacts.py; then
|
|
1990
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/check-f9-artifacts.py: F9 skill-driven artifact check must accept current l2_risk_probes arm"
|
|
1991
|
-
fi
|
|
1992
|
-
if ! grep -Fq '_load_json_object' benchmark/auto-resolve/scripts/check-f9-artifacts.py \
|
|
1993
|
-
|| ! grep -Fq 'expected JSON object' benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh; then
|
|
1994
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/check-f9-artifacts.py: F9 timing/state JSON must fail closed on non-object payloads"
|
|
1995
|
-
fi
|
|
1996
|
-
if grep -RInE 'asserts variant/solo|Variant-only artifact checks|Variant artifact check' \
|
|
1997
|
-
benchmark/auto-resolve/scripts/check-f9-artifacts.py \
|
|
1998
|
-
benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md \
|
|
1999
|
-
benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/spec.md >/dev/null 2>&1; then
|
|
2000
|
-
offenders="${offenders}"$'\n'"F9 artifact docs/checker wording must describe skill-driven arms, not variant-only checks"
|
|
2001
|
-
fi
|
|
2002
|
-
if grep -Fq '<variant|bare>' benchmark/auto-resolve/scripts/run-fixture.sh; then
|
|
2003
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: usage must list current benchmark arms"
|
|
2004
|
-
fi
|
|
2005
|
-
if grep -Fq 'l2_gated/l2_forced' benchmark/auto-resolve/scripts/run-fixture.sh; then
|
|
2006
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: comments must not omit l2_risk_probes from l2_* arm handling"
|
|
2007
|
-
fi
|
|
2008
|
-
if grep -Fq 'ENGINE_CLAUSE="--engine auto"' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2009
|
-
|| grep -Fq 'Run with `--engine auto`' benchmark/auto-resolve/scripts/run-fixture.sh; then
|
|
2010
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: smoke variant arm must use current --engine claude --risk-probes path, not retired --engine auto"
|
|
2011
|
-
fi
|
|
2012
|
-
if ! grep -Fq 'ENGINE_CLAUSE="--engine claude --risk-probes"' benchmark/auto-resolve/scripts/run-fixture.sh; then
|
|
2013
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/run-fixture.sh: variant/l2 risk-probes path must remain available"
|
|
2014
|
-
fi
|
|
2015
|
-
if ! grep -Fq '{bare, solo_claude, selected pair arm}' benchmark/auto-resolve/scripts/judge.sh; then
|
|
2016
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/judge.sh: blind judge topology comment must describe current pair-candidate proof runs"
|
|
2017
|
-
fi
|
|
2018
|
-
if grep -Fq 'blind judge scores l2_gated' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2019
|
-
|| grep -Fq 'l2_gated is clean' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py; then
|
|
2020
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py: gate docstring must describe selected pair arm, not l2_gated-only proof"
|
|
2021
|
-
fi
|
|
2022
|
-
if grep -Fq 'only then spends a `l2_gated` arm' benchmark/auto-resolve/README.md \
|
|
2023
|
-
|| grep -Fq 'fresh `l2_gated` measurement' benchmark/auto-resolve/README.md; then
|
|
2024
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: full-pipeline runner docs must not present l2_gated as the default measured proof path"
|
|
2025
|
-
fi
|
|
2026
|
-
if ! grep -Fq 'Pair arms are limited to current' benchmark/auto-resolve/README.md \
|
|
2027
|
-
|| ! grep -Fq '`l2_forced` is' benchmark/auto-resolve/README.md; then
|
|
2028
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: full-pipeline docs must state allowed pair arms and retired l2_forced"
|
|
2029
|
-
fi
|
|
2030
|
-
if ! grep -Fq 'Current solo<pair' benchmark/auto-resolve/RUBRIC.md \
|
|
2031
|
-
|| ! grep -Fq 'evidence uses the full-pipeline pair gate' benchmark/auto-resolve/RUBRIC.md \
|
|
2032
|
-
|| ! grep -Fq 'explicit selected pair arm' benchmark/auto-resolve/RUBRIC.md \
|
|
2033
|
-
|| ! grep -Fq 'selected pair arm over `solo_claude`' benchmark/auto-resolve/RUBRIC.md; then
|
|
2034
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/RUBRIC.md: rubric must distinguish legacy run-suite ship gate from current selected-pair-arm evidence"
|
|
2035
|
-
fi
|
|
2036
|
-
if grep -Fq 'PHASE 8' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2037
|
-
|| grep -Fq 'security_review on `--engine auto`' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2038
|
-
|| grep -Fq 'BENCHMARK-RESULTS-v3.md' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2039
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/run-real-benchmark.md: real-run docs must describe the current 5-phase resolve and pair score harness"
|
|
2040
|
-
fi
|
|
2041
|
-
if ! grep -Fq 'Archive note (2026-05-14): historical pre-cutover benchmark plan' benchmark/auto-resolve/v3.6-ab-plan.md \
|
|
2042
|
-
|| ! grep -Fq 'Do not use this file for current' benchmark/auto-resolve/v3.6-ab-plan.md \
|
|
2043
|
-
|| ! grep -Fq 'Archive note (2026-05-14): historical pre-cutover results' benchmark/auto-resolve/BENCHMARK-RESULTS-v3.md \
|
|
2044
|
-
|| ! grep -Fq 'Do not treat these projected' benchmark/auto-resolve/BENCHMARK-RESULTS-v3.md \
|
|
2045
|
-
|| ! grep -Fq 'Archive note (2026-05-14): historical v3.2 pilot' benchmark/auto-resolve/PILOT-RESULTS-v3.2.md \
|
|
2046
|
-
|| ! grep -Fq 'This n=1 pilot is not current solo<pair evidence' benchmark/auto-resolve/PILOT-RESULTS-v3.2.md \
|
|
2047
|
-
|| ! grep -Fq 'Archive note (2026-05-14): historical v3.2 strict-route pilot' benchmark/auto-resolve/PILOT-RESULTS-STRICT-v3.2.md \
|
|
2048
|
-
|| ! grep -Fq 'This inline n=1 pilot is not current solo<pair evidence' benchmark/auto-resolve/PILOT-RESULTS-STRICT-v3.2.md \
|
|
2049
|
-
|| ! grep -Fq 'archived static comparison helper for pre-cutover auto-resolve' benchmark/auto-resolve/measure-static.py \
|
|
2050
|
-
|| ! grep -Fq 'is not current solo<pair evidence' benchmark/auto-resolve/measure-static.py; then
|
|
2051
|
-
offenders="${offenders}"$'\n'"archived v3 benchmark artifacts must be clearly marked as non-current solo<pair evidence"
|
|
2052
|
-
fi
|
|
2053
|
-
if ! grep -Fq 'l2_risk_probes` | current measured pair path' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2054
|
-
|| ! grep -Fq 'Dry-runs, lint, and shell tests prove wiring only' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2055
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/run-real-benchmark.md: real-run docs must name the measured pair arm and distinguish wiring checks from scores"
|
|
2056
|
-
fi
|
|
2057
|
-
if grep -RInE 'F27.*unmeasured|unmeasured.*F27' \
|
|
2058
|
-
benchmark/auto-resolve/README.md \
|
|
2059
|
-
benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2060
|
-
benchmark/auto-resolve/run-real-benchmark.md \
|
|
2061
|
-
benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md >/dev/null 2>&1; then
|
|
2062
|
-
offenders="${offenders}"$'\n'"F27 docs must record the measured headroom failure, not stale unmeasured-candidate wording"
|
|
2063
|
-
fi
|
|
2064
|
-
if [ -d benchmark/auto-resolve/fixtures/F27-cli-subscription-proration ]; then
|
|
2065
|
-
offenders="${offenders}"$'\n'"F27 must stay out of active golden fixtures after failing headroom; keep it under fixtures/retired/"
|
|
2066
|
-
fi
|
|
2067
|
-
if [ ! -f benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md ]; then
|
|
2068
|
-
offenders="${offenders}"$'\n'"retired F27 must keep RETIRED.md with the measured rejection reason"
|
|
2069
|
-
fi
|
|
2070
|
-
if ! grep -Fq '"benchmark/auto-resolve/fixtures/retired/F*/**"' package.json; then
|
|
2071
|
-
offenders="${offenders}"$'\n'"package.json must include retired fixtures so replay artifacts ship in npm packages"
|
|
2072
|
-
fi
|
|
2073
|
-
if ! grep -Fq '20260511-f27-headroom-smoke-061401' benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md \
|
|
2074
|
-
|| ! grep -Fq '20260511-f27-headroom-smoke-061401' benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md; then
|
|
2075
|
-
offenders="${offenders}"$'\n'"F27 notes must cite the measured headroom smoke run"
|
|
2076
|
-
fi
|
|
2077
|
-
if grep -Fq 'better candidate for pair risk probes' benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/NOTES.md; then
|
|
2078
|
-
offenders="${offenders}"$'\n'"F27 notes must not describe rejected F27 as a better pair-risk-probe candidate"
|
|
2079
|
-
fi
|
|
2080
|
-
if grep -Fq 'F16-cli-quote-tax-rules F27-cli-subscription-proration' \
|
|
2081
|
-
benchmark/auto-resolve/README.md benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2082
|
-
offenders="${offenders}"$'\n'"benchmark docs must not recommend rejected F27 in headroom/full-pipeline command examples"
|
|
2083
|
-
fi
|
|
2084
|
-
if [ -d benchmark/auto-resolve/fixtures/F28-cli-return-authorization ]; then
|
|
2085
|
-
offenders="${offenders}"$'\n'"F28 must stay out of active golden fixtures after corrected-oracle reverify failed headroom; keep it under fixtures/retired/"
|
|
2086
|
-
fi
|
|
2087
|
-
if [ ! -f benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md ]; then
|
|
2088
|
-
offenders="${offenders}"$'\n'"retired F28 must keep RETIRED.md with the corrected-oracle rejection reason"
|
|
2089
|
-
fi
|
|
2090
|
-
if ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md \
|
|
2091
|
-
|| ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md \
|
|
2092
|
-
|| ! grep -Fq '20260511-f28-policy-oraclefix-reverified-pair' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/NOTES.md \
|
|
2093
|
-
|| ! grep -Fq '20260511-f28-policy-oraclefix-reverified-pair' benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md \
|
|
2094
|
-
|| ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2095
|
-
|| ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2096
|
-
|| ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/README.md \
|
|
2097
|
-
|| ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/README.md \
|
|
2098
|
-
|| ! grep -Fq '20260511-f28-headroom-smoke-085307' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2099
|
-
|| ! grep -Fq '20260511-f28-pair-smoke-091021' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2100
|
-
offenders="${offenders}"$'\n'"F28 docs must cite measured smoke and corrected-oracle rejection run ids before anyone counts it"
|
|
2101
|
-
fi
|
|
2102
|
-
if grep -Fq 'F16-cli-quote-tax-rules F28-cli-return-authorization' \
|
|
2103
|
-
benchmark/auto-resolve/README.md benchmark/auto-resolve/run-real-benchmark.md README.md; then
|
|
2104
|
-
offenders="${offenders}"$'\n'"benchmark docs must not recommend unstable F28 in pair-evidence command examples"
|
|
2105
|
-
fi
|
|
2106
|
-
if ! bash -n benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh >/dev/null 2>&1; then
|
|
2107
|
-
offenders="${offenders}"$'\n'"pair-rejected-fixtures.sh must be valid bash"
|
|
2108
|
-
fi
|
|
2109
|
-
if ! grep -Fq 'trivial calibration fixture' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2110
|
-
|| ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2111
|
-
|| ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2112
|
-
|| ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2113
|
-
|| ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2114
|
-
|| ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2115
|
-
|| ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2116
|
-
|| ! grep -Fq 'known-limit ambiguity fixture' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2117
|
-
|| ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2118
|
-
|| ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2119
|
-
|| ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2120
|
-
|| ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2121
|
-
|| ! grep -Fq 'bare 94 / solo_claude 98 in 20260508-f22-exact-error-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2122
|
-
|| ! grep -Fq '20260508-f26-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2123
|
-
|| ! grep -Fq '20260511-f27-headroom-smoke-061401' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2124
|
-
|| ! grep -Fq '20260511-f28-policy-oraclefix-reverified-pair' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2125
|
-
|| ! grep -Fq '20260510-f29-headroom-v2' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2126
|
-
|| ! grep -Fq '20260511-f30-headroom-v1' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2127
|
-
|| ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2128
|
-
|| ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2129
|
-
|| ! grep -Fq '20260513-s2-inventory-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2130
|
-
|| ! grep -Fq '20260513-s3-ticket-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2131
|
-
|| ! grep -Fq '20260513-s4-return-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2132
|
-
|| ! grep -Fq '20260513-s5-credit-headroom' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2133
|
-
|| ! grep -Fq '20260514-s6-refund-headroom-v1' benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh \
|
|
2134
|
-
|| ! grep -Fq 'source "$BENCH_ROOT/scripts/pair-rejected-fixtures.sh"' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2135
|
-
|| ! grep -Fq 'source "$BENCH_ROOT/scripts/pair-rejected-fixtures.sh"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2136
|
-
|| ! grep -Fq 'fixture_smoke_only' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2137
|
-
|| ! grep -Fq 'fixture_smoke_only' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2138
|
-
|| ! grep -Fq 'smoke-only-s1-provider-run' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2139
|
-
|| ! grep -Fq 'smoke-only-s1-provider-run' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2140
|
-
|| ! grep -Fq 'smoke-only-s1-cli-headroom' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2141
|
-
|| ! grep -Fq 'smoke-only-s1-cli-pair' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2142
|
-
|| ! grep -Fq 'declare -F rejected_pair_fixture_reason' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2143
|
-
|| ! grep -Fq 'declare -F rejected_pair_fixture_reason' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2144
|
-
|| ! grep -Fq 'rejected-f31-fixture' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2145
|
-
|| ! grep -Fq 'rejected-f32-fixture' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2146
|
-
|| ! grep -Fq 'rejected-s6-shadow-fixture' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2147
|
-
|| ! grep -Fq 'rejected-f31-fixture' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2148
|
-
|| ! grep -Fq 'rejected-f32-fixture' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2149
|
-
|| ! grep -Fq 'rejected-s6-shadow-fixture' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2150
|
-
|| ! grep -Fq 'load_rejected_short_ids' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2151
|
-
|| ! grep -Fq 'load_rejected_fixture_reasons' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2152
|
-
|| ! grep -Fq 'rejected_excluded' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2153
|
-
|| ! grep -Fq 'rejected_excluded_reasons' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2154
|
-
|| ! grep -Fq 'rejected_excluded_reasons' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2155
|
-
|| ! grep -Fq 'selection_rule.rejected_excluded_reasons' benchmark/auto-resolve/README.md \
|
|
2156
|
-
|| ! grep -Fq 'rejected_excluded_reasons' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2157
|
-
|| ! grep -Fq 'fixtures_pair_eligible must not be empty' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2158
|
-
|| ! grep -Fq 'gate3_threshold_count must be a positive integer' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2159
|
-
|| ! grep -Fq 'selection_rule.rejected_excluded_reasons keys must match rejected_excluded' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2160
|
-
|| ! grep -Fq 'nan-threshold-manifest' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2161
|
-
|| ! grep -Fq 'bad-rejected-reasons-manifest' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2162
|
-
|| ! grep -Fq 'fixture rejected for pair-candidate runs' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2163
|
-
|| ! grep -Fq 'fixture rejected for pair-candidate runs' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2164
|
-
|| ! grep -Fq 'rejected fixture registry missing' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2165
|
-
|| ! grep -Fq 'rejected fixture registry missing' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2166
|
-
|| ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2167
|
-
|| ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2168
|
-
|| ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
2169
|
-
|| ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
2170
|
-
|| ! grep -Fq '([FS]\d+)-\*\|([FS]\d+)' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2171
|
-
|| ! grep -Fq 'missing-rejected-registry' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2172
|
-
|| ! grep -Fq 'missing-rejected-registry' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2173
|
-
|| ! grep -Fq 'rejected-shadow-direct' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2174
|
-
|| ! grep -Fq 'rejected-shadow-direct' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2175
|
-
|| ! grep -Fq 's-only-registry' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
2176
|
-
|| ! grep -Fq 's-only-registry' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
2177
|
-
|| ! grep -Fq 'shadow solo ceiling' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2178
|
-
|| grep -Fq 'case "$fid" in' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2179
|
-
|| grep -Fq 'case "$fid" in' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh; then
|
|
2180
|
-
offenders="${offenders}"$'\n'"pair candidate runners, audits, and manifest builder must honor the shared rejected fixture registry, including F* fixtures and S* shadow controls, without duplicating the case table"
|
|
2181
|
-
fi
|
|
2182
|
-
if ! grep -Fq 'Pair-candidate status: rejected by design' benchmark/auto-resolve/fixtures/F1-cli-trivial-flag/NOTES.md \
|
|
2183
|
-
|| ! grep -Fq 'Pair-candidate status: rejected by design' benchmark/auto-resolve/fixtures/F8-known-limit-ambiguous/NOTES.md \
|
|
2184
|
-
|| ! grep -Fq 'rejected by design' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2185
|
-
|| ! grep -Fq 'calibration/known-limit controls' benchmark/auto-resolve/README.md \
|
|
2186
|
-
|| ! grep -Fq 'calibration/known-limit controls' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2187
|
-
offenders="${offenders}"$'\n'"F1/F8 docs must mark calibration and known-limit controls as rejected by design for pair evidence"
|
|
2188
|
-
fi
|
|
2189
|
-
if ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand/NOTES.md \
|
|
2190
|
-
|| ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2191
|
-
|| ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/README.md \
|
|
2192
|
-
|| ! grep -Fq '20260512-f2-medium-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2193
|
-
offenders="${offenders}"$'\n'"F2 docs must cite the measured headroom rejection before anyone counts it"
|
|
2194
|
-
fi
|
|
2195
|
-
if ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/fixtures/F12-webhook-raw-body-signature/NOTES.md \
|
|
2196
|
-
|| ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2197
|
-
|| ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/README.md \
|
|
2198
|
-
|| ! grep -Fq '20260511-f12-webhook-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2199
|
-
offenders="${offenders}"$'\n'"F12 docs must cite the measured headroom rejection before anyone counts it"
|
|
2200
|
-
fi
|
|
2201
|
-
if ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/fixtures/F10-persist-write-collision/NOTES.md \
|
|
2202
|
-
|| ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/fixtures/F11-batch-import-all-or-nothing/NOTES.md \
|
|
2203
|
-
|| ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2204
|
-
|| ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/README.md \
|
|
2205
|
-
|| ! grep -Fq '20260507-f10-f11-tier1-full-pipeline' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2206
|
-
offenders="${offenders}"$'\n'"F10/F11 docs must cite the measured headroom rejection before anyone counts them"
|
|
2207
|
-
fi
|
|
2208
|
-
if ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/fixtures/F3-backend-contract-risk/NOTES.md \
|
|
2209
|
-
|| ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2210
|
-
|| ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/README.md \
|
|
2211
|
-
|| ! grep -Fq '20260511-f3-http-error-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2212
|
-
offenders="${offenders}"$'\n'"F3 docs must cite the measured headroom rejection before anyone counts it"
|
|
2213
|
-
fi
|
|
2214
|
-
if ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/fixtures/F4-web-browser-design/NOTES.md \
|
|
2215
|
-
|| ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2216
|
-
|| ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/README.md \
|
|
2217
|
-
|| ! grep -Fq '20260512-f4-web-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2218
|
-
offenders="${offenders}"$'\n'"F4 docs must cite the measured headroom rejection before anyone counts it"
|
|
2219
|
-
fi
|
|
2220
|
-
if ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/fixtures/F5-fix-loop-red-green/NOTES.md \
|
|
2221
|
-
|| ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2222
|
-
|| ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/README.md \
|
|
2223
|
-
|| ! grep -Fq '20260512-f5-fixloop-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2224
|
-
offenders="${offenders}"$'\n'"F5 docs must cite the measured headroom rejection before anyone counts it"
|
|
2225
|
-
fi
|
|
2226
|
-
if ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/fixtures/F6-dep-audit-native-module/NOTES.md \
|
|
2227
|
-
|| ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2228
|
-
|| ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/README.md \
|
|
2229
|
-
|| ! grep -Fq '20260512-f6-checksum-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2230
|
-
offenders="${offenders}"$'\n'"F6 docs must cite the measured headroom rejection before anyone counts it"
|
|
2231
|
-
fi
|
|
2232
|
-
if ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/fixtures/F7-out-of-scope-trap/NOTES.md \
|
|
2233
|
-
|| ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2234
|
-
|| ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/README.md \
|
|
2235
|
-
|| ! grep -Fq '20260512-f7-scope-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2236
|
-
offenders="${offenders}"$'\n'"F7 docs must cite the measured headroom rejection before anyone counts it"
|
|
2237
|
-
fi
|
|
2238
|
-
if ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve/NOTES.md \
|
|
2239
|
-
|| ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2240
|
-
|| ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/README.md \
|
|
2241
|
-
|| ! grep -Fq '20260512-f9-e2e-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2242
|
-
offenders="${offenders}"$'\n'"F9 docs must cite the measured headroom rejection before anyone counts it"
|
|
2243
|
-
fi
|
|
2244
|
-
if ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/fixtures/F15-frozen-diff-race-review/NOTES.md \
|
|
2245
|
-
|| ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2246
|
-
|| ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/README.md \
|
|
2247
|
-
|| ! grep -Fq '20260511-f15-concurrency-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2248
|
-
offenders="${offenders}"$'\n'"F15 docs must cite the measured headroom rejection before anyone counts it"
|
|
2249
|
-
fi
|
|
2250
|
-
if ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md \
|
|
2251
|
-
|| ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2252
|
-
|| ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/README.md \
|
|
2253
|
-
|| ! grep -Fq '20260512-f31-seat-rebalance-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2254
|
-
offenders="${offenders}"$'\n'"F31 docs must cite the measured headroom rejection before anyone counts it"
|
|
2255
|
-
fi
|
|
2256
|
-
if grep -Fq 'execFileSync' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
|
|
2257
|
-
|| ! grep -Fq "assert.strictEqual(result.stderr, '')" benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
|
|
2258
|
-
|| ! grep -Fq 'assert.deepStrictEqual(parsed, {' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
|
|
2259
|
-
|| grep -Fq 'parsed.applied' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js \
|
|
2260
|
-
|| ! grep -Fq 'On success, write exactly one JSON object to stdout and no stderr. Keys: `applied`, `rejected`, `accounts`.' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json \
|
|
2261
|
-
|| [ "$(grep -Fc 'On success, write exactly one JSON object to stdout and no stderr. Keys: `applied`, `rejected`, `accounts`.' benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json)" -ne 1 ]; then
|
|
2262
|
-
offenders="${offenders}"$'\n'"F31 priority verifier must bind success stderr/no-extra-output contract"
|
|
2263
|
-
fi
|
|
2264
|
-
if ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md \
|
|
2265
|
-
|| ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2266
|
-
|| ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/README.md \
|
|
2267
|
-
|| ! grep -Fq '20260512-f32-subscription-renewal-headroom' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2268
|
-
offenders="${offenders}"$'\n'"F32 docs must cite the measured headroom rejection before anyone counts it"
|
|
2269
|
-
fi
|
|
2270
|
-
if ! grep -Fq 'assert.deepEqual(output, {' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js \
|
|
2271
|
-
|| ! grep -Fq 'Output row key names and nested `credits` key names match the visible spec exactly, with no aliased or extra keys.' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json \
|
|
2272
|
-
|| [ "$(grep -Fc 'Output row key names and nested `credits` key names match the visible spec exactly, with no aliased or extra keys.' benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json)" -ne 1 ]; then
|
|
2273
|
-
offenders="${offenders}"$'\n'"F32 priority verifier must own exact output-shape contract without duplicate-error overclaim"
|
|
2274
|
-
fi
|
|
2275
|
-
if ! grep -Fq 'retired_fixture_exists' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2276
|
-
|| ! grep -Fq 'retired_fixture_exists' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2277
|
-
|| ! grep -Fq 'fixture is retired and is not rerun by pair-candidate runners' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2278
|
-
|| ! grep -Fq 'fixture is retired and is not rerun by pair-candidate runners' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2279
|
-
|| ! grep -Fq 'historical artifact replay' benchmark/auto-resolve/README.md \
|
|
2280
|
-
|| ! grep -Fq 'historical artifact replay' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2281
|
-
offenders="${offenders}"$'\n'"pair candidate runners must reject retired fixtures explicitly and docs must reserve retired fixtures for historical artifact replay"
|
|
2282
|
-
fi
|
|
2283
|
-
if ! grep -Fq 'F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2284
|
-
offenders="${offenders}"$'\n'"run-real-benchmark.md examples must use the current measured F16/F23/F25 pair-evidence set"
|
|
2285
|
-
fi
|
|
2286
|
-
if ! grep -Fq '20260510-f16-f23-f25-combined-proof' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2287
|
-
|| ! grep -Fq '20260510-f16-f23-f25-combined-proof' benchmark/auto-resolve/README.md \
|
|
2288
|
-
|| ! grep -Fq '20260510-f16-f23-f25-combined-proof' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2289
|
-
offenders="${offenders}"$'\n'"benchmark docs must cite the current F16/F23/F25 pair proof run"
|
|
2290
|
-
fi
|
|
2291
|
-
if ! grep -Fq 'average pair margin +25.3' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2292
|
-
|| ! grep -Fq 'average pair margin +25.3' benchmark/auto-resolve/README.md \
|
|
2293
|
-
|| ! grep -Fq 'average pair margin was `+25.3`' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2294
|
-
offenders="${offenders}"$'\n'"benchmark docs must cite the F16/F23/F25 average pair margin"
|
|
2295
|
-
fi
|
|
2296
|
-
if ! grep -Fq 'average solo_claude headroom **8.0**' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2297
|
-
|| ! grep -Fq 'minimum solo_claude' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2298
|
-
|| ! grep -Fq '| Fixture | Bare | Solo_claude | Pair (`l2_risk_probes`) | Margin | Pair mode | Wall ratio |' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2299
|
-
|| ! grep -Fq 'include `bare` headroom and `solo_claude` headroom' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2300
|
-
|| ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2301
|
-
|| ! grep -Fq 'average and minimum `bare`/`solo_claude`' benchmark/auto-resolve/README.md \
|
|
2302
|
-
|| ! grep -Fq 'average and minimum headroom' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2303
|
-
|| ! grep -Fq 'fixture pass count' benchmark/auto-resolve/README.md \
|
|
2304
|
-
|| ! grep -Fq 'fixture pass count' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2305
|
-
offenders="${offenders}"$'\n'"benchmark docs must cite and explain headroom set summaries"
|
|
2306
|
-
fi
|
|
2307
|
-
if ! grep -Fq 'both baseline arms evidence-complete' \
|
|
2308
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
|
|
2309
|
-
|| ! grep -Fq 'both baseline arms evidence-complete' \
|
|
2310
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json \
|
|
2311
|
-
|| ! grep -Fq 'Fixtures passed: 3/3 (minimum required: 3)' \
|
|
2312
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
|
|
2313
|
-
|| ! grep -Fq 'Average solo_claude headroom: 8.0' \
|
|
2314
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
|
|
2315
|
-
|| ! grep -Fq '"min_solo_headroom": 5' \
|
|
2316
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json \
|
|
2317
|
-
|| ! grep -Fq '| Fixture | Bare | Bare headroom | Solo_claude | Solo_claude headroom | Status | Reason |' \
|
|
2318
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
|
|
2319
|
-
|| ! grep -Fq '| F16-cli-quote-tax-rules | 50 | 10 | 75 | 5 | PASS | |' \
|
|
2320
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
|
|
2321
|
-
|| ! grep -Fq '"solo_headroom": 14' \
|
|
2322
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json; then
|
|
2323
|
-
offenders="${offenders}"$'\n'"tracked F16/F23/F25 headroom report must use current evidence-complete wording and headroom columns"
|
|
2324
|
-
fi
|
|
2325
|
-
if ! grep -Fq 'l2_risk_probes evidence-clean' \
|
|
2326
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
|
|
2327
|
-
|| ! grep -Fq 'l2_risk_probes must be evidence-clean' \
|
|
2328
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json \
|
|
2329
|
-
|| ! grep -Fq 'Fixtures passed: 3/3 (minimum required: 3)' \
|
|
2330
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
|
|
2331
|
-
|| ! grep -Fq 'Average pair margin: +25.3' \
|
|
2332
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
|
|
2333
|
-
|| ! grep -Fq '"avg_pair_margin": 25.333333333333332' \
|
|
2334
|
-
benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json; then
|
|
2335
|
-
offenders="${offenders}"$'\n'"tracked F16/F23/F25 pair report must use current l2_risk_probes evidence-clean wording and average pair margin"
|
|
2336
|
-
fi
|
|
2337
|
-
if ! grep -Fq 'evidence-complete `bare <= 60`' benchmark/auto-resolve/README.md \
|
|
2338
|
-
|| ! grep -Fq 'evidence-complete `bare <= 60`' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2339
|
-
offenders="${offenders}"$'\n'"benchmark docs must describe baseline headroom arms as evidence-complete, not correctness-clean"
|
|
2340
|
-
fi
|
|
2341
|
-
if ! grep -Fq 'min-bare-headroom' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2342
|
-
|| ! grep -Fq 'min-solo-headroom' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2343
|
-
|| ! grep -Fq 'min-bare-headroom' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2344
|
-
|| ! grep -Fq 'min-solo-headroom' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2345
|
-
|| ! grep -Fq 'headroom >= 5' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md \
|
|
2346
|
-
|| ! grep -Fq 'headroom >= 5' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
|
|
2347
|
-
|| ! grep -Fq 'default minimum 5-point `bare`/`solo_claude` headroom margin' benchmark/auto-resolve/README.md \
|
|
2348
|
-
|| ! grep -Fq 'default minimum 5-point `bare`/`solo_claude` headroom margin' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2349
|
-
|| ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' benchmark/auto-resolve/README.md \
|
|
2350
|
-
|| ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' benchmark/auto-resolve/BENCHMARK-DESIGN.md \
|
|
2351
|
-
|| ! grep -Fq 'bare headroom 1 < 5' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2352
|
-
|| ! grep -Fq 'bare headroom 1 < 5' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh; then
|
|
2353
|
-
offenders="${offenders}"$'\n'"benchmark headroom gates must reject threshold-fragile candidates with default 5-point baseline headroom margins"
|
|
2354
|
-
fi
|
|
2355
|
-
if [ -d benchmark/auto-resolve/results/20260511-f28-headroom-smoke-085307/F28-cli-return-authorization ]; then
|
|
2356
|
-
if python3 benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2357
|
-
--run-id 20260511-f28-headroom-smoke-085307 \
|
|
2358
|
-
--min-fixtures 1 \
|
|
2359
|
-
--out-json /tmp/devlyn-lint-f28-headroom-gate.json \
|
|
2360
|
-
--out-md /tmp/devlyn-lint-f28-headroom-gate.md >/dev/null 2>&1; then
|
|
2361
|
-
offenders="${offenders}"$'\n'"current local F28 headroom artifact must fail the default 5-point headroom margin gate"
|
|
2362
|
-
elif ! grep -Fq 'bare headroom 1 < 5' /tmp/devlyn-lint-f28-headroom-gate.md; then
|
|
2363
|
-
offenders="${offenders}"$'\n'"current local F28 headroom failure must cite bare headroom 1 < 5"
|
|
2364
|
-
fi
|
|
2365
|
-
fi
|
|
2366
|
-
if [ -d benchmark/auto-resolve/results/20260511-f28-pair-smoke-091021/F28-cli-return-authorization ]; then
|
|
2367
|
-
if python3 benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2368
|
-
--run-id 20260511-f28-pair-smoke-091021 \
|
|
2369
|
-
--min-fixtures 1 \
|
|
2370
|
-
--out-json /tmp/devlyn-lint-f28-pair-smoke-headroom-gate.json \
|
|
2371
|
-
--out-md /tmp/devlyn-lint-f28-pair-smoke-headroom-gate.md >/dev/null 2>&1; then
|
|
2372
|
-
offenders="${offenders}"$'\n'"current local F28 pair-smoke artifact must fail before pair evidence can be counted"
|
|
2373
|
-
elif ! grep -Fq 'bare score 65 > 60' /tmp/devlyn-lint-f28-pair-smoke-headroom-gate.md; then
|
|
2374
|
-
offenders="${offenders}"$'\n'"current local F28 pair-smoke failure must cite bare score 65 > 60"
|
|
2375
|
-
fi
|
|
2376
|
-
fi
|
|
2377
|
-
if ! grep -Fq 'startup `Gate:` line' benchmark/auto-resolve/README.md \
|
|
2378
|
-
|| ! grep -Fq 'startup `Gate:` line' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2379
|
-
|| ! grep -Fq 'headroom gate passed — candidate set accepted' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2380
|
-
|| ! grep -Fq 'headroom gate failed — candidate set rejected' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2381
|
-
|| ! grep -Fq 'candidate set was accepted or rejected' benchmark/auto-resolve/README.md \
|
|
2382
|
-
|| ! grep -Fq 'candidate set was accepted or rejected' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2383
|
-
|| ! grep -Fq 'fixture score table with bare score, bare' benchmark/auto-resolve/README.md \
|
|
2384
|
-
|| ! grep -Fq 'headroom, solo_claude score, solo_claude headroom' benchmark/auto-resolve/README.md \
|
|
2385
|
-
|| ! grep -Fq 'remaining headroom against' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2386
|
-
|| ! grep -Fq 'pair minus `solo_claude` margin' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2387
|
-
|| ! grep -Fq 'Fixture Bare Solo_claude Pair Pair-Solo_claude' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2388
|
-
|| ! grep -Fq 'fixture score table with bare,' benchmark/auto-resolve/README.md \
|
|
2389
|
-
|| ! grep -Fq 'solo_claude, pair, margin, pair-mode' benchmark/auto-resolve/README.md \
|
|
2390
|
-
|| ! grep -Fq 'pair above `solo_claude`' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
2391
|
-
|| ! grep -Fq 'Suite average variant-solo_claude margin' benchmark/auto-resolve/BENCHMARK-DESIGN.md \
|
|
2392
|
-
|| ! grep -Fq 'legacy `variant`-`bare` (L2-L0)' benchmark/auto-resolve/RUBRIC.md \
|
|
2393
|
-
|| ! grep -Fq '`solo_claude`-`bare` measures solo harness value; pair-`solo_claude` measures pair value' benchmark/auto-resolve/README.md \
|
|
2394
|
-
|| ! grep -Fq '| fixture | bare | solo_claude | solo_claude-bare |' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2395
|
-
|| ! grep -Fq '| fixture | bare | solo_claude | solo_claude-bare |' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2396
|
-
|| ! grep -Fq '| fixture | bare | solo_claude | pair | pair-solo_claude |' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2397
|
-
|| ! grep -Fq 'startup `Headroom:` and `Pair:` lines' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2398
|
-
|| ! grep -Fq 'startup `Headroom:` / `Pair:` lines' benchmark/auto-resolve/README.md; then
|
|
2399
|
-
offenders="${offenders}"$'\n'"benchmark docs must describe real-run startup gate lines and headroom score columns"
|
|
2400
|
-
fi
|
|
2401
|
-
if ! grep -Fq 'DEVLYN_BENCHMARK_CLI_SUBCOMMAND: benchmarkMode' bin/devlyn.js \
|
|
2402
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark headroom --run-id "$RUN_ID"' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2403
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark pair --run-id "$RUN_ID"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2404
|
-
|| ! grep -Fq 'canonical trigger, margin >= +' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2405
|
-
|| ! grep -Fq 'canonical trigger, margin >= +5' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2406
|
-
|| ! grep -Fq 'headroom gate failed — pair arm not executed' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2407
|
-
|| ! grep -Fq 'pair gate failed — pair evidence rejected' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2408
|
-
|| ! grep -Fq 'headroom gate passed — executing $PAIR_ARM' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2409
|
-
|| ! grep -Fq 'pair gate passed — pair evidence accepted' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2410
|
-
|| ! grep -Fq 'headroom fails, the runner explicitly says the pair arm was not executed' benchmark/auto-resolve/README.md \
|
|
2411
|
-
|| ! grep -Fq 'pair gate fails, it explicitly says pair evidence was rejected' benchmark/auto-resolve/README.md \
|
|
2412
|
-
|| ! grep -Fq 'then that pair evidence was accepted' benchmark/auto-resolve/README.md \
|
|
2413
|
-
|| ! grep -Fq 'If headroom fails, it reports that the pair arm was not executed' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2414
|
-
|| ! grep -Fq 'pair gate fails, it reports that pair evidence was rejected' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2415
|
-
|| ! grep -Fq 'then that pair evidence was' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2416
|
-
|| ! grep -Fq 'accepted. When launched through' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2417
|
-
|| ! grep -Fq 'the replay `Command:` uses the' benchmark/auto-resolve/README.md \
|
|
2418
|
-
|| ! grep -Fq 'same package CLI path' benchmark/auto-resolve/README.md \
|
|
2419
|
-
|| ! grep -Fq 'the replay command uses' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2420
|
-
|| ! grep -Fq 'uses that same' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2421
|
-
|| ! grep -Fq 'package CLI path' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2422
|
-
offenders="${offenders}"$'\n'"benchmark CLI headroom/pair runs must replay as npx devlyn-cli commands and docs must state that"
|
|
2423
|
-
fi
|
|
2424
|
-
if ! grep -Fq 'npx devlyn-cli benchmark headroom --min-fixtures 3 F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' README.md \
|
|
2425
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark frontier --out-md /tmp/devlyn-pair-frontier.md' README.md \
|
|
2426
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark pair --min-fixtures 3 --max-pair-solo-wall-ratio 3 F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' README.md \
|
|
2427
|
-
|| ! grep -Fq 'average pair margin' README.md \
|
|
2428
|
-
|| ! grep -Fq 'default 5-point `bare`/`solo_claude` headroom margins' README.md \
|
|
2429
|
-
|| ! grep -Fq 'Add `--dry-run` to either score runner' README.md \
|
|
2430
|
-
|| ! grep -Fq 'fixture count, and the replay command' README.md \
|
|
2431
|
-
|| ! grep -Fq 'Dry-runs' README.md \
|
|
2432
|
-
|| ! grep -Fq 'and lint prove wiring only; real score claims must cite the run id and fixture' README.md; then
|
|
2433
|
-
offenders="${offenders}"$'\n'"README.md must expose score-focused benchmark headroom/pair CLI paths and distinguish wiring checks from real scores"
|
|
2434
|
-
fi
|
|
2435
|
-
if ! grep -Fq 'npx devlyn-cli benchmark pair --min-fixtures 3 --max-pair-solo-wall-ratio 3 F16-cli-quote-tax-rules F23-cli-fulfillment-wave F25-cli-cart-promotion-rules' bin/devlyn.js \
|
|
2436
|
-
|| ! grep -Fq -- '--max-pair-solo-wall-ratio N default: 3' bin/devlyn.js \
|
|
2437
|
-
|| ! grep -Fq 'MAX_PAIR_SOLO_WALL_RATIO=3' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2438
|
-
|| ! grep -Fq 'parser.add_argument("--max-pair-solo-wall-ratio", type=positive_float, default=3.0)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2439
|
-
|| ! grep -Fq '"max_observed_pair_solo_wall_ratio": max(ratios) if ratios else None' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2440
|
-
|| ! grep -Fq 'Allowed pair/solo wall ratio' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2441
|
-
|| ! grep -Fq 'Maximum observed pair/solo wall ratio' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2442
|
-
|| ! grep -Fq 'Maximum observed pair/solo wall ratio: 2.00x' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2443
|
-
|| ! grep -Fq 'separates the allowed pair/solo wall ratio from the maximum observed pair/solo wall ratio' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2444
|
-
|| ! grep -Fq 'python3 "$GATE" --results-root "$TMP_DIR" --run-id slow-pair' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2445
|
-
|| ! grep -Fq -- '--min-fixtures 3' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2446
|
-
|| ! grep -Fq -- '--min-fixtures 3' benchmark/auto-resolve/README.md \
|
|
2447
|
-
|| ! grep -Fq 'average pair margin' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2448
|
-
|| ! grep -Fq 'average pair margin' benchmark/auto-resolve/README.md \
|
|
2449
|
-
|| ! grep -Fq -- '--dry-run validate args/fixtures and print replay command only' bin/devlyn.js \
|
|
2450
|
-
|| ! grep -Fq 'does not produce' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2451
|
-
|| ! grep -Fq 'scores. When showing' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
2452
|
-
offenders="${offenders}"$'\n'"benchmark pair examples must explicitly gate the current F16/F23/F25 proof set with --min-fixtures 3"
|
|
2453
|
-
fi
|
|
2454
|
-
if ! grep -Fq 'use 3 for F16/F23/F25 proof reruns; audit requires 4 passing evidence rows' bin/devlyn.js \
|
|
2455
|
-
|| ! grep -Fq 'use 3 for F16/F23/F25 proof reruns; audit requires 4 passing evidence rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh; then
|
|
2456
|
-
offenders="${offenders}"$'\n'"benchmark CLI help must distinguish proof reruns from the four-row release audit"
|
|
2457
|
-
fi
|
|
2458
|
-
if ! grep -Fq 'judge blind mapping missing' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2459
|
-
|| ! grep -Fq 'judge blind mapping missing' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2460
|
-
|| ! grep -Fq 'blind_mapping_arm_missing' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2461
|
-
|| ! grep -Fq 'def trusted_winner()' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2462
|
-
|| ! grep -Fq 'raw_findings_by_arm = judge.get("findings_by_arm")' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2463
|
-
|| ! grep -Fq 'def critical_findings_for(arm: str)' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2464
|
-
|| ! grep -Fq 'def exact_bool(value)' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2465
|
-
|| ! grep -Fq '"malformed_boolean_fields": malformed_boolean_fields' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2466
|
-
|| ! grep -Fq '"dq_judge_malformed": judge_dq_malformed' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2467
|
-
|| ! grep -Fq 'def bool_or_none(value)' benchmark/auto-resolve/scripts/ship-gate.py \
|
|
2468
|
-
|| ! grep -Fq 'l1_dq_by_fixture: dict[str, bool]' benchmark/auto-resolve/scripts/ship-gate.py \
|
|
2469
|
-
|| ! grep -Fq 'summary arms_present malformed' benchmark/auto-resolve/scripts/ship-gate.py \
|
|
2470
|
-
|| ! grep -Fq 'summary arms_present.solo_claude malformed' benchmark/auto-resolve/scripts/ship-gate.py \
|
|
2471
|
-
|| ! grep -Fq 'summary margins_avg malformed' benchmark/auto-resolve/scripts/ship-gate.py \
|
|
2472
|
-
|| ! grep -Fq 'raw_findings_letters = chosen.get("critical_findings")' benchmark/auto-resolve/scripts/judge.sh \
|
|
2473
|
-
|| ! grep -Fq 'raw_findings_letters = chosen.get("critical_findings")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2474
|
-
|| ! grep -Fq 'raw_validation = judge.get("_axis_validation")' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2475
|
-
|| ! grep -Fq 'raw_validation = judge.get("_axis_validation")' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2476
|
-
|| ! grep -Fq 'raw_validation = judge.get("_axis_validation")' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2477
|
-
|| ! grep -Fq 'raw_scores_by_arm = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2478
|
-
|| ! grep -Fq 'from pair_evidence_contract import is_score, is_strict_number' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2479
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2480
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/ship-gate.py \
|
|
2481
|
-
|| ! grep -Fq 'if arm in mapped_arms and is_score(score)' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2482
|
-
|| ! grep -Fq 'def strict_number(value)' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2483
|
-
|| ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2484
|
-
|| ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2485
|
-
|| ! grep -Fq 'from pair_evidence_contract import is_score' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2486
|
-
|| ! grep -Fq 'return value if is_score(value) else None' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2487
|
-
|| ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2488
|
-
|| ! grep -Fq 'from pair_evidence_contract import is_score' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2489
|
-
|| ! grep -Fq 'def exact_bool(value: object)' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2490
|
-
|| ! grep -Fq 'def disqualifier_flag(value: object' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2491
|
-
|| ! grep -Fq 'if not is_score(solo) or not is_score(bare):' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2492
|
-
|| ! grep -Fq 'if is_score(score):' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2493
|
-
|| ! grep -Fq 'return legacy if is_score(legacy) else None' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2494
|
-
|| ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
|
|
2495
|
-
|| ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2496
|
-
|| ! grep -Fq 'from pair_evidence_contract import (' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2497
|
-
|| ! grep -Fq 'is_score,' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2498
|
-
|| ! grep -Fq 'is_strict_number,' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2499
|
-
|| ! grep -Fq 'loads_strict_json_object(p.read_text())' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2500
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2501
|
-
|| ! grep -Fq 'PAIR_VERDICTS = {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK", "BLOCKED", "FAIL"}' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2502
|
-
|| ! grep -Fq 'def is_pair_judge_verdict(value: object) -> bool:' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2503
|
-
|| ! grep -Fq 'def exact_bool(value: object)' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2504
|
-
|| ! grep -Fq 'def bool_flag(value: object' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2505
|
-
|| ! grep -Fq 'if is_score(sba.get(arm)):' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2506
|
-
|| ! grep -Fq 'return legacy if is_score(legacy) else None' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2507
|
-
|| ! grep -Fq 'def strict_elapsed_seconds' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2508
|
-
|| ! grep -Fq 'def timeout_flag(result: dict | None) -> bool:' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2509
|
-
|| ! grep -Fq 'raw_scores = judge.get("scores_by_arm")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2510
|
-
|| ! grep -Fq 'raw_dq_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2511
|
-
|| ! grep -Fq 'raw_dq_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2512
|
-
|| ! grep -Fq 'raw_dq_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2513
|
-
|| ! grep -Fq 'raw_by_arm = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2514
|
-
|| ! grep -Fq 'raw_dba = judge.get("disqualifiers_by_arm")' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2515
|
-
|| ! grep -Fq 'raw_legacy = judge.get("disqualifiers")' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2516
|
-
|| ! grep -Fq 'raw_dqs = judge.get("disqualifiers")' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2517
|
-
|| ! grep -Fq 'def is_score(value):' benchmark/auto-resolve/scripts/judge.sh \
|
|
2518
|
-
|| ! grep -Fq 'invalid judge score value(s)' benchmark/auto-resolve/scripts/judge.sh \
|
|
2519
|
-
|| ! grep -Fq 'invalid judge disqualifier value(s)' benchmark/auto-resolve/scripts/judge.sh \
|
|
2520
|
-
|| ! grep -Fq 'invalid opus disqualifier value(s)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2521
|
-
|| ! grep -Fq 'if arm is not None and key in chosen and is_score(chosen[key]):' benchmark/auto-resolve/scripts/judge.sh \
|
|
2522
|
-
|| ! grep -Fq 'raw_dq_letters = chosen.get("disqualifiers")' benchmark/auto-resolve/scripts/judge.sh \
|
|
2523
|
-
|| ! grep -Fq 'raw_dq_letters = chosen.get("disqualifiers")' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2524
|
-
|| ! grep -Fq 'scores_by_arm` alone is not evidence' benchmark/auto-resolve/README.md \
|
|
2525
|
-
|| ! grep -Fq 'without the blind slot mapping is not score evidence' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2526
|
-
|| ! grep -Fq 'absent from the blind mapping is not score evidence' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2527
|
-
|| ! grep -Fq 'wrong-pair-mapping' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2528
|
-
|| ! grep -Fq 'from pair_evidence_contract import (' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2529
|
-
|| ! grep -Fq 'ALLOWED_PAIR_ARMS,' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2530
|
-
|| ! grep -Fq 'loads_strict_json_object,' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2531
|
-
|| ! grep -Fq 'def bool_flag_failure(value: Any, true_reason: str, malformed_reason: str) -> str | None:' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2532
|
-
|| ! grep -Fq 'def bool_flag_failure(value: object, true_reason: str, malformed_reason: str) -> str | None:' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
2533
|
-
|| ! grep -Fq 'return value if is_score(value) else None' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2534
|
-
|| ! grep -Fq 'if not is_strict_number(pair_elapsed) or not is_strict_number(solo_elapsed):' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2535
|
-
|| ! grep -Fq 'return is_strict_number(value) and value >= 1.0' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2536
|
-
|| ! grep -Fq 'value must be finite and > 0' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2537
|
-
|| ! grep -Fq 'overrange-score' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2538
|
-
|| ! grep -Fq 'boolean-score' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2539
|
-
|| ! grep -Fq 'boolean-wall-time' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2540
|
-
|| ! grep -Fq 'invalid-max-wall-ratio' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2541
|
-
|| ! grep -Fq 'boolean-pair-verify-score' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2542
|
-
|| ! grep -Fq 'malformed-pair-bool' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2543
|
-
|| ! grep -Fq 'malformed-judge-bool' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2544
|
-
|| ! grep -Fq 'wrong-mapping' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2545
|
-
|| ! grep -Fq 'overrange-score' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2546
|
-
|| ! grep -Fq 'boolean-score' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2547
|
-
|| ! grep -Fq 'malformed-bare-bool' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2548
|
-
|| ! grep -Fq 'bare judge disqualifier malformed' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2549
|
-
|| ! grep -Fq 'variant-mapping-disqualifies' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2550
|
-
|| ! grep -Fq 'solo-mapping-disqualifies' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2551
|
-
|| ! grep -Fq 'stale-margin' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2552
|
-
|| ! grep -Fq 'stale judge margins must be recomputed from trusted scores' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2553
|
-
|| ! grep -Fq 'winner without blind-mapped trusted score must be null' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2554
|
-
|| ! grep -Fq 'malformed scores_by_arm must not expose' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2555
|
-
|| ! grep -Fq 'out-of-range scores_by_arm must not expose variant score' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2556
|
-
|| ! grep -Fq 'boolean scores_by_arm must not expose solo score' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2557
|
-
|| ! grep -Fq 'boolean result numeric fields must not appear in compile summary' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2558
|
-
|| ! grep -Fq 'malformed-result-bool' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2559
|
-
|| ! grep -Fq 'malformed-judge-bool' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2560
|
-
|| ! grep -Fq 'malformed-l1-dq-summary' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2561
|
-
|| ! grep -Fq 'malformed-arms-present-wrapper' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2562
|
-
|| ! grep -Fq 'malformed-arms-present' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2563
|
-
|| ! grep -Fq 'malformed-margins-avg-wrapper' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2564
|
-
|| ! grep -Fq 'F1 L1 disqualifier malformed' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2565
|
-
|| ! grep -Fq 'overrange C1 row fields must not promote F17' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2566
|
-
|| ! grep -Fq 'string C1 disqualifier must not promote F18' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2567
|
-
|| ! grep -Fq 'f9-overrange-scores' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2568
|
-
|| ! grep -Fq 'f9-boolean-scores' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2569
|
-
|| ! grep -Fq 'f9-string-dq-entry' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2570
|
-
|| ! grep -Fq 'overrange-scores-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2571
|
-
|| ! grep -Fq 'boolean-scores-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2572
|
-
|| ! grep -Fq 'boolean-wall-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2573
|
-
|| ! grep -Fq 'nan-wall-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2574
|
-
|| ! grep -Fq 'string-timeout-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2575
|
-
|| ! grep -Fq 'malformed-pair-state-results' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2576
|
-
|| ! grep -Fq 'non-list finding entry must become a one-item list' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2577
|
-
|| ! grep -Fq 'non-dict findings_by_arm must be ignored' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2578
|
-
|| ! grep -Fq 'non-dict _axis_validation wrapper must not crash' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2579
|
-
|| ! grep -Fq 'load_dict_json' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2580
|
-
|| ! grep -Fq 'measurement invalid: malformed summary.json' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2581
|
-
|| ! grep -Fq 'nan-summary' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2582
|
-
|| ! grep -Fq 'summary rows contain non-object entries' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2583
|
-
|| ! grep -Fq 'malformed-summary-field-types' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2584
|
-
|| ! grep -Fq 'variant axis count malformed' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2585
|
-
|| ! grep -Fq 'result.json malformed' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2586
|
-
|| ! grep -Fq 'result.json malformed' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2587
|
-
|| ! grep -Fq 'non-dict variant result.json must fail closed' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2588
|
-
|| ! grep -Fq 'NaN variant result.json must fail closed as a disqualifier' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2589
|
-
|| ! grep -Fq '("solo_claude (L1)", "solo_claude")' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2590
|
-
|| ! grep -Fq '| Fixture | Category | variant (L2) | solo_claude (L1) | bare (L0) | variant-bare | solo_claude-bare | variant-solo_claude | Winner | Wall variant/solo_claude/bare | Wall variant/solo_claude | Wall variant/bare |' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2591
|
-
|| ! grep -Fq '| Fixture | Category | variant (L2) | solo_claude (L1) | bare (L0) | variant-bare | solo_claude-bare | variant-solo_claude | Winner | Wall variant/solo_claude/bare | Wall variant/solo_claude | Wall variant/bare |' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2592
|
-
|| ! grep -Fq '**solo_claude (L1):**' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2593
|
-
|| ! grep -Fq 'variant (L2) vs solo_claude (L1) margin avg' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2594
|
-
|| ! grep -Fq '**variant (L2) vs solo_claude (L1) margin avg:** +10.0' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2595
|
-
|| ! grep -Fq 'Wall ratio variant (L2) / solo_claude (L1)' benchmark/auto-resolve/scripts/compile-report.py \
|
|
2596
|
-
|| ! grep -Fq '**Wall ratio variant (L2) / solo_claude (L1):** 1.0x' benchmark/auto-resolve/scripts/test-ship-gate.sh \
|
|
2597
|
-
|| ! grep -Fq 'malformed-scores' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2598
|
-
|| ! grep -Fq 'malformed-scores' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2599
|
-
|| ! grep -Fq 'malformed-dq-entry' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2600
|
-
|| ! grep -Fq 'malformed-dq-entry' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2601
|
-
|| ! grep -Fq 'malformed-axis-wrapper' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
2602
|
-
|| ! grep -Fq 'malformed-axis-wrapper' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
2603
|
-
|| ! grep -Fq 'fixture-nan-metadata' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2604
|
-
|| ! grep -Fq 'fixture-nan-expected' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2605
|
-
|| ! grep -Fq 'c1-summary malformed: expected object' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2606
|
-
|| ! grep -Fq 'f9-judge malformed: expected object' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2607
|
-
|| ! grep -Fq 'malformed C1 row fields must not promote F1' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2608
|
-
|| ! grep -Fq 'F9 must not be included when scores_by_arm is malformed' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2609
|
-
|| ! grep -Fq 'truthy malformed disqualifier entry must exclude F9' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2610
|
-
|| ! grep -Fq 'disqualifiers": ["not", "a", "dict"]' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2611
|
-
|| ! grep -Fq 'bad-mapping' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2612
|
-
|| ! grep -Fq 'write_fixture_with_malformed_dq_entry' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2613
|
-
|| ! grep -Fq 'write_fixture_with_malformed_legacy_dq' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2614
|
-
|| ! grep -Fq 'write_fixture_with_string_dq_entry' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2615
|
-
|| ! grep -Fq 'write_fixture_with_malformed_result' benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh \
|
|
2616
|
-
|| ! grep -Fq 'disqualifiers": ["not", "a", "dict"]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2617
|
-
|| ! grep -Fq 'malformed compare.json for malformed-compare' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2618
|
-
|| ! grep -Fq 'malformed compare.json for nan-compare: invalid JSON' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2619
|
-
|| ! grep -Fq 'malformed-compare-sections' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2620
|
-
|| ! grep -Fq 'malformed-verdict-fields' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2621
|
-
|| ! grep -Fq 'malformed-elapsed-fields' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2622
|
-
|| ! grep -Fq 'string-pair-mode' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2623
|
-
|| ! grep -Fq 'def strict_positive_number(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2624
|
-
|| ! grep -Fq 'math.isfinite(value)' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2625
|
-
|| ! grep -Fq 'def strict_nonnegative_int(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2626
|
-
|| ! grep -Fq 'def summary_findings_count(data):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2627
|
-
|| ! grep -Fq 'def severity_count_sum(data):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2628
|
-
|| ! grep -Fq 'def strict_greater(left, right):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2629
|
-
|| ! grep -Fq 'metadata = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2630
|
-
|| ! grep -Fq 'metadata timeout_seconds must be a positive integer' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2631
|
-
|| ! grep -Fq 'state = as_dict(loads_strict_json_object(state_path.read_text())) if state_path.is_file() else {}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2632
|
-
|| ! grep -Fq 'json.loads(line, parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2633
|
-
|| ! grep -Fq 'summary = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2634
|
-
|| ! grep -Fq 'expected = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2635
|
-
|| ! grep -Fq 'out[arm] = loads_strict_json_object(path.read_text()) if path.is_file() else {"missing": True}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2636
|
-
|| ! grep -Fq 'def as_dict(value):' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2637
|
-
|| ! grep -Fq 'metadata = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2638
|
-
|| ! grep -Fq 'metadata timeout_seconds must be a positive integer' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2639
|
-
|| ! grep -Fq 'expected = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2640
|
-
|| ! grep -Fq 'verify = as_dict(loads_strict_json_object(pathlib.Path(result_dir, "verify.json").read_text()))' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2641
|
-
|| ! grep -Fq 'loads_strict_json_object(pathlib.Path(result_dir, "timing.json").read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2642
|
-
|| ! grep -Fq 'loads_strict_json_object(pathlib.Path(state_path).read_text())' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2643
|
-
|| ! grep -Fq 'phases = as_dict(state.get("phases"))' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2644
|
-
|| ! grep -Fq 'legacy_verify = as_dict(state.get("verify"))' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2645
|
-
|| ! grep -Fq 'data = raw_oracle' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2646
|
-
|| ! grep -Fq 'oracle artifact malformed or unreadable' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2647
|
-
|| ! grep -Fq '"type": "oracle-error"' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2648
|
-
|| ! grep -Fq 'verify["oracle_disqualifier"] = True' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2649
|
-
|| ! grep -Fq 'findings = raw_findings if isinstance(raw_findings, list) else []' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2650
|
-
|| ! grep -Fq 'if not isinstance(finding, dict):' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2651
|
-
|| ! grep -Fq 'def as_dict(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2652
|
-
|| ! grep -Fq 'phases = as_dict(state.get("phases"))' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2653
|
-
|| ! grep -Fq 'legacy_verify = as_dict(state.get("verify"))' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2654
|
-
|| ! grep -Fq 'PAIR_VERDICTS = {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK", "BLOCKED", "FAIL"}' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2655
|
-
|| ! grep -Fq 'def has_pair_judge_verdict(sub_verdicts):' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2656
|
-
|| ! grep -Fq 'PAIR_VERDICTS = {"PASS", "PASS_WITH_ISSUES", "NEEDS_WORK", "BLOCKED", "FAIL"}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2657
|
-
|| ! grep -Fq 'def has_pair_judge_verdict(sub_verdicts):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2658
|
-
|| ! grep -Fq 'or verify_phase.get("pair_mode") is True' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2659
|
-
|| ! grep -Fq 'has_pair_judge_verdict(sub_verdicts) or verify_phase.get("pair_mode") is True' benchmark/auto-resolve/scripts/run-fixture.sh \
|
|
2660
|
-
|| ! grep -Fq 'has_pair_judge_verdict(sub_verdicts)' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2661
|
-
|| ! grep -Fq 'raw_pair_sub = pair.get("sub_verdicts")' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2662
|
-
|| ! grep -Fq 'raw_pair_trigger = pair.get("pair_trigger")' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2663
|
-
|| ! grep -Fq 'pair_trigger = raw_pair_trigger if isinstance(raw_pair_trigger, dict) else {}' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2664
|
-
|| ! grep -Fq 'pair_mode_true = pair.get("pair_mode") is True' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2665
|
-
|| ! grep -Fq 'def fmt_trigger_reasons(value):' benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2666
|
-
|| ! grep -Fq "| Arm | Verdict | Pair mode | Triggers | Findings | Elapsed | Invoke exit | Failure |" benchmark/auto-resolve/scripts/run-frozen-verify-pair.sh \
|
|
2667
|
-
|| ! grep -Fq '| Arm | Verdict | Pair mode | Triggers | Findings | Elapsed | Invoke exit | Failure |' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2668
|
-
|| ! grep -Fq 'pair_mode = pair.get("pair_mode") is True' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2669
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2670
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/check-f9-artifacts.py \
|
|
2671
|
-
|| ! grep -Fq 'invalid JSON numeric constant: NaN' benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh \
|
|
2672
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text(encoding="utf8"))' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2673
|
-
|| ! grep -Fq 'def pair_trigger_failures(' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2674
|
-
|| ! grep -Fq 'def pair_trigger_reasons(pair: dict[str, Any]) -> list[str]:' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2675
|
-
|| ! grep -Fq 'pair_trigger_eligible(pair)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2676
|
-
|| ! grep -Fq '"pair_trigger_has_canonical_reason": has_canonical_pair_trigger_reason(trigger_reasons)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2677
|
-
|| ! grep -Fq '"pair_trigger_has_canonical_reason": true' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2678
|
-
|| ! grep -Fq 'failed attempt: pair trigger contract: ' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2679
|
-
|| ! grep -Fq 'has_known_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2680
|
-
|| ! grep -Fq 'all_known_pair_trigger_reasons(trigger["reasons"])' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2681
|
-
|| ! grep -Fq 'has_canonical_pair_trigger_reason(trigger["reasons"])' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2682
|
-
|| ! grep -Fq 'path_has_actionable_solo_headroom_hypothesis(fixture_spec)' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2683
|
-
|| ! grep -Fq 'pair_trigger reasons missing known trigger reason' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2684
|
-
|| ! grep -Fq 'pair_trigger reasons contain unknown trigger reason' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2685
|
-
|| ! grep -Fq 'pair_trigger reasons missing canonical trigger reason' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2686
|
-
|| ! grep -Fq 'pair_trigger missing spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2687
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger requires --fixtures-root' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2688
|
-
|| ! grep -Fq 'swebench-missing-hypothesis-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2689
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger requires --fixtures-root' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2690
|
-
|| ! grep -Fq 'has_known_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2691
|
-
|| ! grep -Fq 'all_known_pair_trigger_reasons(reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2692
|
-
|| ! grep -Fq 'has_canonical_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2693
|
-
|| ! grep -Fq 'def pair_trigger_reasons(pair: dict[str, Any]) -> list[str]:' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2694
|
-
|| ! grep -Fq 'path_has_actionable_solo_headroom_hypothesis(fixtures_root / fixture_id / "spec.md")' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2695
|
-
|| ! grep -Fq 'pair_trigger missing spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2696
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2697
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh \
|
|
2698
|
-
|| ! grep -Fq 'missing-hypothesis-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2699
|
-
|| ! grep -Fq '"pair_trigger_has_canonical_reason": has_canonical_pair_trigger_reason(trigger_reasons)' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2700
|
-
|| ! grep -Fq '"pair_trigger_has_canonical_reason": true' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2701
|
-
|| ! grep -Fq '| Run | Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Triggers | Wall ratio | External lift | Internal lift | Status | Reason |' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2702
|
-
|| ! grep -Fq '| Run | Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Triggers | Wall ratio | External lift | Internal lift | Status | Reason |' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2703
|
-
|| ! grep -Fq 'pair_trigger reasons missing known trigger reason' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2704
|
-
|| ! grep -Fq 'pair_trigger reasons contain unknown trigger reason' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2705
|
-
|| ! grep -Fq 'pair_trigger reasons missing canonical trigger reason' benchmark/auto-resolve/scripts/frozen-verify-gate.py \
|
|
2706
|
-
|| ! grep -Fq 'normalized-canonical-pair-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2707
|
-
|| ! grep -Fq '| Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Pair trigger | Triggers | Wall ratio | External lift | Internal lift | Included | Classification |' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2708
|
-
|| ! grep -Fq '| Fixture | Solo VERIFY | Pair VERIFY | Pair mode | Pair trigger | Triggers | Wall ratio | External lift | Internal lift | Included | Classification |' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2709
|
-
|| ! grep -Fq 'def is_true(value: Any) -> bool:' benchmark/auto-resolve/scripts/swebench-frozen-matrix.py \
|
|
2710
|
-
|| ! grep -Fq 'swebench-bool-elapsed-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2711
|
-
|| ! grep -Fq 'runner-nan-metadata' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2712
|
-
|| ! grep -Fq 'runner-nan-expected' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2713
|
-
|| ! grep -Fq 'json.loads(line, parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/run-swebench-solver-batch.sh \
|
|
2714
|
-
|| ! grep -Fq 'nan-instance-row' benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh \
|
|
2715
|
-
|| ! grep -Fq 'manifest = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-iter-0033c.sh \
|
|
2716
|
-
|| ! grep -Fq 'manifest fixtures_pair_eligible must be a string array' benchmark/auto-resolve/scripts/run-iter-0033c.sh \
|
|
2717
|
-
|| ! grep -Fq 'parsed = json.loads(ln, parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/iter-0033c-compare.py \
|
|
2718
|
-
|| ! grep -Fq 'data = loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/judge.sh \
|
|
2719
|
-
|| ! grep -Fq 'decoder = json.JSONDecoder(parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/judge.sh \
|
|
2720
|
-
|| ! grep -Fq 'judge = loads_strict_json_object(judge_path.read_text())' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2721
|
-
|| ! grep -Fq 'gpt = loads_strict_json_object(pathlib.Path(sys.argv[2]).read_text())' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2722
|
-
|| ! grep -Fq 'decoder = json.JSONDecoder(parse_constant=reject_json_constant)' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2723
|
-
|| ! grep -Fq 'g = loads_strict_json_object(g_f.read_text())' benchmark/auto-resolve/scripts/judge-opus-pass.sh \
|
|
2724
|
-
|| ! grep -Fq 'swebench-malformed-pair-judge-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2725
|
-
|| ! grep -Fq 'swebench-malformed-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2726
|
-
|| ! grep -Fq 'swebench-unknown-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2727
|
-
|| ! grep -Fq 'swebench-normalized-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2728
|
-
|| ! grep -Fq 'swebench-mixed-unknown-pair-trigger-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2729
|
-
|| ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger missing or malformed"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2730
|
-
|| ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger reasons missing known trigger reason"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2731
|
-
|| ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger reasons contain unknown trigger reason"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2732
|
-
|| ! grep -Fq '"classification": "failed attempt: pair trigger contract: pair_trigger reasons missing canonical trigger reason"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2733
|
-
|| ! grep -Fq 'historical-only-pair-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
2734
|
-
|| ! grep -Fq 'HISTORICAL_ONLY_TRIGGER_RUN_ID' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2735
|
-
|| ! grep -Fq '| Pair trigger |' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2736
|
-
|| ! grep -Fq '"verify_findings_count": "2"' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2737
|
-
|| ! grep -Fq '"pair_found_more_low_or_worse": false' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2738
|
-
|| ! grep -Fq 'pair-trigger eligibility/contract failures' benchmark/auto-resolve/README.md \
|
|
2739
|
-
|| ! grep -Fq 'swebench-string-bool-matrix-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2740
|
-
|| ! grep -Fq 'findings = raw_findings if isinstance(raw_findings, list) else []' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2741
|
-
|| ! grep -Fq 'oracle artifact malformed or unreadable' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2742
|
-
|| ! grep -Fq 'scope-tier-a-nan' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2743
|
-
|| ! grep -Fq 'scope-tier-b-nan' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2744
|
-
|| ! grep -Fq 'expected.json malformed: spec_output_files must be a string array' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2745
|
-
|| ! grep -Fq 'loads_strict_json_object(exp_path.read_text())' benchmark/auto-resolve/scripts/oracle-scope-tier-a.py \
|
|
2746
|
-
|| ! grep -Fq 'loads_strict_json_object(pathlib.Path(args.expected).read_text())' benchmark/auto-resolve/scripts/oracle-scope-tier-b.py \
|
|
2747
|
-
|| ! grep -Fq 'failed attempt: malformed compare' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2748
|
-
|| ! grep -Fq 'swebench-nan-matrix-test' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2749
|
-
|| ! grep -Fq 'parse_prepared_case' benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py \
|
|
2750
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text(encoding="utf8"))' benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py \
|
|
2751
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/collect-swebench-predictions.py \
|
|
2752
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py \
|
|
2753
|
-
|| ! grep -Fq 'loads_strict_json_object(stdout)' benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py \
|
|
2754
|
-
|| ! grep -Fq 'batch-bad-timeout' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2755
|
-
|| ! grep -Fq 'prepare-bad-timeout' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2756
|
-
|| ! grep -Fq 'prepare-nan-case' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2757
|
-
|| ! grep -Fq 'missing patch.diff for instance ids' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2758
|
-
|| ! grep -Fq 'no non-empty patches collected' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2759
|
-
|| ! grep -Fq 'collect-nan-instances' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2760
|
-
|| ! grep -Fq 'batch-nan-predictions' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2761
|
-
|| ! grep -Fq 'invalid JSON numeric constant: NaN' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2762
|
-
|| ! grep -Fq 'fetch-empty-limit' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2763
|
-
|| ! grep -Fq 'batch-empty-limit' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2764
|
-
|| ! grep -Fq 'no prediction instances selected' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2765
|
-
|| ! grep -Fq 'manifest malformed: expected JSON object' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2766
|
-
|| ! grep -Fq 'corpus-manifest-nan' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2767
|
-
|| ! grep -Fq 'loads_strict_json_object(manifest_path.read_text())' benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh \
|
|
2768
|
-
|| ! grep -Fq 'loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())' benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh \
|
|
2769
|
-
|| ! grep -Fq 'manifest malformed: prepared must be a non-empty array' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2770
|
-
|| ! grep -Fq 'prepared[1] expected JSON object' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2771
|
-
|| ! grep -Fq 'run ids malformed: no run ids' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2772
|
-
|| ! grep -Fq 'run ids malformed: line 2 is empty' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2773
|
-
|| ! grep -Fq 'run ids malformed: line 1 has unsafe run id' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2774
|
-
|| ! grep -Fq -- '--fixture must match [A-Za-z0-9_.-]+' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2775
|
-
|| ! grep -Fq -- '--run-id must match [A-Za-z0-9_.-]+' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2776
|
-
|| ! grep -Fq 'unsafe SWE-bench repo' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2777
|
-
|| ! grep -Fq 'unsafe SWE-bench base_commit' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2778
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py \
|
|
2779
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/fetch-swebench-instances.py \
|
|
2780
|
-
|| ! grep -Fq 'malformed fetched row 1: row must be object' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2781
|
-
|| ! grep -Fq 'solver-nan' benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh \
|
|
2782
|
-
|| ! grep -Fq 'malformed scores_by_arm must not provide arm scores' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
|
|
2783
|
-
|| ! grep -Fq 'non-dict result.json must not expose result fields' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
|
|
2784
|
-
|| ! grep -Fq 'NaN result numeric fields must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
|
|
2785
|
-
|| ! grep -Fq 'score_for(judge' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
|
|
2786
|
-
|| ! grep -Fq 'from pair_evidence_contract import is_score, is_strict_number' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
|
|
2787
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text(encoding="utf8"))' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
|
|
2788
|
-
|| ! grep -Fq 'return legacy if is_score(legacy) else None' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
|
|
2789
|
-
|| ! grep -Fq 'def strict_number' benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
|
|
2790
|
-
|| ! grep -Fq 'out-of-range scores must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
|
|
2791
|
-
|| ! grep -Fq 'boolean scores must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
|
|
2792
|
-
|| ! grep -Fq 'boolean result numeric fields must not appear in L1 summary' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh \
|
|
2793
|
-
|| ! grep -Fq 'lowercase c_score' benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh; then
|
|
2794
|
-
offenders="${offenders}"$'\n'"benchmark gates must require judge _blind_mapping before accepting score evidence"
|
|
2795
|
-
fi
|
|
2796
|
-
if ! grep -Fq 'solo_claude beats bare' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2797
|
-
|| ! grep -Fq 'mapped_score' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2798
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
|
|
2799
|
-
|| ! grep -Fq 'c1-nan-score' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2800
|
-
|| ! grep -Fq 'dirty F5 L1<=L0 row must not be promoted' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2801
|
-
|| ! grep -Fq 'l1-rerun-summary must not override pre-registered C1 selection grounds' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh \
|
|
2802
|
-
|| ! grep -Fq 'wrong mapping' benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh; then
|
|
2803
|
-
offenders="${offenders}"$'\n'"build-pair-eligible-manifest.py must use arm-mapped clean scores for F9 and L1<=L0 promotion"
|
|
2804
|
-
fi
|
|
2805
|
-
if [ -f benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/judge.json ] \
|
|
2806
|
-
&& ! python3 benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2807
|
-
--run-id 20260510-f16-f23-f25-combined-proof \
|
|
2808
|
-
--pair-arm l2_risk_probes \
|
|
2809
|
-
--min-fixtures 3 \
|
|
2810
|
-
--min-pair-margin 5 \
|
|
2811
|
-
--max-pair-solo-wall-ratio 3 \
|
|
2812
|
-
--out-json /tmp/devlyn-lint-f16-f23-f25-pair-gate.json \
|
|
2813
|
-
--out-md /tmp/devlyn-lint-f16-f23-f25-pair-gate.md >/dev/null 2>&1; then
|
|
2814
|
-
offenders="${offenders}"$'\n'"current local artifacts for 20260510-f16-f23-f25-combined-proof must re-gate as PASS"
|
|
2815
|
-
fi
|
|
2816
|
-
if [ -f benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/judge.json ] \
|
|
2817
|
-
&& ! python3 benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
2818
|
-
--run-id 20260511-f21-current-riskprobes-v1 \
|
|
2819
|
-
--pair-arm l2_risk_probes \
|
|
2820
|
-
--min-fixtures 1 \
|
|
2821
|
-
--min-pair-margin 5 \
|
|
2822
|
-
--max-pair-solo-wall-ratio 3 \
|
|
2823
|
-
--out-json /tmp/devlyn-lint-f21-pair-gate.json \
|
|
2824
|
-
--out-md /tmp/devlyn-lint-f21-pair-gate.md >/dev/null 2>&1; then
|
|
2825
|
-
offenders="${offenders}"$'\n'"current local artifacts for 20260511-f21-current-riskprobes-v1 must re-gate as PASS"
|
|
2826
|
-
fi
|
|
2827
|
-
if ! grep -Fq 'bash scripts/lint-fixtures.sh' benchmark/auto-resolve/README.md; then
|
|
2828
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: gate-change instructions must include fixture schema lint"
|
|
2829
|
-
fi
|
|
2830
|
-
if ! python3 - <<'PY'
|
|
2831
|
-
import runpy
|
|
2832
|
-
|
|
2833
|
-
runtime = runpy.run_path("config/skills/_shared/verify-merge-findings.py")
|
|
2834
|
-
benchmark = runpy.run_path("benchmark/auto-resolve/scripts/pair_evidence_contract.py")
|
|
2835
|
-
|
|
2836
|
-
runtime_known = set(runtime["KNOWN_PAIR_TRIGGER_REASONS"])
|
|
2837
|
-
canonical = set(benchmark["CANONICAL_PAIR_TRIGGER_REASONS"])
|
|
2838
|
-
aliases = set(benchmark["HISTORICAL_PAIR_TRIGGER_REASON_ALIASES"])
|
|
2839
|
-
normalized_aliases = set(benchmark["HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES"])
|
|
2840
|
-
known = set(benchmark["KNOWN_PAIR_TRIGGER_REASONS"])
|
|
2841
|
-
expected_aliases = {"risk_profile.high_risk", "risk_probes_enabled"}
|
|
2842
|
-
expected_normalized_aliases = {
|
|
2843
|
-
"complexity.high.spec.frontmatter",
|
|
2844
|
-
"frontmatter.complexity.high",
|
|
2845
|
-
"high.complexity.spec",
|
|
2846
|
-
"high.risk.profile",
|
|
2847
|
-
"spec.frontmatter.complexity.high",
|
|
2848
|
-
"state.complexity.high",
|
|
2849
|
-
}
|
|
2850
|
-
|
|
2851
|
-
errors = []
|
|
2852
|
-
if canonical != runtime_known:
|
|
2853
|
-
errors.append("benchmark canonical pair-trigger reasons must match runtime reasons")
|
|
2854
|
-
if aliases != expected_aliases:
|
|
2855
|
-
errors.append("benchmark historical pair-trigger aliases changed unexpectedly")
|
|
2856
|
-
if normalized_aliases != expected_normalized_aliases:
|
|
2857
|
-
errors.append("benchmark normalized historical pair-trigger aliases changed unexpectedly")
|
|
2858
|
-
if known != canonical | aliases:
|
|
2859
|
-
errors.append("benchmark known pair-trigger reasons must be canonical plus aliases")
|
|
2860
|
-
if runtime_known & aliases:
|
|
2861
|
-
errors.append("runtime pair-trigger reasons must not accept benchmark-only aliases")
|
|
2862
|
-
if not benchmark["has_historical_pair_trigger_reason"](["risk_profile.high_risk"]):
|
|
2863
|
-
errors.append("benchmark historical alias helper must detect exact aliases")
|
|
2864
|
-
if benchmark["has_historical_pair_trigger_reason"](["complexity.high"]):
|
|
2865
|
-
errors.append("benchmark historical alias helper must not count canonical reasons")
|
|
2866
|
-
if benchmark["is_known_pair_trigger_reason"]("risk high"):
|
|
2867
|
-
errors.append("benchmark pair-trigger reader must reject normalized canonical reason strings")
|
|
2868
|
-
if benchmark["is_canonical_pair_trigger_reason"]("risk high"):
|
|
2869
|
-
errors.append("benchmark pair-trigger canonical helper must be exact-match only")
|
|
2870
|
-
if not benchmark["is_known_pair_trigger_reason"]("high.risk.profile"):
|
|
2871
|
-
errors.append("benchmark pair-trigger reader must preserve documented normalized historical aliases")
|
|
2872
|
-
if benchmark["is_canonical_pair_trigger_reason"]("high.risk.profile"):
|
|
2873
|
-
errors.append("benchmark normalized historical aliases must not count as canonical reasons")
|
|
2874
|
-
if errors:
|
|
2875
|
-
raise SystemExit("\n".join(errors))
|
|
2876
|
-
PY
|
|
2877
|
-
then
|
|
2878
|
-
offenders="${offenders}"$'\n'"benchmark pair-trigger aliases must stay benchmark-only and runtime canonical reasons must stay in sync"
|
|
2879
|
-
fi
|
|
2880
|
-
if ! grep -Fq 'bash scripts/lint-shadow-fixtures.sh' benchmark/auto-resolve/README.md \
|
|
2881
|
-
|| ! grep -Fq 'The headroom and pair candidate runners' benchmark/auto-resolve/README.md \
|
|
2882
|
-
|| ! grep -Fq 'accept explicitly named `S*` ids for dry-run checks and candidate measurement' benchmark/auto-resolve/README.md \
|
|
2883
|
-
|| ! grep -Fq 'Use `run-suite.sh --suite shadow` only with `--dry-run`' benchmark/auto-resolve/README.md \
|
|
2884
|
-
|| ! grep -Fq 'rejected/smoke controls do not' benchmark/auto-resolve/README.md \
|
|
2885
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark headroom --dry-run --min-fixtures 1 S1-cli-lang-flag' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2886
|
-
|| ! grep -Fq 'Use non-dry-run headroom/pair only for' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2887
|
-
|| ! grep -Fq 'explicitly named `S*` candidates with a solo-headroom hypothesis' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2888
|
-
|| ! grep -Fq 'promote a validated `S*` task to an active `F*`' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2889
|
-
|| ! grep -Fq 'shadow suite run-suite is dry-run only' benchmark/auto-resolve/scripts/run-suite.sh \
|
|
2890
|
-
|| ! grep -Fq 'run-suite.sh --suite shadow --dry-run' benchmark/auto-resolve/scripts/run-suite.sh \
|
|
2891
|
-
|| ! grep -Fq 'shadow suite refuses provider/judge runs' benchmark/auto-resolve/scripts/run-suite.sh \
|
|
2892
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark suite --suite shadow --dry-run' bin/devlyn.js \
|
|
2893
|
-
|| ! grep -Fq 'Use benchmark headroom/pair with explicit S* candidates for real provider measurement.' benchmark/auto-resolve/scripts/run-suite.sh \
|
|
2894
|
-
|| ! grep -Fq 'shadow-suite-provider-run' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2895
|
-
|| ! grep -Fq 'shadow-suite-judge-only-provider-run' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2896
|
-
|| ! grep -Fq 'arg-parse-shadow-cli-suite-dry-run' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2897
|
-
|| ! grep -Fq 'shadow suite dry-run must not invite a blocked non-dry-run suite invocation' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2898
|
-
|| ! grep -Fq '`run-suite.sh --suite shadow` is dry-run only' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2899
|
-
|| ! grep -Fq 'parallel to the active golden `F*` fixtures' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2900
|
-
|| ! grep -Fq 'Active golden `F*` fixtures and pair-evidence audits control' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2901
|
-
|| ! grep -Fq 'Default `--suite golden` keeps active golden `F*` behavior' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2902
|
-
|| ! grep -Fq 'before any bare/solo/pair measurement' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2903
|
-
|| ! grep -Fq 'Do not spend' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2904
|
-
|| ! grep -Fq 'real provider calls on S1' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2905
|
-
|| ! grep -Fq 'not a solo<pair evidence candidate' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2906
|
-
|| grep -Fq 'F1-F9 still controls release' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2907
|
-
|| grep -Fq 'existing F1-F9 behavior' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2908
|
-
|| grep -Fq 'L0/L1/L2 measurement' benchmark/auto-resolve/shadow-fixtures/README.md; then
|
|
2909
|
-
offenders="${offenders}"$'\n'"shadow fixture docs must describe lint and S* headroom/pair candidate calibration before golden promotion"
|
|
2910
|
-
fi
|
|
2911
|
-
if ! grep -Fq 'write a solo-headroom hypothesis' benchmark/auto-resolve/README.md \
|
|
2912
|
-
|| ! grep -Fq 'each new pair-candidate shadow fixture needs a' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2913
|
-
|| ! grep -Fq "candidate's \`spec.md\`: name the visible behavior a capable \`solo_claude\`" benchmark/auto-resolve/README.md \
|
|
2914
|
-
|| ! grep -Fq '`solo_claude` baseline is expected to miss' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2915
|
-
|| ! grep -Fq "candidate's \`spec.md\`" benchmark/auto-resolve/README.md \
|
|
2916
|
-
|| ! grep -Fq 'solo-headroom hypothesis in `spec.md`' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2917
|
-
|| ! grep -Fq 'observable command from `expected.json`' benchmark/auto-resolve/README.md \
|
|
2918
|
-
|| ! grep -Fq 'observable command from' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2919
|
-
|| ! grep -Fq 'candidate runners enforce this as an actionable hypothesis' benchmark/auto-resolve/README.md \
|
|
2920
|
-
|| ! grep -Fq 'runners enforce this as an actionable hypothesis' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2921
|
-
|| ! grep -Fq 'backticked observable command matching `expected.json`' benchmark/auto-resolve/README.md \
|
|
2922
|
-
|| ! grep -Fq 'backticked observable command matching `expected.json`' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2923
|
-
|| ! grep -Fq 'command/observable' benchmark/auto-resolve/README.md \
|
|
2924
|
-
|| ! grep -Fq 'command/observable' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2925
|
-
|| ! grep -Fq 'itself containing `miss`' benchmark/auto-resolve/README.md \
|
|
2926
|
-
|| ! grep -Fq 'itself containing `miss`' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2927
|
-
|| ! grep -Fq '## Solo ceiling avoidance' benchmark/auto-resolve/README.md \
|
|
2928
|
-
|| ! grep -Fq '## Solo ceiling avoidance' benchmark/auto-resolve/shadow-fixtures/README.md \
|
|
2929
|
-
|| ! grep -Fq 'solo-saturated `S2`-`S6` controls' benchmark/auto-resolve/README.md \
|
|
2930
|
-
|| ! grep -Fq 'calibrated solo-saturated controls (`S2`-`S6`)' benchmark/auto-resolve/shadow-fixtures/README.md; then
|
|
2931
|
-
offenders="${offenders}"$'\n'"benchmark shadow candidate docs must require solo-headroom hypothesis and solo-ceiling avoidance before provider spend"
|
|
2932
|
-
fi
|
|
2933
|
-
if [ ! -f benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py ] \
|
|
2934
|
-
|| ! grep -Fq 'def has_actionable_solo_headroom_hypothesis_text' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
2935
|
-
|| ! grep -Fq '"solo-headroom hypothesis" in lower' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
2936
|
-
|| ! grep -Fq 'and "solo_claude" in lower' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
2937
|
-
|| ! grep -Fq 'and "miss" in lower' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
2938
|
-
|| ! grep -Fq 'OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
2939
|
-
|| ! grep -Fq 'def is_command_like_backtick' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
2940
|
-
|| ! grep -Fq 'def path_has_actionable_solo_headroom_hypothesis' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
2941
|
-
|| ! grep -Fq 'has_actionable_solo_headroom_hypothesis_text(text)' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
|
|
2942
|
-
|| ! grep -Fq 'actionable_observable_commands(text)' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
|
|
2943
|
-
|| ! grep -Fq -- '--expected-json' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
|
|
2944
|
-
|| ! grep -Fq 'expected_commands(args.expected_json)' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
|
|
2945
|
-
|| ! grep -Fq 'expected UTF-8 text' benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py \
|
|
2946
|
-
|| ! grep -Fq 'solo-headroom-hypothesis.py' scripts/lint-shadow-fixtures.sh \
|
|
2947
|
-
|| ! grep -Fq 'has_solo_ceiling_avoidance_note' scripts/lint-shadow-fixtures.sh \
|
|
2948
|
-
|| ! grep -Fq 'solo-ceiling-avoidance.py' scripts/lint-shadow-fixtures.sh \
|
|
2949
|
-
|| ! grep -Fq 'SECTION_RE' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
|
|
2950
|
-
|| ! grep -Fq 'CONTROL_RE' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
|
|
2951
|
-
|| ! grep -Fq 'REASON_RE' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
|
|
2952
|
-
|| ! grep -Fq 'expected UTF-8 text' benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py \
|
|
2953
|
-
|| ! grep -Fq 'fixture_has_solo_ceiling_avoidance_note' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2954
|
-
|| ! grep -Fq 'fixture_has_solo_ceiling_avoidance_note' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2955
|
-
|| ! grep -Fq 'solo-ceiling-avoidance.py' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2956
|
-
|| ! grep -Fq 'solo-ceiling-avoidance.py' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2957
|
-
|| ! grep -Fq 'shadow fixture NOTES.md needs ## Solo ceiling avoidance' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2958
|
-
|| ! grep -Fq 'shadow fixture NOTES.md needs ## Solo ceiling avoidance' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2959
|
-
|| ! grep -Fq 'solo-headroom-hypothesis.py' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2960
|
-
|| ! grep -Fq 'solo-headroom-hypothesis.py' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2961
|
-
|| ! grep -Fq -- '--expected-json "$dir/expected.json" "$dir/spec.md"' benchmark/auto-resolve/scripts/run-headroom-candidate.sh \
|
|
2962
|
-
|| ! grep -Fq -- '--expected-json "$dir/expected.json" "$dir/spec.md"' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
2963
|
-
|| ! grep -Fq -- '--expected-json "$d/expected.json" "$spec"' scripts/lint-shadow-fixtures.sh \
|
|
2964
|
-
|| ! grep -Fq 'from expected.json before provider spend' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2965
|
-
|| ! grep -Fq 'from expected.json before provider spend' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2966
|
-
|| ! grep -Fq 'unrelated-backtick-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2967
|
-
|| ! grep -Fq 'unrelated-backtick-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2968
|
-
|| ! grep -Fq 'observable-without-miss-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2969
|
-
|| ! grep -Fq 'observable-without-miss-solo-headroom-hypothesis' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2970
|
-
|| ! grep -Fq 'missing-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2971
|
-
|| ! grep -Fq 'missing-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2972
|
-
|| ! grep -Fq 'weak-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh \
|
|
2973
|
-
|| ! grep -Fq 'weak-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
2974
|
-
|| ! grep -Fq 'hypothesis with unrelated backtick must fail' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2975
|
-
|| ! grep -Fq 'weak hypothesis without observable command must fail' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2976
|
-
|| ! grep -Fq 'actionable-hypothesis.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2977
|
-
|| ! grep -Fq 'docs-style-actionable-hypothesis.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2978
|
-
|| ! grep -Fq 'shadow-missing-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2979
|
-
|| ! grep -Fq 'shadow-weak-solo-ceiling-avoidance' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2980
|
-
|| ! grep -Fq 'weak solo ceiling avoidance must fail' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2981
|
-
|| ! grep -Fq 'non-utf8-solo-ceiling.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
2982
|
-
|| ! grep -Fq 'non-utf8-hypothesis.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh; then
|
|
2983
|
-
offenders="${offenders}"$'\n'"solo-headroom hypothesis provider-spend guards must share one actionable checker and test weak-vs-actionable cases"
|
|
2984
|
-
fi
|
|
2985
|
-
if ! grep -Fq 'pair-candidate-frontier.py' benchmark/auto-resolve/README.md \
|
|
2986
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark frontier --out-md /tmp/devlyn-pair-frontier.md' benchmark/auto-resolve/README.md \
|
|
2987
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark frontier --out-md /tmp/devlyn-pair-frontier.md' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2988
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --out-dir /tmp/devlyn-benchmark-audit' README.md \
|
|
2989
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --out-dir /tmp/devlyn-benchmark-audit' benchmark/auto-resolve/README.md \
|
|
2990
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --out-dir /tmp/devlyn-benchmark-audit' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2991
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' README.md \
|
|
2992
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/README.md \
|
|
2993
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/run-real-benchmark.md \
|
|
2994
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' bin/devlyn.js \
|
|
2995
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2996
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark frontier Show pair candidate frontier scores/triggers without providers' bin/devlyn.js \
|
|
2997
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark frontier Show pair candidate frontier scores/triggers without providers' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
2998
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit Audit pair evidence readiness' bin/devlyn.js \
|
|
2999
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit Audit pair evidence readiness' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3000
|
-
|| ! grep -Fq 'Show active rejected/evidence/unmeasured pair candidates, scores, and triggers without providers' bin/devlyn.js \
|
|
3001
|
-
|| ! grep -Fq 'Show active rejected/evidence/unmeasured pair candidates, scores, and triggers without providers' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3002
|
-
|| ! grep -Fq 'Prints pair evidence score rows with trigger reasons; --out-md includes a Triggers column' bin/devlyn.js \
|
|
3003
|
-
|| ! grep -Fq 'Prints pair evidence score rows with trigger reasons; --out-md includes a Triggers column' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3004
|
-
|| ! grep -Fq 'Prints frontier score rows plus headroom and pair quality handoff rows' bin/devlyn.js \
|
|
3005
|
-
|| ! grep -Fq 'Prints frontier score rows plus headroom and pair quality handoff rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3006
|
-
|| ! grep -Fq 'Prints frontier score rows plus headroom_rejections=PASS/FAIL, pair_evidence_quality=PASS/FAIL, pair_trigger_reasons=PASS/FAIL, pair_evidence_hypotheses=PASS/FAIL, pair_evidence_hypothesis_triggers=PASS/WARN/FAIL, historical-alias, and hypothesis-trigger gap handoff rows' bin/devlyn.js \
|
|
3007
|
-
|| ! grep -Fq 'Prints frontier score rows plus headroom_rejections=PASS/FAIL, pair_evidence_quality=PASS/FAIL, pair_trigger_reasons=PASS/FAIL, pair_evidence_hypotheses=PASS/FAIL, pair_evidence_hypothesis_triggers=PASS/WARN/FAIL, historical-alias, and hypothesis-trigger gap handoff rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3008
|
-
|| ! grep -Fq "audit: 'audit-pair-evidence.py'" bin/devlyn.js \
|
|
3009
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit [options]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3010
|
-
|| ! grep -Fq -- '--min-pair-evidence N default: 4' bin/devlyn.js \
|
|
3011
|
-
|| ! grep -Fq -- '--min-pair-evidence N default: 4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3012
|
-
|| ! grep -Fq -- '--min-pair-margin N default: 5' bin/devlyn.js \
|
|
3013
|
-
|| ! grep -Fq -- '--min-pair-margin N default: 5' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3014
|
-
|| ! grep -Fq -- '--max-pair-solo-wall-ratio N default: 3' bin/devlyn.js \
|
|
3015
|
-
|| ! grep -Fq -- '--max-pair-solo-wall-ratio N default: 3' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3016
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger' bin/devlyn.js \
|
|
3017
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3018
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3019
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit-headroom --out-json /tmp/devlyn-headroom-audit.json' README.md \
|
|
3020
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit-headroom --out-json /tmp/devlyn-headroom-audit.json' benchmark/auto-resolve/README.md \
|
|
3021
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit-headroom --out-json /tmp/devlyn-headroom-audit.json' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3022
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit-headroom Audit failed headroom results' bin/devlyn.js \
|
|
3023
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit-headroom Audit failed headroom results' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3024
|
-
|| ! grep -Fq "'audit-headroom': 'audit-headroom-rejections.py'" bin/devlyn.js \
|
|
3025
|
-
|| ! grep -Fq 'npx devlyn-cli benchmark audit-headroom [options]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3026
|
-
|| ! grep -Fq 'benchmarkMode === '\''frontier'\''' bin/devlyn.js \
|
|
3027
|
-
|| ! grep -Fq 'test-pair-candidate-frontier.sh' benchmark/auto-resolve/README.md \
|
|
3028
|
-
|| ! grep -Fq 'audit-pair-evidence.py' benchmark/auto-resolve/README.md \
|
|
3029
|
-
|| ! grep -Fq 'PASS audit-pair-evidence' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3030
|
-
|| ! grep -Fq 'FAIL audit-pair-evidence' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3031
|
-
|| ! grep -Fq 'FAIL audit-pair-evidence' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3032
|
-
|| ! grep -Fq 'audit.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3033
|
-
|| ! grep -Fq 'audit.json' README.md \
|
|
3034
|
-
|| ! grep -Fq 'audit.json' benchmark/auto-resolve/README.md \
|
|
3035
|
-
|| ! grep -Fq 'audit.json' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3036
|
-
|| ! grep -Fq '"frontier_summary": frontier_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3037
|
-
|| ! grep -Fq '"pair_evidence_rows": pair_evidence_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3038
|
-
|| ! grep -Fq 'def load_pair_evidence_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3039
|
-
|| ! grep -Fq 'def load_frontier_stdout_metrics' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3040
|
-
|| ! grep -Fq '"summary_rows": summary_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3041
|
-
|| ! grep -Fq '"aggregate_rows": aggregate_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3042
|
-
|| ! grep -Fq '"trigger_rows": trigger_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3043
|
-
|| ! grep -Fq '"trigger_rows_match_count": trigger_rows == expected_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3044
|
-
|| ! grep -Fq '"hypothesis_trigger_rows_match_count": hypothesis_trigger_rows == expected_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3045
|
-
|| ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3046
|
-
|| ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3047
|
-
|| ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3048
|
-
|| ! grep -Fq 'def reject_json_constant' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3049
|
-
|| ! grep -Fq 'def loads_strict_json_object' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3050
|
-
|| ! grep -Fq 'parse_constant=reject_json_constant' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3051
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3052
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/headroom-gate.py \
|
|
3053
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3054
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3055
|
-
|| ! grep -Fq 'loads_strict_json_object(path.read_text())' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3056
|
-
|| ! grep -Fq 'nan-json-constant' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3057
|
-
|| ! grep -Fq 'nan-result-json' benchmark/auto-resolve/scripts/test-headroom-gate.sh \
|
|
3058
|
-
|| ! grep -Fq 'nan-result-json' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3059
|
-
|| ! grep -Fq 'CANONICAL_PAIR_TRIGGER_REASONS = {' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3060
|
-
|| ! grep -Fq 'HISTORICAL_PAIR_TRIGGER_REASON_ALIASES = {' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3061
|
-
|| ! grep -Fq 'HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES = {' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3062
|
-
|| ! grep -Fq 'Benchmark readers accept historical aliases only for archived artifacts' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3063
|
-
|| ! grep -Fq 'KNOWN_PAIR_TRIGGER_REASONS = (' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3064
|
-
|| ! grep -Fq 'mode.pair-verify' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3065
|
-
|| ! grep -Fq 'def normalized_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3066
|
-
|| ! grep -Fq 'def is_canonical_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3067
|
-
|| ! grep -Fq 'def has_known_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3068
|
-
|| ! grep -Fq 'def all_known_pair_trigger_reasons' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3069
|
-
|| ! grep -Fq 'def has_canonical_pair_trigger_reason' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3070
|
-
|| ! grep -Fq 'spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3071
|
-
|| ! grep -Fq 'risk_profile.high_risk' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3072
|
-
|| ! grep -Fq 'has_known_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3073
|
-
|| ! grep -Fq 'all_known_pair_trigger_reasons(reasons)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3074
|
-
|| ! grep -Fq 'has_canonical_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3075
|
-
|| ! grep -Fq '"pair_trigger_has_canonical_reason": has_canonical_pair_trigger_reason(' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3076
|
-
|| ! grep -Fq '"pair_trigger_has_hypothesis_reason": (' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3077
|
-
|| ! grep -Fq '"require_hypothesis_trigger": args.require_hypothesis_trigger' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3078
|
-
|| ! grep -Fq '"pair_trigger_has_canonical_reason": true' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3079
|
-
|| ! grep -Fq '"pair_trigger_has_hypothesis_reason": true' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3080
|
-
|| ! grep -Fq '"require_hypothesis_trigger": true' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3081
|
-
|| ! grep -Fq 'pair_trigger eligible with a canonical reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3082
|
-
|| ! grep -Fq 'pair_trigger eligible with canonical reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3083
|
-
|| ! grep -Fq 'pair_trigger reasons missing known trigger reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3084
|
-
|| ! grep -Fq 'pair_trigger reasons contain unknown trigger reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3085
|
-
|| ! grep -Fq 'pair_trigger reasons missing canonical trigger reason' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3086
|
-
|| ! grep -Fq 'fixture_spec_has_solo_headroom_hypothesis(fixture_dir.name)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3087
|
-
|| ! grep -Fq 'pair_trigger missing spec.solo_headroom_hypothesis' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3088
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3089
|
-
|| ! grep -Fq -- '--require-hypothesis-trigger' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
3090
|
-
|| ! grep -Fq 'release audit: npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/scripts/run-full-pipeline-pair-candidate.sh \
|
|
3091
|
-
|| ! grep -Fq 'release audit: npx devlyn-cli benchmark audit --require-hypothesis-trigger --out-dir /tmp/devlyn-benchmark-audit-strict' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
3092
|
-
|| ! grep -Fq 'missing-hypothesis-trigger' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3093
|
-
|| ! grep -Fq 'grep -Fq -- '\''--require-hypothesis-trigger'\''' benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh \
|
|
3094
|
-
|| ! grep -Fq 'def fmt_trigger_reasons' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3095
|
-
|| ! grep -Fq 'Hypothesis trigger required: {str(report' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3096
|
-
|| ! grep -Fq '| Fixture | Bare | Bare headroom | Solo_claude | Solo_claude headroom | Pair | Margin | Pair mode | Hypothesis trigger | Triggers | Wall ratio | Status | Reason |' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3097
|
-
|| ! grep -Fq '| Fixture | Bare | Bare headroom | Solo_claude | Solo_claude headroom | Pair | Margin | Pair mode | Hypothesis trigger | Triggers | Wall ratio | Status | Reason |' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3098
|
-
|| ! grep -Fq 'trigger-reason, and wall-ratio columns' benchmark/auto-resolve/README.md \
|
|
3099
|
-
|| ! grep -Fq 'eligible with non-empty reasons and at least one canonical reason' benchmark/auto-resolve/README.md \
|
|
3100
|
-
|| ! grep -Fq 'fixtures with an actionable solo-headroom hypothesis must include `spec.solo_headroom_hypothesis` in the trigger reasons' benchmark/auto-resolve/README.md \
|
|
3101
|
-
|| ! grep -Fq 'pair trigger eligibility, trigger reasons, canonical-trigger coverage, and `spec.solo_headroom_hypothesis` coverage' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3102
|
-
|| ! grep -Fq 'unknown-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3103
|
-
|| ! grep -Fq 'mixed-unknown-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3104
|
-
|| ! grep -Fq 'normalized-canonical-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3105
|
-
|| ! grep -Fq 'historical-only-pair-trigger-reason' benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh \
|
|
3106
|
-
|| ! grep -Fq 'unknown-pair-trigger' benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh \
|
|
3107
|
-
|| ! grep -Fq 'def normalize_pair_evidence_row' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3108
|
-
|| ! grep -Fq 'def best_pair_evidence' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3109
|
-
|| ! grep -Fq 'ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3110
|
-
|| ! grep -Fq 'pair_arm not in ALLOWED_PAIR_ARMS' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3111
|
-
|| ! grep -Fq 'pair_mode = row.get("pair_mode")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3112
|
-
|| ! grep -Fq 'if pair_mode is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3113
|
-
|| ! grep -Fq 'pair_trigger_eligible = row.get("pair_trigger_eligible")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3114
|
-
|| ! grep -Fq 'if pair_trigger_eligible is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3115
|
-
|| ! grep -Fq 'pair_trigger_reasons = row.get("pair_trigger_reasons")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3116
|
-
|| ! grep -Fq 'isinstance(pair_trigger_reasons, list)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3117
|
-
|| ! grep -Fq 'all_known_pair_trigger_reasons(pair_trigger_reasons)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3118
|
-
|| ! grep -Fq 'has_canonical_pair_trigger_reason(pair_trigger_reasons)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3119
|
-
|| ! grep -Fq '"pair_trigger_reasons": pair_trigger_reasons' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3120
|
-
|| ! grep -Fq '"pair_trigger_has_canonical_reason": True' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3121
|
-
|| ! grep -Fq '"pair_trigger_has_hypothesis_reason": (' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3122
|
-
|| ! grep -Fq 'def pair_result_trigger_reasons' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3123
|
-
|| ! grep -Fq 'candidate_row = dict(row)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3124
|
-
|| ! grep -Fq 'all_known_pair_trigger_reasons(reasons)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3125
|
-
|| ! grep -Fq 'has_canonical_pair_trigger_reason(reasons)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3126
|
-
|| ! grep -Fq 'candidate_row["pair_trigger_reasons"] = reasons' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3127
|
-
|| ! grep -Fq '"reasons": ["complexity.high", "looks-hard"]' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3128
|
-
|| ! grep -Fq '"reasons":["risk high"]' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3129
|
-
|| ! grep -Fq 'triggers={triggers}' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3130
|
-
|| ! grep -Fq 'hypothesis_trigger={hypothesis_trigger}' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3131
|
-
|| ! grep -Fq '| Fixture | Status | Verdict | Evidence | Pair arm | Triggers | Hypothesis trigger |' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3132
|
-
|| ! grep -Fq 'pair-arm, and trigger-reason columns' benchmark/auto-resolve/README.md \
|
|
3133
|
-
|| ! grep -Fq '| Fixture | Status | Verdict | Evidence | Pair arm | Triggers | Hypothesis trigger |' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3134
|
-
|| ! grep -Fq 'def check_pair_evidence_hypotheses' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3135
|
-
|| ! grep -Fq 'def pair_evidence_hypothesis_trigger_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3136
|
-
|| ! grep -Fq 'def pair_evidence_hypothesis_trigger_gap_details' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3137
|
-
|| ! grep -Fq 'def check_pair_evidence_hypothesis_triggers' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3138
|
-
|| ! grep -Fq 'def print_pair_evidence_hypothesis_triggers_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3139
|
-
|| ! grep -Fq 'pair_evidence_hypotheses={status}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3140
|
-
|| ! grep -Fq 'pair_evidence_hypothesis_triggers={status}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3141
|
-
|| ! grep -Fq 'pair evidence hypotheses missing for fixture(s)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3142
|
-
|| ! grep -Fq 'pair-evidence-hypotheses' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3143
|
-
|| ! grep -Fq 'pair_evidence_hypotheses=PASS documented=2 total=2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3144
|
-
|| ! grep -Fq 'pair_evidence_hypothesis_triggers=WARN matched=0 documented=2 total=2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3145
|
-
|| ! grep -Fq 'pair_evidence_hypothesis_trigger_gaps=F16-cli-quote-tax-rules=complexity.high;F21-cli-scheduler-priority=complexity.high,risk_profile.high_risk' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3146
|
-
|| ! grep -Fq 'pair_evidence_hypothesis_triggers=FAIL matched=0 documented=2 total=2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3147
|
-
|| ! grep -Fq 'strict benchmark audit must not report current hypothesis-trigger gaps' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3148
|
-
|| ! grep -Fq 'pair_evidence_hypothesis_triggers=PASS matched=4 documented=4 total=4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3149
|
-
|| ! grep -Fq 'verdict=pair_evidence_passed triggers={triggers}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3150
|
-
|| ! grep -Fq 'def format_trigger_reasons' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3151
|
-
|| ! grep -Fq 'missing-frontier-triggers' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3152
|
-
|| ! grep -Fq 'triggers=complexity.high' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3153
|
-
|| ! grep -Fq '"pair_trigger_reasons": pair_trigger_reasons(pair_result)' benchmark/auto-resolve/scripts/full-pipeline-pair-gate.py \
|
|
3154
|
-
|| ! grep -Fq '"pair_trigger_eligible": pair_trigger_eligible' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3155
|
-
|| ! grep -Fq '"verdict": "pair_evidence_passed"' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3156
|
-
|| ! grep -Fq '"pair_mode": pair_mode' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3157
|
-
|| ! grep -Fq 'def pair_result_trigger_reasons' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3158
|
-
|| ! grep -Fq 'candidate_row = dict(row)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3159
|
-
|| ! grep -Fq 'candidate_row["pair_trigger_reasons"] = reasons' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3160
|
-
|| ! grep -Fq 'F16-cli-quote-tax-rules/l2_risk_probes/result.json' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
3161
|
-
|| ! grep -Fq '"reasons":["risk high"]' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
3162
|
-
|| ! grep -Fq 'def is_strict_int' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3163
|
-
|| ! grep -Fq 'def is_score' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3164
|
-
|| ! grep -Fq '0 <= value <= 100' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3165
|
-
|| ! grep -Fq 'def is_strict_number' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3166
|
-
|| ! grep -Fq 'import math' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3167
|
-
|| ! grep -Fq 'math.isfinite(value)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3168
|
-
|| ! grep -Fq 'and value > 0' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3169
|
-
|| ! grep -Fq 'if pair_margin != pair_score - solo_score:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3170
|
-
|| ! grep -Fq '"min_pair_evidence": min_pair_evidence' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3171
|
-
|| ! grep -Fq '"actual_rows": len(pair_evidence_rows)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3172
|
-
|| ! grep -Fq '"rows_match_count": (' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3173
|
-
|| ! grep -Fq '"pair_evidence_quality": {' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3174
|
-
|| ! grep -Fq 'def check_pair_evidence_quality' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3175
|
-
|| ! grep -Fq 'def print_pair_evidence_quality' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3176
|
-
|| ! grep -Fq 'def check_pair_trigger_reasons' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3177
|
-
|| ! grep -Fq 'def print_pair_trigger_reasons_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3178
|
-
|| ! grep -Fq 'def pair_trigger_historical_alias_details' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3179
|
-
|| ! grep -Fq 'pair_evidence_quality={status} min_pair_margin_actual={actual_margin}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3180
|
-
|| ! grep -Fq 'pair_trigger_reasons={status} canonical={canonical} historical_alias={historical_alias}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3181
|
-
|| ! grep -Fq '"historical_alias_details": historical_alias_details' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3182
|
-
|| ! grep -Fq 'frontier pair_margin_min does not match pair evidence rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3183
|
-
|| ! grep -Fq 'frontier pair_solo_wall_ratio_max does not match pair evidence rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3184
|
-
|| ! grep -Fq 'is_strict_int(pair_evidence_count)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3185
|
-
|| ! grep -Fq 'if not is_strict_int(count):' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3186
|
-
|| ! grep -Fq 'pair evidence count missing or malformed from frontier report' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3187
|
-
|| ! grep -Fq 'pair evidence rows {len(rows)} do not match summary count {count}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3188
|
-
|| ! grep -Fq 'def check_frontier_report' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3189
|
-
|| ! grep -Fq 'frontier verdict {verdict!r} is not PASS' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3190
|
-
|| ! grep -Fq 'frontier has {unmeasured_count} unmeasured candidate fixture(s)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3191
|
-
|| ! grep -Fq '"frontier_report": {' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3192
|
-
|| ! grep -Fq '"min_pair_margin": min_pair_margin' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3193
|
-
|| ! grep -Fq '"max_pair_solo_wall_ratio": max_pair_solo_wall_ratio' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3194
|
-
|| ! grep -Fq 'not math.isfinite(args.max_pair_solo_wall_ratio)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3195
|
-
|| ! grep -Fq 'max-pair-solo-wall-ratio must be finite and > 0' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3196
|
-
|| ! grep -Fq 'pair evidence count {count} below required minimum {minimum}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3197
|
-
|| ! grep -Fq 'assert audit["checks"]["min_pair_evidence"]["actual_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3198
|
-
|| ! grep -Fq 'assert audit["checks"]["frontier_report"]["verdict"] == frontier["verdict"]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3199
|
-
|| ! grep -Fq 'pair_evidence_quality=PASS min_pair_margin_actual=+21 min_pair_margin_required=+5 max_wall_actual=2.25x max_wall_allowed=3.00x' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3200
|
-
|| ! grep -Fq 'pair_trigger_reasons=PASS canonical=4 historical_alias=0 exposed=4 total=4 summary=4 rows_match=true' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3201
|
-
|| ! grep -Fq 'current benchmark audit must not report historical aliases or hypothesis-trigger gaps' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3202
|
-
|| ! grep -Fq 'benchmark audit must fail when min pair evidence exceeds current evidence rows' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3203
|
-
|| ! grep -Fq 'actual_pair_evidence=$(python3 - "$TMP/audit/audit.json"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3204
|
-
|| ! grep -Fq 'required_pair_evidence=$((actual_pair_evidence + 1))' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3205
|
-
|| ! grep -Fq 'pair evidence count ${actual_pair_evidence} below required minimum ${required_pair_evidence}' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3206
|
-
|| ! grep -Fq 'pair_margin_avg=+27.25 pair_margin_min=+21 wall_avg=1.66x wall_max=2.25x' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3207
|
-
|| ! grep -Fq 'F16-cli-quote-tax-rules: bare=50 solo_claude=75 pair=96 arm=l2_risk_probes margin=+21' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3208
|
-
|| ! grep -Fq 'FAIL audit-pair-evidence' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3209
|
-
|| ! grep -Fq 'assert audit["checks"]["min_pair_evidence"]["required"] == required' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3210
|
-
|| ! grep -Fq 'assert audit["checks"]["pair_evidence_quality"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3211
|
-
|| ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["summary_pair_evidence_count"] == 4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3212
|
-
|| ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3213
|
-
|| ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["canonical_rows"] == 4' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3214
|
-
|| ! grep -Fq 'assert audit["checks"]["pair_trigger_reasons"]["historical_alias_details"] == []' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3215
|
-
|| ! grep -Fq 'assert len(audit["pair_evidence_rows"]) == frontier["pair_evidence_count"]' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3216
|
-
|| ! grep -Fq 'benchmark frontier must fail when active unmeasured candidates remain' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3217
|
-
|| ! grep -Fq 'PASS pair-candidate-frontier' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3218
|
-
|| ! grep -Fq 'FAIL pair-candidate-frontier' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3219
|
-
|| ! grep -Fq 'assert report["rows"][0]["status"] == "candidate_unmeasured"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3220
|
-
|| ! grep -Fq 'assert row["verdict"] == "pair_evidence_passed"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3221
|
-
|| ! grep -Fq 'assert row["pair_mode"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3222
|
-
|| ! grep -Fq 'assert row["pair_trigger_eligible"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3223
|
-
|| ! grep -Fq 'assert isinstance(row["pair_solo_wall_ratio"], (int, float))' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3224
|
-
|| ! grep -Fq 'assert report["frontier_summary"]["pair_margin_avg"] == 27' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3225
|
-
|| ! grep -Fq 'assert report["pair_evidence_rows"] == [' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3226
|
-
|| ! grep -Fq 'assert report["checks"]["min_pair_evidence"]["actual_rows"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3227
|
-
|| ! grep -Fq 'assert report["checks"]["min_pair_evidence"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3228
|
-
|| ! grep -Fq 'assert report["checks"]["min_pair_evidence"]["status"] == "FAIL"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3229
|
-
|| ! grep -Fq 'assert report["checks"]["pair_evidence_quality"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3230
|
-
|| ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3231
|
-
|| ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["summary_pair_evidence_count"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3232
|
-
|| ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3233
|
-
|| ! grep -Fq 'assert report["checks"]["pair_trigger_reasons"]["historical_alias_details"] == [' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3234
|
-
|| ! grep -Fq 'normalized-canonical-trigger-reason-rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3235
|
-
|| ! grep -Fq 'assert report["checks"]["pair_evidence_quality"]["min_pair_margin_actual"] == 21' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3236
|
-
|| ! grep -Fq 'pair_evidence_quality=PASS min_pair_margin_actual=+21 min_pair_margin_required=+5 max_wall_actual=1.47x max_wall_allowed=3.00x' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3237
|
-
|| ! grep -Fq 'pair_evidence_quality=FAIL min_pair_margin_actual=+4 min_pair_margin_required=+5 max_wall_actual=1.20x max_wall_allowed=3.00x' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3238
|
-
|| ! grep -Fq 'no-quality-rows-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3239
|
-
|| ! grep -Fq 'pair evidence quality check has no complete rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3240
|
-
|| ! grep -Fq 'low-quality-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3241
|
-
|| ! grep -Fq 'pair evidence margin below minimum for fixture(s): F16-cli-quote-tax-rules' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3242
|
-
|| ! grep -Fq 'high-wall-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3243
|
-
|| ! grep -Fq 'pair evidence wall ratio above maximum for fixture(s): F16-cli-quote-tax-rules' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3244
|
-
|| ! grep -Fq 'summary-mismatch-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3245
|
-
|| ! grep -Fq 'frontier pair_margin_min does not match pair evidence rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3246
|
-
|| ! grep -Fq 'summary-wall-mismatch-frontier.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3247
|
-
|| ! grep -Fq 'frontier pair_solo_wall_ratio_max does not match pair evidence rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3248
|
-
|| ! grep -Fq 'frontier-fail-verdict.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3249
|
-
|| ! grep -Fq 'frontier-unmeasured.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3250
|
-
|| ! grep -Fq 'frontier-malformed-unmeasured.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3251
|
-
|| ! grep -Fq 'frontier-incomplete-best.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3252
|
-
|| ! grep -Fq '"run_id": "lower-complete"' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3253
|
-
|| ! grep -Fq 'bool-frontier-count.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3254
|
-
|| ! grep -Fq 'malformed-pair-evidence-count' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3255
|
-
|| ! grep -Fq 'mismatched-frontier-rows.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3256
|
-
|| ! grep -Fq 'pair evidence rows 1 do not match summary count 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3257
|
-
|| ! grep -Fq 'bad-frontier-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3258
|
-
|| ! grep -Fq 'nan-frontier-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3259
|
-
|| ! grep -Fq 'nan-max-wall-ratio' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3260
|
-
|| ! grep -Fq 'mismatched-margin-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3261
|
-
|| ! grep -Fq 'mismatched-margin-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3262
|
-
|| ! grep -Fq 'overrange-score-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3263
|
-
|| ! grep -Fq 'overrange-score-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3264
|
-
|| ! grep -Fq 'invalid-pair-arm-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3265
|
-
|| ! grep -Fq 'invalid-pair-arm-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3266
|
-
|| ! grep -Fq 'false-pair-mode-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3267
|
-
|| ! grep -Fq 'false-pair-mode-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3268
|
-
|| ! grep -Fq 'zero-wall-row-fields.json' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3269
|
-
|| ! grep -Fq 'zero-wall-row-fields' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3270
|
-
|| ! grep -Fq 'pair evidence count 2 below required minimum 4' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3271
|
-
|| ! grep -Fq 'assert rows["F22-cli-low-margin"]["status"] == "candidate_unmeasured"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3272
|
-
|| ! grep -Fq 'assert rows["F22-cli-low-margin"]["status"] == "pair_evidence_passed"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3273
|
-
|| ! grep -Fq 'assert rows["F23-cli-high-wall"]["status"] == "candidate_unmeasured"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3274
|
-
|| ! grep -Fq 'assert rows["F23-cli-high-wall"]["status"] == "pair_evidence_passed"' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3275
|
-
|| ! grep -Fq 'def print_final_verdict' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3276
|
-
|| ! grep -Fq 'PASS pair-candidate-frontier' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3277
|
-
|| ! grep -Fq 'FAIL pair-candidate-frontier' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3278
|
-
|| ! grep -Fq 'PASS pair-candidate-frontier' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3279
|
-
|| ! grep -Fq 'FAIL pair-candidate-frontier' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3280
|
-
|| ! grep -Fq 'requires at least four fixtures with passing pair evidence' README.md \
|
|
3281
|
-
|| ! grep -Fq 'revalidates frontier `verdict: PASS`, zero unmeasured candidates' README.md \
|
|
3282
|
-
|| ! grep -Fq 'requires at least four active fixtures with passing pair evidence' benchmark/auto-resolve/README.md \
|
|
3283
|
-
|| ! grep -Fq 'revalidates frontier `verdict: PASS`' benchmark/auto-resolve/README.md \
|
|
3284
|
-
|| ! grep -Fq 'requires at least four active fixtures with passing pair evidence' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3285
|
-
|| ! grep -Fq 'revalidates frontier `verdict: PASS`, zero unmeasured candidates' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3286
|
-
|| ! grep -Fq 'counted by `benchmark audit` as the fourth passing pair-evidence row' benchmark/auto-resolve/README.md \
|
|
3287
|
-
|| ! grep -Fq 'counted by `benchmark audit` as the fourth passing pair-evidence row' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3288
|
-
|| ! grep -Fq 'counted by `benchmark audit` as the fourth passing pair-evidence row' benchmark/auto-resolve/BENCHMARK-RESULTS.md \
|
|
3289
|
-
|| ! grep -Fq 'the default 5-point pair margin' README.md \
|
|
3290
|
-
|| ! grep -Fq -- '`--pair-verify` and `--no-pair` are mutually exclusive' README.md \
|
|
3291
|
-
|| ! grep -Fq 'default 5-point pair margin' benchmark/auto-resolve/README.md \
|
|
3292
|
-
|| ! grep -Fq 'the default 5-point pair margin' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3293
|
-
|| ! grep -Fq '3x pair/solo wall ratio' README.md \
|
|
3294
|
-
|| ! grep -Fq '3x pair/solo wall ratio' benchmark/auto-resolve/README.md \
|
|
3295
|
-
|| ! grep -Fq '3x pair/solo wall ratio' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3296
|
-
|| ! grep -Fq 'audit.json` with the frontier summary' README.md \
|
|
3297
|
-
|| ! grep -Fq 'audit.json` with the frontier summary' benchmark/auto-resolve/README.md \
|
|
3298
|
-
|| ! grep -Fq 'audit.json` with the frontier summary' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3299
|
-
|| ! grep -Fq 'artifact map' README.md \
|
|
3300
|
-
|| ! grep -Fq 'artifact map' benchmark/auto-resolve/README.md \
|
|
3301
|
-
|| ! grep -Fq 'artifact map' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3302
|
-
|| ! grep -Fq '`checks.frontier_stdout` records summary, aggregate, final-verdict, expected, printed score-row, trigger-visible row, and hypothesis-trigger-visible row counts' README.md \
|
|
3303
|
-
|| ! grep -Fq '`headroom_rejections=...`, `pair_evidence_quality=...`,' README.md \
|
|
3304
|
-
|| ! grep -Fq '`pair_trigger_reasons=...`, `pair_evidence_hypotheses=...`, and' README.md \
|
|
3305
|
-
|| ! grep -Fq '`pair_evidence_hypothesis_triggers=...` handoff rows' README.md \
|
|
3306
|
-
|| ! grep -Fq '`pair_trigger_historical_aliases=...` when archived evidence includes legacy' README.md \
|
|
3307
|
-
|| ! grep -Fq '`pair_evidence_hypothesis_trigger_gaps=...` when documented' README.md \
|
|
3308
|
-
|| ! grep -Fq 'canonical trigger reason coverage' README.md \
|
|
3309
|
-
|| ! grep -Fq '`checks.pair_evidence_quality` records the same quality thresholds from the compact rows' README.md \
|
|
3310
|
-
|| ! grep -Fq '`checks.pair_trigger_reasons` records canonical/historical-alias/exposed/total trigger-reason row counts, fixture-level historical alias details, summary count, and row-match status' README.md \
|
|
3311
|
-
|| ! grep -Fq '`checks.pair_evidence_hypotheses` records documented/total pair-evidence hypothesis row counts' README.md \
|
|
3312
|
-
|| ! grep -Fq '`checks.pair_evidence_hypothesis_triggers` records whether documented hypotheses also appear as `spec.solo_headroom_hypothesis` trigger reasons plus fixture-level gap details' README.md \
|
|
3313
|
-
|| ! grep -Fq 'regenerated pair evidence' README.md \
|
|
3314
|
-
|| ! grep -Fq 'Historical trigger aliases are only reported for archived artifact review' README.md \
|
|
3315
|
-
|| ! grep -Fq 'current pair-evidence gates fail historical-only or unknown trigger reasons' README.md \
|
|
3316
|
-
|| ! grep -Fq '`checks.frontier_stdout`' benchmark/auto-resolve/README.md \
|
|
3317
|
-
|| ! grep -Fq 'aggregate, final-verdict, expected, printed score-row, trigger-visible row, and hypothesis-trigger-visible row counts' benchmark/auto-resolve/README.md \
|
|
3318
|
-
|| ! grep -Fq '`headroom_rejections=...`,' benchmark/auto-resolve/README.md \
|
|
3319
|
-
|| ! grep -Fq '`pair_trigger_reasons=...`, and' benchmark/auto-resolve/README.md \
|
|
3320
|
-
|| ! grep -Fq '`pair_evidence_hypotheses=...` and' benchmark/auto-resolve/README.md \
|
|
3321
|
-
|| ! grep -Fq '`pair_evidence_hypothesis_triggers=...` handoff rows' benchmark/auto-resolve/README.md \
|
|
3322
|
-
|| ! grep -Fq '`pair_trigger_historical_aliases=...` when archived evidence includes legacy' benchmark/auto-resolve/README.md \
|
|
3323
|
-
|| ! grep -Fq '`pair_evidence_hypothesis_trigger_gaps=...` when documented' benchmark/auto-resolve/README.md \
|
|
3324
|
-
|| ! grep -Fq 'canonical trigger reason coverage' benchmark/auto-resolve/README.md \
|
|
3325
|
-
|| ! grep -Fq 'records the same quality thresholds from the compact rows,' benchmark/auto-resolve/README.md \
|
|
3326
|
-
|| ! grep -Fq '`checks.pair_trigger_reasons` records canonical/historical-alias/exposed/total trigger-reason row counts, fixture-level historical alias details, summary count, and row-match status' benchmark/auto-resolve/README.md \
|
|
3327
|
-
|| ! grep -Fq '`checks.pair_evidence_hypotheses` records documented/total pair-evidence hypothesis row counts' benchmark/auto-resolve/README.md \
|
|
3328
|
-
|| ! grep -Fq '`checks.pair_evidence_hypothesis_triggers` records whether documented hypotheses also appear as `spec.solo_headroom_hypothesis` trigger reasons plus fixture-level gap details' benchmark/auto-resolve/README.md \
|
|
3329
|
-
|| ! grep -Fq 'regenerated pair evidence' benchmark/auto-resolve/README.md \
|
|
3330
|
-
|| ! grep -Fq 'Historical trigger aliases are only reported for archived artifact review' benchmark/auto-resolve/README.md \
|
|
3331
|
-
|| ! grep -Fq 'current pair-evidence gates fail historical-only or unknown trigger reasons' benchmark/auto-resolve/README.md \
|
|
3332
|
-
|| ! grep -Fq '`checks.frontier_stdout` records summary, aggregate, final-verdict, expected, printed score-row, trigger-visible row, and hypothesis-trigger-visible row counts' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3333
|
-
|| ! grep -Fq 'also prints `headroom_rejections=...`, `pair_evidence_quality=...`,' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3334
|
-
|| ! grep -Fq '`pair_trigger_reasons=...`, `pair_evidence_hypotheses=...`, and `pair_evidence_hypothesis_triggers=...` handoff rows' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3335
|
-
|| ! grep -Fq '`pair_trigger_historical_aliases=...` when archived evidence includes legacy' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3336
|
-
|| ! grep -Fq '`pair_evidence_hypothesis_trigger_gaps=...` when documented' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3337
|
-
|| ! grep -Fq 'canonical trigger reason coverage' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3338
|
-
|| ! grep -Fq '`checks.pair_evidence_quality` records the same quality thresholds from the compact rows' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3339
|
-
|| ! grep -Fq '`checks.pair_trigger_reasons` records canonical/historical-alias/exposed/total trigger-reason row counts, fixture-level historical alias details, summary count, and row-match status' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3340
|
-
|| ! grep -Fq '`checks.pair_evidence_hypotheses` records documented/total pair-evidence hypothesis row counts' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3341
|
-
|| ! grep -Fq '`checks.pair_evidence_hypothesis_triggers` records whether documented hypotheses also appear as `spec.solo_headroom_hypothesis` trigger reasons plus fixture-level gap details' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3342
|
-
|| ! grep -Fq 'regenerated pair evidence' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3343
|
-
|| ! grep -Fq 'Historical trigger aliases are only reported for archived artifact review' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3344
|
-
|| ! grep -Fq 'current pair-evidence gates fail historical-only or unknown trigger reasons' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3345
|
-
|| ! grep -Fq 'compact trigger-backed verdict-bearing `pair_evidence_rows`' README.md \
|
|
3346
|
-
|| ! grep -Fq 'pair_trigger_eligible: true' README.md \
|
|
3347
|
-
|| ! grep -Fq 'non-empty `pair_trigger_reasons`, `pair_trigger_has_canonical_reason: true`, and `pair_trigger_has_hypothesis_reason`; the audit fails rows missing trigger reasons' README.md \
|
|
3348
|
-
|| ! grep -Fq 'pair_trigger_has_canonical_reason: true' README.md \
|
|
3349
|
-
|| ! grep -Fq '`pair_evidence_rows`' benchmark/auto-resolve/README.md \
|
|
3350
|
-
|| ! grep -Fq 'compact trigger-backed verdict-bearing score rows' benchmark/auto-resolve/README.md \
|
|
3351
|
-
|| ! grep -Fq 'pair_trigger_eligible: true' benchmark/auto-resolve/README.md \
|
|
3352
|
-
|| ! grep -Fq 'non-empty `pair_trigger_reasons`, `pair_trigger_has_canonical_reason: true`, and `pair_trigger_has_hypothesis_reason`; the audit fails rows missing trigger reasons' benchmark/auto-resolve/README.md \
|
|
3353
|
-
|| ! grep -Fq 'pair_trigger_has_canonical_reason: true' benchmark/auto-resolve/README.md \
|
|
3354
|
-
|| ! grep -Fq 'compact trigger-backed verdict-bearing `pair_evidence_rows`' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3355
|
-
|| ! grep -Fq 'pair_trigger_eligible: true' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3356
|
-
|| ! grep -Fq 'non-empty `pair_trigger_reasons`, `pair_trigger_has_canonical_reason: true`, and `pair_trigger_has_hypothesis_reason`; the audit fails rows missing trigger reasons' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3357
|
-
|| ! grep -Fq 'pair_trigger_has_canonical_reason: true' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3358
|
-
|| ! grep -Fq 'revalidates `pair_mode: true`' README.md \
|
|
3359
|
-
|| ! grep -Fq 'satisfy `pair_mode: true`' benchmark/auto-resolve/README.md \
|
|
3360
|
-
|| ! grep -Fq 'revalidates `pair_mode: true`' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3361
|
-
|| ! grep -Fq 'JSON rows expose `pair_trigger_reasons` and' benchmark/auto-resolve/README.md \
|
|
3362
|
-
|| ! grep -Fq 'Markdown output includes a `Triggers`' benchmark/auto-resolve/README.md \
|
|
3363
|
-
|| ! grep -Fq 'trigger reasons, canonical-trigger coverage, classification counts' benchmark/auto-resolve/README.md \
|
|
3364
|
-
|| ! grep -Fq 'Its Markdown table includes a `Triggers` column' benchmark/auto-resolve/README.md \
|
|
3365
|
-
|| ! grep -Fq 'compact evidence row count must match the' README.md \
|
|
3366
|
-
|| ! grep -Fq 'compact evidence row count must match the' benchmark/auto-resolve/README.md \
|
|
3367
|
-
|| ! grep -Fq 'evidence row count must match the frontier evidence count' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3368
|
-
|| ! grep -Fq '"verdict": "PASS" if unmeasured_candidate_total == 0 else "FAIL"' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3369
|
-
|| ! grep -Fq 'assert frontier["unmeasured_count"] == 0' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3370
|
-
|| ! grep -Fq 'pair_margin_avg' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3371
|
-
|| ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3372
|
-
|| ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3373
|
-
|| ! grep -Fq 'from pair_evidence_contract import' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3374
|
-
|| ! grep -Fq 'def normalize_pair_evidence_row' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3375
|
-
|| ! grep -Fq 'def best_pair_evidence' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3376
|
-
|| ! grep -Fq 'ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3377
|
-
|| ! grep -Fq 'pair_arm not in ALLOWED_PAIR_ARMS' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3378
|
-
|| ! grep -Fq 'pair_mode = row.get("pair_mode")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3379
|
-
|| ! grep -Fq 'if pair_mode is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3380
|
-
|| ! grep -Fq 'pair_trigger_eligible = row.get("pair_trigger_eligible")' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3381
|
-
|| ! grep -Fq 'if pair_trigger_eligible is not True:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3382
|
-
|| ! grep -Fq '"pair_trigger_eligible": pair_trigger_eligible' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3383
|
-
|| ! grep -Fq '"pair_mode": pair_mode' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3384
|
-
|| ! grep -Fq 'def is_strict_int' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3385
|
-
|| ! grep -Fq 'def is_score' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3386
|
-
|| ! grep -Fq '0 <= value <= 100' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3387
|
-
|| ! grep -Fq 'def is_strict_number' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3388
|
-
|| ! grep -Fq 'import math' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3389
|
-
|| ! grep -Fq 'math.isfinite(value)' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3390
|
-
|| ! grep -Fq 'and value > 0' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3391
|
-
|| ! grep -Fq 'if pair_margin != pair_score - solo_score:' benchmark/auto-resolve/scripts/pair_evidence_contract.py \
|
|
3392
|
-
|| ! grep -Fq 'not math.isfinite(args.max_pair_solo_wall_ratio)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3393
|
-
|| ! grep -Fq 'nan-wall-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3394
|
-
|| ! grep -Fq 'inflated-margin-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3395
|
-
|| ! grep -Fq 'overrange-score-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3396
|
-
|| ! grep -Fq 'invalid-arm-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3397
|
-
|| ! grep -Fq 'false-pair-mode-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3398
|
-
|| ! grep -Fq 'zero-wall-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3399
|
-
|| ! grep -Fq 'nan-max-wall-ratio' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3400
|
-
|| ! grep -Fq 'pair_margin_avg=+21.00 pair_margin_min=+21 wall_avg=1.28x wall_max=1.28x' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3401
|
-
|| ! grep -Fq 'incomplete-high-run' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3402
|
-
|| ! grep -Fq 'bad-pair-evidence-json' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3403
|
-
|| ! grep -Fq 'pair evidence artifact malformed' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3404
|
-
|| ! grep -Fq 'bad-pair-evidence-rows' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3405
|
-
|| ! grep -Fq 'pair evidence artifact rows malformed' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3406
|
-
|| ! grep -Fq 'assert len(rows["F16-cli-quote-tax-rules"]["passing_pair_evidence"]) == 1' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3407
|
-
|| ! grep -Fq 'plus row-level verdicts' README.md \
|
|
3408
|
-
|| ! grep -Fq 'including pair arm, trigger reasons, average/minimum pair margin' README.md \
|
|
3409
|
-
|| ! grep -Fq 'Markdown frontier artifacts include a `Triggers` column' README.md \
|
|
3410
|
-
|| ! grep -Fq 'pair arm, margin, wall ratio, run id, verdict, and trigger reasons' benchmark/auto-resolve/README.md \
|
|
3411
|
-
|| ! grep -Fq 'with pair arm, verdict, and trigger reasons from the frontier step' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3412
|
-
|| ! grep -Fq 'pair-candidate-frontier.py --fail-on-unmeasured' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3413
|
-
|| ! grep -Fq 'audit-headroom-rejections.py' benchmark/auto-resolve/README.md \
|
|
3414
|
-
|| ! grep -Fq 'audit-headroom-rejections.py' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3415
|
-
|| ! grep -Fq 'unrecorded headroom rejection(s)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3416
|
-
|| ! grep -Fq 'unsupported registry rejection(s)' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3417
|
-
|| ! grep -Fq 'solo_claude={solo_score}' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3418
|
-
|| ! grep -Fq 'expected_solo_claude={expected_solo_score}' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3419
|
-
|| ! grep -Fq 'unsupported_registry_rejections' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
3420
|
-
|| ! grep -Fq 'expected_solo_claude=98' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
3421
|
-
|| ! grep -Fq 'active registry entries whose reason cites a run id or' benchmark/auto-resolve/README.md \
|
|
3422
|
-
|| ! grep -Fq 'active rejected-registry reason is backed' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3423
|
-
|| ! grep -Fq 'invalid headroom rejections' bin/devlyn.js \
|
|
3424
|
-
|| ! grep -Fq 'MALFORMED_JSON' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3425
|
-
|| ! grep -Fq 'MALFORMED_ROWS' benchmark/auto-resolve/scripts/audit-headroom-rejections.py \
|
|
3426
|
-
|| ! grep -Fq 'F33-cli-new-candidate' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
3427
|
-
|| ! grep -Fq 'bad-json-headroom <unknown>' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
3428
|
-
|| ! grep -Fq 'malformed-headroom <unknown>' benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh \
|
|
3429
|
-
|| ! grep -Fq 'candidate_unmeasured' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3430
|
-
|| ! grep -Fq 'pair_evidence_passed' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3431
|
-
|| ! grep -Fq 'print_summary(report)' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3432
|
-
|| ! grep -Fq 'fail_on_unmeasured' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3433
|
-
|| ! grep -Fq 'unmeasured candidate fixture(s)' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3434
|
-
|| ! grep -Fq 'pure JSON stdout must not include final text verdict' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3435
|
-
|| ! grep -Fq 'benchmark frontier pure JSON stdout must not include final text verdict' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3436
|
-
|| ! grep -Fq -- '--fail-on-unmeasured' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3437
|
-
|| ! grep -Fq -- '--fail-on-unmeasured' bin/devlyn.js \
|
|
3438
|
-
|| ! grep -Fq -- '--fail-on-unmeasured' benchmark/auto-resolve/README.md \
|
|
3439
|
-
|| ! grep -Fq -- '--fail-on-unmeasured' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3440
|
-
|| ! grep -Fq 'bare={bare} solo_claude={solo} pair={pair} arm={arm} margin={margin}' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3441
|
-
|| ! grep -Fq 'verdict=pair_evidence_passed' benchmark/auto-resolve/scripts/pair-candidate-frontier.py \
|
|
3442
|
-
|| ! grep -Fq 'F16-cli-quote-tax-rules: bare=50 solo_claude=75 pair=96 arm=l2_risk_probes margin=+21' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3443
|
-
|| ! grep -Fq 'verdict=pair_evidence_passed' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3444
|
-
|| ! grep -Fq '[audit] frontier' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3445
|
-
|| ! grep -Fq 'fixtures=21 rejected=17 candidates=4 pair_evidence=4 unmeasured=0 verdict=PASS' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3446
|
-
|| ! grep -Fq 'F16-cli-quote-tax-rules: bare=50 solo_claude=75 pair=96 arm=l2_risk_probes margin=+21' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3447
|
-
|| ! grep -Fq 'frontier.stdout' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3448
|
-
|| ! grep -Fq 'headroom-rejections.stdout' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3449
|
-
|| ! grep -Fq '"frontier_stdout": "frontier.stdout"' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3450
|
-
|| ! grep -Fq '"headroom_rejections_stdout": "headroom-rejections.stdout"' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3451
|
-
|| ! grep -Fq 'def load_headroom_audit_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3452
|
-
|| ! grep -Fq 'def check_headroom_audit_report' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3453
|
-
|| ! grep -Fq 'def print_headroom_rejections_summary' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3454
|
-
|| ! grep -Fq 'headroom_rejections={status} verdict={verdict}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3455
|
-
|| ! grep -Fq 'headroom_report_status == 0' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3456
|
-
|| ! grep -Fq '"report_check_exit_code": headroom_report_status' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3457
|
-
|| ! grep -Fq 'headroom audit unsupported registry rejection count missing or malformed' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3458
|
-
|| ! grep -Fq 'headroom_rejections=PASS verdict=PASS unrecorded=0 unsupported=0' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3459
|
-
|| ! grep -Fq 'headroom_rejections=PASS verdict=PASS unrecorded=0 unsupported=0' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3460
|
-
|| ! grep -Fq 'pair trigger reason rows {len(rows)} do not match summary count {count}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3461
|
-
|| ! grep -Fq 'pair trigger reasons missing for fixture(s)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3462
|
-
|| ! grep -Fq 'missing-trigger-reasons' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3463
|
-
|| ! grep -Fq 'malformed-trigger-reason-rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3464
|
-
|| ! grep -Fq 'mixed-unknown-trigger-reason-rows' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3465
|
-
|| ! grep -Fq 'pair_trigger_reasons=PASS canonical=2 historical_alias=1 exposed=2 total=2 summary=2 rows_match=true' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3466
|
-
|| ! grep -Fq 'pair_trigger_historical_aliases=F21-cli-scheduler-priority=risk_profile.high_risk' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3467
|
-
|| ! grep -Fq '"unsupported_registry_rejection_count": (' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3468
|
-
|| ! grep -Fq 'len(unsupported) if isinstance(unsupported, list) else None' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3469
|
-
|| ! grep -Fq 'headroom-missing-unsupported' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3470
|
-
|| ! grep -Fq 'unsupported_registry_rejection_count' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3471
|
-
|| ! grep -Fq 'unsupported_registry_rejection_count' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3472
|
-
|| ! grep -Fq '`checks.headroom_rejections` records child verdict plus unrecorded/unsupported counts' README.md \
|
|
3473
|
-
|| ! grep -Fq '`checks.headroom_rejections` records the child verdict plus unrecorded and' benchmark/auto-resolve/README.md \
|
|
3474
|
-
|| ! grep -Fq '`checks.headroom_rejections` records child verdict plus unrecorded/unsupported counts' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3475
|
-
|| ! grep -Fq 'def check_frontier_stdout' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3476
|
-
|| ! grep -Fq '"final_verdict_rows": final_verdict_rows' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3477
|
-
|| ! grep -Fq 'frontier stdout final verdict row count is not exactly 1' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3478
|
-
|| ! grep -Fq 'frontier stdout missing score row' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3479
|
-
|| ! grep -Fq 'wall={wall} run={run_id}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3480
|
-
|| ! grep -Fq 'frontier stdout aggregate score row count is not exactly 1' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3481
|
-
|| ! grep -Fq 'frontier stdout summary score row count is not exactly 1' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3482
|
-
|| ! grep -Fq 'frontier stdout aggregate fields malformed' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3483
|
-
|| ! grep -Fq 'frontier stdout score row count' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3484
|
-
|| ! grep -Fq 'summary_count = stdout.splitlines().count(required_summary)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3485
|
-
|| ! grep -Fq 'aggregate_count = stdout.splitlines().count(required_aggregate)' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3486
|
-
|| ! grep -Fq 'frontier stdout summary counts malformed' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3487
|
-
|| ! grep -Fq 'count_keys = {' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3488
|
-
|| ! grep -Fq 'pair_margin_avg={avg} pair_margin_min={min_margin}' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3489
|
-
|| ! grep -Fq 'def format_decimal_margin' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3490
|
-
|| ! grep -Fq 'def format_wall_ratio' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3491
|
-
|| ! grep -Fq 'frontier stdout check missing summary fields' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3492
|
-
|| ! grep -Fq 'missing-frontier-score-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3493
|
-
|| ! grep -Fq 'missing-frontier-aggregate-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3494
|
-
|| ! grep -Fq 'duplicate-frontier-summary-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3495
|
-
|| ! grep -Fq 'duplicate-frontier-aggregate-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3496
|
-
|| ! grep -Fq 'pair_margin_avg=+27.00 pair_margin_min=+21 wall_avg=1.38x wall_max=1.47x' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3497
|
-
|| ! grep -Fq 'partial-frontier-score-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3498
|
-
|| ! grep -Fq 'extra-frontier-score-row' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3499
|
-
|| ! grep -Fq 'malformed-frontier-stdout-summary' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3500
|
-
|| ! grep -Fq 'malformed-frontier-stdout-counts' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3501
|
-
|| ! grep -Fq 'malformed-frontier-stdout-aggregate' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3502
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["status"] == "PASS"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3503
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["summary_rows"] == 1' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3504
|
-
|| ! grep -Fq 'report["checks"]["frontier_stdout"]["aggregate_rows"] == 1' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3505
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["final_verdict_rows"] == 1' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3506
|
-
|| ! grep -Fq 'missing-final-frontier-verdict' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3507
|
-
|| ! grep -Fq 'duplicate-final-frontier-verdict' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3508
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["expected_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3509
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["trigger_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3510
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["hypothesis_trigger_rows"] == len(audit["pair_evidence_rows"])' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3511
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3512
|
-
|| ! grep -Fq 'audit["checks"]["frontier_stdout"]["hypothesis_trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3513
|
-
|| ! grep -Fq 'report["checks"]["frontier_stdout"]["trigger_rows"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3514
|
-
|| ! grep -Fq 'report["checks"]["frontier_stdout"]["hypothesis_trigger_rows"] == 2' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3515
|
-
|| ! grep -Fq 'report["checks"]["frontier_stdout"]["trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3516
|
-
|| ! grep -Fq 'report["checks"]["frontier_stdout"]["hypothesis_trigger_rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3517
|
-
|| ! grep -Fq 'report["checks"]["frontier_stdout"]["rows_match_count"] is True' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3518
|
-
|| ! grep -Fq 'audit["artifacts"]["frontier_stdout"] == "frontier.stdout"' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3519
|
-
|| ! grep -Fq 'report["artifacts"] == {' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3520
|
-
|| ! grep -Fq 'frontier.stdout' benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh \
|
|
3521
|
-
|| ! grep -Fq 'frontier.stdout' benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh \
|
|
3522
|
-
|| ! grep -Fq 'frontier summary and an artifact map (`artifacts`)' benchmark/auto-resolve/README.md \
|
|
3523
|
-
|| ! grep -Fq 'artifact map' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3524
|
-
|| ! grep -Fq 'child stdout/stderr logs' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3525
|
-
|| ! grep -Fq '| Fixture | Status | Verdict | Evidence | Pair arm | Triggers | Hypothesis trigger | Bare | Solo_claude | Pair | Margin | Wall ratio | Rejected reason |' benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh \
|
|
3526
|
-
|| ! grep -Fq 'benchmark frontier` also prints a stdout score summary for existing complete pair' README.md \
|
|
3527
|
-
|| ! grep -Fq 'plus row-level verdicts' README.md \
|
|
3528
|
-
|| ! grep -Fq 'including pair arm, trigger reasons, average/minimum pair margin' README.md \
|
|
3529
|
-
|| ! grep -Fq 'Markdown frontier artifacts include a `Triggers` column' README.md \
|
|
3530
|
-
|| ! grep -Fq 'Full-pipeline pair gate artifacts record `require_hypothesis_trigger` in JSON' README.md \
|
|
3531
|
-
|| ! grep -Fq 'Full-pipeline pair gate artifacts record `require_hypothesis_trigger` in JSON' benchmark/auto-resolve/README.md \
|
|
3532
|
-
|| ! grep -Fq 'Full-pipeline pair gate artifacts record `require_hypothesis_trigger` in JSON' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3533
|
-
|| ! grep -Fq 'includes a Markdown `Hypothesis trigger` column' benchmark/auto-resolve/README.md \
|
|
3534
|
-
|| ! grep -Fq 'includes a Markdown `Hypothesis trigger` column' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3535
|
-
|| ! grep -Fq 'pair arm, margin, wall ratio, run id, verdict, and trigger reasons' benchmark/auto-resolve/README.md \
|
|
3536
|
-
|| ! grep -Fq 'overall verdict plus row-level verdict, pair-arm, and trigger-reason columns' benchmark/auto-resolve/README.md \
|
|
3537
|
-
|| ! grep -Fq 'with pair arm, verdict, and trigger reasons from the frontier step' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3538
|
-
|| ! grep -Fq 'overall verdict plus row-level verdict, pair-arm, and trigger-reason columns' benchmark/auto-resolve/run-real-benchmark.md \
|
|
3539
|
-
|| ! grep -Fq 'complete pair evidence rows' benchmark/auto-resolve/README.md \
|
|
3540
|
-
|| ! grep -Fq 'complete pair evidence rows' benchmark/auto-resolve/run-real-benchmark.md; then
|
|
3541
|
-
offenders="${offenders}"$'\n'"benchmark docs and CLI must expose the pair-candidate frontier report before new provider spend"
|
|
3542
|
-
fi
|
|
3543
|
-
if ! grep -Fq '"benchmark/auto-resolve/scripts/**"' package.json \
|
|
3544
|
-
|| ! grep -Fq 'benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json' package.json \
|
|
3545
|
-
|| ! grep -Fq 'benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json' package.json \
|
|
3546
|
-
|| ! grep -Fq 'benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json' package.json \
|
|
3547
|
-
|| ! grep -Fq 'benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json' package.json; then
|
|
3548
|
-
offenders="${offenders}"$'\n'"package.json must include benchmark scripts, runner regression tests, current pair evidence, and rejected-headroom evidence in npm packages"
|
|
3549
|
-
fi
|
|
3550
|
-
if ! grep -Fq '"max_observed_pair_solo_wall_ratio": 2.2506234413965087' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json \
|
|
3551
|
-
|| ! grep -Fq '"max_observed_pair_solo_wall_ratio": 1.4728476821192054' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json \
|
|
3552
|
-
|| ! grep -Fq 'pair_trigger eligible with a canonical reason' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json \
|
|
3553
|
-
|| ! grep -Fq 'pair_trigger eligible with a canonical reason' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json \
|
|
3554
|
-
|| ! grep -Fq 'pair_trigger eligible with canonical reason' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
|
|
3555
|
-
|| ! grep -Fq 'pair_trigger eligible with canonical reason' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md \
|
|
3556
|
-
|| ! grep -Fq 'Allowed pair/solo wall ratio: 3.00x' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
|
|
3557
|
-
|| ! grep -Fq 'Maximum observed pair/solo wall ratio: 2.25x' benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md \
|
|
3558
|
-
|| ! grep -Fq 'Allowed pair/solo wall ratio: 3.00x' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md \
|
|
3559
|
-
|| ! grep -Fq 'Maximum observed pair/solo wall ratio: 1.47x' benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md; then
|
|
3560
|
-
offenders="${offenders}"$'\n'"packaged pair evidence artifacts must use the current observed-vs-allowed wall-ratio schema and canonical trigger rule wording"
|
|
3561
|
-
fi
|
|
3562
|
-
if make_temp_dir package_results /tmp/devlyn-lint-package-results.XXXXXX \
|
|
3563
|
-
&& make_temp_dir package_audit /tmp/devlyn-lint-package-audit.XXXXXX \
|
|
3564
|
-
&& make_temp_file package_audit_stdout /tmp/devlyn-lint-package-audit.out.XXXXXX; then
|
|
3565
|
-
cp -R benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof "$package_results/"
|
|
3566
|
-
cp -R benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1 "$package_results/"
|
|
3567
|
-
for rejected_run in \
|
|
3568
|
-
20260507-f10-f11-tier1-full-pipeline \
|
|
3569
|
-
20260508-f22-exact-error-headroom \
|
|
3570
|
-
20260508-f26-headroom \
|
|
3571
|
-
20260511-f3-http-error-headroom \
|
|
3572
|
-
20260511-f12-webhook-headroom \
|
|
3573
|
-
20260511-f15-concurrency-headroom \
|
|
3574
|
-
20260512-f2-medium-headroom \
|
|
3575
|
-
20260512-f4-web-headroom \
|
|
3576
|
-
20260512-f5-fixloop-headroom \
|
|
3577
|
-
20260512-f6-checksum-headroom \
|
|
3578
|
-
20260512-f7-scope-headroom \
|
|
3579
|
-
20260512-f9-e2e-headroom \
|
|
3580
|
-
20260512-f31-seat-rebalance-headroom \
|
|
3581
|
-
20260512-f32-subscription-renewal-headroom; do
|
|
3582
|
-
cp -R "benchmark/auto-resolve/results/$rejected_run" "$package_results/"
|
|
3583
|
-
done
|
|
3584
|
-
if ! python3 benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3585
|
-
--results-root "$package_results" \
|
|
3586
|
-
--out-dir "$package_audit" >"$package_audit_stdout" 2>&1; then
|
|
3587
|
-
offenders="${offenders}"$'\n'"packaged pair evidence subset must pass benchmark audit without relying on unshipped local results"
|
|
3588
|
-
elif ! grep -Fq 'headroom_rejections=PASS verdict=PASS unrecorded=0 unsupported=0' "$package_audit_stdout" \
|
|
3589
|
-
|| ! grep -Fq 'pair_evidence_quality=PASS min_pair_margin_actual=+21 min_pair_margin_required=+5 max_wall_actual=2.25x max_wall_allowed=3.00x' "$package_audit_stdout" \
|
|
3590
|
-
|| ! grep -Fq 'pair_trigger_reasons=PASS canonical=4 historical_alias=0 exposed=4 total=4 summary=4 rows_match=true' "$package_audit_stdout" \
|
|
3591
|
-
|| ! grep -Fq 'pair_evidence_hypothesis_triggers=PASS matched=4 documented=4 total=4' "$package_audit_stdout" \
|
|
3592
|
-
|| grep -Fq 'pair_trigger_historical_aliases=' "$package_audit_stdout" \
|
|
3593
|
-
|| grep -Fq 'pair_evidence_hypothesis_trigger_gaps=' "$package_audit_stdout"; then
|
|
3594
|
-
offenders="${offenders}"$'\n'"packaged pair evidence subset audit stdout must expose headroom, pair-quality, and trigger-reason handoff rows"
|
|
3595
|
-
elif ! python3 - "$package_audit/audit.json" <<'PY'
|
|
3596
|
-
import json
|
|
3597
|
-
import sys
|
|
3598
|
-
|
|
3599
|
-
report = json.load(open(sys.argv[1], encoding="utf8"))
|
|
3600
|
-
rows = report.get("pair_evidence_rows")
|
|
3601
|
-
frontier_summary = report.get("frontier_summary", {})
|
|
3602
|
-
frontier_report = report.get("checks", {}).get("frontier_report", {})
|
|
3603
|
-
frontier_stdout = report.get("checks", {}).get("frontier_stdout", {})
|
|
3604
|
-
min_pair_evidence = report.get("checks", {}).get("min_pair_evidence", {})
|
|
3605
|
-
pair_evidence_quality = report.get("checks", {}).get("pair_evidence_quality", {})
|
|
3606
|
-
pair_trigger_reasons = report.get("checks", {}).get("pair_trigger_reasons", {})
|
|
3607
|
-
headroom_rejections = report.get("checks", {}).get("headroom_rejections", {})
|
|
3608
|
-
artifacts = report.get("artifacts", {})
|
|
3609
|
-
assert report.get("verdict") == "PASS"
|
|
3610
|
-
assert frontier_summary.get("verdict") == "PASS"
|
|
3611
|
-
assert frontier_summary.get("pair_evidence_count") == 4
|
|
3612
|
-
assert frontier_summary.get("unmeasured_count") == 0
|
|
3613
|
-
assert frontier_report.get("status") == "PASS"
|
|
3614
|
-
assert frontier_report.get("verdict") == "PASS"
|
|
3615
|
-
assert frontier_report.get("unmeasured_count") == 0
|
|
3616
|
-
assert frontier_stdout.get("status") == "PASS"
|
|
3617
|
-
assert frontier_stdout.get("report", "").endswith("frontier.stdout")
|
|
3618
|
-
assert frontier_stdout.get("summary_rows") == 1
|
|
3619
|
-
assert frontier_stdout.get("aggregate_rows") == 1
|
|
3620
|
-
assert frontier_stdout.get("final_verdict_rows") == 1
|
|
3621
|
-
assert frontier_stdout.get("expected_rows") == len(rows) == 4
|
|
3622
|
-
assert frontier_stdout.get("stdout_rows") == len(rows) == 4
|
|
3623
|
-
assert frontier_stdout.get("trigger_rows") == len(rows) == 4
|
|
3624
|
-
assert frontier_stdout.get("hypothesis_trigger_rows") == len(rows) == 4
|
|
3625
|
-
assert frontier_stdout.get("rows_match_count") is True
|
|
3626
|
-
assert frontier_stdout.get("trigger_rows_match_count") is True
|
|
3627
|
-
assert frontier_stdout.get("hypothesis_trigger_rows_match_count") is True
|
|
3628
|
-
assert headroom_rejections.get("status") == "PASS"
|
|
3629
|
-
assert headroom_rejections.get("report_check_exit_code") == 0
|
|
3630
|
-
assert headroom_rejections.get("verdict") == "PASS"
|
|
3631
|
-
assert headroom_rejections.get("unrecorded_failure_count") == 0
|
|
3632
|
-
assert headroom_rejections.get("unsupported_registry_rejection_count") == 0
|
|
3633
|
-
assert min_pair_evidence.get("rows_match_count") is True
|
|
3634
|
-
assert min_pair_evidence.get("actual_rows") == len(rows) == 4
|
|
3635
|
-
assert pair_evidence_quality.get("status") == "PASS"
|
|
3636
|
-
assert pair_evidence_quality.get("min_pair_margin_actual") == frontier_summary.get("pair_margin_min")
|
|
3637
|
-
assert pair_evidence_quality.get("max_pair_solo_wall_ratio_actual") == frontier_summary.get("pair_solo_wall_ratio_max")
|
|
3638
|
-
assert pair_trigger_reasons.get("status") == "PASS"
|
|
3639
|
-
assert pair_trigger_reasons.get("summary_pair_evidence_count") == 4
|
|
3640
|
-
assert pair_trigger_reasons.get("canonical_rows") == 4
|
|
3641
|
-
assert pair_trigger_reasons.get("historical_alias_rows") == 0
|
|
3642
|
-
assert pair_trigger_reasons.get("historical_alias_details") == []
|
|
3643
|
-
assert pair_trigger_reasons.get("exposed_rows") == 4
|
|
3644
|
-
assert pair_trigger_reasons.get("total_rows") == 4
|
|
3645
|
-
assert pair_trigger_reasons.get("rows_match_count") is True
|
|
3646
|
-
pair_hypothesis_triggers = report.get("checks", {}).get("pair_evidence_hypothesis_triggers", {})
|
|
3647
|
-
assert pair_hypothesis_triggers.get("status") == "PASS"
|
|
3648
|
-
assert pair_hypothesis_triggers.get("matched_rows") == 4
|
|
3649
|
-
assert pair_hypothesis_triggers.get("documented_rows") == 4
|
|
3650
|
-
assert pair_hypothesis_triggers.get("gap_details") == []
|
|
3651
|
-
assert artifacts.get("frontier_json") == "frontier.json"
|
|
3652
|
-
assert artifacts.get("frontier_stdout") == "frontier.stdout"
|
|
3653
|
-
assert artifacts.get("headroom_audit_json") == "headroom-audit.json"
|
|
3654
|
-
assert artifacts.get("headroom_rejections_stdout") == "headroom-rejections.stdout"
|
|
3655
|
-
for row in rows:
|
|
3656
|
-
assert row.get("verdict") == "pair_evidence_passed"
|
|
3657
|
-
assert row.get("pair_arm") == "l2_risk_probes"
|
|
3658
|
-
assert row.get("pair_mode") is True
|
|
3659
|
-
assert row.get("pair_trigger_eligible") is True
|
|
3660
|
-
assert isinstance(row.get("pair_trigger_reasons"), list)
|
|
3661
|
-
assert row.get("pair_trigger_reasons")
|
|
3662
|
-
assert row.get("pair_trigger_has_canonical_reason") is True
|
|
3663
|
-
assert row.get("pair_trigger_has_hypothesis_reason") is True
|
|
3664
|
-
PY
|
|
3665
|
-
then
|
|
3666
|
-
offenders="${offenders}"$'\n'"packaged pair evidence subset audit.json must expose frontier_report, frontier_stdout, artifact map, and 4 trigger-backed verdict-bearing pair rows with trigger reasons"
|
|
3667
|
-
fi
|
|
3668
|
-
rm -rf "$package_results" "$package_audit" "$package_audit_stdout"
|
|
3669
|
-
else
|
|
3670
|
-
offenders="${offenders}"$'\n'"packaged pair evidence subset could not allocate temporary audit workspace"
|
|
3671
|
-
fi
|
|
3672
|
-
if make_temp_file pack_json /tmp/devlyn-lint-pack.json.XXXXXX \
|
|
3673
|
-
&& make_temp_dir pack_cache /tmp/devlyn-lint-pack-cache.XXXXXX; then
|
|
3674
|
-
if npm_config_cache="$pack_cache" npm pack --dry-run --json > "$pack_json" 2>/dev/null; then
|
|
3675
|
-
if ! node - "$pack_json" <<'NODE'
|
|
3676
|
-
const fs = require("fs");
|
|
3677
|
-
const path = require("path");
|
|
3678
|
-
const packPath = process.argv[2];
|
|
3679
|
-
const pack = JSON.parse(fs.readFileSync(packPath, "utf8"))[0];
|
|
3680
|
-
const files = new Set(pack.files.map((file) => file.path));
|
|
3681
|
-
function listFiles(dir) {
|
|
3682
|
-
return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
|
|
3683
|
-
const full = path.join(dir, entry.name);
|
|
3684
|
-
if (entry.isDirectory()) return listFiles(full);
|
|
3685
|
-
if (!entry.isFile()) return [];
|
|
3686
|
-
return [full.split(path.sep).join("/")];
|
|
3687
|
-
});
|
|
3688
|
-
}
|
|
3689
|
-
const shadowRequired = fs
|
|
3690
|
-
.readdirSync("benchmark/auto-resolve/shadow-fixtures", { withFileTypes: true })
|
|
3691
|
-
.filter((entry) => entry.isDirectory() && /^S/.test(entry.name))
|
|
3692
|
-
.flatMap((entry) => listFiles(path.join("benchmark/auto-resolve/shadow-fixtures", entry.name)));
|
|
3693
|
-
const required = [
|
|
3694
|
-
"benchmark/auto-resolve/BENCHMARK-RESULTS.md",
|
|
3695
|
-
"benchmark/auto-resolve/run-real-benchmark.md",
|
|
3696
|
-
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.md",
|
|
3697
|
-
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/headroom-gate.json",
|
|
3698
|
-
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.md",
|
|
3699
|
-
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/full-pipeline-pair-gate.json",
|
|
3700
|
-
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F16-cli-quote-tax-rules/l2_risk_probes/result.json",
|
|
3701
|
-
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F23-cli-fulfillment-wave/l2_risk_probes/result.json",
|
|
3702
|
-
"benchmark/auto-resolve/results/20260510-f16-f23-f25-combined-proof/F25-cli-cart-promotion-rules/l2_risk_probes/result.json",
|
|
3703
|
-
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.md",
|
|
3704
|
-
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/headroom-gate.json",
|
|
3705
|
-
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.md",
|
|
3706
|
-
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/full-pipeline-pair-gate.json",
|
|
3707
|
-
"benchmark/auto-resolve/results/20260511-f21-current-riskprobes-v1/F21-cli-scheduler-priority/l2_risk_probes/result.json",
|
|
3708
|
-
"benchmark/auto-resolve/results/20260512-f2-medium-headroom/headroom-gate.json",
|
|
3709
|
-
"benchmark/auto-resolve/results/20260512-f31-seat-rebalance-headroom/headroom-gate.json",
|
|
3710
|
-
"benchmark/auto-resolve/results/20260512-f32-subscription-renewal-headroom/headroom-gate.json",
|
|
3711
|
-
"benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/NOTES.md",
|
|
3712
|
-
"benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/spec.md",
|
|
3713
|
-
"benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/expected.json",
|
|
3714
|
-
"benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/metadata.json",
|
|
3715
|
-
"benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/priority-transfer-rollback.js",
|
|
3716
|
-
"benchmark/auto-resolve/fixtures/F31-cli-seat-rebalance/verifiers/duplicate-event-error.js",
|
|
3717
|
-
"benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/NOTES.md",
|
|
3718
|
-
"benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/spec.md",
|
|
3719
|
-
"benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/expected.json",
|
|
3720
|
-
"benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/metadata.json",
|
|
3721
|
-
"benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/priority-credit-rollback.js",
|
|
3722
|
-
"benchmark/auto-resolve/fixtures/F32-cli-subscription-renewal/verifiers/duplicate-renewal-error.js",
|
|
3723
|
-
"benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh",
|
|
3724
|
-
"benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh",
|
|
3725
|
-
"benchmark/auto-resolve/scripts/pair_evidence_contract.py",
|
|
3726
|
-
"benchmark/auto-resolve/scripts/pair-candidate-frontier.py",
|
|
3727
|
-
"benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh",
|
|
3728
|
-
"benchmark/auto-resolve/scripts/audit-pair-evidence.py",
|
|
3729
|
-
"benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py",
|
|
3730
|
-
"benchmark/auto-resolve/scripts/solo-ceiling-avoidance.py",
|
|
3731
|
-
"benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh",
|
|
3732
|
-
"benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh",
|
|
3733
|
-
"benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh",
|
|
3734
|
-
"benchmark/auto-resolve/scripts/test-lint-fixtures.sh",
|
|
3735
|
-
"benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh",
|
|
3736
|
-
"benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh",
|
|
3737
|
-
"benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh",
|
|
3738
|
-
"benchmark/auto-resolve/scripts/test-ship-gate.sh",
|
|
3739
|
-
"benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh",
|
|
3740
|
-
"benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh",
|
|
3741
|
-
"benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/RETIRED.md",
|
|
3742
|
-
"benchmark/auto-resolve/fixtures/retired/F27-cli-subscription-proration/spec.md",
|
|
3743
|
-
"benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/RETIRED.md",
|
|
3744
|
-
"benchmark/auto-resolve/fixtures/retired/F28-cli-return-authorization/spec.md",
|
|
3745
|
-
"benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/RETIRED.md",
|
|
3746
|
-
"benchmark/auto-resolve/fixtures/retired/F30-cli-credit-hold-settlement/spec.md",
|
|
3747
|
-
"benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/RETIRED.md",
|
|
3748
|
-
"scripts/lint-fixtures.sh",
|
|
3749
|
-
"scripts/lint-shadow-fixtures.sh",
|
|
3750
|
-
...shadowRequired,
|
|
3751
|
-
];
|
|
3752
|
-
const missing = required.filter((file) => !files.has(file));
|
|
3753
|
-
if (missing.length > 0) {
|
|
3754
|
-
console.error(missing.join("\n"));
|
|
3755
|
-
process.exit(1);
|
|
3756
|
-
}
|
|
3757
|
-
const forbidden = pack.files
|
|
3758
|
-
.map((file) => file.path)
|
|
3759
|
-
.filter((file) => file.includes("__pycache__") || file.endsWith(".pyc"));
|
|
3760
|
-
if (forbidden.length > 0) {
|
|
3761
|
-
console.error(forbidden.join("\n"));
|
|
3762
|
-
process.exit(2);
|
|
3763
|
-
}
|
|
3764
|
-
NODE
|
|
3765
|
-
then
|
|
3766
|
-
offenders="${offenders}"$'\n'"npm pack dry-run must include benchmark runner/gate regression tests, all shadow fixture files, retired fixture replay docs, and exclude pycache artifacts"
|
|
3767
|
-
fi
|
|
3768
|
-
else
|
|
3769
|
-
offenders="${offenders}"$'\n'"npm pack dry-run failed while checking benchmark runner/gate regression tests"
|
|
3770
|
-
fi
|
|
3771
|
-
else
|
|
3772
|
-
offenders="${offenders}"$'\n'"npm pack dry-run could not allocate temporary package workspace"
|
|
3773
|
-
fi
|
|
3774
|
-
[ -n "${pack_json:-}" ] && rm -f "$pack_json"
|
|
3775
|
-
[ -n "${pack_cache:-}" ] && rm -rf "$pack_cache"
|
|
3776
|
-
non_executable_shell_scripts=$(find benchmark/auto-resolve/scripts -maxdepth 1 -name '*.sh' -type f ! -perm -111 -print | sort)
|
|
3777
|
-
if [ -n "$non_executable_shell_scripts" ]; then
|
|
3778
|
-
while IFS= read -r f; do
|
|
3779
|
-
offenders="${offenders}"$'\n'"benchmark shell script must be executable: $f"
|
|
3780
|
-
done <<< "$non_executable_shell_scripts"
|
|
3781
|
-
fi
|
|
3782
|
-
for gate_test in \
|
|
3783
|
-
test-benchmark-arg-parsing.sh \
|
|
3784
|
-
test-pair-candidate-frontier.sh \
|
|
3785
|
-
test-audit-pair-evidence.sh \
|
|
3786
|
-
test-audit-headroom-rejections.sh \
|
|
3787
|
-
test-build-pair-eligible-manifest.sh \
|
|
3788
|
-
test-ship-gate.sh \
|
|
3789
|
-
test-headroom-gate.sh \
|
|
3790
|
-
test-run-headroom-candidate.sh \
|
|
3791
|
-
test-check-f9-artifacts.sh \
|
|
3792
|
-
test-lint-fixtures.sh \
|
|
3793
|
-
test-run-full-pipeline-pair-candidate.sh \
|
|
3794
|
-
test-full-pipeline-pair-gate.sh \
|
|
3795
|
-
test-iter-0033c-l1-summary.sh \
|
|
3796
|
-
test-iter-0033c-compare.sh \
|
|
3797
|
-
test-run-swebench-solver-batch.sh \
|
|
3798
|
-
test-swebench-frozen-case.sh \
|
|
3799
|
-
test-frozen-verify-gate.sh
|
|
3800
|
-
do
|
|
3801
|
-
if ! grep -Fq "$gate_test" benchmark/auto-resolve/README.md; then
|
|
3802
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/README.md: gate-change instructions must list $gate_test"
|
|
3803
|
-
fi
|
|
3804
|
-
done
|
|
3805
|
-
if ! bash benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh >/dev/null 2>&1; then
|
|
3806
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-benchmark-arg-parsing.sh: failed"
|
|
3807
|
-
fi
|
|
3808
|
-
if ! bash benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh >/dev/null 2>&1; then
|
|
3809
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-pair-candidate-frontier.sh: failed"
|
|
3810
|
-
fi
|
|
3811
|
-
if ! bash benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh >/dev/null 2>&1; then
|
|
3812
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-audit-pair-evidence.sh: failed"
|
|
3813
|
-
fi
|
|
3814
|
-
if ! bash benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh >/dev/null 2>&1; then
|
|
3815
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-audit-headroom-rejections.sh: failed"
|
|
3816
|
-
fi
|
|
3817
|
-
if ! bash benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh >/dev/null 2>&1; then
|
|
3818
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-build-pair-eligible-manifest.sh: failed"
|
|
3819
|
-
fi
|
|
3820
|
-
if ! bash benchmark/auto-resolve/scripts/test-headroom-gate.sh >/dev/null 2>&1; then
|
|
3821
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-headroom-gate.sh: failed"
|
|
3822
|
-
fi
|
|
3823
|
-
if ! bash benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh >/dev/null 2>&1; then
|
|
3824
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh: failed"
|
|
3825
|
-
fi
|
|
3826
|
-
if ! bash benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh >/dev/null 2>&1; then
|
|
3827
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-check-f9-artifacts.sh: failed"
|
|
3828
|
-
fi
|
|
3829
|
-
if ! bash benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh >/dev/null 2>&1; then
|
|
3830
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-run-full-pipeline-pair-candidate.sh: failed"
|
|
3831
|
-
fi
|
|
3832
|
-
if ! bash benchmark/auto-resolve/scripts/test-ship-gate.sh >/dev/null 2>&1; then
|
|
3833
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-ship-gate.sh: failed"
|
|
3834
|
-
fi
|
|
3835
|
-
if ! bash benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh >/dev/null 2>&1; then
|
|
3836
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-full-pipeline-pair-gate.sh: failed"
|
|
3837
|
-
fi
|
|
3838
|
-
if ! bash benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh >/dev/null 2>&1; then
|
|
3839
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-iter-0033c-l1-summary.sh: failed"
|
|
3840
|
-
fi
|
|
3841
|
-
if ! bash benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh >/dev/null 2>&1; then
|
|
3842
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-iter-0033c-compare.sh: failed"
|
|
3843
|
-
fi
|
|
3844
|
-
if ! bash benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh >/dev/null 2>&1; then
|
|
3845
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh: failed"
|
|
3846
|
-
fi
|
|
3847
|
-
if ! bash benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh >/dev/null 2>&1; then
|
|
3848
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-swebench-frozen-case.sh: failed"
|
|
3849
|
-
fi
|
|
3850
|
-
if ! bash benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh >/dev/null 2>&1; then
|
|
3851
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-frozen-verify-gate.sh: failed"
|
|
3852
|
-
fi
|
|
3853
|
-
if ! bash scripts/lint-fixtures.sh >/dev/null 2>&1; then
|
|
3854
|
-
offenders="${offenders}"$'\n'"scripts/lint-fixtures.sh: failed"
|
|
3855
|
-
fi
|
|
3856
|
-
if ! bash scripts/lint-shadow-fixtures.sh >/dev/null 2>&1; then
|
|
3857
|
-
offenders="${offenders}"$'\n'"scripts/lint-shadow-fixtures.sh: failed"
|
|
3858
|
-
fi
|
|
3859
|
-
if ! bash benchmark/auto-resolve/scripts/test-lint-fixtures.sh >/dev/null 2>&1; then
|
|
3860
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/test-lint-fixtures.sh: failed"
|
|
3861
|
-
fi
|
|
3862
|
-
if ! python3 -m py_compile benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py >/dev/null 2>&1; then
|
|
3863
|
-
offenders="${offenders}"$'\n'"benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py: py_compile failed"
|
|
3864
|
-
fi
|
|
3865
|
-
if ! grep -Fq 'high-risk fixture must include a resolve risk-trigger term' scripts/lint-fixtures.sh \
|
|
3866
|
-
|| ! grep -Fq 'SOLO_HEADROOM_CHECK="$REPO_ROOT/benchmark/auto-resolve/scripts/solo-headroom-hypothesis.py"' scripts/lint-fixtures.sh \
|
|
3867
|
-
|| ! grep -Fq 'pair_evidence_passed fixture spec.md must document an actionable solo-headroom hypothesis with solo_claude miss and observable command from expected.json' scripts/lint-fixtures.sh \
|
|
3868
|
-
|| ! grep -Fq -- '--expected-json "$d/expected.json" "$d/spec.md"' scripts/lint-fixtures.sh \
|
|
3869
|
-
|| ! grep -Fq 'str(fixture_dir / "expected.json")' benchmark/auto-resolve/scripts/audit-pair-evidence.py \
|
|
3870
|
-
|| ! grep -Fq 'hypothesis command must match expected.json verification command' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3871
|
-
|| ! grep -Fq 'pair-evidence-hypothesis-fail.out' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3872
|
-
|| ! grep -Fq 'pair-evidence-hypothesis-pass.out' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3873
|
-
|| ! grep -Fq 'DEVLYN_FIXTURES_DIR' scripts/lint-fixtures.sh \
|
|
3874
|
-
|| ! grep -Fq 'DEVLYN_FIXTURE_GLOB' scripts/lint-fixtures.sh \
|
|
3875
|
-
|| ! grep -Fq 'DEVLYN_FIXTURE_GLOB="S*"' scripts/lint-shadow-fixtures.sh \
|
|
3876
|
-
|| ! grep -Fq 'benchmark/auto-resolve/shadow-fixtures' scripts/lint-shadow-fixtures.sh \
|
|
3877
|
-
|| ! grep -Fq 'category") != "high-risk"' scripts/lint-fixtures.sh \
|
|
3878
|
-
|| ! grep -Fq 'permissions?' scripts/lint-fixtures.sh \
|
|
3879
|
-
|| ! grep -Fq 'idempoten\w*' scripts/lint-fixtures.sh \
|
|
3880
|
-
|| ! grep -Fq 'output-shape' scripts/lint-fixtures.sh \
|
|
3881
|
-
|| ! grep -Fq 'high-risk trigger validation' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3882
|
-
|| ! grep -Fq 'DEVLYN_FIXTURES_DIR="$FIXTURES_DIR" bash "$ROOT/scripts/lint-fixtures.sh"' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3883
|
-
|| ! grep -Fq 'DEVLYN_LINT_FIXTURES_NO_JSONSCHEMA=1' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3884
|
-
|| ! grep -Fq 'spec-verify-check --check-expected failed' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3885
|
-
|| ! grep -Fq 'unless spec.expected.json declares "pure_design": true' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3886
|
-
|| ! grep -Fq 'verification_commands must be an array' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3887
|
-
|| ! grep -Fq 'expected.json must be an object' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3888
|
-
|| ! grep -Fq 'hidden oracle missing contract_refs' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3889
|
-
|| ! grep -Fq 'contract_ref not found in spec.md' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3890
|
-
|| ! grep -Fq 'BENCH_FIXTURE_DIR file not found' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3891
|
-
|| ! grep -Fq 'BENCH_FIXTURE_DIR file escapes fixture dir' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3892
|
-
|| ! grep -Fq 'hidden oracle must reference an explicit $BENCH_FIXTURE_DIR/... file' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3893
|
-
|| ! grep -Fq 'hidden oracle must assert stdout_contains includes' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3894
|
-
|| ! grep -Fq 'cd \"$BENCH_FIXTURE_DIR\" && node verifiers/hidden-oracle.js' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3895
|
-
|| ! grep -Fq 'missing-hidden-oracle.js' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3896
|
-
|| ! grep -Fq '../outside-hidden-oracle.js' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3897
|
-
|| ! grep -Fq 'This visible contract is not in the spec.' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3898
|
-
|| ! grep -Fq 'spec-verify-check --check failed' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3899
|
-
|| ! grep -Fq 'frontmatter complexity must be one of' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3900
|
-
|| ! grep -Fq 'resolve spec contract enum' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3901
|
-
|| ! grep -Fq 'metadata.difficulty' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3902
|
-
|| ! grep -Fq "grep -Fq 'Traceback'" benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3903
|
-
|| ! grep -Fq 'hidden oracle missing contract_refs' scripts/lint-fixtures.sh \
|
|
3904
|
-
|| ! grep -Fq 'contract_ref not found in spec.md' scripts/lint-fixtures.sh \
|
|
3905
|
-
|| ! grep -Fq 'BENCH_FIXTURE_DIR file not found' scripts/lint-fixtures.sh \
|
|
3906
|
-
|| ! grep -Fq 'BENCH_FIXTURE_DIR file escapes fixture dir' scripts/lint-fixtures.sh \
|
|
3907
|
-
|| ! grep -Fq 'hidden oracle must reference an explicit $BENCH_FIXTURE_DIR/... file' scripts/lint-fixtures.sh \
|
|
3908
|
-
|| ! grep -Fq 'hidden oracle must assert stdout_contains includes' scripts/lint-fixtures.sh \
|
|
3909
|
-
|| ! grep -Fq 'SPEC_VERIFY_CHECK="$REPO_ROOT/config/skills/_shared/spec-verify-check.py"' scripts/lint-fixtures.sh \
|
|
3910
|
-
|| ! grep -Fq 'python3 "$SPEC_VERIFY_CHECK" --check "$d/spec.md"' scripts/lint-fixtures.sh \
|
|
3911
|
-
|| ! grep -Fq 'spec-verify-check --check failed' scripts/lint-fixtures.sh \
|
|
3912
|
-
|| ! grep -Fq 'python3 "$SPEC_VERIFY_CHECK" --check-expected "$d/expected.json"' scripts/lint-fixtures.sh \
|
|
3913
|
-
|| ! grep -Fq 'spec-verify-check --check-expected failed' scripts/lint-fixtures.sh \
|
|
3914
|
-
|| ! grep -Fq 'REJECTED_REGISTRY="${DEVLYN_REJECTED_FIXTURE_REGISTRY:-$REPO_ROOT/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh}"' scripts/lint-fixtures.sh \
|
|
3915
|
-
|| ! grep -Fq 'declare -F rejected_pair_fixture_reason' scripts/lint-fixtures.sh \
|
|
3916
|
-
|| ! grep -Fq 'rejected fixture registry must define rejected_pair_fixture_reason' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3917
|
-
|| ! grep -Fq 'malformed-rejected.sh' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3918
|
-
|| ! grep -Fq 'NOTES.md records pair-candidate rejection but pair-rejected-fixtures.sh has no rejected reason' scripts/lint-fixtures.sh \
|
|
3919
|
-
|| ! grep -Fq 'test-active-headroom' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3920
|
-
|| ! grep -Fq 'active-calibration-rejected-missing.out' benchmark/auto-resolve/scripts/test-lint-fixtures.sh \
|
|
3921
|
-
|| ! grep -Fq 'Rejected controls should remain replayable' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3922
|
-
|| ! grep -Fq 'pair-rejected-fixtures.sh' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3923
|
-
|| ! grep -Fq 'target.relative_to(fixture_root)' scripts/lint-fixtures.sh \
|
|
3924
|
-
|| ! grep -Fq '$BENCH_FIXTURE_DIR/...' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3925
|
-
|| ! grep -Fq 'must not escape the fixture directory' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3926
|
-
|| ! grep -Fq 'cd "$BENCH_FIXTURE_DIR"` indirection' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3927
|
-
|| ! grep -Fq 'success sentinel' benchmark/auto-resolve/fixtures/SCHEMA.md \
|
|
3928
|
-
|| ! grep -Fq 'def fallback_validate():' scripts/lint-fixtures.sh \
|
|
3929
|
-
|| ! grep -Fq 'expected.json must be an object' scripts/lint-fixtures.sh \
|
|
3930
|
-
|| ! grep -Fq 'schema_ok=0' scripts/lint-fixtures.sh \
|
|
3931
|
-
|| ! grep -Fq 'DEVLYN_LINT_FIXTURES_NO_JSONSCHEMA' scripts/lint-fixtures.sh \
|
|
3932
|
-
|| ! grep -Fq 'conditional risk-probe triggers' benchmark/auto-resolve/fixtures/SCHEMA.md; then
|
|
3933
|
-
offenders="${offenders}"$'\n'"fixture lint must require high-risk fixtures to include resolve pair/risk trigger terms"
|
|
3934
|
-
fi
|
|
3935
|
-
unsafe_json_parser_refs=$(python3 - <<'PY'
|
|
3936
|
-
import pathlib
|
|
3937
|
-
import re
|
|
3938
|
-
|
|
3939
|
-
root = pathlib.Path("benchmark/auto-resolve/scripts")
|
|
3940
|
-
patterns = [
|
|
3941
|
-
re.compile(r"json\.load\(open\("),
|
|
3942
|
-
re.compile(r"json\.load\(response\)"),
|
|
3943
|
-
re.compile(r"json\.loads\([^,\n]*(?:read_text\(\)|\bline\b|\bln\b)[^,\n]*\)"),
|
|
3944
|
-
]
|
|
3945
|
-
for path in sorted(root.glob("*")):
|
|
3946
|
-
if path.name.startswith("test-") or path.suffix not in {".py", ".sh"}:
|
|
3947
|
-
continue
|
|
3948
|
-
text = path.read_text(errors="ignore")
|
|
3949
|
-
for line_no, line in enumerate(text.splitlines(), 1):
|
|
3950
|
-
if any(pattern.search(line) for pattern in patterns):
|
|
3951
|
-
print(f"{path}:{line_no}:{line.strip()}")
|
|
3952
|
-
PY
|
|
3953
|
-
)
|
|
3954
|
-
if [ -n "$unsafe_json_parser_refs" ]; then
|
|
3955
|
-
while IFS= read -r f; do
|
|
3956
|
-
offenders="${offenders}"$'\n'"benchmark script uses unsafe JSON parser without strict constants: $f"
|
|
3957
|
-
done <<< "$unsafe_json_parser_refs"
|
|
3958
|
-
fi
|
|
3959
|
-
if [ -z "$offenders" ]; then
|
|
3960
|
-
ok "benchmark docs use current bare / solo_claude / pair-arm topology"
|
|
3961
|
-
else
|
|
3962
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
3963
|
-
fi
|
|
3964
|
-
|
|
3965
|
-
# ---------------------------------------------------------------------------
|
|
3966
|
-
# 9. Engine availability fails closed; stale silent-downgrade wording is forbidden.
|
|
3967
|
-
# ---------------------------------------------------------------------------
|
|
3968
|
-
section "Check 9: Engine availability fails closed"
|
|
3969
|
-
offenders=$(grep -RInE 'codex-ping failed|codex-ping fail|engine downgraded: codex-unavailable|downgrades to Claude-only|silently downgrades|silently downgrade|silently switch to Claude|Codex CLI availability downgrade' \
|
|
3970
|
-
config/skills CLAUDE.md README.md bin/ benchmark/auto-resolve/run-real-benchmark.md 2>/dev/null \
|
|
3971
|
-
| grep -v 'roadmap-archival-workspace/' \
|
|
3972
|
-
| grep -v 'devlyn:auto-resolve-workspace/' \
|
|
3973
|
-
| grep -v 'devlyn:ideate-workspace/' \
|
|
3974
|
-
| grep -v 'preflight-workspace/' \
|
|
3975
|
-
|| true)
|
|
3976
|
-
if [ -z "$offenders" ]; then
|
|
3977
|
-
ok "engine availability fail-closed wording canonical"
|
|
3978
|
-
else
|
|
3979
|
-
while IFS= read -r f; do bad "$f"; done <<< "$offenders"
|
|
3980
|
-
fi
|
|
3981
|
-
|
|
3982
|
-
# ---------------------------------------------------------------------------
|
|
3983
|
-
# 9b. Release tag and package version parity.
|
|
3984
|
-
# v2.2.3 was tagged while package.json still said 2.2.2. A mismatched npm
|
|
3985
|
-
# package makes Codex/Claude installed-skill drift hard to diagnose because
|
|
3986
|
-
# users can be on a tag whose package metadata points at a different build.
|
|
3987
|
-
# ---------------------------------------------------------------------------
|
|
3988
|
-
section "Check 9b: package.json version matches exact release tag"
|
|
3989
|
-
exact_tag=$(git describe --tags --exact-match HEAD 2>/dev/null || true)
|
|
3990
|
-
pkg_version=$(node -p "require('./package.json').version" 2>/dev/null || true)
|
|
3991
|
-
if [ -z "$exact_tag" ]; then
|
|
3992
|
-
ok "HEAD is not an exact tag — package/tag parity not applicable"
|
|
3993
|
-
elif [ "$pkg_version" = "${exact_tag#v}" ]; then
|
|
3994
|
-
ok "package.json version matches $exact_tag"
|
|
3995
|
-
else
|
|
3996
|
-
bad "package.json version '$pkg_version' does not match HEAD tag '$exact_tag'"
|
|
3997
|
-
fi
|
|
3998
|
-
|
|
3999
|
-
# ---------------------------------------------------------------------------
|
|
4000
|
-
# (Check 7 retired iter-0034 Phase 4 cutover: the 4 findings-producing
|
|
4001
|
-
# standalones — evaluate / review / clean / team-review — were deleted; the
|
|
4002
|
-
# JSONL sidecar contract no longer has a surface to enforce.)
|
|
4003
|
-
# ---------------------------------------------------------------------------
|
|
4004
|
-
# 12. CLAUDE.md ↔ _shared/runtime-principles.md per-section excerpt parity (iter-0019.A).
|
|
4005
|
-
# Sub-agent prompts inline the runtime contract from runtime-principles.md; that file
|
|
4006
|
-
# must mirror the corresponding CLAUDE.md sections. Drift in one source-of-truth without
|
|
4007
|
-
# the other produces silent behavioral divergence between session-level and sub-agent
|
|
4008
|
-
# enforcement. Per-section markers `<!-- runtime-principles:section=NAME:begin/end -->`
|
|
4009
|
-
# wrap each of the 3 mirrored sections (subtractive-first, goal-locked, evidence)
|
|
4010
|
-
# in BOTH files. No-workaround discipline lives in CLAUDE.md Core Principle #1
|
|
4011
|
-
# only and is not mirrored to runtime-principles.md.
|
|
4012
|
-
# Check 12 extracts each named block from both files and diffs.
|
|
4013
|
-
# ---------------------------------------------------------------------------
|
|
4014
|
-
section "Check 12: CLAUDE.md ↔ runtime-principles.md per-section excerpt parity"
|
|
4015
|
-
rp_src="config/skills/_shared/runtime-principles.md"
|
|
4016
|
-
claude_src="CLAUDE.md"
|
|
4017
|
-
rp_drift=0
|
|
4018
|
-
expected_sections="subtractive-first goal-locked evidence"
|
|
4019
|
-
|
|
4020
|
-
if [ ! -f "$rp_src" ]; then
|
|
4021
|
-
bad "$rp_src — missing"
|
|
4022
|
-
rp_drift=1
|
|
4023
|
-
elif [ ! -f "$claude_src" ]; then
|
|
4024
|
-
bad "$claude_src — missing"
|
|
4025
|
-
rp_drift=1
|
|
4026
|
-
else
|
|
4027
|
-
# Topology: each marker appears exactly once per file.
|
|
4028
|
-
for name in $expected_sections; do
|
|
4029
|
-
for kind in begin end; do
|
|
4030
|
-
marker="<!-- runtime-principles:section=${name}:${kind} -->"
|
|
4031
|
-
for f in "$rp_src" "$claude_src"; do
|
|
4032
|
-
count=$(grep -Fxc "$marker" "$f" 2>/dev/null || echo 0)
|
|
4033
|
-
if [ "$count" -ne 1 ]; then
|
|
4034
|
-
bad "${f}: marker '${marker}' appears ${count} times (expected 1)"
|
|
4035
|
-
rp_drift=1
|
|
4036
|
-
fi
|
|
4037
|
-
done
|
|
4038
|
-
done
|
|
4039
|
-
done
|
|
4040
|
-
|
|
4041
|
-
# Topology: in runtime-principles.md, all 4 sections must sit INSIDE the
|
|
4042
|
-
# outer `:contract:` block AND appear in the canonical order. CLAUDE.md
|
|
4043
|
-
# placement is free (sections may live in any order, anywhere in the file).
|
|
4044
|
-
contract_begin_line=$(grep -Fxn '<!-- runtime-principles:contract:begin -->' "$rp_src" | head -1 | cut -d: -f1)
|
|
4045
|
-
contract_end_line=$(grep -Fxn '<!-- runtime-principles:contract:end -->' "$rp_src" | head -1 | cut -d: -f1)
|
|
4046
|
-
if [ -z "$contract_begin_line" ] || [ -z "$contract_end_line" ]; then
|
|
4047
|
-
bad "${rp_src}: outer ':contract:begin/end' markers missing"
|
|
4048
|
-
rp_drift=1
|
|
4049
|
-
else
|
|
4050
|
-
prev_line=0
|
|
4051
|
-
for name in $expected_sections; do
|
|
4052
|
-
sec_begin_line=$(grep -Fxn "<!-- runtime-principles:section=${name}:begin -->" "$rp_src" | head -1 | cut -d: -f1)
|
|
4053
|
-
sec_end_line=$(grep -Fxn "<!-- runtime-principles:section=${name}:end -->" "$rp_src" | head -1 | cut -d: -f1)
|
|
4054
|
-
if [ -n "$sec_begin_line" ] && [ -n "$sec_end_line" ]; then
|
|
4055
|
-
if [ "$sec_begin_line" -le "$contract_begin_line" ] || [ "$sec_end_line" -ge "$contract_end_line" ]; then
|
|
4056
|
-
bad "${rp_src}: section '${name}' is outside the ':contract:' block"
|
|
4057
|
-
rp_drift=1
|
|
4058
|
-
fi
|
|
4059
|
-
if [ "$sec_begin_line" -lt "$prev_line" ]; then
|
|
4060
|
-
bad "${rp_src}: section '${name}' is out of canonical order (expected: ${expected_sections})"
|
|
4061
|
-
rp_drift=1
|
|
4062
|
-
fi
|
|
4063
|
-
prev_line=$sec_end_line
|
|
4064
|
-
fi
|
|
4065
|
-
done
|
|
4066
|
-
fi
|
|
4067
|
-
|
|
4068
|
-
# Content: byte-compare each section block via diff over temp files.
|
|
4069
|
-
# awk-into-tmpfile preserves trailing newlines (command substitution strips them).
|
|
4070
|
-
if make_temp_file tmp_rp && make_temp_file tmp_claude; then
|
|
4071
|
-
for name in $expected_sections; do
|
|
4072
|
-
begin="<!-- runtime-principles:section=${name}:begin -->"
|
|
4073
|
-
end="<!-- runtime-principles:section=${name}:end -->"
|
|
4074
|
-
awk -v b="$begin" -v e="$end" '$0==b{f=1;next}$0==e{f=0}f' "$rp_src" > "$tmp_rp"
|
|
4075
|
-
awk -v b="$begin" -v e="$end" '$0==b{f=1;next}$0==e{f=0}f' "$claude_src" > "$tmp_claude"
|
|
4076
|
-
if [ ! -s "$tmp_rp" ]; then
|
|
4077
|
-
bad "${name}: empty/missing block in $rp_src"
|
|
4078
|
-
rp_drift=1
|
|
4079
|
-
continue
|
|
4080
|
-
fi
|
|
4081
|
-
if [ ! -s "$tmp_claude" ]; then
|
|
4082
|
-
bad "${name}: empty/missing block in $claude_src"
|
|
4083
|
-
rp_drift=1
|
|
4084
|
-
continue
|
|
4085
|
-
fi
|
|
4086
|
-
if ! diff -q "$tmp_rp" "$tmp_claude" >/dev/null 2>&1; then
|
|
4087
|
-
bad "${name}: CLAUDE.md and runtime-principles.md content differ"
|
|
4088
|
-
rp_drift=1
|
|
4089
|
-
fi
|
|
4090
|
-
done
|
|
4091
|
-
rm -f "$tmp_rp" "$tmp_claude"
|
|
4092
|
-
else
|
|
4093
|
-
rp_drift=1
|
|
4094
|
-
fi
|
|
4095
|
-
|
|
4096
|
-
if [ $rp_drift -eq 0 ]; then
|
|
4097
|
-
ok "all 3 mirrored contract sections in parity (subtractive-first / goal-locked / evidence) — markers, topology, content"
|
|
4098
|
-
fi
|
|
4099
|
-
fi
|
|
4100
|
-
|
|
4101
|
-
# ---------------------------------------------------------------------------
|
|
4102
|
-
# 13. pair-plan idgen output is deterministic across consecutive runs (iter-0022).
|
|
4103
|
-
# Same input → byte-identical canonical_id_registry.json. Catches accidental
|
|
4104
|
-
# dict-order, float-printing, or timestamp-leak regressions in idgen.
|
|
4105
|
-
# Runs twice on F2 with --generated-at pinned and compares sha256.
|
|
4106
|
-
# ---------------------------------------------------------------------------
|
|
4107
|
-
section "Check 13: pair-plan-idgen.py output deterministic across runs (F2)"
|
|
4108
|
-
idgen="benchmark/auto-resolve/scripts/pair-plan-idgen.py"
|
|
4109
|
-
fixture="benchmark/auto-resolve/fixtures/F2-cli-medium-subcommand"
|
|
4110
|
-
if [ ! -x "$idgen" ] && [ ! -f "$idgen" ]; then
|
|
4111
|
-
bad "Check 13 prerequisite missing: $idgen"
|
|
4112
|
-
elif [ ! -d "$fixture" ]; then
|
|
4113
|
-
bad "Check 13 prerequisite missing: $fixture"
|
|
4114
|
-
else
|
|
4115
|
-
if make_temp_file tmp1 && make_temp_file tmp2; then
|
|
4116
|
-
if python3 "$idgen" --fixture "$fixture" --generated-at 2026-04-29T18:30:00Z --output "$tmp1" >/dev/null 2>&1 \
|
|
4117
|
-
&& python3 "$idgen" --fixture "$fixture" --generated-at 2026-04-29T18:30:00Z --output "$tmp2" >/dev/null 2>&1; then
|
|
4118
|
-
sha1=$(shasum -a 256 "$tmp1" | awk '{print $1}')
|
|
4119
|
-
sha2=$(shasum -a 256 "$tmp2" | awk '{print $1}')
|
|
4120
|
-
if [ "$sha1" = "$sha2" ]; then
|
|
4121
|
-
ok "F2 registry sha256 stable across two idgen runs ($sha1)"
|
|
4122
|
-
else
|
|
4123
|
-
bad "F2 registry sha256 drift: run1=$sha1 run2=$sha2"
|
|
4124
|
-
fi
|
|
4125
|
-
else
|
|
4126
|
-
bad "idgen invocation failed; cannot verify determinism"
|
|
4127
|
-
fi
|
|
4128
|
-
else
|
|
4129
|
-
bad "idgen temp files unavailable; cannot verify determinism"
|
|
4130
|
-
fi
|
|
4131
|
-
if make_temp_dir tmp_bad && make_temp_file idgen_nan_out /tmp/pair-plan-idgen-nan.out.XXXXXX; then
|
|
4132
|
-
cp -R "$fixture/." "$tmp_bad/"
|
|
4133
|
-
printf '{"verification_commands": NaN}\n' > "$tmp_bad/expected.json"
|
|
4134
|
-
if python3 "$idgen" --fixture "$tmp_bad" --generated-at 2026-04-29T18:30:00Z >"$idgen_nan_out" 2>&1; then
|
|
4135
|
-
bad "pair-plan-idgen.py accepted NaN in expected.json"
|
|
4136
|
-
elif ! grep -Fq 'invalid JSON numeric constant: NaN' "$idgen_nan_out"; then
|
|
4137
|
-
bad "pair-plan-idgen.py NaN failure did not cite invalid JSON numeric constant"
|
|
4138
|
-
fi
|
|
4139
|
-
rm -rf "$tmp_bad" "$idgen_nan_out"
|
|
4140
|
-
fi
|
|
4141
|
-
|
|
4142
|
-
if make_temp_file bad_plan && make_temp_file plan_lint_nan_out /tmp/pair-plan-lint-nan.out.XXXXXX; then
|
|
4143
|
-
printf '{"schema_version": NaN}\n' > "$bad_plan"
|
|
4144
|
-
if python3 benchmark/auto-resolve/scripts/pair-plan-lint.py --plan "$bad_plan" --quiet >"$plan_lint_nan_out" 2>&1; then
|
|
4145
|
-
bad "pair-plan-lint.py accepted NaN in pair-plan.json"
|
|
4146
|
-
elif ! grep -Fq '"code": "plan_invalid_json"' "$plan_lint_nan_out" \
|
|
4147
|
-
|| ! grep -Fq 'invalid JSON numeric constant: NaN' "$plan_lint_nan_out"; then
|
|
4148
|
-
bad "pair-plan-lint.py NaN failure did not report plan_invalid_json with invalid numeric constant"
|
|
4149
|
-
fi
|
|
4150
|
-
rm -f "$bad_plan" "$plan_lint_nan_out"
|
|
4151
|
-
fi
|
|
4152
|
-
rm -f "${tmp1:-}" "${tmp2:-}"
|
|
4153
|
-
fi
|
|
4154
|
-
|
|
4155
|
-
# ---------------------------------------------------------------------------
|
|
4156
|
-
# 14. F9 fixture id matches the shipped 2-skill contract (iter-0033a, 2026-04-30).
|
|
4157
|
-
# `/devlyn:preflight` was folded into `/devlyn:resolve`'s VERIFY phase; the
|
|
4158
|
-
# legacy F9 dir name (`F9-e2e-ideate-to-preflight`) is misleading once
|
|
4159
|
-
# preflight is gone. The retired copy lives under `fixtures/retired/` for
|
|
4160
|
-
# replay; the live fixture must be `F9-e2e-ideate-to-resolve`. Any other
|
|
4161
|
-
# non-retired reference to the old id is a stale rename.
|
|
4162
|
-
# ---------------------------------------------------------------------------
|
|
4163
|
-
section "Check 14: F9 fixture id matches 2-skill contract"
|
|
4164
|
-
f9_drift=0
|
|
4165
|
-
if [ ! -d "benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve" ]; then
|
|
4166
|
-
bad "live F9 fixture missing at benchmark/auto-resolve/fixtures/F9-e2e-ideate-to-resolve"
|
|
4167
|
-
f9_drift=1
|
|
4168
|
-
fi
|
|
4169
|
-
# Stale references outside fixtures/retired/ are bugs. Examine line content
|
|
4170
|
-
# (not just filename) so files that legitimately mention the retired *path*
|
|
4171
|
-
# (e.g. fixtures/F9-e2e-ideate-to-resolve/NOTES.md explaining where the OLD
|
|
4172
|
-
# version lives) pass while genuine stale references fail. Excluded scopes:
|
|
4173
|
-
# benchmark/auto-resolve/results/ (historical run artifacts, frozen) and
|
|
4174
|
-
# scripts/lint-skills.sh itself (carries the pattern in this check).
|
|
4175
|
-
stale=$(git grep -In -- 'F9-e2e-ideate-to-preflight' -- \
|
|
4176
|
-
config/skills \
|
|
4177
|
-
benchmark \
|
|
4178
|
-
scripts \
|
|
4179
|
-
CLAUDE.md \
|
|
4180
|
-
README.md \
|
|
4181
|
-
':!benchmark/auto-resolve/results/**' \
|
|
4182
|
-
2>/dev/null \
|
|
4183
|
-
| grep -v '^benchmark/auto-resolve/fixtures/retired/F9-e2e-ideate-to-preflight/' \
|
|
4184
|
-
| grep -v '^scripts/lint-skills\.sh:' \
|
|
4185
|
-
| grep -v 'fixtures/retired/F9-e2e-ideate-to-preflight' \
|
|
4186
|
-
|| true)
|
|
4187
|
-
if [ -n "$stale" ]; then
|
|
4188
|
-
while IFS= read -r f; do bad "stale F9-e2e-ideate-to-preflight reference: $f"; done <<< "$stale"
|
|
4189
|
-
f9_drift=1
|
|
4190
|
-
fi
|
|
4191
|
-
if [ $f9_drift -eq 0 ]; then
|
|
4192
|
-
ok "F9 fixture id is canonical (F9-e2e-ideate-to-resolve); no stale refs outside retired/"
|
|
4193
|
-
fi
|
|
4194
|
-
|
|
4195
|
-
# ---------------------------------------------------------------------------
|
|
4196
|
-
# 15. Current pair-evidence fixtures carry their local evidence handoff notes.
|
|
4197
|
-
# The audit artifacts are canonical, but fixture-level notes keep future
|
|
4198
|
-
# fixture edits from losing why a candidate currently counts as solo<pair
|
|
4199
|
-
# evidence.
|
|
4200
|
-
# ---------------------------------------------------------------------------
|
|
4201
|
-
section "Check 15: Pair evidence fixture notes cite current passing runs"
|
|
4202
|
-
if python3 - <<'PY'
|
|
4203
|
-
import importlib.util
|
|
4204
|
-
import pathlib
|
|
4205
|
-
import sys
|
|
4206
|
-
|
|
4207
|
-
script = pathlib.Path("benchmark/auto-resolve/scripts/pair-candidate-frontier.py")
|
|
4208
|
-
spec = importlib.util.spec_from_file_location("pair_candidate_frontier", script)
|
|
4209
|
-
module = importlib.util.module_from_spec(spec)
|
|
4210
|
-
assert spec.loader is not None
|
|
4211
|
-
spec.loader.exec_module(module)
|
|
4212
|
-
|
|
4213
|
-
report = module.build_report(
|
|
4214
|
-
fixtures_root=pathlib.Path("benchmark/auto-resolve/fixtures"),
|
|
4215
|
-
registry=pathlib.Path("benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh"),
|
|
4216
|
-
results_root=pathlib.Path("benchmark/auto-resolve/results"),
|
|
4217
|
-
)
|
|
4218
|
-
errors = []
|
|
4219
|
-
for row in report["rows"]:
|
|
4220
|
-
if row.get("status") != "pair_evidence_passed":
|
|
4221
|
-
continue
|
|
4222
|
-
fixture = row["fixture"]
|
|
4223
|
-
best = module.best_pair_evidence(row.get("passing_pair_evidence", []))
|
|
4224
|
-
if best is None:
|
|
4225
|
-
errors.append(f"{fixture}: missing complete pair evidence")
|
|
4226
|
-
continue
|
|
4227
|
-
notes_path = pathlib.Path("benchmark/auto-resolve/fixtures") / fixture / "NOTES.md"
|
|
4228
|
-
try:
|
|
4229
|
-
notes = notes_path.read_text(encoding="utf8")
|
|
4230
|
-
except OSError:
|
|
4231
|
-
errors.append(f"{fixture}: NOTES.md missing")
|
|
4232
|
-
continue
|
|
4233
|
-
required = [
|
|
4234
|
-
str(best["run_id"]),
|
|
4235
|
-
"pair_evidence_passed",
|
|
4236
|
-
f"bare `{best['bare_score']}`",
|
|
4237
|
-
f"solo_claude `{best['solo_score']}`",
|
|
4238
|
-
f"pair `{best['pair_score']}`",
|
|
4239
|
-
f"margin `{best['pair_margin']:+d}`",
|
|
4240
|
-
f"wall `{best['pair_solo_wall_ratio']:.2f}x`",
|
|
4241
|
-
f"arm `{best['pair_arm']}`",
|
|
4242
|
-
]
|
|
4243
|
-
missing = [item for item in required if item not in notes]
|
|
4244
|
-
if missing:
|
|
4245
|
-
errors.append(f"{fixture}: NOTES.md missing {', '.join(missing)}")
|
|
4246
|
-
if errors:
|
|
4247
|
-
raise SystemExit("\n".join(errors))
|
|
4248
|
-
PY
|
|
4249
|
-
then
|
|
4250
|
-
ok "current pair-evidence fixture notes cite passing run ids"
|
|
4251
|
-
else
|
|
4252
|
-
bad "current pair-evidence fixture NOTES.md files must cite passing run ids and pair_evidence_passed"
|
|
4253
|
-
fi
|
|
4254
|
-
|
|
4255
|
-
# ---------------------------------------------------------------------------
|
|
4256
|
-
# Summary.
|
|
4257
|
-
# ---------------------------------------------------------------------------
|
|
4258
|
-
echo
|
|
4259
|
-
if [ $fail -eq 0 ]; then
|
|
4260
|
-
printf '%sAll checks passed.%s\n' "$green" "$reset"
|
|
4261
|
-
exit 0
|
|
4262
|
-
else
|
|
4263
|
-
printf '%sLint failed.%s Fix the offenders above.\n' "$red" "$reset"
|
|
4264
|
-
exit 1
|
|
4265
|
-
fi
|